Kubernetes StatefulSet Replica Health and Quorum Degradation Sentinel

id: kubernetes-statefulset-replica-degradation-sentinel
namespace: company.sre

inputs:
  - id: kube_context
    type: STRING
    required: false
    description: Optional Kubernetes context name.
  - id: degradation_tolerance
    type: INT
    defaults: 0
    description: Maximum acceptable count of degraded StatefulSets with unready replicas.

tasks:
  - id: audit_statefulsets
    type: io.kestra.plugin.scripts.shell.Commands
    containerImage: bitnami/kubectl:latest
    commands:
      - |
        CONTEXT_ARG=""
        if [ -n "{{ inputs.kube_context }}" ]; then
          CONTEXT_ARG="--context={{ inputs.kube_context }}"
        fi

        kubectl $CONTEXT_ARG get statefulsets -A -o json > /tmp/sts.json

        DEGRADED_COUNT=$(jq '[.items[] | select((.status.readyReplicas // 0) < .spec.replicas)] | length' /tmp/sts.json)
        DEGRADED_DETAILS=$(jq '[.items[] | select((.status.readyReplicas // 0) < .spec.replicas) | {namespace: .metadata.namespace, name: .metadata.name, desired: .spec.replicas, ready: (.status.readyReplicas // 0), current: (.status.currentReplicas // 0)}]' /tmp/sts.json)

        echo "::set-output name=degraded_count::$DEGRADED_COUNT::"
        echo "::set-output name=degraded_details::$DEGRADED_DETAILS::"

  - id: evaluate_statefulset_health
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.audit_statefulsets.vars.degraded_count | int >
      inputs.degradation_tolerance }}"
    then:
      - id: notify_oncall
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#infrastructure-alerts",
            "attachments": [
              {
                "color": "danger",
                "title": "🚨 Kubernetes StatefulSet Replica Degradation Alert",
                "text": "Detected *{{ outputs.audit_statefulsets.vars.degraded_count }}* degraded StatefulSet(s) where ready replicas do not match desired spec. Degradation details: {{ outputs.audit_statefulsets.vars.degraded_details }}. Impact: Cluster quorum loss or read/write degradation for stateful applications.",
                "footer": "Kestra SRE Orchestrator • StatefulSet Sentinel"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#infrastructure-alerts",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ StatefulSet Sentinel Workflow Execution Failed",
            "text": "The StatefulSet replica audit workflow encountered an execution error. Please check cluster RBAC and API server connectivity.",
            "footer": "Kestra Workflow Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"
    disabled: true