Kubernetes Mutating Admission Webhook Timeout and Failure Policy Sentinel

id: kubernetes-mutating-webhook-timeout-sentinel
namespace: company.sre

inputs:
  - id: kube_context
    type: STRING
    required: false
    description: Optional Kubernetes context name.
  - id: max_timeout_seconds
    type: INT
    defaults: 10
    description: Maximum acceptable timeoutSeconds before webhook risks stalling API
      server requests.

tasks:
  - id: audit_mutating_webhooks
    type: io.kestra.plugin.scripts.shell.Commands
    containerImage: bitnami/kubectl:latest
    commands:
      - |
        CONTEXT_ARG=""
        if [ -n "{{ inputs.kube_context }}" ]; then
          CONTEXT_ARG="--context={{ inputs.kube_context }}"
        fi

        kubectl $CONTEXT_ARG get mutatingwebhookconfigurations -o json > /tmp/mwhc.json

        FAILING_WEBHOOKS=$(jq '[.items[] | {name: .metadata.name, webhooks: [.webhooks[] | select((.timeoutSeconds // 10) >= {{ inputs.max_timeout_seconds }} and .failurePolicy == "Fail") | {name: .name, timeout: (.timeoutSeconds // 10), failure_policy: .failurePolicy}]}] | map(select(.webhooks | length > 0))' /tmp/mwhc.json)
        FAILING_COUNT=$(echo "$FAILING_WEBHOOKS" | jq 'length')

        echo "::set-output name=failing_count::$FAILING_COUNT::"
        echo "::set-output name=failing_webhooks::$FAILING_WEBHOOKS::"

  - id: evaluate_webhook_risk
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.audit_mutating_webhooks.vars.failing_count | int > 0 }}"
    then:
      - id: notify_platform_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#infrastructure-alerts",
            "attachments": [
              {
                "color": "warning",
                "title": "⚠️ Kubernetes MutatingWebhook High Timeout and Failure Risk Alert",
                "text": "Detected *{{ outputs.audit_mutating_webhooks.vars.failing_count }}* MutatingWebhookConfiguration(s) with `failurePolicy: Fail` and high `timeoutSeconds` (>={{ inputs.max_timeout_seconds }}s). At-risk webhooks: {{ outputs.audit_mutating_webhooks.vars.failing_webhooks }}. Impact: If the admission webhook endpoint experiences latency or becomes unreachable, the Kubernetes API server will reject all incoming pod creations, breaking deployments.",
                "footer": "Kestra SRE Sentinel • Admission Webhooks"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#infrastructure-alerts",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ MutatingWebhook Sentinel Execution Failed",
            "text": "The MutatingWebhookConfiguration audit workflow encountered an execution error. Please verify cluster RBAC permissions and kubeconfig.",
            "footer": "Kestra Workflow Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "0 8 * * *"
    disabled: true