Kubernetes API Server Request Latency Spike Sentinel

id: kubernetes-apiserver-request-latency-spike-sentinel
namespace: company.ops

inputs:
  - id: apiserver_url
    type: STRING
    defaults: "https://kubernetes.default.svc"
    description: Kubernetes API server internal service endpoint URL.
  - id: latency_p99_threshold_seconds
    type: FLOAT
    defaults: 1.0
    description: Maximum acceptable p99 API request duration in seconds before alerting.

tasks:
  - id: scrape_apiserver_metrics
    type: io.kestra.plugin.scripts.shell.Commands
    containerImage: bitnami/kubectl:latest
    commands:
      - |
        TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token 2>/dev/null || echo "")

        curl -s -k -H "Authorization: Bearer $TOKEN" "{{ inputs.apiserver_url }}/metrics" > /tmp/k8s_metrics.prom

        P99_LATENCY=$(grep -E '^apiserver_request_duration_seconds\{.*verb="LIST",.*quantile="0.99".*\}' /tmp/k8s_metrics.prom | awk '{print $2}' | head -n1)
        P99_LATENCY=${P99_LATENCY:-0}

        echo "::set-output name=p99_latency::$P99_LATENCY::"

  - id: evaluate_apiserver_latency
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.scrape_apiserver_metrics.vars.p99_latency | float >
      inputs.latency_p99_threshold_seconds }}"
    then:
      - id: notify_platform_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#kubernetes-alerts",
            "attachments": [
              {
                "color": "danger",
                "title": "🚨 Kubernetes API Server Request Latency Spike Alert",
                "text": "Kubernetes API Server at `{{ inputs.apiserver_url }}` recorded p99 request duration of *{{ outputs.scrape_apiserver_metrics.vars.p99_latency }}s* (Threshold: {{ inputs.latency_p99_threshold_seconds }}s)! Impact: Control plane saturation, etcd bottlenecking, or heavy unindexed LIST calls are throttling kubelet, ingress, and controller reconciliation loops.",
                "footer": "Kestra Kubernetes Sentinel • Control Plane"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#kubernetes-alerts",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ Kubernetes API Server Request Latency Spike Sentinel Pipeline Failure",
            "text": "Failed to scrape Kubernetes API server /metrics endpoint. Verify service account permissions.",
            "footer": "Kestra Workflow Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"
    disabled: true