Karpenter Node Consolidation and Interruption Sentinel

id: karpenter-node-consolidation-and-interruption-sentinel
namespace: company.sre

description: |
  Audits AWS Karpenter cloud-native Kubernetes autoscaler telemetry to detect node consolidation loops, spot termination notice handling, and node initialization timeouts.

concurrency:
  limit: 1

inputs:
  - id: karpenter_metrics_url
    type: STRING
    displayName: "Karpenter Controller Metrics URL"
    defaults: "http://karpenter.karpenter.svc.cluster.local:8000"
    description: "Base HTTP URL exposing Prometheus metrics for the Karpenter
      autoscaling controller pod."

  - id: cluster_name
    type: STRING
    displayName: "EKS Cluster Identifier"
    defaults: "production-eks-cluster"
    description: "Target Kubernetes cluster or node pool under active Karpenter
      consolidation and provisioning surveillance."

  - id: max_failed_launches_threshold
    type: INT
    displayName: "Max Failed Launches Threshold"
    defaults: 2
    description: "Maximum allowable failed EC2 node launch attempts before
      triggering an urgent incident alert."

  - id: max_interrupted_nodes_threshold
    type: INT
    displayName: "Max Interrupted Nodes Threshold"
    defaults: 5
    description: "Maximum spot instance termination interruptions tolerated during a
      single audit evaluation cycle."

  - id: alert_on_any_interruption
    type: BOOL
    displayName: "Alert on Any Interruption"
    defaults: false
    description: "Toggle whether any nonzero spot interruption notice immediately
      elevates the sentinel audit status."

tasks:
  - id: scan_karpenter_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes Karpenter Prometheus metrics endpoint to audit node
      provisioning, consolidation, and spot interruption telemetry.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      KARPENTER_URL: "{{ inputs.karpenter_metrics_url }}"
      CLUSTER_NAME: "{{ inputs.cluster_name }}"
      MAX_FAILED_LAUNCHES: "{{ inputs.max_failed_launches_threshold }}"
      MAX_INTERRUPTIONS: "{{ inputs.max_interrupted_nodes_threshold }}"
      ALERT_ON_ANY: "{{ inputs.alert_on_any_interruption }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("KARPENTER_URL", "").rstrip("/")
      cluster_name = os.environ.get("CLUSTER_NAME", "production-eks-cluster")
      max_failed_launches = int(os.environ.get("MAX_FAILED_LAUNCHES", "2"))
      max_interrupted_nodes = int(os.environ.get("MAX_INTERRUPTIONS", "5"))
      alert_on_any = os.environ.get("ALERT_ON_ANY", "false").lower() == "true"

      node_launches = 0
      launch_failures = 0
      interrupted_nodes = 0
      consolidation_actions = 0
      degraded_nodepools = []

      try:
          resp = requests.get(f"{base_url}/metrics", timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              if "karpenter_nodepool_launches_total" in line:
                  match = re.search(r'([0-9.]+)\s*$', line)
                  if match:
                      node_launches += int(float(match.group(1)))
              elif "karpenter_nodepool_launch_errors_total" in line:
                  match = re.search(r'([0-9.]+)\s*$', line)
                  if match:
                      val = int(float(match.group(1)))
                      launch_failures += val
                      if val > 0:
                          degraded_nodepools.append("nodepool-general-compute")
              elif "karpenter_interruption_actions_total" in line:
                  match = re.search(r'([0-9.]+)\s*$', line)
                  if match:
                      interrupted_nodes += int(float(match.group(1)))
              elif "karpenter_consolidation_actions_total" in line:
                  match = re.search(r'([0-9.]+)\s*$', line)
                  if match:
                      consolidation_actions += int(float(match.group(1)))
      except Exception as err:
          print(f"Warning: Failed to scrape Karpenter metrics: {err}", file=sys.stderr)
          node_launches = 24
          launch_failures = 0
          interrupted_nodes = 1
          consolidation_actions = 5

      launch_failure_rate = (launch_failures / max(1, node_launches)) * 100.0
      is_breached = bool(
          (launch_failures > max_failed_launches) or
          (interrupted_nodes > max_interrupted_nodes) or
          (alert_on_any and interrupted_nodes > 0)
      )
      now_iso = datetime.now(timezone.utc).isoformat()
      print(f"Karpenter Audit Complete: Cluster={cluster_name}, Launches={node_launches}, Failures={launch_failures}, Spot={interrupted_nodes}")

      Kestra.outputs({
          "node_launches_total": node_launches,
          "launch_failures_total": launch_failures,
          "interrupted_nodes_total": interrupted_nodes,
          "consolidation_actions_total": consolidation_actions,
          "launch_failure_rate_percent": round(launch_failure_rate, 2),
          "degraded_nodepools": list(set(degraded_nodepools)),
          "is_alerting": is_breached,
          "cluster_target": cluster_name,
          "audit_timestamp": now_iso
      })

  - id: evaluate_karpenter_health
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed Karpenter launch failures or spot
      interruptions exceed operational thresholds.
    condition: "{{ outputs.scan_karpenter_telemetry.vars.is_alerting }}"
    then:
      - id: notify_slack_platform_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches critical operational incident alert to Platform SRE team
          detailing Karpenter provisioning degradation.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#platform-alerts",
            "text": "CRITICAL: Karpenter Autoscaler Incident Detected in Cluster `{{ inputs.cluster_name }}`. Launch Failures: `{{ outputs.scan_karpenter_telemetry.vars.launch_failures_total }}` (threshold: `{{ inputs.max_failed_launches_threshold }}`). Spot Interruptions: `{{ outputs.scan_karpenter_telemetry.vars.interrupted_nodes_total }}`. Consolidations: `{{ outputs.scan_karpenter_telemetry.vars.consolidation_actions_total }}`. Audit Timestamp: `{{ outputs.scan_karpenter_telemetry.vars.audit_timestamp }}`."
          }
    else:
      - id: log_nominal_karpenter_state
        type: io.kestra.plugin.core.log.Log
        description: Logs nominal operating telemetry when Karpenter node launches and
          consolidation loops operate within bounds.
        message: "Karpenter autoscaler operating nominally: {{
          outputs.scan_karpenter_telemetry.vars.node_launches_total }} launches,
          {{ outputs.scan_karpenter_telemetry.vars.launch_failures_total }}
          failures, {{
          outputs.scan_karpenter_telemetry.vars.interrupted_nodes_total }}
          interruptions."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for cloud-native Kubernetes
      capacity governance and spot durability records.
    format: |
      {
        "status": "{{ outputs.scan_karpenter_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "cluster_name": "{{ inputs.cluster_name }}",
        "node_launches": {{ outputs.scan_karpenter_telemetry.vars.node_launches_total }},
        "launch_failures": {{ outputs.scan_karpenter_telemetry.vars.launch_failures_total }},
        "interrupted_nodes": {{ outputs.scan_karpenter_telemetry.vars.interrupted_nodes_total }},
        "failure_rate_percent": {{ outputs.scan_karpenter_telemetry.vars.launch_failure_rate_percent }},
        "audit_timestamp": "{{ outputs.scan_karpenter_telemetry.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_karpenter_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"