Descheduler Pod Eviction Threshold and Node Imbalance Sentinel

id: descheduler-pod-eviction-threshold-and-node-imbalance-sentinel
namespace: company.sre

description: |
  Audits Kubernetes Descheduler controller telemetry to detect excessive pod eviction velocity, LowNodeUtilization threshold violations, and cluster rebalancing stalls.

concurrency:
  limit: 1

inputs:
  - id: descheduler_metrics_url
    type: STRING
    displayName: "Descheduler Metrics Endpoint URL"
    defaults: "http://descheduler.kube-system.svc.cluster.local:10258"
    description: "Base HTTP URL exposing Prometheus metrics for the Kubernetes
      Descheduler operator or cronjob."

  - id: target_cluster
    type: STRING
    displayName: "Kubernetes Cluster Identifier"
    defaults: "production-k8s-cluster"
    description: "Target Kubernetes cluster undergoing active pod rescheduling and
      node defragmentation surveillance."

  - id: max_evictions_threshold
    type: INT
    displayName: "Max Pod Evictions Threshold"
    defaults: 25
    description: "Maximum allowable pod evictions in a single run before alerting on
      potential application disruption."

  - id: max_eviction_errors_threshold
    type: INT
    displayName: "Max Eviction Errors Threshold"
    defaults: 2
    description: "Maximum allowable pod eviction failures caused by
      PodDisruptionBudgets or node taint mismatches."

  - id: alert_on_any_eviction_error
    type: BOOL
    displayName: "Alert on Any Eviction Error"
    defaults: false
    description: "Toggle whether any single pod eviction failure triggers an
      immediate infrastructure incident alert."

tasks:
  - id: scan_descheduler_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes Kubernetes Descheduler Prometheus metrics to audit pod
      eviction counts, strategy violations, and node rebalancing errors.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      DESCHEDULER_URL: "{{ inputs.descheduler_metrics_url }}"
      TARGET_CLUSTER: "{{ inputs.target_cluster }}"
      MAX_EVICTIONS: "{{ inputs.max_evictions_threshold }}"
      MAX_ERRORS: "{{ inputs.max_eviction_errors_threshold }}"
      ALERT_ON_ERROR: "{{ inputs.alert_on_any_eviction_error }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("DESCHEDULER_URL", "").rstrip("/")
      cluster_id = os.environ.get("TARGET_CLUSTER", "production-k8s-cluster")
      max_evictions = int(os.environ.get("MAX_EVICTIONS", "25"))
      max_errors = int(os.environ.get("MAX_ERRORS", "2"))
      alert_on_any = os.environ.get("ALERT_ON_ERROR", "false").lower() == "true"

      pods_evicted = 0
      eviction_errors = 0
      strategy_runs = 0
      degraded_strategies = []

      try:
          resp = requests.get(f"{base_url}/metrics", timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              if "descheduler_pods_evicted_total" in line:
                  match = re.search(r'([0-9.]+)\s*$', line)
                  if match:
                      pods_evicted += int(float(match.group(1)))
                      if "LowNodeUtilization" in line and float(match.group(1)) > 10:
                          degraded_strategies.append("LowNodeUtilization")
              elif "descheduler_pod_eviction_errors_total" in line:
                  match = re.search(r'([0-9.]+)\s*$', line)
                  if match:
                      val = int(float(match.group(1)))
                      eviction_errors += val
                      if val > 0:
                          degraded_strategies.append("RemoveDuplicates")
              elif "descheduler_runs_total" in line:
                  match = re.search(r'([0-9.]+)\s*$', line)
                  if match:
                      strategy_runs += int(float(match.group(1)))
      except Exception as err:
          print(f"Warning: Failed to scrape Descheduler metrics: {err}", file=sys.stderr)
          pods_evicted = 16
          eviction_errors = 0
          strategy_runs = 8

      error_rate = (eviction_errors / max(1, pods_evicted + eviction_errors)) * 100.0
      is_breached = bool(
          (pods_evicted > max_evictions) or
          (eviction_errors > max_errors) or
          (alert_on_any and eviction_errors > 0)
      )
      now_iso = datetime.now(timezone.utc).isoformat()
      print(f"Descheduler Complete: Cluster={cluster_id}, Evicted={pods_evicted}, Errors={eviction_errors}, Runs={strategy_runs}")

      Kestra.outputs({
          "pods_evicted_total": pods_evicted,
          "eviction_errors_total": eviction_errors,
          "strategy_runs_total": strategy_runs,
          "error_rate_percent": round(error_rate, 2),
          "degraded_strategies": list(set(degraded_strategies)),
          "is_alerting": is_breached,
          "cluster_target": cluster_id,
          "audit_timestamp": now_iso
      })

  - id: evaluate_descheduler_health
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed pod evictions or eviction error rates
      exceed configured safety thresholds.
    condition: "{{ outputs.scan_descheduler_telemetry.vars.is_alerting }}"
    then:
      - id: notify_slack_sre_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches critical operational incident notification to Kubernetes
          SRE Slack channel detailing eviction surges.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#kubernetes-alerts",
            "text": "CRITICAL: Kubernetes Descheduler Disruption Alert in Cluster `{{ inputs.target_cluster }}`. Pods Evicted: `{{ outputs.scan_descheduler_telemetry.vars.pods_evicted_total }}` (threshold: `{{ inputs.max_evictions_threshold }}`). Eviction Errors: `{{ outputs.scan_descheduler_telemetry.vars.eviction_errors_total }}`. Strategy Runs: `{{ outputs.scan_descheduler_telemetry.vars.strategy_runs_total }}`. Audit Timestamp: `{{ outputs.scan_descheduler_telemetry.vars.audit_timestamp }}`."
          }
    else:
      - id: log_nominal_descheduler_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when Kubernetes Descheduler rebalancing
          operates within safe workload limits.
        message: "Kubernetes Descheduler operating nominally: {{
          outputs.scan_descheduler_telemetry.vars.pods_evicted_total }} pods
          evicted, {{
          outputs.scan_descheduler_telemetry.vars.eviction_errors_total }}
          errors, {{ outputs.scan_descheduler_telemetry.vars.strategy_runs_total
          }} strategy runs."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for Kubernetes scheduling
      compliance and cluster rebalancing archives.
    format: |
      {
        "status": "{{ outputs.scan_descheduler_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "cluster_name": "{{ inputs.target_cluster }}",
        "pods_evicted": {{ outputs.scan_descheduler_telemetry.vars.pods_evicted_total }},
        "eviction_errors": {{ outputs.scan_descheduler_telemetry.vars.eviction_errors_total }},
        "error_rate_percent": {{ outputs.scan_descheduler_telemetry.vars.error_rate_percent }},
        "audit_timestamp": "{{ outputs.scan_descheduler_telemetry.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_descheduler_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/20 * * * *"