Kubernetes Pod CrashLoopBackOff Sentinel

id: kubernetes-pod-crashloopbackoff-sentinel
namespace: company.team
description: |
  Automated Kubernetes workload reliability sentinel. Audits cluster namespaces for pods trapped in
  CrashLoopBackOff or OOMKilled states, isolates container termination exit codes, and alerts SREs on Slack.

concurrency:
  limit: 1

inputs:
  - id: target_namespaces
    type: STRING
    displayName: Target Namespaces
    defaults: "default,production"
    description: "Comma-separated list of Kubernetes namespaces to inspect for
      unhealthy pods."

  - id: max_restart_threshold
    type: INT
    displayName: Restart Count Threshold
    defaults: 3
    description: "Minimum restart count required to flag a container as crash-looping."

  - id: slack_channel
    type: STRING
    displayName: Slack Channel
    defaults: "#sre-alerts"
    description: "Slack channel destination for Kubernetes workload failure alerts."

triggers:
  - id: scheduled_pod_audit
    type: io.kestra.plugin.core.trigger.Schedule
    description: "Audits cluster pod states every 15 minutes."
    cron: "*/15 * * * *"

tasks:
  - id: scan_unhealthy_pods
    type: io.kestra.plugin.scripts.python.Script
    description: "Queries the Kubernetes cluster API to discover pods in
      CrashLoopBackOff, OOMKilled, or Error states."
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    outputFiles:
      - unhealthy_pods_report.json
    script: |
      import os
      import json
      from kestra import Kestra

      # Simulated or real Kubernetes client invocation
      namespaces = [ns.strip() for ns in "{{ inputs.target_namespaces }}".split(",") if ns.strip()]
      restart_limit = int("{{ inputs.max_restart_threshold }}")

      # Pod diagnostic parser
      unhealthy_pods = []

      # In production, uses kubernetes python client with in-cluster config or kubeconfig
      # Here we simulate structured discovery across specified namespaces
      mock_candidates = [
          {
              "namespace": "production",
              "pod_name": "checkout-service-7f8d9b-x2k41",
              "container_name": "checkout-api",
              "state_reason": "CrashLoopBackOff",
              "last_exit_code": 137,
              "termination_reason": "OOMKilled",
              "restart_count": 8,
              "node_name": "ip-10-0-12-84.ec2.internal"
          }
      ]

      for candidate in mock_candidates:
          if candidate["namespace"] in namespaces and candidate["restart_count"] >= restart_limit:
              unhealthy_pods.append(candidate)

      has_unhealthy = len(unhealthy_pods) > 0
      top_pod = unhealthy_pods[0]["pod_name"] if unhealthy_pods else "none"
      top_reason = unhealthy_pods[0]["termination_reason"] if unhealthy_pods else "none"
      top_restarts = unhealthy_pods[0]["restart_count"] if unhealthy_pods else 0

      report = {
          "total_scanned_namespaces": len(namespaces),
          "unhealthy_pods_count": len(unhealthy_pods),
          "unhealthy_pods": unhealthy_pods
      }

      with open("unhealthy_pods_report.json", "w") as f:
          json.dump(report, f, indent=2)

      Kestra.outputs({
          "has_unhealthy_pods": has_unhealthy,
          "unhealthy_count": len(unhealthy_pods),
          "top_pod_name": top_pod,
          "top_termination_reason": top_reason,
          "top_restart_count": top_restarts
      })

  - id: check_unhealthy_pods
    type: io.kestra.plugin.core.flow.If
    description: "Evaluates whether any pods are trapped in crash-looping or
      OOMKilled states."
    condition: "{{ outputs.scan_unhealthy_pods.vars.has_unhealthy_pods }}"
    then:
      - id: notify_slack_sre
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: "Dispatches actionable triage card to SRE team with termination
          cause and remediation steps."
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "{{ inputs.slack_channel }}",
            "text": "*CRITICAL: Kubernetes Pod CrashLoopBackOff Detected*\n• *Impacted Pod*: `{{ outputs.scan_unhealthy_pods.vars.top_pod_name }}`\n• *Failure Reason*: {{ outputs.scan_unhealthy_pods.vars.top_termination_reason }}\n• *Restart Count*: {{ outputs.scan_unhealthy_pods.vars.top_restart_count }} restarts (Threshold: {{ inputs.max_restart_threshold }})\n• *Operational Impact*: Workload is unable to achieve healthy readiness status, degrading replica set availability and request processing.\n• *Recommended Remediation*: Inspect container memory limits in Deployment manifest if OOMKilled (exit code 137), or review application startup logs with `kubectl logs {{ outputs.scan_unhealthy_pods.vars.top_pod_name }} --previous`."
          }

    else:
      - id: log_healthy_cluster
        type: io.kestra.plugin.core.log.Log
        description: "Logs nominal status confirmation across audited namespaces."
        message: "Kubernetes workload audit compliant. Zero pods in CrashLoopBackOff or
          OOMKilled states across namespaces: {{ inputs.target_namespaces }}."

  - id: export_sentinel_manifest
    type: io.kestra.plugin.core.debug.Return
    description: "Outputs structured JSON compliance manifest for SRE operational logs."
    format: |
      {
        "pipeline": "kubernetes-pod-crashloopbackoff-sentinel",
        "audited_at": "{{ execution.startDate }}",
        "unhealthy_pods_count": {{ outputs.scan_unhealthy_pods.vars.unhealthy_count }},
        "status": "{{ outputs.scan_unhealthy_pods.vars.has_unhealthy_pods ? 'ACTION_REQUIRED' : 'HEALTHY' }}"
      }