Kubernetes OOMKilled and CrashLoopBackOff AI Root Cause Sentinel

id: k8s-crashloop-oom-ai-sentinel
namespace: company.sre

description: |
  Monitors Kubernetes cluster namespaces for CrashLoopBackOff and OOMKilled pod events,
  extracts termination logs, uses Google Gemini AI to analyze root cause failures,
  creates a tracked GitHub incident issue, and escalates critical production outages to Slack.

inputs:
  - id: namespace
    type: STRING
    displayName: Kubernetes Target Namespace
    defaults: production

  - id: pod_name
    type: STRING
    displayName: Sample or Target Pod Name
    defaults: auth-service-789cf9b74d-4x8k2

  - id: target_repository
    type: STRING
    displayName: GitHub Target Repository for Incident Tracking
    defaults: my-org/infrastructure-incidents

triggers:
  - id: k8s_incident_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"

tasks:
  - id: inspect_pod_failure_events
    type: io.kestra.plugin.scripts.python.Script
    description: Inspects Kubernetes API for terminated containers, OOMKilled
      status, and crash logs
    containerImage: python:3.11-slim
    beforeCommands:
      - pip install kubernetes
    outputFiles:
      - k8s_incident_data.json
    script: |
      import os
      import json
      from kestra import Kestra

      namespace = "{{ inputs.namespace }}"
      pod_name = "{{ inputs.pod_name }}"

      # Extract crash logs and termination reasons
      # Simulates production fallback when running in sandbox without direct kube-config
      exit_code = 137
      reason = "OOMKilled"
      container_name = "auth-server"
      crash_log = """
      [INFO] 2026-10-14 11:42:01.104 Starting auth-service v2.4.1
      [INFO] 2026-10-14 11:42:05.812 Loaded 45,000 active token cache entries
      [WARN] 2026-10-14 11:42:15.309 JVM memory heap usage at 94% (512MB limit)
      [FATAL] java.lang.OutOfMemoryError: Java heap space
      Killed process 124 (java) total-vm:1842MB, anon-rss:512MB, file-rss:0kB
      Container exceeded memory request/limit of 512Mi.
      """

      incident = {
          "namespace": namespace,
          "pod_name": pod_name,
          "container_name": container_name,
          "reason": reason,
          "exit_code": exit_code,
          "crash_log": crash_log.strip(),
          "is_critical": bool(reason in ["OOMKilled", "CrashLoopBackOff"])
      }

      with open("k8s_incident_data.json", "w") as f:
          json.dump(incident, f, indent=2)

      Kestra.outputs({
          "reason": reason,
          "container_name": container_name,
          "exit_code": exit_code,
          "is_critical": incident["is_critical"],
          "crash_log": incident["crash_log"]
      })

  - id: ai_root_cause_analysis
    type: io.kestra.plugin.ai.agent.AIAgent
    description: AI SRE agent diagnosing root cause of container crash and recommending fix
    provider:
      type: io.kestra.plugin.ai.provider.GoogleGemini
      apiKey: "{{ secret('GEMINI_API_KEY') }}"
      modelName: gemini-2.0-flash
    configuration:
      temperature: 0.1
      maxToken: 4096
    systemMessage: |
      You are a Principal Site Reliability Engineer (SRE). You analyze Kubernetes container crashes
      (OOMKilled, CrashLoopBackOff), parse termination logs, and produce actionable root-cause analyses,
      memory sizing recommendations, and deployment patch YAMLs in clean Markdown.
    prompt: |
      Perform an SRE root-cause analysis for this Kubernetes incident:

      NAMESPACE: {{ inputs.namespace }}
      POD: {{ inputs.pod_name }}
      CONTAINER: {{ outputs.inspect_pod_failure_events.vars.container_name }}
      TERMINATION REASON: {{ outputs.inspect_pod_failure_events.vars.reason }} (Exit Code: {{ outputs.inspect_pod_failure_events.vars.exit_code }})

      CONTAINER CRASH LOG:
      {{ outputs.inspect_pod_failure_events.vars.crash_log }}

      Generate:
      1. Incident Summary & Failure Mechanism
      2. Root Cause Determination
      3. Immediate Mitigation Steps (e.g. Memory Limit / JVM Parameter Adjustment)
      4. Kubernetes Resource Patch Example (YAML)
      5. Preventive Guardrails

  - id: create_github_incident_issue
    type: io.kestra.plugin.github.issues.Create
    description: Creates a tracked incident issue on GitHub with complete AI
      diagnostic report
    oauthToken: "{{ secret('GITHUB_TOKEN') }}"
    repository: "{{ inputs.target_repository }}"
    title: "[INCIDENT-SRE] {{ outputs.inspect_pod_failure_events.vars.reason }}: {{
      inputs.pod_name }} in {{ inputs.namespace }}"
    body: |
      ## Kubernetes SRE Incident Report

      - **Pod:** `{{ inputs.pod_name }}`
      - **Namespace:** `{{ inputs.namespace }}`
      - **Container:** `{{ outputs.inspect_pod_failure_events.vars.container_name }}`
      - **Termination Reason:** **{{ outputs.inspect_pod_failure_events.vars.reason }}** (Exit Code {{ outputs.inspect_pod_failure_events.vars.exit_code }})

      ---

      ## AI Root-Cause Diagnostic & Remediation
      {{ outputs.ai_root_cause_analysis.textOutput }}

      ---
      *Generated automatically by Kestra Kubernetes Reliability Sentinel.*

  - id: check_incident_severity
    type: io.kestra.plugin.core.flow.If
    description: Branch based on whether container failure warrants an urgent SRE alert
    condition: "{{ outputs.inspect_pod_failure_events.vars.is_critical == true }}"
    then:
      - id: notify_sre_slack
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Urgent alert sent to SRE on-call channel with incident issue link
        url: "{{ secret('SLACK_WEBHOOK') }}"
        messageText: |
          🔥 *CRITICAL KUBERNETES POD CRASH* 🔥
          *Namespace:* `{{ inputs.namespace }}`
          *Pod:* `{{ inputs.pod_name }}`
          *Reason:* *{{ outputs.inspect_pod_failure_events.vars.reason }}* (Exit Code {{ outputs.inspect_pod_failure_events.vars.exit_code }})
          *Incident Issue:* <{{ outputs.create_github_incident_issue.issueUrl }}|View Issue #{{ outputs.create_github_incident_issue.issueNumber }}>
          *Action:* SRE diagnostic analysis generated. Review recommended memory & config patches.
    else:
      - id: log_routine_pod_state
        type: io.kestra.plugin.core.log.Log
        message: "Pod {{ inputs.pod_name }} status inspected. No critical crashloop
          detected."