Kubernetes OOMKilled Pod Termination Storm Sentinel

id: kubernetes-event-exporter-oom-storm-sentinel
namespace: company.ops

inputs:
  - id: namespace
    type: STRING
    defaults: "default"
    description: Kubernetes namespace to audit for OOMKilled pod events.
  - id: oom_event_threshold
    type: INT
    defaults: 5
    description: Maximum acceptable count of OOMKilled termination events in the
      last 15 minutes.

tasks:
  - id: query_oom_events
    type: io.kestra.plugin.scripts.shell.Commands
    containerImage: bitnami/kubectl:latest
    commands:
      - |
        kubectl get events -n "{{ inputs.namespace }}" --field-selector reason=OOMKilled -o json > /tmp/oom_events.json

        OOM_COUNT=$(jq '.items | length // 0' /tmp/oom_events.json)

        echo "::set-output name=oom_count::$OOM_COUNT::"

  - id: evaluate_oom_storm
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.query_oom_events.vars.oom_count | int >=
      inputs.oom_event_threshold }}"
    then:
      - id: notify_platform_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#kubernetes-alerts",
            "attachments": [
              {
                "color": "danger",
                "title": "🚨 Kubernetes OOMKilled Pod Termination Storm Alert",
                "text": "Namespace `{{ inputs.namespace }}` recorded *{{ outputs.query_oom_events.vars.oom_count }}* container terminations due to OOMKilled! Impact: Pods are being terminated by the Linux kernel cgroup OOM killer due to memory exhaustion, triggering CrashLoopBackOff cascades and customer service downtime.",
                "footer": "Kestra Kubernetes Sentinel • OOMKilled"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#kubernetes-alerts",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ Kubernetes OOMKilled Pod Termination Storm Sentinel Pipeline Failure",
            "text": "Failed to query Kubernetes events for OOMKilled terminations. Verify cluster RBAC permissions.",
            "footer": "Kestra Workflow Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"
    disabled: true