Kubernetes CNI IPAM Node Pod Subnet Exhaustion Sentinel

id: kubernetes-cni-ipam-subnet-exhaustion-sentinel
namespace: company.ops

inputs:
  - id: node_utilization_threshold_pct
    type: INT
    defaults: 85
    description: "Maximum acceptable node pod allocation percentage before alerting
      on IPAM subnet exhaustion (default 85%)."

tasks:
  - id: audit_node_ipam_capacity
    type: io.kestra.plugin.scripts.shell.Commands
    containerImage: bitnami/kubectl:latest
    commands:
      - |
        kubectl get nodes -o json > /tmp/nodes.json

        TOTAL_NODES=$(jq '.items | length // 0' /tmp/nodes.json)

        cat << 'EOF' > /tmp/check_capacity.py
        import json, sys
        with open('/tmp/nodes.json') as f:
            data = json.load(f)

        exhausted = 0
        for node in data.get('items', []):
            name = node['metadata']['name']
            allocatable_pods = int(node['status']['allocatable'].get('pods', 110))
            if allocatable_pods == 0:
                continue
            # Query active pods on this node
            import subprocess
            res = subprocess.run(['kubectl', 'get', 'pods', '-A', '--field-selector', f'spec.nodeName={name}', '-o', 'json'], capture_output=True, text=True)
            if res.returncode == 0:
                pods_data = json.loads(res.stdout)
                active_pods = len(pods_data.get('items', []))
                pct = (active_pods * 100) / allocatable_pods
                if pct >= float(sys.argv[1]):
                    exhausted += 1
        print(exhausted)
        EOF

        EXHAUSTED_NODES=$(python3 /tmp/check_capacity.py "{{ inputs.node_utilization_threshold_pct }}" 2>/dev/null || echo 0)

        echo "::set-output name=total_nodes::$TOTAL_NODES::"
        echo "::set-output name=exhausted_nodes::$EXHAUSTED_NODES::"

  - id: evaluate_ipam_exhaustion
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.audit_node_ipam_capacity.vars.exhausted_nodes | int > 0 }}"
    then:
      - id: notify_platform_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#kubernetes-alerts",
            "attachments": [
              {
                "color": "danger",
                "title": "🚨 Kubernetes CNI IPAM Pod Subnet Exhaustion Alert",
                "text": "Cluster recorded *{{ outputs.audit_node_ipam_capacity.vars.exhausted_nodes }}* worker nodes (out of *{{ outputs.audit_node_ipam_capacity.vars.total_nodes }}*) exceeding {{ inputs.node_utilization_threshold_pct }}% pod allocation! Impact: CNI IPAM secondary subnets are exhausted, causing incoming pods to stall indefinitely in `Pending` state with `FailedCreatePodSandBox` IP allocation errors.",
                "footer": "Kestra Kubernetes Sentinel • CNI IPAM"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#kubernetes-alerts",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ Kubernetes CNI IPAM Node Pod Subnet Exhaustion Sentinel Pipeline Failure",
            "text": "Failed to audit Kubernetes node IPAM capacity. Check cluster RBAC permissions.",
            "footer": "Kestra Workflow Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"
    disabled: true