Kubernetes Headless Service Orphaned Endpoints and DNS Resolution Sentinel

id: kubernetes-orphaned-headless-endpoints-sentinel
namespace: company.sre

inputs:
  - id: kube_context
    type: STRING
    required: false
    description: Optional Kubernetes context name.
  - id: orphaned_threshold
    type: INT
    defaults: 0
    description: Maximum acceptable count of headless services with zero ready
      backend endpoints.

tasks:
  - id: audit_headless_endpoints
    type: io.kestra.plugin.scripts.shell.Commands
    containerImage: bitnami/kubectl:latest
    commands:
      - |
        CONTEXT_ARG=""
        if [ -n "{{ inputs.kube_context }}" ]; then
          CONTEXT_ARG="--context={{ inputs.kube_context }}"
        fi

        kubectl $CONTEXT_ARG get services -A -o json > /tmp/services.json
        kubectl $CONTEXT_ARG get endpoints -A -o json > /tmp/endpoints.json

        HEADLESS_SERVICES=$(jq '[.items[] | select(.spec.clusterIP == "None" and (.spec.selector | length > 0)) | {namespace: .metadata.namespace, name: .metadata.name}]' /tmp/services.json)

        ORPHANED_LIST=$(jq -n --argjson svc "$HEADLESS_SERVICES" --slurpfile ep /tmp/endpoints.json '
          [ $svc[] | . as $s | 
            ($ep[0].items[] | select(.metadata.namespace == $s.namespace and .metadata.name == $s.name)) as $e |
            select(($e.subsets // []) | map(.addresses // [] | length) | add // 0 == 0) |
            {namespace: $s.namespace, service: $s.name, reason: "Zero ready endpoint addresses"}
          ]
        ')

        ORPHANED_COUNT=$(echo "$ORPHANED_LIST" | jq 'length')

        echo "::set-output name=orphaned_count::$ORPHANED_COUNT::"
        echo "::set-output name=orphaned_details::$ORPHANED_LIST::"

  - id: evaluate_endpoint_health
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.audit_headless_endpoints.vars.orphaned_count | int >
      inputs.orphaned_threshold }}"
    then:
      - id: notify_network_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#infrastructure-alerts",
            "attachments": [
              {
                "color": "warning",
                "title": "⚠️ Kubernetes Orphaned Headless Service Endpoints Alert",
                "text": "Detected *{{ outputs.audit_headless_endpoints.vars.orphaned_count }}* headless Service(s) with *zero ready backend endpoints*. Orphaned services: {{ outputs.audit_headless_endpoints.vars.orphaned_details }}. Impact: DNS lookups for these headless services will return NXDOMAIN or empty SRV/A records, breaking internal RPC routing.",
                "footer": "Kestra SRE Orchestrator • Endpoints Sentinel"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#infrastructure-alerts",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ Headless Endpoints Sentinel Execution Failed",
            "text": "The Kubernetes headless service audit encountered an execution error. Please check cluster RBAC permissions for services and endpoints.",
            "footer": "Kestra Workflow Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/30 * * * *"
    disabled: true