Kubernetes CoreDNS Request Latency Spike and Upstream RTT Sentinel

id: coredns-dns-request-duration-latency-spike-sentinel
namespace: company.networking

inputs:
  - id: coredns_metrics_url
    type: STRING
    defaults: "http://coredns.kube-system:9153/metrics"
    description: Prometheus scrape endpoint for Kubernetes CoreDNS.

tasks:
  - id: scrape_coredns_metrics
    type: io.kestra.plugin.scripts.shell.Commands
    commands:
      - |
        curl -s -f "{{ inputs.coredns_metrics_url }}" > /tmp/coredns_telemetry.txt

        # Scrape coredns_dns_request_duration_seconds_sum and count to compute average latency
        SUM=$(grep -E "^coredns_dns_request_duration_seconds_sum" /tmp/coredns_telemetry.txt | awk '{sum+=$2} END {print sum+0}')
        COUNT=$(grep -E "^coredns_dns_request_duration_seconds_count" /tmp/coredns_telemetry.txt | awk '{sum+=$2} END {print sum+0}')

        AVG_LATENCY=$(awk -v sum="$SUM" -v count="$COUNT" 'BEGIN {if (count>0) print sum/count; else print 0}')
        echo "::set-output name=avg_latency::$AVG_LATENCY::"

  - id: evaluate_dns_latency
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.scrape_coredns_metrics.vars.avg_latency | float > 0.050 }}"
    then:
      - id: notify_platform_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#networking-alerts",
            "attachments": [
              {
                "color": "danger",
                "title": "🚨 CoreDNS Elevated Request Latency Alert",
                "text": "CoreDNS at `{{ inputs.coredns_metrics_url }}` recorded an average DNS request duration of *{{ outputs.scrape_coredns_metrics.vars.avg_latency }} seconds* (> 50ms)! Cluster-wide microservice inter-communication is experiencing DNS resolution stalls.",
                "footer": "Kestra Networking Sentinel • CoreDNS"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#networking-alerts",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ CoreDNS Sentinel Failure",
            "text": "Failed to scrape CoreDNS metrics endpoint. Verify kube-system network access.",
            "footer": "Kestra Workflow Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"
    disabled: true