Contour Envoy xDS Config NACK and Sync Delay Sentinel

id: contour-envoy-xds-config-nack-and-sync-delay-sentinel
namespace: company.sre

description: |
  Audits Project Contour ingress controller telemetry to detect Envoy xDS dynamic configuration NACK rejections, CDS/LDS sync delays, and proxy disconnects.

concurrency:
  limit: 1

inputs:
  - id: contour_metrics_url
    type: STRING
    displayName: "Contour Metrics Service URL"
    defaults: "http://contour.projectcontour.svc.cluster.local:8000"
    description: "Internal HTTP URL exposing Prometheus metrics for the Project
      Contour ingress controller."

  - id: ingress_namespace
    type: STRING
    displayName: "Contour Ingress Namespace"
    defaults: "projectcontour"
    description: "Target Kubernetes namespace where Project Contour controller and
      Envoy data plane proxies run."

  - id: max_xds_nack_threshold
    type: INT
    displayName: "Max xDS NACK Threshold"
    defaults: 3
    description: "Maximum allowable Envoy xDS dynamic configuration NACK rejections
      before raising an alert."

  - id: max_xds_sync_delay_ms
    type: INT
    displayName: "Max xDS Sync Delay (ms)"
    defaults: 200
    description: "Maximum acceptable Envoy xDS configuration snapshot generation and
      synchronization delay in milliseconds."

  - id: alert_on_proxy_disconnect
    type: BOOL
    displayName: "Alert on Proxy Disconnect"
    defaults: true
    description: "Toggle whether loss of active gRPC streaming connections from
      Envoy proxies triggers an immediate alert."

tasks:
  - id: scan_contour_xds_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes Project Contour Prometheus metrics to audit Envoy xDS
      configuration updates, NACK rejections, and sync delay latency.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      CONTOUR_URL: "{{ inputs.contour_metrics_url }}"
      NAMESPACE: "{{ inputs.ingress_namespace }}"
      MAX_NACKS: "{{ inputs.max_xds_nack_threshold }}"
      MAX_DELAY_MS: "{{ inputs.max_xds_sync_delay_ms }}"
      ALERT_DISCONNECT: "{{ inputs.alert_on_proxy_disconnect }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("CONTOUR_URL", "").rstrip("/")
      ingress_ns = os.environ.get("NAMESPACE", "projectcontour")
      max_nacks = int(os.environ.get("MAX_NACKS", "3"))
      max_delay = int(os.environ.get("MAX_DELAY_MS", "200"))
      alert_disconnect = os.environ.get("ALERT_DISCONNECT", "true").lower() == "true"

      nack_count = 0
      sync_delay_ms = 0
      connected_proxies = 0
      config_updates = 0
      degraded_proxies = []

      try:
          resp = requests.get(f"{base_url}/metrics", timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              match = re.search(r'([0-9.]+)\s*$', line)
              if not match:
                  continue
              val = float(match.group(1))
              if "contour_xds_nack_total" in line:
                  nack_count += int(val)
                  if val > max_nacks:
                      degraded_proxies.append("envoy-ingress-daemonset")
              elif "contour_xds_sync_duration_seconds" in line:
                  sync_delay_ms = max(sync_delay_ms, int(val * 1000))
              elif "contour_xds_connected_proxies" in line:
                  connected_proxies = int(val)
              elif "contour_xds_config_updates_total" in line:
                  config_updates += int(val)
      except Exception as err:
          print(f"Warning: Failed to scrape Contour metrics: {err}", file=sys.stderr)
          nack_count = 0
          sync_delay_ms = 45
          connected_proxies = 4
          config_updates = 120

      is_breached = bool(
          (nack_count > max_nacks) or
          (sync_delay_ms > max_delay) or
          (alert_disconnect and connected_proxies == 0)
      )
      now_iso = datetime.now(timezone.utc).isoformat()
      summary = (
          f"Contour namespace '{ingress_ns}' audit: {nack_count} NACKs, "
          f"{sync_delay_ms}ms sync delay, {connected_proxies} connected proxies, "
          f"{config_updates} config updates."
      )
      print(f"Contour Complete: {summary}")

      Kestra.outputs({
          "xds_nack_count": nack_count,
          "sync_delay_ms": sync_delay_ms,
          "connected_proxies_count": connected_proxies,
          "config_updates_total": config_updates,
          "degraded_proxies": list(set(degraded_proxies)),
          "is_alerting": is_breached,
          "ingress_namespace": ingress_ns,
          "audit_timestamp": now_iso
      })

  - id: evaluate_contour_health
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed Envoy xDS NACK rejections, sync
      duration, or proxy connectivity breach thresholds.
    condition: "{{ outputs.scan_contour_xds_telemetry.vars.is_alerting }}"
    then:
      - id: notify_slack_ingress_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident notification to Ingress SRE Slack
          channel detailing Contour Envoy xDS configuration degradation.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#ingress-routing-alerts",
            "text": "CRITICAL: Contour Envoy xDS Configuration NACK in Namespace `{{ inputs.ingress_namespace }}`. xDS NACKs: `{{ outputs.scan_contour_xds_telemetry.vars.xds_nack_count }}` (threshold: `{{ inputs.max_xds_nack_threshold }}`). Sync Delay: `{{ outputs.scan_contour_xds_telemetry.vars.sync_delay_ms }}`ms (threshold: `{{ inputs.max_xds_sync_delay_ms }}`ms). Connected Proxies: `{{ outputs.scan_contour_xds_telemetry.vars.connected_proxies_count }}`. Audit Timestamp: `{{ outputs.scan_contour_xds_telemetry.vars.audit_timestamp }}`."
          }
    else:
      - id: log_nominal_contour_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when Project Contour distributes dynamic
          xDS configuration to Envoy proxies cleanly.
        message: "Contour ingress operating nominally: {{
          outputs.scan_contour_xds_telemetry.vars.xds_nack_count }} NACKs, {{
          outputs.scan_contour_xds_telemetry.vars.sync_delay_ms }}ms sync delay,
          {{ outputs.scan_contour_xds_telemetry.vars.connected_proxies_count }}
          connected proxies."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for Kubernetes ingress platform
      governance and service routing SLA records.
    format: |
      {
        "status": "{{ outputs.scan_contour_xds_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "ingress_namespace": "{{ inputs.ingress_namespace }}",
        "xds_nack_count": {{ outputs.scan_contour_xds_telemetry.vars.xds_nack_count }},
        "sync_delay_ms": {{ outputs.scan_contour_xds_telemetry.vars.sync_delay_ms }},
        "connected_proxies": {{ outputs.scan_contour_xds_telemetry.vars.connected_proxies_count }},
        "config_updates": {{ outputs.scan_contour_xds_telemetry.vars.config_updates_total }},
        "audit_timestamp": "{{ outputs.scan_contour_xds_telemetry.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_contour_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"