Istio Service Mesh Control Plane xDS Push Error Sentinel

id: istio-service-mesh-pilot-sync-failure-sentinel
namespace: company.mesh

inputs:
  - id: istiod_metrics_url
    type: STRING
    defaults: "http://istiod.istio-system:15014/metrics"
    description: Prometheus scrape endpoint for Istio Pilot control plane.
  - id: max_xds_errors
    type: INT
    defaults: 0
    description: "Maximum allowable xDS push errors before alerting (default 0)."

tasks:
  - id: scrape_istiod_metrics
    type: io.kestra.plugin.scripts.shell.Commands
    commands:
      - |
        curl -s -f "{{ inputs.istiod_metrics_url }}" > /tmp/istio_telemetry.txt

        # Scrape pilot_xds_push_errors counter
        XDS_ERRORS=$(grep -E "^pilot_xds_push_errors" /tmp/istio_telemetry.txt | awk '{sum+=$2} END {print sum+0}')
        echo "::set-output name=xds_errors::$XDS_ERRORS::"

  - id: evaluate_xds_errors
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.scrape_istiod_metrics.vars.xds_errors | int >
      inputs.max_xds_errors }}"
    then:
      - id: notify_mesh_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#service-mesh-alerts",
            "attachments": [
              {
                "color": "danger",
                "title": "🚨 Istio Control Plane xDS Push Error Alert",
                "text": "Istio Pilot at `{{ inputs.istiod_metrics_url }}` recorded *{{ outputs.scrape_istiod_metrics.vars.xds_errors }} xDS configuration push errors*! Sidecar Envoy proxies are failing to receive dynamic routing, mTLS certificates, or cluster endpoints.",
                "footer": "Kestra Service Mesh Sentinel • Istio"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#service-mesh-alerts",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ Istio Sentinel Failure",
            "text": "Failed to scrape Istio Pilot metrics endpoint. Verify istiod service reachability.",
            "footer": "Kestra Workflow Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"
    disabled: true