Apache APISIX Route Health & Upstream Drift Sentinel

id: apisix-route-health-and-upstream-drift-sentinel
namespace: company.team
description: |
  Audits Apache APISIX API Gateway instances on a schedule by probing Control
  API health and verifying canary route status across the Admin API, detecting
  route deletion, unauthorized disabling, or configuration drift, and dispatching
  actionable Slack alerts or safe remediation re-provisions.

triggers:
  - id: periodic_gateway_audit
    type: io.kestra.plugin.core.trigger.Schedule
    description: "Periodically probes APISIX control plane and canary route
      integrity. Shipped disabled by default; enable once apisix_host and admin
      credentials point to a running APISIX instance."
    cron: "*/15 * * * *"
    disabled: true
    inputs:
      apisix_host: "localhost"
      apisix_control_port: 9092
      apisix_admin_port: 9180
      canary_route_id: "canary_health_route"
      canary_upstream_target: "127.0.0.1:8080"
      auto_restore_route: "false"
      dry_run: "true"

inputs:
  - id: apisix_host
    type: STRING
    displayName: "APISIX Gateway Host"
    defaults: "localhost"
    description: "Hostname or IP address of the Apache APISIX API Gateway instance."

  - id: apisix_control_port
    type: INT
    displayName: "Control API Port"
    defaults: 9092
    description: "Port exposed by the APISIX Control API for health checks (default: 9092)."

  - id: apisix_admin_port
    type: INT
    displayName: "Admin API Port"
    defaults: 9180
    description: "Port exposed by the APISIX Admin API for route and upstream
      management (default: 9180)."

  - id: canary_route_id
    type: STRING
    displayName: "Canary Route ID"
    defaults: "canary_health_route"
    description: "Identifier of the canary route monitored for drift and
      configuration state."

  - id: canary_upstream_target
    type: STRING
    displayName: "Canary Upstream Target"
    defaults: "127.0.0.1:8080"
    description: "Upstream host and port configured for the canary verification route."

  - id: auto_restore_route
    type: STRING
    displayName: "Auto-Restore Canary Route"
    defaults: "false"
    description: "Set to 'true' to automatically re-provision or re-enable the
      canary route when missing or disabled."

  - id: dry_run
    type: STRING
    displayName: "Dry Run Mode"
    defaults: "true"
    description: "When 'true', alerts on drift and logs intended remediations
      without altering APISIX route tables."

tasks:
  - id: probe_apisix_control_health
    type: io.kestra.plugin.core.http.Request
    description: "Queries APISIX Control API GET /v1/healthcheck endpoint to verify
      gateway process readiness and health."
    uri: "http://{{ inputs.apisix_host }}:{{ inputs.apisix_control_port
      }}/v1/healthcheck"
    method: GET
    headers:
      Accept: "application/json"
    options:
      allowFailed: true

  - id: probe_canary_route_status
    type: io.kestra.plugin.core.http.Request
    description: "Queries APISIX Admin API GET /apisix/admin/routes/{id} to audit
      canary route existence and enabled status."
    uri: "http://{{ inputs.apisix_host }}:{{ inputs.apisix_admin_port
      }}/apisix/admin/routes/{{ inputs.canary_route_id }}"
    method: GET
    headers:
      X-API-KEY: "{{ secret('APISIX_ADMIN_KEY') }}"
      Accept: "application/json"
    options:
      allowFailed: true

  - id: classify_gateway_verdict
    type: io.kestra.plugin.core.flow.Switch
    description: "Evaluates Control API health and Admin API route metadata to
      classify gateway operational state."
    value: >-
      {% if outputs.probe_apisix_control_health.status != 200 or
      (fromJson(outputs.probe_apisix_control_health.body).status ?? '') != 'ok'
      %} CONTROL_PLANE_UNHEALTHY {% elif
      outputs.probe_canary_route_status.status == 404 %} CANARY_ROUTE_MISSING {%
      elif (fromJson(outputs.probe_canary_route_status.body).value.status ?? 1)
      == 0 %} CANARY_ROUTE_DISABLED {% else %} HEALTHY {% endif %}
    cases:
      CONTROL_PLANE_UNHEALTHY:
        - id: alert_control_plane_unhealthy
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          description: "Dispatches high-severity alert when APISIX Control API reports
            degraded or unresponsive status."
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "channel": "#gateway-alerts",
              "text": ":rotating_light: *Apache APISIX Control Plane Unhealthy*",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": ":rotating_light: *APISIX Control Plane Unhealthy*\n*Host:* `{{ inputs.apisix_host }}:{{ inputs.apisix_control_port }}`\n*HTTP Status:* `{{ outputs.probe_apisix_control_health.status }}`\n*Reported Status:* `{{ fromJson(outputs.probe_apisix_control_health.body).status ?? 'UNKNOWN' }}`\nGateway data plane may fail to process traffic or reload configuration."
                  }
                }
              ]
            }

      CANARY_ROUTE_MISSING:
        - id: alert_canary_route_missing
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          description: "Dispatches alert when monitored canary route is absent from the
            APISIX route table."
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "channel": "#gateway-alerts",
              "text": ":warning: *Apache APISIX Canary Route Missing*",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": ":warning: *Canary Route Missing from Route Table*\n*Host:* `{{ inputs.apisix_host }}:{{ inputs.apisix_admin_port }}`\n*Route ID:* `{{ inputs.canary_route_id }}`\n*HTTP Status:* `{{ outputs.probe_canary_route_status.status }}`\nThe route was deleted or never synchronized. Ingress traffic for this endpoint will return 404."
                  }
                }
              ]
            }

        - id: evaluate_remediation_route_missing
          type: io.kestra.plugin.core.flow.If
          description: "Evaluates safety gates before re-provisioning missing canary route
            via Admin API."
          condition: "{{ inputs.auto_restore_route == 'true' and inputs.dry_run == 'false'
            }}"
          then:
            - id: restore_canary_route
              type: io.kestra.plugin.core.http.Request
              description: "Issues PUT /apisix/admin/routes/{id} to re-create the missing
                canary route with baseline upstream."
              uri: "http://{{ inputs.apisix_host }}:{{ inputs.apisix_admin_port
                }}/apisix/admin/routes/{{ inputs.canary_route_id }}"
              method: PUT
              headers:
                X-API-KEY: "{{ secret('APISIX_ADMIN_KEY') }}"
                Content-Type: "application/json"
              body: |
                {
                  "name": "{{ inputs.canary_route_id }}",
                  "uri": "/healthz",
                  "status": 1,
                  "upstream": {
                    "type": "roundrobin",
                    "nodes": {
                      "{{ inputs.canary_upstream_target }}": 1
                    }
                  }
                }
            - id: log_restore_missing
              type: io.kestra.plugin.core.log.Log
              message: "Re-provisioned missing canary route '{{ inputs.canary_route_id }}' on
                APISIX Admin API."
          else:
            - id: log_dry_run_missing
              type: io.kestra.plugin.core.log.Log
              message: "Dry-run active or auto_restore_route disabled. Suppressed automated
                route creation."

      CANARY_ROUTE_DISABLED:
        - id: alert_canary_route_disabled
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          description: "Dispatches alert when canary route exists but is administratively
            disabled (status=0)."
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "channel": "#gateway-alerts",
              "text": ":warning: *Apache APISIX Canary Route Disabled*",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": ":warning: *Canary Route Disabled (Status 0)*\n*Host:* `{{ inputs.apisix_host }}:{{ inputs.apisix_admin_port }}`\n*Route ID:* `{{ inputs.canary_route_id }}`\nThe route status is 0 (disabled). Ingress traffic is being dropped or bypassed."
                  }
                }
              ]
            }

        - id: evaluate_remediation_route_disabled
          type: io.kestra.plugin.core.flow.If
          description: "Evaluates safety gates before re-enabling disabled canary route
            via Admin API."
          condition: "{{ inputs.auto_restore_route == 'true' and inputs.dry_run == 'false'
            }}"
          then:
            - id: enable_canary_route
              type: io.kestra.plugin.core.http.Request
              description: "Issues PATCH /apisix/admin/routes/{id} to re-enable the route
                (status=1)."
              uri: "http://{{ inputs.apisix_host }}:{{ inputs.apisix_admin_port
                }}/apisix/admin/routes/{{ inputs.canary_route_id }}"
              method: PATCH
              headers:
                X-API-KEY: "{{ secret('APISIX_ADMIN_KEY') }}"
                Content-Type: "application/json"
              body: |
                {
                  "status": 1
                }
            - id: log_enable_route
              type: io.kestra.plugin.core.log.Log
              message: "Re-enabled canary route '{{ inputs.canary_route_id }}' (status=1) on
                APISIX Admin API."
          else:
            - id: log_dry_run_disabled
              type: io.kestra.plugin.core.log.Log
              message: "Dry-run active or auto_restore_route disabled. Suppressed automated
                route enablement."

      HEALTHY:
        - id: log_gateway_healthy
          type: io.kestra.plugin.core.log.Log
          message: "APISIX Gateway and canary route '{{ inputs.canary_route_id }}' are
            healthy (control_status=ok, route_status=enabled)."

  - id: log_audit_summary
    type: io.kestra.plugin.core.log.Log
    description: "Records execution audit lineage and parameter state across gateway
      verification steps."
    message: "APISIX Sentinel Audit Complete: Host={{ inputs.apisix_host }},
      ControlPort={{ inputs.apisix_control_port }}, AdminPort={{
      inputs.apisix_admin_port }}, CanaryRoute={{ inputs.canary_route_id }},
      DryRun={{ inputs.dry_run }}."

errors:
  - id: alert_connectivity_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: "Dispatches critical outage alert when network connection to APISIX
      endpoints fails completely."
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "channel": "#gateway-alerts",
        "text": ":rotating_light: *Apache APISIX Gateway Connection Failure*",
        "blocks": [
          {
            "type": "section",
            "text": {
              "type": "mrkdwn",
              "text": ":rotating_light: *APISIX Ingress Unreachable*\n*Target Host:* `{{ inputs.apisix_host }}`\n*Control Port:* `{{ inputs.apisix_control_port }}`\n*Admin Port:* `{{ inputs.apisix_admin_port }}`\n*Execution ID:* `{{ execution.id }}`\nKestra was unable to establish an HTTP connection. Check container status, network routes, or firewall rules."
            }
          }
        ]
      }