Caddy Reverse Proxy Upstream Health and Error Rate Sentinel

id: caddy-reverse-proxy-upstream-health-sentinel
namespace: company.infrastructure

inputs:
  - id: caddy_admin_endpoint
    type: STRING
    displayName: Caddy Admin API Endpoint
    description: Base URL of the Caddy server admin API endpoint.
    defaults: "http://localhost:2019"

  - id: max_unhealthy_threshold_percent
    type: INT
    displayName: Maximum Unhealthy Upstream Threshold Percent
    description: Percentage of unhealthy upstream backends allowed before triggering
      a critical incident alert.
    defaults: 20

  - id: send_slack_alert
    type: BOOL
    displayName: Dispatch Slack Incident Alert
    description: Whether to transmit real-time Slack notifications upon reverse
      proxy upstream backend degradation.
    defaults: true

triggers:
  - id: caddy_health_audit_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    description: Recurring audit polling Caddy reverse proxy upstream health every 5
      minutes.
    cron: "*/5 * * * *"
    disabled: true

tasks:
  - id: probe_caddy_upstreams
    type: io.kestra.plugin.core.http.Request
    description: Query Caddy Admin API reverse proxy upstream pool status.
    uri: "{{ inputs.caddy_admin_endpoint }}/reverse_proxy/upstreams"
    method: GET
    headers:
      Accept: "application/json"
    options:
      allowFailed: true
      timeout:
        connectTimeout: PT10S
        readIdleTimeout: PT30S
    retry:
      type: constant
      interval: PT5S
      maxAttempts: 3
    allowFailure: true

  - id: evaluate_upstream_health
    type: io.kestra.plugin.scripts.python.Script
    description: Interrogate upstream pool JSON response, calculate degradation
      percentage, and classify health status.
    taskRunner:
      type: io.kestra.plugin.core.runner.Process
    inputFiles:
      upstreams.json: "{{ outputs.probe_caddy_upstreams.body }}"
    script: |
      import json

      max_unhealthy_pct = float({{ inputs.max_unhealthy_threshold_percent }})

      try:
          with open("upstreams.json", "r") as f:
              raw = f.read().strip()
              data = json.loads(raw) if raw else []
      except Exception as e:
          data = [{"address": "CADDY_API_UNREACHABLE", "status": "unhealthy", "fails": 1}]

      if not isinstance(data, list):
          data = []

      total_upstreams = len(data)
      unhealthy_hosts = []

      for item in data:
          status = str(item.get("status", "healthy")).lower()
          fails = int(item.get("fails", 0))
          if status == "unhealthy" or fails > 0:
              unhealthy_hosts.append(item.get("address", "unknown"))

      unhealthy_count = len(unhealthy_hosts)
      unhealthy_pct = (unhealthy_count / total_upstreams * 100.0) if total_upstreams > 0 else 0.0

      if total_upstreams == 0 or unhealthy_count == 0:
          classification = "ALL_UPSTREAMS_HEALTHY"
      elif unhealthy_pct <= max_unhealthy_pct:
          classification = "PARTIAL_BACKEND_DEGRADATION"
      else:
          classification = "CRITICAL_UPSTREAM_OUTAGE"

      unhealthy_summary = ", ".join(unhealthy_hosts[:5]) if unhealthy_hosts else "none"

      output_payload = {
          "outputs": {
              "total_upstreams": total_upstreams,
              "unhealthy_count": unhealthy_count,
              "unhealthy_percent": round(unhealthy_pct, 2),
              "unhealthy_hosts": unhealthy_summary,
              "classification": classification
          }
      }
      print(f"::{json.dumps(output_payload)}::")

  - id: classify_pool_status
    type: io.kestra.plugin.core.log.Log
    description: Log evaluated Caddy reverse proxy upstream pool metrics and status
      classification.
    message: |
      Caddy Endpoint: {{ inputs.caddy_admin_endpoint }}
      Total Upstreams: {{ outputs.evaluate_upstream_health.vars.total_upstreams }}
      Unhealthy Upstreams: {{ outputs.evaluate_upstream_health.vars.unhealthy_count }} ({{ outputs.evaluate_upstream_health.vars.unhealthy_percent }}%)
      Failing Hosts: {{ outputs.evaluate_upstream_health.vars.unhealthy_hosts }}
      Classification: {{ outputs.evaluate_upstream_health.vars.classification }}

  - id: route_upstream_gate
    type: io.kestra.plugin.core.flow.Switch
    description: Route flow execution based on Caddy upstream pool health classification.
    value: "{{ outputs.evaluate_upstream_health.vars.classification }}"
    cases:
      ALL_UPSTREAMS_HEALTHY:
        - id: log_nominal_pool
          type: io.kestra.plugin.core.log.Log
          description: Log nominal verification confirming all reverse proxy backends are
            healthy.
          message: "HEALTHY: All {{ outputs.evaluate_upstream_health.vars.total_upstreams
            }} Caddy reverse proxy upstream hosts are responsive."

      PARTIAL_BACKEND_DEGRADATION:
        - id: log_degradation_advisory
          type: io.kestra.plugin.core.log.Log
          description: Log advisory warning when backend degradation is within acceptable
            threshold limits.
          message: "WARNING: Caddy upstream degradation detected. Unhealthy hosts: {{
            outputs.evaluate_upstream_health.vars.unhealthy_hosts }} ({{
            outputs.evaluate_upstream_health.vars.unhealthy_percent }}%)."

      CRITICAL_UPSTREAM_OUTAGE:
        - id: alert_slack_upstream_outage
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          description: Dispatch urgent incident alert to infrastructure engineering channel.
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "text": ":rotating_light: *CRITICAL Caddy Reverse Proxy Upstream Outage Alert*\n*Endpoint:* `{{ inputs.caddy_admin_endpoint }}`\n*Total Upstreams:* `{{ outputs.evaluate_upstream_health.vars.total_upstreams }}`\n*Unhealthy Count:* `{{ outputs.evaluate_upstream_health.vars.unhealthy_count }}` (`{{ outputs.evaluate_upstream_health.vars.unhealthy_percent }}%`)\n*Failing Hosts:* `{{ outputs.evaluate_upstream_health.vars.unhealthy_hosts }}`\n*Execution:* <{{ flow.url }}|{{ execution.id }}>"
            }

        - id: remediation_outage_gate
          type: io.kestra.plugin.core.execution.Fail
          description: Fail execution to trigger incident escalation and prevent cascading
            gateway failure.

  - id: final_audit_log
    type: io.kestra.plugin.core.log.Log
    description: Record immutable audit log entry documenting Caddy upstream pool
      health verification.
    message: |
      [AUDIT] Caddy Reverse Proxy Upstream Health Sentinel completed.
      Endpoint: {{ inputs.caddy_admin_endpoint }}
      Total: {{ outputs.evaluate_upstream_health.vars.total_upstreams }}
      Unhealthy: {{ outputs.evaluate_upstream_health.vars.unhealthy_count }}
      Verdict: {{ outputs.evaluate_upstream_health.vars.classification }}