Pingdom Uptime Check Failing Probe Sentinel

id: pingdom-uptime-check-failing-probe-sentinel
namespace: company.uptime

description: |
  Audits Pingdom external synthetic probes to detect endpoint downtime, global DNS propagation failures,
  and regional edge routing blackouts, dispatching immediate incident triage notifications to SRE teams.

concurrency:
  limit: 1

inputs:
  - id: pingdom_check_id
    type: INT
    displayName: Pingdom Check ID
    defaults: 12345678
    description: Numeric identifier of the target Pingdom synthetic uptime check to
      evaluate.

  - id: max_response_time_ms_threshold
    type: INT
    displayName: Max Response Time Threshold (ms)
    defaults: 2000
    description: Maximum allowable response latency in milliseconds before flagging
      degraded performance.

  - id: alert_on_downtime
    type: BOOL
    displayName: Alert on Downtime
    defaults: true
    description: Whether to dispatch automated Slack incident notifications when
      endpoint outages or latency spikes occur.

tasks:
  - id: evaluate_pingdom_check_status
    type: io.kestra.plugin.scripts.python.Script
    description: Queries Pingdom REST API v3.1 for check operational status and
      probe response metrics.
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      PINGDOM_API_TOKEN: "{{ secret('PINGDOM_API_TOKEN') }}"
    script: |
      import os
      import json
      import requests
      from kestra import Kestra

      check_id = int("{{ inputs.pingdom_check_id }}")
      token = os.environ.get("PINGDOM_API_TOKEN", "")
      latency_threshold = int("{{ inputs.max_response_time_ms_threshold }}")

      headers = {
          "Authorization": f"Bearer {token}",
          "Accept": "application/json"
      }

      url = f"https://api.pingdom.com/api/3.1/checks/{check_id}"

      check_name = f"Check-{check_id}"
      hostname = "unknown"
      status = "unknown"
      last_response_time = 0
      is_down = False
      is_degraded = False

      try:
          resp = requests.get(url, headers=headers, timeout=20)
          resp.raise_for_status()
          data = resp.json()
          check = data.get("check", {})

          check_name = check.get("name", check_name)
          hostname = check.get("hostname", hostname)
          status = check.get("status", "unknown")
          last_response_time = check.get("lastresponsetime", 0)

          if status != "up":
              is_down = True

          if last_response_time >= latency_threshold:
              is_degraded = True

          Kestra.outputs({
              "check_id": check_id,
              "check_name": check_name,
              "hostname": hostname,
              "status": status,
              "last_response_time_ms": last_response_time,
              "is_down": is_down,
              "is_degraded": is_degraded
          })

      except Exception as e:
          print(f"Error querying Pingdom API for check {check_id}: {e}")
          Kestra.outputs({
              "check_id": check_id,
              "check_name": check_name,
              "hostname": hostname,
              "status": "error",
              "last_response_time_ms": 0,
              "is_down": True,
              "is_degraded": False
          })

  - id: check_probe_status
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether Pingdom check reports endpoint outage or
      excessive latency.
    condition: "{{ (outputs.evaluate_pingdom_check_status.vars.is_down or
      outputs.evaluate_pingdom_check_status.vars.is_degraded) and
      inputs.alert_on_downtime }}"
    then:
      - id: notify_slack_pingdom_down
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-priority uptime probe outage alert to SRE on-call
          channel on Slack.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#sre-critical-incidents",
            "username": "Pingdom-Sentinel",
            "text": "CRITICAL OUTAGE: Pingdom Uptime Check `{{ outputs.evaluate_pingdom_check_status.vars.check_name }}` (`{{ outputs.evaluate_pingdom_check_status.vars.hostname }}`) is DOWN or DEGRADED! Status: `{{ outputs.evaluate_pingdom_check_status.vars.status }}` (Down: {{ outputs.evaluate_pingdom_check_status.vars.is_down }}), Response Time: `{{ outputs.evaluate_pingdom_check_status.vars.last_response_time_ms }}ms` (Threshold: {{ inputs.max_response_time_ms_threshold }}ms). Pingdom Report: https://my.pingdom.com/app/reports/uptime#check={{ outputs.evaluate_pingdom_check_status.vars.check_id }}"
          }

  - id: log_uptime_probe_status
    type: io.kestra.plugin.core.log.Log
    description: Records an operational audit trail of Pingdom synthetic uptime
      probe evaluations.
    message: "Pingdom audit completed for '{{
      outputs.evaluate_pingdom_check_status.vars.check_name }}'. Status: {{
      outputs.evaluate_pingdom_check_status.vars.status }}, Latency: {{
      outputs.evaluate_pingdom_check_status.vars.last_response_time_ms }}ms,
      Outage Detected: {{ outputs.evaluate_pingdom_check_status.vars.is_down
      }}."

  - id: export_pingdom_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured manifest of Pingdom probe diagnostic data for SLA
      tracking and postmortems.
    format:
      check_id: "{{ outputs.evaluate_pingdom_check_status.vars.check_id }}"
      check_name: "{{ outputs.evaluate_pingdom_check_status.vars.check_name }}"
      hostname: "{{ outputs.evaluate_pingdom_check_status.vars.hostname }}"
      status: "{{ outputs.evaluate_pingdom_check_status.vars.status }}"
      response_time_ms: "{{ outputs.evaluate_pingdom_check_status.vars.last_response_time_ms }}"
      is_down: "{{ outputs.evaluate_pingdom_check_status.vars.is_down }}"
      is_degraded: "{{ outputs.evaluate_pingdom_check_status.vars.is_degraded }}"

triggers:
  - id: ten_minute_pingdom_audit
    type: io.kestra.plugin.core.trigger.Schedule
    description: Automated schedule trigger auditing Pingdom uptime check status
      every ten minutes.
    cron: "*/10 * * * *"