Datadog Active Monitor Alert and Incident Sync Sentinel

id: datadog-alert-incident-sync-sentinel
namespace: company.sre

description: |
  Polls Datadog monitors in Alert state via containerized Python script,
  evaluates critical production breaches, files a prioritized GitHub incident
  tracking issue, and dispatches rich triage alerts to on-call engineers via Slack.

inputs:
  - id: datadog_site
    type: STRING
    displayName: Datadog Site
    defaults: datadoghq.com

  - id: monitor_tags
    type: STRING
    displayName: Monitor Tags Filter
    defaults: "env:prod,service:backend"

  - id: github_repository
    type: STRING
    displayName: Incident Tracking GitHub Repository
    defaults: my-org/sre-incidents

triggers:
  - id: every_fifteen_minutes
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"
    disabled: true

tasks:
  - id: probe_datadog_alerts
    type: io.kestra.plugin.scripts.python.Script
    description: Queries the Datadog API for all monitors actively in Alert status
    containerImage: python:3.11-slim
    beforeCommands:
      - pip install requests kestra
    script: |
      import os
      import requests
      from kestra import Kestra

      site = "{{ inputs.datadog_site }}"
      tags = "{{ inputs.monitor_tags }}"
      api_key = os.environ.get("DATADOG_API_KEY")
      app_key = os.environ.get("DATADOG_APP_KEY")

      headers = {
          "DD-API-KEY": api_key,
          "DD-APPLICATION-KEY": app_key,
          "Content-Type": "application/json"
      }

      url = f"https://api.{site}/api/v1/monitor"
      params = {
          "monitor_tags": tags,
          "group_states": "Alert"
      }

      response = requests.get(url, headers=headers, params=params, timeout=30)
      if response.status_code != 200:
          print(f"Datadog API returned status {response.status_code}: {response.text}")
          # Raise or emit safe probe failure
          response.raise_for_status()

      monitors = response.json()
      alerting_monitors = [m for m in monitors if m.get("overall_state") == "Alert"]

      summary = []
      for m in alerting_monitors:
          summary.append({
              "id": m.get("id"),
              "name": m.get("name"),
              "type": m.get("type"),
              "message": m.get("message", "")[:200]
          })

      alert_count = len(alerting_monitors)
      print(f"Found {alert_count} active monitors in Alert state.")

      Kestra.outputs({
          "alert_count": alert_count,
          "has_alerts": alert_count > 0,
          "alerting_monitors": summary
      })

  - id: branch_on_active_alerts
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether any Datadog monitors are actively in Alert state
    condition: "{{ outputs.probe_datadog_alerts.vars.has_alerts == true }}"
    then:
      - id: create_incident_github_issue
        type: io.kestra.plugin.github.issues.Create
        description: Creates a GitHub incident triage ticket for active Datadog alerts
        oauthToken: "{{ secret('GITHUB_TOKEN') }}"
        repository: "{{ inputs.github_repository }}"
        title: "[Incident] Active Datadog Monitor Alerts Detected ({{
          outputs.probe_datadog_alerts.vars.alert_count }} Active)"
        body: |
          ## 🚨 Active Datadog Monitor Alert Incident

          The automated SRE sentinel detected **{{ outputs.probe_datadog_alerts.vars.alert_count }}** monitors in `Alert` state matching filter `{{ inputs.monitor_tags }}`.

          ### Alerting Monitors:
          {% for m in outputs.probe_datadog_alerts.vars.alerting_monitors %}
          - **Monitor ID:** `{{ m.id }}` - **{{ m.name }}** (Type: `{{ m.type }}`)
          {% endfor %}

          ---
          *Generated automatically by Kestra Datadog Alert Incident Sentinel.*

      - id: notify_sre_slack
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Alerts on-call SREs on Slack with incident details and ticket link
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {{ {
            "text": "🚨 *Datadog Active Monitor Alerts Detected*\n*Filter:* `" ~ inputs.monitor_tags ~ "`\n*Alert Count:* *" ~ outputs.probe_datadog_alerts.vars.alert_count ~ "*\n*Incident Ticket:* <" ~ (outputs.create_incident_github_issue.issueUrl ?? "#") ~ "|View GitHub Incident>"
          } | toJson }}

    else:
      - id: log_all_monitors_healthy
        type: io.kestra.plugin.core.log.Log
        message: "All Datadog monitors matching {{ inputs.monitor_tags }} are OK. Zero
          active alerts."

  - id: log_audit_complete
    type: io.kestra.plugin.core.log.Log
    description: Final status log recording the Datadog alert audit results
    message: "Datadog alert audit completed. active_alerts={{
      outputs.probe_datadog_alerts.vars.alert_count }}."

errors:
  - id: alert_probe_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alerts Slack when the Datadog API probe or ticket creation fails
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {{ {
        "text": "⚠️ *Datadog Alert Sentinel Failed* in execution `" ~ execution.id ~ "`. Unable to query Datadog monitors or dispatch notifications."
      } | toJson }}