AI Incident Triager with Gemini, PagerDuty, and Slack

id: ai-incident-triager-gemini-pagerduty-slack
namespace: company.sre

inputs:
  - id: service_name
    type: STRING
    displayName: Service Name
    required: true
    defaults: "payment-gateway"
    description: "The name of the service experiencing an anomaly or failure."

  - id: raw_error_logs
    type: STRING
    displayName: Raw Error Logs
    required: true
    defaults: "ConnectionTimeout: upstream database replica db-east-02 failed health
      check after 30000ms. Error 504 Gateway Timeout on /v1/charge."
    description: "Raw stack trace or log excerpt captured during the incident."

  - id: severity_hint
    type: SELECT
    displayName: Severity Hint
    values: [ "LOW", "MEDIUM", "HIGH", "CRITICAL" ]
    defaults: "HIGH"
    description: "Initial automated alert severity level from monitoring source."

tasks:
  - id: log_incident_received
    type: io.kestra.plugin.core.log.Log
    message: "🚨 Triaging incoming incident on service: {{ inputs.service_name }}
      (Initial Severity: {{ inputs.severity_hint }})"

  - id: gemini_root_cause_analysis
    type: io.kestra.plugin.gemini.StructuredOutputCompletion
    description: "Perform LLM log parsing and structured incident triage using
      Google Gemini"
    apiKey: "{{ secret('GEMINI_API_KEY') }}"
    model: "gemini-2.5-flash"
    retry:
      type: constant
      interval: PT10S
      maxAttempts: 3
    prompt: |
      You are a Principal Site Reliability Engineer (SRE).
      Analyze the following production incident logs:

      Service: {{ inputs.service_name }}
      Reported Severity: {{ inputs.severity_hint }}
      Raw Error Logs:
      {{ inputs.raw_error_logs }}

      Analyze the likely root cause, produce a concise summary, recommend immediate actionable recovery steps, and evaluate if this requires on-call engineer paging.
    jsonResponseSchema: |
      {
        "type": "object",
        "properties": {
          "summary": {
            "type": "string",
            "description": "Concise 1-sentence summary of the incident"
          },
          "root_cause_hypothesis": {
            "type": "string",
            "description": "Likely culprit behind the failure"
          },
          "recommended_actions": {
            "type": "array",
            "items": { "type": "string" },
            "description": "2-3 immediate recovery steps for the on-call engineer"
          },
          "escalate_to_pagerduty": {
            "type": "boolean",
            "description": "True if critical service impairment requires human intervention"
          }
        },
        "required": ["summary", "root_cause_hypothesis", "recommended_actions", "escalate_to_pagerduty"]
      }

  - id: notify_slack_channel
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: "Post structured AI root cause analysis to incident channel"
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "blocks": [
          {
            "type": "header",
            "text": {
              "type": "plain_text",
              "text": "🚨 Incident Triage: {{ inputs.service_name }}",
              "emoji": true
            }
          },
          {
            "type": "section",
            "fields": [
              {
                "type": "mrkdwn",
                "text": "*Severity:* {{ inputs.severity_hint }}"
              },
              {
                "type": "mrkdwn",
                "text": "*Execution ID:* `{{ execution.id }}`"
              }
            ]
          },
          {
            "type": "section",
            "text": {
              "type": "mrkdwn",
              "text": "*🤖 AI Analysis Summary:*\n{{ outputs.gemini_root_cause_analysis.predictions | first | jq('.summary') | first }}\n\n*Likely Root Cause:*\n{{ outputs.gemini_root_cause_analysis.predictions | first | jq('.root_cause_hypothesis') | first }}"
            }
          }
        ]
      }

  - id: check_and_escalate
    type: io.kestra.plugin.core.flow.If
    description: "Conditionally page on-call if AI marks as requiring immediate
      human escalation"
    condition: "{{ (outputs.gemini_root_cause_analysis.predictions | first |
      jq('.escalate_to_pagerduty') | first == true) or (inputs.severity_hint ==
      'CRITICAL') }}"
    then:
      - id: trigger_pagerduty_incident
        type: io.kestra.plugin.pagerduty.PagerDutyAlert
        url: "https://events.pagerduty.com/v2/enqueue"
        payload: |
          {
            "routing_key": "{{ secret('PAGERDUTY_ROUTING_KEY') }}",
            "event_action": "trigger",
            "dedup_key": "kestra-incident-{{ inputs.service_name }}-{{ execution.id }}",
            "payload": {
              "summary": "[CRITICAL SRE ALERT] {{ inputs.service_name }}: {{ outputs.gemini_root_cause_analysis.predictions | first | jq('.summary') | first }}",
              "source": "kestra-flow-{{ flow.id }}",
              "severity": "critical",
              "custom_details": {
                "execution_id": "{{ execution.id }}",
                "service": "{{ inputs.service_name }}",
                "recommended_actions": "{{ outputs.gemini_root_cause_analysis.predictions | first | jq('.recommended_actions') | first }}"
              }
            }
          }

errors:
  - id: alert_flow_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: "Dead-letter alerting if triage workflow fails"
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "⚠️ *Kestra Triage Pipeline Failed!* Execution `{{ execution.id }}` encountered an unhandled error. Please inspect logs manually."
      }