CloudWatch Metric Breach Auto-Triage

id: cloudwatch-metric-breach-auto-triage
namespace: company.team
description: |
  Turn a raw CloudWatch metric breach into an actionable incident alert.
  When the metric trigger fires, the flow pulls the surrounding metric
  datapoints, runs a set of opt-in AWS diagnostic commands (log tail,
  instance state, recent deployments), and posts everything to Slack with a
  link to your runbook. No more "CPU > 80%" alerts with zero context , 
  the on-call engineer gets the triage bundle in one message.

triggers:
  - id: metric_breach
    type: io.kestra.plugin.aws.cloudwatch.Trigger
    description: Fires when the watched metric breaches the threshold.
    region: "{{ inputs.aws_region }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
    namespace: "{{ inputs.metric_namespace }}"
    metricName: "{{ inputs.metric_name }}"
    statistic: Average
    periodSeconds: 300
    window: PT15M

inputs:
  - id: aws_region
    type: STRING
    defaults: us-east-1
    description: AWS region of the watched resources.
  - id: metric_namespace
    type: STRING
    defaults: AWS/EC2
    description: CloudWatch namespace of the watched metric.
  - id: metric_name
    type: STRING
    defaults: CPUUtilization
    description: CloudWatch metric name that triggers the triage.
  - id: diagnostic_commands
    type: ARRAY
    itemType: STRING
    defaults:
      - aws ec2 describe-instances --filters
        "Name=instance-state-name,Values=running" --query
        "Reservations[].Instances[].[InstanceId,InstanceType,LaunchTime]"
        --output table
      - aws logs tail /aws/lambda/app-errors --since 15m --format short 2>&1 |
        head -n 50
    description: Read-only AWS CLI diagnostic commands to run on every breach. Keep
      them read-only, this flow never mutates infrastructure.
  - id: runbook_url
    type: STRING
    defaults: https://wiki.company.team/runbooks/metric-breach
    description: Link to the team runbook, included in the Slack alert.

tasks:
  - id: capture_breach_context
    type: io.kestra.plugin.aws.cli.AwsCLI
    description: Pull the metric datapoints surrounding the breach window.
    region: "{{ inputs.aws_region }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
    commands:
      - |
        END=$(date -u +%Y-%m-%dT%H:%M:%SZ)
        START=$(date -u -d "30 minutes ago" +%Y-%m-%dT%H:%M:%SZ)
        echo "Breach context for {{ inputs.metric_namespace }}/{{ inputs.metric_name }} ($START -> $END)"
        aws cloudwatch get-metric-statistics --namespace "{{ inputs.metric_namespace }}" \
          --metric-name "{{ inputs.metric_name }}" --start-time "$START" --end-time "$END" \
          --period 300 --statistics Average Maximum --region "{{ inputs.aws_region }}"

  - id: run_diagnostics
    type: io.kestra.plugin.core.flow.Loop
    description: Run each configured read-only diagnostic command.
    values: "{{ inputs.diagnostic_commands }}"
    tasks:
      - id: diagnostic
        type: io.kestra.plugin.aws.cli.AwsCLI
        description: Execute one diagnostic command.
        region: "{{ inputs.aws_region }}"
        accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
        secretKeyId: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
        commands:
          - "{{ item.value }}"

  - id: notify_slack
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Post the triage bundle to Slack with the runbook link.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": ":rotating_light: *CloudWatch breach triage*, `{{ inputs.metric_namespace }}/{{ inputs.metric_name }}` in `{{ inputs.aws_region }}`\n*Metric context:*\n```{{ outputs.capture_breach_context.stdout }}```\n*Runbook:* {{ inputs.runbook_url }}"
      }