AWS EventBridge Rule Dead Letter Queue Invocation Error Sentinel

id: aws-eventbridge-dead-letter-queue-sentinel
namespace: company.cloudops

inputs:
  - id: aws_region
    type: STRING
    defaults: us-east-1
    description: AWS region where the EventBridge rule resides.
  - id: rule_name
    type: STRING
    defaults: order-event-router-rule
    description: Name of the EventBridge rule to monitor.
  - id: max_failed_invocations_threshold
    type: INT
    defaults: 0
    description: Maximum permissible failed target invocations in a 15-minute window
      before alerting.

tasks:
  - id: query_eventbridge_failures
    type: io.kestra.plugin.aws.cli.AwsCLI
    region: "{{ inputs.aws_region }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
    commands:
      - |
        aws cloudwatch get-metric-statistics           --namespace AWS/Events           --metric-name FailedInvocations           --dimensions Name=RuleName,Value="{{ inputs.rule_name }}"           --start-time $(date -u -v-15m +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d "15 minutes ago" +%Y-%m-%dT%H:%M:%SZ)           --end-time $(date -u +%Y-%m-%dT%H:%M:%SZ)           --period 900           --statistics Sum           --output json > eb_failures.json
        python3 -c "
        import json
        with open('eb_failures.json') as f:
            dps = json.load(f).get('Datapoints', [])
        total_failed = int(sum(dp.get('Sum', 0) for dp in dps))
        result = {
            'rule_name': '{{ inputs.rule_name }}',
            'failed_invocations': total_failed,
            'alert_triggered': total_failed > {{ inputs.max_failed_invocations_threshold }}
        }
        with open('eb_summary.json', 'w') as out:
            json.dump(result, out)
        print('EventBridge audit:', result)
        "
    outputFiles:
      - eb_summary.json

  - id: evaluate_invocation_failures
    type: io.kestra.plugin.core.flow.If
    condition: "{{
      fromJson(read(outputs.query_eventbridge_failures.outputFiles['eb_summary.\
      json'])).alert_triggered }}"
    then:
      - id: alert_eda_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "text": ":warning: *AWS EventBridge Invocation Failure Alert*: Rule `{{ inputs.rule_name }}` recorded {{ fromJson(read(outputs.query_eventbridge_failures.outputFiles['eb_summary.json'])).failed_invocations }} failed target invocations in the last 15 minutes. Check target permissions, concurrency limits, or DLQ delivery."
          }

errors:
  - id: notify_sentinel_error
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "text": ":warning: *EventBridge Sentinel Failure*: Failed to query CloudWatch metrics for rule `{{ inputs.rule_name }}`."
      }

triggers:
  - id: scheduled_eb_check
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"
    disabled: true