AWS ECS Container Stopped Task Non-Zero Exit Code Sentinel

id: aws-ecs-stopped-task-exit-code-sentinel
namespace: company.cloudops

inputs:
  - id: aws_region
    type: STRING
    defaults: us-east-1
    description: AWS region where the ECS cluster is hosted.
  - id: cluster_name
    type: STRING
    defaults: production-workloads
    description: Name or ARN of the ECS cluster to inspect.

tasks:
  - id: inspect_stopped_tasks
    type: io.kestra.plugin.aws.cli.AwsCLI
    region: "{{ inputs.aws_region }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
    commands:
      - |
        aws ecs list-tasks           --cluster "{{ inputs.cluster_name }}"           --desired-status STOPPED           --output json > task_arns.json
        python3 -c "
        import json, subprocess
        with open('task_arns.json') as f:
            arns = json.load(f).get('taskArns', [])
        failed_tasks = []
        if arns:
            cmd = f'aws ecs describe-tasks --cluster {{ inputs.cluster_name }} --tasks {" ".join(arns[:15])} --output json'
            res = subprocess.run(cmd, shell=True, capture_output=True, text=True)
            if res.returncode == 0:
                tasks = json.loads(res.stdout).get('tasks', [])
                for t in tasks:
                    task_arn = t.get('taskArn', '')
                    stopped_reason = t.get('stoppedReason', '')
                    containers = t.get('containers', [])
                    for c in containers:
                        exit_code = c.get('exitCode')
                        if exit_code is not None and exit_code != 0:
                            failed_tasks.append({
                                'task': task_arn.split('/')[-1],
                                'container': c.get('name'),
                                'exit_code': exit_code,
                                'reason': stopped_reason
                            })
        result = {
            'cluster': '{{ inputs.cluster_name }}',
            'failed_tasks_count': len(failed_tasks),
            'failed_tasks': failed_tasks[:5],
            'alert_triggered': len(failed_tasks) > 0
        }
        with open('ecs_summary.json', 'w') as out:
            json.dump(result, out)
        print('ECS audit:', result)
        "
    outputFiles:
      - ecs_summary.json

  - id: evaluate_task_failures
    type: io.kestra.plugin.core.flow.If
    condition: "{{
      fromJson(read(outputs.inspect_stopped_tasks.outputFiles['ecs_summary.json\
      '])).alert_triggered }}"
    then:
      - id: alert_container_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "text": ":rotating_light: *AWS ECS Container Failure Alert*: Detected {{ fromJson(read(outputs.inspect_stopped_tasks.outputFiles['ecs_summary.json'])).failed_tasks_count }} stopped tasks with non-zero exit codes in cluster `{{ inputs.cluster_name }}`. Potential application crashloops or memory limits exceeded."
          }

errors:
  - id: notify_sentinel_error
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "text": ":warning: *ECS Sentinel Execution Failure*: Failed to query tasks in cluster `{{ inputs.cluster_name }}`."
      }

triggers:
  - id: scheduled_ecs_check
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"
    disabled: true