AWS API Gateway Endpoint Latency and 5XX Spike Sentinel

id: aws-apigateway-endpoint-latency-and-5xx-spike-sentinel
namespace: company.cloud

description: |
  Audits AWS API Gateway REST and HTTP API stages to detect surges in HTTP 5XX server errors
  and p99 integration latency breaches, delivering actionable diagnostic incident notifications to DevOps teams.

concurrency:
  limit: 1

inputs:
  - id: aws_region
    type: STRING
    displayName: AWS Region
    defaults: us-east-1
    description: Target AWS region where API Gateway instances and CloudWatch
      metrics reside.

  - id: api_name_filter
    type: STRING
    displayName: API Name Filter
    defaults: all
    description: Name or prefix of target API Gateway APIs to audit, or all to scan
      all regional APIs.

  - id: error_5xx_rate_threshold_percent
    type: FLOAT
    displayName: 5XX Error Rate Threshold
    defaults: 2.0
    description: Percentage threshold of 5XX responses relative to total requests
      that triggers incident alerting.

  - id: p99_latency_threshold_ms
    type: INT
    displayName: P99 Latency Threshold (ms)
    defaults: 1500
    description: Maximum allowable p99 integration latency in milliseconds before
      flagging degraded performance.

  - id: alert_on_degraded
    type: BOOL
    displayName: Alert on Degraded
    defaults: true
    description: Whether to dispatch automated Slack incident notifications when
      threshold breaches are detected.

tasks:
  - id: scan_apigateway_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Queries AWS API Gateway and CloudWatch metrics to detect 5XX error
      spikes and integration latency anomalies.
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install boto3
    env:
      AWS_ACCESS_KEY_ID: "{{ secret('AWS_ACCESS_KEY_ID') }}"
      AWS_SECRET_ACCESS_KEY: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
      AWS_DEFAULT_REGION: "{{ inputs.aws_region }}"
    script: |
      import os
      import json
      import boto3
      from datetime import datetime, timedelta, timezone
      from kestra import Kestra

      region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
      api_filter = "{{ inputs.api_name_filter }}".strip().lower()
      error_threshold = float("{{ inputs.error_5xx_rate_threshold_percent }}")
      latency_threshold = int("{{ inputs.p99_latency_threshold_ms }}")

      apigw_client = boto3.client("apigateway", region_name=region)
      cloudwatch = boto3.client("cloudwatch", region_name=region)

      end_time = datetime.now(timezone.utc)
      start_time = end_time - timedelta(minutes=15)

      apis_audited = 0
      degraded_apis = []

      paginator = apigw_client.get_paginator("get_rest_apis")
      for page in paginator.paginate():
          for api in page.get("items", []):
              api_id = api["id"]
              api_name = api["name"]

              if api_filter != "all" and api_filter not in api_name.lower():
                  continue

              apis_audited += 1

              # Query 5XX Error Rate and Count
              metric_data_queries = [
                  {
                      "Id": "count",
                      "MetricStat": {
                          "Metric": {
                              "Namespace": "AWS/ApiGateway",
                              "MetricName": "Count",
                              "Dimensions": [{"Name": "ApiName", "Value": api_name}]
                          },
                          "Period": 900,
                          "Stat": "Sum"
                      },
                      "ReturnData": True
                  },
                  {
                      "Id": "errors_5xx",
                      "MetricStat": {
                          "Metric": {
                              "Namespace": "AWS/ApiGateway",
                              "MetricName": "5XXError",
                              "Dimensions": [{"Name": "ApiName", "Value": api_name}]
                          },
                          "Period": 900,
                          "Stat": "Sum"
                      },
                      "ReturnData": True
                  },
                  {
                      "Id": "p99_latency",
                      "MetricStat": {
                          "Metric": {
                              "Namespace": "AWS/ApiGateway",
                              "MetricName": "IntegrationLatency",
                              "Dimensions": [{"Name": "ApiName", "Value": api_name}]
                          },
                          "Period": 900,
                          "Stat": "p99"
                      },
                      "ReturnData": True
                  }
              ]

              try:
                  response = cloudwatch.get_metric_data(
                      MetricDataQueries=metric_data_queries,
                      StartTime=start_time,
                      EndTime=end_time
                  )
                  results = {r["Id"]: (r["Values"][0] if r["Values"] else 0.0) for r in response.get("MetricDataResults", [])}
                  total_requests = results.get("count", 0.0)
                  errors_5xx = results.get("errors_5xx", 0.0)
                  p99_latency = results.get("p99_latency", 0.0)

                  error_rate = (errors_5xx / total_requests * 100.0) if total_requests > 0 else 0.0

                  if (total_requests > 10 and error_rate >= error_threshold) or (p99_latency >= latency_threshold):
                      degraded_apis.append({
                          "api_id": api_id,
                          "api_name": api_name,
                          "total_requests": int(total_requests),
                          "errors_5xx": int(errors_5xx),
                          "error_rate_percent": round(error_rate, 2),
                          "p99_integration_latency_ms": round(p99_latency, 2),
                          "remediation": f"aws apigateway get-stage --rest-api-id {api_id} --stage-name prod"
                      })
              except Exception as e:
                  print(f"Error querying CloudWatch for API {api_name}: {e}")

      Kestra.outputs({
          "total_apis_audited": apis_audited,
          "degraded_apis_count": len(degraded_apis),
          "has_degraded_apis": len(degraded_apis) > 0,
          "degraded_apis_details": json.dumps(degraded_apis)
      })

  - id: check_degraded_api_routes
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether degraded API Gateway stages or 5XX error spikes
      were detected.
    condition: "{{ outputs.scan_apigateway_telemetry.vars.has_degraded_apis and
      inputs.alert_on_degraded }}"
    then:
      - id: notify_slack_api_incident
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-priority incident notifications to Slack with
          degraded API endpoints and error rates.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#devops-incidents",
            "username": "APIGateway-Sentinel",
            "text": "CRITICAL: AWS API Gateway 5XX error spike or p99 latency degradation detected in region `{{ inputs.aws_region }}`. Total APIs audited: `{{ outputs.scan_apigateway_telemetry.vars.total_apis_audited }}`, Degraded APIs: `{{ outputs.scan_apigateway_telemetry.vars.degraded_apis_count }}`."
          }

  - id: log_telemetry_status
    type: io.kestra.plugin.core.log.Log
    description: Records an operational audit trail of API Gateway telemetry evaluations.
    message: "AWS API Gateway audit completed for region '{{ inputs.aws_region }}'.
      Total APIs: {{ outputs.scan_apigateway_telemetry.vars.total_apis_audited
      }}, Degraded: {{
      outputs.scan_apigateway_telemetry.vars.degraded_apis_count }}."

  - id: export_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured manifest of API Gateway latency metrics and error
      rates for observability platforms.
    format:
      region: "{{ inputs.aws_region }}"
      api_name_filter: "{{ inputs.api_name_filter }}"
      total_apis_audited: "{{ outputs.scan_apigateway_telemetry.vars.total_apis_audited }}"
      degraded_apis_count: "{{ outputs.scan_apigateway_telemetry.vars.degraded_apis_count }}"
      details: "{{ outputs.scan_apigateway_telemetry.vars.degraded_apis_details }}"

triggers:
  - id: scheduled_apigateway_audit
    type: io.kestra.plugin.core.trigger.Schedule
    description: Automated schedule trigger auditing API Gateway telemetry every 15 minutes.
    cron: "*/15 * * * *"