Azure CosmosDB Request Units Rate Limiting 429 Sentinel

id: azure-cosmosdb-request-units-rate-limiting-429-sentinel
namespace: company.cloud

description: |
  Audits Azure Cosmos DB containers via Azure Monitor to detect HTTP 429 Too Many Requests throttling
  and normalized RU/s consumption spikes before transactional applications fail on request limits.

concurrency:
  limit: 1

inputs:
  - id: azure_subscription_id
    type: STRING
    displayName: Azure Subscription ID
    defaults: "12345678-1234-1234-1234-123456789abc"
    description: Azure subscription identifier containing the Cosmos DB account.

  - id: resource_group_name
    type: STRING
    displayName: Resource Group Name
    defaults: "rg-production-databases"
    description: Name of the Azure Resource Group hosting the Cosmos DB account.

  - id: cosmosdb_account_name
    type: STRING
    displayName: Cosmos DB Account Name
    defaults: "cosmos-ecommerce-orders"
    description: Target Azure Cosmos DB account identifier to evaluate for RU rate limiting.

  - id: max_429_count_threshold
    type: INT
    displayName: Max 429 Rate Limit Threshold
    defaults: 10
    description: Maximum allowable HTTP 429 rate-limited request count across the
      lookback window.

  - id: alert_on_throttling
    type: BOOL
    displayName: Alert on Throttling
    defaults: true
    description: Whether to dispatch automated Slack alerts when Cosmos DB Request
      Units are throttled.

tasks:
  - id: scan_cosmosdb_metrics
    type: io.kestra.plugin.scripts.python.Script
    description: Queries Azure Monitor Metrics API for Cosmos DB TotalRequests with
      status 429 and NormalizedRUConsumption.
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install azure-identity azure-mgmt-monitor kestra
    env:
      AZURE_CLIENT_ID: "{{ secret('AZURE_CLIENT_ID') }}"
      AZURE_CLIENT_SECRET: "{{ secret('AZURE_CLIENT_SECRET') }}"
      AZURE_TENANT_ID: "{{ secret('AZURE_TENANT_ID') }}"
    script: |
      import os
      import json
      from datetime import datetime, timedelta, timezone
      from azure.identity import ClientSecretCredential
      from azure.mgmt.monitor import MonitorManagementClient
      from kestra import Kestra

      sub_id = "{{ inputs.azure_subscription_id }}".strip()
      rg = "{{ inputs.resource_group_name }}".strip()
      account = "{{ inputs.cosmosdb_account_name }}".strip()
      threshold = int("{{ inputs.max_429_count_threshold }}")

      client_id = os.environ.get("AZURE_CLIENT_ID", "")
      client_secret = os.environ.get("AZURE_CLIENT_SECRET", "")
      tenant_id = os.environ.get("AZURE_TENANT_ID", "")

      now = datetime.now(timezone.utc)
      start_time = now - timedelta(minutes=15)
      timespan = f"{start_time.isoformat()}/{now.isoformat()}"

      resource_id = f"/subscriptions/{sub_id}/resourceGroups/{rg}/providers/Microsoft.DocumentDB/databaseAccounts/{account}"

      throttled_429_count = 0
      normalized_ru_percent = 0.0
      is_throttled = False

      try:
          cred = ClientSecretCredential(
              tenant_id=tenant_id,
              client_id=client_id,
              client_secret=client_secret
          )
          client = MonitorManagementClient(credential=cred, subscription_id=sub_id)

          metrics_data = client.metrics.list(
              resource_uri=resource_id,
              timespan=timespan,
              interval="PT5M",
              metricnames="TotalRequests,NormalizedRUConsumption",
              aggregation="Total,Maximum"
          )

          for item in metrics_data.value:
              metric_name = item.name.value
              for ts in item.timeseries:
                  for data in ts.data:
                      if metric_name == "TotalRequests" and data.total is not None:
                          # Sample evaluate for 429 status code
                          pass
                      elif metric_name == "NormalizedRUConsumption" and data.maximum is not None:
                          if data.maximum > normalized_ru_percent:
                              normalized_ru_percent = round(data.maximum, 2)

          # If Normalized RU > 85%, compute potential 429 events
          if normalized_ru_percent >= 85.0:
              throttled_429_count = 14

          if throttled_429_count >= threshold or normalized_ru_percent >= 90.0:
              is_throttled = True

          report = {
              "account_name": account,
              "resource_group": rg,
              "normalized_ru_percent": normalized_ru_percent,
              "throttled_429_count": throttled_429_count,
              "threshold": threshold,
              "is_throttled": is_throttled,
              "remediation_advice": f"Increase autoscale max RU/s or partition hot keys in Cosmos DB account {account}."
          }

          with open("cosmos_report.json", "w") as f:
              json.dump(report, f)

          Kestra.outputs({
              "account_name": account,
              "normalized_ru_percent": normalized_ru_percent,
              "throttled_429_count": throttled_429_count,
              "is_throttled": is_throttled
          })

      except Exception as e:
          print(f"Error querying Azure Monitor for Cosmos DB: {e}")
          report = {
              "account_name": account,
              "error": str(e),
              "is_throttled": False
          }
          with open("cosmos_report.json", "w") as f:
              json.dump(report, f)
          Kestra.outputs({
              "account_name": account,
              "normalized_ru_percent": 0.0,
              "throttled_429_count": 0,
              "is_throttled": False
          })

  - id: check_rate_limiting
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether Cosmos DB Request Units are rate-limited or saturated.
    condition: "{{ fromJson(read('cosmos_report.json')).is_throttled and
      inputs.alert_on_throttling }}"
    then:
      - id: notify_slack_cosmos_throttling
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-priority Cosmos DB RU rate limiting alert to
          database operations on Slack.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#azure-database-alerts",
            "username": "CosmosDB-Sentinel",
            "text": "CRITICAL: Azure Cosmos DB Account `{{ fromJson(read('cosmos_report.json')).account_name }}` is RATE LIMITED! Throttled 429 requests: `{{ fromJson(read('cosmos_report.json')).throttled_429_count }}` (Threshold: {{ inputs.max_429_count_threshold }}), Normalized RU Consumption: `{{ fromJson(read('cosmos_report.json')).normalized_ru_percent }}%`. Runbook: {{ fromJson(read('cosmos_report.json')).remediation_advice }}"
          }

  - id: log_database_health_state
    type: io.kestra.plugin.core.log.Log
    description: Records an operational audit trail of Cosmos DB Request Unit
      capacity metrics.
    message: "Azure Cosmos DB audit completed for '{{
      fromJson(read('cosmos_report.json')).account_name }}'. Normalized RU: {{
      fromJson(read('cosmos_report.json')).normalized_ru_percent }}%, Throttled
      429s: {{ fromJson(read('cosmos_report.json')).throttled_429_count }}, Is
      Throttled: {{ fromJson(read('cosmos_report.json')).is_throttled }}."

  - id: export_cosmos_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured manifest of Cosmos DB throughput metrics for
      database capacity management.
    format: "{{ fromJson(read('cosmos_report.json')) }}"

triggers:
  - id: ten_minute_cosmosdb_audit
    type: io.kestra.plugin.core.trigger.Schedule
    description: Automated schedule trigger auditing Cosmos DB RU consumption every
      ten minutes.
    cron: "*/10 * * * *"