Azure AppService Plan Memory Leak and Restart Sentinel

id: azure-appservice-plan-memory-leak-and-restart-sentinel
namespace: company.cloud

description: |
  Audits Azure App Service Plans via Azure Monitor to detect MemoryPercentage saturation
  and container restart loops before production web applications crash on out-of-memory errors.

concurrency:
  limit: 1

inputs:
  - id: azure_subscription_id
    type: STRING
    displayName: Azure Subscription ID
    defaults: "12345678-1234-1234-1234-123456789abc"
    description: Azure subscription identifier containing the App Service Plan.

  - id: resource_group_name
    type: STRING
    displayName: Resource Group Name
    defaults: "rg-production-apps"
    description: Name of the Azure Resource Group hosting the App Service Plan.

  - id: app_service_plan_name
    type: STRING
    displayName: App Service Plan Name
    defaults: "asp-ecommerce-prod"
    description: Name of the target Azure App Service Plan to audit for memory saturation.

  - id: max_memory_percent_threshold
    type: FLOAT
    displayName: Max Memory Threshold (%)
    defaults: 85.0
    description: Maximum acceptable memory utilization percentage before triggering
      exhaustion alerts.

  - id: alert_on_saturation
    type: BOOL
    displayName: Alert on Saturation
    defaults: true
    description: Whether to dispatch automated Slack alerts when App Service Plan
      memory is saturated.

tasks:
  - id: scan_appservice_memory
    type: io.kestra.plugin.scripts.python.Script
    description: Queries Azure Monitor Metrics API for App Service Plan
      MemoryPercentage and CpuPercentage.
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install azure-identity azure-mgmt-monitor kestra
    env:
      AZURE_CLIENT_ID: "{{ secret('AZURE_CLIENT_ID') }}"
      AZURE_CLIENT_SECRET: "{{ secret('AZURE_CLIENT_SECRET') }}"
      AZURE_TENANT_ID: "{{ secret('AZURE_TENANT_ID') }}"
    script: |
      import os
      import json
      from datetime import datetime, timedelta, timezone
      from azure.identity import ClientSecretCredential
      from azure.mgmt.monitor import MonitorManagementClient
      from kestra import Kestra

      sub_id = "{{ inputs.azure_subscription_id }}".strip()
      rg = "{{ inputs.resource_group_name }}".strip()
      plan = "{{ inputs.app_service_plan_name }}".strip()
      threshold = float("{{ inputs.max_memory_percent_threshold }}")

      client_id = os.environ.get("AZURE_CLIENT_ID", "")
      client_secret = os.environ.get("AZURE_CLIENT_SECRET", "")
      tenant_id = os.environ.get("AZURE_TENANT_ID", "")

      now = datetime.now(timezone.utc)
      start_time = now - timedelta(minutes=15)
      timespan = f"{start_time.isoformat()}/{now.isoformat()}"

      resource_id = f"/subscriptions/{sub_id}/resourceGroups/{rg}/providers/Microsoft.Web/serverfarms/{plan}"

      max_memory_percent = 0.0
      avg_cpu_percent = 0.0
      is_saturated = False

      try:
          cred = ClientSecretCredential(
              tenant_id=tenant_id,
              client_id=client_id,
              client_secret=client_secret
          )
          client = MonitorManagementClient(credential=cred, subscription_id=sub_id)

          metrics_data = client.metrics.list(
              resource_uri=resource_id,
              timespan=timespan,
              interval="PT5M",
              metricnames="MemoryPercentage,CpuPercentage",
              aggregation="Average,Maximum"
          )

          for item in metrics_data.value:
              metric_name = item.name.value
              for ts in item.timeseries:
                  for data in ts.data:
                      if metric_name == "MemoryPercentage" and data.maximum is not None:
                          if data.maximum > max_memory_percent:
                              max_memory_percent = round(data.maximum, 2)
                      elif metric_name == "CpuPercentage" and data.average is not None:
                          avg_cpu_percent = round(data.average, 2)

          if max_memory_percent >= threshold:
              is_saturated = True

          report = {
              "plan_name": plan,
              "resource_group": rg,
              "max_memory_percent": max_memory_percent,
              "avg_cpu_percent": avg_cpu_percent,
              "threshold_percent": threshold,
              "is_saturated": is_saturated,
              "remediation_advice": f"Scale out instance count or scale up tier for App Service Plan {plan}. Review Node.js/Java heap dumps."
          }

          with open("appservice_report.json", "w") as f:
              json.dump(report, f)

          Kestra.outputs({
              "plan_name": plan,
              "max_memory_percent": max_memory_percent,
              "is_saturated": is_saturated
          })

      except Exception as e:
          print(f"Error querying Azure Monitor for App Service Plan: {e}")
          report = {
              "plan_name": plan,
              "error": str(e),
              "is_saturated": False
          }
          with open("appservice_report.json", "w") as f:
              json.dump(report, f)
          Kestra.outputs({
              "plan_name": plan,
              "max_memory_percent": 0.0,
              "is_saturated": False
          })

  - id: check_memory_saturation
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether App Service Plan memory percentage exceeds the
      safety threshold.
    condition: "{{ fromJson(read('appservice_report.json')).is_saturated and
      inputs.alert_on_saturation }}"
    then:
      - id: notify_slack_memory_saturation
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-priority memory saturation alert to DevOps on Slack.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#azure-compute-alerts",
            "username": "AppService-Sentinel",
            "text": "CRITICAL: Azure App Service Plan `{{ fromJson(read('appservice_report.json')).plan_name }}` is SATURATED at `{{ fromJson(read('appservice_report.json')).max_memory_percent }}%` memory (Threshold: {{ fromJson(read('appservice_report.json')).threshold_percent }}%)! Worker instances are at risk of Out-Of-Memory container recycling. CPU: {{ fromJson(read('appservice_report.json')).avg_cpu_percent }}%. Runbook: {{ fromJson(read('appservice_report.json')).remediation_advice }}"
          }

  - id: log_compute_health_state
    type: io.kestra.plugin.core.log.Log
    description: Records an operational audit trail of App Service Plan compute health.
    message: "Azure App Service Plan audit completed for '{{
      fromJson(read('appservice_report.json')).plan_name }}'. Memory: {{
      fromJson(read('appservice_report.json')).max_memory_percent }}%, CPU: {{
      fromJson(read('appservice_report.json')).avg_cpu_percent }}%, Is
      Saturated: {{ fromJson(read('appservice_report.json')).is_saturated }}."

  - id: export_appservice_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured manifest of App Service Plan performance metrics
      for platform autoscaling.
    format: "{{ fromJson(read('appservice_report.json')) }}"

triggers:
  - id: ten_minute_appservice_audit
    type: io.kestra.plugin.core.trigger.Schedule
    description: Automated schedule trigger auditing App Service Plan memory every
      ten minutes.
    cron: "*/10 * * * *"