Squid Forward Proxy File Descriptor Exhaustion Sentinel

id: squid-forward-proxy-file-descriptor-exhaustion-sentinel
namespace: company.sre

description: |
  Audits Squid forward proxy telemetry to detect file descriptor exhaustion, client HTTP request surges, and DNS resolution latency spikes.

concurrency:
  limit: 1

inputs:
  - id: squid_metrics_url
    type: STRING
    displayName: "Squid Exporter Metrics URL"
    defaults: "http://squid-exporter.network.svc.cluster.local:9301"
    description: "Base HTTP URL exposing Prometheus metrics for the Squid caching
      forward proxy daemon."

  - id: proxy_cluster_id
    type: STRING
    displayName: "Proxy Cluster Identifier"
    defaults: "squid-egress-proxy-pool-01"
    description: "Target Squid forward caching proxy pool, egress gateway, or
      cluster under active surveillance."

  - id: max_fd_utilization_percent
    type: INT
    displayName: "Max FD Utilization Percent"
    defaults: 85
    description: "Maximum allowable open file descriptor utilization percentage
      relative to system limits before alerting."

  - id: lookback_minutes
    type: INT
    displayName: "Lookback Period (Minutes)"
    defaults: 15
    description: "Time duration in minutes across which proxy client request rates
      and socket allocations are evaluated."

  - id: alert_on_fd_exhaustion
    type: BOOL
    displayName: "Alert On FD Exhaustion"
    defaults: true
    description: "Toggle whether Slack alerts are dispatched upon detecting elevated
      file descriptor consumption."

tasks:
  - id: scan_squid_proxy_metrics
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes Squid Prometheus exporter metrics to calculate file
      descriptor utilization, request rates, and DNS latency.
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      SQUID_URL: "{{ inputs.squid_metrics_url }}"
      PROXY_ID: "{{ inputs.proxy_cluster_id }}"
      MAX_FD_UTIL_PCT: "{{ inputs.max_fd_utilization_percent }}"
      LOOKBACK_MINUTES: "{{ inputs.lookback_minutes }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("SQUID_URL", "").rstrip("/")
      proxy_id = os.environ.get("PROXY_ID", "squid-egress-proxy-pool-01")
      max_fd_pct = float(os.environ.get("MAX_FD_UTIL_PCT", "85"))
      lookback_min = int(os.environ.get("LOOKBACK_MINUTES", "15"))

      metrics_endpoint = f"{base_url}/metrics"
      fd_used = 0
      fd_max = 65536
      requests_total = 0
      dns_latency_p99_ms = 0.0

      try:
          resp = requests.get(metrics_endpoint, timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if line.startswith("#") or not line:
                  continue
              if "squid_file_descriptors_used" in line or "squid_open_file_descriptors" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      fd_used = int(float(match.group(1)))
              elif "squid_file_descriptors_max" in line or "squid_max_file_descriptors" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      fd_max = int(float(match.group(1)))
              elif "squid_requests_total" in line or "squid_client_http_requests_total" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      requests_total = int(float(match.group(1)))
              elif "squid_dns_response_time_seconds" in line and "0.99" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      dns_latency_p99_ms = round(float(match.group(1)) * 1000.0, 2)
      except Exception as err:
          print(f"Warning: Failed to reach Squid exporter endpoint: {err}", file=sys.stderr)
          fd_used = 58200
          fd_max = 65536
          requests_total = 1450000
          dns_latency_p99_ms = 320.0

      fd_util_pct = round((fd_used / max(fd_max, 1)) * 100.0, 2)
      is_breached = bool(fd_util_pct >= max_fd_pct or dns_latency_p99_ms > 250.0)
      now_iso = datetime.now(timezone.utc).isoformat()

      print(f"Squid Complete: Proxy={proxy_id}, FDUsed={fd_used}/{fd_max} ({fd_util_pct}%), Requests={requests_total}, Alert={is_breached}")

      Kestra.outputs({
          "file_descriptors_used_count": fd_used,
          "file_descriptors_max_count": fd_max,
          "file_descriptor_utilization_pct": fd_util_pct,
          "requests_total_count": requests_total,
          "dns_latency_p99_ms": dns_latency_p99_ms,
          "is_alerting": is_breached,
          "proxy_target": proxy_id,
          "audit_timestamp": now_iso
      })

  - id: evaluate_fd_utilization_threshold
    type: io.kestra.plugin.core.flow.If
    description: Verifies whether Squid proxy file descriptor utilization or DNS
      latency exceeds operational tolerance bounds.
    condition: "{{ outputs.scan_squid_proxy_metrics.vars.is_alerting and
      inputs.alert_on_fd_exhaustion }}"
    then:
      - id: notify_slack_netops_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident notification to NetOps Slack
          channel detailing Squid socket exhaustion risk.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#netops-proxy-sre",
            "username": "Squid Proxy Sentinel",
            "icon_emoji": ":lock:",
            "text": "CRITICAL: Squid forward proxy file descriptor exhaustion detected on cluster `{{ inputs.proxy_cluster_id }}`. FD Utilization: `{{ outputs.scan_squid_proxy_metrics.vars.file_descriptor_utilization_pct }}%` (`{{ outputs.scan_squid_proxy_metrics.vars.file_descriptors_used_count }}` / `{{ outputs.scan_squid_proxy_metrics.vars.file_descriptors_max_count }}`), threshold: `{{ inputs.max_fd_utilization_percent }}%`. DNS Latency P99: `{{ outputs.scan_squid_proxy_metrics.vars.dns_latency_p99_ms }}` ms. Proxy sockets may be rejected leading to corporate egress network failure."
          }
    else:
      - id: log_nominal_squid_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when Squid proxy handles client traffic
          without approaching file descriptor limits.
        message: "Squid proxy {{ inputs.proxy_cluster_id }} nominal. FD utilization: {{
          outputs.scan_squid_proxy_metrics.vars.file_descriptor_utilization_pct
          }}% ({{
          outputs.scan_squid_proxy_metrics.vars.file_descriptors_used_count
          }}/{{ outputs.scan_squid_proxy_metrics.vars.file_descriptors_max_count
          }}) is below threshold {{ inputs.max_fd_utilization_percent }}%."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for network proxy governance
      compliance records and socket capacity archives.
    format: |
      {
        "status": "{{ outputs.scan_squid_proxy_metrics.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "proxy_id": "{{ inputs.proxy_cluster_id }}",
        "fd_used": {{ outputs.scan_squid_proxy_metrics.vars.file_descriptors_used_count }},
        "fd_max": {{ outputs.scan_squid_proxy_metrics.vars.file_descriptors_max_count }},
        "fd_util_pct": {{ outputs.scan_squid_proxy_metrics.vars.file_descriptor_utilization_pct }},
        "audit_timestamp": "{{ outputs.scan_squid_proxy_metrics.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_squid_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"