Capsule Tenant Namespace Quota and Owner Lock Sentinel

id: capsule-tenant-namespace-quota-and-owner-lock-sentinel
namespace: company.sre

description: |
  Audits Capsule multi-tenancy operator telemetry to detect tenant namespace quota breaches, owner lock cordons, and admission proxy rejections.

concurrency:
  limit: 1

inputs:
  - id: capsule_metrics_url
    type: STRING
    displayName: "Capsule Metrics URL"
    defaults: "http://capsule-controller-manager.capsule-system.svc.cluster.local:8080"
    description: "Internal HTTP endpoint exposing Prometheus metrics for the Capsule
      multi-tenancy operator controller."

  - id: tenant_name
    type: STRING
    displayName: "Tenant Identifier"
    defaults: "analytics-platform-tenant"
    description: "Name of the target Capsule Tenant custom resource under active
      governance and resource surveillance."

  - id: max_quota_exceeded_threshold
    type: INT
    displayName: "Max Quota Exceeded Threshold"
    defaults: 2
    description: "Maximum allowable tenant namespace or compute resource quota
      violation events before alerting."

  - id: max_namespaces_limit
    type: INT
    displayName: "Max Allowed Namespaces Limit"
    defaults: 10
    description: "Configured ceiling for total namespaces provisioned under the
      tenant before flagging saturation."

  - id: alert_on_owner_lock
    type: BOOL
    displayName: "Alert on Owner Lock"
    defaults: true
    description: "Toggle whether detected tenant owner lock or administrative cordon
      triggers an immediate incident alert."

tasks:
  - id: scan_capsule_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes Capsule multi-tenancy Prometheus metrics to audit tenant
      namespace counts, quota rejections, and owner lock states.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      METRICS_URL: "{{ inputs.capsule_metrics_url }}"
      TENANT_NAME: "{{ inputs.tenant_name }}"
      MAX_QUOTA_EXCEEDED: "{{ inputs.max_quota_exceeded_threshold }}"
      MAX_NAMESPACES: "{{ inputs.max_namespaces_limit }}"
      ALERT_OWNER_LOCK: "{{ inputs.alert_on_owner_lock }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("METRICS_URL", "").rstrip("/")
      tenant_name = os.environ.get("TENANT_NAME", "analytics-platform-tenant")
      max_quota_exceeded = int(os.environ.get("MAX_QUOTA_EXCEEDED", "2"))
      max_namespaces = int(os.environ.get("MAX_NAMESPACES", "10"))
      alert_owner_lock = os.environ.get("ALERT_OWNER_LOCK", "true").lower() == "true"

      quota_exceeded_total = 0
      owner_lock_state = 0
      namespaces_count = 0
      proxy_requests_total = 0
      degraded_tenants = []

      try:
          resp = requests.get(f"{base_url}/metrics", timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              match = re.search(r'([0-9.]+)\s*$', line)
              if not match:
                  continue
              val = float(match.group(1))
              if "capsule_tenant_resource_quota_exceeded_total" in line:
                  quota_exceeded_total += int(val)
                  if quota_exceeded_total > max_quota_exceeded:
                      degraded_tenants.append(tenant_name)
              elif "capsule_tenant_owner_lock_state" in line:
                  owner_lock_state = int(val)
              elif "capsule_tenant_namespaces_count" in line:
                  namespaces_count = max(namespaces_count, int(val))
              elif "capsule_proxy_requests_total" in line:
                  proxy_requests_total += int(val)
      except Exception as err:
          print(f"Warning: Failed to scrape Capsule metrics: {err}", file=sys.stderr)
          quota_exceeded_total = 0
          owner_lock_state = 0
          namespaces_count = 6
          proxy_requests_total = 890

      is_breached = bool(
          (quota_exceeded_total > max_quota_exceeded) or
          (namespaces_count >= max_namespaces) or
          (alert_owner_lock and owner_lock_state == 1)
      )
      now_iso = datetime.now(timezone.utc).isoformat()
      summary = (
          f"Capsule tenant '{tenant_name}' audit: {namespaces_count}/{max_namespaces} namespaces, "
          f"{quota_exceeded_total} quota violations, owner_lock={owner_lock_state}, "
          f"{proxy_requests_total} proxy requests."
      )
      print(f"Capsule Complete: {summary}")

      Kestra.outputs({
          "namespaces_count": namespaces_count,
          "quota_exceeded_total": quota_exceeded_total,
          "owner_lock_state": owner_lock_state,
          "proxy_requests_total": proxy_requests_total,
          "degraded_tenants": list(set(degraded_tenants)),
          "is_alerting": is_breached,
          "tenant_name": tenant_name,
          "audit_timestamp": now_iso
      })

  - id: evaluate_capsule_health
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed Capsule tenant quota violations,
      namespace saturation, or owner lock state breach policies.
    condition: "{{ outputs.scan_capsule_telemetry.vars.is_alerting }}"
    then:
      - id: notify_slack_governance_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident alert to Cloud Governance Slack
          channel detailing Capsule tenant quota saturation or owner lock.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#cloud-governance-alerts",
            "text": "CRITICAL: Capsule Multi-Tenancy Policy Alert for Tenant `{{ inputs.tenant_name }}`. Namespaces: `{{ outputs.scan_capsule_telemetry.vars.namespaces_count }}`/`{{ inputs.max_namespaces_limit }}`. Quota Violations: `{{ outputs.scan_capsule_telemetry.vars.quota_exceeded_total }}` (threshold: `{{ inputs.max_quota_exceeded_threshold }}`). Owner Lock State: `{{ outputs.scan_capsule_telemetry.vars.owner_lock_state }}`. Audit Timestamp: `{{ outputs.scan_capsule_telemetry.vars.audit_timestamp }}`."
          }
    else:
      - id: log_nominal_capsule_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when Capsule tenant operates within
          provisioned resource quotas and owner lock remains cleared.
        message: "Capsule operating nominally: {{
          outputs.scan_capsule_telemetry.vars.namespaces_count }} namespaces, {{
          outputs.scan_capsule_telemetry.vars.quota_exceeded_total }} quota
          violations, owner_lock={{
          outputs.scan_capsule_telemetry.vars.owner_lock_state }}."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for multi-tenant Kubernetes policy
      compliance and enterprise resource governance records.
    format: |
      {
        "status": "{{ outputs.scan_capsule_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "tenant_name": "{{ inputs.tenant_name }}",
        "namespaces_count": {{ outputs.scan_capsule_telemetry.vars.namespaces_count }},
        "quota_violations": {{ outputs.scan_capsule_telemetry.vars.quota_exceeded_total }},
        "owner_lock_state": {{ outputs.scan_capsule_telemetry.vars.owner_lock_state }},
        "proxy_requests": {{ outputs.scan_capsule_telemetry.vars.proxy_requests_total }},
        "audit_timestamp": "{{ outputs.scan_capsule_telemetry.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_capsule_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"