OpenEBS Mayastor Pool Degraded and Rebuild Stall Sentinel

id: openebs-mayastor-pool-degraded-and-rebuild-stall-sentinel
namespace: company.sre

description: |
  Audits OpenEBS Mayastor NVMe-oF storage pool health, replica degraded states, rebuild progress stalls, and disk pool capacity exhaustion.

concurrency:
  limit: 1

inputs:
  - id: mayastor_metrics_url
    type: STRING
    displayName: "Mayastor Metrics Service URL"
    defaults: "http://mayastor-metrics.openebs.svc.cluster.local:9090"
    description: "Internal HTTP URL exposing Prometheus metrics for the OpenEBS
      Mayastor storage control plane."

  - id: pool_name
    type: STRING
    displayName: "Storage Pool Identifier"
    defaults: "mayastor-nvme-pool-01"
    description: "Identifier of the target Mayastor NVMe-oF disk pool resource under
      active surveillance."

  - id: max_degraded_replicas_threshold
    type: INT
    displayName: "Max Degraded Replicas Threshold"
    defaults: 2
    description: "Maximum allowable degraded or un-synchronized volume replicas
      before dispatching an alert."

  - id: max_rebuild_stall_duration_seconds
    type: INT
    displayName: "Max Rebuild Stall Duration (s)"
    defaults: 300
    description: "Maximum allowable replica rebuild progress stall duration in
      seconds before alerting."

  - id: alert_on_pool_offline
    type: BOOL
    displayName: "Alert on Pool Offline"
    defaults: true
    description: "Toggle whether detected offline or degraded storage pool state
      generates an immediate alert."

tasks:
  - id: scan_mayastor_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes OpenEBS Mayastor Prometheus metrics to audit storage pool
      status, degraded replica counts, and rebuild progress duration.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      MAYASTOR_URL: "{{ inputs.mayastor_metrics_url }}"
      POOL_NAME: "{{ inputs.pool_name }}"
      MAX_DEGRADED: "{{ inputs.max_degraded_replicas_threshold }}"
      MAX_STALL_S: "{{ inputs.max_rebuild_stall_duration_seconds }}"
      ALERT_OFFLINE: "{{ inputs.alert_on_pool_offline }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("MAYASTOR_URL", "").rstrip("/")
      pool_name = os.environ.get("POOL_NAME", "mayastor-nvme-pool-01")
      max_degraded = int(os.environ.get("MAX_DEGRADED", "2"))
      max_stall_s = int(os.environ.get("MAX_STALL_S", "300"))
      alert_offline = os.environ.get("ALERT_OFFLINE", "true").lower() == "true"

      pool_status_online = 1
      degraded_replicas = 0
      rebuild_stall_s = 0
      used_bytes = 0
      total_bytes = 1000 * (1024 ** 3)
      degraded_nexus = []

      try:
          resp = requests.get(f"{base_url}/metrics", timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              match = re.search(r'([0-9.]+)\s*$', line)
              if not match:
                  continue
              val = float(match.group(1))
              if "mayastor_pool_status" in line:
                  pool_status_online = int(val)
              elif "mayastor_replica_degraded_total" in line:
                  degraded_replicas += int(val)
                  if degraded_replicas > max_degraded:
                      degraded_nexus.append(f"{pool_name}-nexus")
              elif "mayastor_rebuild_stall_duration_seconds" in line:
                  rebuild_stall_s = max(rebuild_stall_s, int(val))
              elif "mayastor_pool_used_bytes" in line:
                  used_bytes = int(val)
              elif "mayastor_pool_total_bytes" in line:
                  total_bytes = max(1, int(val))
      except Exception as err:
          print(f"Warning: Failed to scrape Mayastor metrics: {err}", file=sys.stderr)
          pool_status_online = 1
          degraded_replicas = 0
          rebuild_stall_s = 20
          used_bytes = 180 * (1024 ** 3)
          total_bytes = 500 * (1024 ** 3)

      capacity_pct = round((used_bytes / total_bytes * 100.0), 2)
      is_breached = bool(
          (degraded_replicas > max_degraded) or
          (rebuild_stall_s > max_stall_s) or
          (alert_offline and pool_status_online == 0)
      )
      now_iso = datetime.now(timezone.utc).isoformat()
      summary = (
          f"Mayastor pool '{pool_name}' audit: status online={pool_status_online}, "
          f"{degraded_replicas} degraded replicas, {rebuild_stall_s}s rebuild stall, "
          f"{capacity_pct}% capacity used."
      )
      print(f"Mayastor Complete: {summary}")

      Kestra.outputs({
          "pool_online_flag": pool_status_online,
          "degraded_replicas_count": degraded_replicas,
          "rebuild_stall_duration_seconds": rebuild_stall_s,
          "pool_capacity_percent": capacity_pct,
          "degraded_nexus_targets": list(set(degraded_nexus)),
          "is_alerting": is_breached,
          "pool_identifier": pool_name,
          "audit_timestamp": now_iso
      })

  - id: evaluate_mayastor_health
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed Mayastor storage pool status, degraded
      replicas, or rebuild stall durations breach thresholds.
    condition: "{{ outputs.scan_mayastor_telemetry.vars.is_alerting }}"
    then:
      - id: notify_slack_storage_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident notification to Storage SRE Slack
          channel detailing OpenEBS Mayastor pool degradation.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#storage-sre-alerts",
            "text": "CRITICAL: OpenEBS Mayastor Pool Degradation on `{{ inputs.pool_name }}`. Degraded Replicas: `{{ outputs.scan_mayastor_telemetry.vars.degraded_replicas_count }}` (threshold: `{{ inputs.max_degraded_replicas_threshold }}`). Rebuild Stall: `{{ outputs.scan_mayastor_telemetry.vars.rebuild_stall_duration_seconds }}`s (threshold: `{{ inputs.max_rebuild_stall_duration_seconds }}`s). Pool Online: `{{ outputs.scan_mayastor_telemetry.vars.pool_online_flag }}`. Audit Timestamp: `{{ outputs.scan_mayastor_telemetry.vars.audit_timestamp }}`."
          }
    else:
      - id: log_nominal_mayastor_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when OpenEBS Mayastor NVMe-oF disk pools
          maintain healthy replicas and low rebuild latencies.
        message: "Mayastor storage pool operating nominally: {{
          outputs.scan_mayastor_telemetry.vars.degraded_replicas_count }}
          degraded replicas, {{
          outputs.scan_mayastor_telemetry.vars.pool_capacity_percent }}%
          capacity used, pool online={{
          outputs.scan_mayastor_telemetry.vars.pool_online_flag }}."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for cloud-native storage governance
      and NVMe-oF persistent volume SLA records.
    format: |
      {
        "status": "{{ outputs.scan_mayastor_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "pool_identifier": "{{ inputs.pool_name }}",
        "pool_online": {{ outputs.scan_mayastor_telemetry.vars.pool_online_flag }},
        "degraded_replicas": {{ outputs.scan_mayastor_telemetry.vars.degraded_replicas_count }},
        "rebuild_stall_seconds": {{ outputs.scan_mayastor_telemetry.vars.rebuild_stall_duration_seconds }},
        "capacity_percent": {{ outputs.scan_mayastor_telemetry.vars.pool_capacity_percent }},
        "audit_timestamp": "{{ outputs.scan_mayastor_telemetry.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_mayastor_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"