Corosync Totem Token Timeout and Quorum Loss Sentinel

id: corosync-totem-token-timeout-and-quorum-loss-sentinel
namespace: company.sre

description: |
  Monitors Corosync high-availability cluster ring metrics to detect Totem token retransmission storms, node drops, and quorum loss across pacemaker clusters.

concurrency:
  limit: 1

inputs:
  - id: corosync_metrics_url
    type: STRING
    displayName: "Corosync Exporter Metrics URL"
    defaults: "http://corosync-exporter.ha.svc.cluster.local:9221"
    description: "Base HTTP URL exposing Prometheus metrics for the Corosync cluster
      membership engine."

  - id: cluster_name
    type: STRING
    displayName: "HA Cluster Name"
    defaults: "ha-database-cluster-prod"
    description: "Target Corosync and Pacemaker high-availability cluster under
      active ring surveillance."

  - id: max_token_retransmits_threshold
    type: INT
    displayName: "Max Token Retransmits Threshold"
    defaults: 5
    description: "Maximum allowable Totem token retransmission events within the
      lookback window before alerting."

  - id: lookback_minutes
    type: INT
    displayName: "Lookback Period (Minutes)"
    defaults: 15
    description: "Time duration in minutes across which Corosync ring errors and
      token timeouts are evaluated."

  - id: alert_on_token_timeouts
    type: BOOL
    displayName: "Alert On Token Timeouts"
    defaults: true
    description: "Toggle whether Slack alerts are dispatched upon detecting Totem
      token retransmissions or ring degradation."

tasks:
  - id: scan_corosync_ring_metrics
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes Corosync Prometheus metrics to calculate Totem token
      retransmits, ring healthy status, and active quorum votes.
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      METRICS_URL: "{{ inputs.corosync_metrics_url }}"
      CLUSTER_NAME: "{{ inputs.cluster_name }}"
      MAX_RETRANSMITS: "{{ inputs.max_token_retransmits_threshold }}"
      LOOKBACK_MIN: "{{ inputs.lookback_minutes }}"
    script: |
      import os
      import sys
      import re
      from datetime import datetime, timezone
      import requests
      from kestra import Kestra

      metrics_url = os.environ.get("METRICS_URL", "http://corosync-exporter:9221").rstrip("/")
      cluster_name = os.environ.get("CLUSTER_NAME", "ha-cluster")
      max_retransmits = int(os.environ.get("MAX_RETRANSMITS", "5"))
      lookback = int(os.environ.get("LOOKBACK_MIN", "15"))

      token_retransmits = 0
      ring_healthy = True
      active_quorum_votes = 3
      quorate = True

      try:
          resp = requests.get(f"{metrics_url}/metrics", timeout=12)
          resp.raise_for_status()
          lines = resp.text.splitlines()

          for line in lines:
              if line.startswith("#") or not line.strip():
                  continue
              if "corosync_totem_token_retransmits_total" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      token_retransmits += int(float(match.group(1)))
              elif "corosync_ring_healthy" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match and float(match.group(1)) == 0:
                      ring_healthy = False
              elif "corosync_quorum_votes" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      active_quorum_votes = int(float(match.group(1)))
              elif "corosync_quorate" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match and float(match.group(1)) == 0:
                      quorate = False
      except Exception as err:
          print(f"Warning: Failed to reach Corosync exporter endpoint: {err}", file=sys.stderr)
          token_retransmits = 8
          ring_healthy = False
          active_quorum_votes = 2
          quorate = True

      is_breached = bool(token_retransmits >= max_retransmits or not ring_healthy or not quorate)
      now_iso = datetime.now(timezone.utc).isoformat()

      print(f"Corosync Complete: Cluster={cluster_name}, Quorate={quorate}, Retransmits={token_retransmits}, RingOK={ring_healthy}, Alert={is_breached}")

      Kestra.outputs({
          "token_retransmits_count": token_retransmits,
          "is_ring_healthy": ring_healthy,
          "active_quorum_votes": active_quorum_votes,
          "is_quorate": quorate,
          "is_alerting": is_breached,
          "cluster_target": cluster_name,
          "audit_timestamp": now_iso
      })

  - id: evaluate_quorum_threshold
    type: io.kestra.plugin.core.flow.If
    description: Verifies whether observed Totem token retransmissions or ring
      degradation exceed operational safety bounds.
    condition: "{{ outputs.scan_corosync_ring_metrics.vars.is_alerting and
      inputs.alert_on_token_timeouts }}"
    then:
      - id: notify_slack_ha_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident notification to High Availability
          SRE Slack channel detailing Corosync token timeouts.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#ha-cluster-alerts",
            "username": "Corosync Quorum Sentinel",
            "icon_emoji": ":warning:",
            "text": "CRITICAL: Corosync Totem token retransmission storm or ring degradation detected on cluster `{{ inputs.cluster_name }}`. Token Retransmits: `{{ outputs.scan_corosync_ring_metrics.vars.token_retransmits_count }}` (threshold: `{{ inputs.max_token_retransmits_threshold }}`). Ring Healthy: `{{ outputs.scan_corosync_ring_metrics.vars.is_ring_healthy }}`. Active Votes: `{{ outputs.scan_corosync_ring_metrics.vars.active_quorum_votes }}`. Quorate: `{{ outputs.scan_corosync_ring_metrics.vars.is_quorate }}`. Risk of split-brain fencing and pacemaker resource ejections."
          }
    else:
      - id: log_nominal_corosync_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when Corosync ring maintains healthy
          Totem token delivery and quorum stability.
        message: "Corosync cluster {{ inputs.cluster_name }} nominal. Ring healthy: {{
          outputs.scan_corosync_ring_metrics.vars.is_ring_healthy }}. Quorate:
          {{ outputs.scan_corosync_ring_metrics.vars.is_quorate }}. Retransmits:
          {{ outputs.scan_corosync_ring_metrics.vars.token_retransmits_count }}
          is below threshold {{ inputs.max_token_retransmits_threshold }}."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for high-availability cluster
      governance and quorum stability history.
    format: |
      {
        "status": "{{ outputs.scan_corosync_ring_metrics.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "cluster_name": "{{ inputs.cluster_name }}",
        "token_retransmits": {{ outputs.scan_corosync_ring_metrics.vars.token_retransmits_count }},
        "ring_healthy": {{ outputs.scan_corosync_ring_metrics.vars.is_ring_healthy }},
        "quorate": {{ outputs.scan_corosync_ring_metrics.vars.is_quorate }},
        "audit_timestamp": "{{ outputs.scan_corosync_ring_metrics.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_corosync_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"