Keepalived VRRP State Transition Flapping Sentinel

id: keepalived-vrrp-state-transition-flapping-sentinel
namespace: company.sre

description: |
  Monitors Keepalived VRRP daemon telemetry to detect virtual IP state flapping, split-brain master elections, and advertisement packet timeouts.

concurrency:
  limit: 1

inputs:
  - id: keepalived_metrics_url
    type: STRING
    displayName: "Keepalived Metrics URL"
    defaults: "http://keepalived-exporter.networking.svc.cluster.local:9650"
    description: "Base HTTP URL exposing Prometheus metrics for the Keepalived VRRP
      failover daemon."

  - id: vrrp_instance_id
    type: STRING
    displayName: "VRRP Instance Identifier"
    defaults: "VI_LOAD_BALANCER_EXTERNAL"
    description: "Target VRRP virtual router instance name under active failover
      surveillance."

  - id: max_state_transitions_threshold
    type: INT
    displayName: "Max State Transitions"
    defaults: 3
    description: "Maximum allowable VRRP master-backup state transitions within the
      lookback window before alerting."

  - id: lookback_minutes
    type: INT
    displayName: "Lookback Period (Minutes)"
    defaults: 15
    description: "Time duration in minutes across which VRRP transition counters and
      error metrics are evaluated."

  - id: alert_on_vrrp_flapping
    type: BOOL
    displayName: "Alert On VRRP Flapping"
    defaults: true
    description: "Toggle whether Slack alerts are dispatched upon detecting VRRP
      state flapping or split-brain conditions."

tasks:
  - id: scan_keepalived_vrrp_metrics
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes Keepalived Prometheus metrics to calculate VRRP state
      transitions, advert errors, and master state stability.
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      METRICS_URL: "{{ inputs.keepalived_metrics_url }}"
      INSTANCE_ID: "{{ inputs.vrrp_instance_id }}"
      MAX_TRANSITIONS: "{{ inputs.max_state_transitions_threshold }}"
      LOOKBACK_MIN: "{{ inputs.lookback_minutes }}"
    script: |
      import os
      import sys
      import re
      from datetime import datetime, timezone
      import requests
      from kestra import Kestra

      metrics_url = os.environ.get("METRICS_URL", "http://keepalived-exporter:9650").rstrip("/")
      instance_id = os.environ.get("INSTANCE_ID", "VI_LB")
      max_transitions = int(os.environ.get("MAX_TRANSITIONS", "3"))
      lookback = int(os.environ.get("LOOKBACK_MIN", "15"))

      state_transitions = 0
      advert_errors = 0
      current_state_code = 1
      is_master = False

      try:
          resp = requests.get(f"{metrics_url}/metrics", timeout=12)
          resp.raise_for_status()
          lines = resp.text.splitlines()

          for line in lines:
              if line.startswith("#") or not line.strip():
                  continue
              if "keepalived_vrrp_state_transitions_total" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      state_transitions += int(float(match.group(1)))
              elif "keepalived_vrrp_advert_errors_total" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      advert_errors += int(float(match.group(1)))
              elif "keepalived_vrrp_state" in line and 'state="master"' in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match and float(match.group(1)) > 0:
                      is_master = True
                      current_state_code = 2
      except Exception as err:
          print(f"Warning: Failed to reach Keepalived exporter endpoint: {err}", file=sys.stderr)
          state_transitions = 5
          advert_errors = 3
          is_master = False
          current_state_code = 1

      state_name = "MASTER" if is_master else "BACKUP"
      is_breached = bool(state_transitions >= max_transitions or advert_errors > 2)
      now_iso = datetime.now(timezone.utc).isoformat()

      print(f"Keepalived Audit Complete: Instance={instance_id}, State={state_name}, Transitions={state_transitions}, Errors={advert_errors}, Alert={is_breached}")

      Kestra.outputs({
          "state_transitions_count": state_transitions,
          "advert_errors_count": advert_errors,
          "current_vrrp_state": state_name,
          "is_master": is_master,
          "is_alerting": is_breached,
          "instance_target": instance_id,
          "audit_timestamp": now_iso
      })

  - id: evaluate_flapping_threshold
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed VRRP state transitions or advertisement
      packet errors exceed stability thresholds.
    condition: "{{ outputs.scan_keepalived_vrrp_metrics.vars.is_alerting and
      inputs.alert_on_vrrp_flapping }}"
    then:
      - id: notify_slack_netops_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident notification to NetOps Slack
          channel detailing Keepalived VRRP flapping.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#netops-high-priority",
            "username": "Keepalived VRRP Sentinel",
            "icon_emoji": ":twisted_rightwards_arrows:",
            "text": "CRITICAL: Keepalived VRRP state flapping detected on instance `{{ inputs.vrrp_instance_id }}`. Transitions: `{{ outputs.scan_keepalived_vrrp_metrics.vars.state_transitions_count }}` (threshold: `{{ inputs.max_state_transitions_threshold }}`). Current State: `{{ outputs.scan_keepalived_vrrp_metrics.vars.current_vrrp_state }}`. Advert Errors: `{{ outputs.scan_keepalived_vrrp_metrics.vars.advert_errors_count }}`. Risk of split-brain virtual IP collisions and ARP churn."
          }
    else:
      - id: log_nominal_vrrp_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when Keepalived VRRP failover daemon
          maintains stable single-master quorum.
        message: "Keepalived VRRP instance {{ inputs.vrrp_instance_id }} nominal. State:
          {{ outputs.scan_keepalived_vrrp_metrics.vars.current_vrrp_state }}.
          State transitions {{
          outputs.scan_keepalived_vrrp_metrics.vars.state_transitions_count }}
          is below threshold {{ inputs.max_state_transitions_threshold }}."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for high-availability networking
      logs and gateway failover history.
    format: |
      {
        "status": "{{ outputs.scan_keepalived_vrrp_metrics.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "instance_id": "{{ inputs.vrrp_instance_id }}",
        "state_transitions": {{ outputs.scan_keepalived_vrrp_metrics.vars.state_transitions_count }},
        "advert_errors": {{ outputs.scan_keepalived_vrrp_metrics.vars.advert_errors_count }},
        "current_state": "{{ outputs.scan_keepalived_vrrp_metrics.vars.current_vrrp_state }}",
        "audit_timestamp": "{{ outputs.scan_keepalived_vrrp_metrics.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_keepalived_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"