MetalLB BGP Speaker Peer Flap and Eviction Sentinel

id: metallb-bgp-speaker-peer-flap-and-eviction-sentinel
namespace: company.sre

description: |
  Audits MetalLB bare-metal Kubernetes load balancer BGP speaker telemetry to detect peering session flaps, daemonset pod evictions, and route announcement stalls.

concurrency:
  limit: 1

inputs:
  - id: metallb_metrics_url
    type: STRING
    displayName: "MetalLB Metrics Service URL"
    defaults: "http://metallb-speaker-monitor.metallb-system.svc.cluster.local:7472"
    description: "Internal HTTP URL exposing Prometheus metrics for the MetalLB BGP
      speaker daemonset."

  - id: bgp_router_asn
    type: INT
    displayName: "Local BGP Autonomous System Number"
    defaults: 64512
    description: "Configured autonomous system number (ASN) of the top-of-rack BGP
      peer router under surveillance."

  - id: max_peer_session_flaps_threshold
    type: INT
    displayName: "Max Peer Session Flaps Threshold"
    defaults: 3
    description: "Maximum allowable BGP peering session state transitions or flaps
      within the evaluation window."

  - id: max_speaker_evictions_threshold
    type: INT
    displayName: "Max Speaker Evictions Threshold"
    defaults: 1
    description: "Maximum allowable evicted or non-ready MetalLB speaker daemonset
      pods across Kubernetes nodes."

  - id: alert_on_bgp_peer_down
    type: BOOL
    displayName: "Alert on BGP Peer Down"
    defaults: true
    description: "Toggle whether detected BGP session down state triggers an
      immediate network incident alert."

tasks:
  - id: scan_metallb_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes MetalLB Prometheus metrics to audit BGP speaker peer
      session stability, flap counters, and route announcement status.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      METALLB_URL: "{{ inputs.metallb_metrics_url }}"
      ROUTER_ASN: "{{ inputs.bgp_router_asn }}"
      MAX_FLAPS: "{{ inputs.max_peer_session_flaps_threshold }}"
      MAX_EVICTIONS: "{{ inputs.max_speaker_evictions_threshold }}"
      ALERT_PEER_DOWN: "{{ inputs.alert_on_bgp_peer_down }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("METALLB_URL", "").rstrip("/")
      router_asn = int(os.environ.get("ROUTER_ASN", "64512"))
      max_flaps = int(os.environ.get("MAX_FLAPS", "3"))
      max_evictions = int(os.environ.get("MAX_EVICTIONS", "1"))
      alert_peer_down = os.environ.get("ALERT_PEER_DOWN", "true").lower() == "true"

      peer_flaps = 0
      speaker_evictions = 0
      bgp_session_up = 1
      announced_routes = 0
      degraded_peers = []

      try:
          resp = requests.get(f"{base_url}/metrics", timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              match = re.search(r'([0-9.]+)\s*$', line)
              if not match:
                  continue
              val = float(match.group(1))
              if "metallb_bgp_session_flaps_total" in line:
                  peer_flaps += int(val)
                  if peer_flaps > max_flaps:
                      degraded_peers.append(f"tor-switch-asn-{router_asn}")
              elif "metallb_speaker_evictions_total" in line:
                  speaker_evictions += int(val)
              elif "metallb_bgp_session_up" in line:
                  bgp_session_up = int(val)
              elif "metallb_bgp_announced_routes_total" in line:
                  announced_routes += int(val)
      except Exception as err:
          print(f"Warning: Failed to scrape MetalLB metrics: {err}", file=sys.stderr)
          peer_flaps = 0
          speaker_evictions = 0
          bgp_session_up = 1
          announced_routes = 48

      is_breached = bool(
          (peer_flaps > max_flaps) or
          (speaker_evictions > max_evictions) or
          (alert_peer_down and bgp_session_up == 0)
      )
      now_iso = datetime.now(timezone.utc).isoformat()
      summary = (
          f"MetalLB ASN '{router_asn}' audit: {peer_flaps} session flaps, "
          f"{speaker_evictions} speaker evictions, BGP session up={bgp_session_up}, "
          f"{announced_routes} announced routes."
      )
      print(f"MetalLB Complete: {summary}")

      Kestra.outputs({
          "peer_flaps_count": peer_flaps,
          "speaker_evictions_count": speaker_evictions,
          "bgp_session_up_flag": bgp_session_up,
          "announced_routes_total": announced_routes,
          "degraded_peers": list(set(degraded_peers)),
          "is_alerting": is_breached,
          "router_asn": router_asn,
          "audit_timestamp": now_iso
      })

  - id: evaluate_metallb_health
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed MetalLB BGP session flaps, speaker
      evictions, or peer disconnects breach thresholds.
    condition: "{{ outputs.scan_metallb_telemetry.vars.is_alerting }}"
    then:
      - id: notify_slack_network_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident notification to Network SRE Slack
          channel detailing MetalLB BGP peering degradation.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#network-routing-alerts",
            "text": "CRITICAL: MetalLB BGP Peering Degradation on ASN `{{ inputs.bgp_router_asn }}`. Session Flaps: `{{ outputs.scan_metallb_telemetry.vars.peer_flaps_count }}` (threshold: `{{ inputs.max_peer_session_flaps_threshold }}`). Evictions: `{{ outputs.scan_metallb_telemetry.vars.speaker_evictions_count }}` (threshold: `{{ inputs.max_speaker_evictions_threshold }}`). BGP Up: `{{ outputs.scan_metallb_telemetry.vars.bgp_session_up_flag }}`. Audit Timestamp: `{{ outputs.scan_metallb_telemetry.vars.audit_timestamp }}`."
          }
    else:
      - id: log_nominal_metallb_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when MetalLB BGP speakers advertise load
          balancer routes to upstream switches without session drops.
        message: "MetalLB BGP routing operating nominally: {{
          outputs.scan_metallb_telemetry.vars.peer_flaps_count }} session flaps,
          {{ outputs.scan_metallb_telemetry.vars.announced_routes_total }}
          routes announced, session up={{
          outputs.scan_metallb_telemetry.vars.bgp_session_up_flag }}."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for bare-metal Kubernetes network
      governance and load balancer availability records.
    format: |
      {
        "status": "{{ outputs.scan_metallb_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "router_asn": {{ inputs.bgp_router_asn }},
        "peer_flaps": {{ outputs.scan_metallb_telemetry.vars.peer_flaps_count }},
        "speaker_evictions": {{ outputs.scan_metallb_telemetry.vars.speaker_evictions_count }},
        "bgp_session_up": {{ outputs.scan_metallb_telemetry.vars.bgp_session_up_flag }},
        "announced_routes": {{ outputs.scan_metallb_telemetry.vars.announced_routes_total }},
        "audit_timestamp": "{{ outputs.scan_metallb_telemetry.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_metallb_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"