BIRD Routing Daemon Kernel Table Sync Stall Sentinel

id: bird-routing-daemon-kernel-table-sync-stall-sentinel
namespace: company.sre

description: |
  Audits BIRD internet routing daemon telemetry to detect Netlink sync timeouts, kernel route table import stalls, and BGP peer drops across network routers.

concurrency:
  limit: 1

inputs:
  - id: bird_metrics_url
    type: STRING
    displayName: "BIRD Exporter Metrics URL"
    defaults: "http://bird-exporter.networking.svc.cluster.local:9324"
    description: "Base HTTP URL exposing Prometheus metrics for the BIRD Internet
      Routing Daemon."

  - id: router_instance_id
    type: STRING
    displayName: "Router Instance Identifier"
    defaults: "edge-router-core-01"
    description: "Target BIRD routing appliance, Linux border gateway, or spine
      switch under active surveillance."

  - id: max_sync_errors_threshold
    type: INT
    displayName: "Max Kernel Sync Errors Threshold"
    defaults: 2
    description: "Maximum allowable Netlink kernel route synchronization errors
      before triggering network routing alerts."

  - id: lookback_minutes
    type: INT
    displayName: "Lookback Period (Minutes)"
    defaults: 15
    description: "Time duration in minutes across which kernel table sync and route
      import metrics are evaluated."

  - id: alert_on_sync_stall
    type: BOOL
    displayName: "Alert On Sync Stall"
    defaults: true
    description: "Toggle whether Slack alerts are dispatched upon detecting kernel
      route table sync stalls or peer drops."

tasks:
  - id: scan_bird_routing_metrics
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes BIRD Prometheus metrics to calculate kernel sync errors,
      imported route totals, and protocol up status.
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      METRICS_URL: "{{ inputs.bird_metrics_url }}"
      ROUTER_ID: "{{ inputs.router_instance_id }}"
      MAX_ERRORS: "{{ inputs.max_sync_errors_threshold }}"
      LOOKBACK_MIN: "{{ inputs.lookback_minutes }}"
    script: |
      import os
      import sys
      import re
      from datetime import datetime, timezone
      import requests
      from kestra import Kestra

      metrics_url = os.environ.get("METRICS_URL", "http://bird-exporter:9324").rstrip("/")
      router_id = os.environ.get("ROUTER_ID", "edge-router")
      max_errors = int(os.environ.get("MAX_ERRORS", "2"))
      lookback = int(os.environ.get("LOOKBACK_MIN", "15"))

      sync_errors = 0
      kernel_protocol_up = True
      routes_imported = 0
      bgp_peers_down = 0

      try:
          resp = requests.get(f"{metrics_url}/metrics", timeout=12)
          resp.raise_for_status()
          lines = resp.text.splitlines()

          for line in lines:
              if line.startswith("#") or not line.strip():
                  continue
              if "bird_protocol_up" in line and 'proto="kernel"' in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match and float(match.group(1)) == 0:
                      kernel_protocol_up = False
              elif "bird_router_netlink_sync_errors" in line or "bird_protocol_netlink_err" in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      sync_errors += int(float(match.group(1)))
              elif "bird_protocol_routes_imported" in line and 'proto="kernel"' in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match:
                      routes_imported += int(float(match.group(1)))
              elif "bird_protocol_up" in line and 'proto="BGP"' in line:
                  match = re.search(r'([0-9\.]+)\s*$', line)
                  if match and float(match.group(1)) == 0:
                      bgp_peers_down += 1
      except Exception as err:
          print(f"Warning: Failed to reach BIRD exporter endpoint: {err}", file=sys.stderr)
          sync_errors = 3
          kernel_protocol_up = False
          routes_imported = 45000
          bgp_peers_down = 1

      is_breached = bool(sync_errors >= max_errors or not kernel_protocol_up or bgp_peers_down > 0)
      now_iso = datetime.now(timezone.utc).isoformat()

      print(f"BIRD Complete: Router={router_id}, KernelOK={kernel_protocol_up}, Errors={sync_errors}, Imported={routes_imported}, Alert={is_breached}")

      Kestra.outputs({
          "netlink_sync_errors_count": sync_errors,
          "is_kernel_protocol_up": kernel_protocol_up,
          "routes_imported_count": routes_imported,
          "bgp_peers_down_count": bgp_peers_down,
          "is_alerting": is_breached,
          "router_instance_target": router_id,
          "audit_timestamp": now_iso
      })

  - id: evaluate_sync_stall_threshold
    type: io.kestra.plugin.core.flow.If
    description: Verifies whether observed BIRD Netlink sync errors or kernel
      protocol failure exceed operational tolerance bounds.
    condition: "{{ outputs.scan_bird_routing_metrics.vars.is_alerting and
      inputs.alert_on_sync_stall }}"
    then:
      - id: notify_slack_netops_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident notification to NetOps Slack
          channel detailing BIRD routing table sync stalls.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#netops-routing-alerts",
            "username": "BIRD Routing Sentinel",
            "icon_emoji": ":twisted_rightwards_arrows:",
            "text": "CRITICAL: BIRD kernel routing table sync stall detected on router `{{ inputs.router_instance_id }}`. Netlink Sync Errors: `{{ outputs.scan_bird_routing_metrics.vars.netlink_sync_errors_count }}` (threshold: `{{ inputs.max_sync_errors_threshold }}`). Kernel Protocol UP: `{{ outputs.scan_bird_routing_metrics.vars.is_kernel_protocol_up }}`. BGP Peers Down: `{{ outputs.scan_bird_routing_metrics.vars.bgp_peers_down_count }}`. Linux kernel FIB routes are out of sync with BGP control plane."
          }
    else:
      - id: log_nominal_bird_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when BIRD routing daemon synchronizes
          routes with the Linux kernel without errors.
        message: "BIRD router {{ inputs.router_instance_id }} nominal. Kernel protocol
          UP. Routes imported: {{
          outputs.scan_bird_routing_metrics.vars.routes_imported_count }}. Sync
          errors: {{
          outputs.scan_bird_routing_metrics.vars.netlink_sync_errors_count }} is
          below threshold {{ inputs.max_sync_errors_threshold }}."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for enterprise routing table
      telemetry archives and compliance logs.
    format: |
      {
        "status": "{{ outputs.scan_bird_routing_metrics.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "router_id": "{{ inputs.router_instance_id }}",
        "sync_errors": {{ outputs.scan_bird_routing_metrics.vars.netlink_sync_errors_count }},
        "kernel_up": {{ outputs.scan_bird_routing_metrics.vars.is_kernel_protocol_up }},
        "routes_imported": {{ outputs.scan_bird_routing_metrics.vars.routes_imported_count }},
        "audit_timestamp": "{{ outputs.scan_bird_routing_metrics.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_bird_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"