id: tailscale-subnet-router-failover-and-ha-sentinel
namespace: company.networking
description: "Continuously monitors Tailscale subnet routers for high
availability and failover readiness, verifies advertised and enabled CIDR
routes, detects node outages, and alerts on VPC connectivity loss."
inputs:
- id: tailnet
type: STRING
defaults: "example.org"
description: "Tailscale tailnet name (e.g., example.org, or '-' to use the
default tailnet associated with the API key)."
- id: target_subnet
type: STRING
defaults: "10.0.0.0/16"
description: "Target VPC / subnet CIDR prefix advertised by Tailscale subnet routers."
- id: primary_router_hostname
type: STRING
defaults: "subnet-router-primary"
description: "Hostname or identifier of the designated primary subnet router node."
- id: max_offline_minutes
type: INT
defaults: 5
description: "Threshold in minutes beyond which an unresponsive node is marked offline."
- id: slack_channel
type: STRING
defaults: "#network-alerts"
description: "Target Slack channel for failover notices and connectivity outage alerts."
- id: enable_pagerduty
type: BOOL
defaults: false
description: "When true, triggers high-severity incident dispatch to PagerDuty
upon total VPC connectivity loss."
triggers:
- id: periodic_ha_sentinel
type: io.kestra.plugin.core.trigger.Schedule
cron: "*/5 * * * *"
disabled: true
tasks:
- id: probe_tailscale_devices
type: io.kestra.plugin.core.http.Request
description: "Queries Tailscale REST API to retrieve registered devices,
advertised routes, and connectivity states."
uri: "https://api.tailscale.com/api/v2/tailnet/{{ inputs.tailnet
}}/devices?fields=all"
method: GET
headers:
Authorization: "Bearer {{ secret('TAILSCALE_API_KEY') }}"
Accept: "application/json"
options:
allowFailed: true
timeout:
readIdleTimeout: PT30S
- id: audit_subnet_routers
type: io.kestra.plugin.scripts.python.Script
description: "Analyzes advertised versus enabled subnet routes, router heartbeat
timestamps, and failover status."
containerImage: python:3.12-slim
inputFiles:
devices.json: "{{ outputs.probe_tailscale_devices.body | json }}"
script: |
import json
from datetime import datetime, timezone
now = datetime.now(timezone.utc)
target_subnet = "{{ inputs.target_subnet }}"
primary_hostname = "{{ inputs.primary_router_hostname }}"
max_offline_minutes = {{ inputs.max_offline_minutes }}
try:
with open("devices.json", "r", encoding="utf-8") as f:
raw_content = f.read().strip()
devices_data = json.loads(raw_content) if raw_content else {}
except Exception:
devices_data = {}
if isinstance(devices_data, dict) and "message" in devices_data and "devices" not in devices_data:
err_msg = devices_data.get("message", "Unknown API error")
report = {
"status": "API_ERROR",
"severity": "CRITICAL",
"target_subnet": target_subnet,
"vpc_connectivity_intact": False,
"active_routing_node": "NONE",
"total_routers": 0,
"healthy_routers_count": 0,
"healthy_primary_count": 0,
"healthy_standby_count": 0,
"primary_routers": [],
"standby_routers": [],
"unapproved_nodes": [],
"summary": f"Tailscale API error: {err_msg}"
}
else:
raw_devices = []
if isinstance(devices_data, dict):
raw_devices = devices_data.get("devices", [])
elif isinstance(devices_data, list):
raw_devices = devices_data
subnet_routers = []
unapproved_nodes = []
for dev in raw_devices:
adv = dev.get("advertisedRoutes") or []
enb = dev.get("enabledRoutes") or []
matches_subnet = False
if not target_subnet:
matches_subnet = bool(adv or enb)
else:
matches_subnet = (target_subnet in adv) or (target_subnet in enb)
if not matches_subnet:
continue
is_online = bool(dev.get("online", False))
last_seen_str = dev.get("lastSeen", "")
last_seen_age_sec = 0
last_seen_valid = False
if last_seen_str:
try:
clean_ts = last_seen_str.replace("Z", "+00:00")
seen_dt = datetime.fromisoformat(clean_ts)
last_seen_age_sec = max(0, int((now - seen_dt).total_seconds()))
last_seen_valid = True
except Exception:
pass
offline_by_time = last_seen_valid and (last_seen_age_sec > max_offline_minutes * 60)
is_node_online = is_online and not offline_by_time
is_route_enabled = (target_subnet in enb) if target_subnet else bool(enb)
if (target_subnet in adv) and not is_route_enabled:
unapproved_nodes.append(dev.get("hostname") or dev.get("name") or str(dev.get("id")))
hostname = dev.get("hostname") or dev.get("name") or str(dev.get("id")) or "unknown"
tags = dev.get("tags") or []
is_primary = False
if primary_hostname and (primary_hostname.lower() in hostname.lower() or "tag:primary" in tags):
is_primary = True
router_info = {
"id": str(dev.get("id")),
"name": dev.get("name", hostname),
"hostname": hostname,
"addresses": dev.get("addresses") or [],
"online": is_node_online,
"last_seen": last_seen_str,
"last_seen_age_sec": last_seen_age_sec,
"is_healthy": is_node_online and is_route_enabled,
"is_primary": is_primary,
"route_enabled": is_route_enabled,
"advertised_routes": adv,
"enabled_routes": enb
}
subnet_routers.append(router_info)
if subnet_routers and not any(r["is_primary"] for r in subnet_routers):
subnet_routers[0]["is_primary"] = True
primary_routers = [r for r in subnet_routers if r["is_primary"]]
standby_routers = [r for r in subnet_routers if not r["is_primary"]]
healthy_primary = [r for r in primary_routers if r["is_healthy"]]
healthy_standby = [r for r in standby_routers if r["is_healthy"]]
total_healthy = len(healthy_primary) + len(healthy_standby)
active_routing_node = "NONE"
if healthy_primary:
active_routing_node = healthy_primary[0]["hostname"]
elif healthy_standby:
active_routing_node = healthy_standby[0]["hostname"]
if not subnet_routers:
status = "NO_ROUTERS_FOUND"
severity = "CRITICAL"
vpc_connectivity = False
summary = f"No Tailscale subnet routers found advertising target subnet '{target_subnet}'."
elif total_healthy == 0:
status = "TOTAL_SUBNET_OUTAGE"
severity = "CRITICAL"
vpc_connectivity = False
summary = f"CRITICAL: 0 healthy subnet routers online for subnet {target_subnet}. Complete loss of VPC network connectivity!"
elif not healthy_primary and healthy_standby:
status = "HA_FAILOVER_ACTIVE"
severity = "WARNING"
vpc_connectivity = True
summary = f"WARNING: Primary router is OFFLINE. Subnet traffic failed over to standby router '{active_routing_node}'. VPC connectivity intact but HA redundancy lost."
elif healthy_primary and standby_routers and not healthy_standby:
status = "STANDBY_ROUTER_DEGRADED"
severity = "WARNING"
vpc_connectivity = True
summary = f"WARNING: Standby router is OFFLINE. Primary router '{active_routing_node}' is active, but standby redundancy is lost."
elif unapproved_nodes and not total_healthy:
status = "ROUTE_APPROVAL_PENDING"
severity = "WARNING"
vpc_connectivity = False
summary = f"WARNING: Subnet routes are advertised but not approved in Tailscale console for nodes: {', '.join(unapproved_nodes)}."
elif unapproved_nodes:
status = "ROUTE_APPROVAL_PENDING"
severity = "WARNING"
vpc_connectivity = True
summary = f"WARNING: Route approval pending for nodes: {', '.join(unapproved_nodes)}."
else:
status = "NOMINAL_HEALTHY"
severity = "INFO"
vpc_connectivity = True
summary = f"NOMINAL: Primary router '{active_routing_node}' and {len(healthy_standby)} standby router(s) online for subnet {target_subnet}."
report = {
"timestamp": now.isoformat(),
"status": status,
"severity": severity,
"target_subnet": target_subnet,
"vpc_connectivity_intact": vpc_connectivity,
"active_routing_node": active_routing_node,
"total_routers": len(subnet_routers),
"healthy_routers_count": total_healthy,
"healthy_primary_count": len(healthy_primary),
"healthy_standby_count": len(healthy_standby),
"primary_routers": primary_routers,
"standby_routers": standby_routers,
"unapproved_nodes": unapproved_nodes,
"summary": summary
}
output_data = {
"status": report["status"],
"severity": report["severity"],
"target_subnet": report["target_subnet"],
"vpc_connectivity_intact": report["vpc_connectivity_intact"],
"active_routing_node": report["active_routing_node"],
"total_routers": report["total_routers"],
"healthy_routers_count": report["healthy_routers_count"],
"healthy_primary_count": report["healthy_primary_count"],
"healthy_standby_count": report["healthy_standby_count"],
"summary": report["summary"],
"health_report": report
}
try:
from kestra import Kestra
Kestra.outputs(output_data)
except ImportError:
pass
print(f"::{{'outputs': {json.dumps(output_data)}}}::")
- id: record_subnet_health_kv
type: io.kestra.plugin.core.kv.Set
description: "Stores evaluated subnet router health status, failover state, and
active router node in Kestra KV store."
key: "tailscale_subnet_{{ inputs.target_subnet | replace({'/': '_', '.': '_'})
}}"
kvType: JSON
value: "{{ outputs.audit_subnet_routers.vars.health_report | json }}"
- id: evaluate_sentinel_state
type: io.kestra.plugin.core.flow.Switch
description: "Dispatches alerts or logs based on the assessed health and
failover state of Tailscale subnet routers."
value: "{{ outputs.audit_subnet_routers.vars.status }}"
cases:
TOTAL_SUBNET_OUTAGE:
- id: alert_total_outage_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Dispatches critical Slack alert when zero healthy subnet routers
are available for the target subnet."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": ":rotating_light: [CRITICAL] Tailscale Subnet Router Outage - Loss of VPC Network Connectivity!",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "[CRITICAL] Tailscale Subnet Outage"
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Target Subnet:*\n`{{ inputs.target_subnet }}`"
},
{
"type": "mrkdwn",
"text": "*Tailnet:*\n`{{ inputs.tailnet }}`"
},
{
"type": "mrkdwn",
"text": "*Healthy Routers:*\n`0 online`"
},
{
"type": "mrkdwn",
"text": "*VPC Connectivity:*\n`DOWN`"
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Summary:*\n{{ outputs.audit_subnet_routers.vars.summary }}"
}
},
{
"type": "context",
"elements": [
{
"type": "mrkdwn",
"text": "Execution ID: `{{ execution.id }}` | Immediate action: Inspect host connectivity on primary and standby router instances, verify Tailscale daemon service, or check Tailscale Admin Console route approval."
}
]
}
]
}
- id: alert_total_outage_pagerduty
type: io.kestra.plugin.pagerduty.PagerDutyAlert
description: "Escalates total subnet connectivity loss to PagerDuty on-call
responder."
runIf: "{{ inputs.enable_pagerduty == true }}"
url: "https://events.pagerduty.com/v2/enqueue"
payload: |
{
"routing_key": "{{ secret('PAGERDUTY_ROUTING_KEY') }}",
"event_action": "trigger",
"dedup_key": "tailscale-subnet-outage-{{ inputs.target_subnet | replace({'/': '-', '.': '-'}) }}",
"payload": {
"summary": "[CRITICAL] Total Tailscale Subnet Router Outage for {{ inputs.target_subnet }} in tailnet {{ inputs.tailnet }}",
"source": "kestra-flow-{{ flow.id }}",
"severity": "critical",
"custom_details": {
"target_subnet": "{{ inputs.target_subnet }}",
"tailnet": "{{ inputs.tailnet }}",
"healthy_routers": {{ outputs.audit_subnet_routers.vars.healthy_routers_count }},
"execution_id": "{{ execution.id }}",
"status": "{{ outputs.audit_subnet_routers.vars.status }}"
}
}
}
- id: fail_on_outage
type: io.kestra.plugin.core.execution.Fail
description: "Terminates the execution in failed status to register an SLA
breach."
NO_ROUTERS_FOUND:
- id: alert_no_routers_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Alerts when no devices in the tailnet advertise the target subnet."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": ":warning: [ALERT] No Tailscale subnet routers found advertising `{{ inputs.target_subnet }}` in tailnet `{{ inputs.tailnet }}`."
}
- id: fail_on_missing_routers
type: io.kestra.plugin.core.execution.Fail
description: "Fails workflow because configured subnet routers do not exist."
HA_FAILOVER_ACTIVE:
- id: alert_ha_failover_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Alerts network operations that the primary router is offline and
traffic has failed over to the standby router."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": ":warning: [WARNING] Tailscale Subnet Router Failover Triggered - Primary Offline",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "[WARNING] Subnet Router Failover Active"
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Target Subnet:*\n`{{ inputs.target_subnet }}`"
},
{
"type": "mrkdwn",
"text": "*Primary Node:*\n`{{ inputs.primary_router_hostname }}` (OFFLINE)"
},
{
"type": "mrkdwn",
"text": "*Active Standby Node:*\n`{{ outputs.audit_subnet_routers.vars.active_routing_node }}` (ONLINE)"
},
{
"type": "mrkdwn",
"text": "*Redundancy State:*\n`DEGRADED (N=0 Standby)`"
}
]
},
{
"type": "context",
"elements": [
{
"type": "mrkdwn",
"text": "Execution ID: `{{ execution.id }}` | Secondary router is actively carrying traffic. Investigate primary host outage to restore high-availability redundancy."
}
]
}
]
}
- id: log_failover_state
type: io.kestra.plugin.core.log.Log
message: "[WARNING] Subnet router failover active! Primary router '{{
inputs.primary_router_hostname }}' is OFFLINE. Standby router '{{
outputs.audit_subnet_routers.vars.active_routing_node }}' is serving
traffic for {{ inputs.target_subnet }}."
STANDBY_ROUTER_DEGRADED:
- id: alert_standby_degraded_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Warns that secondary standby router is offline, removing HA
redundancy."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": ":warning: [WARNING] Tailscale Standby Subnet Router Offline - Redundancy Lost for `{{ inputs.target_subnet }}`."
}
- id: log_standby_degraded
type: io.kestra.plugin.core.log.Log
message: "[WARNING] Primary router is online, but secondary standby router is
offline for subnet {{ inputs.target_subnet }}."
ROUTE_APPROVAL_PENDING:
- id: alert_unapproved_routes_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Alerts when advertised routes are not enabled or approved by
tailnet administrator."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": ":warning: [WARNING] Tailscale Subnet Routes Advertised but Not Approved in Admin Console for `{{ inputs.target_subnet }}`."
}
- id: log_route_unapproved
type: io.kestra.plugin.core.log.Log
message: "[WARNING] Advertised routes are not approved in Tailscale console: {{
outputs.audit_subnet_routers.vars.summary }}"
API_ERROR:
- id: alert_api_error_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Alerts when Tailscale REST API returns an error response."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": ":x: [ERROR] Tailscale API error while auditing subnet routers: {{ outputs.audit_subnet_routers.vars.summary }}"
}
- id: fail_on_api_error
type: io.kestra.plugin.core.execution.Fail
description: "Terminates execution because Tailscale API could not be queried."
NOMINAL_HEALTHY:
- id: log_nominal_health
type: io.kestra.plugin.core.log.Log
message: "[OK] Tailscale subnet routing healthy for {{ inputs.target_subnet }}.
Active router: {{
outputs.audit_subnet_routers.vars.active_routing_node }}. Healthy
routers: {{ outputs.audit_subnet_routers.vars.healthy_routers_count
}}/{{ outputs.audit_subnet_routers.vars.total_routers }}."
- id: log_sentinel_audit
type: io.kestra.plugin.core.log.Log
description: "Records final audit telemetry for the sentinel execution."
message: "[AUDIT] Subnet sentinel execution {{ execution.id }} completed.
Subnet: {{ inputs.target_subnet }}, Status: {{
outputs.audit_subnet_routers.vars.status }}, Active Router: {{
outputs.audit_subnet_routers.vars.active_routing_node }}, Healthy Routers:
{{ outputs.audit_subnet_routers.vars.healthy_routers_count }}."
errors:
- id: alert_flow_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Alerts on unexpected sentinel flow failures, such as network
timeouts or API credential issues."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": ":x: [ERROR] Tailscale Subnet Router Sentinel execution `{{ execution.id }}` failed unexpectedly. Check worker connectivity and API token configuration."
}