id: multi-region-failover-orchestrator
namespace: company.team
description: |
Probe three regions on a synthetic schedule, cut Cloudflare DNS over to a
healthy secondary the moment the primary goes dark, verify the switched
record before declaring victory, and page the war room - with a hard
execution failure when verification fails.
triggers:
- id: synthetic_health_check
type: io.kestra.plugin.core.trigger.Schedule
description: Synthetic probe every five minutes (enable as needed).
cron: "*/5 * * * *"
disabled: true
- id: on_demand_failover
type: io.kestra.plugin.core.trigger.Webhook
description: Fire the failover runbook on demand from an incident channel command.
key: "{{ secret('FAILOVER_WEBHOOK_KEY') }}"
inputs:
- id: primary_url
type: STRING
displayName: Primary region health URL
description: Health endpoint of the region currently serving production traffic.
defaults: https://primary.example.com/healthz
- id: secondary_url
type: STRING
displayName: Secondary region health URL
description: Health endpoint of the region that receives traffic after cutover.
defaults: https://secondary.example.com/healthz
- id: tertiary_url
type: STRING
displayName: Tertiary region health URL
description: Third region probed for situational awareness - never receives the
cutover itself.
defaults: https://tertiary.example.com/healthz
- id: host_name
type: STRING
displayName: Hostname to switch
description: DNS record that moves to the failover origin when the primary is dark.
defaults: app.example.com
- id: zone_id
type: STRING
defaults: "023e105f4ecef8ad9ca31a8372d0c353"
displayName: Cloudflare zone id
description: Zone containing the record to switch.
- id: failover_ip
type: STRING
required: false
displayName: Failover origin IP
description: Content published on the record during cutover - the secondary
region's origin address.
- id: record_type
type: STRING
displayName: Record type
description: DNS record type being switched.
defaults: A
- id: verify_url
type: STRING
displayName: Post-cutover verification URL
description: URL checked after the switch - it must be served through the
changed record to prove the cutover took.
defaults: https://app.example.com/healthz
tasks:
- id: probe_regions
type: io.kestra.plugin.core.flow.Parallel
description: Probe all three regional health endpoints concurrently - serial
probes would triple the detection latency.
tasks:
- id: probe_primary
type: io.kestra.plugin.core.http.Request
description: Probe the region currently serving traffic.
uri: "{{ inputs.primary_url }}"
method: GET
timeout: PT5S
- id: probe_secondary
type: io.kestra.plugin.core.http.Request
description: Probe the failover candidate.
uri: "{{ inputs.secondary_url }}"
method: GET
timeout: PT5S
- id: probe_tertiary
type: io.kestra.plugin.core.http.Request
description: Probe the third region for situational awareness.
uri: "{{ inputs.tertiary_url }}"
method: GET
timeout: PT5S
- id: assess
type: io.kestra.plugin.scripts.python.Script
description: Turn the three probe codes into a verdict - only a dark primary
with a live secondary authorizes cutover.
taskRunner:
type: io.kestra.plugin.core.runner.Process
env:
PRIMARY_CODE: "{{ outputs.probe_primary.code }}"
SECONDARY_CODE: "{{ outputs.probe_secondary.code }}"
TERTIARY_CODE: "{{ outputs.probe_tertiary.code }}"
script: |
import json
import os
def emit(outputs):
print("::" + json.dumps({"outputs": outputs}) + "::")
def healthy(name):
raw = (os.environ.get(name, "") or "").strip()
try:
code = int(float(raw))
except Exception:
return False
return 200 <= code < 300
regions = [
("us_east", "PRIMARY_CODE"),
("eu_west", "SECONDARY_CODE"),
("ap_south", "TERTIARY_CODE"),
]
verdicts = {label: healthy(env_name) for label, env_name in regions}
healthy_regions = [label for label, _ in regions if verdicts[label]]
degraded_regions = [label for label, _ in regions if not verdicts[label]]
primary_ok = verdicts["us_east"]
secondary_ok = verdicts["eu_west"]
tertiary_ok = verdicts["ap_south"]
cutover_needed = (not primary_ok) and secondary_ok
cutover_blocked = (not primary_ok) and (not secondary_ok)
summary = "primary=%s secondary=%s tertiary=%s healthy=%d/3" % (
"up" if primary_ok else "DOWN",
"up" if secondary_ok else "DOWN",
"up" if tertiary_ok else "DOWN",
len(healthy_regions),
)
if degraded_regions:
summary += " degraded=" + ",".join(degraded_regions)
if cutover_blocked:
summary += " (cutover blocked: failover target also down)"
emit({
"primary_ok": primary_ok,
"secondary_ok": secondary_ok,
"tertiary_ok": tertiary_ok,
"healthy_regions": healthy_regions,
"degraded_regions": degraded_regions,
"healthy_count": len(healthy_regions),
"cutover_needed": cutover_needed,
"cutover_blocked": cutover_blocked,
"summary": summary,
})
- id: cutover_gate
type: io.kestra.plugin.core.flow.If
description: Only a dark primary with a live secondary authorizes the DNS switch.
condition: "{{ outputs.assess.vars.cutover_needed == true }}"
then:
- id: switch_dns
type: io.kestra.plugin.cloudflare.dns.records.Upsert
description: Point the hostname at the failover origin - a short TTL keeps the
rollback cheap.
apiToken: "{{ secret('CLOUDFLARE_API_TOKEN') }}"
zoneId: "{{ inputs.zone_id }}"
name: "{{ inputs.host_name }}"
recordType: "{{ inputs.record_type }}"
ttl: 60
proxied: false
content: "{{ inputs.failover_ip }}"
- id: verify_failover
type: io.kestra.plugin.core.http.Request
description: Re-check through the switched record - the cutover is only real
when the answer serves traffic.
uri: "{{ inputs.verify_url }}"
method: GET
timeout: PT5S
- id: verify_gate
type: io.kestra.plugin.core.flow.If
description: Verified cutover posts a war-room all-clear; a failed verification
pages hard and fails the runbook.
condition: "{{ outputs.verify_failover.code == 200 }}"
then:
- id: slack_cutover
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: War-room all-clear with the switched record and the degraded
regions.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "FAILOVER COMPLETE: {{ inputs.host_name }} -> {{ inputs.failover_ip }} ({{ inputs.record_type }}, ttl 60). Primary region dark, verification {{ inputs.verify_url }} returned 200. {{ outputs.assess.vars.summary }} (execution {{ execution.id }})"
}
else:
- id: slack_cutover_failed
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Critical page - DNS moved but the verification URL is not serving.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "CRITICAL: failover verification FAILED for {{ inputs.host_name }} -> {{ inputs.failover_ip }}: {{ inputs.verify_url }} returned {{ outputs.verify_failover.code }}. Roll back the record manually. {{ outputs.assess.vars.summary }} (execution {{ execution.id }})"
}
- id: fail_execution
type: io.kestra.plugin.core.execution.Fail
description: Fail on purpose so SLA dashboards and dependent flows see red, not
green, after a broken cutover.
errorMessage: "Failover verification failed: {{ inputs.verify_url }} returned {{
outputs.verify_failover.code }} after switching {{
inputs.host_name }} to {{ inputs.failover_ip }}."
else:
- id: blocked_gate
type: io.kestra.plugin.core.flow.If
description: Primary AND secondary both down means cutover is impossible - that
is an incident, not a healthy run.
condition: "{{ outputs.assess.vars.cutover_blocked == true }}"
then:
- id: slack_regions_down
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Page when there is nowhere left to fail over to.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "CRITICAL: failover impossible - primary and secondary regions are both down, no DNS cutover available. {{ outputs.assess.vars.summary }} (execution {{ execution.id }})"
}
else:
- id: log_healthy
type: io.kestra.plugin.core.log.Log
description: Quiet line when traffic is fine or the primary is healthy.
message: "failover_check {{ outputs.assess.vars.summary }} cutover=none"
errors:
- id: slack_errors
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: The runbook itself failing is an incident - page the channel.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "multi-region-failover-orchestrator FAILED (execution {{ execution.id }}): {{ error is defined ? error : 'see execution logs' }}"
}
outputs:
- id: primary_ok
type: BOOL
value: "{{ outputs.assess.vars.primary_ok }}"
- id: healthy_regions
type: JSON
value: "{{ outputs.assess.vars.healthy_regions }}"
- id: degraded_regions
type: JSON
value: "{{ outputs.assess.vars.degraded_regions }}"
- id: healthy_count
type: INT
value: "{{ outputs.assess.vars.healthy_count }}"
- id: cutover_performed
type: BOOL
value: "{{ outputs.switch_dns is defined ? true : false }}"
- id: verified
type: BOOL
value: "{{ outputs.verify_failover is defined ? outputs.verify_failover.code ==
200 : false }}"
- id: summary
type: STRING
value: "{{ outputs.assess.vars.summary }}"
