id: nats-jetstream-mirror-cluster-migration-verified-cutover
namespace: company.infrastructure
description: |
Moves durable JetStream consumers from a source NATS cluster to a target
cluster that already mirrors the source stream natively, refusing the
cutover unless the mirror has fully caught up, capturing each consumer's
exact acknowledged sequence on the source, recreating it on the target at
that precise stream sequence after operator approval, then reading every
new consumer back to verify the starting sequence actually matches before
declaring success. The source stream and its consumers are never modified.
inputs:
- id: source_nats_url
type: STRING
displayName: "Source NATS Server URL"
description: NATS server URL(s) for the source cluster, e.g.
nats://source.internal:4222.
- id: target_nats_url
type: STRING
displayName: "Target NATS Server URL"
description: NATS server URL(s) for the target cluster, which already runs a
stream mirroring source_stream.
- id: source_stream
type: STRING
displayName: "Source Stream Name"
description: The JetStream stream on the source cluster whose consumers are
being migrated.
- id: target_stream
type: STRING
displayName: "Target Mirror Stream Name"
description: The JetStream stream on the target cluster that mirrors
source_stream (created ahead of time with `nats stream add
--mirror=<source_stream>`).
- id: consumer_names
type: ARRAY
itemType: STRING
displayName: "Durable Consumer Names"
description: Durable consumer names on the source stream to migrate to the
target mirror, resuming each at its exact last-acknowledged sequence.
tasks:
- id: inspect_source_and_mirror
type: io.kestra.plugin.scripts.shell.Commands
description: Reads real JetStream state from both clusters via the official nats
CLI (natsio/nats-box image) - the stream info JSON directly reflects
nats-server's own StreamInfo/StreamState/StreamSourceInfo structs,
confirmed from nats-server's source.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: natsio/nats-box:latest
outputFiles:
- "source_stream.json"
- "target_stream.json"
env:
NATS_URL: "{{ inputs.source_nats_url }}"
NATS_CREDS: "{{ secret('SOURCE_NATS_CREDS') }}"
commands:
- nats stream info "{{ inputs.source_stream }}" --json > source_stream.json
- NATS_URL="{{ inputs.target_nats_url }}" NATS_CREDS="{{
secret('TARGET_NATS_CREDS') }}" nats stream info "{{
inputs.target_stream }}" --json > target_stream.json
- id: evaluate_mirror_readiness
type: io.kestra.plugin.scripts.python.Script
description: Refuses the cutover unless the target is a real mirror of the named
source stream and has fully caught up (mirror.lag == 0), confirmed from
nats-server's own StreamSourceInfo.Lag field - the same role the Kafka
reference blueprint's "quiet check" plays before it allows a cutover.
inputFiles:
source_stream.json: "{{
outputs.inspect_source_and_mirror.outputFiles['source_stream.json'] }}"
target_stream.json: "{{
outputs.inspect_source_and_mirror.outputFiles['target_stream.json'] }}"
script: |
import json
with open("source_stream.json") as f:
source = json.load(f)
with open("target_stream.json") as f:
target = json.load(f)
source_state = source.get("state", {})
target_mirror = target.get("mirror")
refusal_reasons = []
if target_mirror is None:
refusal_reasons.append("Target stream has no mirror configuration at all.")
else:
mirror_name = target_mirror.get("name")
mirror_lag = target_mirror.get("lag", -1)
if mirror_name != "{{ inputs.source_stream }}":
refusal_reasons.append(
f"Target stream mirrors '{mirror_name}', not the requested source '{{ inputs.source_stream }}'."
)
if mirror_lag != 0:
refusal_reasons.append(
f"Target mirror is not caught up: lag={mirror_lag} messages behind the source."
)
is_safe = len(refusal_reasons) == 0
print("::" + json.dumps({
"outputs": {
"is_safe": is_safe,
"refusal_reasons": refusal_reasons,
"source_first_seq": source_state.get("first_seq"),
"source_last_seq": source_state.get("last_seq"),
"source_messages": source_state.get("messages"),
}
}) + "::")
- id: evaluate_safety_gate
type: io.kestra.plugin.core.flow.If
description: Stops the whole migration before touching any consumer if the
mirror is not in a provably safe state.
condition: "{{ outputs.evaluate_mirror_readiness.vars.is_safe == false }}"
then:
- id: alert_unsafe_to_cutover
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Refuses and explains exactly why, instead of silently skipping or
half-applying.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "NATS JetStream cutover REFUSED for stream {{ inputs.source_stream }} -> {{ inputs.target_stream }}. Reasons: {{ outputs.evaluate_mirror_readiness.vars.refusal_reasons | toJson }}. Execution {{ execution.id }}."
}
- id: fail_unsafe_cutover
type: io.kestra.plugin.core.execution.Fail
description: Stops the flow here; nothing downstream runs when the mirror is not
safely caught up.
errorMessage: "Target mirror is not in a safe state for cutover. See the Slack
alert and this execution's logs for the exact reasons."
- id: capture_consumer_positions
type: io.kestra.plugin.core.flow.Loop
description: Reads each source consumer's real AckFloor.Stream sequence -
confirmed from nats-server's ConsumerInfo/SequenceInfo structs - which is
the exact point the new target consumer must resume from to avoid a gap or
a duplicate redelivery.
values: "{{ inputs.consumer_names }}"
concurrencyLimit: 1
outputs:
- id: consumer_name
type: STRING
value: "{{ item.value }}"
- id: ack_floor_seq
type: INT
value: "{{ outputs.read_source_consumer_info.vars.ack_floor_seq }}"
tasks:
- id: fetch_source_consumer_info
type: io.kestra.plugin.scripts.shell.Commands
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: natsio/nats-box:latest
outputFiles:
- "consumer_info.json"
env:
NATS_URL: "{{ inputs.source_nats_url }}"
NATS_CREDS: "{{ secret('SOURCE_NATS_CREDS') }}"
commands:
- nats consumer info "{{ inputs.source_stream }}" "{{ item.value }}"
--json > consumer_info.json
- id: read_source_consumer_info
type: io.kestra.plugin.scripts.python.Script
inputFiles:
consumer_info.json: "{{
outputs.fetch_source_consumer_info.outputFiles['consumer_info.json']
}}"
script: |
import json
with open("consumer_info.json") as f:
info = json.load(f)
ack_floor_seq = info.get("ack_floor", {}).get("stream_seq", 0)
print("::" + json.dumps({
"outputs": {
"ack_floor_seq": ack_floor_seq,
}
}) + "::")
- id: save_migration_plan_kv
type: io.kestra.plugin.core.kv.Set
description: Persists every consumer's captured source ack-floor sequence before
anything on the target is touched, as the audit record for this cutover.
key: "nats_mirror_cutover_{{ inputs.source_stream | slugify }}_to_{{
inputs.target_stream | slugify }}"
value: "{{ outputs.capture_consumer_positions.outputs | toJson }}"
kvType: JSON
- id: pause_for_operator_approval
type: io.kestra.plugin.core.flow.Pause
description: Waits indefinitely for an operator to review the captured sequences
before any consumer is created on the target cluster. No pauseDuration is
set, so an unattended execution waits rather than silently proceeding.
- id: apply_cutover
type: io.kestra.plugin.core.flow.Loop
description: Creates each consumer on the target mirror starting at the exact
next sequence after the source's ack floor, confirmed from natscli's own
setStartPolicy source - a plain numeric --deliver value sets
DeliverPolicy=by_start_sequence and OptStartSeq to that number. The loop
iterates over capture_consumer_positions' own declared Loop outputs,
confirmed from Loop.java's real Output.outputs shape (a List<LoopOutput>,
each wrapping {item, outputs}), so each map-typed element arrives
JSON-stringified in item.value and is parsed with
fromJson(...).outputs.<field> rather than direct dot access.
values: "{{ outputs.capture_consumer_positions.outputs }}"
concurrencyLimit: 1
tasks:
- id: create_target_consumer
type: io.kestra.plugin.scripts.shell.Commands
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: natsio/nats-box:latest
env:
NATS_URL: "{{ inputs.target_nats_url }}"
NATS_CREDS: "{{ secret('TARGET_NATS_CREDS') }}"
commands:
- nats consumer add "{{ inputs.target_stream }}" "{{
fromJson(item.value).outputs.consumer_name }}" --deliver "{{
fromJson(item.value).outputs.ack_floor_seq + 1 }}" --ack explicit
--pull --max-deliver -1 --replay instant --json
- id: verify_cutover
type: io.kestra.plugin.core.flow.Loop
description: Reads each newly created target consumer back and fails the
verification unless its configured opt_start_seq exactly matches what was
requested, instead of trusting the create command's exit code alone. Same
fromJson(item.value).outputs.<field> access pattern as apply_cutover, for
the same verified reason.
values: "{{ outputs.capture_consumer_positions.outputs }}"
concurrencyLimit: 1
outputs:
- id: consumer_name
type: STRING
value: "{{ fromJson(item.value).outputs.consumer_name }}"
- id: expected_start_seq
type: INT
value: "{{ fromJson(item.value).outputs.ack_floor_seq + 1 }}"
- id: verified
type: BOOL
value: "{{ outputs.check_target_consumer_start_seq.vars.verified }}"
- id: actual_start_seq
type: INT
value: "{{ outputs.check_target_consumer_start_seq.vars.actual_start_seq }}"
tasks:
- id: fetch_target_consumer_info
type: io.kestra.plugin.scripts.shell.Commands
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: natsio/nats-box:latest
outputFiles:
- "target_consumer_info.json"
env:
NATS_URL: "{{ inputs.target_nats_url }}"
NATS_CREDS: "{{ secret('TARGET_NATS_CREDS') }}"
commands:
- nats consumer info "{{ inputs.target_stream }}" "{{
fromJson(item.value).outputs.consumer_name }}" --json >
target_consumer_info.json
- id: check_target_consumer_start_seq
type: io.kestra.plugin.scripts.python.Script
env:
EXPECTED_START_SEQ: "{{ fromJson(item.value).outputs.ack_floor_seq + 1 }}"
inputFiles:
target_consumer_info.json: "{{
outputs.fetch_target_consumer_info.outputFiles['target_consumer_inf\
o.json'] }}"
script: |
import json
import os
with open("target_consumer_info.json") as f:
info = json.load(f)
expected = int(os.environ["EXPECTED_START_SEQ"])
actual = info.get("config", {}).get("opt_start_seq", -1)
verified = actual == expected
print("::" + json.dumps({
"outputs": {
"verified": verified,
"actual_start_seq": actual,
}
}) + "::")
- id: evaluate_verification_results
type: io.kestra.plugin.core.flow.If
description: Branches on whether every migrated consumer verified cleanly.
verify_cutover's own outputs are also wrapped List<LoopOutput> ({item,
outputs}), confirmed from Loop.java, so the jq expression selects on
.outputs.verified, not a flat .verified.
condition: "{{ outputs.verify_cutover.outputs | jq('[.[] |
select(.outputs.verified == false)] | length > 0') | first }}"
then:
- id: alert_verification_failed
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Names exactly which consumer(s) did not verify, so a human
investigates instead of trusting a partial cutover.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "NATS JetStream cutover for {{ inputs.source_stream }} -> {{ inputs.target_stream }} completed with FAILED verification on at least one consumer. Details: {{ outputs.verify_cutover.outputs | toJson }}. Investigate before routing traffic to the target consumers. Execution {{ execution.id }}."
}
else:
- id: notify_cutover_verified
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Confirms the real, read-back-verified outcome.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "NATS JetStream cutover for {{ inputs.source_stream }} -> {{ inputs.target_stream }} verified: {{ outputs.verify_cutover.outputs | length }} consumer(s) recreated on the target at their exact source ack-floor sequence. Source stream and consumers were never modified. Execution {{ execution.id }}."
}
errors:
- id: alert_execution_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alerts Slack separately if the flow itself fails, so a broken
migration attempt is never mistaken for a completed or refused one.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "NATS JetStream cutover FAILED in execution {{ execution.id }} for {{ inputs.source_stream }} -> {{ inputs.target_stream }}. Check whether any target consumers were partially created before fixing and retrying."
}
outputs:
- id: migration_plan
type: JSON
value: "{{ outputs.capture_consumer_positions.outputs | toJson }}"