id: pulumi-pending-operations-recovery-guard
namespace: company.team
description: |
Recover a Pulumi stack that an interrupted update left wedged. A killed CI job, a dropped
connection or a crashed CLI stops `pulumi up` partway, so the stack keeps `pending_operations`
in its deployment and often a held lock. Every later `pulumi up` then refuses with "the stack is
currently locked", so deploys are blocked until a human steps in. This blueprint reads the stack
with `pulumi stack export`, counts the pending operations and any lock, then behind a dry run and
an approval runs `pulumi cancel` to drop the lock and `pulumi refresh --clear-pending-creates` to
reconcile state and clear the stale operations. It exports the stack again and fails the run if
any pending operation survived.
inputs:
- id: project_name
type: STRING
displayName: Pulumi project name
description: The project the stack belongs to, as written in Pulumi.yaml. Used
to select the stack on the self-managed backend.
defaults: recovery-demo
- id: stack_name
type: STRING
displayName: Stack name
description: The stack to inspect and recover, for example dev or prod. On a
file backend the fully qualified name is organization/<project>/<stack>.
defaults: dev
- id: min_pending_operations
type: INT
displayName: Pending operation threshold
description: Remediation is considered only when the stack carries at least this
many pending operations or a lock is held. One is the right value for
recovery, raise it to ignore a single transient operation.
defaults: 1
- id: backend_snapshot_b64
type: STRING
displayName: Backend snapshot (base64 tar.gz)
description: Optional base64 of a tar.gz of the Pulumi project directory,
including its .pulumi self-managed backend. The flow restores it, so
recovery runs against a point in time snapshot and the approval pause
never holds the live backend. Leave empty to inspect the backend already
staged in the task working directory.
defaults: ""
- id: auto_remediate
type: BOOL
displayName: Auto remediate
description: When false the flow only reports. Must be true, with dry_run false
and the pause approved, before cancel or refresh runs.
defaults: false
- id: dry_run
type: BOOL
displayName: Dry run
description: When true the plan is only logged. Must be false, with
auto_remediate true, before anything is changed.
defaults: true
tasks:
- id: detect
type: io.kestra.plugin.scripts.shell.Commands
description: Read-only. Logs in to the self-managed backend, exports the stack
and counts pending operations and any lock. Nothing is changed here.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: pulumi/pulumi:latest
env:
PULUMI_CONFIG_PASSPHRASE: ""
PULUMI_SKIP_UPDATE_CHECK: "true"
PROJECT: "{{ inputs.project_name }}"
STACK: "{{ inputs.stack_name }}"
SEED_B64: "{{ inputs.backend_snapshot_b64 }}"
outputFiles:
- backend.tgz
- export.json
commands:
- |
set -eu
mkdir -p proj
if [ -n "${SEED_B64:-}" ]; then
printf '%s' "$SEED_B64" | base64 -d > seed.tgz
tar -xzf seed.tgz -C proj
fi
cd proj
if [ ! -f Pulumi.yaml ]; then
printf 'name: %s\nruntime: yaml\n' "$PROJECT" > Pulumi.yaml
fi
pulumi login "file://$PWD" >/dev/null 2>&1
pulumi stack select "$STACK" 2>/dev/null || pulumi stack init "$STACK" >/dev/null 2>&1
pulumi stack export --file ../export.json
LOCKDIR=".pulumi/locks"
LOCKS=0
if [ -d "$LOCKDIR" ]; then
LOCKS=$(find "$LOCKDIR" -name '*.json' 2>/dev/null | wc -l | tr -d ' ')
fi
tar -czf ../backend.tgz .
LOCKS="$LOCKS" python3 - <<'PY'
import json, os
d = json.load(open("../export.json"))
dep = d.get("deployment", {}) or {}
pend = dep.get("pending_operations") or d.get("pending_operations") or []
summ = []
for op in pend:
t = op.get("type", "op")
urn = (op.get("resource") or {}).get("urn", "")
summ.append(t + " " + urn.split("::")[-1])
out = {
"pending_count": str(len(pend)),
"lock_count": os.environ.get("LOCKS", "0"),
"ops_summary": ("; ".join(summ) or "none"),
}
print("::" + json.dumps({"outputs": out}) + "::")
PY
- id: summary
type: io.kestra.plugin.core.log.Log
message: >-
Pulumi recovery check for {{ inputs.project_name }}/{{ inputs.stack_name
}}: {{ outputs.detect.vars.pending_count }} pending operations, {{
outputs.detect.vars.lock_count }} lock files. Operations: {{
outputs.detect.vars.ops_summary }}.
- id: route
type: io.kestra.plugin.core.flow.If
condition: "{{ (outputs.detect.vars.pending_count | number >=
inputs.min_pending_operations) or (outputs.detect.vars.lock_count | number
> 0) }}"
then:
- id: findings_alert
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{{ {'text': 'Pulumi stack ' ~ inputs.project_name ~ '/' ~ inputs.stack_name ~ ' is wedged: ' ~ outputs.detect.vars.pending_count ~ ' pending operations and ' ~ outputs.detect.vars.lock_count ~ ' lock files. ' ~ (inputs.auto_remediate and not inputs.dry_run ? 'Recovery waits for approval on this execution.' : 'Nothing changed (auto_remediate=' ~ inputs.auto_remediate ~ ', dry_run=' ~ inputs.dry_run ~ ').')} | toJson }}
- id: remediate_gate
type: io.kestra.plugin.core.flow.If
description: cancel and refresh run only when auto_remediate is true, dry_run is
false and the pause below is approved.
condition: "{{ inputs.auto_remediate and not inputs.dry_run }}"
then:
- id: hold
type: io.kestra.plugin.core.flow.Pause
description: >-
pulumi cancel drops the lock, it does not roll back a resource.
pulumi refresh reconciles state with what the provider actually
has, which can show resources the interrupted update really
created. Read the exported state above and the pending operations,
then resume with approved true to recover the stack.
onResume:
- id: approved
type: BOOL
required: false
description: Set true to run cancel and refresh. Anything else leaves the stack
untouched.
- id: approval_gate
type: io.kestra.plugin.core.flow.If
condition: "{{ outputs.hold.onResume.approved ?? false }}"
then:
- id: remediate
type: io.kestra.plugin.scripts.shell.Commands
description: Drop the lock with pulumi cancel, then reconcile the stack with
pulumi refresh so the stale pending operations clear. Both are
logged with their exit codes.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: pulumi/pulumi:latest
env:
PULUMI_CONFIG_PASSPHRASE: ""
PULUMI_SKIP_UPDATE_CHECK: "true"
STACK: "{{ inputs.stack_name }}"
inputFiles:
backend.tgz: "{{ outputs.detect.outputFiles['backend.tgz'] }}"
outputFiles:
- backend.tgz
commands:
- |
set -eu
mkdir -p proj
tar -xzf backend.tgz -C proj
cd proj
pulumi login "file://$PWD" >/dev/null 2>&1
pulumi stack select "$STACK"
set +e
pulumi cancel --yes; echo "cancel_rc=$?"
pulumi refresh --yes --skip-preview --clear-pending-creates; echo "refresh_rc=$?"
set -e
tar -czf ../backend.tgz .
- id: reverify
type: io.kestra.plugin.scripts.shell.Commands
description: Export the stack again and count pending operations and any lock.
This is the read that proves the recovery landed.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: pulumi/pulumi:latest
env:
PULUMI_CONFIG_PASSPHRASE: ""
PULUMI_SKIP_UPDATE_CHECK: "true"
STACK: "{{ inputs.stack_name }}"
inputFiles:
backend.tgz: "{{ outputs.remediate.outputFiles['backend.tgz'] }}"
outputFiles:
- export_after.json
commands:
- |
set -eu
mkdir -p proj
tar -xzf backend.tgz -C proj
cd proj
pulumi login "file://$PWD" >/dev/null 2>&1
pulumi stack select "$STACK"
pulumi stack export --file ../export_after.json
LOCKS=0
if [ -d ".pulumi/locks" ]; then
LOCKS=$(find ".pulumi/locks" -name '*.json' 2>/dev/null | wc -l | tr -d ' ')
fi
LOCKS="$LOCKS" python3 - <<'PY'
import json, os
d = json.load(open("../export_after.json"))
dep = d.get("deployment", {}) or {}
pend = dep.get("pending_operations") or d.get("pending_operations") or []
out = {"pending_after": str(len(pend)), "lock_after": os.environ.get("LOCKS", "0")}
print("::" + json.dumps({"outputs": out}) + "::")
PY
- id: verify
type: io.kestra.plugin.core.flow.If
description: The run fails when a pending operation or a lock survived the
recovery. Reporting success without reading the stack again is
what this gate prevents.
condition: "{{ (outputs.reverify.vars.pending_after | number > 0) or
(outputs.reverify.vars.lock_after | number > 0) }}"
then:
- id: recovery_incomplete
type: io.kestra.plugin.core.execution.Fail
errorMessage: >-
Recovery did not land on {{ inputs.project_name }}/{{
inputs.stack_name }}. Still {{
outputs.reverify.vars.pending_after }} pending operations
and {{ outputs.reverify.vars.lock_after }} lock files
after cancel and refresh. Read the refresh output and the
exported state before the next deploy.
else:
- id: recovered
type: io.kestra.plugin.core.log.Log
message: >-
Recovered {{ inputs.project_name }}/{{ inputs.stack_name
}}: {{ outputs.detect.vars.pending_count }} pending
operations and {{ outputs.detect.vars.lock_count }} lock
files before, 0 and 0 after.
else:
- id: approval_declined
type: io.kestra.plugin.core.log.Log
message: "Approval not granted on execution {{ execution.id }}. Nothing was
changed on {{ inputs.project_name }}/{{ inputs.stack_name }}."
else:
- id: report_only
type: io.kestra.plugin.core.log.Log
message: >-
Report only, nothing changed (auto_remediate={{
inputs.auto_remediate }}, dry_run={{ inputs.dry_run }}). Run with
auto_remediate true and dry_run false, then approve the pause, to
recover {{ inputs.project_name }}/{{ inputs.stack_name }}.
else:
- id: nothing_pending
type: io.kestra.plugin.core.log.Log
message: >-
{{ inputs.project_name }}/{{ inputs.stack_name }} is clean: {{
outputs.detect.vars.pending_count }} pending operations and {{
outputs.detect.vars.lock_count }} lock files, below the threshold.
Nothing to recover.
errors:
- id: alert_flow_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{{ {'text': 'Pulumi pending operations recovery failed on ' ~ inputs.project_name ~ '/' ~ inputs.stack_name ~ ' (execution ' ~ execution.id ~ '). Either the backend was unreachable or a pending operation survived the recovery. Check the execution logs.'} | toJson }}
triggers:
- id: nightly
type: io.kestra.plugin.core.trigger.Schedule
description: Shipped disabled and in safe mode. Enabling it only reports until
you change these inputs.
cron: "40 2 * * *"
disabled: true
inputs:
dry_run: true
auto_remediate: false