id: acme-certificate-renew-deploy-rollback
namespace: company.team
description: |
Certificate lifecycle on one schedule: probe what every endpoint really serves, renew what is
missing, expiring, untrusted or missing a name through ACME, verify the new certificates before
they go anywhere, deploy them to the TLS edge, check what clients now see, and roll back to the
previous configuration when that check fails.
inputs:
- id: certificates
type: JSON
displayName: Certificate inventory
description: One entry per certificate. The first domain is the common name.
Every domain must be served by tls_endpoint.
defaults: |
[
{"name": "shop", "domains": ["shop.example.test", "www.shop.example.test"]},
{"name": "api", "domains": ["api.example.test"]}
]
- id: renew_before_days
type: INT
displayName: Renew before (days)
description: Renew a certificate when fewer days than this are left.
defaults: 30
- id: tls_endpoint
type: STRING
displayName: TLS endpoint
description: host:port that serves the certificates, probed with each domain as SNI.
defaults: web:443
- id: caddy_admin_url
type: STRING
displayName: Caddy admin API
description: The deploy target. Its /load endpoint swaps the configuration atomically.
defaults: http://web:2019
- id: acme_directory
type: STRING
displayName: ACME directory
description: Pebble in the demo. https://acme-v02.api.letsencrypt.org/directory
in production, and https://acme-staging-v02.api.letsencrypt.org/directory
to test.
defaults: https://pebble:14000/dir
- id: acme_email
type: STRING
displayName: ACME account email
defaults: ops@example.test
- id: acme_profile
type: STRING
displayName: ACME profile
description: >
Certificate profile to request. Pebble offers default (90 days) and
shortlived (6 days). Let's Encrypt offers classic, tlsserver and
shortlived. Leave empty to let the CA choose, but note that Pebble then
picks a profile at random.
defaults: default
- id: key_type
type: SELECT
displayName: Key type
values:
- ec256
- ec384
- rsa2048
- rsa3072
- rsa4096
defaults: ec256
- id: trust_bundle_url
type: STRING
displayName: Trust bundle URL
description: >
Where the roots that clients trust come from. Set to system for a public
CA such as Let's Encrypt: the operating system trust store is used. A URL
is only for a private or test CA such as Pebble, whose root changes on
every start.
defaults: https://pebble:15000/roots/0
- id: force_renew
type: BOOL
displayName: Force renewal
description: Renew every certificate in the inventory, even when it is healthy.
defaults: false
- id: dry_run
type: BOOL
displayName: Dry run
description: Probe and report what would be renewed, and change nothing.
defaults: false
- id: demo_drop_intermediate
type: BOOL
displayName: "Demo: deploy without the intermediate"
description: Deploys only the leaf certificate, the classic broken-chain
mistake. Browsers that cached the intermediate still work, others fail.
The health check must catch it and roll back.
defaults: false
variables:
# Shell helpers shared by the probe, the verification and the health checks.
tls_lib: |
fetch_trust() {
if [ "${TRUST_URL:-system}" != "system" ]; then
# A test CA serves its root over a self-signed management port. Never do this for a public CA.
curl -sfk "$TRUST_URL" -o trust.pem
else
cp /etc/ssl/certs/ca-certificates.crt trust.pem
fi
}
# served <domain> <prefix>: what the endpoint presents for this SNI name.
served() {
rm -f "$2.leaf.pem" "$2.chain.pem"
openssl s_client -connect "$ENDPOINT" -servername "$1" -showcerts </dev/null >"$2.raw" 2>"$2.err" || true
awk -v leaf="$2.leaf.pem" -v chain="$2.chain.pem" '
/-----BEGIN CERTIFICATE-----/ { n++; inside = 1 }
inside { if (n == 1) print > leaf; else print > chain }
/-----END CERTIFICATE-----/ { inside = 0 }' "$2.raw"
touch "$2.chain.pem"
[ -s "$2.leaf.pem" ]
}
serial_of() { openssl x509 -in "$1" -noout -serial | cut -d= -f2; }
end_epoch() { date -u -d "$(openssl x509 -in "$1" -noout -enddate -dateopt iso_8601 | cut -d= -f2 | sed 's/Z$//')" +%s; }
start_epoch() { date -u -d "$(openssl x509 -in "$1" -noout -startdate -dateopt iso_8601 | cut -d= -f2 | sed 's/Z$//')" +%s; }
days_left() { echo $(( ($(end_epoch "$1") - $(date -u +%s)) / 86400 )); }
sans_of() { openssl x509 -in "$1" -noout -ext subjectAltName 2>/dev/null | tail -n +2 | tr ',' '\n' | sed -n 's/^ *DNS://p' | sort -u; }
# covers <cert> <domain>: exact name, or a wildcard one label up.
covers() { sans_of "$1" | grep -qxF -e "$2" -e "*.${2#*.}"; }
# chain_ok <leaf> <chain>: prints the openssl error when the chain does not verify.
chain_ok() {
if [ -s "$2" ]; then out=$(openssl verify -CAfile trust.pem -untrusted "$2" "$1" 2>&1) || { echo "$out" | grep -m1 -i error | sed 's/^.*error [0-9]* at [0-9]* depth lookup: //'; return 1; }
else out=$(openssl verify -CAfile trust.pem "$1" 2>&1) || { echo "$out" | grep -m1 -i error | sed 's/^.*error [0-9]* at [0-9]* depth lookup: //'; return 1; }
fi
}
# handshake_ok <domain>: what a strict client sees, with only the roots it trusts.
handshake_ok() {
openssl s_client -connect "$ENDPOINT" -servername "$1" -verify_hostname "$1" -verify_return_error -CAfile trust.pem </dev/null >hs.out 2>&1
}
emit() { printf '::{"outputs":%s}::\n' "$(jq -c . "$1")"; }
concurrency:
limit: 1
triggers:
- id: daily
type: io.kestra.plugin.core.trigger.Schedule
description: Daily. Renewal starts renew_before_days ahead, so a failed day
leaves weeks to fix it.
cron: "17 4 * * *"
disabled: true
tasks:
# ---------------------------------------------------------------------------
# 1. Probe: judge certificates by what the endpoint serves, not by a file on disk
# ---------------------------------------------------------------------------
- id: probe
type: io.kestra.plugin.scripts.shell.Commands
description: >
For every domain in the inventory, open a TLS connection with that SNI
name and classify the certificate the endpoint presents. The worst domain
decides the status of the certificate.
containerImage: alpine:3.20
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- apk add --no-cache -q openssl curl jq
env:
ENDPOINT: "{{ inputs.tls_endpoint }}"
TRUST_URL: "{{ inputs.trust_bundle_url }}"
RENEW_BEFORE: "{{ inputs.renew_before_days }}"
inputFiles:
lib.sh: "{{ vars.tls_lib }}"
inventory.json: "{{ inputs.certificates | toJson }}"
outputFiles:
- probe.json
commands:
- |
. ./lib.sh
fetch_trust
: > probe.ndjson
jq -c '.[]' inventory.json | while read -r c; do
name=$(echo "$c" | jq -r .name)
status=OK; reason="healthy"; serial=""; days=""; rank=0
# Higher rank wins: MISSING 6, EXPIRED 5, UNTRUSTED 4, SAN_MISMATCH 3, SPLIT 2, EXPIRING 1.
set_status() { if [ "$1" -gt "$rank" ]; then rank=$1; status=$2; reason=$3; fi; }
for d in $(echo "$c" | jq -r '.domains[]'); do
if ! served "$d" p; then
set_status 6 MISSING "no certificate served for $d"; continue
fi
s=$(serial_of p.leaf.pem); dl=$(days_left p.leaf.pem)
if [ -z "$serial" ]; then serial=$s; days=$dl
elif [ "$s" != "$serial" ]; then set_status 2 SPLIT "$d is served by a different certificate ($s)"; fi
if [ "$dl" -lt 0 ]; then set_status 5 EXPIRED "expired $(( -dl )) day(s) ago"; fi
if ! err=$(chain_ok p.leaf.pem p.chain.pem); then set_status 4 UNTRUSTED "chain does not verify for $d: $err"; fi
if ! covers p.leaf.pem "$d"; then set_status 3 SAN_MISMATCH "served certificate does not cover $d"; fi
if [ "$dl" -lt "$RENEW_BEFORE" ]; then set_status 1 EXPIRING "$dl day(s) left, renewal starts at $RENEW_BEFORE"; fi
done
jq -nc --arg name "$name" --arg status "$status" --arg reason "$reason" --arg serial "$serial" \
--arg days "$days" --argjson domains "$(echo "$c" | jq -c .domains)" \
'{name: $name, domains: $domains, status: $status, reason: $reason, serial: $serial,
days_left: (if $days == "" then null else ($days | tonumber) end)}' >> probe.ndjson
echo "$name: $status, $reason"
done
jq -s '.' probe.ndjson > probe.json
jq '{probe: .}' probe.json > out.json
emit out.json
- id: plan
type: io.kestra.plugin.core.output.OutputValues
description: The certificates to renew, and a lookup of what each one was before
this run.
values:
due: "{{ outputs.probe.vars.probe | jq(inputs.force_renew ? '[.[].name]' : '[.[]
| select(.status != \"OK\") | .name]') | first | toJson }}"
before: "{{ outputs.probe.vars.probe | jq('map({key: .name, value: {serial,
status, days_left}}) | from_entries') | first | toJson }}"
- id: log_plan
type: io.kestra.plugin.core.log.Log
message: >-
{{ outputs.probe.vars.probe | length }} certificate(s) probed on {{
inputs.tls_endpoint }}. {{ (fromJson(outputs.plan.values.due) | length) ==
0 ? 'Nothing to renew.' : 'To renew: ' ~
(fromJson(outputs.plan.values.due) | join(', ')) ~ (inputs.force_renew ? '
(forced).' : '.') }} {{ inputs.dry_run ? 'Dry run: nothing will change.' :
'' }}
- id: lifecycle
type: io.kestra.plugin.core.flow.If
condition: "{{ (fromJson(outputs.plan.values.due) | length) > 0 and not
inputs.dry_run }}"
then:
# -----------------------------------------------------------------------
# 2. Renew through ACME with a DNS-01 challenge
# -----------------------------------------------------------------------
- id: renew
type: io.kestra.plugin.scripts.shell.Commands
description: >
One ACME order per due certificate with lego. The demo solves DNS-01
with the exec provider against Pebble's test DNS server. For a real
zone, set DNS_PROVIDER to your provider (cloudflare, route53, gcloud,
azuredns...) and add its credentials to env from secrets.
containerImage: goacme/lego:v4.25.2
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint: []
beforeCommands: []
env:
LEGO_SERVER: "{{ inputs.acme_directory }}"
LEGO_EMAIL: "{{ inputs.acme_email }}"
DNS_PROVIDER: exec
# exec provider (demo): the hook below publishes the TXT record.
EXEC_PATH: ./dns-hook.sh
EXEC_PROPAGATION_TIMEOUT: "60"
EXEC_POLLING_INTERVAL: "2"
EXEC_SEQUENCE_INTERVAL: "2"
# Trust Pebble's API certificate. Remove for a public CA.
LEGO_CA_CERTIFICATES: ./pebble-api-ca.pem
DNS_RESOLVERS: challtestsrv:8053
inputFiles:
due.txt: |
{% for c in inputs.certificates %}{% if (fromJson(outputs.plan.values.due)) contains c.name %}{{ c.name }} {{ c.domains | join(' ') }}
{% endif %}{% endfor %}
dns-hook.sh: |
#!/bin/sh
# lego exec contract: $1 = present | cleanup, $2 = record FQDN, $3 = record value.
case "$1" in
present) wget -qO- --post-data "{\"host\":\"$2\",\"value\":\"$3\"}" http://challtestsrv:8055/set-txt ;;
cleanup) wget -qO- --post-data "{\"host\":\"$2\"}" http://challtestsrv:8055/clear-txt ;;
esac
# Pebble's fixed test CA for its own API (test/certs/pebble.minica.pem in the Pebble repository).
pebble-api-ca.pem: |
-----BEGIN CERTIFICATE-----
MIIDPzCCAiegAwIBAgIIU0Xm9UFdQxUwDQYJKoZIhvcNAQELBQAwIDEeMBwGA1UE
AxMVbWluaWNhIHJvb3QgY2EgNTM0NWU2MCAXDTI1MDkwMzIzNDAwNVoYDzIxMjUw
OTAzMjM0MDA1WjAgMR4wHAYDVQQDExVtaW5pY2Egcm9vdCBjYSA1MzQ1ZTYwggEi
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5WgZNoVJandj43kkLyU50vzCZ
alozvdRo3OFiKoDtmqKPNWRNO2hC9AUNxTDJco51Yc42u/WV3fPbbhSznTiOOVtn
Ajm6iq4I5nZYltGGZetGDOQWr78y2gWY+SG078MuOO2hyDIiKtVc3xiXYA+8Hluu
9F8KbqSS1h55yxZ9b87eKR+B0zu2ahzBCIHKmKWgc6N13l7aDxxY3D6uq8gtJRU0
toumyLbdzGcupVvjbjDP11nl07RESDWBLG1/g3ktJvqIa4BWgU2HMh4rND6y8OD3
Hy3H8MY6CElL+MOCbFJjWqhtOxeFyZZV9q3kYnk9CAuQJKMEGuN4GU6tzhW1AgMB
AAGjezB5MA4GA1UdDwEB/wQEAwIChDATBgNVHSUEDDAKBggrBgEFBQcDATASBgNV
HRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBSu8RGpErgYUoYnQuwCq+/ggTiEjDAf
BgNVHSMEGDAWgBSu8RGpErgYUoYnQuwCq+/ggTiEjDANBgkqhkiG9w0BAQsFAAOC
AQEAXDVYov1+f6EL7S41LhYQkEX/GyNNzsEvqxE9U0+3Iri5JfkcNOiA9O9L6Z+Y
bqcsXV93s3vi4r4WSWuc//wHyJYrVe5+tK4nlFpbJOvfBUtnoBDyKNxXzZCxFJVh
f9uc8UejRfQMFbDbhWY/x83y9BDufJHHq32OjCIN7gp2UR8rnfYvlz7Zg4qkJBsn
DG4dwd+pRTCFWJOVIG0JoNhK3ZmE7oJ1N4H38XkZ31NPcMksKxpsLLIS9+mosZtg
4olL7tMPJklx5ZaeMFaKRDq4Gdxkbw4+O4vRgNm3Z8AXWKknOdfgdpqLUPPhRcP4
v1lhy71EhBuXXwRQJry0lTdF+w==
-----END CERTIFICATE-----
outputFiles:
- issued.tar
commands:
- |
set -eu
chmod +x dns-hook.sh
mkdir -p issued
profile="{{ inputs.acme_profile }}"
grep -v '^ *$' due.txt | while read -r name domains; do
args=""
for d in $domains; do args="$args -d $d"; done
echo "Ordering $name for: $domains"
# The account key lives in ./lego for this run only. See the notes on keeping it.
/lego --accept-tos --path ./lego --key-type "{{ inputs.key_type }}" --filename "$name" \
--dns "$DNS_PROVIDER" --dns.resolvers "$DNS_RESOLVERS" --dns.propagation-disable-ans \
$args run ${profile:+--profile "$profile"} 2>&1 | grep -E 'INFO|error|Error' | sed 's/^[0-9/ :]*//' | grep -vE 'Could not find solver|AuthURL'
mkdir -p "issued/$name"
cp "lego/certificates/$name.crt" "issued/$name/fullchain.pem"
cp "lego/certificates/$name.key" "issued/$name/key.pem"
done
tar -cf issued.tar issued
# -----------------------------------------------------------------------
# 3. Verify before anything is deployed
# -----------------------------------------------------------------------
- id: verify_issued
type: io.kestra.plugin.scripts.shell.Commands
description: >
Chain to a trusted root, every requested name covered, private key
matching the certificate, already valid. Any failure stops the run
with nothing deployed.
containerImage: alpine:3.20
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- apk add --no-cache -q openssl curl jq
env:
ENDPOINT: "{{ inputs.tls_endpoint }}"
TRUST_URL: "{{ inputs.trust_bundle_url }}"
RENEW_BEFORE: "{{ inputs.renew_before_days }}"
inputFiles:
lib.sh: "{{ vars.tls_lib }}"
issued.tar: "{{ outputs.renew.outputFiles['issued.tar'] }}"
inventory.json: "{{ inputs.certificates | toJson }}"
commands:
- |
. ./lib.sh
fetch_trust
tar -xf issued.tar
echo '{}' > certs.json; : > problems.txt; : > warnings.txt
for dir in issued/*/; do
name=$(basename "$dir")
awk '/BEGIN CERT/{n++} n==1' "$dir/fullchain.pem" > leaf.pem
awk '/BEGIN CERT/{n++} n>1' "$dir/fullchain.pem" > chain.pem
err=$(chain_ok leaf.pem chain.pem) || echo "$name: chain does not verify: $err" >> problems.txt
for d in $(jq -r --arg n "$name" '.[] | select(.name == $n) | .domains[]' inventory.json); do
covers leaf.pem "$d" || echo "$name: $d is not in the certificate" >> problems.txt
done
[ "$(openssl x509 -in leaf.pem -noout -pubkey | openssl sha256)" = "$(openssl pkey -in "$dir/key.pem" -pubout | openssl sha256)" ] \
|| echo "$name: private key does not match the certificate" >> problems.txt
[ "$(start_epoch leaf.pem)" -le "$(( $(date -u +%s) + 300 ))" ] || echo "$name: not valid yet" >> problems.txt
dl=$(days_left leaf.pem)
[ "$dl" -gt "$RENEW_BEFORE" ] || echo "$name: new certificate has only $dl day(s), it will be due again on the next run" >> warnings.txt
key=$(openssl x509 -in leaf.pem -noout -text | sed -n 's/^ *Public Key Algorithm: //p;s/^ *Public-Key: (\(.*\))/\1/p' | paste -sd' ')
jq --arg n "$name" --arg s "$(serial_of leaf.pem)" --argjson d "$dl" --arg k "$key" \
--arg i "$(openssl x509 -in leaf.pem -noout -issuer | sed 's/^issuer=//')" \
--arg e "$(openssl x509 -in leaf.pem -noout -enddate -dateopt iso_8601 | cut -d= -f2)" \
--arg sans "$(sans_of leaf.pem | paste -sd' ')" \
'.[$n] = {serial: $s, days_left: $d, not_after: $e, key: $k, issuer: $i, sans: $sans}' certs.json > t && mv t certs.json
echo "$name: serial $(serial_of leaf.pem), $dl days, $key, SANs $(sans_of leaf.pem | paste -sd' ')"
done
cat warnings.txt
if [ -s problems.txt ]; then cat problems.txt; echo "Verification failed, nothing deployed."; exit 1; fi
jq -n --slurpfile c certs.json --rawfile w warnings.txt '{certs: $c[0], warnings: $w}' > out.json
emit out.json
# -----------------------------------------------------------------------
# 4. Deploy: snapshot, then one atomic configuration swap
# -----------------------------------------------------------------------
- id: deploy
type: io.kestra.plugin.scripts.shell.Commands
description: >
Save the running Caddy configuration as the rollback point, replace
only the certificates being renewed (matched by tag) and load the
result in one call. Caddy applies /load atomically: a rejected
configuration leaves the old one running.
containerImage: alpine:3.20
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- apk add --no-cache -q openssl curl jq
inputFiles:
issued.tar: "{{ outputs.renew.outputFiles['issued.tar'] }}"
outputFiles:
- snapshot.json
env:
ADMIN: "{{ inputs.caddy_admin_url }}"
DROP_INTERMEDIATE: "{{ inputs.demo_drop_intermediate }}"
commands:
- |
set -eu
tar -xf issued.tar
curl -sf "$ADMIN/config/" -o snapshot.json
[ -s snapshot.json ] && [ "$(cat snapshot.json)" != "null" ] || { echo "The target has no configuration to snapshot"; exit 1; }
echo '[]' > new.json
for dir in issued/*/; do
name=$(basename "$dir")
if [ "$DROP_INTERMEDIATE" = "true" ]; then
awk '/BEGIN CERT/{n++} n==1' "$dir/fullchain.pem" > deploy.pem
echo "$name: DEMO deploying the leaf only, without the intermediate"
else
cp "$dir/fullchain.pem" deploy.pem
fi
jq --arg n "$name" --rawfile c deploy.pem --rawfile k "$dir/key.pem" \
'. + [{certificate: $c, key: $k, tags: [$n]}]' new.json > t && mv t new.json
done
names=$(jq -c '[.[].tags[0]]' new.json)
jq --slurpfile new new.json --argjson names "$names" '
.apps.tls.certificates.load_pem =
([(.apps.tls.certificates.load_pem // [])[] | select(([.tags[]?] - $names | length) == ((.tags // []) | length))] + $new[0])
| .apps.http.servers |= map_values(if ((.listen // []) | tostring | test(":443")) then .tls_connection_policies //= [{}] else . end)
' snapshot.json > next.json
echo "Certificates in the new configuration: $(jq -r '[.apps.tls.certificates.load_pem[].tags[0]] | join(", ")' next.json)"
code=$(curl -s -o resp.txt -w '%{http_code}' -X POST -H 'Content-Type: application/json' --data-binary @next.json "$ADMIN/load")
[ "$code" = "200" ] || { echo "Caddy refused the configuration ($code): $(cat resp.txt)"; exit 1; }
echo "Loaded into $ADMIN"
# -----------------------------------------------------------------------
# 5. Health check: what does a strict client see now?
# -----------------------------------------------------------------------
- id: health_check
type: io.kestra.plugin.scripts.shell.Commands
description: >
For every domain of every renewed certificate: the endpoint serves the
new serial and a strict client, trusting only the roots, completes the
handshake with hostname checking. Certificates that were not renewed
must still be served unchanged. Reports instead of failing, so the
rollback can run.
containerImage: alpine:3.20
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- apk add --no-cache -q openssl curl jq
env:
ENDPOINT: "{{ inputs.tls_endpoint }}"
TRUST_URL: "{{ inputs.trust_bundle_url }}"
EXECUTION_ID: "{{ execution.id }}"
inputFiles:
lib.sh: "{{ vars.tls_lib }}"
inventory.json: "{{ inputs.certificates | toJson }}"
issued.json: "{{ outputs.verify_issued.vars.certs | toJson }}"
before.json: "{{ outputs.plan.values.before }}"
commands:
- |
. ./lib.sh
fetch_trust
: > failures.txt; checks=0
for attempt in 1 2 3; do
: > failures.txt; checks=0
for row in $(jq -r '.[] | .name as $n | .domains[] | "\($n)|\(.)"' inventory.json); do
name=${row%%|*}; d=${row#*|}; checks=$((checks + 1))
want=$(jq -r --arg n "$name" '.[$n].serial // empty' issued.json)
[ -n "$want" ] || want=$(jq -r --arg n "$name" '.[$n].serial // empty' before.json)
if ! served "$d" h; then echo "$d: no certificate served" >> failures.txt; continue; fi
got=$(serial_of h.leaf.pem)
[ -z "$want" ] || [ "$got" = "$want" ] || echo "$d: serves $got, expected $want" >> failures.txt
if [ -n "$(jq -r --arg n "$name" '.[$n].serial // empty' issued.json)" ] && ! handshake_ok "$d"; then
echo "$d: strict client handshake failed: $(grep -m1 -E 'verify error|error:' hs.out | sed 's/^.*verify error://')" >> failures.txt
fi
done
[ -s failures.txt ] || break
echo "Attempt $attempt: $(wc -l < failures.txt) problem(s), checking again in 3 seconds"; sleep 3
done
cat failures.txt
ok=true; [ -s failures.txt ] && ok=false
echo "Health check: $checks domain check(s), healthy: $ok"
jq -n --slurpfile i issued.json --slurpfile b before.json --arg at "$(date -u +%FT%TZ)" --arg ex "$EXECUTION_ID" \
'$i[0] | with_entries(.value += {replaced_serial: $b[0][.key].serial, replaced_status: $b[0][.key].status, renewed_at: $at, execution_id: $ex})' > record.json
jq -n --argjson ok "$ok" --argjson n "$checks" --arg f "$(sed 's/$/;/' failures.txt | paste -sd' ' | sed 's/;$//')" --slurpfile r record.json '{healthy: $ok, checks: $n, failures: $f, record: $r[0]}' > out.json
emit out.json
- id: outcome
type: io.kestra.plugin.core.flow.If
condition: "{{ outputs.health_check.vars.healthy }}"
then:
- id: record
type: io.kestra.plugin.core.kv.Set
description: The certificate register, one entry per certificate with its
serial, expiry and the serial it replaced. Entries not renewed in
this run are kept.
key: certificate_register
kvType: JSON
value: "{{ {'old': (kv('certificate_register', errorOnMissing=false) ?? {}),
'new': outputs.health_check.vars.record} | jq('.old + .new') |
first | toJson }}"
else:
# ---------------------------------------------------------------------
# 6. Rollback to the snapshot and prove it took
# ---------------------------------------------------------------------
- id: rollback
type: io.kestra.plugin.scripts.shell.Commands
description: Load the configuration saved before the deploy.
containerImage: alpine:3.20
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- apk add --no-cache -q openssl curl jq
inputFiles:
snapshot.json: "{{ outputs.deploy.outputFiles['snapshot.json'] }}"
env:
ADMIN: "{{ inputs.caddy_admin_url }}"
commands:
- |
set -eu
code=$(curl -s -o resp.txt -w '%{http_code}' -X POST -H 'Content-Type: application/json' --data-binary @snapshot.json "$ADMIN/load")
[ "$code" = "200" ] || { echo "ROLLBACK FAILED ($code): $(cat resp.txt)"; exit 1; }
echo "Previous configuration loaded into $ADMIN"
- id: rollback_check
type: io.kestra.plugin.scripts.shell.Commands
description: Every domain serves exactly what it served before the run.
containerImage: alpine:3.20
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- apk add --no-cache -q openssl curl jq
env:
ENDPOINT: "{{ inputs.tls_endpoint }}"
inputFiles:
lib.sh: "{{ vars.tls_lib }}"
inventory.json: "{{ inputs.certificates | toJson }}"
before.json: "{{ outputs.plan.values.before }}"
commands:
- |
. ./lib.sh
: > diff.txt
for row in $(jq -r '.[] | .name as $n | .domains[] | "\($n)|\(.)"' inventory.json); do
name=${row%%|*}; d=${row#*|}
want=$(jq -r --arg n "$name" '.[$n].serial' before.json)
if served "$d" r; then got=$(serial_of r.leaf.pem); else got=""; fi
[ "$got" = "$want" ] || echo "$d: serves '${got:-nothing}', before the run '${want:-nothing}'" >> diff.txt
done
if [ -s diff.txt ]; then cat diff.txt; echo "Rollback incomplete"; exit 1; fi
echo "Rollback confirmed: every domain serves the same certificate as before the run."
- id: deploy_rolled_back
type: io.kestra.plugin.core.execution.Fail
errorMessage: >-
New certificates failed the health check and were rolled back. The
endpoint serves what it served before. Problems: {{
outputs.health_check.vars.failures }}
# ---------------------------------------------------------------------------
# 7. Report
# ---------------------------------------------------------------------------
- id: report
type: io.kestra.plugin.core.log.Log
message: |
Certificate inventory on {{ inputs.tls_endpoint }}:
{% for p in outputs.probe.vars.probe %}- {{ p.name }} ({{ p.domains | join(', ') }}): {{ p.status }}, {{ p.reason }}{% if p.serial != '' %}, serial {{ p.serial }}{% endif %}{% if (fromJson(outputs.plan.values.due)) contains p.name and not inputs.dry_run %} -> renewed, serial {{ outputs.verify_issued.vars.certs[p.name].serial }}, {{ outputs.verify_issued.vars.certs[p.name].days_left }} days{% endif %}
{% endfor %}
finally:
- id: purge_key_material
type: io.kestra.plugin.core.storage.PurgeCurrentExecutionFiles
description: The private keys and the configuration snapshot passed through
internal storage. Delete them when the run ends, whatever the outcome.
