id: dynamodb-unexpiring-item-ttl-backfill-guard
namespace: company.team
description: >-
Find DynamoDB items that the table's own TTL can never expire, backfill the
TTL attribute as a real epoch Number derived from each item's creation
timestamp, then re-read the table and fail the run if any repaired item is
still unexpirable.
labels:
owner: platform-team
system: dynamodb
inputs:
- id: dry_run
type: BOOL
defaults: true
description: Report only. No write reaches DynamoDB while this is true.
- id: auto_remediate
type: BOOL
defaults: false
description: Allow the backfill. A write needs this true, dry_run false and an
approved Pause.
- id: table_name
type: STRING
defaults: sessions
validator: "^[A-Za-z0-9_.-]{3,255}$"
description: DynamoDB table to audit.
- id: ttl_attribute
type: STRING
defaults: expires_at
validator: "^[A-Za-z_][A-Za-z0-9_]{0,254}$"
description: >-
Attribute DynamoDB is configured to read TTL from. It must not be a
DynamoDB reserved word, because the Scan task exposes no
expressionAttributeNames to alias it with.
- id: created_at_attribute
type: STRING
defaults: created_at
description: >-
Existing creation timestamp the expiry is derived from. Accepts an epoch
Number or an ISO 8601 String.
- id: retention_days
type: INT
defaults: 90
description: Retention window added to the creation timestamp to get the expiry epoch.
- id: max_items_per_run
type: INT
defaults: 25
description: >-
Cap on items repaired per run. Keep it small so the read to write window
stays short, because each write is a conditional single-attribute update.
- id: scan_limit
type: INT
defaults: 500
description: Maximum items the detection Scan reads in one pass.
- id: aws_region
type: STRING
defaults: us-east-1
description: Region the table lives in.
- id: endpoint_url
type: STRING
defaults: ""
validator: "^(https?://[A-Za-z0-9._:/-]+)?$"
description: >-
Override the DynamoDB endpoint. Leave empty for AWS. Set it to a local
endpoint to run this against DynamoDB Local or LocalStack.
- id: access_key_id
type: STRING
defaults: ""
description: Overrides the AWS_ACCESS_KEY_ID secret when set. Leave empty in production.
- id: secret_access_key
type: STRING
defaults: ""
description: Overrides the AWS_SECRET_ACCESS_KEY secret when set. Leave empty in
production.
- id: settle_duration
type: DURATION
defaults: PT2S
description: Pause before the re-read, because a Scan is eventually consistent.
variables:
cli_image: public.ecr.aws/aws-cli/aws-cli:latest
ttl_filter: "attribute_not_exists({{ inputs.ttl_attribute }}) or
attribute_type({{ inputs.ttl_attribute }}, :s)"
tasks:
- id: read_ttl_config
type: io.kestra.plugin.aws.cli.AwsCLI
description: >-
Read DescribeTimeToLive. Backfilling an attribute the table is not
configured to read expires nothing, so this runs before anything else.
containerImage: "{{ vars.cli_image }}"
region: "{{ inputs.aws_region }}"
accessKeyId: "{{ inputs.access_key_id != '' ? inputs.access_key_id :
secret('AWS_ACCESS_KEY_ID') }}"
secretKeyId: "{{ inputs.secret_access_key != '' ? inputs.secret_access_key :
secret('AWS_SECRET_ACCESS_KEY') }}"
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint: []
commands:
- set -e
- >-
EP="{{ inputs.endpoint_url != '' ? '--endpoint-url ' ~
inputs.endpoint_url : '' }}"
- aws dynamodb describe-time-to-live --table-name "{{ inputs.table_name
}}" $EP --output json > ttl.json
- aws dynamodb describe-table --table-name "{{ inputs.table_name }}" $EP
--query 'Table.KeySchema[].AttributeName' --output json > keys.json
- cat ttl.json
- |
STATUS=$(jq -r '.TimeToLiveDescription.TimeToLiveStatus // "UNKNOWN"' ttl.json)
ATTR=$(jq -r '.TimeToLiveDescription.AttributeName // ""' ttl.json)
printf '::%s::\n' "$(jq -nc --arg s "$STATUS" --arg a "$ATTR" --argjson k "$(cat keys.json)" '{outputs:{ttl_status:$s,ttl_attribute:$a,key_schema:($k|tojson)}}')"
- id: log_ttl_config
type: io.kestra.plugin.core.log.Log
message: >-
TTL config on {{ inputs.table_name }}: status={{
outputs.read_ttl_config.vars.ttl_status }} attribute={{
outputs.read_ttl_config.vars.ttl_attribute }} expected={{
inputs.ttl_attribute }} key_schema={{
outputs.read_ttl_config.vars.key_schema }}
- id: refuse_misconfigured_ttl
type: io.kestra.plugin.core.flow.If
description: >-
Refuse the run when TTL is off or set on a different attribute. Writing
the attribute anyway would look like a fix while nothing ever expires.
condition: "{{ outputs.read_ttl_config.vars.ttl_status != 'ENABLED' or
outputs.read_ttl_config.vars.ttl_attribute != inputs.ttl_attribute }}"
then:
- id: refuse_reason
type: io.kestra.plugin.core.log.Log
level: ERROR
message: >-
Refusing to backfill. DynamoDB reports TTL status {{
outputs.read_ttl_config.vars.ttl_status }} on attribute "{{
outputs.read_ttl_config.vars.ttl_attribute }}" for table {{
inputs.table_name }}, so writing "{{ inputs.ttl_attribute }}" would
expire nothing. Enable TTL on that attribute first. You can also point
ttl_attribute at the one the table uses.
- id: fail_misconfigured
type: io.kestra.plugin.core.execution.Fail
errorMessage: >-
TTL is not enabled on {{ inputs.table_name }}.{{ inputs.ttl_attribute
}} (reported status {{ outputs.read_ttl_config.vars.ttl_status }},
reported attribute "{{ outputs.read_ttl_config.vars.ttl_attribute
}}").
- id: detect_unexpiring
type: io.kestra.plugin.aws.dynamodb.Scan
description: >-
Scan for items DynamoDB will never expire. The filter catches both a
missing TTL attribute and one stored as a String, which TTL skips silently
because it only acts on a Number.
region: "{{ inputs.aws_region }}"
accessKeyId: "{{ inputs.access_key_id != '' ? inputs.access_key_id :
secret('AWS_ACCESS_KEY_ID') }}"
secretKeyId: "{{ inputs.secret_access_key != '' ? inputs.secret_access_key :
secret('AWS_SECRET_ACCESS_KEY') }}"
endpointOverride: "{{ inputs.endpoint_url != '' ? inputs.endpoint_url : null }}"
tableName: "{{ inputs.table_name }}"
fetchType: FETCH
limit: "{{ inputs.scan_limit }}"
filterExpression: "{{ render(vars.ttl_filter) }}"
expressionAttributeValues:
":s": S
- id: log_detection
type: io.kestra.plugin.core.log.Log
message: >-
Scan found {{ outputs.detect_unexpiring.size }} item(s) on {{
inputs.table_name }} that TTL cannot expire.
- id: plan_backfill
type: io.kestra.plugin.aws.cli.AwsCLI
description: >-
Re-read the matching items as raw DynamoDB JSON, classify each one and
build a capped write plan. Nothing is written here.
containerImage: "{{ vars.cli_image }}"
region: "{{ inputs.aws_region }}"
accessKeyId: "{{ inputs.access_key_id != '' ? inputs.access_key_id :
secret('AWS_ACCESS_KEY_ID') }}"
secretKeyId: "{{ inputs.secret_access_key != '' ? inputs.secret_access_key :
secret('AWS_SECRET_ACCESS_KEY') }}"
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint: []
env:
TTL_ATTR: "{{ inputs.ttl_attribute }}"
CREATED_ATTR: "{{ inputs.created_at_attribute }}"
RETENTION_DAYS: "{{ inputs.retention_days }}"
MAX_ITEMS: "{{ inputs.max_items_per_run }}"
KEY_SCHEMA: "{{ outputs.read_ttl_config.vars.key_schema }}"
NOW_EPOCH: "{{ execution.startDate | timestamp }}"
outputFiles:
- plan.json
inputFiles:
classify.py: |
import json, os, sys
from datetime import datetime, timezone
ttl_attr = os.environ["TTL_ATTR"]
created_attr = os.environ["CREATED_ATTR"]
retention = int(os.environ["RETENTION_DAYS"]) * 86400
cap = int(os.environ["MAX_ITEMS"])
now = int(os.environ["NOW_EPOCH"])
key_names = json.loads(os.environ["KEY_SCHEMA"])
with open("scan.json") as fh:
items = json.load(fh).get("Items") or []
ISO = ["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%dT%H:%M:%S%z",
"%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%dT%H:%M:%S",
"%Y-%m-%d %H:%M:%S", "%Y-%m-%d"]
def to_epoch(text):
raw = text.strip()
if raw.isdigit():
return int(raw)
probe = raw[:-1] + "+0000" if raw.endswith("Z") else raw
for fmt in ISO:
try:
parsed = datetime.strptime(probe, fmt)
except ValueError:
continue
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return int(parsed.timestamp())
return None
def label(item):
return "/".join(str(list(item.get(k, {}).values())[0]) for k in key_names if k in item)
plan, refused, already_past = [], [], 0
missing = wrong_type = 0
for item in items:
key = dict((k, item[k]) for k in key_names if k in item)
name = label(item)
if len(key) != len(key_names):
refused.append(name + " (incomplete key)")
continue
current = item.get(ttl_attr)
if current is None:
missing += 1
source = item.get(created_attr)
if source is None:
refused.append(name + " (no " + created_attr + ", refusing to guess)")
continue
raw = source.get("N") or source.get("S")
if raw is None:
refused.append(name + " (" + created_attr + " is not a Number or String)")
continue
base = to_epoch(str(raw).split(".")[0] if source.get("N") else str(raw))
if base is None:
refused.append(name + " (" + created_attr + "=" + str(raw) + " is unparseable)")
continue
ttl, reason = base + retention, "missing_attribute"
elif "S" in current:
wrong_type += 1
kept = to_epoch(current["S"])
if kept is None:
refused.append(name + " (" + ttl_attr + "=" + current["S"] + " is not an epoch or a date)")
continue
ttl, reason = kept, "wrong_type_string"
else:
refused.append(name + " (unexpected " + ttl_attr + " type " + ",".join(current) + ")")
continue
if ttl <= now:
already_past += 1
plan.append({"key_json": json.dumps(key, separators=(",", ":")),
"ttl": ttl, "reason": reason, "label": name})
plan.sort(key=lambda row: row["ttl"])
capped = plan[:cap]
with open("plan.json", "w") as fh:
json.dump(capped, fh)
out = {
"unexpiring_total": len(items),
"missing_attribute": missing,
"wrong_type_string": wrong_type,
"planned": len(capped),
"deferred": len(plan) - len(capped),
"refused": len(refused),
"already_past_retention": already_past,
"planned_labels": ", ".join(row["label"] for row in capped) or "none",
"refused_labels": "; ".join(refused) or "none",
}
print("::" + json.dumps({"outputs": out}) + "::")
for row in capped:
print("PLAN " + row["label"] + " reason=" + row["reason"] + " ttl=" + str(row["ttl"]))
for row in refused:
print("REFUSED " + row, file=sys.stderr)
commands:
- set -e
- >-
EP="{{ inputs.endpoint_url != '' ? '--endpoint-url ' ~
inputs.endpoint_url : '' }}"
- |
aws dynamodb scan --table-name "{{ inputs.table_name }}" $EP \
--filter-expression "{{ render(vars.ttl_filter) }}" \
--expression-attribute-values '{":s":{"S":"S"}}' \
--max-items {{ inputs.scan_limit }} --output json > scan.json
- python3 classify.py
- id: report_findings
type: io.kestra.plugin.core.log.Log
message: >-
{{ inputs.table_name }}: {{ outputs.plan_backfill.vars.unexpiring_total }}
unexpiring, {{ outputs.plan_backfill.vars.missing_attribute }} missing {{
inputs.ttl_attribute }}, {{ outputs.plan_backfill.vars.wrong_type_string
}} holding it as a String, {{ outputs.plan_backfill.vars.planned }}
planned ({{ outputs.plan_backfill.vars.already_past_retention }} already
past retention), {{ outputs.plan_backfill.vars.deferred }} deferred by the
cap, {{ outputs.plan_backfill.vars.refused }} refused. Refused: {{
outputs.plan_backfill.vars.refused_labels }}
- id: alert_findings
type: io.kestra.plugin.core.flow.If
condition: "{{ outputs.plan_backfill.vars.unexpiring_total > 0 }}"
then:
- id: slack_findings
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "DynamoDB TTL backfill guard on {{ inputs.table_name }}: {{ outputs.plan_backfill.vars.unexpiring_total }} item(s) cannot be expired by TTL ({{ outputs.plan_backfill.vars.missing_attribute }} missing {{ inputs.ttl_attribute }}, {{ outputs.plan_backfill.vars.wrong_type_string }} String typed). Planned {{ outputs.plan_backfill.vars.planned }}, refused {{ outputs.plan_backfill.vars.refused }}. dry_run={{ inputs.dry_run }} auto_remediate={{ inputs.auto_remediate }}. Execution {{ execution.id }}."
}
- id: remediate_gate
type: io.kestra.plugin.core.flow.If
description: Writes happen only with auto_remediate true, dry_run false and a
plan to apply.
condition: "{{ inputs.auto_remediate and not inputs.dry_run and
outputs.plan_backfill.vars.planned > 0 }}"
then:
- id: hold_for_approval
type: io.kestra.plugin.core.flow.Pause
description: >-
A TTL attribute arms a delete, so an operator confirms the plan before
anything is written.
onResume:
- id: approved
type: BOOL
required: false
description: Set true to apply the planned TTL values.
- id: apply_or_skip
type: io.kestra.plugin.core.flow.If
condition: "{{ outputs.hold_for_approval.onResume.approved ?? false }}"
then:
- id: backfill_ttl
type: io.kestra.plugin.aws.cli.AwsCLI
description: >-
Write the TTL attribute one item at a time with a condition
expression, so an item somebody else has already fixed is left
alone and an item deleted since the scan is not recreated.
containerImage: "{{ vars.cli_image }}"
region: "{{ inputs.aws_region }}"
accessKeyId: "{{ inputs.access_key_id != '' ? inputs.access_key_id :
secret('AWS_ACCESS_KEY_ID') }}"
secretKeyId: "{{ inputs.secret_access_key != '' ? inputs.secret_access_key :
secret('AWS_SECRET_ACCESS_KEY') }}"
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint: []
env:
TTL_ATTR: "{{ inputs.ttl_attribute }}"
TABLE: "{{ inputs.table_name }}"
ENDPOINT: "{{ inputs.endpoint_url }}"
inputFiles:
plan.json: "{{ outputs.plan_backfill.outputFiles['plan.json'] }}"
backfill.py: |
import json, os, subprocess, sys
ttl_attr = os.environ["TTL_ATTR"]
table = os.environ["TABLE"]
endpoint = os.environ.get("ENDPOINT") or ""
with open("plan.json") as fh:
plan = json.load(fh)
written, skipped, failed, applied = 0, 0, 0, []
for row in plan:
key_name = next(iter(json.loads(row["key_json"])))
args = ["aws", "dynamodb", "update-item", "--table-name", table]
if endpoint:
args += ["--endpoint-url", endpoint]
args += [
"--key", row["key_json"],
"--update-expression", "SET #t = :v",
"--condition-expression",
"attribute_exists(#k) and (attribute_not_exists(#t) or attribute_type(#t, :s))",
"--expression-attribute-names", json.dumps({"#t": ttl_attr, "#k": key_name}),
"--expression-attribute-values",
json.dumps({":v": {"N": str(row["ttl"])}, ":s": {"S": "S"}}),
]
done = subprocess.run(args, capture_output=True, text=True)
if done.returncode == 0:
written += 1
applied.append(row["label"])
print("WROTE " + row["label"] + " " + ttl_attr + "=" + str(row["ttl"]))
elif "ConditionalCheckFailedException" in done.stderr:
skipped += 1
print("SKIPPED " + row["label"] + " changed since the plan (Number TTL set or item deleted)")
else:
failed += 1
print("FAILED " + row["label"] + " " + done.stderr.strip()[:300],
file=sys.stderr)
out = {"written": written, "skipped": skipped, "failed": failed,
"attempted": len(plan),
"applied_labels": ", ".join(applied) or "none"}
print("::" + json.dumps({"outputs": out}) + "::")
commands:
- set -e
- python3 backfill.py
- id: settle
type: io.kestra.plugin.core.flow.Sleep
duration: "{{ inputs.settle_duration }}"
- id: reverify_scan
type: io.kestra.plugin.aws.dynamodb.Scan
description: Re-run the detection Scan so the remaining population is measured,
not assumed.
region: "{{ inputs.aws_region }}"
accessKeyId: "{{ inputs.access_key_id != '' ? inputs.access_key_id :
secret('AWS_ACCESS_KEY_ID') }}"
secretKeyId: "{{ inputs.secret_access_key != '' ? inputs.secret_access_key :
secret('AWS_SECRET_ACCESS_KEY') }}"
endpointOverride: "{{ inputs.endpoint_url != '' ? inputs.endpoint_url : null }}"
tableName: "{{ inputs.table_name }}"
fetchType: FETCH
limit: "{{ inputs.scan_limit }}"
filterExpression: "{{ render(vars.ttl_filter) }}"
expressionAttributeValues:
":s": S
- id: reverify_keys
type: io.kestra.plugin.aws.cli.AwsCLI
description: >-
Read every item the backfill touched and confirm the TTL attribute
now exists as a Number.
containerImage: "{{ vars.cli_image }}"
region: "{{ inputs.aws_region }}"
accessKeyId: "{{ inputs.access_key_id != '' ? inputs.access_key_id :
secret('AWS_ACCESS_KEY_ID') }}"
secretKeyId: "{{ inputs.secret_access_key != '' ? inputs.secret_access_key :
secret('AWS_SECRET_ACCESS_KEY') }}"
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint: []
env:
TTL_ATTR: "{{ inputs.ttl_attribute }}"
TABLE: "{{ inputs.table_name }}"
ENDPOINT: "{{ inputs.endpoint_url }}"
inputFiles:
plan.json: "{{ outputs.plan_backfill.outputFiles['plan.json'] }}"
verify.py: |
import json, os, subprocess, sys
ttl_attr = os.environ["TTL_ATTR"]
table = os.environ["TABLE"]
endpoint = os.environ.get("ENDPOINT") or ""
with open("plan.json") as fh:
plan = json.load(fh)
good, bad = 0, []
for row in plan:
args = ["aws", "dynamodb", "get-item", "--table-name", table,
"--consistent-read", "--key", row["key_json"], "--output", "json"]
if endpoint:
args += ["--endpoint-url", endpoint]
done = subprocess.run(args, capture_output=True, text=True)
if done.returncode != 0:
bad.append(row["label"] + " (read failed)")
continue
item = (json.loads(done.stdout or "{}") or {}).get("Item")
if item is None:
print("GONE " + row["label"] + " no longer present")
good += 1
continue
value = item.get(ttl_attr)
if value and "N" in value:
good += 1
print("OK " + row["label"] + " " + ttl_attr + "=" + value["N"])
else:
shape = ",".join(value) if value else "absent"
bad.append(row["label"] + " (" + ttl_attr + " is " + shape + ")")
print("STILL UNEXPIRING " + row["label"] + " " + shape, file=sys.stderr)
out = {"verified": good, "unrepaired": len(bad),
"unrepaired_labels": "; ".join(bad) or "none"}
print("::" + json.dumps({"outputs": out}) + "::")
commands:
- set -e
- python3 verify.py
- id: log_reverify
type: io.kestra.plugin.core.log.Log
message: >-
Backfill wrote {{ outputs.backfill_ttl.vars.written }} of {{
outputs.backfill_ttl.vars.attempted }} item(s) (skipped {{
outputs.backfill_ttl.vars.skipped }}, failed {{
outputs.backfill_ttl.vars.failed }}). Unexpiring population {{
outputs.plan_backfill.vars.unexpiring_total }} before, {{
outputs.reverify_scan.size }} after. Re-read verified {{
outputs.reverify_keys.vars.verified }}, unrepaired {{
outputs.reverify_keys.vars.unrepaired }}: {{
outputs.reverify_keys.vars.unrepaired_labels }}
- id: fail_if_unrepaired
type: io.kestra.plugin.core.flow.If
condition: "{{ outputs.reverify_keys.vars.unrepaired > 0 or
outputs.backfill_ttl.vars.failed > 0 }}"
then:
- id: fail_backfill
type: io.kestra.plugin.core.execution.Fail
errorMessage: >-
Backfill did not land. {{
outputs.reverify_keys.vars.unrepaired }} item(s) still carry
no Number TTL after the write ({{
outputs.reverify_keys.vars.unrepaired_labels }}) and {{
outputs.backfill_ttl.vars.failed }} write(s) errored.
else:
- id: log_declined
type: io.kestra.plugin.core.log.Log
level: WARN
message: >-
Approval declined or not granted. No TTL value was written to {{
inputs.table_name }}. {{ outputs.plan_backfill.vars.planned }}
item(s) stay unexpirable: {{
outputs.plan_backfill.vars.planned_labels }}
- id: summary
type: io.kestra.plugin.core.log.Log
message: >-
Done on {{ inputs.table_name }}. dry_run={{ inputs.dry_run }}
auto_remediate={{ inputs.auto_remediate }} unexpiring={{
outputs.plan_backfill.vars.unexpiring_total }} planned={{
outputs.plan_backfill.vars.planned }} refused={{
outputs.plan_backfill.vars.refused }}
errors:
- id: slack_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "DynamoDB TTL backfill guard FAILED on table {{ inputs.table_name }} (execution {{ execution.id }}). Either the table does not have TTL enabled on {{ inputs.ttl_attribute }}; a backfilled item is still unexpirable; or DynamoDB rejected a call. Nothing further was written."
}
triggers:
- id: weekly_audit
type: io.kestra.plugin.core.trigger.Schedule
description: Weekly read-only audit. Enable it after you have run the flow by hand once.
cron: "0 6 * * 1"
disabled: true
inputs:
dry_run: true
auto_remediate: false