id: backup-coverage-auditor
namespace: company.team
description: |
Backup systems report success for everything they run while whole
databases and stores sit outside the protection scope and nobody ever
audits the gap. A keyed webhook accepts the two lists that matter, the
asset registry that must be protected and the backup job targets that
actually run, a shared workspace materializes them, and a Python step
set-diffs the two sides and scores coverage. Uncovered assets open a
GitHub issue on the tracking repository and page Slack with the coverage
percentage, while the verdict, the uncovered asset names, and the
orphaned backup targets are published as outputs.
triggers:
- id: post_coverage_snapshot
type: io.kestra.plugin.core.trigger.Webhook
description: The backup reporting job POSTs the asset registry and the backup
job targets here after each inventory run.
key: "{{ secret('BACKUP_AUDIT_WEBHOOK_KEY') }}"
- id: daily_coverage_sweep
type: io.kestra.plugin.core.trigger.Schedule
description: Fallback net, a daily coverage sweep over the registered assets,
shipped disabled until an operator switches it on.
cron: "0 5 * * *"
disabled: true
inputs:
- id: asset_registry
type: JSON
defaults:
- name: orders-postgres
type: postgresql
tier: gold
- name: clicks-warehouse
type: snowflake
tier: gold
- name: sessions-redis
type: redis
tier: silver
- name: billing-ledger-mysql
type: mysql
tier: gold
- name: archive-blobs
type: s3
tier: bronze
description: Assets that must be protected, each carrying a name, a store type,
and a tier, used when no webhook body is posted.
- id: backup_targets
type: JSON
defaults:
- target: orders-postgres
policy: nightly
last_success: 2026-10-15T02:11:00Z
- target: clicks-warehouse
policy: nightly
last_success: 2026-10-15T02:40:00Z
- target: sessions-redis
policy: hourly
last_success: 2026-10-15T12:00:00Z
- target: legacy-s3-archive
policy: weekly
last_success: 2026-10-12T03:00:00Z
description: Storage locations the backup jobs actually protect, each carrying a
target name, a policy, and the last success timestamp.
- id: repository
type: STRING
defaults: acme/data-platform
description: GitHub repository in owner slash name form that receives the
coverage gap issue when assets go unprotected.
tasks:
# --- 1. coverage diff ---
- id: coverage_workspace
type: io.kestra.plugin.core.flow.WorkingDirectory
description: One shared directory so the shell step that writes the snapshot and
the Python step that diffs it read the same file.
tasks:
- id: write_coverage_report
type: io.kestra.plugin.scripts.shell.Commands
description: Materialize the posted coverage snapshot, or the registry and
target inputs on manual runs, into the shared workspace with a quoted
heredoc.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint: []
containerImage: python:3.12-slim
commands:
- |
set -u
cat > posted_report.json <<'REPORT_EOF'
{{ (trigger.body ?? '') | toJson }}
REPORT_EOF
echo "wrote $(wc -c < posted_report.json) bytes of backup coverage report to the shared workspace"
- id: diff_coverage
type: io.kestra.plugin.scripts.python.Script
description: Set-diff the registry against the protected targets, score the
coverage percentage, and emit the coverage verdict through the outputs
protocol.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: python:3.12-slim
env:
ASSET_REGISTRY: "{{ inputs.asset_registry | toJson }}"
BACKUP_TARGETS: "{{ inputs.backup_targets | toJson }}"
script: |
import json
import os
POSTED_FILE = "posted_report.json"
def parse_json_env(name):
raw = os.environ.get(name, "")
if not raw.strip():
return None
try:
document = json.loads(raw)
except ValueError:
return None
return document if isinstance(document, (list, dict)) else None
def load_posted(path):
try:
if not os.path.exists(path) or os.path.getsize(path) == 0:
return "empty", None, "posted coverage report is missing or empty"
with open(path, "r", encoding="utf-8") as handle:
document = json.load(handle)
if isinstance(document, str):
if document == "":
return "empty", None, ""
document = json.loads(document)
if isinstance(document, dict):
return "object", document, ""
return "invalid", None, "posted coverage report is not a JSON object"
except ValueError:
return "invalid", None, "posted coverage report is not valid JSON"
def norm(value):
return str(value or "").strip().lower()
kind, posted, problem = load_posted(POSTED_FILE)
registry = None
backups = []
if kind == "object":
registry = posted.get("registry")
backups = posted.get("backups") or []
elif kind == "empty":
registry = parse_json_env("ASSET_REGISTRY")
parsed_backups = parse_json_env("BACKUP_TARGETS")
backups = parsed_backups if isinstance(parsed_backups, list) else []
if registry is None:
problem = "asset registry could not be read from the inputs"
asset_count = 0
covered_count = 0
coverage_pct = 0.0
uncovered = []
uncovered_details = []
orphan_targets = []
backups_checked = 0
error = ""
verdict = "covered"
if registry is None:
verdict = "error"
error = problem or "asset registry could not be loaded"
elif not isinstance(registry, list) or not registry:
verdict = "error"
error = "coverage report contains no registry assets"
else:
backup_rows = [b for b in backups if isinstance(b, dict)] if isinstance(backups, list) else []
covered = {norm(b.get("target")) for b in backup_rows if norm(b.get("target"))}
backups_checked = len(backup_rows)
registry_keys = set()
for asset in registry:
if not isinstance(asset, dict):
continue
name = str(asset.get("name") or "unnamed-asset")
key = norm(name)
registry_keys.add(key)
if key in covered:
covered_count += 1
else:
uncovered.append(name)
uncovered_details.append({
"name": name,
"type": str(asset.get("type") or "unknown"),
"tier": str(asset.get("tier") or "unknown"),
})
asset_count = len([a for a in registry if isinstance(a, dict)])
if asset_count:
coverage_pct = round(covered_count * 100.0 / asset_count, 2)
else:
verdict = "error"
error = "coverage report contains no registry assets"
orphan_targets = sorted({
str(b.get("target")) for b in backup_rows
if norm(b.get("target")) and norm(b.get("target")) not in registry_keys
})
if verdict != "error":
verdict = "uncovered" if uncovered else "covered"
payload = {
"verdict": verdict,
"asset_count": asset_count,
"covered_count": covered_count,
"coverage_pct": coverage_pct,
"uncovered_count": len(uncovered),
"uncovered_assets": sorted(uncovered),
"uncovered_details": uncovered_details,
"orphan_targets": orphan_targets,
"backups_checked": backups_checked,
"error": error,
}
print("assets=%d covered=%d uncovered=%d orphans=%d verdict=%s" % (
asset_count, covered_count, len(uncovered), len(orphan_targets), verdict))
print("::" + json.dumps({"outputs": payload}) + "::")
# --- 2. coverage verdict ---
- id: check_coverage
type: io.kestra.plugin.core.flow.If
description: Escalate when at least one registered asset has no backup job
covering it, and verify the error branch instead of trusting a parse
failure.
condition: "{{ outputs.diff_coverage.vars.verdict == 'uncovered' }}"
then:
- id: file_coverage_issue
type: io.kestra.plugin.github.issues.Create
description: Open a tracked issue on the platform repository that lists every
unprotected asset and every orphaned backup target.
oauthToken: "{{ secret('GITHUB_TOKEN') }}"
repository: "{{ inputs.repository }}"
title: "Backup coverage gap at {{ outputs.diff_coverage.vars.coverage_pct }}
percent"
body: |
## Backup coverage audit
- Execution {{ execution.id }}
- Registry assets {{ outputs.diff_coverage.vars.asset_count }}, protected {{ outputs.diff_coverage.vars.covered_count }}
- Coverage {{ outputs.diff_coverage.vars.coverage_pct }}%
- Unprotected assets ({{ outputs.diff_coverage.vars.uncovered_count }}) {{ outputs.diff_coverage.vars.uncovered_assets | join(', ') }}
- Backup targets outside the registry ({{ outputs.diff_coverage.vars.orphan_targets | length }}) {{ outputs.diff_coverage.vars.orphan_targets | join(', ') }}
labels:
- backup
- coverage
- id: notify_uncovered
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Tell the ops channel the coverage percentage and the exact asset
names no backup job protects.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "BACKUP GAP: {{ outputs.diff_coverage.vars.coverage_pct }}% coverage across {{ outputs.diff_coverage.vars.asset_count }} registered asset(s) - {{ outputs.diff_coverage.vars.uncovered_count }} unprotected ({{ outputs.diff_coverage.vars.uncovered_assets | join(', ') }}), {{ outputs.diff_coverage.vars.orphan_targets | length }} orphaned target(s). Execution {{ execution.id }}."
}
else:
- id: check_diff_error
type: io.kestra.plugin.core.flow.If
description: Surface a coverage report that could not be diffed as its own
branch so a broken feed never reads as full coverage.
condition: "{{ outputs.diff_coverage.vars.verdict == 'error' }}"
then:
- id: alert_unreadable_report
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Tell the ops channel the coverage report could not be diffed and
carry the load problem named by the grader.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "BACKUP coverage report unreadable - {{ outputs.diff_coverage.vars.error }}. Execution {{ execution.id }}."
}
else:
- id: log_full_coverage
type: io.kestra.plugin.core.log.Log
description: Record the full coverage grade so the execution history doubles as
protection evidence.
message: "full coverage - {{ outputs.diff_coverage.vars.covered_count }} of {{
outputs.diff_coverage.vars.asset_count }} registered asset(s)
protected across {{ outputs.diff_coverage.vars.backups_checked }}
backup target(s)."
errors:
- id: notify_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Escalate any failure to the ops channel with the execution url.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{"text": "FAILED: {{ flow.id }} {{ execution.id }}"}
outputs:
- id: verdict
type: STRING
description: covered when every registered asset has a backup target, uncovered
when one is missing, error when the report could not be diffed.
value: "{{ outputs.diff_coverage.vars.verdict }}"
- id: coverage_pct
type: FLOAT
description: Share of registered assets protected by at least one backup target,
for example 60.0.
value: "{{ outputs.diff_coverage.vars.coverage_pct }}"
- id: uncovered_assets
type: ARRAY
description: Registry asset names no backup job protects, for example the
primary billing database.
value: "{{ outputs.diff_coverage.vars.uncovered_assets | toJson }}"
- id: asset_count
type: INT
description: Number of assets in the registry that were audited during this run.
value: "{{ outputs.diff_coverage.vars.asset_count }}"
- id: orphan_targets
type: ARRAY
description: Backup targets that protect nothing in the registry, which usually
means retired work is still consuming storage.
value: "{{ outputs.diff_coverage.vars.orphan_targets | toJson }}"