id: keda-scaledobject-error-budget-gate
namespace: company.team
description: |
Score KEDA ScaledObject health against a burn based error budget so HPA
thrashing is gated instead of ignored. A status exporter or CI job POSTs
ScaledObject and HPA status JSON to a keyed webhook, the flow stages that
status as a deterministic fixture when none has landed, Python burns budget
points for every failed trigger and every replica flap, and an If splits an
exhausted budget from the rest. An exhausted budget alerts Slack and fails
the run, while a Switch on the remaining state separates a within budget
burn from an unparseable status, and the state, the consumed share and the
worst trigger are published as outputs.
triggers:
- id: on_status_publish
type: io.kestra.plugin.core.trigger.Webhook
description: The KEDA status exporter or a CI job POSTs ScaledObject and HPA
status JSON here so the budget is scored the moment new metrics land.
key: "{{ secret('KEDA_STATUS_WEBHOOK_KEY') }}"
- id: daily_budget_rollup
type: io.kestra.plugin.core.trigger.Schedule
description: Daily 06:00 UTC net that rescores the standing status file, shipped
disabled until the webhook path is trusted in your environment.
cron: "0 6 * * *"
disabled: true
inputs:
- id: exhausted_threshold_pct
type: FLOAT
defaults: 70.0
description: Share of the error budget that must be burned before the gate
reports exhausted instead of within.
- id: window_hours
type: INT
defaults: 24
description: Lookback window the publisher aggregated trigger failures and
replica flaps over, echoed in the alerts for context.
- id: scaledobject_name
type: STRING
defaults: checkout-api-scaler
description: Name of the KEDA ScaledObject under audit, used in the alert text
when the posted status omits one.
tasks:
# --- 1. status workspace ---
- id: status_workspace
type: io.kestra.plugin.core.flow.WorkingDirectory
description: One shared directory so the staged status fixture and the budget
scorer read the same file for the whole run.
tasks:
# --- 2. status fixture ---
- id: write_status_fixture
type: io.kestra.plugin.scripts.shell.Commands
description: Write a deterministic ScaledObject status fixture with a heredoc,
one failing queue trigger and a thrashing HPA, so the scorer always
has a status file to read when no publisher has posted.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint: []
containerImage: alpine:3.20
commands:
- |
set -u
cat > status.json <<'JSON'
{
"scaledobject": "checkout-api-scaler",
"namespace": "prod",
"window_hours": 24,
"scale_events": 22,
"replica_flaps": 6,
"triggers": [
{"type": "aws-sqs-queue", "name": "checkout-sqs", "status": "failing", "failure_count": 10},
{"type": "prometheus", "name": "http-latency", "status": "active", "failure_count": 0}
]
}
JSON
echo "wrote $(wc -c < status.json) bytes of ScaledObject status"
# --- 3. budget score ---
- id: score_error_budget
type: io.kestra.plugin.scripts.python.Script
description: Read the posted body or the staged status, burn five budget points
per trigger failure and four per replica flap against a hundred point
budget, and emit the state, the consumed share, the worst trigger and
the counters through the outputs protocol.
containerImage: python:3.12-slim
env:
KEDA_STATUS_JSON: "{{ trigger.body ?? '' }}"
EXHAUSTED_THRESHOLD_PCT: "{{ inputs.exhausted_threshold_pct }}"
script: |
import json
import os
FAILURE_BURN_PCT = 5.0
FLAP_BURN_PCT = 4.0
DEFAULT_THRESHOLD_PCT = 70.0
def as_int(value, fallback=0):
try:
return int(value)
except (TypeError, ValueError):
try:
return int(float(value))
except (TypeError, ValueError):
return fallback
def as_float(value, fallback=0.0):
try:
return float(value)
except (TypeError, ValueError):
return fallback
def load_status():
raw = (os.environ.get("KEDA_STATUS_JSON") or "").strip()
if raw:
try:
return json.loads(raw)
except ValueError:
return None
path = "status.json"
if not os.path.exists(path) or os.path.getsize(path) == 0:
return None
try:
with open(path, "r", encoding="utf-8") as handle:
return json.load(handle)
except ValueError:
return None
threshold = as_float(os.environ.get("EXHAUSTED_THRESHOLD_PCT"), DEFAULT_THRESHOLD_PCT)
outputs = {
"parse_ok": False,
"budget_state": "unknown",
"budget_consumed_pct": 0.0,
"failed_trigger_count": 0,
"total_trigger_failures": 0,
"replica_flaps": 0,
"worst_trigger": "none",
}
status = load_status()
if status is not None:
triggers = status.get("triggers")
if not isinstance(triggers, list):
triggers = []
total_failures = 0
failed_count = 0
worst_name = "none"
worst_failures = 0
for trigger in triggers:
if not isinstance(trigger, dict):
continue
failures = max(0, as_int(trigger.get("failure_count")))
state = str(trigger.get("status") or "").strip().lower()
if failures > 0 or state == "failing":
failed_count += 1
total_failures += failures
if failures > worst_failures:
worst_failures = failures
worst_name = str(trigger.get("name") or trigger.get("type") or "unknown")
flaps = max(0, as_int(status.get("replica_flaps")))
consumed = min(100.0, total_failures * FAILURE_BURN_PCT + flaps * FLAP_BURN_PCT)
outputs.update({
"parse_ok": True,
"budget_state": "exhausted" if consumed >= threshold else "within",
"budget_consumed_pct": round(consumed, 1),
"failed_trigger_count": failed_count,
"total_trigger_failures": total_failures,
"replica_flaps": flaps,
"worst_trigger": worst_name,
})
print("parse_ok=%s consumed=%.1f%% state=%s failures=%d flaps=%d" % (
outputs["parse_ok"], outputs["budget_consumed_pct"], outputs["budget_state"],
outputs["total_trigger_failures"], outputs["replica_flaps"]))
print("::" + json.dumps({"outputs": outputs}) + "::")
# --- 4. burn gate ---
- id: burn_gate
type: io.kestra.plugin.core.flow.If
description: Branch on whether the burned share met the exhausted threshold, so
only an exhausted budget reaches the Slack alert and the failing gate
below.
condition: "{{ outputs.score_error_budget.vars.budget_state == 'exhausted' }}"
then:
# --- 5. exhausted escalation ---
- id: alert_exhausted
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Post the consumed share, the failure counts, the flap count and the
worst trigger so the platform team knows what to repair first.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "KEDA error budget EXHAUSTED for {{ inputs.scaledobject_name }} - {{ outputs.score_error_budget.vars.budget_consumed_pct }} percent burned in {{ inputs.window_hours }} hour(s), {{ outputs.score_error_budget.vars.total_trigger_failures }} trigger failure(s) across {{ outputs.score_error_budget.vars.failed_trigger_count }} trigger(s), {{ outputs.score_error_budget.vars.replica_flaps }} replica flap(s), worst trigger {{ outputs.score_error_budget.vars.worst_trigger }}. HPA thrash continues until the trigger is repaired. Execution {{ execution.id }}."
}
- id: fail_gate
type: io.kestra.plugin.core.execution.Fail
description: Turn the execution red so the deploy pipeline watching this gate
stops instead of shipping on a thrashing scaler.
errorMessage: "KEDA error budget exhausted for {{ inputs.scaledobject_name }} at
{{ outputs.score_error_budget.vars.budget_consumed_pct }} percent
consumed."
else:
# --- 6. remaining states ---
- id: route_remaining
type: io.kestra.plugin.core.flow.Switch
description: Route the states that did not exhaust the budget, recording a
within budget burn with its counters and flagging an unparseable
status for investigation.
value: "{{ outputs.score_error_budget.vars.budget_state }}"
cases:
within:
- id: log_within_budget
type: io.kestra.plugin.core.log.Log
description: Record the within budget burn so the execution history still shows
the scaler was checked and how much budget it used.
message: "KEDA budget within for {{ inputs.scaledobject_name }} - {{
outputs.score_error_budget.vars.budget_consumed_pct }} percent
burned in {{ inputs.window_hours }} hour(s), {{
outputs.score_error_budget.vars.total_trigger_failures }}
trigger failure(s), {{
outputs.score_error_budget.vars.replica_flaps }} flap(s), worst
trigger {{ outputs.score_error_budget.vars.worst_trigger }}."
defaults:
- id: log_unknown_budget
type: io.kestra.plugin.core.log.Log
description: Flag the unparseable status so a broken publisher is investigated
instead of being mistaken for a quiet scaler.
message: "KEDA budget state unknown for {{ inputs.scaledobject_name }} - the
posted status could not be parsed, fix the publisher and re-run
the gate."
errors:
- id: notify_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Escalate any failure to the ops channel with the execution url.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{"text": "FAILED: {{ flow.id }} {{ execution.id }}"}
outputs:
- id: budget_state
type: STRING
description: Budget verdict for the window, exhausted when the burned share met
the threshold, within when it stayed under it, unknown when the status
could not be parsed.
value: "{{ outputs.score_error_budget.vars.budget_state }}"
- id: budget_consumed_pct
type: FLOAT
description: Share of the hundred point error budget burned this window, for
example 74.0 when ten failures and six flaps were scored.
value: "{{ outputs.score_error_budget.vars.budget_consumed_pct }}"
- id: failed_trigger_count
type: INT
description: Number of ScaledObject triggers reporting failures or a failing
status in the scored window.
value: "{{ outputs.score_error_budget.vars.failed_trigger_count }}"
- id: total_trigger_failures
type: INT
description: Sum of per trigger failure counts across every trigger in the
posted status.
value: "{{ outputs.score_error_budget.vars.total_trigger_failures }}"
- id: replica_flaps
type: INT
description: Number of HPA scale up and scale down reversals the publisher
observed inside the window.
value: "{{ outputs.score_error_budget.vars.replica_flaps }}"
- id: worst_trigger
type: STRING
description: Name of the trigger with the highest failure count, for example
checkout-sqs, or none when no trigger failed.
value: "{{ outputs.score_error_budget.vars.worst_trigger }}"