id: kafka-topic-disk-budget-guard
namespace: company.team
description: |
Keep Kafka topics inside a disk budget: read the on-disk size of every topic with
DescribeLogDirs, compare it with a default budget and per-topic overrides, and flag topics
over budget, especially those with no size or time limit at all. Alert Slack, and optionally
set retention.bytes so each topic fits its budget (retention.bytes is per partition, so the
budget is divided by the partition count), describe the topic again to verify, and optionally
wait for the broker to delete the old segments. Compacted topics are only reported.
inputs:
- id: bootstrap_servers
type: STRING
displayName: Kafka bootstrap servers
defaults: "localhost:9092"
- id: default_budget_mb
type: INT
displayName: Default budget per topic (MB)
description: Total on-disk size allowed for a topic, all partitions and replicas
included.
defaults: 20
- id: budgets_mb
type: JSON
displayName: Per-topic budgets (MB)
defaults: '{"orders": 50}'
- id: wait_for_reclaim
type: BOOL
displayName: Wait for the space to come back
description: After remediation, poll DescribeLogDirs until the topics are within
budget (the broker deletes closed segments on its own schedule).
defaults: false
- id: auto_remediate
type: BOOL
displayName: Auto remediate
defaults: false
- id: dry_run
type: BOOL
displayName: Dry run
description: Must be false, together with auto_remediate, for TopicUpdate to run.
defaults: true
tasks:
- id: log_dirs
type: io.kestra.plugin.kafka.DescribeLogDirs
description: Read-only. On-disk size per topic, summed over partitions, replicas
and brokers.
properties:
bootstrap.servers: "{{ inputs.bootstrap_servers }}"
- id: over
type: io.kestra.plugin.core.output.OutputValues
values:
topics: >-
{{ {'sizes': outputs.log_dirs.topicSizes ?? {}, 'dflt':
inputs.default_budget_mb, 'b': inputs.budgets_mb} | jq('
[.sizes | to_entries[] | select(.key | startswith("__") | not)] as $s | .dflt as $dflt | .b as $b
| [$s[] | {topic: .key, size: .value, budget: (($b[.key] // $dflt) * 1048576)} | select(.size > .budget)] | sort_by(-.size)') | first | toJson }}
- id: describe_over
type: io.kestra.plugin.core.flow.Loop
description: Read the partition count and retention configs of every topic over budget.
values: "{{ fromJson(outputs.over.values.topics) | jq('map(.topic)') | first |
toJson }}"
outputs:
- id: info
type: STRING
value: >-
{{ {'topic': item.value, 'partitions':
outputs.describe.partitionCount, 'replicas':
outputs.describe.replicationFactor, 'retention_ms':
outputs.describe.configs['retention.ms'], 'retention_bytes':
outputs.describe.configs['retention.bytes'], 'cleanup':
outputs.describe.configs['cleanup.policy']} | toJson }}
tasks:
- id: describe
type: io.kestra.plugin.kafka.TopicDescribe
topic: "{{ item.value }}"
properties:
bootstrap.servers: "{{ inputs.bootstrap_servers }}"
- id: plan
type: io.kestra.plugin.core.output.OutputValues
description: retention.bytes applies per partition and per replica, so the
target is budget / (partitions x replication factor). A topic already
capped at or below its target is left out and its cap is never raised. Its
overshoot is the active segments.
values:
items: >-
{{ {'over': fromJson(outputs.over.values.topics), 'info':
(outputs.describe_over.outputs ?? []) | jq('map(.outputs.info | fromjson
| {(.topic): .}) | add // {}') | first} | jq('
.info as $i | [.over[] | . + $i[.topic] | . + {
unbounded: (.retention_ms == "-1" and .retention_bytes == "-1"),
target_retention_bytes: ((.budget / (.partitions * .replicas)) | floor | tostring | sub("\\.0$"; "")),
action: (if (.cleanup | contains("compact")) then "REPORT_COMPACTED" else "SET_RETENTION_BYTES" end)}
| select(.action == "REPORT_COMPACTED" or (((.retention_bytes // "-1") | tonumber) as $rb | ($rb < 0 or $rb > (.target_retention_bytes | tonumber))))]') | first | toJson }}
- id: summary
type: io.kestra.plugin.core.log.Log
message: >-
Kafka disk budget on {{ inputs.bootstrap_servers }}: {{
outputs.log_dirs.topicSizes | length }} topics measured, {{
fromJson(outputs.plan.values.items) | length }} over budget and not yet
capped. {{ fromJson(outputs.plan.values.items) | jq('map("\(.topic)
\((.size / 1048576 * 10 | floor) / 10) MB of \(.budget / 1048576 | floor |
tostring | sub("\\.0$"; "")) MB\(if .unbounded then " (no time or size
limit)" else "" end), \(.partitions) partitions: \(if .action ==
"REPORT_COMPACTED" then "compacted, report only" else "retention.bytes ->
\(.target_retention_bytes)" end)") | join("; ")') | first }}
- id: route
type: io.kestra.plugin.core.flow.If
condition: "{{ fromJson(outputs.plan.values.items) | length > 0 }}"
then:
- id: alert
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{{ {'text': 'Kafka topics over their disk budget on ' ~ inputs.bootstrap_servers ~ ': ' ~ (fromJson(outputs.plan.values.items) | jq('map("\(.topic) \((.size / 1048576) | floor) MB / \(.budget / 1048576) MB\(if .unbounded then ", unbounded retention" else "" end)") | join("; ")') | first) ~ '.' ~ (inputs.auto_remediate and not inputs.dry_run ? ' Setting retention.bytes now.' : ' No change applied (auto_remediate=' ~ inputs.auto_remediate ~ ', dry_run=' ~ inputs.dry_run ~ ').')} | toJson }}
- id: remediate_gate
type: io.kestra.plugin.core.flow.If
condition: "{{ inputs.auto_remediate and not inputs.dry_run and
fromJson(outputs.plan.values.items) | jq('map(select(.action ==
\"SET_RETENTION_BYTES\")) | length') | first > 0 }}"
then:
- id: remediate
type: io.kestra.plugin.core.flow.Loop
concurrencyLimit: 1
values: "{{ fromJson(outputs.plan.values.items) | jq('map(select(.action ==
\"SET_RETENTION_BYTES\"))') | first | toJson }}"
tasks:
- id: update
type: io.kestra.plugin.kafka.TopicUpdate
topic: "{{ fromJson(item.value).topic }}"
configs:
retention.bytes: "{{ fromJson(item.value).target_retention_bytes }}"
properties:
bootstrap.servers: "{{ inputs.bootstrap_servers }}"
- id: verify
type: io.kestra.plugin.core.flow.Loop
values: "{{ fromJson(outputs.plan.values.items) | jq('map(select(.action ==
\"SET_RETENTION_BYTES\") | .topic)') | first | toJson }}"
outputs:
- id: live
type: STRING
value: "{{ {'topic': item.value, 'retention_bytes':
outputs.recheck.configs['retention.bytes']} | toJson }}"
tasks:
- id: recheck
type: io.kestra.plugin.kafka.TopicDescribe
topic: "{{ item.value }}"
properties:
bootstrap.servers: "{{ inputs.bootstrap_servers }}"
- id: verify_result
type: io.kestra.plugin.core.flow.If
condition: >-
{{ {'p': fromJson(outputs.plan.values.items), 'l':
outputs.verify.outputs} | jq('(.l | map(.outputs.live | fromjson |
{(.topic): .retention_bytes}) | add) as $l
| [.p[] | select(.action == "SET_RETENTION_BYTES") | select($l[.topic] != .target_retention_bytes)] | length') | first > 0 }}
then:
- id: remediation_failed
type: io.kestra.plugin.core.execution.Fail
errorMessage: "TopicUpdate ran, but retention.bytes is not the planned value on
every topic."
else:
- id: remediated
type: io.kestra.plugin.core.log.Log
message: "retention.bytes set and verified on {{
fromJson(outputs.plan.values.items) | jq('map(select(.action
== \"SET_RETENTION_BYTES\") |
\"\\(.topic)=\\(.target_retention_bytes)\") | join(\", \")') |
first }}."
- id: wait_reclaim
type: io.kestra.plugin.core.flow.LoopUntil
runIf: "{{ inputs.wait_for_reclaim }}"
description: The broker deletes closed segments at its next retention check. The
active segment always stays, so up to one segment per partition
above the budget is accepted.
condition: >-
{{ {'s': outputs.measure.topicSizes ?? {}, 'p':
fromJson(outputs.plan.values.items)} | jq('.s as $s | [.p[] |
select(.action == "SET_RETENTION_BYTES") | select(($s[.topic] //
0) > .budget * 1.25)] | length') | first == 0 }}
checkFrequency:
interval: PT15S
maxIterations: 40
tasks:
- id: measure
type: io.kestra.plugin.kafka.DescribeLogDirs
properties:
bootstrap.servers: "{{ inputs.bootstrap_servers }}"
- id: reclaimed
type: io.kestra.plugin.core.log.Log
runIf: "{{ inputs.wait_for_reclaim }}"
message: >-
Space reclaimed: {{ {'s': outputs.measure.topicSizes, 'p':
fromJson(outputs.plan.values.items)} | jq('.s as $s | [.p[] |
select(.action == "SET_RETENTION_BYTES") | "\(.topic) \((.size /
1048576) | floor | tostring | sub("\\.0$"; "")) MB ->
\(($s[.topic] / 1048576 * 10 | floor) / 10) MB"] | join(", ")') |
first }}.
else:
- id: within_budget
type: io.kestra.plugin.core.log.Log
message: "Every topic is within its disk budget, or already capped at it."
errors:
- id: alert_flow_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{{ {'text': 'Kafka topic disk budget guard failed on ' ~ inputs.bootstrap_servers ~ ' (execution ' ~ execution.id ~ '). Disk usage unknown, check the logs.'} | toJson }}
triggers:
- id: hourly
type: io.kestra.plugin.core.trigger.Schedule
description: Shipped disabled, and always in safe mode.
cron: "30 * * * *"
disabled: true
inputs:
auto_remediate: false
dry_run: true