id: pushgateway-stale-metric-group-reclaim-guard
namespace: company.team
description: |
Repair a live Pushgateway batch group whose metric labels drifted after a flow rename.
The flow only writes when the group is stale and Kestra has a recent successful execution
for the intended flow. It rewrites the three batch metrics and proves the old labels are gone.
inputs:
- id: prometheus_url
type: STRING
displayName: Prometheus URL
description: Prometheus server that scrapes the Pushgateway.
defaults: http://prometheus:9090
- id: pushgateway_url
type: STRING
displayName: Pushgateway URL
description: Pushgateway base URL reachable by the Kestra worker.
defaults: http://pushgateway:9091
- id: kestra_url
type: STRING
displayName: Kestra API URL
description: Kestra API used to confirm that the intended flow still runs.
defaults: http://localhost:8080
- id: kestra_user
type: STRING
displayName: Kestra API user
defaults: admin@kestra.io
- id: kestra_password
type: STRING
displayName: Kestra API password
defaults: "{{ secret('KESTRA_PASSWORD') }}"
- id: namespace
type: STRING
displayName: Kestra namespace
description: Exact namespace used to select recent successful executions.
defaults: company.team
- id: expected_flow_id
type: STRING
displayName: Expected flow label
description: Flow id that must appear in every repaired metric.
defaults: prometheus-batch-job-metrics
- id: group_job
type: STRING
displayName: Pushgateway group job
description: Stable Pushgateway grouping key. This flow never deletes or renames
a group.
defaults: prometheus-batch-job-metrics
- id: group_instance
type: STRING
displayName: Pushgateway group instance
description: Stable Pushgateway grouping instance.
defaults: kestra
- id: stale_after_seconds
type: INT
displayName: Staleness window
description: A push older than this many seconds is a candidate.
defaults: 3600
- id: execution_lookback_hours
type: INT
displayName: Execution lookback
description: Recent successful execution window used to prove the job is live.
defaults: 24
- id: replacement_duration_seconds
type: FLOAT
displayName: Replacement duration
description: Duration value written when the old group lacks a trustworthy
duration value.
defaults: 0
- id: auto_remediate
type: BOOL
displayName: Auto remediate
description: Allow the Push replacement after approval when dry_run is false.
defaults: false
- id: dry_run
type: BOOL
displayName: Dry run
description: Report the plan without writing to Pushgateway.
defaults: true
variables:
cutoff: "{{ now(timeZone='UTC') | dateAdd(-1 * inputs.execution_lookback_hours,
'HOURS') | date(\"yyyy-MM-dd'T'HH:mm:ss'Z'\", timeZone='UTC') }}"
expected_label_check: >-
[.[] | select((.labels.namespace // \"\") == \"{{ inputs.namespace }}\" and
(.labels.flow // \"\") == \"{{ inputs.expected_flow_id }}\")]
drifted_label_check: >-
[.[] | select((.labels.namespace // \"\") != \"{{ inputs.namespace }}\" or
(.labels.flow // \"\") != \"{{ inputs.expected_flow_id }}\")]
tasks:
- id: detect_stale_push
type: io.kestra.plugin.prometheus.Query
description: Read-only detection of the stable group’s stale Pushgateway timestamp.
url: "{{ inputs.prometheus_url }}"
query: 'time() - push_time_seconds{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}"} > {{ inputs.stale_after_seconds }}'
fetchType: FETCH
- id: read_success_metric
type: io.kestra.plugin.prometheus.Query
description: Read the success series and its labels from the candidate group.
url: "{{ inputs.prometheus_url }}"
query: 'job_success{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}"}'
fetchType: FETCH
- id: read_last_success_metric
type: io.kestra.plugin.prometheus.Query
description: Read the last-success series and its labels from the candidate group.
url: "{{ inputs.prometheus_url }}"
query: 'job_last_success_timestamp{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}"}'
fetchType: FETCH
- id: read_duration_metric
type: io.kestra.plugin.prometheus.Query
description: Read the duration series so the plan can prove the complete metric set.
url: "{{ inputs.prometheus_url }}"
query: 'job_duration_seconds{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}"}'
fetchType: FETCH
- id: recent_executions
type: io.kestra.plugin.core.http.Request
description: Read recent successful executions for the intended flow from Kestra’s API.
uri: "{{ inputs.kestra_url }}/api/v1/main/executions/search"
method: GET
contentType: application/json
options:
auth:
type: BASIC
username: "{{ inputs.kestra_user }}"
password: "{{ inputs.kestra_password }}"
params:
filters[state][EQUALS]: SUCCESS
filters[namespace][EQUALS]: "{{ inputs.namespace }}"
filters[flowId][EQUALS]: "{{ inputs.expected_flow_id }}"
filters[startDate][GREATER_THAN_OR_EQUAL_TO]: "{{ render(vars.cutoff) }}"
size: "50"
- id: plan
type: io.kestra.plugin.core.output.OutputValues
description: Build the repair plan from independent Pushgateway and Kestra reads.
values:
stale_count: "{{ outputs.detect_stale_push.total }}"
success_count: "{{ outputs.read_success_metric.total }}"
last_success_count: "{{ outputs.read_last_success_metric.total }}"
duration_count: "{{ outputs.read_duration_metric.total }}"
live_count: "{{ outputs.recent_executions.body | jq('.results | length') | first }}"
drifted_label_count: "{{ ({'a': outputs.read_success_metric.metrics, 'b':
outputs.read_last_success_metric.metrics, 'c':
outputs.read_duration_metric.metrics} | toJson) | jq('[.a[], .b[], .c[]]
| map(select((.labels.namespace // \"\") != \"' ~ inputs.namespace ~ '\"
or (.labels.flow // \"\") != \"' ~ inputs.expected_flow_id ~ '\")) |
length') | first }}"
live_candidate: "{{ outputs.detect_stale_push.total > 0 and
(outputs.recent_executions.body | jq('.results | length') | first) > 0
}}"
complete_before: "{{ outputs.read_success_metric.total > 0 and
outputs.read_last_success_metric.total > 0 and
outputs.read_duration_metric.total > 0 }}"
- id: summary
type: io.kestra.plugin.core.log.Log
description: Record the stale group, live execution evidence and label plan.
message: >-
Pushgateway group {{ inputs.group_job }}/{{ inputs.group_instance }}:
stale={{ outputs.plan.values.stale_count }}, recent_successes={{
outputs.plan.values.live_count }}, complete_before={{
outputs.plan.values.complete_before }}, drifted_series={{
outputs.plan.values.drifted_label_count }}. Retired groups are reported
only because Push has no delete operation. dry_run={{ inputs.dry_run }}
auto_remediate={{ inputs.auto_remediate }}.
- id: route
type: io.kestra.plugin.core.flow.If
description: Continue only when a stale group maps to a live intended flow and
carries drifted labels.
condition: "{{ outputs.plan.values.live_candidate == 'true' and
outputs.plan.values.drifted_label_count | number > 0 }}"
then:
- id: findings_alert
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alert the platform team with the bounded live repair plan.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: >-
{{ {'text': 'Pushgateway stale group ' ~ inputs.group_job ~ '/' ~
inputs.group_instance ~ ': ' ~ outputs.plan.values.stale_count ~ '
stale push timestamp(s), ' ~ outputs.plan.values.drifted_label_count ~
' drifted metric series and ' ~ outputs.plan.values.live_count ~ '
recent successful Kestra execution(s). ' ~ (inputs.auto_remediate and
not inputs.dry_run ? 'A full-group label repair waits for approval.' :
'No write will run in report-only mode.')} | toJson }}
- id: remediation_mode
type: io.kestra.plugin.core.flow.If
description: A write requires both remediation gates.
condition: "{{ inputs.auto_remediate and not inputs.dry_run }}"
then:
- id: approval
type: io.kestra.plugin.core.flow.Pause
description: Review the stale group and live execution evidence before rewriting
the metrics.
pauseDuration: PT1H
behavior: RESUME
onResume:
- id: approved
type: BOOL
required: false
description: Set true to permit the Pushgateway metric rewrite.
- id: note
type: STRING
required: false
description: Optional approval note.
- id: approval_gate
type: io.kestra.plugin.core.flow.If
description: Continue only after an explicit approval.
condition: "{{ outputs.approval.onResume.approved ?? false }}"
then:
- id: recheck_stale_push
type: io.kestra.plugin.prometheus.Query
description: Re-read staleness immediately before the write.
url: "{{ inputs.prometheus_url }}"
query: 'time() - push_time_seconds{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}"} > {{ inputs.stale_after_seconds }}'
fetchType: FETCH
- id: recheck_success_metric
type: io.kestra.plugin.prometheus.Query
description: Re-read the success labels immediately before the write.
url: "{{ inputs.prometheus_url }}"
query: 'job_success{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}"}'
fetchType: FETCH
- id: recheck_last_success_metric
type: io.kestra.plugin.prometheus.Query
description: Re-read the last-success labels immediately before the write.
url: "{{ inputs.prometheus_url }}"
query: 'job_last_success_timestamp{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}"}'
fetchType: FETCH
- id: recheck_duration_metric
type: io.kestra.plugin.prometheus.Query
description: Re-read the duration labels immediately before the write.
url: "{{ inputs.prometheus_url }}"
query: 'job_duration_seconds{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}"}'
fetchType: FETCH
- id: recheck_executions
type: io.kestra.plugin.core.http.Request
description: Re-read recent successful Kestra executions before the write.
uri: "{{ inputs.kestra_url }}/api/v1/main/executions/search"
method: GET
contentType: application/json
options:
auth:
type: BASIC
username: "{{ inputs.kestra_user }}"
password: "{{ inputs.kestra_password }}"
params:
filters[state][EQUALS]: SUCCESS
filters[namespace][EQUALS]: "{{ inputs.namespace }}"
filters[flowId][EQUALS]: "{{ inputs.expected_flow_id }}"
filters[startDate][GREATER_THAN_OR_EQUAL_TO]: "{{ render(vars.cutoff) }}"
size: "50"
- id: fresh_plan
type: io.kestra.plugin.core.output.OutputValues
description: Compare the fresh reads with the paused plan and reject a changed
target.
values:
stale_count: "{{ outputs.recheck_stale_push.total }}"
live_count: "{{ outputs.recheck_executions.body | jq('.results | length') |
first }}"
success_count: "{{ outputs.recheck_success_metric.total }}"
last_success_count: "{{ outputs.recheck_last_success_metric.total }}"
duration_count: "{{ outputs.recheck_duration_metric.total }}"
drifted_label_count: "{{ ({'a': outputs.recheck_success_metric.metrics, 'b':
outputs.recheck_last_success_metric.metrics, 'c':
outputs.recheck_duration_metric.metrics} | toJson) |
jq('[.a[], .b[], .c[]] | map(select((.labels.namespace //
\"\") != \"' ~ inputs.namespace ~ '\" or (.labels.flow //
\"\") != \"' ~ inputs.expected_flow_id ~ '\")) | length') |
first }}"
- id: stale_plan_gate
type: io.kestra.plugin.core.flow.If
description: Fail closed if any eligibility fact changed during approval.
condition: "{{ outputs.fresh_plan.values.stale_count ==
outputs.plan.values.stale_count and
outputs.fresh_plan.values.live_count ==
outputs.plan.values.live_count and
outputs.fresh_plan.values.success_count ==
outputs.plan.values.success_count and
outputs.fresh_plan.values.last_success_count ==
outputs.plan.values.last_success_count and
outputs.fresh_plan.values.duration_count ==
outputs.plan.values.duration_count and
outputs.fresh_plan.values.drifted_label_count ==
outputs.plan.values.drifted_label_count }}"
then:
- id: replace_group
type: io.kestra.plugin.prometheus.Push
description: Rewrite the three batch metrics of the stable group with the
expected labels. Push sends a POST, so a metric with any
other name in the group is kept.
url: "{{ inputs.pushgateway_url }}"
job: "{{ inputs.group_job }}"
instance: "{{ inputs.group_instance }}"
metrics:
- name: job_last_success_timestamp
value: "{{ now() | timestamp }}"
labels:
namespace: "{{ inputs.namespace }}"
flow: "{{ inputs.expected_flow_id }}"
- name: job_success
value: "1"
labels:
namespace: "{{ inputs.namespace }}"
flow: "{{ inputs.expected_flow_id }}"
- name: job_duration_seconds
value: "{{ inputs.replacement_duration_seconds }}"
labels:
namespace: "{{ inputs.namespace }}"
flow: "{{ inputs.expected_flow_id }}"
- id: wait_for_scrape
type: io.kestra.plugin.core.flow.Sleep
description: Allow Prometheus one scrape interval to observe the Push
replacement.
duration: PT2S
- id: verify_success_metric
type: io.kestra.plugin.prometheus.Query
description: Verify the repaired success series carries the expected labels.
url: "{{ inputs.prometheus_url }}"
query: 'job_success{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}",namespace="{{ inputs.namespace
}}",flow="{{ inputs.expected_flow_id }}"}'
fetchType: FETCH
- id: verify_last_success_metric
type: io.kestra.plugin.prometheus.Query
description: Verify the repaired last-success series carries the expected
labels.
url: "{{ inputs.prometheus_url }}"
query: 'job_last_success_timestamp{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}",namespace="{{ inputs.namespace
}}",flow="{{ inputs.expected_flow_id }}"}'
fetchType: FETCH
- id: verify_duration_metric
type: io.kestra.plugin.prometheus.Query
description: Verify the repaired duration series carries the expected labels.
url: "{{ inputs.prometheus_url }}"
query: 'job_duration_seconds{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}",namespace="{{ inputs.namespace
}}",flow="{{ inputs.expected_flow_id }}"}'
fetchType: FETCH
- id: verify_fresh_push
type: io.kestra.plugin.prometheus.Query
description: Verify the replacement produced a fresh Pushgateway timestamp.
url: "{{ inputs.prometheus_url }}"
query: 'time() - push_time_seconds{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}"} < {{ inputs.stale_after_seconds
}}'
fetchType: FETCH
- id: verify_drifted_labels
type: io.kestra.plugin.prometheus.Query
description: Verify that the old flow label no longer has a metric series.
url: "{{ inputs.prometheus_url }}"
query: 'job_success{job="{{ inputs.group_job }}",instance="{{
inputs.group_instance }}",flow!="{{
inputs.expected_flow_id }}"}'
fetchType: FETCH
- id: verify_result
type: io.kestra.plugin.core.flow.If
description: Fail unless every required series is present, fresh and free of
drift.
condition: "{{ outputs.verify_success_metric.total > 0 and
outputs.verify_last_success_metric.total > 0 and
outputs.verify_duration_metric.total > 0 and
outputs.verify_fresh_push.total > 0 and
outputs.verify_drifted_labels.total == 0 }}"
then:
- id: repaired
type: io.kestra.plugin.core.log.Log
description: Record the verified replacement.
message: "Repaired {{ inputs.group_job }}/{{ inputs.group_instance }} with the
expected labels and complete metric set."
else:
- id: verification_failed
type: io.kestra.plugin.core.execution.Fail
errorMessage: "Pushgateway replacement did not verify: success={{
outputs.verify_success_metric.total }},
last_success={{
outputs.verify_last_success_metric.total }},
duration={{ outputs.verify_duration_metric.total }},
fresh_push={{ outputs.verify_fresh_push.total }},
drifted={{ outputs.verify_drifted_labels.total }}."
else:
- id: stale_plan_changed
type: io.kestra.plugin.core.execution.Fail
errorMessage: "Eligibility changed during approval. The Pushgateway group was
not written. Re-run detection against the current labels
and execution history."
else:
- id: declined
type: io.kestra.plugin.core.log.Log
description: Keep the stale group untouched when approval is declined.
message: "Approval declined. The stale Pushgateway group was not written."
else:
- id: report_only
type: io.kestra.plugin.core.log.Log
description: Keep the group untouched in dry-run or non-remediating mode.
message: "Report only. A live stale group would be replaced only when
auto_remediate=true and dry_run=false with approval."
else:
- id: clean
type: io.kestra.plugin.core.log.Log
description: Record a healthy or non-live group without writing.
message: "No stale live Pushgateway group matched the repair scope. No write was
attempted."
errors:
- id: alert_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alert Slack when detection, reconciliation or verification fails.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: >-
{{ {'text': 'Pushgateway stale metric repair FAILED for ' ~
inputs.group_job ~ '/' ~ inputs.group_instance ~ ' in execution ' ~
execution.id ~ '. The group may still be stale or drifted. Check the
Kestra task logs.'} | toJson }}
triggers:
- id: stale_push_event
type: io.kestra.plugin.prometheus.Trigger
disabled: true
description: Start the repair flow when the stable group’s push timestamp
crosses the staleness window. Triggers cannot read flow inputs, so the
query is fixed to the default group and window. Edit it to match your
inputs.
interval: PT1M
url: "{{ secret('PROMETHEUS_URL') }}"
query: 'time() -
push_time_seconds{job="prometheus-batch-job-metrics",instance="kestra"} >
3600'
fetchType: FETCH
- id: hourly_backstop
type: io.kestra.plugin.core.trigger.Schedule
description: Disabled backstop. Manual execution or the Prometheus trigger is
the event path.
cron: "0 * * * *"
disabled: true
inputs:
auto_remediate: false
dry_run: true