id: elasticsearch-runaway-task-cancel-guard
namespace: company.team
description: |
Find Elasticsearch reindex and by-query tasks that were launched with
wait_for_completion=false and have been running past a threshold, cancel
them behind a dry run and an approval gate, wait out the cooperative
cancellation window, then re-read the task list and fail the run if a
targeted task is still there.
inputs:
- id: elasticsearch_url
type: STRING
defaults: "http://localhost:9200"
description: Base URL of the Elasticsearch cluster, scheme host and port.
- id: elasticsearch_username
type: STRING
defaults: "{{ secret('ELASTICSEARCH_USERNAME') }}"
description: Username for basic auth. Clusters with security disabled ignore it.
- id: elasticsearch_password
type: STRING
defaults: "{{ secret('ELASTICSEARCH_PASSWORD') }}"
description: Password for basic auth. Clusters with security disabled ignore it.
- id: task_actions
type: STRING
defaults: "indices:data/write/reindex,indices:data/write/update/byquery,indices\
:data/write/delete/byquery"
description: >-
Comma separated allowlist of task action names the flow may cancel,
matched exactly. This is the safety boundary of the whole flow.
Elasticsearch runs internal tasks that keep the cluster alive, so a
wildcard here matches no action name at all rather than widening the blast
radius.
- id: max_running_seconds
type: INT
defaults: 1800
description: >-
A task is flagged once running_time_in_nanos is above this many seconds.
Set it above the longest reindex you expect to finish on its own.
- id: max_cancel_per_run
type: INT
defaults: 5
description: >-
The flow refuses to cancel anything when more tasks are flagged than this.
A large list usually means the threshold is wrong, not that the cluster is
full of runaway work.
- id: settle_timeout
type: STRING
defaults: "PT1M"
description: >-
How long to wait for cancelled tasks to leave the task list before the
re-read, as an ISO-8601 duration. Cancellation is cooperative, so a task
keeps draining until it reaches its next checkpoint.
- id: dry_run
type: BOOL
defaults: true
description: When true the flow only reports, it never cancels a task.
- id: auto_remediate
type: BOOL
defaults: false
description: When true and dry_run is false the flow cancels the flagged tasks
after approval.
triggers:
- id: hourly
type: io.kestra.plugin.core.trigger.Schedule
description: Scan for long running by-query tasks every hour in safe report-only mode.
cron: "0 * * * *"
disabled: true
inputs:
dry_run: true
auto_remediate: false
tasks:
- id: detect_cluster
type: io.kestra.plugin.elasticsearch.Request
description: Read cluster health for the context a long running task shows up
in, the pending task count and the queue wait.
connection:
hosts:
- "{{ inputs.elasticsearch_url }}"
basicAuth:
username: "{{ inputs.elasticsearch_username }}"
password: "{{ inputs.elasticsearch_password }}"
method: GET
endpoint: "_cluster/health"
- id: detect_tasks
type: io.kestra.plugin.elasticsearch.Request
description: >-
List the running tasks whose action is in the allowlist. group_by=none
returns one flat array so each entry carries its own node id, running time
and cancellable flag.
connection:
hosts:
- "{{ inputs.elasticsearch_url }}"
basicAuth:
username: "{{ inputs.elasticsearch_username }}"
password: "{{ inputs.elasticsearch_password }}"
method: GET
endpoint: "_tasks"
parameters:
actions: "{{ inputs.task_actions }}"
detailed: "true"
group_by: "none"
filter_path: "tasks.node,tasks.id,tasks.action,tasks.description,tasks.running_\
time_in_nanos,tasks.start_time_in_millis,tasks.cancellable,tasks.cancel\
led,tasks.parent_task_id,tasks.status.total,tasks.status.batches,tasks.\
status.requests_per_second"
- id: check_reads
type: io.kestra.plugin.core.flow.If
description: >-
The Request task reports task success on any HTTP status, so every read is
gated on its status here. A 404 on a mistyped endpoint would otherwise
read as an idle cluster.
condition: "{{ outputs.detect_cluster.status != 200 or
outputs.detect_tasks.status != 200 }}"
then:
- id: fail_reads
type: io.kestra.plugin.core.execution.Fail
description: Fail when the cluster could not be read so nothing downstream acts
on a partial picture.
errorMessage: >-
Could not read the cluster. health {{ outputs.detect_cluster.status
}}, tasks {{ outputs.detect_tasks.status }}.
- id: plan
type: io.kestra.plugin.core.output.OutputValues
description: >-
Keep the cancellable top-level tasks whose action is in the allowlist and
whose running time is past the threshold. A child task of a sliced reindex
is dropped because cancelling the parent takes its slices with it.
values:
cluster_status: "{{ outputs.detect_cluster.response | jq('.status') | first }}"
pending_tasks: "{{ outputs.detect_cluster.response |
jq('.number_of_pending_tasks') | first }}"
queue_wait_ms: "{{ outputs.detect_cluster.response |
jq('.task_max_waiting_in_queue_millis') | first }}"
active_shards: "{{ outputs.detect_cluster.response | jq('.active_shards') | first }}"
inspected: "{{ outputs.detect_tasks.response | jq('((.tasks) // []) | length') |
first }}"
targets_json: >-
{{ {'r': outputs.detect_tasks.response, 'a': inputs.task_actions, 'max':
inputs.max_running_seconds} | toJson | jq('. as $x | ($x.a | split(",")
| map(ltrimstr(" ") | rtrimstr(" "))) as $allow | (($x.max | tonumber) *
1000000000) as $limit | [(($x.r.tasks) // [])[] | select(.cancellable ==
true) | select(has("parent_task_id") | not) | select(.action as $act |
$allow | index($act)) | select(.running_time_in_nanos > $limit) |
{task_id: (.node + ":" + (.id | tostring)), action: .action,
running_seconds: ((.running_time_in_nanos / 1000000000) | tostring |
split(".") | .[0]), cancelled: (.cancelled // false), batches:
((.status.batches) // 0), total: ((.status.total) // 0), rps:
((.status.requests_per_second) // 0), note: ((.description) // "")}]') |
first | toJson }}
target_names: >-
{{ {'r': outputs.detect_tasks.response, 'a': inputs.task_actions, 'max':
inputs.max_running_seconds} | toJson | jq('. as $x | ($x.a | split(",")
| map(ltrimstr(" ") | rtrimstr(" "))) as $allow | (($x.max | tonumber) *
1000000000) as $limit | [(($x.r.tasks) // [])[] | select(.cancellable ==
true) | select(has("parent_task_id") | not) | select(.action as $act |
$allow | index($act)) | select(.running_time_in_nanos > $limit) | (.node
+ ":" + (.id | tostring)) + " (" + .action + ", running " +
((.running_time_in_nanos / 1000000000) | tostring | split(".") | .[0]) +
"s, " + (((.status.batches) // 0) | tostring) + " batches of " +
(((.status.total) // 0) | tostring) + " docs, rps " +
(((.status.requests_per_second) // 0) | tostring) + (if (.cancelled //
false) then ", cancel already requested" else "" end) + ")"] | join(" |
")') | first }}
- id: report
type: io.kestra.plugin.core.log.Log
description: Record the cluster context, the flagged tasks and the mode.
message: |
Runaway task scan on {{ inputs.elasticsearch_url }}.
Allowlisted actions: {{ inputs.task_actions }}. Threshold: {{ inputs.max_running_seconds }}s of running time.
Cluster status {{ outputs.plan.values.cluster_status }} with {{ outputs.plan.values.active_shards }} active shards, {{ outputs.plan.values.pending_tasks }} pending cluster tasks, queue wait {{ outputs.plan.values.queue_wait_ms }}ms.
Tasks in the allowlisted actions right now: {{ outputs.plan.values.inspected }}.
Flagged: {{ outputs.plan.values.targets_json | jq('length') | first }}. {{ outputs.plan.values.target_names }}
Mode dry_run={{ inputs.dry_run }} auto_remediate={{ inputs.auto_remediate }} max_cancel_per_run={{ inputs.max_cancel_per_run }} settle_timeout={{ inputs.settle_timeout }}.
- id: alert_detected_gate
type: io.kestra.plugin.core.flow.If
description: Alert Slack the moment a runaway task is found, before and apart
from any cancellation decision.
condition: "{{ (outputs.plan.values.targets_json | jq('length') | first) > 0 }}"
then:
- id: alert_detected
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: >-
Post the flagged tasks and whether a cancel will run on this pass. The
payload is built as a map then serialised, so a task description
carrying quotes or a newline cannot break the JSON.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: >-
{{ {'text': 'Elasticsearch runaway task guard: ' ~
(outputs.plan.values.targets_json | jq('length') | first) ~ ' task(s)
on ' ~ inputs.elasticsearch_url ~ ' have been running longer than ' ~
inputs.max_running_seconds ~ 's with nobody waiting on them. ' ~
outputs.plan.values.target_names ~ '. Cluster ' ~
outputs.plan.values.cluster_status ~ ' with ' ~
outputs.plan.values.pending_tasks ~ ' pending cluster tasks and ' ~
outputs.plan.values.queue_wait_ms ~ 'ms queue wait. dry_run=' ~
inputs.dry_run ~ ' auto_remediate=' ~ inputs.auto_remediate ~ '.'} |
toJson }}
- id: route
type: io.kestra.plugin.core.flow.If
description: Act only when a runaway task is present and the operator asked for
a real change.
condition: "{{ (outputs.plan.values.targets_json | jq('length') | first) > 0 and
inputs.auto_remediate and not inputs.dry_run }}"
then:
- id: cap_gate
type: io.kestra.plugin.core.flow.If
description: >-
Cancel only while the flagged list is small. A long list means the
threshold is below the normal runtime of this cluster's jobs, so
cancelling everything would take out work somebody is waiting for.
condition: "{{ (outputs.plan.values.targets_json | jq('length') | first) <=
inputs.max_cancel_per_run }}"
then:
- id: approval_gate
type: io.kestra.plugin.core.flow.Pause
description: >-
Hold the run until a human approves the cancel. A cancelled update
or delete by query keeps the documents it already changed, so this
is never an automatic decision.
pauseDuration: PT2H
behavior: FAIL
onResume:
- id: approved
type: BOOL
defaults: false
description: >-
Set true to cancel the listed tasks. Anything else leaves them
running.
- id: apply
type: io.kestra.plugin.core.flow.If
description: Run the cancel only on an explicit approval. An unanswered gate
never reads as approval.
condition: "{{ outputs.approval_gate.onResume.approved ?? false }}"
then:
- id: remediate_cancel_tasks
type: io.kestra.plugin.core.flow.Loop
description: >-
Remediation loop. One POST _tasks/{task_id}/_cancel per
flagged task, concurrencyLimit 1 so the cancels go in one at a
time. allowFailure so a task that finished on its own between
the scan and the cancel does not stop the pass, the re-read
stays the real gate.
concurrencyLimit: 1
values: "{{ outputs.plan.values.targets_json }}"
tasks:
- id: cancel_task
type: io.kestra.plugin.elasticsearch.Request
description: Ask Elasticsearch to cancel one task. The call returns as soon as
the flag is set, not when the task stops.
allowFailure: true
connection:
hosts:
- "{{ inputs.elasticsearch_url }}"
basicAuth:
username: "{{ inputs.elasticsearch_username }}"
password: "{{ inputs.elasticsearch_password }}"
method: POST
endpoint: "_tasks/{{ fromJson(item.value).task_id }}/_cancel"
- id: cancel_log
type: io.kestra.plugin.core.log.Log
message: >-
POST _tasks/{{ fromJson(item.value).task_id }}/_cancel on
{{ fromJson(item.value).action }} after {{
fromJson(item.value).running_seconds }}s returned HTTP {{
outputs.cancel_task.status }}.
- id: settle
type: io.kestra.plugin.core.flow.LoopUntil
description: >-
Bounded settle wait. Cancellation is cooperative, so a task
stays in the task list until it reaches its next checkpoint
and stops itself. This polls until every targeted task is gone
or settle_timeout runs out, then hands the verdict to the
re-read.
condition: "{{ ({'t': outputs.plan.values.targets_json, 'r':
outputs.settle_read.response} | toJson | jq('. as $x | ($x.t |
fromjson) as $t | [(($x.r.tasks) // [])[] | (.node + \":\" +
(.id | tostring)) as $id | select([$t[].task_id] |
index($id))] | length') | first) == 0 }}"
checkFrequency:
interval: PT5S
maxDuration: "{{ inputs.settle_timeout }}"
failOnMaxReached: false
tasks:
- id: settle_read
type: io.kestra.plugin.elasticsearch.Request
description: Read the task list again while waiting for the cancelled tasks to
drain.
connection:
hosts:
- "{{ inputs.elasticsearch_url }}"
basicAuth:
username: "{{ inputs.elasticsearch_username }}"
password: "{{ inputs.elasticsearch_password }}"
method: GET
endpoint: "_tasks"
parameters:
actions: "{{ inputs.task_actions }}"
detailed: "true"
group_by: "none"
filter_path: "tasks.node,tasks.id,tasks.action,tasks.running_time_in_nanos,task\
s.cancellable,tasks.cancelled,tasks.parent_task_id,task\
s.status.total,tasks.status.batches"
- id: reverify_tasks
type: io.kestra.plugin.elasticsearch.Request
description: Re-read the task list so the verdict comes from Elasticsearch
itself, not from the cancel call returning 200.
connection:
hosts:
- "{{ inputs.elasticsearch_url }}"
basicAuth:
username: "{{ inputs.elasticsearch_username }}"
password: "{{ inputs.elasticsearch_password }}"
method: GET
endpoint: "_tasks"
parameters:
actions: "{{ inputs.task_actions }}"
detailed: "true"
group_by: "none"
filter_path: "tasks.node,tasks.id,tasks.action,tasks.running_time_in_nanos,task\
s.cancellable,tasks.cancelled,tasks.parent_task_id,tasks.st\
atus.total,tasks.status.batches"
- id: verify_plan
type: io.kestra.plugin.core.output.OutputValues
description: >-
Keep the targeted task ids that are still in the task list. A
task still present with its cancel flag set is draining rather
than ignoring the request, so the message says which of the
two happened.
values:
still_json: >-
{{ {'t': outputs.plan.values.targets_json, 'r':
outputs.reverify_tasks.response} | toJson | jq('. as $x |
($x.t | fromjson) as $t | [(($x.r.tasks) // [])[] | (.node +
":" + (.id | tostring)) as $id | select([$t[].task_id] |
index($id)) | {task_id: $id, action: .action,
running_seconds: ((.running_time_in_nanos / 1000000000) |
tostring | split(".") | .[0]), cancelled: (.cancelled //
false)}]') | first | toJson }}
still_names: >-
{{ {'t': outputs.plan.values.targets_json, 'r':
outputs.reverify_tasks.response} | toJson | jq('. as $x |
($x.t | fromjson) as $t | [(($x.r.tasks) // [])[] | (.node +
":" + (.id | tostring)) as $id | select([$t[].task_id] |
index($id)) | $id + " (" + .action + ", still running " +
((.running_time_in_nanos / 1000000000) | tostring |
split(".") | .[0]) + "s, cancel flag " + ((.cancelled //
false) | tostring) + ")"] | join(" | ")') | first }}
gone_names: >-
{{ {'t': outputs.plan.values.targets_json, 'r':
outputs.reverify_tasks.response} | toJson | jq('. as $x |
($x.t | fromjson) as $t | [(($x.r.tasks) // [])[] | (.node +
":" + (.id | tostring))] as $live | [$t[] | .task_id as $tid
| select(($live | index($tid)) == null) | $tid] | join(" |
")') | first }}
- id: verify_report
type: io.kestra.plugin.core.log.Log
description: Record the state after the pass so the audit trail carries the
result.
message: |
After the pass on {{ inputs.elasticsearch_url }}.
Targeted {{ outputs.plan.values.targets_json | jq('length') | first }} task(s). Gone: {{ outputs.verify_plan.values.gone_names }}
Still present: {{ outputs.verify_plan.values.still_json | jq('length') | first }}. {{ outputs.verify_plan.values.still_names }}
Tasks in the allowlisted actions now: {{ outputs.reverify_tasks.response | jq('((.tasks) // []) | length') | first }}. Re-read HTTP {{ outputs.reverify_tasks.status }}.
A cancelled by-query task keeps every document it already changed and a cancelled reindex keeps every document already copied. Nothing was rolled back here.
- id: verify_result
type: io.kestra.plugin.core.flow.If
description: Pass only when every targeted task has left the task list,
otherwise fail loudly.
condition: "{{ outputs.reverify_tasks.status != 200 or
(outputs.verify_plan.values.still_json | jq('length') | first)
> 0 }}"
then:
- id: remediation_failed
type: io.kestra.plugin.core.execution.Fail
description: Fail when a targeted task outlived the settle window so a cancel
that did not take is never reported as done.
errorMessage: >-
{{ outputs.verify_plan.values.still_json | jq('length') |
first }} of {{ outputs.plan.values.targets_json |
jq('length') | first }} targeted task(s) are still in the
task list after settle_timeout {{ inputs.settle_timeout
}}: {{ outputs.verify_plan.values.still_names }}. Re-read
HTTP {{ outputs.reverify_tasks.status }}. A cancel flag of
true means the task is still draining to its next
checkpoint, so raise settle_timeout and run again. A
cancel flag of false means Elasticsearch never accepted
the cancel, so check the cancel call status in the loop
logs.
else:
- id: remediated
type: io.kestra.plugin.core.log.Log
description: Record that every targeted task stopped.
message: >-
Cancelled and verified. {{
outputs.plan.values.targets_json | jq('length') | first }}
task(s) were past {{ inputs.max_running_seconds }}s of
running time, 0 of them are in the task list now. Gone: {{
outputs.verify_plan.values.gone_names }}. Cluster {{
outputs.plan.values.cluster_status }}. Documents already
written by a cancelled by-query task are still there.
else:
- id: not_approved
type: io.kestra.plugin.core.log.Log
description: Record that approval was withheld so nothing was cancelled.
message: >-
Approval not granted, so no task was cancelled. {{
outputs.plan.values.targets_json | jq('length') | first }}
task(s) are still running: {{ outputs.plan.values.target_names
}}.
else:
- id: refuse_report
type: io.kestra.plugin.core.log.Log
description: Record that the flow refused to cancel because too many tasks were
flagged.
message: >-
Refusing to cancel. {{ outputs.plan.values.targets_json |
jq('length') | first }} task(s) are past {{
inputs.max_running_seconds }}s against max_cancel_per_run {{
inputs.max_cancel_per_run }}. Raise max_running_seconds to match
how long this cluster's jobs normally take. Raise
max_cancel_per_run if the list is genuinely all runaway work.
Flagged: {{ outputs.plan.values.target_names }}.
- id: fail_cap
type: io.kestra.plugin.core.execution.Fail
description: Fail so the alert fires, cancelling a list this long would take out
work somebody is waiting for.
errorMessage: >-
{{ outputs.plan.values.targets_json | jq('length') | first }}
flagged task(s) exceed max_cancel_per_run {{
inputs.max_cancel_per_run }}, so nothing was cancelled. Review the
threshold before re-running.
else:
- id: report_only
type: io.kestra.plugin.core.log.Log
description: Record a no-change outcome, either nothing is flagged or the run is
in report mode.
message: >-
No task was cancelled on {{ inputs.elasticsearch_url }}. Flagged {{
outputs.plan.values.targets_json | jq('length') | first }}, dry_run {{
inputs.dry_run }}, auto_remediate {{ inputs.auto_remediate }}.
errors:
- id: alert
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alert Slack on any failure so a refused pass or a cancel that did
not take is never silent.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: >-
{{ {'text': 'elasticsearch-runaway-task-cancel-guard failed in flow ' ~
flow.id ~ ' (execution ' ~ execution.id ~ ') on ' ~
inputs.elasticsearch_url ~ '. One of these happened: the cluster could not
be read, more tasks were flagged than max_cancel_per_run allows, a
targeted task was still in the task list after the settle window. Check
the execution logs.'} | toJson }}