id: kubernetes-prolonged-termination-watch
namespace: company.team
description: |
Read-only monitoring of prolonged Pod deletion, with Slack alerts and clearing
messages. Duration starts when this watcher first observes deletion in progress.
It does not require a finalizer, sleep hook or particular application.
The schedule is disabled until the first manual execution has been verified.
Use a unique cluster_id per cluster and keep it stable between executions.
Keep one copy of this flow responsible for each saved state key.
Alerts report observed delay, not a proven root cause or failed application.
A Pod removed between scans cannot be detected. Resetting KV state resets timers.
State is saved after Slack succeeds. A crash between sending and saving can
produce a repeated message; normal successful scans suppress repeats.
inputs:
- id: cluster_id
type: STRING
defaults: docker-desktop
description: Stable identifier for the Kubernetes cluster being monitored.
- id: scan_namespace
type: STRING
defaults: kestra-terminating-demo
description: One Kubernetes namespace that the ServiceAccount may read.
- id: threshold_seconds
type: INT
defaults: 60
description: Seconds since first observation before alerting. Use a suitable
production threshold.
concurrency:
limit: 1
behavior: QUEUE
variables:
state_key: "{{ flow.id }}-{{ inputs.cluster_id }}-{{ inputs.scan_namespace }}"
tasks:
- id: list_pods
type: io.kestra.plugin.kubernetes.kubectl.Get
connection:
masterUrl: "{{ secret('POD_WATCH_K8S_URL') }}"
oauthToken: "{{ secret('POD_WATCH_K8S_TOKEN') }}"
caCertData: "{{ secret('POD_WATCH_K8S_CA_CERT_DATA') }}"
trustCerts: false
disableHostnameVerification: false
namespace: "{{ inputs.scan_namespace }}"
resourceType: pods
fetchType: FETCH
- id: load_state
type: io.kestra.plugin.core.kv.Get
key: "{{ render(vars.state_key) }}"
errorOnMissing: false
- id: evaluate
type: io.kestra.plugin.scripts.python.Script
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: python:3.12-slim
dependencies:
- kestra
inputFiles:
pods.json: "{{ outputs.list_pods.metadataItems | toJson }}"
status.json: "{{ outputs.list_pods.statusItems | toJson }}"
previous.json: "{{ (outputs.load_state.value ?? {}) | toJson }}"
env:
CLUSTER_ID: "{{ inputs.cluster_id }}"
SCAN_NAMESPACE: "{{ inputs.scan_namespace }}"
THRESHOLD_SECONDS: "{{ inputs.threshold_seconds }}"
outputFiles:
- report.json
script: |
import json
import os
from datetime import datetime, timezone
from pathlib import Path
from kestra import Kestra
def timestamp(value):
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
if parsed.tzinfo is None:
raise ValueError("A timestamp has no timezone")
return parsed
def evaluate(pods, statuses, previous, cluster, namespace, threshold, now):
if not cluster.strip() or not namespace.strip() or threshold < 1:
raise ValueError("Provide a cluster ID, one namespace and a positive threshold")
if not isinstance(pods, list) or not isinstance(statuses, list):
raise ValueError("Expected Pod metadata and status lists from Kubernetes")
if len(pods) != len(statuses):
raise ValueError("Pod metadata and status counts do not match")
scope = {"cluster": cluster, "namespace": namespace}
if not isinstance(previous, dict):
raise ValueError("Saved state must be a JSON object")
if previous and (previous.get("version") != 1 or previous.get("scope") != scope):
raise ValueError("Saved state belongs to a different scope or version")
old = previous.get("pods", {})
if not isinstance(old, dict):
raise ValueError("Invalid saved Pod state")
current, present = {}, set()
alerts, cleared = [], []
waiting = suppressed = 0
for pod, wrapped_status in zip(pods, statuses):
uid = pod.get("uid")
name = pod.get("name")
if not uid or not name or pod.get("namespace") != namespace or uid in present:
raise ValueError("Missing, duplicate or out-of-scope Pod identity")
present.add(uid)
deletion = pod.get("deletionTimestamp")
if not deletion:
if uid in old:
raise ValueError("A previously deleting Pod lost its deletion timestamp")
continue
deadline = timestamp(deletion)
prior = old.get(uid, {})
first_seen = prior.get("first_seen", now.isoformat())
age = max(0, int((now - timestamp(first_seen)).total_seconds()))
alerted = prior.get("alerted", False)
if not isinstance(alerted, bool):
raise ValueError("Invalid saved alert flag")
status = (wrapped_status or {}).get("status") or {}
entry = {
"uid": uid, "name": name, "namespace": namespace,
"first_seen": first_seen, "observed_seconds": age,
"deletion_timestamp": deletion,
"deletion_grace_seconds": pod.get("deletionGracePeriodSeconds"),
"recorded_deadline_passed": now >= deadline,
"phase": status.get("phase", "Unknown"),
"finalizers": pod.get("finalizers") or [],
"alerted": alerted,
}
if age >= threshold and not alerted:
alerts.append(entry.copy())
entry["alerted"] = True
elif alerted:
suppressed += 1
else:
waiting += 1
current[uid] = entry
for uid, entry in old.items():
if uid not in present and entry.get("alerted"):
cleared.append(entry)
lines = []
if alerts or cleared:
lines.append(f"Pod termination watch | cluster {cluster} | namespace {namespace}")
for entry in alerts:
deadline_note = ("recorded deletion deadline has passed" if
entry["recorded_deadline_passed"] else
"still before recorded deletion deadline")
finalizers = ", ".join(entry["finalizers"]) or "none"
lines.append(
f"ALERT: {namespace}/{entry['name']} (UID {entry['uid']})\n"
f"Deletion observed for {entry['observed_seconds']}s; threshold {threshold}s.\n"
f"Phase: {entry['phase']}; finalizers: {finalizers}; {deadline_note}.\n"
f"Inspect with: kubectl describe pod {entry['name']} -n {namespace}"
)
for entry in cleared:
lines.append(
f"CLEARED: {entry['namespace']}/{entry['name']} (UID {entry['uid']}) "
"is no longer present in a successful Kubernetes read."
)
message = "\n\n".join(lines)
if len(message) > 35000:
raise ValueError("Report exceeds one Slack message; state has not been committed")
summary = (
f"Checked {len(pods)} Pod(s); terminating={len(current)}; "
f"waiting={waiting}; new_alerts={len(alerts)}; "
f"already_alerted={suppressed}; cleared={len(cleared)}."
)
return {
"summary": summary,
"send_notification": bool(alerts or cleared),
"message": message,
"new_alerts": alerts,
"cleared": cleared,
"next_state": {"version": 1, "scope": scope, "pods": current},
}
result = evaluate(
json.loads(Path("pods.json").read_text()),
json.loads(Path("status.json").read_text()),
json.loads(Path("previous.json").read_text()),
os.environ["CLUSTER_ID"], os.environ["SCAN_NAMESPACE"],
int(os.environ["THRESHOLD_SECONDS"]), datetime.now(timezone.utc),
)
Path("report.json").write_text(json.dumps(result, indent=2))
Kestra.outputs(result)
- id: log_summary
type: io.kestra.plugin.core.log.Log
message: "{{ outputs.evaluate.vars.summary }}"
- id: notify_changes
type: io.kestra.plugin.core.flow.If
condition: "{{ outputs.evaluate.vars.send_notification }}"
then:
- id: send_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('POD_WATCH_SLACK_WEBHOOK_URL') }}"
messageText: "{{ outputs.evaluate.vars.message }}"
- id: save_state
type: io.kestra.plugin.core.kv.Set
key: "{{ render(vars.state_key) }}"
kvType: JSON
value: "{{ outputs.evaluate.vars.next_state | toJson }}"
triggers:
- id: every_minute
type: io.kestra.plugin.core.trigger.Schedule
cron: "* * * * *"
disabled: true
errors:
- id: alert_watch_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('POD_WATCH_SLACK_WEBHOOK_URL') }}"
messageText: >-
Pod termination watcher failed for {{ inputs.cluster_id }}/{{
inputs.scan_namespace }}. Check execution {{ execution.id }} in Kestra.
This is a monitoring failure, not confirmation that the Pods are healthy
or that deletion completed.