id: kubernetes-csi-volume-attachment-recovery-guard
namespace: company.platform
description: |
Diagnose potentially stuck Kubernetes CSI volume attachments by correlating
VolumeAttachment, node, persistent volume, persistent volume claim, and pod
state. This workflow is read-only: it never patches or deletes Kubernetes
resources or force-detaches volumes.
inputs:
- id: workload_namespace
type: STRING
displayName: Workload namespace
description: Namespace to inspect for pods and persistent volume claims.
defaults: default
- id: stale_after_minutes
type: INT
displayName: Unattached age threshold
description: Flag unattached VolumeAttachments older than this threshold for review.
defaults: 15
triggers:
- id: hourly_inspection
type: io.kestra.plugin.core.trigger.Schedule
description: Optional hourly inspection. Enable after a successful manual test.
cron: "0 * * * *"
disabled: true
tasks:
- id: collect_volume_attachments
type: io.kestra.plugin.kubernetes.kubectl.Get
description: Collect full cluster-wide CSI VolumeAttachment objects.
connection:
masterUrl: "{{ secret('K8S_MASTER_URL') }}"
oauthToken: "{{ secret('K8S_TOKEN') }}"
namespace: ""
resourceType: volumeattachments
apiGroup: storage.k8s.io
apiVersion: v1
fetchType: STORE
- id: collect_nodes
type: io.kestra.plugin.kubernetes.kubectl.Get
description: Collect node conditions to correlate attachment failures with node health.
connection:
masterUrl: "{{ secret('K8S_MASTER_URL') }}"
oauthToken: "{{ secret('K8S_TOKEN') }}"
namespace: ""
resourceType: nodes
fetchType: STORE
- id: collect_persistent_volumes
type: io.kestra.plugin.kubernetes.kubectl.Get
description: Collect PV claim references to map attachments to claims.
connection:
masterUrl: "{{ secret('K8S_MASTER_URL') }}"
oauthToken: "{{ secret('K8S_TOKEN') }}"
namespace: ""
resourceType: persistentvolumes
fetchType: STORE
- id: collect_persistent_volume_claims
type: io.kestra.plugin.kubernetes.kubectl.Get
description: Collect claims from the selected workload namespace.
connection:
masterUrl: "{{ secret('K8S_MASTER_URL') }}"
oauthToken: "{{ secret('K8S_TOKEN') }}"
namespace: "{{ inputs.workload_namespace }}"
resourceType: persistentvolumeclaims
fetchType: STORE
- id: collect_pods
type: io.kestra.plugin.kubernetes.kubectl.Get
description: Collect pod volume references and scheduling state.
connection:
masterUrl: "{{ secret('K8S_MASTER_URL') }}"
oauthToken: "{{ secret('K8S_TOKEN') }}"
namespace: "{{ inputs.workload_namespace }}"
resourceType: pods
fetchType: STORE
- id: evaluate_attachment_evidence
type: io.kestra.plugin.scripts.python.Script
description: Correlate stored Kubernetes objects and identify conditions needing
operator review.
dependencies:
- kestra
inputFiles:
volumeattachments.json: "{{ outputs.collect_volume_attachments.uri }}"
nodes.json: "{{ outputs.collect_nodes.uri }}"
persistentvolumes.json: "{{ outputs.collect_persistent_volumes.uri }}"
claims.json: "{{ outputs.collect_persistent_volume_claims.uri }}"
pods.json: "{{ outputs.collect_pods.uri }}"
script: |
import json
from datetime import datetime, timezone
from kestra import Kestra
THRESHOLD_MINUTES = int("{{ inputs.stale_after_minutes }}")
def read_items(path):
with open(path, "r", encoding="utf-8") as file:
data = json.load(file)
if isinstance(data, list):
return data
if isinstance(data, dict) and isinstance(data.get("items"), list):
return data["items"]
if isinstance(data, dict) and data.get("metadata") is not None:
return [data]
return []
def age_minutes(timestamp):
if not timestamp:
return None
try:
value = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
return max(0, int((datetime.now(timezone.utc) - value).total_seconds() / 60))
except (ValueError, TypeError):
return None
attachments = read_items("volumeattachments.json")
nodes = read_items("nodes.json")
pvs = read_items("persistentvolumes.json")
claims = read_items("claims.json")
pods = read_items("pods.json")
node_ready = {}
for node in nodes:
metadata = node.get("metadata") or {}
status = node.get("status") or {}
conditions = status.get("conditions") or []
ready = next(
(c.get("status") == "True"
for c in conditions if c.get("type") == "Ready"),
False
)
node_ready[metadata.get("name", "")] = ready
claim_to_pv = {}
for claim in claims:
metadata = claim.get("metadata") or {}
spec = claim.get("spec") or {}
volume_name = spec.get("volumeName")
if volume_name:
claim_to_pv[(metadata.get("namespace", ""), metadata.get("name", ""))] = volume_name
pv_to_claim = {}
for pv in pvs:
metadata = pv.get("metadata") or {}
claim_ref = (pv.get("spec") or {}).get("claimRef") or {}
if metadata.get("name") and claim_ref.get("name"):
pv_to_claim[metadata["name"]] = (
claim_ref.get("namespace", ""),
claim_ref.get("name", "")
)
pods_by_pv = {}
for pod in pods:
metadata = pod.get("metadata") or {}
spec = pod.get("spec") or {}
pod_namespace = metadata.get("namespace", "")
pod_name = metadata.get("name", "")
for volume in spec.get("volumes") or []:
pvc = (volume.get("persistentVolumeClaim") or {}).get("claimName")
if not pvc:
continue
pv_name = claim_to_pv.get((pod_namespace, pvc))
if pv_name:
pods_by_pv.setdefault(pv_name, []).append(
f"{pod_namespace}/{pod_name}"
)
findings = []
for attachment in attachments:
metadata = attachment.get("metadata") or {}
spec = attachment.get("spec") or {}
status = attachment.get("status") or {}
name = metadata.get("name", "unknown")
node_name = spec.get("nodeName", "")
pv_name = (spec.get("source") or {}).get("persistentVolumeName", "")
attached = status.get("attached")
age = age_minutes(metadata.get("creationTimestamp"))
reasons = []
attach_error = status.get("attachError") or {}
detach_error = status.get("detachError") or {}
if attach_error:
reasons.append("attach_error_reported")
if detach_error:
reasons.append("detach_error_reported")
if metadata.get("deletionTimestamp"):
reasons.append("deletion_in_progress")
if node_name and node_name not in node_ready:
reasons.append("referenced_node_not_found")
elif node_name and node_ready.get(node_name) is False:
reasons.append("referenced_node_not_ready")
if attached is None:
reasons.append("attachment_status_unknown")
elif attached is False and age is not None and age >= THRESHOLD_MINUTES:
reasons.append("unattached_beyond_threshold")
if not reasons:
continue
claim = pv_to_claim.get(pv_name, ("", ""))
claim_label = f"{claim[0]}/{claim[1]}" if claim[0] and claim[1] else "unresolved"
related_pods = pods_by_pv.get(pv_name, [])
findings.append({
"attachment": name,
"persistent_volume": pv_name or "unknown",
"node": node_name or "unknown",
"attached": attached,
"age_minutes": age,
"claim": claim_label,
"related_pods": related_pods,
"reasons": reasons
})
summary_parts = []
for finding in findings[:8]:
summary_parts.append(
f"{finding['attachment']} [{','.join(finding['reasons'])}]"
)
summary = "; ".join(summary_parts) if summary_parts else "No review findings."
if len(findings) > 8:
summary += f"; plus {len(findings) - 8} additional findings."
report = {
"attachment_count": len(attachments),
"review_count": len(findings),
"threshold_minutes": THRESHOLD_MINUTES,
"findings": findings
}
with open("attachment-review-report.json", "w", encoding="utf-8") as file:
json.dump(report, file, indent=2)
Kestra.outputs({
"attachment_count": len(attachments),
"review_count": len(findings),
"summary": summary
})
print(json.dumps(report, indent=2))
outputFiles:
- attachment-review-report.json
- id: route_findings
type: io.kestra.plugin.core.flow.If
description: Send an alert only when attachment evidence needs operator review.
condition: "{{ outputs.evaluate_attachment_evidence.vars.review_count > 0 }}"
then:
- id: notify_operator
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Send a concise report of attachments that need investigation.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "Kubernetes CSI Volume Attachment Recovery Guard: {{ outputs.evaluate_attachment_evidence.vars.review_count }} attachment(s) need review. {{ outputs.evaluate_attachment_evidence.vars.summary }}"
}
else:
- id: log_no_findings
type: io.kestra.plugin.core.log.Log
message: "No attachment conditions requiring review were found. Attachments
inspected: {{
outputs.evaluate_attachment_evidence.vars.attachment_count }}."