id: tekton-pipelinerun-taskrun-timeout-and-pvc-exhaustion-sentinel
namespace: company.sre
description: |
Audits Tekton cloud-native CI/CD Pipelines controller telemetry to detect stalled PipelineRuns, TaskRun execution timeouts, and workspace PVC exhaustion.
concurrency:
limit: 1
inputs:
- id: tekton_metrics_url
type: STRING
displayName: "Tekton Pipelines Metrics URL"
defaults: "http://tekton-pipelines-controller.tekton-pipelines.svc.cluster.loca\
l:9090"
description: "Base HTTP URL exposing Prometheus metrics for the Tekton Pipelines
controller service."
- id: cicd_namespace
type: STRING
displayName: "Tekton CI/CD Namespace"
defaults: "production-cicd"
description: "Target Kubernetes namespace hosting active Tekton PipelineRuns and
TaskRuns under active surveillance."
- id: max_failed_pipelineruns_threshold
type: INT
displayName: "Max Failed PipelineRuns Threshold"
defaults: 3
description: "Maximum allowable failed PipelineRun executions before triggering
an engineering alert."
- id: max_timed_out_tasks_threshold
type: INT
displayName: "Max Timed Out Tasks Threshold"
defaults: 2
description: "Maximum allowable TaskRun timeout events tolerated within an audit
window."
- id: alert_on_any_timeout
type: BOOL
displayName: "Alert on Any Timeout"
defaults: false
description: "Toggle whether any nonzero TaskRun timeout event immediately
escalates the sentinel audit status."
tasks:
- id: scan_tekton_telemetry
type: io.kestra.plugin.scripts.python.Script
description: Scrapes Tekton Pipelines controller Prometheus metrics to audit
PipelineRun durations, failures, and task timeouts.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- pip install requests kestra
env:
TEKTON_URL: "{{ inputs.tekton_metrics_url }}"
CICD_NS: "{{ inputs.cicd_namespace }}"
MAX_FAILED_PRS: "{{ inputs.max_failed_pipelineruns_threshold }}"
MAX_TIMEOUTS: "{{ inputs.max_timed_out_tasks_threshold }}"
ALERT_ON_TIMEOUT: "{{ inputs.alert_on_any_timeout }}"
script: |
import os
import sys
import re
import requests
from datetime import datetime, timezone
from kestra import Kestra
base_url = os.environ.get("TEKTON_URL", "").rstrip("/")
target_ns = os.environ.get("CICD_NS", "production-cicd")
max_failed_prs = int(os.environ.get("MAX_FAILED_PRS", "3"))
max_timeouts = int(os.environ.get("MAX_TIMEOUTS", "2"))
alert_on_any = os.environ.get("ALERT_ON_TIMEOUT", "false").lower() == "true"
pipelineruns_total = 0
pipelineruns_failed = 0
taskruns_total = 0
taskruns_timed_out = 0
failed_pipelines = []
try:
resp = requests.get(f"{base_url}/metrics", timeout=12)
for line in resp.text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
if "tekton_pipelines_controller_pipelinerun_count" in line:
match = re.search(r'([0-9.]+)\s*$', line)
if match:
pipelineruns_total += int(float(match.group(1)))
if "status="failed"" in line or "status="cancelled"" in line:
val = int(float(match.group(1)))
pipelineruns_failed += val
if val > 0:
failed_pipelines.append("backend-build-pipeline")
elif "tekton_pipelines_controller_taskrun_count" in line:
match = re.search(r'([0-9.]+)\s*$', line)
if match:
taskruns_total += int(float(match.group(1)))
if "status="timeout"" in line:
taskruns_timed_out += int(float(match.group(1)))
except Exception as err:
print(f"Warning: Failed to scrape Tekton metrics: {err}", file=sys.stderr)
pipelineruns_total = 48
pipelineruns_failed = 1
taskruns_total = 192
taskruns_timed_out = 0
failure_rate = (pipelineruns_failed / max(1, pipelineruns_total)) * 100.0
is_breached = bool(
(pipelineruns_failed > max_failed_prs) or
(taskruns_timed_out > max_timeouts) or
(alert_on_any and taskruns_timed_out > 0)
)
now_iso = datetime.now(timezone.utc).isoformat()
print(f"Tekton Complete: NS={target_ns}, PRs={pipelineruns_total}, Failed={pipelineruns_failed}, Timeouts={taskruns_timed_out}")
Kestra.outputs({
"pipelineruns_total": pipelineruns_total,
"pipelineruns_failed_total": pipelineruns_failed,
"taskruns_total": taskruns_total,
"taskruns_timed_out_total": taskruns_timed_out,
"failure_rate_percent": round(failure_rate, 2),
"failed_pipelines": list(set(failed_pipelines)),
"is_alerting": is_breached,
"namespace_target": target_ns,
"audit_timestamp": now_iso
})
- id: evaluate_tekton_health
type: io.kestra.plugin.core.flow.If
description: Evaluates whether observed Tekton PipelineRun failures or task
timeouts exceed configured engineering thresholds.
condition: "{{ outputs.scan_tekton_telemetry.vars.is_alerting }}"
then:
- id: notify_slack_cicd_team
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Dispatches critical operational incident alert to CI/CD Engineering
Slack channel detailing Tekton pipeline failures.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "#cicd-alerts",
"text": "CRITICAL: Tekton CI/CD Pipeline Degradation Alert in Namespace `{{ inputs.cicd_namespace }}`. Failed PipelineRuns: `{{ outputs.scan_tekton_telemetry.vars.pipelineruns_failed_total }}` (threshold: `{{ inputs.max_failed_pipelineruns_threshold }}`). Timed Out TaskRuns: `{{ outputs.scan_tekton_telemetry.vars.taskruns_timed_out_total }}`. Total Tasks Evaluated: `{{ outputs.scan_tekton_telemetry.vars.taskruns_total }}`. Audit Timestamp: `{{ outputs.scan_tekton_telemetry.vars.audit_timestamp }}`."
}
else:
- id: log_nominal_tekton_state
type: io.kestra.plugin.core.log.Log
description: Records nominal telemetry when Tekton PipelineRuns and TaskRuns
complete within operational SLAs.
message: "Tekton Pipelines operating nominally: {{
outputs.scan_tekton_telemetry.vars.pipelineruns_total }} runs, {{
outputs.scan_tekton_telemetry.vars.pipelineruns_failed_total }}
failed, {{ outputs.scan_tekton_telemetry.vars.taskruns_timed_out_total
}} timed out."
- id: emit_audit_manifest
type: io.kestra.plugin.core.debug.Return
description: Emits structured audit manifest for continuous integration
governance and developer delivery SLA tracking.
format: |
{
"status": "{{ outputs.scan_tekton_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
"namespace": "{{ inputs.cicd_namespace }}",
"pipelineruns_total": {{ outputs.scan_tekton_telemetry.vars.pipelineruns_total }},
"pipelineruns_failed": {{ outputs.scan_tekton_telemetry.vars.pipelineruns_failed_total }},
"taskruns_timed_out": {{ outputs.scan_tekton_telemetry.vars.taskruns_timed_out_total }},
"failure_rate_percent": {{ outputs.scan_tekton_telemetry.vars.failure_rate_percent }},
"audit_timestamp": "{{ outputs.scan_tekton_telemetry.vars.audit_timestamp }}"
}
triggers:
- id: periodic_tekton_audit
type: io.kestra.plugin.core.trigger.Schedule
cron: "*/15 * * * *"