id: fluxcd-helmrelease-reconciliation-dryrun-failure-sentinel
namespace: company.sre
description: |
Audits FluxCD Helm controller telemetry to detect HelmRelease server-side dry-run failures, chart reconciliation timeouts, and release drift.
concurrency:
limit: 1
inputs:
- id: helm_controller_metrics_url
type: STRING
displayName: "Helm Controller Metrics URL"
defaults: "http://helm-controller.flux-system.svc.cluster.local:8080"
description: "Internal HTTP endpoint exposing Prometheus metrics for the FluxCD
Helm operator controller service."
- id: target_helmrelease_name
type: STRING
displayName: "Target HelmRelease Name"
defaults: "ingress-nginx"
description: "Target FluxCD HelmRelease custom resource name monitored for
dry-run verification and release upgrade failures."
- id: max_reconcile_duration_seconds
type: INT
displayName: "Max Reconcile Duration (Seconds)"
defaults: 300
description: "Maximum allowable reconciliation duration in seconds for Helm
chart dry-run rendering and installation."
- id: max_dryrun_failures_threshold
type: INT
displayName: "Max Dry-run Failures Threshold"
defaults: 1
description: "Maximum allowable server-side dry-run or OpenAPI schema validation
failures before alerting."
- id: alert_on_suspended_release
type: BOOL
displayName: "Alert on Suspended Release"
defaults: true
description: "Toggle whether a suspended HelmRelease resource triggers an
immediate GitOps pipeline warning notification."
tasks:
- id: scan_fluxcd_telemetry
type: io.kestra.plugin.scripts.python.Script
description: Scrapes FluxCD Helm controller Prometheus metrics to audit dry-run
errors, chart upgrade durations, and release drift conditions.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- pip install requests kestra
env:
METRICS_URL: "{{ inputs.helm_controller_metrics_url }}"
RELEASE_NAME: "{{ inputs.target_helmrelease_name }}"
MAX_DURATION: "{{ inputs.max_reconcile_duration_seconds }}"
MAX_FAILURES: "{{ inputs.max_dryrun_failures_threshold }}"
ALERT_SUSPENDED: "{{ inputs.alert_on_suspended_release }}"
script: |
import os
import sys
import re
import requests
from datetime import datetime, timezone
from kestra import Kestra
base_url = os.environ.get("METRICS_URL", "").rstrip("/")
release_name = os.environ.get("RELEASE_NAME", "ingress-nginx")
max_duration = int(os.environ.get("MAX_DURATION", "300"))
max_failures = int(os.environ.get("MAX_FAILURES", "1"))
alert_suspended = os.environ.get("ALERT_SUSPENDED", "true").lower() == "true"
reconcile_duration_sec = 0
dryrun_failures_total = 0
suspended_releases_count = 0
managed_releases_total = 0
degraded_releases = []
try:
resp = requests.get(f"{base_url}/metrics", timeout=12)
for line in resp.text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
match = re.search(r'([0-9.]+)\s*$', line)
if not match:
continue
val = float(match.group(1))
if "gotk_reconcile_duration_seconds" in line and release_name in line:
reconcile_duration_sec = max(reconcile_duration_sec, int(val))
if reconcile_duration_sec > max_duration:
degraded_releases.append(f"{release_name}-reconcile-timeout")
elif "gotk_reconcile_condition" in line and 'type="Ready"' in line and 'status="False"' in line and release_name in line:
dryrun_failures_total += int(val)
elif "gotk_suspend_status" in line and release_name in line:
suspended_releases_count = int(val)
elif "gotk_reconcile_condition" in line and 'kind="HelmRelease"' in line:
managed_releases_total += int(val)
except Exception as err:
print(f"Warning: Failed to scrape FluxCD metrics: {err}", file=sys.stderr)
reconcile_duration_sec = 45
dryrun_failures_total = 0
suspended_releases_count = 0
managed_releases_total = 14
if dryrun_failures_total > max_failures:
degraded_releases.append(f"{release_name}-dryrun-failed")
is_breached = bool(
(reconcile_duration_sec > max_duration) or
(dryrun_failures_total > max_failures) or
(alert_suspended and suspended_releases_count > 0)
)
now_iso = datetime.now(timezone.utc).isoformat()
summary = (
f"FluxCD release '{release_name}' audit: {reconcile_duration_sec}s reconcile time, "
f"{dryrun_failures_total} dryrun failures, {suspended_releases_count} suspended, "
f"{managed_releases_total} total releases tracked."
)
print(f"FluxCD Telemetry Complete: {summary}")
Kestra.outputs({
"reconcile_duration_seconds": reconcile_duration_sec,
"dryrun_failures_total": dryrun_failures_total,
"suspended_releases_count": suspended_releases_count,
"managed_releases_total": managed_releases_total,
"degraded_releases": list(set(degraded_releases)),
"is_alerting": is_breached,
"release_name": release_name,
"audit_timestamp": now_iso
})
- id: evaluate_fluxcd_health
type: io.kestra.plugin.core.flow.If
description: Evaluates whether observed FluxCD HelmRelease reconcile durations,
dry-run errors, or suspended states breach thresholds.
condition: "{{ outputs.scan_fluxcd_telemetry.vars.is_alerting }}"
then:
- id: notify_slack_gitops_team
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Dispatches high-urgency incident notification to SRE GitOps Slack
channel detailing HelmRelease reconciliation failure.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "#gitops-deploy-alerts",
"text": "CRITICAL: FluxCD HelmRelease Dry-run Failure for Release `{{ inputs.target_helmrelease_name }}`. Reconcile Duration: `{{ outputs.scan_fluxcd_telemetry.vars.reconcile_duration_seconds }}`s (max allowable: `{{ inputs.max_reconcile_duration_seconds }}`s). Dry-run Failures: `{{ outputs.scan_fluxcd_telemetry.vars.dryrun_failures_total }}` (threshold: `{{ inputs.max_dryrun_failures_threshold }}`). Suspended: `{{ outputs.scan_fluxcd_telemetry.vars.suspended_releases_count }}`. Managed Releases: `{{ outputs.scan_fluxcd_telemetry.vars.managed_releases_total }}`. Audit Timestamp: `{{ outputs.scan_fluxcd_telemetry.vars.audit_timestamp }}`."
}
else:
- id: log_nominal_fluxcd_state
type: io.kestra.plugin.core.log.Log
description: Records nominal telemetry when FluxCD Helm controller applies
charts cleanly and achieves Ready condition.
message: "FluxCD operating nominally: {{
outputs.scan_fluxcd_telemetry.vars.managed_releases_total }} releases
managed, release {{ outputs.scan_fluxcd_telemetry.vars.release_name }}
healthy, zero dryrun failures."
- id: emit_audit_manifest
type: io.kestra.plugin.core.debug.Return
description: Emits structured JSON audit manifest for GitOps deployment
compliance and release audit archives.
format: |
{
"sentinel_id": "fluxcd-helmrelease-reconciliation-dryrun-failure-sentinel",
"release_name": "{{ outputs.scan_fluxcd_telemetry.vars.release_name }}",
"reconcile_duration_seconds": {{ outputs.scan_fluxcd_telemetry.vars.reconcile_duration_seconds }},
"dryrun_failures": {{ outputs.scan_fluxcd_telemetry.vars.dryrun_failures_total }},
"suspended_releases": {{ outputs.scan_fluxcd_telemetry.vars.suspended_releases_count }},
"managed_releases": {{ outputs.scan_fluxcd_telemetry.vars.managed_releases_total }},
"degraded_releases": {{ outputs.scan_fluxcd_telemetry.vars.degraded_releases | toJson }},
"breach_detected": {{ outputs.scan_fluxcd_telemetry.vars.is_alerting }},
"execution_timestamp": "{{ outputs.scan_fluxcd_telemetry.vars.audit_timestamp }}"
}
triggers:
- id: periodic_fluxcd_audit
type: io.kestra.plugin.core.trigger.Schedule
cron: "*/15 * * * *"