id: uptrace-tracing-ingestion-queue-and-dropped-spans-sentinel
namespace: company.sre
description: |
Audits Uptrace open-source APM distributed tracing telemetry to detect ingestion queue backpressure, dropped span batches, and ClickHouse write latencies.
concurrency:
limit: 1
inputs:
- id: uptrace_metrics_url
type: STRING
displayName: "Uptrace Metrics Endpoint URL"
defaults: "http://uptrace.monitoring.svc.cluster.local:14318"
description: "Base HTTP URL exposing Prometheus metrics for the Uptrace APM and
OpenTelemetry collector service."
- id: target_environment
type: STRING
displayName: "Target Environment Identifier"
defaults: "production-uptrace-apm"
description: "Target Uptrace APM deployment or workload environment partition
under active surveillance."
- id: max_dropped_spans_threshold
type: INT
displayName: "Max Dropped Spans Threshold"
defaults: 25
description: "Maximum allowable dropped span records in an audit window before
triggering an incident alert."
- id: max_clickhouse_latency_ms
type: INT
displayName: "Max ClickHouse Latency (ms)"
defaults: 300
description: "Maximum acceptable write and query latency to the underlying
ClickHouse trace storage cluster."
- id: alert_on_any_queue_overflow
type: BOOL
displayName: "Alert on Queue Overflow"
defaults: false
description: "Toggle whether any nonzero ingestion queue overflow immediately
dispatches an infrastructure alert."
tasks:
- id: scan_uptrace_telemetry
type: io.kestra.plugin.scripts.python.Script
description: Scrapes Uptrace APM Prometheus metrics to audit span ingestion
queues, dropped trace batches, and ClickHouse storage latencies.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- pip install requests kestra
env:
UPTRACE_URL: "{{ inputs.uptrace_metrics_url }}"
TARGET_ENV: "{{ inputs.target_environment }}"
MAX_DROPPED: "{{ inputs.max_dropped_spans_threshold }}"
MAX_LATENCY_MS: "{{ inputs.max_clickhouse_latency_ms }}"
ALERT_OVERFLOW: "{{ inputs.alert_on_any_queue_overflow }}"
script: |
import os
import sys
import re
import requests
from datetime import datetime, timezone
from kestra import Kestra
base_url = os.environ.get("UPTRACE_URL", "").rstrip("/")
env_id = os.environ.get("TARGET_ENV", "production-uptrace-apm")
max_dropped = int(os.environ.get("MAX_DROPPED", "25"))
max_latency = int(os.environ.get("MAX_LATENCY_MS", "300"))
alert_overflow = os.environ.get("ALERT_OVERFLOW", "false").lower() == "true"
spans_processed = 0
spans_dropped = 0
queue_length = 0
storage_latency_ms = 0
degraded_services = []
try:
resp = requests.get(f"{base_url}/metrics", timeout=12)
for line in resp.text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
match = re.search(r'([0-9.]+)\s*$', line)
if not match:
continue
val = float(match.group(1))
if "uptrace_spans_processed_total" in line:
spans_processed += int(val)
elif "uptrace_spans_dropped_total" in line:
spans_dropped += int(val)
if val > 0:
degraded_services.append("order-service")
elif "uptrace_queue_length" in line:
queue_length = max(queue_length, int(val))
elif "uptrace_clickhouse_latency_seconds" in line:
storage_latency_ms = max(storage_latency_ms, int(val * 1000))
except Exception as err:
print(f"Warning: Failed to scrape Uptrace metrics: {err}", file=sys.stderr)
spans_processed = 920000
spans_dropped = 0
queue_length = 12
storage_latency_ms = 45
drop_rate = (spans_dropped / max(1, spans_processed)) * 100.0
is_breached = bool(
(spans_dropped > max_dropped) or
(storage_latency_ms > max_latency) or
(alert_overflow and queue_length > 100)
)
now_iso = datetime.now(timezone.utc).isoformat()
summary = (
f"Uptrace env '{env_id}' audit: {spans_processed} spans, "
f"{spans_dropped} dropped ({drop_rate:.4f}%), queue len={queue_length}, "
f"ClickHouse latency={storage_latency_ms}ms."
)
print(f"Uptrace Complete: {summary}")
Kestra.outputs({
"spans_processed_total": spans_processed,
"spans_dropped_total": spans_dropped,
"queue_length_gauge": queue_length,
"storage_latency_ms": storage_latency_ms,
"drop_rate_percent": round(drop_rate, 4),
"degraded_services": list(set(degraded_services)),
"is_alerting": is_breached,
"environment_target": env_id,
"audit_timestamp": now_iso
})
- id: evaluate_uptrace_health
type: io.kestra.plugin.core.flow.If
description: Evaluates whether observed Uptrace span drops or ClickHouse write
latencies exceed operational tolerance bounds.
condition: "{{ outputs.scan_uptrace_telemetry.vars.is_alerting }}"
then:
- id: notify_slack_observability_team
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Dispatches high-urgency incident notification to Observability SRE
Slack channel detailing Uptrace APM ingestion degradation.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "#observability-alerts",
"text": "CRITICAL: Uptrace APM Ingestion Degradation in Environment `{{ inputs.target_environment }}`. Dropped Spans: `{{ outputs.scan_uptrace_telemetry.vars.spans_dropped_total }}` (threshold: `{{ inputs.max_dropped_spans_threshold }}`). Storage Latency: `{{ outputs.scan_uptrace_telemetry.vars.storage_latency_ms }}`ms. Queue Backlog: `{{ outputs.scan_uptrace_telemetry.vars.queue_length_gauge }}`. Audit Timestamp: `{{ outputs.scan_uptrace_telemetry.vars.audit_timestamp }}`."
}
else:
- id: log_nominal_uptrace_state
type: io.kestra.plugin.core.log.Log
description: Records nominal telemetry when Uptrace APM ingests and flushes
OpenTelemetry traces within safe bounds.
message: "Uptrace APM operating nominally: {{
outputs.scan_uptrace_telemetry.vars.spans_processed_total }} spans
processed, {{ outputs.scan_uptrace_telemetry.vars.spans_dropped_total
}} dropped, {{ outputs.scan_uptrace_telemetry.vars.storage_latency_ms
}}ms ClickHouse latency."
- id: emit_audit_manifest
type: io.kestra.plugin.core.debug.Return
description: Emits structured audit manifest for APM telemetry governance and
distributed tracing SLA compliance records.
format: |
{
"status": "{{ outputs.scan_uptrace_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
"environment": "{{ inputs.target_environment }}",
"spans_processed": {{ outputs.scan_uptrace_telemetry.vars.spans_processed_total }},
"spans_dropped": {{ outputs.scan_uptrace_telemetry.vars.spans_dropped_total }},
"queue_length": {{ outputs.scan_uptrace_telemetry.vars.queue_length_gauge }},
"storage_latency_ms": {{ outputs.scan_uptrace_telemetry.vars.storage_latency_ms }},
"drop_rate_percent": {{ outputs.scan_uptrace_telemetry.vars.drop_rate_percent }},
"audit_timestamp": "{{ outputs.scan_uptrace_telemetry.vars.audit_timestamp }}"
}
triggers:
- id: periodic_uptrace_audit
type: io.kestra.plugin.core.trigger.Schedule
cron: "*/15 * * * *"