id: signoz-opentelemetry-collector-and-ingestion-sentinel
namespace: company.observability
description: |
Infrastructure sentinel that continuously audits SigNoz OpenTelemetry collector health,
evaluates exporter buffer queue saturation and dropped span counts, classifies ingestion
backpressure through a Switch gate, dispatches actionable Slack alerts without telemetry
loss, and exports structured observability audit records.
inputs:
- id: signoz_collector_url
type: STRING
displayName: "SigNoz Collector Metrics URL"
defaults: "http://localhost:8888"
description: "Prometheus telemetry endpoint for the SigNoz OpenTelemetry
Collector daemon."
- id: signoz_query_service_url
type: STRING
displayName: "SigNoz Query Service URL"
defaults: "http://localhost:8080"
description: "Base URL for the SigNoz Query Service API."
- id: queue_threshold_pct
type: INT
displayName: "Buffer Queue Warning Threshold %"
defaults: 75
description: "Collector exporter buffer queue saturation percentage above which
alerts are raised."
- id: alert_on_refused_spans
type: BOOL
displayName: "Alert on Refused Spans"
defaults: true
description: "When true, escalates immediately if the collector reports refused
or dropped telemetry spans."
- id: slack_channel
type: STRING
displayName: "Slack Alert Channel"
defaults: "#observability-alerts"
description: "Target Slack channel for SigNoz ingestion and collector buffer alerts."
triggers:
- id: periodic_collector_audit
type: io.kestra.plugin.core.trigger.Schedule
description: "Executes continuous OpenTelemetry ingestion audits every ten
minutes. Shipped disabled by default for initial connection verification."
cron: "*/10 * * * *"
disabled: true
tasks:
- id: probe_query_service_health
type: io.kestra.plugin.core.http.Request
description: "Queries SigNoz query-service API health endpoint to confirm core
platform availability."
uri: "{{ inputs.signoz_query_service_url }}/api/v1/health"
method: GET
options:
allowFailed: true
- id: probe_collector_metrics
type: io.kestra.plugin.core.http.Request
description: "Fetches internal Prometheus telemetry from the SigNoz
OpenTelemetry collector."
uri: "{{ inputs.signoz_collector_url }}/metrics"
method: GET
options:
allowFailed: false
- id: evaluate_ingestion_buffer
type: io.kestra.plugin.scripts.python.Script
description: "Parses Prometheus text format to extract queue capacity, current
buffer depth, and dropped span counters."
containerImage: python:3.11-slim
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- pip install kestra
env:
METRICS_TEXT: "{{ outputs.probe_collector_metrics.body }}"
THRESHOLD_PCT: "{{ inputs.queue_threshold_pct }}"
ALERT_ON_DROPS: "{{ inputs.alert_on_refused_spans }}"
script: |
import os
import sys
from kestra import Kestra
metrics_raw = os.environ.get("METRICS_TEXT", "")
threshold = float(os.environ.get("THRESHOLD_PCT", 75))
alert_drops = os.environ.get("ALERT_ON_DROPS", "true").lower() == "true"
queue_size = 0
queue_capacity = 0
refused_spans = 0
enqueue_failed = 0
for line in metrics_raw.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split()
if len(parts) < 2:
continue
metric_header = parts[0]
try:
val = float(parts[1])
except ValueError:
continue
if metric_header.startswith("otelcol_exporter_queue_size"):
queue_size += val
elif metric_header.startswith("otelcol_exporter_queue_capacity"):
queue_capacity += val
elif "otelcol_receiver_refused_spans" in metric_header:
refused_spans += val
elif "otelcol_exporter_enqueue_failed_spans" in metric_header:
enqueue_failed += val
total_dropped_or_refused = refused_spans + enqueue_failed
queue_pct = 0.0
if queue_capacity > 0:
queue_pct = round((queue_size / queue_capacity) * 100, 2)
status = "HEALTHY"
if (alert_drops and total_dropped_or_refused > 0) or queue_pct >= 90:
status = "CRITICAL_INGESTION_SATURATION"
elif queue_pct >= threshold:
status = "BUFFER_QUEUE_WARNING"
Kestra.outputs({
"queue_size": int(queue_size),
"queue_capacity": int(queue_capacity),
"queue_saturation_pct": queue_pct,
"refused_spans": int(refused_spans),
"enqueue_failed_spans": int(enqueue_failed),
"total_dropped_or_refused": int(total_dropped_or_refused),
"status": status
})
- id: classify_ingestion_state
type: io.kestra.plugin.core.flow.Switch
description: "Evaluates buffer saturation and dropped span figures to route into
critical breach, warning advisory, or healthy state."
value: "{{ outputs.evaluate_ingestion_buffer.vars.status }}"
cases:
CRITICAL_INGESTION_SATURATION:
- id: notify_critical_saturation
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Dispatches high-priority Slack incident briefing card when
collector buffers saturate or spans are dropped."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": "*[CRITICAL ALERT] SigNoz OpenTelemetry Ingestion Buffer Saturation*",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "[CRITICAL ALERT] SigNoz Ingestion Saturation",
"emoji": false
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Collector Target:*\n`{{ inputs.signoz_collector_url }}`"
},
{
"type": "mrkdwn",
"text": "*Status:*\n`CRITICAL BUFFER SATURATION`"
},
{
"type": "mrkdwn",
"text": "*Buffer Queue Utilization:*\n`{{ outputs.evaluate_ingestion_buffer.vars.queue_saturation_pct }}%` (Threshold: `{{ inputs.queue_threshold_pct }}%`)"
},
{
"type": "mrkdwn",
"text": "*Queue Depth / Capacity:*\n`{{ outputs.evaluate_ingestion_buffer.vars.queue_size }} / {{ outputs.evaluate_ingestion_buffer.vars.queue_capacity }}`"
},
{
"type": "mrkdwn",
"text": "*Dropped / Refused Spans:*\n`{{ outputs.evaluate_ingestion_buffer.vars.total_dropped_or_refused }}`"
},
{
"type": "mrkdwn",
"text": "*Execution ID:*\n`{{ execution.id }}`"
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Remediation Guidance:*\n1. Check ClickHouse storage disk I/O and table lock contention.\n2. Scale OpenTelemetry collector replicas or increase exporter sending queue batch sizes.\n3. Verify network throughput between collectors and ClickHouse database nodes."
}
}
]
}
- id: log_critical_saturation
type: io.kestra.plugin.core.log.Log
description: "Logs critical ingestion buffer breach in Kestra execution history."
message: "[CRITICAL] SigNoz collector at {{ inputs.signoz_collector_url }} queue
is at {{ outputs.evaluate_ingestion_buffer.vars.queue_saturation_pct
}}% capacity. Total dropped or refused spans: {{
outputs.evaluate_ingestion_buffer.vars.total_dropped_or_refused }}."
BUFFER_QUEUE_WARNING:
- id: notify_buffer_warning
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Dispatches advisory warning to Slack when collector queue depth
approaches capacity boundaries."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": "*[WARNING] SigNoz Collector Buffer Queue Approaching Limit*",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "[WARNING] SigNoz Buffer Queue Approaching Limit",
"emoji": false
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Collector Target:*\n`{{ inputs.signoz_collector_url }}`"
},
{
"type": "mrkdwn",
"text": "*Queue Utilization:*\n`{{ outputs.evaluate_ingestion_buffer.vars.queue_saturation_pct }}%`"
},
{
"type": "mrkdwn",
"text": "*Current Queue Depth:*\n`{{ outputs.evaluate_ingestion_buffer.vars.queue_size }} / {{ outputs.evaluate_ingestion_buffer.vars.queue_capacity }}`"
}
]
}
]
}
- id: log_buffer_warning
type: io.kestra.plugin.core.log.Log
description: "Logs advisory queue saturation warning in execution history."
message: "[WARNING] SigNoz collector queue at {{
outputs.evaluate_ingestion_buffer.vars.queue_saturation_pct }}%
utilization. Monitoring for continued backpressure."
HEALTHY:
- id: log_healthy_ingestion
type: io.kestra.plugin.core.log.Log
description: "Records healthy status when collector queues are operating within
normal tolerances."
message: "[HEALTHY] SigNoz OpenTelemetry ingestion at {{
inputs.signoz_collector_url }} is nominal. Queue utilization: {{
outputs.evaluate_ingestion_buffer.vars.queue_saturation_pct }}%,
dropped spans: 0."
- id: finalize_ingestion_telemetry
type: io.kestra.plugin.core.output.OutputValues
description: "Publishes structured ingestion metrics for capacity planning, SLA
verification, and observability auditing."
values:
execution_id: "{{ execution.id }}"
collector_url: "{{ inputs.signoz_collector_url }}"
queue_saturation_pct: "{{ outputs.evaluate_ingestion_buffer.vars.queue_saturation_pct }}"
queue_size: "{{ outputs.evaluate_ingestion_buffer.vars.queue_size }}"
queue_capacity: "{{ outputs.evaluate_ingestion_buffer.vars.queue_capacity }}"
total_dropped_or_refused: "{{ outputs.evaluate_ingestion_buffer.vars.total_dropped_or_refused }}"
classification: "{{ outputs.classify_ingestion_state.value }}"
errors:
- id: notify_collector_unreachable
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Alerts SRE team immediately if the SigNoz collector metrics
endpoint is unreachable."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": "*[FAILED] SigNoz Collector Telemetry Probe Outage*\nTarget: `{{ inputs.signoz_collector_url }}`\nFailure: Sentinel was unable to collect Prometheus metrics from the OpenTelemetry collector.\nExecution ID: `{{ execution.id }}`"
}