id: litestream-sqlite-replication-lag-and-staleness-sentinel
namespace: company.storage
description: |
Infrastructure sentinel that periodically audits Litestream SQLite continuous replication
daemon metrics, evaluates S3 replica lag and sync error counters, classifies replication
staleness through a Switch gate, dispatches actionable Slack alerts before data loss occurs,
and exports structured telemetry for disaster recovery compliance.
inputs:
- id: litestream_metrics_url
type: STRING
displayName: "Litestream Metrics URL"
defaults: "http://localhost:9090"
description: "Base URL for the Litestream Prometheus metrics endpoint (default
port 9090)."
- id: database_name
type: STRING
displayName: "Monitored Database Name"
defaults: "production.db"
description: "Database identifier or relative path configured in Litestream
replication."
- id: warning_lag_seconds
type: INT
displayName: "Warning Lag Threshold (Seconds)"
defaults: 60
description: "Maximum allowable replication lag in seconds before raising an
advisory warning."
- id: critical_lag_seconds
type: INT
displayName: "Critical Lag Threshold (Seconds)"
defaults: 300
description: "Maximum allowable replication lag in seconds before escalating to
a critical incident."
- id: max_snapshot_age_hours
type: INT
displayName: "Max Snapshot Age (Hours)"
defaults: 24
description: "Maximum allowable elapsed hours since the last full SQLite
database snapshot."
- id: slack_channel
type: STRING
displayName: "Slack Alert Channel"
defaults: "#db-ops"
description: "Target Slack channel for database replication and disaster
recovery alerts."
triggers:
- id: periodic_replication_check
type: io.kestra.plugin.core.trigger.Schedule
description: "Executes continuous replication lag audits every ten minutes.
Shipped disabled by default for initial connection verification."
cron: "*/10 * * * *"
disabled: true
tasks:
- id: probe_litestream_metrics
type: io.kestra.plugin.core.http.Request
description: "Scrapes Prometheus metrics from the Litestream replication daemon
endpoint."
uri: "{{ inputs.litestream_metrics_url }}/metrics"
method: GET
options:
allowFailed: false
- id: evaluate_replication_lag
type: io.kestra.plugin.scripts.python.Script
description: "Parses Litestream Prometheus exposition metrics for replica lag,
sync error counts, and snapshot timestamps."
containerImage: python:3.11-slim
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- pip install kestra
env:
METRICS_TEXT: "{{ outputs.probe_litestream_metrics.body }}"
WARNING_LAG_SEC: "{{ inputs.warning_lag_seconds }}"
CRITICAL_LAG_SEC: "{{ inputs.critical_lag_seconds }}"
MAX_SNAPSHOT_HOURS: "{{ inputs.max_snapshot_age_hours }}"
script: |
import os
import sys
import time
from kestra import Kestra
metrics_raw = os.environ.get("METRICS_TEXT", "")
warn_lag = float(os.environ.get("WARNING_LAG_SEC", 60))
crit_lag = float(os.environ.get("CRITICAL_LAG_SEC", 300))
max_snap_hrs = float(os.environ.get("MAX_SNAPSHOT_HOURS", 24))
replica_lag_ms = 0.0
error_count = 0.0
last_snapshot_ts = 0.0
sync_count = 0.0
for line in metrics_raw.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split()
if len(parts) < 2:
continue
metric_header = parts[0]
try:
val = float(parts[1])
except ValueError:
continue
if "litestream_replica_lag_ms" in metric_header:
replica_lag_ms = max(replica_lag_ms, val)
elif "litestream_replica_error_count" in metric_header:
error_count += val
elif "litestream_snapshot_last_timestamp" in metric_header:
last_snapshot_ts = max(last_snapshot_ts, val)
elif "litestream_sync_count" in metric_header:
sync_count += val
lag_seconds = round(replica_lag_ms / 1000.0, 2)
now_epoch = time.time()
snapshot_age_hours = 0.0
if last_snapshot_ts > 0:
snapshot_age_hours = round((now_epoch - last_snapshot_ts) / 3600.0, 2)
status = "HEALTHY"
if lag_seconds >= crit_lag or error_count > 0:
status = "CRITICAL_REPLICATION_STALL"
elif lag_seconds >= warn_lag or (snapshot_age_hours > max_snap_hrs and last_snapshot_ts > 0):
status = "REPLICATION_LAG_WARNING"
Kestra.outputs({
"lag_seconds": lag_seconds,
"lag_ms": int(replica_lag_ms),
"error_count": int(error_count),
"sync_count": int(sync_count),
"snapshot_age_hours": snapshot_age_hours,
"status": status
})
- id: classify_replication_health
type: io.kestra.plugin.core.flow.Switch
description: "Evaluates calculated replica lag and error counters to route into
critical stall, warning, or healthy state."
value: "{{ outputs.evaluate_replication_lag.vars.status }}"
cases:
CRITICAL_REPLICATION_STALL:
- id: notify_critical_replication_stall
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Dispatches high-priority Slack incident briefing card when
replication lag stalls or sync errors occur."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": "*[CRITICAL ALERT] Litestream SQLite Replication Stall*",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "[CRITICAL ALERT] Litestream Replication Stall",
"emoji": false
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Database:*\n`{{ inputs.database_name }}`"
},
{
"type": "mrkdwn",
"text": "*Status:*\n`CRITICAL REPLICATION STALL`"
},
{
"type": "mrkdwn",
"text": "*Replica Lag:*\n`{{ outputs.evaluate_replication_lag.vars.lag_seconds }} seconds` (Limit: `{{ inputs.critical_lag_seconds }}s`)"
},
{
"type": "mrkdwn",
"text": "*Sync Errors:*\n`{{ outputs.evaluate_replication_lag.vars.error_count }}`"
},
{
"type": "mrkdwn",
"text": "*Snapshot Age:*\n`{{ outputs.evaluate_replication_lag.vars.snapshot_age_hours }} hours`"
},
{
"type": "mrkdwn",
"text": "*Execution ID:*\n`{{ execution.id }}`"
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Recommended Remediation:*\n1. Check network connectivity between host and S3/MinIO bucket endpoint.\n2. Verify cloud storage IAM access keys and write permissions.\n3. Inspect Litestream daemon logs on target server for WAL lock contention."
}
}
]
}
- id: log_critical_stall
type: io.kestra.plugin.core.log.Log
description: "Logs critical replication stall in Kestra execution history."
message: "[CRITICAL] Litestream replication for {{ inputs.database_name }} is
stalled. Lag: {{ outputs.evaluate_replication_lag.vars.lag_seconds
}}s, Sync Errors: {{
outputs.evaluate_replication_lag.vars.error_count }}."
REPLICATION_LAG_WARNING:
- id: notify_replication_warning
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Dispatches advisory warning to Slack when replication lag exceeds
nominal thresholds."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": "*[WARNING] Litestream Replication Lag Approaching Threshold*",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "[WARNING] Litestream Replication Lag Warning",
"emoji": false
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Database:*\n`{{ inputs.database_name }}`"
},
{
"type": "mrkdwn",
"text": "*Replica Lag:*\n`{{ outputs.evaluate_replication_lag.vars.lag_seconds }} seconds` (Threshold: `{{ inputs.warning_lag_seconds }}s`)"
},
{
"type": "mrkdwn",
"text": "*Snapshot Age:*\n`{{ outputs.evaluate_replication_lag.vars.snapshot_age_hours }} hours`"
}
]
}
]
}
- id: log_warning_lag
type: io.kestra.plugin.core.log.Log
description: "Logs advisory replication lag warning in execution history."
message: "[WARNING] Litestream replication for {{ inputs.database_name }} lag
elevated at {{ outputs.evaluate_replication_lag.vars.lag_seconds
}}s. Monitoring for resolution."
HEALTHY:
- id: log_healthy_replication
type: io.kestra.plugin.core.log.Log
description: "Records healthy status when SQLite replication lag and snapshot
age are within normal bounds."
message: "[HEALTHY] Litestream replication for {{ inputs.database_name }} is
nominal. Lag: {{ outputs.evaluate_replication_lag.vars.lag_seconds
}}s, Errors: 0, Snapshot age: {{
outputs.evaluate_replication_lag.vars.snapshot_age_hours }}h."
- id: finalize_replication_audit
type: io.kestra.plugin.core.output.OutputValues
description: "Publishes structured replication telemetry for disaster recovery
dashboards and compliance audits."
values:
execution_id: "{{ execution.id }}"
database_name: "{{ inputs.database_name }}"
lag_seconds: "{{ outputs.evaluate_replication_lag.vars.lag_seconds }}"
error_count: "{{ outputs.evaluate_replication_lag.vars.error_count }}"
sync_count: "{{ outputs.evaluate_replication_lag.vars.sync_count }}"
snapshot_age_hours: "{{ outputs.evaluate_replication_lag.vars.snapshot_age_hours }}"
classification: "{{ outputs.classify_replication_health.value }}"
errors:
- id: notify_probe_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Alerts SRE team immediately if the Litestream metrics endpoint is
unreachable."
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": "*[FAILED] Litestream Replication Metrics Probe Outage*\nTarget: `{{ inputs.litestream_metrics_url }}`\nDatabase: `{{ inputs.database_name }}`\nFailure: Sentinel was unable to collect Prometheus metrics from the Litestream daemon.\nExecution ID: `{{ execution.id }}`"
}