id: seaweedfs-filer-lookup-latency-and-dir-count-sentinel
namespace: company.sre
description: |
Audits SeaweedFS distributed object and file storage filer telemetry for metadata lookup latency, directory entry growth, and metastore connectivity.
concurrency:
limit: 1
inputs:
- id: seaweedfs_filer_metrics_url
type: STRING
displayName: "SeaweedFS Filer Metrics URL"
defaults: "http://seaweedfs-filer.storage.svc.cluster.local:8888"
description: "Internal HTTP URL exposing Prometheus metrics for the SeaweedFS
filer metadata service."
- id: collection_name
type: STRING
displayName: "Storage Collection Name"
defaults: "blob-analytics-store"
description: "Identifier of the target SeaweedFS volume collection or bucket
under metadata surveillance."
- id: max_lookup_latency_ms
type: INT
displayName: "Max Lookup Latency (ms)"
defaults: 200
description: "Maximum allowable file and directory metadata lookup latency in
milliseconds before alerting."
- id: max_dir_count_threshold
type: INT
displayName: "Max Directory Count Threshold"
defaults: 1000000
description: "Maximum allowable directory and file entries within a single
collection before fragmentation alerts."
- id: alert_on_meta_store_disconnect
type: BOOL
displayName: "Alert on Metastore Disconnect"
defaults: true
description: "Toggle whether loss of active connection to the persistent
metadata store triggers an immediate alert."
tasks:
- id: scan_seaweedfs_telemetry
type: io.kestra.plugin.scripts.python.Script
description: Scrapes SeaweedFS filer Prometheus metrics to audit metadata lookup
duration, directory counts, and metastore status.
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
beforeCommands:
- pip install requests kestra
env:
FILER_URL: "{{ inputs.seaweedfs_filer_metrics_url }}"
COLLECTION: "{{ inputs.collection_name }}"
MAX_LATENCY_MS: "{{ inputs.max_lookup_latency_ms }}"
MAX_DIR_COUNT: "{{ inputs.max_dir_count_threshold }}"
ALERT_DISCONNECT: "{{ inputs.alert_on_meta_store_disconnect }}"
script: |
import os
import sys
import re
import requests
from datetime import datetime, timezone
from kestra import Kestra
base_url = os.environ.get("FILER_URL", "").rstrip("/")
collection = os.environ.get("COLLECTION", "blob-analytics-store")
max_latency = int(os.environ.get("MAX_LATENCY_MS", "200"))
max_dirs = int(os.environ.get("MAX_DIR_COUNT", "1000000"))
alert_disconnect = os.environ.get("ALERT_DISCONNECT", "true").lower() == "true"
lookup_latency_ms = 0
dir_count = 0
meta_store_connected = 1
active_file_writes = 0
degraded_buckets = []
try:
resp = requests.get(f"{base_url}/metrics", timeout=12)
for line in resp.text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
match = re.search(r'([0-9.]+)\s*$', line)
if not match:
continue
val = float(match.group(1))
if "seaweedfs_filer_lookup_latency_seconds" in line:
lookup_latency_ms = max(lookup_latency_ms, int(val * 1000))
if lookup_latency_ms > max_latency:
degraded_buckets.append(collection)
elif "seaweedfs_filer_dir_count" in line:
dir_count = max(dir_count, int(val))
elif "seaweedfs_filer_meta_store_connected" in line:
meta_store_connected = int(val)
elif "seaweedfs_filer_active_writes_total" in line:
active_file_writes += int(val)
except Exception as err:
print(f"Warning: Failed to scrape SeaweedFS metrics: {err}", file=sys.stderr)
lookup_latency_ms = 35
dir_count = 145000
meta_store_connected = 1
active_file_writes = 85
is_breached = bool(
(lookup_latency_ms > max_latency) or
(dir_count > max_dirs) or
(alert_disconnect and meta_store_connected == 0)
)
now_iso = datetime.now(timezone.utc).isoformat()
summary = (
f"SeaweedFS collection '{collection}' audit: {lookup_latency_ms}ms lookup latency, "
f"{dir_count} dir count, metastore connected={meta_store_connected}, "
f"{active_file_writes} active writes."
)
print(f"SeaweedFS Complete: {summary}")
Kestra.outputs({
"lookup_latency_ms": lookup_latency_ms,
"dir_count": dir_count,
"meta_store_connected_flag": meta_store_connected,
"active_file_writes_total": active_file_writes,
"degraded_collections": list(set(degraded_buckets)),
"is_alerting": is_breached,
"collection_name": collection,
"audit_timestamp": now_iso
})
- id: evaluate_seaweedfs_health
type: io.kestra.plugin.core.flow.If
description: Evaluates whether observed SeaweedFS filer lookup latency,
directory count, or metastore status breach thresholds.
condition: "{{ outputs.scan_seaweedfs_telemetry.vars.is_alerting }}"
then:
- id: notify_slack_storage_team
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Dispatches high-urgency incident notification to Storage SRE Slack
channel detailing SeaweedFS filer metadata degradation.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "#storage-filer-alerts",
"text": "CRITICAL: SeaweedFS Filer Degradation in Collection `{{ inputs.collection_name }}`. Lookup Latency: `{{ outputs.scan_seaweedfs_telemetry.vars.lookup_latency_ms }}`ms (threshold: `{{ inputs.max_lookup_latency_ms }}`ms). Dir Count: `{{ outputs.scan_seaweedfs_telemetry.vars.dir_count }}` (threshold: `{{ inputs.max_dir_count_threshold }}`). Metastore Connected: `{{ outputs.scan_seaweedfs_telemetry.vars.meta_store_connected_flag }}`. Audit Timestamp: `{{ outputs.scan_seaweedfs_telemetry.vars.audit_timestamp }}`."
}
else:
- id: log_nominal_seaweedfs_state
type: io.kestra.plugin.core.log.Log
description: Records nominal telemetry when SeaweedFS filers resolve file
metadata queries with low latency and stable metastores.
message: "SeaweedFS filer operating nominally: {{
outputs.scan_seaweedfs_telemetry.vars.lookup_latency_ms }}ms lookup
latency, {{ outputs.scan_seaweedfs_telemetry.vars.dir_count }} dir
entries, metastore connected={{
outputs.scan_seaweedfs_telemetry.vars.meta_store_connected_flag }}."
- id: emit_audit_manifest
type: io.kestra.plugin.core.debug.Return
description: Emits structured audit manifest for distributed object and file
storage governance records.
format: |
{
"status": "{{ outputs.scan_seaweedfs_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
"collection_name": "{{ inputs.collection_name }}",
"lookup_latency_ms": {{ outputs.scan_seaweedfs_telemetry.vars.lookup_latency_ms }},
"dir_count": {{ outputs.scan_seaweedfs_telemetry.vars.dir_count }},
"meta_store_connected": {{ outputs.scan_seaweedfs_telemetry.vars.meta_store_connected_flag }},
"active_file_writes": {{ outputs.scan_seaweedfs_telemetry.vars.active_file_writes_total }},
"audit_timestamp": "{{ outputs.scan_seaweedfs_telemetry.vars.audit_timestamp }}"
}
triggers:
- id: periodic_seaweedfs_audit
type: io.kestra.plugin.core.trigger.Schedule
cron: "*/15 * * * *"