id: redis-memory-fragmentation-and-eviction-circuit-breaker
namespace: company.cache
description: |
Monitor Redis memory fragmentation ratio and key eviction velocity, remediate allocator
bloat with MEMORY PURGE or active defragmentation, track audit records in Kestra KV,
and dispatch multi-channel alerts to Slack and PagerDuty.
concurrency:
limit: 1
behavior: QUEUE
inputs:
- id: redis_host
type: STRING
displayName: Redis Host
description: Hostname or IP address of the target Redis instance.
defaults: "localhost"
- id: redis_port
type: INT
displayName: Redis Port
description: TCP port of the target Redis instance.
defaults: 6379
min: 1
max: 65535
- id: redis_password
type: STRING
displayName: Redis Password
description: Optional Redis AUTH password; defaults to empty string for
unauthenticated instances or uses secret expressions.
defaults: ""
- id: fragmentation_warn_threshold
type: FLOAT
displayName: Fragmentation Warning Threshold
description: Ratio of used_memory_rss to used_memory (e.g. 1.5) that trips a
warning alert.
defaults: 1.5
- id: fragmentation_critical_threshold
type: FLOAT
displayName: Fragmentation Critical Threshold
description: Critical ratio of used_memory_rss to used_memory (e.g. 1.8) that
initiates automated defragmentation or memory purge.
defaults: 1.8
- id: min_fragmented_bytes_mb
type: INT
displayName: Minimum Fragmented Memory (MiB)
description: Minimum difference between RSS and used memory in MiB required
before triggering remediation, avoiding false alarms on idle or tiny
caches.
defaults: 64
- id: eviction_spike_threshold_rate
type: FLOAT
displayName: Eviction Spike Rate (keys/sec)
description: Key eviction velocity between consecutive audits signaling cache
thrashing or inadequate maxmemory sizing.
defaults: 50.0
- id: remediation_strategy
type: SELECT
displayName: Remediation Strategy
description: Action to take on critical fragmentation. MEMORY_PURGE flushes
jemalloc dirty pages; ACTIVE_DEFRAG enables background defragmentation;
AUTO attempts purge first and enables active defragmentation if ratio
remains above threshold.
values:
- AUTO
- MEMORY_PURGE
- ACTIVE_DEFRAG
- NONE
defaults: AUTO
- id: auto_remediate
type: BOOL
displayName: Auto Remediate
description: When true, executes the configured remediation commands directly on
the Redis instance.
defaults: true
- id: dry_run
type: BOOL
displayName: Dry Run
description: When true, audits metrics and emits notifications without executing
any mutation or purge commands against Redis.
defaults: false
- id: enable_slack
type: BOOL
displayName: Enable Slack Alerts
description: Whether to dispatch alerts to Slack when anomalies are detected.
defaults: true
- id: slack_channel
type: STRING
displayName: Slack Channel
description: Target Slack channel for alerts.
defaults: "#redis-alerts"
- id: enable_pagerduty
type: BOOL
displayName: Enable PagerDuty Alerts
description: Whether to escalate critical fragmentation and cache thrashing
incidents to PagerDuty.
defaults: false
- id: pagerduty_severity
type: SELECT
displayName: PagerDuty Incident Severity
description: Severity level passed to the PagerDuty Events API v2 on critical breaches.
values:
- critical
- error
- warning
- info
defaults: critical
variables:
kv_state_key: "redis_circuit_breaker_{{ inputs.redis_host }}_{{ inputs.redis_port }}"
tasks:
- id: read_previous_baseline
type: io.kestra.plugin.core.kv.Get
description: Fetch previous audit state from Kestra KV store to calculate
eviction rate deltas and track fragmentation history.
key: "{{ render(vars.kv_state_key) }}"
errorOnMissing: false
- id: inspect_and_remediate_redis
type: io.kestra.plugin.scripts.python.Script
description: Connect to Redis over RESP protocol, parse INFO memory and stats,
evaluate circuit breaker rules, and conditionally execute remediation
commands.
taskRunner:
type: io.kestra.plugin.core.runner.Process
env:
REDIS_HOST: "{{ inputs.redis_host }}"
REDIS_PORT: "{{ inputs.redis_port }}"
REDIS_PASSWORD: "{{ inputs.redis_password ?? '' }}"
WARN_RATIO: "{{ inputs.fragmentation_warn_threshold }}"
CRIT_RATIO: "{{ inputs.fragmentation_critical_threshold }}"
MIN_FRAGMENTED_MB: "{{ inputs.min_fragmented_bytes_mb }}"
EVICTION_SPIKE_RATE: "{{ inputs.eviction_spike_threshold_rate }}"
REMEDIATION_STRATEGY: "{{ inputs.remediation_strategy }}"
AUTO_REMEDIATE: "{{ inputs.auto_remediate }}"
DRY_RUN: "{{ inputs.dry_run }}"
PREV_BASELINE_JSON: "{{ outputs.read_previous_baseline.value ?? '{}' }}"
EXECUTION_ID: "{{ execution.id }}"
script: |
import socket
import json
import time
import os
import sys
host = os.environ.get("REDIS_HOST", "localhost")
port = int(os.environ.get("REDIS_PORT", 6379))
password = os.environ.get("REDIS_PASSWORD", "")
warn_ratio = float(os.environ.get("WARN_RATIO", 1.5))
crit_ratio = float(os.environ.get("CRIT_RATIO", 1.8))
min_fragmented_mb = float(os.environ.get("MIN_FRAGMENTED_MB", 64))
eviction_spike_rate = float(os.environ.get("EVICTION_SPIKE_RATE", 50.0))
remediation_strategy = os.environ.get("REMEDIATION_STRATEGY", "AUTO")
auto_remediate = os.environ.get("AUTO_REMEDIATE", "true").lower() == "true"
dry_run = os.environ.get("DRY_RUN", "false").lower() == "true"
prev_raw = os.environ.get("PREV_BASELINE_JSON", "{}")
execution_id = os.environ.get("EXECUTION_ID", "local")
try:
prev_baseline = json.loads(prev_raw) if prev_raw and prev_raw != "{}" else {}
except Exception:
prev_baseline = {}
def send_resp(sock, *args):
buf = [f"*{len(args)}\r\n".encode("utf-8")]
for a in args:
val = str(a).encode("utf-8")
buf.append(f"${len(val)}\r\n".encode("utf-8"))
buf.append(val)
buf.append(b"\r\n")
sock.sendall(b"".join(buf))
def read_line(sock):
buf = bytearray()
while True:
c = sock.recv(1)
if not c:
break
if c == b"\r":
sock.recv(1) # discard trailing \n
break
buf.extend(c)
return buf.decode("utf-8", errors="replace")
def read_resp(sock):
prefix = sock.recv(1).decode("latin-1")
if not prefix:
return None
if prefix in ("+", "-", ":"):
line = read_line(sock)
if prefix == "-":
raise RuntimeError(f"Redis error: {line}")
elif prefix == ":":
return int(line)
return line
elif prefix == "$":
length = int(read_line(sock))
if length == -1:
return None
data = bytearray()
while len(data) < length:
chunk = sock.recv(min(4096, length - len(data)))
if not chunk:
break
data.extend(chunk)
sock.recv(2) # discard \r\n
return data.decode("utf-8", errors="replace")
elif prefix == "*":
count = int(read_line(sock))
if count == -1:
return None
return [read_resp(sock) for _ in range(count)]
else:
raise ValueError(f"Unknown RESP prefix: {prefix}")
def parse_info_map(info_str):
result = {}
if not info_str:
return result
for line in info_str.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
if ":" in line:
k, v = line.split(":", 1)
result[k.strip()] = v.strip()
return result
now_ts = int(time.time())
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(12)
sock.connect((host, port))
if password:
send_resp(sock, "AUTH", password)
read_resp(sock)
send_resp(sock, "INFO", "memory")
mem_info = parse_info_map(read_resp(sock))
send_resp(sock, "INFO", "stats")
stats_info = parse_info_map(read_resp(sock))
try:
send_resp(sock, "CONFIG", "GET", "activedefrag")
defrag_config = read_resp(sock)
current_activedefrag = defrag_config[1] if isinstance(defrag_config, list) and len(defrag_config) > 1 else "unknown"
except Exception:
current_activedefrag = "unsupported"
used_memory = int(mem_info.get("used_memory", 0))
used_memory_rss = int(mem_info.get("used_memory_rss", 0))
mem_fragmentation_ratio = float(mem_info.get("mem_fragmentation_ratio", 1.0))
evicted_keys = int(stats_info.get("evicted_keys", 0))
fragmented_bytes = max(0, used_memory_rss - used_memory)
fragmented_mb = round(fragmented_bytes / (1024 * 1024), 2)
delta_evicted = 0
eviction_rate_per_sec = 0.0
if prev_baseline and isinstance(prev_baseline, dict):
prev_ts = prev_baseline.get("timestamp", now_ts)
prev_evicted = prev_baseline.get("evicted_keys", evicted_keys)
delta_time = max(1, now_ts - prev_ts)
delta_evicted = max(0, evicted_keys - prev_evicted)
eviction_rate_per_sec = round(delta_evicted / float(delta_time), 2)
warn_fragmentation = (mem_fragmentation_ratio >= warn_ratio) and (fragmented_mb >= min_fragmented_mb)
crit_fragmentation = (mem_fragmentation_ratio >= crit_ratio) and (fragmented_mb >= min_fragmented_mb)
cache_thrashing = (eviction_rate_per_sec >= eviction_spike_rate)
if crit_fragmentation and cache_thrashing:
status = "CRITICAL_SATURATION"
elif crit_fragmentation:
status = "FRAGMENTATION_CRITICAL"
elif cache_thrashing:
status = "CACHE_THRASHING"
elif warn_fragmentation:
status = "FRAGMENTATION_WARNING"
else:
status = "HEALTHY"
action_taken = "NONE"
post_ratio = None
post_rss = None
if crit_fragmentation:
if dry_run:
action_taken = "DRY_RUN_REMEDIATION_SKIPPED"
elif auto_remediate:
actions = []
if remediation_strategy in ("AUTO", "MEMORY_PURGE"):
try:
send_resp(sock, "MEMORY", "PURGE")
read_resp(sock)
actions.append("MEMORY_PURGE")
except Exception as ex:
actions.append(f"PURGE_FAILED({ex})")
# Read metrics after purge
try:
send_resp(sock, "INFO", "memory")
post_mem = parse_info_map(read_resp(sock))
post_ratio = float(post_mem.get("mem_fragmentation_ratio", mem_fragmentation_ratio))
post_rss = int(post_mem.get("used_memory_rss", used_memory_rss))
except Exception:
post_ratio = mem_fragmentation_ratio
post_rss = used_memory_rss
if remediation_strategy in ("AUTO", "ACTIVE_DEFRAG"):
if remediation_strategy == "ACTIVE_DEFRAG" or (post_ratio >= crit_ratio):
try:
send_resp(sock, "CONFIG", "SET", "activedefrag", "yes")
read_resp(sock)
actions.append("ACTIVE_DEFRAG_ENABLED")
except Exception as ex:
actions.append(f"DEFRAG_FAILED({ex})")
action_taken = "+".join(actions) if actions else "NONE"
sock.close()
circuit_breaker_tripped = (status != "HEALTHY")
audit_record = {
"timestamp": now_ts,
"iso_timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now_ts)),
"host": host,
"port": port,
"status": status,
"mem_fragmentation_ratio": mem_fragmentation_ratio,
"used_memory": used_memory,
"used_memory_rss": used_memory_rss,
"fragmented_mb": fragmented_mb,
"evicted_keys": evicted_keys,
"delta_evicted_keys": delta_evicted,
"eviction_rate_per_sec": eviction_rate_per_sec,
"action_taken": action_taken,
"post_remediation_ratio": post_ratio,
"post_remediation_rss": post_rss,
"activedefrag_setting": current_activedefrag,
"circuit_breaker_tripped": circuit_breaker_tripped,
"execution_id": execution_id
}
print(f"Audit Result: Status={status}, Ratio={mem_fragmentation_ratio}, Evictions={evicted_keys}, Rate={eviction_rate_per_sec}/s, Action={action_taken}")
output_map = {
"status": status,
"circuit_breaker_tripped": circuit_breaker_tripped,
"mem_fragmentation_ratio": mem_fragmentation_ratio,
"used_memory_bytes": used_memory,
"used_memory_rss_bytes": used_memory_rss,
"fragmented_mb": fragmented_mb,
"evicted_keys": evicted_keys,
"delta_evicted_keys": delta_evicted,
"eviction_rate_per_sec": eviction_rate_per_sec,
"action_taken": action_taken,
"post_remediation_ratio": post_ratio,
"post_remediation_rss": post_rss,
"audit_payload": audit_record
}
try:
from kestra import Kestra
Kestra.outputs(output_map)
except Exception:
pass
print("::" + json.dumps({"outputs": output_map}) + "::")
- id: record_audit_kv
type: io.kestra.plugin.core.kv.Set
description: Persist audit state record to Kestra KV store for historical
trending and rate analysis.
key: "{{ render(vars.kv_state_key) }}"
value: "{{ outputs.inspect_and_remediate_redis.vars.audit_payload | toJson }}"
kvType: JSON
- id: evaluate_circuit_breaker_alert
type: io.kestra.plugin.core.flow.If
description: Dispatch alerts when the circuit breaker detects fragmentation or
eviction anomalies.
condition: "{{ outputs.inspect_and_remediate_redis.vars.circuit_breaker_tripped
== true }}"
then:
- id: notify_slack_circuit_breaker
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Post structured operational incident to Slack.
runIf: "{{ inputs.enable_slack == true }}"
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "⚠️ Redis Circuit Breaker Tripped: {{ outputs.inspect_and_remediate_redis.vars.status }}",
"emoji": true
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Instance:* `{{ inputs.redis_host }}:{{ inputs.redis_port }}`"
},
{
"type": "mrkdwn",
"text": "*Status:* `{{ outputs.inspect_and_remediate_redis.vars.status }}`"
},
{
"type": "mrkdwn",
"text": "*Fragmentation Ratio:* `{{ outputs.inspect_and_remediate_redis.vars.mem_fragmentation_ratio }}`"
},
{
"type": "mrkdwn",
"text": "*Fragmented Memory:* `{{ outputs.inspect_and_remediate_redis.vars.fragmented_mb }} MB`"
},
{
"type": "mrkdwn",
"text": "*Eviction Velocity:* `{{ outputs.inspect_and_remediate_redis.vars.eviction_rate_per_sec }} keys/s`"
},
{
"type": "mrkdwn",
"text": "*Total Evicted:* `{{ outputs.inspect_and_remediate_redis.vars.evicted_keys }}`"
},
{
"type": "mrkdwn",
"text": "*Remediation Action:* `{{ outputs.inspect_and_remediate_redis.vars.action_taken }}`"
},
{
"type": "mrkdwn",
"text": "*Execution ID:* `{{ execution.id }}`"
}
]
}
]
}
- id: escalate_pagerduty_circuit_breaker
type: io.kestra.plugin.pagerduty.PagerDutyAlert
description: Escalate critical saturation or critical fragmentation breaches to
PagerDuty.
runIf: "{{ inputs.enable_pagerduty == true and
(outputs.inspect_and_remediate_redis.vars.status ==
'FRAGMENTATION_CRITICAL' or
outputs.inspect_and_remediate_redis.vars.status ==
'CRITICAL_SATURATION') }}"
url: "https://events.pagerduty.com/v2/enqueue"
payload: |
{
"routing_key": "{{ secret('PAGERDUTY_ROUTING_KEY') }}",
"event_action": "trigger",
"dedup_key": "redis-breaker-{{ inputs.redis_host }}-{{ inputs.redis_port }}",
"payload": {
"summary": "[REDIS CRITICAL] Memory fragmentation or cache thrashing on {{ inputs.redis_host }}:{{ inputs.redis_port }} (Status: {{ outputs.inspect_and_remediate_redis.vars.status }})",
"source": "kestra-flow-{{ flow.id }}",
"severity": "{{ inputs.pagerduty_severity }}",
"custom_details": {
"instance": "{{ inputs.redis_host }}:{{ inputs.redis_port }}",
"status": "{{ outputs.inspect_and_remediate_redis.vars.status }}",
"mem_fragmentation_ratio": "{{ outputs.inspect_and_remediate_redis.vars.mem_fragmentation_ratio }}",
"fragmented_mb": "{{ outputs.inspect_and_remediate_redis.vars.fragmented_mb }}",
"evicted_keys": "{{ outputs.inspect_and_remediate_redis.vars.evicted_keys }}",
"eviction_rate_per_sec": "{{ outputs.inspect_and_remediate_redis.vars.eviction_rate_per_sec }}",
"action_taken": "{{ outputs.inspect_and_remediate_redis.vars.action_taken }}",
"execution_id": "{{ execution.id }}"
}
}
}
else:
- id: log_nominal_health
type: io.kestra.plugin.core.log.Log
description: Log nominal health metrics when Redis operating parameters are
within bounds.
message: "Redis memory and key eviction state nominal on {{ inputs.redis_host
}}:{{ inputs.redis_port }}. Fragmentation ratio {{
outputs.inspect_and_remediate_redis.vars.mem_fragmentation_ratio }},
evicted keys {{ outputs.inspect_and_remediate_redis.vars.evicted_keys
}} (rate {{
outputs.inspect_and_remediate_redis.vars.eviction_rate_per_sec }}/s).
KV baseline updated."
errors:
- id: alert_flow_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alert engineering team if circuit breaker execution fails unexpectedly.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"channel": "{{ inputs.slack_channel }}",
"text": "❌ Redis Circuit Breaker execution {{ execution.id }} failed on instance {{ inputs.redis_host }}:{{ inputs.redis_port }}. Check execution logs."
}
triggers:
- id: periodic_audit_schedule
type: io.kestra.plugin.core.trigger.Schedule
description: Run every 5 minutes to audit Redis memory and protect against
runaway fragmentation.
cron: "*/5 * * * *"
disabled: true
- id: webhook_diagnostic_trigger
type: io.kestra.plugin.core.trigger.Webhook
description: On-demand execution via secure webhook from Alertmanager or
incident triage pipelines.
key: "{{ secret('REDIS_CIRCUIT_BREAKER_WEBHOOK_KEY') }}"
outputs:
- id: status
type: STRING
value: "{{ outputs.inspect_and_remediate_redis.vars.status ?? 'UNKNOWN' }}"
- id: circuit_breaker_tripped
type: BOOL
value: "{{ outputs.inspect_and_remediate_redis.vars.circuit_breaker_tripped ??
false }}"
- id: mem_fragmentation_ratio
type: FLOAT
value: "{{ outputs.inspect_and_remediate_redis.vars.mem_fragmentation_ratio ??
1.0 }}"
- id: used_memory_bytes
type: INT
value: "{{ outputs.inspect_and_remediate_redis.vars.used_memory_bytes ?? 0 }}"
- id: used_memory_rss_bytes
type: INT
value: "{{ outputs.inspect_and_remediate_redis.vars.used_memory_rss_bytes ?? 0 }}"
- id: fragmented_mb
type: FLOAT
value: "{{ outputs.inspect_and_remediate_redis.vars.fragmented_mb ?? 0.0 }}"
- id: evicted_keys
type: INT
value: "{{ outputs.inspect_and_remediate_redis.vars.evicted_keys ?? 0 }}"
- id: eviction_rate_per_sec
type: FLOAT
value: "{{ outputs.inspect_and_remediate_redis.vars.eviction_rate_per_sec ?? 0.0
}}"
- id: action_taken
type: STRING
value: "{{ outputs.inspect_and_remediate_redis.vars.action_taken ?? 'NONE' }}"
- id: post_remediation_ratio
type: FLOAT
value: "{{ outputs.inspect_and_remediate_redis.vars.post_remediation_ratio ??
0.0 }}"