id: quickwit-index-retention-policy-compliance-sentinel
namespace: company.data
labels:
env: production
system: quickwit
category: data-platform
inputs:
- id: quickwit_url
type: STRING
displayName: Quickwit Base URL
description: REST API endpoint of the Quickwit cluster (default http://localhost:7280).
defaults: http://localhost:7280
- id: index_id
type: STRING
displayName: Index ID
description: ID of the Quickwit index to audit for retention policy compliance
and search health.
defaults: app-logs
- id: approved_retention_period
type: STRING
displayName: Approved Retention Period
description: Approved retention period string (e.g., "30 days", "60 days") that
the index must adhere to under data governance policies.
defaults: 30 days
- id: enforce_strict_period
type: BOOL
displayName: Enforce Strict Period
description: When true, classifies any retention period differing from
approved_retention_period as non-compliant.
defaults: true
- id: max_query_latency_ms
type: INT
displayName: Max Query Latency (ms)
description: Maximum acceptable canary search latency in milliseconds before
flagging query degradation.
defaults: 500
- id: alert_on_warning
type: BOOL
displayName: Alert on Warning
description: Whether to dispatch Slack alerts for non-critical policy warnings
in addition to critical retention breaches.
defaults: true
triggers:
- id: periodic_retention_audit_schedule
type: io.kestra.plugin.core.trigger.Schedule
description: Scheduled surveillance trigger auditing Quickwit index retention
configuration every 6 hours.
cron: "0 */6 * * *"
disabled: true
inputs:
quickwit_url: "http://localhost:7280"
index_id: "app-logs"
approved_retention_period: "30 days"
enforce_strict_period: true
max_query_latency_ms: 500
alert_on_warning: true
tasks:
- id: probe_index_metadata
type: io.kestra.plugin.quickwit.index.Get
description: Fetch index configuration, document mappings, and effective
retention policy definitions from the Quickwit cluster.
url: "{{ inputs.quickwit_url }}"
index: "{{ inputs.index_id }}"
- id: probe_search_availability
type: io.kestra.plugin.quickwit.search.Search
description: Execute a lightweight canary search query to measure cluster query
liveness and response latency.
url: "{{ inputs.quickwit_url }}"
index: "{{ inputs.index_id }}"
query: "*"
fetchType: FETCH
- id: classify_retention_compliance
type: io.kestra.plugin.core.log.Log
description: Evaluate retrieved index metadata and search latency against
organizational governance benchmarks.
message: |
Quickwit Retention Audit Evaluation for Index '{{ inputs.index_id }}':
- Index UID: {{ outputs.probe_index_metadata.indexUid }}
- Configured Retention: {{ outputs.probe_index_metadata.metadata.indexConfig.retention | json }}
- Approved Retention Period: {{ inputs.approved_retention_period }}
- Canary Hits Count: {{ outputs.probe_search_availability.numHits }}
- Search Latency: {{ outputs.probe_search_availability.elapsedTimeMicros / 1000 }} ms (Threshold: {{ inputs.max_query_latency_ms }} ms)
- id: route_by_compliance_status
type: io.kestra.plugin.core.flow.Switch
description: Branch flow execution based on Quickwit retention policy compliance
and search health.
value: "{{ (outputs.probe_index_metadata.metadata == null or
outputs.probe_index_metadata.metadata.indexConfig == null or
outputs.probe_index_metadata.metadata.indexConfig.retention == null) ?
'CRITICAL_RETENTION_UNCONFIGURED' : (inputs.enforce_strict_period and
outputs.probe_index_metadata.metadata.indexConfig.retention.period !=
inputs.approved_retention_period ? 'POLICY_RETENTION_EXCEEDED' :
'COMPLIANT') }}"
cases:
CRITICAL_RETENTION_UNCONFIGURED:
- id: log_critical_retention_breach
type: io.kestra.plugin.core.log.Log
level: ERROR
message: "CRITICAL: Quickwit index '{{ inputs.index_id }}' has NO retention
policy configured! Immutable splits will accumulate unbounded in
object storage."
- id: alert_slack_critical
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Dispatch high-priority incident alert to Slack when an index lacks
retention configuration.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "CRITICAL: Quickwit index '{{ inputs.index_id }}' has NO retention policy configured!",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Quickwit Sentinel Alert: Missing Retention Policy",
"emoji": true
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Cluster URL:*\n{{ inputs.quickwit_url }}"
},
{
"type": "mrkdwn",
"text": "*Index ID:*\n`{{ inputs.index_id }}`"
},
{
"type": "mrkdwn",
"text": "*Index UID:*\n`{{ outputs.probe_index_metadata.indexUid }}`"
},
{
"type": "mrkdwn",
"text": "*Retention Status:*\n*UNCONFIGURED / INFINITE*"
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Remediation Required:* Update the index configuration to define a `retention` block with `period` (e.g. `\"30 days\"`) and `schedule` (`\"@daily\"`) to prevent runaway cloud storage billing."
}
},
{
"type": "context",
"elements": [
{
"type": "mrkdwn",
"text": "Execution: {{ execution.id }} | Flow: {{ flow.id }} | Namespace: {{ flow.namespace }}"
}
]
}
]
}
POLICY_RETENTION_EXCEEDED:
- id: log_policy_warning
type: io.kestra.plugin.core.log.Log
level: WARN
message: "WARNING: Quickwit index '{{ inputs.index_id }}' configured retention
period '{{
outputs.probe_index_metadata.metadata.indexConfig.retention.period
}}' does not match approved period '{{
inputs.approved_retention_period }}'."
- id: alert_slack_warning
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Dispatch warning alert to Slack when retention period drifts from
approved governance baseline.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
runIf: "{{ inputs.alert_on_warning }}"
payload: |
{
"text": "WARNING: Quickwit index '{{ inputs.index_id }}' retention policy drift detected.",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Quickwit Governance Warning: Retention Policy Drift",
"emoji": true
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Index:*\n`{{ inputs.index_id }}`"
},
{
"type": "mrkdwn",
"text": "*Configured Period:*\n`{{ outputs.probe_index_metadata.metadata.indexConfig.retention.period }}`"
},
{
"type": "mrkdwn",
"text": "*Approved Period:*\n`{{ inputs.approved_retention_period }}`"
},
{
"type": "mrkdwn",
"text": "*Canary Search Latency:*\n{{ outputs.probe_search_availability.elapsedTimeMicros / 1000 }} ms"
}
]
},
{
"type": "context",
"elements": [
{
"type": "mrkdwn",
"text": "Execution: {{ execution.id }} | Namespace: {{ flow.namespace }}"
}
]
}
]
}
COMPLIANT:
- id: log_compliant_status
type: io.kestra.plugin.core.log.Log
message: "Quickwit index '{{ inputs.index_id }}' is COMPLIANT: retention period
is '{{
outputs.probe_index_metadata.metadata.indexConfig.retention.period
}}' on schedule '{{
outputs.probe_index_metadata.metadata.indexConfig.retention.schedule
}}'. Canary query executed in {{
outputs.probe_search_availability.elapsedTimeMicros / 1000 }} ms."
- id: log_audit_summary
type: io.kestra.plugin.core.log.Log
description: Record a permanent audit log entry summarizing index retention
health and query metrics for governance history.
message: |
QUICKWIT INDEX RETENTION AUDIT SUMMARY:
- Execution ID: {{ execution.id }}
- Cluster: {{ inputs.quickwit_url }}
- Index: {{ inputs.index_id }} (UID: {{ outputs.probe_index_metadata.indexUid }})
- Retention Defined: {{ outputs.probe_index_metadata.metadata.indexConfig.retention != null }}
- Retention Period: {{ outputs.probe_index_metadata.metadata.indexConfig.retention.period ?? 'NONE' }}
- Status: {{ (outputs.probe_index_metadata.metadata == null or outputs.probe_index_metadata.metadata.indexConfig == null or outputs.probe_index_metadata.metadata.indexConfig.retention == null) ? 'CRITICAL_RETENTION_UNCONFIGURED' : (inputs.enforce_strict_period and outputs.probe_index_metadata.metadata.indexConfig.retention.period != inputs.approved_retention_period ? 'POLICY_RETENTION_EXCEEDED' : 'COMPLIANT') }}
- Canary Hits: {{ outputs.probe_search_availability.numHits }}
- Canary Latency Micros: {{ outputs.probe_search_availability.elapsedTimeMicros }}
- Timestamp: {{ now() }}
errors:
- id: alert_flow_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Dispatch emergency alert to Slack if Quickwit cluster connection or
probe execution fails.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "ERROR: Quickwit Retention Sentinel failed during execution {{ execution.id }}. Check Kestra execution logs."
}
- id: log_failure_context
type: io.kestra.plugin.core.log.Log
level: ERROR
message: "Quickwit Index Retention Sentinel failed during probe execution for
index {{ inputs.index_id }} on cluster {{ inputs.quickwit_url }}."