id: vespa-cluster-convergence-and-node-health-sentinel
namespace: company.ai
description: |
Continuously audits Vespa big data serving and vector search cluster node readiness, detects application config generation desynchronization, inspects JVM heap headroom, and dispatches actionable Slack incident alerts.
inputs:
- id: vespa_host
type: STRING
displayName: Vespa Container Endpoint Host
defaults: "http://vespa.internal:8080"
description: "Base HTTP URL of the target Vespa stateless container node or
query endpoint."
- id: expected_application_generation
type: INT
displayName: Expected Config Generation
defaults: -1
description: "Expected application config generation number. Set to -1 to
automatically track and accept current active generation."
- id: min_free_heap_bytes
type: INT
displayName: Minimum Free Heap Threshold (Bytes)
defaults: 268435456
description: "Minimum tolerated free JVM heap memory in bytes (default 256 MB:
268435456) before raising saturation warnings."
- id: slack_channel
type: STRING
displayName: Operational Slack Channel
defaults: "#ai-ops"
description: "Target Slack channel or webhook destination for cluster health and
convergence incident alerts."
triggers:
- id: scheduled_sentinel_audit
type: io.kestra.plugin.core.trigger.Schedule
cron: "0 */3 * * *"
disabled: true
inputs:
vespa_host: "http://vespa.internal:8080"
expected_application_generation: -1
min_free_heap_bytes: 268435456
slack_channel: "#ai-ops"
tasks:
- id: probe_node_health
type: io.kestra.plugin.core.http.Request
uri: "{{ inputs.vespa_host }}/state/v1/health"
method: GET
headers:
Accept: application/json
timeout: PT15S
description: "Queries the Vespa container state/health endpoint to evaluate
service readiness and HTTP process status."
- id: probe_application_status
type: io.kestra.plugin.core.http.Request
uri: "{{ inputs.vespa_host }}/ApplicationStatus"
method: GET
headers:
Accept: application/json
timeout: PT15S
description: "Retrieves active deployment metadata, generation numbers, and
application model details from the local container cluster."
- id: probe_cluster_metrics
type: io.kestra.plugin.core.http.Request
uri: "{{ inputs.vespa_host }}/state/v1/metrics"
method: GET
headers:
Accept: application/json
timeout: PT30S
description: "Collects real-time container metrics including JVM free memory,
active document counters, and indexing throughput."
- id: evaluate_cluster_health
type: io.kestra.plugin.core.flow.Switch
value: >-
{% set health = outputs.probe_node_health.body %} {% set status_code =
health.status.code | default('down') %} {% set app_status =
outputs.probe_application_status.body %} {% set current_gen =
app_status.generation | default(-1) %} {% set metrics =
outputs.probe_cluster_metrics.body %} {% set free_heap = 1073741824 %} {%
if metrics.metrics is defined and metrics.metrics.values is iterable %}
{% for metric in metrics.metrics.values %}
{% if metric.name == 'mem.heap.free' %}
{% set free_heap = metric.values.average | default(metric.values.last | default(free_heap)) %}
{% endif %}
{% endfor %}
{% endif %} {% if status_code != 'up' %}
NODE_DOWN
{% elseif inputs.expected_application_generation > 0 and current_gen !=
inputs.expected_application_generation %}
GENERATION_DESYNC
{% elseif free_heap < inputs.min_free_heap_bytes %}
DEGRADED_HEAP
{% else %}
COMPLIANT
{% endif %}
cases:
COMPLIANT:
- id: log_compliant_status
type: io.kestra.plugin.core.log.Log
message: "Vespa cluster node '{{ inputs.vespa_host }}' is healthy and converged.
Active generation: {{
outputs.probe_application_status.body.generation |
default('unknown') }}."
DEGRADED_HEAP:
- id: alert_heap_saturation_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"attachments": [
{
"color": "#FFA726",
"title": "⚠️ Vespa JVM Heap Saturation Warning",
"text": "Vespa container node `{{ inputs.vespa_host }}` reported low free JVM heap headroom below threshold `{{ inputs.min_free_heap_bytes }} bytes`.\n*Active Generation:* `{{ outputs.probe_application_status.body.generation | default('unknown') }}`\n*Status Code:* `{{ outputs.probe_node_health.body.status.code | default('unknown') }}`",
"footer": "Kestra Sentinel | Vector Search Resource Guard"
}
]
}
- id: log_heap_warning
type: io.kestra.plugin.core.log.Log
message: "Vespa container node '{{ inputs.vespa_host }}' has insufficient heap
headroom. Review garbage collection and cache allocation."
GENERATION_DESYNC:
- id: alert_generation_desync_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"attachments": [
{
"color": "#E53935",
"title": "❌ Vespa Config Generation Desynchronization",
"text": "Vespa container node `{{ inputs.vespa_host }}` is serving config generation `{{ outputs.probe_application_status.body.generation | default(-1) }}`, which differs from expected generation `{{ inputs.expected_application_generation }}`.\nApplication convergence has stalled or an unapplied schema revision exists.",
"footer": "Kestra Sentinel | Schema Convergence Gate"
}
]
}
- id: log_desync_warning
type: io.kestra.plugin.core.log.Log
message: "Application generation mismatch on '{{ inputs.vespa_host }}'.
Investigation required to verify config model convergence."
NODE_DOWN:
- id: alert_node_down_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"attachments": [
{
"color": "#D32F2F",
"title": "🚨 Vespa Node Outage Detected",
"text": "Vespa health probe returned status `{{ outputs.probe_node_health.body.status.code | default('down') }}` on `{{ inputs.vespa_host }}`.\n*Description:* `{{ outputs.probe_node_health.body.status.description | default('Node unreachable or unhealthy') }}`",
"footer": "Kestra Sentinel | Infrastructure Availability Alert"
}
]
}
- id: log_audit_summary
type: io.kestra.plugin.core.log.Log
message: "Completed Vespa cluster health audit on {{ inputs.vespa_host }} at {{
now() }}. Node status: {{ outputs.probe_node_health.body.status.code |
default('down') }}, Generation: {{
outputs.probe_application_status.body.generation | default('unknown') }}."
errors:
- id: alert_probe_failure_slack
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"attachments": [
{
"color": "#D32F2F",
"title": "🚨 Vespa Sentinel Probe Failure",
"text": "The Vespa compliance probe encountered an unhandled network error or connection timeout connecting to `{{ inputs.vespa_host }}`.\n*Execution ID:* `{{ flow.executionId }}`\n*Error Message:* `{{ flow.errorMessage | default('Unknown execution error') }}`",
"footer": "Kestra Orchestrator | Automated Incident Management"
}
]
}