Milvus Vector Database Compaction Backlog and Segment Growth Sentinel

id: milvus-compaction-backlog-segment-growth-sentinel
namespace: company.ai

inputs:
  - id: milvus_metrics_url
    type: STRING
    defaults: "http://milvus.internal:9091/metrics"
    description: Prometheus scrape endpoint for Milvus DataCoord metrics.
  - id: max_compacting_segments
    type: INT
    defaults: 20
    description: "Maximum allowable compacting segments in backlog before alerting
      (default 20)."

tasks:
  - id: query_milvus_metrics
    type: io.kestra.plugin.scripts.shell.Commands
    commands:
      - |
        curl -s -f "{{ inputs.milvus_metrics_url }}" > /tmp/milvus_metrics.txt

        # Scrape milvus_datacoord_compacting_segment_count or segment count metrics
        COMPACTING=$(grep -E "^milvus_datacoord_compacting_segment_count" /tmp/milvus_metrics.txt | awk '{print $2+0}' | head -n 1)
        echo "::set-output name=compacting_segments::${COMPACTING:-0}::"

  - id: evaluate_compaction_backlog
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.query_milvus_metrics.vars.compacting_segments | int >
      inputs.max_compacting_segments }}"
    then:
      - id: notify_platform_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#ai-platform",
            "attachments": [
              {
                "color": "warning",
                "title": "🚨 Milvus Compaction Backlog Saturated Alert",
                "text": "Milvus instance at `{{ inputs.milvus_metrics_url }}` has *{{ outputs.query_milvus_metrics.vars.compacting_segments }} compacting segments* in backlog! Uncompacted small segments and delayed merges are elevating vector search latency.",
                "footer": "Kestra AI Sentinel • Milvus"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#ai-platform",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ Milvus Sentinel Failure",
            "text": "Failed to scrape Milvus metrics endpoint. Verify DataCoord service availability.",
            "footer": "Kestra Workflow Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/30 * * * *"
    disabled: true