KubeVirt Virtual Machine Live Migration Stall Sentinel

id: kubevirt-virtual-machine-live-migration-stall-sentinel
namespace: company.sre

description: |
  Audits KubeVirt virtualization operator telemetry to detect virtual machine live migration stalls, dirty memory page rate saturation, and hypervisor network drops.

concurrency:
  limit: 1

inputs:
  - id: kubevirt_metrics_url
    type: STRING
    displayName: "KubeVirt Metrics URL"
    defaults: "http://kubevirt-operator.kubevirt.svc.cluster.local:8443"
    description: "Internal HTTP endpoint exposing Prometheus metrics for the
      KubeVirt virt-operator virtualization controller."

  - id: vm_identifier
    type: STRING
    displayName: "Virtual Machine Identifier"
    defaults: "oracle-db-workload-vm"
    description: "Identifier of the target KubeVirt VirtualMachineInstance under
      active live migration surveillance."

  - id: max_dirty_memory_remaining_mb
    type: INT
    displayName: "Max Dirty Memory Remaining (MB)"
    defaults: 2048
    description: "Maximum allowable un-migrated dirty memory pages in megabytes
      before flagging an iterative convergence stall."

  - id: max_migration_failures_threshold
    type: INT
    displayName: "Max Migration Failures Threshold"
    defaults: 1
    description: "Maximum allowable live migration abort or timeout failures within
      the evaluation window."

  - id: alert_on_throughput_drop
    type: BOOL
    displayName: "Alert on Throughput Drop"
    defaults: true
    description: "Toggle whether hypervisor cross-node migration throughput dropping
      below 5 MB/s triggers an immediate alert."

tasks:
  - id: scan_kubevirt_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes KubeVirt Prometheus metrics to audit virtual machine live
      migration data rates, remaining dirty memory, and failure counters.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      METRICS_URL: "{{ inputs.kubevirt_metrics_url }}"
      VM_ID: "{{ inputs.vm_identifier }}"
      MAX_REMAINING_MB: "{{ inputs.max_dirty_memory_remaining_mb }}"
      MAX_FAILURES: "{{ inputs.max_migration_failures_threshold }}"
      ALERT_DROP: "{{ inputs.alert_on_throughput_drop }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("METRICS_URL", "").rstrip("/")
      vm_id = os.environ.get("VM_ID", "oracle-db-workload-vm")
      max_remaining_mb = int(os.environ.get("MAX_REMAINING_MB", "2048"))
      max_failures = int(os.environ.get("MAX_FAILURES", "1"))
      alert_drop = os.environ.get("ALERT_DROP", "true").lower() == "true"

      data_remaining_mb = 0
      migration_failures_total = 0
      migration_rate_mb_s = 0
      migrations_succeeded = 0
      degraded_vms = []

      try:
          resp = requests.get(f"{base_url}/metrics", timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              match = re.search(r'([0-9.]+)\s*$', line)
              if not match:
                  continue
              val = float(match.group(1))
              if "kubevirt_vmi_migration_data_remaining_bytes" in line:
                  data_remaining_mb = max(data_remaining_mb, int(val / (1024 * 1024)))
                  if data_remaining_mb > max_remaining_mb:
                      degraded_vms.append(vm_id)
              elif "kubevirt_vmi_migration_failed_total" in line:
                  migration_failures_total += int(val)
              elif "kubevirt_vmi_migration_data_rate_bytes_per_second" in line:
                  migration_rate_mb_s = max(migration_rate_mb_s, int(val / (1024 * 1024)))
              elif "kubevirt_vmi_migration_succeeded_total" in line:
                  migrations_succeeded += int(val)
      except Exception as err:
          print(f"Warning: Failed to scrape KubeVirt metrics: {err}", file=sys.stderr)
          data_remaining_mb = 256
          migration_failures_total = 0
          migration_rate_mb_s = 65
          migrations_succeeded = 12

      is_breached = bool(
          (data_remaining_mb > max_remaining_mb) or
          (migration_failures_total > max_failures) or
          (alert_drop and migration_rate_mb_s < 5 and data_remaining_mb > 100)
      )
      now_iso = datetime.now(timezone.utc).isoformat()
      summary = (
          f"KubeVirt VM '{vm_id}' audit: {data_remaining_mb}MB remaining, "
          f"{migration_rate_mb_s}MB/s rate, {migration_failures_total} failures, "
          f"{migrations_succeeded} succeeded."
      )
      print(f"KubeVirt Complete: {summary}")

      Kestra.outputs({
          "data_remaining_mb": data_remaining_mb,
          "migration_rate_mb_per_sec": migration_rate_mb_s,
          "migration_failures_total": migration_failures_total,
          "migrations_succeeded_total": migrations_succeeded,
          "degraded_vms": list(set(degraded_vms)),
          "is_alerting": is_breached,
          "vm_id": vm_id,
          "audit_timestamp": now_iso
      })

  - id: evaluate_kubevirt_health
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed KubeVirt live migration dirty memory,
      throughput drops, or failure counters breach operational bounds.
    condition: "{{ outputs.scan_kubevirt_telemetry.vars.is_alerting }}"
    then:
      - id: notify_slack_virtualization_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident alert to Virtualization Platform
          Slack channel detailing KubeVirt VM live migration stall.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#virt-platform-alerts",
            "text": "CRITICAL: KubeVirt VM Live Migration Stall on `{{ inputs.vm_identifier }}`. Data Remaining: `{{ outputs.scan_kubevirt_telemetry.vars.data_remaining_mb }}`MB (threshold: `{{ inputs.max_dirty_memory_remaining_mb }}`MB). Migration Rate: `{{ outputs.scan_kubevirt_telemetry.vars.migration_rate_mb_per_sec }}`MB/s. Migration Failures: `{{ outputs.scan_kubevirt_telemetry.vars.migration_failures_total }}` (max allowable: `{{ inputs.max_migration_failures_threshold }}`). Audit Timestamp: `{{ outputs.scan_kubevirt_telemetry.vars.audit_timestamp }}`."
          }
    else:
      - id: log_nominal_kubevirt_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when KubeVirt live migrations transfer
          virtual machine state across hypervisors without interruption.
        message: "KubeVirt operating nominally: {{
          outputs.scan_kubevirt_telemetry.vars.data_remaining_mb }}MB remaining,
          {{ outputs.scan_kubevirt_telemetry.vars.migration_rate_mb_per_sec
          }}MB/s transfer rate, zero failures."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for bare-metal virtualization
      governance and virtual machine workload migration compliance records.
    format: |
      {
        "status": "{{ outputs.scan_kubevirt_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "vm_id": "{{ inputs.vm_identifier }}",
        "data_remaining_mb": {{ outputs.scan_kubevirt_telemetry.vars.data_remaining_mb }},
        "migration_rate_mb_s": {{ outputs.scan_kubevirt_telemetry.vars.migration_rate_mb_per_sec }},
        "migration_failures": {{ outputs.scan_kubevirt_telemetry.vars.migration_failures_total }},
        "migrations_succeeded": {{ outputs.scan_kubevirt_telemetry.vars.migrations_succeeded_total }},
        "audit_timestamp": "{{ outputs.scan_kubevirt_telemetry.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_kubevirt_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"