Self-healing KVM VM monitor with auto-restart

id: kvm-self-healing-monitor
namespace: company.infra

inputs:
  - id: monitored_vm
    type: STRING
    defaults: critical-db-01
    description: >
      Libvirt domain to watch and auto-recover. Treat as a trusted,
      operator-supplied identifier.
    validator: ^[a-z0-9][a-z0-9-]{0,62}$

  - id: max_restarts_per_window
    type: INT
    defaults: 3
    description: >
      Maximum auto-restarts allowed inside the flap window before the flow stops
      restarting and escalates to on-call instead. Prevents endlessly rebooting
      a crash-looping VM.

  - id: flap_window
    type: DURATION
    defaults: PT30M
    description: >
      Sliding window over which restarts are counted. The restart counter in the
      KV store expires after this duration (TTL), so flapping resets once the VM
      stays healthy for a full window.

variables:
  libvirt_uri: "{{ secret('LIBVIRT_URI') }}"

pluginDefaults:
  - type: io.kestra.plugin.kvm
    values:
      uri: "{{ vars.libvirt_uri }}"

concurrency:
  limit: 1

tasks:
  - id: read_maintenance_flag
    type: io.kestra.plugin.core.kv.Get
    description: >
      Read an optional per-VM maintenance flag. When set, the domain was stopped
      on purpose (backup, failover, planned maintenance) and must not be
      auto-booted.
    key: "maintenance_{{ inputs.monitored_vm }}"
    errorOnMissing: false

  - id: read_restart_counter
    type: io.kestra.plugin.core.kv.Get
    description: Read how many times this VM has been auto-restarted within the
      current flap window.
    key: "restart_count_{{ inputs.monitored_vm }}"
    errorOnMissing: false

  - id: decide
    type: io.kestra.plugin.core.flow.If
    description: >
      Recover only when the VM is down unexpectedly AND it has not exceeded the
      flap threshold. Otherwise escalate or stand down.
    condition: "{{ (outputs.read_maintenance_flag.value ?? 'false') != 'true' and
      ((outputs.read_restart_counter.value ?? 0) | number) <
      (inputs.max_restarts_per_window | number) }}"
    then:
      - id: bump_restart_counter
        type: io.kestra.plugin.core.kv.Set
        description: >
          Increment the flap counter before restarting, with a TTL equal to the
          flap window so it resets automatically after a healthy period.
        key: "restart_count_{{ inputs.monitored_vm }}"
        kvType: NUMBER
        value: "{{ ((outputs.read_restart_counter.value ?? 0) | number) + 1 }}"
        ttl: "{{ inputs.flap_window }}"

      - id: recover_vm
        type: io.kestra.plugin.kvm.StartVm
        description: Boot the unexpectedly stopped domain and wait for it to reach RUNNING.
        name: "{{ inputs.monitored_vm }}"
        waitForRunning: true
        timeToWait: PT2M
        retry:
          type: constant
          interval: PT15S
          maxAttempt: 3

      - id: alert_recovered
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Slack alert that the domain was found down and auto-restarted.
        url: "{{ secret('SLACK_WEBHOOK') }}"
        retry:
          type: constant
          interval: PT10S
          maxAttempt: 3
        payload: |
          {
            "blocks": [
              {
                "type": "header",
                "text": {
                  "type": "plain_text",
                  "text": "Auto-recovery: {{ inputs.monitored_vm }} was down"
                }
              },
              {
                "type": "section",
                "fields": [
                  {
                    "type": "mrkdwn",
                    "text": "*Detected state:*\n{{ trigger.state }}"
                  },
                  {
                    "type": "mrkdwn",
                    "text": "*State after restart:*\n{{ outputs.recover_vm.state }}"
                  },
                  {
                    "type": "mrkdwn",
                    "text": "*Restarts in window:*\n{{ ((outputs.read_restart_counter.value ?? 0) | number) + 1 }} / {{ inputs.max_restarts_per_window }}"
                  },
                  {
                    "type": "mrkdwn",
                    "text": "*Execution:*\n{{ execution.id }}"
                  }
                ]
              }
            ]
          }
    else:
      - id: classify
        type: io.kestra.plugin.core.flow.If
        description: Distinguish an intentional stop from a flapping VM that hit the
          restart ceiling.
        condition: "{{ (outputs.read_maintenance_flag.value ?? 'false') == 'true' }}"
        then:
          - id: log_maintenance
            type: io.kestra.plugin.core.log.Log
            description: The VM is down on purpose; do not auto-restart.
            level: INFO
            message: "Domain {{ inputs.monitored_vm }} is SHUTOFF but the maintenance flag
              is set; skipping auto-restart."
        else:
          - id: escalate_flapping
            type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
            description: >
              The VM has hit the restart ceiling inside the flap window. Stop
              restarting and page on-call instead of rebooting a sick VM
              forever.
            url: "{{ secret('SLACK_WEBHOOK') }}"
            retry:
              type: constant
              interval: PT10S
              maxAttempt: 3
            messageText: ":rotating_light: <!channel> {{ inputs.monitored_vm }} has flapped
              {{ outputs.read_restart_counter.value ?? 0 }} times within {{
              inputs.flap_window }} and was NOT auto-restarted. On-call must
              investigate (execution {{ execution.id }})."

errors:
  - id: alert_recovery_failed
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: >
      A failed self-heal is more urgent than a successful one: if the restart
      itself fails or never reaches RUNNING, page loudly so the VM does not stay
      down silently.
    url: "{{ secret('SLACK_WEBHOOK') }}"
    retry:
      type: constant
      interval: PT10S
      maxAttempt: 3
    messageText: ":rotating_light: <!channel> Auto-recovery FAILED for {{
      inputs.monitored_vm }} in {{ flow.namespace }}.{{ flow.id }} (execution {{
      execution.id }}). The VM is likely still down; investigate now."

triggers:
  - id: watch_vm
    type: io.kestra.plugin.kvm.VmEventTrigger
    description: >
      Poll the monitored domain's state every minute, but only create an
      execution when it is SHUTOFF, so a healthy VM does not flood the execution
      history with no-op runs.
    uri: "{{ vars.libvirt_uri }}"
    name: "{{ inputs.monitored_vm }}"
    interval: PT1M
    conditions:
      - type: io.kestra.plugin.core.condition.Expression
        expression: "{{ trigger.state == 'SHUTOFF' }}"