Scheduled KVM maintenance window orchestration

id: kvm-scheduled-maintenance-window
namespace: company.infra

triggers:
  - id: weekly_window
    type: io.kestra.plugin.core.trigger.Schedule
    description: Run every Sunday at 02:00 UTC, an off-hours maintenance window.
    cron: "0 2 * * 0"
    timezone: Etc/UTC

inputs:
  - id: target_vm
    type: STRING
    defaults: app-server-01
    description: >
      Libvirt domain to patch during the maintenance window. Treat as a trusted,
      operator-supplied identifier.
    validator: ^[a-z0-9][a-z0-9-]{0,62}$

variables:
  libvirt_uri: "{{ secret('LIBVIRT_URI') }}"

concurrency:
  limit: 1

tasks:
  - id: inventory
    type: io.kestra.plugin.kvm.ListVms
    description: Enumerate domains on the hypervisor before any changes for an audit record.
    uri: "{{ vars.libvirt_uri }}"
    retry:
      type: constant
      interval: PT15S
      maxAttempts: 3

  - id: assert_target_present
    type: io.kestra.plugin.core.execution.Fail
    description: >
      Fail early (before stopping anything) if the target domain is not present
      in the inventory, so the window does not shut down a VM it cannot verify.
    condition: "{{ inputs.target_vm not in (outputs.inventory.vms | jq('.[].name')) }}"
    errorMessage: "Domain {{ inputs.target_vm }} was not found in the hypervisor
      inventory; aborting maintenance window."

  - id: stop_window
    type: io.kestra.plugin.core.flow.Sequential
    description: >
      Gracefully stop the VM, and hard power-off as a local fallback if the ACPI
      shutdown stalls. The fallback lives in this flowable's errors block
      because a leaf task cannot carry its own errors handler.

    tasks:
      - id: graceful_stop
        type: io.kestra.plugin.kvm.StopVm
        description: Send a graceful ACPI shutdown and wait until the domain reaches
          SHUTOFF.
        uri: "{{ vars.libvirt_uri }}"
        name: "{{ inputs.target_vm }}"
        force: false
        waitForStopped: true
        timeToWait: PT5M
        retry:
          type: constant
          interval: PT15S
          maxAttempts: 2
    errors:
      - id: force_stop
        type: io.kestra.plugin.kvm.StopVm
        description: >
          ACPI shutdown did not complete in time (no ACPI daemon, hung kernel,
          stuck app). Hard power-off so maintenance can proceed instead of
          stranding the VM in an unknown state.

        uri: "{{ vars.libvirt_uri }}"
        name: "{{ inputs.target_vm }}"
        force: true
        waitForStopped: true
        timeToWait: PT2M
        retry:
          type: constant
          interval: PT15S
          maxAttempts: 2

  - id: run_maintenance
    type: io.kestra.plugin.scripts.shell.Commands
    description: >
      Run a real offline maintenance action against the stopped domain: take a
      libvirt-external disk snapshot and report the qcow2 image health.
    # Script tasks default to the Docker runner. This one needs host-side libvirt
    # tooling (qemu-img/virsh against the local hypervisor), so it uses the Process
    # runner to execute on the host. Remove this block to run in Docker, or switch
    # the type to Kubernetes/Docker if your hypervisor tooling lives in a container.
    taskRunner:
      type: io.kestra.plugin.core.runner.Process
    retry:
      type: constant
      interval: PT30S
      maxAttempts: 2
    env:
      LIBVIRT_DEFAULT_URI: "{{ vars.libvirt_uri }}"
      TARGET_VM: "{{ inputs.target_vm }}"
      SNAPSHOT_NAME: "maint-{{ (trigger.date ?? execution.startDate) |
        date('yyyyMMdd-HHmmss') }}"
    commands:
      - set -euo pipefail
      - echo "Maintenance window started for ${TARGET_VM} at $(date -u)"
      - echo "Reported domain state is {{ outputs.graceful_stop.state }};
        running offline maintenance"
      - virsh snapshot-create-as --domain "${TARGET_VM}" --name
        "${SNAPSHOT_NAME}" --disk-only --atomic
      - |
        for disk in $(virsh domblklist "${TARGET_VM}" --details | awk '$2=="disk"{print $4}'); do
          echo "Inspecting ${disk}"
          qemu-img info "${disk}" || true
        done
      - echo "Maintenance complete for ${TARGET_VM} (snapshot ${SNAPSHOT_NAME})"

  - id: restart_vm
    type: io.kestra.plugin.kvm.StartVm
    description: Boot the domain again and wait for it to return to the RUNNING state.
    uri: "{{ vars.libvirt_uri }}"
    name: "{{ inputs.target_vm }}"
    waitForRunning: true
    timeToWait: PT3M
    retry:
      type: constant
      interval: PT15S
      maxAttempts: 3

  - id: notify_complete
    type: io.kestra.plugin.email.MailSend
    description: Email the operations team a maintenance summary once the VM is back online.
    host: "{{ secret('SMTP_HOST') }}"
    port: 587
    transportStrategy: SMTP_TLS
    username: "{{ secret('SMTP_USERNAME') }}"
    password: "{{ secret('SMTP_PASSWORD') }}"
    from: kestra@company.local
    to: ops@company.local
    retry:
      type: constant
      interval: PT1M
      maxAttempts: 3
    subject: "Maintenance complete for {{ inputs.target_vm }}"
    htmlTextContent: |
      Maintenance window finished for domain {{ inputs.target_vm }}.<br>
      Final state: {{ outputs.restart_vm.state }}.<br>
      Execution: {{ execution.id }}

errors:
  - id: alert_failure
    type: io.kestra.plugin.email.MailSend
    description: >
      Alert ops on any failure. A maintenance window is defined by what happens
      when it goes wrong: if the stop, the maintenance script, or the restart
      fails, ops must hear about it because the VM may be left down.

    host: "{{ secret('SMTP_HOST') }}"
    port: 587
    transportStrategy: SMTP_TLS
    username: "{{ secret('SMTP_USERNAME') }}"
    password: "{{ secret('SMTP_PASSWORD') }}"
    from: kestra@company.local
    to: ops@company.local
    retry:
      type: constant
      interval: PT1M
      maxAttempts: 3
    subject: ":rotating_light: MAINTENANCE FAILED for {{ inputs.target_vm }}"
    htmlTextContent: |
      Maintenance window FAILED for domain {{ inputs.target_vm }}.<br>
      Flow: {{ flow.namespace }}.{{ flow.id }}<br>
      Execution: {{ execution.id }}<br>
      Last known stop state: {{ outputs.graceful_stop.state ?? 'n/a' }}<br>
      Last known restart state: {{ outputs.restart_vm.state ?? 'n/a' }}<br>
      The VM may be stopped; verify and recover it manually.