Alert on Worker Disk Usage Thresholds

id: docker-host-disk-usage-alert
namespace: company.team
description: |
  Check root filesystem usage on the worker host, and alert Slack before the
  disk fills up and takes pipelines down with it.

triggers:
  - id: hourly_disk_check
    type: io.kestra.plugin.core.trigger.Schedule
    description: Hourly check catches logs and build artifacts ballooning between cleanups.
    cron: "15 * * * *"

inputs:
  - id: threshold_percent
    type: INT
    defaults: 85
    description: Usage percentage that triggers the alert.
  - id: path
    type: STRING
    defaults: "/"
    description: Filesystem path to check.

tasks:
  - id: check_disk
    type: io.kestra.plugin.scripts.shell.Commands
    description: Parse df output for the given path and emit the integer usage
      percent via the stdout outputs protocol. A parse failure reports worst
      case so it cannot read as healthy.
    containerImage: alpine:3.20
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    commands:
      - |
        apk add --no-cache python3 >/dev/null 2>&1
        cat > disk.py <<'PYEOF'
        import os, json, subprocess
        path = os.environ.get("CHECK_PATH", "/")
        try:
            out = subprocess.run(["df", "-P", path], capture_output=True, text=True, timeout=15).stdout.splitlines()
            used = int(out[1].split()[4].rstrip("%")) if len(out) > 1 else 100
        except Exception:
            used = 100
        print(f"{path} usage: {used}%")
        print("::" + json.dumps({"outputs": {"used_percent": used}}) + "::")
        PYEOF
        python3 disk.py
    env:
      CHECK_PATH: "{{ inputs.path }}"

  - id: over_threshold
    type: io.kestra.plugin.core.flow.If
    description: Branch so only a real breach alerts; a passing check is logged for
      the trend.
    condition: "{{ outputs.check_disk.vars.used_percent >= inputs.threshold_percent }}"
    then:
      - id: alert_disk
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: State the numbers and the path so the on-call sees exactly which
          disk is in trouble.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "text": ":floppy_disk: Disk usage on {{ inputs.path }} is {{ outputs.check_disk.vars.used_percent }}% (threshold {{ inputs.threshold_percent }}%). Free space before a worker starts failing. Execution {{ execution.id }}."
          }
    else:
      - id: log_ok
        type: io.kestra.plugin.core.log.Log
        description: Record the passing reading on every run so the history is your
          usage trend.
        message: "Disk on {{ inputs.path }} at {{ outputs.check_disk.vars.used_percent
          }}%, under the {{ inputs.threshold_percent }}% threshold."

errors:
  - id: alert_check_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alert when the check itself fails - a dead probe is not a clean bill.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "Disk usage check FAILED in flow {{ flow.id }} (execution {{ execution.id }}). A broken check must not look like healthy disks."
      }

outputs:
  - id: disk_usage
    type: INT
    description: 'Current usage percent for the checked path, e.g. 72.'
    value: "{{ outputs.check_disk.vars.used_percent }}"