Ollama Model Storage Headroom and Cache Eviction Sentinel

id: ollama-model-storage-headroom-and-prune-sentinel
namespace: company.ai

description: |
  Audit an Ollama local model registry for disk storage consumption and headroom exhaustion.
  On schedule or webhook, the flow queries the daemon for installed models, parameter sizes,
  and currently active VRAM workloads, calculates total gigabytes consumed against a ceiling threshold,
  flags capacity breaches to Slack, and optionally prunes stale unpinned model weights before
  worker disks crash.

triggers:
  - id: nightly_headroom_scan
    type: io.kestra.plugin.core.trigger.Schedule
    description: Nightly capacity audit running at 03:00 UTC, shipped disabled so
      nothing runs until an operator enables it.
    cron: "0 3 * * *"
    disabled: true

  - id: on_demand_audit
    type: io.kestra.plugin.core.trigger.Webhook
    description: Authenticated webhook endpoint allowing ChatOps bots, CI/CD
      pipelines, or alerts to trigger an on-demand audit.
    key: "{{ secret('OLLAMA_WEBHOOK_KEY') }}"

inputs:
  - id: ollama_host
    type: STRING
    defaults: "http://localhost:11434"
    description: Base URL of the Ollama API service (e.g., http://localhost:11434 or
      http://host.docker.internal:11434).

  - id: max_disk_usage_gb
    type: FLOAT
    defaults: 50.0
    description: Maximum tolerable disk footprint in gigabytes consumed by Ollama
      models before triggering a capacity breach alert.

  - id: min_retention_days
    type: INT
    defaults: 7
    description: Minimum age in days before an unpinned, inactive model qualifies as
      an eviction candidate.

  - id: protected_models
    type: ARRAY
    itemType: STRING
    defaults:
      - "llama3.2:latest"
      - "nomic-embed-text:latest"
    description: Pinned models that are permanently protected and must never be
      deleted by automated eviction.

  - id: auto_prune
    type: BOOL
    defaults: false
    description: When true, automatically deletes candidate models via the Ollama
      DELETE API until storage is back below the threshold. Shipped false for
      safety.

tasks:
  - id: audit_workspace
    type: io.kestra.plugin.core.flow.WorkingDirectory
    description: Shared workspace isolating query results, capacity calculations,
      and generated audit reports.
    tasks:
      - id: probe_ollama
        type: io.kestra.plugin.scripts.shell.Commands
        description: Probe the Ollama daemon for installed models and running processes.
          The task fails when the endpoint doesn't answer. The curl image runs
          as a non-root user, so the runner starts it as root to let it write
          tags.json and ps.json.
        taskRunner:
          type: io.kestra.plugin.scripts.runner.docker.Docker
          entryPoint: []
          user: root
        containerImage: curlimages/curl:8.9.1
        env:
          OLLAMA_HOST: "{{ inputs.ollama_host }}"
        outputFiles:
          - tags.json
          - ps.json
        commands:
          - |
            set -eu
            echo "Querying Ollama tags from ${OLLAMA_HOST}/api/tags"
            curl -fsS --max-time 10 "${OLLAMA_HOST}/api/tags" > tags.json

            echo "Querying active Ollama models from ${OLLAMA_HOST}/api/ps"
            curl -fsS --max-time 10 "${OLLAMA_HOST}/api/ps" > ps.json

            echo "Tags payload size: $(wc -c < tags.json) bytes"
            echo "PS payload size: $(wc -c < ps.json) bytes"

      - id: analyze_capacity
        type: io.kestra.plugin.scripts.python.Script
        description: Calculate total model footprint, cross-reference running workloads
          and pinned lists, and emit structured outputs.
        taskRunner:
          type: io.kestra.plugin.scripts.runner.docker.Docker
        containerImage: python:3.12-slim
        env:
          MAX_DISK_GB: "{{ inputs.max_disk_usage_gb }}"
          MIN_RETENTION_DAYS: "{{ inputs.min_retention_days }}"
          PROTECTED_MODELS: "{{ inputs.protected_models | toJson }}"
        inputFiles:
          tags.json: "{{ outputs.probe_ollama.outputFiles['tags.json'] }}"
          ps.json: "{{ outputs.probe_ollama.outputFiles['ps.json'] }}"
        outputFiles:
          - ollama-storage-report.txt
          - prune-plan.json
        script: |
          import json
          import os
          from datetime import datetime, timezone, timedelta

          def parse_iso(ts):
              if not ts:
                  return datetime.min.replace(tzinfo=timezone.utc)
              ts = ts.split(".")[0]
              if ts.endswith("Z"):
                  ts = ts[:-1]
              try:
                  return datetime.fromisoformat(ts).replace(tzinfo=timezone.utc)
              except Exception:
                  return datetime.min.replace(tzinfo=timezone.utc)

          max_disk_gb = float(os.environ.get("MAX_DISK_GB", "50.0"))
          min_retention_days = int(os.environ.get("MIN_RETENTION_DAYS", "7"))
          raw_protected = os.environ.get("PROTECTED_MODELS", "[]")

          try:
              protected_models = set(json.loads(raw_protected))
          except Exception:
              protected_models = set()

          with open("tags.json", "r") as f:
              tags_data = json.load(f)

          with open("ps.json", "r") as f:
              ps_data = json.load(f)

          installed_models = tags_data.get("models", [])
          active_models = {m.get("name") for m in ps_data.get("models", []) if m.get("name")}

          total_bytes = sum(m.get("size", 0) for m in installed_models)
          total_gb = round(total_bytes / (1024 ** 3), 2)
          excess_gb = round(max(0.0, total_gb - max_disk_gb), 2)

          now = datetime.now(timezone.utc)
          retention_cutoff = now - timedelta(days=min_retention_days)

          model_records = []
          for m in installed_models:
              name = m.get("name", "unknown")
              size_bytes = m.get("size", 0)
              size_gb = round(size_bytes / (1024 ** 3), 2)
              mod_time = parse_iso(m.get("modified_at"))
              is_active = name in active_models
              is_protected = name in protected_models
              is_stale = mod_time < retention_cutoff

              can_evict = (not is_active) and (not is_protected) and is_stale

              model_records.append({
                  "name": name,
                  "size_gb": size_gb,
                  "size_bytes": size_bytes,
                  "modified_at": m.get("modified_at", ""),
                  "is_active": is_active,
                  "is_protected": is_protected,
                  "can_evict": can_evict,
              })

          model_records.sort(key=lambda x: parse_iso(x["modified_at"]))

          prune_candidates = []
          reclaimed_accumulator = 0.0
          for m in model_records:
              if m["can_evict"]:
                  prune_candidates.append({
                      "name": m["name"],
                      "size_gb": m["size_gb"],
                      "modified_at": m["modified_at"]
                  })
                  reclaimed_accumulator += m["size_gb"]
                  if (total_gb - reclaimed_accumulator) <= max_disk_gb:
                      break

          verdict = "CAPACITY_BREACH" if total_gb > max_disk_gb else "HEALTHY"

          with open("prune-plan.json", "w") as f:
              json.dump(prune_candidates, f, indent=2)

          with open("ollama-storage-report.txt", "w") as f:
              f.write("=== Ollama Model Storage Capacity Report ===\n")
              f.write(f"Generated at: {now.strftime('%Y-%m-%d %H:%M:%S UTC')}\n")
              f.write(f"Verdict: {verdict}\n")
              f.write(f"Total Models Installed: {len(installed_models)}\n")
              f.write(f"Currently Active in VRAM: {len(active_models)}\n")
              f.write(f"Total Disk Usage: {total_gb} GB\n")
              f.write(f"Configured Headroom Ceiling: {max_disk_gb} GB\n")
              f.write(f"Excess Storage: {excess_gb} GB\n\n")
              f.write("Installed Models Inventory:\n")
              for m in model_records:
                  status_flag = "[ACTIVE]" if m["is_active"] else ("[PROTECTED]" if m["is_protected"] else ("[CANDIDATE]" if m["can_evict"] else "[RECENT]"))
                  f.write(f"  - {m['name']} ({m['size_gb']} GB) {status_flag}\n")
              if prune_candidates:
                  f.write("\nEviction Candidates for Reclaim:\n")
                  for c in prune_candidates:
                      f.write(f"  * {c['name']} ({c['size_gb']} GB)\n")

          print(f"Capacity audit completed: total_gb={total_gb}, ceiling={max_disk_gb}, verdict={verdict}")
          print("::" + json.dumps({
              "outputs": {
                  "verdict": verdict,
                  "total_disk_gb": total_gb,
                  "model_count": len(installed_models),
                  "active_model_count": len(active_models),
                  "excess_disk_gb": excess_gb,
                  "candidate_count": len(prune_candidates),
                  "prune_candidates": prune_candidates
              }
          }) + "::")

  - id: evaluate_headroom
    type: io.kestra.plugin.core.flow.If
    description: Escalate and remediate whenever model disk usage breaches the
      configured ceiling.
    condition: "{{ outputs.analyze_capacity.vars.verdict == 'CAPACITY_BREACH' }}"
    then:
      - id: check_auto_prune
        type: io.kestra.plugin.core.flow.If
        description: Check if the operator authorized automatic weight eviction.
        condition: "{{ inputs.auto_prune }}"
        then:
          - id: prune_excess_models
            type: io.kestra.plugin.core.flow.Loop
            description: Loop through identified stale candidate models and purge them via
              the Ollama DELETE API.
            values: "{{ outputs.analyze_capacity.vars.prune_candidates }}"
            concurrencyLimit: 1
            tasks:
              - id: delete_model
                type: io.kestra.plugin.scripts.shell.Commands
                description: Execute DELETE /api/delete against Ollama to reclaim host storage.
                taskRunner:
                  type: io.kestra.plugin.scripts.runner.docker.Docker
                  entryPoint: []
                containerImage: curlimages/curl:8.9.1
                env:
                  OLLAMA_HOST: "{{ inputs.ollama_host }}"
                  MODEL_NAME: "{{ fromJson(item.value).name }}"
                commands:
                  - |
                    set -u
                    echo "Reclaiming disk: deleting unpinned model ${MODEL_NAME} from ${OLLAMA_HOST}"
                    curl -fsS -X DELETE "${OLLAMA_HOST}/api/delete" \
                      -H "Content-Type: application/json" \
                      -d "{\"name\": \"${MODEL_NAME}\"}"

        else:
          - id: log_prune_disabled
            type: io.kestra.plugin.core.log.Log
            message: "Auto-prune is disabled (auto_prune=false). Identified {{
              outputs.analyze_capacity.vars.candidate_count }} eviction
              candidates but no deletions were performed."

      - id: notify_slack_alert
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Post high-priority capacity breach details to Slack.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "text": "⚠️ *Ollama Storage Capacity Alert*: Host model cache has reached *{{ outputs.analyze_capacity.vars.total_disk_gb }} GB*, exceeding the *{{ inputs.max_disk_usage_gb }} GB* ceiling (excess: {{ outputs.analyze_capacity.vars.excess_disk_gb }} GB). Auto-prune is *{{ inputs.auto_prune ? 'ENABLED' : 'DISABLED' }}*. Candidates identified: {{ outputs.analyze_capacity.vars.candidate_count }}."
          }

    else:
      - id: log_healthy
        type: io.kestra.plugin.core.log.Log
        description: Log nominal storage capacity when disk usage stays within headroom
          thresholds.
        message: "Ollama storage healthy: {{ outputs.analyze_capacity.vars.total_disk_gb
          }} GB consumed across {{ outputs.analyze_capacity.vars.model_count }}
          models (ceiling: {{ inputs.max_disk_usage_gb }} GB)."

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alert Slack if any query or analysis task unexpectedly fails.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "❌ *Ollama Capacity Sentinel Failed*: Execution {{ execution.id }} failed during storage evaluation. Check worker logs immediately."
      }

outputs:
  - id: verdict
    type: STRING
    description: Capacity evaluation verdict (CAPACITY_BREACH or HEALTHY).
    value: "{{ outputs.analyze_capacity.vars.verdict }}"

  - id: total_disk_gb
    type: FLOAT
    description: Total gigabytes consumed by all installed Ollama model files.
    value: "{{ outputs.analyze_capacity.vars.total_disk_gb }}"

  - id: model_count
    type: INT
    description: Count of installed models found on the Ollama host.
    value: "{{ outputs.analyze_capacity.vars.model_count }}"

  - id: excess_disk_gb
    type: FLOAT
    description: Gigabytes exceeding the maximum allowed storage threshold.
    value: "{{ outputs.analyze_capacity.vars.excess_disk_gb }}"

  - id: report_uri
    type: STRING
    description: Internal storage URI of the generated ollama-storage-report.txt report.
    value: "{{ outputs.analyze_capacity.outputFiles['ollama-storage-report.txt'] }}"