Portainer Container Crash-Loop Sentinel

id: portainer-container-crash-loop-sentinel
namespace: company.team

description: |
  Authenticate against a Portainer instance on a schedule, list every
  container on a given Docker endpoint, classify the environment as
  HEALTHY, AT_RISK, CRITICAL, or UNREACHABLE based on how many
  containers are stuck restarting or have exited, log clean checks,
  and alert Slack the moment a container enters a crash loop or
  Portainer itself cannot be reached.

inputs:
  - id: portainer_url
    type: STRING
    defaults: "http://localhost:9000"
    displayName: "Portainer Base URL"
    description: "Base URL of the Portainer instance, no trailing slash."
  - id: endpoint_id
    type: INT
    defaults: 1
    displayName: "Portainer Endpoint ID"
    description: "ID of the Docker environment (endpoint) to list containers for, as
      shown under Environments in Portainer."

triggers:
  - id: container_health_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    description: "Check every Docker environment's container states every 5 minutes."
    cron: "*/5 * * * *"
    disabled: true
    inputs:
      portainer_url: "http://localhost:9000"
      endpoint_id: 1

tasks:
  - id: login_to_portainer
    type: io.kestra.plugin.core.http.Request
    description: "Authenticate against the Portainer REST API to obtain the JWT used
      by every subsequent call."
    uri: "{{ inputs.portainer_url }}/api/auth"
    method: POST
    contentType: application/json
    body: |
      {
        "Username": "{{ secret('PORTAINER_USERNAME') }}",
        "Password": "{{ secret('PORTAINER_PASSWORD') }}"
      }
    allowFailure: true

  - id: list_containers
    type: io.kestra.plugin.core.http.Request
    description: "List every container (running and stopped) on the target Docker
      endpoint via Portainer's Docker API proxy."
    uri: "{{ inputs.portainer_url }}/api/endpoints/{{ inputs.endpoint_id
      }}/docker/containers/json?all=true"
    method: GET
    headers:
      Authorization: "{{ outputs.login_to_portainer.body is defined ? ('Bearer ' ~
        (outputs.login_to_portainer.body | jq('.jwt') | first | default(''))) :
        '' }}"
    allowFailure: true

  - id: classify_container_health
    type: io.kestra.plugin.core.debug.Return
    description: "Classify UNREACHABLE when the container list could not be fetched,
      CRITICAL when at least one container is stuck in the restarting state,
      AT_RISK when at least one container has exited, otherwise HEALTHY."
    format: "{{ outputs.list_containers.body is defined ?
      (((outputs.list_containers.body | jq('[.[] | select(.State ==
      \"restarting\")] | length') | first | default(0)) > 0) ? 'CRITICAL' :
      (((outputs.list_containers.body | jq('[.[] | select(.State == \"exited\")]
      | length') | first | default(0)) > 0) ? 'AT_RISK' : 'HEALTHY')) :
      'UNREACHABLE' }}"

  - id: route_by_container_health
    type: io.kestra.plugin.core.flow.Switch
    description: "Log a quiet confirmation when every container is healthy, page
      Slack when a container is crash-looping or has exited, or page Slack when
      Portainer could not be reached."
    value: "{{ outputs.classify_container_health.value }}"
    cases:
      "HEALTHY":
        - id: log_healthy
          type: io.kestra.plugin.core.log.Log
          message: "All containers on endpoint {{ inputs.endpoint_id }} are healthy: none
            restarting or exited."
      "AT_RISK":
        - id: alert_at_risk
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "text": "Portainer sentinel AT_RISK on endpoint {{ inputs.endpoint_id }}: exited container(s): {{ outputs.list_containers.body | jq('[.[] | select(.State == \"exited\") | (.Names[0] // .Id)] | join(\", \")') | first }}. Execution {{ execution.id }}."
            }
      "CRITICAL":
        - id: alert_critical
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "text": "Portainer sentinel CRITICAL on endpoint {{ inputs.endpoint_id }}: container(s) stuck restarting: {{ outputs.list_containers.body | jq('[.[] | select(.State == \"restarting\") | (.Names[0] // .Id)] | join(\", \")') | first }}. Execution {{ execution.id }}."
            }
      "UNREACHABLE":
        - id: alert_unreachable
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "text": "Portainer sentinel UNREACHABLE for endpoint {{ inputs.endpoint_id }}: the container list could not be fetched (login failure, invalid credentials, or Portainer/the Docker endpoint is down). Execution {{ execution.id }}."
            }
    defaults:
      - id: log_unclassified
        type: io.kestra.plugin.core.log.Log
        message: "Completed Portainer container health check with status: {{
          outputs.classify_container_health.value }}"

  - id: log_audit_status
    type: io.kestra.plugin.core.log.Log
    description: "Post-check status line recorded on every run regardless of branch,
      for a continuous container-health trend in the execution history."
    message: "Portainer audit: endpoint={{ inputs.endpoint_id }}, status={{
      outputs.classify_container_health.value }}, execution={{ execution.id }}."

errors:
  - id: alert_flow_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: "Alert Slack when the sentinel flow itself fails, so a broken check
      never silently stops monitoring container health."
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "Portainer container crash-loop sentinel flow {{ flow.id }} FAILED (execution {{ execution.id }}). Container health for endpoint {{ inputs.endpoint_id }} is now unknown; check the task logs and the Portainer UI manually."
      }

outputs:
  - id: status
    type: STRING
    value: "{{ outputs.classify_container_health.value }}"