id: dhis2-scheduled-job-failure-sentinel
namespace: company.team
description: |
On a schedule, check the background jobs of a DHIS2 health information
system (analytics tables, data synchronisation, imports, notifications) and
classify them as HEALTHY, WARNING, CRITICAL or UNREACHABLE. Log quiet runs,
page Slack when jobs fail or stay stuck, then pause for someone to approve
re-running the failed jobs through the DHIS2 API. Nothing is run without
that approval.
inputs:
- id: dhis2_url
type: STRING
defaults: "https://dhis2.example.org"
displayName: "DHIS2 URL"
description: "Base URL of the DHIS2 instance (the one that serves /api), without
a trailing slash."
- id: lookback_hours
type: INT
defaults: 24
displayName: "Lookback (hours)"
description: "Jobs whose last run failed within this many hours, by the server's
own clock, are counted."
- id: stuck_hours
type: INT
defaults: 6
displayName: "Stuck After (hours)"
description: "A job still RUNNING this many hours after its last start counts as stuck."
- id: critical_failed
type: INT
defaults: 3
displayName: "Critical Failed Jobs"
description: "At or above this many failed jobs in the lookback window, the
check is classified CRITICAL. A failed enabled scheduled job is always
CRITICAL."
- id: approval_wait
type: DURATION
defaults: PT2H
displayName: "Retry Approval Window"
description: "How long the execution waits for someone to approve re-running the
failed jobs. With no answer in time, nothing is re-run."
- id: max_retry_jobs
type: INT
defaults: 5
displayName: "Max Jobs to Retry"
description: "At most this many failed jobs, most recent first, are offered for
a retry in one run."
triggers:
- id: job_check
type: io.kestra.plugin.core.trigger.Schedule
description: Check DHIS2 jobs every hour. Shipped disabled; enable it once
dhis2_url and the DHIS2_USERNAME and DHIS2_PASSWORD secrets point at a
real instance and a manual run has been verified.
cron: "20 * * * *"
disabled: true
inputs:
dhis2_url: "https://dhis2.example.org"
tasks:
- id: read_system_info
type: io.kestra.plugin.core.http.Request
description: Read /api/system/info to confirm the endpoint is DHIS2 and to get
the server's own date, which every time window below is measured against.
This is a GET; nothing is changed.
uri: "{{ inputs.dhis2_url }}/api/system/info"
method: GET
headers:
Accept: application/json
options:
auth:
type: BASIC
username: "{{ secret('DHIS2_USERNAME') }}"
password: "{{ secret('DHIS2_PASSWORD') }}"
allowFailed: true
timeout:
connectTimeout: PT10S
readIdleTimeout: PT30S
retry:
type: constant
interval: PT15S
maxAttempts: 3
allowFailure: true
- id: fetch_job_configurations
type: io.kestra.plugin.core.http.Request
description: Read every job configuration with its type, scheduling, status,
last run and last result from /api/jobConfigurations. Skipped when
/system/info did not answer.
runIf: "{{ outputs.read_system_info.code is defined and
outputs.read_system_info.code == 200 and (outputs.read_system_info.body |
trim | startsWith('{')) }}"
uri: "{{ inputs.dhis2_url
}}/api/jobConfigurations?paging=false&fields=id,name,jobType,enabled,sche\
dulingType,jobStatus,lastExecutedStatus,lastExecuted"
method: GET
headers:
Accept: application/json
options:
auth:
type: BASIC
username: "{{ secret('DHIS2_USERNAME') }}"
password: "{{ secret('DHIS2_PASSWORD') }}"
allowFailed: true
timeout:
connectTimeout: PT10S
readIdleTimeout: PT60S
retry:
type: constant
interval: PT15S
maxAttempts: 3
allowFailure: true
- id: classify_jobs
type: io.kestra.plugin.core.debug.Return
description: Classify the jobs in one jq query. DHIS2 returns times in the
server's local time without an offset, so the lookback and stuck windows
are measured against the server's own serverDate rather than Kestra's
clock. A missing response, a non-200 code, a body that is not JSON, or an
answer without a DHIS2 version become UNREACHABLE instead of looking
healthy.
format: >-
{{ (outputs.read_system_info.code is not defined or
outputs.read_system_info.code != 200 or not (outputs.read_system_info.body
| trim | startsWith('{')) or outputs.fetch_job_configurations is not
defined or outputs.fetch_job_configurations.code is not defined or
outputs.fetch_job_configurations.code != 200 or not
(outputs.fetch_job_configurations.body | trim | startsWith('{'))) ?
'{"status":"UNREACHABLE","jobs":0,"failed":0,"failed_scheduled":0,"stuck":0,"failed_jobs":"none","stuck_jobs":"none","retry_ids":[],"retry_jobs":"none"}'
: ({'sys': fromJson(outputs.read_system_info.body), 'cfg':
fromJson(outputs.fetch_job_configurations.body)} | jq(' if (.sys.version
// "") == "" or (.cfg.jobConfigurations | type) != "array" then {status:
"UNREACHABLE", jobs: 0, failed: 0, failed_scheduled: 0, stuck: 0,
failed_jobs: "none", stuck_jobs: "none", retry_ids: [], retry_jobs:
"none"} else def epoch: .[0:19] + "Z" | fromdateiso8601; (.sys.serverDate
| epoch) as $now | ($now - ' ~ (inputs.lookback_hours * 3600) ~ ') as
$since | ($now - ' ~ (inputs.stuck_hours * 3600) ~ ') as $stuck_before |
.cfg.jobConfigurations as $jobs | [$jobs[] | select(.lastExecutedStatus ==
"FAILED" and .lastExecuted != null and (.lastExecuted | epoch) >= $since)]
as $failed | [$failed[] | select(.enabled == true and (.schedulingType ==
"CRON" or .schedulingType == "FIXED_DELAY"))] as $failed_scheduled |
[$jobs[] | select(.jobStatus == "RUNNING" and .lastExecuted != null and
(.lastExecuted | epoch) < $stuck_before)] as $stuck | ($failed |
sort_by(.lastExecuted) | reverse | .[0:' ~ inputs.max_retry_jobs ~ ']) as
$retry | {dhis2_version: .sys.version, server_date: .sys.serverDate, jobs:
($jobs | length), failed: ($failed | length), failed_scheduled:
($failed_scheduled | length), stuck: ($stuck | length), failed_jobs: (if
($failed | length) == 0 then "none" else ([$failed[] | .jobType] |
group_by(.) | map(.[0] + " (" + (length | tostring) + ")") | join(", "))
end), failed_scheduled_jobs: (if ($failed_scheduled | length) == 0 then
"none" else ([$failed_scheduled[] | .name] | join(", ")) end), stuck_jobs:
(if ($stuck | length) == 0 then "none" else ([$stuck[] | .name + " since "
+ .lastExecuted[0:16]] | join(", ")) end), retry_ids: [$retry[] | .id],
retry_jobs: (if ($retry | length) == 0 then "none" else ([$retry[] | .name
+ " (" + .id + ")"] | join(", ")) end), status: (if ($failed_scheduled |
length) > 0 or ($stuck | length) > 0 or ($failed | length) >= ' ~
inputs.critical_failed ~ ' then "CRITICAL" elif ($failed | length) > 0
then "WARNING" else "HEALTHY" end)} end ') | first | toJson) }}
- id: route_by_job_status
type: io.kestra.plugin.core.flow.Switch
description: Route to a quiet log when no job failed in the window and none is
stuck, an informational Slack message for a few failed one-off jobs, or a
Slack page when a scheduled job failed, failures pile up, a job is stuck,
or DHIS2 could not be read.
value: "{{ fromJson(outputs.classify_jobs.value).status }}"
cases:
"HEALTHY":
- id: log_healthy
type: io.kestra.plugin.core.log.Log
description: Log a quiet confirmation when no job failed in the lookback window
and none is stuck.
message: "DHIS2 jobs healthy: {{ fromJson(outputs.classify_jobs.value).jobs }}
job configuration(s), none failed in the last {{
inputs.lookback_hours }} hour(s), none stuck (DHIS2 {{
fromJson(outputs.classify_jobs.value).dhis2_version }})."
"WARNING":
- id: alert_warning_jobs
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Post an informational Slack message listing the job types that
failed.
url: "{{ secret('SLACK_WEBHOOK') }}"
payload: |
{
"text": "DHIS2 jobs WARNING: {{ fromJson(outputs.classify_jobs.value).failed }} job(s) failed in the last {{ inputs.lookback_hours }} hour(s): {{ fromJson(outputs.classify_jobs.value).failed_jobs }}. Check Data Administration > Scheduler on {{ inputs.dhis2_url }}. To re-run {{ fromJson(outputs.classify_jobs.value).retry_jobs }}, resume execution {{ execution.id }} in Kestra with retry set to true before the approval window ends."
}
"CRITICAL":
- id: alert_critical_jobs
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Page Slack when an enabled scheduled job failed, failures crossed
critical_failed, or a job is stuck; analytics and synced data may be
stale.
url: "{{ secret('SLACK_WEBHOOK') }}"
payload: |
{
"text": "DHIS2 jobs CRITICAL: {{ fromJson(outputs.classify_jobs.value).failed }} failed job(s) in the last {{ inputs.lookback_hours }} hour(s) ({{ fromJson(outputs.classify_jobs.value).failed_jobs }}).{% if fromJson(outputs.classify_jobs.value).failed_scheduled > 0 %} Scheduled jobs that failed: {{ fromJson(outputs.classify_jobs.value).failed_scheduled_jobs }}.{% endif %}{% if fromJson(outputs.classify_jobs.value).stuck > 0 %} Stuck: {{ fromJson(outputs.classify_jobs.value).stuck_jobs }}.{% endif %} Dashboards and reports may show stale data.{% if fromJson(outputs.classify_jobs.value).retry_ids | length > 0 %} To re-run {{ fromJson(outputs.classify_jobs.value).retry_jobs }}, resume execution {{ execution.id }} in Kestra with retry set to true before the approval window ends.{% else %} Execution {{ execution.id }}.{% endif %}"
}
"UNREACHABLE":
- id: alert_unreachable_dhis2
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Page Slack when DHIS2 could not be read, rejected the credentials,
or did not answer like DHIS2.
url: "{{ secret('SLACK_WEBHOOK') }}"
payload: |
{
"text": "DHIS2 jobs UNREACHABLE: {{ inputs.dhis2_url }} did not return its system information and job configurations (server down, wrong URL, rejected credentials, or a page that is not DHIS2). Job status is unknown. Execution {{ execution.id }}."
}
defaults:
- id: log_unclassified
type: io.kestra.plugin.core.log.Log
description: Log any classification outside the four known values, so a change
in the jq query never fails silently.
message: "DHIS2 scheduled job failure sentinel completed with an unexpected
classification: {{ outputs.classify_jobs.value }}."
- id: wait_for_retry_approval
type: io.kestra.plugin.core.flow.Pause
description: When failed jobs were found, pause until an on-call person resumes
the execution with retry set to true or false. If nobody answers within
approval_wait, the execution resumes on its own and nothing is re-run.
Skipped for HEALTHY, UNREACHABLE and stuck-only runs.
runIf: "{{ fromJson(outputs.classify_jobs.value).retry_ids | length > 0 }}"
pauseDuration: "{{ inputs.approval_wait }}"
behavior: RESUME
onResume:
- id: retry
type: BOOL
defaults: false
displayName: "Re-run the failed jobs"
description: "Set to true to re-run the failed jobs listed in the alert through
the DHIS2 API."
- id: retry_if_approved
type: io.kestra.plugin.core.flow.If
description: Re-run the failed jobs only when the resume answer was true. A
declined or timed-out pause (which has no answer) goes to else, so nothing
is re-run without an explicit approval.
condition: "{{ (outputs.wait_for_retry_approval.onResume.retry ?? false) == true }}"
then:
- id: retry_each_failed_job
type: io.kestra.plugin.core.flow.Loop
description: Only after an explicit approval, re-run each failed job one at a
time, most recent first, up to max_retry_jobs. A timed-out pause has
no resume answer and counts as not approved.
values: "{{ fromJson(outputs.classify_jobs.value).retry_ids | toJson }}"
concurrencyLimit: 1
tasks:
- id: run_job
type: io.kestra.plugin.core.http.Request
description: Ask DHIS2 to run the job now with POST
/api/jobConfigurations/{id}/execute, the same as Run now in the
scheduler. The job keeps its schedule.
uri: "{{ inputs.dhis2_url }}/api/jobConfigurations/{{ item.value }}/execute"
method: POST
headers:
Accept: application/json
options:
auth:
type: BASIC
username: "{{ secret('DHIS2_USERNAME') }}"
password: "{{ secret('DHIS2_PASSWORD') }}"
allowFailed: true
timeout:
connectTimeout: PT10S
readIdleTimeout: PT30S
allowFailure: true
outputs:
- id: job_id
type: STRING
description: The DHIS2 job configuration id that was re-run.
value: "{{ item.value }}"
- id: http_code
type: INT
description: The HTTP code DHIS2 answered the run request with; 0 when it did
not answer.
value: "{{ outputs.run_job.code ?? 0 }}"
- id: alert_retry_requested
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Tell Slack which jobs DHIS2 accepted for a re-run; the next check
reports whether they succeeded.
url: "{{ secret('SLACK_WEBHOOK') }}"
payload: |
{
"text": "DHIS2 retry approved: asked {{ inputs.dhis2_url }} to re-run {{ fromJson(outputs.classify_jobs.value).retry_jobs }}. Answers: {% for run in outputs.retry_each_failed_job.outputs %}{{ run.outputs.job_id }} HTTP {{ run.outputs.http_code }}{% if not loop.last %}, {% endif %}{% endfor %}. The next check shows whether they completed. Execution {{ execution.id }}."
}
else:
- id: log_retry_not_approved
type: io.kestra.plugin.core.log.Log
description: Record that a retry was offered but declined or not answered in
time, so nothing was re-run.
runIf: "{{ fromJson(outputs.classify_jobs.value).retry_ids | length > 0 }}"
message: "DHIS2 retry not approved for {{
fromJson(outputs.classify_jobs.value).retry_jobs }}; nothing was
re-run."
- id: log_audit_result
type: io.kestra.plugin.core.log.Log
description: Record one audit line for every run, whichever branch fired, with
both HTTP codes and the classification, so the check's own history stays
queryable even if Slack delivery fails.
message: "DHIS2 jobs audit: server={{ inputs.dhis2_url }} system_http={{
outputs.read_system_info.code ?? 'n/a' }} jobs_http={{
outputs.fetch_job_configurations.code ?? 'n/a' }} result={{
outputs.classify_jobs.value }} retry_approved={{
fromJson(outputs.classify_jobs.value).retry_ids | length == 0 ? 'not
offered' : (outputs.wait_for_retry_approval.onResume.retry ?? 'no answer')
}} execution={{ execution.id }}."
errors:
- id: alert_flow_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alert Slack when the sentinel flow itself fails outright, so a
broken check is never mistaken for healthy DHIS2 jobs.
url: "{{ secret('SLACK_WEBHOOK') }}"
payload: |
{
"text": "DHIS2 scheduled job failure sentinel flow {{ flow.id }} FAILED (execution {{ execution.id }}). Job status at {{ inputs.dhis2_url }} is unknown; check the task logs and the DHIS2 scheduler manually."
}
outputs:
- id: status
type: STRING
value: "{{ outputs.classify_jobs is defined ?
fromJson(outputs.classify_jobs.value).status : 'UNREACHABLE' }}"
- id: jobs
type: JSON
value: "{{ outputs.classify_jobs is defined ? outputs.classify_jobs.value :
'{\"status\":\"UNREACHABLE\"}' }}"