id: dagster-missing-partition-backfill
namespace: company.team
description: >-
Reconcile the materialized partitions of one Dagster asset. The flow reads the
asset's materialization events through GraphQL, reports missing keys in safe
mode and only launches one run per missing key after explicit approval. The
final GraphQL read is the success gate.
inputs:
- id: dagster_base_url
type: STRING
defaults: "{{ secret('DAGSTER_GRAPHQL_URL') }}"
description: Dagster GraphQL endpoint reachable by the Kestra worker.
- id: job_name
type: STRING
defaults: partition_backfill_job
description: Dagster job that materializes one requested partition per run.
- id: repository
type: STRING
defaults: repository_name
description: Dagster repository containing job_name.
- id: location
type: STRING
defaults: code_location
description: Dagster code location containing repository.
- id: asset_key
type: STRING
defaults: daily_orders
description: Exact Dagster asset key whose partition coverage is reconciled.
- id: partition_keys
type: ARRAY
itemType: STRING
defaults:
- "2026-10-14"
- "2026-10-15"
- "2026-10-16"
- "2026-10-17"
description: Expected partition keys for the bounded reconciliation window.
- id: protected_partition_keys
type: ARRAY
itemType: STRING
defaults: []
description: Expected keys to report but never launch a run for.
- id: partition_mode
type: STRING
defaults: materialize
description: Job fixture mode. Production jobs can ignore this input; the local
proof uses noop to prove an ineffective write.
- id: auto_remediate
type: BOOL
defaults: false
description: Arm the approved remediation branch.
- id: dry_run
type: BOOL
defaults: true
description: Keep true to report gaps without launching Dagster runs.
triggers:
- id: daily_partition_reconciliation
type: io.kestra.plugin.core.trigger.Schedule
description: Disabled by default and pinned to safe mode until the asset and job
have been verified manually.
cron: "0 4 * * *"
disabled: true
inputs:
auto_remediate: false
dry_run: true
tasks:
- id: read_status
type: io.kestra.plugin.core.http.Request
description: Read exact asset materialization events from Dagster's GraphQL API
before planning.
uri: "{{ inputs.dagster_base_url }}"
method: POST
contentType: application/json
headers:
Content-Type: application/json
body: |
{
"query": "query Materializations($assetKey: AssetKeyInput!) { assetOrError(assetKey: $assetKey) { __typename ... on Asset { assetMaterializations(limit: 100) { partition runId } } } }",
"variables": {"assetKey": {"path": ["{{ inputs.asset_key }}"]}}
}
options:
timeout:
connectTimeout: PT10S
readIdleTimeout: PT30S
- id: build_plan
type: io.kestra.plugin.core.output.OutputValues
description: Compute the target set from Dagster's response. Protected keys and
an exact asset match stay out of the write plan.
values:
plan: >-
{{ {'r': fromJson(outputs.read_status.body), 'expected':
inputs.partition_keys, 'protected': inputs.protected_partition_keys,
'armed': (inputs.auto_remediate and not inputs.dry_run)} | jq('
.r.data.assetOrError as $a
| ($a.assetMaterializations // [] | map(select(.partition != null) | .partition) | unique) as $have
| (.expected // []) as $expected
| (.protected // []) as $protected
| [ $expected[] as $p | select(($protected | index($p)) == null) | $p ] as $scope
| [ $scope[] as $p | select(($have | index($p)) == null) | $p ] as $missing
| {typename: ($a.__typename // "UNKNOWN"), materialized: $have, expected: $expected,
protected: $protected, scope: $scope, missing: $missing,
spared: [ $expected[] as $p | select(($protected | index($p)) != null) | $p ],
armed: .armed, count: ($missing | length)}
') | first | toJson }}
- id: log_plan
type: io.kestra.plugin.core.log.Log
message: >-
Dagster asset {{ inputs.asset_key }} coverage: {{
fromJson(outputs.build_plan.values.plan) | jq('.count') | first }}
missing, {{ fromJson(outputs.build_plan.values.plan) | jq('.materialized |
length') | first }} materialized, {{
fromJson(outputs.build_plan.values.plan) | jq('.spared | length') | first
}} protected. Missing keys: {{ fromJson(outputs.build_plan.values.plan) |
jq('if (.missing | length) == 0 then "none" else (.missing | join(", "))
end') | first }}. dry_run={{ inputs.dry_run }} auto_remediate={{
inputs.auto_remediate }}.
- id: validate_asset
type: io.kestra.plugin.core.flow.If
condition: "{{ fromJson(outputs.build_plan.values.plan) | jq('.typename !=
\"Asset\"') | first }}"
then:
- id: asset_read_failed
type: io.kestra.plugin.core.execution.Fail
errorMessage: >-
Dagster did not return the requested asset {{ inputs.asset_key }}.
GraphQL typename was {{ fromJson(outputs.build_plan.values.plan) |
jq('.typename') | first }}. No run was launched.
- id: findings
type: io.kestra.plugin.core.flow.If
condition: "{{ fromJson(outputs.build_plan.values.plan) | jq('.count > 0') | first }}"
then:
- id: alert_findings
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alert on every detected gap so a dry run is visible to the on-call
team.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: >-
{{ {'text': 'Dagster partition gap on ' ~ inputs.asset_key ~ ': ' ~
(fromJson(outputs.build_plan.values.plan) | jq('.missing | join(",
")') | first) ~ '. Protected: ' ~
(fromJson(outputs.build_plan.values.plan) | jq('if (.spared | length)
== 0 then "none" else (.spared | join(", ")) end') | first) ~ '.
dry_run=' ~ inputs.dry_run ~ ' auto_remediate=' ~
inputs.auto_remediate ~ '.'} | toJson }}
- id: remediation_gate
type: io.kestra.plugin.core.flow.If
description: Only armed non-dry runs reach the approval gate.
condition: "{{ inputs.auto_remediate and not inputs.dry_run }}"
then:
- id: approval
type: io.kestra.plugin.core.flow.Pause
description: A human must approve launching one Dagster run for each currently
missing partition.
pauseDuration: PT2H
behavior: FAIL
onResume:
- id: approved
type: BOOL
defaults: false
description: Approve the listed partition runs.
- id: note
type: STRING
defaults: ""
description: Optional approval note.
- id: apply_if_approved
type: io.kestra.plugin.core.flow.If
condition: "{{ outputs.approval.onResume.approved ?? false }}"
then:
- id: reread_after_approval
type: io.kestra.plugin.core.http.Request
description: Re-read eligibility after approval so a stale plan cannot launch a
duplicate or overwrite a recovered partition.
uri: "{{ inputs.dagster_base_url }}"
method: POST
contentType: application/json
headers:
Content-Type: application/json
body: |
{
"query": "query Materializations($assetKey: AssetKeyInput!) { assetOrError(assetKey: $assetKey) { __typename ... on Asset { assetMaterializations(limit: 100) { partition runId } } } }",
"variables": {"assetKey": {"path": ["{{ inputs.asset_key }}"]}}
}
options:
timeout:
connectTimeout: PT10S
readIdleTimeout: PT30S
- id: recheck_plan
type: io.kestra.plugin.core.output.OutputValues
description: Skip keys filled during approval and fail before writing if an
earlier materialization disappeared.
values:
items: >-
{{ {'a': fromJson(outputs.build_plan.values.plan), 'r':
fromJson(outputs.reread_after_approval.body)} | jq('
.a as $a
| (.r.data.assetOrError.assetMaterializations // [] | map(select(.partition != null) | .partition) | unique) as $now
| [ $a.materialized[] as $p | select(($now | index($p)) == null) | $p ] as $lost
| [ $a.missing[] as $p | select(($now | index($p)) == null) | $p ] as $eligible
| [ $a.missing[] as $p | select(($now | index($p)) != null) | $p ] as $skipped
| {now: $now, eligible: $eligible, skipped: $skipped, lost: $lost,
note: ("After approval, " + ($eligible | length | tostring) + " missing key(s) remain eligible; "
+ (if ($skipped | length) == 0 then "none were filled during approval" else "skipped keys filled during approval: " + ($skipped | join(", ")) end)
+ ". Previously materialized keys lost during approval: " + (if ($lost | length) == 0 then "none" else ($lost | join(", ")) end) + ".")}
') | first | toJson }}
- id: recheck_log
type: io.kestra.plugin.core.log.Log
message: "{{ fromJson(outputs.recheck_plan.values.items) | jq('.note') | first
}}"
- id: stale_plan
type: io.kestra.plugin.core.flow.If
condition: "{{ fromJson(outputs.recheck_plan.values.items) | jq('.lost | length
> 0') | first }}"
then:
- id: stale_plan_failed
type: io.kestra.plugin.core.execution.Fail
errorMessage: >-
Refusing to write because materialized partition(s)
disappeared during approval: {{
fromJson(outputs.recheck_plan.values.items) | jq('.lost |
join(", ")') | first }}.
- id: launch_missing
type: io.kestra.plugin.core.flow.If
condition: "{{ fromJson(outputs.recheck_plan.values.items) | jq('.eligible |
length > 0') | first }}"
then:
- id: launch_one_partition
type: io.kestra.plugin.core.flow.Loop
description: Launch each approved partition sequentially. The loop never runs
two Dagster jobs at once.
concurrencyLimit: 1
values: "{{ fromJson(outputs.recheck_plan.values.items) | jq('.eligible') |
first | toJson }}"
tasks:
- id: trigger_partition
type: io.kestra.plugin.dagster.TriggerRun
description: Trigger and wait for the Dagster job that materializes this one
partition.
baseUrl: "{{ inputs.dagster_base_url }}"
jobName: "{{ inputs.job_name }}"
repository: "{{ inputs.repository }}"
location: "{{ inputs.location }}"
wait: true
pollFrequency: PT1S
maxDuration: PT10M
allowFailure: true
body:
runConfig:
ops:
materialize_partition:
config:
partition_key: "{{ item.value }}"
asset_key: "{{ inputs.asset_key }}"
mode: "{{ inputs.partition_mode }}"
- id: verify_status
type: io.kestra.plugin.core.http.Request
description: Query Dagster again after all launches. This read determines
whether the gap really closed.
uri: "{{ inputs.dagster_base_url }}"
method: POST
contentType: application/json
headers:
Content-Type: application/json
body: |
{
"query": "query Materializations($assetKey: AssetKeyInput!) { assetOrError(assetKey: $assetKey) { __typename ... on Asset { assetMaterializations(limit: 100) { partition runId } } } }",
"variables": {"assetKey": {"path": ["{{ inputs.asset_key }}"]}}
}
options:
timeout:
connectTimeout: PT10S
readIdleTimeout: PT30S
- id: verify_plan
type: io.kestra.plugin.core.output.OutputValues
values:
result: >-
{{ {'r': fromJson(outputs.verify_status.body), 'expected':
inputs.partition_keys, 'protected':
inputs.protected_partition_keys} | jq('
.r.data.assetOrError as $a
| ($a.assetMaterializations // [] | map(select(.partition != null) | .partition) | unique) as $have
| [ (.expected // [])[] as $p | select((.protected // []) | index($p) | not) | select(($have | index($p)) == null) | $p ] as $missing
| {typename: ($a.__typename // "UNKNOWN"), materialized: $have, missing: $missing, count: ($missing | length)}
') | first | toJson }}
- id: fail_if_gap_remains
type: io.kestra.plugin.core.flow.If
condition: "{{ fromJson(outputs.verify_plan.values.result) | jq('.count > 0') |
first }}"
then:
- id: ineffective_write
type: io.kestra.plugin.core.execution.Fail
errorMessage: >-
Dagster remediation did not close the partition gap. Still
missing: {{ fromJson(outputs.verify_plan.values.result) |
jq('.missing | join(", ")') | first }}.
else:
- id: remediated
type: io.kestra.plugin.core.log.Log
message: >-
Dagster partition coverage recovered. Final materialized
count: {{ fromJson(outputs.verify_plan.values.result) |
jq('.materialized | length') | first }}.
else:
- id: declined
type: io.kestra.plugin.core.log.Log
message: "Approval declined{{ (outputs.approval.onResume.note ?? '') == '' ? ''
: ': ' ~ outputs.approval.onResume.note }}. No Dagster run was
launched and the detected gap remains."
else:
- id: dry_run_log
type: io.kestra.plugin.core.log.Log
message: "Dry run or auto remediation disabled. Would launch one Dagster run for
each missing key: {{ fromJson(outputs.build_plan.values.plan) |
jq('.missing | join(\", \")') | first }}."
else:
- id: healthy
type: io.kestra.plugin.core.log.Log
message: "Dagster asset {{ inputs.asset_key }} has complete coverage for the
requested window. No run launched."
errors:
- id: alert_flow_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alert when the reconciliation flow fails before proving coverage.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: >-
{{ {'text': 'Dagster missing partition backfill failed for asset ' ~
inputs.asset_key ~ ' in execution ' ~ execution.id ~ '. Final coverage was
not proven; inspect the execution before retrying.'} | toJson }}