id: rabbitmq-retry-ladder-consumer
namespace: company.messaging
variables:
rabbitmq_host: localhost
rabbitmq_port: "5672"
rabbitmq_vhost: /
work_queue: orders.work
parking_lot_queue: orders.parking-lot
retry_delays_seconds: [ 10, 60, 600 ]
handler_url: http://localhost:8000/orders
dedupe_window: P1D
concurrency:
limit: 1
behavior: QUEUE
tasks:
- id: declare_topology
type: io.kestra.plugin.core.flow.If
description: A manual run has no trigger, so it declares the queues and stops.
Running it again changes nothing.
condition: "{{ trigger is not defined }}"
then:
- id: create_work_queue
type: io.kestra.plugin.amqp.CreateQueue
host: "{{ vars.rabbitmq_host }}"
port: "{{ vars.rabbitmq_port }}"
virtualHost: "{{ vars.rabbitmq_vhost }}"
username: "{{ secret('RABBITMQ_USERNAME') }}"
password: "{{ secret('RABBITMQ_PASSWORD') }}"
name: "{{ vars.work_queue }}"
- id: create_parking_lot_queue
type: io.kestra.plugin.amqp.CreateQueue
host: "{{ vars.rabbitmq_host }}"
port: "{{ vars.rabbitmq_port }}"
virtualHost: "{{ vars.rabbitmq_vhost }}"
username: "{{ secret('RABBITMQ_USERNAME') }}"
password: "{{ secret('RABBITMQ_PASSWORD') }}"
name: "{{ vars.parking_lot_queue }}"
- id: create_retry_queues
type: io.kestra.plugin.core.flow.Loop
values: "{{ vars.retry_delays_seconds }}"
tasks:
- id: create_retry_queue
type: io.kestra.plugin.amqp.CreateQueue
description: |
Messages wait here for the delay, then expire and are dead-lettered through the default exchange straight into the work queue, so other queues bound to the producer's exchange never see the retry.
The arguments are one Pebble expression because RabbitMQ rejects a TTL rendered as a string.
host: "{{ vars.rabbitmq_host }}"
port: "{{ vars.rabbitmq_port }}"
virtualHost: "{{ vars.rabbitmq_vhost }}"
username: "{{ secret('RABBITMQ_USERNAME') }}"
password: "{{ secret('RABBITMQ_PASSWORD') }}"
name: "{{ vars.work_queue }}.retry-{{ item.value }}s"
args: "{{ {'x-message-ttl': item.value * 1000, 'x-dead-letter-exchange': '',
'x-dead-letter-routing-key': vars.work_queue} | toJson }}"
- id: topology_ready
type: io.kestra.plugin.core.execution.Exit
- id: identify
type: io.kestra.plugin.core.output.OutputValues
description: A message without a message_id is identified by the hash of its
body. KV keys only allow a few characters, so the id is hashed again for
the key.
values:
message_id: "{{ trigger.messageId ?? ('body-sha256-' ~ (trigger.data | sha256)) }}"
key: "{{ (trigger.messageId ?? ('body-sha256-' ~ (trigger.data | sha256))) |
sha256 }}"
- id: drop_duplicate
type: io.kestra.plugin.core.flow.If
condition: "{{ kv('delivered-' ~ outputs.identify.values.key,
errorOnMissing=false) is not null }}"
then:
- id: label_duplicate
type: io.kestra.plugin.core.execution.Labels
labels:
messageId: "{{ outputs.identify.values.message_id }}"
outcome: duplicate
- id: log_duplicate
type: io.kestra.plugin.core.log.Log
level: WARN
message: "{{ outputs.identify.values.message_id }} was already delivered at {{
kv('delivered-' ~ outputs.identify.values.key, errorOnMissing=false)
}}, so this copy is dropped."
- id: stop_duplicate
type: io.kestra.plugin.core.execution.Exit
- id: count_attempt
type: io.kestra.plugin.core.output.OutputValues
description: RabbitMQ's x-death header would hold this, but the AMQP plugin
turns every header into a "key=value" string, so attempts are counted in
KV.
values:
attempt: "{{ (kv('attempts-' ~ outputs.identify.values.key,
errorOnMissing=false) ?? 0) + 1 }}"
- id: deliver
type: io.kestra.plugin.core.http.Request
description: Non-2xx responses are returned instead of failing the task
(allowFailed), and a connection error only fails this task (allowFailure),
so `decide` sees every result.
method: POST
uri: "{{ vars.handler_url }}"
contentType: "{{ trigger.contentType ?? 'application/json' }}"
headers:
Idempotency-Key: "{{ outputs.identify.values.message_id }}"
X-Delivery-Attempt: "{{ outputs.count_attempt.values.attempt }}"
body: "{{ trigger.data }}"
allowFailure: true
options:
allowFailed: true
timeout:
connectTimeout: PT5S
readIdleTimeout: PT30S
- id: decide
type: io.kestra.plugin.core.output.OutputValues
description: 2xx is delivered. Other 4xx codes are permanent, so the message is
parked at once. 408, 425, 429, 5xx and no response climb the ladder until
it runs out.
values:
outcome: >-
{%- set code = outputs.deliver.code ?? 0 -%} {%- set attempt =
outputs.count_attempt.values.attempt | number -%} {%- if code >= 200 and
code < 300 -%}DELIVERED {%- elseif code >= 400 and code < 500 and not
([408, 425, 429] contains code) -%}PARK {%- elseif attempt <=
(vars.retry_delays_seconds | length) -%}RETRY {%- else -%}PARK{%- endif
-%}
reason: >-
{%- set code = outputs.deliver.code ?? 0 -%} {%- set result = code > 0 ?
('HTTP ' ~ code) : ('no response: ' ~ (errorLogs()[0].message ??
'unknown error') | split('\n') | first) -%} {%- if code >= 400 and code
< 500 and not ([408, 425, 429] contains code) -%}handler rejected the
message with {{ result }} {%- elseif code >= 300 or code == 0 -%}attempt
{{ outputs.count_attempt.values.attempt }} failed with {{ result }} {%-
else -%}{{ result }}{%- endif -%}
- id: label_outcome
type: io.kestra.plugin.core.execution.Labels
labels:
messageId: "{{ outputs.identify.values.message_id }}"
attempt: "{{ outputs.count_attempt.values.attempt }}"
outcome: "{{ outputs.decide.values.outcome | lower }}"
- id: route
type: io.kestra.plugin.core.flow.Switch
value: "{{ outputs.decide.values.outcome }}"
cases:
DELIVERED:
- id: remember_delivery
type: io.kestra.plugin.core.kv.Set
key: "delivered-{{ outputs.identify.values.key }}"
kvDescription: "{{ outputs.identify.values.message_id }}"
value: "{{ now() }}"
ttl: "{{ vars.dedupe_window }}"
- id: forget_attempts
type: io.kestra.plugin.core.kv.Delete
key: "attempts-{{ outputs.identify.values.key }}"
RETRY:
- id: schedule_retry
type: io.kestra.plugin.amqp.Publish
description: Persistent (deliveryMode 2), so a message waiting on the ladder
survives a broker restart. The retry is the only copy, so a broker
outage is retried here instead of losing it.
host: "{{ vars.rabbitmq_host }}"
port: "{{ vars.rabbitmq_port }}"
virtualHost: "{{ vars.rabbitmq_vhost }}"
username: "{{ secret('RABBITMQ_USERNAME') }}"
password: "{{ secret('RABBITMQ_PASSWORD') }}"
exchange: ""
routingKey: "{{ vars.work_queue }}.retry-{{
vars.retry_delays_seconds[(outputs.count_attempt.values.attempt |
number) - 1] }}s"
retry:
type: exponential
interval: PT5S
maxInterval: PT1M
maxAttempts: 20
from:
data: "{{ trigger.data }}"
messageId: "{{ outputs.identify.values.message_id }}"
contentType: "{{ trigger.contentType ?? 'application/json' }}"
deliveryMode: 2
- id: remember_attempt
type: io.kestra.plugin.core.kv.Set
key: "attempts-{{ outputs.identify.values.key }}"
kvDescription: "{{ outputs.identify.values.message_id }}"
kvType: NUMBER
value: "{{ outputs.count_attempt.values.attempt }}"
ttl: P7D
- id: retry_scheduled
type: io.kestra.plugin.core.execution.Exit
state: WARNING
PARK:
- id: park
type: io.kestra.plugin.amqp.Publish
description: The parked message carries why it was parked, because headers
cannot be read back with this plugin.
host: "{{ vars.rabbitmq_host }}"
port: "{{ vars.rabbitmq_port }}"
virtualHost: "{{ vars.rabbitmq_vhost }}"
username: "{{ secret('RABBITMQ_USERNAME') }}"
password: "{{ secret('RABBITMQ_PASSWORD') }}"
exchange: ""
routingKey: "{{ vars.parking_lot_queue }}"
serdeType: JSON
retry:
type: exponential
interval: PT5S
maxInterval: PT1M
maxAttempts: 20
from:
messageId: "{{ outputs.identify.values.message_id }}"
type: kestra.parked-message
contentType: application/json
deliveryMode: 2
data:
messageId: "{{ outputs.identify.values.message_id }}"
sourceQueue: "{{ vars.work_queue }}"
reason: "{{ outputs.decide.values.reason }}"
attempts: "{{ outputs.count_attempt.values.attempt }}"
lastStatus: "{{ outputs.deliver.code ?? 0 }}"
parkedAt: "{{ now() }}"
consumerExecutionId: "{{ execution.id }}"
contentType: "{{ trigger.contentType ?? 'application/json' }}"
payload: "{{ trigger.data }}"
- id: forget_attempts_after_park
type: io.kestra.plugin.core.kv.Delete
description: A message replayed from the parking lot starts the ladder again
from the first rung.
key: "attempts-{{ outputs.identify.values.key }}"
- id: parked
type: io.kestra.plugin.core.execution.Exit
state: WARNING
errors:
- id: log_lost_message
type: io.kestra.plugin.core.log.Log
level: ERROR
message: "Processing {{ outputs.identify.values.message_id ?? 'a message' }}
failed in {{ (errorLogs() | last).taskId ?? 'an unknown task' }}: {{
(errorLogs() | last).message ?? 'see the execution logs' }}. The trigger
already acknowledged the message, so this execution holds the only copy:
fix the cause, then replay the execution."
triggers:
- id: on_message
type: io.kestra.plugin.amqp.RealtimeTrigger
description: serdeType STRING (the default) passes any payload through. With
JSON, one undecodable message would stop the trigger and be redelivered
forever.
host: "{{ vars.rabbitmq_host }}"
port: "{{ vars.rabbitmq_port }}"
virtualHost: "{{ vars.rabbitmq_vhost }}"
username: "{{ secret('RABBITMQ_USERNAME') }}"
password: "{{ secret('RABBITMQ_PASSWORD') }}"
queue: "{{ vars.work_queue }}"