Meilisearch Content Freshness Probe with a Sentinel Document

id: meilisearch-content-freshness-probe
namespace: company.team
description: |
  Fetch a sentinel document that the ingest pipeline touches on every run,
  compare its updated_at timestamp against a maximum age, and alert Discord
  when the docs index has not been refreshed in time.

inputs:
  - id: sentinel_document_id
    type: STRING
    defaults: docs-freshness-sentinel
    description: Identifier of the sentinel document the ingest pipeline upserts on
      every run with a fresh updated_at timestamp.

  - id: max_age_hours
    type: INT
    defaults: 26
    description: Maximum acceptable age of the sentinel document in hours. Set it
      slightly above the ingest schedule interval to absorb normal jitter.

tasks:
  - id: get_sentinel
    type: io.kestra.plugin.meilisearch.DocumentGet
    description: Fetch the sentinel document from the docs_site index; its
      updated_at field records the last successful ingest run.
    url: "{{ secret('MEILISEARCH_URL') }}"
    key: "{{ secret('MEILISEARCH_KEY') }}"
    index: docs_site
    documentId: "{{ inputs.sentinel_document_id }}"

  - id: check_freshness
    type: io.kestra.plugin.scripts.python.Script
    description: Compute the sentinel age in hours from its updated_at timestamp and
      emit an is_stale boolean plus the age through Kestra's output protocol.
    taskRunner:
      type: io.kestra.plugin.core.runner.Process
    env:
      SENTINEL_UPDATED_AT: "{{ outputs.get_sentinel.document.updated_at }}"
      MAX_AGE_HOURS: "{{ inputs.max_age_hours }}"
    script: |
      import json
      import os
      from datetime import datetime, timezone

      updated_at = datetime.fromisoformat(os.environ["SENTINEL_UPDATED_AT"].replace("Z", "+00:00"))
      max_age_hours = float(os.environ["MAX_AGE_HOURS"])

      age_hours = (datetime.now(timezone.utc) - updated_at).total_seconds() / 3600
      is_stale = age_hours > max_age_hours

      print("::" + json.dumps({"outputs": {"is_stale": is_stale, "age_hours": round(age_hours, 1)}}) + "::")

  - id: check_stale
    type: io.kestra.plugin.core.flow.If
    description: Branch on the staleness verdict computed by the Python step.
    condition: "{{ outputs.check_freshness.vars.is_stale }}"
    then:
      - id: alert_stale
        type: io.kestra.plugin.discord.DiscordIncomingWebhook
        description: Tell the channel exactly how old the index content is and what the
          limit was, so responders can judge severity at a glance.
        url: "{{ secret('DISCORD_WEBHOOK_URL') }}"
        payload: |
          {
            "content": "Docs index is STALE: sentinel document '{{ inputs.sentinel_document_id }}' was last updated {{ outputs.check_freshness.vars.age_hours }} hours ago, limit is {{ inputs.max_age_hours }} hours. The ingest pipeline likely stopped running (execution {{ execution.id }})."
          }
    else:
      - id: log_fresh
        type: io.kestra.plugin.core.log.Log
        description: Record the sentinel age on healthy runs so the execution history
          shows how close the pipeline runs to its freshness budget.
        message: "Docs index is fresh, sentinel '{{ inputs.sentinel_document_id }}' is
          {{ outputs.check_freshness.vars.age_hours }} hours old (limit {{
          inputs.max_age_hours }} hours)."

errors:
  - id: alert_failure
    type: io.kestra.plugin.discord.DiscordIncomingWebhook
    description: Alert Discord when the probe itself fails, including the case where
      the sentinel document is missing entirely, which usually means the index
      was rebuilt without it.
    url: "{{ secret('DISCORD_WEBHOOK_URL') }}"
    payload: |
      {
        "content": "Docs freshness probe FAILED in flow {{ flow.id }} (execution {{ execution.id }}). The sentinel document may be missing or Meilisearch is unreachable, check the task logs."
      }

triggers:
  - id: hourly
    type: io.kestra.plugin.core.trigger.Schedule
    description: Probe freshness every hour so a stalled ingest is caught the same
      day it stops.
    cron: "0 * * * *"
    disabled: true