Hourly OpenSearch Error Spike Gate with Discord Alerting

id: opensearch-error-spike-gate
namespace: company.team
description: |
  Count server errors in the last hour with one OpenSearch query and gate
  on the result. Above the threshold the flow alerts Discord and fails
  loudly; below it, it logs an all-clear and exits green.

inputs:
  - id: error_threshold
    type: INT
    defaults: 50
    description: Maximum number of 5xx documents tolerated in the last hour before
      the flow alerts and fails.

tasks:
  - id: count_recent_errors
    type: io.kestra.plugin.opensearch.Search
    description: Run a size-zero search counting documents with a 5xx status in the
      last hour, using date math so the window needs no templating.
    connection:
      hosts:
        - "{{ secret('OPENSEARCH_HOST') }}"
      basicAuth:
        username: "{{ secret('OPENSEARCH_USERNAME') }}"
        password: "{{ secret('OPENSEARCH_PASSWORD') }}"
    indexes:
      - app-logs
    fetchType: FETCH
    request:
      size: 0
      query:
        bool:
          filter:
            - range:
                status:
                  gte: 500
                  lt: 600
            - range:
                "@timestamp":
                  gte: now-1h

  - id: gate
    type: io.kestra.plugin.core.flow.If
    description: Compare the hourly 5xx count against the threshold input and branch
      into alerting or an all-clear log line.
    condition: "{{ outputs.count_recent_errors.total >= inputs.error_threshold }}"
    then:
      - id: alert_spike
        type: io.kestra.plugin.discord.DiscordIncomingWebhook
        description: Page the channel with the exact count and threshold before the flow
          fails, so responders see the numbers without opening Kestra.
        url: "{{ secret('DISCORD_WEBHOOK_URL') }}"
        payload: |
          {
            "content": "OpenSearch error spike: {{ outputs.count_recent_errors.total }} server errors in app-logs over the last hour, threshold is {{ inputs.error_threshold }}. Execution {{ execution.id }} is failing on purpose."
          }

      - id: fail_execution
        type: io.kestra.plugin.core.execution.Fail
        description: Fail the execution deliberately so the spike is visible as a red
          run in the UI and in any downstream flow triggers watching this flow's
          state.
        errorMessage: "Hourly 5xx count crossed the configured threshold, see the
          alert_spike message for the numbers."
    else:
      - id: all_clear
        type: io.kestra.plugin.core.log.Log
        description: Record the healthy count so quiet hours leave an auditable trail
          instead of an empty execution.
        message: "All clear, {{ outputs.count_recent_errors.total }} server errors in
          the last hour, below the threshold of {{ inputs.error_threshold }}."

errors:
  - id: alert_failure
    type: io.kestra.plugin.discord.DiscordIncomingWebhook
    description: Alert Discord whenever the execution fails, covering both a
      deliberate threshold breach and an unexpected search error.
    url: "{{ secret('DISCORD_WEBHOOK_URL') }}"
    payload: |
      {
        "content": "OpenSearch error spike gate finished RED in flow {{ flow.id }} (execution {{ execution.id }}). Either the 5xx threshold was crossed or the search itself failed, check the execution logs."
      }

triggers:
  - id: hourly
    type: io.kestra.plugin.core.trigger.Schedule
    description: Check the last hour of logs at the top of every hour.
    cron: "0 * * * *"
    disabled: true