Alert on nginx 5xx Error Rate

id: nginx-error-rate-alert
namespace: company.team
description: |
  Count 5xx responses in the recent nginx access log and alert Slack when the
  error rate crosses your ceiling.

triggers:
  - id: every_ten_minutes
    type: io.kestra.plugin.core.trigger.Schedule
    description: Ten-minute cadence catches a bad deploy before the queue of tickets
      builds up.
    cron: "*/10 * * * *"
    disabled: true

inputs:
  - id: log_path
    type: STRING
    defaults: "/var/log/nginx/access.log"
    description: Path to the nginx access log on the host.
  - id: max_error_rate_percent
    type: INT
    defaults: 2
    description: 5xx percentage that triggers the alert.
  - id: window_lines
    type: INT
    defaults: 1000
    description: Trailing lines of the log to sample.

tasks:
  - id: rate_errors
    type: io.kestra.plugin.scripts.shell.Commands
    description: tail the access log, count 5xx status codes, and emit the computed
      percentage via the stdout outputs protocol. A missing log reports worst
      case.
    containerImage: alpine:3.20
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    commands:
      - |
        set -u
        if [ ! -r "{{ inputs.log_path }}" ]; then
          echo "log not readable at {{ inputs.log_path }}; treating as breach"
          echo '::{"outputs": {"error_rate": 100}}::'
          exit 0
        fi
        TOTAL=$(tail -n "{{ inputs.window_lines }}" "{{ inputs.log_path }}" | wc -l)
        ERRS=$(tail -n "{{ inputs.window_lines }}" "{{ inputs.log_path }}" | grep -cE ' 5[0-9]{2} ' || true)
        if [ "$TOTAL" -eq 0 ]; then
          echo "no log lines; treating as breach"
          echo '::{"outputs": {"error_rate": 100}}::'
        else
          PCT=$(( ERRS * 100 / TOTAL ))
          echo "$ERRS of $TOTAL responses were 5xx ($PCT%)"
          echo "::{\"outputs\": {\"error_rate\": $PCT}}::"
        fi

  - id: rate_breached
    type: io.kestra.plugin.core.flow.If
    description: Only alert when the rate crosses the ceiling; otherwise log the reading.
    condition: "{{ outputs.rate_errors.vars.error_rate >=
      inputs.max_error_rate_percent }}"
    then:
      - id: alert_high_rate
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: State the rate and threshold so the triage starts with the numbers.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "text": ":no_entry: nginx 5xx rate at {{ outputs.rate_errors.vars.error_rate }}% over the last {{ inputs.window_lines }} log lines (threshold {{ inputs.max_error_rate_percent }}%). Check upstreams and the latest deploy. Execution {{ execution.id }}."
          }
    else:
      - id: log_ok
        type: io.kestra.plugin.core.log.Log
        description: Record the passing rate for the trend.
        message: "nginx 5xx rate {{ outputs.rate_errors.vars.error_rate }}%, under {{
          inputs.max_error_rate_percent }}%."

errors:
  - id: alert_rate_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alert when the check itself fails - a broken probe must not read as
      zero errors.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "nginx error rate check FAILED in flow {{ flow.id }} (execution {{ execution.id }}). Check the log path and the Worker's file access."
      }

outputs:
  - id: error_rate_percent
    type: INT
    description: '5xx percentage over the sampled window, e.g. 1.'
    value: "{{ outputs.rate_errors.vars.error_rate }}"