Detect Newly Flaky Buildkite Test Steps and Ratchet Them Open as Issues

id: buildkite-flaky-test-ratchet
namespace: company.team
description: |
  Watch a Buildkite pipeline for steps that swing between pass and fail, compare the
  recent run set against the previous one, and ratchet newly flaky steps open as a
  GitHub issue with a Slack quarantine suggestion.

triggers:
  - id: daily_ratchet
    type: io.kestra.plugin.core.trigger.Schedule
    description: Run the ratchet once a day. Shipped disabled so the first run is
      always the manual one.
    cron: "0 6 * * *"
    disabled: true

  - id: on_build_finished
    type: io.kestra.plugin.core.trigger.Webhook
    description: Buildkite posts here on build finished, so the ratchet also runs
      the moment a build settles.
    key: "{{ secret('BUILDKITE_WEBHOOK_KEY') }}"

inputs:
  - id: organization
    type: STRING
    defaults: your-buildkite-org
    description: Buildkite organization slug, used in the builds and the test
      analytics API paths.
  - id: pipeline
    type: STRING
    defaults: your-pipeline
    description: Buildkite pipeline slug watched for step level pass and fail variance.
  - id: github_repository
    type: STRING
    defaults: your-org/your-repo
    description: GitHub repository in owner slash repo form where the ratchet issue
      is opened.

tasks:
  # --- 1. collect the evidence ---
  - id: fetch_builds
    type: io.kestra.plugin.core.http.Request
    description: Pull the last 40 builds of the pipeline, newest first,
      authenticated with the Buildkite API token.
    uri: "https://api.buildkite.com/v2/organizations/{{ inputs.organization
      }}/pipelines/{{ inputs.pipeline }}/builds?per_page=40"
    method: GET
    headers:
      Authorization: "Bearer {{ secret('BUILDKITE_TOKEN') }}"
      Accept: application/json

  - id: fetch_test_analytics
    type: io.kestra.plugin.core.http.Request
    description: Pull Test Engine records for the newest build. Failure here is
      tolerated so the step level check still runs.
    uri: "https://api.buildkite.com/v2/analytics/organizations/{{
      inputs.organization }}/builds/{{ outputs.fetch_builds.body | jq('.[0].id')
      | first }}/tests"
    method: GET
    allowFailure: true
    headers:
      Authorization: "Bearer {{ secret('BUILDKITE_TOKEN') }}"
      Accept: application/json

  # --- 2. analyse the two windows ---
  - id: analyze_flakes
    type: io.kestra.plugin.scripts.python.Script
    description: Split the finished builds into a recent window and a previous
      window, then emit the ratchet verdict through the outputs protocol.
    taskRunner:
      type: io.kestra.plugin.core.runner.Process
    inputFiles:
      builds.json: "{{ outputs.fetch_builds.body }}"
      tests.json: "{{ outputs.fetch_test_analytics is defined ?
        (outputs.fetch_test_analytics.body ?? '[]') : '[]' }}"
    script: |
      import json

      WINDOW = 10
      PASSED = "passed"
      FAILED = "failed"
      FINISHED = (PASSED, FAILED)

      def read_json(path, default):
          try:
              handle = open(path)
          except OSError:
              return default
          with handle:
              raw = handle.read().strip()
          if not raw:
              return default
          try:
              data = json.loads(raw)
          except ValueError:
              return default
          if isinstance(data, str):
              try:
                  data = json.loads(data)
              except ValueError:
                  return default
          return data

      def finished(builds):
          if not isinstance(builds, list):
              return []
          keep = []
          for build in builds:
              if not isinstance(build, dict):
                  continue
              if build.get("state") not in FINISHED:
                  continue
              jobs = build.get("jobs")
              if isinstance(jobs, list) and jobs:
                  keep.append(build)
          return keep

      def step_counts(builds):
          counts = {}
          for build in finished(builds):
              for job in build.get("jobs") or []:
                  if not isinstance(job, dict):
                      continue
                  name = job.get("name")
                  state = job.get("state")
                  if not name or state not in FINISHED:
                      continue
                  bucket = counts.setdefault(name, {PASSED: 0, FAILED: 0})
                  bucket[state] += 1
          return counts

      def flaky_steps(counts):
          return sorted(
              name for name, bucket in counts.items()
              if bucket[PASSED] > 0 and bucket[FAILED] > 0
          )

      def step_rate(counts, name):
          bucket = counts[name]
          total = bucket[PASSED] + bucket[FAILED]
          if total == 0:
              return 0.0
          return round(100.0 * bucket[FAILED] / total, 1)

      def worst_step(counts):
          name, rate = "none", 0.0
          for candidate in sorted(counts):
              current = step_rate(counts, candidate)
              if current > rate:
                  name, rate = candidate, current
          return name, rate

      def window_share(counts):
          failed = sum(bucket[FAILED] for bucket in counts.values())
          total = sum(bucket[PASSED] + bucket[FAILED] for bucket in counts.values())
          if total == 0:
              return 0.0
          return round(100.0 * failed / total, 1)

      def quarantine_candidates(tests):
          if not isinstance(tests, list):
              return []
          names = set()
          for test in tests:
              if not isinstance(test, dict):
                  continue
              name = test.get("name")
              if not name:
                  continue
              labels = [str(label).lower() for label in (test.get("labels") or [])]
              results = test.get("executions_count_by_result") or {}
              passed = results.get(PASSED) or 0
              failed = results.get(FAILED) or 0
              if "flaky" in labels or (passed > 0 and failed > 0):
                  names.add(name)
          return sorted(names)

      def analyze(recent_builds, baseline_builds, tests):
          recent = finished(recent_builds)
          baseline = finished(baseline_builds)
          recent_counts = step_counts(recent)
          baseline_counts = step_counts(baseline)
          newly_flaky = sorted(
              set(flaky_steps(recent_counts)) - set(flaky_steps(baseline_counts))
          )
          worst_name, worst_rate = worst_step(recent_counts)
          _, baseline_rate = worst_step(baseline_counts)
          rate_worsened = worst_rate > baseline_rate
          quarantine = quarantine_candidates(tests)
          escalate = bool(newly_flaky) or rate_worsened
          verdict = "RATCHET" if escalate else "STABLE"
          step_lines = [
              "- `%s` : %.1f%% failed (%d of %d builds)" % (
                  name,
                  step_rate(recent_counts, name),
                  recent_counts[name][FAILED],
                  recent_counts[name][PASSED] + recent_counts[name][FAILED],
              )
              for name in newly_flaky
          ]
          test_lines = ["- `%s`" % name for name in quarantine]
          summary = "%s: %d newly flaky of %d tracked steps, worst %s at %.1f%% failed" % (
              verdict,
              len(newly_flaky),
              len(recent_counts),
              worst_name,
              worst_rate,
          )
          return {
              "verdict": verdict,
              "new_flake_count": len(newly_flaky),
              "newly_flaky": newly_flaky,
              "newly_flaky_text": ", ".join(newly_flaky) or "none",
              "newly_flaky_md": "\n".join(step_lines) or "- none",
              "worst_step": worst_name,
              "worst_rate": "%.1f%%" % worst_rate,
              "baseline_worst_rate": "%.1f%%" % baseline_rate,
              "rate_worsened": rate_worsened,
              "recent_rate": "%.1f%%" % window_share(recent_counts),
              "baseline_rate": "%.1f%%" % window_share(baseline_counts),
              "quarantine_candidates": quarantine,
              "quarantine_text": ", ".join(quarantine) or "none",
              "quarantine_md": "\n".join(test_lines) or "- none",
              "builds_analyzed": len(recent),
              "baseline_builds": len(baseline),
              "steps_tracked": len(recent_counts),
              "summary": summary,
          }

      builds = read_json("builds.json", [])
      tests = read_json("tests.json", [])
      ordered = finished(builds)
      result = analyze(ordered[:WINDOW], ordered[WINDOW:2 * WINDOW], tests)
      print("::" + json.dumps({"outputs": result}) + "::")
      print(result["summary"])

  # --- 3. ratchet gate ---
  - id: ratchet_gate
    type: io.kestra.plugin.core.flow.If
    description: Escalate only when a step is newly flaky or the worst failed share
      got worse than in the previous window.
    condition: "{{ outputs.analyze_flakes.vars.new_flake_count > 0 or
      outputs.analyze_flakes.vars.rate_worsened }}"
    then:
      - id: open_ratchet_issue
        type: io.kestra.plugin.github.issues.Create
        description: Open one issue listing the newly flaky steps, their failed shares,
          and the quarantine candidates.
        jwtToken: "{{ secret('GITHUB_TOKEN') }}"
        repository: "{{ inputs.github_repository }}"
        title: "Flaky steps detected in {{ inputs.pipeline }} by the Kestra ratchet"
        body: |
          ## Newly flaky steps

          Pipeline `{{ inputs.organization }}/{{ inputs.pipeline }}` shows **{{ outputs.analyze_flakes.vars.new_flake_count }} newly flaky step(s)** against the previous run set.

          {{ outputs.analyze_flakes.vars.newly_flaky_md }}

          ### Rates

          - Worst step now `{{ outputs.analyze_flakes.vars.worst_step }}` at {{ outputs.analyze_flakes.vars.worst_rate }} failed, against {{ outputs.analyze_flakes.vars.baseline_worst_rate }} in the previous window.
          - Overall failed share {{ outputs.analyze_flakes.vars.recent_rate }} now, {{ outputs.analyze_flakes.vars.baseline_rate }} before.
          - Builds analysed {{ outputs.analyze_flakes.vars.builds_analyzed }} recent and {{ outputs.analyze_flakes.vars.baseline_builds }} previous, tracking {{ outputs.analyze_flakes.vars.steps_tracked }} distinct steps.

          ### Quarantine suggestion

          {{ outputs.analyze_flakes.vars.quarantine_md }}

          Fix or quarantine every step listed above, then close this issue. The ratchet only reopens for a step that is newly flaky or whose failed share got worse, so a step that stays flaky does not keep generating issues.

          Execution {{ execution.id }}.
        labels:
          - flaky-tests
          - quarantine

      - id: quarantine_hint
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Post the quarantine suggestion so the fix or quarantine decision
          starts immediately.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "text": "Flake ratchet OPENED for {{ inputs.pipeline }} ({{ inputs.organization }}): {{ outputs.analyze_flakes.vars.new_flake_count }} newly flaky step(s) - {{ outputs.analyze_flakes.vars.newly_flaky_text }}. Worst {{ outputs.analyze_flakes.vars.worst_step }} at {{ outputs.analyze_flakes.vars.worst_rate }} failed, previous window {{ outputs.analyze_flakes.vars.baseline_worst_rate }}. Quarantine suggestion - {{ outputs.analyze_flakes.vars.quarantine_text }}. Issue opened in {{ inputs.github_repository }}. Execution {{ execution.id }}."
          }

    else:
      - id: log_stable
        type: io.kestra.plugin.core.log.Log
        description: Record the clean verdict so the execution history proves the
          ratchet ran and found nothing new.
        message: "STABLE for {{ inputs.pipeline }}: no newly flaky step and no worse
          failed share across {{ outputs.analyze_flakes.vars.builds_analyzed }}
          recent builds."

errors:
  - id: slack_alert
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alert when the API call or the analysis itself breaks, because a
      dead ratchet looks exactly like a clean pipeline.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "FAILED {{ flow.id }} run {{ execution.id }} : the Buildkite flake ratchet could not run for {{ inputs.pipeline }}. Check the BUILDKITE_TOKEN secret and Python on the worker."
      }

outputs:
  - id: verdict
    type: STRING
    description: RATCHET when a step is newly flaky or the worst failed share got
      worse, otherwise STABLE.
    value: "{{ outputs.analyze_flakes.vars.verdict }}"
  - id: newly_flaky
    type: JSON
    description: Steps that swing pass to fail in the recent window but were clean
      in the previous one.
    value: "{{ outputs.analyze_flakes.vars.newly_flaky | toJson }}"
  - id: worst_rate
    type: STRING
    description: Highest per-step failed share in the recent window, for example 40.0%.
    value: "{{ outputs.analyze_flakes.vars.worst_rate }}"
  - id: new_flake_count
    type: INT
    description: How many steps crossed into flaky since the previous run set, for
      example 2.
    value: "{{ outputs.analyze_flakes.vars.new_flake_count }}"