Detect Flaky GitHub Actions Workflows

id: github-actions-flaky-workflow-detector
namespace: company.team
description: |
  Detect GitHub Actions workflows that alternate between success and failure
  across recent runs, then alert Slack when the failure rate suggests flaky CI.

triggers:
  - id: daily_check
    type: io.kestra.plugin.core.trigger.Schedule
    description: Checks recent workflow runs once per day; disabled until secrets
      are configured.
    cron: "30 9 * * 1-5"
    disabled: true
  - id: manual_webhook
    type: io.kestra.plugin.core.trigger.Webhook
    description: Trigger an on-demand flakiness audit from CI or operations tooling.
    key: actions-flaky-check

inputs:
  - id: repository
    type: STRING
    displayName: GitHub Repository
    defaults: kestra-io/kestra
    description: Owner/repository whose workflow runs should be analyzed.
  - id: workflow_file
    type: STRING
    displayName: Workflow File
    defaults: main.yml
    description: Workflow file name under .github/workflows, for example main.yml.
  - id: run_window
    type: INT
    displayName: Recent Runs
    defaults: 20
    description: Number of recent workflow runs to inspect.
  - id: min_completed_runs
    type: INT
    displayName: Minimum Runs
    defaults: 8
    description: Minimum completed runs required before a flakiness decision is made.
  - id: failure_rate_threshold
    type: FLOAT
    displayName: Failure Rate Threshold
    defaults: 0.4
    description: Failure rate at or above this value can trigger the flaky-workflow alert.
  - id: min_transitions
    type: INT
    displayName: Minimum Outcome Transitions
    defaults: 3
    description: Minimum success/failure transitions required to classify a workflow
      as flaky.

tasks:
  - id: fetch_runs
    type: io.kestra.plugin.core.http.Request
    description: Fetch recent completed workflow runs from the GitHub REST API.
    uri: "https://api.github.com/repos/{{ inputs.repository }}/actions/workflows/{{
      inputs.workflow_file }}/runs?per_page={{ inputs.run_window
      }}&status=completed"
    method: GET
    headers:
      Accept: "application/vnd.github+json"
      User-Agent: "Kestra-Actions-Flaky-Detector"
      Authorization: "Bearer {{ secret('GITHUB_TOKEN') }}"

  - id: analyze_flakiness
    type: io.kestra.plugin.scripts.python.Script
    description: Analyze completed runs for alternating outcomes and failure rate.
    containerImage: python:3.11-slim
    inputFiles:
      runs.json: "{{ outputs.fetch_runs.body | toJson }}"
    script: |
      import json

      with open("runs.json") as f:
          raw = f.read().strip()
          data = json.loads(raw) if raw else {}

      runs = data.get("workflow_runs", []) if isinstance(data, dict) else []
      completed = [r for r in runs if r.get("conclusion") in ("success", "failure")]
      outcomes = [r.get("conclusion") for r in completed]

      failures = sum(1 for x in outcomes if x == "failure")
      total = len(outcomes)
      failure_rate = (failures / total) if total else 0.0

      transitions = sum(
          1 for previous, current in zip(outcomes, outcomes[1:])
          if previous != current
      )

      enough_data = total >= int("{{ inputs.min_completed_runs }}")
      flaky = (
          enough_data
          and failure_rate >= float("{{ inputs.failure_rate_threshold }}")
          and transitions >= int("{{ inputs.min_transitions }}")
      )

      latest = completed[0] if completed else {}
      summary = {
          "total_runs": total,
          "failures": failures,
          "failure_rate": round(failure_rate, 3),
          "transitions": transitions,
          "enough_data": enough_data,
          "flaky": flaky,
          "latest_url": latest.get("html_url", ""),
          "latest_conclusion": latest.get("conclusion", "none")
      }

      print("Analyzed %s runs: %s failures, %.1f%% failure rate, %s transitions."
            % (total, failures, failure_rate * 100, transitions))
      print("::" + json.dumps({"outputs": summary}) + "::")

  - id: evaluate_flakiness
    type: io.kestra.plugin.core.flow.If
    description: Alert only when enough history exists and the workflow repeatedly
      changes outcome.
    condition: "{{ outputs.analyze_flakiness.vars.flaky == true }}"
    then:
      - id: alert_flaky_workflow
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Send flakiness evidence and the latest run link to Slack.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "text": "Possible flaky GitHub Actions workflow: {{ inputs.repository }} / {{ inputs.workflow_file }}. Failure rate: {{ outputs.analyze_flakiness.vars.failure_rate }}; outcome transitions: {{ outputs.analyze_flakiness.vars.transitions }}; latest run: {{ outputs.analyze_flakiness.vars.latest_url }}. Review recent run history before treating this as deterministic failure."
          }
    else:
      - id: log_healthy
        type: io.kestra.plugin.core.log.Log
        description: Record the evidence when the workflow does not meet flakiness
          thresholds.
        message: "Flakiness audit for {{ inputs.repository }}/{{ inputs.workflow_file
          }}: {{ outputs.analyze_flakiness.vars.total_runs }} runs, {{
          outputs.analyze_flakiness.vars.failure_rate }} failure rate, {{
          outputs.analyze_flakiness.vars.transitions }} transitions, flaky={{
          outputs.analyze_flakiness.vars.flaky }}."

errors:
  - id: alert_audit_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alert when the GitHub API or analysis fails so missing data cannot
      look healthy.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "GitHub Actions flakiness audit FAILED for {{ inputs.repository }}/{{ inputs.workflow_file }} in execution {{ execution.id }}. Check GITHUB_TOKEN permissions, workflow_file, and API rate limits."
      }

outputs:
  - id: flakiness_summary
    type: JSON
    description: Summary of run count, failure rate, transitions, latest run URL,
      and flaky decision.
    value: "{{ outputs.analyze_flakiness.vars | toJson }}"