id: github-issue-local-duplicate-review
namespace: company.ai
inputs:
- id: repository
type: STRING
defaults: kestra-io/kestra
description: Public GitHub owner/repository. No token or write access is used.
- id: issue_limit
type: INT
defaults: 20
description: Fetch the most recently updated open issues, from 2 to 50.
- id: similarity_threshold
type: FLOAT
defaults: 0.82
description: Cosine-similarity cutoff from 0 to 1, not a confidence probability.
- id: max_pairs
type: INT
defaults: 10
description: Maximum candidate pairs in the review report, from 1 to 30.
tasks:
- id: validate_inputs
type: io.kestra.plugin.scripts.python.Script
containerImage: python:3.12-slim
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
inputFiles:
settings.json: "{{ inputs | toJson }}"
script: |
import json
import math
import re
from pathlib import Path
settings = json.loads(Path("settings.json").read_text())
repository = settings["repository"]
if not isinstance(repository, str) or not re.fullmatch(
r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repository
):
raise ValueError("repository must be a public GitHub owner/repository")
for key, minimum, maximum in (("issue_limit", 2, 50), ("max_pairs", 1, 30)):
value = settings[key]
if type(value) is not int or not minimum <= value <= maximum:
raise ValueError(f"{key} must be an integer from {minimum} to {maximum}")
threshold = settings["similarity_threshold"]
if (
type(threshold) not in (int, float)
or not math.isfinite(threshold)
or not 0 <= threshold <= 1
):
raise ValueError("similarity_threshold must be a finite number from 0 to 1")
print("Repository and bounded review settings validated")
- id: fetch_issues
type: io.kestra.plugin.core.http.Download
uri: "https://api.github.com/search/issues?q=repo:{{ inputs.repository
}}+is:issue+is:open&sort=updated&order=desc&per_page={{ inputs.issue_limit
}}"
headers:
Accept: application/vnd.github+json
X-GitHub-Api-Version: "2022-11-28"
User-Agent: Kestra-Local-Issue-Review
retry:
type: constant
interval: PT5S
maxAttempts: 3
- id: normalize
type: io.kestra.plugin.scripts.python.Script
containerImage: python:3.12-slim
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
dependencies:
- kestra==2.0.0
inputFiles:
response.json: "{{ outputs.fetch_issues.uri }}"
settings.json: "{{ inputs | toJson }}"
outputFiles:
- issues.json
script: |
import json
from pathlib import Path
from kestra import Kestra
response = json.loads(Path("response.json").read_text())
settings = json.loads(Path("settings.json").read_text())
if (
not isinstance(response, dict)
or not isinstance(response.get("items"), list)
or type(response.get("total_count")) is not int
or response["total_count"] < 0
):
raise ValueError("GitHub did not return the expected search result; no empty success substituted")
if response.get("incomplete_results") is True:
raise ValueError("GitHub search was incomplete; narrow the repository query and retry")
issues = []
seen = set()
for item in response["items"]:
if not isinstance(item, dict):
raise ValueError("GitHub returned a non-object issue")
if "pull_request" in item:
continue
number = item.get("number")
title = item.get("title")
if type(number) is not int or number <= 0 or number in seen:
raise ValueError("Issue numbers must be positive and unique")
if not isinstance(title, str) or not title.strip():
raise ValueError("An issue has no usable title")
seen.add(number)
issues.append({"number": number, "title": " ".join(title.split())})
issues.sort(key=lambda issue: issue["number"])
document = {
"repository": settings["repository"],
"issues": issues,
"scope": {
"requested_limit": settings["issue_limit"],
"returned_items": len(response["items"]),
"reviewed_open_issues": len(issues),
"total_open_search_results": response["total_count"],
},
}
Path("issues.json").write_text(json.dumps(document, ensure_ascii=False))
Kestra.outputs({"issue_count": len(issues)})
- id: enough_issues
type: io.kestra.plugin.core.flow.If
condition: "{{ outputs.normalize.vars.issue_count >= 2 }}"
then:
- id: independent_analysis
type: io.kestra.plugin.core.flow.Parallel
tasks:
- id: semantic_vectors
type: io.kestra.plugin.scripts.python.Script
containerImage: python:3.12-slim
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
dependencies:
- numpy==2.2.6
- onnxruntime==1.22.0
- tokenizers==0.21.1
- huggingface-hub==0.33.0
env:
HF_HUB_DISABLE_TELEMETRY: "1"
HF_HUB_DISABLE_IMPLICIT_TOKEN: "1"
HF_HUB_DISABLE_XET: "1"
inputFiles:
issues.json: "{{ outputs.normalize.outputFiles['issues.json'] }}"
outputFiles:
- semantic.json
script: |
import json
from pathlib import Path
import numpy as np
import onnxruntime as ort
from huggingface_hub import hf_hub_download
from tokenizers import Tokenizer
model = "Xenova/all-MiniLM-L6-v2"
revision = "751bff37182d3f1213fa05d7196b954e230abad9"
issues = json.loads(Path("issues.json").read_text())["issues"]
tokenizer_path = hf_hub_download(model, "tokenizer.json", revision=revision, token=False)
model_path = hf_hub_download(model, "onnx/model_quantized.onnx", revision=revision, token=False)
tokenizer = Tokenizer.from_file(tokenizer_path)
tokenizer.enable_truncation(max_length=256)
tokenizer.enable_padding()
encoded = tokenizer.encode_batch([issue["title"] for issue in issues])
arrays = {
"input_ids": np.asarray([row.ids for row in encoded], dtype=np.int64),
"attention_mask": np.asarray([row.attention_mask for row in encoded], dtype=np.int64),
"token_type_ids": np.asarray([row.type_ids for row in encoded], dtype=np.int64),
}
options = ort.SessionOptions()
options.intra_op_num_threads = 1
session = ort.InferenceSession(model_path, sess_options=options, providers=["CPUExecutionProvider"])
names = [item.name for item in session.get_inputs()]
if any(name not in arrays for name in names):
raise ValueError("The pinned model has an unexpected input contract")
hidden = session.run(None, {name: arrays[name] for name in names})[0]
if hidden.ndim != 3 or hidden.shape[0] != len(issues) or hidden.shape[2] != 384:
raise ValueError("The model did not return the expected token embeddings")
mask = arrays["attention_mask"][..., None]
counts = mask.sum(axis=1)
if np.any(counts == 0) or not np.all(np.isfinite(hidden)):
raise ValueError("Empty attention masks or non-finite model output")
pooled = (hidden * mask).sum(axis=1) / counts
norms = np.linalg.norm(pooled, axis=1, keepdims=True)
if np.any(norms == 0):
raise ValueError("The model returned a zero-length sentence embedding")
vectors = pooled / norms
Path("semantic.json").write_text(json.dumps({
"model": model,
"revision": revision,
"dimensions": 384,
"numbers": [issue["number"] for issue in issues],
"vectors": vectors.tolist(),
}))
print(f"Computed {len(issues)} local 384-dimensional sentence embeddings")
- id: lexical_overlap
type: io.kestra.plugin.scripts.python.Script
containerImage: python:3.12-slim
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
inputFiles:
issues.json: "{{ outputs.normalize.outputFiles['issues.json'] }}"
outputFiles:
- lexical.json
script: |
import itertools
import json
import re
from pathlib import Path
issues = json.loads(Path("issues.json").read_text())["issues"]
tokens = {
issue["number"]: set(re.findall(r"\w+", issue["title"].casefold()))
for issue in issues
}
scores = {}
for left, right in itertools.combinations(issues, 2):
first, second = tokens[left["number"]], tokens[right["number"]]
union = first | second
key = f'{left["number"]}:{right["number"]}'
scores[key] = len(first & second) / len(union) if union else 0.0
Path("lexical.json").write_text(json.dumps(scores))
print(f"Computed {len(scores)} explanatory word-overlap scores")
- id: rank_report
type: io.kestra.plugin.scripts.python.Script
containerImage: python:3.12-slim
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
dependencies:
- kestra==2.0.0
inputFiles:
issues.json: "{{ outputs.normalize.outputFiles['issues.json'] }}"
semantic.json: "{{ outputs.semantic_vectors.outputFiles['semantic.json'] }}"
lexical.json: "{{ outputs.lexical_overlap.outputFiles['lexical.json'] }}"
settings.json: "{{ inputs | toJson }}"
outputFiles:
- review.json
- review.md
script: |
import itertools
import json
import math
from pathlib import Path
from kestra import Kestra
document = json.loads(Path("issues.json").read_text())
semantic = json.loads(Path("semantic.json").read_text())
lexical = json.loads(Path("lexical.json").read_text())
settings = json.loads(Path("settings.json").read_text())
issues, vectors = document["issues"], semantic["vectors"]
if semantic["numbers"] != [issue["number"] for issue in issues] or len(vectors) != len(issues):
raise ValueError("Embedding identities do not match the fetched issue sample")
for vector in vectors:
if len(vector) != 384 or any(not math.isfinite(value) for value in vector):
raise ValueError("Invalid sentence embedding")
if abs(math.fsum(value * value for value in vector) - 1) > 0.001:
raise ValueError("Sentence embeddings are not normalized")
candidates = []
for left, right in itertools.combinations(range(len(issues)), 2):
score = max(-1.0, min(1.0, math.fsum(a * b for a, b in zip(vectors[left], vectors[right]))))
if score < settings["similarity_threshold"]:
continue
first, second = issues[left], issues[right]
key = f'{first["number"]}:{second["number"]}'
overlap = lexical[key]
if not math.isfinite(overlap) or not 0 <= overlap <= 1:
raise ValueError("Invalid explanatory lexical score")
candidates.append({
"left": first, "right": second,
"cosine_similarity": round(score, 6),
"lexical_jaccard": round(overlap, 6),
})
candidates.sort(key=lambda pair: (
-pair["cosine_similarity"], -pair["lexical_jaccard"],
pair["left"]["number"], pair["right"]["number"],
))
candidates = candidates[:settings["max_pairs"]]
result = {
"status": "review_candidates" if candidates else "no_pairs_above_threshold",
"repository": document["repository"],
"scope": document["scope"],
"model": semantic["model"], "revision": semantic["revision"],
"dimensions": semantic["dimensions"],
"threshold": settings["similarity_threshold"],
"read_only": True, "candidates": candidates,
}
lines = [
"# GitHub issue duplicate-candidate review",
f'Repository: {document["repository"]}',
f'Scope: {len(issues)} recent open issue titles; not the complete repository history.',
f'Cosine cutoff: {settings["similarity_threshold"]}. Scores are not probabilities.',
"Human review only. No GitHub labels, comments, assignments or closures were changed.",
"",
]
for pair in candidates:
lines.append(f'## Cosine {pair["cosine_similarity"]:.3f}; word overlap {pair["lexical_jaccard"]:.3f}')
for side in ("left", "right"):
issue = pair[side]
title = issue["title"].replace("`", "'").replace("\n", " ")
url = f'https://github.com/{document["repository"]}/issues/{issue["number"]}'
lines.append(f'- [#{issue["number"]}]({url}): `{title}`')
lines.append("")
if not candidates:
lines.append("No pairs exceeded this cutoff in this bounded sample; this is not proof that no duplicates exist.")
Path("review.json").write_text(json.dumps(result, ensure_ascii=False, indent=2))
Path("review.md").write_text("\n".join(lines), encoding="utf-8")
Kestra.outputs({"candidate_count": len(candidates)})
else:
- id: no_comparison
type: io.kestra.plugin.scripts.python.Script
containerImage: python:3.12-slim
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
inputFiles:
issues.json: "{{ outputs.normalize.outputFiles['issues.json'] }}"
outputFiles:
- review.json
- review.md
script: |
import json
from pathlib import Path
document = json.loads(Path("issues.json").read_text())
result = {
"status": "not_enough_open_issues",
"repository": document["repository"],
"scope": document["scope"], "read_only": True, "candidates": [],
}
Path("review.json").write_text(json.dumps(result, indent=2))
Path("review.md").write_text(
"# No comparison performed\n\nFewer than two usable open issues were returned. "
"No model was run and no claim about duplicate absence is made.\n"
)
print("No comparison performed: fewer than two open issues")
triggers:
- id: weekly_review
type: io.kestra.plugin.core.trigger.Schedule
cron: "0 8 * * 1"
disabled: true
