id: huggingface-model-upstream-change-watch
namespace: company.team
description: |
Watch the Hugging Face models your services load from the Hub. When a model's main branch gets new
commits, its license or gating changes, or it disappears, post the changed files and commits in Slack
with an AI note on what the change means for how you use the model.
inputs:
- id: models
type: JSON
displayName: Models to watch
description: List of objects with a Hub model id, a one-line note on how you use
it, and an owner to mention.
defaults: |
[
{"id": "Qwen/Qwen2.5-1.5B-Instruct", "usage": "Support chatbot, loaded by vLLM from main with its chat template.", "owner": "@ml-platform"},
{"id": "BAAI/bge-small-en-v1.5", "usage": "Embeddings for our RAG index; changing weights means re-embedding everything.", "owner": "@search"},
{"id": "sentence-transformers/all-MiniLM-L6-v2", "usage": "Duplicate-ticket detection; we compare new embeddings with stored ones.", "owner": "@support-tools"}
]
- id: model_name
type: STRING
displayName: Gemini model
defaults: gemini-2.5-flash
tasks:
- id: for_each_model
type: io.kestra.plugin.core.flow.Loop
description: Check every model independently, four at a time.
values: "{{ inputs.models }}"
concurrencyLimit: 4
tasks:
- id: fetch_model
type: io.kestra.plugin.core.http.Download
description: Model metadata from the public Hub API (current commit, license,
gating).
uri: "https://huggingface.co/api/models/{{ fromJson(item.value).id }}"
- id: fetch_files
type: io.kestra.plugin.core.http.Download
description: Every file on the main branch, including subfolders, with its
content hash.
uri: "https://huggingface.co/api/models/{{ fromJson(item.value).id
}}/tree/main?recursive=true"
- id: fetch_commits
type: io.kestra.plugin.core.http.Download
description: Recent commits on the main branch.
uri: "https://huggingface.co/api/models/{{ fromJson(item.value).id
}}/commits/main"
- id: load_previous
type: io.kestra.plugin.core.kv.Get
description: Model state recorded at the last run. Missing on the first run.
key: "hf_model_{{ fromJson(item.value).id | replace({'/': '-'}) | slugify }}"
errorOnMissing: false
- id: compare
type: io.kestra.plugin.scripts.python.Script
description: List changed files, new commits, and license or gating changes
since the last run.
containerImage: python:3.12-slim
inputFiles:
model.json: "{{ outputs.fetch_model.uri }}"
files.json: "{{ outputs.fetch_files.uri }}"
commits.json: "{{ outputs.fetch_commits.uri }}"
previous.json: "{{ (outputs.load_previous.value ?? {}) | toJson }}"
script: |
import json
def load(name):
with open(name, encoding="utf-8") as f:
data = json.load(f)
return json.loads(data) if isinstance(data, str) else data
model = load("model.json")
files = load("files.json")
commits = load("commits.json")
previous = load("previous.json") or {}
def kind(path):
p = path.lower()
if p.endswith((".safetensors", ".bin", ".gguf", ".pt", ".pth", ".onnx", ".h5", ".msgpack")):
return "weights"
if "tokenizer" in p or p.endswith(("vocab.json", "merges.txt", ".model", "chat_template.jinja", "chat_template.json")):
return "tokenizer / chat template"
if p.endswith(("config.json", "generation_config.json")):
return "config"
if p in ("readme.md", "license", "license.md", "license.txt", "use_policy.md"):
return "card / license"
return "other"
state = {
"sha": model.get("sha"),
"license": (model.get("cardData") or {}).get("license") or "unknown",
"gated": str(model.get("gated", False)),
"disabled": bool(model.get("disabled")),
"files": {f["path"]: (f.get("lfs") or {}).get("oid") or f.get("oid") for f in files if f.get("type") == "file"},
}
changes, new_commits = [], []
if previous and state["sha"] != previous.get("sha"):
old = previous.get("files", {})
for path, oid in sorted(state["files"].items()):
if path not in old:
changes.append({"file": path, "kind": kind(path), "change": "added"})
elif old[path] != oid:
changes.append({"file": path, "kind": kind(path), "change": "modified"})
for path in sorted(set(old) - set(state["files"])):
changes.append({"file": path, "kind": kind(path), "change": "deleted"})
for c in commits:
if c["id"] == previous.get("sha"):
break
new_commits.append({"id": c["id"][:10], "date": c["date"][:10], "title": c["title"][:120],
"authors": [a.get("user") for a in c.get("authors", [])]})
facts = []
if previous:
if state["license"] != previous.get("license"):
facts.append(f"License changed: {previous.get('license')} -> {state['license']}")
if state["gated"] != previous.get("gated"):
facts.append(f"Gating changed: {previous.get('gated')} -> {state['gated']}")
if state["disabled"] and not previous.get("disabled"):
facts.append("The model was disabled on the Hub.")
print(f"{model.get('id')}: commit {str(state['sha'])[:10]}, license {state['license']}, gated {state['gated']}, "
f"{len(state['files'])} files, {len(new_commits)} new commits, {len(changes)} changed files, {len(facts)} other changes")
print("::" + json.dumps({"outputs": {
"changed": bool(changes or new_commits or facts),
"changes": changes,
"commits": new_commits,
"facts": facts,
"previous_sha": (previous.get("sha") or "")[:10],
"previous_revision": previous.get("sha") or "",
"sha": str(state["sha"])[:10],
"license": state["license"],
"state": state,
}}) + "::")
- id: if_changed
type: io.kestra.plugin.core.flow.If
description: Only call the LLM and Slack when the model changed.
condition: "{{ outputs.compare.vars.changed }}"
then:
- id: assess_impact
type: io.kestra.plugin.ai.completion.ChatCompletion
description: Explain what the upstream change means for the way the team uses
the model.
provider:
type: io.kestra.plugin.ai.provider.GoogleGemini
apiKey: "{{ secret('GEMINI_API_KEY') }}"
modelName: "{{ inputs.model_name }}"
configuration:
temperature: 0.1
responseFormat:
type: JSON
jsonSchema:
type: object
properties:
impact:
type: string
enum:
- NONE
- LOW
- HIGH
summary:
type: string
action:
type: string
required:
- impact
- summary
- action
messages:
- type: SYSTEM
content: |
You are an ML platform engineer. You receive the files and commits that changed upstream in a
Hugging Face model repository, plus any license or gating change, and how the team uses the model.
Rate the impact on the team: HIGH if weights, tokenizer, chat template or license changed in a way
that alters outputs or terms of use for that usage; LOW for config or behaviour-adjacent changes;
NONE for README-only or metadata-only edits. Summarize in at most 2 sentences and give one concrete
action (for example pin the previous revision, re-run the evaluation suite, re-embed, or review the
new license). Use only the files, commits and changes you are given.
- type: USER
content: |
Model: {{ fromJson(item.value).id }}
How we use it: {{ fromJson(item.value).usage }}
Changed files: {{ outputs.compare.vars.changes | toJson }}
New commits: {{ outputs.compare.vars.commits | toJson }}
Other changes: {{ outputs.compare.vars.facts | toJson }}
- id: alert_owner
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Post the Hub facts (never from the model) with the AI assessment.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
messageText: |
{{ outputs.assess_impact.jsonOutput.impact == 'HIGH' ? ':rotating_light:' : ':hugging_face:' }} *<https://huggingface.co/{{ fromJson(item.value).id }}|{{ fromJson(item.value).id }}> changed upstream* (impact for us: {{ outputs.assess_impact.jsonOutput.impact }}, owner: {{ fromJson(item.value).owner }})
Commit {{ outputs.compare.vars.previous_sha }} -> *{{ outputs.compare.vars.sha }}* | license: {{ outputs.compare.vars.license }}
{% for f in outputs.compare.vars.facts %}- {{ f }}
{% endfor %}{% for c in outputs.compare.vars.changes %}- `{{ c.file }}` {{ c.change }} ({{ c.kind }})
{% endfor %}{% for c in outputs.compare.vars.commits %}- {{ c.date }} {{ c.id }}: {{ c.title }}
{% endfor %}*What it means:* {{ outputs.assess_impact.jsonOutput.summary }}
*Suggested action:* {{ outputs.assess_impact.jsonOutput.action }}
To keep the previous version, load it with `revision="{{ outputs.compare.vars.previous_revision }}"`.
- id: remember_state
type: io.kestra.plugin.core.kv.Set
description: Store the model state for the next comparison. Runs after the alert.
key: "hf_model_{{ fromJson(item.value).id | replace({'/': '-'}) | slugify }}"
kvType: JSON
overwrite: true
value: "{{ outputs.compare.vars.state | toJson }}"
errors:
- id: alert_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: A model that cannot be checked must not look unchanged.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
messageText: ":warning: Hugging Face model watch failed in execution {{
execution.id }} ({{ flow.namespace }}.{{ flow.id }}). Open the failed loop
iteration: an HTTP 401 or 404 from the Hub means the model was deleted,
renamed or made private, which needs a look before the next deploy pulls
it."
triggers:
- id: daily
type: io.kestra.plugin.core.trigger.Schedule
description: Check every morning. Disabled until the secrets are set.
cron: "0 7 * * *"
disabled: true
