Find the Kubernetes manifests that will break on your next cluster upgrade

id: kubernetes-upgrade-readiness-gate
namespace: company.team

inputs:
  - id: current_version
    type: STRING
    displayName: Current Kubernetes version
    defaults: 1.24.0

  - id: target_version
    type: STRING
    displayName: Target Kubernetes version
    description: The version the cluster is being upgraded to. Manifests that
      validate today but not on this version are upgrade blockers.
    defaults: 1.29.0

  - id: min_score_grade
    type: SELECT
    displayName: kube-score gate
    description: CRITICAL also blocks on critical kube-score findings. NONE reports
      them without blocking, which suits a first pass on an existing repository.
    values:
      - CRITICAL
      - NONE
    defaults: NONE

  - id: demo_migrated
    type: BOOL
    displayName: Demo with migrated API versions
    description: Rewrite the demo manifests to the current API versions (policy/v1,
      autoscaling/v2, batch/v1) to see a ready verdict.
    defaults: false

triggers:
  - id: on_pull_request
    type: io.kestra.plugin.core.trigger.Webhook
    description: Call from CI on every change to the manifests. Replace the key first.
    key: replace-with-a-long-random-key

tasks:
  - id: check
    type: io.kestra.plugin.scripts.shell.Commands
    description: >
      Validate every manifest with kubeconform against the current and the
      target Kubernetes schemas, then score it with kube-score. A manifest valid
      today but invalid on the target uses an API the upgrade removes.
    containerImage: python:3.12-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    retry:
      type: constant
      interval: PT15S
      maxAttempts: 2
    env:
      CURRENT: "{{ inputs.current_version }}"
      MIGRATED: "{{ inputs.demo_migrated }}"
      DEBIAN_FRONTEND: noninteractive
      TARGET: "{{ inputs.target_version }}"
    # Demo manifests. Replace with a git.Clone of your manifests in a WorkingDirectory.
    inputFiles:
      manifests/api.yaml: |
        apiVersion: apps/v1
        kind: Deployment
        metadata: {name: api, labels: {app: api}}
        spec:
          replicas: 2
          selector: {matchLabels: {app: api}}
          template:
            metadata: {labels: {app: api}}
            spec:
              containers:
                - name: api
                  image: ghcr.io/acme/api:1.8.2
                  resources: {requests: {cpu: 100m, memory: 128Mi}, limits: {cpu: 500m, memory: 256Mi}}
                  securityContext: {runAsNonRoot: true, runAsUser: 10001, runAsGroup: 10001, readOnlyRootFilesystem: true, allowPrivilegeEscalation: false}
                  readinessProbe: {httpGet: {path: /ready, port: 8080}}
                  livenessProbe: {httpGet: {path: /health, port: 8081}}
      manifests/api-pdb.yaml: |
        apiVersion: policy/v1beta1
        kind: PodDisruptionBudget
        metadata: {name: api}
        spec:
          minAvailable: 1
          selector: {matchLabels: {app: api}}
      manifests/api-hpa.yaml: |
        apiVersion: autoscaling/v2beta2
        kind: HorizontalPodAutoscaler
        metadata: {name: api}
        spec:
          scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: api}
          minReplicas: 2
          maxReplicas: 10
          metrics:
            - type: Resource
              resource: {name: cpu, target: {type: Utilization, averageUtilization: 70}}
      manifests/worker.yaml: |
        apiVersion: apps/v1
        kind: Deployment
        metadata: {name: worker}
        spec:
          replicas: 1
          selector: {matchLabels: {app: worker}}
          template:
            metadata: {labels: {app: worker}}
            spec:
              containers:
                - name: worker
                  image: ghcr.io/acme/worker:latest
      manifests/cron.yaml: |
        apiVersion: batch/v1beta1
        kind: CronJob
        metadata: {name: nightly-report}
        spec:
          schedule: "0 3 * * *"
          jobTemplate:
            spec:
              template:
                spec:
                  restartPolicy: OnFailure
                  containers:
                    - name: report
                      image: ghcr.io/acme/report:2.1.0
    beforeCommands:
      - apt-get -qq update >/dev/null && apt-get -qq install -y curl >/dev/null
      - curl -fsSL -o kc.tgz
        https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz
      - echo
        "95f14e87aa28c09d5941f11bd024c1d02fdc0303ccaa23f61cef67bc92619d73  kc.tgz"
        | sha256sum -c - >/dev/null
      - curl -fsSL -o ks.tgz
        https://github.com/zegl/kube-score/releases/download/v1.19.0/kube-score_1.19.0_linux_amd64.tar.gz
      - echo
        "a5b10e509bd845f0bc32a529f4e8165c021877f924ee6f6be66678039f75a761  ks.tgz"
        | sha256sum -c - >/dev/null
      - tar -xzf kc.tgz kubeconform && tar -xzf ks.tgz kube-score
    outputFiles:
      - readiness.md
    commands:
      - |
        set +e
        if [ "$MIGRATED" = true ]; then
          sed -i 's#policy/v1beta1#policy/v1#; s#autoscaling/v2beta2#autoscaling/v2#; s#batch/v1beta1#batch/v1#' manifests/*.yaml
        fi
        ./kubeconform -strict -output json -kubernetes-version "$CURRENT" manifests/ > current.json
        ./kubeconform -strict -output json -kubernetes-version "$TARGET" manifests/ > target.json
        ./kube-score score -o json manifests/*.yaml > score.json
        python3 - <<'PY'
        import json, os
        def by_file(path):
            out = {}
            for r in json.load(open(path))["resources"]:
                out[os.path.basename(r["filename"])] = (r["status"], r.get("msg", ""), r.get("kind", ""))
            return out
        files = sorted(f for f in os.listdir("manifests"))
        cur, tgt = by_file("current.json"), by_file("target.json")
        ok = "statusValid"
        status = lambda d, f: d.get(f, (ok, "", ""))
        broken_now = [f for f in files if status(cur, f)[0] != ok]
        blockers = [f for f in files if status(cur, f)[0] == ok and status(tgt, f)[0] != ok]
        critical = []
        for obj in json.load(open("score.json")):
            name = f"{obj['type_meta']['kind']}/{obj['object_meta']['name']}"
            for c in obj["checks"]:
                if c.get("skipped") or c["grade"] != 1:
                    continue
                for comment in c.get("comments") or []:
                    critical.append(f"{name}: {c['check']['name']} ({comment['summary']})")
        lines = [f"# Kubernetes {os.environ['CURRENT']} to {os.environ['TARGET']}", "",
                 "| manifest | valid now | valid on target |", "|---|---|---|"]
        for f in files:
            lines.append(f"| `{f}` | {'yes' if status(cur, f)[0] == ok else 'NO'} | {'yes' if status(tgt, f)[0] == ok else 'NO: ' + status(tgt, f)[1][:90]} |")
        lines += ["", f"kube-score critical findings ({len(critical)}):"] + [f"- {c}" for c in critical]
        open("readiness.md", "w").write("\n".join(lines) + "\n")
        print("\n".join(lines))
        print("::" + json.dumps({"outputs": {
            "manifests": len(files), "upgrade_blockers": len(blockers), "blockers": ", ".join(blockers),
            "invalid_now": ", ".join(broken_now), "critical_count": len(critical), "critical": "; ".join(critical[:12])}}) + "::")
        PY

  - id: publish_report
    type: io.kestra.plugin.core.namespace.UploadFiles
    namespace: "{{ flow.namespace }}"
    filesMap:
      "k8s/upgrade-readiness-{{ inputs.target_version }}.md": "{{ outputs.check.outputFiles['readiness.md'] }}"

  - id: gate
    type: io.kestra.plugin.core.flow.If
    description: Block when a manifest is already invalid, would break on the target
      version, or has a critical finding when the kube-score gate is on.
    condition: >-
      {{ outputs.check.vars.upgrade_blockers == 0 and
      outputs.check.vars.invalid_now == '' and (inputs.min_score_grade == 'NONE'
      or outputs.check.vars.critical_count == 0) }}
    then:
      - id: ready
        type: io.kestra.plugin.core.log.Log
        message: "All {{ outputs.check.vars.manifests }} manifests are valid on {{
          inputs.target_version }}. Critical kube-score findings: {{
          outputs.check.vars.critical_count }}."
    else:
      - id: not_ready
        type: io.kestra.plugin.core.execution.Fail
        errorMessage: >-
          Not ready for {{ inputs.target_version }}: {{
          outputs.check.vars.upgrade_blockers }} manifest(s) use APIs removed on
          the target ({{ outputs.check.vars.blockers }}). Invalid today: {{
          outputs.check.vars.invalid_now == '' ? 'none' :
          outputs.check.vars.invalid_now }}. Critical findings ({{
          outputs.check.vars.critical_count }}): {{ outputs.check.vars.critical
          }}