Recover a Helm release stuck pending after an interrupted upgrade

# SPDX-License-Identifier: Apache-2.0
id: helm-stuck-pending-release-recovery-guard
namespace: company.team
description: Recover an interrupted Helm upgrade or rollback after a stable
  grace observation and explicit approval under a cooperative release fence.

concurrency:
  limit: 1
  behavior: QUEUE

inputs:
  - id: release_name
    type: STRING
    defaults: application
  - id: release_namespace
    type: STRING
    defaults: applications
  - id: master_url
    type: STRING
    defaults: "{{ secret('K8S_MASTER_URL') }}"
  - id: ca_cert
    type: STRING
    defaults: "{{ secret('K8S_CA_CERT') }}"
  - id: token
    type: SECRET
    defaults: "{{ secret('K8S_TOKEN') }}"
  - id: docker_network
    type: STRING
    defaults: bridge
  - id: grace_seconds
    type: INT
    defaults: 1800
    min: 1
    max: 86400
  - id: history_window
    type: INT
    defaults: 20
    min: 1
    max: 100
  - id: dry_run
    type: BOOL
    defaults: true
  - id: auto_remediate
    type: BOOL
    defaults: false

variables:
  fence_name: "helm-recovery-{{ inputs.release_name }}"
  observation_key: "helm-pending-{{ inputs.master_url | sha256 }}-{{
    inputs.release_namespace }}-{{ inputs.release_name }}"

tasks:
  - id: validate_identifiers
    type: io.kestra.plugin.core.execution.Fail
    condition: >-
      {{ inputs.release_name | length == 0 or inputs.release_name | length > 53
      or
         inputs.release_name | replace({'^[a-z0-9]([a-z0-9-]*[a-z0-9])?$': ''}, regexp=true) | length > 0 or
         inputs.release_namespace | length == 0 or inputs.release_namespace | length > 63 or
         inputs.release_namespace | replace({'^[a-z0-9]([a-z0-9-]*[a-z0-9])?$': ''}, regexp=true) | length > 0 }}
    errorMessage: Release and namespace must be bounded lowercase DNS labels.

  - id: inspect_release
    type: io.kestra.plugin.helm.Status
    releaseName: "{{ inputs.release_name }}"
    namespace: "{{ inputs.release_namespace }}"
    containerImage: alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
      entryPoint: []
      networkMode: "{{ inputs.docker_network }}"
      memory:
        memory: 512MB
    connection:
      masterUrl: "{{ inputs.master_url }}"
      caCertData: "{{ inputs.ca_cert }}"
      oauthToken: "{{ inputs.token }}"

  - id: observe
    type: io.kestra.plugin.core.output.OutputValues
    values:
      signature: "{{ [outputs.inspect_release.revision,
        outputs.inspect_release.status, outputs.inspect_release.lastDeployed] |
        toJson }}"
      now_seconds: "{{ now() | timestamp }}"
      pending: "{{ outputs.inspect_release.status == 'pending-upgrade' or
        outputs.inspect_release.status == 'pending-rollback' }}"

  - id: reconcile_observation
    type: io.kestra.plugin.core.execution.Fail
    condition: "{{ (kv(render(vars.observation_key), errorOnMissing=false) ?? '{}')
      | toJson | length > 4096 }}"
    errorMessage: Observation exceeds the 4096-byte budget. Review the KV record
      before running again.

  - id: compute_observation
    type: io.kestra.plugin.core.output.OutputValues
    values:
      observation: >-
        {{ {'old': (kv(render(vars.observation_key), errorOnMissing=false) ??
        '{}'), 'signature': outputs.observe.values.signature, 'now':
        outputs.observe.values.now_seconds, 'pending':
        outputs.observe.values.pending} | toJson | jq('(.old | if type ==
        "string" then fromjson else . end) as $old | (.now | tonumber) as $now |
        {signature: .signature, first_seen: (if .pending == "true" and
        $old.signature == .signature and ($old.first_seen | type) == "number"
        and $old.first_seen >= 0 and $old.first_seen <= $now then
        $old.first_seen else $now end)}') | first | toJson }}

  - id: store_observation
    type: io.kestra.plugin.core.kv.Set
    key: "{{ render(vars.observation_key) }}"
    value: "{{ outputs.compute_observation.values.observation }}"

  - id: recovery_candidate
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.observe.values.pending == 'true' and
      ((outputs.observe.values.now_seconds | number) -
      fromJson(outputs.compute_observation.values.observation).first_seen) >=
      inputs.grace_seconds }}"
    then:
      - id: history_names
        type: io.kestra.plugin.core.output.OutputValues
        values:
          names: >-
            {{ {'name': inputs.release_name, 'revision':
            outputs.inspect_release.revision, 'window': inputs.history_window} |
            toJson | jq('. as $x | [range(([1, ($x.revision - $x.window)] |
            max); $x.revision) | "sh.helm.release.v1." + $x.name + ".v" +
            tostring]') | first | toJson }}

      - id: require_prior_revision
        type: io.kestra.plugin.core.execution.Fail
        condition: "{{ fromJson(outputs.history_names.values.names) | length == 0 }}"
        errorMessage: No prior revision exists. First installs require manual recovery.

      - id: inspect_history_metadata
        type: io.kestra.plugin.kubernetes.kubectl.Get
        namespace: "{{ inputs.release_namespace }}"
        resourceType: Secret
        resourcesNames: "{{ outputs.history_names.values.names }}"
        fetchType: FETCH
        connection:
          masterUrl: "{{ inputs.master_url }}"
          caCertData: "{{ inputs.ca_cert }}"
          oauthToken: "{{ inputs.token }}"

      - id: select_deployed_revision
        type: io.kestra.plugin.core.output.OutputValues
        values:
          revision: >-
            {{ {'items': outputs.inspect_history_metadata.metadataItems,
            'release': inputs.release_name, 'namespace':
            inputs.release_namespace, 'current':
            outputs.inspect_release.revision} | toJson | jq('. as $x | [.items[]
            | select(.namespace == $x.namespace and .labels.owner == "helm" and
            .labels.name == $x.release and .labels.status == "deployed") |
            select(.labels.version | test("^[1-9][0-9]*$")) | (.labels.version |
            tonumber) as $v | select($v < $x.current and .name ==
            ("sh.helm.release.v1." + $x.release + ".v" + ($v | tostring))) | $v]
            | max // 0') | first }}

      - id: require_deployed_history
        type: io.kestra.plugin.core.execution.Fail
        condition: "{{ (outputs.select_deployed_revision.values.revision | number) < 1 }}"
        errorMessage: No deployed revision found inside the bounded history window.
          Nothing is rolled back.

      - id: inspect_baseline
        type: io.kestra.plugin.helm.Status
        releaseName: "{{ inputs.release_name }}"
        namespace: "{{ inputs.release_namespace }}"
        revision: "{{ outputs.select_deployed_revision.values.revision }}"
        containerImage: alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2
        taskRunner:
          type: io.kestra.plugin.scripts.runner.docker.Docker
          entryPoint: []
          networkMode: "{{ inputs.docker_network }}"
          memory:
            memory: 512MB
        connection:
          masterUrl: "{{ inputs.master_url }}"
          caCertData: "{{ inputs.ca_cert }}"
          oauthToken: "{{ inputs.token }}"

      - id: require_baseline_deployed
        type: io.kestra.plugin.core.execution.Fail
        condition: "{{ outputs.inspect_baseline.status != 'deployed' }}"
        errorMessage: History label and native status disagree. Recovery requires manual
          review.

      - id: report_plan
        type: io.kestra.plugin.core.log.Log
        message: "Release {{ inputs.release_namespace }}/{{ inputs.release_name }}
          remains {{ outputs.inspect_release.status }} at revision {{
          outputs.inspect_release.revision }}. Proposed recovery revision {{
          outputs.inspect_baseline.revision }}. dry_run={{ inputs.dry_run }},
          auto_remediate={{ inputs.auto_remediate }}."

      - id: permitted_mode
        type: io.kestra.plugin.core.flow.If
        condition: "{{ inputs.auto_remediate and not inputs.dry_run }}"
        then:
          - id: approve_recovery
            type: io.kestra.plugin.core.flow.Pause
            pauseDuration: PT1H
            behavior: FAIL
            onResume:
              - id: approved
                type: BOOL
                defaults: false
                description: Approve the proposed revision after checking chart/data
                  compatibility and hook effects.
              - id: original_runner_stopped
                type: BOOL
                defaults: false
                description: Confirm the original Helm process is stopped and every release
                  writer honors the dedicated ConfigMap fence.

          - id: approved_change
            type: io.kestra.plugin.core.flow.If
            condition: "{{ (outputs.approve_recovery.onResume.approved ?? false) and
              (outputs.approve_recovery.onResume.original_runner_stopped ??
              false) }}"
            then:
              - id: read_fence
                type: io.kestra.plugin.kubernetes.kubectl.Get
                namespace: "{{ inputs.release_namespace }}"
                resourceType: ConfigMap
                resourcesNames:
                  - "{{ render(vars.fence_name) }}"
                fetchType: FETCH_ONE
                connection:
                  masterUrl: "{{ inputs.master_url }}"
                  caCertData: "{{ inputs.ca_cert }}"
                  oauthToken: "{{ inputs.token }}"

              - id: require_idle_fence
                type: io.kestra.plugin.core.execution.Fail
                condition: "{{ outputs.read_fence.size != 1 or
                  (outputs.read_fence.metadataItem.annotations['helm-recovery-r\
                  elease'] ?? '') != inputs.release_name or
                  (outputs.read_fence.metadataItem.annotations['helm-recovery-h\
                  older'] ?? 'missing') != '' }}"
                errorMessage: Dedicated release fence is absent, mismatched or held. Reconcile
                  its owner manually.

              - id: acquire_fence
                type: io.kestra.plugin.kubernetes.kubectl.Patch
                namespace: "{{ inputs.release_namespace }}"
                resourceType: ConfigMap
                resourceName: "{{ render(vars.fence_name) }}"
                patchStrategy: JSON_PATCH
                patch: >-
                  {{ [{'op': 'test', 'path': '/metadata/uid', 'value':
                  outputs.read_fence.metadataItem.uid}, {'op': 'test', 'path':
                  '/metadata/resourceVersion', 'value':
                  outputs.read_fence.metadataItem.resourceVersion}, {'op':
                  'test', 'path': '/metadata/annotations/helm-recovery-holder',
                  'value': ''}, {'op': 'replace', 'path':
                  '/metadata/annotations/helm-recovery-holder', 'value':
                  execution.id}] | toJson }}
                connection:
                  masterUrl: "{{ inputs.master_url }}"
                  caCertData: "{{ inputs.ca_cert }}"
                  oauthToken: "{{ inputs.token }}"

              - id: fenced_release
                type: io.kestra.plugin.helm.Status
                releaseName: "{{ inputs.release_name }}"
                namespace: "{{ inputs.release_namespace }}"
                containerImage: alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2
                taskRunner:
                  type: io.kestra.plugin.scripts.runner.docker.Docker
                  entryPoint: []
                  networkMode: "{{ inputs.docker_network }}"
                  memory:
                    memory: 512MB
                connection:
                  masterUrl: "{{ inputs.master_url }}"
                  caCertData: "{{ inputs.ca_cert }}"
                  oauthToken: "{{ inputs.token }}"

              - id: reject_stale_plan
                type: io.kestra.plugin.core.execution.Fail
                condition: "{{ outputs.fenced_release.revision !=
                  outputs.inspect_release.revision or
                  outputs.fenced_release.status !=
                  outputs.inspect_release.status or
                  outputs.fenced_release.lastDeployed !=
                  outputs.inspect_release.lastDeployed }}"
                errorMessage: Release changed during approval. Fence is retained. Inspect state
                  and restart with a fresh observation.

              - id: fenced_baseline
                type: io.kestra.plugin.helm.Status
                releaseName: "{{ inputs.release_name }}"
                namespace: "{{ inputs.release_namespace }}"
                revision: "{{ outputs.inspect_baseline.revision }}"
                containerImage: alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2
                taskRunner:
                  type: io.kestra.plugin.scripts.runner.docker.Docker
                  entryPoint: []
                  networkMode: "{{ inputs.docker_network }}"
                  memory:
                    memory: 512MB
                connection:
                  masterUrl: "{{ inputs.master_url }}"
                  caCertData: "{{ inputs.ca_cert }}"
                  oauthToken: "{{ inputs.token }}"

              - id: reject_changed_baseline
                type: io.kestra.plugin.core.execution.Fail
                condition: "{{ outputs.fenced_baseline.status != 'deployed' or
                  outputs.fenced_baseline.revision !=
                  outputs.inspect_baseline.revision or
                  read(outputs.fenced_baseline.manifest) !=
                  read(outputs.inspect_baseline.manifest) }}"
                errorMessage: Approved baseline changed. Fence is retained for manual review.

              - id: recover_release
                type: io.kestra.plugin.helm.Rollback
                releaseName: "{{ inputs.release_name }}"
                namespace: "{{ inputs.release_namespace }}"
                revision: "{{ outputs.inspect_baseline.revision }}"
                wait: WATCHER
                timeout: PT2M
                cleanupOnFail: false
                forceReplace: false
                forceConflicts: false
                noHooks: false
                containerImage: alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2
                taskRunner:
                  type: io.kestra.plugin.scripts.runner.docker.Docker
                  entryPoint: []
                  networkMode: "{{ inputs.docker_network }}"
                  memory:
                    memory: 512MB
                connection:
                  masterUrl: "{{ inputs.master_url }}"
                  caCertData: "{{ inputs.ca_cert }}"
                  oauthToken: "{{ inputs.token }}"

              - id: verify_release
                type: io.kestra.plugin.helm.Status
                releaseName: "{{ inputs.release_name }}"
                namespace: "{{ inputs.release_namespace }}"
                containerImage: alpine/helm:4.3.0@sha256:a6cf54599ccb99d90cf0712b30f03fdb3cab062e6b94e0418cc4db7e8a1464b2
                taskRunner:
                  type: io.kestra.plugin.scripts.runner.docker.Docker
                  entryPoint: []
                  networkMode: "{{ inputs.docker_network }}"
                  memory:
                    memory: 512MB
                connection:
                  masterUrl: "{{ inputs.master_url }}"
                  caCertData: "{{ inputs.ca_cert }}"
                  oauthToken: "{{ inputs.token }}"

              - id: require_recovery
                type: io.kestra.plugin.core.execution.Fail
                condition: "{{ outputs.verify_release.status != 'deployed' or
                  outputs.verify_release.revision !=
                  outputs.inspect_release.revision + 1 or
                  read(outputs.verify_release.manifest) !=
                  read(outputs.inspect_baseline.manifest) }}"
                errorMessage: Recovery did not restore the approved deployed manifest. Fence is
                  retained for manual reconciliation.

              - id: release_fence
                type: io.kestra.plugin.kubernetes.kubectl.Patch
                namespace: "{{ inputs.release_namespace }}"
                resourceType: ConfigMap
                resourceName: "{{ render(vars.fence_name) }}"
                patchStrategy: JSON_PATCH
                patch: >-
                  {{ [{'op': 'test', 'path': '/metadata/uid', 'value':
                  outputs.acquire_fence.metadata.uid}, {'op': 'test', 'path':
                  '/metadata/annotations/helm-recovery-holder', 'value':
                  execution.id}, {'op': 'replace', 'path':
                  '/metadata/annotations/helm-recovery-holder', 'value': ''}] |
                  toJson }}
                connection:
                  masterUrl: "{{ inputs.master_url }}"
                  caCertData: "{{ inputs.ca_cert }}"
                  oauthToken: "{{ inputs.token }}"

  - id: summary
    type: io.kestra.plugin.core.log.Log
    message: "Helm pending recovery check complete for {{ inputs.release_namespace
      }}/{{ inputs.release_name }}. Initial status {{
      outputs.inspect_release.status }}, initial revision {{
      outputs.inspect_release.revision }}."

errors:
  - id: notify_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: >-
      {{ {'text': 'Helm pending recovery failed for ' ~ inputs.release_namespace
      ~ '/' ~ inputs.release_name ~ '. Execution ' ~ execution.id ~ '. Inspect
      the release and any retained recovery fence before retrying.'} | toJson }}

triggers:
  - id: periodic_check
    type: io.kestra.plugin.core.trigger.Schedule
    disabled: true
    cron: "*/10 * * * *"
    inputs:
      dry_run: true
      auto_remediate: false