Export All Kestra Namespaces to GitLab with Releases for GitOps Best Practices

id: git-flows-files-kv-sync-to-gitlab
namespace: company.team
description: "This flow saves all Kestra namespaces (flows, files, and KV pairs)
  to a Git repository on a weekly basis. Optionally, it can create a release and
  automate deployment."

variables:
  your_git_repository: https://gitlab.com/your_orga/some_kestra_repo
  kestra_local_uri: http://localhost:8080
  your_project_id: "254"

inputs:
  - id: make_new_release
    type: BOOL
    defaults: false

  - id: tag_name
    type: STRING
    dependsOn:
      inputs:
        - make_new_release
      condition: "{{ inputs.make_new_release }}"
    required: true

  - id: release_name
    type: STRING
    dependsOn:
      inputs:
        - make_new_release
      condition: "{{ inputs.make_new_release }}"
    required: true
    defaults: "Latest"
    description: "If set to 'Latest', the flow will create a pull request from this
      latest state to the MAIN branch from which saves are pulled by default."

  - id: additional_description
    type: STRING
    dependsOn:
      inputs:
        - make_new_release
      condition: "{{ inputs.make_new_release }}"
    required: false

labels:
  type: ADMIN_PROCESS

tasks:
  - id: get_namespaces
    type: io.kestra.plugin.core.flow.Subflow
    flowId: get-all-namespaces
    namespace: company.team

  - id: for_each_namespace
    type: io.kestra.plugin.core.flow.ForEach
    values: "{{ outputs.get_namespaces['outputs']['namespaces'] }}"
    concurrencyLimit: 1
    tasks:
      - id: pushes_and_kv_storing
        type: io.kestra.plugin.core.flow.Parallel
        tasks:
          - id: kv_storing
            type: io.kestra.plugin.core.flow.Sequential
            tasks:
              - id: get_keys
                type: io.kestra.plugin.core.http.Request
                uri: "{{ vars.kestra_local_uri }}/api/v1/namespaces/{{ parent.taskrun.value
                  }}/kv/"
              - id: for_each_key
                type: io.kestra.plugin.core.flow.ForEach
                concurrencyLimit: 0
                values: "{{ outputs.get_keys[parent.taskrun.value].body }}"
                tasks:
                  - id: retrieve_values
                    type: io.kestra.plugin.core.http.Request
                    uri: "{{ vars.kestra_local_uri }}/api/v1/namespaces/{{ parents[0].taskrun.value
                      }}/kv/{{ json(taskrun.value).key }}"
          - id: pushes
            type: io.kestra.plugin.core.flow.Sequential
            tasks:
              - id: push_files
                type: io.kestra.plugin.git.PushNamespaceFiles
                commitMessage: Add files from `{{ parents[0].taskrun.value }}` namespace.
                gitDirectory: "_files/{{ parents[0].taskrun.value }}"
                namespace: "{{ parents[0].taskrun.value }}"
                url: "{{ vars.your_git_repository }}"
                password: "{{ secret('GITLAB_API_TOKEN') }}"
                username: "Depends on if you use a token or not."
                dryRun: false
              - id: push_flows
                type: io.kestra.plugin.git.PushFlows
                gitDirectory: "_flows/{{ parents[0].taskrun.value }}"
                commitMessage: Add flows from `{{ parents[0].taskrun.value }}` namespace.
                includeChildNamespaces: false
                sourceNamespace: "{{ parents[0].taskrun.value }}"
                targetNamespace: "{{ parents[0].taskrun.value }}"
                url: "{{ vars.your_git_repository }}"
                password: "{{ secret('GITLAB_API_TOKEN') }}"
                username: "Depends on if you use a token or not."

  - id: set_wd
    type: io.kestra.plugin.core.flow.WorkingDirectory
    inputFiles:
      kvStore.json: |
        {% for elem in outputs.retrieve_values %}{"namespace":"{{ elem.key }}","keys":[{% for ele in (elem.value) %}{"key":"{{ json(ele.key).key }}","body":{{ ele.value.body }}}{% if not loop.last %},{% endif %}{% endfor %}]}
        {% endfor %}
    tasks:
      - id: storeKv
        type: io.kestra.plugin.core.namespace.UploadFiles
        conflict: OVERWRITE
        files:
          - kvStore.json
        namespace: kestra_export_import
        destination: "/_kv/"

  - id: push_kv
    type: io.kestra.plugin.git.PushNamespaceFiles
    commitMessage: Add KV pairs
    gitDirectory: "."
    files: /_kv/**
    url: "{{ vars.your_git_repository }}"
    password: "{{ secret('GITLAB_API_TOKEN') }}"
    username: "Depends on if you use a token or not."

  - id: delete_stored_kv
    type: io.kestra.plugin.core.namespace.DeleteFiles
    namespace: kestra_export_import
    files: /_kv/**

  - id: make_release
    type: io.kestra.plugin.core.http.Request
    runIf: "{{ inputs.make_new_release }}"
    method: POST
    contentType: application/json
    uri: "https://gitlab.com/api/v4/projects/{{ vars.your_project_id
      }}/releases?ref=kestra&access_token={{ secret('GITLAB_API_TOKEN') }}"
    body: |
      { "name": "{{ inputs.release_name }}", "tag_name": "{{ inputs.tag_name }}", "description": "{{ inputs.additional_description }}"}

  - id: create_pull_request
    type: io.kestra.plugin.core.http.Request
    runIf: "{{ inputs.release_name equals 'Latest' }}"
    method: POST
    contentType: application/json
    uri: "https://gitlab.com/api/v4/projects/{{ vars.your_project_id
      }}/merge_requests?title=lts_up&source_branch=kestra&target_branch=main&ac\
      cess_token={{ secret('GITLAB_API_TOKEN') }}"

pluginDefaults:
  - type: io.kestra.plugin.core.http.Request
    values:
      method: GET
      headers:
        Authorization: "{{ secret('KESTRA_API_TOKEN') }}"

triggers:
  - id: weekly_saves
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "0 0 7,14,21,28 * *"