Sync Flows, Files, and KV Stores from GitLab with GitOps Best Practices

id: git-flows-files-kv-sync-from-gitlab
namespace: company.team
description: Sync Kestra flows, files, and key-value pairs from a GitLab
  repository to Kestra

variables:
  your_git_repository: https://gitlab.com/your_orga/some_kestra_repo
  kestra_local_uri: http://localhost:8080
  your_project_id: "254"

labels:
  type: ADMIN_PROCESS

inputs:
  - id: branch_target
    type: SELECT
    allowCustomValue: false
    values:
      - main
      - kestra
    required: true

tasks:
  - id: get_flows_targets
    type: io.kestra.plugin.core.http.Request
    method: GET
    uri: "https://gitlab.com/api/v4/projects/{{ vars.your_project_id
      }}/repository/tree/?ref={{ inputs.branch_target
      }}&recursive=false&path=_flows&access_token={{ secret('GITLAB_API_TOKEN')
      }}"

  - id: for_each_flows_folder
    type: io.kestra.plugin.core.flow.ForEach
    values: |
      [{% for elem in json(outputs.get_flows_targets['body']) %}"{{ elem.name }}"{% if not loop.last %},{% endif %}{% endfor %}]
    tasks:
      - id: pull_flows
        type: io.kestra.plugin.git.SyncFlows
        branch: "{{ inputs.branch_target }}"
        gitDirectory: "_flows/{{ taskrun.value }}"
        targetNamespace: "{{ taskrun.value }}"
        url: "{{ vars.your_git_repository }}"
        password: "{{ secret('GITLAB_API_TOKEN') }}"
        username: "Depends on if you use a token or not."
        dryRun: false

  - id: git_target
    type: io.kestra.plugin.core.http.Request
    method: GET
    uri: "https://gitlab.com/api/v4/projects/{{ vars.your_project_id
      }}/repository/tree/?ref={{ inputs.branch_target
      }}&recursive=false&path=_files&access_token={{ secret('GITLAB_API_TOKEN')
      }}"

  - id: for_each_files_folder
    type: io.kestra.plugin.core.flow.ForEach
    values: |
      [{% for elem in json(outputs.git_target['body']) %}"{{ elem.name }}"{% if not loop.last %},{% endif %}{% endfor %}]
    tasks:
      - id: pull_files
        type: io.kestra.plugin.git.SyncNamespaceFiles
        branch: "{{ inputs.branch_target }}"
        gitDirectory: "_files/{{ taskrun.value }}"
        namespace: "{{ taskrun.value }}"
        url: "{{ vars.your_git_repository }}"
        password: "{{ secret('GITLAB_API_TOKEN') }}"
        username: "Depends on if you use a token or not."
        dryRun: false

  - id: fetch_kv
    type: io.kestra.plugin.core.http.Request
    method: GET
    uri: "https://gitlab.com/api/v4/projects/{{ vars.your_project_id
      }}/repository/files/_kv%2FkvStore.json?ref={{ inputs.branch_target
      }}&recursive=false&path=_flows&access_token={{ secret('GITLAB_API_TOKEN')
      }}"

  - id: for_each_namespace_store
    type: io.kestra.plugin.core.flow.ForEach
    values: "{{ json(outputs.fetch_kv['body']).content | base64decode | split('

      ') }}"
    tasks:
      - id: for_each_key
        type: io.kestra.plugin.core.flow.ForEach
        values: "{{ json(taskrun.value).keys }}"
        tasks:
          - id: put_key
            type: io.kestra.plugin.core.http.Request
            method: PUT
            contentType: application/json
            uri: "{{ kv('kestra_local_uri') }}/api/v1/namespaces/{{
              json(parent.taskrun.value).namespace }}/kv/{{
              json(taskrun.value).key }}"
            headers:
              Authorization: "{{ secret('KESTRA_API_TOKEN') }}"
            body: |
              {% if json(taskrun.value).body.type == "STRING" %}"{% endif %}{{ json(taskrun.value).body.value }}{% if json(taskrun.value).body.type == "STRING" %}"{% endif %}