id: dlt-zendesk-to-duckdb
namespace: company.team

tasks:
  - id: dlt_pipeline
    type: io.kestra.plugin.scripts.python.Script
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    containerImage: python:3.11
    beforeCommands:
      - pip install dlt[duckdb]
      - dlt --non-interactive init zendesk duckdb
    warningOnStdErr: false
    env:
      SOURCES__ZENDESK__ZENDESK_SUPPORT__CREDENTIALS__PASSWORD: "{{ secret('ZENDESK_PASSWORD') }}"
      SOURCES__ZENDESK__ZENDESK_SUPPORT__CREDENTIALS__SUBDOMAIN: "{{ secret('ZENDESK_SUBDOMAIN') }}"
      SOURCES__ZENDESK__ZENDESK_SUPPORT__CREDENTIALS__EMAIL: "{{ secret('ZENDESK_EMAIL') }}"
    script: |
      import dlt
      from zendesk import zendesk_support

      pipeline = dlt.pipeline(
          pipeline_name="zendesk_pipeline",
          destination="duckdb",
          dataset_name="zendesk",
      )

      zendesk_source = zendesk_support(load_all=False)
      tickets = zendesk_source.tickets

      load_info = pipeline.run(tickets)