Source
yaml
id: airbyte-sync-parallel-with-dbt
namespace: company.team
tasks:
- id: data_ingestion
type: io.kestra.plugin.core.flow.Parallel
tasks:
- id: salesforce
type: io.kestra.plugin.airbyte.connections.Sync
connectionId: e3b1ce92-547c-436f-b1e8-23b6936c12ab
- id: google_analytics
type: io.kestra.plugin.airbyte.connections.Sync
connectionId: e3b1ce92-547c-436f-b1e8-23b6936c12cd
- id: facebook_ads
type: io.kestra.plugin.airbyte.connections.Sync
connectionId: e3b1ce92-547c-436f-b1e8-23b6936c12ef
- id: dbt
type: io.kestra.plugin.core.flow.WorkingDirectory
tasks:
- id: clone_repository
type: io.kestra.plugin.git.Clone
url: https://github.com/kestra-io/dbt-demo
branch: main
- id: dbt_build
type: io.kestra.plugin.dbt.cli.Build
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
containerImage: ghcr.io/kestra-io/dbt-bigquery:latest
dbtPath: /usr/local/bin/dbt
inputFiles:
.profile/profiles.yml: |
jaffle_shop:
outputs:
dev:
type: bigquery
dataset: your_big_query_dataset_name
project: your_big_query_project
fixed_retries: 1
keyfile: sa.json
location: EU
method: service-account
priority: interactive
threads: 8
timeout_seconds: 300
target: dev
sa.json: "{{ secret('GCP_CREDS') }}"
pluginDefaults:
- type: io.kestra.plugin.airbyte.connections.Sync
values:
url: http://host.docker.internal:8000/
username: "{{ secret('AIRBYTE_USERNAME') }}"
password: "{{ secret('AIRBYTE_PASSWORD') }}"
About this blueprint
Parallel Ingest BigQuery Git dbt
This flow executes several Airbyte syncs in parallel and then runs dbt Core's CLI commands.
It's recommended to set GCP and Airbyte server credentials as secrets.
More Related Blueprints