Esql

id: bulk_load_and_query
namespace: company.team

tasks:
  - id: extract
    type: io.kestra.plugin.core.http.Download
    uri: https://huggingface.co/datasets/kestra/datasets/resolve/main/jsonl/books.jsonl

  - id: load
    type: io.kestra.plugin.elasticsearch.Bulk
    from: "{{ outputs.extract.uri }}"

  - id: sleep
    type: io.kestra.plugin.core.flow.Sleep
    duration: PT5S
    description: Pause needed after load before we can query

  - id: query
    type: io.kestra.plugin.elasticsearch.Esql
    fetchType: STORE
    query: |
      FROM books
        | KEEP author, name, page_count, release_date
        | SORT page_count DESC
        | LIMIT 5

pluginDefaults:
  - type: io.kestra.plugin.elasticsearch
    values:
      connection:
        headers:
          - "Authorization: ApiKey yourEncodedApiKey"
        hosts:
          - https://yourCluster.us-central1.gcp.cloud.es.io:443