id: elastic-esql-query-to-python-pandas-dataframe
namespace: company.team

tasks:
  - id: esql_query
    type: io.kestra.plugin.elasticsearch.Request
    connection:
      hosts:
        - "https://b08318eb65f34bfdadcca4a633ea1602.us-central1.gcp.cloud.es.io\
          :443"
      headers:
        - "Authorization: ApiKey {{ secret('ES_API_TOKEN') }}"
    method: "POST"
    endpoint: /_query?format=csv
    body:
      query: |
        FROM test
        | WHERE title IS NOT NULL

  - id: pandas
    type: io.kestra.plugin.scripts.python.Script
    inputFiles:
      data.csv: "{{ outputs.esql_query.response }}"
    beforeCommands:
      - pip install pandas
    script: |
      import pandas as pd
      data = pd.read_csv("data.csv", sep=',')
      print(data.head())