IngestDocument

yaml
type: "io.kestra.plugin.ai.rag.IngestDocument"

Examples

yaml
id: document_ingestion
namespace: company.ai

tasks:
  - id: ingest
    type: io.kestra.plugin.ai.rag.IngestDocument
    provider:
      type: io.kestra.plugin.ai.provider.GoogleGemini
      modelName: gemini-embedding-exp-03-07
      apiKey: "{{ kv('GEMINI_API_KEY') }}"
    embeddings:
      type: io.kestra.plugin.ai.embeddings.KestraKVStore
    drop: true
    fromExternalURLs:
      - https://raw.githubusercontent.com/kestra-io/docs/refs/heads/main/content/blogs/release-0-24.md

Properties

embeddings *

Definitions

Store embeddings in Chroma

baseUrl*string

collectionName*string

type*object

Possible Values

io.kestra.plugin.ai.embeddings.Chromaio.kestra.plugin.langchain4j.embeddings.Chroma

Store embeddings in Elasticsearch

connection*

io.kestra.plugin.ai.embeddings.Elasticsearch-ElasticsearchConnection

hosts*array

SubTypestring

Min items1

basicAuth

io.kestra.plugin.ai.embeddings.Elasticsearch-ElasticsearchConnection-BasicAuth

passwordstring

usernamestring

headersarray

SubTypestring

pathPrefixstring

strictDeprecationModebooleanstring

trustAllSslbooleanstring

indexName*string

type*object

Possible Values

io.kestra.plugin.ai.embeddings.Elasticsearchio.kestra.plugin.langchain4j.embeddings.Elasticsearch

Prototype embeddings in Kestra KV

type*object

Possible Values

io.kestra.plugin.ai.embeddings.KestraKVStoreio.kestra.plugin.langchain4j.embeddings.KestraKVStore

kvNamestring

Default{{flow.id}}-embedding-store

Store embeddings in MariaDB

createTable*booleanstring

databaseUrl*string

fieldName*string

password*string

tableName*string

type*object

username*string

columnDefinitionsarray

SubTypestring

indexesarray

SubTypestring

metadataStorageModestring

DefaultCOLUMN_PER_KEY

Store embeddings in Milvus

token*string

type*object

Possible Values

io.kestra.plugin.ai.embeddings.Milvusio.kestra.plugin.langchain4j.embeddings.Milvus

autoFlushOnDeletebooleanstring

autoFlushOnInsertbooleanstring

collectionNamestring

consistencyLevelstring

databaseNamestring

hoststring

idFieldNamestring

indexTypestring

metadataFieldNamestring

metricTypestring

passwordstring

portintegerstring

retrieveEmbeddingsOnSearchbooleanstring

textFieldNamestring

uristring

usernamestring

vectorFieldNamestring

Store embeddings in MongoDB Atlas

collectionName*string

host*string

indexName*string

scheme*string

type*object

Possible Values

io.kestra.plugin.ai.embeddings.MongoDBAtlasio.kestra.plugin.langchain4j.embeddings.MongoDBAtlas

createIndexbooleanstring

databasestring

metadataFieldNamesarray

SubTypestring

optionsobject

passwordstring

usernamestring

Store embeddings with pgvector

database*string

host*string

password*string

port*integerstring

table*string

type*object

Possible Values

io.kestra.plugin.ai.embeddings.PGVectorio.kestra.plugin.langchain4j.embeddings.PGVector

user*string

useIndexbooleanstring

Defaultfalse

Store embeddings in Pinecone

apiKey*string

cloud*string

index*string

region*string

type*object

Possible Values

io.kestra.plugin.ai.embeddings.Pineconeio.kestra.plugin.langchain4j.embeddings.Pinecone

namespacestring

Store embeddings in Qdrant

apiKey*string

collectionName*string

host*string

port*integerstring

type*object

Possible Values

io.kestra.plugin.ai.embeddings.Qdrantio.kestra.plugin.langchain4j.embeddings.Qdrant

Store embeddings in Redis

host*string

port*integerstring

type*object

indexNamestring

Defaultembedding-index

Store embeddings in Alibaba Tablestore

accessKeyId*string

accessKeySecret*string

endpoint*string

instanceName*string

type*object

metadataSchemaListarray

com.alicloud.openservices.tablestore.model.search.FieldSchema

analyzerstring

Possible Values

SingleWordMaxWordMinWordSplitFuzzy

analyzerParameter

com.alicloud.openservices.tablestore.model.search.analysis.AnalyzerParameter

dateFormatsarray

SubTypestring

enableHighlightingboolean

enableSortAndAggboolean

fieldNamestring

fieldTypestring

Possible Values

LONGDOUBLEBOOLEANKEYWORDTEXTNESTEDGEO_POINTDATEVECTORFUZZY_KEYWORDIPJSONUNKNOWN

indexboolean

indexOptionsstring

Possible Values

DOCSFREQSPOSITIONSOFFSETS

isArrayboolean

jsonTypestring

Possible Values

FLATTENNESTED

sourceFieldNamesarray

SubTypestring

storeboolean

subFieldSchemasarray

com.alicloud.openservices.tablestore.model.search.FieldSchema

analyzerstring

Possible Values

SingleWordMaxWordMinWordSplitFuzzy

analyzerParameter

dateFormatsarray

SubTypestring

enableHighlightingboolean

enableSortAndAggboolean

fieldNamestring

fieldTypestring

Possible Values

LONGDOUBLEBOOLEANKEYWORDTEXTNESTEDGEO_POINTDATEVECTORFUZZY_KEYWORDIPJSONUNKNOWN

indexboolean

indexOptionsstring

Possible Values

DOCSFREQSPOSITIONSOFFSETS

isArrayboolean

jsonTypestring

Possible Values

FLATTENNESTED

sourceFieldNamesarray

SubTypestring

storeboolean

subFieldSchemasarray

vectorOptions

com.alicloud.openservices.tablestore.model.search.vector.VectorOptions

dataTypestring

dimensioninteger

metricTypestring

Possible Values

EUCLIDEANCOSINEDOT_PRODUCT

Store embeddings in Weaviate

apiKey*string

host*string

type*object

Possible Values

io.kestra.plugin.ai.embeddings.Weaviateio.kestra.plugin.langchain4j.embeddings.Weaviate

avoidDupsbooleanstring

consistencyLevelstring

Possible Values

ONEQUORUMALL

grpcPortintegerstring

metadataFieldNamestring

metadataKeysarray

SubTypestring

objectClassstring

portintegerstring

schemestring

securedGrpcbooleanstring

useGrpcForInsertsbooleanstring

provider *

Definitions

Use Amazon Bedrock models

accessKeyId*string

modelName*string

secretAccessKey*string

type*object

Possible Values

io.kestra.plugin.ai.provider.AmazonBedrockio.kestra.plugin.langchain4j.provider.AmazonBedrock

baseUrlstring

caPemstring

clientPemstring

modelTypestring

DefaultCOHERE

Possible Values

COHERETITAN

Use Anthropic Claude models

apiKey*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.Anthropicio.kestra.plugin.langchain4j.provider.Anthropic

baseUrlstring

caPemstring

clientPemstring

maxTokensintegerstring

Use Azure OpenAI deployments

endpoint*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.AzureOpenAIio.kestra.plugin.langchain4j.provider.AzureOpenAI

apiKeystring

baseUrlstring

caPemstring

clientIdstring

clientPemstring

clientSecretstring

serviceVersionstring

tenantIdstring

Use DashScope (Qwen) models

apiKey*string

modelName*string

type*object

baseUrlstring

Defaulthttps://dashscope-intl.aliyuncs.com/api/v1

caPemstring

clientPemstring

enableSearchbooleanstring

maxTokensintegerstring

repetitionPenaltynumberstring

Use DeepSeek models

apiKey*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.DeepSeekio.kestra.plugin.langchain4j.provider.DeepSeek

baseUrlstring

Defaulthttps://api.deepseek.com/v1

caPemstring

clientPemstring

Use GitHub Models via Azure AI Inference

gitHubToken*string

modelName*string

type*object

baseUrlstring

caPemstring

clientPemstring

Use Google Gemini models

apiKey*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.GoogleGeminiio.kestra.plugin.langchain4j.provider.GoogleGemini

baseUrlstring

caPemstring

clientPemstring

embeddingModelConfiguration

io.kestra.plugin.ai.provider.GoogleGemini-EmbeddingModelConfiguration

maxRetriesintegerstring

outputDimensionalityintegerstring

taskTypestring

Possible Values

RETRIEVAL_QUERYRETRIEVAL_DOCUMENTSEMANTIC_SIMILARITYCLASSIFICATIONCLUSTERINGQUESTION_ANSWERINGFACT_VERIFICATION

timeoutstring

titleMetadataKeystring

Use Google Vertex AI models

endpoint*string

location*string

modelName*string

project*string

type*object

Possible Values

io.kestra.plugin.ai.provider.GoogleVertexAIio.kestra.plugin.langchain4j.provider.GoogleVertexAI

baseUrlstring

caPemstring

clientPemstring

Use Hugging Face Inference endpoints

apiKey*string

modelName*string

type*object

baseUrlstring

Defaulthttps://router.huggingface.co/v1

caPemstring

clientPemstring

Use LocalAI OpenAI-compatible server

baseUrl*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.LocalAIio.kestra.plugin.langchain4j.provider.LocalAI

caPemstring

clientPemstring

Use Mistral models

apiKey*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.MistralAIio.kestra.plugin.langchain4j.provider.MistralAI

baseUrlstring

caPemstring

clientPemstring

Use OCI Generative AI models

compartmentId*string

modelName*string

region*string

type*object

authProviderstring

baseUrlstring

caPemstring

clientPemstring

Use local Ollama models

endpoint*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.Ollamaio.kestra.plugin.langchain4j.provider.Ollama

baseUrlstring

caPemstring

clientPemstring

Use OpenAI models

apiKey*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.OpenAIio.kestra.plugin.langchain4j.provider.OpenAI

baseUrlstring

Defaulthttps://api.openai.com/v1

caPemstring

clientPemstring

Use OpenRouter models

apiKey*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.OpenRouterio.kestra.plugin.langchain4j.provider.OpenRouter

baseUrlstring

caPemstring

clientPemstring

Use IBM watsonx.ai models

apiKey*string

modelName*string

projectId*string

type*object

baseUrlstring

caPemstring

clientPemstring

Use Cloudflare Workers AI models

accountId*string

apiKey*string

modelName*string

type*object

Possible Values

io.kestra.plugin.ai.provider.WorkersAIio.kestra.plugin.langchain4j.provider.WorkersAI

baseUrlstring

caPemstring

clientPemstring

Use ZhiPu AI models

apiKey*string

modelName*string

type*object

baseUrlstring

Defaulthttps://open.bigmodel.cn/

caPemstring

clientPemstring

maxRetriesintegerstring

maxTokenintegerstring

stopsarray

SubTypestring

bulkSize integerstring

Default500

documentSplitter

Definitions

io.kestra.plugin.ai.rag.IngestDocument-DocumentSplitter

maxOverlapSizeInChars*integer

maxSegmentSizeInChars*integer

splitterstring

DefaultRECURSIVE

Possible Values

RECURSIVEPARAGRAPHLINESENTENCEWORD

drop booleanstring

Defaultfalse

fromDocuments array

Definitions

io.kestra.plugin.ai.rag.IngestDocument-InlineDocument

content*string

metadataobject

fromExternalURLs array

SubTypestring

fromInternalURIs array

SubTypestring

fromPath string

metadata object

SubTypestring

Outputs

embeddingStoreOutputs object

ingestedDocuments integer

inputTokenCount integer

outputTokenCount integer

totalTokenCount integer

Metrics

indexed.documents counter

Unitrecords

input.token.count counter

Unittoken

output.token.count counter

Unittoken

total.token.count counter

Unittoken

Run DeepSeek chat and JSON completions

Tasks that call DeepSeek chat models for conversational or schema-guided JSON responses.

Perplexity

Tasks that request chat completions from Perplexity models.

Google Cloud

Execute tasks and orchestrate workflows across Google Cloud Platform services. Centralize authentication and resource management for seamless GCP integration.

CloudDataInfrastructureAI

IngestDocument

More Plugins in this Category

Run DeepSeek chat and JSON completions

Perplexity

Google Cloud

1.6.0