id: pinecone-embeddings-upsert-pipeline
namespace: company.team
description: |
Load precomputed embeddings into Pinecone on a schedule. The flow downloads
a JSON export of embeddings, reshapes each record into Pinecone's vector
format with a small Python script, upserts the file in one call, and posts
the upserted count to Slack.
triggers:
- id: daily
type: io.kestra.plugin.core.trigger.Schedule
description: Refresh the vector index every morning after the upstream embedding
job completes.
cron: "0 5 * * *"
disabled: true
inputs:
- id: index_name
type: STRING
defaults: rag-docs
description: Name of the Pinecone index to load vectors into. Must already exist
with a dimension matching the embeddings.
- id: embeddings_url
type: STRING
defaults: "https://data.example.com/embeddings/latest.json"
description: URL of a JSON array of embedding records, each with an id field, an
embedding field holding the vector, and optional metadata fields. Point at
your embedding job's export.
tasks:
- id: download_embeddings
type: io.kestra.plugin.core.http.Download
description: Pull the embedding export into Kestra's internal storage so the
reshape step works on a local copy.
uri: "{{ inputs.embeddings_url }}"
- id: reshape_vectors
type: io.kestra.plugin.scripts.python.Script
description: Reshape each embedding record into Pinecone's vector format, one
map per line with id, values, and scalar metadata, written as a file for
the upsert.
taskRunner:
type: io.kestra.plugin.core.runner.Process
inputFiles:
embeddings.json: "{{ outputs.download_embeddings.uri }}"
outputFiles:
- vectors.ion
script: |
import json
with open("embeddings.json") as f:
records = json.load(f)
count = 0
with open("vectors.ion", "w") as out:
for record in records:
vector = {"id": str(record["id"]), "values": record["embedding"]}
metadata = {
key: value
for key, value in record.items()
if key not in ("id", "embedding")
and isinstance(value, (str, int, float, bool))
}
if metadata:
vector["metadata"] = metadata
out.write(json.dumps(vector) + "\n")
count += 1
print(f"reshaped {count} vectors")
- id: upsert_vectors
type: io.kestra.plugin.pinecone.Upsert
description: Upsert the reshaped file into the index in one call; reruns
overwrite the same vector ids instead of duplicating them.
apiKey: "{{ secret('PINECONE_API_KEY') }}"
indexName: "{{ inputs.index_name }}"
from: "{{ outputs.reshape_vectors.outputFiles['vectors.ion'] }}"
- id: notify
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Report how many vectors landed so the team sees every load at a glance.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "Pinecone ingestion finished: {{ outputs.upsert_vectors.upsertedCount }} vectors upserted into index {{ inputs.index_name }} from {{ inputs.embeddings_url }} (execution {{ execution.id }})."
}
errors:
- id: alert_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alert Slack when the pipeline fails so the retrieval corpus never
goes silently stale.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "Pinecone ingestion FAILED in flow {{ flow.id }} (execution {{ execution.id }}). Check the download, reshape, or upsert task logs; the index {{ inputs.index_name }} was not refreshed."
}