id: verified-ocr-pdf-archive
namespace: company.documents
description: OCR one PDF, independently validate PDF/A and page text, then
publish an atomic archive or quarantine bundle.
labels:
team: documents
use-case: verified-ocr
concurrency:
limit: 1
behavior: QUEUE
inputs:
- id: source_bucket
type: STRING
defaults: "{{ vars.inbox_bucket }}"
- id: source_key
type: STRING
defaults: samples/scanned.pdf
description: PDF object key for manual runs; the webhook may override only source_key.
- id: archive_bucket
type: STRING
defaults: ocr-archive
- id: endpoint
type: STRING
defaults: "{{ vars.s3_endpoint }}"
description: S3-compatible endpoint reachable from the Kestra worker; use the
regional S3 endpoint for AWS.
- id: region
type: STRING
defaults: "{{ vars.s3_region }}"
- id: dry_run
type: BOOL
defaults: true
description: Perform real OCR and validation; do not write accepted or
quarantine objects.
- id: language
type: STRING
defaults: eng
description: Tesseract language codes, such as eng or eng+deu; every requested
pack must be installed in ocr_image.
- id: max_pages
type: INT
defaults: 25
min: 1
max: 100
- id: max_size_mb
type: INT
defaults: 20
min: 1
max: 100
- id: min_text_chars_per_page
type: INT
defaults: 10
min: 1
max: 1000
variables:
inbox_bucket: ocr-inbox
inbox_prefix: samples/
s3_endpoint: http://seaweedfs:8333
s3_region: us-east-1
ocr_image: jbarlow83/ocrmypdf-alpine@sha256:a176f57611dc0537c54fa8583ab03aa563085ab85cfad2e24aeed8818ee5c8f3
validator_image: ghcr.io/verapdf/cli@sha256:de5843855864e20b2ffffe8aab9336810725468eecf06dccdc15df7a43527ef2
tasks:
- id: route_request
type: io.kestra.plugin.core.flow.If
description: S3 batches queue one independent execution per PDF; manual and
webhook runs process one object directly.
condition: "{{ trigger.objects is defined }}"
then:
- id: each_pdf
type: io.kestra.plugin.core.flow.Loop
values: "{{ trigger.objects | jq('.[].key') }}"
tasks:
- id: queue_pdf
type: io.kestra.plugin.core.flow.Subflow
description: Queue with the flow input defaults, without waiting for completion;
children validate and alert independently.
namespace: "{{ flow.namespace }}"
flowId: "{{ flow.id }}"
wait: false
inputs:
source_key: "{{ item.value }}"
else:
- id: validate_request
type: io.kestra.plugin.scripts.shell.Commands
description: Validate manual settings and the sole permitted webhook field
before any object download.
containerImage: "{{ vars.ocr_image }}"
env:
REQUEST: "{{ {'source_bucket': inputs.source_bucket, 'source_key':
trigger.body.source_key ?? inputs.source_key, 'archive_bucket':
inputs.archive_bucket, 'endpoint': inputs.endpoint, 'region':
inputs.region, 'language': inputs.language} | toJson }}"
commands:
- |
python3 - <<'PY'
import json, os, re
from urllib.parse import urlsplit
request = json.loads(os.environ['REQUEST'])
for field in ('source_bucket', 'archive_bucket'):
if not isinstance(request[field], str) or not re.fullmatch(r'[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]', request[field]):
raise ValueError('Invalid bucket: ' + field)
if not re.fullmatch(r'[A-Za-z0-9_]+(?:\+[A-Za-z0-9_]+)*', request['language']):
raise ValueError('language must contain Tesseract codes separated by +')
key = request['source_key']
if not isinstance(key, str) or not key.lower().endswith('.pdf') or len(key.encode()) > 1024 or any(ord(c) < 32 for c in key):
raise ValueError('source_key must be a PDF object key without control characters')
endpoint = urlsplit(request['endpoint'])
if endpoint.scheme not in ('http', 'https') or not endpoint.hostname or endpoint.username or endpoint.query or endpoint.fragment:
raise ValueError('Invalid S3 endpoint')
if request['source_bucket'] == request['archive_bucket'] or not request['region'].strip():
raise ValueError('Use distinct source/archive buckets and a nonempty region')
print('::' + json.dumps({'outputs': {'source_key': key}}) + '::')
PY
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint:
- ''
networkMode: none
delete: true
cpu:
cpus: 2
memory:
memory: 2GB
- id: download_source
type: io.kestra.plugin.aws.s3.Download
description: Read the requested source once; the original object is never changed.
bucket: "{{ inputs.source_bucket }}"
key: "{{ outputs.validate_request.vars.source_key }}"
endpointOverride: "{{ inputs.endpoint }}"
region: "{{ inputs.region }}"
accessKeyId: "{{ secret('OCR_S3_ACCESS_KEY_ID') }}"
secretKeyId: "{{ secret('OCR_S3_SECRET_ACCESS_KEY') }}"
compatibilityMode: true
forcePathStyle: true
retry:
type: constant
interval: PT5S
maxAttempts: 3
- id: process_pdf
type: io.kestra.plugin.core.flow.WorkingDirectory
description: Share this execution's private files between OCR and independent
validators; Kestra manages temporary-directory cleanup.
inputFiles:
source.pdf: "{{ outputs.download_source.uri }}"
tasks:
- id: inspect_source
type: io.kestra.plugin.scripts.shell.Commands
description: Hash the downloaded bytes and reject oversized, encrypted, damaged
or over-limit PDFs before OCR.
containerImage: "{{ vars.ocr_image }}"
env:
MAX_BYTES: "{{ inputs.max_size_mb * 1048576 }}"
MAX_PAGES: "{{ inputs.max_pages }}"
commands:
- |
python3 - <<'PY'
import hashlib, json, os
from pathlib import Path
import pikepdf
source = Path('source.pdf')
result = {'sha256': hashlib.file_digest(source.open('rb'), 'sha256').hexdigest(), 'bytes': source.stat().st_size, 'errors': []}
if result['bytes'] > int(os.environ['MAX_BYTES']):
result['errors'].append('Source exceeds max_size_mb (checked after download)')
else:
try:
with pikepdf.open(source, attempt_recovery=False) as pdf:
result['pages'] = len(pdf.pages)
if pdf.is_encrypted:
result['errors'].append('Encrypted PDF is not accepted')
if not 1 <= result['pages'] <= int(os.environ['MAX_PAGES']):
result['errors'].append('Source page count outside configured limit')
except (pikepdf.PdfError, pikepdf.PasswordError) as exc:
result['errors'].append(type(exc).__name__ + ': ' + str(exc))
result['accepted'] = not result['errors']
Path('inspection.json').write_text(json.dumps(result, indent=2))
PY
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint:
- ''
networkMode: none
user: "0:0" # Kestra VOLUME staging is root-owned; these tasks write shared files.
delete: true
cpu:
cpus: 2
memory:
memory: 2GB
- id: run_ocr
type: io.kestra.plugin.scripts.shell.Commands
description: Invoke real OCRmyPDF only for accepted sources; record document
rejections without losing their diagnostics.
containerImage: "{{ vars.ocr_image }}"
env:
LANGUAGE: "{{ inputs.language }}"
commands:
- |
python3 - <<'PY'
import json, os, subprocess
from pathlib import Path
import ocrmypdf
result = {'version': ocrmypdf.__version__, 'exit_code': None}
if json.loads(Path('inspection.json').read_text())['accepted']:
args = ['ocrmypdf', '--skip-text', '--output-type', 'pdfa-2', '--jobs', '2', '--language', os.environ['LANGUAGE'], 'source.pdf', 'searchable.pdf']
with open('ocr.log', 'w') as log:
result['exit_code'] = subprocess.run(args, stdout=log, stderr=subprocess.STDOUT, timeout=600).returncode
# Missing tools, invalid invocation, permissions and engine failures are operational errors.
if result['exit_code'] not in (0, 2, 4, 6, 8, 10):
raise RuntimeError('OCRmyPDF operational failure; see task logs: ' + Path('ocr.log').read_text()[-4000:])
Path('ocr.json').write_text(json.dumps(result, indent=2))
PY
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint:
- ''
networkMode: none
user: "0:0" # Kestra VOLUME staging is root-owned; these tasks write shared files.
delete: true
cpu:
cpus: 2
memory:
memory: 2GB
- id: validate_pdfa
type: io.kestra.plugin.scripts.shell.Commands
description: Independently validate the candidate as PDF/A-2b; absent XML or any
noncompliance is rejected by the next task.
containerImage: "{{ vars.validator_image }}"
commands:
- |
set -eu
/opt/verapdf/verapdf --version > validator-version.txt
if [ -s searchable.pdf ]; then
set +e
/opt/verapdf/verapdf --flavour 2b --format xml searchable.pdf > validation.xml 2> validator.log
status=$?
set -e
printf '%s' "$status" > validator-exit.txt
fi
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint:
- ''
networkMode: none
user: "0:0" # Kestra VOLUME staging is root-owned; these tasks write shared files.
delete: true
cpu:
cpus: 2
memory:
memory: 2GB
- id: verify_and_bundle
type: io.kestra.plugin.scripts.shell.Commands
description: Require OCR success, PDF/A compliance, unchanged page count and
text on every page; package measured evidence and checksums.
containerImage: "{{ vars.ocr_image }}"
env:
CONFIG: "{{ {'language': inputs.language, 'max_pages': inputs.max_pages,
'max_size_mb': inputs.max_size_mb, 'min_text_chars_per_page':
inputs.min_text_chars_per_page, 'ocr_image': vars.ocr_image,
'validator_image': vars.validator_image, 'pipeline_version': 2}
| toJson }}"
SOURCE_KEY: "{{ outputs.validate_request.vars.source_key }}"
SOURCE_BUCKET: "{{ inputs.source_bucket }}"
outputFiles:
- bundle.zip
- report.json
- searchable.pdf
- text.txt
- validation.xml
commands:
- |
python3 - <<'PY'
import hashlib, json, os, zipfile
from pathlib import Path
from defusedxml import ElementTree as ET
import pypdfium2 as pdfium
inspection = json.loads(Path('inspection.json').read_text())
ocr = json.loads(Path('ocr.json').read_text())
config = json.loads(os.environ['CONFIG'])
errors = list(inspection['errors'])
report = {'source_bucket': os.environ['SOURCE_BUCKET'], 'source_key': os.environ['SOURCE_KEY'], 'inspection': inspection, 'ocr': ocr, 'config': config, 'errors': errors, 'pdfa_compliant': False}
if inspection['accepted']:
if ocr['exit_code'] != 0:
errors.append('OCRmyPDF rejected document: exit ' + str(ocr['exit_code']))
if not Path('validation.xml').exists():
errors.append('Missing independent PDF/A report')
else:
root = ET.parse('validation.xml').getroot()
report['validator_exit_code'] = int(Path('validator-exit.txt').read_text())
if report['validator_exit_code'] != 0:
errors.append('veraPDF returned a nonzero exit code')
validations = root.findall('.//validationReport')
summary = root.find('.//batchSummary')
compliant = len(validations) == 1 and validations[0].get('isCompliant') == 'true'
clean = summary is not None and summary.get('totalJobs') == '1' and all(summary.get(k) == '0' for k in ('failedToParse', 'encrypted', 'outOfMemory', 'veraExceptions'))
report['pdfa_compliant'] = compliant and clean
if not report['pdfa_compliant']:
errors.append('Independent PDF/A-2b validation failed')
if ocr['exit_code'] == 0:
with pdfium.PdfDocument('searchable.pdf') as pdf:
texts = []
for page in pdf:
textpage = page.get_textpage()
texts.append(textpage.get_text_range())
textpage.close()
page.close()
report['text_chars_per_page'] = [sum(not c.isspace() for c in text) for text in texts]
if len(texts) != inspection['pages']:
errors.append('Output page count differs from source')
if any(n < config['min_text_chars_per_page'] for n in report['text_chars_per_page']):
errors.append('One or more pages below min_text_chars_per_page')
Path('text.txt').write_text('\n\f\n'.join(texts), encoding='utf-8')
accepted = not errors
report['status'] = 'ACCEPTED' if accepted else 'REJECTED'
identity = hashlib.sha256((inspection['sha256'] + json.dumps(config, sort_keys=True)).encode()).hexdigest()
key = ('accepted/' if accepted else 'quarantine/') + identity + '.zip'
report['destination_key'] = key
Path('report.json').write_text(json.dumps(report, indent=2), encoding='utf-8')
names = ['report.json', 'inspection.json', 'ocr.json', 'ocr.log', 'validation.xml', 'validator.log', 'validator-version.txt', 'validator-exit.txt']
names += ['searchable.pdf', 'text.txt'] if accepted else ['source.pdf']
names = [name for name in names if Path(name).is_file()]
manifest = {name: hashlib.sha256(Path(name).read_bytes()).hexdigest() for name in names}
with zipfile.ZipFile('bundle.zip', 'w', zipfile.ZIP_DEFLATED) as archive:
for name in names:
archive.write(name)
archive.writestr('manifest.json', json.dumps(manifest, indent=2))
print('::' + json.dumps({'outputs': {'accepted': accepted, 'destination_key': key, 'source_sha256': inspection['sha256'], 'errors': errors}}) + '::')
PY
taskRunner:
type: io.kestra.plugin.scripts.runner.docker.Docker
entryPoint:
- ''
networkMode: none
user: "0:0" # Kestra VOLUME staging is root-owned; these tasks write shared files.
delete: true
cpu:
cpus: 2
memory:
memory: 2GB
- id: find_existing
type: io.kestra.plugin.aws.s3.List
description: Check the exact content/configuration key before publishing; reruns
do not overwrite an existing archive.
bucket: "{{ inputs.archive_bucket }}"
prefix: "{{ outputs.verify_and_bundle.vars.destination_key }}"
regexp: "{{ '^' ~ (outputs.verify_and_bundle.vars.destination_key |
replace({'.': '\\.'})) ~ '$' }}"
maxFiles: 1
endpointOverride: "{{ inputs.endpoint }}"
region: "{{ inputs.region }}"
accessKeyId: "{{ secret('OCR_S3_ACCESS_KEY_ID') }}"
secretKeyId: "{{ secret('OCR_S3_SECRET_ACCESS_KEY') }}"
compatibilityMode: true
forcePathStyle: true
retry:
type: constant
interval: PT5S
maxAttempts: 3
- id: publication_plan
type: io.kestra.plugin.core.output.OutputValues
values:
dry_run: "{{ inputs.dry_run }}"
accepted: "{{ outputs.verify_and_bundle.vars.accepted }}"
action: "{{ (outputs.find_existing.objects | length) > 0 ? 'KEEP_EXISTING' :
'UPLOAD_BUNDLE' }}"
destination: "{{ 's3://' ~ inputs.archive_bucket ~ '/' ~
outputs.verify_and_bundle.vars.destination_key }}"
bundle_uri: "{{ outputs.verify_and_bundle.outputFiles['bundle.zip'] }}"
report_uri: "{{ outputs.verify_and_bundle.outputFiles['report.json'] }}"
- id: publish_if_live
type: io.kestra.plugin.core.flow.If
description: A dry run never writes to object storage; publish each
accepted/quarantine bundle with a single PUT.
condition: "{{ inputs.dry_run == false and (outputs.find_existing.objects |
length) == 0 }}"
then:
- id: upload_bundle
type: io.kestra.plugin.aws.s3.Upload
from: "{{ outputs.verify_and_bundle.outputFiles['bundle.zip'] }}"
bucket: "{{ inputs.archive_bucket }}"
key: "{{ outputs.verify_and_bundle.vars.destination_key }}"
contentType: application/zip
endpointOverride: "{{ inputs.endpoint }}"
region: "{{ inputs.region }}"
accessKeyId: "{{ secret('OCR_S3_ACCESS_KEY_ID') }}"
secretKeyId: "{{ secret('OCR_S3_SECRET_ACCESS_KEY') }}"
compatibilityMode: true
forcePathStyle: true
retry:
type: constant
interval: PT5S
maxAttempts: 3
- id: reject_invalid
type: io.kestra.plugin.core.flow.If
description: Fail rejected documents even in dry-run mode, after retaining their
report and any permitted quarantine upload.
condition: "{{ outputs.verify_and_bundle.vars.accepted == false }}"
then:
- id: document_rejected
type: io.kestra.plugin.core.execution.Fail
errorMessage: "Document rejected: {{ outputs.verify_and_bundle.vars.errors |
join('; ') }}. Report: {{
outputs.verify_and_bundle.outputFiles['report.json'] }}"
errors:
- id: first_error
type: io.kestra.plugin.core.output.OutputValues
values:
task: "{{ (errorLogs() | length) > 0 ? (errorLogs()[0].taskId ?? 'execution') :
'execution' }}"
message: "{{ (errorLogs() | length) > 0 ? errorLogs()[0].message : 'No task
error logged' }}"
link: "{{ (kestra.url ?? 'http://localhost:8080/') ~ 'ui/main/executions/' ~
flow.namespace ~ '/' ~ flow.id ~ '/' ~ execution.id }}"
- id: log_failure
type: io.kestra.plugin.core.log.Log
level: ERROR
message: "{{ flow.id }} / {{ execution.id }} failed in {{
outputs.first_error.values.task }}: {{ outputs.first_error.values.message
}} - {{ outputs.first_error.values.link }}"
- id: alert_failure
type: io.kestra.plugin.core.http.Request
description: Deliver structured failure evidence to a local HTTP receiver or an
organization's alert endpoint.
uri: "{{ secret('OCR_ALERT_WEBHOOK_URL') }}"
method: POST
contentType: application/json
body: "{{ {'flow_id': flow.id, 'execution_id': execution.id, 'failed_task':
outputs.first_error.values.task, 'message':
outputs.first_error.values.message, 'execution_url':
outputs.first_error.values.link} | toJson }}"
retry:
type: constant
interval: PT5S
maxAttempts: 3
triggers:
- id: inbox_pdf
type: io.kestra.plugin.aws.s3.Trigger
description: Poll the configured inbox and queue new or changed PDFs without
moving or deleting sources. Enable after a dry run.
disabled: true
interval: PT1M
bucket: "{{ vars.inbox_bucket }}"
prefix: "{{ vars.inbox_prefix }}"
regexp: '(?i).*\.pdf$'
filter: FILES
action: NONE
on: CREATE_OR_UPDATE
maxFiles: 1000
maxKeys: 1000
endpointOverride: "{{ vars.s3_endpoint }}"
region: "{{ vars.s3_region }}"
accessKeyId: "{{ secret('OCR_S3_ACCESS_KEY_ID') }}"
secretKeyId: "{{ secret('OCR_S3_SECRET_ACCESS_KEY') }}"
compatibilityMode: true
forcePathStyle: true
- id: pdf_arrived
type: io.kestra.plugin.core.trigger.Webhook
disabled: true
key: "{{ secret('OCR_WEBHOOK_KEY') }}"
fetchType: FETCH