Document Ingestion Quality Gate with Apache Tika and Slack Alerts

id: tika-document-quality-gate
namespace: company.team
description: |
  Audit web documents with Apache Tika, normalize Content-Type MIME metadata,
  enforce character length thresholds, and send Slack alerts for quality failures.

inputs:
  - id: document_urls
    type: ARRAY
    itemType: STRING
    displayName: Document URLs to audit
    description: List of public document URLs (PDF, TXT, HTML) to parse and evaluate.
    defaults:
      - "https://huggingface.co/datasets/kestra/datasets/resolve/main/pdf/app_s\
        tore.pdf"
      - "https://www.rfc-editor.org/rfc/rfc2324.txt"
      - "https://docs.python.org/3/license.html"
      - "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"

  - id: min_chars
    type: SELECT
    displayName: Minimum character count threshold
    description: Minimum number of extracted text characters required for a document
      to pass quality audit.
    values:
      - "500"
      - "1000"
      - "5000"
    defaults: "1000"

triggers:
  - id: weekly_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "0 8 * * 1"
    disabled: true

tasks:
  - id: process_documents
    type: io.kestra.plugin.core.flow.Loop
    description: Iterate through input document URLs, extract text with Tika, and
      enforce quality criteria.
    values: "{{ inputs.document_urls }}"
    tasks:
      - id: download_document
        type: io.kestra.plugin.core.http.Download
        description: Stream document from web source into internal storage.
        uri: "{{ item.value }}"

      - id: parse_document
        type: io.kestra.plugin.tika.Parse
        description: Parse document text and metadata with Apache Tika.
        from: "{{ outputs.download_document.uri }}"
        contentType: TEXT
        store: false

      - id: normalize_mime
        type: io.kestra.plugin.core.debug.Return
        description: Normalize Tika Content-Type metadata header by stripping parameters.
        format: "{{ ((outputs.parse_document.result.metadata['Content-Type'] ?? '') |
          string) | split(';') | first | trim }}"

      - id: check_quality
        type: io.kestra.plugin.core.flow.If
        description: Validate character count threshold and normalized MIME type.
        condition: "{{ ((outputs.parse_document.result.content ?? '') | length) <
          (inputs.min_chars | int) or not (['application/pdf', 'text/plain',
          'text/html'] contains outputs.normalize_mime.value) }}"
        then:
          - id: alert_quality_failure
            type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
            url: "{{ secret('SLACK_WEBHOOK_URL') }}"
            payload: |
              {
                "text": "Document Quality Gate Failure Alert\nURL: {{ item.value }}\nExtracted Characters: {{ (outputs.parse_document.result.content ?? '') | length }} (Threshold: {{ inputs.min_chars }})\nDetected MIME: {{ outputs.normalize_mime.value }}\nStatus: FAILED quality gate inspection (character threshold or MIME boundary violation)."
              }
        else:
          - id: log_quality_pass
            type: io.kestra.plugin.core.log.Log
            message: "PASS: Document {{ item.value }} passed quality audit ({{
              (outputs.parse_document.result.content ?? '') | length }} chars,
              MIME: {{ outputs.normalize_mime.value }})."

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "Document Quality Pipeline Execution Error\nFlow tika-document-quality-gate encountered an uncaught exception during execution. Please inspect run logs."
      }