Firecrawl Web Scraping Job Error and Stagnation Sentinel

id: firecrawl-web-scraping-job-error-sentinel
namespace: company.ai

inputs:
  - id: firecrawl_api_url
    type: STRING
    defaults: "https://api.firecrawl.dev/v1"
    description: Base URL of the Firecrawl web scraping API.
  - id: crawl_job_id
    type: STRING
    description: Target crawl job ID to audit.

tasks:
  - id: inspect_crawl_status
    type: io.kestra.plugin.scripts.shell.Commands
    commands:
      - |
        # Query Firecrawl API for crawl job status
        curl -s -H "Authorization: Bearer {{ secret('FIRECRAWL_API_KEY') }}" \
          "{{ inputs.firecrawl_api_url }}/crawl/{{ inputs.crawl_job_id }}" > /tmp/firecrawl_res.json

        python3 -c "
        import json
        try:
            with open('/tmp/firecrawl_res.json') as f:
                data = json.load(f)
            status = data.get('status', 'unknown')
            total = data.get('total', 0)
            completed = data.get('completed', 0)
            is_failed = 1 if status in ['failed', 'cancelled'] else 0
            print(f'{is_failed}|{status}|{total}|{completed}')
        except Exception:
            print('0|unknown|0|0')
        " > /tmp/firecrawl_summary.txt

        IS_FAILED=$(cut -d '|' -f 1 /tmp/firecrawl_summary.txt)
        JOB_STATUS=$(cut -d '|' -f 2 /tmp/firecrawl_summary.txt)
        TOTAL_PAGES=$(cut -d '|' -f 3 /tmp/firecrawl_summary.txt)
        COMPLETED_PAGES=$(cut -d '|' -f 4 /tmp/firecrawl_summary.txt)

        echo "::set-output name=is_failed::${IS_FAILED:-0}::"
        echo "::set-output name=job_status::$JOB_STATUS::"
        echo "::set-output name=total_pages::$TOTAL_PAGES::"
        echo "::set-output name=completed_pages::$COMPLETED_PAGES::"

  - id: evaluate_crawl_health
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.inspect_crawl_status.vars.is_failed | int == 1 }}"
    then:
      - id: notify_ai_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {
            "channel": "#ai-alerts",
            "attachments": [
              {
                "color": "danger",
                "title": "🚨 Firecrawl Web Scraping Job Failure Alert",
                "text": "Crawl Job {{ inputs.crawl_job_id }} entered status {{ outputs.inspect_crawl_status.vars.job_status }} ({{ outputs.inspect_crawl_status.vars.completed_pages }} of {{ outputs.inspect_crawl_status.vars.total_pages }} pages scraped). Downstream RAG embedding pipelines are missing documentation context.",
                "footer": "Kestra AI Ingestion Sentinel • Firecrawl"
              }
            ]
          }

errors:
  - id: alert_pipeline_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {
        "channel": "#ai-alerts",
        "attachments": [
          {
            "color": "danger",
            "title": "⚠️ Firecrawl Sentinel Failure",
            "text": "Failed to communicate with Firecrawl API at {{ inputs.firecrawl_api_url }}.",
            "footer": "Kestra Failure Sentinel"
          }
        ]
      }

triggers:
  - id: periodic_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/30 * * * *"
    disabled: true