Check Markdown Files for Dead Links

id: python-markdown-link-checker
namespace: company.team

triggers:
  - id: weekly_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    description: Disabled by default. Enable this schedule to continuously monitor
      the links in your repository.
    cron: "0 9 * * 1"
    disabled: true

inputs:
  - id: markdown_url
    type: URI
    defaults: https://raw.githubusercontent.com/kestra-io/kestra/develop/README.md
    description: Public raw URL of the Markdown file to check.

  - id: timeout_seconds
    type: INT
    defaults: 10
    description: Timeout for checking each link in seconds.

  - id: fail_on_dead_links
    type: BOOL
    defaults: true
    description: Whether to fail the flow execution if dead links are found.

tasks:
  - id: download
    type: io.kestra.plugin.core.http.Download
    description: Download the Markdown file to Kestra's internal storage.
    uri: "{{ inputs.markdown_url }}"

  - id: check_links
    type: io.kestra.plugin.scripts.python.Script
    description: Extract URLs from the Markdown file, deduplicate them, and check
      each link for accessibility using HEAD and GET fallbacks.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    containerImage: python:3.11-slim
    inputFiles:
      file.md: "{{ outputs.download.uri }}"
    env:
      TIMEOUT: "{{ inputs.timeout_seconds }}"
    script: |
      import os
      import json
      import re
      import urllib.request
      import urllib.error
      import time

      timeout = int(os.environ["TIMEOUT"])

      with open("file.md", "r", encoding="utf-8") as f:
          content = f.read()

      urls = re.findall(r'(https?://[^\s)\]"\']+)', content)
      urls = list(set(urls))
      urls = [u for u in urls if not u.startswith("mailto:") and "localhost" not in u and "127.0.0.1" not in u]

      headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'}

      dead_links = []

      for u in urls:
          success = False
          error_msg = ""
          for attempt in range(3):
              try:
                  req = urllib.request.Request(u, headers=headers, method='HEAD')
                  try:
                      with urllib.request.urlopen(req, timeout=timeout) as response:
                          success = True
                          break
                  except (urllib.error.HTTPError, urllib.error.URLError) as e:
                      if isinstance(e, urllib.error.HTTPError) and e.code in [403, 405, 503, 429]:
                          req = urllib.request.Request(u, headers=headers, method='GET')
                          with urllib.request.urlopen(req, timeout=timeout) as response:
                              success = True
                              break
                      else:
                          raise e
              except Exception as e:
                  error_msg = str(e)
                  time.sleep(2)
          if not success:
              dead_links.append({"url": u, "error": error_msg})

      total = len(urls)
      dead_count = len(dead_links)

      print(f"Checked {total} links. Found {dead_count} dead links.")

      outputs = {
          "outputs": {
              "total": total,
              "dead_count": dead_count,
              "dead_links": dead_links
          }
      }
      print("::" + json.dumps(outputs) + "::")

  - id: evaluate
    type: io.kestra.plugin.core.flow.If
    description: If dead links are found, post them to Slack. Otherwise, log a
      success message.
    condition: "{{ outputs.check_links.vars.dead_count > 0 }}"
    then:
      - id: notify_slack
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Post the detailed list of dead links to Slack.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "text": "Dead links found in {{ inputs.markdown_url }} (Execution {{ execution.id }}):\n{{ outputs.check_links.vars.dead_links | toJson }}"
          }

      - id: fail_flow
        type: io.kestra.plugin.core.execution.Fail
        description: Fail the flow execution since fail_on_dead_links is true.
        runIf: "{{ inputs.fail_on_dead_links }}"
        errorMessage: "Flow failed because {{ outputs.check_links.vars.dead_count }}
          dead links were found."
    else:
      - id: log_success
        type: io.kestra.plugin.core.log.Log
        description: Log a success message when all links are accessible.
        message: "Success: All {{ outputs.check_links.vars.total }} links in {{
          inputs.markdown_url }} are valid."

errors:
  - id: alert_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alert the team if the flow execution fails due to a task error or
      dead links.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "Link checker FAILED in flow {{ flow.id }} (execution {{ execution.id }}) for {{ inputs.markdown_url }}."
      }