Nightly Competitor Pricing Monitor with Puppeteer

id: competitor-pricing-monitor-puppeteer
namespace: company.team
description: |
  Scrape a list of competitor pricing pages every night with a headless Chrome
  browser driven by Puppeteer, since most modern pricing pages render their
  price tables client-side and never appear in a plain HTTP response. Diff the
  freshly scraped prices against yesterday's snapshot stored in the KV Store,
  post a Slack alert only when a price actually moved, then persist the new
  snapshot so tomorrow's run has something to compare against.

inputs:
  - id: competitors
    type: JSON
    description: List of competitor pages to scrape, each with a name, url, and a
      CSS selector pointing at the rendered price element.
    defaults:
      - name: Acme Corp
        url: https://acme.example.com/pricing
        selector: .price-value
      - name: Globex Inc
        url: https://globex.example.com/pricing
        selector: "[data-testid='plan-price']"

tasks:
  - id: read_previous_snapshot
    type: io.kestra.plugin.core.kv.Get
    description: Read yesterday's scraped prices; falls through to an empty list on
      the first run.
    key: competitor_pricing_snapshot
    errorOnMissing: false

  - id: scrape_pricing_pages
    type: io.kestra.plugin.scripts.node.Script
    description: Launch one headless Chrome instance and scrape every competitor
      page in turn, since standing up a new browser per page would be wasteful.
    containerImage: ghcr.io/puppeteer/puppeteer:latest
    beforeCommands:
      - npm install puppeteer @kestra-io/libs
    inputFiles:
      competitors.json: "{{ inputs.competitors }}"
    script: |
      const fs = require('fs');
      const puppeteer = require('puppeteer');
      const Kestra = require('@kestra-io/libs');

      (async () => {
        const competitors = JSON.parse(fs.readFileSync('competitors.json', 'utf8'));
        const browser = await puppeteer.launch({
          headless: 'new',
          args: ['--no-sandbox', '--disable-setuid-sandbox']
        });

        const results = [];
        for (const competitor of competitors) {
          const page = await browser.newPage();
          await page.goto(competitor.url, { waitUntil: 'networkidle0', timeout: 30000 });
          await page.waitForSelector(competitor.selector, { timeout: 15000 });
          const price = await page.$eval(competitor.selector, (el) => el.textContent.trim());
          results.push({ name: competitor.name, price, scrapedAt: new Date().toISOString() });
          await page.close();
        }

        await browser.close();
        Kestra.outputs({ pricesJson: JSON.stringify(results) });
        console.log(`Scraped ${results.length} competitor page(s).`);
      })();

  - id: detect_price_changes
    type: io.kestra.plugin.scripts.node.Script
    description: Compare today's scrape against yesterday's snapshot and summarize
      what actually changed.
    beforeCommands:
      - npm install @kestra-io/libs
    inputFiles:
      previous.json: "{{ outputs.read_previous_snapshot.value ?? '[]' }}"
      current.json: "{{ outputs.scrape_pricing_pages.vars.pricesJson }}"
    script: |
      const fs = require('fs');
      const Kestra = require('@kestra-io/libs');

      const previous = JSON.parse(fs.readFileSync('previous.json', 'utf8'));
      const current = JSON.parse(fs.readFileSync('current.json', 'utf8'));

      const previousByName = Object.fromEntries(previous.map((p) => [p.name, p.price]));
      const changes = current
        .filter((c) => previousByName[c.name] !== undefined && previousByName[c.name] !== c.price)
        .map((c) => ({ name: c.name, previousPrice: previousByName[c.name], newPrice: c.price }));

      Kestra.outputs({
        changeCount: changes.length,
        changesSummary: changes.map((c) => `${c.name}: ${c.previousPrice} -> ${c.newPrice}`).join('; ')
      });

      console.log(`Detected ${changes.length} price change(s).`);

  - id: alert_on_price_change
    type: io.kestra.plugin.core.flow.If
    description: Only bother the team when a price actually moved.
    condition: "{{ outputs.detect_price_changes.vars.changeCount > 0 }}"
    then:
      - id: notify_price_change
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "text": "Competitor pricing changed: {{ outputs.detect_price_changes.vars.changesSummary }}"
          }

  - id: update_snapshot
    type: io.kestra.plugin.core.kv.Set
    description: Persist today's scrape so tomorrow's run has a baseline to diff against.
    key: competitor_pricing_snapshot
    value: "{{ outputs.scrape_pricing_pages.vars.pricesJson }}"

triggers:
  - id: nightly
    type: io.kestra.plugin.core.trigger.Schedule
    description: Scrape once a night. Adjust or disable as needed.
    cron: "0 6 * * *"
    disabled: true

errors:
  - id: alert_on_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alert Slack if the scrape or diff step fails, so a broken selector
      does not fail silently.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "Competitor pricing monitor failed (execution {{ execution.id }}). A page layout may have changed."
      }