browsertrix-crawler/tests/add-exclusion.test.js

import { exec } from "child_process";
import Redis from "ioredis";

test("dynamically add exclusion while crawl is running", async () => {
  let callback = null;

  const p = new Promise((resolve) => {
    callback = (error, stdout, stderr) => {
      resolve({ error, stdout, stderr });
    };
  });

  try {
    exec(
      "docker run -p 36379:6379 -e CRAWL_ID=test -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection add-exclusion --url https://webrecorder.net/ --scopeType prefix --limit 20 --logging debug --debugAccessRedis",
      { shell: "/bin/bash" },
      callback,
    );
  } catch (error) {
    console.log(error);
  }

  await new Promise((resolve) => setTimeout(resolve, 3000));

  const redis = new Redis("redis://127.0.0.1:36379/0", { lazyConnect: true });

  await redis.connect({ maxRetriesPerRequest: 50 });

  while (true) {
    if (Number(await redis.zcard("test:q")) > 1) {
      break;
    }

    await new Promise((resolve) => setTimeout(resolve, 500));
  }

  const uids = await redis.hkeys("test:status");

  // exclude all pages containing 'webrecorder', should clear out the queue and end the crawl
  await redis.rpush(
    `${uids[0]}:msg`,
    JSON.stringify({ type: "addExclusion", regex: "webrecorder" }),
  );

  // ensure 'Add Exclusion is contained in the debug logs
  const { stdout } = await p;

  expect(stdout.indexOf("Add Exclusion") > 0).toBe(true);

  expect(stdout.indexOf("Removing excluded URL") > 0).toBe(true);

  await redis.disconnect();
});