browsertrix-crawler/tests/config_file.test.js

const yaml = require("js-yaml");
const util = require("util");
const exec = util.promisify(require("child_process").exec);
const fs = require("fs");

test("check yaml config file with seed list is used", async () => {
  jest.setTimeout(30000);

  try{

    await exec("docker-compose run -v $PWD/tests/fixtures:/tests/fixtures crawler crawl --collection configtest --config /tests/fixtures/crawl-1.yaml --depth 0");
  }
  catch (error) {
    console.log(error);
  }

  const crawledPages = fs.readFileSync("crawls/collections/configtest/pages/pages.jsonl", "utf8");
  const pages = new Set();

  for (const line of crawledPages.trim().split("\n")) {
    const url = JSON.parse(line).url;
    if (url) {
      pages.add(url);
    }
  }

  const config = yaml.load(fs.readFileSync("tests/fixtures/crawl-1.yaml", "utf8"));

  let foundAllSeeds = true; 

  for (const seed of config.seeds) {
    const url = new URL(seed).href;
    if (!pages.has(url)) {
      foundAllSeeds = false;
    }
  }
  expect(foundAllSeeds).toBe(true);

  expect(fs.existsSync("crawls/collections/configtest/configtest.wacz")).toBe(true);

});

test("check yaml config file will be overwritten by command line", async () => {
  jest.setTimeout(30000);

  try{

    await exec("docker-compose run -v $PWD/crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures crawler crawl --collection configtest --config /tests/fixtures/crawl-1.yaml --url https://www.example.com --timeout 20000");
  }
  catch (error) {
    console.log(error);
  }

  const crawledPages = fs.readFileSync("crawls/collections/configtest/pages/pages.jsonl", "utf8");
  const pages = new Set();

  for (const line of crawledPages.trim().split("\n")) {
    pages.add(JSON.parse(line).url);
  }


  
  expect(pages.has("https://www.example.com/")).toBe(true);


});