browsertrix-crawler/tests/config_file.test.js

import fs from "fs";
import yaml from "js-yaml";

import util from "util";
import {exec as execCallback } from "child_process";

const exec = util.promisify(execCallback);


test("check yaml config file with seed list is used", async () => {
  try{

    await exec("docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --config /tests/fixtures/crawl-1.yaml --depth 0");
  }
  catch (error) {
    console.log(error);
  }

  const crawledPages = fs.readFileSync("test-crawls/collections/configtest/pages/pages.jsonl", "utf8");
  const pages = new Set();

  for (const line of crawledPages.trim().split("\n")) {
    const url = JSON.parse(line).url;
    if (url) {
      pages.add(url);
    }
  }

  const config = yaml.load(fs.readFileSync("tests/fixtures/crawl-1.yaml", "utf8"));

  let foundAllSeeds = true; 

  for (const seed of config.seeds) {
    const url = new URL(seed).href;
    if (!pages.has(url)) {
      foundAllSeeds = false;
    }
  }
  expect(foundAllSeeds).toBe(true);

  expect(fs.existsSync("test-crawls/collections/configtest/configtest.wacz")).toBe(true);

});

test("check yaml config file will be overwritten by command line", async () => {
  try{

    await exec("docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection configtest-2 --config /tests/fixtures/crawl-1.yaml --url https://specs.webrecorder.net/ --scopeType page --timeout 20000");
  }
  catch (error) {
    console.log(error);
  }

  const crawledPages = fs.readFileSync("test-crawls/collections/configtest-2/pages/pages.jsonl", "utf8");
  const pages = new Set();

  for (const line of crawledPages.trim().split("\n")) {
    const url = JSON.parse(line).url;
    if (url) {
      pages.add(url);
    }
  }

  expect(pages.has("https://specs.webrecorder.net/")).toBe(true);
  expect(pages.size).toBe(1);

});