browsertrix-crawler/tests/mult_url_crawl_with_favicon.test.js

import child_process from "child_process";
import fs from "fs";

test("ensure multi url crawl run with docker run passes", async () => {
  child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://webrecorder.net/ --generateWACZ --text --collection advanced --combineWARC --rolloverSize 10000 --workers 2 --title \"test title\" --description \"test description\" --pages 2 --limit 2");

  child_process.execSync("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler wacz validate --file collections/advanced/advanced.wacz");

});

test("check that the favicon made it into the pages jsonl file", () => {
  expect(fs.existsSync("test-crawls/collections/advanced/pages/pages.jsonl")).toBe(true);

  const data1 = JSON.parse(fs.readFileSync("test-crawls/collections/advanced/pages/pages.jsonl", "utf8").split("\n")[1]);
  const data2 = JSON.parse(fs.readFileSync("test-crawls/collections/advanced/pages/pages.jsonl", "utf8").split("\n")[2]);
  const data = [ data1, data2 ];
  for (const d of data) {
    if (d.url === "https://webrecorder.net/") {
      expect(d.favIconUrl).toEqual("https://webrecorder.net/assets/favicon.ico");
    }
    if (d.url === "https://iana.org/") {
      expect(d.favIconUrl).toEqual("https://www.iana.org/_img/bookmark_icon.ico");
    }
  }
});