browsertrix-crawler/tests/crawl_overwrite.js

import child_process from "child_process";
import fs from "fs";

test("ensure --overwrite with existing collection results in a successful crawl", async () => {
  child_process.execSync(
    "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ  --collection overwrite",
  );

  child_process.execSync(
    "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ  --collection overwrite --overwrite",
  );
});

test("check that the pages.jsonl file exists in the collection under the pages folder", () => {
  expect(
    fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl"),
  ).toBe(true);
});

test("check that the WACZ file exists in the collection", () => {
  expect(
    fs.existsSync("test-crawls/collections/overwrite/pages/pages.jsonl"),
  ).toBe(true);
});

//-----------

test("ensure --overwrite results in a successful crawl even if collection didn't exist", async () => {
  child_process.execSync(
    "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://www.example.com/ --generateWACZ  --collection overwrite-nothing --overwrite",
  );
});

test("check that the pages.jsonl file exists in the collection under the pages folder", () => {
  expect(
    fs.existsSync(
      "test-crawls/collections/overwrite-nothing/pages/pages.jsonl",
    ),
  ).toBe(true);
});

test("check that the WACZ file exists in the collection", () => {
  expect(
    fs.existsSync(
      "test-crawls/collections/overwrite-nothing/pages/pages.jsonl",
    ),
  ).toBe(true);
});