browsertrix-crawler/tests/limit_reached.test.js

import fs from "fs";
import util from "util";
import { exec as execCallback, execSync } from "child_process";

const exec = util.promisify(execCallback);

test("ensure page limit reached", async () => {
  execSync(
    'docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --scopeType prefix --behaviors "" --url https://old.webrecorder.net/ --limit 12 --workers 2 --collection limit-test --statsFilename stats.json --exclude community',
  );
});

test("check limit written to stats file is as expected", () => {
  const data = fs.readFileSync("test-crawls/stats.json", "utf8");
  const dataJSON = JSON.parse(data);
  expect(dataJSON.crawled).toEqual(12);
  expect(dataJSON.total).toEqual(12);
  expect(dataJSON.limit.hit).toBe(true);
});

test("ensure crawl fails if failOnFailedLimit is reached", async () => {
  let passed = true;
  try {
    await exec(
      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/will404 --url https://specs.webrecorder.net --failOnInvalidStatus --failOnFailedLimit 1 --limit 10 --collection faillimitreached",
    );
  } catch (error) {
    expect(error.code).toEqual(12);
    passed = false;
  }
  expect(passed).toBe(false);
});

test("ensure crawl fails if timeLimit is reached", async () => {
  let passed = true;
  try {
    await exec(
      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net --failOnInvalidStatus --timeLimit 1 --limit 2 --collection failontimelimitreached",
    );
  } catch (error) {
    expect(error.code).toEqual(15);
    passed = false;
  }
  expect(passed).toBe(false);
});

test("ensure crawl fails if sizeLimit is reached", async () => {
  let passed = true;
  try {
    await exec(
      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net --failOnInvalidStatus --sizeLimit 1 --limit 2 --collection failonsizelimitreached",
    );
  } catch (error) {
    expect(error.code).toEqual(14);
    passed = false;
  }
  expect(passed).toBe(false);
});

test("ensure crawl fails if diskUtilizationLimit is reached", async () => {
  let passed = true;
  try {
    await exec(
      "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net --failOnInvalidStatus --diskUtilization 1 --limit 2 --collection failonsizelimitreached",
    );
  } catch (error) {
    expect(error.code).toEqual(16);
    passed = false;
  }
  expect(passed).toBe(false);
});