mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 14:33:17 +00:00
8318039ae3
Fixes #563 This PR makes a few changes to fix a regression in behavior around `failOnFailedSeed` for the 1.x releases: - Fail with exit code 1, not 17, when pages are unreachable due to DNS not resolving or other network errors if the page is a seed and `failOnFailedSeed` is set - Extend tests, add test to ensure crawl succeeds on 404 seed status code if `failOnINvalidStatus` isn't set
82 lines
2.9 KiB
JavaScript
82 lines
2.9 KiB
JavaScript
import util from "util";
|
|
import { exec as execCallback } from "child_process";
|
|
|
|
const exec = util.promisify(execCallback);
|
|
|
|
test("ensure one invalid seed doesn't end crawl if failOnFailedSeed is not set", async () => {
|
|
let passed = true;
|
|
try {
|
|
await exec(
|
|
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://example.invalid --generateWACZ --limit 2 --collection invalidseed",
|
|
);
|
|
} catch (error) {
|
|
console.log(error);
|
|
passed = false;
|
|
}
|
|
expect(passed).toBe(true);
|
|
});
|
|
|
|
test("ensure one invalid seed fails crawl if failOnFailedSeed is set", async () => {
|
|
let passed = true;
|
|
try {
|
|
await exec(
|
|
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url example.invalid --generateWACZ --limit 2 --failOnFailedSeed --collection failseed",
|
|
);
|
|
} catch (error) {
|
|
expect(error.code).toEqual(1);
|
|
passed = false;
|
|
}
|
|
expect(passed).toBe(false);
|
|
});
|
|
|
|
test("ensure seed with network error fails crawl if failOnFailedSeed and failOnInvalidStatus is set", async () => {
|
|
let passed = true;
|
|
try {
|
|
await exec(
|
|
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://example.invalid --generateWACZ --limit 2 --failOnFailedSeed --failOnInvalidStatus --collection failseedstatus",
|
|
);
|
|
} catch (error) {
|
|
expect(error.code).toEqual(1);
|
|
passed = false;
|
|
}
|
|
expect(passed).toBe(false);
|
|
});
|
|
|
|
test("ensure seed with 4xx/5xx response fails crawl if failOnFailedSeed and failOnInvalidStatus are set", async () => {
|
|
let passed = true;
|
|
try {
|
|
await exec(
|
|
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://webrecorder.net/doesntexist --generateWACZ --limit 2 --failOnFailedSeed --failOnInvalidStatus --collection failseed404status",
|
|
);
|
|
} catch (error) {
|
|
expect(error.code).toEqual(1);
|
|
passed = false;
|
|
}
|
|
expect(passed).toBe(false);
|
|
});
|
|
|
|
test("ensure seed with 4xx/5xx response succeeds if failOnInvalidStatus is not set", async () => {
|
|
let passed = true;
|
|
try {
|
|
await exec(
|
|
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://webrecorder.net/doesntexist --generateWACZ --limit 2 --failOnFailedSeed --collection failseedwithoutinvalidstatus",
|
|
);
|
|
} catch (error) {
|
|
console.log(error);
|
|
passed = false;
|
|
}
|
|
expect(passed).toBe(true);
|
|
});
|
|
|
|
test("ensure crawl fails if no valid seeds are passed", async () => {
|
|
let passed = true;
|
|
try {
|
|
await exec(
|
|
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url iana.org/ --url example.invalid --generateWACZ --limit 2 --collection allinvalidseeds",
|
|
);
|
|
} catch (error) {
|
|
expect(error.code).toEqual(17);
|
|
passed = false;
|
|
}
|
|
expect(passed).toBe(false);
|
|
});
|