mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 06:23:16 +00:00
a00866bbab
- if redirected page is excluded, block loading of page - mark page as excluded, don't retry, and don't write to page list - support generic blocking of pages based on initial page response - fixes #744
21 lines
836 B
JavaScript
21 lines
836 B
JavaScript
import fs from "fs";
|
|
import { execSync } from "child_process";
|
|
|
|
// example.com includes a link to 'https://www.iana.org/domains/example' which redirects to 'https://www.iana.org/help/example-domains'
|
|
// pgae loading should be blocked on redirected due to exclusion of 'help', though the initial link is loaded
|
|
|
|
test("ensure exclusion is applied on redirected URL, which contains 'help', so it is not crawled", () => {
|
|
execSync(
|
|
"docker run -p 9037:9037 -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://example.com/ --exclude help --collection redir-exclude-test --extraHops 1");
|
|
|
|
// no entries besides header
|
|
expect(
|
|
fs
|
|
.readFileSync(
|
|
"test-crawls/collections/redir-exclude-test/pages/extraPages.jsonl",
|
|
"utf8",
|
|
).trim().split("\n").length
|
|
).toBe(1);
|
|
|
|
});
|
|
|