mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 14:33:17 +00:00
473de8c49f
* scope fixes: - remove default prefix scopeType, ensure scope include and exclude take precedence - add new 'custom' scopeType, when include or exclude are used - use --scopeIncludeRx and --scopeExcludeRx for better consistency for scope include and exclude (also allow --include/--exclude) - ensure per-seed scope include/exclude used when present, and scopeType set to 'custom' - ensure default scope is set to 'prefix' if no scopeType and no include/exclude regexes specified - rename --type to --scopeType in seed to maintain consistency - add sitemap param as alias for useSitemap tests: - add seed scope resolution tests for argParse, testing per-scope seed resolution, inheritance and overrides - fix screencaster to use relative paths to work with tests - ci: use yarn instead of npm * update README with new flags * bump version to 0.4.0-beta.3
47 lines
1.3 KiB
JavaScript
47 lines
1.3 KiB
JavaScript
const yaml = require("js-yaml");
|
|
const child_process = require("child_process");
|
|
const fs = require("fs");
|
|
|
|
test("pass config file via stdin", async () => {
|
|
jest.setTimeout(30000);
|
|
|
|
const configYaml = fs.readFileSync("tests/fixtures/crawl-2.yaml", "utf8");
|
|
const config = yaml.load(configYaml);
|
|
|
|
try {
|
|
const version = require("../package.json").version;
|
|
const proc = child_process.execSync(`docker run -i -v $PWD/crawls:/crawls webrecorder/browsertrix-crawler:${version} crawl --config stdin --scopeExcludeRx webrecorder.net/202`, {input: configYaml, stdin: "inherit", encoding: "utf8"});
|
|
|
|
console.log(proc);
|
|
}
|
|
catch (error) {
|
|
console.log(error);
|
|
}
|
|
|
|
const crawledPages = fs.readFileSync("crawls/collections/config-stdin/pages/pages.jsonl", "utf8");
|
|
const pages = new Set();
|
|
|
|
for (const line of crawledPages.trim().split("\n")) {
|
|
const url = JSON.parse(line).url;
|
|
if (!url) {
|
|
continue;
|
|
}
|
|
pages.add(url);
|
|
expect(url.indexOf("webrecorder.net/202")).toEqual(-1);
|
|
}
|
|
|
|
let foundAllSeeds = true;
|
|
|
|
for (const seed of config.seeds) {
|
|
const url = new URL(seed).href;
|
|
if (!pages.has(url)) {
|
|
foundAllSeeds = false;
|
|
}
|
|
}
|
|
expect(foundAllSeeds).toBe(true);
|
|
|
|
expect(fs.existsSync("crawls/collections/config-stdin/config-stdin.wacz")).toBe(true);
|
|
|
|
});
|
|
|
|
|