mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 06:23:16 +00:00
bb9c82493b
Initial (beta) support for QA/replay crawling!
- Supports running a crawl over a given WACZ / list of WACZ (multi WACZ) input, hosted in ReplayWeb.page
- Runs local http server with full-page, ui-less ReplayWeb.page embed
- ReplayWeb.page release version configured in the Dockerfile, pinned ui.js and sw.js fetched directly from cdnjs
Can be deployed with `webrecorder/browsertrix-crawler qa` entrypoint.
- Requires `--qaSource`, pointing to WACZ or multi-WACZ json that will be replay/QAd
- Also supports `--qaRedisKey` where QA comparison data will be pushed, if specified.
- Supports `--qaDebugImageDiff` for outputting crawl / replay/ diff
images.
- If using --writePagesToRedis, a `comparison` key is added to existing page data where:
```
comparison: {
screenshotMatch?: number;
textMatch?: number;
resourceCounts: {
crawlGood?: number;
crawlBad?: number;
replayGood?: number;
replayBad?: number;
};
};
```
- bump version to 1.1.0-beta.2
72 lines
1.9 KiB
JavaScript
72 lines
1.9 KiB
JavaScript
import child_process from "child_process";
|
|
import Redis from "ioredis";
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async function waitContainer(containerId) {
|
|
try {
|
|
child_process.execSync(`docker kill -s SIGINT ${containerId}`);
|
|
} catch (e) {
|
|
return;
|
|
}
|
|
|
|
// containerId is initially the full id, but docker ps
|
|
// only prints the short id (first 12 characters)
|
|
containerId = containerId.slice(0, 12);
|
|
|
|
while (true) {
|
|
try {
|
|
const res = child_process.execSync("docker ps -q", { encoding: "utf-8" });
|
|
if (res.indexOf(containerId) < 0) {
|
|
return;
|
|
}
|
|
} catch (e) {
|
|
console.error(e);
|
|
}
|
|
await sleep(500);
|
|
}
|
|
}
|
|
|
|
async function runCrawl(numExpected, url, sitemap="", limit=0) {
|
|
const containerId = child_process.execSync(`docker run -d -p 36381:6379 -e CRAWL_ID=test webrecorder/browsertrix-crawler crawl --url ${url} --sitemap ${sitemap} --limit ${limit} --context sitemap --logging debug --debugAccessRedis`, {encoding: "utf-8"});
|
|
|
|
await sleep(3000);
|
|
|
|
const redis = new Redis("redis://127.0.0.1:36381/0", { lazyConnect: true, retryStrategy: () => null });
|
|
|
|
let finished = 0;
|
|
|
|
try {
|
|
await redis.connect({
|
|
maxRetriesPerRequest: 100,
|
|
});
|
|
|
|
while (true) {
|
|
finished = await redis.zcard("test:q");
|
|
|
|
if (finished >= numExpected) {
|
|
break;
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.error(e);
|
|
} finally {
|
|
await waitContainer(containerId);
|
|
}
|
|
|
|
expect(finished).toBeGreaterThanOrEqual(numExpected);
|
|
}
|
|
|
|
test("test sitemap fully finish", async () => {
|
|
await runCrawl(8036, "https://www.mozilla.org/", "", 0);
|
|
});
|
|
|
|
test("test sitemap with limit", async () => {
|
|
await runCrawl(1900, "https://www.mozilla.org/", "", 2000);
|
|
});
|
|
|
|
test("test sitemap with limit, specific URL", async () => {
|
|
await runCrawl(1900, "https://www.mozilla.org/", "https://www.mozilla.org/sitemap.xml", 2000);
|
|
});
|