Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Page-reuse concurrency + Browser Repair + Screencaster Cleanup Improvements (#157)

Browse source 
* new window: use cdp instead of window.open

* new window tweaks: add reuseCount, use browser.target() instead of opening a new blank page

* rename NewWindowPage -> ReuseWindowConcurrency, move to windowconcur.js
potential fix for #156

* browser repair:
- when using window-concurrency, attempt to repair / relaunch browser if cdp errors occur
- mark pages as failed and don't reuse if page error or cdp errors occur
- screencaster: clear previous targets if screencasting when repairing browser

* bump version to 0.7.0-beta.3
This commit is contained in:
Ilya Kreymer 2022-08-19 09:23:40 -07:00 • committed by GitHub
parent 827c153679
commit 6cc38bf511
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 163 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

6
util/argParser.js
View file

@ -8,7 +8,7 @@ const { Cluster } = require("puppeteer-cluster");
const yargs = require("yargs/yargs");
const { hideBin } = require("yargs/helpers");
const { NewWindowPage} = require("./screencaster");
const { ReuseWindowConcurrency } = require("./windowconcur");
const { BEHAVIOR_LOG_FUNC, WAIT_UNTIL_OPTS } = require("./constants");
const { ScopedSeed } = require("./seeds");
const { interpolateFilename } = require("./storage");
@ -374,7 +374,7 @@ class ArgParser {
argv.newContext = Cluster.CONCURRENCY_PAGE;
if (argv.screencastPort && argv.workers > 1) {
console.log("Note: to support screencasting with >1 workers, newContext set to 'window' instead of 'page'");
argv.newContext = NewWindowPage;
argv.newContext = ReuseWindowConcurrency;
}
break;
@ -387,7 +387,7 @@ class ArgParser {
break;
case "window":
argv.newContext = NewWindowPage;
argv.newContext = ReuseWindowConcurrency;
break;
default: