Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 06:23:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

update puppeteer, update disablefeature

Browse source 
logging for stall detection
This commit is contained in:
Ilya Kreymer 2025-02-27 08:13:56 -08:00
parent 6f081b24f9
commit 9eae1d4e6c
6 changed files with 22 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -1,4 +1,4 @@
ARG BROWSER_VERSION=1.74.51
ARG BROWSER_VERSION=1.77.52
ARG BROWSER_IMAGE_BASE=webrecorder/browsertrix-browser-base:brave-${BROWSER_VERSION}
FROM ${BROWSER_IMAGE_BASE}

2
package.json
View file

@ -31,7 +31,7 @@
"p-queue": "^7.3.4",
"pixelmatch": "^5.3.0",
"pngjs": "^7.0.0",
"puppeteer-core": "^24.2.0",
"puppeteer-core": "^24.3.0",
"sax": "^1.3.0",
"sharp": "^0.32.6",
"tsc": "^2.0.4",

2
src/util/browser.ts
View file

@ -629,7 +629,7 @@ export const defaultArgs = [
// AvoidUnnecessaryBeforeUnloadCheckSync - https://github.com/microsoft/playwright/issues/14047
// Translate - https://github.com/microsoft/playwright/issues/16126
// Optimization* - https://bugs.chromium.org/p/chromium/issues/detail?id=1311753
"--disable-features=ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,DialMediaRouteProvider,AcceptCHFrame,AutoExpandDetailsElement,CertificateTransparencyComponentUpdater,AvoidUnnecessaryBeforeUnloadCheckSync,Translate,OptimizationGuideModelDownloading,OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints",
"--disable-features=ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,DialMediaRouteProvider,AcceptCHFrame,AutoExpandDetailsElement,CertificateTransparencyComponentUpdater,AvoidUnnecessaryBeforeUnloadCheckSync,Translate,OptimizationGuideModelDownloading,OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints,SkiaGraphite",
"--allow-pre-commit-input",
"--disable-hang-monitor",
"--disable-ipc-flooding-protection",

4
src/util/recorder.ts
View file

@ -192,6 +192,8 @@ export class Recorder {
patterns: [{ urlPattern: "*", requestStage: "Response" }],
});
logger.debug("Recorder fetch.enable finished", {}, "worker");
// Response
cdp.on("Network.responseReceived", (params) =>
this.handleResponseReceived(params),
@ -225,6 +227,7 @@ export class Recorder {
);
await cdp.send("Network.enable");
logger.debug("Recorder Network.enable finished", {}, "worker");
// Target
cdp.on("Target.attachedToTarget", async (params) => {
@ -249,6 +252,7 @@ export class Recorder {
waitForDebuggerOnStart: false,
flatten: true,
});
logger.debug("Recorder Target.setAutoAttach finished", {}, "worker");
// Console
cdp.on("Console.messageAdded", (params) => {

2
src/util/worker.ts
View file

@ -232,8 +232,8 @@ export class PageWorker {
}
if (retry >= 2) {
this.crawler.markBrowserCrashed();
if (this.crawler.params.restartsOnError) {
this.crawler.markBrowserCrashed();
throw new Error("Unable to load new page, browser needs restart");
} else {
this.crawler.browser.killBrowser();

28
yarn.lock
View file

@ -1559,10 +1559,10 @@ chownr@^1.1.1:
resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
chromium-bidi@1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-1.2.0.tgz#e481b3eee0bf0f2d940a60b83faa79414f4752b8"
integrity sha512-XtdJ1GSN6S3l7tO7F77GhNsw0K367p0IsLYf2yZawCVAKKC3lUvDhPdMVrB2FNhmhfW43QGYbEX3Wg6q0maGwQ==
chromium-bidi@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-2.0.0.tgz#f2f59a0c89cd1b2965e89e36b82c9e18e847deed"
integrity sha512-8VmyVj0ewSY4pstZV0Y3rCUUwpomam8uWgHZf1XavRxJEP4vU9/dcpNuoyB+u4AQxPo96CASXz5CHPvdH+dSeQ==
dependencies:
mitt "^3.0.1"
zod "^3.24.1"
@ -4119,17 +4119,17 @@ punycode@^2.1.0:
resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5"
integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==
puppeteer-core@^24.2.0:
version "24.2.0"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.2.0.tgz#4f935d1417777b335ca747bf1c3ae997989ed6e4"
integrity sha512-e4A4/xqWdd4kcE6QVHYhJ+Qlx/+XpgjP4d8OwBx0DJoY/nkIRhSgYmKQnv7+XSs1ofBstalt+XPGrkaz4FoXOQ==
puppeteer-core@^24.3.0:
version "24.3.0"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.3.0.tgz#7fe2e8b69923825640d1452b1083f90d2f21619e"
integrity sha512-x8kQRP/xxtiFav6wWuLzrctO0HWRpSQy+JjaHbqIl+d5U2lmRh2pY9vh5AzDFN0EtOXW2pzngi9RrryY1vZGig==
dependencies:
"@puppeteer/browsers" "2.7.1"
chromium-bidi "1.2.0"
chromium-bidi "2.0.0"
debug "^4.4.0"
devtools-protocol "0.0.1402036"
typed-query-selector "^2.12.0"
ws "^8.18.0"
ws "^8.18.1"
pure-rand@^6.0.0:
version "6.1.0"
@ -5155,10 +5155,10 @@ ws@^7.4.4:
resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9"
integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==
ws@^8.18.0:
version "8.18.0"
resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
ws@^8.18.1:
version "8.18.1"
resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.1.tgz#ea131d3784e1dfdff91adb0a4a116b127515e3cb"
integrity sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==
xml2js@^0.5.0:
version "0.5.0"