Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-12-07 13:49:47 +00:00
Code Issues Projects Releases Packages Wiki Activity

remove --disable-component-update flag, fixes shields not working (#915)

Browse source 
should fix main cause of slow down in #913 
deps: update to brave 1.84.139, puppeteer 24.30.0
bump to 1.9.1
This commit is contained in:
Ilya Kreymer 2025-11-14 20:30:42 -08:00 • committed by GitHub
parent bb11147234
commit 8c8fd6be08
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 39 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -1,4 +1,4 @@
ARG BROWSER_VERSION=1.84.135 ARG BROWSER_VERSION=1.84.139
ARG BROWSER_IMAGE_BASE=webrecorder/browsertrix-browser-base:brave-${BROWSER_VERSION} ARG BROWSER_IMAGE_BASE=webrecorder/browsertrix-browser-base:brave-${BROWSER_VERSION}
FROM ${BROWSER_IMAGE_BASE} FROM ${BROWSER_IMAGE_BASE}

4
package.json
View file

@ -1,6 +1,6 @@
{ {
"name": "browsertrix-crawler", "name": "browsertrix-crawler",
"version": "1.9.0", "version": "1.9.1",
"main": "browsertrix-crawler", "main": "browsertrix-crawler",
"type": "module", "type": "module",
"repository": "https://github.com/webrecorder/browsertrix-crawler", "repository": "https://github.com/webrecorder/browsertrix-crawler",
@ -33,7 +33,7 @@
"p-queue": "^7.3.4", "p-queue": "^7.3.4",
"pixelmatch": "^5.3.0", "pixelmatch": "^5.3.0",
"pngjs": "^7.0.0", "pngjs": "^7.0.0",
"puppeteer-core": "^24.22.0", "puppeteer-core": "^24.30.0",
"sax": "^1.3.0", "sax": "^1.3.0",
"sharp": "^0.32.6", "sharp": "^0.32.6",
"tsc": "^2.0.4", "tsc": "^2.0.4",

2
src/util/browser.ts
View file

@ -729,7 +729,7 @@ export const defaultArgs = [
"--disable-breakpad", "--disable-breakpad",
"--disable-client-side-phishing-detection", "--disable-client-side-phishing-detection",
"--disable-component-extensions-with-background-pages", "--disable-component-extensions-with-background-pages",
"--disable-component-update", // Avoids unneeded network activity after startup. //"--disable-component-update", // now required for chrome/brave
"--no-default-browser-check", "--no-default-browser-check",
"--disable-default-apps", "--disable-default-apps",
"--disable-dev-shm-usage", "--disable-dev-shm-usage",

70
yarn.lock
View file

@ -772,17 +772,17 @@
tslib "^2.7.0" tslib "^2.7.0"
tsyringe "^4.8.0" tsyringe "^4.8.0"
"@puppeteer/browsers@2.10.10": "@puppeteer/browsers@2.10.13":
version "2.10.10" version "2.10.13"
resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.10.10.tgz#f806f92d966918c931fb9c48052eba2db848beaa" resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.10.13.tgz#42c8b7df14e992f311ca9dca5fed3f0c2182fd17"
integrity sha512-3ZG500+ZeLql8rE0hjfhkycJjDj0pI/btEh3L9IkWUYcOrgP0xCNRq3HbtbqOPbvDhFaAWD88pDFtlLv8ns8gA== integrity sha512-a9Ruw3j3qlnB5a/zHRTkruppynxqaeE4H9WNj5eYGRWqw0ZauZ23f4W2ARf3hghF5doozyD+CRtt7XSYuYRI/Q==
dependencies: dependencies:
debug "^4.4.3" debug "^4.4.3"
extract-zip "^2.0.1" extract-zip "^2.0.1"
progress "^2.0.3" progress "^2.0.3"
proxy-agent "^6.5.0" proxy-agent "^6.5.0"
semver "^7.7.2" semver "^7.7.3"
tar-fs "^3.1.0" tar-fs "^3.1.1"
yargs "^17.7.2" yargs "^17.7.2"
"@puppeteer/browsers@2.8.0": "@puppeteer/browsers@2.8.0":
@ -1715,18 +1715,18 @@ chrome-launcher@^1.1.2:
is-wsl "^2.2.0" is-wsl "^2.2.0"
lighthouse-logger "^2.0.1" lighthouse-logger "^2.0.1"
chromium-bidi@2.1.2: chromium-bidi@11.0.0:
version "2.1.2" version "11.0.0"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-2.1.2.tgz#b0710279f993128d4e0b41c892209ea093217d97" resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-11.0.0.tgz#193433d0722095abca0cada2fa0c5111b447bea3"
integrity sha512-vtRWBK2uImo5/W2oG6/cDkkHSm+2t6VHgnj+Rcwhb0pP74OoUb4GipyRX/T/y39gYQPhioP0DPShn+A7P6CHNw== integrity sha512-cM3DI+OOb89T3wO8cpPSro80Q9eKYJ7hGVXoGS3GkDPxnYSqiv+6xwpIf6XERyJ9Tdsl09hmNmY94BkgZdVekw==
dependencies: dependencies:
mitt "^3.0.1" mitt "^3.0.1"
zod "^3.24.1" zod "^3.24.1"
chromium-bidi@8.0.0: chromium-bidi@2.1.2:
version "8.0.0" version "2.1.2"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-8.0.0.tgz#d73c9beed40317adf2bcfeb9a47087003cd467ec" resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-2.1.2.tgz#b0710279f993128d4e0b41c892209ea093217d97"
integrity sha512-d1VmE0FD7lxZQHzcDUCKZSNRtRwISXDsdg4HjdTR5+Ll5nQ/vzU12JeNmupD6VWffrPSlrnGhEWlLESKH3VO+g== integrity sha512-vtRWBK2uImo5/W2oG6/cDkkHSm+2t6VHgnj+Rcwhb0pP74OoUb4GipyRX/T/y39gYQPhioP0DPShn+A7P6CHNw==
dependencies: dependencies:
mitt "^3.0.1" mitt "^3.0.1"
zod "^3.24.1" zod "^3.24.1"
@ -2059,10 +2059,10 @@ devtools-protocol@0.0.1436416:
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1436416.tgz#ce8af8a210b8bcac83c5c8f095b9f977a9570df0" resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1436416.tgz#ce8af8a210b8bcac83c5c8f095b9f977a9570df0"
integrity sha512-iGLhz2WOrlBLcTcoVsFy5dPPUqILG6cc8MITYd5lV6i38gWG14bMXRH/d8G5KITrWHBnbsOnWHfc9Qs4/jej9Q== integrity sha512-iGLhz2WOrlBLcTcoVsFy5dPPUqILG6cc8MITYd5lV6i38gWG14bMXRH/d8G5KITrWHBnbsOnWHfc9Qs4/jej9Q==
devtools-protocol@0.0.1495869: devtools-protocol@0.0.1521046:
version "0.0.1495869" version "0.0.1521046"
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1495869.tgz#f68daef77a48d5dcbcdd55dbfa3265a51989c91b" resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1521046.tgz#918e6175ea83100fefcb2b78779f15a77aa8a41b"
integrity sha512-i+bkd9UYFis40RcnkW7XrOprCujXRAHg62IVh/Ah3G8MmNXpCGt1m0dTFhSdx/AVs8XEMbdOGRwdkR1Bcta8AA== integrity sha512-vhE6eymDQSKWUXwwA37NtTTVEzjtGVfDr3pRbsWEQ5onH/Snp2c+2xZHWJJawG/0hCCJLRGt4xVtEVUVILol4w==
diff-sequences@^29.6.3: diff-sequences@^29.6.3:
version "29.6.3" version "29.6.3"
@ -4567,17 +4567,17 @@ puppeteer-core@24.4.0, puppeteer-core@^24.4.0:
typed-query-selector "^2.12.0" typed-query-selector "^2.12.0"
ws "^8.18.1" ws "^8.18.1"
puppeteer-core@^24.22.0: puppeteer-core@^24.30.0:
version "24.22.0" version "24.30.0"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.22.0.tgz#4d576b1a2b7699c088d3f0e843c32d81df82c3a6" resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.30.0.tgz#7d0d15ce6aee4f1aa8a8f046bf0198f025ee6c81"
integrity sha512-oUeWlIg0pMz8YM5pu0uqakM+cCyYyXkHBxx9di9OUELu9X9+AYrNGGRLK9tNME3WfN3JGGqQIH3b4/E9LGek/w== integrity sha512-2S3Smy0t0W4wJnNvDe7W0bE7wDmZjfZ3ljfMgJd6hn2Hq/f0jgN+x9PULZo2U3fu5UUIJ+JP8cNUGllu8P91Pg==
dependencies: dependencies:
"@puppeteer/browsers" "2.10.10" "@puppeteer/browsers" "2.10.13"
chromium-bidi "8.0.0" chromium-bidi "11.0.0"
debug "^4.4.3" debug "^4.4.3"
devtools-protocol "0.0.1495869" devtools-protocol "0.0.1521046"
typed-query-selector "^2.12.0" typed-query-selector "^2.12.0"
webdriver-bidi-protocol "0.2.11" webdriver-bidi-protocol "0.3.8"
ws "^8.18.3" ws "^8.18.3"
puppeteer@^24.4.0: puppeteer@^24.4.0:
@ -4853,10 +4853,10 @@ semver@^7.7.1:
resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.1.tgz#abd5098d82b18c6c81f6074ff2647fd3e7220c9f" resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.1.tgz#abd5098d82b18c6c81f6074ff2647fd3e7220c9f"
integrity sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA== integrity sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==
semver@^7.7.2: semver@^7.7.3:
version "7.7.2" version "7.7.3"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.2.tgz#67d99fdcd35cec21e6f8b87a7fd515a33f982b58" resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.3.tgz#4b5f4143d007633a8dc671cd0a6ef9147b8bb946"
integrity sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA== integrity sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==
set-function-length@^1.2.1: set-function-length@^1.2.1:
version "1.2.2" version "1.2.2"
@ -5220,7 +5220,7 @@ tar-fs@^3.0.8:
bare-fs "^4.0.1" bare-fs "^4.0.1"
bare-path "^3.0.0" bare-path "^3.0.0"
tar-fs@^3.1.0: tar-fs@^3.1.1:
version "3.1.1" version "3.1.1"
resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.1.1.tgz#4f164e59fb60f103d472360731e8c6bb4a7fe9ef" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.1.1.tgz#4f164e59fb60f103d472360731e8c6bb4a7fe9ef"
integrity sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg== integrity sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==
@ -5585,10 +5585,10 @@ web-encoding@^1.1.5:
optionalDependencies: optionalDependencies:
"@zxing/text-encoding" "0.9.0" "@zxing/text-encoding" "0.9.0"
webdriver-bidi-protocol@0.2.11: webdriver-bidi-protocol@0.3.8:
version "0.2.11" version "0.3.8"
resolved "https://registry.yarnpkg.com/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.2.11.tgz#dba18d9b0a33aed33fab272dbd6e42411ac753cc" resolved "https://registry.yarnpkg.com/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.8.tgz#9c822b2647fd16d22b1b6fd730d4a3b863c93b93"
integrity sha512-Y9E1/oi4XMxcR8AT0ZC4OvYntl34SPgwjmELH+owjBr0korAX4jKgZULBWILGCVGdVCQ0dodTToIETozhG8zvA== integrity sha512-21Yi2GhGntMc671vNBCjiAeEVknXjVRoyu+k+9xOMShu+ZQfpGQwnBqbNz/Sv4GXZ6JmutlPAi2nIJcrymAWuQ==
whatwg-encoding@^2.0.0: whatwg-encoding@^2.0.0:
version "2.0.0" version "2.0.0"