Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Deps update 1.6.1 (#826)

Browse source
This commit is contained in:
Ilya Kreymer 2025-05-02 00:43:37 -07:00 • committed by GitHub
parent d47812d139
commit fc59d04231
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 43 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Dockerfile
View file

@ -1,4 +1,4 @@
ARG BROWSER_VERSION=1.77.95 ARG BROWSER_VERSION=1.78.94
ARG BROWSER_IMAGE_BASE=webrecorder/browsertrix-browser-base:brave-${BROWSER_VERSION} ARG BROWSER_IMAGE_BASE=webrecorder/browsertrix-browser-base:brave-${BROWSER_VERSION}
FROM ${BROWSER_IMAGE_BASE} FROM ${BROWSER_IMAGE_BASE}
@ -39,7 +39,7 @@ ADD config/ /app/
ADD html/ /app/html/ ADD html/ /app/html/
ARG RWP_VERSION=2.3.4 ARG RWP_VERSION=2.3.6
ADD https://cdn.jsdelivr.net/npm/replaywebpage@${RWP_VERSION}/ui.js /app/html/rwp/ ADD https://cdn.jsdelivr.net/npm/replaywebpage@${RWP_VERSION}/ui.js /app/html/rwp/
ADD https://cdn.jsdelivr.net/npm/replaywebpage@${RWP_VERSION}/sw.js /app/html/rwp/ ADD https://cdn.jsdelivr.net/npm/replaywebpage@${RWP_VERSION}/sw.js /app/html/rwp/
ADD https://cdn.jsdelivr.net/npm/replaywebpage@${RWP_VERSION}/adblock/adblock.gz /app/html/rwp/adblock.gz ADD https://cdn.jsdelivr.net/npm/replaywebpage@${RWP_VERSION}/adblock/adblock.gz /app/html/rwp/adblock.gz

4
package.json
View file

@ -18,7 +18,7 @@
"dependencies": { "dependencies": {
"@novnc/novnc": "1.4.0", "@novnc/novnc": "1.4.0",
"@puppeteer/replay": "^3.1.1", "@puppeteer/replay": "^3.1.1",
"@webrecorder/wabac": "^2.22.9", "@webrecorder/wabac": "^2.22.15",
"browsertrix-behaviors": "^0.8.4", "browsertrix-behaviors": "^0.8.4",
"client-zip": "^2.4.5", "client-zip": "^2.4.5",
"css-selector-parser": "^3.0.5", "css-selector-parser": "^3.0.5",
@ -32,7 +32,7 @@
"p-queue": "^7.3.4", "p-queue": "^7.3.4",
"pixelmatch": "^5.3.0", "pixelmatch": "^5.3.0",
"pngjs": "^7.0.0", "pngjs": "^7.0.0",
"puppeteer-core": "^24.2.0", "puppeteer-core": "^24.7.2",
"sax": "^1.3.0", "sax": "^1.3.0",
"sharp": "^0.32.6", "sharp": "^0.32.6",
"tsc": "^2.0.4", "tsc": "^2.0.4",

6
tests/sitemap-parse.test.js
View file

@ -68,15 +68,15 @@ async function runCrawl(numExpected, url, sitemap="", limit=0, numExpectedLessTh
} }
test("test sitemap fully finish", async () => { test("test sitemap fully finish", async () => {
await runCrawl(3500, "https://www.mozilla.org/", "", 0); await runCrawl(3500, "https://developer.mozilla.org/", "", 0);
}); });
test("test sitemap with limit", async () => { test("test sitemap with limit", async () => {
await runCrawl(1900, "https://www.mozilla.org/", "", 2000); await runCrawl(1900, "https://developer.mozilla.org/", "", 2000);
}); });
test("test sitemap with limit, specific URL", async () => { test("test sitemap with limit, specific URL", async () => {
await runCrawl(1900, "https://www.mozilla.org/", "https://www.mozilla.org/sitemap.xml", 2000); await runCrawl(1900, "https://developer.mozilla.org/", "https://developer.mozilla.org/sitemap.xml", 2000);
}); });
test("test sitemap with application/xml content-type", async () => { test("test sitemap with application/xml content-type", async () => {

77
yarn.lock
View file

@ -772,16 +772,16 @@
tslib "^2.7.0" tslib "^2.7.0"
tsyringe "^4.8.0" tsyringe "^4.8.0"
"@puppeteer/browsers@2.7.1": "@puppeteer/browsers@2.10.2":
version "2.7.1" version "2.10.2"
resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.7.1.tgz#6df07e95d8e22239b77599f3ceaef4041b933e62" resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.10.2.tgz#c2a63cee699c6b5b971b9fcba9095098970f1648"
integrity sha512-MK7rtm8JjaxPN7Mf1JdZIZKPD2Z+W7osvrC1vjpvfOX1K0awDIHYbNi89f7eotp7eMUn2shWnt03HwVbriXtKQ== integrity sha512-i4Ez+s9oRWQbNjtI/3+jxr7OH508mjAKvza0ekPJem0ZtmsYHP3B5dq62+IaBHKaGCOuqJxXzvFLUhJvQ6jtsQ==
dependencies: dependencies:
debug "^4.4.0" debug "^4.4.0"
extract-zip "^2.0.1" extract-zip "^2.0.1"
progress "^2.0.3" progress "^2.0.3"
proxy-agent "^6.5.0" proxy-agent "^6.5.0"
semver "^7.7.0" semver "^7.7.1"
tar-fs "^3.0.8" tar-fs "^3.0.8"
yargs "^17.7.2" yargs "^17.7.2"
@ -1134,16 +1134,16 @@
resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406"
integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==
"@webrecorder/wabac@^2.22.9": "@webrecorder/wabac@^2.22.15":
version "2.22.9" version "2.22.15"
resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.22.9.tgz#c48fae9bc10015789999e87e68d14d52ad99515f" resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.22.15.tgz#2b5c31cf16fdd2055e7bae331fa26488359df9cd"
integrity sha512-FsOP9TAZtS1kFAcBMv4yTwcj0VvE2Ke25gDnxlZ/WQIDjZsjWaEK0ZRlY645Simi40byFQ2dry44X3dwtKWhEA== integrity sha512-GJvFNCnDdvpij1mBNRRpT4zfr66W5td4NpNKEPmBtEwTPdlRyNlU96ZSL3NRVtzvg7RG6LpI6Oa86lKd6EUpBA==
dependencies: dependencies:
"@peculiar/asn1-ecc" "^2.3.4" "@peculiar/asn1-ecc" "^2.3.4"
"@peculiar/asn1-schema" "^2.3.3" "@peculiar/asn1-schema" "^2.3.3"
"@peculiar/x509" "^1.9.2" "@peculiar/x509" "^1.9.2"
"@types/js-levenshtein" "^1.1.3" "@types/js-levenshtein" "^1.1.3"
"@webrecorder/wombat" "^3.8.8" "@webrecorder/wombat" "^3.8.11"
acorn "^8.10.0" acorn "^8.10.0"
auto-js-ipfs "^2.1.1" auto-js-ipfs "^2.1.1"
base64-js "^1.5.1" base64-js "^1.5.1"
@ -1164,10 +1164,10 @@
stream-browserify "^3.0.0" stream-browserify "^3.0.0"
warcio "^2.4.3" warcio "^2.4.3"
"@webrecorder/wombat@^3.8.8": "@webrecorder/wombat@^3.8.11":
version "3.8.8" version "3.8.11"
resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.8.tgz#aab4dd8eea6d6cb17bfefb7ee1802e7b45b11ed7" resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.11.tgz#392a3a7003fdb41f5b214d4571d5f38f9978f2c8"
integrity sha512-XkJOZAyHrdXNkAVoISQEh/NHzaBMekQZfWqes/k2vYkW6v9DmZ0wjP7Kf6MHCuajKX8uSH+caB2tv1kJgvnv3Q== integrity sha512-Sg97woFtMFPh0PJ60WV9sze4nw5aPEq+BE9dFwdd6WpA45da0pkjCVX9z+YXejQLxdvKy24U9+rDPA9W6XqIGQ==
dependencies: dependencies:
warcio "^2.4.0" warcio "^2.4.0"
@ -1704,14 +1704,6 @@ chrome-launcher@^1.1.2:
is-wsl "^2.2.0" is-wsl "^2.2.0"
lighthouse-logger "^2.0.1" lighthouse-logger "^2.0.1"
chromium-bidi@1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-1.2.0.tgz#e481b3eee0bf0f2d940a60b83faa79414f4752b8"
integrity sha512-XtdJ1GSN6S3l7tO7F77GhNsw0K367p0IsLYf2yZawCVAKKC3lUvDhPdMVrB2FNhmhfW43QGYbEX3Wg6q0maGwQ==
dependencies:
mitt "^3.0.1"
zod "^3.24.1"
chromium-bidi@2.1.2: chromium-bidi@2.1.2:
version "2.1.2" version "2.1.2"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-2.1.2.tgz#b0710279f993128d4e0b41c892209ea093217d97" resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-2.1.2.tgz#b0710279f993128d4e0b41c892209ea093217d97"
@ -1720,6 +1712,14 @@ chromium-bidi@2.1.2:
mitt "^3.0.1" mitt "^3.0.1"
zod "^3.24.1" zod "^3.24.1"
chromium-bidi@4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-4.1.1.tgz#e1c34154ddd94473f180fd15158a24d36049e3d5"
integrity sha512-biR7t4vF3YluE6RlMSk9IWk+b9U+WWyzHp+N2pL9vRTk+UXHYRTVp7jTK58ZNzMLBgoLMHY4QyJMbeuw3eKxqg==
dependencies:
mitt "^3.0.1"
zod "^3.24.1"
ci-info@^3.2.0: ci-info@^3.2.0:
version "3.9.0" version "3.9.0"
resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4" resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4"
@ -2031,16 +2031,16 @@ detect-newline@^3.0.0:
resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651"
integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==
devtools-protocol@0.0.1402036:
version "0.0.1402036"
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1402036.tgz#6f55044daed0ae5f0ec5c80c834b7b8edc822441"
integrity sha512-JwAYQgEvm3yD45CHB+RmF5kMbWtXBaOGwuxa87sZogHcLCv8c/IqnThaoQ1y60d7pXWjSKWQphPEc+1rAScVdg==
devtools-protocol@0.0.1413902: devtools-protocol@0.0.1413902:
version "0.0.1413902" version "0.0.1413902"
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1413902.tgz#a0f00fe9eb25ab337a8f9656a29e0a1a69f42401" resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1413902.tgz#a0f00fe9eb25ab337a8f9656a29e0a1a69f42401"
integrity sha512-yRtvFD8Oyk7C9Os3GmnFZLu53yAfsnyw1s+mLmHHUK0GQEc9zthHWvS1r67Zqzm5t7v56PILHIVZ7kmFMaL2yQ== integrity sha512-yRtvFD8Oyk7C9Os3GmnFZLu53yAfsnyw1s+mLmHHUK0GQEc9zthHWvS1r67Zqzm5t7v56PILHIVZ7kmFMaL2yQ==
devtools-protocol@0.0.1425554:
version "0.0.1425554"
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz#51ed2fed1405f56783d24a393f7c75b6bbb58029"
integrity sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==
devtools-protocol@0.0.1436416: devtools-protocol@0.0.1436416:
version "0.0.1436416" version "0.0.1436416"
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1436416.tgz#ce8af8a210b8bcac83c5c8f095b9f977a9570df0" resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1436416.tgz#ce8af8a210b8bcac83c5c8f095b9f977a9570df0"
@ -4544,17 +4544,17 @@ puppeteer-core@24.4.0, puppeteer-core@^24.4.0:
typed-query-selector "^2.12.0" typed-query-selector "^2.12.0"
ws "^8.18.1" ws "^8.18.1"
puppeteer-core@^24.2.0: puppeteer-core@^24.7.2:
version "24.2.0" version "24.7.2"
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.2.0.tgz#4f935d1417777b335ca747bf1c3ae997989ed6e4" resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.7.2.tgz#734e377a5634ce1e419fa3ce20ad297a7e1a99ff"
integrity sha512-e4A4/xqWdd4kcE6QVHYhJ+Qlx/+XpgjP4d8OwBx0DJoY/nkIRhSgYmKQnv7+XSs1ofBstalt+XPGrkaz4FoXOQ== integrity sha512-P9pZyTmJqKODFCnkZgemCpoFA4LbAa8+NumHVQKyP5X9IgdNS1ZnAnIh1sMAwhF8/xEUGf7jt+qmNLlKieFw1Q==
dependencies: dependencies:
"@puppeteer/browsers" "2.7.1" "@puppeteer/browsers" "2.10.2"
chromium-bidi "1.2.0" chromium-bidi "4.1.1"
debug "^4.4.0" debug "^4.4.0"
devtools-protocol "0.0.1402036" devtools-protocol "0.0.1425554"
typed-query-selector "^2.12.0" typed-query-selector "^2.12.0"
ws "^8.18.0" ws "^8.18.1"
puppeteer@^24.4.0: puppeteer@^24.4.0:
version "24.4.0" version "24.4.0"
@ -4824,7 +4824,7 @@ semver@^7.3.5, semver@^7.5.3, semver@^7.5.4:
resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
semver@^7.7.0, semver@^7.7.1: semver@^7.7.1:
version "7.7.1" version "7.7.1"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.1.tgz#abd5098d82b18c6c81f6074ff2647fd3e7220c9f" resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.1.tgz#abd5098d82b18c6c81f6074ff2647fd3e7220c9f"
integrity sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA== integrity sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==
@ -5652,11 +5652,6 @@ ws@^7.0.0, ws@^7.4.4:
resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9" resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9"
integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ== integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==
ws@^8.18.0:
version "8.18.0"
resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
ws@^8.18.1: ws@^8.18.1:
version "8.18.1" version "8.18.1"
resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.1.tgz#ea131d3784e1dfdff91adb0a4a116b127515e3cb" resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.1.tgz#ea131d3784e1dfdff91adb0a4a116b127515e3cb"