diff --git a/Dockerfile b/Dockerfile index 7ec76f05..22fdada7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG BROWSER_VERSION=1.74.48 +ARG BROWSER_VERSION=1.74.51 ARG BROWSER_IMAGE_BASE=webrecorder/browsertrix-browser-base:brave-${BROWSER_VERSION} FROM ${BROWSER_IMAGE_BASE} diff --git a/package.json b/package.json index 766d3e71..f23bef14 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "p-queue": "^7.3.4", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", - "puppeteer-core": "^24.1.0", + "puppeteer-core": "^24.2.0", "sax": "^1.3.0", "sharp": "^0.32.6", "tsc": "^2.0.4", diff --git a/src/replaycrawler.ts b/src/replaycrawler.ts index 5d360dfc..0dbcf66c 100644 --- a/src/replaycrawler.ts +++ b/src/replaycrawler.ts @@ -438,6 +438,11 @@ export class ReplayCrawler extends Crawler { ); } + while (replayFrame.url().indexOf("about:blank") >= 0) { + logger.debug("Waiting for replay frame to update"); + await sleep(2); + } + // optionally reload (todo: reevaluate if this is needed) // await page.reload(); diff --git a/tests/qa_compare.test.js b/tests/qa_compare.test.js index 8daed468..3867ea03 100644 --- a/tests/qa_compare.test.js +++ b/tests/qa_compare.test.js @@ -59,6 +59,8 @@ test("run QA comparison, with write pages to redis", async () => { expect(json.comparison).toHaveProperty("screenshotMatch"); expect(json.comparison).toHaveProperty("textMatch"); expect(json.comparison).toHaveProperty("resourceCounts"); + expect(json.comparison.screenshotMatch).toBe(1); + expect(json.comparison.textMatch).toBe(1); expect(json.comparison.resourceCounts).toHaveProperty("crawlGood"); expect(json.comparison.resourceCounts).toHaveProperty("crawlBad"); diff --git a/yarn.lock b/yarn.lock index 683c81e9..6389bfa5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -719,18 +719,17 @@ tslib "^2.7.0" tsyringe "^4.8.0" -"@puppeteer/browsers@2.7.0": - version "2.7.0" - resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.7.0.tgz#dad70b30458f4e0855b2f402055f408823cc67c5" - integrity sha512-bO61XnTuopsz9kvtfqhVbH6LTM1koxK0IlBR+yuVrM2LB7mk8+5o1w18l5zqd5cs8xlf+ntgambqRqGifMDjog== +"@puppeteer/browsers@2.7.1": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.7.1.tgz#6df07e95d8e22239b77599f3ceaef4041b933e62" + integrity sha512-MK7rtm8JjaxPN7Mf1JdZIZKPD2Z+W7osvrC1vjpvfOX1K0awDIHYbNi89f7eotp7eMUn2shWnt03HwVbriXtKQ== dependencies: debug "^4.4.0" extract-zip "^2.0.1" progress "^2.0.3" proxy-agent "^6.5.0" - semver "^7.6.3" - tar-fs "^3.0.6" - unbzip2-stream "^1.4.3" + semver "^7.7.0" + tar-fs "^3.0.8" yargs "^17.7.2" "@sinclair/typebox@^0.27.8": @@ -1337,11 +1336,25 @@ bare-fs@^2.1.1: bare-path "^2.0.0" bare-stream "^2.0.0" +bare-fs@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/bare-fs/-/bare-fs-4.0.1.tgz#85844f34da819c76754d545323a8b23ed3617c76" + integrity sha512-ilQs4fm/l9eMfWY2dY0WCIUplSUp7U0CT1vrqMg1MUdeZl4fypu5UP0XcDBK5WBQPJAKP1b7XEodISmekH/CEg== + dependencies: + bare-events "^2.0.0" + bare-path "^3.0.0" + bare-stream "^2.0.0" + bare-os@^2.1.0: version "2.4.4" resolved "https://registry.yarnpkg.com/bare-os/-/bare-os-2.4.4.tgz#01243392eb0a6e947177bb7c8a45123d45c9b1a9" integrity sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ== +bare-os@^3.0.1: + version "3.4.0" + resolved "https://registry.yarnpkg.com/bare-os/-/bare-os-3.4.0.tgz#97be31503f3095beb232a6871f0118859832eb0c" + integrity sha512-9Ous7UlnKbe3fMi7Y+qh0DwAup6A1JkYgPnjvMDNOlmnxNRQvQ/7Nst+OnUQKzk0iAT0m9BisbDVp9gCv8+ETA== + bare-path@^2.0.0, bare-path@^2.1.0: version "2.1.3" resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-2.1.3.tgz#594104c829ef660e43b5589ec8daef7df6cedb3e" @@ -1349,6 +1362,13 @@ bare-path@^2.0.0, bare-path@^2.1.0: dependencies: bare-os "^2.1.0" +bare-path@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-3.0.0.tgz#b59d18130ba52a6af9276db3e96a2e3d3ea52178" + integrity sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw== + dependencies: + bare-os "^3.0.1" + bare-stream@^2.0.0: version "2.4.2" resolved "https://registry.yarnpkg.com/bare-stream/-/bare-stream-2.4.2.tgz#5a4241ff8a3bdd6d037fc459ab3e41189d2f2576" @@ -1469,7 +1489,7 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer@^5.2.1, buffer@^5.5.0: +buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== @@ -1539,13 +1559,13 @@ chownr@^1.1.1: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== -chromium-bidi@0.11.0: - version "0.11.0" - resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.11.0.tgz#9c3c42ee7b42d8448e9fce8d649dc8bfbcc31153" - integrity sha512-6CJWHkNRoyZyjV9Rwv2lYONZf1Xm0IuDyNq97nwSsxxP3wf5Bwy15K5rOvVKMtJ127jJBmxFUanSAOjgFRxgrA== +chromium-bidi@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-1.2.0.tgz#e481b3eee0bf0f2d940a60b83faa79414f4752b8" + integrity sha512-XtdJ1GSN6S3l7tO7F77GhNsw0K367p0IsLYf2yZawCVAKKC3lUvDhPdMVrB2FNhmhfW43QGYbEX3Wg6q0maGwQ== dependencies: - mitt "3.0.1" - zod "3.23.8" + mitt "^3.0.1" + zod "^3.24.1" ci-info@^3.2.0: version "3.9.0" @@ -1795,10 +1815,10 @@ detect-newline@^3.0.0: resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -devtools-protocol@0.0.1380148: - version "0.0.1380148" - resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1380148.tgz#7dcdad06515135b244ff05878ca8019e041c1c55" - integrity sha512-1CJABgqLxbYxVI+uJY/UDUHJtJ0KZTSjNYJYKqd9FRoXT33WDakDHNxRapMEgzeJ/C3rcs01+avshMnPmKQbvA== +devtools-protocol@0.0.1402036: + version "0.0.1402036" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1402036.tgz#6f55044daed0ae5f0ec5c80c834b7b8edc822441" + integrity sha512-JwAYQgEvm3yD45CHB+RmF5kMbWtXBaOGwuxa87sZogHcLCv8c/IqnThaoQ1y60d7pXWjSKWQphPEc+1rAScVdg== diff-sequences@^29.6.3: version "29.6.3" @@ -3644,7 +3664,7 @@ minio@^7.1.3: web-encoding "^1.1.5" xml2js "^0.5.0" -mitt@3.0.1: +mitt@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/mitt/-/mitt-3.0.1.tgz#ea36cf0cc30403601ae074c8f77b7092cdab36d1" integrity sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw== @@ -4099,15 +4119,15 @@ punycode@^2.1.0: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== -puppeteer-core@^24.1.0: - version "24.1.0" - resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.1.0.tgz#4ea006ab26077dfbf6c72e2cf74797a7ff6db468" - integrity sha512-ReefWoQgqdyl67uWEBy/TMZ4mAB7hP0JB5HIxSE8B1ot/4ningX1gmzHCOSNfMbTiS/VJHCvaZAe3oJTXph7yw== +puppeteer-core@^24.2.0: + version "24.2.0" + resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.2.0.tgz#4f935d1417777b335ca747bf1c3ae997989ed6e4" + integrity sha512-e4A4/xqWdd4kcE6QVHYhJ+Qlx/+XpgjP4d8OwBx0DJoY/nkIRhSgYmKQnv7+XSs1ofBstalt+XPGrkaz4FoXOQ== dependencies: - "@puppeteer/browsers" "2.7.0" - chromium-bidi "0.11.0" + "@puppeteer/browsers" "2.7.1" + chromium-bidi "1.2.0" debug "^4.4.0" - devtools-protocol "0.0.1380148" + devtools-protocol "0.0.1402036" typed-query-selector "^2.12.0" ws "^8.18.0" @@ -4352,11 +4372,16 @@ semver@^6.3.0, semver@^6.3.1: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.3.5, semver@^7.5.3, semver@^7.5.4, semver@^7.6.3: +semver@^7.3.5, semver@^7.5.3, semver@^7.5.4: version "7.6.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== +semver@^7.7.0: + version "7.7.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.1.tgz#abd5098d82b18c6c81f6074ff2647fd3e7220c9f" + integrity sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA== + set-function-length@^1.2.1: version "1.2.2" resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" @@ -4688,7 +4713,7 @@ tar-fs@^2.0.0: pump "^3.0.0" tar-stream "^2.1.4" -tar-fs@^3.0.4, tar-fs@^3.0.6: +tar-fs@^3.0.4: version "3.0.6" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.6.tgz#eaccd3a67d5672f09ca8e8f9c3d2b89fa173f217" integrity sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w== @@ -4699,6 +4724,17 @@ tar-fs@^3.0.4, tar-fs@^3.0.6: bare-fs "^2.1.1" bare-path "^2.1.0" +tar-fs@^3.0.8: + version "3.0.8" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.8.tgz#8f62012537d5ff89252d01e48690dc4ebed33ab7" + integrity sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg== + dependencies: + pump "^3.0.0" + tar-stream "^3.1.5" + optionalDependencies: + bare-fs "^4.0.1" + bare-path "^3.0.0" + tar-stream@^2.1.4: version "2.2.0" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" @@ -4760,11 +4796,6 @@ through2@^4.0.2: dependencies: readable-stream "3" -through@^2.3.8: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== - tmpl@1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" @@ -4913,14 +4944,6 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" -unbzip2-stream@^1.4.3: - version "1.4.3" - resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7" - integrity sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg== - dependencies: - buffer "^5.2.1" - through "^2.3.8" - undici-types@~6.19.2, undici-types@~6.19.8: version "6.19.8" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" @@ -5191,7 +5214,7 @@ yocto-queue@^0.1.0: resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== -zod@3.23.8: - version "3.23.8" - resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d" - integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g== +zod@^3.24.1: + version "3.24.1" + resolved "https://registry.yarnpkg.com/zod/-/zod-3.24.1.tgz#27445c912738c8ad1e9de1bea0359fa44d9d35ee" + integrity sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==