mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 06:23:16 +00:00
Autoclick Support (#729)
Adds support for autoclick behavior:
- Adds new `autoclick` behavior option to `--behaviors`, but not
enabling by default
- Adds support for new exposed function `__bx_addSet` which allows
autoclick behavior to persist state about links that have already been
clicked to avoid duplicates, only used if link has an href
- Adds a new pageFinished flag on the worker state.
- Adds a on('dialog') handler to reject onbeforeunload page navigations,
when in behavior (page not finished), but accept when page is finished -
to allow navigation away only when behaviors are done
- Update to browsertrix-behaviors 0.7.0, which supports autoclick
- Add --clickSelector option to customize elements that will be clicked,
defaulting to `a`.
- Add --linkSelector as alias for --selectLinks for consistency
- Unknown options for --behaviors printed as warnings, instead of hard
exit, for forward compatibility for new behavior types in the future
Fixes #728, also #216, #665, #31
This commit is contained in:
Ilya Kreymer
GitHub
parent
871490758a
commit
b7150f1343
14 changed files with 259 additions and 108 deletions
124
yarn.lock
124
yarn.lock
|
|
@ -719,15 +719,15 @@
|
|||
tslib "^2.7.0"
|
||||
tsyringe "^4.8.0"
|
||||
|
||||
"@puppeteer/browsers@2.4.1":
|
||||
version "2.4.1"
|
||||
resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.4.1.tgz#7afd271199cc920ece2ff25109278be0a3e8a225"
|
||||
integrity sha512-0kdAbmic3J09I6dT8e9vE2JOCSt13wHCW5x/ly8TSt2bDtuIWe2TgLZZDHdcziw9AVCzflMAXCrVyRIhIs44Ng==
|
||||
"@puppeteer/browsers@2.7.0":
|
||||
version "2.7.0"
|
||||
resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.7.0.tgz#dad70b30458f4e0855b2f402055f408823cc67c5"
|
||||
integrity sha512-bO61XnTuopsz9kvtfqhVbH6LTM1koxK0IlBR+yuVrM2LB7mk8+5o1w18l5zqd5cs8xlf+ntgambqRqGifMDjog==
|
||||
dependencies:
|
||||
debug "^4.3.7"
|
||||
debug "^4.4.0"
|
||||
extract-zip "^2.0.1"
|
||||
progress "^2.0.3"
|
||||
proxy-agent "^6.4.0"
|
||||
proxy-agent "^6.5.0"
|
||||
semver "^7.6.3"
|
||||
tar-fs "^3.0.6"
|
||||
unbzip2-stream "^1.4.3"
|
||||
|
|
@ -1062,13 +1062,18 @@ acorn@^8.10.0, acorn@^8.9.0:
|
|||
resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.14.0.tgz#063e2c70cac5fb4f6467f0b11152e04c682795b0"
|
||||
integrity sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==
|
||||
|
||||
agent-base@^7.0.2, agent-base@^7.1.0, agent-base@^7.1.1:
|
||||
agent-base@^7.1.0:
|
||||
version "7.1.1"
|
||||
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.1.tgz#bdbded7dfb096b751a2a087eeeb9664725b2e317"
|
||||
integrity sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==
|
||||
dependencies:
|
||||
debug "^4.3.4"
|
||||
|
||||
agent-base@^7.1.2:
|
||||
version "7.1.3"
|
||||
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.3.tgz#29435eb821bc4194633a5b89e5bc4703bafc25a1"
|
||||
integrity sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==
|
||||
|
||||
ajv@^6.12.4:
|
||||
version "6.12.6"
|
||||
resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
|
||||
|
|
@ -1435,10 +1440,10 @@ browserslist@^4.24.0:
|
|||
node-releases "^2.0.18"
|
||||
update-browserslist-db "^1.1.1"
|
||||
|
||||
browsertrix-behaviors@^0.6.6:
|
||||
version "0.6.6"
|
||||
resolved "https://registry.yarnpkg.com/browsertrix-behaviors/-/browsertrix-behaviors-0.6.6.tgz#10bcccfb091c051f5c886d5f69487e6d184078de"
|
||||
integrity sha512-UPNcU9dV0nAvUwJHKKYCkuqdYdlMjK7AWYDyr4xBpSq55xmEh2wQlwQyDyJuUUUrhJNII4NqXK24hVXPdvf5VA==
|
||||
browsertrix-behaviors@^0.7.0:
|
||||
version "0.7.0"
|
||||
resolved "https://registry.yarnpkg.com/browsertrix-behaviors/-/browsertrix-behaviors-0.7.0.tgz#a08b7d3e9cd449d0d76b14a438e28472124fd1a4"
|
||||
integrity sha512-t0X74puXJsH8sVkkVZwEdo8L5E1PYtzX/RkVXM4fwwBIL804bOB8WIV+5Dfwov/odaukhB67KZhM00hN60SiBA==
|
||||
dependencies:
|
||||
query-selector-shadow-dom "^1.0.1"
|
||||
|
||||
|
|
@ -1534,13 +1539,12 @@ chownr@^1.1.1:
|
|||
resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
|
||||
integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
|
||||
|
||||
chromium-bidi@0.8.0:
|
||||
version "0.8.0"
|
||||
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.8.0.tgz#ffd79dad7db1fcc874f1c55fcf46ded05a884269"
|
||||
integrity sha512-uJydbGdTw0DEUjhoogGveneJVWX/9YuqkWePzMmkBYwtdAqo5d3J/ovNKFr+/2hWXYmYCr6it8mSSTIj6SS6Ug==
|
||||
chromium-bidi@0.11.0:
|
||||
version "0.11.0"
|
||||
resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.11.0.tgz#9c3c42ee7b42d8448e9fce8d649dc8bfbcc31153"
|
||||
integrity sha512-6CJWHkNRoyZyjV9Rwv2lYONZf1Xm0IuDyNq97nwSsxxP3wf5Bwy15K5rOvVKMtJ127jJBmxFUanSAOjgFRxgrA==
|
||||
dependencies:
|
||||
mitt "3.0.1"
|
||||
urlpattern-polyfill "10.0.0"
|
||||
zod "3.23.8"
|
||||
|
||||
ci-info@^3.2.0:
|
||||
|
|
@ -1696,7 +1700,7 @@ data-view-byte-offset@^1.0.0:
|
|||
es-errors "^1.3.0"
|
||||
is-data-view "^1.0.1"
|
||||
|
||||
debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.7:
|
||||
debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4:
|
||||
version "4.3.7"
|
||||
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52"
|
||||
integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==
|
||||
|
|
@ -1710,6 +1714,13 @@ debug@^3.2.7:
|
|||
dependencies:
|
||||
ms "^2.1.1"
|
||||
|
||||
debug@^4.4.0:
|
||||
version "4.4.0"
|
||||
resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.0.tgz#2b3f2aea2ffeb776477460267377dc8710faba8a"
|
||||
integrity sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==
|
||||
dependencies:
|
||||
ms "^2.1.3"
|
||||
|
||||
decode-uri-component@^0.2.2:
|
||||
version "0.2.2"
|
||||
resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.2.tgz#e69dbe25d37941171dd540e024c444cd5188e1e9"
|
||||
|
|
@ -1784,10 +1795,10 @@ detect-newline@^3.0.0:
|
|||
resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651"
|
||||
integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==
|
||||
|
||||
devtools-protocol@0.0.1367902:
|
||||
version "0.0.1367902"
|
||||
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz#7333bfc4466c5a54a4c6de48a9dfbcb4b811660c"
|
||||
integrity sha512-XxtPuC3PGakY6PD7dG66/o8KwJ/LkH2/EKe19Dcw58w53dv4/vSQEkn/SzuyhHE2q4zPgCkxQBxus3VV4ql+Pg==
|
||||
devtools-protocol@0.0.1380148:
|
||||
version "0.0.1380148"
|
||||
resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1380148.tgz#7dcdad06515135b244ff05878ca8019e041c1c55"
|
||||
integrity sha512-1CJABgqLxbYxVI+uJY/UDUHJtJ0KZTSjNYJYKqd9FRoXT33WDakDHNxRapMEgzeJ/C3rcs01+avshMnPmKQbvA==
|
||||
|
||||
diff-sequences@^29.6.3:
|
||||
version "29.6.3"
|
||||
|
|
@ -2603,12 +2614,12 @@ http-status-codes@^2.1.4:
|
|||
resolved "https://registry.yarnpkg.com/http-status-codes/-/http-status-codes-2.3.0.tgz#987fefb28c69f92a43aecc77feec2866349a8bfc"
|
||||
integrity sha512-RJ8XvFvpPM/Dmc5SV+dC4y5PCeOhT3x1Hq0NU3rjGeg5a/CqlhZ7uudknPwZFz4aeAXDcbAyaeP7GAo9lvngtA==
|
||||
|
||||
https-proxy-agent@^7.0.3, https-proxy-agent@^7.0.5:
|
||||
version "7.0.5"
|
||||
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz#9e8b5013873299e11fab6fd548405da2d6c602b2"
|
||||
integrity sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==
|
||||
https-proxy-agent@^7.0.6:
|
||||
version "7.0.6"
|
||||
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz#da8dfeac7da130b05c2ba4b59c9b6cd66611a6b9"
|
||||
integrity sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==
|
||||
dependencies:
|
||||
agent-base "^7.0.2"
|
||||
agent-base "^7.1.2"
|
||||
debug "4"
|
||||
|
||||
human-signals@^2.1.0:
|
||||
|
|
@ -3834,19 +3845,19 @@ p-try@^2.0.0:
|
|||
resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
|
||||
integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==
|
||||
|
||||
pac-proxy-agent@^7.0.1:
|
||||
version "7.0.2"
|
||||
resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz#0fb02496bd9fb8ae7eb11cfd98386daaac442f58"
|
||||
integrity sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==
|
||||
pac-proxy-agent@^7.1.0:
|
||||
version "7.1.0"
|
||||
resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.1.0.tgz#da7c3b5c4cccc6655aaafb701ae140fb23f15df2"
|
||||
integrity sha512-Z5FnLVVZSnX7WjBg0mhDtydeRZ1xMcATZThjySQUHqr+0ksP8kqaw23fNKkaaN/Z8gwLUs/W7xdl0I75eP2Xyw==
|
||||
dependencies:
|
||||
"@tootallnate/quickjs-emscripten" "^0.23.0"
|
||||
agent-base "^7.0.2"
|
||||
agent-base "^7.1.2"
|
||||
debug "^4.3.4"
|
||||
get-uri "^6.0.1"
|
||||
http-proxy-agent "^7.0.0"
|
||||
https-proxy-agent "^7.0.5"
|
||||
https-proxy-agent "^7.0.6"
|
||||
pac-resolver "^7.0.1"
|
||||
socks-proxy-agent "^8.0.4"
|
||||
socks-proxy-agent "^8.0.5"
|
||||
|
||||
pac-resolver@^7.0.1:
|
||||
version "7.0.1"
|
||||
|
|
@ -4056,19 +4067,19 @@ prop-types@^15.8.1:
|
|||
object-assign "^4.1.1"
|
||||
react-is "^16.13.1"
|
||||
|
||||
proxy-agent@^6.4.0:
|
||||
version "6.4.0"
|
||||
resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.4.0.tgz#b4e2dd51dee2b377748aef8d45604c2d7608652d"
|
||||
integrity sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==
|
||||
proxy-agent@^6.5.0:
|
||||
version "6.5.0"
|
||||
resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.5.0.tgz#9e49acba8e4ee234aacb539f89ed9c23d02f232d"
|
||||
integrity sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==
|
||||
dependencies:
|
||||
agent-base "^7.0.2"
|
||||
agent-base "^7.1.2"
|
||||
debug "^4.3.4"
|
||||
http-proxy-agent "^7.0.1"
|
||||
https-proxy-agent "^7.0.3"
|
||||
https-proxy-agent "^7.0.6"
|
||||
lru-cache "^7.14.1"
|
||||
pac-proxy-agent "^7.0.1"
|
||||
pac-proxy-agent "^7.1.0"
|
||||
proxy-from-env "^1.1.0"
|
||||
socks-proxy-agent "^8.0.2"
|
||||
socks-proxy-agent "^8.0.5"
|
||||
|
||||
proxy-from-env@^1.1.0:
|
||||
version "1.1.0"
|
||||
|
|
@ -4088,15 +4099,15 @@ punycode@^2.1.0:
|
|||
resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5"
|
||||
integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==
|
||||
|
||||
puppeteer-core@^23.7.1:
|
||||
version "23.9.0"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-23.9.0.tgz#24add69fb58dde4ac49d165872b44a30d2bf5b32"
|
||||
integrity sha512-hLVrav2HYMVdK0YILtfJwtnkBAwNOztUdR4aJ5YKDvgsbtagNr6urUJk9HyjRA9e+PaLI3jzJ0wM7A4jSZ7Qxw==
|
||||
puppeteer-core@^24.1.0:
|
||||
version "24.1.0"
|
||||
resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.1.0.tgz#4ea006ab26077dfbf6c72e2cf74797a7ff6db468"
|
||||
integrity sha512-ReefWoQgqdyl67uWEBy/TMZ4mAB7hP0JB5HIxSE8B1ot/4ningX1gmzHCOSNfMbTiS/VJHCvaZAe3oJTXph7yw==
|
||||
dependencies:
|
||||
"@puppeteer/browsers" "2.4.1"
|
||||
chromium-bidi "0.8.0"
|
||||
debug "^4.3.7"
|
||||
devtools-protocol "0.0.1367902"
|
||||
"@puppeteer/browsers" "2.7.0"
|
||||
chromium-bidi "0.11.0"
|
||||
debug "^4.4.0"
|
||||
devtools-protocol "0.0.1380148"
|
||||
typed-query-selector "^2.12.0"
|
||||
ws "^8.18.0"
|
||||
|
||||
|
|
@ -4445,12 +4456,12 @@ smart-buffer@^4.2.0:
|
|||
resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae"
|
||||
integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==
|
||||
|
||||
socks-proxy-agent@^8.0.2, socks-proxy-agent@^8.0.4:
|
||||
version "8.0.4"
|
||||
resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz#9071dca17af95f483300316f4b063578fa0db08c"
|
||||
integrity sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==
|
||||
socks-proxy-agent@^8.0.5:
|
||||
version "8.0.5"
|
||||
resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz#b9cdb4e7e998509d7659d689ce7697ac21645bee"
|
||||
integrity sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==
|
||||
dependencies:
|
||||
agent-base "^7.1.1"
|
||||
agent-base "^7.1.2"
|
||||
debug "^4.3.4"
|
||||
socks "^2.8.3"
|
||||
|
||||
|
|
@ -4959,11 +4970,6 @@ url-join@^4.0.1:
|
|||
resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.1.tgz#b642e21a2646808ffa178c4c5fda39844e12cde7"
|
||||
integrity sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==
|
||||
|
||||
urlpattern-polyfill@10.0.0:
|
||||
version "10.0.0"
|
||||
resolved "https://registry.yarnpkg.com/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz#f0a03a97bfb03cdf33553e5e79a2aadd22cac8ec"
|
||||
integrity sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==
|
||||
|
||||
util-deprecate@^1.0.1:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue