Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-12-08 06:09:48 +00:00
Code Issues Projects Releases Packages Wiki Activity

try custom click

Browse source
This commit is contained in:
Ilya Kreymer 2025-11-10 22:25:35 -08:00
parent 8b1d6ae5e9
commit 38035be235
5 changed files with 32 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
behaviors.js
View file

File diff suppressed because one or more lines are too long

1
package.json
View file

@ -24,6 +24,7 @@
"css-selector-parser": "^3.0.5",
"fetch-socks": "^1.3.0",
"get-folder-size": "^4.0.0",
"ghost-cursor": "^1.4.1",
"husky": "^8.0.3",
"ioredis": "^5.3.2",
"iso-639-1": "^3.1.5",

8
src/crawler.ts
View file

@ -3,6 +3,7 @@ import path from "path";
import fs, { WriteStream } from "fs";
import os from "os";
import fsp from "fs/promises";
import { createCursor } from "ghost-cursor";
import {
RedisCrawlState,
@ -835,6 +836,13 @@ export class Crawler {
(url: string) => callbacks.addLink && callbacks.addLink(url),
);
await page.exposeFunction(BxFunctionBindings.Click, async (sel: string) => {
//@ts-expect-error: ignore
const cursor = createCursor(page);
await cursor.click(sel);
logger.info("Custom Click Made", {}, "behaviorScript");
});
// used for both behaviors and link extraction now
await this.browser.addInitScript(page, btrixBehaviors);

2
src/util/constants.ts
View file

@ -32,6 +32,8 @@ export enum BxFunctionBindings {
NextFlowStep = "__bx_nextFlowStep",
ContentCheckFailed = "__bx_contentCheckFailed",
Click = "__bx_click",
}
export const MAX_DEPTH = 1000000;

19
yarn.lock
View file

@ -914,6 +914,11 @@
dependencies:
"@babel/types" "^7.20.7"
"@types/bezier-js@4":
version "4.1.3"
resolved "https://registry.yarnpkg.com/@types/bezier-js/-/bezier-js-4.1.3.tgz#237d4fe7e9aae7edd0c27a71f9f236f4ddc1c562"
integrity sha512-FNVVCu5mx/rJCWBxLTcL7oOajmGtWtBTDjq6DSUWUI12GeePivrZZXz+UgE0D6VYsLEjvExRO03z4hVtu3pTEQ==
"@types/graceful-fs@^4.1.3":
version "4.1.9"
resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.9.tgz#2a06bc0f68a20ab37b3e36aa238be6abdf49e8b4"
@ -1534,6 +1539,11 @@ basic-ftp@^5.0.2:
resolved "https://registry.yarnpkg.com/basic-ftp/-/basic-ftp-5.0.5.tgz#14a474f5fffecca1f4f406f1c26b18f800225ac0"
integrity sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==
bezier-js@^6.1.3:
version "6.1.4"
resolved "https://registry.yarnpkg.com/bezier-js/-/bezier-js-6.1.4.tgz#c7828f6c8900562b69d5040afb881bcbdad82001"
integrity sha512-PA0FW9ZpcHbojUCMu28z9Vg/fNkwTj5YhusSAjHHDfHDGLxJ6YUKrAN2vk1fP2MMOxVw4Oko16FMlRGVBGqLKg==
bl@^4.0.3:
version "4.1.0"
resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
@ -2697,6 +2707,15 @@ get-uri@^6.0.1:
debug "^4.3.4"
fs-extra "^11.2.0"
ghost-cursor@^1.4.1:
version "1.4.1"
resolved "https://registry.yarnpkg.com/ghost-cursor/-/ghost-cursor-1.4.1.tgz#dbdf38132459bb385334d1093958134b57551a87"
integrity sha512-K8A8/Co/Jbdqee694qrNsGWBG51DVK5UF2gGKEoZBDx9F1WmoD2SzUoDHWoY7O+TY84s1VrWwwfkVKxI2FoV2Q==
dependencies:
"@types/bezier-js" "4"
bezier-js "^6.1.3"
debug "^4.3.4"
github-from-package@0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce"