Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Disable behaviors entirely if --behaviors array is empty (#672)

Browse source 
Fixes #651
This commit is contained in:
Tessa Walsh 2024-08-27 16:20:19 -04:00 • committed by GitHub
parent c61a03de6e
commit 39c8f48bb2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 15 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/docs/user-guide/behaviors.md
View file

@ -10,6 +10,8 @@ See [Browsertrix Behaviors](https://github.com/webrecorder/browsertrix-behaviors
Browsertrix Crawler includes a `--pageExtraDelay`/`--delay` option, which can be used to have the crawler sleep for a configurable number of seconds after behaviors before moving on to the next page.
To disable behaviors for a crawl, use `--behaviors ""`.
## Additional Custom Behaviors
Custom behaviors can be mounted into the crawler and loaded from there. For example:

4
src/crawler.ts
View file

@ -484,7 +484,11 @@ export class Crawler {
logger.info("Seeds", this.seeds);
if (this.params.behaviorOpts) {
logger.info("Behavior Options", this.params.behaviorOpts);
} else {
logger.info("Behaviors disabled");
}
if (this.params.profile) {
logger.info("With Browser Profile", { url: this.params.profile });

4
src/util/argParser.ts
View file

@ -630,10 +630,14 @@ class ArgParser {
// background behaviors to apply
const behaviorOpts: { [key: string]: string | boolean } = {};
if (argv.behaviors.length > 0) {
argv.behaviors.forEach((x: string) => (behaviorOpts[x] = true));
behaviorOpts.log = BEHAVIOR_LOG_FUNC;
behaviorOpts.startEarly = true;
argv.behaviorOpts = JSON.stringify(behaviorOpts);
} else {
argv.behaviorOpts = "";
}
argv.text = argv.text || [];