Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Disable behaviors entirely if --behaviors array is empty (#672)

Browse source 
Fixes #651
This commit is contained in:
Tessa Walsh 2024-08-27 16:20:19 -04:00 • committed by GitHub
parent c61a03de6e
commit 39c8f48bb2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 15 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/docs/user-guide/behaviors.md
View file

@ -10,6 +10,8 @@ See [Browsertrix Behaviors](https://github.com/webrecorder/browsertrix-behaviors
Browsertrix Crawler includes a `--pageExtraDelay`/`--delay` option, which can be used to have the crawler sleep for a configurable number of seconds after behaviors before moving on to the next page. Browsertrix Crawler includes a `--pageExtraDelay`/`--delay` option, which can be used to have the crawler sleep for a configurable number of seconds after behaviors before moving on to the next page.
To disable behaviors for a crawl, use `--behaviors ""`.
## Additional Custom Behaviors ## Additional Custom Behaviors
Custom behaviors can be mounted into the crawler and loaded from there. For example: Custom behaviors can be mounted into the crawler and loaded from there. For example:

6
src/crawler.ts
View file

@ -484,7 +484,11 @@ export class Crawler {
logger.info("Seeds", this.seeds); logger.info("Seeds", this.seeds);
logger.info("Behavior Options", this.params.behaviorOpts); if (this.params.behaviorOpts) {
logger.info("Behavior Options", this.params.behaviorOpts);
} else {
logger.info("Behaviors disabled");
}
if (this.params.profile) { if (this.params.profile) {
logger.info("With Browser Profile", { url: this.params.profile }); logger.info("With Browser Profile", { url: this.params.profile });

12
src/util/argParser.ts
View file

@ -630,10 +630,14 @@ class ArgParser {
// background behaviors to apply // background behaviors to apply
const behaviorOpts: { [key: string]: string | boolean } = {}; const behaviorOpts: { [key: string]: string | boolean } = {};
argv.behaviors.forEach((x: string) => (behaviorOpts[x] = true)); if (argv.behaviors.length > 0) {
behaviorOpts.log = BEHAVIOR_LOG_FUNC; argv.behaviors.forEach((x: string) => (behaviorOpts[x] = true));
behaviorOpts.startEarly = true; behaviorOpts.log = BEHAVIOR_LOG_FUNC;
argv.behaviorOpts = JSON.stringify(behaviorOpts); behaviorOpts.startEarly = true;
argv.behaviorOpts = JSON.stringify(behaviorOpts);
} else {
argv.behaviorOpts = "";
}
argv.text = argv.text || []; argv.text = argv.text || [];