mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 06:23:16 +00:00
misc tweaks: (#650)
- logging: log behavior options that are enabled on startup, after seeds - redis: launch local redis only if --redisStoreUrl starts with redis://localhost or redis://127.0.0.1 - interrupt: check that crawler is not 'done' before exiting with exit code 13, if already done, exit with 0
This commit is contained in:
parent
48716c172d
commit
d620eb8e31
1 changed files with 15 additions and 4 deletions
|
@ -458,7 +458,14 @@ export class Crawler {
|
|||
|
||||
this.proxyServer = initProxy(this.params.proxyServer);
|
||||
|
||||
subprocesses.push(this.launchRedis());
|
||||
const redisUrl = this.params.redisStoreUrl || "redis://localhost:6379/0";
|
||||
|
||||
if (
|
||||
redisUrl.startsWith("redis://localhost:") ||
|
||||
redisUrl.startsWith("redis://127.0.0.1:")
|
||||
) {
|
||||
subprocesses.push(this.launchRedis());
|
||||
}
|
||||
|
||||
await fsp.mkdir(this.logDir, { recursive: true });
|
||||
|
||||
|
@ -477,6 +484,8 @@ export class Crawler {
|
|||
|
||||
logger.info("Seeds", this.seeds);
|
||||
|
||||
logger.info("Behavior Options", this.params.behaviorOpts);
|
||||
|
||||
if (this.params.profile) {
|
||||
logger.info("With Browser Profile", { url: this.params.profile });
|
||||
}
|
||||
|
@ -1242,10 +1251,12 @@ self.__bx_behaviors.selectMainBehavior();
|
|||
await this.browser.close();
|
||||
await closeWorkers(0);
|
||||
await this.closeFiles();
|
||||
await this.setStatusAndExit(13, "interrupted");
|
||||
} else {
|
||||
await this.setStatusAndExit(0, "done");
|
||||
if (!this.done) {
|
||||
await this.setStatusAndExit(13, "interrupted");
|
||||
return;
|
||||
}
|
||||
}
|
||||
await this.setStatusAndExit(0, "done");
|
||||
}
|
||||
|
||||
async isCrawlRunning() {
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue