Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

optimize shutdown: if after interrupt signal was received, redis connection is gone, assume crawler is being terminated and exit quickly, (#292)

Browse source 
don't attemtpt to reconnect to redis (assume crawler is also being shutdown)
This commit is contained in:
Ilya Kreymer 2023-04-24 09:50:49 -07:00 • committed by GitHub
parent 5c497f4fa4
commit 3c7c7bfbc4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

4
main.js
View file

@ -1,8 +1,10 @@
#!/usr/bin/env -S node --experimental-global-webcrypto #!/usr/bin/env -S node --experimental-global-webcrypto
import { logger } from "./util/logger.js"; import { logger } from "./util/logger.js";
import { setExitOnRedisError } from "./util/redis.js";
import { Crawler } from "./crawler.js"; import { Crawler } from "./crawler.js";
var crawler = null; var crawler = null;
var lastSigInt = 0; var lastSigInt = 0;
@ -21,6 +23,8 @@ async function handleTerminate(signame) {
process.exit(0); process.exit(0);
} }
setExitOnRedisError(true);
try { try {
if (!crawler.interrupted) { if (!crawler.interrupted) {
logger.info("SIGNAL: gracefully finishing current pages..."); logger.info("SIGNAL: gracefully finishing current pages...");

8
util/redis.js
View file

@ -4,6 +4,7 @@ import { logger } from "./logger.js";
const error = console.error; const error = console.error;
let lastLogTime = 0; let lastLogTime = 0;
let exitOnError = false;
// log only once every 10 seconds // log only once every 10 seconds
const REDIS_ERROR_LOG_INTERVAL_SECS = 10000; const REDIS_ERROR_LOG_INTERVAL_SECS = 10000;
@ -17,6 +18,9 @@ console.error = function (...args) {
let now = Date.now(); let now = Date.now();
if ((now - lastLogTime) > REDIS_ERROR_LOG_INTERVAL_SECS) { if ((now - lastLogTime) > REDIS_ERROR_LOG_INTERVAL_SECS) {
if (lastLogTime && exitOnError) {
logger.fatal("Crawl interrupted, redis gone, exiting", {}, "redis");
}
logger.warn("ioredis error", {error: args[0]}, "redis"); logger.warn("ioredis error", {error: args[0]}, "redis");
lastLogTime = now; lastLogTime = now;
} }
@ -30,3 +34,7 @@ export async function initRedis(url) {
await redis.connect(); await redis.connect();
return redis; return redis;
} }
export function setExitOnRedisError() {
exitOnError = true;
}