optimize shutdown: if after interrupt signal was received, redis connection is gone, assume crawler is being terminated and exit quickly, (#292)

don't attemtpt to reconnect to redis (assume crawler is also being shutdown)

main.js

@@ -1,8 +1,10 @@
 #!/usr/bin/env -S node --experimental-global-webcrypto
 
 import { logger } from "./util/logger.js";
+import { setExitOnRedisError } from "./util/redis.js";
 import { Crawler } from "./crawler.js";
 
+
 var crawler = null;
 
 var lastSigInt = 0;
@@ -21,6 +23,8 @@ async function handleTerminate(signame) {
     process.exit(0);
   }
 
+  setExitOnRedisError(true);
+
   try {
     if (!crawler.interrupted) {
       logger.info("SIGNAL: gracefully finishing current pages...");

util/redis.js

@@ -4,6 +4,7 @@ import { logger } from "./logger.js";
 const error = console.error;
 
 let lastLogTime = 0;
+let exitOnError = false;
 
 // log only once every 10 seconds
 const REDIS_ERROR_LOG_INTERVAL_SECS = 10000;
@@ -17,6 +18,9 @@ console.error = function (...args) {
     let now = Date.now();
 
     if ((now - lastLogTime) > REDIS_ERROR_LOG_INTERVAL_SECS) {
+      if (lastLogTime && exitOnError) {
+        logger.fatal("Crawl interrupted, redis gone, exiting", {}, "redis");
+      }
       logger.warn("ioredis error", {error: args[0]}, "redis");
       lastLogTime = now;
     }
@@ -30,3 +34,7 @@ export async function initRedis(url) {
   await redis.connect();
   return redis;
 }
+
+export function setExitOnRedisError() {
+  exitOnError = true;
+}