From 3c7c7bfbc4918f95de46d7f7cd1bac6e6c39a14f Mon Sep 17 00:00:00 2001
From: Ilya Kreymer <ikreymer@users.noreply.github.com>
Date: Mon, 24 Apr 2023 09:50:49 -0700
Subject: [PATCH] optimize shutdown: if after interrupt signal was received,
 redis connection is gone, assume crawler is being terminated and exit
 quickly, (#292)

don't attemtpt to reconnect to redis (assume crawler is also being shutdown)
---
 main.js       | 4 ++++
 util/redis.js | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/main.js b/main.js
index 785716d1..80682cc2 100755
--- a/main.js
+++ b/main.js
@@ -1,8 +1,10 @@
 #!/usr/bin/env -S node --experimental-global-webcrypto
 
 import { logger } from "./util/logger.js";
+import { setExitOnRedisError } from "./util/redis.js";
 import { Crawler } from "./crawler.js";
 
+
 var crawler = null;
 
 var lastSigInt = 0;
@@ -21,6 +23,8 @@ async function handleTerminate(signame) {
     process.exit(0);
   }
 
+  setExitOnRedisError(true);
+
   try {
     if (!crawler.interrupted) {
       logger.info("SIGNAL: gracefully finishing current pages...");
diff --git a/util/redis.js b/util/redis.js
index 9ba85c6c..4bd839b3 100644
--- a/util/redis.js
+++ b/util/redis.js
@@ -4,6 +4,7 @@ import { logger } from "./logger.js";
 const error = console.error;
 
 let lastLogTime = 0;
+let exitOnError = false;
 
 // log only once every 10 seconds
 const REDIS_ERROR_LOG_INTERVAL_SECS = 10000;
@@ -17,6 +18,9 @@ console.error = function (...args) {
     let now = Date.now();
 
     if ((now - lastLogTime) > REDIS_ERROR_LOG_INTERVAL_SECS) {
+      if (lastLogTime && exitOnError) {
+        logger.fatal("Crawl interrupted, redis gone, exiting", {}, "redis");
+      }
       logger.warn("ioredis error", {error: args[0]}, "redis");
       lastLogTime = now;
     }
@@ -30,3 +34,7 @@ export async function initRedis(url) {
   await redis.connect();
   return redis;
 }
+
+export function setExitOnRedisError() {
+  exitOnError = true;
+}