tests: reduce logging (#596)

remove logging of crawl logs by default for clearer output from tests, only log in case of error.

src/util/storage.ts

@@ -208,6 +208,7 @@ export async function checkDiskUtilization(
   params: Record<string, any>,
   archiveDirSize: number,
   dfOutput = null,
+  doLog = true,
 ) {
   const diskUsage: Record<string, string> = await getDiskUsage(
     collDir,
@@ -217,9 +218,11 @@ export async function checkDiskUtilization(
 
   // Check that disk usage isn't already above threshold
   if (usedPercentage >= params.diskUtilization) {
+    if (doLog) {
       logger.info(
         `Disk utilization threshold reached ${usedPercentage}% > ${params.diskUtilization}%, stopping`,
       );
+    }
     return {
       stop: true,
       used: usedPercentage,
@@ -246,9 +249,11 @@ export async function checkDiskUtilization(
   );
 
   if (projectedUsedPercentage >= params.diskUtilization) {
+    if (doLog) {
       logger.info(
         `Disk utilization projected to reach threshold ${projectedUsedPercentage}% > ${params.diskUtilization}%, stopping`,
       );
+    }
     return {
       stop: true,
       used: usedPercentage,

tests/adblockrules.test.js

@@ -15,7 +15,7 @@ function runCrawl(name, config, commandExtra = "") {
       { input: configYaml, stdin: "inherit", encoding: "utf8" },
     );
 
-    console.log(proc);
+    //console.log(proc);
   } catch (error) {
     console.log(error);
   }

tests/blockrules.test.js

@@ -15,7 +15,7 @@ function runCrawl(name, config, commandExtra = "") {
       { input: configYaml, stdin: "inherit", encoding: "utf8" },
     );
 
-    console.log(proc);
+    //console.log(proc);
   } catch (error) {
     console.log(error);
   }

tests/config_stdin.test.js

@@ -12,7 +12,7 @@ test("pass config file via stdin", async () => {
       { input: configYaml, stdin: "inherit", encoding: "utf8" },
     );
 
-    console.log(proc);
+    //console.log(proc);
   } catch (error) {
     console.log(error);
   }

tests/rollover-writer.test.js

@@ -11,8 +11,6 @@ test("set rollover to 500K and ensure individual WARCs rollover, including scree
   let main = 0;
   let screenshots = 0;
 
-  console.log(warcLists);
-
   for (const name of warcLists) {
     if (name.startsWith("rec-")) {
       main++;

tests/storage.test.js

@@ -33,6 +33,7 @@ grpcfuse       1000000      285000    715000  28% /crawls`;
     params,
     5000 * 1024,
     mockDfOutput,
+    false
   );
   expect(returnValue).toEqual({
     stop: false,
@@ -60,6 +61,7 @@ grpcfuse       100000    85000     15000  85% /crawls`;
     params,
     3000 * 1024,
     mockDfOutput,
+    false
   );
   expect(returnValue).toEqual({
     stop: true,

tests/warcinfo.test.js

@@ -13,7 +13,7 @@ test("run crawl", async() => {
       { input: configYaml, stdin: "inherit", encoding: "utf8" },
     );
 
-    console.log(proc);
+    //console.log(proc);
     success = true;
   } catch (error) {
     console.log(error);