Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

tests: reduce logging (#596)

Browse source 
remove logging of crawl logs by default for clearer output from tests, only log in case of error.
This commit is contained in:
Ilya Kreymer 2024-06-26 13:05:13 -07:00 • committed by GitHub
parent 4495532606
commit a3396adba2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 17 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/util/storage.ts
View file

@ -208,6 +208,7 @@ export async function checkDiskUtilization(
params: Record<string, any>, params: Record<string, any>,
archiveDirSize: number, archiveDirSize: number,
dfOutput = null, dfOutput = null,
doLog = true,
) { ) {
const diskUsage: Record<string, string> = await getDiskUsage( const diskUsage: Record<string, string> = await getDiskUsage(
collDir, collDir,
@ -217,9 +218,11 @@ export async function checkDiskUtilization(
// Check that disk usage isn't already above threshold // Check that disk usage isn't already above threshold
if (usedPercentage >= params.diskUtilization) { if (usedPercentage >= params.diskUtilization) {
logger.info( if (doLog) {
`Disk utilization threshold reached ${usedPercentage}% > ${params.diskUtilization}%, stopping`, logger.info(
); `Disk utilization threshold reached ${usedPercentage}% > ${params.diskUtilization}%, stopping`,
);
}
return { return {
stop: true, stop: true,
used: usedPercentage, used: usedPercentage,
@ -246,9 +249,11 @@ export async function checkDiskUtilization(
); );
if (projectedUsedPercentage >= params.diskUtilization) { if (projectedUsedPercentage >= params.diskUtilization) {
logger.info( if (doLog) {
`Disk utilization projected to reach threshold ${projectedUsedPercentage}% > ${params.diskUtilization}%, stopping`, logger.info(
); `Disk utilization projected to reach threshold ${projectedUsedPercentage}% > ${params.diskUtilization}%, stopping`,
);
}
return { return {
stop: true, stop: true,
used: usedPercentage, used: usedPercentage,

2
tests/adblockrules.test.js
View file

@ -15,7 +15,7 @@ function runCrawl(name, config, commandExtra = "") {
{ input: configYaml, stdin: "inherit", encoding: "utf8" }, { input: configYaml, stdin: "inherit", encoding: "utf8" },
); );
console.log(proc); //console.log(proc);
} catch (error) { } catch (error) {
console.log(error); console.log(error);
} }

2
tests/blockrules.test.js
View file

@ -15,7 +15,7 @@ function runCrawl(name, config, commandExtra = "") {
{ input: configYaml, stdin: "inherit", encoding: "utf8" }, { input: configYaml, stdin: "inherit", encoding: "utf8" },
); );
console.log(proc); //console.log(proc);
} catch (error) { } catch (error) {
console.log(error); console.log(error);
} }

2
tests/config_stdin.test.js
View file

@ -12,7 +12,7 @@ test("pass config file via stdin", async () => {
{ input: configYaml, stdin: "inherit", encoding: "utf8" }, { input: configYaml, stdin: "inherit", encoding: "utf8" },
); );
console.log(proc); //console.log(proc);
} catch (error) { } catch (error) {
console.log(error); console.log(error);
} }

2
tests/rollover-writer.test.js
View file

@ -11,8 +11,6 @@ test("set rollover to 500K and ensure individual WARCs rollover, including scree
let main = 0; let main = 0;
let screenshots = 0; let screenshots = 0;
console.log(warcLists);
for (const name of warcLists) { for (const name of warcLists) {
if (name.startsWith("rec-")) { if (name.startsWith("rec-")) {
main++; main++;

2
tests/storage.test.js
View file

@ -33,6 +33,7 @@ grpcfuse 1000000 285000 715000 28% /crawls`;
params, params,
5000 * 1024, 5000 * 1024,
mockDfOutput, mockDfOutput,
false
); );
expect(returnValue).toEqual({ expect(returnValue).toEqual({
stop: false, stop: false,
@ -60,6 +61,7 @@ grpcfuse 100000 85000 15000 85% /crawls`;
params, params,
3000 * 1024, 3000 * 1024,
mockDfOutput, mockDfOutput,
false
); );
expect(returnValue).toEqual({ expect(returnValue).toEqual({
stop: true, stop: true,

2
tests/warcinfo.test.js
View file

@ -13,7 +13,7 @@ test("run crawl", async() => {
{ input: configYaml, stdin: "inherit", encoding: "utf8" }, { input: configYaml, stdin: "inherit", encoding: "utf8" },
); );
console.log(proc); //console.log(proc);
success = true; success = true;
} catch (error) { } catch (error) {
console.log(error); console.log(error);