Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add Prettier to the repo, and format all the files! (#428)

Browse source 
This adds prettier to the repo, and sets up the pre-commit hook to
auto-format as well as lint.
Also updates ignores files to exclude crawls, test-crawls, scratch, dist as needed.
This commit is contained in:
Emma Segal-Grossman 2023-11-09 19:11:11 -05:00 • committed by GitHub
parent af1e0860e4
commit 2a49406df7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
70 changed files with 3192 additions and 2026 deletions
Download patch file Download diff file Expand all files Collapse all files

18
tests/file_stats.test.js
View file

@ -2,17 +2,18 @@ import child_process from "child_process";
import fs from "fs";
test("ensure that stats file is modified", async () => {
const child = child_process.exec("docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://webrecorder.net/ --generateWACZ --text --limit 3 --collection file-stats --statsFilename progress.json");
const child = child_process.exec(
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://webrecorder.net/ --generateWACZ --text --limit 3 --collection file-stats --statsFilename progress.json",
);
// detect crawler exit
let crawler_exited = false;
child.on("exit", function() {
child.on("exit", function () {
crawler_exited = true;
});
// helper function to sleep
const sleep = ms => new Promise(res => setTimeout(res, ms));
const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
// wait for stats file creation up to 30 secs (to not wait indefinitely)
let counter = 0;
@ -23,7 +24,9 @@ test("ensure that stats file is modified", async () => {
}
// get initial modification time
const initial_mtime = fs.fstatSync(fs.openSync("test-crawls/progress.json", "r")).mtime;
const initial_mtime = fs.fstatSync(
fs.openSync("test-crawls/progress.json", "r"),
).mtime;
// wait for crawler exit
while (!crawler_exited) {
@ -31,12 +34,13 @@ test("ensure that stats file is modified", async () => {
}
// get final modification time
const final_mtime = fs.fstatSync(fs.openSync("test-crawls/progress.json", "r")).mtime;
const final_mtime = fs.fstatSync(
fs.openSync("test-crawls/progress.json", "r"),
).mtime;
// compare initial and final modification time
const diff = Math.abs(final_mtime - initial_mtime);
expect(diff > 0).toBe(true);
});
test("check that stats file format is correct", () => {