Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

remove crc32 computation, fixes #653 (#657)

Browse source 
Removes crc32 computation, which was incorrect, and no longer needed
This commit is contained in:
Ilya Kreymer 2024-07-29 16:19:44 -07:00 • committed by GitHub
parent 717dd138ec
commit 539730d54e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 4 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -20,7 +20,6 @@
"@types/sax": "^1.2.7", "@types/sax": "^1.2.7",
"@webrecorder/wabac": "^2.19.4", "@webrecorder/wabac": "^2.19.4",
"browsertrix-behaviors": "^0.6.2", "browsertrix-behaviors": "^0.6.2",
"crc": "^4.3.2",
"fetch-socks": "^1.3.0", "fetch-socks": "^1.3.0",
"get-folder-size": "^4.0.0", "get-folder-size": "^4.0.0",
"husky": "^8.0.3", "husky": "^8.0.3",

15
src/util/storage.ts
View file

@ -6,8 +6,6 @@ import util from "util";
import os from "os"; import os from "os";
import { createHash } from "crypto"; import { createHash } from "crypto";
import crc32 from "crc/crc32";
import * as Minio from "minio"; import * as Minio from "minio";
import { initRedis } from "./redis.js"; import { initRedis } from "./redis.js";
@ -98,13 +96,13 @@ export class S3StorageSync {
srcFilename, srcFilename,
); );
const { hash, crc32 } = await checksumFile("sha256", srcFilename); const hash = await checksumFile("sha256", srcFilename);
const path = targetFilename; const path = targetFilename;
const size = await getFileSize(srcFilename); const size = await getFileSize(srcFilename);
// for backwards compatibility, keep 'bytes' // for backwards compatibility, keep 'bytes'
return { path, size, hash, crc32, bytes: size }; return { path, size, hash, bytes: size };
} }
async downloadFile(srcFilename: string, destFilename: string) { async downloadFile(srcFilename: string, destFilename: string) {
@ -296,21 +294,16 @@ export function calculatePercentageUsed(used: number, total: number) {
return Math.round((used / total) * 100); return Math.round((used / total) * 100);
} }
function checksumFile( function checksumFile(hashName: string, path: string): Promise<string> {
hashName: string,
path: string,
): Promise<{ hash: string; crc32: number }> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const hash = createHash(hashName); const hash = createHash(hashName);
let crc: number = 0;
const stream = fs.createReadStream(path); const stream = fs.createReadStream(path);
stream.on("error", (err) => reject(err)); stream.on("error", (err) => reject(err));
stream.on("data", (chunk) => { stream.on("data", (chunk) => {
hash.update(chunk); hash.update(chunk);
crc = crc32(chunk, crc);
}); });
stream.on("end", () => resolve({ hash: hash.digest("hex"), crc32: crc })); stream.on("end", () => resolve(hash.digest("hex")));
}); });
} }