Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-12-08 06:09:48 +00:00
Code Issues Projects Releases Packages Wiki Activity

update to latest warcio (2.4.7) to fix issus when returning payload only size

Browse source
This commit is contained in:
Ilya Kreymer 2025-09-18 02:04:28 -07:00
parent db9e78e823
commit 2f81798f09
3 changed files with 16 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -28,7 +28,7 @@ RUN mkdir -p /tmp/ads && cd /tmp/ads && \
cat ad-hosts.txt | grep '^0.0.0.0 '| awk '{ print $2; }' | grep -v '0.0.0.0' | jq --raw-input --slurp 'split("\n")' > /app/ad-hosts.json && \ cat ad-hosts.txt | grep '^0.0.0.0 '| awk '{ print $2; }' | grep -v '0.0.0.0' | jq --raw-input --slurp 'split("\n")' > /app/ad-hosts.json && \
rm /tmp/ads/ad-hosts.txt rm /tmp/ads/ad-hosts.txt
RUN yarn install --network-timeout 1000000 RUN yarn install --network-timeout 1000000 --network-concurrency 1
ADD tsconfig.json /app/ ADD tsconfig.json /app/
ADD src /app/src ADD src /app/src

4
src/util/recorder.ts
View file

@ -1524,7 +1524,7 @@ export class Recorder extends EventEmitter {
const { logDetails } = this; const { logDetails } = this;
try { try {
reqresp.readSize = await serializer.digestRecord({ reqresp.readSize = await serializer.digestRecord({
includeHeadersSize: false, returnPayloadOnlySize: true,
}); });
// set truncated field and recompute header buff // set truncated field and recompute header buff
if (reqresp.truncated) { if (reqresp.truncated) {
@ -1667,7 +1667,7 @@ export class Recorder extends EventEmitter {
} }
} else { } else {
reqresp.readSize = await serializer.digestRecord({ reqresp.readSize = await serializer.digestRecord({
includeHeadersSize: false, returnPayloadOnlySize: true,
}); });
} }

13
yarn.lock
View file

@ -5576,6 +5576,19 @@ warcio@^2.4.7:
uuid-random "^1.3.2" uuid-random "^1.3.2"
yargs "^17.7.2" yargs "^17.7.2"
"warcio@git+https://github.com/webrecorder/warcio.js#fix-digestrecord-no-headers":
version "2.4.7"
resolved "git+https://github.com/webrecorder/warcio.js#0e95227bb6af3b089dc397827ac2db175a5b3b9e"
dependencies:
"@types/pako" "^1.0.7"
"@types/stream-buffers" "^3.0.7"
base32-encode "^2.0.0"
hash-wasm "^4.9.0"
pako "^1.0.11"
tempy "^3.1.0"
uuid-random "^1.3.2"
yargs "^17.7.2"
web-encoding@^1.1.5: web-encoding@^1.1.5:
version "1.1.5" version "1.1.5"
resolved "https://registry.yarnpkg.com/web-encoding/-/web-encoding-1.1.5.tgz#fc810cf7667364a6335c939913f5051d3e0c4864" resolved "https://registry.yarnpkg.com/web-encoding/-/web-encoding-1.1.5.tgz#fc810cf7667364a6335c939913f5051d3e0c4864"