Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

dep: update to wabac.js 2.20 (#704)

Browse source 
Update imports for new TS-based wabac.js
This commit is contained in:
Ilya Kreymer 2024-10-16 21:02:04 -07:00 • committed by GitHub
parent a45b85dd74
commit 0d39ea3590
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 26 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

4
package.json
View file

@ -17,7 +17,7 @@
}, },
"dependencies": { "dependencies": {
"@novnc/novnc": "^1.4.0", "@novnc/novnc": "^1.4.0",
"@webrecorder/wabac": "^2.19.8", "@webrecorder/wabac": "^2.20.0-beta.4",
"browsertrix-behaviors": "^0.6.4", "browsertrix-behaviors": "^0.6.4",
"client-zip": "^2.4.5", "client-zip": "^2.4.5",
"fetch-socks": "^1.3.0", "fetch-socks": "^1.3.0",
@ -58,7 +58,7 @@
"jest": "^29.7.0", "jest": "^29.7.0",
"md5": "^2.3.0", "md5": "^2.3.0",
"prettier": "3.0.3", "prettier": "3.0.3",
"typescript": "^5.2.2" "typescript": "^5.5.4"
}, },
"jest": { "jest": {
"transform": {}, "transform": {},

9
src/replaycrawler.ts
View file

@ -10,10 +10,7 @@ import { PageInfoRecord, PageInfoValue, Recorder } from "./util/recorder.js";
import fsp from "fs/promises"; import fsp from "fs/promises";
import path from "path"; import path from "path";
// @ts-expect-error wabac.js import { ZipRangeReader, createLoader } from "@webrecorder/wabac";
import { ZipRangeReader } from "@webrecorder/wabac/src/wacz/ziprangereader.js";
// @ts-expect-error wabac.js
import { createLoader } from "@webrecorder/wabac/src/blockloaders.js";
import { AsyncIterReader } from "warcio"; import { AsyncIterReader } from "warcio";
import { parseArgs } from "./util/argParser.js"; import { parseArgs } from "./util/argParser.js";
@ -783,7 +780,7 @@ export class ReplayCrawler extends Crawler {
class WACZLoader { class WACZLoader {
url: string; url: string;
zipreader: ZipRangeReader; zipreader: ZipRangeReader | null;
constructor(url: string) { constructor(url: string) {
this.url = url; this.url = url;
@ -802,7 +799,7 @@ class WACZLoader {
} }
async loadFile(fileInZip: string) { async loadFile(fileInZip: string) {
const { reader } = await this.zipreader.loadFile(fileInZip); const { reader } = await this.zipreader!.loadFile(fileInZip);
if (!reader) { if (!reader) {
return null; return null;

8
src/util/recorder.ts
View file

@ -10,13 +10,7 @@ import {
import { fetch, getGlobalDispatcher, Response } from "undici"; import { fetch, getGlobalDispatcher, Response } from "undici";
// @ts-expect-error TODO fill in why error is expected import { getCustomRewriter, rewriteDASH, rewriteHLS } from "@webrecorder/wabac";
import { getCustomRewriter } from "@webrecorder/wabac/src/rewrite/index.js";
import {
rewriteDASH,
rewriteHLS,
// @ts-expect-error TODO fill in why error is expected
} from "@webrecorder/wabac/src/rewrite/rewriteVideo.js";
import { WARCRecord } from "warcio"; import { WARCRecord } from "warcio";
import { TempFileBuffer, WARCSerializer } from "warcio/node"; import { TempFileBuffer, WARCSerializer } from "warcio/node";

5
src/util/reqresp.ts
View file

@ -1,7 +1,4 @@
// @ts-expect-error TODO fill in why error is expected import { getCustomRewriter, getStatusText } from "@webrecorder/wabac";
import { getStatusText } from "@webrecorder/wabac/src/utils.js";
// @ts-expect-error TODO fill in why error is expected
import { getCustomRewriter } from "@webrecorder/wabac/src/rewrite/index.js";
import { Protocol } from "puppeteer-core"; import { Protocol } from "puppeteer-core";
import { postToGetUrl } from "warcio"; import { postToGetUrl } from "warcio";

4
tsconfig.json
View file

@ -29,9 +29,9 @@
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
/* Modules */ /* Modules */
"module": "NodeNext" /* Specify what module code is generated. */, "module": "esnext" /* Specify what module code is generated. */,
"rootDir": "./src" /* Specify the root folder within your source files. */, "rootDir": "./src" /* Specify the root folder within your source files. */,
"moduleResolution": "NodeNext" /* Specify how TypeScript looks up a file from a given module specifier. */, "moduleResolution": "bundler" /* Specify how TypeScript looks up a file from a given module specifier. */,
//"baseUrl": "./src", /* Specify the base directory to resolve non-relative module names. */ //"baseUrl": "./src", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */

33
yarn.lock
View file

@ -1312,15 +1312,16 @@
resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406"
integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==
"@webrecorder/wabac@^2.19.8": "@webrecorder/wabac@^2.20.0-beta.4":
version "2.19.8" version "2.20.0-beta.4"
resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.19.8.tgz#302ade200953a1c76f0b355983ae4081428fb933" resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.20.0-beta.4.tgz#c60fcd00f449cca52ce1a0bef305a06922c9e3e8"
integrity sha512-WjyfsGK8JWKeeDsGrOIT8ZLjMcOOAN93OMnRLO214jSV18SHEOY4JRvXzFOLF+OWYC5kJIMjl05gurTLq18jOA== integrity sha512-enHYcZoqs7cOu2tdTqVeB/zB27uL4wmCMzvF55bJqdB8d5zgPpY+/fpRA3eLxGrPc0nFYAjsI/aNaa62FH7WKQ==
dependencies: dependencies:
"@peculiar/asn1-ecc" "^2.3.4" "@peculiar/asn1-ecc" "^2.3.4"
"@peculiar/asn1-schema" "^2.3.3" "@peculiar/asn1-schema" "^2.3.3"
"@peculiar/x509" "^1.9.2" "@peculiar/x509" "^1.9.2"
"@webrecorder/wombat" "^3.8.0" "@types/js-levenshtein" "^1.1.3"
"@webrecorder/wombat" "^3.8.2"
acorn "^8.10.0" acorn "^8.10.0"
auto-js-ipfs "^2.1.1" auto-js-ipfs "^2.1.1"
base64-js "^1.5.1" base64-js "^1.5.1"
@ -1339,14 +1340,14 @@
path-parser "^6.1.0" path-parser "^6.1.0"
process "^0.11.10" process "^0.11.10"
stream-browserify "^3.0.0" stream-browserify "^3.0.0"
warcio "^2.3.0" warcio "^2.3.1"
"@webrecorder/wombat@^3.8.0": "@webrecorder/wombat@^3.8.2":
version "3.8.0" version "3.8.2"
resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.0.tgz#63ed3df199f11223b23c9ce66202590b8511ae2a" resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.2.tgz#e46e18719834d633175eec52ce753a4dc4e48e27"
integrity sha512-MpzNu9+ClCHjOER9XCrsEIsJk15L6qGO+PxeBPiOtaFJmNUiz0auMT5AQwiPqJgKEAniZTlPx1O4kNCVJu9f2Q== integrity sha512-uUZr9V4UYpVOpM64Tm27ND/hMjDbT37+/qyNaNV6loqDuVzBVQh5w7SfTEy0Bbjj1MYyNZP244mOtWtotTpUEA==
dependencies: dependencies:
warcio "^2.3.0" warcio "^2.3.1"
"@zxing/text-encoding@0.9.0": "@zxing/text-encoding@0.9.0":
version "0.9.0" version "0.9.0"
@ -5154,10 +5155,10 @@ typed-query-selector@^2.12.0:
resolved "https://registry.yarnpkg.com/typed-query-selector/-/typed-query-selector-2.12.0.tgz#92b65dbc0a42655fccf4aeb1a08b1dddce8af5f2" resolved "https://registry.yarnpkg.com/typed-query-selector/-/typed-query-selector-2.12.0.tgz#92b65dbc0a42655fccf4aeb1a08b1dddce8af5f2"
integrity sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg== integrity sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==
typescript@^5.2.2: typescript@^5.5.4:
version "5.2.2" version "5.5.4"
resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.4.tgz#d9852d6c82bad2d2eda4fd74a5762a8f5909e9ba"
integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== integrity sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==
unbox-primitive@^1.0.0, unbox-primitive@^1.0.1: unbox-primitive@^1.0.0, unbox-primitive@^1.0.1:
version "1.0.1" version "1.0.1"
@ -5282,7 +5283,7 @@ walker@^1.0.8:
dependencies: dependencies:
makeerror "1.0.12" makeerror "1.0.12"
warcio@^2.3.0, warcio@^2.3.1: warcio@^2.3.1:
version "2.3.1" version "2.3.1"
resolved "https://registry.yarnpkg.com/warcio/-/warcio-2.3.1.tgz#8ac9de897de1a556161168f2a3938b60929908ca" resolved "https://registry.yarnpkg.com/warcio/-/warcio-2.3.1.tgz#8ac9de897de1a556161168f2a3938b60929908ca"
integrity sha512-PjcWqzXfs6HdWfHi1V/i8MoMmV5M0Csg3rOa2mqCJ1dmCJXswVfQ0VXbEVumwavNIW2oFFj6LJoCHHeL4Ls/zw== integrity sha512-PjcWqzXfs6HdWfHi1V/i8MoMmV5M0Csg3rOa2mqCJ1dmCJXswVfQ0VXbEVumwavNIW2oFFj6LJoCHHeL4Ls/zw==