diff --git a/.eslintrc.cjs b/.eslintrc.cjs index afdf7f28..43cf3170 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -1,39 +1,32 @@ module.exports = { - "env": { - "browser": true, - "es2021": true, - "node": true, - "jest": true - }, - "extends": "eslint:recommended", - "parserOptions": { - "ecmaVersion": 12, - "sourceType": "module" - }, - "rules": { - "indent": [ - "error", - 2 - ], - "linebreak-style": [ - "error", - "unix" - ], - "quotes": [ - "error", - "double" - ], - "semi": [ - "error", - "always" - ], - "no-constant-condition": [ - "error", - {"checkLoops": false } - ], - "no-use-before-define": [ - "error", - {"variables": true, "functions": false, "classes": false, "allowNamedExports": true} - ] - } + env: { + browser: true, + es2021: true, + node: true, + jest: true, + }, + extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"], + parser: "@typescript-eslint/parser", + plugins: ["@typescript-eslint"], + parserOptions: { + ecmaVersion: 12, + sourceType: "module", + }, + rules: { + indent: ["error", 2], + "linebreak-style": ["error", "unix"], + quotes: ["error", "double"], + semi: ["error", "always"], + "no-constant-condition": ["error", { checkLoops: false }], + "no-use-before-define": [ + "error", + { + variables: true, + functions: false, + classes: false, + allowNamedExports: true, + }, + ], + }, + reportUnusedDisableDirectives: true, }; diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 10d55f45..51614467 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,6 +40,8 @@ jobs: node-version: ${{ matrix.node-version }} - name: install requirements run: yarn install + - name: build js + run: yarn run tsc - name: build docker run: docker-compose build - name: run jest diff --git a/Dockerfile b/Dockerfile index c1ff54d4..2f803fcd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,14 +38,18 @@ RUN mkdir -p /tmp/ads && cd /tmp/ads && \ RUN yarn install --network-timeout 1000000 -ADD *.js /app/ -ADD util/*.js /app/util/ +ADD tsconfig.json /app/ +ADD src /app/src + +RUN yarn run tsc ADD config/ /app/ ADD html/ /app/html/ -RUN ln -s /app/main.js /usr/bin/crawl; ln -s /app/create-login-profile.js /usr/bin/create-login-profile +RUN chmod u+x /app/dist/main.js /app/dist/create-login-profile.js + +RUN ln -s /app/dist/main.js /usr/bin/crawl; ln -s /app/dist/create-login-profile.js /usr/bin/create-login-profile WORKDIR /crawls diff --git a/defaultDriver.js b/defaultDriver.js deleted file mode 100644 index fb8376c3..00000000 --- a/defaultDriver.js +++ /dev/null @@ -1,4 +0,0 @@ - -export default async ({data, page, crawler}) => { - await crawler.loadPage(page, data); -}; diff --git a/package.json b/package.json index ca073e91..39663cc7 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "author": "Ilya Kreymer , Webrecorder Software", "license": "AGPL-3.0-or-later", "scripts": { - "lint": "eslint *.js util/*.js tests/*.test.js", + "tsc": "tsc", + "lint": "eslint *.js tests/*.test.js", "test": "yarn node --experimental-vm-modules $(yarn bin jest --bail 1)", "prepare": "husky install" }, @@ -18,23 +19,31 @@ "crc": "^4.3.2", "get-folder-size": "^4.0.0", "husky": "^8.0.3", - "ioredis": "^4.27.1", + "ioredis": "^5.3.2", "js-yaml": "^4.1.0", - "minio": "7.0.26", + "minio": "^7.1.3", "p-queue": "^7.3.4", "puppeteer-core": "^20.7.4", "sharp": "^0.32.1", - "sitemapper": "^3.2.5", + "sitemapper": "^3.2.6", + "tsc": "^2.0.4", "uuid": "8.3.2", - "warcio": "^2.2.0", + "warcio": "^2.2.1", "ws": "^7.4.4", "yargs": "^17.7.2" }, "devDependencies": { - "eslint": "^8.37.0", + "@types/js-yaml": "^4.0.8", + "@types/node": "^20.8.7", + "@types/uuid": "^9.0.6", + "@types/ws": "^8.5.8", + "@typescript-eslint/eslint-plugin": "^6.10.0", + "@typescript-eslint/parser": "^6.10.0", + "eslint": "^8.53.0", "eslint-plugin-react": "^7.22.0", "jest": "^29.2.1", - "md5": "^2.3.0" + "md5": "^2.3.0", + "typescript": "^5.2.2" }, "jest": { "transform": {}, diff --git a/crawler.js b/src/crawler.ts similarity index 61% rename from crawler.js rename to src/crawler.ts index c98d9275..775bd382 100644 --- a/crawler.js +++ b/src/crawler.ts @@ -1,10 +1,11 @@ -import child_process from "child_process"; +import child_process, { ChildProcess, StdioOptions } from "child_process"; import path from "path"; -import fs from "fs"; +import fs, { WriteStream } from "fs"; import os from "os"; -import fsp from "fs/promises"; +import fsp, { FileHandle } from "fs/promises"; + +import { RedisCrawlState, LoadState, QueueState, PageState, WorkerId } from "./util/state.js"; -import { RedisCrawlState, LoadState, QueueState } from "./util/state.js"; import Sitemapper from "sitemapper"; import yaml from "js-yaml"; @@ -12,13 +13,13 @@ import * as warcio from "warcio"; import { HealthChecker } from "./util/healthcheck.js"; import { TextExtractViaSnapshot } from "./util/textextract.js"; -import { initStorage, getFileSize, getDirSize, interpolateFilename, checkDiskUtilization } from "./util/storage.js"; -import { ScreenCaster, WSTransport, RedisPubSubTransport } from "./util/screencaster.js"; +import { initStorage, getFileSize, getDirSize, interpolateFilename, checkDiskUtilization, S3StorageSync } from "./util/storage.js"; +import { ScreenCaster, WSTransport } from "./util/screencaster.js"; import { Screenshots } from "./util/screenshots.js"; import { parseArgs } from "./util/argParser.js"; import { initRedis } from "./util/redis.js"; import { logger, errJSON } from "./util/logger.js"; -import { runWorkers } from "./util/worker.js"; +import { WorkerOpts, WorkerState, runWorkers } from "./util/worker.js"; import { sleep, timedRun, secondsElapsed } from "./util/timing.js"; import { collectAllFileSources } from "./util/file_reader.js"; @@ -32,32 +33,120 @@ import { OriginOverride } from "./util/originoverride.js"; // to ignore HTTPS error for HEAD check import { Agent as HTTPAgent } from "http"; import { Agent as HTTPSAgent } from "https"; +import { CDPSession, Frame, HTTPRequest, Page } from "puppeteer-core"; -const HTTPS_AGENT = HTTPSAgent({ +const HTTPS_AGENT = new HTTPSAgent({ rejectUnauthorized: false, }); -const HTTP_AGENT = HTTPAgent(); +const HTTP_AGENT = new HTTPAgent(); -const behaviors = fs.readFileSync(new URL("./node_modules/browsertrix-behaviors/dist/behaviors.js", import.meta.url), {encoding: "utf8"}); +const behaviors = fs.readFileSync(new URL("../node_modules/browsertrix-behaviors/dist/behaviors.js", import.meta.url), {encoding: "utf8"}); const FETCH_TIMEOUT_SECS = 30; const PAGE_OP_TIMEOUT_SECS = 5; const POST_CRAWL_STATES = ["generate-wacz", "uploading-wacz", "generate-cdx", "generate-warc"]; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type LogDetails = Record; + +type PageEntry = { + id: string; + url: string; + title?: string; + loadState?: number; + mime?: string; + seed?: boolean; + text?: string; + favIconUrl?: string; +}; + // ============================================================================ export class Crawler { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + params: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + origConfig: any; + + collDir: string; + logDir: string; + logFilename: string; + + headers: Record = {}; + + crawlState!: RedisCrawlState; + + pagesFH?: FileHandle | null = null; + logFH!: WriteStream; + + crawlId: string; + + startTime: number; + + limitHit = false; + pageLimit: number; + + saveStateFiles: string[] = []; + lastSaveTime: number; + + maxPageTime: number; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + emulateDevice: any = {}; + + captureBasePrefix = ""; + + infoString!: string; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + gotoOpts: Record; + + pagesDir: string; + pagesFile: string; + + blockRules: BlockRules | null; + adBlockRules: AdBlockRules | null; + + healthChecker: HealthChecker | null = null; + originOverride: OriginOverride | null = null; + + screencaster: ScreenCaster | null = null; + + interrupted = false; + finalExit = false; + uploadAndDeleteLocal = false; + done = false; + + customBehaviors = ""; + behaviorLastLine?: string; + + browser: Browser; + storage: S3StorageSync | null = null; + + maxHeapUsed = 0; + maxHeapTotal = 0; + + // eslint-disable-next-line no-use-before-define + driver!: (opts: { page: Page; data: PageState; crawler: Crawler }) => NonNullable; + constructor() { const res = parseArgs(); this.params = res.parsed; this.origConfig = res.origConfig; // root collections dir - this.collDir = path.join(this.params.cwd, "collections", this.params.collection); + this.collDir = path.join( + this.params.cwd, + "collections", + this.params.collection + ); this.logDir = path.join(this.collDir, "logs"); - this.logFilename = path.join(this.logDir, `crawl-${new Date().toISOString().replace(/[^\d]/g, "")}.log`); + this.logFilename = path.join( + this.logDir, + `crawl-${new Date().toISOString().replace(/[^\d]/g, "")}.log` + ); const debugLogging = this.params.logging.includes("debug"); logger.setDebugLogging(debugLogging); @@ -74,7 +163,6 @@ export class Crawler { logger.debug("Writing log to: " + this.logFilename, {}, "init"); this.headers = {}; - this.crawlState = null; // pages file this.pagesFH = null; @@ -89,7 +177,9 @@ export class Crawler { // resolve maxPageLimit and ensure pageLimit is no greater than maxPageLimit if (this.params.maxPageLimit) { - this.pageLimit = this.pageLimit ? Math.min(this.pageLimit, this.params.maxPageLimit) : this.params.maxPageLimit; + this.pageLimit = this.pageLimit + ? Math.min(this.pageLimit, this.params.maxPageLimit) + : this.params.maxPageLimit; } this.saveStateFiles = []; @@ -97,18 +187,22 @@ export class Crawler { // sum of page load + behavior timeouts + 2 x fetch + cloudflare + link extraction timeouts + extra page delay // if exceeded, will interrupt and move on to next page (likely behaviors or some other operation is stuck) - this.maxPageTime = this.params.pageLoadTimeout + this.params.behaviorTimeout + - FETCH_TIMEOUT_SECS*2 + PAGE_OP_TIMEOUT_SECS*2 + this.params.pageExtraDelay; + this.maxPageTime = + this.params.pageLoadTimeout + + this.params.behaviorTimeout + + FETCH_TIMEOUT_SECS * 2 + + PAGE_OP_TIMEOUT_SECS * 2 + + this.params.pageExtraDelay; this.emulateDevice = this.params.emulateDevice || {}; //this.captureBasePrefix = `http://${process.env.PROXY_HOST}:${process.env.PROXY_PORT}/${this.params.collection}/record`; //this.capturePrefix = "";//process.env.NO_PROXY ? "" : this.captureBasePrefix + "/id_/"; - this.captureBasePrefix = null; + //this.captureBasePrefix = ""; this.gotoOpts = { waitUntil: this.params.waitUntil, - timeout: this.params.pageLoadTimeout * 1000 + timeout: this.params.pageLoadTimeout * 1000, }; // pages directory @@ -129,7 +223,6 @@ export class Crawler { this.done = false; this.customBehaviors = ""; - this.behaviorLastLine = null; this.browser = new Browser(); } @@ -158,7 +251,9 @@ export class Crawler { const redisUrl = this.params.redisStoreUrl || "redis://localhost:6379/0"; if (!redisUrl.startsWith("redis://")) { - logger.fatal("stateStoreUrl must start with redis:// -- Only redis-based store currently supported"); + logger.fatal( + "stateStoreUrl must start with redis:// -- Only redis-based store currently supported" + ); } let redis; @@ -174,17 +269,30 @@ export class Crawler { } } - logger.debug(`Storing state via Redis ${redisUrl} @ key prefix "${this.crawlId}"`, {}, "state"); + logger.debug( + `Storing state via Redis ${redisUrl} @ key prefix "${this.crawlId}"`, + {}, + "state" + ); logger.debug(`Max Page Time: ${this.maxPageTime} seconds`, {}, "state"); - this.crawlState = new RedisCrawlState(redis, this.params.crawlId, this.maxPageTime, os.hostname()); + this.crawlState = new RedisCrawlState( + redis, + this.params.crawlId, + this.maxPageTime, + os.hostname() + ); // clear any pending URLs from this instance await this.crawlState.clearOwnPendingLocks(); if (this.params.saveState === "always" && this.params.saveStateInterval) { - logger.debug(`Saving crawl state every ${this.params.saveStateInterval} seconds, keeping last ${this.params.saveStateHistory} states`, {}, "state"); + logger.debug( + `Saving crawl state every ${this.params.saveStateInterval} seconds, keeping last ${this.params.saveStateHistory} states`, + {}, + "state" + ); } if (this.params.logErrorsToRedis) { @@ -200,11 +308,16 @@ export class Crawler { if (this.params.screencastPort) { transport = new WSTransport(this.params.screencastPort); - logger.debug(`Screencast server started on: ${this.params.screencastPort}`, {}, "screencast"); - } else if (this.params.redisStoreUrl && this.params.screencastRedis) { - transport = new RedisPubSubTransport(this.params.redisStoreUrl, this.crawlId); - logger.debug("Screencast enabled via redis pubsub", {}, "screencast"); + logger.debug( + `Screencast server started on: ${this.params.screencastPort}`, + {}, + "screencast" + ); } + // } else if (this.params.redisStoreUrl && this.params.screencastRedis) { + // transport = new RedisPubSubTransport(this.params.redisStoreUrl, this.crawlId); + // logger.debug("Screencast enabled via redis pubsub", {}, "screencast"); + // } if (!transport) { return null; @@ -214,26 +327,28 @@ export class Crawler { } launchRedis() { - let redisStdio; + let redisStdio: StdioOptions; if (this.params.logging.includes("redis")) { const redisStderr = fs.openSync(path.join(this.logDir, "redis.log"), "a"); redisStdio = [process.stdin, redisStderr, redisStderr]; - } else { redisStdio = "ignore"; } - let redisArgs = []; + let redisArgs: string[] = []; if (this.params.debugAccessRedis) { redisArgs = ["--protected-mode", "no"]; } - return child_process.spawn("redis-server", redisArgs,{cwd: "/tmp/", stdio: redisStdio}); + return child_process.spawn("redis-server", redisArgs, { + cwd: "/tmp/", + stdio: redisStdio, + }); } async bootstrap() { - const subprocesses = []; + const subprocesses: ChildProcess[] = []; subprocesses.push(this.launchRedis()); @@ -243,7 +358,7 @@ export class Crawler { // logger.info("wb-manager init failed, collection likely already exists"); //} - fs.mkdirSync(this.logDir, {recursive: true}); + fs.mkdirSync(this.logDir, { recursive: true }); this.logFH = fs.createWriteStream(this.logFilename); logger.setExternalLogStream(this.logFH); @@ -253,23 +368,26 @@ export class Crawler { logger.info("Seeds", this.params.scopedSeeds); if (this.params.profile) { - logger.info("With Browser Profile", {url: this.params.profile}); + logger.info("With Browser Profile", { url: this.params.profile }); } if (this.params.overwrite) { logger.debug(`Clearing ${this.collDir} before starting`); try { fs.rmSync(this.collDir, { recursive: true, force: true }); - } catch(e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.error(`Unable to clear ${this.collDir}`, e); } } if (this.params.customBehaviors) { - this.customBehaviors = this.loadCustomBehaviors(this.params.customBehaviors); + this.customBehaviors = this.loadCustomBehaviors( + this.params.customBehaviors + ); } - this.headers = {"User-Agent": this.configureUA()}; + this.headers = { "User-Agent": this.configureUA() }; process.on("exit", () => { for (const proc of subprocesses) { @@ -277,19 +395,22 @@ export class Crawler { } }); - child_process.spawn("socat", ["tcp-listen:9222,reuseaddr,fork", "tcp:localhost:9221"]); + child_process.spawn("socat", [ + "tcp-listen:9222,reuseaddr,fork", + "tcp:localhost:9221", + ]); if (!this.params.headless && !process.env.NO_XVFB) { child_process.spawn("Xvfb", [ - process.env.DISPLAY, + process.env.DISPLAY || "", "-listen", "tcp", "-screen", "0", - process.env.GEOMETRY, + process.env.GEOMETRY || "", "-ac", "+extension", - "RANDR" + "RANDR", ]); } } @@ -324,40 +445,47 @@ export class Crawler { exitCode = 11; } } - } catch(e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.error("Crawl failed", e); exitCode = 9; status = "failing"; if (await this.crawlState.incFailCount()) { status = "failed"; } - } finally { await this.setStatusAndExit(exitCode, status); } } - _behaviorLog({data, type}, pageUrl, workerid) { + _behaviorLog( + { data, type }: { data: string; type: string }, + pageUrl: string, + workerid: WorkerId + ) { let behaviorLine; let message; let details; - const logDetails = {page: pageUrl, workerid}; + const logDetails = { page: pageUrl, workerid }; - if (typeof(data) === "string") { + if (typeof data === "string") { message = data; details = logDetails; } else { message = type === "info" ? "Behavior log" : "Behavior debug"; - details = typeof(data) === "object" ? {...data, ...logDetails} : logDetails; + details = + typeof data === "object" + ? { ...(data as object), ...logDetails } + : logDetails; } switch (type) { case "info": behaviorLine = JSON.stringify(data); - if (behaviorLine != this._behaviorLastLine) { + if (behaviorLine !== this.behaviorLastLine) { logger.info(message, details, "behaviorScript"); - this._behaviorLastLine = behaviorLine; + this.behaviorLastLine = behaviorLine; } break; @@ -371,18 +499,39 @@ export class Crawler { } } - isInScope({seedId, url, depth, extraHops} = {}, logDetails = {}) { + isInScope( + { + seedId, + url, + depth, + extraHops, + }: { seedId: number; url: string; depth: number; extraHops: number }, + logDetails = {} + ) { const seed = this.params.scopedSeeds[seedId]; return seed.isIncluded(url, depth, extraHops, logDetails); } - async setupPage({page, cdp, workerid, callbacks}) { - await this.browser.setupPage({page, cdp}); - - if ((this.adBlockRules && this.params.blockAds) || - this.blockRules || this.originOverride) { + async setupPage({ + page, + cdp, + workerid, + callbacks, + }: { + page: Page; + cdp: CDPSession; + workerid: WorkerId; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + callbacks: any; + }) { + await this.browser.setupPage({ page, cdp }); + if ( + (this.adBlockRules && this.params.blockAds) || + this.blockRules || + this.originOverride + ) { await page.setRequestInterception(true); if (this.adBlockRules && this.params.blockAds) { @@ -401,24 +550,39 @@ export class Crawler { if (this.params.logging.includes("jserrors")) { page.on("console", (msg) => { if (msg.type() === "error") { - logger.warn(msg.text(), {"location": msg.location(), "page": page.url(), workerid}, "jsError"); + logger.warn( + msg.text(), + { location: msg.location(), page: page.url(), workerid }, + "jsError" + ); } }); page.on("pageerror", (e) => { - logger.warn("Page Error", {...errJSON(e), "page": page.url(), workerid}, "jsError"); + logger.warn( + "Page Error", + { ...errJSON(e), page: page.url(), workerid }, + "jsError" + ); }); } if (this.screencaster) { - logger.debug("Start Screencast", {workerid}, "screencast"); + logger.debug("Start Screencast", { workerid }, "screencast"); await this.screencaster.screencastPage(page, cdp, workerid); } - await page.exposeFunction(ADD_LINK_FUNC, (url) => callbacks.addLink && callbacks.addLink(url)); + await page.exposeFunction( + ADD_LINK_FUNC, + (url: string) => callbacks.addLink && callbacks.addLink(url) + ); if (this.params.behaviorOpts) { - await page.exposeFunction(BEHAVIOR_LOG_FUNC, (logdata) => this._behaviorLog(logdata, page.url(), workerid)); + await page.exposeFunction( + BEHAVIOR_LOG_FUNC, + (logdata: { data: string; type: string }) => + this._behaviorLog(logdata, page.url(), workerid) + ); await this.browser.addInitScript(page, behaviors); const initScript = ` @@ -431,7 +595,7 @@ self.__bx_behaviors.selectMainBehavior(); } } - loadCustomBehaviors(filename) { + loadCustomBehaviors(filename: string) { let str = ""; for (const source of collectAllFileSources(filename, ".js")) { @@ -441,13 +605,14 @@ self.__bx_behaviors.selectMainBehavior(); return str; } - async getFavicon(page, logDetails) { + async getFavicon(page: Page, logDetails: LogDetails): Promise { try { const resp = await fetch("http://127.0.0.1:9221/json"); if (resp.status === 200) { const browserJson = await resp.json(); for (const jsons of browserJson) { - if (jsons.id === page.target()._targetId) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if (jsons.id === (page.target() as any)._targetId) { return jsons.faviconUrl; } } @@ -455,18 +620,22 @@ self.__bx_behaviors.selectMainBehavior(); } catch (e) { // ignore } - logger.warn("Failed to fetch favicon from browser /json endpoint", logDetails); + logger.warn( + "Failed to fetch favicon from browser /json endpoint", + logDetails + ); + return ""; } - async crawlPage(opts) { + async crawlPage(opts: WorkerState) { await this.writeStats(); - const {page, cdp, data, workerid, callbacks, directFetchCapture} = opts; + const { page, cdp, data, workerid, callbacks, directFetchCapture } = opts; data.callbacks = callbacks; - const {url} = data; + const { url } = data; - const logDetails = {page: url, workerid}; + const logDetails = { page: url, workerid }; data.logDetails = logDetails; data.workerid = workerid; @@ -481,7 +650,7 @@ self.__bx_behaviors.selectMainBehavior(); if (!data.isHTMLPage && directFetchCapture) { try { - const {fetched, mime} = await timedRun( + const { fetched, mime } = await timedRun( directFetchCapture(url), FETCH_TIMEOUT_SECS, "Direct fetch capture attempt timed out", @@ -494,7 +663,11 @@ self.__bx_behaviors.selectMainBehavior(); if (mime) { data.mime = mime; } - logger.info("Direct fetch successful", {url, ...logDetails}, "fetch"); + logger.info( + "Direct fetch successful", + { url, ...logDetails }, + "fetch" + ); return true; } } catch (e) { @@ -503,7 +676,7 @@ self.__bx_behaviors.selectMainBehavior(); } // run custom driver here - await this.driver({page, data, crawler: this}); + await this.driver({ page, data, crawler: this }); data.title = await page.title(); data.favicon = await this.getFavicon(page, logDetails); @@ -514,7 +687,12 @@ self.__bx_behaviors.selectMainBehavior(); if (!data.isHTMLPage) { logger.debug("Skipping screenshots for non-HTML page", logDetails); } - const screenshots = new Screenshots({browser: this.browser, page, url, directory: archiveDir}); + const screenshots = new Screenshots({ + browser: this.browser, + page, + url, + directory: archiveDir, + }); if (this.params.screenshot.includes("view")) { await screenshots.take(); } @@ -529,8 +707,15 @@ self.__bx_behaviors.selectMainBehavior(); let textextract = null; if (data.isHTMLPage) { - textextract = new TextExtractViaSnapshot(cdp, {url, directory: archiveDir}); - const {changed, text} = await textextract.extractAndStoreText("text", false, this.params.text.includes("to-warc")); + textextract = new TextExtractViaSnapshot(cdp, { + url, + directory: archiveDir, + }); + const { changed, text } = await textextract.extractAndStoreText( + "text", + false, + this.params.text.includes("to-warc") + ); if (changed && text && this.params.text.includes("to-pages")) { data.text = text; @@ -541,7 +726,11 @@ self.__bx_behaviors.selectMainBehavior(); if (this.params.behaviorOpts) { if (!data.isHTMLPage) { - logger.debug("Skipping behaviors for non-HTML page", logDetails, "behavior"); + logger.debug( + "Skipping behaviors for non-HTML page", + logDetails, + "behavior" + ); } else if (data.skipBehaviors) { logger.info("Skipping behaviors for slow page", logDetails, "behavior"); } else { @@ -566,14 +755,17 @@ self.__bx_behaviors.selectMainBehavior(); } if (this.params.pageExtraDelay) { - logger.info(`Waiting ${this.params.pageExtraDelay} seconds before moving on to next page`, logDetails); + logger.info( + `Waiting ${this.params.pageExtraDelay} seconds before moving on to next page`, + logDetails + ); await sleep(this.params.pageExtraDelay); } return true; } - async pageFinished(data) { + async pageFinished(data: PageState) { await this.writePage(data); // if page loaded, considered page finished successfully @@ -581,7 +773,7 @@ self.__bx_behaviors.selectMainBehavior(); const { loadState, logDetails } = data; if (data.loadState >= LoadState.FULL_PAGE_LOADED) { - logger.info("Page Finished", {loadState, ...logDetails}, "pageStatus"); + logger.info("Page Finished", { loadState, ...logDetails }, "pageStatus"); await this.crawlState.markFinished(data.url); @@ -589,7 +781,11 @@ self.__bx_behaviors.selectMainBehavior(); this.healthChecker.resetErrors(); } } else { - logger.warn("Page Load Failed", {loadState, ...logDetails}, "pageStatus"); + logger.warn( + "Page Load Failed", + { loadState, ...logDetails }, + "pageStatus" + ); await this.crawlState.markFailed(data.url); @@ -603,51 +799,84 @@ self.__bx_behaviors.selectMainBehavior(); await this.checkLimits(); } - async teardownPage({workerid}) { + async teardownPage({ workerid }: WorkerOpts) { if (this.screencaster) { await this.screencaster.stopById(workerid); } } - async workerIdle(workerid) { + async workerIdle(workerid: WorkerId) { if (this.screencaster) { //logger.debug("End Screencast", {workerid}, "screencast"); await this.screencaster.stopById(workerid, true); } } - async runBehaviors(page, cdp, frames, logDetails) { + async runBehaviors( + page: Page, + cdp: CDPSession, + frames: Frame[], + logDetails: LogDetails + ) { try { frames = frames || page.frames(); - logger.info("Running behaviors", {frames: frames.length, frameUrls: frames.map(frame => frame.url()), ...logDetails}, "behavior"); + logger.info( + "Running behaviors", + { + frames: frames.length, + frameUrls: frames.map((frame) => frame.url()), + ...logDetails, + }, + "behavior" + ); const results = await Promise.allSettled( - frames.map(frame => this.browser.evaluateWithCLI(page, frame, cdp, ` + frames.map((frame) => + this.browser.evaluateWithCLI( + page, + frame, + cdp, + ` if (!self.__bx_behaviors) { console.error("__bx_behaviors missing, can't run behaviors"); } else { self.__bx_behaviors.run(); - }` - , logDetails, "behavior")) + }`, + logDetails, + "behavior" + ) + ) ); - for (const {status, reason} in results) { + for (const res of results) { + const { status, reason }: { status: string; reason?: string } = res; if (status === "rejected") { - logger.warn("Behavior run partially failed", {reason, ...logDetails}, "behavior"); + logger.warn( + "Behavior run partially failed", + { reason, ...logDetails }, + "behavior" + ); } } - logger.info("Behaviors finished", {finished: results.length, ...logDetails}, "behavior"); + logger.info( + "Behaviors finished", + { finished: results.length, ...logDetails }, + "behavior" + ); return true; - } catch (e) { - logger.warn("Behavior run failed", {...errJSON(e), ...logDetails}, "behavior"); + logger.warn( + "Behavior run failed", + { ...errJSON(e), ...logDetails }, + "behavior" + ); return false; } } - async shouldIncludeFrame(frame, logDetails) { + async shouldIncludeFrame(frame: Frame, logDetails: LogDetails) { if (!frame.parentFrame()) { return frame; } @@ -656,10 +885,16 @@ self.__bx_behaviors.selectMainBehavior(); // this is all designed to detect and skip PDFs, and other frames that are actually EMBEDs // if there's no tag or an iframe tag, then assume its a regular frame - const tagName = await frame.evaluate("self && self.frameElement && self.frameElement.tagName"); + const tagName = await frame.evaluate( + "self && self.frameElement && self.frameElement.tagName" + ); if (tagName && tagName !== "IFRAME" && tagName !== "FRAME") { - logger.debug("Skipping processing non-frame object", {tagName, frameUrl, ...logDetails}, "behavior"); + logger.debug( + "Skipping processing non-frame object", + { tagName, frameUrl, ...logDetails }, + "behavior" + ); return null; } @@ -668,35 +903,53 @@ self.__bx_behaviors.selectMainBehavior(); if (frameUrl === "about:blank") { res = false; } else { - res = !this.adBlockRules.isAdUrl(frameUrl); + res = this.adBlockRules && !this.adBlockRules.isAdUrl(frameUrl); } if (!res) { - logger.debug("Skipping processing frame", {frameUrl, ...logDetails}, "behavior"); + logger.debug( + "Skipping processing frame", + { frameUrl, ...logDetails }, + "behavior" + ); } return res ? frame : null; } async getInfoString() { - const packageFileJSON = JSON.parse(await fsp.readFile("../app/package.json")); - const warcioPackageJSON = JSON.parse(await fsp.readFile("/app/node_modules/warcio/package.json")); + const packageFileJSON = JSON.parse( + await fsp.readFile(new URL("../package.json", import.meta.url), { + encoding: "utf-8", + }) + ); + const warcioPackageJSON = JSON.parse( + await fsp.readFile( + new URL("../node_modules/warcio/package.json", import.meta.url), + { encoding: "utf-8" } + ) + ); return `Browsertrix-Crawler ${packageFileJSON.version} (with warcio.js ${warcioPackageJSON.version})`; } - async createWARCInfo(filename) { + async createWARCInfo(filename: string) { const warcVersion = "WARC/1.0"; const type = "warcinfo"; const info = { - "software": this.infoString, - "format": "WARC File Format 1.0" + software: this.infoString, + format: "WARC File Format 1.0", }; - const warcInfo = {...info, ...this.params.warcInfo, }; - const record = await warcio.WARCRecord.createWARCInfo({filename, type, warcVersion}, warcInfo); - const buffer = await warcio.WARCSerializer.serialize(record, {gzip: true}); + const warcInfo = { ...info, ...this.params.warcInfo }; + const record = await warcio.WARCRecord.createWARCInfo( + { filename, type, warcVersion }, + warcInfo + ); + const buffer = await warcio.WARCSerializer.serialize(record, { + gzip: true, + }); return buffer; } @@ -710,7 +963,9 @@ self.__bx_behaviors.selectMainBehavior(); if (this.params.sizeLimit) { if (size >= this.params.sizeLimit) { - logger.info(`Size threshold reached ${size} >= ${this.params.sizeLimit}, stopping`); + logger.info( + `Size threshold reached ${size} >= ${this.params.sizeLimit}, stopping` + ); interrupt = true; } } @@ -718,7 +973,9 @@ self.__bx_behaviors.selectMainBehavior(); if (this.params.timeLimit) { const elapsed = secondsElapsed(this.startTime); if (elapsed >= this.params.timeLimit) { - logger.info(`Time threshold reached ${elapsed} > ${this.params.timeLimit}, stopping`); + logger.info( + `Time threshold reached ${elapsed} > ${this.params.timeLimit}, stopping` + ); interrupt = true; } } @@ -734,7 +991,9 @@ self.__bx_behaviors.selectMainBehavior(); if (this.params.failOnFailedLimit) { const numFailed = this.crawlState.numFailed(); if (numFailed >= this.params.failOnFailedLimit) { - logger.fatal(`Failed threshold reached ${numFailed} >= ${this.params.failedLimit}, failing crawl`); + logger.fatal( + `Failed threshold reached ${numFailed} >= ${this.params.failedLimit}, failing crawl` + ); } } @@ -753,12 +1012,12 @@ self.__bx_behaviors.selectMainBehavior(); } async checkCanceled() { - if (this.crawlState && await this.crawlState.isCrawlCanceled()) { + if (this.crawlState && (await this.crawlState.isCrawlCanceled())) { await this.setStatusAndExit(0, "canceled"); } } - async setStatusAndExit(exitCode, status) { + async setStatusAndExit(exitCode: number, status: string) { logger.info(`Exiting, Crawl status: ${status}`); await this.closeLog(); @@ -799,13 +1058,17 @@ self.__bx_behaviors.selectMainBehavior(); async crawl() { if (this.params.healthCheckPort) { - this.healthChecker = new HealthChecker(this.params.healthCheckPort, this.params.workers); + this.healthChecker = new HealthChecker( + this.params.healthCheckPort, + this.params.workers + ); } try { const driverUrl = new URL(this.params.driver, import.meta.url); - this.driver = (await import(driverUrl)).default; - } catch(e) { + this.driver = (await import(driverUrl.href)).default; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.warn(`Error importing driver ${this.params.driver}`, e); return; } @@ -841,7 +1104,9 @@ self.__bx_behaviors.selectMainBehavior(); } if (POST_CRAWL_STATES.includes(initState)) { - logger.info("crawl already finished, running post-crawl tasks", {state: initState}); + logger.info("crawl already finished, running post-crawl tasks", { + state: initState, + }); await this.postCrawl(); return; } else if (await this.crawlState.isCrawlStopped()) { @@ -857,15 +1122,26 @@ self.__bx_behaviors.selectMainBehavior(); await this.crawlState.setStatus("running"); if (this.params.state) { - await this.crawlState.load(this.params.state, this.params.scopedSeeds, true); + await this.crawlState.load( + this.params.state, + this.params.scopedSeeds, + true + ); } await this.initPages(); - this.adBlockRules = new AdBlockRules(this.captureBasePrefix, this.params.adBlockMessage); + this.adBlockRules = new AdBlockRules( + this.captureBasePrefix, + this.params.adBlockMessage + ); if (this.params.blockRules && this.params.blockRules.length) { - this.blockRules = new BlockRules(this.params.blockRules, this.captureBasePrefix, this.params.blockMessage); + this.blockRules = new BlockRules( + this.params.blockRules, + this.captureBasePrefix, + this.params.blockMessage + ); } this.screencaster = this.initScreenCaster(); @@ -876,7 +1152,7 @@ self.__bx_behaviors.selectMainBehavior(); for (let i = 0; i < this.params.scopedSeeds.length; i++) { const seed = this.params.scopedSeeds[i]; - if (!await this.queueUrl(i, seed.url, 0, 0)) { + if (!(await this.queueUrl(i, seed.url, 0, 0))) { if (this.limitHit) { break; } @@ -894,14 +1170,19 @@ self.__bx_behaviors.selectMainBehavior(); chromeOptions: { proxy: false, userAgent: this.emulateDevice.userAgent, - extraArgs: this.extraChromeArgs() + extraArgs: this.extraChromeArgs(), }, - ondisconnect: (err) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ondisconnect: (err: any) => { this.interrupted = true; - logger.error("Browser disconnected (crashed?), interrupting crawl", err, "browser"); - } - }); - + logger.error( + "Browser disconnected (crashed?), interrupting crawl", + err, + "browser" + ); + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any); // -------------- // Run Crawl Here! @@ -919,7 +1200,6 @@ self.__bx_behaviors.selectMainBehavior(); await this.writeStats(); - // if crawl has been stopped, mark as final exit for post-crawl tasks if (await this.crawlState.isCrawlStopped()) { this.finalExit = true; @@ -935,36 +1215,48 @@ self.__bx_behaviors.selectMainBehavior(); if (this.params.generateCDX) { logger.info("Generating CDX"); - await fsp.mkdir(path.join(this.collDir, "indexes"), {recursive: true}); + await fsp.mkdir(path.join(this.collDir, "indexes"), { recursive: true }); await this.crawlState.setStatus("generate-cdx"); const warcList = await fsp.readdir(path.join(this.collDir, "archive")); - const warcListFull = warcList.map((filename) => path.join(this.collDir, "archive", filename)); + const warcListFull = warcList.map((filename) => + path.join(this.collDir, "archive", filename) + ); //const indexResult = await this.awaitProcess(child_process.spawn("wb-manager", ["reindex", this.params.collection], {cwd: this.params.cwd})); const params = [ "-o", path.join(this.collDir, "indexes", "index.cdxj"), - ...warcListFull + ...warcListFull, ]; - const indexResult = await this.awaitProcess(child_process.spawn("cdxj-indexer", params, {cwd: this.params.cwd})); + const indexResult = await this.awaitProcess( + child_process.spawn("cdxj-indexer", params, { cwd: this.params.cwd }) + ); if (indexResult === 0) { logger.debug("Indexing complete, CDX successfully created"); } else { - logger.error("Error indexing and generating CDX", {"status code": indexResult}); + logger.error("Error indexing and generating CDX", { + "status code": indexResult, + }); } } logger.info("Crawling done"); - if (this.params.generateWACZ && (!this.interrupted || this.finalExit || this.uploadAndDeleteLocal)) { + if ( + this.params.generateWACZ && + (!this.interrupted || this.finalExit || this.uploadAndDeleteLocal) + ) { const uploaded = await this.generateWACZ(); if (uploaded && this.uploadAndDeleteLocal) { - logger.info(`Uploaded WACZ, deleting local data to free up space: ${this.collDir}`); + logger.info( + `Uploaded WACZ, deleting local data to free up space: ${this.collDir}` + ); try { fs.rmSync(this.collDir, { recursive: true, force: true }); - } catch(e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.warn(`Unable to clear ${this.collDir} before exit`, e); } } @@ -980,14 +1272,14 @@ self.__bx_behaviors.selectMainBehavior(); } } - async closeLog() { + async closeLog(): Promise { // close file-based log logger.setExternalLogStream(null); if (!this.logFH) { return; } try { - await new Promise(resolve => this.logFH.close(() => resolve())); + await new Promise((resolve) => this.logFH.close(() => resolve())); } catch (e) { // ignore } @@ -1008,7 +1300,7 @@ self.__bx_behaviors.selectMainBehavior(); logger.info(`Num WARC Files: ${warcFileList.length}`); if (!warcFileList.length) { // if finished, just return - if (isFinished || await this.crawlState.isCrawlCanceled()) { + if (isFinished || (await this.crawlState.isCrawlCanceled())) { return; } // if stopped, won't get anymore data @@ -1031,9 +1323,12 @@ self.__bx_behaviors.selectMainBehavior(); const createArgs = [ "create", "--split-seeds", - "-o", waczPath, - "--pages", this.pagesFile, - "--log-directory", this.logDir + "-o", + waczPath, + "--pages", + this.pagesFile, + "--log-directory", + this.logDir, ]; if (process.env.WACZ_SIGN_URL) { @@ -1057,13 +1352,17 @@ self.__bx_behaviors.selectMainBehavior(); createArgs.push("-f"); - warcFileList.forEach((val, index) => createArgs.push(path.join(archiveDir, val))); // eslint-disable-line no-unused-vars + warcFileList.forEach((val) => + createArgs.push(path.join(archiveDir, val)) + ); // create WACZ - const waczResult = await this.awaitProcess(child_process.spawn("wacz" , createArgs)); + const waczResult = await this.awaitProcess( + child_process.spawn("wacz", createArgs) + ); if (waczResult !== 0) { - logger.error("Error creating WACZ", {"status code": waczResult}); + logger.error("Error creating WACZ", { "status code": waczResult }); logger.fatal("Unable to write WACZ successfully"); } @@ -1094,15 +1393,15 @@ self.__bx_behaviors.selectMainBehavior(); return false; } - awaitProcess(proc) { - let stdout = []; - let stderr = []; + awaitProcess(proc: ChildProcess) { + const stdout: string[] = []; + const stderr: string[] = []; - proc.stdout.on("data", (data) => { + proc.stdout!.on("data", (data) => { stdout.push(data.toString()); }); - proc.stderr.on("data", (data) => { + proc.stderr!.on("data", (data) => { stderr.push(data.toString()); }); @@ -1124,7 +1423,15 @@ self.__bx_behaviors.selectMainBehavior(); const { heapUsed, heapTotal } = memUsage; this.maxHeapUsed = Math.max(this.maxHeapUsed || 0, heapUsed); this.maxHeapTotal = Math.max(this.maxHeapTotal || 0, heapTotal); - logger.debug("Memory", {maxHeapUsed: this.maxHeapUsed, maxHeapTotal: this.maxHeapTotal, ...memUsage}, "memory"); + logger.debug( + "Memory", + { + maxHeapUsed: this.maxHeapUsed, + maxHeapTotal: this.maxHeapTotal, + ...memUsage, + }, + "memory" + ); } async writeStats() { @@ -1137,14 +1444,14 @@ self.__bx_behaviors.selectMainBehavior(); const done = await this.crawlState.numDone(); const failed = await this.crawlState.numFailed(); const total = realSize + pendingList.length + done; - const limit = {max: this.pageLimit || 0, hit: this.limitHit}; + const limit = { max: this.pageLimit || 0, hit: this.limitHit }; const stats = { - "crawled": done, - "total": total, - "pending": pendingList.length, - "failed": failed, - "limit": limit, - "pendingPages": pendingList.map(x => JSON.stringify(x)) + crawled: done, + total: total, + pending: pendingList.length, + failed: failed, + limit: limit, + pendingPages: pendingList.map((x) => JSON.stringify(x)), }; logger.info("Crawl statistics", stats, "crawlStatus"); @@ -1152,25 +1459,33 @@ self.__bx_behaviors.selectMainBehavior(); if (this.params.statsFilename) { try { - await fsp.writeFile(this.params.statsFilename, JSON.stringify(stats, null, 2)); - } catch (err) { + await fsp.writeFile( + this.params.statsFilename, + JSON.stringify(stats, null, 2) + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { logger.warn("Stats output failed", err); } } } - async loadPage(page, data, selectorOptsList = DEFAULT_SELECTORS) { - const {url, seedId, depth} = data; + async loadPage( + page: Page, + data: PageState, + selectorOptsList = DEFAULT_SELECTORS + ) { + const { url, seedId, depth } = data; const logDetails = data.logDetails; - const failCrawlOnError = ((depth === 0) && this.params.failOnFailedSeed); + const failCrawlOnError = depth === 0 && this.params.failOnFailedSeed; let ignoreAbort = false; // Detect if ERR_ABORTED is actually caused by trying to load a non-page (eg. downloadable PDF), // if so, don't report as an error - page.once("requestfailed", (req) => { + page.once("requestfailed", (req: HTTPRequest) => { ignoreAbort = shouldIgnoreAbort(req); }); @@ -1182,20 +1497,35 @@ self.__bx_behaviors.selectMainBehavior(); }); } - const gotoOpts = isHTMLPage ? this.gotoOpts : {waitUntil: "domcontentloaded"}; + const gotoOpts = isHTMLPage + ? this.gotoOpts + : { waitUntil: "domcontentloaded" }; logger.info("Awaiting page load", logDetails); try { const resp = await page.goto(url, gotoOpts); + if (!resp) { + throw new Error("page response missing"); + } + // Handle 4xx or 5xx response as a page load error const statusCode = resp.status(); - if (statusCode.toString().startsWith("4") || statusCode.toString().startsWith("5")) { + if ( + statusCode.toString().startsWith("4") || + statusCode.toString().startsWith("5") + ) { if (failCrawlOnError) { - logger.fatal("Seed Page Load Error, failing crawl", {statusCode, ...logDetails}); + logger.fatal("Seed Page Load Error, failing crawl", { + statusCode, + ...logDetails, + }); } else { - logger.error("Non-200 Status Code, skipping page", {statusCode, ...logDetails}); + logger.error("Non-200 Status Code, skipping page", { + statusCode, + ...logDetails, + }); throw new Error("logged"); } } @@ -1203,21 +1533,33 @@ self.__bx_behaviors.selectMainBehavior(); const contentType = resp.headers()["content-type"]; isHTMLPage = this.isHTMLContentType(contentType); - - } catch (e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { const msg = e.message || ""; if (!msg.startsWith("net::ERR_ABORTED") || !ignoreAbort) { // if timeout error, and at least got to content loaded, continue on - if (e.name === "TimeoutError" && data.loadState == LoadState.CONTENT_LOADED) { - logger.warn("Page Loading Slowly, skipping behaviors", {msg, ...logDetails}); + if ( + e.name === "TimeoutError" && + data.loadState == LoadState.CONTENT_LOADED + ) { + logger.warn("Page Loading Slowly, skipping behaviors", { + msg, + ...logDetails, + }); data.skipBehaviors = true; } else if (failCrawlOnError) { // if fail on error, immediately fail here - logger.fatal("Page Load Timeout, failing crawl", {msg, ...logDetails}); + logger.fatal("Page Load Timeout, failing crawl", { + msg, + ...logDetails, + }); } else { // log if not already log and rethrow if (msg !== "logged") { - logger.error("Page Load Timeout, skipping page", {msg, ...logDetails}); + logger.error("Page Load Timeout, skipping page", { + msg, + ...logDetails, + }); e.message = "logged"; } throw e; @@ -1230,16 +1572,24 @@ self.__bx_behaviors.selectMainBehavior(); data.isHTMLPage = isHTMLPage; if (isHTMLPage) { - let frames = await page.frames(); - frames = await Promise.allSettled(frames.map((frame) => this.shouldIncludeFrame(frame, logDetails))); + const frames = await page.frames(); - data.filteredFrames = frames.filter((x) => { - if (x.status === "fulfilled") { - return !!x.value; - } - logger.warn("Error in iframe check", {reason: x.reason, ...logDetails}); - return false; - }).map(x => x.value); + const filteredFrames = await Promise.allSettled( + frames.map((frame) => this.shouldIncludeFrame(frame, logDetails)) + ); + + data.filteredFrames = filteredFrames + .filter((x: PromiseSettledResult) => { + if (x.status === "fulfilled") { + return !!x.value; + } + logger.warn("Error in iframe check", { + reason: x.reason, + ...logDetails, + }); + return false; + }) + .map((x) => (x as PromiseFulfilledResult).value); //data.filteredFrames = await page.frames().filter(frame => this.shouldIncludeFrame(frame, logDetails)); } else { @@ -1268,7 +1618,7 @@ self.__bx_behaviors.selectMainBehavior(); await this.extractLinks(page, data, selectorOptsList, logDetails); } - async netIdle(page, details) { + async netIdle(page: Page, details: LogDetails) { if (!this.params.netIdleWait) { return; } @@ -1277,40 +1627,57 @@ self.__bx_behaviors.selectMainBehavior(); await sleep(0.5); try { - await this.browser.waitForNetworkIdle(page, {timeout: this.params.netIdleWait * 1000}); + await this.browser.waitForNetworkIdle(page, { + timeout: this.params.netIdleWait * 1000, + }); } catch (e) { logger.debug("waitForNetworkIdle timed out, ignoring", details); // ignore, continue } } - async extractLinks(page, data, selectors = DEFAULT_SELECTORS, logDetails) { - const {seedId, depth, extraHops = 0, filteredFrames, callbacks} = data; + async extractLinks( + page: Page, + data: PageState, + selectors = DEFAULT_SELECTORS, + logDetails: LogDetails + ) { + const { seedId, depth, extraHops = 0, filteredFrames, callbacks } = data; - let links = []; + let links: string[] = []; const promiseList = []; - callbacks.addLink = (url) => { + callbacks.addLink = (url: string) => { links.push(url); if (links.length == 500) { - promiseList.push(this.queueInScopeUrls(seedId, links, depth, extraHops, logDetails)); + promiseList.push( + this.queueInScopeUrls(seedId, links, depth, extraHops, logDetails) + ); links = []; } }; - const loadLinks = (options) => { + const loadLinks = (options: { + selector: string; + extract: string; + isAttribute: boolean; + addLinkFunc: string; + }) => { const { selector, extract, isAttribute, addLinkFunc } = options; - const urls = new Set(); + const urls = new Set(); - const getAttr = elem => urls.add(elem.getAttribute(extract)); - const getProp = elem => urls.add(elem[extract]); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const getAttr = (elem: any) => urls.add(elem.getAttribute(extract)); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const getProp = (elem: any) => urls.add(elem[extract]); const getter = isAttribute ? getAttr : getProp; document.querySelectorAll(selector).forEach(getter); - const func = window[addLinkFunc]; - urls.forEach(url => func.call(this, url)); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const func = (window as any)[addLinkFunc] as (url: string) => NonNullable; + urls.forEach((url) => func.call(this, url)); return true; }; @@ -1318,35 +1685,60 @@ self.__bx_behaviors.selectMainBehavior(); const frames = filteredFrames || page.frames(); try { - for (const {selector = "a[href]", extract = "href", isAttribute = false} of selectors) { + for (const { + selector = "a[href]", + extract = "href", + isAttribute = false, + } of selectors) { const promiseResults = await Promise.allSettled( - frames.map(frame => timedRun( - frame.evaluate(loadLinks, {selector, extract, isAttribute, addLinkFunc: ADD_LINK_FUNC}), - PAGE_OP_TIMEOUT_SECS, - "Link extraction timed out", - logDetails, - )) + frames.map((frame) => + timedRun( + frame.evaluate(loadLinks, { + selector, + extract, + isAttribute, + addLinkFunc: ADD_LINK_FUNC, + }), + PAGE_OP_TIMEOUT_SECS, + "Link extraction timed out", + logDetails + ) + ) ); for (let i = 0; i < promiseResults.length; i++) { - const {status, reason} = promiseResults[i]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const { status, reason } = promiseResults[i] as any; if (status === "rejected") { - logger.warn("Link Extraction failed in frame", {reason, frameUrl: frames[i].url, ...logDetails}); + logger.warn("Link Extraction failed in frame", { + reason, + frameUrl: frames[i].url, + ...logDetails, + }); } } } - } catch (e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.warn("Link Extraction failed", e); } if (links.length) { - promiseList.push(this.queueInScopeUrls(seedId, links, depth, extraHops, logDetails)); + promiseList.push( + this.queueInScopeUrls(seedId, links, depth, extraHops, logDetails) + ); } await Promise.allSettled(promiseList); } - async queueInScopeUrls(seedId, urls, depth, extraHops = 0, logDetails = {}) { + async queueInScopeUrls( + seedId: number, + urls: string[], + depth: number, + extraHops = 0, + logDetails: LogDetails = {} + ) { try { depth += 1; @@ -1354,36 +1746,51 @@ self.__bx_behaviors.selectMainBehavior(); const newExtraHops = extraHops + 1; for (const possibleUrl of urls) { - const res = this.isInScope({url: possibleUrl, extraHops: newExtraHops, depth, seedId}, logDetails); + const res = this.isInScope( + { url: possibleUrl, extraHops: newExtraHops, depth, seedId }, + logDetails + ); if (!res) { continue; } - const {url, isOOS} = res; + const { url, isOOS } = res; if (url) { - await this.queueUrl(seedId, url, depth, isOOS ? newExtraHops : extraHops, logDetails); + await this.queueUrl( + seedId, + url, + depth, + isOOS ? newExtraHops : extraHops, + logDetails + ); } } - } catch (e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.error("Queuing Error", e); } } - async checkCF(page, logDetails) { + async checkCF(page: Page, logDetails: LogDetails) { try { logger.debug("Check CF Blocking", logDetails); - while (await timedRun( - page.$("div.cf-browser-verification.cf-im-under-attack"), - PAGE_OP_TIMEOUT_SECS, - "Cloudflare check timed out", - logDetails, - "general", - true - )) { - logger.debug("Cloudflare Check Detected, waiting for reload...", logDetails); + while ( + await timedRun( + page.$("div.cf-browser-verification.cf-im-under-attack"), + PAGE_OP_TIMEOUT_SECS, + "Cloudflare check timed out", + logDetails, + "general", + true + ) + ) { + logger.debug( + "Cloudflare Check Detected, waiting for reload...", + logDetails + ); await sleep(5.5); } } catch (e) { @@ -1391,25 +1798,42 @@ self.__bx_behaviors.selectMainBehavior(); } } - async queueUrl(seedId, url, depth, extraHops, logDetails = {}) { + async queueUrl( + seedId: number, + url: string, + depth: number, + extraHops: number, + logDetails: LogDetails = {} + ) { if (this.limitHit) { return false; } - const result = await this.crawlState.addToQueue({url, seedId, depth, extraHops}, this.pageLimit); + const result = await this.crawlState.addToQueue( + { url, seedId, depth, extraHops }, + this.pageLimit + ); switch (result) { case QueueState.ADDED: - logger.debug("Queued new page url", {url, ...logDetails}, "links"); + logger.debug("Queued new page url", { url, ...logDetails }, "links"); return true; case QueueState.LIMIT_HIT: - logger.debug("Not queued page url, at page limit", {url, ...logDetails}, "links"); + logger.debug( + "Not queued page url, at page limit", + { url, ...logDetails }, + "links" + ); this.limitHit = true; return false; case QueueState.DUPE_URL: - logger.debug("Not queued page url, already seen", {url, ...logDetails}, "links"); + logger.debug( + "Not queued page url, already seen", + { url, ...logDetails }, + "links" + ); return false; } @@ -1421,7 +1845,7 @@ self.__bx_behaviors.selectMainBehavior(); let createNew = false; // create pages dir if doesn't exist and write pages.jsonl header - if (fs.existsSync(this.pagesDir) != true){ + if (fs.existsSync(this.pagesDir) != true) { await fsp.mkdir(this.pagesDir); createNew = true; } @@ -1429,7 +1853,11 @@ self.__bx_behaviors.selectMainBehavior(); this.pagesFH = await fsp.open(this.pagesFile, "a"); if (createNew) { - const header = {"format": "json-pages-1.0", "id": "pages", "title": "All Pages"}; + const header: Record = { + format: "json-pages-1.0", + id: "pages", + title: "All Pages", + }; header["hasText"] = this.params.text.includes("to-pages"); if (this.params.text.length) { logger.debug("Text Extraction: " + this.params.text.join(",")); @@ -1439,14 +1867,23 @@ self.__bx_behaviors.selectMainBehavior(); const header_formatted = JSON.stringify(header).concat("\n"); await this.pagesFH.writeFile(header_formatted); } - - } catch(err) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { logger.error("pages/pages.jsonl creation failed", err); } } - async writePage({pageid, url, depth, title, text, loadState, mime, favicon}) { - const row = {id: pageid, url, title, loadState}; + async writePage({ + pageid, + url, + depth, + title, + text, + loadState, + mime, + favicon, + }: PageState) { + const row: PageEntry = { id: pageid!, url, title, loadState }; if (mime) { row.mime = mime; @@ -1466,38 +1903,42 @@ self.__bx_behaviors.selectMainBehavior(); const processedRow = JSON.stringify(row) + "\n"; try { - await this.pagesFH.writeFile(processedRow); - } catch (err) { + await this.pagesFH!.writeFile(processedRow); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { logger.warn("pages/pages.jsonl append failed", err); } } - resolveAgent(urlParsed) { + resolveAgent(urlParsed: URL) { return urlParsed.protocol === "https:" ? HTTPS_AGENT : HTTP_AGENT; } - async isHTML(url, logDetails) { + async isHTML(url: string, logDetails: LogDetails) { try { const resp = await fetch(url, { method: "HEAD", headers: this.headers, - agent: this.resolveAgent - }); + agent: this.resolveAgent, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any); if (resp.status !== 200) { - logger.debug("HEAD response code != 200, loading in browser", {status: resp.status, ...logDetails}); + logger.debug("HEAD response code != 200, loading in browser", { + status: resp.status, + ...logDetails, + }); return true; } return this.isHTMLContentType(resp.headers.get("Content-Type")); - - } catch(e) { + } catch (e) { // can't confirm not html, so try in browser - logger.debug("HEAD request failed", {...errJSON(e), ...logDetails}); + logger.debug("HEAD request failed", { ...errJSON(e), ...logDetails }); return true; } } - isHTMLContentType(contentType) { + isHTMLContentType(contentType: string | null) { // just load if no content-type if (!contentType) { return true; @@ -1512,29 +1953,39 @@ self.__bx_behaviors.selectMainBehavior(); return false; } - async parseSitemap(url, seedId, sitemapFromDate) { + async parseSitemap(url: string, seedId: number, sitemapFromDate: number) { // handle sitemap last modified date if passed - let lastmodFromTimestamp = null; + let lastmodFromTimestamp = undefined; const dateObj = new Date(sitemapFromDate); if (isNaN(dateObj.getTime())) { - logger.info("Fetching full sitemap (fromDate not specified/valid)", {url, sitemapFromDate}, "sitemap"); + logger.info( + "Fetching full sitemap (fromDate not specified/valid)", + { url, sitemapFromDate }, + "sitemap" + ); } else { lastmodFromTimestamp = dateObj.getTime(); - logger.info("Fetching and filtering sitemap by date", {url, sitemapFromDate}, "sitemap"); + logger.info( + "Fetching and filtering sitemap by date", + { url, sitemapFromDate }, + "sitemap" + ); } - const sitemapper = new Sitemapper({ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const sitemapper = new (Sitemapper as any)({ url, timeout: 15000, requestHeaders: this.headers, - lastmod: lastmodFromTimestamp + lastmod: lastmodFromTimestamp, }); try { const { sites } = await sitemapper.fetch(); - logger.info("Sitemap Urls Found", {urls: sites.length}, "sitemap"); + logger.info("Sitemap Urls Found", { urls: sites.length }, "sitemap"); await this.queueInScopeUrls(seedId, sites, 0); - } catch(e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.warn("Error fetching sites from sitemap", e, "sitemap"); } } @@ -1554,7 +2005,7 @@ self.__bx_behaviors.selectMainBehavior(); for (let i = 0; i < warcLists.length; i++) { const fileName = path.join(this.collDir, "archive", warcLists[i]); const fileSize = await getFileSize(fileName); - fileSizeObjects.push({"fileSize": fileSize, "fileName": fileName}); + fileSizeObjects.push({ fileSize: fileSize, fileName: fileName }); fileSizeObjects.sort((a, b) => b.fileSize - a.fileSize); } @@ -1571,7 +2022,6 @@ self.__bx_behaviors.selectMainBehavior(); // Iterate through the sorted file size array. for (let j = 0; j < fileSizeObjects.length; j++) { - // if need to rollover to new warc let doRollover = false; @@ -1583,9 +2033,10 @@ self.__bx_behaviors.selectMainBehavior(); const currentCombinedWarcSize = await getFileSize(combinedWarcFullPath); // If adding the current warc to the existing combined file creates a file smaller than the rollover size add the data to the combinedWarc - const proposedWarcSize = fileSizeObjects[j].fileSize + currentCombinedWarcSize; + const proposedWarcSize = + fileSizeObjects[j].fileSize + currentCombinedWarcSize; - doRollover = (proposedWarcSize >= this.params.rolloverSize); + doRollover = proposedWarcSize >= this.params.rolloverSize; } if (doRollover) { @@ -1605,7 +2056,7 @@ self.__bx_behaviors.selectMainBehavior(); fh.end(); } - fh = fs.createWriteStream(combinedWarcFullPath, {flags: "a"}); + fh = fs.createWriteStream(combinedWarcFullPath, { flags: "a" }); generatedCombinedWarcs.push(combinedWarcName); @@ -1617,11 +2068,13 @@ self.__bx_behaviors.selectMainBehavior(); const reader = fs.createReadStream(fileSizeObjects[j].fileName); - const p = new Promise((resolve) => { + const p = new Promise((resolve) => { reader.on("end", () => resolve()); }); - reader.pipe(fh, {end: false}); + if (fh) { + reader.pipe(fh, { end: false }); + } await p; } @@ -1656,18 +2109,20 @@ self.__bx_behaviors.selectMainBehavior(); if (!done) { // if not done, save state only after specified interval has elapsed - if (secondsElapsed(this.lastSaveTime, now) < this.params.saveStateInterval) { + if ( + secondsElapsed(this.lastSaveTime, now) < this.params.saveStateInterval + ) { return; } } this.lastSaveTime = now.getTime(); - const ts = now.toISOString().slice(0,19).replace(/[T:-]/g, ""); + const ts = now.toISOString().slice(0, 19).replace(/[T:-]/g, ""); const crawlDir = path.join(this.collDir, "crawls"); - await fsp.mkdir(crawlDir, {recursive: true}); + await fsp.mkdir(crawlDir, { recursive: true }); const filenameOnly = `crawl-${ts}-${this.params.crawlId}.yaml`; @@ -1678,11 +2133,12 @@ self.__bx_behaviors.selectMainBehavior(); if (this.origConfig) { this.origConfig.state = state; } - const res = yaml.dump(this.origConfig, {lineWidth: -1}); + const res = yaml.dump(this.origConfig, { lineWidth: -1 }); try { logger.info(`Saving crawl state to: ${filename}`); await fsp.writeFile(filename, res); - } catch (e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.error(`Failed to write save state file: ${filename}`, e); return; } @@ -1693,7 +2149,7 @@ self.__bx_behaviors.selectMainBehavior(); const oldFilename = this.saveStateFiles.shift(); logger.info(`Removing old save-state: ${oldFilename}`); try { - await fsp.unlink(oldFilename); + await fsp.unlink(oldFilename || ""); } catch (e) { logger.error(`Failed to delete old save state file: ${oldFilename}`); } @@ -1707,10 +2163,11 @@ self.__bx_behaviors.selectMainBehavior(); } } -function shouldIgnoreAbort(req) { +function shouldIgnoreAbort(req: HTTPRequest) { try { - const failure = req.failure() && req.failure().errorText; - if (failure !== "net::ERR_ABORTED" || req.resourceType() !== "document") { + const failure = req.failure(); + const failureText = failure && failure.errorText || ""; + if (failureText !== "net::ERR_ABORTED" || req.resourceType() !== "document") { return false; } @@ -1728,5 +2185,7 @@ function shouldIgnoreAbort(req) { } catch (e) { return false; } + + return false; } diff --git a/create-login-profile.js b/src/create-login-profile.ts similarity index 63% rename from create-login-profile.js rename to src/create-login-profile.ts index 182287df..eeb7482d 100755 --- a/create-login-profile.js +++ b/src/create-login-profile.ts @@ -2,24 +2,25 @@ import fs from "fs"; import path from "path"; -import http from "http"; +import http, { IncomingMessage, ServerResponse } from "http"; import readline from "readline"; import child_process from "child_process"; -import yargs from "yargs"; +import yargs, { Options } from "yargs"; import { logger } from "./util/logger.js"; import { Browser } from "./util/browser.js"; import { initStorage } from "./util/storage.js"; +import { CDPSession, Page, PuppeteerLifeCycleEvent } from "puppeteer-core"; -const profileHTML = fs.readFileSync(new URL("html/createProfile.html", import.meta.url), {encoding: "utf8"}); -const vncHTML = fs.readFileSync(new URL("html/vnc_lite.html", import.meta.url), {encoding: "utf8"}); +const profileHTML = fs.readFileSync(new URL("../html/createProfile.html", import.meta.url), {encoding: "utf8"}); +const vncHTML = fs.readFileSync(new URL("../html/vnc_lite.html", import.meta.url), {encoding: "utf8"}); -const behaviors = fs.readFileSync(new URL("./node_modules/browsertrix-behaviors/dist/behaviors.js", import.meta.url), {encoding: "utf8"}); +const behaviors = fs.readFileSync(new URL("../node_modules/browsertrix-behaviors/dist/behaviors.js", import.meta.url), {encoding: "utf8"}); -function cliOpts() { +function cliOpts(): { [key: string]: Options } { return { "url": { describe: "The URL of the login page", @@ -93,7 +94,7 @@ function cliOpts() { } function getDefaultWindowSize() { - const values = process.env.GEOMETRY.split("x"); + const values = (process.env.GEOMETRY || "").split("x"); const x = Number(values[0]); const y = Number(values[1]); return `${x},${y}`; @@ -102,23 +103,23 @@ function getDefaultWindowSize() { async function main() { - const params = yargs(process.argv) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const params : any = yargs(process.argv) .usage("browsertrix-crawler profile [options]") .option(cliOpts()) .argv; logger.setDebugLogging(true); - if (!params.headless) { logger.debug("Launching XVFB"); child_process.spawn("Xvfb", [ - process.env.DISPLAY, + process.env.DISPLAY || "", "-listen", "tcp", "-screen", "0", - process.env.GEOMETRY, + process.env.GEOMETRY || "", "-ac", "+extension", "RANDR" @@ -137,9 +138,9 @@ async function main() { "-rfbport", "6080", "-passwd", - process.env.VNC_PASS, + process.env.VNC_PASS || "", "-display", - process.env.DISPLAY + process.env.DISPLAY || "" ]); } @@ -178,7 +179,7 @@ async function main() { const { page, cdp } = await browser.newWindowPageWithCDP(); - const waitUntil = "load"; + const waitUntil : PuppeteerLifeCycleEvent = "load"; await page.setCacheEnabled(false); @@ -203,7 +204,9 @@ async function main() { } } -async function automatedProfile(params, browser, page, cdp, waitUntil) { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +async function automatedProfile(params: any, browser: Browser, page: Page, cdp: CDPSession, + waitUntil: PuppeteerLifeCycleEvent) { let u, p; logger.debug("Looking for username and password entry fields on page..."); @@ -222,12 +225,12 @@ async function automatedProfile(params, browser, page, cdp, waitUntil) { return; } - await u.type(params.user); + await u!.type(params.user); - await p.type(params.password); + await p!.type(params.password); await Promise.allSettled([ - p.press("Enter"), + p!.press("Enter"), page.waitForNavigation({waitUntil}) ]); @@ -240,7 +243,8 @@ async function automatedProfile(params, browser, page, cdp, waitUntil) { process.exit(0); } -async function createProfile(params, browser, page, cdp, targetFilename = "") { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +async function createProfile(params: any, browser: Browser, page: Page, cdp: CDPSession, targetFilename = "") { await cdp.send("Network.clearBrowserCache"); await browser.close(); @@ -268,8 +272,9 @@ async function createProfile(params, browser, page, cdp, targetFilename = "") { return resource; } -function promptInput(msg, hidden = false) { - const rl = readline.createInterface({ +function promptInput(msg: string, hidden = false) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const rl : any = readline.createInterface({ input: process.stdin, output: process.stdout }); @@ -290,8 +295,8 @@ function promptInput(msg, hidden = false) { }); } - return new Promise((resolve) => { - rl.question(msg, function (res) { + return new Promise((resolve) => { + rl.question(msg, function (res: string) { rl.close(); resolve(res); }); @@ -300,9 +305,31 @@ function promptInput(msg, hidden = false) { class InteractiveBrowser { - constructor(params, browser, page, cdp, targetId) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + params: any; + browser: Browser; + page: Page; + cdp: CDPSession; + + targetId: string; + originSet = new Set(); + + shutdownWait: number; + shutdownTimer: NodeJS.Timer | null; + + constructor( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + params: any, + browser: Browser, + page: Page, + cdp: CDPSession, + targetId: string + ) { logger.info("Creating Profile Interactively..."); - child_process.spawn("socat", ["tcp-listen:9222,reuseaddr,fork", "tcp:localhost:9221"]); + child_process.spawn("socat", [ + "tcp-listen:9222,reuseaddr,fork", + "tcp:localhost:9221", + ]); this.params = params; this.browser = browser; @@ -311,8 +338,6 @@ class InteractiveBrowser { this.targetId = targetId; - this.originSet = new Set(); - this.addOrigin(); page.on("load", () => this.handlePageLoad()); @@ -323,25 +348,31 @@ class InteractiveBrowser { cdp.on("Page.windowOpen", async (resp) => { if (resp.url) { - await cdp.send("Target.activateTarget", {targetId: this.targetId}); + await cdp.send("Target.activateTarget", { targetId: this.targetId }); await page.goto(resp.url); } }); } this.shutdownWait = params.shutdownWait * 1000; - + if (this.shutdownWait) { this.shutdownTimer = setTimeout(() => process.exit(0), this.shutdownWait); - logger.debug(`Shutting down in ${this.shutdownWait}ms if no ping received`); + logger.debug( + `Shutting down in ${this.shutdownWait}ms if no ping received` + ); } else { - this.shutdownTimer = 0; + this.shutdownTimer = null; } - const httpServer = http.createServer((req, res) => this.handleRequest(req, res)); + const httpServer = http.createServer((req, res) => + this.handleRequest(req, res) + ); const port = 9223; httpServer.listen(port); - logger.info(`Browser Profile UI Server started. Load http://localhost:${port}/ to interact with a Chromium-based browser, click 'Create Profile' when done.`); + logger.info( + `Browser Profile UI Server started. Load http://localhost:${port}/ to interact with a Chromium-based browser, click 'Create Profile' when done.` + ); if (!params.headless) { logger.info("Screencasting with VNC on port 6080"); @@ -363,18 +394,26 @@ class InteractiveBrowser { } } - async saveCookiesFor(url) { + async saveCookiesFor(url: string) { try { if (this.params.cookieDays <= 0) { return; } - const cookies = await this.browser.getCookies(this.page, url); - for (const cookie of cookies) { - cookie.expires = (new Date().getTime() / 1000) + this.params.cookieDays * 86400; + const cookies = await this.browser.getCookies(this.page); + for (const cookieOrig of cookies) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const cookie = cookieOrig as any; + cookie.expires = + new Date().getTime() / 1000 + this.params.cookieDays * 86400; + delete cookie.size; delete cookie.session; - if (cookie.sameSite && cookie.sameSite !== "Lax" && cookie.sameSite !== "Strict") { + if ( + cookie.sameSite && + cookie.sameSite !== "Lax" && + cookie.sameSite !== "Strict" + ) { delete cookie.sameSite; } if (!cookie.domain && !cookie.path) { @@ -382,64 +421,76 @@ class InteractiveBrowser { } } await this.browser.setCookies(this.page, cookies); - } catch (e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.error("Save Cookie Error: ", e); } } addOrigin() { const url = this.page.url(); - logger.debug("Adding origin", {url}); + logger.debug("Adding origin", { url }); if (url.startsWith("http:") || url.startsWith("https:")) { this.originSet.add(new URL(url).origin); } } - async handleRequest(req, res) { - const parsedUrl = new URL(req.url, `http://${req.headers.host}`); + async handleRequest(req: IncomingMessage, res: ServerResponse) { + const parsedUrl = new URL(req.url || "", `http://${req.headers.host}`); const pathname = parsedUrl.pathname; let targetUrl; let origins; switch (pathname) { case "/": - res.writeHead(200, {"Content-Type": "text/html"}); + res.writeHead(200, { "Content-Type": "text/html" }); if (this.params.headless) { targetUrl = `http://$HOST:9222/devtools/inspector.html?ws=$HOST:9222/devtools/page/${this.targetId}&panel=resources`; } else { targetUrl = `http://$HOST:9223/vnc/?host=$HOST&port=6080&password=${process.env.VNC_PASS}`; } - res.end(profileHTML.replace("$DEVTOOLS_SRC", targetUrl.replaceAll("$HOST", parsedUrl.hostname))); + res.end( + profileHTML.replace( + "$DEVTOOLS_SRC", + targetUrl.replaceAll("$HOST", parsedUrl.hostname) + ) + ); return; case "/vnc/": case "/vnc/index.html": - res.writeHead(200, {"Content-Type": "text/html"}); + res.writeHead(200, { "Content-Type": "text/html" }); res.end(vncHTML); return; case "/ping": if (this.shutdownWait) { - clearInterval(this.shutdownTimer); - this.shutdownTimer = setTimeout(() => process.exit(0), this.shutdownWait); - logger.debug(`Ping received, delaying shutdown for ${this.shutdownWait}ms`); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + clearTimeout(this.shutdownTimer as any); + this.shutdownTimer = setTimeout( + () => process.exit(0), + this.shutdownWait + ); + logger.debug( + `Ping received, delaying shutdown for ${this.shutdownWait}ms` + ); } origins = Array.from(this.originSet.values()); - res.writeHead(200, {"Content-Type": "application/json"}); + res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({pong: true, origins})); + res.end(JSON.stringify({ pong: true, origins })); return; case "/target": - res.writeHead(200, {"Content-Type": "application/json"}); - res.end(JSON.stringify({targetId: this.targetId})); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ targetId: this.targetId })); return; case "/vncpass": - res.writeHead(200, {"Content-Type": "application/json"}); - res.end(JSON.stringify({password: process.env.VNC_PASS})); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ password: process.env.VNC_PASS })); return; case "/navigate": @@ -451,14 +502,14 @@ class InteractiveBrowser { const postData = await this.readBodyJson(req); const url = new URL(postData.url).href; - res.writeHead(200, {"Content-Type": "application/json"}); - res.end(JSON.stringify({success: true})); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ success: true })); this.page.goto(url); - - } catch (e) { - res.writeHead(400, {"Content-Type": "application/json"}); - res.end(JSON.stringify({"error": e.toString()})); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { + res.writeHead(400, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: e.toString() })); logger.warn("HTTP Error", e); } return; @@ -474,14 +525,21 @@ class InteractiveBrowser { await this.saveAllCookies(); - const resource = await createProfile(this.params, this.browser, this.page, this.cdp, targetFilename); + const resource = await createProfile( + this.params, + this.browser, + this.page, + this.cdp, + targetFilename + ); origins = Array.from(this.originSet.values()); - res.writeHead(200, {"Content-Type": "application/json"}); - res.end(JSON.stringify({resource, origins})); - } catch (e) { - res.writeHead(500, {"Content-Type": "application/json"}); - res.end(JSON.stringify({"error": e.toString()})); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ resource, origins })); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { + res.writeHead(500, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: e.toString() })); logger.warn("HTTP Error", e); } @@ -498,11 +556,16 @@ class InteractiveBrowser { await createProfile(this.params, this.browser, this.page, this.cdp); - res.writeHead(200, {"Content-Type": "text/html"}); - res.end("Profile Created! You may now close this window."); - } catch (e) { - res.writeHead(500, {"Content-Type": "text/html"}); - res.end("Profile creation failed! See the browsertrix-crawler console for more info"); + res.writeHead(200, { "Content-Type": "text/html" }); + res.end( + "Profile Created! You may now close this window." + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { + res.writeHead(500, { "Content-Type": "text/html" }); + res.end( + "Profile creation failed! See the browsertrix-crawler console for more info" + ); logger.warn("HTTP Error", e); } @@ -511,18 +574,21 @@ class InteractiveBrowser { } if (pathname.startsWith("/vnc/")) { - const fileUrl = new URL("node_modules/@novnc/novnc/" + pathname.slice("/vnc/".length), import.meta.url); - const file = fs.readFileSync(fileUrl, {encoding: "utf-8"}); - res.writeHead(200, {"Content-Type": "application/javascript"}); + const fileUrl = new URL( + "../node_modules/@novnc/novnc/" + pathname.slice("/vnc/".length), + import.meta.url + ); + const file = fs.readFileSync(fileUrl, { encoding: "utf-8" }); + res.writeHead(200, { "Content-Type": "application/javascript" }); res.end(file); return; } - res.writeHead(404, {"Content-Type": "text/html"}); + res.writeHead(404, { "Content-Type": "text/html" }); res.end("Not Found"); } - async readBodyJson(req) { + async readBodyJson(req: IncomingMessage) { const buffers = []; for await (const chunk of req) { diff --git a/src/defaultDriver.ts b/src/defaultDriver.ts new file mode 100644 index 00000000..e22ca63d --- /dev/null +++ b/src/defaultDriver.ts @@ -0,0 +1,7 @@ +import { Page } from "puppeteer-core"; +import { PageState } from "./util/state.js"; +import { Crawler } from "./crawler.js"; + +export default async ({data, page, crawler} : {data: PageState, page: Page, crawler: Crawler}) => { + await crawler.loadPage(page, data); +}; diff --git a/main.js b/src/main.ts similarity index 80% rename from main.js rename to src/main.ts index 56afe7e0..80bf63d9 100755 --- a/main.js +++ b/src/main.ts @@ -5,13 +5,13 @@ import { setExitOnRedisError } from "./util/redis.js"; import { Crawler } from "./crawler.js"; -var crawler = null; +let crawler : Crawler | null = null; -var lastSigInt = 0; +let lastSigInt = 0; let forceTerm = false; -async function handleTerminate(signame) { +async function handleTerminate(signame: string) { logger.info(`${signame} received...`); if (!crawler || !crawler.crawlState) { logger.error("error: no crawler running, exiting"); @@ -23,7 +23,7 @@ async function handleTerminate(signame) { process.exit(0); } - setExitOnRedisError(true); + setExitOnRedisError(); try { crawler.checkCanceled(); @@ -31,13 +31,13 @@ async function handleTerminate(signame) { if (!crawler.interrupted) { logger.info("SIGNAL: gracefully finishing current pages..."); crawler.gracefulFinishOnInterrupt(); - - } else if (forceTerm || (Date.now() - lastSigInt) > 200) { + } else if (forceTerm || Date.now() - lastSigInt > 200) { logger.info("SIGNAL: stopping crawl now..."); await crawler.serializeAndExit(); } lastSigInt = Date.now(); - } catch (e) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { logger.error("Error stopping crawl after receiving termination signal", e); } } diff --git a/util/argParser.js b/src/util/argParser.ts similarity index 93% rename from util/argParser.js rename to src/util/argParser.ts index 5ca6cbd0..6d527a53 100644 --- a/util/argParser.js +++ b/src/util/argParser.ts @@ -4,7 +4,7 @@ import os from "os"; import yaml from "js-yaml"; import { KnownDevices as devices } from "puppeteer-core"; -import yargs from "yargs"; +import yargs, { Options } from "yargs"; import { hideBin } from "yargs/helpers"; import { BEHAVIOR_LOG_FUNC, WAIT_UNTIL_OPTS, EXTRACT_TEXT_TYPES } from "./constants.js"; @@ -16,8 +16,8 @@ import { logger } from "./logger.js"; // ============================================================================ class ArgParser { - get cliOpts() { - const coerce = array => { + get cliOpts() : { [key: string]: Options } { + const coerce = (array : string[]) => { return array.flatMap(v => v.split(",")).filter(x => !!x); }; @@ -305,7 +305,7 @@ class ArgParser { "warcInfo": { alias: ["warcinfo"], describe: "Optional fields added to the warcinfo record in combined WARCs", - type: "object" + //type: "object" }, "redisStoreUrl": { @@ -423,7 +423,7 @@ class ArgParser { "customBehaviors": { describe: "injects a custom behavior file or set of behavior files in a directory", - type: ["string"] + type: "string" }, "debugAccessRedis": { @@ -433,8 +433,8 @@ class ArgParser { }; } - parseArgs(argv) { - argv = argv || process.argv; + parseArgs(argvParams?: string[]) { + let argv = argvParams || process.argv; if (process.env.CRAWL_ARGS) { argv = argv.concat(this.splitCrawlArgsQuoteSafe(process.env.CRAWL_ARGS)); @@ -445,11 +445,12 @@ class ArgParser { const parsed = yargs(hideBin(argv)) .usage("crawler [options]") .option(this.cliOpts) - .config("config", "Path to YAML config file", (configPath) => { + .config("config", "Path to YAML config file", (configPath : string | number) => { if (configPath === "/crawls/stdin") { configPath = process.stdin.fd; } - origConfig = yaml.load(fs.readFileSync(configPath, "utf8")); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + origConfig = yaml.load(fs.readFileSync(configPath, "utf8")) as any; return origConfig; }) .check((argv) => this.validateArgs(argv)) @@ -458,13 +459,15 @@ class ArgParser { return {parsed, origConfig}; } - splitCrawlArgsQuoteSafe(crawlArgs) { + splitCrawlArgsQuoteSafe(crawlArgs: string) : string[] { // Split process.env.CRAWL_ARGS on spaces but retaining spaces within double quotes const regex = /"[^"]+"|[^\s]+/g; - return crawlArgs.match(regex).map(e => e.replace(/"(.+)"/, "$1")); + const res = crawlArgs.match(regex); + return res ? res.map(e => e.replace(/"(.+)"/, "$1")) : []; } - validateArgs(argv) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + validateArgs(argv: Record) { argv.crawlId = argv.crawlId || process.env.CRAWL_ID || os.hostname; argv.collection = interpolateFilename(argv.collection, argv.crawlId); @@ -474,15 +477,16 @@ class ArgParser { } // background behaviors to apply - const behaviorOpts = {}; - argv.behaviors.forEach((x) => behaviorOpts[x] = true); + const behaviorOpts : {[key: string]: string | boolean} = {}; + argv.behaviors.forEach((x: string) => behaviorOpts[x] = true); behaviorOpts.log = BEHAVIOR_LOG_FUNC; argv.behaviorOpts = JSON.stringify(behaviorOpts); argv.text = argv.text || []; if (argv.mobileDevice) { - argv.emulateDevice = devices[argv.mobileDevice.replace("-", " ")]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + argv.emulateDevice = (devices as Record)[argv.mobileDevice.replace("-", " ")]; if (!argv.emulateDevice) { logger.fatal("Unknown device: " + argv.mobileDevice); } @@ -556,6 +560,6 @@ class ArgParser { } } -export function parseArgs(argv) { +export function parseArgs(argv?: string[]) { return new ArgParser().parseArgs(argv); } diff --git a/util/blockrules.js b/src/util/blockrules.ts similarity index 77% rename from util/blockrules.js rename to src/util/blockrules.ts index c50b4234..6116280e 100644 --- a/util/blockrules.js +++ b/src/util/blockrules.ts @@ -1,6 +1,8 @@ import fs from "fs"; import { logger, errJSON } from "./logger.js"; +import { HTTPRequest, Page } from "puppeteer-core"; +import { Browser } from "./browser.js"; const RULE_TYPES = ["block", "allowOnly"]; @@ -14,11 +16,23 @@ const BlockState = { BLOCK_AD: "advertisement" }; +type BlockRuleDecl = { + url?: string; + frameTextMatch?: string; + inFrameUrl?: string; + type?: string; +} + // =========================================================================== class BlockRule { - constructor(data) { + type: string; + url: RegExp | null; + frameTextMatch?: RegExp | null; + inFrameUrl?: RegExp | null; + + constructor(data: string | BlockRuleDecl) { if (typeof(data) === "string") { this.url = new RegExp(data); this.type = "block"; @@ -49,7 +63,12 @@ ${this.frameTextMatch ? "Frame Text Regex: " + this.frameTextMatch : ""} // =========================================================================== export class BlockRules { - constructor(blockRules, blockPutUrl, blockErrMsg) { + rules: BlockRule[]; + blockPutUrl: string; + blockErrMsg: string; + blockedUrlSet = new Set(); + + constructor(blockRules: BlockRuleDecl[], blockPutUrl: string, blockErrMsg: string) { this.rules = []; this.blockPutUrl = blockPutUrl; this.blockErrMsg = blockErrMsg; @@ -68,8 +87,8 @@ export class BlockRules } } - async initPage(browser, page) { - const onRequest = async (request) => { + async initPage(browser: Browser, page: Page) { + const onRequest = async (request: HTTPRequest) => { const logDetails = {page: page.url()}; try { await this.handleRequest(request, logDetails); @@ -80,7 +99,8 @@ export class BlockRules await browser.interceptRequest(page, onRequest); } - async handleRequest(request, logDetails) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async handleRequest(request: HTTPRequest, logDetails: Record) { const url = request.url(); let blockState; @@ -99,7 +119,8 @@ export class BlockRules } } - async shouldBlock(request, url, logDetails) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async shouldBlock(request: HTTPRequest, url: string, logDetails: Record) { if (!url.startsWith("http:") && !url.startsWith("https:")) { return BlockState.ALLOW; } @@ -107,6 +128,9 @@ export class BlockRules const isNavReq = request.isNavigationRequest(); const frame = request.frame(); + if (!frame) { + return BlockState.ALLOW; + } let frameUrl = ""; let blockState; @@ -157,7 +181,8 @@ export class BlockRules return BlockState.ALLOW; } - async ruleCheck(rule, request, reqUrl, frameUrl, isNavReq, logDetails) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async ruleCheck(rule: BlockRule, request: HTTPRequest, reqUrl: string, frameUrl: string, isNavReq: boolean, logDetails: Record) { const {url, inFrameUrl, frameTextMatch} = rule; const type = rule.type || "block"; @@ -187,7 +212,8 @@ export class BlockRules return {block, done: false}; } - async isTextMatch(request, reqUrl, frameTextMatch, logDetails) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async isTextMatch(request: HTTPRequest, reqUrl: string, frameTextMatch: RegExp, logDetails: Record) { try { const res = await fetch(reqUrl); const text = await res.text(); @@ -199,7 +225,7 @@ export class BlockRules } } - async recordBlockMsg(url) { + async recordBlockMsg(url: string) { if (this.blockedUrlSet.has(url)) { return; } @@ -221,18 +247,21 @@ export class BlockRules // =========================================================================== export class AdBlockRules extends BlockRules { - constructor(blockPutUrl, blockErrMsg, adhostsFilePath = "../ad-hosts.json") { + adhosts: string[]; + + constructor(blockPutUrl: string, blockErrMsg: string, adhostsFilePath = "../../ad-hosts.json") { super([], blockPutUrl, blockErrMsg); - this.adhosts = JSON.parse(fs.readFileSync(new URL(adhostsFilePath, import.meta.url))); + this.adhosts = JSON.parse(fs.readFileSync(new URL(adhostsFilePath, import.meta.url), {"encoding": "utf-8"})); } - isAdUrl(url) { + isAdUrl(url: string) { const fragments = url.split("/"); const domain = fragments.length > 2 ? fragments[2] : null; - return this.adhosts.includes(domain); + return domain && this.adhosts.includes(domain); } - async shouldBlock(request, url, logDetails) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async shouldBlock(request: HTTPRequest, url: string, logDetails: Record) { if (this.isAdUrl(url)) { logger.debug("URL blocked for being an ad", {url, ...logDetails}, "blocking"); await this.recordBlockMsg(url); diff --git a/util/browser.js b/src/util/browser.ts similarity index 66% rename from util/browser.js rename to src/util/browser.ts index 5dee4b20..4d74c3cf 100644 --- a/util/browser.js +++ b/src/util/browser.ts @@ -9,61 +9,85 @@ import path from "path"; import { logger } from "./logger.js"; import { initStorage } from "./storage.js"; -import puppeteer from "puppeteer-core"; +import puppeteer, { Frame, HTTPRequest, Page, PuppeteerLaunchOptions, Viewport } from "puppeteer-core"; +import { CDPSession, Target, Browser as PptrBrowser } from "puppeteer-core"; + +type LaunchOpts = { + profileUrl: string; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + chromeOptions: Record + signals: boolean; + headless: boolean; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + emulateDevice?: Record + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ondisconnect?: ((err: any) => NonNullable) | null +}; // ================================================================== -export class BaseBrowser +export class Browser { + profileDir: string; + customProfile = false; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + emulateDevice: Record | null = null; + + browser?: PptrBrowser | null = null; + firstCDP: CDPSession | null = null; + + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + recorders: any[] = []; + constructor() { this.profileDir = fs.mkdtempSync(path.join(os.tmpdir(), "profile-")); - this.customProfile = false; - this.emulateDevice = null; - - this.recorders = []; } - async launch({profileUrl, chromeOptions, signals = false, headless = false, emulateDevice = {}, ondisconnect = null} = {}) { - if (this.isLaunched()) { - return; - } - - if (profileUrl) { - this.customProfile = await this.loadProfile(profileUrl); - } - - this.emulateDevice = emulateDevice; - - const args = this.chromeArgs(chromeOptions); - - let defaultViewport = null; - - if (process.env.GEOMETRY) { - const geom = process.env.GEOMETRY.split("x"); - - defaultViewport = {width: Number(geom[0]), height: Number(geom[1])}; - } - - const launchOpts = { - args, - headless: headless ? "new" : false, - executablePath: this.getBrowserExe(), - ignoreDefaultArgs: ["--enable-automation", "--hide-scrollbars"], - ignoreHTTPSErrors: true, - handleSIGHUP: signals, - handleSIGINT: signals, - handleSIGTERM: signals, - protocolTimeout: 0, - - defaultViewport, - waitForInitialPage: false, - userDataDir: this.profileDir - }; - - await this._init(launchOpts, ondisconnect); + async launch({profileUrl, chromeOptions, signals = false, headless = false, emulateDevice = {}, ondisconnect = null} : LaunchOpts) { if (this.isLaunched()) { + return; } - async setupPage({page}) { + if (profileUrl) { + this.customProfile = await this.loadProfile(profileUrl); + } + + this.emulateDevice = emulateDevice; + + const args = this.chromeArgs(chromeOptions); + + let defaultViewport = null; + + if (process.env.GEOMETRY) { + const geom = process.env.GEOMETRY.split("x"); + + defaultViewport = {width: Number(geom[0]), height: Number(geom[1])}; + } + + const launchOpts : PuppeteerLaunchOptions = { + args, + headless: headless ? "new" : false, + executablePath: this.getBrowserExe(), + ignoreDefaultArgs: ["--enable-automation", "--hide-scrollbars"], + ignoreHTTPSErrors: true, + handleSIGHUP: signals, + handleSIGINT: signals, + handleSIGTERM: signals, + protocolTimeout: 0, + + defaultViewport, + waitForInitialPage: false, + userDataDir: this.profileDir + }; + + await this._init(launchOpts, ondisconnect); + } + + async setupPage({page} : {page: Page, cdp: CDPSession}) { await this.addInitScript(page, "Object.defineProperty(navigator, \"webdriver\", {value: false});"); if (this.customProfile) { @@ -73,7 +97,7 @@ export class BaseBrowser } } - async loadProfile(profileFilename) { + async loadProfile(profileFilename: string) : Promise { const targetFilename = "/tmp/profile.tar.gz"; if (profileFilename && @@ -83,16 +107,19 @@ export class BaseBrowser const resp = await fetch(profileFilename); await pipeline( - Readable.fromWeb(resp.body), + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + Readable.fromWeb(resp.body as any), fs.createWriteStream(targetFilename) ); profileFilename = targetFilename; } else if (profileFilename && profileFilename.startsWith("@")) { - const storage = initStorage(""); + const storage = initStorage(); if (!storage) { logger.fatal("Profile specified relative to s3 storage, but no S3 storage defined"); + return false; } await storage.downloadFile(profileFilename.slice(1), targetFilename); @@ -112,7 +139,7 @@ export class BaseBrowser return false; } - saveProfile(profileFilename) { + saveProfile(profileFilename: string) { child_process.execFileSync("tar", ["cvfz", profileFilename, "./"], {cwd: this.profileDir}); } @@ -142,11 +169,17 @@ export class BaseBrowser } getDefaultUA() { - let version = process.env.BROWSER_VERSION; + let version : string | undefined = process.env.BROWSER_VERSION; try { - version = child_process.execFileSync(this.getBrowserExe(), ["--version"], {encoding: "utf8"}); - version = version.match(/[\d.]+/)[0]; + const browser = this.getBrowserExe(); + if (browser) { + version = child_process.execFileSync(browser, ["--version"], {encoding: "utf8"}); + const match = version && version.match(/[\d.]+/); + if (match) { + version = match[0]; + } + } } catch(e) { console.error(e); } @@ -161,13 +194,13 @@ export class BaseBrowser return file; } } - - return null; } - async evaluateWithCLI_(cdp, frame, cdpContextId, funcString, logData, contextName) { + async evaluateWithCLI_(cdp: CDPSession, frame: Frame, cdpContextId: number, funcString: string, logData: Record, contextName: string) { const frameUrl = frame.url(); - let details = {frameUrl, ...logData}; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let details : Record = {frameUrl, ...logData}; if (!frameUrl || frame.isDetached()) { logger.info("Run Script Skipped, frame no longer attached or has no URL", details, contextName); @@ -201,18 +234,6 @@ export class BaseBrowser return result.value; } -} - - -// ================================================================== -export class Browser extends BaseBrowser -{ - constructor() { - super(); - this.browser = null; - - this.firstCDP = null; - } isLaunched() { if (this.browser) { @@ -231,11 +252,12 @@ export class Browser extends BaseBrowser } } - addInitScript(page, script) { + addInitScript(page: Page, script: string) { return page.evaluateOnNewDocument(script); } - async _init(launchOpts, ondisconnect = null) { + // eslint-disable-next-line @typescript-eslint/ban-types + async _init(launchOpts: PuppeteerLaunchOptions, ondisconnect : Function | null = null) { this.browser = await puppeteer.launch(launchOpts); const target = this.browser.target(); @@ -252,21 +274,29 @@ export class Browser extends BaseBrowser }); } - async newWindowPageWithCDP() { + async newWindowPageWithCDP() : Promise<{cdp: CDPSession, page: Page}> { // unique url to detect new pages const startPage = "about:blank?_browsertrix" + Math.random().toString(36).slice(2); - const p = new Promise((resolve) => { - const listener = (target) => { + const p = new Promise((resolve) => { + const listener = (target: Target) => { if (target.url() === startPage) { resolve(target); - this.browser.removeListener("targetcreated", listener); + if (this.browser) { + this.browser.removeListener("targetcreated", listener); + } } }; - this.browser.on("targetcreated", listener); + if (this.browser) { + this.browser.on("targetcreated", listener); + } }); + if (!this.firstCDP) { + throw new Error("CDP missing"); + } + try { await this.firstCDP.send("Target.createTarget", {url: startPage, newWindow: true}); } catch (e) { @@ -283,12 +313,17 @@ export class Browser extends BaseBrowser const target = await p; const page = await target.page(); + if (!page) { + throw new Error("page missing"); + } const device = this.emulateDevice; - if (device) { + if (device && page) { if (device.viewport && device.userAgent) { - await page.emulate(device); + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await page.emulate(device as any); } else if (device.userAgent) { await page.setUserAgent(device.userAgent); } @@ -300,9 +335,17 @@ export class Browser extends BaseBrowser } async serviceWorkerFetch() { + if (!this.firstCDP) { + return; + } + this.firstCDP.on("Fetch.requestPaused", async (params) => { const { frameId, requestId, networkId, request } = params; + if (!this.firstCDP) { + throw new Error("CDP missing"); + } + if (networkId) { try { await this.firstCDP.send("Fetch.continueResponse", {requestId}); @@ -343,30 +386,44 @@ export class Browser extends BaseBrowser await this.firstCDP.send("Fetch.enable", {patterns: [{urlPattern: "*", requestStage: "Response"}]}); } - async evaluateWithCLI(_, frame, cdp, funcString, logData, contextName) { - const context = await frame.executionContext(); + // TODO: Fix this the next time the file is edited. + + async evaluateWithCLI( + _: unknown, + frame: Frame, + cdp: CDPSession, + funcString: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + logData: Record, + contextName: string + ) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const context = await (frame as any).executionContext(); cdp = context._client; const cdpContextId = context._contextId; return await this.evaluateWithCLI_(cdp, frame, cdpContextId, funcString, logData, contextName); } - interceptRequest(page, callback) { + interceptRequest(page: Page, callback: (event: HTTPRequest) => void) { page.on("request", callback); } - async waitForNetworkIdle(page, params) { + async waitForNetworkIdle(page: Page, params: {timeout?: number}) { return await page.waitForNetworkIdle(params); } - async setViewport(page, params) { + async setViewport(page: Page, params: Viewport) { await page.setViewport(params); } - async getCookies(page) { + async getCookies(page: Page) { return await page.cookies(); } - async setCookies(page, cookies) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async setCookies(page: Page, cookies: any) { return await page.setCookie(...cookies); } } diff --git a/util/constants.js b/src/util/constants.ts similarity index 100% rename from util/constants.js rename to src/util/constants.ts diff --git a/util/file_reader.js b/src/util/file_reader.ts similarity index 72% rename from util/file_reader.js rename to src/util/file_reader.ts index e8038675..33a26b74 100644 --- a/util/file_reader.js +++ b/src/util/file_reader.ts @@ -3,7 +3,7 @@ import path from "path"; const MAX_DEPTH = 2; -export function collectAllFileSources(fileOrDir, ext = null, depth = 0) { +export function collectAllFileSources(fileOrDir: string, ext?: string, depth = 0) : string[] { const resolvedPath = path.resolve(fileOrDir); if (depth >= MAX_DEPTH) { @@ -13,14 +13,14 @@ export function collectAllFileSources(fileOrDir, ext = null, depth = 0) { const stat = fs.statSync(resolvedPath); - if (stat.isFile && (ext === null || path.extname(resolvedPath) === ext)) { + if (stat.isFile() && (ext === null || path.extname(resolvedPath) === ext)) { const contents = fs.readFileSync(resolvedPath); return [`/* src: ${resolvedPath} */\n\n${contents}`]; } - if (stat.isDirectory) { + if (stat.isDirectory()) { const files = fs.readdirSync(resolvedPath); - return files.reduce((acc, next) => { + return files.reduce((acc: string[], next: string) => { const nextPath = path.join(fileOrDir, next); return [...acc, ...collectAllFileSources(nextPath, ext, depth + 1)]; }, []); @@ -28,6 +28,7 @@ export function collectAllFileSources(fileOrDir, ext = null, depth = 0) { if (depth === 0) { console.warn(`WARN: The provided path "${resolvedPath}" is not a .js file or directory.`); - return []; } + + return []; } diff --git a/util/healthcheck.js b/src/util/healthcheck.ts similarity index 79% rename from util/healthcheck.js rename to src/util/healthcheck.ts index 36506ffc..65169499 100644 --- a/util/healthcheck.js +++ b/src/util/healthcheck.ts @@ -6,9 +6,14 @@ import { logger } from "./logger.js"; // =========================================================================== export class HealthChecker { - constructor(port, errorThreshold) { + port: number; + errorThreshold: number; + healthServer: http.Server; + + errorCount = 0; + + constructor(port: number, errorThreshold: number) { this.port = port; - this.errorCount = 0; this.errorThreshold = errorThreshold; this.healthServer = http.createServer((...args) => this.healthCheck(...args)); @@ -16,8 +21,8 @@ export class HealthChecker this.healthServer.listen(port); } - async healthCheck(req, res) { - const pathname = url.parse(req.url).pathname; + async healthCheck(req: http.IncomingMessage, res: http.ServerResponse) { + const pathname = req.url ? url.parse(req.url).pathname : ""; switch (pathname) { case "/healthz": if (this.errorCount < this.errorThreshold) { diff --git a/util/logger.js b/src/util/logger.ts similarity index 53% rename from util/logger.js rename to src/util/logger.ts index 70e29998..45824544 100644 --- a/util/logger.js +++ b/src/util/logger.ts @@ -1,57 +1,73 @@ // =========================================================================== // to fix serialization of regexes for logging purposes -RegExp.prototype.toJSON = RegExp.prototype.toString; + +import { Writable } from "node:stream"; +import { RedisCrawlState } from "./state.js"; + +// RegExp.prototype.toJSON = RegExp.prototype.toString; +Object.defineProperty(RegExp.prototype, "toJSON", { value: RegExp.prototype.toString }); // =========================================================================== -export function errJSON(e) { - return {"type": "exception", "message": e.message, "stack": e.stack}; +// TODO: Fix this the next time the file is edited. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function errJSON(e: any) { + if (e instanceof Error) { + return {"type": "exception", "message": e.message, "stack": e.stack}; + } else { + return {"message": e.toString()}; + } } // =========================================================================== class Logger { - constructor() { - this.logStream = null; - this.debugLogging = null; - this.logErrorsToRedis = false; - this.logLevels = []; - this.contexts = []; - this.crawlState = null; + logStream : Writable | null = null; + debugLogging = false; + logErrorsToRedis = false; + logLevels : string[] = []; + contexts : string[] = []; + crawlState? : RedisCrawlState | null = null; + fatalExitCode = 17; - this.fatalExitCode = 17; - } - - setDefaultFatalExitCode(exitCode) { + setDefaultFatalExitCode(exitCode: number) { this.fatalExitCode = exitCode; } - setExternalLogStream(logFH) { + setExternalLogStream(logFH: Writable | null) { this.logStream = logFH; } - setDebugLogging(debugLog) { + setDebugLogging(debugLog: boolean) { this.debugLogging = debugLog; } - setLogErrorsToRedis(logErrorsToRedis) { + setLogErrorsToRedis(logErrorsToRedis: boolean) { this.logErrorsToRedis = logErrorsToRedis; } - setLogLevel(logLevels) { + setLogLevel(logLevels: string[]) { this.logLevels = logLevels; } - setContext(contexts) { + setContext(contexts: string[]) { this.contexts = contexts; } - setCrawlState(crawlState) { + setCrawlState(crawlState: RedisCrawlState) { this.crawlState = crawlState; } - logAsJSON(message, data, context, logLevel="info") { + // TODO: Fix this the next time the file is edited. + + logAsJSON( + message: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + data: Record | Error | any, + context: string, + logLevel="info" + ) { if (data instanceof Error) { data = errJSON(data); } else if (typeof data !== "object") { @@ -70,7 +86,7 @@ class Logger } } - let dataToLog = { + const dataToLog = { "timestamp": new Date().toISOString(), "logLevel": logLevel, "context": context, @@ -84,30 +100,30 @@ class Logger } const toLogToRedis = ["error", "fatal"]; - if (this.logErrorsToRedis && toLogToRedis.includes(logLevel)) { + if (this.logErrorsToRedis && this.crawlState && toLogToRedis.includes(logLevel)) { this.crawlState.logError(string); } } - info(message, data={}, context="general") { + info(message: string, data={}, context="general") { this.logAsJSON(message, data, context); } - error(message, data={}, context="general") { + error(message: string, data={}, context="general") { this.logAsJSON(message, data, context, "error"); } - warn(message, data={}, context="general") { + warn(message: string, data={}, context="general") { this.logAsJSON(message, data, context, "warn"); } - debug(message, data={}, context="general") { + debug(message: string, data={}, context="general") { if (this.debugLogging) { this.logAsJSON(message, data, context, "debug"); } } - fatal(message, data={}, context="general", exitCode=0) { + fatal(message: string, data={}, context="general", exitCode=0) { exitCode = exitCode || this.fatalExitCode; this.logAsJSON(`${message}. Quitting`, data, context, "fatal"); diff --git a/util/originoverride.js b/src/util/originoverride.ts similarity index 80% rename from util/originoverride.js rename to src/util/originoverride.ts index e2190330..7188405e 100644 --- a/util/originoverride.js +++ b/src/util/originoverride.ts @@ -1,10 +1,14 @@ +import { HTTPRequest, Page } from "puppeteer-core"; import { errJSON, logger } from "./logger.js"; +import { Browser } from "./browser.js"; export class OriginOverride { - constructor(originOverride) { + originOverride: {origUrl: URL, destUrl: URL}[]; + + constructor(originOverride: string[]) { this.originOverride = originOverride.map((override) => { - let [orig, dest] = override.split("="); + const [orig, dest] = override.split("="); const origUrl = new URL(orig); const destUrl = new URL(dest); @@ -12,8 +16,8 @@ export class OriginOverride }); } - async initPage(browser, page) { - const onRequest = async (request) => { + async initPage(browser: Browser, page: Page) { + const onRequest = async (request: HTTPRequest) => { try { const url = request.url(); @@ -28,12 +32,13 @@ export class OriginOverride } } - if (!newUrl) { + if (!newUrl || !orig) { request.continue({}, -1); return; } const headers = new Headers(request.headers()); + headers.set("host", orig.host); if (headers.get("origin")) { headers.set("origin", orig.origin); diff --git a/util/recorder.js b/src/util/recorder.ts similarity index 80% rename from util/recorder.js rename to src/util/recorder.ts index eac522ef..88d7cffd 100644 --- a/util/recorder.js +++ b/src/util/recorder.ts @@ -10,12 +10,16 @@ import { logger, errJSON } from "./logger.js"; import { sleep, timestampNow } from "./timing.js"; import { RequestResponseInfo } from "./reqresp.js"; +// @ts-expect-error TODO fill in why error is expected import { baseRules as baseDSRules } from "@webrecorder/wabac/src/rewrite/index.js"; +// @ts-expect-error TODO fill in why error is expected import { rewriteDASH, rewriteHLS } from "@webrecorder/wabac/src/rewrite/rewriteVideo.js"; import { WARCRecord } from "warcio"; -import { WARCSerializer } from "warcio/node"; +import { TempFileBuffer, WARCSerializer } from "warcio/node"; import { WARCWriter } from "./warcwriter.js"; +import { RedisCrawlState, WorkerId } from "./state.js"; +import { CDPSession, Protocol } from "puppeteer-core"; const MAX_BROWSER_FETCH_SIZE = 2_000_000; const MAX_NETWORK_LOAD_SIZE = 200_000_000; @@ -26,15 +30,58 @@ const WRITE_DUPE_KEY = "s:writedupe"; const encoder = new TextEncoder(); + // ================================================================= -function logNetwork(/*msg, data*/) { +// TODO: Fix this the next time the file is edited. +// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unused-vars +function logNetwork(msg: string, data: any) { // logger.debug(msg, data, "recorderNetwork"); } // ================================================================= export class Recorder { - constructor({workerid, collDir, crawler}) { + workerid: WorkerId; + collDir: string; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + crawler: any; + + crawlState: RedisCrawlState; + + warcQ: PQueue; + fetcherQ: PQueue; + + pendingRequests!: Map; + skipIds!: Set; + + swSessionId?: string | null; + swFrameIds = new Set(); + swUrls = new Set(); + + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + logDetails: Record = {}; + skipping = false; + + allowFull206 = false; + + archivesDir: string; + tempdir: string; + tempCdxDir: string; + + gzip = true; + + writer: WARCWriter; + + pageid!: string; + + // TODO: Fix this the next time the file is edited. + + constructor( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + {workerid, collDir, crawler} : {workerid: WorkerId, collDir: string, crawler: any} + ) { this.workerid = workerid; this.crawler = crawler; this.crawlState = crawler.crawlState; @@ -43,19 +90,6 @@ export class Recorder this.fetcherQ = new PQueue({concurrency: 1}); - this.pendingRequests = null; - this.skipIds = null; - - this.swSessionId = null; - this.swFrameIds = new Set(); - this.swUrls = new Set(); - - - this.logDetails = {}; - this.skipping = false; - - this.allowFull206 = true; - this.collDir = collDir; this.archivesDir = path.join(this.collDir, "archive"); @@ -68,7 +102,6 @@ export class Recorder const crawlId = process.env.CRAWL_ID || os.hostname(); const filename = `rec-${crawlId}-${timestampNow()}-${this.workerid}.warc`; - this.gzip = true; this.writer = new WARCWriter({ archivesDir: this.archivesDir, @@ -79,7 +112,7 @@ export class Recorder }); } - async onCreatePage({cdp}) { + async onCreatePage({cdp} : {cdp: CDPSession}) { // Fetch cdp.on("Fetch.requestPaused", async (params) => { @@ -159,7 +192,7 @@ export class Recorder await cdp.send("Target.setAutoAttach", {autoAttach: true, waitForDebuggerOnStart: false, flatten: true}); } - handleResponseReceived(params) { + handleResponseReceived(params: Protocol.Network.ResponseReceivedEvent) { const { requestId, response } = params; const reqresp = this.pendingReqResp(requestId); @@ -170,7 +203,7 @@ export class Recorder reqresp.fillResponse(response); } - handleRequestExtraInfo(params) { + handleRequestExtraInfo(params: Protocol.Network.RequestWillBeSentExtraInfoEvent) { if (!this.shouldSkip(params.headers)) { const reqresp = this.pendingReqResp(params.requestId, true); if (reqresp) { @@ -179,13 +212,13 @@ export class Recorder } } - handleRedirectResponse(params) { + handleRedirectResponse(params: Protocol.Network.RequestWillBeSentEvent) { const { requestId, redirectResponse } = params; // remove and serialize, but allow reusing requestId // as redirect chain may reuse same requestId for subsequent request const reqresp = this.removeReqResp(requestId, true); - if (!reqresp) { + if (!reqresp || !redirectResponse) { return; } @@ -199,7 +232,7 @@ export class Recorder this.serializeToWARC(reqresp); } - handleLoadingFailed(params) { + handleLoadingFailed(params: Protocol.Network.LoadingFailedEvent) { const { errorText, type, requestId } = params; const reqresp = this.pendingReqResp(requestId, true); @@ -211,13 +244,13 @@ export class Recorder switch (errorText) { case "net::ERR_BLOCKED_BY_CLIENT": - logNetwork("Request blocked", {url, errorText, ...this.logDetails}, "recorder"); + logNetwork("Request blocked", {url, errorText, ...this.logDetails}); break; case "net::ERR_ABORTED": // check if this is a false positive -- a valid download that's already been fetched // the abort is just for page, but download will succeed - if (url && type === "Document" && reqresp.isValidBinary()) { + if (type === "Document" && reqresp.isValidBinary()) { this.serializeToWARC(reqresp); //} else if (url) { } else if (url && reqresp.requestHeaders && reqresp.requestHeaders["x-browsertrix-fetch"]) { @@ -235,7 +268,7 @@ export class Recorder this.removeReqResp(requestId); } - handleLoadingFinished(params) { + handleLoadingFinished(params: Protocol.Network.LoadingFinishedEvent) { const reqresp = this.pendingReqResp(params.requestId, true); if (!reqresp || reqresp.asyncLoading) { @@ -251,7 +284,7 @@ export class Recorder this.serializeToWARC(reqresp); } - async handleRequestPaused(params, cdp, isSWorker = false) { + async handleRequestPaused(params: Protocol.Fetch.RequestPausedEvent, cdp: CDPSession, isSWorker = false) { const { requestId, request, responseStatusCode, responseErrorReason, resourceType, networkId } = params; const { method, headers, url } = request; @@ -276,7 +309,7 @@ export class Recorder } } - async handleFetchResponse(params, cdp, isSWorker) { + async handleFetchResponse(params: Protocol.Fetch.RequestPausedEvent, cdp: CDPSession, isSWorker: boolean) { const { request } = params; const { url } = request; const {requestId, responseErrorReason, responseStatusCode, responseHeaders} = params; @@ -341,7 +374,7 @@ export class Recorder // if not consumed via takeStream, attempt async loading if (!streamingConsume) { - let fetcher = null; + let fetcher : AsyncFetcher; if (reqresp.method !== "GET" || contentLen > MAX_NETWORK_LOAD_SIZE) { fetcher = new AsyncFetcher(opts); @@ -388,12 +421,12 @@ export class Recorder try { await cdp.send("Fetch.fulfillRequest", { requestId, - responseCode: responseStatusCode, + responseCode: responseStatusCode || 0, responseHeaders, body }); } catch (e) { - const type = reqresp.type; + const type = reqresp.resourceType; if (type === "Document") { logger.debug("document not loaded in browser, possibly other URLs missing", {url, type: reqresp.resourceType}, "recorder"); } else { @@ -404,7 +437,7 @@ export class Recorder return true; } - startPage({pageid, url}) { + startPage({pageid, url} : {pageid: string, url: string}) { this.pageid = pageid; this.logDetails = {page: url, workerid: this.workerid}; if (this.pendingRequests && this.pendingRequests.size) { @@ -431,8 +464,8 @@ export class Recorder while (numPending && !this.crawler.interrupted) { const pending = []; for (const [requestId, reqresp] of this.pendingRequests.entries()) { - const url = reqresp.url; - const entry = {requestId, url}; + const url = reqresp.url || ""; + const entry : {requestId: string, url: string, expectedSize?: number, readSize?: number} = {requestId, url}; if (reqresp.expectedSize) { entry.expectedSize = reqresp.expectedSize; } @@ -464,7 +497,7 @@ export class Recorder await this.writer.flush(); } - shouldSkip(headers, url, method, resourceType) { + shouldSkip(headers: Protocol.Network.Headers, url?: string, method?: string, resourceType?: string) { if (headers && !method) { method = headers[":method"]; } @@ -477,7 +510,7 @@ export class Recorder return true; } - if (["EventSource", "WebSocket", "Ping"].includes(resourceType)) { + if (["EventSource", "WebSocket", "Ping"].includes(resourceType || "")) { return true; } @@ -494,7 +527,7 @@ export class Recorder return false; } - async rewriteResponse(reqresp) { + async rewriteResponse(reqresp: RequestResponseInfo) { const { url, responseHeadersList, extraOpts, payload } = reqresp; if (!payload || !payload.length) { @@ -509,12 +542,12 @@ export class Recorder switch (ct) { case "application/x-mpegURL": case "application/vnd.apple.mpegurl": - string = payload.toString("utf-8"); + string = payload.toString(); newString = rewriteHLS(string, {save: extraOpts}); break; case "application/dash+xml": - string = payload.toString("utf-8"); + string = payload.toString(); newString = rewriteDASH(string, {save: extraOpts}); break; @@ -526,7 +559,7 @@ export class Recorder const rw = baseDSRules.getRewriter(url); if (rw !== baseDSRules.defaultRewriter) { - string = payload.toString("utf-8"); + string = payload.toString(); newString = rw.rewrite(string, {live: true, save: extraOpts}); } break; @@ -549,8 +582,11 @@ export class Recorder //return Buffer.from(newString).toString("base64"); } - _getContentType(headers) { - for (let header of headers) { + _getContentType(headers? : Protocol.Fetch.HeaderEntry[] | {name: string, value: string}[]) { + if (!headers) { + return null; + } + for (const header of headers) { if (header.name.toLowerCase() === "content-type") { return header.value.split(";")[0]; } @@ -559,8 +595,11 @@ export class Recorder return null; } - _getContentLen(headers) { - for (let header of headers) { + _getContentLen(headers? : Protocol.Fetch.HeaderEntry[]) { + if (!headers) { + return -1; + } + for (const header of headers) { if (header.name.toLowerCase() === "content-length") { return Number(header.value); } @@ -569,8 +608,11 @@ export class Recorder return -1; } - _getContentRange(headers) { - for (let header of headers) { + _getContentRange(headers? : Protocol.Fetch.HeaderEntry[]) { + if (!headers) { + return null; + } + for (const header of headers) { if (header.name.toLowerCase() === "content-range") { return header.value; } @@ -579,15 +621,15 @@ export class Recorder return null; } - noResponseForStatus(status) { + noResponseForStatus(status: number | undefined | null) { return (!status || status === 204 || (status >= 300 && status < 400)); } - isValidUrl(url) { + isValidUrl(url?: string) { return url && (url.startsWith("https:") || url.startsWith("http:")); } - pendingReqResp(requestId, reuseOnly = false) { + pendingReqResp(requestId: string, reuseOnly = false) { if (!this.pendingRequests.has(requestId)) { if (reuseOnly || !requestId) { return null; @@ -605,14 +647,14 @@ export class Recorder return reqresp; } else { const reqresp = this.pendingRequests.get(requestId); - if (requestId !== reqresp.requestId) { + if (reqresp && requestId !== reqresp.requestId) { logger.warn("Invalid request id", {requestId, actualRequestId: reqresp.requestId}, "recorder"); } return reqresp; } } - removeReqResp(requestId, allowReuse=false) { + removeReqResp(requestId: string, allowReuse=false) { const reqresp = this.pendingRequests.get(requestId); this.pendingRequests.delete(requestId); if (!allowReuse) { @@ -621,13 +663,13 @@ export class Recorder return reqresp; } - async serializeToWARC(reqresp) { + async serializeToWARC(reqresp: RequestResponseInfo) { if (!reqresp.payload) { logNetwork("Not writing, no payload", {url: reqresp.url}); return; } - if (reqresp.method === "GET" && !await this.crawlState.addIfNoDupe(WRITE_DUPE_KEY, reqresp.url)) { + if (reqresp.url && reqresp.method === "GET" && !(await this.crawlState.addIfNoDupe(WRITE_DUPE_KEY, reqresp.url))) { logNetwork("Skipping dupe", {url: reqresp.url}); return; } @@ -638,21 +680,21 @@ export class Recorder this.warcQ.add(() => this.writer.writeRecordPair(responseRecord, requestRecord)); } - async directFetchCapture(url) { - const reqresp = new RequestResponseInfo(0); + async directFetchCapture(url: string) : Promise<{fetched: boolean, mime: string}>{ + const reqresp = new RequestResponseInfo("0"); reqresp.url = url; reqresp.method = "GET"; logger.debug("Directly fetching page URL without browser", {url, ...this.logDetails}, "recorder"); - const filter = (resp) => resp.status === 200 && !resp.headers.get("set-cookie"); + const filter = (resp: Response) => resp.status === 200 && !resp.headers.get("set-cookie"); // ignore dupes: if previous URL was not a page, still load as page. if previous was page, // should not get here, as dupe pages tracked via seen list - const fetcher = new AsyncFetcher({tempdir: this.tempdir, reqresp, recorder: this, networkId: 0, filter, ignoreDupe: true}); + const fetcher = new AsyncFetcher({tempdir: this.tempdir, reqresp, recorder: this, networkId: "0", filter, ignoreDupe: true}); const res = await fetcher.load(); - const mime = reqresp && reqresp.responseHeaders["content-type"] && reqresp.responseHeaders["content-type"].split(";")[0]; + const mime = reqresp && reqresp.responseHeaders && reqresp.responseHeaders["content-type"] && reqresp.responseHeaders["content-type"].split(";")[0] || ""; return {fetched: res === "fetched", mime}; } @@ -661,7 +703,20 @@ export class Recorder // ================================================================= class AsyncFetcher { - constructor({tempdir, reqresp, expectedSize = -1, recorder, networkId, filter = null, ignoreDupe = false}) { + reqresp: RequestResponseInfo; + + networkId: string; + filter?: (resp: Response) => boolean; + ignoreDupe = false; + + recorder: Recorder; + + tempdir: string; + filename: string; + + constructor({tempdir, reqresp, expectedSize = -1, recorder, networkId, filter = undefined, ignoreDupe = false} : + {tempdir: string, reqresp: RequestResponseInfo, expectedSize?: number, recorder: Recorder, + networkId: string, filter?: (resp: Response) => boolean, ignoreDupe?: boolean }) { this.reqresp = reqresp; this.reqresp.expectedSize = expectedSize; this.reqresp.asyncLoading = true; @@ -685,7 +740,7 @@ class AsyncFetcher let fetched = "notfetched"; try { - if (reqresp.method === "GET" && !await crawlState.addIfNoDupe(ASYNC_FETCH_DUPE_KEY, url)) { + if (reqresp.method === "GET" && url && !(await crawlState.addIfNoDupe(ASYNC_FETCH_DUPE_KEY, url))) { if (!this.ignoreDupe) { this.reqresp.asyncLoading = false; return "dupe"; @@ -719,7 +774,7 @@ class AsyncFetcher //return fetched; } - const externalBuffer = serializer.externalBuffer; + const externalBuffer : TempFileBuffer = serializer.externalBuffer as TempFileBuffer; if (externalBuffer) { const { currSize, buffers, fh } = externalBuffer; @@ -731,14 +786,14 @@ class AsyncFetcher } if (Object.keys(reqresp.extraOpts).length) { - responseRecord.warcHeaders["WARC-JSON-Metadata"] = JSON.stringify(reqresp.extraOpts); + responseRecord.warcHeaders.headers.set("WARC-JSON-Metadata", JSON.stringify(reqresp.extraOpts)); } recorder.warcQ.add(() => recorder.writer.writeRecordPair(responseRecord, requestRecord, serializer)); } catch (e) { logger.error("Streaming Fetch Error", {url, networkId, filename, ...errJSON(e), ...logDetails}, "recorder"); - await crawlState.removeDupe(ASYNC_FETCH_DUPE_KEY, url); + await crawlState.removeDupe(ASYNC_FETCH_DUPE_KEY, url!); } finally { recorder.removeReqResp(networkId); } @@ -761,9 +816,9 @@ class AsyncFetcher signal = abort.signal; } - const resp = await fetch(url, {method, headers, body: reqresp.postData || undefined, signal}); + const resp = await fetch(url!, {method, headers, body: reqresp.postData || undefined, signal}); - if (this.filter && !this.filter(resp)) { + if (this.filter && !this.filter(resp) && abort) { abort.abort(); throw new Error("invalid response, ignoring fetch"); } @@ -778,7 +833,7 @@ class AsyncFetcher } else if (!resp.body) { logger.error("Empty body, stopping fetch", {url}, "recorder"); - await this.recorder.crawlState.removeDupe(ASYNC_FETCH_DUPE_KEY, url); + await this.recorder.crawlState.removeDupe(ASYNC_FETCH_DUPE_KEY, url!); return; } @@ -787,7 +842,7 @@ class AsyncFetcher return this.takeReader(resp.body.getReader()); } - async* takeReader(reader) { + async* takeReader(reader: ReadableStreamDefaultReader) { try { while (true) { const { value, done } = await reader.read(); @@ -803,7 +858,7 @@ class AsyncFetcher } } - async* takeStreamIter(cdp, stream) { + async* takeStreamIter(cdp: CDPSession, stream: Protocol.IO.StreamHandle) { try { while (true) { const {data, base64Encoded, eof} = await cdp.send("IO.read", {handle: stream}); @@ -825,7 +880,12 @@ class AsyncFetcher // ================================================================= class ResponseStreamAsyncFetcher extends AsyncFetcher { - constructor(opts) { + cdp: CDPSession; + requestId: string; + + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(opts: any) { super(opts); this.cdp = opts.cdp; this.requestId = opts.requestId; @@ -845,7 +905,11 @@ class ResponseStreamAsyncFetcher extends AsyncFetcher // ================================================================= class NetworkLoadStreamAsyncFetcher extends AsyncFetcher { - constructor(opts) { + cdp: CDPSession; + + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(opts: any) { super(opts); this.cdp = opts.cdp; } @@ -883,7 +947,7 @@ class NetworkLoadStreamAsyncFetcher extends AsyncFetcher return; } - reqresp.status = httpStatusCode; + reqresp.status = httpStatusCode || 0; reqresp.responseHeaders = headers || {}; return this.takeStreamIter(cdp, stream); @@ -892,15 +956,15 @@ class NetworkLoadStreamAsyncFetcher extends AsyncFetcher // ================================================================= // response -function createResponse(reqresp, pageid, contentIter) { +function createResponse(reqresp: RequestResponseInfo, pageid: string, contentIter?: AsyncIterable | Iterable) { const url = reqresp.url; const warcVersion = "WARC/1.1"; const statusline = `HTTP/1.1 ${reqresp.status} ${reqresp.statusText}`; const date = new Date().toISOString(); - const httpHeaders = reqresp.getResponseHeadersDict(reqresp.payload ? reqresp.payload.length : null); + const httpHeaders = reqresp.getResponseHeadersDict(reqresp.payload ? reqresp.payload.length : 0); - const warcHeaders = { + const warcHeaders : Record = { "WARC-Page-ID": pageid, }; @@ -909,7 +973,7 @@ function createResponse(reqresp, pageid, contentIter) { } if (!contentIter) { - contentIter = [reqresp.payload]; + contentIter = [reqresp.payload] as Iterable; } if (Object.keys(reqresp.extraOpts).length) { @@ -923,7 +987,7 @@ function createResponse(reqresp, pageid, contentIter) { // ================================================================= // request -function createRequest(reqresp, responseRecord, pageid) { +function createRequest(reqresp: RequestResponseInfo, responseRecord: WARCRecord, pageid: string) { const url = reqresp.url; const warcVersion = "WARC/1.1"; const method = reqresp.method; @@ -936,12 +1000,12 @@ function createRequest(reqresp, responseRecord, pageid) { const httpHeaders = reqresp.getRequestHeadersDict(); - const warcHeaders = { - "WARC-Concurrent-To": responseRecord.warcHeader("WARC-Record-ID"), + const warcHeaders : Record = { + "WARC-Concurrent-To": responseRecord.warcHeader("WARC-Record-ID")!, "WARC-Page-ID": pageid, }; - const date = responseRecord.warcDate; + const date = responseRecord.warcDate || undefined; return WARCRecord.create({ url, date, warcVersion, type: "request", warcHeaders, diff --git a/util/redis.js b/src/util/redis.ts similarity index 88% rename from util/redis.js rename to src/util/redis.ts index 4bd839b3..3e99f723 100644 --- a/util/redis.js +++ b/src/util/redis.ts @@ -1,4 +1,4 @@ -import Redis from "ioredis"; +import { Redis } from "ioredis"; import { logger } from "./logger.js"; const error = console.error; @@ -15,7 +15,7 @@ console.error = function (...args) { args[0].indexOf("[ioredis] Unhandled error event") === 0 ) { - let now = Date.now(); + const now = Date.now(); if ((now - lastLogTime) > REDIS_ERROR_LOG_INTERVAL_SECS) { if (lastLogTime && exitOnError) { @@ -29,7 +29,7 @@ console.error = function (...args) { error.call(console, ...args); }; -export async function initRedis(url) { +export async function initRedis(url: string) { const redis = new Redis(url, {lazyConnect: true}); await redis.connect(); return redis; diff --git a/util/reqresp.js b/src/util/reqresp.ts similarity index 63% rename from util/reqresp.js rename to src/util/reqresp.ts index d5c86020..9d95f94d 100644 --- a/util/reqresp.js +++ b/src/util/reqresp.ts @@ -1,5 +1,8 @@ +// @ts-expect-error TODO fill in why error is expected import { getStatusText } from "@webrecorder/wabac/src/utils.js"; +import { Protocol } from "puppeteer-core"; + const CONTENT_LENGTH = "content-length"; const CONTENT_TYPE = "content-type"; const EXCLUDE_HEADERS = ["content-encoding", "transfer-encoding"]; @@ -8,53 +11,63 @@ const EXCLUDE_HEADERS = ["content-encoding", "transfer-encoding"]; // =========================================================================== export class RequestResponseInfo { - constructor(requestId) { - this._created = new Date(); - + _created: Date = new Date(); + + requestId: string; + + ts?: string; + + method?: string; + url!: string; + protocol?: string = "HTTP/1.1"; + + // request data + requestHeaders?: Record; + requestHeadersText?: string; + + postData?: string; + hasPostData: boolean = false; + + // response data + status: number = 0; + statusText?: string; + + responseHeaders?: Record; + responseHeadersList?: {name: string, value: string}[]; + responseHeadersText?: string; + + payload?: Uint8Array; + + // misc + fromServiceWorker: boolean = false; + + frameId?: string; + + fetch: boolean = false; + + resourceType?: string; + + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + extraOpts: Record = {}; + + // stats + readSize: number = 0; + expectedSize: number = 0; + + // set to true to indicate async loading in progress + asyncLoading: boolean = false; + + // set to add truncated message + truncated?: string; + + constructor(requestId: string) { this.requestId = requestId; - - this.ts = null; - - // request data - this.method = null; - this.url = null; - this.protocol = "HTTP/1.1"; - - this.requestHeaders = null; - this.requestHeadersText = null; - - this.postData = null; - this.hasPostData = false; - - // response data - this.status = 0; - this.statusText = null; - - this.responseHeaders = null; - this.responseHeadersList = null; - this.responseHeadersText = null; - - this.payload = null; - - this.fromServiceWorker = false; - - this.fetch = false; - - this.resourceType = null; - - this.extraOpts = {}; - - this.readSize = 0; - this.expectedSize = 0; - - // set to true to indicate async loading in progress - this.asyncLoading = false; - - // set to add truncated message - this.truncated = null; } - fillRequest(params) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fillRequest(params: Record) { this.url = params.request.url; this.method = params.request.method; if (!this.requestHeaders) { @@ -69,7 +82,9 @@ export class RequestResponseInfo } - fillFetchRequestPaused(params) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fillFetchRequestPaused(params: Record) { this.fillRequest(params); this.status = params.responseStatusCode; @@ -83,7 +98,7 @@ export class RequestResponseInfo this.frameId = params.frameId; } - fillResponse(response) { + fillResponse(response: Protocol.Network.Response) { // if initial fetch was a 200, but now replacing with 304, don't! if (response.status == 304 && this.status && this.status != 304 && this.url) { return; @@ -112,8 +127,8 @@ export class RequestResponseInfo this.fromServiceWorker = !!response.fromServiceWorker; if (response.securityDetails) { - const issuer = response.securityDetails.issuer || ""; - const ctc = response.securityDetails.certificateTransparencyCompliance === "compliant" ? "1" : "0"; + const issuer : string = response.securityDetails.issuer || ""; + const ctc : string = response.securityDetails.certificateTransparencyCompliance === "compliant" ? "1" : "0"; this.extraOpts.cert = {issuer, ctc}; } } @@ -124,14 +139,15 @@ export class RequestResponseInfo } try { const headers = new Headers(this.responseHeaders); - const redirUrl = new URL(headers.get("location"), this.url).href; + const location = headers.get("location") || ""; + const redirUrl = new URL(location, this.url).href; return this.url === redirUrl; } catch (e) { return false; } } - fillResponseReceivedExtraInfo(params) { + fillResponseReceivedExtraInfo(params: Record) { // this.responseHeaders = params.headers; // if (params.headersText) { // this.responseHeadersText = params.headersText; @@ -139,22 +155,28 @@ export class RequestResponseInfo this.extraOpts.ipType = params.resourceIPAddressSpace; } - fillFetchResponse(response) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fillFetchResponse(response: Record) { this.responseHeaders = Object.fromEntries(response.headers); this.status = response.status; this.statusText = response.statusText || getStatusText(this.status); } - fillRequestExtraInfo(params) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + fillRequestExtraInfo(params: Record) { this.requestHeaders = params.headers; } getResponseHeadersText() { let headers = `${this.protocol} ${this.status} ${this.statusText}\r\n`; - for (const header of Object.keys(this.responseHeaders)) { - headers += `${header}: ${this.responseHeaders[header].replace(/\n/g, ", ")}\r\n`; + if (this.responseHeaders) { + for (const header of Object.keys(this.responseHeaders)) { + headers += `${header}: ${this.responseHeaders[header].replace(/\n/g, ", ")}\r\n`; + } } headers += "\r\n"; return headers; @@ -165,14 +187,14 @@ export class RequestResponseInfo } getRequestHeadersDict() { - return this._getHeadersDict(this.requestHeaders, null); + return this._getHeadersDict(this.requestHeaders); } - getResponseHeadersDict(length) { + getResponseHeadersDict(length = 0) { return this._getHeadersDict(this.responseHeaders, this.responseHeadersList, length); } - _getHeadersDict(headersDict, headersList, actualContentLength) { + _getHeadersDict(headersDict?: Record, headersList?: {name: string, value: string}[], actualContentLength = 0) { if (!headersDict && headersList) { headersDict = {}; diff --git a/util/screencaster.js b/src/util/screencaster.ts similarity index 63% rename from util/screencaster.js rename to src/util/screencaster.ts index 63a43246..69c111b3 100644 --- a/util/screencaster.js +++ b/src/util/screencaster.ts @@ -1,29 +1,39 @@ -import ws from "ws"; -import http from "http"; +import ws, { WebSocket } from "ws"; +import http, { IncomingMessage, ServerResponse } from "http"; import url from "url"; import fs from "fs"; import { initRedis } from "./redis.js"; import { logger } from "./logger.js"; +import { Duplex } from "stream"; +import { CDPSession, Page } from "puppeteer-core"; +import { WorkerId } from "./state.js"; -const indexHTML = fs.readFileSync(new URL("../html/screencast.html", import.meta.url), {encoding: "utf8"}); +const indexHTML = fs.readFileSync(new URL("../../html/screencast.html", import.meta.url), {encoding: "utf8"}); // =========================================================================== class WSTransport { - constructor(port) { - this.allWS = new Set(); + allWS = new Set(); + // eslint-disable-next-line no-use-before-define + caster!: ScreenCaster; + wss: ws.Server; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + httpServer: any; + - this.caster = null; + constructor(port: number) { + this.allWS = new Set(); this.wss = new ws.Server({ noServer: true }); - this.wss.on("connection", (ws) => this.initWebSocket(ws)); + this.wss.on("connection", (ws: WebSocket) => this.initWebSocket(ws)); this.httpServer = http.createServer((...args) => this.handleRequest(...args)); - this.httpServer.on("upgrade", (request, socket, head) => { - const pathname = url.parse(request.url).pathname; + this.httpServer.on("upgrade", (request: IncomingMessage, socket: Duplex, head: Buffer) => { + const pathname = url.parse(request.url || "").pathname; if (pathname === "/ws") { this.wss.handleUpgrade(request, socket, head, (ws) => { @@ -35,8 +45,8 @@ class WSTransport this.httpServer.listen(port); } - async handleRequest(req, res) { - const pathname = url.parse(req.url).pathname; + async handleRequest(req: IncomingMessage, res: ServerResponse) { + const pathname = url.parse(req.url || "").pathname; switch (pathname) { case "/": res.writeHead(200, {"Content-Type": "text/html"}); @@ -48,7 +58,7 @@ class WSTransport res.end("Not Found"); } - initWebSocket(ws) { + initWebSocket(ws: WebSocket) { for (const packet of this.caster.iterCachedData()) { ws.send(JSON.stringify(packet)); } @@ -71,10 +81,12 @@ class WSTransport }); } - sendAll(packet) { - packet = JSON.stringify(packet); + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + sendAll(packet: Record) { + const packetStr = JSON.stringify(packet); for (const ws of this.allWS) { - ws.send(packet); + ws.send(packetStr); } } @@ -87,22 +99,30 @@ class WSTransport // =========================================================================== class RedisPubSubTransport { - constructor(redisUrl, crawlId) { - this.numConnections = 0; + numConnections: number = 0; + castChannel: string; + // eslint-disable-next-line no-use-before-define + caster!: ScreenCaster; + ctrlChannel: string; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + redis: any; + + constructor(redisUrl: string, crawlId: string) { this.castChannel = `c:${crawlId}:cast`; this.ctrlChannel = `c:${crawlId}:ctrl`; this.init(redisUrl); } - async init(redisUrl) { + async init(redisUrl: string) { this.redis = await initRedis(redisUrl); const subRedis = await initRedis(redisUrl); await subRedis.subscribe(this.ctrlChannel); - subRedis.on("message", async (channel, message) => { + subRedis.on("message", async (channel: string, message: string) => { if (channel !== this.ctrlChannel) { return; } @@ -129,7 +149,9 @@ class RedisPubSubTransport }); } - async sendAll(packet) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async sendAll(packet: Record) { await this.redis.publish(this.castChannel, JSON.stringify(packet)); } @@ -143,19 +165,20 @@ class RedisPubSubTransport // =========================================================================== class ScreenCaster { - constructor(transport, numWorkers) { + transport: WSTransport; + caches = new Map(); + urls = new Map(); + cdps = new Map(); + maxWidth = 640; + maxHeight = 480; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + initMsg: {[key: string]: any}; + + constructor(transport: WSTransport, numWorkers: number) { this.transport = transport; this.transport.caster = this; - this.caches = new Map(); - this.urls = new Map(); - - this.cdps = new Map(); - - // todo: make customizable - this.maxWidth = 640; - this.maxHeight = 480; - this.initMsg = { msg: "init", width: this.maxWidth, @@ -174,7 +197,7 @@ class ScreenCaster } } - async screencastPage(page, cdp, id) { + async screencastPage(page: Page, cdp: CDPSession, id: WorkerId) { this.urls.set(id, page.url()); // shouldn't happen, getting duplicate cdp @@ -220,7 +243,7 @@ class ScreenCaster } } - async stopById(id, sendClose=false) { + async stopById(id: WorkerId, sendClose=false) { this.caches.delete(id); this.urls.delete(id); @@ -241,24 +264,32 @@ class ScreenCaster this.cdps.delete(id); } - async startCast(cdp, id) { - if (cdp._startedCast) { + async startCast(cdp: CDPSession, id: WorkerId) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((cdp as any)._startedCast) { return; } - cdp._startedCast = true; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (cdp as any)._startedCast = true; logger.info("Started Screencast", {workerid: id}, "screencast"); await cdp.send("Page.startScreencast", {format: "png", everyNthFrame: 1, maxWidth: this.maxWidth, maxHeight: this.maxHeight}); } - async stopCast(cdp, id) { - if (!cdp._startedCast) { + async stopCast(cdp: CDPSession, id: WorkerId) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if (!(cdp as any)._startedCast) { return; } - cdp._startedCast = false; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (cdp as any)._startedCast = false; logger.info("Stopping Screencast", {workerid: id}, "screencast"); diff --git a/util/screenshots.js b/src/util/screenshots.ts similarity index 81% rename from util/screenshots.js rename to src/util/screenshots.ts index df17fd34..abb999ce 100644 --- a/util/screenshots.js +++ b/src/util/screenshots.ts @@ -2,11 +2,18 @@ import sharp from "sharp"; import { WARCResourceWriter } from "./warcresourcewriter.js"; import { logger, errJSON } from "./logger.js"; +import { Browser } from "./browser.js"; // ============================================================================ -export const screenshotTypes = { +type ScreenShotType = { + type: string; + omitBackground: boolean; + fullPage: boolean; +} + +export const screenshotTypes : Record = { "view": { type: "png", omitBackground: true, @@ -24,10 +31,15 @@ export const screenshotTypes = { } }; - export class Screenshots extends WARCResourceWriter { + browser: Browser; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + page: any; - constructor(opts) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(opts: any) { super({...opts, warcName: "screenshots.warc.gz"}); this.browser = opts.browser; this.page = opts.page; diff --git a/util/seeds.js b/src/util/seeds.ts similarity index 70% rename from util/seeds.js rename to src/util/seeds.ts index d4a6546c..44a7bb52 100644 --- a/util/seeds.js +++ b/src/util/seeds.ts @@ -1,10 +1,34 @@ import { logger } from "./logger.js"; import { MAX_DEPTH } from "./constants.js"; +type ScopeType = + | "prefix" + | "host" + | "domain" + | "page" + | "page-spa" + | "any" + | "custom"; export class ScopedSeed { - constructor({url, scopeType, include, exclude = [], allowHash = false, depth = -1, sitemap = false, extraHops = 0} = {}) { + url: string; + scopeType: ScopeType; + include: RegExp[]; + exclude: RegExp[] = []; + allowHash = false; + depth = -1; + sitemap?: string | null; + extraHops = 0; + + maxExtraHops = 0; + maxDepth = 0; + + + constructor( + {url, scopeType, include, exclude = [], allowHash = false, depth = -1, sitemap = false, extraHops = 0} : + {url: string, scopeType: ScopeType, include: string[], exclude?: string[], allowHash?: boolean, depth?: number, sitemap?: string | boolean | null, extraHops?: number} + ) { const parsedUrl = this.parseUrl(url); if (!parsedUrl) { throw new Error("Invalid URL"); @@ -19,8 +43,9 @@ export class ScopedSeed } if (this.scopeType !== "custom") { - [include, allowHash] = this.scopeFromType(this.scopeType, parsedUrl); - this.include = [...include, ...this.include]; + const [includeNew, allowHashNew] = this.scopeFromType(this.scopeType, parsedUrl); + this.include = [...includeNew, ...this.include]; + allowHash = allowHashNew; } // for page scope, the depth is set to extraHops, as no other @@ -35,7 +60,10 @@ export class ScopedSeed this.maxDepth = depth < 0 ? MAX_DEPTH : depth; } - parseRx(value) { + //parseRx(value? : union[string[], string, RegExp[]]) -> RegExp[] { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + parseRx(value : any) { if (value === null || value === undefined || value === "") { return []; } else if (!(value instanceof Array)) { @@ -45,7 +73,7 @@ export class ScopedSeed } } - addExclusion(value) { + addExclusion(value: string | RegExp) { if (!value) { return; } @@ -55,7 +83,7 @@ export class ScopedSeed this.exclude.push(value); } - removeExclusion(value) { + removeExclusion(value: string) { for (let i = 0; i < this.exclude.length; i++) { if (this.exclude[i].toString() == value.toString()) { this.exclude.splice(i, 1); @@ -64,7 +92,7 @@ export class ScopedSeed } } - parseUrl(url, logDetails = {}) { + parseUrl(url: string, logDetails = {}) { let parsedUrl = null; try { parsedUrl = new URL(url.trim()); @@ -81,18 +109,21 @@ export class ScopedSeed return parsedUrl; } - resolveSiteMap(sitemap) { + resolveSiteMap(sitemap: boolean | string | null) : string | null { if (sitemap === true) { const url = new URL(this.url); url.pathname = "/sitemap.xml"; return url.href; + } else if (typeof(sitemap) === "string") { + const url = new URL(sitemap, this.url); + return url.href; } - return sitemap; + return null; } - scopeFromType(scopeType, parsedUrl) { - let include; + scopeFromType(scopeType: ScopeType, parsedUrl: URL) : [RegExp[], boolean] { + let include : RegExp[] = []; let allowHash = false; switch (scopeType) { @@ -132,26 +163,26 @@ export class ScopedSeed return [include, allowHash]; } - isAtMaxDepth(depth) { + isAtMaxDepth(depth: number) { return depth >= this.maxDepth; } - isIncluded(url, depth, extraHops = 0, logDetails = {}) { + isIncluded(url: string, depth: number, extraHops = 0, logDetails = {}) { if (depth > this.maxDepth) { return false; } - url = this.parseUrl(url, logDetails); - if (!url) { + const urlParsed = this.parseUrl(url, logDetails); + if (!urlParsed) { return false; } if (!this.allowHash) { // remove hashtag - url.hash = ""; + urlParsed.hash = ""; } - url = url.href; + url = urlParsed.href; if (url === this.url) { return true; @@ -194,11 +225,11 @@ export class ScopedSeed } } -export function rxEscape(string) { +export function rxEscape(string: string) { return string.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&"); } -export function urlRxEscape(url, parsedUrl) { +export function urlRxEscape(url: string, parsedUrl: URL) { return rxEscape(url).replace(parsedUrl.protocol, "https?:"); } diff --git a/util/state.js b/src/util/state.ts similarity index 70% rename from util/state.js rename to src/util/state.ts index 8c2473b7..52a43314 100644 --- a/util/state.js +++ b/src/util/state.ts @@ -1,60 +1,144 @@ +import { Redis, Result, Callback } from "ioredis"; + import { logger } from "./logger.js"; import { MAX_DEPTH } from "./constants.js"; +import { ScopedSeed } from "./seeds.js"; +import { Frame } from "puppeteer-core"; // ============================================================================ -export const LoadState = { - FAILED: 0, - CONTENT_LOADED: 1, - FULL_PAGE_LOADED: 2, - EXTRACTION_DONE: 3, - BEHAVIORS_DONE: 4, -}; +export enum LoadState { + FAILED = 0, + CONTENT_LOADED = 1, + FULL_PAGE_LOADED = 2, + EXTRACTION_DONE = 3, + BEHAVIORS_DONE = 4, +} // ============================================================================ -export const QueueState = { - ADDED: 0, - LIMIT_HIT: 1, - DUPE_URL: 2, -}; +export enum QueueState { + ADDED = 0, + LIMIT_HIT = 1, + DUPE_URL = 2, +} + + +// ============================================================================ +export type WorkerId = number; // ============================================================================ export class PageState { - constructor(redisData) { + url: string; + seedId: number; + depth: number; + extraHops: number; + + workerid!: WorkerId; + + pageid?: string; + title?: string; + mime?: string; + + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + callbacks: any; + + isHTMLPage?: boolean; + text?: string; + favicon?: string; + + skipBehaviors = false; + filteredFrames: Frame[] = []; + loadState : LoadState = LoadState.FAILED; + + logDetails = {}; + + constructor(redisData: {url: string, seedId: number, depth: number, extraHops: number}) { this.url = redisData.url; this.seedId = redisData.seedId; this.depth = redisData.depth; this.extraHops = redisData.extraHops; - - this.workerid = null; - this.pageid = null; - this.title = null; - - this.isHTMLPage = null; - this.text = null; - - this.skipBehaviors = false; - this.filteredFrames = []; - - this.loadState = LoadState.FAILED; - this.logDetails = {}; } } +// ============================================================================ +declare module "ioredis" { + interface RedisCommander { + addqueue( + pkey: string, + qkey: string, + skey: string, + url: string, + score: number, + data: string, + limit: number, + ): Result; + + getnext( + qkey: string, + pkey: string, + ): Result; + + markstarted( + pkey: string, + pkeyUrl: string, + url: string, + started: string, + maxPageTime: number, + uid: string, + ): Result; + + movefailed( + pkey: string, + fkey: string, + url: string, + value: string, + state: string, + ): Result; + + unlockpending( + pkeyUrl: string, + uid: string, + callback?: Callback + ): Result; + + requeue( + pkey: string, + qkey: string, + pkeyUrl: string, + url: string, + maxRetryPending: number, + ): Result; + + } +} // ============================================================================ export class RedisCrawlState { - constructor(redis, key, maxPageTime, uid) { + redis: Redis; + maxRetryPending = 1; + _lastSize = 0; + + uid: string; + key: string; + maxPageTime: number; + + qkey: string; + pkey: string; + skey: string; + dkey: string; + fkey: string; + ekey: string; + + constructor(redis: Redis, key: string, maxPageTime: number, uid: string) { this.redis = redis; - this.maxRetryPending = 1; - this._lastSize = 0; this.uid = uid; this.key = key; @@ -73,7 +157,7 @@ export class RedisCrawlState this._initLuaCommands(this.redis); } - _initLuaCommands(redis) { + _initLuaCommands(redis: Redis) { redis.defineCommand("addqueue", { numberOfKeys: 3, lua: ` @@ -184,58 +268,58 @@ return 0; return new Date().toISOString(); } - async markStarted(url) { + async markStarted(url: string) { const started = this._timestamp(); return await this.redis.markstarted(this.pkey, this.pkey + ":" + url, url, started, this.maxPageTime, this.uid); } - async markFinished(url) { + async markFinished(url: string) { await this.redis.hdel(this.pkey, url); return await this.redis.incr(this.dkey); } - async markFailed(url) { + async markFailed(url: string) { await this.redis.movefailed(this.pkey, this.fkey, url, "1", "failed"); return await this.redis.incr(this.dkey); } - async markExcluded(url) { + async markExcluded(url: string) { await this.redis.hdel(this.pkey, url); await this.redis.srem(this.skey, url); } - recheckScope(data, seeds) { + recheckScope(data: {url: string, depth: number, extraHops: number, seedId: number}, seeds: ScopedSeed[]) { const seed = seeds[data.seedId]; return seed.isIncluded(data.url, data.depth, data.extraHops); } async isFinished() { - return (await this.queueSize() == 0) && (await this.numDone() > 0); + return ((await this.queueSize()) == 0) && ((await this.numDone()) > 0); } - async setStatus(status_) { + async setStatus(status_: string) { await this.redis.hset(`${this.key}:status`, this.uid, status_); } - async getStatus() { - return await this.redis.hget(`${this.key}:status`, this.uid); + async getStatus() : Promise { + return (await this.redis.hget(`${this.key}:status`, this.uid)) || ""; } - async setArchiveSize(size) { + async setArchiveSize(size: number) { return await this.redis.hset(`${this.key}:size`, this.uid, size); } async isCrawlStopped() { - if (await this.redis.get(`${this.key}:stopping`) === "1") { + if ((await this.redis.get(`${this.key}:stopping`)) === "1") { return true; } - if (await this.redis.hget(`${this.key}:stopone`, this.uid) === "1") { + if ((await this.redis.hget(`${this.key}:stopone`, this.uid)) === "1") { return true; } @@ -243,7 +327,7 @@ return 0; } async isCrawlCanceled() { - return await this.redis.get(`${this.key}:canceled`) === "1"; + return (await this.redis.get(`${this.key}:canceled`)) === "1"; } // note: not currently called in crawler, but could be @@ -252,7 +336,7 @@ return 0; await this.redis.set(`${this.key}:stopping`, "1"); } - async processMessage(seeds) { + async processMessage(seeds: ScopedSeed[]) { while (true) { const result = await this.redis.lpop(`${this.uid}:msg`); if (!result) { @@ -285,18 +369,20 @@ return 0; } break; } - } catch (e) { + } // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + catch (e: any) { logger.warn("Error processing message", e, "redisMessage"); } } } - isStrMatch(s) { + isStrMatch(s: string) { // if matches original string, then consider not a regex return s.replace(/\\/g, "").replace(/[\\^$*+?.()|[\]{}]/g, "\\$&") === s; } - filterQueue(regexStr) { + filterQueue(regexStr: string) { const regex = new RegExp(regexStr); let matcher = undefined; @@ -325,7 +411,7 @@ return 0; stream.resume(); }); - return new Promise(resolve => { + return new Promise(resolve => { stream.on("end", () => { resolve(); }); @@ -341,9 +427,12 @@ return 0; return (res >= 3); } - async addToQueue({url, seedId, depth = 0, extraHops = 0} = {}, limit = 0) { + //async addToQueue({url : string, seedId, depth = 0, extraHops = 0} = {}, limit = 0) { + async addToQueue({url, seedId, depth = 0, extraHops = 0} : {url: string, seedId: number, depth?: number, extraHops?: number}, limit = 0) { const added = this._timestamp(); - const data = {added, url, seedId, depth}; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const data : any = {added, url, seedId, depth}; if (extraHops) { data.extraHops = extraHops; } @@ -375,8 +464,8 @@ return 0; return new PageState(data); } - async has(url) { - return !!await this.redis.sismember(this.skey, url); + async has(url: string) { + return !!(await this.redis.sismember(this.skey, url)); } async serialize() { @@ -390,25 +479,25 @@ return 0; return {done, queued, pending, failed, errors}; } - _getScore(data) { + _getScore(data: {depth: number, extraHops: number}) { return (data.depth || 0) + (data.extraHops || 0) * MAX_DEPTH; } - async _iterSortedKey(key, inc = 100) { - const results = []; + async _iterSortedKey(key: string, inc = 100) { + const results : string[] = []; const len = await this.redis.zcard(key); for (let i = 0; i < len; i += inc) { - const someResults = await this.redis.zrangebyscore(key, 0, "inf", "limit", i, inc); + const someResults = await this.redis.zrangebyscore(key, 0, "inf", "LIMIT", i, inc); results.push(...someResults); } return results; } - async _iterListKeys(key, inc = 100) { - const results = []; + async _iterListKeys(key: string, inc = 100) { + const results : string[] = []; const len = await this.redis.llen(key); @@ -419,8 +508,10 @@ return 0; return results; } - async load(state, seeds, checkScope) { - const seen = []; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async load(state: Record, seeds: ScopedSeed[], checkScope: boolean) { + const seen : string[] = []; // need to delete existing keys, if exist to fully reset state await this.redis.del(this.qkey); @@ -486,7 +577,7 @@ return 0; async numDone() { const done = await this.redis.get(this.dkey); - return parseInt(done); + return parseInt(done || "0"); } async numSeen() { @@ -524,7 +615,9 @@ return 0; for (const url of pendingUrls) { await this.redis.unlockpending(this.pkey + ":" + url, this.uid); } - } catch (e) { + } // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + catch (e: any) { logger.error("Redis Del Pending Failed", e, "state"); } } @@ -551,15 +644,15 @@ return 0; return this._lastSize; } - async addIfNoDupe(key, value) { - return await this.redis.sadd(key, value) === 1; + async addIfNoDupe(key: string, value: string) { + return (await this.redis.sadd(key, value)) === 1; } - async removeDupe(key, value) { + async removeDupe(key: string, value: string) { return await this.redis.srem(key, value); } - async logError(error) { + async logError(error: string) { return await this.redis.lpush(this.ekey, error); } } diff --git a/util/storage.js b/src/util/storage.ts similarity index 76% rename from util/storage.js rename to src/util/storage.ts index ffef90ce..5e2d7173 100644 --- a/util/storage.js +++ b/src/util/storage.ts @@ -5,20 +5,40 @@ import util from "util"; import os from "os"; import { createHash } from "crypto"; + import crc32 from "crc/crc32"; -import Minio from "minio"; +import * as Minio from "minio"; import { initRedis } from "./redis.js"; import { logger } from "./logger.js"; +// @ts-expect-error TODO fill in why error is expected import getFolderSize from "get-folder-size"; // =========================================================================== export class S3StorageSync { - constructor(urlOrData, {webhookUrl, userId, crawlId} = {}) { + fullPrefix: string; + client: Minio.Client; + + bucketName: string; + objectPrefix: string; + resources: object[] = []; + + userId: string; + crawlId: string; + webhookUrl?: string; + + // TODO: Fix this the next time the file is edited. + + constructor( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + urlOrData: string | any, + {webhookUrl, userId, crawlId} : + {webhookUrl?: string, userId: string, crawlId: string} + ) { let url; let accessKey; let secretKey; @@ -47,8 +67,6 @@ export class S3StorageSync partSize: 100*1024*1024 }); - this.client.enableSHA256 = true; - this.bucketName = url.pathname.slice(1).split("/")[0]; this.objectPrefix = url.pathname.slice(this.bucketName.length + 2); @@ -60,12 +78,12 @@ export class S3StorageSync this.webhookUrl = webhookUrl; } - async uploadFile(srcFilename, targetFilename) { + async uploadFile(srcFilename: string, targetFilename: string) { const fileUploadInfo = { "bucket": this.bucketName, "crawlId": this.crawlId, "prefix": this.objectPrefix, - "targetFilename": this.targetFilename + targetFilename }; logger.info("S3 file upload information", fileUploadInfo, "s3Upload"); @@ -80,13 +98,13 @@ export class S3StorageSync return {path, size, hash, crc32, bytes: size}; } - async downloadFile(srcFilename, destFilename) { + async downloadFile(srcFilename: string, destFilename: string) { await this.client.fGetObject(this.bucketName, this.objectPrefix + srcFilename, destFilename); } - async uploadCollWACZ(srcFilename, targetFilename, completed = true) { + async uploadCollWACZ(srcFilename: string, targetFilename: string, completed = true) { const resource = await this.uploadFile(srcFilename, targetFilename); - logger.info("WACZ S3 file upload resource", {...targetFilename, resource}, "s3Upload"); + logger.info("WACZ S3 file upload resource", {targetFilename, resource}, "s3Upload"); if (this.webhookUrl) { const body = { @@ -130,8 +148,8 @@ export function initStorage() { const opts = { crawlId: process.env.CRAWL_ID || os.hostname(), - webhookUrl: process.env.WEBHOOK_URL, - userId: process.env.STORE_USER, + webhookUrl: process.env.WEBHOOK_URL || "", + userId: process.env.STORE_USER || "", }; logger.info("Initing Storage..."); @@ -139,12 +157,12 @@ export function initStorage() { } -export async function getFileSize(filename) { +export async function getFileSize(filename: string) { const stats = await fsp.stat(filename); return stats.size; } -export async function getDirSize(dir) { +export async function getDirSize(dir: string) { const { size, errors } = await getFolderSize(dir); if (errors && errors.length) { logger.warn("Size check errors", {errors}, "sizecheck"); @@ -152,8 +170,10 @@ export async function getDirSize(dir) { return size; } -export async function checkDiskUtilization(params, archiveDirSize, dfOutput=null) { - const diskUsage = await getDiskUsage("/crawls", dfOutput); +// TODO: Fix this the next time the file is edited. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function checkDiskUtilization(params: Record, archiveDirSize: number, dfOutput=null) { + const diskUsage : Record = await getDiskUsage("/crawls", dfOutput); const usedPercentage = parseInt(diskUsage["Use%"].slice(0, -1)); // Check that disk usage isn't already above threshold @@ -199,19 +219,21 @@ export async function checkDiskUtilization(params, archiveDirSize, dfOutput=null }; } -export async function getDFOutput(path) { +export async function getDFOutput(path: string) { const exec = util.promisify(child_process.exec); const res = await exec(`df ${path}`); return res.stdout; } export async function getDiskUsage(path="/crawls", dfOutput = null) { - const result = dfOutput || await getDFOutput(path); + const result = dfOutput || (await getDFOutput(path)); const lines = result.split("\n"); const keys = lines[0].split(/\s+/ig); const rows = lines.slice(1).map(line => { const values = line.split(/\s+/ig); - return keys.reduce((o, k, index) => { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return keys.reduce((o: Record, k, index) => { o[k] = values[index]; return o; }, {}); @@ -219,14 +241,14 @@ export async function getDiskUsage(path="/crawls", dfOutput = null) { return rows[0]; } -export function calculatePercentageUsed(used, total) { +export function calculatePercentageUsed(used: number, total: number) { return Math.round((used/total) * 100); } -function checksumFile(hashName, path) { +function checksumFile(hashName: string, path: string) : Promise<{hash: string, crc32: number}>{ return new Promise((resolve, reject) => { const hash = createHash(hashName); - let crc = null; + let crc : number = 0; const stream = fs.createReadStream(path); stream.on("error", err => reject(err)); @@ -238,7 +260,7 @@ function checksumFile(hashName, path) { }); } -export function interpolateFilename(filename, crawlId) { +export function interpolateFilename(filename: string, crawlId: string) { filename = filename.replace("@ts", new Date().toISOString().replace(/[:TZz.-]/g, "")); filename = filename.replace("@hostname", os.hostname()); filename = filename.replace("@hostsuffix", os.hostname().slice(-14)); diff --git a/util/textextract.js b/src/util/textextract.ts similarity index 68% rename from util/textextract.js rename to src/util/textextract.ts index 846c4329..141db2be 100644 --- a/util/textextract.js +++ b/src/util/textextract.ts @@ -1,16 +1,21 @@ import { WARCResourceWriter } from "./warcresourcewriter.js"; import { logger } from "./logger.js"; - +import { CDPSession, Protocol } from "puppeteer-core"; // ============================================================================ -export class BaseTextExtract extends WARCResourceWriter { - constructor(cdp, opts) { +export abstract class BaseTextExtract extends WARCResourceWriter { + cdp: CDPSession; + lastText: string | null = null; + text: string | null = null; + + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(cdp: CDPSession, opts: any) { super({...opts, warcName: "text.warc.gz"}); this.cdp = cdp; - this.lastText = null; } - async extractAndStoreText(resourceType, ignoreIfMatchesLast = false, saveToWarc = false) { + async extractAndStoreText(resourceType: string, ignoreIfMatchesLast = false, saveToWarc = false) { try { const text = await this.doGetText(); @@ -26,26 +31,26 @@ export class BaseTextExtract extends WARCResourceWriter { this.lastText = text; return {changed: true, text}; - } catch (e) { + } // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + catch (e: any) { logger.debug("Error extracting text", e, "text"); return {changed: false, text: null}; } } - async doGetText() { - throw new Error("unimplemented"); - } + abstract doGetText() : Promise; } // ============================================================================ export class TextExtractViaSnapshot extends BaseTextExtract { - async doGetText() { + async doGetText() : Promise { const result = await this.cdp.send("DOMSnapshot.captureSnapshot", {computedStyles: []}); return this.parseTextFromDOMSnapshot(result); } - parseTextFromDOMSnapshot(result) { + parseTextFromDOMSnapshot(result: Protocol.DOMSnapshot.CaptureSnapshotResponse) : string { const TEXT_NODE = 3; const ELEMENT_NODE = 1; @@ -53,13 +58,13 @@ export class TextExtractViaSnapshot extends BaseTextExtract { const {strings, documents} = result; - const accum = []; + const accum : string[] = []; for (const doc of documents) { - const nodeValues = doc.nodes.nodeValue; - const nodeNames = doc.nodes.nodeName; - const nodeTypes = doc.nodes.nodeType; - const parentIndex = doc.nodes.parentIndex; + const nodeValues = doc.nodes.nodeValue || []; + const nodeNames = doc.nodes.nodeName || []; + const nodeTypes = doc.nodes.nodeType || []; + const parentIndex = doc.nodes.parentIndex || []; for (let i = 0; i < nodeValues.length; i++) { if (nodeValues[i] === -1) { @@ -74,28 +79,28 @@ export class TextExtractViaSnapshot extends BaseTextExtract { if (!SKIPPED_NODES.includes(name)) { const value = strings[nodeValues[i]].trim(); if (value) { - accum.push(value); + accum.push(value as string); } } } } } - - return accum.join("\n"); } + + return accum.join("\n"); } } // ============================================================================ export class TextExtractViaDocument extends BaseTextExtract { - async doGetText() { + async doGetText() : Promise { const result = await this.cdp.send("DOM.getDocument", {"depth": -1, "pierce": true}); return this.parseTextFromDOM(result); } - async parseTextFromDom(dom) { - const accum = []; + parseTextFromDOM(dom: Protocol.DOM.GetDocumentResponse) : string { + const accum : string[] = []; const metadata = {}; this.parseText(dom.root, metadata, accum); @@ -103,9 +108,9 @@ export class TextExtractViaDocument extends BaseTextExtract { return accum.join("\n"); } - async parseText(node, metadata, accum) { + parseText(node: Protocol.DOM.Node, metadata: Record | null, accum: string[]) { const SKIPPED_NODES = ["head", "script", "style", "header", "footer", "banner-div", "noscript"]; - const EMPTY_LIST = []; + const EMPTY_LIST : Protocol.DOM.Node[] = []; const TEXT = "#text"; const TITLE = "title"; @@ -123,9 +128,9 @@ export class TextExtractViaDocument extends BaseTextExtract { accum.push(value); } } else if (name === TITLE) { - const title = []; + const title : string[] = []; - for (let child of children) { + for (const child of children) { this.parseText(child, null, title); } @@ -135,7 +140,7 @@ export class TextExtractViaDocument extends BaseTextExtract { accum.push(title.join(" ")); } } else { - for (let child of children) { + for (const child of children) { this.parseText(child, metadata, accum); } diff --git a/util/timing.js b/src/util/timing.ts similarity index 67% rename from util/timing.js rename to src/util/timing.ts index 56b7f9c1..9d005b3a 100644 --- a/util/timing.js +++ b/src/util/timing.ts @@ -1,14 +1,24 @@ import { logger } from "./logger.js"; -export function sleep(seconds) { +export function sleep(seconds: number) { return new Promise(resolve => setTimeout(resolve, seconds * 1000)); } -export function timedRun(promise, seconds, message="Promise timed out", logDetails={}, context="general", isWarn=false) { +// TODO: Fix this the next time the file is edited. + +export function timedRun( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + promise: Promise, + seconds: number, + message="Promise timed out", + logDetails={}, + context="general", + isWarn=false +) { // return Promise return value or log error if timeout is reached first const timeout = seconds * 1000; - const rejectPromiseOnTimeout = (timeout) => { + const rejectPromiseOnTimeout = (timeout: number) => { return new Promise((resolve, reject) => { setTimeout(() => (reject("timeout reached")), timeout); }); @@ -26,7 +36,7 @@ export function timedRun(promise, seconds, message="Promise timed out", logDetai }); } -export function secondsElapsed(startTime, nowDate = null) { +export function secondsElapsed(startTime: number, nowDate: Date | null = null) { nowDate = nowDate || new Date(); return (nowDate.getTime() - startTime) / 1000; diff --git a/util/warcresourcewriter.js b/src/util/warcresourcewriter.ts similarity index 61% rename from util/warcresourcewriter.js rename to src/util/warcresourcewriter.ts index e173609a..a14187c4 100644 --- a/util/warcresourcewriter.js +++ b/src/util/warcresourcewriter.ts @@ -4,27 +4,35 @@ import * as warcio from "warcio"; export class WARCResourceWriter { - constructor({url, directory, date, warcName}) { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + page: any; + url: string; + directory: string; + warcName: string; + date: Date; + + constructor({url, directory, date, warcName} : {url: string, directory: string, date: Date, warcName: string}) { this.url = url; this.directory = directory; this.warcName = path.join(this.directory, warcName); this.date = date ? date : new Date(); } - async writeBufferToWARC(contents, resourceType, contentType) { + async writeBufferToWARC(contents: Uint8Array, resourceType: string, contentType: string) { const warcRecord = await this.wrap(contents, resourceType, contentType); const warcRecordBuffer = await warcio.WARCSerializer.serialize(warcRecord, {gzip: true}); fs.appendFileSync(this.warcName, warcRecordBuffer); } - async wrap(buffer, resourceType, contentType) { + async wrap(buffer: Uint8Array, resourceType: string, contentType: string) { const warcVersion = "WARC/1.1"; const warcRecordType = "resource"; const warcHeaders = {"Content-Type": contentType}; async function* content() { yield buffer; } - let resourceUrl = `urn:${resourceType}:${this.url}`; + const resourceUrl = `urn:${resourceType}:${this.url}`; return warcio.WARCRecord.create({ url: resourceUrl, diff --git a/util/warcwriter.js b/src/util/warcwriter.ts similarity index 69% rename from util/warcwriter.js rename to src/util/warcwriter.ts index 0ff26666..33b65246 100644 --- a/util/warcwriter.js +++ b/src/util/warcwriter.ts @@ -1,15 +1,32 @@ import fs from "fs"; +import { Writable } from "stream"; import path from "path"; import { CDXIndexer } from "warcio"; import { WARCSerializer } from "warcio/node"; import { logger, errJSON } from "./logger.js"; +import type { IndexerOffsetLength, WARCRecord } from "warcio"; // ================================================================= -export class WARCWriter +export class WARCWriter implements IndexerOffsetLength { - constructor({archivesDir, tempCdxDir, filename, gzip, logDetails}) { + archivesDir: string; + tempCdxDir: string; + filename: string; + gzip: boolean; + logDetails: Record; + + offset = 0; + recordLength = 0; + + indexer?: CDXIndexer; + + fh?: Writable | null; + cdxFH?: Writable | null; + + constructor({archivesDir, tempCdxDir, filename, gzip, logDetails} : + {archivesDir: string, tempCdxDir: string, filename: string, gzip: boolean, logDetails: Record}) { this.archivesDir = archivesDir; this.tempCdxDir = tempCdxDir; this.filename = filename; @@ -21,12 +38,7 @@ export class WARCWriter if (this.tempCdxDir) { this.indexer = new CDXIndexer({format: "cdxj"}); - } else { - this.indexer = null; } - - this.fh = null; - this.cdxFH = null; } async initFH() { @@ -38,7 +50,7 @@ export class WARCWriter } } - async writeRecordPair(responseRecord, requestRecord, responseSerializer = null) { + async writeRecordPair(responseRecord: WARCRecord, requestRecord: WARCRecord, responseSerializer: WARCSerializer | undefined = undefined) { const opts = {gzip: this.gzip}; if (!responseSerializer) { @@ -58,10 +70,14 @@ export class WARCWriter } - async _writeRecord(record, serializer) { + async _writeRecord(record: WARCRecord, serializer: WARCSerializer) { let total = 0; const url = record.warcTargetURI; + if (!this.fh) { + throw new Error("writer not initialized"); + } + for await (const chunk of serializer) { total += chunk.length; try { @@ -74,12 +90,12 @@ export class WARCWriter return total; } - _writeCDX(record) { + _writeCDX(record: WARCRecord | null) { if (this.indexer) { const cdx = this.indexer.indexRecord(record, this, this.filename); if (this.indexer && this.cdxFH && cdx) { - this.indexer.write(cdx, this.cdxFH); + this.indexer.write(cdx, this.cdxFH as NodeJS.WriteStream); } } @@ -102,8 +118,8 @@ export class WARCWriter } // ================================================================= -export function streamFinish(fh) { - const p = new Promise(resolve => { +export function streamFinish(fh: Writable) { + const p = new Promise(resolve => { fh.once("finish", () => resolve()); }); fh.end(); diff --git a/util/worker.js b/src/util/worker.ts similarity index 72% rename from util/worker.js rename to src/util/worker.ts index 148b6983..3a98204a 100644 --- a/util/worker.js +++ b/src/util/worker.ts @@ -6,6 +6,8 @@ import { logger, errJSON } from "./logger.js"; import { sleep, timedRun } from "./timing.js"; import { Recorder } from "./recorder.js"; import { rxEscape } from "./seeds.js"; +import { CDPSession, Page } from "puppeteer-core"; +import { PageState, WorkerId } from "./state.js"; const MAX_REUSE = 5; @@ -14,7 +16,9 @@ const TEARDOWN_TIMEOUT = 10; const FINISHED_TIMEOUT = 60; // =========================================================================== -export function runWorkers(crawler, numWorkers, maxPageTime, collDir) { +// TODO: Fix this the next time the file is edited. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function runWorkers(crawler: any, numWorkers: number, maxPageTime: number, collDir: string) { logger.info(`Creating ${numWorkers} workers`, {}, "worker"); const workers = []; @@ -29,40 +33,73 @@ export function runWorkers(crawler, numWorkers, maxPageTime, collDir) { const rx = new RegExp(rxEscape(process.env.CRAWL_ID) + "\\-([\\d]+)$"); const m = os.hostname().match(rx); if (m) { - offset = m[1] * numWorkers; + offset = Number(m[1]) * numWorkers; logger.info("Starting workerid index at " + offset, "worker"); } } for (let i = 0; i < numWorkers; i++) { - workers.push(new PageWorker(i + offset, crawler, maxPageTime, collDir)); + workers.push(new PageWorker((i + offset), crawler, maxPageTime, collDir)); } return Promise.allSettled(workers.map((worker) => worker.run())); } +// =========================================================================== +// TODO: Fix this the next time the file is edited. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type WorkerOpts = Record & { + page: Page; + cdp: CDPSession; + workerid: WorkerId; + // eslint-disable-next-line @typescript-eslint/ban-types + callbacks: Record; + directFetchCapture?: ((url: string) => Promise<{fetched: boolean, mime: string}>) | null; +}; + +// =========================================================================== +export type WorkerState = WorkerOpts & { + data: PageState +}; + // =========================================================================== export class PageWorker { - constructor(id, crawler, maxPageTime, collDir) { + id: WorkerId; + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + crawler: any; + maxPageTime: number; + + reuseCount = 0; + page?: Page | null; + cdp?: CDPSession | null; + + // eslint-disable-next-line @typescript-eslint/ban-types + callbacks?: Record; + + opts?: WorkerOpts; + + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + logDetails: Record = {}; + + crashed = false; + markCrashed?: (reason: string) => void; + crashBreak?: Promise; + + recorder: Recorder; + + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor(id: WorkerId, crawler: any, maxPageTime: number, collDir: string) { this.id = id; this.crawler = crawler; this.maxPageTime = maxPageTime; - this.reuseCount = 0; - this.page = null; - this.cdp = null; - this.callbacks = null; - - this.opts = null; - this.logDetails = {workerid: this.id}; - this.crashed = false; - this.markCrashed = null; - this.crashBreak = null; - this.recorder = new Recorder({workerid: id, collDir, crawler: this.crawler}); this.crawler.browser.recorders.push(this.recorder); @@ -108,9 +145,9 @@ export class PageWorker } } - isSameOrigin(url) { + isSameOrigin(url: string) { try { - const currURL = new URL(this.page.url()); + const currURL = new URL(this.page ? this.page.url() : ""); const newURL = new URL(url); return currURL.origin === newURL.origin; } catch (e) { @@ -118,8 +155,8 @@ export class PageWorker } } - async initPage(url) { - if (!this.crashed && this.page && ++this.reuseCount <= MAX_REUSE && this.isSameOrigin(url)) { + async initPage(url: string) : Promise { + if (!this.crashed && this.page && this.opts && ++this.reuseCount <= MAX_REUSE && this.isSameOrigin(url)) { logger.debug("Reusing page", {reuseCount: this.reuseCount, ...this.logDetails}, "worker"); return this.opts; } else if (this.page) { @@ -151,10 +188,10 @@ export class PageWorker this.page = page; this.cdp = cdp; this.callbacks = {}; - const directFetchCapture = this.recorder ? (x) => this.recorder.directFetchCapture(x) : null; + const directFetchCapture = this.recorder ? (x: string) => this.recorder.directFetchCapture(x) : null; this.opts = { - page: this.page, - cdp: this.cdp, + page, + cdp, workerid, callbacks: this.callbacks, directFetchCapture, @@ -168,15 +205,19 @@ export class PageWorker this.crashed = false; this.crashBreak = new Promise((resolve, reject) => this.markCrashed = reject); - this.logDetails = {page: this.page.url(), workerid}; + this.logDetails = {page: page.url(), workerid}; // more serious page crash, mark as failed - this.page.on("error", (err) => { + // TODO: Fix this the next time the file is edited. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + page.on("error", (err: any) => { // ensure we're still on this page, otherwise ignore! if (this.page === page) { logger.error("Page Crashed", {...errJSON(err), ...this.logDetails}, "worker"); this.crashed = true; - this.markCrashed("crashed"); + if (this.markCrashed) { + this.markCrashed("crashed"); + } } }); @@ -204,9 +245,11 @@ export class PageWorker } } } + + throw new Error("no page available, shouldn't get here"); } - async crawlPage(opts) { + async crawlPage(opts: WorkerState) { const res = await this.crawler.crawlPage(opts); if (this.recorder) { await this.recorder.finishPage(); @@ -214,7 +257,7 @@ export class PageWorker return res; } - async timedCrawlPage(opts) { + async timedCrawlPage(opts: WorkerState) { const workerid = this.id; const { data } = opts; const { url } = data; @@ -244,7 +287,7 @@ export class PageWorker ]); } catch (e) { - if (e.message !== "logged" && !this.crashed) { + if (e instanceof Error && e.message !== "logged" && !this.crashed) { logger.error("Worker Exception", {...errJSON(e), ...this.logDetails}, "worker"); } } finally { @@ -317,7 +360,7 @@ export class PageWorker await sleep(0.5); } else { // if no pending and queue size is still empty, we're done! - if (!await crawlState.queueSize()) { + if (!(await crawlState.queueSize())) { break; } } diff --git a/tests/custom-behaviors/custom-2.js b/tests/custom-behaviors/custom-2.js index f08e31d3..26abad6a 100644 --- a/tests/custom-behaviors/custom-2.js +++ b/tests/custom-behaviors/custom-2.js @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ class TestBehavior2 { static init() { diff --git a/tests/custom-behaviors/custom.js b/tests/custom-behaviors/custom.js index 86358d7d..c0604508 100644 --- a/tests/custom-behaviors/custom.js +++ b/tests/custom-behaviors/custom.js @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ class TestBehavior { static init() { diff --git a/tests/scopes.test.js b/tests/scopes.test.js index 0e6f0422..d41dea2a 100644 --- a/tests/scopes.test.js +++ b/tests/scopes.test.js @@ -1,4 +1,4 @@ -import { parseArgs } from "../util/argParser.js"; +import { parseArgs } from "../dist/util/argParser.js"; import fs from "fs"; diff --git a/tests/storage.test.js b/tests/storage.test.js index 82f4f8d7..b9943778 100644 --- a/tests/storage.test.js +++ b/tests/storage.test.js @@ -1,4 +1,4 @@ -import { calculatePercentageUsed, checkDiskUtilization } from "../util/storage.js"; +import { calculatePercentageUsed, checkDiskUtilization } from "../dist/util/storage.js"; test("ensure calculatePercentageUsed returns expected values", () => { diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..86808655 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,107 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "lib": ["es2022", "dom", "dom.iterable"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "NodeNext", /* Specify what module code is generated. */ + "rootDir": "./src", /* Specify the root folder within your source files. */ + "moduleResolution": "NodeNext", /* Specify how TypeScript looks up a file from a given module specifier. */ + //"baseUrl": "./src", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + "outDir": "./dist/", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + //"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + + "include": [ + "src/**/*", + ] +} diff --git a/yarn.lock b/yarn.lock index a00f0e3d..8ec57c71 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@aashutoshrathi/word-wrap@^1.2.3": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf" + integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA== + "@ampproject/remapping@^2.1.0": version "2.2.0" resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.0.tgz#56c133824780de3174aed5ab6834f3026790154d" @@ -345,26 +350,26 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@eslint-community/eslint-utils@^4.2.0": +"@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": version "4.4.0" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== dependencies: eslint-visitor-keys "^3.3.0" -"@eslint-community/regexpp@^4.4.0": - version "4.5.0" - resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.5.0.tgz#f6f729b02feee2c749f57e334b7a1b5f40a81724" - integrity sha512-vITaYzIcNmjn5tF5uxcZ/ft7/RXGrMUIS9HalWckEOF6ESiwXKoMzAQf2UW0aVd6rnOeExTJVd5hmWXucBKGXQ== +"@eslint-community/regexpp@^4.5.1", "@eslint-community/regexpp@^4.6.1": + version "4.10.0" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.10.0.tgz#548f6de556857c8bb73bbee70c35dc82a2e74d63" + integrity sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA== -"@eslint/eslintrc@^2.0.2": - version "2.0.2" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.0.2.tgz#01575e38707add677cf73ca1589abba8da899a02" - integrity sha512-3W4f5tDUra+pA+FzgugqL2pRimUTDJWKr7BINqOpkZrC0uYI0NIc0/JFgBROCU07HR6GieA5m3/rsPIhDmCXTQ== +"@eslint/eslintrc@^2.1.3": + version "2.1.3" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.1.3.tgz#797470a75fe0fbd5a53350ee715e85e87baff22d" + integrity sha512-yZzuIG+jnVu6hNSzFEN07e8BxF3uAzYtQb6uDkaYZLo6oYZDCq454c5kB8zxnzfCYyP4MIuyBn10L0DqwujTmA== dependencies: ajv "^6.12.4" debug "^4.3.2" - espree "^9.5.1" + espree "^9.6.0" globals "^13.19.0" ignore "^5.2.0" import-fresh "^3.2.1" @@ -372,17 +377,17 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.37.0": - version "8.37.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.37.0.tgz#cf1b5fa24217fe007f6487a26d765274925efa7d" - integrity sha512-x5vzdtOOGgFVDCUs81QRB2+liax8rFg3+7hqM+QhBG0/G3F1ZsoYl97UrqgHgQ9KKT7G6c4V+aTUCgu/n22v1A== +"@eslint/js@8.53.0": + version "8.53.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.53.0.tgz#bea56f2ed2b5baea164348ff4d5a879f6f81f20d" + integrity sha512-Kn7K8dx/5U6+cT1yEhpX1w4PCSg0M+XyRILPgvwcEBjerFWCwQj5sbr3/VmxqV0JGHCBCzyd6LxypEuehypY1w== -"@humanwhocodes/config-array@^0.11.8": - version "0.11.8" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.8.tgz#03595ac2075a4dc0f191cc2131de14fbd7d410b9" - integrity sha512-UybHIJzJnR5Qc/MsD9Kr+RpO2h+/P1GhOwdiLPXK5TWk5sgTdu88bTD9UP+CKbPPh5Rni1u0GjAdYQLemG8g+g== +"@humanwhocodes/config-array@^0.11.13": + version "0.11.13" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz#075dc9684f40a531d9b26b0822153c1e832ee297" + integrity sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ== dependencies: - "@humanwhocodes/object-schema" "^1.2.1" + "@humanwhocodes/object-schema" "^2.0.1" debug "^4.1.1" minimatch "^3.0.5" @@ -391,10 +396,15 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== -"@humanwhocodes/object-schema@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" - integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@humanwhocodes/object-schema@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz#e5211452df060fa8522b55c7b3c0c4d1981cb044" + integrity sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw== + +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" @@ -652,12 +662,12 @@ "@nodelib/fs.stat" "2.0.5" run-parallel "^1.1.9" -"@nodelib/fs.stat@2.0.5": +"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": version "2.0.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b" integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== -"@nodelib/fs.walk@^1.2.8": +"@nodelib/fs.walk@^1.2.3", "@nodelib/fs.walk@^1.2.8": version "1.2.8" resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a" integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== @@ -917,6 +927,16 @@ dependencies: "@types/istanbul-lib-report" "*" +"@types/js-yaml@^4.0.8": + version "4.0.8" + resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.8.tgz#7574e422d70d4a1b41f517d1d9abc61be2299a97" + integrity sha512-m6jnPk1VhlYRiLFm3f8X9Uep761f+CK8mHyS65LutH2OhmBF0BeMEjHgg05usH8PLZMWWc/BUR9RPmkvpWnyRA== + +"@types/json-schema@^7.0.12": + version "7.0.15" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" + integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== + "@types/keyv@*": version "3.1.1" resolved "https://registry.yarnpkg.com/@types/keyv/-/keyv-3.1.1.tgz#e45a45324fca9dab716ab1230ee249c9fb52cfa7" @@ -929,6 +949,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-15.3.0.tgz#d6fed7d6bc6854306da3dea1af9f874b00783e26" integrity sha512-8/bnjSZD86ZfpBsDlCIkNXIvm+h6wi9g7IqL+kmFkQ+Wvu3JrasgLElfiPgoo8V8vVfnEi0QVS12gbl94h9YsQ== +"@types/node@^20.8.7": + version "20.8.7" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.7.tgz#ad23827850843de973096edfc5abc9e922492a25" + integrity sha512-21TKHHh3eUHIi2MloeptJWALuCu5H7HQTdTrWIFReA8ad+aggoX+lRes3ex7/FtpC+sVUpFMQ+QTfYr74mruiQ== + dependencies: + undici-types "~5.25.1" + "@types/prettier@^2.1.5": version "2.7.1" resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.1.tgz#dfd20e2dc35f027cdd6c1908e80a5ddc7499670e" @@ -941,11 +968,28 @@ dependencies: "@types/node" "*" +"@types/semver@^7.5.0": + version "7.5.5" + resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.5.tgz#deed5ab7019756c9c90ea86139106b0346223f35" + integrity sha512-+d+WYC1BxJ6yVOgUgzK8gWvp5qF8ssV5r4nsDcZWKRWcDQLQ619tvWAxJQYGgBrO1MnLJC7a5GtiYsAoQ47dJg== + "@types/stack-utils@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.0.tgz#7036640b4e21cc2f259ae826ce843d277dad8cff" integrity sha512-RJJrrySY7A8havqpGObOB4W92QXKJo63/jFLLgpvOtsGUqbQZ9Sbgl35KMm1DjC6j7AvmmU2bIno+3IyEaemaw== +"@types/uuid@^9.0.6": + version "9.0.6" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.6.tgz#c91ae743d8344a54b2b0c691195f5ff5265f6dfb" + integrity sha512-BT2Krtx4xaO6iwzwMFUYvWBWkV2pr37zD68Vmp1CDV196MzczBRxuEpD6Pr395HAgebC/co7hOphs53r8V7jew== + +"@types/ws@^8.5.8": + version "8.5.8" + resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.8.tgz#13efec7bd439d0bdf2af93030804a94f163b1430" + integrity sha512-flUksGIQCnJd6sZ1l5dqCEG/ksaoAg/eUwiLAGTJQcfgvZJKF++Ta4bJA6A5aPSJmsr+xlseHn4KLgVlNnvPTg== + dependencies: + "@types/node" "*" + "@types/yargs-parser@*": version "20.2.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.0.tgz#dd3e6699ba3237f0348cd085e4698780204842f9" @@ -965,6 +1009,96 @@ dependencies: "@types/node" "*" +"@typescript-eslint/eslint-plugin@^6.10.0": + version "6.10.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.10.0.tgz#cfe2bd34e26d2289212946b96ab19dcad64b661a" + integrity sha512-uoLj4g2OTL8rfUQVx2AFO1hp/zja1wABJq77P6IclQs6I/m9GLrm7jCdgzZkvWdDCQf1uEvoa8s8CupsgWQgVg== + dependencies: + "@eslint-community/regexpp" "^4.5.1" + "@typescript-eslint/scope-manager" "6.10.0" + "@typescript-eslint/type-utils" "6.10.0" + "@typescript-eslint/utils" "6.10.0" + "@typescript-eslint/visitor-keys" "6.10.0" + debug "^4.3.4" + graphemer "^1.4.0" + ignore "^5.2.4" + natural-compare "^1.4.0" + semver "^7.5.4" + ts-api-utils "^1.0.1" + +"@typescript-eslint/parser@^6.10.0": + version "6.10.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.10.0.tgz#578af79ae7273193b0b6b61a742a2bc8e02f875a" + integrity sha512-+sZwIj+s+io9ozSxIWbNB5873OSdfeBEH/FR0re14WLI6BaKuSOnnwCJ2foUiu8uXf4dRp1UqHP0vrZ1zXGrog== + dependencies: + "@typescript-eslint/scope-manager" "6.10.0" + "@typescript-eslint/types" "6.10.0" + "@typescript-eslint/typescript-estree" "6.10.0" + "@typescript-eslint/visitor-keys" "6.10.0" + debug "^4.3.4" + +"@typescript-eslint/scope-manager@6.10.0": + version "6.10.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.10.0.tgz#b0276118b13d16f72809e3cecc86a72c93708540" + integrity sha512-TN/plV7dzqqC2iPNf1KrxozDgZs53Gfgg5ZHyw8erd6jd5Ta/JIEcdCheXFt9b1NYb93a1wmIIVW/2gLkombDg== + dependencies: + "@typescript-eslint/types" "6.10.0" + "@typescript-eslint/visitor-keys" "6.10.0" + +"@typescript-eslint/type-utils@6.10.0": + version "6.10.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.10.0.tgz#1007faede067c78bdbcef2e8abb31437e163e2e1" + integrity sha512-wYpPs3hgTFblMYwbYWPT3eZtaDOjbLyIYuqpwuLBBqhLiuvJ+9sEp2gNRJEtR5N/c9G1uTtQQL5AhV0fEPJYcg== + dependencies: + "@typescript-eslint/typescript-estree" "6.10.0" + "@typescript-eslint/utils" "6.10.0" + debug "^4.3.4" + ts-api-utils "^1.0.1" + +"@typescript-eslint/types@6.10.0": + version "6.10.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.10.0.tgz#f4f0a84aeb2ac546f21a66c6e0da92420e921367" + integrity sha512-36Fq1PWh9dusgo3vH7qmQAj5/AZqARky1Wi6WpINxB6SkQdY5vQoT2/7rW7uBIsPDcvvGCLi4r10p0OJ7ITAeg== + +"@typescript-eslint/typescript-estree@6.10.0": + version "6.10.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.10.0.tgz#667381eed6f723a1a8ad7590a31f312e31e07697" + integrity sha512-ek0Eyuy6P15LJVeghbWhSrBCj/vJpPXXR+EpaRZqou7achUWL8IdYnMSC5WHAeTWswYQuP2hAZgij/bC9fanBg== + dependencies: + "@typescript-eslint/types" "6.10.0" + "@typescript-eslint/visitor-keys" "6.10.0" + debug "^4.3.4" + globby "^11.1.0" + is-glob "^4.0.3" + semver "^7.5.4" + ts-api-utils "^1.0.1" + +"@typescript-eslint/utils@6.10.0": + version "6.10.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.10.0.tgz#4d76062d94413c30e402c9b0df8c14aef8d77336" + integrity sha512-v+pJ1/RcVyRc0o4wAGux9x42RHmAjIGzPRo538Z8M1tVx6HOnoQBCX/NoadHQlZeC+QO2yr4nNSFWOoraZCAyg== + dependencies: + "@eslint-community/eslint-utils" "^4.4.0" + "@types/json-schema" "^7.0.12" + "@types/semver" "^7.5.0" + "@typescript-eslint/scope-manager" "6.10.0" + "@typescript-eslint/types" "6.10.0" + "@typescript-eslint/typescript-estree" "6.10.0" + semver "^7.5.4" + +"@typescript-eslint/visitor-keys@6.10.0": + version "6.10.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.10.0.tgz#b9eaf855a1ac7e95633ae1073af43d451e8f84e3" + integrity sha512-xMGluxQIEtOM7bqFCo+rCMh5fqI+ZxV5RUUOa29iVPz1OgCZrtc7rFnz5cLUazlkPKYqX+75iuDq7m0HQ48nCg== + dependencies: + "@typescript-eslint/types" "6.10.0" + eslint-visitor-keys "^3.4.1" + +"@ungap/structured-clone@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" + integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== + "@webrecorder/wabac@^2.16.12": version "2.16.12" resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.16.12.tgz#cf9ce5490cffcc34f0c1c4a30245276a094d78b2" @@ -1016,10 +1150,10 @@ acorn@^8.10.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.10.0.tgz#8be5b3907a67221a81ab23c7889c4c5526b62ec5" integrity sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw== -acorn@^8.8.0: - version "8.8.2" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" - integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== +acorn@^8.9.0: + version "8.11.2" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.2.tgz#ca0d78b51895be5390a5903c5b3bdcdaf78ae40b" + integrity sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w== agent-base@^7.0.1, agent-base@^7.0.2, agent-base@^7.1.0: version "7.1.0" @@ -1028,7 +1162,7 @@ agent-base@^7.0.1, agent-base@^7.0.2, agent-base@^7.1.0: dependencies: debug "^4.3.4" -ajv@^6.10.0, ajv@^6.12.4: +ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -1105,6 +1239,11 @@ array-includes@^3.1.2, array-includes@^3.1.3: get-intrinsic "^1.1.1" is-string "^1.0.5" +array-union@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" + integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== + array.prototype.flatmap@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/array.prototype.flatmap/-/array.prototype.flatmap-1.2.4.tgz#94cfd47cc1556ec0747d97f7c7738c58122004c9" @@ -1115,16 +1254,6 @@ array.prototype.flatmap@^1.2.4: es-abstract "^1.18.0-next.1" function-bind "^1.1.1" -asn1.js@^5.2.0: - version "5.4.1" - resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-5.4.1.tgz#11a980b84ebb91781ce35b0fdc2ee294e3783f07" - integrity sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA== - dependencies: - bn.js "^4.0.0" - inherits "^2.0.1" - minimalistic-assert "^1.0.0" - safer-buffer "^2.1.0" - asn1js@^3.0.5: version "3.0.5" resolved "https://registry.yarnpkg.com/asn1js/-/asn1js-3.0.5.tgz#5ea36820443dbefb51cc7f88a2ebb5b462114f38" @@ -1141,10 +1270,10 @@ ast-types@^0.13.4: dependencies: tslib "^2.0.1" -async@^3.1.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" - integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== +async@^3.2.4: + version "3.2.4" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== auto-js-ipfs@^2.1.1: version "2.3.0" @@ -1252,23 +1381,13 @@ bl@^4.0.3: inherits "^2.0.4" readable-stream "^3.4.0" -block-stream2@^2.0.0: +block-stream2@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/block-stream2/-/block-stream2-2.1.0.tgz#ac0c5ef4298b3857796e05be8ebed72196fa054b" integrity sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg== dependencies: readable-stream "^3.4.0" -bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.11.9: - version "4.12.0" - resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88" - integrity sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA== - -bn.js@^5.0.0, bn.js@^5.1.1: - version "5.2.0" - resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.2.0.tgz#358860674396c6997771a9d051fcc1b57d4ae002" - integrity sha512-D7iWRBvnZE8ecXiLj/9wbxH7Tk79fAh8IHaTNq1RWRixsS02W+5qS+iE9yq6RYl0asXx5tw0bLhmT5pIfbSquw== - brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -1284,11 +1403,6 @@ braces@^3.0.2: dependencies: fill-range "^7.0.1" -brorand@^1.0.1, brorand@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/brorand/-/brorand-1.1.0.tgz#12c25efe40a45e3c323eb8675a0a0ce57b22371f" - integrity sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8= - brotli@^1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/brotli/-/brotli-1.3.3.tgz#7365d8cc00f12cf765d2b2c898716bcf4b604d48" @@ -1296,64 +1410,10 @@ brotli@^1.3.3: dependencies: base64-js "^1.1.2" -browser-or-node@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/browser-or-node/-/browser-or-node-1.3.0.tgz#f2a4e8568f60263050a6714b2cc236bb976647a7" - integrity sha512-0F2z/VSnLbmEeBcUrSuDH5l0HxTXdQQzLjkmBR4cYfvg1zJrKSlmIZFqyFR8oX0NrwPhy3c3HQ6i3OxMbew4Tg== - -browserify-aes@^1.0.0, browserify-aes@^1.0.4: - version "1.2.0" - resolved "https://registry.yarnpkg.com/browserify-aes/-/browserify-aes-1.2.0.tgz#326734642f403dabc3003209853bb70ad428ef48" - integrity sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA== - dependencies: - buffer-xor "^1.0.3" - cipher-base "^1.0.0" - create-hash "^1.1.0" - evp_bytestokey "^1.0.3" - inherits "^2.0.1" - safe-buffer "^5.0.1" - -browserify-cipher@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/browserify-cipher/-/browserify-cipher-1.0.1.tgz#8d6474c1b870bfdabcd3bcfcc1934a10e94f15f0" - integrity sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w== - dependencies: - browserify-aes "^1.0.4" - browserify-des "^1.0.0" - evp_bytestokey "^1.0.0" - -browserify-des@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/browserify-des/-/browserify-des-1.0.2.tgz#3af4f1f59839403572f1c66204375f7a7f703e9c" - integrity sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A== - dependencies: - cipher-base "^1.0.1" - des.js "^1.0.0" - inherits "^2.0.1" - safe-buffer "^5.1.2" - -browserify-rsa@^4.0.0, browserify-rsa@^4.0.1: - version "4.1.0" - resolved "https://registry.yarnpkg.com/browserify-rsa/-/browserify-rsa-4.1.0.tgz#b2fd06b5b75ae297f7ce2dc651f918f5be158c8d" - integrity sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog== - dependencies: - bn.js "^5.0.0" - randombytes "^2.0.1" - -browserify-sign@^4.0.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/browserify-sign/-/browserify-sign-4.2.1.tgz#eaf4add46dd54be3bb3b36c0cf15abbeba7956c3" - integrity sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg== - dependencies: - bn.js "^5.1.1" - browserify-rsa "^4.0.1" - create-hash "^1.2.0" - create-hmac "^1.1.7" - elliptic "^6.5.3" - inherits "^2.0.4" - parse-asn1 "^5.1.5" - readable-stream "^3.6.0" - safe-buffer "^5.2.0" +browser-or-node@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/browser-or-node/-/browser-or-node-2.1.1.tgz#738790b3a86a8fc020193fa581273fbe65eaea0f" + integrity sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg== browserslist@^4.21.3: version "4.21.4" @@ -1377,7 +1437,7 @@ bser@2.1.1: dependencies: node-int64 "^0.4.0" -buffer-crc32@~0.2.3: +buffer-crc32@^0.2.13, buffer-crc32@~0.2.3: version "0.2.13" resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== @@ -1387,11 +1447,6 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== -buffer-xor@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" - integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= - buffer@^5.2.1, buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" @@ -1498,14 +1553,6 @@ ci-info@^3.2.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.5.0.tgz#bfac2a29263de4c829d806b1ab478e35091e171f" integrity sha512-yH4RezKOGlOhxkmhbeNuC4eYZKAUsEaGtBuBzDDP1eFUKiccDWzBABxBfOx31IDwDIXMTxWuwAxUGModvkbuVw== -cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: - version "1.0.4" - resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de" - integrity sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q== - dependencies: - inherits "^2.0.1" - safe-buffer "^5.0.1" - cjs-module-lexer@^1.0.0: version "1.2.2" resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.2.tgz#9f84ba3244a512f3a54e5277e8eef4c489864e40" @@ -1599,37 +1646,6 @@ crc@^4.3.2: resolved "https://registry.yarnpkg.com/crc/-/crc-4.3.2.tgz#49b7821cbf2cf61dfd079ed93863bbebd5469b9a" integrity sha512-uGDHf4KLLh2zsHa8D8hIQ1H/HtFQhyHrc0uhHBcoKGol/Xnb+MPYfUMw7cvON6ze/GUESTudKayDcJC5HnJv1A== -create-ecdh@^4.0.0: - version "4.0.4" - resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.4.tgz#d6e7f4bffa66736085a0762fd3a632684dabcc4e" - integrity sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A== - dependencies: - bn.js "^4.1.0" - elliptic "^6.5.3" - -create-hash@^1.1.0, create-hash@^1.1.2, create-hash@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/create-hash/-/create-hash-1.2.0.tgz#889078af11a63756bcfb59bd221996be3a9ef196" - integrity sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg== - dependencies: - cipher-base "^1.0.1" - inherits "^2.0.1" - md5.js "^1.3.4" - ripemd160 "^2.0.1" - sha.js "^2.4.0" - -create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7: - version "1.1.7" - resolved "https://registry.yarnpkg.com/create-hmac/-/create-hmac-1.1.7.tgz#69170c78b3ab957147b2b8b04572e47ead2243ff" - integrity sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg== - dependencies: - cipher-base "^1.0.3" - create-hash "^1.1.0" - inherits "^2.0.1" - ripemd160 "^2.0.0" - safe-buffer "^5.0.1" - sha.js "^2.4.8" - cross-fetch@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-4.0.0.tgz#f037aef1580bb3a1a35164ea2a848ba81b445983" @@ -1651,23 +1667,6 @@ crypt@0.0.2: resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" integrity sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs= -crypto-browserify@^3.12.0: - version "3.12.0" - resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec" - integrity sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg== - dependencies: - browserify-cipher "^1.0.0" - browserify-sign "^4.0.0" - create-ecdh "^4.0.0" - create-hash "^1.1.0" - create-hmac "^1.1.0" - diffie-hellman "^5.0.0" - inherits "^2.0.1" - pbkdf2 "^3.0.3" - public-encrypt "^4.0.0" - randombytes "^2.0.0" - randomfill "^1.0.3" - crypto-random-string@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-4.0.0.tgz#5a3cc53d7dd86183df5da0312816ceeeb5bb1fc2" @@ -1687,13 +1686,18 @@ debug@4, debug@4.3.4, debug@^4.3.2, debug@^4.3.4: dependencies: ms "2.1.2" -debug@^4.1.0, debug@^4.1.1, debug@^4.3.1: +debug@^4.1.0, debug@^4.1.1: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" integrity sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ== dependencies: ms "2.1.2" +decode-uri-component@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.2.tgz#e69dbe25d37941171dd540e024c444cd5188e1e9" + integrity sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ== + decompress-response@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" @@ -1742,18 +1746,10 @@ degenerator@^5.0.0: escodegen "^2.1.0" esprima "^4.0.1" -denque@^1.1.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de" - integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ== - -des.js@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/des.js/-/des.js-1.0.1.tgz#5382142e1bdc53f85d86d53e5f4aa7deb91e0843" - integrity sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA== - dependencies: - inherits "^2.0.1" - minimalistic-assert "^1.0.0" +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== detect-libc@^2.0.0, detect-libc@^2.0.1: version "2.0.1" @@ -1775,14 +1771,12 @@ diff-sequences@^29.2.0: resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.2.0.tgz#4c55b5b40706c7b5d2c5c75999a50c56d214e8f6" integrity sha512-413SY5JpYeSBZxmenGEmCVQ8mCgtFJF0w9PROdaS6z987XC2Pd2GOKqOITLtMftmyFZqgtCOb/QA7/Z3ZXfzIw== -diffie-hellman@^5.0.0: - version "5.0.3" - resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875" - integrity sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg== +dir-glob@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" + integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== dependencies: - bn.js "^4.1.0" - miller-rabin "^4.0.0" - randombytes "^2.0.0" + path-type "^4.0.0" doctrine@^2.1.0: version "2.1.0" @@ -1803,19 +1797,6 @@ electron-to-chromium@^1.4.251: resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz#61046d1e4cab3a25238f6bf7413795270f125592" integrity sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA== -elliptic@^6.5.3: - version "6.5.4" - resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.4.tgz#da37cebd31e79a1367e941b592ed1fbebd58abbb" - integrity sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ== - dependencies: - bn.js "^4.11.9" - brorand "^1.1.0" - hash.js "^1.0.0" - hmac-drbg "^1.0.1" - inherits "^2.0.4" - minimalistic-assert "^1.0.1" - minimalistic-crypto-utils "^1.0.1" - emittery@^0.10.2: version "0.10.2" resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.10.2.tgz#902eec8aedb8c41938c46e9385e9db7e03182933" @@ -1902,11 +1883,6 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" -es6-error@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" - integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg== - escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" @@ -1956,40 +1932,46 @@ eslint-plugin-react@^7.22.0: resolve "^2.0.0-next.3" string.prototype.matchall "^4.0.4" -eslint-scope@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.1.1.tgz#fff34894c2f65e5226d3041ac480b4513a163642" - integrity sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw== +eslint-scope@^7.2.2: + version "7.2.2" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.2.2.tgz#deb4f92563390f32006894af62a22dba1c46423f" + integrity sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg== dependencies: esrecurse "^4.3.0" estraverse "^5.2.0" -eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.0: +eslint-visitor-keys@^3.3.0: version "3.4.0" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz#c7f0f956124ce677047ddbc192a68f999454dedc" integrity sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ== -eslint@^8.37.0: - version "8.37.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.37.0.tgz#1f660ef2ce49a0bfdec0b0d698e0b8b627287412" - integrity sha512-NU3Ps9nI05GUoVMxcZx1J8CNR6xOvUT4jAUMH5+z8lpp3aEdPVCImKw6PWG4PY+Vfkpr+jvMpxs/qoE7wq0sPw== +eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4.3: + version "3.4.3" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" + integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== + +eslint@^8.53.0: + version "8.53.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.53.0.tgz#14f2c8244298fcae1f46945459577413ba2697ce" + integrity sha512-N4VuiPjXDUa4xVeV/GC/RV3hQW9Nw+Y463lkWaKKXKYMvmRiRDAtfpuPFLN+E1/6ZhyR8J2ig+eVREnYgUsiag== dependencies: "@eslint-community/eslint-utils" "^4.2.0" - "@eslint-community/regexpp" "^4.4.0" - "@eslint/eslintrc" "^2.0.2" - "@eslint/js" "8.37.0" - "@humanwhocodes/config-array" "^0.11.8" + "@eslint-community/regexpp" "^4.6.1" + "@eslint/eslintrc" "^2.1.3" + "@eslint/js" "8.53.0" + "@humanwhocodes/config-array" "^0.11.13" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" - ajv "^6.10.0" + "@ungap/structured-clone" "^1.2.0" + ajv "^6.12.4" chalk "^4.0.0" cross-spawn "^7.0.2" debug "^4.3.2" doctrine "^3.0.0" escape-string-regexp "^4.0.0" - eslint-scope "^7.1.1" - eslint-visitor-keys "^3.4.0" - espree "^9.5.1" + eslint-scope "^7.2.2" + eslint-visitor-keys "^3.4.3" + espree "^9.6.1" esquery "^1.4.2" esutils "^2.0.2" fast-deep-equal "^3.1.3" @@ -1997,32 +1979,29 @@ eslint@^8.37.0: find-up "^5.0.0" glob-parent "^6.0.2" globals "^13.19.0" - grapheme-splitter "^1.0.4" + graphemer "^1.4.0" ignore "^5.2.0" - import-fresh "^3.0.0" imurmurhash "^0.1.4" is-glob "^4.0.0" is-path-inside "^3.0.3" - js-sdsl "^4.1.4" js-yaml "^4.1.0" json-stable-stringify-without-jsonify "^1.0.1" levn "^0.4.1" lodash.merge "^4.6.2" minimatch "^3.1.2" natural-compare "^1.4.0" - optionator "^0.9.1" + optionator "^0.9.3" strip-ansi "^6.0.1" - strip-json-comments "^3.1.0" text-table "^0.2.0" -espree@^9.5.1: - version "9.5.1" - resolved "https://registry.yarnpkg.com/espree/-/espree-9.5.1.tgz#4f26a4d5f18905bf4f2e0bd99002aab807e96dd4" - integrity sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg== +espree@^9.6.0, espree@^9.6.1: + version "9.6.1" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.6.1.tgz#a2a17b8e434690a5432f2f8018ce71d331a48c6f" + integrity sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ== dependencies: - acorn "^8.8.0" + acorn "^8.9.0" acorn-jsx "^5.3.2" - eslint-visitor-keys "^3.4.0" + eslint-visitor-keys "^3.4.1" esprima@^4.0.0, esprima@^4.0.1: version "4.0.1" @@ -2058,14 +2037,6 @@ eventemitter3@^4.0.7: resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== -evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz#7fcbdb198dc71959432efe13842684e0525acb02" - integrity sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA== - dependencies: - md5.js "^1.3.4" - safe-buffer "^5.1.1" - execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" @@ -2123,6 +2094,17 @@ fast-fifo@^1.1.0, fast-fifo@^1.2.0: resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.3.0.tgz#03e381bcbfb29932d7c3afde6e15e83e05ab4d8b" integrity sha512-IgfweLvEpwyA4WgiQe9Nx6VV2QkML2NkvZnk1oKnIzXgXdWxuhF7zw4DvLTPZJn6PIUneiAXPF24QmoEqHTjyw== +fast-glob@^3.2.9: + version "3.3.2" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129" + integrity sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow== + dependencies: + "@nodelib/fs.stat" "^2.0.2" + "@nodelib/fs.walk" "^1.2.3" + glob-parent "^5.1.2" + merge2 "^1.3.0" + micromatch "^4.0.4" + fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" @@ -2133,10 +2115,12 @@ fast-levenshtein@^2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= -fast-xml-parser@^3.17.5: - version "3.19.0" - resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.19.0.tgz#cb637ec3f3999f51406dd8ff0e6fc4d83e520d01" - integrity sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg== +fast-xml-parser@^4.2.2: + version "4.3.2" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.3.2.tgz#761e641260706d6e13251c4ef8e3f5694d4b0d79" + integrity sha512-rmrXUXwbJedoXkStenj1kkljNF7ugn5ZjR9FJcwmCfcCbtOMDghPajbc+Tck6vE6F5XsDmx+Pr2le9fw8+pXBg== + dependencies: + strnum "^1.0.5" fast-xml-parser@^4.2.5: version "4.2.6" @@ -2180,6 +2164,11 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" +filter-obj@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/filter-obj/-/filter-obj-1.1.0.tgz#9b311112bc6c6127a16e016c6c5d7f19e0805c5b" + integrity sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ== + find-up@^4.0.0, find-up@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" @@ -2314,6 +2303,13 @@ github-from-package@0.0.0: resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" integrity sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw== +glob-parent@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== + dependencies: + is-glob "^4.0.1" + glob-parent@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3" @@ -2345,6 +2341,18 @@ globals@^13.19.0: dependencies: type-fest "^0.20.2" +globby@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" + integrity sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g== + dependencies: + array-union "^2.1.0" + dir-glob "^3.0.1" + fast-glob "^3.2.9" + ignore "^5.2.0" + merge2 "^1.4.1" + slash "^3.0.0" + got@^11.8.0: version "11.8.2" resolved "https://registry.yarnpkg.com/got/-/got-11.8.2.tgz#7abb3959ea28c31f3576f1576c1effce23f33599" @@ -2372,10 +2380,10 @@ graceful-fs@^4.2.9: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== -grapheme-splitter@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz#9cf3a665c6247479896834af35cf1dbb4400767e" - integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ== +graphemer@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" + integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== has-bigints@^1.0.1: version "1.0.1" @@ -2411,37 +2419,11 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" -hash-base@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/hash-base/-/hash-base-3.1.0.tgz#55c381d9e06e1d2997a883b4a3fddfe7f0d3af33" - integrity sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA== - dependencies: - inherits "^2.0.4" - readable-stream "^3.6.0" - safe-buffer "^5.2.0" - hash-wasm@^4.9.0: version "4.9.0" resolved "https://registry.yarnpkg.com/hash-wasm/-/hash-wasm-4.9.0.tgz#7e9dcc9f7d6bd0cc802f2a58f24edce999744206" integrity sha512-7SW7ejyfnRxuOc7ptQHSf4LDoZaWOivfzqw+5rpcQku0nHfmicPKE51ra9BiRLAmT8+gGLestr1XroUkqdjL6w== -hash.js@^1.0.0, hash.js@^1.0.3: - version "1.1.7" - resolved "https://registry.yarnpkg.com/hash.js/-/hash.js-1.1.7.tgz#0babca538e8d4ee4a0f8988d68866537a003cf42" - integrity sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA== - dependencies: - inherits "^2.0.3" - minimalistic-assert "^1.0.1" - -hmac-drbg@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz#d2745701025a6c775a6c545793ed502fc0c649a1" - integrity sha1-0nRXAQJabHdabFRXk+1QL8DGSaE= - dependencies: - hash.js "^1.0.3" - minimalistic-assert "^1.0.0" - minimalistic-crypto-utils "^1.0.1" - html-escaper@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" @@ -2514,12 +2496,12 @@ ieee754@^1.1.13, ieee754@^1.2.1: resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== -ignore@^5.2.0: +ignore@^5.2.0, ignore@^5.2.4: version "5.2.4" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ== -import-fresh@^3.0.0, import-fresh@^3.2.1: +import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== @@ -2548,7 +2530,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.4: +inherits@2, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -2567,18 +2549,17 @@ internal-slot@^1.0.3: has "^1.0.3" side-channel "^1.0.4" -ioredis@^4.27.1: - version "4.27.2" - resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.27.2.tgz#6a79bca05164482da796f8fa010bccefd3bf4811" - integrity sha512-7OpYymIthonkC2Jne5uGWXswdhlua1S1rWGAERaotn0hGJWTSURvxdHA9G6wNbT/qKCloCja/FHsfKXW8lpTmg== +ioredis@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7" + integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== dependencies: + "@ioredis/commands" "^1.1.1" cluster-key-slot "^1.1.0" - debug "^4.3.1" - denque "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" lodash.defaults "^4.2.0" - lodash.flatten "^4.4.0" - p-map "^2.1.0" - redis-commands "1.7.0" + lodash.isarguments "^3.1.0" redis-errors "^1.2.0" redis-parser "^3.0.0" standard-as-callback "^2.1.0" @@ -2691,7 +2672,7 @@ is-glob@^4.0.0: dependencies: is-extglob "^2.1.1" -is-glob@^4.0.3: +is-glob@^4.0.1, is-glob@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== @@ -3209,11 +3190,6 @@ js-levenshtein@^1.1.6: resolved "https://registry.yarnpkg.com/js-levenshtein/-/js-levenshtein-1.1.6.tgz#c6cee58eb3550372df8deb85fad5ce66ce01d59d" integrity sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g== -js-sdsl@^4.1.4: - version "4.4.0" - resolved "https://registry.yarnpkg.com/js-sdsl/-/js-sdsl-4.4.0.tgz#8b437dbe642daa95760400b602378ed8ffea8430" - integrity sha512-FfVSdx6pJ41Oa+CF7RDaFmTnCaFhua+SNYQX74riGOpl96x+2jQCqEfQ2bnXu/5DPCqlRuiqyvTJM0Qjz26IVg== - "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -3333,10 +3309,10 @@ lodash.defaults@^4.2.0: resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw= -lodash.flatten@^4.4.0: - version "4.4.0" - resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f" - integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8= +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== lodash.merge@^4.6.2: version "4.6.2" @@ -3386,15 +3362,6 @@ makeerror@1.0.12: dependencies: tmpl "1.0.5" -md5.js@^1.3.4: - version "1.3.5" - resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" - integrity sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg== - dependencies: - hash-base "^3.0.0" - inherits "^2.0.1" - safe-buffer "^5.1.2" - md5@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" @@ -3409,6 +3376,11 @@ merge-stream@^2.0.0: resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== +merge2@^1.3.0, merge2@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" + integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== + micromatch@^4.0.4: version "4.0.5" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" @@ -3417,25 +3389,17 @@ micromatch@^4.0.4: braces "^3.0.2" picomatch "^2.3.1" -miller-rabin@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/miller-rabin/-/miller-rabin-4.0.1.tgz#f080351c865b0dc562a8462966daa53543c78a4d" - integrity sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA== - dependencies: - bn.js "^4.0.0" - brorand "^1.0.1" +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-db@1.48.0: - version "1.48.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d" - integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ== - -mime-types@^2.1.14: - version "2.1.31" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b" - integrity sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg== +mime-types@^2.1.35: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== dependencies: - mime-db "1.48.0" + mime-db "1.52.0" mimic-fn@^2.1.0: version "2.1.0" @@ -3452,16 +3416,6 @@ mimic-response@^3.1.0: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== -minimalistic-assert@^1.0.0, minimalistic-assert@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz#2e194de044626d4a10e7f7fbc00ce73e83e4d5c7" - integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A== - -minimalistic-crypto-utils@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a" - integrity sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo= - minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -3481,32 +3435,25 @@ minimist@^1.2.0, minimist@^1.2.3: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== -minimist@^1.2.5: - version "1.2.5" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" - integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== - -minio@7.0.26: - version "7.0.26" - resolved "https://registry.yarnpkg.com/minio/-/minio-7.0.26.tgz#83bcda813ed486a8bc7f028efa135d49b02d9bc8" - integrity sha512-knutnEZZMIUB/Xln6psVDrqObFKXDcF9m4IfFIX+zgDHYg3AlcF88DY1wdgg7bUkf+uU8iHkzP2q5CXAhia73w== +minio@^7.1.3: + version "7.1.3" + resolved "https://registry.yarnpkg.com/minio/-/minio-7.1.3.tgz#86dc95f3671045d6956920db757bb63f25bf20ee" + integrity sha512-xPrLjWkTT5E7H7VnzOjF//xBp9I40jYB4aWhb2xTFopXXfw+Wo82DDWngdUju7Doy3Wk7R8C4LAgwhLHHnf0wA== dependencies: - async "^3.1.0" - block-stream2 "^2.0.0" - browser-or-node "^1.3.0" - crypto-browserify "^3.12.0" - es6-error "^4.1.1" - fast-xml-parser "^3.17.5" + async "^3.2.4" + block-stream2 "^2.1.0" + browser-or-node "^2.1.1" + buffer-crc32 "^0.2.13" + fast-xml-parser "^4.2.2" ipaddr.js "^2.0.1" json-stream "^1.0.0" lodash "^4.17.21" - mime-types "^2.1.14" - mkdirp "^0.5.1" - querystring "0.2.0" - through2 "^3.0.1" + mime-types "^2.1.35" + query-string "^7.1.3" + through2 "^4.0.2" web-encoding "^1.1.5" - xml "^1.0.0" - xml2js "^0.4.15" + xml "^1.0.1" + xml2js "^0.5.0" mitt@3.0.0: version "3.0.0" @@ -3518,13 +3465,6 @@ mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== -mkdirp@^0.5.1: - version "0.5.5" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" - integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== - dependencies: - minimist "^1.2.5" - ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" @@ -3665,17 +3605,17 @@ onetime@^5.1.2: dependencies: mimic-fn "^2.1.0" -optionator@^0.9.1: - version "0.9.1" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" - integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== +optionator@^0.9.3: + version "0.9.3" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.3.tgz#007397d44ed1872fdc6ed31360190f81814e2c64" + integrity sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg== dependencies: + "@aashutoshrathi/word-wrap" "^1.2.3" deep-is "^0.1.3" fast-levenshtein "^2.0.6" levn "^0.4.1" prelude-ls "^1.2.1" type-check "^0.4.0" - word-wrap "^1.2.3" p-cancelable@^2.0.0: version "2.1.1" @@ -3710,11 +3650,6 @@ p-locate@^5.0.0: dependencies: p-limit "^3.0.2" -p-map@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" - integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== - p-queue@^7.3.4: version "7.3.4" resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-7.3.4.tgz#7ef7d89b6c1a0563596d98adbc9dc404e9ed4a84" @@ -3768,17 +3703,6 @@ parent-module@^1.0.0: dependencies: callsites "^3.0.0" -parse-asn1@^5.0.0, parse-asn1@^5.1.5: - version "5.1.6" - resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.6.tgz#385080a3ec13cb62a62d39409cb3e88844cdaed4" - integrity sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw== - dependencies: - asn1.js "^5.2.0" - browserify-aes "^1.0.0" - evp_bytestokey "^1.0.0" - pbkdf2 "^3.0.3" - safe-buffer "^5.1.1" - parse-json@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" @@ -3845,16 +3769,10 @@ path-parser@^6.1.0: search-params "3.0.0" tslib "^1.10.0" -pbkdf2@^3.0.3: - version "3.1.2" - resolved "https://registry.yarnpkg.com/pbkdf2/-/pbkdf2-3.1.2.tgz#dd822aa0887580e52f1a039dc3eda108efae3075" - integrity sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA== - dependencies: - create-hash "^1.1.2" - create-hmac "^1.1.4" - ripemd160 "^2.0.1" - safe-buffer "^5.0.1" - sha.js "^2.4.8" +path-type@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" + integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== pend@~1.2.0: version "1.2.0" @@ -3966,18 +3884,6 @@ proxy-from-env@^1.1.0: resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== -public-encrypt@^4.0.0: - version "4.0.3" - resolved "https://registry.yarnpkg.com/public-encrypt/-/public-encrypt-4.0.3.tgz#4fcc9d77a07e48ba7527e7cbe0de33d0701331e0" - integrity sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q== - dependencies: - bn.js "^4.1.0" - browserify-rsa "^4.0.0" - create-hash "^1.1.0" - parse-asn1 "^5.0.0" - randombytes "^2.0.1" - safe-buffer "^5.1.2" - pump@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" @@ -4015,10 +3921,15 @@ pvutils@^1.1.3: resolved "https://registry.yarnpkg.com/pvutils/-/pvutils-1.1.3.tgz#f35fc1d27e7cd3dfbd39c0826d173e806a03f5a3" integrity sha512-pMpnA0qRdFp32b1sJl1wOJNxZLQ2cbQx+k6tjNtZ8CpvVhNqEPRgivZ2WOUev2YMajecdH7ctUPDvEe87nariQ== -querystring@0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= +query-string@^7.1.3: + version "7.1.3" + resolved "https://registry.yarnpkg.com/query-string/-/query-string-7.1.3.tgz#a1cf90e994abb113a325804a972d98276fe02328" + integrity sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg== + dependencies: + decode-uri-component "^0.2.2" + filter-obj "^1.1.0" + split-on-first "^1.0.0" + strict-uri-encode "^2.0.0" queue-microtask@^1.2.2: version "1.2.3" @@ -4035,21 +3946,6 @@ quick-lru@^5.1.1: resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932" integrity sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA== -randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5: - version "2.1.0" - resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" - integrity sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ== - dependencies: - safe-buffer "^5.1.0" - -randomfill@^1.0.3: - version "1.0.4" - resolved "https://registry.yarnpkg.com/randomfill/-/randomfill-1.0.4.tgz#c92196fc86ab42be983f1bf31778224931d61458" - integrity sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw== - dependencies: - randombytes "^2.0.5" - safe-buffer "^5.1.0" - rc@^1.2.7: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" @@ -4070,16 +3966,7 @@ react-is@^18.0.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b" integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== -"readable-stream@2 || 3", readable-stream@^3.4.0, readable-stream@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" - integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - -readable-stream@^3.1.1, readable-stream@^3.5.0: +readable-stream@3, readable-stream@^3.1.1, readable-stream@^3.5.0: version "3.6.2" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== @@ -4088,10 +3975,14 @@ readable-stream@^3.1.1, readable-stream@^3.5.0: string_decoder "^1.1.1" util-deprecate "^1.0.1" -redis-commands@1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89" - integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ== +readable-stream@^3.4.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" + integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" redis-errors@^1.0.0, redis-errors@^1.2.0: version "1.2.0" @@ -4186,14 +4077,6 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" -ripemd160@^2.0.0, ripemd160@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c" - integrity sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA== - dependencies: - hash-base "^3.0.0" - inherits "^2.0.1" - run-parallel@^1.1.9: version "1.2.0" resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" @@ -4201,7 +4084,7 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.0, safe-buffer@~5.2.0: +safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -4211,11 +4094,6 @@ safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safer-buffer@^2.1.0: - version "2.1.2" - resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" - integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -4245,13 +4123,12 @@ semver@^7.5.0: dependencies: lru-cache "^6.0.0" -sha.js@^2.4.0, sha.js@^2.4.8: - version "2.4.11" - resolved "https://registry.yarnpkg.com/sha.js/-/sha.js-2.4.11.tgz#37a5cf0b81ecbc6943de109ba2960d1b26584ae7" - integrity sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ== +semver@^7.5.4: + version "7.5.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== dependencies: - inherits "^2.0.1" - safe-buffer "^5.0.1" + lru-cache "^6.0.0" sharp@^0.32.1: version "0.32.1" @@ -4319,7 +4196,7 @@ sisteransi@^1.0.5: resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg== -sitemapper@^3.2.5: +sitemapper@^3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/sitemapper/-/sitemapper-3.2.6.tgz#892ebdade9a1b0839bd3dee3b67f3d57b10b3a89" integrity sha512-AZbim4lmKgchUj6yyJ9ru0eLJ4/S6QAqy5QEbpCpvBbBnXxTERLMC6rzgKy1gHM19YUEtYJFTC2t8lxDWO0wkQ== @@ -4378,6 +4255,11 @@ source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.1: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== +split-on-first@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/split-on-first/-/split-on-first-1.1.0.tgz#f610afeee3b12bce1d0c30425e76398b78249a5f" + integrity sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw== + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" @@ -4411,6 +4293,11 @@ streamx@^2.15.0: fast-fifo "^1.1.0" queue-tick "^1.0.1" +strict-uri-encode@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546" + integrity sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ== + string-length@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" @@ -4497,7 +4384,7 @@ strip-final-newline@^2.0.0: resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== -strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: +strip-json-comments@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== @@ -4606,13 +4493,12 @@ text-table@^0.2.0: resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= -through2@^3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4" - integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ== +through2@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764" + integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw== dependencies: - inherits "^2.0.4" - readable-stream "2 || 3" + readable-stream "3" through@^2.3.8: version "2.3.8" @@ -4646,6 +4532,16 @@ tr46@~0.0.3: resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== +ts-api-utils@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.0.3.tgz#f12c1c781d04427313dbac808f453f050e54a331" + integrity sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg== + +tsc@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/tsc/-/tsc-2.0.4.tgz#5f6499146abea5dca4420b451fa4f2f9345238f5" + integrity sha512-fzoSieZI5KKJVBYGvwbVZs/J5za84f2lSTLPYf6AGiIf43tZ3GNrI1QzTLcjtyDDP4aLxd46RTZq1nQxe7+k5Q== + tslib@^1.10.0, tslib@^1.9.3: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" @@ -4707,6 +4603,11 @@ type-fest@^2.12.2: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-2.19.0.tgz#88068015bb33036a598b952e55e9311a60fd3a9b" integrity sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA== +typescript@^5.2.2: + version "5.2.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" + integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== + unbox-primitive@^1.0.0, unbox-primitive@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471" @@ -4725,6 +4626,11 @@ unbzip2-stream@1.4.3: buffer "^5.2.1" through "^2.3.8" +undici-types@~5.25.1: + version "5.25.3" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.25.3.tgz#e044115914c85f0bcbb229f346ab739f064998c3" + integrity sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA== + unique-string@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-3.0.0.tgz#84a1c377aff5fd7a8bc6b55d8244b2bd90d75b9a" @@ -4807,6 +4713,18 @@ warcio@^2.2.0: uuid-random "^1.3.2" yargs "^17.6.2" +warcio@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/warcio/-/warcio-2.2.1.tgz#3619728fde716291c9b364744c276362a94bacec" + integrity sha512-KPLoz3aFtdTjexG+QQaubMyuLiNANzvcadGMyNKdpcmhl0k6lBHQQVpxZw3Hx9+4pbyqDXyiF4cr/h2tS8kvcw== + dependencies: + base32-encode "^2.0.0" + hash-wasm "^4.9.0" + pako "^1.0.11" + tempy "^3.1.0" + uuid-random "^1.3.2" + yargs "^17.6.2" + web-encoding@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/web-encoding/-/web-encoding-1.1.5.tgz#fc810cf7667364a6335c939913f5051d3e0c4864" @@ -4859,11 +4777,6 @@ which@^2.0.1: dependencies: isexe "^2.0.0" -word-wrap@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" - integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== - wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" @@ -4896,7 +4809,7 @@ ws@^7.4.4: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== -xml2js@^0.4.15, xml2js@^0.4.23: +xml2js@^0.4.23: version "0.4.23" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== @@ -4904,10 +4817,18 @@ xml2js@^0.4.15, xml2js@^0.4.23: sax ">=0.6.0" xmlbuilder "~11.0.0" -xml@^1.0.0: +xml2js@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xml@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" - integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= + integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw== xmlbuilder@~11.0.0: version "11.0.1"