Support for uploading to S3 (#95)

- support uploading WACZ to s3-compatible storage (via minio client) - config storage loaded from env vars, enabled when WACZ output is used. - support pinging either or an http or a redis key-based webhook, - webhook: include 'completed' bool to indicate if fully completed crawl or partial (eg. interrupted via signal) - consolidate redis init to redis.js - support upload filename with custom variables: can interpolate current timestamp (@ts), hostname (@hostname) and user provided id (@crawlId) - README: add docs for s3 storage, remove unused args - update to pywb 2.6.2, browsertrix-behaviors 0.2.4 * fix to `limit` option, ensure limit check uses shared state * bump version to 0.5.0-beta.1
2025-10-19 14:33:17 +00:00 · 2021-11-23 12:53:30 -08:00 · 2021-11-23 12:53:30 -08:00 · 9f541ab011
commit 9f541ab011
parent f5d0328ac0
8 changed files with 288 additions and 19 deletions
--- a/README.md
+++ b/README.md
@ -405,6 +405,45 @@ docker run -p 9037:9037 -v $PWD/crawls:/crawls/ webrecorder/browsertrix-crawler

 will start a crawl with 3 workers, and show the screen of each of the workers from `http://localhost:9037/`.

+### Uploading crawl output to S3-Compatible Storage
+
+Browsertrix Crawler also includes support for uploading WACZ files to S3-compatible storage, and notifying a webhook when the upload succeeds.
+
+(At this time, S3 upload is supported only when WACZ output is enabled, but WARC uploads may be added in the future).
+
+This feature can currently be enabled by setting environment variables (for security reasons, these settings are not passed in as part of the command-line or YAML config at this time).
+
+<details>
+
+<summary>Environment variables for S3-uploads include:</summary>
+
+- `STORE_ACCESS_KEY` / `STORE_SECRET_KEY` - S3 credentials
+- `STORE_ENDPOINT_URL` - S3 endpoint URL
+- `STORE_PATH` - optional path appended to endpoint, if provided
+- `STORE_FILENAME` - filename or template for filename to put on S3
+- `STORE_USER` - optional username to pass back as part of the webhook callback
+- `CRAWL_ID` - unique crawl id (defaults to container hostname)
+- `WEBHOOK_URL` - the URL of the webhook (can be http://, https:// or redis://)
+
+</details>
+
+#### Webhook Notification
+
+The webhook URL can be an HTTP URL which receives a JSON POST request OR a Redis URL, which specifies a redis list key to which the JSON data is pushed as a string.
+
+<details>
+
+<summary>Webhook notification JSON includes:</summary>
+
+- `id` - crawl id (value of `CRAWL_ID`)
+- `userId` - user id (value of `STORE_USER`)
+- `filename` - bucket path + filename of the file
+- `size` - size of WACZ file
+- `hash` - SHA-256 of WACZ file
+- `completed` - boolean of whether crawl fully completed or partially (due to interrupt signal or other error).
+
+</details>
+
 ## Interrupting and Restarting the Crawl

 With version 0.5.0, a crawl can be gracefully interrupted with Ctrl-C (SIGINT) or a SIGTERM.
--- a/crawler.js
+++ b/crawler.js
@ -1,6 +1,7 @@
 const child_process = require("child_process");
 const path = require("path");
 const fs = require("fs");
+const os = require("os");
 const fsp = require("fs/promises");

 // to ignore HTTPS error for HEAD check
@ -17,7 +18,6 @@ const { RedisCrawlState, MemoryCrawlState } = require("./util/state");
 const AbortController = require("abort-controller");
 const Sitemapper = require("sitemapper");
 const { v4: uuidv4 } = require("uuid");
-const Redis = require("ioredis");
 const yaml = require("js-yaml");

 const warcio = require("warcio");
@ -25,8 +25,10 @@ const warcio = require("warcio");
 const behaviors = fs.readFileSync(path.join(__dirname, "node_modules", "browsertrix-behaviors", "dist", "behaviors.js"), {encoding: "utf8"});

 const  TextExtract  = require("./util/textextract");
+const { S3StorageSync } = require("./util/storage");
 const { ScreenCaster } = require("./util/screencaster");
 const { parseArgs } = require("./util/argParser");
+const { initRedis } = require("./util/redis");

 const { getBrowserExe, loadProfile } = require("./util/browser");

@ -43,9 +45,6 @@ class Crawler {

    this.emulateDevice = null;

-    // links crawled counter
-    this.numLinks = 0;
-
    // pages file
    this.pagesFH = null;

@ -148,10 +147,10 @@ class Crawler {
        throw new Error("stateStoreUrl must start with redis:// -- Only redis-based store currently supported");
      }

-      const redis = new Redis(redisUrl, {lazyConnect: true});
+      let redis;

      try {
-        await redis.connect();
+        redis = await initRedis(redisUrl);
      } catch (e) {
        throw new Error("Unable to connect to state store Redis: " + redisUrl);
      }
@ -350,6 +349,25 @@ class Crawler {
      return;
    }

+    if (this.params.generateWACZ && process.env.STORE_ENDPOINT_URL) {
+      const endpointUrl = process.env.STORE_ENDPOINT_URL + (process.env.STORE_PATH || "");
+      const storeInfo = {
+        endpointUrl,
+        accessKey: process.env.STORE_ACCESS_KEY,
+        secretKey: process.env.STORE_SECRET_KEY,
+      };
+
+      const opts = {
+        crawlId: process.env.CRAWL_ID || os.hostname(),
+        webhookUrl: process.env.WEBHOOK_URL,
+        userId: process.env.STORE_USER,
+        filename: process.env.STORE_FILENAME || "@ts-@id.wacz",
+      };
+
+      console.log("Initing Storage...");
+      this.storage = new S3StorageSync(storeInfo, opts);
+    }
+
    // Puppeteer Cluster init and options
    this.cluster = await Cluster.launch({
      concurrency: this.params.newContext,
@ -436,6 +454,11 @@ class Crawler {
      // Run the wacz create command
      child_process.spawnSync("wacz" , argument_list, {stdio: "inherit"});
      this.debugLog(`WACZ successfully generated and saved to: ${waczPath}`);
+
+      if (this.storage) {
+        const finished = await this.crawlState.finished();
+        await this.storage.uploadCollWACZ(waczPath, finished);
+      }
    }
  }

@ -543,7 +566,7 @@ class Crawler {
      return false;
    }

-    if (this.numLinks >= this.params.limit && this.params.limit > 0) {
+    if (this.params.limit > 0 && (await this.crawlState.numRealSeen() >= this.params.limit)) {
      this.limitHit = true;
      return false;
    }
@ -553,7 +576,6 @@ class Crawler {
    }

    await this.crawlState.add(url);
-    this.numLinks++;
    this.cluster.queue({url, seedId, depth});
    return true;
  }
@ -656,7 +678,7 @@ class Crawler {
  async awaitPendingClear() {
    this.statusLog("Waiting to ensure pending data is written to WARCs...");

-    const redis = new Redis("redis://localhost/0");
+    const redis = await initRedis("redis://localhost/0");

    while (true) {
      const res = await redis.get(`pywb:${this.params.collection}:pending`);
--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
  "name": "browsertrix-crawler",
-  "version": "0.5.0-beta.0",
+  "version": "0.5.0-beta.1",
  "main": "browsertrix-crawler",
  "repository": "https://github.com/webrecorder/browsertrix-crawler",
  "author": "Ilya Kreymer <ikreymer@gmail.com>, Webrecorder Software",
@ -10,9 +10,10 @@
  },
  "dependencies": {
    "abort-controller": "^3.0.0",
-    "browsertrix-behaviors": "github:webrecorder/browsertrix-behaviors#skip-mp4-video",
+    "browsertrix-behaviors": "^0.2.4",
    "ioredis": "^4.27.1",
    "js-yaml": "^4.1.0",
+    "minio": "^7.0.18",
    "node-fetch": "^2.6.1",
    "puppeteer-cluster": "github:ikreymer/puppeteer-cluster#async-job-queue",
    "puppeteer-core": "^8.0.0",
--- a/screencast/index.html
+++ b/screencast/index.html
@ -14,7 +14,7 @@
  }
 </style>
 <script>
-  const ws = new WebSocket(window.location.origin.replace("http", "ws") + "/ws");
+  const ws = new WebSocket(window.location.href.replace("http", "ws") + "ws");
  ws.addEventListener("message", (event) => handleMessage(event.data));

  const unusedElems = [];
--- a/tests/custom_driver.test.js
+++ b/tests/custom_driver.test.js
@ -22,10 +22,12 @@ test("ensure custom driver with custom selector crawls JS files as pages", async
    pages.add(url);
  }

+  console.log(pages);
+
  const expectedPages = new Set([
    "https://www.iana.org/",
-    "https://www.iana.org/_js/2013.1/jquery.js",
-    "https://www.iana.org/_js/2013.1/iana.js"
+    "https://www.iana.org/_js/jquery.js",
+    "https://www.iana.org/_js/iana.js"
  ]);

  expect(pages).toEqual(expectedPages);
--- a/util/redis.js
+++ b/util/redis.js
@ -0,0 +1,7 @@
+const Redis = require("ioredis");
+
+module.exports.initRedis = async function(url) {
+  const redis = new Redis(url, {lazyConnect: true});
+  await redis.connect();
+  return redis;
+};
--- a/util/storage.js
+++ b/util/storage.js
@ -0,0 +1,115 @@
+const fs = require("fs");
+const os = require("os");
+const { Transform } = require("stream");
+const { createHash } = require("crypto");
+
+const fetch = require("node-fetch");
+const Minio = require("minio");
+
+const { initRedis } = require("./redis");
+
+class S3StorageSync
+{
+  constructor(urlOrData, {filename, webhookUrl, userId, crawlId} = {}) {
+    let url;
+    let accessKey;
+    let secretKey;
+
+    if (typeof(urlOrData) === "string") {
+      url = new URL(urlOrData);
+      accessKey = url.username;
+      secretKey = url.password;
+      url.username = "";
+      url.password = "";
+      this.fullPrefix = url.href;
+
+    } else {
+      url = new URL(urlOrData.endpointUrl);
+      accessKey = urlOrData.accessKey;
+      secretKey = urlOrData.secretKey;
+      this.fullPrefix = url.href;
+    }
+
+    this.client = new Minio.Client({
+      endPoint: url.hostname,
+      port: Number(url.port) || (url.protocol === "https:" ? 443 : 80),
+      useSSL: url.protocol === "https:",
+      accessKey,
+      secretKey
+    });
+
+    this.bucketName = url.pathname.slice(1).split("/")[0];
+
+    this.objectPrefix = url.pathname.slice(this.bucketName.length + 2);
+
+    this.resources = [];
+
+    this.userId = userId;
+    this.crawlId = crawlId;
+    this.webhookUrl = webhookUrl;
+
+    filename = filename.replace("@ts", new Date().toISOString().replace(/[:TZz.]/g, ""));
+    filename = filename.replace("@hostname", os.hostname());
+    filename = filename.replace("@id", this.crawlId);
+
+    this.waczFilename = "data/" + filename;
+  }
+
+  async uploadCollWACZ(filename, completed = true) {
+    const origStream = fs.createReadStream(filename);
+
+    const hash = createHash("sha256");
+    let size = 0;
+    let finalHash;
+
+    const hashTrans = new Transform({
+      transform(chunk, encoding, callback) {
+        size += chunk.length;
+        hash.update(chunk);
+        this.push(chunk);
+        callback();
+      },
+
+      flush(callback) {
+        finalHash = "sha256:" + hash.digest("hex");
+        callback();
+      }
+    });
+
+    const fsStream = origStream.pipe(hashTrans);
+    const res = await this.client.putObject(this.bucketName, this.objectPrefix + this.waczFilename, fsStream);
+    console.log(res);
+
+    const resource = {"path": this.waczFilename, "hash": finalHash, "bytes": size};
+
+    if (this.webhookUrl) {
+      const body = {
+        id: this.crawlId,
+        user: this.userId,
+
+        //filename: `s3://${this.bucketName}/${this.objectPrefix}${this.waczFilename}`,
+        filename: this.fullPrefix + this.waczFilename,
+
+        hash: resource.hash,
+        size: resource.bytes,
+
+        completed
+      };
+
+      console.log("Pinging Webhook: " + this.webhookUrl);
+
+      if (this.webhookUrl.startsWith("http://") || this.webhookUrl.startsWith("https://")) {
+        await fetch(this.webhookUrl, {method: "POST", body: JSON.stringify(body)});
+      } else if (this.webhookUrl.startsWith("redis://")) {
+        const parts = this.webhookUrl.split("/");
+        if (parts.length !== 5) {
+          throw new Error("redis webhook url must be in format: redis://<host>:<port>/<db>/<key>");
+        }
+        const redis = await initRedis(parts.slice(0, 4).join("/"));
+        await redis.rpush(parts[4], JSON.stringify(body));
+      }
+    }
+  }
+}
+
+module.exports.S3StorageSync = S3StorageSync;
--- a/yarn.lock
+++ b/yarn.lock
@ -886,6 +886,11 @@ astral-regex@^2.0.0:
  resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31"
  integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==

+async@^3.1.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720"
+  integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==
+
 asynckit@^0.4.0:
  version "0.4.0"
  resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@ -1006,6 +1011,13 @@ bl@^4.0.3:
    inherits "^2.0.4"
    readable-stream "^3.4.0"

+block-stream2@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/block-stream2/-/block-stream2-2.1.0.tgz#ac0c5ef4298b3857796e05be8ebed72196fa054b"
+  integrity sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==
+  dependencies:
+    readable-stream "^3.4.0"
+
 brace-expansion@^1.1.7:
  version "1.1.11"
  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@ -1053,9 +1065,10 @@ browserslist@^4.14.5:
    escalade "^3.1.1"
    node-releases "^1.1.71"

-"browsertrix-behaviors@github:webrecorder/browsertrix-behaviors#skip-mp4-video":
+browsertrix-behaviors@^0.2.4:
  version "0.2.4"
-  resolved "https://codeload.github.com/webrecorder/browsertrix-behaviors/tar.gz/50a0538f0a19fba786a7af62ef6c0946e21038b4"
+  resolved "https://registry.yarnpkg.com/browsertrix-behaviors/-/browsertrix-behaviors-0.2.4.tgz#171705e264c094927026cc7d47bc2a4ba817ebcc"
+  integrity sha512-jOtI1mJ/57PXk+JaHvvxbR8D+20lsVWkemkFlb5hpqJve22UuUYyJcS7aopoyCIYFsNnavdx41nM+aAe6pZWZg==

 bser@2.1.1:
  version "2.1.1"
@ -1577,6 +1590,11 @@ es-to-primitive@^1.2.1:
    is-date-object "^1.0.1"
    is-symbol "^1.0.2"

+es6-error@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d"
+  integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==
+
 escalade@^3.1.1:
  version "3.1.1"
  resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
@ -1876,6 +1894,11 @@ fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6:
  resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
  integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=

+fast-xml-parser@^3.17.5:
+  version "3.19.0"
+  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.19.0.tgz#cb637ec3f3999f51406dd8ff0e6fc4d83e520d01"
+  integrity sha512-4pXwmBplsCPv8FOY1WRakF970TjNGnGnfbOnLqjlYvMiF1SR3yOHyxMR/YCXpPTOspNF5gwudqktIP4VsWkvBg==
+
 fb-watchman@^2.0.0:
  version "2.0.1"
  resolved "https://registry.yarnpkg.com/fb-watchman/-/fb-watchman-2.0.1.tgz#fc84fb39d2709cf3ff6d743706157bb5708a8a85"
@ -3050,6 +3073,11 @@ json-stable-stringify-without-jsonify@^1.0.1:
  resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"
  integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=

+json-stream@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/json-stream/-/json-stream-1.0.0.tgz#1a3854e28d2bbeeab31cc7ddf683d2ddc5652708"
+  integrity sha1-GjhU4o0rvuqzHMfd9oPS3cVlJwg=
+
 json-stringify-safe@~5.0.1:
  version "5.0.1"
  resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
@ -3169,7 +3197,7 @@ lodash.truncate@^4.4.2:
  resolved "https://registry.yarnpkg.com/lodash.truncate/-/lodash.truncate-4.4.2.tgz#5a350da0b1113b837ecfffd5812cbe58d6eae193"
  integrity sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=

-lodash@^4.17.19, lodash@^4.17.21, lodash@^4.7.0:
+lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0:
  version "4.17.21"
  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
  integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@ -3265,6 +3293,11 @@ mime-db@1.47.0:
  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.47.0.tgz#8cb313e59965d3c05cfbf898915a267af46a335c"
  integrity sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==

+mime-db@1.48.0:
+  version "1.48.0"
+  resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d"
+  integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==
+
 mime-types@^2.1.12, mime-types@~2.1.19:
  version "2.1.30"
  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.30.tgz#6e7be8b4c479825f85ed6326695db73f9305d62d"
@ -3272,6 +3305,13 @@ mime-types@^2.1.12, mime-types@~2.1.19:
  dependencies:
    mime-db "1.47.0"

+mime-types@^2.1.14:
+  version "2.1.31"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b"
+  integrity sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg==
+  dependencies:
+    mime-db "1.48.0"
+
 mimic-fn@^2.1.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
@ -3299,6 +3339,24 @@ minimist@^1.1.1, minimist@^1.2.0, minimist@^1.2.5:
  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==

+minio@^7.0.18:
+  version "7.0.18"
+  resolved "https://registry.yarnpkg.com/minio/-/minio-7.0.18.tgz#a2a6dae52a4dde9e35ed47cdf2accc21df4a512d"
+  integrity sha512-jVRjkw8A5Spf+ETY5OXQUcQckHriuUA3u2+MAcX36btLT8EytlOVivxIseXvyFf9cNn3dy5w1F1UyjMvHU+nqg==
+  dependencies:
+    async "^3.1.0"
+    block-stream2 "^2.0.0"
+    es6-error "^4.1.1"
+    fast-xml-parser "^3.17.5"
+    json-stream "^1.0.0"
+    lodash "^4.17.20"
+    mime-types "^2.1.14"
+    mkdirp "^0.5.1"
+    querystring "0.2.0"
+    through2 "^3.0.1"
+    xml "^1.0.0"
+    xml2js "^0.4.15"
+
 mixin-deep@^1.2.0:
  version "1.3.2"
  resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566"
@ -3312,6 +3370,13 @@ mkdirp-classic@^0.5.2:
  resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
  integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==

+mkdirp@^0.5.1:
+  version "0.5.5"
+  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"
+  integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==
+  dependencies:
+    minimist "^1.2.5"
+
 ms@2.0.0:
  version "2.0.0"
  resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@ -3784,6 +3849,11 @@ qs@~6.5.2:
  resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
  integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==

+querystring@0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
+  integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
+
 quick-lru@^5.1.1:
  version "5.1.1"
  resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932"
@ -3818,7 +3888,7 @@ read-pkg@^5.2.0:
    parse-json "^5.0.0"
    type-fest "^0.6.0"

-readable-stream@^3.1.1, readable-stream@^3.4.0:
+"readable-stream@2 || 3", readable-stream@^3.1.1, readable-stream@^3.4.0:
  version "3.6.0"
  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
@ -4469,6 +4539,14 @@ throat@^5.0.0:
  resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b"
  integrity sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==

+through2@^3.0.1:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4"
+  integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==
+  dependencies:
+    inherits "^2.0.4"
+    readable-stream "2 || 3"
+
 through@^2.3.8:
  version "2.3.8"
  resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
@ -4866,7 +4944,7 @@ xml-name-validator@^3.0.0:
  resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
  integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==

-xml2js@^0.4.23:
+xml2js@^0.4.15, xml2js@^0.4.23:
  version "0.4.23"
  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
@ -4874,6 +4952,11 @@ xml2js@^0.4.23:
    sax ">=0.6.0"
    xmlbuilder "~11.0.0"

+xml@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
+  integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=
+
 xmlbuilder@~11.0.0:
  version "11.0.1"
  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"