refactor crawler and default driver:

- add extensible defaultDriver, wrap crawling functionality in Crawler class - support headless/non-headless, custom driver - support custom collection name for pywb, generate-cdx option - autoplay: add slightly delay for splash loading
2025-10-19 14:33:17 +00:00 · 2020-11-01 19:22:53 -08:00 · 2020-11-01 19:22:53 -08:00 · 91b8994a08
commit 91b8994a08
parent ded83b52b3
12 changed files with 1131 additions and 363 deletions
--- a/.eslintrc.js
+++ b/.eslintrc.js
@ -0,0 +1,30 @@
+module.exports = {
+    "env": {
+        "es6": true,
+        "node": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+        "ecmaVersion": 2018
+    },
+    "rules": {
+        "indent": [
+            "error",
+            2
+        ],
+        "linebreak-style": [
+            "error",
+            "unix"
+        ],
+        "quotes": [
+            "error",
+            "double"
+        ],
+        "semi": [
+            "error",
+            "always"
+        ],
+        "no-console": "off",
+        "no-unused-vars": "off",
+    }
+};
--- a/.gitignore
+++ b/.gitignore
@ -3,3 +3,4 @@ __pycache__
 *.egg-info/
 collections/
 node_modules/
+output/
--- a/16
+++ b/16
@ -3,7 +3,7 @@ FROM oldwebtoday/chrome:84 as chrome
 FROM nikolaik/python-nodejs:python3.8-nodejs14

 RUN apt-get update -y \
-    && apt-get install --no-install-recommends -qqy fonts-stix locales-all redis-server \
+    && apt-get install --no-install-recommends -qqy fonts-stix locales-all redis-server xvfb \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

@ -11,12 +11,13 @@ ENV PROXY_HOST=localhost \
    PROXY_PORT=8080 \
    PROXY_CA_URL=http://wsgiprox/download/pem \
    PROXY_CA_FILE=/tmp/proxy-ca.pem \
-    NO_SOCAT=1
-
-RUN pip install uwsgi
+    DISPLAY=:99 \
+    GEOMETRY=1360x1020x16

 RUN pip install git+https://github.com/webrecorder/pywb@patch-work

+RUN pip install uwsgi 'gevent>=20.9.0'
+
 COPY --from=chrome /usr/lib/x86_64-linux-gnu/ /usr/lib/x86_64-linux-gnu/
 COPY --from=chrome /lib/x86_64-linux-gnu/libdbus* /lib/x86_64-linux-gnu/
 COPY --from=chrome /opt/google/chrome/ /opt/google/chrome/
@ -29,8 +30,9 @@ RUN yarn install

 ADD config.yaml /app/
 ADD uwsgi.ini /app/
-ADD crawler.js /app/
-ADD autoplay.js /app/
+ADD *.js /app/

-ENTRYPOINT ["crawler.js"]
+WORKDIR /output
+
+ENTRYPOINT ["node", "/app/main.js"]

--- a/autoplay.js
+++ b/autoplay.js
@ -1,12 +1,19 @@
 (() => {
-
  function run() {
-    if (window.navigator.__crawler_autoplay) {
+    if (self.navigator.__crawler_autoplay) {
      return;
    }

+    function loadAutoplay(url) {
+      if (self.__crawler_autoplayLoad) {
+        self.__crawler_autoplayLoad(url);
+      }
+      // delay to allow splash image to load
+      setTimeout(() => self.location.href = url, 1000);
+    }
+
    //console.log("checking autoplay for " + document.location.href);
-    window.navigator.__crawler_autoplay = true;
+    self.navigator.__crawler_autoplay = true;

    const specialActions = [
      {
@ -20,7 +27,7 @@
          // set continuous_play to true in order to handle
          // a playlist etc
          url.searchParams.set('continuous_play', 'true');
-          self.location.href = url.href;
+          loadAutoplay(url.href);
        },
      },
      {
@ -31,10 +38,7 @@
        },
        handle(url) {
          url.searchParams.set('autoplay', '1');
-          if (window.__crawler_autoplayLoad) {
-            window.__crawler_autoplayLoad(url.href);
-          }
-          self.location.href = url.href;
+          loadAutoplay(url.href);
        },
      },
    ];
@ -55,9 +59,9 @@
    }
  }

-  document.addEventListener("readystatechange", run);
+  self.document.addEventListener("readystatechange", run);

-  if (document.readyState === "complete") {
+  if (self.document.readyState === "complete") {
    run();
  }

@ -65,13 +69,13 @@
  const mediaSet = new Set();

  setInterval(() => {
-    const medias = document.querySelectorAll("video, audio");
+    const medias = self.document.querySelectorAll("video, audio");

    for (const media of medias) {
      try {
        if (media.src && !mediaSet.has(media.src)) {
-          if (window.__crawler_queueUrls && (media.src.startsWith("http:") || media.src.startsWith("https:"))) {
-            window.__crawler_queueUrls(media.src);
+          if (self.__crawler_queueUrls && (media.src.startsWith("http:") || media.src.startsWith("https:"))) {
+            self.__crawler_queueUrls(media.src);
          }
          mediaSet.add(media.src);
        } else if (!media.src) {
@ -83,7 +87,5 @@
    }
  }, 3000);

- 
-
 })();

--- a/config.yaml
+++ b/config.yaml
@ -2,7 +2,7 @@ debug: true

 
 proxy:
-  coll: capture
+  coll: ${COLL}
  recording: true

  enable_banner: false
--- a/crawler.js
+++ b/crawler.js
@ -1,13 +1,12 @@
-const fs = require("fs");
 const puppeteer = require("puppeteer-core");
 const { Cluster } = require("puppeteer-cluster");
 const child_process = require("child_process");
 const fetch = require("node-fetch");
 const AbortController = require("abort-controller");
+const path = require("path");

 const HTML_TYPES = ["text/html", "application/xhtml", "application/xhtml+xml"];
 const WAIT_UNTIL_OPTS = ["load", "domcontentloaded", "networkidle0", "networkidle2"];
-const NEW_CONTEXT_OPTS = ["page", "session", "browser"];
 const CHROME_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36";

 // to ignore HTTPS error for HEAD check
@ -15,422 +14,442 @@ const HTTPS_AGENT = require("https").Agent({
  rejectUnauthorized: false,
 });

-process.once('SIGINT', (code) => {
-  console.log('SIGINT received, exiting');
+const HTTP_AGENT = require("http").Agent();
+
+process.once("SIGINT", () => {
+  console.log("SIGINT received, exiting");
  process.exit(1);
 });

-process.once('SIGTERM', (code) => {
-  console.log('SIGTERM received, exiting');
+process.once("SIGTERM", () => {
+  console.log("SIGTERM received, exiting");
  process.exit(1);
 });


-const autoplayScript = fs.readFileSync("./autoplay.js", "utf-8");
+// ============================================================================
+class Crawler {
+  constructor() {
+    this.headers = {"User-Agent": CHROME_USER_AGENT};

+    this.seenList = new Set();

-// prefix for direct capture via pywb
-const capturePrefix = `http://${process.env.PROXY_HOST}:${process.env.PROXY_PORT}/capture/record/id_/`;
-const headers = {"User-Agent": CHROME_USER_AGENT};
+    // links crawled counter
+    this.numLinks = 0;

+    this.monitor = true;

-async function run(params) {
-  // Chrome Flags, including proxy server
-  const args = [
-    "--no-xshm", // needed for Chrome >80 (check if puppeteer adds automatically)
-    `--proxy-server=http://${process.env.PROXY_HOST}:${process.env.PROXY_PORT}`,
-    "--no-sandbox",
-    "--disable-background-media-suspend",
-    "--autoplay-policy=no-user-gesture-required",
-  ];
+    const params = require("yargs")
+      .usage("browsertrix-crawler [options]")
+      .option(this.cliOpts)
+      .check((argv) => this.validateArgs(argv)).argv;

-  // Puppeter Options
-  const puppeteerOptions = {
-    headless: true,
-    executablePath: "/opt/google/chrome/google-chrome",
-    ignoreHTTPSErrors: true,
-    args
-  };
+    console.log("Exclusions Regexes: ", params.exclude);
+    console.log("Scope Regexes: ", params.scope);

-  // params
-  const { url, waitUntil, timeout, scope, limit, exclude, scroll, newContext } = params;
+    this.params = params;
+    this.capturePrefix = `http://${process.env.PROXY_HOST}:${process.env.PROXY_PORT}/${this.params.collection}/record/id_/`;
+  }

-  let concurrency = Cluster.CONCURRENCY_PAGE;
+  bootstrap() {
+    const opts = {stdio: "ignore"};

-  switch (newContext) {
+    child_process.spawn("redis-server", {...opts, cwd: "/tmp/"});
+
+    child_process.spawnSync("wb-manager", ["init", this.params.collection], opts);
+
+    opts.env = {...process.env, COLL: this.params.collection};
+
+    child_process.spawn("uwsgi", [path.join(__dirname, "uwsgi.ini")], opts);
+
+    if (!this.params.headless) {
+      child_process.spawn("Xvfb", [
+        process.env.DISPLAY,
+        "-listen",
+        "tcp",
+        "-screen",
+        "0",
+        process.env.GEOMETRY,
+        "-ac",
+        "+extension",
+        "RANDR"
+      ]);
+    }
+  }
+
+  get cliOpts() {
+    return {
+      "url": {
+        alias: "u",
+        describe: "The URL to start crawling from",
+        type: "string",
+      },
+
+      "workers": {
+        alias: "w",
+        describe: "The number of workers to run in parallel",
+        default: 1,
+        type: "number",
+      },
+
+      "newContext": {
+        describe: "The context for each new capture, can be a new: page, session or browser.",
+        default: "page",
+        type: "string"
+      },
+
+      "waitUntil": {
+        describe: "Puppeteer page.goto() condition to wait for before continuing",
+        default: "load",
+      },
+
+      "limit": {
+        describe: "Limit crawl to this number of pages",
+        default: 0,
+        type: "number",
+      },
+
+      "timeout": {
+        describe: "Timeout for each page to load (in seconds)",
+        default: 90,
+        type: "number",
+      },
+
+      "scope": {
+        describe: "Regex of page URLs that should be included in the crawl (defaults to the immediate directory of URL)",
+      },
+
+      "exclude": {
+        describe: "Regex of page URLs that should be excluded from the crawl."
+      },
+
+      "scroll": {
+        describe: "If set, will autoscroll to bottom of the page",
+        type: "boolean",
+        default: false,
+      },
+
+      "collection": {
+        alias: "c",
+        describe: "Collection Name",
+        type: "string",
+        default: "capture"
+      },
+
+      "headless": {
+        describe: "Run in headless mode, otherwise start xvfb",
+        type: "boolean",
+        default: false,
+      },
+
+      "driver": {
+        describe: "JS driver for the crawler",
+        type: "string",
+        default: path.join(__dirname, "defaultDriver.js"),
+      },
+
+      "generate-cdx": {
+        describe: "If set, generate index (CDXJ) for use with pywb after crawl is done",
+        type: "boolean",
+        default: false,
+      }
+    };
+  }
+
+  validateUserUrl(url) {
+    url = new URL(url);
+
+    if (url.protocol !== "http:" && url.protocol != "https:") {
+      throw new Error("URL must start with http:// or https://");
+    }
+
+    return url.href;
+  }
+
+  validateArgs(argv) {
+    if (argv.url) {
+      // Scope for crawl, default to the domain of the URL
+      // ensure valid url is used (adds trailing slash if missing)
+      //argv.seeds = [Crawler.validateUserUrl(argv.url)];
+      argv.url = this.validateUserUrl(argv.url);
+    }
+
+    if (!argv.scope) {
+      //argv.scope = url.href.slice(0, url.href.lastIndexOf("/") + 1);
+      argv.scope = [new RegExp("^" + this.rxEscape(argv.url.slice(0, argv.url.lastIndexOf("/") + 1)))];
+    }
+
+    argv.timeout *= 1000;
+
+    // waitUntil condition must be: load, domcontentloaded, networkidle0, networkidle2
+    // (see: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagegotourl-options)
+    if (!WAIT_UNTIL_OPTS.includes(argv.waitUntil)) {
+      throw new Error("Invalid waitUntil, must be one of: " + WAIT_UNTIL_OPTS.join(","));
+    }
+
+    if (!argv.newContext) {
+      argv.newContext = "page";
+    }
+
+    switch (argv.newContext) {
    case "page":
-      concurrency = Cluster.CONCURRENCY_PAGE;
+      argv.newContext = Cluster.CONCURRENCY_PAGE;
      break;

    case "session":
-      concurrency = Cluster.CONCURRENCY_CONTEXT;
+      argv.newContext = Cluster.CONCURRENCY_CONTEXT;
      break;

    case "browser":
-      concurrency = Cluster.CONCURRENCY_BROWSER;
+      argv.newContext = Cluster.CONCURRENCY_BROWSER;
      break;
+
+    default:
+      throw new Error("Invalid newContext, must be one of: page, session, browser");
+    }
+
+    // Support one or multiple exclude
+    if (argv.exclude) {
+      if (typeof(argv.exclude) === "string") {
+        argv.exclude = [new RegExp(argv.exclude)];
+      } else {
+        argv.exclude = argv.exclude.map(e => new RegExp(e));
+      }
+    } else {
+      argv.exclude = [];
+    }
+
+    // Support one or multiple scopes
+    if (argv.scope) {
+      if (typeof(argv.scope) === "string") {
+        argv.scope = [new RegExp(argv.scope)];
+      } else {
+        argv.scope = argv.scope.map(e => new RegExp(e));
+      }
+    } else {
+      argv.scope = [];
+    }
+
+    return true;
  }

-  // Puppeteer Cluster init and options
-  const cluster = await Cluster.launch({
-    concurrency,
-    maxConcurrency: Number(params.workers) || 1,
-    skipDuplicateUrls: true,
-    // total timeout for cluster
-    timeout: timeout * 2,
-    puppeteerOptions,
-    puppeteer,
-    monitor: true
-  });
+  get chromeArgs() {
+    // Chrome Flags, including proxy server
+    return [
+      "--no-xshm", // needed for Chrome >80 (check if puppeteer adds automatically)
+      `--proxy-server=http://${process.env.PROXY_HOST}:${process.env.PROXY_PORT}`,
+      "--no-sandbox",
+      "--disable-background-media-suspend",
+      "--autoplay-policy=no-user-gesture-required",
+    ];
+  }

-  // Maintain own seen list
-  const seenList = new Set();
+  get puppeteerArgs() {
+    // Puppeter Options
+    return {
+      headless: this.params.headless,
+      executablePath: "/opt/google/chrome/google-chrome",
+      ignoreHTTPSErrors: true,
+      args: this.chromeArgs
+    };
+  }

-  //console.log("Limit: " + limit);
+  async run() {
+    this.bootstrap();

-  // links crawled counter
-  let numLinks = 0;
+    try {
+      await this.crawl();
+      process.exit(0);
+    } catch(e) {
+      console.error("Crawl failed");
+      console.error(e);
+      process.exit(1);
+    }
+  }

-  // Crawl Task
-  cluster.task(async ({page, data}) => {
-    const {url} = data;
-
-    if (!await htmlCheck(url, capturePrefix)) {
+  async crawl() {
+    try {
+      this.driver = require(this.params.driver);
+    } catch(e) {
+      console.log(e);
      return;
    }

-    //page.on('console', message => console.log(`${message.type()} ${message.text()}`));
-    //page.on('pageerror', message => console.warn(message));
-    //page.on('error', message => console.warn(message));
-    //page.on('requestfailed', message => console.warn(message._failureText));
-    const mediaResults = [];
-
-    await page.exposeFunction('__crawler_queueUrls', (url) => {
-      mediaResults.push(directCapture(url));
+    // Puppeteer Cluster init and options
+    this.cluster = await Cluster.launch({
+      concurrency: this.params.newContext,
+      maxConcurrency: this.params.workers,
+      skipDuplicateUrls: true,
+      timeout: this.params.timeout * 2,
+      puppeteerOptions: this.puppeteerArgs,
+      puppeteer,
+      monitor: this.monitor
    });

-    let waitForVideo = false;
-
-    await page.exposeFunction('__crawler_autoplayLoad', (url) => {
-      console.log("*** Loading autoplay URL: " + url);
-      waitForVideo = true;
-    });
-
-    try {
-      await page.evaluateOnNewDocument(autoplayScript);
-    } catch(e) {
-      console.log(e);
-    }
-
-    try {
-      await page.goto(url, {waitUntil, timeout});
-    } catch (e) {
-      console.log(`Load timeout for ${url}`);
-    }
-
-    try {
-      await Promise.all(mediaResults);
-    } catch (e) {
-      console.log(`Error loading media URLs`, e);
-    }
-
-    if (waitForVideo) {
-      console.log("Extra wait 15s for video loading");
-      await sleep(15000);
-    }
-
-    if (scroll) {
+    this.cluster.task(async (opts) => {
      try {
-        await Promise.race([page.evaluate(autoScroll), sleep(30000)]);
+        await this.driver({...opts, crawler: this});
      } catch (e) {
-        console.warn("Behavior Failed", e);
+        console.warn(e);
      }
-    }
+    });

+    this.queueUrl(this.params.url);
+
+    await this.cluster.idle();
+    await this.cluster.close();
+
+    // extra wait for all resources to land into WARCs
+    console.log("Waiting 5s to ensure WARCs are finished");
+    await this.sleep(5000);
+
+    if (this.params.generateCdx) {
+      console.log("Generate CDX");
+
+      child_process.spawnSync("wb-manager", ["reindex", this.params.collection], {stdio: "inherit"});
+    }
+  }
+
+  async extractLinks(page, selector = "a[href]") {
    let results = null;

    try {
-      results = await page.evaluate(() => {
-        return [...document.querySelectorAll('a[href]')].map(el => ({ url: el.href}))
-      });
+      results = await page.evaluate((selector) => {
+        /* eslint-disable-next-line no-undef */
+        return [...document.querySelectorAll(selector)].map(elem => elem.href);
+      }, selector);
    } catch (e) {
      console.warn("Link Extraction failed", e);
      return;
    }

    try {
-      for (data of results) {
-        const newUrl = shouldCrawl(scope, seenList, data.url, exclude);
+      for (const url of results) {
+        const captureUrl = this.shouldCrawl(url);

-        if (newUrl) {
-          seenList.add(newUrl);
-          if (numLinks++ >= limit && limit > 0) {
+        if (captureUrl) {
+          if (!this.queueUrl(captureUrl)) {
            break;
          }
-          cluster.queue({url: newUrl});
        }
      }
    } catch (e) {
-      console.log("Queuing Error: " + e);
-    }
-  });
-
-  numLinks++;
-  cluster.queue({url});
-
-  await cluster.idle();
-  await cluster.close();
-
-  // extra wait for all resources to land into WARCs
-  console.log("Waiting 30s to ensure WARCs are finished");
-  await sleep(30000);
-}
-
-
-function shouldCrawl(scope, seenList, url, exclude) {
-  try {
-    url = new URL(url);
-  } catch(e) {
-    return false;
-  }
-
-  // remove hashtag
-  url.hash = "";
-
-  // only queue http/https URLs
-  if (url.protocol != "http:" && url.protocol != "https:") {
-    return false;
-  }
-
-  url = url.href;
-
-  // skip already crawled
-  if (seenList.has(url)) {
-    return false;
-  }
-
-  let inScope = false;
-
-  // check scopes
-  for (const s of scope) {
-    if (s.exec(url)) {
-      inScope = true;
-      break;
+      console.log("Queuing Error: ", e);
    }
  }

-  if (!inScope) {
-    //console.log(`Not in scope ${url} ${scope}`);
-    return false;
-  }
-
-  // check exclusions
-  for (const e of exclude) {
-    if (e.exec(url)) {
-      //console.log(`Skipping ${url} excluded by ${e}`);
+  queueUrl(url) {
+    this.seenList.add(url);
+    if (this.numLinks >= this.params.limit && this.params.limit > 0) {
      return false;
    }
-  }
-
-  return url;
-}
-
-async function htmlCheck(url, capturePrefix) {
-  try {
-    const agent = url.startsWith("https:") ? HTTPS_AGENT : null;
-
-    const resp = await fetch(url, {method: "HEAD", headers, agent});
-
-    if (resp.status >= 400) {
-      console.log(`Skipping ${url}, invalid status ${resp.status}`);
-      return false;
-    }
-
-    const contentType = resp.headers.get("Content-Type");
-
-    // just load if no content-type
-    if (!contentType) {
-      return true;
-    }
-
-    const mime = contentType.split(";")[0];
-
-    if (HTML_TYPES.includes(mime)) {
-      return true;
-    }
-
-    // capture directly
-    await directCapture(url);
-
-    return false;
-  } catch(e) {
-    console.log("HTML Check error", e);
-    // can't confirm not html, so try in browser
+    this.numLinks++;
+    this.cluster.queue({url});
    return true;
  }
-}

-async function directCapture(url) {
-  console.log(`Direct capture: ${capturePrefix}${url}`);
-  const abort = new AbortController();
-  const signal = abort.signal;
-  const resp2 = await fetch(capturePrefix + url, {signal, headers});
-  abort.abort();
-}
+  shouldCrawl(url) {
+    try {
+      url = new URL(url);
+    } catch(e) {
+      return false;
+    }

+    // remove hashtag
+    url.hash = "";

+    // only queue http/https URLs
+    if (url.protocol != "http:" && url.protocol != "https:") {
+      return false;
+    }

-async function autoScroll() {
-  const canScrollMore = () =>
-    self.scrollY + self.innerHeight <
-    Math.max(
-      self.document.body.scrollHeight,
-      self.document.body.offsetHeight,
-      self.document.documentElement.clientHeight,
-      self.document.documentElement.scrollHeight,
-      self.document.documentElement.offsetHeight
-    );
+    url = url.href;

-  const scrollOpts = { top: 250, left: 0, behavior: 'auto' };
+    // skip already crawled
+    if (this.seenList.has(url)) {
+      return false;
+    }

-  while (canScrollMore()) {
-    self.scrollBy(scrollOpts);
-    await new Promise(resolve => setTimeout(resolve, 500));
+    let inScope = false;
+
+    // check scopes
+    for (const s of this.params.scope) {
+      if (s.exec(url)) {
+        inScope = true;
+        break;
+      }
+    }
+
+    if (!inScope) {
+      //console.log(`Not in scope ${url} ${scope}`);
+      return false;
+    }
+
+    // check exclusions
+    for (const e of this.params.exclude) {
+      if (e.exec(url)) {
+        //console.log(`Skipping ${url} excluded by ${e}`);
+        return false;
+      }
+    }
+
+    return url;
  }
-}

-function sleep(time) {
-  return new Promise(resolve => setTimeout(resolve, time));
-}
+  resolveAgent(urlParsed) {
+    return urlParsed.protocol === "https:" ? HTTPS_AGENT : HTTP_AGENT;
+  }

+  async isHTML(url) {
+    try {
+      const resp = await fetch(url, {
+        method: "HEAD",
+        headers: this.headers,
+        agent: this.resolveAgent
+      });

-async function main() {
-  const params = require('yargs')
-  .usage("browsertrix-crawler [options]")
-  .options({
-    "url": {
-      alias: "u",
-      describe: "The URL to start crawling from",
-      demandOption: true,
-      type: "string",
-    },
-
-    "workers": {
-      alias: "w",
-      describe: "The number of workers to run in parallel",
-      demandOption: false,
-      default: 1,
-      type: "number",
-    },
-
-    "newContext": {
-      describe: "The context for each new capture, can be a new: page, session or browser.",
-      default: "page",
-      type: "string"
-    },
-
-    "waitUntil": {
-      describe: "Puppeteer page.goto() condition to wait for before continuing",
-      default: "load",
-    },
-
-    "limit": {
-      describe: "Limit crawl to this number of pages",
-      default: 0,
-      type: "number",
-    },
-
-    "timeout": {
-      describe: "Timeout for each page to load (in seconds)",
-      default: 90,
-      type: "number",
-    },
-
-    "scope": {
-      describe: "Regex of page URLs that should be included in the crawl (defaults to the immediate directory of URL)",
-    },
-
-    "exclude": {
-      describe: "Regex of page URLs that should be excluded from the crawl."
-    },
-
-    "scroll": {
-      describe: "If set, will autoscroll to bottom of the page",
-      type: "boolean",
-      default: false,
-
-    }}).check((argv, option) => {
-      // Scope for crawl, default to the domain of the URL
-      const url = new URL(argv.url);
-
-      if (url.protocol !== "http:" && url.protocol != "https:") {
-        throw new Error("URL must start with http:// or https://");
+      if (resp.status >= 400) {
+        console.log(`Skipping ${url}, invalid status ${resp.status}`);
+        return false;
      }

-      // ensure valid url is used (adds trailing slash if missing)
-      argv.url = url.href;
+      const contentType = resp.headers.get("Content-Type");

-      if (!argv.scope) {
-        //argv.scope = url.href.slice(0, url.href.lastIndexOf("/") + 1);
-        argv.scope = [new RegExp("^" + rxEscape(url.href.slice(0, url.href.lastIndexOf("/") + 1)))];
+      // just load if no content-type
+      if (!contentType) {
+        return true;
      }

-      argv.timeout *= 1000;
+      const mime = contentType.split(";")[0];

-      // waitUntil condition must be: load, domcontentloaded, networkidle0, networkidle2
-      // (see: https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagegotourl-options)
-      if (!WAIT_UNTIL_OPTS.includes(argv.waitUntil)) {
-        throw new Error("Invalid waitUntil, must be one of: " + WAIT_UNTIL_OPTS.join(","));
-      }
-
-      if (!NEW_CONTEXT_OPTS.includes(argv.newContext)) {
-        throw new Error("Invalid newContext, must be one of: " + NEW_CONTEXT_OPTS.join(","));
-      }
-
-      // Support one or multiple exclude
-      if (argv.exclude) {
-        if (typeof(argv.exclude) === "string") {
-          argv.exclude = [new RegExp(argv.exclude)];
-        } else {
-          argv.exclude = argv.exclude.map(e => new RegExp(e));
-        }
-      } else {
-        argv.exclude = [];
-      }
-
-      // Support one or multiple scopes
-      if (argv.scope) {
-        if (typeof(argv.scope) === "string") {
-          argv.scope = [new RegExp(argv.scope)];
-        } else {
-          argv.scope = argv.scope.map(e => new RegExp(e));
-        }
-      } else {
-        argv.scope = [];
+      if (HTML_TYPES.includes(mime)) {
+        return true;
      }

+      return false;
+    } catch(e) {
+      console.log("HTML Check error", e);
+      // can't confirm not html, so try in browser
      return true;
-    })
-  .argv;
+    }
+  }

-  console.log("Exclusions Regexes: ", params.exclude);
-  console.log("Scope Regexes: ", params.scope);
+  async directFetchCapture(url) {
+    //console.log(`Direct capture: ${this.capturePrefix}${url}`);
+    const abort = new AbortController();
+    const signal = abort.signal;
+    const resp2 = await fetch(this.capturePrefix + url, {signal, headers: this.headers});
+    abort.abort();
+  }

-  try {
-    await run(params);
-    process.exit(0);
-  } catch(e) {
-    console.error("Crawl failed");
-    console.error(e);
-    process.exit(1);
+  sleep(time) {
+    return new Promise(resolve => setTimeout(resolve, time));
+  }
+
+  rxEscape(string) {
+    return string.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&");
  }
 }

-function rxEscape(string) {
-  return string.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
-}
-
-
-main();
-
+module.exports.Crawler = Crawler;

--- a/defaultDriver.js
+++ b/defaultDriver.js
@ -0,0 +1,89 @@
+const fs = require("fs");
+
+const autoplayScript = fs.readFileSync("/app/autoplay.js", "utf-8");
+//const autoplayScript = require("/app/autoplay.js");
+
+/* eslint-disable no-undef */
+
+module.exports = async ({data, page, crawler}) => {
+  const {url} = data;
+
+  //page.on("requestfailed", message => console.warn(message._failureText));
+
+  if (!await crawler.isHTML(url)) {
+    await crawler.directFetchCapture(url);
+    return;
+  }
+
+  const mediaResults = [];
+
+  await page.exposeFunction("__crawler_queueUrls", async (url) => {
+    mediaResults.push(await crawler.directFetchCapture(url));
+  });
+
+  let waitForVideo = false;
+
+  await page.exposeFunction("__crawler_autoplayLoad", (url) => {
+    console.log("*** Loading autoplay URL: " + url);
+    waitForVideo = true;
+  });
+
+  try {
+    await page.evaluateOnNewDocument(autoplayScript);
+  } catch(e) {
+    console.log(e);
+  }
+
+  const gotoOpts = {
+    waitUntil: crawler.params.waitUntil,
+    timeout: crawler.params.timeout
+  };
+
+  try {
+    await page.goto(url, gotoOpts);
+  } catch (e) {
+    console.log(`Load timeout for ${url}`, e);
+  }
+
+  try {
+    await Promise.all(mediaResults);
+  } catch (e) {
+    console.log("Error loading media URLs", e);
+  }
+
+  if (waitForVideo) {
+    console.log("Extra wait 15s for video loading");
+    await crawler.sleep(15000);
+  }
+
+  if (crawler.params.scroll) {
+    try {
+      await Promise.race([page.evaluate(autoScroll), crawler.sleep(30000)]);
+    } catch (e) {
+      console.warn("Behavior Failed", e);
+    }
+  }
+
+  await crawler.extractLinks(page, "a[href]");
+};
+
+async function autoScroll() {
+  const canScrollMore = () =>
+    self.scrollY + self.innerHeight <
+    Math.max(
+      self.document.body.scrollHeight,
+      self.document.body.offsetHeight,
+      self.document.documentElement.clientHeight,
+      self.document.documentElement.scrollHeight,
+      self.document.documentElement.offsetHeight
+    );
+
+  const scrollOpts = { top: 250, left: 0, behavior: "auto" };
+
+  while (canScrollMore()) {
+    self.scrollBy(scrollOpts);
+    await new Promise(resolve => setTimeout(resolve, 500));
+  }
+}
+
+
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,6 +1,13 @@
-version: '3.3'
+version: '3.5'
  
 services:
    browsertrix-crawler:
        image: webrecorder/browsertrix-crawler
-        build: ./
+        build:
+          context: ./
+
+        cap_add:
+          - NET_ADMIN
+          - SYS_ADMIN
+
+        shm_size: 1gb
--- a/main.js
+++ b/main.js
@ -0,0 +1,5 @@
+
+const { Crawler } = require("./crawler");
+
+new Crawler().run();
+
--- a/package.json
+++ b/package.json
@ -11,5 +11,8 @@
    "puppeteer-cluster": "^0.22.0",
    "puppeteer-core": "^5.3.1",
    "yargs": "^16.0.3"
+  },
+  "devDependencies": {
+    "eslint": "^7.12.1"
  }
 }
--- a/uwsgi.ini
+++ b/uwsgi.ini
@ -23,5 +23,6 @@ processes = 8

 # specify config file here
 env = PYWB_CONFIG_FILE=/app/config.yaml
+env = COLLECTION=$(COLLECTION)
 wsgi = pywb.apps.wayback

--- a/yarn.lock
+++ b/yarn.lock
@ -2,6 +2,43 @@
 # yarn lockfile v1


+"@babel/code-frame@^7.0.0":
+  version "7.10.4"
+  resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a"
+  integrity sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==
+  dependencies:
+    "@babel/highlight" "^7.10.4"
+
+"@babel/helper-validator-identifier@^7.10.4":
+  version "7.10.4"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz#a78c7a7251e01f616512d31b10adcf52ada5e0d2"
+  integrity sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==
+
+"@babel/highlight@^7.10.4":
+  version "7.10.4"
+  resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.10.4.tgz#7d1bdfd65753538fabe6c38596cdb76d9ac60143"
+  integrity sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==
+  dependencies:
+    "@babel/helper-validator-identifier" "^7.10.4"
+    chalk "^2.0.0"
+    js-tokens "^4.0.0"
+
+"@eslint/eslintrc@^0.2.1":
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.2.1.tgz#f72069c330461a06684d119384435e12a5d76e3c"
+  integrity sha512-XRUeBZ5zBWLYgSANMpThFddrZZkEbGHgUdt5UJjZfnlN9BGCiUBrf+nvbRupSjMvqzwnQN0qwCmOxITt1cfywA==
+  dependencies:
+    ajv "^6.12.4"
+    debug "^4.1.1"
+    espree "^7.3.0"
+    globals "^12.1.0"
+    ignore "^4.0.6"
+    import-fresh "^3.2.1"
+    js-yaml "^3.13.1"
+    lodash "^4.17.19"
+    minimatch "^3.0.4"
+    strip-json-comments "^3.1.1"
+
 "@types/color-name@^1.1.1":
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0"
@ -26,16 +63,53 @@ abort-controller@^3.0.0:
  dependencies:
    event-target-shim "^5.0.0"

+acorn-jsx@^5.2.0:
+  version "5.3.1"
+  resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.1.tgz#fc8661e11b7ac1539c47dbfea2e72b3af34d267b"
+  integrity sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==
+
+acorn@^7.4.0:
+  version "7.4.1"
+  resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
+  integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==
+
 agent-base@5:
  version "5.1.1"
  resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-5.1.1.tgz#e8fb3f242959db44d63be665db7a8e739537a32c"
  integrity sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==

+ajv@^6.10.0, ajv@^6.10.2, ajv@^6.12.4:
+  version "6.12.6"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
+  integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==
+  dependencies:
+    fast-deep-equal "^3.1.1"
+    fast-json-stable-stringify "^2.0.0"
+    json-schema-traverse "^0.4.1"
+    uri-js "^4.2.2"
+
+ansi-colors@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348"
+  integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==
+
+ansi-regex@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997"
+  integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==
+
 ansi-regex@^5.0.0:
  version "5.0.0"
  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75"
  integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==

+ansi-styles@^3.2.0, ansi-styles@^3.2.1:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d"
+  integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==
+  dependencies:
+    color-convert "^1.9.0"
+
 ansi-styles@^4.0.0:
  version "4.2.1"
  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.2.1.tgz#90ae75c424d008d2624c5bf29ead3177ebfcf359"
@ -44,6 +118,25 @@ ansi-styles@^4.0.0:
    "@types/color-name" "^1.1.1"
    color-convert "^2.0.1"

+ansi-styles@^4.1.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937"
+  integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==
+  dependencies:
+    color-convert "^2.0.1"
+
+argparse@^1.0.7:
+  version "1.0.10"
+  resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
+  integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==
+  dependencies:
+    sprintf-js "~1.0.2"
+
+astral-regex@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-1.0.0.tgz#6c8c3fb827dd43ee3918f27b82782ab7658a6fd9"
+  integrity sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==
+
 balanced-match@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
@ -84,6 +177,28 @@ buffer@^5.2.1, buffer@^5.5.0:
    base64-js "^1.0.2"
    ieee754 "^1.1.4"

+callsites@^3.0.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
+  integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
+
+chalk@^2.0.0:
+  version "2.4.2"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
+  integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
+  dependencies:
+    ansi-styles "^3.2.1"
+    escape-string-regexp "^1.0.5"
+    supports-color "^5.3.0"
+
+chalk@^4.0.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a"
+  integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==
+  dependencies:
+    ansi-styles "^4.1.0"
+    supports-color "^7.1.0"
+
 chownr@^1.1.1:
  version "1.1.4"
  resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
@ -98,6 +213,13 @@ cliui@^7.0.0:
    strip-ansi "^6.0.0"
    wrap-ansi "^7.0.0"

+color-convert@^1.9.0:
+  version "1.9.3"
+  resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
+  integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==
+  dependencies:
+    color-name "1.1.3"
+
 color-convert@^2.0.1:
  version "2.0.1"
  resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3"
@ -105,6 +227,11 @@ color-convert@^2.0.1:
  dependencies:
    color-name "~1.1.4"

+color-name@1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25"
+  integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=
+
 color-name@~1.1.4:
  version "1.1.4"
  resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
@ -115,18 +242,44 @@ concat-map@0.0.1:
  resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
  integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=

-debug@4, debug@^4.1.0, debug@^4.1.1:
+cross-spawn@^7.0.2:
+  version "7.0.3"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
+  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
+  dependencies:
+    path-key "^3.1.0"
+    shebang-command "^2.0.0"
+    which "^2.0.1"
+
+debug@4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1:
  version "4.2.0"
  resolved "https://registry.yarnpkg.com/debug/-/debug-4.2.0.tgz#7f150f93920e94c58f5574c2fd01a3110effe7f1"
  integrity sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==
  dependencies:
    ms "2.1.2"

+deep-is@^0.1.3:
+  version "0.1.3"
+  resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
+  integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=
+
 devtools-protocol@0.0.799653:
  version "0.0.799653"
  resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.799653.tgz#86fc95ce5bf4fdf4b77a58047ba9d2301078f119"
  integrity sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg==

+doctrine@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961"
+  integrity sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==
+  dependencies:
+    esutils "^2.0.2"
+
+emoji-regex@^7.0.1:
+  version "7.0.3"
+  resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156"
+  integrity sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==
+
 emoji-regex@^8.0.0:
  version "8.0.0"
  resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
@ -139,11 +292,134 @@ end-of-stream@^1.1.0, end-of-stream@^1.4.1:
  dependencies:
    once "^1.4.0"

+enquirer@^2.3.5:
+  version "2.3.6"
+  resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d"
+  integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==
+  dependencies:
+    ansi-colors "^4.1.1"
+
 escalade@^3.0.2:
  version "3.1.0"
  resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.0.tgz#e8e2d7c7a8b76f6ee64c2181d6b8151441602d4e"
  integrity sha512-mAk+hPSO8fLDkhV7V0dXazH5pDc6MrjBTPyD3VeKzxnVFjH1MIxbCdqGZB9O8+EwWakZs3ZCbDS4IpRt79V1ig==

+escape-string-regexp@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4"
+  integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=
+
+eslint-scope@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c"
+  integrity sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==
+  dependencies:
+    esrecurse "^4.3.0"
+    estraverse "^4.1.1"
+
+eslint-utils@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-2.1.0.tgz#d2de5e03424e707dc10c74068ddedae708741b27"
+  integrity sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==
+  dependencies:
+    eslint-visitor-keys "^1.1.0"
+
+eslint-visitor-keys@^1.1.0, eslint-visitor-keys@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e"
+  integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==
+
+eslint-visitor-keys@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz#21fdc8fbcd9c795cc0321f0563702095751511a8"
+  integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==
+
+eslint@^7.12.1:
+  version "7.12.1"
+  resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.12.1.tgz#bd9a81fa67a6cfd51656cdb88812ce49ccec5801"
+  integrity sha512-HlMTEdr/LicJfN08LB3nM1rRYliDXOmfoO4vj39xN6BLpFzF00hbwBoqHk8UcJ2M/3nlARZWy/mslvGEuZFvsg==
+  dependencies:
+    "@babel/code-frame" "^7.0.0"
+    "@eslint/eslintrc" "^0.2.1"
+    ajv "^6.10.0"
+    chalk "^4.0.0"
+    cross-spawn "^7.0.2"
+    debug "^4.0.1"
+    doctrine "^3.0.0"
+    enquirer "^2.3.5"
+    eslint-scope "^5.1.1"
+    eslint-utils "^2.1.0"
+    eslint-visitor-keys "^2.0.0"
+    espree "^7.3.0"
+    esquery "^1.2.0"
+    esutils "^2.0.2"
+    file-entry-cache "^5.0.1"
+    functional-red-black-tree "^1.0.1"
+    glob-parent "^5.0.0"
+    globals "^12.1.0"
+    ignore "^4.0.6"
+    import-fresh "^3.0.0"
+    imurmurhash "^0.1.4"
+    is-glob "^4.0.0"
+    js-yaml "^3.13.1"
+    json-stable-stringify-without-jsonify "^1.0.1"
+    levn "^0.4.1"
+    lodash "^4.17.19"
+    minimatch "^3.0.4"
+    natural-compare "^1.4.0"
+    optionator "^0.9.1"
+    progress "^2.0.0"
+    regexpp "^3.1.0"
+    semver "^7.2.1"
+    strip-ansi "^6.0.0"
+    strip-json-comments "^3.1.0"
+    table "^5.2.3"
+    text-table "^0.2.0"
+    v8-compile-cache "^2.0.3"
+
+espree@^7.3.0:
+  version "7.3.0"
+  resolved "https://registry.yarnpkg.com/espree/-/espree-7.3.0.tgz#dc30437cf67947cf576121ebd780f15eeac72348"
+  integrity sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==
+  dependencies:
+    acorn "^7.4.0"
+    acorn-jsx "^5.2.0"
+    eslint-visitor-keys "^1.3.0"
+
+esprima@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71"
+  integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==
+
+esquery@^1.2.0:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.3.1.tgz#b78b5828aa8e214e29fb74c4d5b752e1c033da57"
+  integrity sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ==
+  dependencies:
+    estraverse "^5.1.0"
+
+esrecurse@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921"
+  integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==
+  dependencies:
+    estraverse "^5.2.0"
+
+estraverse@^4.1.1:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d"
+  integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==
+
+estraverse@^5.1.0, estraverse@^5.2.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.2.0.tgz#307df42547e6cc7324d3cf03c155d5cdb8c53880"
+  integrity sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==
+
+esutils@^2.0.2:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64"
+  integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==
+
 event-target-shim@^5.0.0:
  version "5.0.1"
  resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
@ -160,6 +436,21 @@ extract-zip@^2.0.0:
  optionalDependencies:
    "@types/yauzl" "^2.9.1"

+fast-deep-equal@^3.1.1:
+  version "3.1.3"
+  resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525"
+  integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==
+
+fast-json-stable-stringify@^2.0.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633"
+  integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==
+
+fast-levenshtein@^2.0.6:
+  version "2.0.6"
+  resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
+  integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=
+
 fd-slicer@~1.1.0:
  version "1.1.0"
  resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e"
@ -167,6 +458,13 @@ fd-slicer@~1.1.0:
  dependencies:
    pend "~1.2.0"

+file-entry-cache@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-5.0.1.tgz#ca0f6efa6dd3d561333fb14515065c2fafdf439c"
+  integrity sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==
+  dependencies:
+    flat-cache "^2.0.1"
+
 find-up@^4.0.0:
  version "4.1.0"
  resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19"
@ -175,6 +473,20 @@ find-up@^4.0.0:
    locate-path "^5.0.0"
    path-exists "^4.0.0"

+flat-cache@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-2.0.1.tgz#5d296d6f04bda44a4630a301413bdbc2ec085ec0"
+  integrity sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==
+  dependencies:
+    flatted "^2.0.0"
+    rimraf "2.6.3"
+    write "1.0.3"
+
+flatted@^2.0.0:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138"
+  integrity sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==
+
 fs-constants@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
@ -185,6 +497,11 @@ fs.realpath@^1.0.0:
  resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
  integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=

+functional-red-black-tree@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327"
+  integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=
+
 get-caller-file@^2.0.5:
  version "2.0.5"
  resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
@ -197,6 +514,13 @@ get-stream@^5.1.0:
  dependencies:
    pump "^3.0.0"

+glob-parent@^5.0.0:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.1.tgz#b6c1ef417c4e5663ea498f1c45afac6916bbc229"
+  integrity sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==
+  dependencies:
+    is-glob "^4.0.1"
+
 glob@^7.1.3:
  version "7.1.6"
  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
@ -209,6 +533,23 @@ glob@^7.1.3:
    once "^1.3.0"
    path-is-absolute "^1.0.0"

+globals@^12.1.0:
+  version "12.4.0"
+  resolved "https://registry.yarnpkg.com/globals/-/globals-12.4.0.tgz#a18813576a41b00a24a97e7f815918c2e19925f8"
+  integrity sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==
+  dependencies:
+    type-fest "^0.8.1"
+
+has-flag@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd"
+  integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0=
+
+has-flag@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b"
+  integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==
+
 https-proxy-agent@^4.0.0:
  version "4.0.0"
  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz#702b71fb5520a132a66de1f67541d9e62154d82b"
@ -222,6 +563,24 @@ ieee754@^1.1.4:
  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
  integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==

+ignore@^4.0.6:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc"
+  integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==
+
+import-fresh@^3.0.0, import-fresh@^3.2.1:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.2.2.tgz#fc129c160c5d68235507f4331a6baad186bdbc3e"
+  integrity sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==
+  dependencies:
+    parent-module "^1.0.0"
+    resolve-from "^4.0.0"
+
+imurmurhash@^0.1.4:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"
+  integrity sha1-khi5srkoojixPcT7a21XbyMUU+o=
+
 inflight@^1.0.4:
  version "1.0.6"
  resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"
@ -235,11 +594,64 @@ inherits@2, inherits@^2.0.3, inherits@^2.0.4:
  resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
  integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==

+is-extglob@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
+  integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=
+
+is-fullwidth-code-point@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f"
+  integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=
+
 is-fullwidth-code-point@^3.0.0:
  version "3.0.0"
  resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d"
  integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==

+is-glob@^4.0.0, is-glob@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc"
+  integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==
+  dependencies:
+    is-extglob "^2.1.1"
+
+isexe@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
+  integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=
+
+js-tokens@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
+  integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==
+
+js-yaml@^3.13.1:
+  version "3.14.0"
+  resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.0.tgz#a7a34170f26a21bb162424d8adacb4113a69e482"
+  integrity sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==
+  dependencies:
+    argparse "^1.0.7"
+    esprima "^4.0.0"
+
+json-schema-traverse@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
+  integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==
+
+json-stable-stringify-without-jsonify@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"
+  integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=
+
+levn@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade"
+  integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==
+  dependencies:
+    prelude-ls "^1.2.1"
+    type-check "~0.4.0"
+
 locate-path@^5.0.0:
  version "5.0.0"
  resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0"
@ -247,6 +659,11 @@ locate-path@^5.0.0:
  dependencies:
    p-locate "^4.1.0"

+lodash@^4.17.14, lodash@^4.17.19:
+  version "4.17.20"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52"
+  integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==
+
 minimatch@^3.0.4:
  version "3.0.4"
  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
@ -254,16 +671,33 @@ minimatch@^3.0.4:
  dependencies:
    brace-expansion "^1.1.7"

+minimist@^1.2.5:
+  version "1.2.5"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
+  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+
 mkdirp-classic@^0.5.2:
  version "0.5.3"
  resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
  integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==

+mkdirp@^0.5.1:
+  version "0.5.5"
+  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"
+  integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==
+  dependencies:
+    minimist "^1.2.5"
+
 ms@2.1.2:
  version "2.1.2"
  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
  integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==

+natural-compare@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
+  integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
+
 node-fetch@^2.6.1:
  version "2.6.1"
  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
@ -276,6 +710,18 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0:
  dependencies:
    wrappy "1"

+optionator@^0.9.1:
+  version "0.9.1"
+  resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499"
+  integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==
+  dependencies:
+    deep-is "^0.1.3"
+    fast-levenshtein "^2.0.6"
+    levn "^0.4.1"
+    prelude-ls "^1.2.1"
+    type-check "^0.4.0"
+    word-wrap "^1.2.3"
+
 p-limit@^2.2.0:
  version "2.3.0"
  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1"
@ -295,6 +741,13 @@ p-try@^2.0.0:
  resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
  integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==

+parent-module@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2"
+  integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==
+  dependencies:
+    callsites "^3.0.0"
+
 path-exists@^4.0.0:
  version "4.0.0"
  resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3"
@ -305,6 +758,11 @@ path-is-absolute@^1.0.0:
  resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
  integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18=

+path-key@^3.1.0:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"
+  integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==
+
 pend@~1.2.0:
  version "1.2.0"
  resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
@ -317,7 +775,12 @@ pkg-dir@^4.2.0:
  dependencies:
    find-up "^4.0.0"

-progress@^2.0.1:
+prelude-ls@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396"
+  integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==
+
+progress@^2.0.0, progress@^2.0.1:
  version "2.0.3"
  resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
  integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
@ -335,6 +798,11 @@ pump@^3.0.0:
    end-of-stream "^1.1.0"
    once "^1.3.1"

+punycode@^2.1.0:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
+  integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
+
 puppeteer-cluster@^0.22.0:
  version "0.22.0"
  resolved "https://registry.yarnpkg.com/puppeteer-cluster/-/puppeteer-cluster-0.22.0.tgz#4ab214671f414f15ad6a94a4b61ed0b4172e86e6"
@ -368,11 +836,28 @@ readable-stream@^3.1.1, readable-stream@^3.4.0:
    string_decoder "^1.1.1"
    util-deprecate "^1.0.1"

+regexpp@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.1.0.tgz#206d0ad0a5648cffbdb8ae46438f3dc51c9f78e2"
+  integrity sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==
+
 require-directory@^2.1.1:
  version "2.1.1"
  resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
  integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I=

+resolve-from@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
+  integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
+
+rimraf@2.6.3:
+  version "2.6.3"
+  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab"
+  integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==
+  dependencies:
+    glob "^7.1.3"
+
 rimraf@^3.0.2:
  version "3.0.2"
  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a"
@ -385,6 +870,46 @@ safe-buffer@~5.2.0:
  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
  integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==

+semver@^7.2.1:
+  version "7.3.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938"
+  integrity sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==
+
+shebang-command@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea"
+  integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==
+  dependencies:
+    shebang-regex "^3.0.0"
+
+shebang-regex@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172"
+  integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
+
+slice-ansi@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-2.1.0.tgz#cacd7693461a637a5788d92a7dd4fba068e81636"
+  integrity sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==
+  dependencies:
+    ansi-styles "^3.2.0"
+    astral-regex "^1.0.0"
+    is-fullwidth-code-point "^2.0.0"
+
+sprintf-js@~1.0.2:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
+  integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=
+
+string-width@^3.0.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961"
+  integrity sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==
+  dependencies:
+    emoji-regex "^7.0.1"
+    is-fullwidth-code-point "^2.0.0"
+    strip-ansi "^5.1.0"
+
 string-width@^4.1.0, string-width@^4.2.0:
  version "4.2.0"
  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.0.tgz#952182c46cc7b2c313d1596e623992bd163b72b5"
@ -401,6 +926,13 @@ string_decoder@^1.1.1:
  dependencies:
    safe-buffer "~5.2.0"

+strip-ansi@^5.1.0:
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae"
+  integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==
+  dependencies:
+    ansi-regex "^4.1.0"
+
 strip-ansi@^6.0.0:
  version "6.0.0"
  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532"
@ -408,6 +940,35 @@ strip-ansi@^6.0.0:
  dependencies:
    ansi-regex "^5.0.0"

+strip-json-comments@^3.1.0, strip-json-comments@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"
+  integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==
+
+supports-color@^5.3.0:
+  version "5.5.0"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f"
+  integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==
+  dependencies:
+    has-flag "^3.0.0"
+
+supports-color@^7.1.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da"
+  integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==
+  dependencies:
+    has-flag "^4.0.0"
+
+table@^5.2.3:
+  version "5.4.6"
+  resolved "https://registry.yarnpkg.com/table/-/table-5.4.6.tgz#1292d19500ce3f86053b05f0e8e7e4a3bb21079e"
+  integrity sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==
+  dependencies:
+    ajv "^6.10.2"
+    lodash "^4.17.14"
+    slice-ansi "^2.1.0"
+    string-width "^3.0.0"
+
 tar-fs@^2.0.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.0.tgz#d1cdd121ab465ee0eb9ccde2d35049d3f3daf0d5"
@ -429,11 +990,28 @@ tar-stream@^2.0.0:
    inherits "^2.0.3"
    readable-stream "^3.1.1"

+text-table@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
+  integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=
+
 through@^2.3.8:
  version "2.3.8"
  resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
  integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=

+type-check@^0.4.0, type-check@~0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1"
+  integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==
+  dependencies:
+    prelude-ls "^1.2.1"
+
+type-fest@^0.8.1:
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
+  integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
+
 unbzip2-stream@^1.3.3:
  version "1.4.3"
  resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7"
@ -442,11 +1020,35 @@ unbzip2-stream@^1.3.3:
    buffer "^5.2.1"
    through "^2.3.8"

+uri-js@^4.2.2:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.0.tgz#aa714261de793e8a82347a7bcc9ce74e86f28602"
+  integrity sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g==
+  dependencies:
+    punycode "^2.1.0"
+
 util-deprecate@^1.0.1:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
  integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=

+v8-compile-cache@^2.0.3:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz#9471efa3ef9128d2f7c6a7ca39c4dd6b5055b132"
+  integrity sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==
+
+which@^2.0.1:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"
+  integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==
+  dependencies:
+    isexe "^2.0.0"
+
+word-wrap@^1.2.3:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
+  integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==
+
 wrap-ansi@^7.0.0:
  version "7.0.0"
  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
@ -461,6 +1063,13 @@ wrappy@1:
  resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
  integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=

+write@1.0.3:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/write/-/write-1.0.3.tgz#0800e14523b923a387e415123c865616aae0f5c3"
+  integrity sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==
+  dependencies:
+    mkdirp "^0.5.1"
+
 ws@^7.2.3:
  version "7.3.1"
  resolved "https://registry.yarnpkg.com/ws/-/ws-7.3.1.tgz#d0547bf67f7ce4f12a72dfe31262c68d7dc551c8"