browsertrix-crawler/util/browser.js

const child_process = require("child_process");
const fs = require("fs");
const path = require("path");
const os = require("os");
const request = require("request");
const { initStorage } = require("./storage");

const profileDir = fs.mkdtempSync(path.join(os.tmpdir(), "profile-"));

module.exports.loadProfile = async function(profileFilename) {
  const targetFilename = "/tmp/profile.tar.gz";

  if (profileFilename &&
      (profileFilename.startsWith("http:") || profileFilename.startsWith("https:"))) {

    console.log(`Downloading ${profileFilename} to ${targetFilename}`);

    const p = new Promise((resolve, reject) => {
      request.get(profileFilename).
        on("error", (err) => reject(err)).
        pipe(fs.createWriteStream(targetFilename)).
        on("finish", () => resolve());
    });

    await p;
      
    profileFilename = targetFilename;
  } else if (profileFilename && profileFilename.startsWith("@")) {
    const storage = initStorage("");

    if (!storage) {
      throw new Error("Profile specified relative to s3 storage, but no S3 storage defined");
    }

    await storage.downloadFile(profileFilename.slice(1), targetFilename);

    profileFilename = targetFilename;
  }

  if (profileFilename) {
    try {
      child_process.execSync("tar xvfz " + profileFilename, {cwd: profileDir});
    } catch (e) {
      console.error(`Profile filename ${profileFilename} not a valid tar.gz`);
    }
  }

  return profileDir;
};

module.exports.saveProfile = function(profileFilename) {
  child_process.execFileSync("tar", ["cvfz", profileFilename, "./"], {cwd: profileDir});
};

function getBrowserExe() {
  const files = [process.env.BROWSER_BIN, "/usr/bin/google-chrome", "/usr/bin/chromium-browser"];
  for (const file of files) {
    if (file && fs.existsSync(file)) {
      return file;
    }
  }

  return null;
}


module.exports.getBrowserExe = getBrowserExe;


function getDefaultUA() {
  let version = process.env.BROWSER_VERSION;

  try {
    version = child_process.execFileSync(getBrowserExe(), ["--version"], {encoding: "utf8"});
    version = version.match(/[\d.]+/)[0];
  } catch(e) {
    console.error(e);
  }

  return `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${version} Safari/537.36`;
}


module.exports.getDefaultUA = getDefaultUA;


module.exports.chromeArgs = (proxy, userAgent=null, extraArgs=[]) => {
  // Chrome Flags, including proxy server
  const args = [
    ...(process.env.CHROME_FLAGS ?? "").split(" ").filter(Boolean),
    "--no-xshm", // needed for Chrome >80 (check if puppeteer adds automatically)
    "--no-sandbox",
    "--disable-background-media-suspend",
    "--enable-features=NetworkService,NetworkServiceInProcess",
    "--autoplay-policy=no-user-gesture-required",
    "--disable-features=IsolateOrigins,site-per-process,ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,AcceptCHFrame,AutoExpandDetailsElement",
    "--disable-site-isolation-trials",
    "--disable-popup-blocking",
    "--disable-backgrounding-occluded-windows",
    `--user-agent=${userAgent || getDefaultUA()}`,
    ...extraArgs,
  ];

  if (proxy) {
    args.push(`--proxy-server=http://${process.env.PROXY_HOST}:${process.env.PROXY_PORT}`);
  }

  return args;
};


module.exports.evaluateWithCLI = async (frame, funcString) => {
  const context = await frame.executionContext();

  // from puppeteer _evaluateInternal() but with includeCommandLineAPI: true
  const contextId = context._contextId;
  const expression = funcString + "\n//# sourceURL=__puppeteer_evaluation_script__";

  const { exceptionDetails, result: remoteObject } = await context._client
    .send("Runtime.evaluate", {
      expression,
      contextId,
      returnByValue: true,
      awaitPromise: true,
      userGesture: true,
      includeCommandLineAPI: true,
    });

  if (exceptionDetails) {
    throw new Error(
      "Behavior Evaluation Failed" + exceptionDetails.text
    );
  }

  return remoteObject.value;
};


module.exports.sleep = async (time) => {
  return new Promise(resolve => setTimeout(resolve, time));
};