bump version to 1.8.1

fix some observed errors that occur when saving profile:
- use browser.cookies instead of page.cookies to get all cookies, not
just from page
- catch exception when clearing cache and ignore
- logging: log when proxy init is happening on all paths, in case error
in proxy connection

docs/docs/user-guide/behaviors.md

@@ -266,6 +266,7 @@ Some of these functions which may be of use to behaviors authors are:
 - `scrollToOffset`: scroll to particular offset
 - `scrollIntoView`: smoothly scroll particular element into view
 - `getState`: increment a state counter and return all state counters + string message
+* `addLink`: add a given URL to the crawl queue
 
 More detailed references will be added in the future.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "browsertrix-crawler",
-  "version": "1.8.0",
+  "version": "1.8.1",
   "main": "browsertrix-crawler",
   "type": "module",
   "repository": "https://github.com/webrecorder/browsertrix-crawler",

src/crawler.ts

@@ -129,8 +129,6 @@ export class Crawler {
   limitHit = false;
   pageLimit: number;
 
-  dupeSeedsFound = false;
-
   saveStateFiles: string[] = [];
   lastSaveTime: number;
 
@@ -854,31 +852,34 @@ self.__bx_behaviors.selectMainBehavior();
       await this.browser.addInitScript(page, initScript);
     }
 
-    // only add if running with autoclick behavior
+    // Handle JS dialogs:
-    if (this.params.behaviors.includes("autoclick")) {
+    // - Ensure off-page navigation is canceled while behavior is running
-      // Ensure off-page navigation is canceled while behavior is running
+    // - dismiss close all other dialogs if not blocking unload
-      page.on("dialog", async (dialog) => {
+    page.on("dialog", async (dialog) => {
-        let accepted = true;
+      let accepted = true;
-        if (dialog.type() === "beforeunload") {
+      if (dialog.type() === "beforeunload") {
-          if (opts.pageBlockUnload) {
+        if (opts.pageBlockUnload) {
-            accepted = false;
+          accepted = false;
-            await dialog.dismiss();
+          await dialog.dismiss();
-          } else {
-            await dialog.accept();
-          }
         } else {
           await dialog.accept();
         }
-        logger.debug("JS Dialog", {
+      } else {
-          accepted,
+        // other JS dialog, just dismiss
-          blockingUnload: opts.pageBlockUnload,
+        await dialog.dismiss();
-          message: dialog.message(),
+      }
-          type: dialog.type(),
+      logger.debug("JS Dialog", {
-          page: page.url(),
+        accepted,
-          workerid,
+        blockingUnload: opts.pageBlockUnload,
-        });
+        message: dialog.message(),
+        type: dialog.type(),
+        page: page.url(),
+        workerid,
       });
+    });
 
+    // only add if running with autoclick behavior
+    if (this.params.behaviors.includes("autoclick")) {
       // Close any windows opened during navigation from autoclick
       await cdp.send("Target.setDiscoverTargets", { discover: true });
 
@@ -2487,10 +2488,6 @@ self.__bx_behaviors.selectMainBehavior();
         return false;
 
       case QueueState.DUPE_URL:
-        if (!this.dupeSeedsFound && depth === 0) {
-          logger.error("Duplicate seed URLs found and skipped");
-          this.dupeSeedsFound = true;
-        }
         logger.debug(
           "Page URL not queued, already seen",
           { url, ...logDetails },

src/create-login-profile.ts

@@ -339,7 +339,11 @@ async function createProfile(
   cdp: CDPSession,
   targetFilename = "",
 ) {
-  await cdp.send("Network.clearBrowserCache");
+  try {
+    await cdp.send("Network.clearBrowserCache");
+  } catch (e) {
+    logger.warn("Error clearing cache", e, "browser");
+  }
 
   await browser.close();
 
@@ -546,7 +550,8 @@ class InteractiveBrowser {
         return;
       }
 
-      const cookies = await this.browser.getCookies(this.page);
+      const cookies = await this.browser.getCookies();
+
       for (const cookieOrig of cookies) {
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const cookie = cookieOrig as any;
@@ -566,7 +571,7 @@ class InteractiveBrowser {
           cookie.url = url;
         }
       }
-      await this.browser.setCookies(this.page, cookies);
+      await this.browser.setCookies(cookies);
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
     } catch (e: any) {
       logger.error("Save Cookie Error: ", e);

src/util/browser.ts

@@ -22,6 +22,7 @@ import puppeteer, {
   Page,
   LaunchOptions,
   Viewport,
+  CookieData,
 } from "puppeteer-core";
 import { CDPSession, Target, Browser as PptrBrowser } from "puppeteer-core";
 import { Recorder } from "./recorder.js";
@@ -616,14 +617,12 @@ export class Browser {
     await page.setViewport(params);
   }
 
-  async getCookies(page: Page) {
+  async getCookies() {
-    return await page.cookies();
+    return (await this.browser?.cookies()) || [];
   }
 
-  // TODO: Fix this the next time the file is edited.
+  async setCookies(cookies: CookieData[]) {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return await this.browser?.setCookie(...cookies);
-  async setCookies(page: Page, cookies: any) {
-    return await page.setCookie(...cookies);
   }
 }
 

src/util/flowbehavior.ts

@@ -368,7 +368,7 @@ class Flow {
       case StepType.DoubleClick:
         await locator(step)
           .setTimeout(timeout * 1000)
-          //.on('action', () => startWaitingForEvents())
+          .setEnsureElementIsInTheViewport(true)
           .click({
             count: 2,
             button: step.button && mouseButtonMap.get(step.button),
@@ -392,7 +392,7 @@ class Flow {
 
         await locator(step)
           .setTimeout(timeout * 1000)
-          //.on('action', () => startWaitingForEvents())
+          .setEnsureElementIsInTheViewport(true)
           .click({
             delay: step.duration,
             button: step.button && mouseButtonMap.get(step.button),
@@ -410,7 +410,7 @@ class Flow {
       case StepType.Hover:
         await locator(step)
           .setTimeout(timeout * 1000)
-          //.on('action', () => startWaitingForEvents())
+          .setEnsureElementIsInTheViewport(true)
           .hover();
         break;
 
@@ -426,15 +426,14 @@ class Flow {
 
       case StepType.Change:
         await locator(step)
-          //.on('action', () => startWaitingForEvents())
           .setTimeout(timeout * 1000)
+          .setEnsureElementIsInTheViewport(true)
           .fill(step.value);
         break;
 
       case StepType.Scroll: {
         if ("selectors" in step) {
           await locator(step)
-            //.on('action', () => startWaitingForEvents())
             .setTimeout(timeout * 1000)
             .scroll({
               scrollLeft: step.x || 0,

src/util/proxy.ts

@@ -153,13 +153,6 @@ export async function initProxy(
     privateKeyFile = privateKeyFile || sshProxyPrivateKeyFile;
     publicHostsFile = publicHostsFile || sshProxyKnownHostsFile;
 
-    logger.debug("Initing proxy", {
-      url: getSafeProxyString(proxyUrl),
-      localPort,
-      privateKeyFile,
-      publicHostsFile,
-    });
-
     const entry = await initSingleProxy(
       proxyUrl,
       localPort++,
@@ -200,6 +193,13 @@ export async function initSingleProxy(
   sshProxyPrivateKeyFile?: string,
   sshProxyKnownHostsFile?: string,
 ): Promise<{ proxyUrl: string; dispatcher: Dispatcher }> {
+  logger.debug("Initing proxy", {
+    url: getSafeProxyString(proxyUrl),
+    localPort,
+    sshProxyPrivateKeyFile,
+    sshProxyKnownHostsFile,
+  });
+
   if (proxyUrl && proxyUrl.startsWith("ssh://")) {
     proxyUrl = await runSSHD(
       proxyUrl,

tests/custom_selector.test.js

@@ -43,8 +43,8 @@ test("test custom selector crawls JS files as pages", async () => {
   ]);
 
   const expectedExtraPages = new Set([
-    "https://www.iana.org/_js/jquery.js",
+    "https://www.iana.org/static/_js/jquery.js",
-    "https://www.iana.org/_js/iana.js",
+    "https://www.iana.org/static/_js/iana.js",
   ]);
 
   expect(pages).toEqual(expectedPages);