mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 06:23:16 +00:00
Profile Saving Improvements (#894)
fix some observed errors that occur when saving profile: - use browser.cookies instead of page.cookies to get all cookies, not just from page - catch exception when clearing cache and ignore - logging: log when proxy init is happening on all paths, in case error in proxy connection
This commit is contained in:
parent
002feb287b
commit
4f234040ce
3 changed files with 20 additions and 16 deletions
|
@ -339,7 +339,11 @@ async function createProfile(
|
|||
cdp: CDPSession,
|
||||
targetFilename = "",
|
||||
) {
|
||||
await cdp.send("Network.clearBrowserCache");
|
||||
try {
|
||||
await cdp.send("Network.clearBrowserCache");
|
||||
} catch (e) {
|
||||
logger.warn("Error clearing cache", e, "browser");
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
|
||||
|
@ -546,7 +550,8 @@ class InteractiveBrowser {
|
|||
return;
|
||||
}
|
||||
|
||||
const cookies = await this.browser.getCookies(this.page);
|
||||
const cookies = await this.browser.getCookies();
|
||||
|
||||
for (const cookieOrig of cookies) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const cookie = cookieOrig as any;
|
||||
|
@ -566,7 +571,7 @@ class InteractiveBrowser {
|
|||
cookie.url = url;
|
||||
}
|
||||
}
|
||||
await this.browser.setCookies(this.page, cookies);
|
||||
await this.browser.setCookies(cookies);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} catch (e: any) {
|
||||
logger.error("Save Cookie Error: ", e);
|
||||
|
|
|
@ -22,6 +22,7 @@ import puppeteer, {
|
|||
Page,
|
||||
LaunchOptions,
|
||||
Viewport,
|
||||
CookieData,
|
||||
} from "puppeteer-core";
|
||||
import { CDPSession, Target, Browser as PptrBrowser } from "puppeteer-core";
|
||||
import { Recorder } from "./recorder.js";
|
||||
|
@ -616,14 +617,12 @@ export class Browser {
|
|||
await page.setViewport(params);
|
||||
}
|
||||
|
||||
async getCookies(page: Page) {
|
||||
return await page.cookies();
|
||||
async getCookies() {
|
||||
return (await this.browser?.cookies()) || [];
|
||||
}
|
||||
|
||||
// TODO: Fix this the next time the file is edited.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
async setCookies(page: Page, cookies: any) {
|
||||
return await page.setCookie(...cookies);
|
||||
async setCookies(cookies: CookieData[]) {
|
||||
return await this.browser?.setCookie(...cookies);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -153,13 +153,6 @@ export async function initProxy(
|
|||
privateKeyFile = privateKeyFile || sshProxyPrivateKeyFile;
|
||||
publicHostsFile = publicHostsFile || sshProxyKnownHostsFile;
|
||||
|
||||
logger.debug("Initing proxy", {
|
||||
url: getSafeProxyString(proxyUrl),
|
||||
localPort,
|
||||
privateKeyFile,
|
||||
publicHostsFile,
|
||||
});
|
||||
|
||||
const entry = await initSingleProxy(
|
||||
proxyUrl,
|
||||
localPort++,
|
||||
|
@ -200,6 +193,13 @@ export async function initSingleProxy(
|
|||
sshProxyPrivateKeyFile?: string,
|
||||
sshProxyKnownHostsFile?: string,
|
||||
): Promise<{ proxyUrl: string; dispatcher: Dispatcher }> {
|
||||
logger.debug("Initing proxy", {
|
||||
url: getSafeProxyString(proxyUrl),
|
||||
localPort,
|
||||
sshProxyPrivateKeyFile,
|
||||
sshProxyKnownHostsFile,
|
||||
});
|
||||
|
||||
if (proxyUrl && proxyUrl.startsWith("ssh://")) {
|
||||
proxyUrl = await runSSHD(
|
||||
proxyUrl,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue