mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 14:33:17 +00:00
profile creation: when creating a profile, force all cookies to have a duration to avoid expiring session cookies (#139)
- save cookies on page load and also before profile creation - default cookie duration is 7 days, configurable via --cookieDays option
This commit is contained in:
parent
93b6dad7b9
commit
6ec47cdd14
1 changed files with 47 additions and 1 deletions
|
@ -73,6 +73,12 @@ function cliOpts() {
|
||||||
"proxy": {
|
"proxy": {
|
||||||
type: "boolean",
|
type: "boolean",
|
||||||
default: false
|
default: false
|
||||||
|
},
|
||||||
|
|
||||||
|
"cookieDays": {
|
||||||
|
type: "number",
|
||||||
|
describe: "If >0, set all cookies, including session cookies, to have this duration in days before saving profile",
|
||||||
|
default: 7
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -267,7 +273,7 @@ class InteractiveBrowser {
|
||||||
|
|
||||||
this.addOrigin();
|
this.addOrigin();
|
||||||
|
|
||||||
page.on("load", () => this.addOrigin());
|
page.on("load", () => this.handlePageLoad());
|
||||||
|
|
||||||
page.on("popup", async () => {
|
page.on("popup", async () => {
|
||||||
await this.page._client.send("Target.activateTarget", {targetId: this.targetId});
|
await this.page._client.send("Target.activateTarget", {targetId: this.targetId});
|
||||||
|
@ -294,6 +300,41 @@ class InteractiveBrowser {
|
||||||
console.log(`Browser Profile UI Server started. Load http://localhost:${port}/ to interact with a Chromium-based browser, click 'Create Profile' when done.`);
|
console.log(`Browser Profile UI Server started. Load http://localhost:${port}/ to interact with a Chromium-based browser, click 'Create Profile' when done.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
handlePageLoad() {
|
||||||
|
this.addOrigin();
|
||||||
|
this.saveCookiesFor(this.page.url());
|
||||||
|
}
|
||||||
|
|
||||||
|
async saveAllCookies() {
|
||||||
|
console.log("Saving all cookies");
|
||||||
|
|
||||||
|
for (const origin of this.originSet.values()) {
|
||||||
|
await this.saveCookiesFor(origin + "/");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async saveCookiesFor(url) {
|
||||||
|
try {
|
||||||
|
if (this.params.cookieDays <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cookies = await this.page.cookies(url);
|
||||||
|
for (const cookie of cookies) {
|
||||||
|
cookie.url = url;
|
||||||
|
cookie.expires = (new Date().getTime() / 1000) + this.params.cookieDays * 7;
|
||||||
|
delete cookie.size;
|
||||||
|
delete cookie.session;
|
||||||
|
if (cookie.sameSite && cookie.sameSite !== "Lax" && cookie.sameSite !== "Strict") {
|
||||||
|
delete cookie.sameSite;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await this.page.setCookie(...cookies);
|
||||||
|
} catch (e) {
|
||||||
|
console.log("Save Cookie Error: " + e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
addOrigin() {
|
addOrigin() {
|
||||||
const url = this.page.url();
|
const url = this.page.url();
|
||||||
console.log("Adding origin for", url);
|
console.log("Adding origin for", url);
|
||||||
|
@ -363,6 +404,9 @@ class InteractiveBrowser {
|
||||||
try {
|
try {
|
||||||
const postData = await this.readBodyJson(req);
|
const postData = await this.readBodyJson(req);
|
||||||
const targetFilename = postData.filename || "";
|
const targetFilename = postData.filename || "";
|
||||||
|
|
||||||
|
await this.saveAllCookies();
|
||||||
|
|
||||||
const resource = await createProfile(this.params, this.browser, this.page, targetFilename);
|
const resource = await createProfile(this.params, this.browser, this.page, targetFilename);
|
||||||
origins = Array.from(this.originSet.values());
|
origins = Array.from(this.originSet.values());
|
||||||
|
|
||||||
|
@ -383,6 +427,8 @@ class InteractiveBrowser {
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
await this.saveAllCookies();
|
||||||
|
|
||||||
await createProfile(this.params, this.browser, this.page);
|
await createProfile(this.params, this.browser, this.page);
|
||||||
|
|
||||||
res.writeHead(200, {"Content-Type": "text/html"});
|
res.writeHead(200, {"Content-Type": "text/html"});
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue