Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 14:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

logging and beheaviors improvements (#389)

Browse source 
- run behaviors: check if behaviors object exists before trying to run behaviors to avoid failure message
- skip behaviors if frame no longer attached / has empty URL
This commit is contained in:
Ilya Kreymer 2023-09-20 14:02:37 -05:00 • committed by GitHub
parent c6cbbc1a17
commit 165a9787af
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

12
crawler.js
View file

@ -589,7 +589,13 @@ self.__bx_behaviors.selectMainBehavior();
logger.info("Running behaviors", {frames: frames.length, frameUrls: frames.map(frame => frame.url()), ...logDetails}, "behavior"); logger.info("Running behaviors", {frames: frames.length, frameUrls: frames.map(frame => frame.url()), ...logDetails}, "behavior");
const results = await Promise.allSettled( const results = await Promise.allSettled(
frames.map(frame => this.browser.evaluateWithCLI(page, frame, cdp, "self.__bx_behaviors.run();", logDetails, "behavior")) frames.map(frame => this.browser.evaluateWithCLI(page, frame, cdp, `
if (!self.__bx_behaviors) {
console.error("__bx_behaviors missing, can't run behaviors");
} else {
self.__bx_behaviors.run();
}`
, logDetails, "behavior"))
); );
for (const {status, reason} in results) { for (const {status, reason} in results) {
@ -1177,8 +1183,8 @@ self.__bx_behaviors.selectMainBehavior();
frames = await Promise.allSettled(frames.map((frame) => this.shouldIncludeFrame(frame, logDetails))); frames = await Promise.allSettled(frames.map((frame) => this.shouldIncludeFrame(frame, logDetails)));
data.filteredFrames = frames.filter((x) => { data.filteredFrames = frames.filter((x) => {
if (x.status === "fulfilled" && x.value) { if (x.status === "fulfilled") {
return true; return !!x.value;
} }
logger.warn("Error in iframe check", {reason: x.reason, ...logDetails}); logger.warn("Error in iframe check", {reason: x.reason, ...logDetails});
return false; return false;

8
util/browser.js
View file

@ -163,7 +163,13 @@ export class BaseBrowser
} }
async evaluateWithCLI_(cdp, frame, cdpContextId, funcString, logData, contextName) { async evaluateWithCLI_(cdp, frame, cdpContextId, funcString, logData, contextName) {
let details = {frameUrl: frame.url(), ...logData}; const frameUrl = frame.url();
let details = {frameUrl, ...logData};
if (!frameUrl || frame.isDetached()) {
logger.info("Run Script Skipped, frame no longer attached or has no URL", details, contextName);
return false;
}
logger.info("Run Script Started", details, contextName); logger.info("Run Script Started", details, contextName);