Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 06:23:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

don't remove 200 status code

Browse source 
use 'status' instead of 'statusCode'
This commit is contained in:
Ilya Kreymer 2024-02-28 15:12:54 -08:00
parent ebab39cdf5
commit 10a42d658f
2 changed files with 12 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

20
src/crawler.ts
View file

@ -91,7 +91,7 @@ type PageEntry = {
text?: string; text?: string;
favIconUrl?: string; favIconUrl?: string;
ts?: string; ts?: string;
statusCode?: number; status?: number;
}; };
// ============================================================================ // ============================================================================
@ -720,7 +720,7 @@ self.__bx_behaviors.selectMainBehavior();
if (mime) { if (mime) {
data.mime = mime; data.mime = mime;
} }
data.statusCode = 200; data.status = 200;
logger.info( logger.info(
"Direct fetch successful", "Direct fetch successful",
{ url, ...logDetails }, { url, ...logDetails },
@ -789,7 +789,7 @@ self.__bx_behaviors.selectMainBehavior();
data.loadState = LoadState.EXTRACTION_DONE; data.loadState = LoadState.EXTRACTION_DONE;
if (data.statusCode >= 400) { if (data.status >= 400) {
return; return;
} }
@ -1587,17 +1587,17 @@ self.__bx_behaviors.selectMainBehavior();
} }
// Handle 4xx or 5xx response as a page load error // Handle 4xx or 5xx response as a page load error
const statusCode = resp.status(); const status = resp.status();
data.statusCode = statusCode; data.status = status;
if (isChromeError) { if (isChromeError) {
if (failCrawlOnError) { if (failCrawlOnError) {
logger.fatal("Seed Page Load Error, failing crawl", { logger.fatal("Seed Page Load Error, failing crawl", {
statusCode, status,
...logDetails, ...logDetails,
}); });
} else { } else {
logger.error("Page Crashed on Load", { logger.error("Page Crashed on Load", {
statusCode, status,
...logDetails, ...logDetails,
}); });
throw new Error("logged"); throw new Error("logged");
@ -1960,7 +1960,7 @@ self.__bx_behaviors.selectMainBehavior();
mime, mime,
favicon, favicon,
ts, ts,
statusCode, status,
}: PageState) { }: PageState) {
const row: PageEntry = { id: pageid!, url, title, loadState }; const row: PageEntry = { id: pageid!, url, title, loadState };
@ -1972,8 +1972,8 @@ self.__bx_behaviors.selectMainBehavior();
row.mime = mime; row.mime = mime;
} }
if (statusCode && statusCode !== 200) { if (status) {
row.statusCode = statusCode; row.status = status;
} }
if (this.params.writePagesToRedis) { if (this.params.writePagesToRedis) {

4
src/util/state.ts
View file

@ -46,7 +46,7 @@ export class PageState {
depth: number; depth: number;
extraHops: number; extraHops: number;
statusCode: number; status: number;
workerid!: WorkerId; workerid!: WorkerId;
@ -72,7 +72,7 @@ export class PageState {
this.seedId = redisData.seedId; this.seedId = redisData.seedId;
this.depth = redisData.depth; this.depth = redisData.depth;
this.extraHops = redisData.extraHops || 0; this.extraHops = redisData.extraHops || 0;
this.statusCode = 0; this.status = 0;
} }
} }