Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 06:23:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

cleanup after merge, add setRequestHeaders

Browse source
This commit is contained in:
Ilya Kreymer 2024-05-26 12:13:00 -07:00
parent f729bba031
commit abc735cd5a
2 changed files with 15 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/util/recorder.ts
View file

@ -1083,11 +1083,7 @@ export class Recorder {
headers["Cookie"] = cookie;
}
reqresp.url = url;
reqresp.method = "GET";
reqresp.requestHeaders = headers;
reqresp.ts = ts;
reqresp.setRequestHeaders(url, "GET", headers);
logger.debug(
"Directly fetching page URL without browser",
{ url, ...this.logDetails },
@ -1123,10 +1119,6 @@ export class Recorder {
});
const res = await fetcher.load();
const mime = (reqresp.responseHeaders.get("content-type") || "").split(
";",
)[0];
if (url === this.pageUrl && !this.pageInfo.ts) {
logger.debug("Setting page timestamp", { ts, url });
this.pageInfo.ts = ts;

20
src/util/reqresp.ts
View file

@ -90,12 +90,7 @@ export class RequestResponseInfo {
}
fillRequest(request: Protocol.Network.Request, resourceType: string) {
this.url = request.url;
this.method = request.method;
if (!this.requestHeadersDict) {
this.requestHeadersDict = request.headers;
}
this.requestHeaders = this.getRequestHeaders();
this.setRequestHeaders(request.url, request.method, request.headers);
this.postData = request.postData;
this.hasPostData = request.hasPostData || false;
@ -204,6 +199,19 @@ export class RequestResponseInfo {
);
}
setRequestHeaders(
url: string,
method: string,
headers: Record<string, string>,
) {
this.url = url;
this.method = method;
if (!this.requestHeadersDict) {
this.requestHeadersDict = headers;
}
this.requestHeaders = this.getRequestHeaders();
}
setResponseHeaders(
headersDict: Record<string, string>,
status: number,