mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 06:23:16 +00:00
cleanup after merge, add setRequestHeaders
This commit is contained in:
parent
f729bba031
commit
abc735cd5a
2 changed files with 15 additions and 15 deletions
|
@ -1083,11 +1083,7 @@ export class Recorder {
|
||||||
headers["Cookie"] = cookie;
|
headers["Cookie"] = cookie;
|
||||||
}
|
}
|
||||||
|
|
||||||
reqresp.url = url;
|
reqresp.setRequestHeaders(url, "GET", headers);
|
||||||
reqresp.method = "GET";
|
|
||||||
reqresp.requestHeaders = headers;
|
|
||||||
reqresp.ts = ts;
|
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Directly fetching page URL without browser",
|
"Directly fetching page URL without browser",
|
||||||
{ url, ...this.logDetails },
|
{ url, ...this.logDetails },
|
||||||
|
@ -1123,10 +1119,6 @@ export class Recorder {
|
||||||
});
|
});
|
||||||
const res = await fetcher.load();
|
const res = await fetcher.load();
|
||||||
|
|
||||||
const mime = (reqresp.responseHeaders.get("content-type") || "").split(
|
|
||||||
";",
|
|
||||||
)[0];
|
|
||||||
|
|
||||||
if (url === this.pageUrl && !this.pageInfo.ts) {
|
if (url === this.pageUrl && !this.pageInfo.ts) {
|
||||||
logger.debug("Setting page timestamp", { ts, url });
|
logger.debug("Setting page timestamp", { ts, url });
|
||||||
this.pageInfo.ts = ts;
|
this.pageInfo.ts = ts;
|
||||||
|
|
|
@ -90,12 +90,7 @@ export class RequestResponseInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
fillRequest(request: Protocol.Network.Request, resourceType: string) {
|
fillRequest(request: Protocol.Network.Request, resourceType: string) {
|
||||||
this.url = request.url;
|
this.setRequestHeaders(request.url, request.method, request.headers);
|
||||||
this.method = request.method;
|
|
||||||
if (!this.requestHeadersDict) {
|
|
||||||
this.requestHeadersDict = request.headers;
|
|
||||||
}
|
|
||||||
this.requestHeaders = this.getRequestHeaders();
|
|
||||||
this.postData = request.postData;
|
this.postData = request.postData;
|
||||||
this.hasPostData = request.hasPostData || false;
|
this.hasPostData = request.hasPostData || false;
|
||||||
|
|
||||||
|
@ -204,6 +199,19 @@ export class RequestResponseInfo {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setRequestHeaders(
|
||||||
|
url: string,
|
||||||
|
method: string,
|
||||||
|
headers: Record<string, string>,
|
||||||
|
) {
|
||||||
|
this.url = url;
|
||||||
|
this.method = method;
|
||||||
|
if (!this.requestHeadersDict) {
|
||||||
|
this.requestHeadersDict = headers;
|
||||||
|
}
|
||||||
|
this.requestHeaders = this.getRequestHeaders();
|
||||||
|
}
|
||||||
|
|
||||||
setResponseHeaders(
|
setResponseHeaders(
|
||||||
headersDict: Record<string, string>,
|
headersDict: Record<string, string>,
|
||||||
status: number,
|
status: number,
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue