Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-12-08 06:09:48 +00:00
Code Issues Projects Releases Packages Wiki Activity

adjust name

Browse source
This commit is contained in:
Ilya Kreymer 2025-12-06 13:13:08 -08:00
parent 4c59c6d70e
commit f7c0adacf3
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/util/state.ts
View file

@ -30,7 +30,7 @@ export enum QueueState {
} }
// ============================================================================ // ============================================================================
const normalizeOpts: Options = { const normalizeUrlOpts: Options = {
defaultProtocol: "https", defaultProtocol: "https",
stripAuthentication: false, stripAuthentication: false,
stripTextFragment: false, stripTextFragment: false,
@ -699,7 +699,7 @@ return inx;
}: QueueEntry, }: QueueEntry,
limit = 0, limit = 0,
) { ) {
url = normalizeUrl(url, normalizeOpts); url = normalizeUrl(url, normalizeUrlOpts);
const added = this._timestamp(); const added = this._timestamp();
const data: QueueEntry = { added, url, seedId, depth, extraHops }; const data: QueueEntry = { added, url, seedId, depth, extraHops };
@ -1025,7 +1025,7 @@ return inx;
} }
async addIfNoDupe(key: string, url: string, status: number) { async addIfNoDupe(key: string, url: string, status: number) {
url = normalizeUrl(url, normalizeOpts); url = normalizeUrl(url, normalizeUrlOpts);
return ( return (
(await this.redis.sadd(key, normalizeDedupStatus(status) + "|" + url)) === (await this.redis.sadd(key, normalizeDedupStatus(status) + "|" + url)) ===
1 1