Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-12-08 06:09:48 +00:00
Code Issues Projects Releases Packages Wiki Activity

rename --dedupStoreUrl -> redisDedupUrl

Browse source 
bump version to 1.9.0
fix typo
This commit is contained in:
Ilya Kreymer 2025-09-17 23:36:25 -07:00
parent bbe084daa0
commit db9e78e823
4 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/crawler.ts
View file

@ -341,7 +341,7 @@ export class Crawler {
async initCrawlState() { async initCrawlState() {
const redisUrl = this.params.redisStoreUrl || "redis://localhost:6379/0"; const redisUrl = this.params.redisStoreUrl || "redis://localhost:6379/0";
const dedupRedisUrl = this.params.dedupStoreUrl || redisUrl; const dedupRedisUrl = this.params.redisDedupUrl || redisUrl;
if (!redisUrl.startsWith("redis://")) { if (!redisUrl.startsWith("redis://")) {
logger.fatal( logger.fatal(

4
src/indexer.ts
View file

@ -17,7 +17,7 @@ export class CrawlIndexer {
return yargs(process.argv) return yargs(process.argv)
.usage("indexer [options]") .usage("indexer [options]")
.options({ .options({
dedupStoreUrl: { redisDedupUrl: {
describe: "URL for remote redis instance to index into", describe: "URL for remote redis instance to index into",
type: "string", type: "string",
required: true, required: true,
@ -43,7 +43,7 @@ export class CrawlIndexer {
const params = this.initArgs(); const params = this.initArgs();
const redis = await initRedisWaitForSuccess(params.dedupStoreUrl); const redis = await initRedisWaitForSuccess(params.redisDedupUrl);
const dedupIndex = new RedisDedupIndex(redis); const dedupIndex = new RedisDedupIndex(redis);
const allFiles = []; const allFiles = [];

2
src/util/argParser.ts
View file

@ -445,7 +445,7 @@ class ArgParser {
default: "redis://localhost:6379/0", default: "redis://localhost:6379/0",
}, },
dedupStoreUrl: { redisDedupUrl: {
describe: describe:
"If set, url for remote redis server to store state. Otherwise, using local redis instance", "If set, url for remote redis server to store state. Otherwise, using local redis instance",
type: "string", type: "string",

2
src/util/recorder.ts
View file

@ -2118,7 +2118,7 @@ async function createRevisitForResponse(
const origWarcHeaders = responseRecord.warcHeaders.headers; const origWarcHeaders = responseRecord.warcHeaders.headers;
for (const header in REVISIT_COPY_HEADERS) { for (const header of REVISIT_COPY_HEADERS) {
if (origWarcHeaders.has(header)) { if (origWarcHeaders.has(header)) {
warcHeaders[header] = origWarcHeaders.get(header)!; warcHeaders[header] = origWarcHeaders.get(header)!;
} }