Stowage/browsertrix-crawler
2
0
Fork 0
mirror of https://github.com/webrecorder/browsertrix-crawler.git synced 2025-10-19 06:23:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

sitemap: only parse sitemap if original seed was added, skip if seed is already in the list/was skipped

Browse source
This commit is contained in:
Ilya Kreymer 2023-08-14 22:00:31 -07:00
parent 5ba6c33bff
commit 5d1a391d5e
1 changed files with 2 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crawler.js
View file

@ -758,9 +758,8 @@ self.__bx_behaviors.selectMainBehavior();
if (this.limitHit) {
break;
}
}
if (seed.sitemap) {
// parse sitemap if seed is queued, otherwise assume already parsed
} else if (seed.sitemap) {
await this.parseSitemap(seed.sitemap, i);
}
}