From b98aa8aea1f54c2cc354a691cd988fadeeab7c83 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Thu, 20 Nov 2025 13:04:52 -0800 Subject: [PATCH] update behaviors --- Dockerfile | 2 +- behaviors.js | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 behaviors.js diff --git a/Dockerfile b/Dockerfile index ec5de684..a815b9d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,7 +55,7 @@ RUN mkdir -p /app/behaviors WORKDIR /crawls # enable to test custom behaviors build (from browsertrix-behaviors) -# COPY behaviors.js /app/node_modules/browsertrix-behaviors/dist/behaviors.js +COPY behaviors.js /app/node_modules/browsertrix-behaviors/dist/behaviors.js # add brave/chromium group policies RUN mkdir -p /etc/brave/policies/managed/ diff --git a/behaviors.js b/behaviors.js new file mode 100644 index 00000000..9eaca3c9 --- /dev/null +++ b/behaviors.js @@ -0,0 +1,55 @@ +/*! behaviors.js is part of Webrecorder project. Copyright (C) 2021-2025, Webrecorder Software. Licensed under the Affero General Public License v3. */(()=>{var __webpack_modules__={"./node_modules/query-selector-shadow-dom/src/normalize.js": +/*!*****************************************************************!*\ + !*** ./node_modules/query-selector-shadow-dom/src/normalize.js ***! + \*****************************************************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "normalizeSelector": () => (/* binding */ normalizeSelector)\n/* harmony export */ });\n/* istanbul ignore file */\n\n\n// normalize-selector-rev-02.js\n/*\n author: kyle simpson (@getify)\n original source: https://gist.github.com/getify/9679380\n\n modified for tests by david kaye (@dfkaye)\n 21 march 2014\n\n rev-02 incorporate kyle\'s changes 3/2/42014\n*/\n\nfunction normalizeSelector(sel) {\n // save unmatched text, if any\n function saveUnmatched() {\n if (unmatched) {\n // whitespace needed after combinator?\n if (tokens.length > 0 && /^[~+>]$/.test(tokens[tokens.length - 1])) {\n tokens.push(" ");\n }\n\n // save unmatched text\n tokens.push(unmatched);\n }\n }\n\n var tokens = [],\n match,\n unmatched,\n regex,\n state = [0],\n next_match_idx = 0,\n prev_match_idx,\n not_escaped_pattern = /(?:[^\\\\]|(?:^|[^\\\\])(?:\\\\\\\\)+)$/,\n whitespace_pattern = /^\\s+$/,\n state_patterns = [\n /\\s+|\\/\\*|["\'>~+[(]/g, // general\n /\\s+|\\/\\*|["\'[\\]()]/g, // [..] set\n /\\s+|\\/\\*|["\'[\\]()]/g, // (..) set\n null, // string literal (placeholder)\n /\\*\\//g, // comment\n ];\n sel = sel.trim();\n\n // eslint-disable-next-line no-constant-condition\n while (true) {\n unmatched = "";\n\n regex = state_patterns[state[state.length - 1]];\n\n regex.lastIndex = next_match_idx;\n match = regex.exec(sel);\n\n // matched text to process?\n if (match) {\n prev_match_idx = next_match_idx;\n next_match_idx = regex.lastIndex;\n\n // collect the previous string chunk not matched before this token\n if (prev_match_idx < next_match_idx - match[0].length) {\n unmatched = sel.substring(\n prev_match_idx,\n next_match_idx - match[0].length\n );\n }\n\n // general, [ ] pair, ( ) pair?\n if (state[state.length - 1] < 3) {\n saveUnmatched();\n\n // starting a [ ] pair?\n if (match[0] === "[") {\n state.push(1);\n }\n // starting a ( ) pair?\n else if (match[0] === "(") {\n state.push(2);\n }\n // starting a string literal?\n else if (/^["\']$/.test(match[0])) {\n state.push(3);\n state_patterns[3] = new RegExp(match[0], "g");\n }\n // starting a comment?\n else if (match[0] === "/*") {\n state.push(4);\n }\n // ending a [ ] or ( ) pair?\n else if (/^[\\])]$/.test(match[0]) && state.length > 0) {\n state.pop();\n }\n // handling whitespace or a combinator?\n else if (/^(?:\\s+|[~+>])$/.test(match[0])) {\n // need to insert whitespace before?\n if (\n tokens.length > 0 &&\n !whitespace_pattern.test(tokens[tokens.length - 1]) &&\n state[state.length - 1] === 0\n ) {\n // add normalized whitespace\n tokens.push(" ");\n }\n\n // case-insensitive attribute selector CSS L4\n if (\n state[state.length - 1] === 1 &&\n tokens.length === 5 &&\n tokens[2].charAt(tokens[2].length - 1) === "="\n ) {\n tokens[4] = " " + tokens[4];\n }\n\n // whitespace token we can skip?\n if (whitespace_pattern.test(match[0])) {\n continue;\n }\n }\n\n // save matched text\n tokens.push(match[0]);\n }\n // otherwise, string literal or comment\n else {\n // save unmatched text\n tokens[tokens.length - 1] += unmatched;\n\n // unescaped terminator to string literal or comment?\n if (not_escaped_pattern.test(tokens[tokens.length - 1])) {\n // comment terminator?\n if (state[state.length - 1] === 4) {\n // ok to drop comment?\n if (\n tokens.length < 2 ||\n whitespace_pattern.test(tokens[tokens.length - 2])\n ) {\n tokens.pop();\n }\n // otherwise, turn comment into whitespace\n else {\n tokens[tokens.length - 1] = " ";\n }\n\n // handled already\n match[0] = "";\n }\n\n state.pop();\n }\n\n // append matched text to existing token\n tokens[tokens.length - 1] += match[0];\n }\n }\n // otherwise, end of processing (no more matches)\n else {\n unmatched = sel.substr(next_match_idx);\n saveUnmatched();\n\n break;\n }\n }\n\n return tokens.join("").trim();\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./node_modules/query-selector-shadow-dom/src/normalize.js?')},"./node_modules/query-selector-shadow-dom/src/querySelectorDeep.js": +/*!*************************************************************************!*\ + !*** ./node_modules/query-selector-shadow-dom/src/querySelectorDeep.js ***! + \*************************************************************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval("__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ \"collectAllElementsDeep\": () => (/* binding */ collectAllElementsDeep),\n/* harmony export */ \"querySelectorAllDeep\": () => (/* binding */ querySelectorAllDeep),\n/* harmony export */ \"querySelectorDeep\": () => (/* binding */ querySelectorDeep)\n/* harmony export */ });\n/* harmony import */ var _normalize__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./normalize */ \"./node_modules/query-selector-shadow-dom/src/normalize.js\");\n\n\n/**\n* Finds first matching elements on the page that may be in a shadow root using a complex selector of n-depth\n*\n* Don't have to specify all shadow roots to button, tree is travered to find the correct element\n*\n* Example querySelectorAllDeep('downloads-item:nth-child(4) #remove');\n*\n* Example should work on chrome://downloads outputting the remove button inside of a download card component\n*\n* Example find first active download link element querySelectorDeep('#downloads-list .is-active a[href^=\"https://\"]');\n*\n* Another example querySelectorAllDeep('#downloads-list div#title-area + a');\ne.g.\n*/\nfunction querySelectorAllDeep(selector, root = document, allElements = null) {\n return _querySelectorDeep(selector, true, root, allElements);\n}\n\nfunction querySelectorDeep(selector, root = document, allElements = null) {\n return _querySelectorDeep(selector, false, root, allElements);\n}\n\nfunction _querySelectorDeep(selector, findMany, root, allElements = null) {\n selector = (0,_normalize__WEBPACK_IMPORTED_MODULE_0__.normalizeSelector)(selector);\n let lightElement = root.querySelector(selector);\n\n if (document.head.createShadowRoot || document.head.attachShadow) {\n // no need to do any special if selector matches something specific in light-dom\n if (!findMany && lightElement) {\n return lightElement;\n }\n\n // split on commas because those are a logical divide in the operation\n const selectionsToMake = splitByCharacterUnlessQuoted(selector, ',');\n\n return selectionsToMake.reduce((acc, minimalSelector) => {\n // if not finding many just reduce the first match\n if (!findMany && acc) {\n return acc;\n }\n // do best to support complex selectors and split the query\n const splitSelector = splitByCharacterUnlessQuoted(minimalSelector\n //remove white space at start of selector\n .replace(/^\\s+/g, '')\n .replace(/\\s*([>+~]+)\\s*/g, '$1'), ' ')\n // filter out entry white selectors\n .filter((entry) => !!entry)\n // convert \"a > b\" to [\"a\", \"b\"]\n .map((entry) => splitByCharacterUnlessQuoted(entry, '>'));\n\n const possibleElementsIndex = splitSelector.length - 1;\n const lastSplitPart = splitSelector[possibleElementsIndex][splitSelector[possibleElementsIndex].length - 1];\n const possibleElements = collectAllElementsDeep(lastSplitPart, root, allElements);\n const findElements = findMatchingElement(splitSelector, possibleElementsIndex, root);\n if (findMany) {\n acc = acc.concat(possibleElements.filter(findElements));\n return acc;\n } else {\n acc = possibleElements.find(findElements);\n return acc || null;\n }\n }, findMany ? [] : null);\n\n\n } else {\n if (!findMany) {\n return lightElement;\n } else {\n return root.querySelectorAll(selector);\n }\n }\n\n}\n\nfunction findMatchingElement(splitSelector, possibleElementsIndex, root) {\n return (element) => {\n let position = possibleElementsIndex;\n let parent = element;\n let foundElement = false;\n while (parent && !isDocumentNode(parent)) {\n let foundMatch = true;\n if (splitSelector[position].length === 1) {\n foundMatch = parent.matches(splitSelector[position]);\n } else {\n // selector is in the format \"a > b\"\n // make sure a few parents match in order\n const reversedParts = ([]).concat(splitSelector[position]).reverse();\n let newParent = parent;\n for (const part of reversedParts) {\n if (!newParent || !newParent.matches(part)) {\n foundMatch = false;\n break;\n }\n newParent = findParentOrHost(newParent, root);\n }\n }\n\n if (foundMatch && position === 0) {\n foundElement = true;\n break;\n }\n if (foundMatch) {\n position--;\n }\n parent = findParentOrHost(parent, root);\n }\n return foundElement;\n };\n\n}\n\nfunction splitByCharacterUnlessQuoted(selector, character) {\n return selector.match(/\\\\?.|^$/g).reduce((p, c) => {\n if (c === '\"' && !p.sQuote) {\n p.quote ^= 1;\n p.a[p.a.length - 1] += c;\n } else if (c === '\\'' && !p.quote) {\n p.sQuote ^= 1;\n p.a[p.a.length - 1] += c;\n\n } else if (!p.quote && !p.sQuote && c === character) {\n p.a.push('');\n } else {\n p.a[p.a.length - 1] += c;\n }\n return p;\n }, { a: [''] }).a;\n}\n\n/**\n * Checks if the node is a document node or not.\n * @param {Node} node\n * @returns {node is Document | DocumentFragment}\n */\nfunction isDocumentNode(node) {\n return node.nodeType === Node.DOCUMENT_FRAGMENT_NODE || node.nodeType === Node.DOCUMENT_NODE;\n}\n\nfunction findParentOrHost(element, root) {\n const parentNode = element.parentNode;\n return (parentNode && parentNode.host && parentNode.nodeType === 11) ? parentNode.host : parentNode === root ? null : parentNode;\n}\n\n/**\n * Finds all elements on the page, inclusive of those within shadow roots.\n * @param {string=} selector Simple selector to filter the elements by. e.g. 'a', 'div.main'\n * @return {!Array} List of anchor hrefs.\n * @author ebidel@ (Eric Bidelman)\n * License Apache-2.0\n */\nfunction collectAllElementsDeep(selector = null, root, cachedElements = null) {\n let allElements = [];\n\n if (cachedElements) {\n allElements = cachedElements;\n } else {\n const findAllElements = function(nodes) {\n for (let i = 0; i < nodes.length; i++) {\n const el = nodes[i];\n allElements.push(el);\n // If the element has a shadow root, dig deeper.\n if (el.shadowRoot) {\n findAllElements(el.shadowRoot.querySelectorAll('*'));\n }\n }\n };\n if(root.shadowRoot) {\n findAllElements(root.shadowRoot.querySelectorAll('*'));\n }\n findAllElements(root.querySelectorAll('*'));\n }\n\n return selector ? allElements.filter(el => el.matches(selector)) : allElements;\t}\n\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./node_modules/query-selector-shadow-dom/src/querySelectorDeep.js?")},"./index.ts": +/*!******************!*\ + !*** ./index.ts ***! + \******************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "BehaviorManager": () => (/* reexport safe */ _src__WEBPACK_IMPORTED_MODULE_0__.BehaviorManager)\n/* harmony export */ });\n/* harmony import */ var _src__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./src */ "./src/index.ts");\n\n\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./index.ts?')},"./src/autoclick.ts": +/*!**************************!*\ + !*** ./src/autoclick.ts ***! + \**************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "AutoClick": () => (/* binding */ AutoClick)\n/* harmony export */ });\n/* harmony import */ var _lib_behavior__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./lib/behavior */ "./src/lib/behavior.ts");\n/* harmony import */ var _lib_utils__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./lib/utils */ "./src/lib/utils.ts");\n\n\nclass AutoClick extends _lib_behavior__WEBPACK_IMPORTED_MODULE_0__.BackgroundBehavior {\n _donePromise;\n _markDone;\n selector;\n seenElem = new WeakSet();\n static id = "Autoclick";\n constructor(selector = "a") {\n super();\n this.selector = selector;\n this._donePromise = new Promise((resolve) => (this._markDone = resolve));\n }\n nextSameOriginLink() {\n try {\n const allLinks = document.querySelectorAll(this.selector);\n for (const el of allLinks) {\n const elem = el;\n if (elem.href && !elem.href.startsWith(self.location.origin)) {\n continue;\n }\n if (!elem.isConnected) {\n continue;\n }\n if (!elem.checkVisibility()) {\n continue;\n }\n if (this.seenElem.has(elem)) {\n continue;\n }\n this.seenElem.add(elem);\n return elem;\n }\n }\n catch (e) {\n this.debug(e.toString());\n }\n return null;\n }\n async start() {\n const beforeUnload = (event) => {\n event.preventDefault();\n return false;\n };\n window.addEventListener("beforeunload", beforeUnload);\n while (true) {\n const elem = this.nextSameOriginLink();\n if (!elem) {\n break;\n }\n await this.processElem(elem);\n }\n window.removeEventListener("beforeunload", beforeUnload);\n this._markDone();\n }\n async processElem(elem) {\n if (elem.target) {\n return;\n }\n if (elem.href) {\n if (!(await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.addToExternalSet)(elem.href))) {\n return;\n }\n this.debug("Clicking on link: " + elem.href);\n }\n else {\n this.debug("Click empty link");\n }\n const origHref = self.location.href;\n const origHistoryLen = self.history.length;\n if (elem.click) {\n elem.click();\n }\n else if (elem.dispatchEvent) {\n elem.dispatchEvent(new MouseEvent("click"));\n }\n await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.sleep)(250);\n if (self.history.length === origHistoryLen + 1 &&\n self.location.href != origHref) {\n await new Promise((resolve) => {\n window.addEventListener("popstate", () => {\n resolve(null);\n }, { once: true });\n window.history.back();\n });\n }\n }\n catch(e) {\n this.debug(e.toString());\n }\n done() {\n return this._donePromise;\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/autoclick.ts?')},"./src/autofetcher.ts": +/*!****************************!*\ + !*** ./src/autofetcher.ts ***! + \****************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "AutoFetcher": () => (/* binding */ AutoFetcher)\n/* harmony export */ });\n/* harmony import */ var query_selector_shadow_dom__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! query-selector-shadow-dom */ "./node_modules/query-selector-shadow-dom/src/querySelectorDeep.js");\n/* harmony import */ var _lib_behavior__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./lib/behavior */ "./src/lib/behavior.ts");\n/* harmony import */ var _lib_utils__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./lib/utils */ "./src/lib/utils.ts");\n\n\n\nconst SRC_SET_SELECTOR = "img[srcset], img[data-srcset], img[data-src], noscript > img[src], img[loading=\'lazy\'], " +\n "video[srcset], video[data-srcset], video[data-src], audio[srcset], audio[data-srcset], audio[data-src], " +\n "picture > source[srcset], picture > source[data-srcset], picture > source[data-src], " +\n "video > source[srcset], video > source[data-srcset], video > source[data-src], " +\n "audio > source[srcset], audio > source[data-srcset], audio > source[data-src]";\nconst SRCSET_REGEX = /\\s*(\\S*\\s+[\\d.]+[wx]),|(?:\\s*,(?:\\s+|(?=https?:)))/;\nconst STYLE_REGEX = /(url\\s*\\(\\s*[\\\\"\']*)([^)\'"]+)([\\\\"\']*\\s*\\))/gi;\nconst IMPORT_REGEX = /(@import\\s*[\\\\"\']*)([^)\'";]+)([\\\\"\']*\\s*;?)/gi;\nconst MAX_CONCURRENT = 6;\nclass AutoFetcher extends _lib_behavior__WEBPACK_IMPORTED_MODULE_1__.BackgroundBehavior {\n urlSet = new Set();\n pendingQueue = [];\n waitQueue = [];\n mutationObserver;\n numPending = 0;\n numDone = 0;\n headers;\n _donePromise;\n _markDone;\n active;\n running = false;\n static id = "Autofetcher";\n constructor(active = false, headers = null, startEarly = false) {\n super();\n this.headers = headers || {};\n this._donePromise = new Promise((resolve) => (this._markDone = resolve));\n this.active = active;\n if (this.active && startEarly) {\n document.addEventListener("DOMContentLoaded", () => this.initObserver());\n }\n }\n get numFetching() {\n return this.numDone + this.numPending + this.pendingQueue.length;\n }\n async start() {\n if (!this.active) {\n return;\n }\n this.initObserver();\n this.run();\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_2__.sleep)(500).then(() => {\n if (!this.pendingQueue.length && !this.numPending) {\n this._markDone(null);\n }\n });\n }\n done() {\n return this._donePromise;\n }\n async run() {\n this.running = true;\n for (const url of this.waitQueue) {\n this.doFetch(url);\n }\n this.waitQueue = [];\n this.extractSrcSrcSetAll(document);\n this.extractStyleSheets();\n this.extractDataAttributes(document);\n }\n isValidUrl(url) {\n return url && (url.startsWith("http:") || url.startsWith("https:"));\n }\n queueUrl(url, immediate = false) {\n try {\n url = new URL(url, document.baseURI).href;\n }\n catch (e) {\n return false;\n }\n if (!this.isValidUrl(url)) {\n return false;\n }\n if (this.urlSet.has(url)) {\n return false;\n }\n this.urlSet.add(url);\n if (this.running || immediate) {\n this.doFetch(url);\n }\n else {\n this.waitQueue.push(url);\n }\n return true;\n }\n async doFetchStream(url) {\n try {\n const resp = await fetch(url, {\n credentials: "include",\n referrerPolicy: "origin-when-cross-origin",\n });\n this.debug(`Autofetch: started ${url}`);\n const reader = resp.body.getReader();\n let res = null;\n while ((res = await reader.read()) && !res.done)\n ;\n this.debug(`Autofetch: finished ${url}`);\n return true;\n }\n catch (e) {\n this.debug(e);\n return false;\n }\n }\n async doFetchNonCors(url) {\n try {\n const abort = new AbortController();\n await fetch(url, {\n mode: "no-cors",\n credentials: "include",\n referrerPolicy: "origin-when-cross-origin",\n headers: this.headers,\n abort,\n });\n abort.abort();\n this.debug(`Autofetch: started non-cors stream for ${url}`);\n }\n catch (e) {\n this.debug(`Autofetch: failed non-cors for ${url}`);\n }\n }\n async doFetch(url) {\n this.pendingQueue.push(url);\n if (this.numPending <= MAX_CONCURRENT) {\n while (this.pendingQueue.length > 0) {\n const url = this.pendingQueue.shift();\n this.numPending++;\n let success = await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_2__.doExternalFetch)(url);\n if (!success) {\n await this.doFetchNonCors(url);\n }\n this.numPending--;\n this.numDone++;\n }\n if (!this.numPending) {\n this._markDone(null);\n }\n }\n }\n initObserver() {\n if (this.mutationObserver) {\n return;\n }\n this.mutationObserver = new MutationObserver((changes) => this.observeChange(changes));\n this.mutationObserver.observe(document.documentElement, {\n characterData: false,\n characterDataOldValue: false,\n attributes: true,\n attributeOldValue: true,\n subtree: true,\n childList: true,\n attributeFilter: ["srcset", "loading"],\n });\n }\n processChangedNode(target) {\n switch (target.nodeType) {\n case Node.ATTRIBUTE_NODE:\n if (target.nodeName === "srcset") {\n this.extractSrcSetAttr(target.nodeValue);\n }\n if (target.nodeName === "loading" && target.nodeValue === "lazy") {\n const elem = target.parentNode;\n if (elem.tagName === "IMG") {\n elem.setAttribute("loading", "eager");\n }\n }\n break;\n case Node.TEXT_NODE:\n if (target.parentNode && target.parentNode.tagName === "STYLE") {\n this.extractStyleText(target.nodeValue);\n }\n break;\n case Node.ELEMENT_NODE:\n if (target.sheet) {\n this.extractStyleSheet(target.sheet);\n }\n this.extractSrcSrcSet(target);\n setTimeout(() => this.extractSrcSrcSetAll(target), 1000);\n setTimeout(() => this.extractDataAttributes(target), 1000);\n break;\n }\n }\n observeChange(changes) {\n for (const change of changes) {\n this.processChangedNode(change.target);\n if (change.type === "childList") {\n for (const node of change.addedNodes) {\n this.processChangedNode(node);\n }\n }\n }\n }\n extractSrcSrcSetAll(root) {\n const elems = (0,query_selector_shadow_dom__WEBPACK_IMPORTED_MODULE_0__.querySelectorAllDeep)(SRC_SET_SELECTOR, root);\n for (const elem of elems) {\n this.extractSrcSrcSet(elem);\n }\n }\n extractSrcSrcSet(elem) {\n if (!elem || elem.nodeType !== Node.ELEMENT_NODE) {\n console.warn("No elem to extract from");\n return;\n }\n const data_src = elem.getAttribute("data-src");\n if (data_src) {\n this.queueUrl(data_src);\n }\n if (elem.getAttribute("loading") === "lazy") {\n elem.setAttribute("loading", "eager");\n }\n const srcset = elem.getAttribute("srcset");\n if (srcset) {\n this.extractSrcSetAttr(srcset);\n }\n const data_srcset = elem.getAttribute("data-srcset");\n if (data_srcset) {\n this.extractSrcSetAttr(data_srcset);\n }\n const src = elem.getAttribute("src");\n if (src &&\n (srcset || data_srcset || elem.parentElement.tagName === "NOSCRIPT")) {\n this.queueUrl(src);\n }\n }\n extractSrcSetAttr(srcset) {\n for (const v of srcset.split(SRCSET_REGEX)) {\n if (v) {\n const parts = v.trim().split(" ");\n this.queueUrl(parts[0]);\n }\n }\n }\n extractStyleSheets(root) {\n root = root || document;\n for (const sheet of root.styleSheets) {\n this.extractStyleSheet(sheet);\n }\n }\n extractStyleSheet(sheet) {\n let rules;\n try {\n rules = sheet.cssRules || sheet.rules;\n }\n catch (e) {\n this.debug("Can\'t access stylesheet");\n return;\n }\n for (const rule of rules) {\n if (rule.type === CSSRule.MEDIA_RULE) {\n this.extractStyleText(rule.cssText);\n }\n }\n }\n extractStyleText(text) {\n const urlExtractor = (m, n1, n2, n3) => {\n this.queueUrl(n2);\n return n1 + n2 + n3;\n };\n text.replace(STYLE_REGEX, urlExtractor).replace(IMPORT_REGEX, urlExtractor);\n }\n extractDataAttributes(root) {\n const QUERY = "//@*[starts-with(name(), \'data-\') and " +\n "(starts-with(., \'http\') or starts-with(., \'/\') or starts-with(., \'./\') or starts-with(., \'../\'))]";\n for (const attr of (0,_lib_utils__WEBPACK_IMPORTED_MODULE_2__.xpathNodes)(QUERY, root)) {\n this.queueUrl(attr.value);\n }\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/autofetcher.ts?')},"./src/autoplay.ts": +/*!*************************!*\ + !*** ./src/autoplay.ts ***! + \*************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "Autoplay": () => (/* binding */ Autoplay)\n/* harmony export */ });\n/* harmony import */ var query_selector_shadow_dom__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! query-selector-shadow-dom */ "./node_modules/query-selector-shadow-dom/src/querySelectorDeep.js");\n/* harmony import */ var _lib_behavior__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./lib/behavior */ "./src/lib/behavior.ts");\n/* harmony import */ var _lib_utils__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./lib/utils */ "./src/lib/utils.ts");\n\n\n\nclass Autoplay extends _lib_behavior__WEBPACK_IMPORTED_MODULE_1__.BackgroundBehavior {\n mediaSet;\n autofetcher;\n numPlaying;\n promises;\n _initDone;\n running = false;\n polling = false;\n static id = "Autoplay";\n constructor(autofetcher, startEarly = false) {\n super();\n this.mediaSet = new Set();\n this.autofetcher = autofetcher;\n this.numPlaying = 0;\n this.promises = [];\n this._initDone = () => null;\n this.promises.push(new Promise((resolve) => (this._initDone = resolve)));\n if (startEarly) {\n document.addEventListener("DOMContentLoaded", () => this.pollAudioVideo());\n }\n }\n async start() {\n this.running = true;\n this.pollAudioVideo();\n this._initDone();\n }\n async pollAudioVideo() {\n const run = true;\n if (this.polling) {\n return;\n }\n this.polling = true;\n while (run) {\n for (const [, elem] of (0,query_selector_shadow_dom__WEBPACK_IMPORTED_MODULE_0__.querySelectorAllDeep)("video, audio, picture").entries()) {\n if (!elem["__bx_autoplay_found"]) {\n if (!this.running) {\n if (this.processFetchableUrl(elem)) {\n elem["__bx_autoplay_found"] = true;\n }\n continue;\n }\n await this.loadMedia(elem);\n elem["__bx_autoplay_found"] = true;\n }\n }\n await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_2__.sleep)(500);\n }\n this.polling = false;\n }\n fetchSrcUrl(source) {\n const url = source.src || source.currentSrc;\n if (!url) {\n return false;\n }\n if (!url.startsWith("http:") && !url.startsWith("https:")) {\n return false;\n }\n if (this.mediaSet.has(url)) {\n return true;\n }\n this.debug("fetch media source URL: " + url);\n this.mediaSet.add(url);\n this.autofetcher.queueUrl(url);\n return true;\n }\n processFetchableUrl(media) {\n let found = this.fetchSrcUrl(media);\n const sources = media.querySelectorAll("source");\n for (const source of sources) {\n const foundSource = this.fetchSrcUrl(source);\n found = found || foundSource;\n }\n return found;\n }\n async loadMedia(media) {\n this.debug("processing media element: " + media.outerHTML);\n const found = this.processFetchableUrl(media);\n if (!media.play) {\n this.debug("media not playable, skipping");\n return;\n }\n if (found) {\n if (!media.paused) {\n media.pause();\n this.debug("media URL found, pausing playback");\n }\n return;\n }\n if (media.paused || media.currentTime) {\n if (media.paused) {\n this.debug("no src url found, attempting to click or play: " + media.outerHTML);\n }\n else {\n this.debug("media already playing, waiting for full playback to finish: " +\n media.outerHTML);\n }\n this.attemptMediaPlay(media).then(async (finished) => {\n let check = true;\n if (finished) {\n finished.then(() => (check = false));\n }\n while (check) {\n if (this.processFetchableUrl(media)) {\n check = false;\n }\n this.debug("Waiting for fixed URL or media to finish: " + media.currentSrc);\n await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_2__.sleep)(1000);\n }\n });\n }\n else if (media.currentSrc) {\n this.debug("media playing from non-URL source: " + media.currentSrc);\n }\n }\n async attemptMediaPlay(media) {\n let resolveFinished;\n const finished = new Promise((res) => {\n resolveFinished = res;\n });\n let resolveStarted;\n const started = new Promise((res) => {\n resolveStarted = res;\n });\n started.then(() => this.promises.push(finished));\n if (!media.paused && media.currentTime > 0) {\n resolveStarted();\n }\n media.addEventListener("loadstart", () => {\n this.debug("media event: loadstart");\n resolveStarted(true);\n });\n media.addEventListener("playing", () => {\n this.debug("media event: playing");\n resolveStarted(true);\n });\n media.addEventListener("loadeddata", () => this.debug("media event: loadeddata"));\n media.addEventListener("ended", () => {\n this.debug("media event: ended");\n resolveFinished();\n });\n media.addEventListener("pause", () => {\n this.debug("media event: pause");\n resolveFinished();\n });\n media.addEventListener("abort", () => {\n this.debug("media event: abort");\n resolveFinished();\n });\n media.addEventListener("error", () => {\n this.debug("media event: error");\n resolveFinished();\n });\n media.addEventListener("stalled", () => {\n this.debug("media event: stalled");\n resolveFinished();\n });\n media.addEventListener("suspend", () => {\n this.debug("media event: suspend");\n resolveFinished();\n });\n media.muted = true;\n if (!media.paused && media.currentTime > 0) {\n return finished;\n }\n const hasA = media.closest("a");\n if (!hasA) {\n media.click();\n if (await Promise.race([started, (0,_lib_utils__WEBPACK_IMPORTED_MODULE_2__.sleep)(1000)])) {\n this.debug("play started after media.click()");\n return finished;\n }\n }\n media.play();\n if (await Promise.race([started, (0,_lib_utils__WEBPACK_IMPORTED_MODULE_2__.sleep)(1000)])) {\n this.debug("play started after media.play()");\n }\n return finished;\n }\n done() {\n return Promise.allSettled(this.promises);\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/autoplay.ts?')},"./src/autoscroll.ts": +/*!***************************!*\ + !*** ./src/autoscroll.ts ***! + \***************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "AutoScroll": () => (/* binding */ AutoScroll)\n/* harmony export */ });\n/* harmony import */ var _lib_behavior__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./lib/behavior */ "./src/lib/behavior.ts");\n/* harmony import */ var _lib_utils__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./lib/utils */ "./src/lib/utils.ts");\n\n\nclass AutoScroll extends _lib_behavior__WEBPACK_IMPORTED_MODULE_0__.Behavior {\n autoFetcher;\n showMoreQuery;\n state = { segments: 1 };\n lastScrollPos;\n samePosCount;\n origPath;\n lastMsg = "";\n constructor(autofetcher) {\n super();\n this.autoFetcher = autofetcher;\n this.showMoreQuery =\n "//*[contains(text(), \'show more\') or contains(text(), \'Show more\')]";\n this.lastScrollPos = -1;\n this.samePosCount = 0;\n this.origPath = document.location.pathname;\n }\n static id = "Autoscroll";\n currScrollPos() {\n return Math.round(self.scrollY + self.innerHeight);\n }\n canScrollMore() {\n const scrollElem = self.document.scrollingElement || self.document.body;\n return (this.currScrollPos() <\n Math.max(scrollElem.clientHeight, scrollElem.scrollHeight));\n }\n debug(msg) {\n if (this.lastMsg === msg) {\n return;\n }\n super.debug(msg);\n this.lastMsg = msg;\n }\n hasScrollEL(obj) {\n try {\n return !!self["getEventListeners"](obj).scroll;\n }\n catch (_) {\n this.debug("getEventListeners() not available");\n return true;\n }\n }\n async shouldScroll() {\n if (!this.hasScrollEL(self.window) &&\n !this.hasScrollEL(self.document) &&\n !this.hasScrollEL(self.document.body)) {\n return false;\n }\n if (window.frames.length >= 2) {\n return true;\n }\n const lastScrollHeight = self.document.scrollingElement.scrollHeight;\n const numFetching = this.autoFetcher.numFetching;\n const scrollEnd = document.scrollingElement.scrollHeight * 0.98 - self.innerHeight;\n window.scrollTo({ top: scrollEnd, left: 0, behavior: "smooth" });\n await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.sleep)(500);\n if (lastScrollHeight !== self.document.scrollingElement.scrollHeight ||\n numFetching < this.autoFetcher.numFetching) {\n window.scrollTo({ top: 0, left: 0, behavior: "auto" });\n return true;\n }\n return false;\n }\n shouldScrollUp() {\n if (self.window.scrollY === 0) {\n return false;\n }\n if ((self.window.scrollY + self["scrollHeight"]) /\n self.document.scrollingElement.scrollHeight <\n 0.9) {\n return false;\n }\n return true;\n }\n async *[Symbol.asyncIterator]() {\n if (this.shouldScrollUp()) {\n yield* this.scrollUp();\n return;\n }\n if (await this.shouldScroll()) {\n yield* this.scrollDown();\n return;\n }\n yield this.getState("Skipping autoscroll, page seems to not be responsive to scrolling events");\n }\n async *scrollDown() {\n const scrollInc = Math.min(self.document.scrollingElement.clientHeight * 0.1, 30);\n const interval = 75;\n let elapsedWait = 0;\n let showMoreElem = null;\n let ignoreShowMoreElem = false;\n const scrollOpts = { top: scrollInc, left: 0, behavior: "auto" };\n let lastScrollHeight = self.document.scrollingElement.scrollHeight;\n while (this.canScrollMore()) {\n if (document.location.pathname !== this.origPath) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.behaviorLog)("Location Changed, stopping scroll: " +\n `${document.location.pathname} != ${this.origPath}`, "info");\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.addLink)(document.location.href);\n return;\n }\n const scrollHeight = self.document.scrollingElement.scrollHeight;\n if (scrollHeight > lastScrollHeight) {\n this.state.segments++;\n lastScrollHeight = scrollHeight;\n }\n if (!showMoreElem && !ignoreShowMoreElem) {\n showMoreElem = (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.xpathNode)(this.showMoreQuery);\n }\n if (showMoreElem && (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.isInViewport)(showMoreElem)) {\n yield this.getState("Clicking \'Show More\', awaiting more content");\n showMoreElem["click"]();\n await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.sleep)(_lib_utils__WEBPACK_IMPORTED_MODULE_1__.waitUnit);\n await Promise.race([\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.waitUntil)(() => self.document.scrollingElement.scrollHeight > scrollHeight, 500),\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.sleep)(30000),\n ]);\n if (self.document.scrollingElement.scrollHeight === scrollHeight) {\n ignoreShowMoreElem = true;\n }\n showMoreElem = null;\n }\n self.scrollBy(scrollOpts);\n await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.sleep)(interval);\n if (this.state.segments === 1) {\n yield this.getState(`Scrolling down by ${scrollOpts.top} pixels every ${interval / 1000.0} seconds`);\n elapsedWait = 2.0;\n }\n else {\n const waitSecs = elapsedWait / (this.state.segments - 1);\n this.debug(`Waiting up to ${waitSecs} seconds for more scroll segments`);\n const startTime = Date.now();\n await Promise.race([\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.waitUntil)(() => this.canScrollMore(), interval),\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.sleep)(waitSecs),\n ]);\n elapsedWait += (Date.now() - startTime) * 2;\n }\n const currPos = this.currScrollPos();\n if (currPos === this.lastScrollPos) {\n if (++this.samePosCount >= 2) {\n break;\n }\n }\n else {\n this.samePosCount = 0;\n }\n this.lastScrollPos = currPos;\n }\n }\n async *scrollUp() {\n const scrollInc = Math.min(self.document.scrollingElement.clientHeight * 0.1, 30);\n const interval = 75;\n const scrollOpts = { top: -scrollInc, left: 0, behavior: "auto" };\n let lastScrollHeight = self.document.scrollingElement.scrollHeight;\n while (self.scrollY > 0) {\n const scrollHeight = self.document.scrollingElement.scrollHeight;\n if (scrollHeight > lastScrollHeight) {\n this.state.segments++;\n lastScrollHeight = scrollHeight;\n }\n self.scrollBy(scrollOpts);\n await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.sleep)(interval);\n if (this.state.segments === 1) {\n yield this.getState(`Scrolling up by ${scrollOpts.top} pixels every ${interval / 1000.0} seconds`);\n }\n else {\n await Promise.race([\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.waitUntil)(() => self.scrollY > 0, interval),\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_1__.sleep)((this.state.segments - 1) * 2000),\n ]);\n }\n }\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/autoscroll.ts?')},"./src/index.ts": +/*!**********************!*\ + !*** ./src/index.ts ***! + \**********************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "BehaviorManager": () => (/* binding */ BehaviorManager)\n/* harmony export */ });\n/* harmony import */ var _autofetcher__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./autofetcher */ "./src/autofetcher.ts");\n/* harmony import */ var _autoplay__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./autoplay */ "./src/autoplay.ts");\n/* harmony import */ var _autoscroll__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./autoscroll */ "./src/autoscroll.ts");\n/* harmony import */ var _autoclick__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! ./autoclick */ "./src/autoclick.ts");\n/* harmony import */ var _lib_utils__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(/*! ./lib/utils */ "./src/lib/utils.ts");\n/* harmony import */ var _lib_behavior__WEBPACK_IMPORTED_MODULE_5__ = __webpack_require__(/*! ./lib/behavior */ "./src/lib/behavior.ts");\n/* harmony import */ var _site__WEBPACK_IMPORTED_MODULE_6__ = __webpack_require__(/*! ./site */ "./src/site/index.ts");\n\n\n\n\n\n\n\n\nconst DEFAULT_OPTS = {\n autofetch: true,\n autoplay: true,\n autoscroll: true,\n autoclick: true,\n siteSpecific: true,\n};\nconst DEFAULT_CLICK_SELECTOR = "a";\nconst DEFAULT_LINK_SELECTOR = "a[href]";\nconst DEFAULT_LINK_EXTRACT = "href";\nclass BehaviorManager {\n autofetch;\n behaviors;\n loadedBehaviors;\n mainBehavior;\n mainBehaviorClass;\n inited;\n started;\n timeout;\n opts;\n linkOpts;\n constructor() {\n this.behaviors = [];\n this.loadedBehaviors = _site__WEBPACK_IMPORTED_MODULE_6__["default"].reduce((behaviors, next) => {\n behaviors[next.id] = next;\n return behaviors;\n }, {});\n this.mainBehavior = null;\n this.inited = false;\n this.started = false;\n this.linkOpts = {\n selector: DEFAULT_LINK_SELECTOR,\n extractName: DEFAULT_LINK_EXTRACT,\n };\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Loaded behaviors for: " + self.location.href);\n }\n init(opts = DEFAULT_OPTS, restart = false, customBehaviors = null) {\n if (this.inited && !restart) {\n return;\n }\n this.inited = true;\n this.opts = opts;\n if (!self.window) {\n return;\n }\n this.timeout = opts.timeout;\n if (opts.log !== undefined) {\n let logger = opts.log;\n if (typeof logger === "string") {\n logger = self[logger];\n }\n if (typeof logger === "function") {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__._setLogFunc)(logger);\n }\n else if (logger === false) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__._setLogFunc)(null);\n }\n }\n this.autofetch = new _autofetcher__WEBPACK_IMPORTED_MODULE_0__.AutoFetcher(!!opts.autofetch, opts.fetchHeaders, opts.startEarly);\n if (opts.autofetch) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Using AutoFetcher");\n this.behaviors.push(this.autofetch);\n }\n if (opts.autoplay) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Using Autoplay");\n this.behaviors.push(new _autoplay__WEBPACK_IMPORTED_MODULE_1__.Autoplay(this.autofetch, opts.startEarly));\n }\n if (opts.autoclick) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Using AutoClick");\n this.behaviors.push(new _autoclick__WEBPACK_IMPORTED_MODULE_3__.AutoClick(opts.clickSelector || DEFAULT_CLICK_SELECTOR));\n }\n if (customBehaviors) {\n for (const behaviorClass of customBehaviors) {\n try {\n this.load(behaviorClass);\n }\n catch (e) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)(`Failed to load custom behavior: ${e} ${behaviorClass}`);\n }\n }\n }\n }\n selectMainBehavior() {\n if (this.mainBehavior) {\n return;\n }\n const opts = this.opts;\n let siteMatch = false;\n if (opts.siteSpecific) {\n for (const name in this.loadedBehaviors) {\n const siteBehaviorClass = this.loadedBehaviors[name];\n if (siteBehaviorClass.isMatch()) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Using Site-Specific Behavior: " + name);\n this.mainBehaviorClass = siteBehaviorClass;\n const siteSpecificOpts = typeof opts.siteSpecific === "object"\n ? opts.siteSpecific[name] || {}\n : {};\n try {\n this.mainBehavior = new _lib_behavior__WEBPACK_IMPORTED_MODULE_5__.BehaviorRunner(siteBehaviorClass, siteSpecificOpts);\n }\n catch (e) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)({ msg: e.toString(), siteSpecific: true }, "error");\n }\n siteMatch = true;\n break;\n }\n }\n }\n if (!siteMatch && opts.autoscroll) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Using Autoscroll");\n this.mainBehaviorClass = _autoscroll__WEBPACK_IMPORTED_MODULE_2__.AutoScroll;\n this.mainBehavior = new _autoscroll__WEBPACK_IMPORTED_MODULE_2__.AutoScroll(this.autofetch);\n }\n if (this.mainBehavior) {\n this.behaviors.push(this.mainBehavior);\n if (this.mainBehavior instanceof _lib_behavior__WEBPACK_IMPORTED_MODULE_5__.BehaviorRunner) {\n return this.mainBehavior.behaviorProps.id;\n }\n }\n return "";\n }\n load(behaviorClass) {\n if (typeof behaviorClass.id !== "string") {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)(\'Behavior class must have a string string "id" property\', "error");\n return;\n }\n const name = behaviorClass.id;\n if (typeof behaviorClass !== "function") {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)(`Must pass a class object, got ${behaviorClass}`, "error");\n return;\n }\n if (typeof behaviorClass.isMatch !== "function" ||\n typeof behaviorClass.init !== "function") {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Behavior class must have an is `isMatch()` and `init()` static methods", "error");\n return;\n }\n if (!this.isInTopFrame()) {\n if (!behaviorClass.runInIframe) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)(`Behavior class ${name}: not running in iframes (.runInIframe not set)`, "debug");\n return;\n }\n }\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)(`Behavior class ${name}: loaded`, "debug");\n this.loadedBehaviors[name] = behaviorClass;\n }\n async resolve(target) {\n const imported = await __webpack_require__("./src lazy recursive ^.*$")(`${target}`);\n if (Array.isArray(imported)) {\n for (const behavior of imported) {\n this.load(behavior);\n }\n }\n else {\n this.load(imported);\n }\n }\n async awaitPageLoad() {\n this.selectMainBehavior();\n if (this.mainBehavior?.awaitPageLoad) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Waiting for custom page load via behavior");\n await this.mainBehavior.awaitPageLoad({ Lib: _lib_utils__WEBPACK_IMPORTED_MODULE_4__ });\n }\n else {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("No custom wait behavior");\n }\n }\n async run(opts = DEFAULT_OPTS, restart = false) {\n if (restart) {\n this.started = false;\n }\n if (this.started) {\n this.unpause();\n return;\n }\n this.init(opts, restart);\n this.selectMainBehavior();\n await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.awaitLoad)();\n this.behaviors.forEach((x) => {\n const id = x.id || x.constructor.id || "(Unnamed)";\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Starting behavior: " + id, "debug");\n x.start();\n });\n this.started = true;\n await (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.sleep)(500);\n const allBehaviors = Promise.allSettled(this.behaviors.map((x) => x.done()));\n if (this.timeout) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)(`Waiting for behaviors to finish or ${this.timeout}ms timeout`, "debug");\n await Promise.race([allBehaviors, (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.sleep)(this.timeout)]);\n }\n else {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Waiting for behaviors to finish", "debug");\n await allBehaviors;\n }\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("All Behaviors Done for " + self.location.href, "debug");\n if (this.mainBehavior && this.mainBehaviorClass.cleanup) {\n this.mainBehavior.cleanup();\n }\n }\n async runOne(name, behaviorOpts = {}) {\n const siteBehaviorClass = _site__WEBPACK_IMPORTED_MODULE_6__["default"].find((b) => b.name === name);\n if (typeof siteBehaviorClass === "undefined") {\n console.error(`No behavior of name ${name} found`);\n return;\n }\n const behavior = new _lib_behavior__WEBPACK_IMPORTED_MODULE_5__.BehaviorRunner(siteBehaviorClass, behaviorOpts);\n behavior.start();\n console.log(`Running behavior: ${name}`);\n await behavior.done();\n console.log(`Behavior ${name} completed`);\n }\n pause() {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Pausing Main Behavior" + this.mainBehaviorClass.name);\n this.behaviors.forEach((x) => x.pause());\n }\n unpause() {\n this.behaviors.forEach((x) => x.unpause());\n }\n doAsyncFetch(url) {\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.behaviorLog)("Queueing Async Fetch Url: " + url);\n return this.autofetch.queueUrl(url, true);\n }\n isInTopFrame() {\n return (self.window.top === self.window ||\n window["__WB_replay_top"] === self.window);\n }\n async extractLinks(selector = DEFAULT_LINK_SELECTOR, extractName = "href", attrOnly = false) {\n this.linkOpts = { selector, extractName, attrOnly };\n (0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.checkToJsonOverride)();\n return await this.extractLinksActual();\n }\n async extractLinksActual() {\n const { selector = DEFAULT_LINK_SELECTOR, extractName = DEFAULT_LINK_EXTRACT, attrOnly = false, } = this.linkOpts;\n const urls = new Set();\n document.querySelectorAll(selector).forEach((elem) => {\n let value = !attrOnly ? elem[extractName] : null;\n if (!value) {\n value = elem.getAttribute(extractName);\n }\n if (typeof value === "string") {\n urls.add(value);\n }\n });\n const promises = [];\n for (const url of urls) {\n promises.push((0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.addLink)(url));\n }\n await Promise.allSettled(promises);\n }\n}\n(0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__._setBehaviorManager)(BehaviorManager);\n(0,_lib_utils__WEBPACK_IMPORTED_MODULE_4__.installBehaviors)(self);\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/index.ts?')},"./src/lib/behavior.ts": +/*!*****************************!*\ + !*** ./src/lib/behavior.ts ***! + \*****************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "BackgroundBehavior": () => (/* binding */ BackgroundBehavior),\n/* harmony export */ "Behavior": () => (/* binding */ Behavior),\n/* harmony export */ "BehaviorRunner": () => (/* binding */ BehaviorRunner)\n/* harmony export */ });\n/* harmony import */ var _utils__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./utils */ "./src/lib/utils.ts");\n\n\nclass BackgroundBehavior {\n debug(msg) {\n (0,_utils__WEBPACK_IMPORTED_MODULE_0__.behaviorLog)(msg, "debug");\n }\n error(msg) {\n (0,_utils__WEBPACK_IMPORTED_MODULE_0__.behaviorLog)(msg, "error");\n }\n log(msg, type = "info") {\n (0,_utils__WEBPACK_IMPORTED_MODULE_0__.behaviorLog)(msg, type);\n }\n}\nclass Behavior extends BackgroundBehavior {\n _running;\n paused;\n _unpause;\n state;\n scrollOpts;\n constructor() {\n super();\n this._running = null;\n this.paused = null;\n this._unpause = null;\n this.state = {};\n this.scrollOpts = { behavior: "smooth", block: "center", inline: "center" };\n }\n start() {\n this._running = this.run();\n }\n done() {\n return this._running ? this._running : Promise.resolve();\n }\n async run() {\n try {\n for await (const step of this) {\n this.debug(step);\n if (this.paused) {\n await this.paused;\n }\n }\n this.debug(this.getState("done!"));\n }\n catch (e) {\n this.error(e.toString());\n }\n }\n pause() {\n if (this.paused) {\n return;\n }\n this.paused = new Promise((resolve) => {\n this._unpause = resolve;\n });\n }\n unpause() {\n if (this._unpause) {\n this._unpause();\n this.paused = null;\n this._unpause = null;\n }\n }\n getState(msg, incrValue) {\n if (incrValue) {\n if (this.state[incrValue] === undefined) {\n this.state[incrValue] = 1;\n }\n else {\n this.state[incrValue]++;\n }\n }\n return { state: this.state, msg };\n }\n cleanup() { }\n async awaitPageLoad(_) {\n }\n static load() {\n if (self["__bx_behaviors"]) {\n self["__bx_behaviors"].load(this);\n }\n else {\n console.warn(`Could not load ${this.name} behavior: window.__bx_behaviors is not initialized`);\n }\n }\n async *[Symbol.asyncIterator]() {\n yield;\n }\n}\nclass AbstractBehaviorInst {\n}\nclass BehaviorRunner extends BackgroundBehavior {\n inst;\n behaviorProps;\n ctx;\n _running;\n paused;\n _unpause;\n get id() {\n return (this.inst?.constructor).id;\n }\n constructor(behavior, mainOpts = {}) {\n super();\n this.behaviorProps = behavior;\n this.inst = new behavior();\n if (typeof this.inst.run !== "function" ||\n this.inst.run.constructor.name !== "AsyncGeneratorFunction") {\n throw Error("Invalid behavior: missing `async run*` instance method");\n }\n let { state, opts } = behavior.init();\n state = state || {};\n opts = opts ? { ...opts, ...mainOpts } : mainOpts;\n const log = async (data, type) => this.wrappedLog(data, type);\n this.ctx = { Lib: _utils__WEBPACK_IMPORTED_MODULE_0__, state, opts, log };\n this._running = null;\n this.paused = null;\n this._unpause = null;\n }\n wrappedLog(data, type = "info") {\n let logData;\n if (typeof data === "string" || data instanceof String) {\n logData = { msg: data };\n }\n else {\n logData = data;\n }\n this.log({ ...logData, behavior: this.behaviorProps.id, siteSpecific: true }, type);\n }\n start() {\n this._running = this.run();\n }\n done() {\n return this._running ? this._running : Promise.resolve();\n }\n async run() {\n try {\n for await (const step of this.inst.run(this.ctx)) {\n if (step) {\n this.wrappedLog(step);\n }\n if (this.paused) {\n await this.paused;\n }\n }\n this.debug({ msg: "done!", behavior: this.behaviorProps.id });\n }\n catch (e) {\n this.error({ msg: e.toString(), behavior: this.behaviorProps.id });\n }\n }\n pause() {\n if (this.paused) {\n return;\n }\n this.paused = new Promise((resolve) => {\n this._unpause = resolve;\n });\n }\n unpause() {\n if (this._unpause) {\n this._unpause();\n this.paused = null;\n this._unpause = null;\n }\n }\n cleanup() { }\n async awaitPageLoad() {\n if (this.inst.awaitPageLoad) {\n await this.inst.awaitPageLoad(this.ctx);\n }\n }\n static load() {\n if (self["__bx_behaviors"]) {\n self["__bx_behaviors"].load(this);\n }\n else {\n console.warn(`Could not load ${this.name} behavior: window.__bx_behaviors is not initialized`);\n }\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/lib/behavior.ts?')},"./src/lib/utils.ts": +/*!**************************!*\ + !*** ./src/lib/utils.ts ***! + \**************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "HistoryState": () => (/* binding */ HistoryState),\n/* harmony export */ "RestoreState": () => (/* binding */ RestoreState),\n/* harmony export */ "_setBehaviorManager": () => (/* binding */ _setBehaviorManager),\n/* harmony export */ "_setLogFunc": () => (/* binding */ _setLogFunc),\n/* harmony export */ "addLink": () => (/* binding */ addLink),\n/* harmony export */ "addToExternalSet": () => (/* binding */ addToExternalSet),\n/* harmony export */ "assertContentValid": () => (/* binding */ assertContentValid),\n/* harmony export */ "awaitLoad": () => (/* binding */ awaitLoad),\n/* harmony export */ "behaviorLog": () => (/* binding */ behaviorLog),\n/* harmony export */ "callBinding": () => (/* binding */ callBinding),\n/* harmony export */ "checkToJsonOverride": () => (/* binding */ checkToJsonOverride),\n/* harmony export */ "doExternalFetch": () => (/* binding */ doExternalFetch),\n/* harmony export */ "getState": () => (/* binding */ getState),\n/* harmony export */ "initFlow": () => (/* binding */ initFlow),\n/* harmony export */ "installBehaviors": () => (/* binding */ installBehaviors),\n/* harmony export */ "isInViewport": () => (/* binding */ isInViewport),\n/* harmony export */ "iterChildElem": () => (/* binding */ iterChildElem),\n/* harmony export */ "iterChildMatches": () => (/* binding */ iterChildMatches),\n/* harmony export */ "nextFlowStep": () => (/* binding */ nextFlowStep),\n/* harmony export */ "openWindow": () => (/* binding */ openWindow),\n/* harmony export */ "scrollAndClick": () => (/* binding */ scrollAndClick),\n/* harmony export */ "scrollIntoView": () => (/* binding */ scrollIntoView),\n/* harmony export */ "scrollToOffset": () => (/* binding */ scrollToOffset),\n/* harmony export */ "sleep": () => (/* binding */ sleep),\n/* harmony export */ "waitForNetworkIdle": () => (/* binding */ waitForNetworkIdle),\n/* harmony export */ "waitUnit": () => (/* binding */ waitUnit),\n/* harmony export */ "waitUntil": () => (/* binding */ waitUntil),\n/* harmony export */ "waitUntilNode": () => (/* binding */ waitUntilNode),\n/* harmony export */ "xpathNode": () => (/* binding */ xpathNode),\n/* harmony export */ "xpathNodes": () => (/* binding */ xpathNodes),\n/* harmony export */ "xpathString": () => (/* binding */ xpathString)\n/* harmony export */ });\nlet _logFunc = console.log;\nlet _behaviorMgrClass = null;\nconst scrollOpts = { behavior: "smooth", block: "center", inline: "center" };\nasync function scrollAndClick(node, interval = 500, opts = scrollOpts) {\n node.scrollIntoView(opts);\n await sleep(interval);\n node.click();\n}\nconst waitUnit = 200;\nasync function sleep(timeout) {\n return new Promise((resolve) => setTimeout(resolve, timeout));\n}\nasync function waitUntil(pred, interval = waitUnit) {\n while (!pred()) {\n await sleep(interval);\n }\n}\nasync function waitUntilNode(path, root = document, old = null, timeout = 1000, interval = waitUnit) {\n let node = null;\n let stop = false;\n const waitP = waitUntil(() => {\n node = xpathNode(path, root);\n return stop || (node !== old && node !== null);\n }, interval);\n const timeoutP = new Promise((r) => setTimeout(() => {\n stop = true;\n r("TIMEOUT");\n }, timeout));\n await Promise.race([waitP, timeoutP]);\n return node;\n}\nasync function awaitLoad(iframe) {\n const doc = iframe ? iframe.contentDocument : document;\n const win = iframe ? iframe.contentWindow : window;\n return new Promise((resolve) => {\n if (doc.readyState === "complete") {\n resolve(null);\n }\n else {\n win.addEventListener("load", resolve);\n }\n });\n}\nfunction unsetToJson(obj) {\n if (obj.toJSON) {\n try {\n obj.__bx__toJSON = obj.toJSON;\n delete obj.toJSON;\n }\n catch (_) {\n }\n }\n}\nfunction restoreToJson(obj) {\n if (obj.__bx__toJSON) {\n try {\n obj.toJSON = obj.__bx__toJSON;\n delete obj.__bx__toJSON;\n }\n catch (_) {\n }\n }\n}\nfunction unsetAllJson() {\n unsetToJson(Object);\n unsetToJson(Object.prototype);\n unsetToJson(Array);\n unsetToJson(Array.prototype);\n}\nfunction restoreAllJson() {\n restoreToJson(Object);\n restoreToJson(Object.prototype);\n restoreToJson(Array);\n restoreToJson(Array.prototype);\n}\nlet needUnsetToJson = false;\nfunction checkToJsonOverride() {\n needUnsetToJson =\n !!Object.toJSON ||\n !!Object.prototype.toJSON ||\n !!Array.toJSON ||\n !!Array.prototype.toJSON;\n}\nasync function callBinding(binding, obj) {\n try {\n if (needUnsetToJson) {\n unsetAllJson();\n }\n return binding(obj);\n }\n catch (_) {\n return binding(JSON.stringify(obj));\n }\n finally {\n if (needUnsetToJson) {\n restoreAllJson();\n }\n }\n}\nasync function behaviorLog(data, type = "debug") {\n if (_logFunc) {\n await callBinding(_logFunc, { data, type });\n }\n}\nasync function addLink(url) {\n if (typeof self["__bx_addLink"] === "function") {\n return await callBinding(self["__bx_addLink"], url);\n }\n}\nasync function doExternalFetch(url) {\n if (typeof self["__bx_fetch"] === "function") {\n return await callBinding(self["__bx_fetch"], url);\n }\n return false;\n}\nasync function addToExternalSet(url) {\n if (typeof self["__bx_addSet"] === "function") {\n return await callBinding(self["__bx_addSet"], url);\n }\n return true;\n}\nasync function waitForNetworkIdle(idleTime = 500, concurrency = 0) {\n if (typeof self["__bx_netIdle"] === "function") {\n return await callBinding(self["__bx_netIdle"], { idleTime, concurrency });\n }\n}\nasync function initFlow(params) {\n if (typeof self["__bx_initFlow"] === "function") {\n return await callBinding(self["__bx_initFlow"], params);\n }\n return -1;\n}\nasync function nextFlowStep(id) {\n if (typeof self["__bx_nextFlowStep"] === "function") {\n return await callBinding(self["__bx_nextFlowStep"], id);\n }\n return { done: true, msg: "" };\n}\nfunction assertContentValid(assertFunc, reason = "invalid") {\n if (typeof self["__bx_contentCheckFailed"] === "function") {\n if (!assertFunc()) {\n behaviorLog("Behavior content check failed: " + reason, "error");\n callBinding(self["__bx_contentCheckFailed"], reason);\n }\n }\n}\nasync function openWindow(url) {\n if (self["__bx_open"]) {\n const p = new Promise((resolve) => (self["__bx_openResolve"] = resolve));\n await callBinding(self["__bx_open"], { url });\n let win = null;\n try {\n win = await p;\n if (win) {\n return win;\n }\n }\n catch (e) {\n console.warn(e);\n }\n finally {\n delete self["__bx_openResolve"];\n }\n }\n return window.open(url);\n}\nfunction _setLogFunc(func) {\n _logFunc = func;\n}\nfunction _setBehaviorManager(cls) {\n _behaviorMgrClass = cls;\n}\nfunction installBehaviors(obj) {\n obj.__bx_behaviors = new _behaviorMgrClass();\n}\nclass RestoreState {\n matchValue;\n constructor(childMatchSelect, child) {\n this.matchValue = xpathString(childMatchSelect, child);\n }\n async restore(rootPath, childMatch) {\n let root = null;\n while (((root = xpathNode(rootPath)), !root)) {\n await sleep(100);\n }\n return xpathNode(childMatch.replace("$1", this.matchValue), root);\n }\n}\nclass HistoryState {\n loc;\n constructor(op) {\n this.loc = window.location.href;\n op();\n }\n get changed() {\n return window.location.href !== this.loc;\n }\n async goBack(backButtonQuery) {\n if (!this.changed) {\n return Promise.resolve(true);\n }\n const backButton = xpathNode(backButtonQuery);\n return new Promise((resolve) => {\n window.addEventListener("popstate", () => {\n resolve(null);\n }, { once: true });\n if (backButton) {\n backButton["click"]();\n }\n else {\n window.history.back();\n }\n });\n }\n}\nfunction xpathNode(path, root) {\n root = root || document;\n return document.evaluate(path, root, null, XPathResult.FIRST_ORDERED_NODE_TYPE).singleNodeValue;\n}\nfunction* xpathNodes(path, root) {\n root = root || document;\n const iter = document.evaluate(path, root, null, XPathResult.ORDERED_NODE_ITERATOR_TYPE);\n let result = null;\n while ((result = iter.iterateNext()) !== null) {\n yield result;\n }\n}\nfunction xpathString(path, root) {\n root = root || document;\n return document.evaluate(path, root, null, XPathResult.STRING_TYPE)\n .stringValue;\n}\nasync function* iterChildElem(root, timeout, totalTimeout) {\n let child = root.firstElementChild;\n while (child) {\n yield child;\n if (!child.nextElementSibling) {\n await Promise.race([\n waitUntil(() => !!child.nextElementSibling, timeout),\n sleep(totalTimeout),\n ]);\n }\n child = child.nextElementSibling;\n }\n}\nasync function* iterChildMatches(path, root, interval = waitUnit, timeout = 5000) {\n let node = xpathNode(`.//${path}`, root);\n const getMatch = (node) => xpathNode(`./following-sibling::${path}`, node);\n while (node) {\n yield node;\n let next = getMatch(node);\n if (next) {\n node = next;\n continue;\n }\n await Promise.race([\n waitUntil(() => {\n next = getMatch(node);\n return next;\n }, interval),\n sleep(timeout),\n ]);\n node = next;\n }\n}\nfunction isInViewport(elem) {\n const bounding = elem.getBoundingClientRect();\n return (bounding.top >= 0 &&\n bounding.left >= 0 &&\n bounding.bottom <=\n (window.innerHeight || document.documentElement.clientHeight) &&\n bounding.right <=\n (window.innerWidth || document.documentElement.clientWidth));\n}\nfunction scrollToOffset(element, offset = 0) {\n const elPosition = element.getBoundingClientRect().top;\n const topPosition = elPosition + window.pageYOffset - offset;\n window.scrollTo({ top: topPosition, behavior: "smooth" });\n}\nfunction scrollIntoView(element, opts = {\n behavior: "smooth",\n block: "center",\n inline: "center",\n}) {\n element.scrollIntoView(opts);\n}\nfunction getState(ctx, msg, incrValue) {\n if (typeof ctx.state === "undefined") {\n ctx.state = {};\n }\n if (incrValue) {\n if (ctx.state[incrValue] === undefined) {\n ctx.state[incrValue] = 1;\n }\n else {\n ctx.state[incrValue]++;\n }\n }\n return { state: ctx.state, msg };\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/lib/utils.ts?')},"./src/site/facebook.ts": +/*!******************************!*\ + !*** ./src/site/facebook.ts ***! + \******************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "FacebookTimelineBehavior": () => (/* binding */ FacebookTimelineBehavior)\n/* harmony export */ });\nconst Q = {\n feed: "//div[@role=\'feed\']",\n article: ".//div[@role=\'article\']",\n pageletPostList: "//div[@data-pagelet=\'page\']/div[@role=\'main\']//div[@role=\'main\']/div",\n pageletProfilePostList: "//div[@data-pagelet=\'page\']//div[@data-pagelet=\'ProfileTimeline\']",\n articleToPostList: "//div[@role=\'article\']/../../../../div",\n photosOrVideos: `.//a[(contains(@href, \'/photos/\') or contains(@href, \'/photo/?\') or contains(@href, \'/videos/\')) and (starts-with(@href, \'${window.location.origin}/\') or starts-with(@href, \'/\'))]`,\n postQuery: ".//a[contains(@href, \'/posts/\')]",\n extraLabel: "//*[starts-with(text(), \'+\')]",\n nextSlideQuery: "//div[@data-name=\'media-viewer-nav-container\']/div[@data-visualcompletion][2]//div[@role=\'button\']",\n nextSlide: "//div[@aria-hidden=\'false\']//div[@role=\'button\' and not(@aria-hidden) and @aria-label]",\n commentList: ".//ul[(../h3) or (../h4)]",\n commentMoreReplies: "./div[2]/div[1]/div[2]/div[@role=\'button\']",\n commentMoreComments: "./following-sibling::div/div/div[2][@role=\'button\'][./span/span]",\n viewComments: ".//h4/..//div[@role=\'button\']",\n photoCommentList: "//ul[../h2]",\n firstPhotoThumbnail: "//div[@role=\'main\']//div[3]//div[contains(@style, \'border-radius\')]//div[contains(@style, \'max-width\') and contains(@style, \'min-width\')]//a[@role=\'link\']",\n firstVideoThumbnail: "//div[@role=\'main\']//div[contains(@style, \'z-index\')]/following-sibling::div/div/div/div[last()]//a[contains(@href, \'/videos/\') and @aria-hidden!=\'true\']",\n firstVideoSimple: "//div[@role=\'main\']//a[contains(@href, \'/videos/\') and @aria-hidden!=\'true\']",\n mainVideo: "//div[@data-pagelet=\'root\']//div[@role=\'dialog\']//div[@role=\'main\']//video",\n nextVideo: "following::a[contains(@href, \'/videos/\') and @aria-hidden!=\'true\']",\n isPhotoVideoPage: /^.*facebook\\.com\\/[^/]+\\/(photos|videos)\\/.+/,\n isPhotosPage: /^.*facebook\\.com\\/[^/]+\\/photos\\/?($|\\?)/,\n isVideosPage: /^.*facebook\\.com\\/[^/]+\\/videos\\/?($|\\?)/,\n pageLoadWaitUntil: "//div[@role=\'main\']",\n};\nclass FacebookTimelineBehavior {\n extraWindow;\n allowNewWindow;\n static id = "Facebook";\n static isMatch() {\n return !!window.location.href.match(/https:\\/\\/(www\\.)?facebook\\.com\\/.*\\/posts\\//);\n }\n static init() {\n return {\n state: {},\n };\n }\n constructor() {\n this.extraWindow = null;\n this.allowNewWindow = false;\n }\n async *iterPostFeeds(ctx) {\n const { iterChildElem, waitUnit, waitUntil, xpathNode, xpathNodes } = ctx.Lib;\n const feeds = Array.from(xpathNodes(Q.feed));\n if (feeds && feeds.length) {\n for (const feed of feeds) {\n for await (const post of iterChildElem(feed, waitUnit, waitUntil * 10)) {\n yield* this.viewPost(ctx, xpathNode(Q.article, post));\n }\n }\n }\n else {\n const feed = xpathNode(Q.pageletPostList) ||\n xpathNode(Q.pageletProfilePostList) ||\n xpathNode(Q.articleToPostList);\n if (!feed) {\n return;\n }\n for await (const post of iterChildElem(feed, waitUnit, waitUntil * 10)) {\n yield* this.viewPost(ctx, xpathNode(Q.article, post));\n }\n }\n if (this.extraWindow) {\n this.extraWindow.close();\n }\n }\n async *viewPost(ctx, post, maxExpands = 2) {\n const { getState, scrollIntoView, sleep, waitUnit, xpathNode } = ctx.Lib;\n if (!post) {\n return;\n }\n const postLink = xpathNode(Q.postQuery, post);\n let url = null;\n if (postLink) {\n url = new URL(postLink.href, window.location.href);\n url.search = "";\n }\n yield getState(ctx, "Viewing post " + (url || ""), "posts");\n scrollIntoView(post);\n await sleep(waitUnit * 2);\n if (xpathNode(".//video", post)) {\n yield getState(ctx, "Playing inline video", "videos");\n await sleep(waitUnit * 2);\n }\n let commentRootUL = xpathNode(Q.commentList, post);\n if (!commentRootUL) {\n const viewCommentsButton = xpathNode(Q.viewComments, post);\n if (viewCommentsButton) {\n viewCommentsButton.click();\n await sleep(waitUnit * 2);\n }\n commentRootUL = xpathNode(Q.commentList, post);\n }\n yield* this.iterComments(ctx, commentRootUL, maxExpands);\n await sleep(waitUnit * 5);\n }\n async *viewPhotosOrVideos(ctx, post) {\n const { getState, sleep, waitUnit, xpathNode, xpathNodes } = ctx.Lib;\n const objects = Array.from(xpathNodes(Q.photosOrVideos, post));\n const objHrefs = new Set();\n let count = 0;\n for (const obj of objects) {\n const url = new URL(obj.href, window.location.href);\n if (obj.href.indexOf("?fbid") === -1) {\n url.search = "";\n }\n if (objHrefs.has(url.href)) {\n continue;\n }\n const type = obj.href.indexOf("/video") >= 0 ? "videos" : "photos";\n ++count;\n objHrefs.add(url.href);\n yield getState(ctx, `Viewing ${type} ${url.href}`, type);\n obj.scrollIntoView();\n await sleep(waitUnit * 5);\n obj.click();\n await sleep(waitUnit * 10);\n if (this.allowNewWindow) {\n await this.openNewWindow(ctx, url.href);\n }\n if (count === objects.length) {\n yield* this.viewExtraObjects(ctx, obj, type, this.allowNewWindow);\n }\n const close = xpathNode(Q.nextSlide);\n if (close) {\n close.click();\n await sleep(waitUnit * 2);\n }\n }\n }\n async *viewExtraObjects(ctx, obj, type, openNew) {\n const { getState, sleep, waitUnit, waitUntil, xpathNode } = ctx.Lib;\n const extraLabel = xpathNode(Q.extraLabel, obj);\n if (!extraLabel) {\n return;\n }\n const num = Number(extraLabel.innerText.slice(1));\n if (isNaN(num)) {\n return;\n }\n let lastHref;\n for (let i = 0; i < num; i++) {\n const nextSlideButton = xpathNode(Q.nextSlideQuery);\n if (!nextSlideButton) {\n continue;\n }\n lastHref = window.location.href;\n nextSlideButton.click();\n await sleep(waitUnit * 5);\n await waitUntil(() => window.location.href !== lastHref, waitUnit * 2);\n yield getState(ctx, `Viewing extra ${type} ${window.location.href}`);\n if (openNew) {\n await this.openNewWindow(ctx, window.location.href);\n }\n }\n }\n async openNewWindow(ctx, url) {\n if (!this.extraWindow) {\n this.extraWindow = await ctx.Lib.openWindow(url);\n }\n else {\n this.extraWindow.location.href = url;\n }\n }\n async *iterComments(ctx, commentRootUL, maxExpands = 2) {\n const { getState, scrollIntoView, sleep, waitUnit, xpathNode } = ctx.Lib;\n if (!commentRootUL) {\n await sleep(waitUnit * 5);\n return;\n }\n let commentBlock = commentRootUL.firstElementChild;\n let lastBlock = null;\n let count = 0;\n while (commentBlock && count < maxExpands) {\n while (commentBlock && count < maxExpands) {\n yield getState(ctx, "Loading comments", "comments");\n scrollIntoView(commentBlock);\n await sleep(waitUnit * 2);\n const moreReplies = xpathNode(Q.commentMoreReplies, commentBlock);\n if (moreReplies) {\n moreReplies.click();\n await sleep(waitUnit * 5);\n }\n lastBlock = commentBlock;\n commentBlock = lastBlock.nextElementSibling;\n count++;\n }\n if (count === maxExpands) {\n break;\n }\n const moreButton = xpathNode(Q.commentMoreComments, commentRootUL);\n if (moreButton) {\n scrollIntoView(moreButton);\n moreButton.click();\n await sleep(waitUnit * 5);\n if (lastBlock) {\n commentBlock = lastBlock.nextElementSibling;\n await sleep(waitUnit * 5);\n }\n }\n }\n await sleep(waitUnit * 2);\n }\n async *iterPhotoSlideShow(ctx) {\n const { getState, scrollIntoView, sleep, waitUnit, waitUntil, xpathNode } = ctx.Lib;\n const firstPhoto = xpathNode(Q.firstPhotoThumbnail);\n if (!firstPhoto) {\n return;\n }\n let lastHref = window.location.href;\n scrollIntoView(firstPhoto);\n firstPhoto.click();\n await sleep(waitUnit * 5);\n await waitUntil(() => window.location.href !== lastHref, waitUnit * 2);\n let nextSlideButton = null;\n while ((nextSlideButton = xpathNode(Q.nextSlideQuery))) {\n lastHref = window.location.href;\n await sleep(waitUnit);\n nextSlideButton.click();\n await sleep(waitUnit * 5);\n await Promise.race([\n waitUntil(() => window.location.href !== lastHref, waitUnit * 2),\n sleep(3000),\n ]);\n if (window.location.href === lastHref) {\n break;\n }\n yield getState(ctx, `Viewing photo ${window.location.href}`, "photos");\n const root = xpathNode(Q.photoCommentList);\n yield* this.iterComments(ctx, root, 2);\n await sleep(waitUnit * 5);\n }\n }\n async *iterAllVideos(ctx) {\n const { getState, scrollIntoView, sleep, waitUnit, waitUntil, xpathNode, xpathNodes, } = ctx.Lib;\n const firstInlineVideo = xpathNode("//video");\n if (firstInlineVideo) {\n scrollIntoView(firstInlineVideo);\n await sleep(waitUnit * 5);\n }\n let videoLink = xpathNode(Q.firstVideoThumbnail) || xpathNode(Q.firstVideoSimple);\n if (!videoLink) {\n return;\n }\n while (videoLink) {\n scrollIntoView(videoLink);\n let lastHref = window.location.href;\n videoLink.click();\n await waitUntil(() => window.location.href !== lastHref, waitUnit * 2);\n yield getState(ctx, "Viewing video: " + window.location.href, "videos");\n await sleep(waitUnit * 10);\n await Promise.race([\n waitUntil(() => {\n for (const video of xpathNodes("//video")) {\n if (video.readyState >= 3) {\n return true;\n }\n }\n return false;\n }, waitUnit * 2),\n sleep(20000),\n ]);\n await sleep(waitUnit * 10);\n const close = xpathNode(Q.nextSlide);\n if (!close) {\n break;\n }\n lastHref = window.location.href;\n close.click();\n await waitUntil(() => window.location.href !== lastHref, waitUnit * 2);\n videoLink = xpathNode(Q.nextVideo, videoLink);\n }\n }\n async *run(ctx) {\n const { getState, sleep, xpathNode } = ctx.Lib;\n yield getState(ctx, "Starting...");\n await sleep(2000);\n if (Q.isPhotosPage.exec(window.location.href)) {\n ctx.state = { photos: 0, comments: 0 };\n yield* this.iterPhotoSlideShow(ctx);\n return;\n }\n if (Q.isVideosPage.exec(window.location.href)) {\n ctx.state = { videos: 0, comments: 0 };\n yield* this.iterAllVideos(ctx);\n return;\n }\n if (Q.isPhotoVideoPage.exec(window.location.href)) {\n ctx.state = { comments: 0 };\n const root = xpathNode(Q.photoCommentList);\n yield* this.iterComments(ctx, root, 1000);\n return;\n }\n ctx.state = { posts: 0, comments: 0, videos: 0 };\n yield* this.iterPostFeeds(ctx);\n }\n async awaitPageLoad(ctx) {\n const { Lib, log } = ctx;\n const { assertContentValid, waitUntilNode } = Lib;\n log("Waiting for Facebook to fully load", "info");\n await waitUntilNode(Q.pageLoadWaitUntil, document, null, 10000);\n assertContentValid(() => !!document.querySelector("div[aria-label*=\'Account Controls\' i]"), "not_logged_in");\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/site/facebook.ts?')},"./src/site/index.ts": +/*!***************************!*\ + !*** ./src/site/index.ts ***! + \***************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "default": () => (__WEBPACK_DEFAULT_EXPORT__)\n/* harmony export */ });\n/* harmony import */ var _facebook__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ./facebook */ "./src/site/facebook.ts");\n/* harmony import */ var _instagram__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! ./instagram */ "./src/site/instagram.ts");\n/* harmony import */ var _telegram__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! ./telegram */ "./src/site/telegram.ts");\n/* harmony import */ var _twitter__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! ./twitter */ "./src/site/twitter.ts");\n/* harmony import */ var _tiktok__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(/*! ./tiktok */ "./src/site/tiktok.ts");\n\n\n\n\n\nconst siteBehaviors = [\n _instagram__WEBPACK_IMPORTED_MODULE_1__.InstagramPostsBehavior,\n _twitter__WEBPACK_IMPORTED_MODULE_3__.TwitterTimelineBehavior,\n _facebook__WEBPACK_IMPORTED_MODULE_0__.FacebookTimelineBehavior,\n _telegram__WEBPACK_IMPORTED_MODULE_2__.TelegramBehavior,\n _tiktok__WEBPACK_IMPORTED_MODULE_4__.TikTokVideoBehavior,\n _tiktok__WEBPACK_IMPORTED_MODULE_4__.TikTokProfileBehavior,\n];\n/* harmony default export */ const __WEBPACK_DEFAULT_EXPORT__ = (siteBehaviors);\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/site/index.ts?')},"./src/site/instagram.ts": +/*!*******************************!*\ + !*** ./src/site/instagram.ts ***! + \*******************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "InstagramPostsBehavior": () => (/* binding */ InstagramPostsBehavior)\n/* harmony export */ });\nconst subpostNextOnlyChevron = "//article[@role=\'presentation\']//div[@role=\'presentation\']/following-sibling::button";\nconst Q = {\n rootPath: "//main//div/div[2]/div/div/div/div",\n childMatchSelect: "string(.//a[starts-with(@href, \'/\')]/@href)",\n childMatch: "child::div[.//a[@href=\'$1\']]",\n firstPostInRow: "div[1]//a",\n postCloseButton: "//div[last() - 2]//div[@role=\'button\']",\n nextPost: "//button[.//*[local-name() = \'svg\' and @aria-label=\'Next\']]",\n postLoading: "//*[@aria-label=\'Loading...\']",\n subpostNextOnlyChevron,\n subpostPrevNextChevron: subpostNextOnlyChevron + "[2]",\n commentRoot: "//article[@role=\'presentation\']/div[1]/div[2]//ul/div[last()]/div/div",\n viewReplies: "ul/li//button[span[not(count(*)) and contains(text(), \'(\')]]",\n loadMore: "//button[span[@aria-label]]",\n pageLoadWaitUntil: "//main",\n};\nclass InstagramPostsBehavior {\n maxCommentsTime;\n postOnlyWindow;\n static id = "Instagram";\n static isMatch() {\n return !!window.location.href.match(/https:\\/\\/(www\\.)?instagram\\.com\\//);\n }\n static init() {\n return {\n state: {\n posts: 0,\n slides: 0,\n rows: 0,\n comments: 0,\n },\n };\n }\n constructor() {\n this.maxCommentsTime = 10000;\n this.postOnlyWindow = null;\n }\n cleanup() {\n if (this.postOnlyWindow) {\n this.postOnlyWindow.close();\n this.postOnlyWindow = null;\n }\n }\n async waitForNext(ctx, child) {\n if (!child) {\n return null;\n }\n await ctx.Lib.sleep(ctx.Lib.waitUnit);\n if (!child.nextElementSibling) {\n return null;\n }\n return child.nextElementSibling;\n }\n async *iterRow(ctx) {\n const { RestoreState, sleep, waitUnit, xpathNode } = ctx.Lib;\n const root = xpathNode(Q.rootPath);\n if (!root) {\n return;\n }\n let child = root.firstElementChild;\n if (!child) {\n return;\n }\n while (child) {\n await sleep(waitUnit);\n const restorer = new RestoreState(Q.childMatchSelect, child);\n if (restorer.matchValue) {\n yield child;\n child = await restorer.restore(Q.rootPath, Q.childMatch);\n }\n child = await this.waitForNext(ctx, child);\n }\n }\n async *viewStandalonePost(ctx, origLoc) {\n const { getState, sleep, waitUnit, waitUntil, xpathNode, xpathString } = ctx.Lib;\n const root = xpathNode(Q.rootPath);\n if (!root?.firstElementChild) {\n return;\n }\n const firstPostHref = xpathString(Q.childMatchSelect, root.firstElementChild);\n yield getState(ctx, "Loading single post view for first post: " + firstPostHref);\n window.history.replaceState({}, "", firstPostHref);\n window.dispatchEvent(new PopStateEvent("popstate", { state: {} }));\n let root2 = null;\n let root3 = null;\n await sleep(waitUnit * 5);\n await waitUntil(() => (root2 = xpathNode(Q.rootPath)) !== root && root2, waitUnit * 5);\n await sleep(waitUnit * 5);\n window.history.replaceState({}, "", origLoc);\n window.dispatchEvent(new PopStateEvent("popstate", { state: {} }));\n await waitUntil(() => (root3 = xpathNode(Q.rootPath)) !== root2 && root3, waitUnit * 5);\n }\n async *iterSubposts(ctx) {\n const { getState, sleep, waitUnit, xpathNode } = ctx.Lib;\n let next = xpathNode(Q.subpostNextOnlyChevron);\n let count = 1;\n while (next) {\n next.click();\n await sleep(waitUnit * 5);\n yield getState(ctx, `Loading Slide ${++count} for ${window.location.href}`, "slides");\n next = xpathNode(Q.subpostPrevNextChevron);\n }\n await sleep(waitUnit * 5);\n }\n async iterComments(ctx) {\n const { scrollIntoView, sleep, waitUnit, waitUntil, xpathNode } = ctx.Lib;\n const root = xpathNode(Q.commentRoot);\n if (!root) {\n return;\n }\n let child = root.firstElementChild;\n let commentsLoaded = false;\n const getViewRepliesButton = (child) => {\n return xpathNode(Q.viewReplies, child);\n };\n while (child) {\n scrollIntoView(child);\n commentsLoaded = true;\n let viewReplies = getViewRepliesButton(child);\n while (viewReplies) {\n const orig = viewReplies.textContent;\n viewReplies.click();\n ctx.state.comments++;\n await sleep(waitUnit * 2.5);\n await waitUntil(() => orig !== viewReplies.textContent, waitUnit);\n viewReplies = getViewRepliesButton(child);\n }\n if (child.nextElementSibling &&\n child.nextElementSibling.tagName === "LI" &&\n !child.nextElementSibling.nextElementSibling) {\n const loadMore = xpathNode(Q.loadMore, child.nextElementSibling);\n if (loadMore) {\n loadMore.click();\n ctx.state.comments++;\n await sleep(waitUnit * 5);\n }\n }\n child = child.nextElementSibling;\n await sleep(waitUnit * 2.5);\n }\n return commentsLoaded;\n }\n async *iterPosts(ctx, next) {\n const { getState, sleep, waitUnit, xpathNode } = ctx.Lib;\n while (next) {\n next.click();\n await sleep(waitUnit * 10);\n yield getState(ctx, "Loading Post: " + window.location.href, "posts");\n await fetch(window.location.href);\n yield* this.handleSinglePost(ctx);\n next = xpathNode(Q.nextPost);\n while (!next && xpathNode(Q.postLoading)) {\n await sleep(waitUnit * 2.5);\n }\n }\n await sleep(waitUnit * 5);\n }\n async *handleSinglePost(ctx) {\n const { getState, sleep } = ctx.Lib;\n yield* this.iterSubposts(ctx);\n yield getState(ctx, "Loaded Comments", "comments");\n await Promise.race([this.iterComments(ctx), sleep(this.maxCommentsTime)]);\n }\n async *run(ctx) {\n if (window.location.pathname.startsWith("/p/")) {\n yield* this.handleSinglePost(ctx);\n return;\n }\n const { getState, scrollIntoView, sleep, waitUnit, xpathNode } = ctx.Lib;\n for await (const row of this.iterRow(ctx)) {\n scrollIntoView(row);\n await sleep(waitUnit * 2.5);\n yield getState(ctx, "Loading Row", "rows");\n const first = xpathNode(Q.firstPostInRow, row);\n yield* this.iterPosts(ctx, first);\n const close = xpathNode(Q.postCloseButton);\n if (close) {\n close.click();\n }\n await sleep(waitUnit * 5);\n }\n }\n async awaitPageLoad(ctx) {\n const { Lib, log } = ctx;\n const { assertContentValid, waitUntilNode } = Lib;\n log("Waiting for Instagram to fully load");\n await waitUntilNode(Q.pageLoadWaitUntil, document, null, 10000);\n assertContentValid(() => !!document.querySelector("*[aria-label=\'New post\']"), "not_logged_in");\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/site/instagram.ts?')},"./src/site/telegram.ts": +/*!******************************!*\ + !*** ./src/site/telegram.ts ***! + \******************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "TelegramBehavior": () => (/* binding */ TelegramBehavior)\n/* harmony export */ });\nconst Q = {\n telegramContainer: "//main//section[@class=\'tgme_channel_history js-message_history\']",\n postId: "string(./div[@data-post]/@data-post)",\n linkExternal: "string(.//a[@class=\'tgme_widget_message_link_preview\' and @href]/@href)",\n};\nclass TelegramBehavior {\n static id = "Telegram";\n static isMatch() {\n return !!window.location.href.match(/https:\\/\\/t.me\\/s\\/\\w[\\w]+/);\n }\n static init() {\n return {\n state: { messages: 0 },\n };\n }\n async waitForPrev(ctx, child) {\n if (!child) {\n return null;\n }\n await ctx.Lib.sleep(ctx.Lib.waitUnit * 5);\n if (!child.previousElementSibling) {\n return null;\n }\n return child.previousElementSibling;\n }\n async *run(ctx) {\n const { getState, scrollIntoView, sleep, waitUnit, xpathNode, xpathString, } = ctx.Lib;\n const root = xpathNode(Q.telegramContainer);\n if (!root) {\n return;\n }\n let child = root.lastElementChild;\n while (child) {\n scrollIntoView(child);\n const postId = xpathString(Q.postId, child) || "unknown";\n const linkUrl = xpathString(Q.linkExternal, child);\n if (linkUrl?.endsWith(".jpg") || linkUrl.endsWith(".png")) {\n yield getState(ctx, "Loading External Image: " + linkUrl);\n const image = new Image();\n image.src = linkUrl;\n document.body.appendChild(image);\n await sleep(waitUnit * 2.5);\n document.body.removeChild(image);\n }\n yield getState(ctx, "Loading Message: " + postId, "messages");\n child = await this.waitForPrev(ctx, child);\n }\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/site/telegram.ts?')},"./src/site/tiktok.ts": +/*!****************************!*\ + !*** ./src/site/tiktok.ts ***! + \****************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "BREADTH_ALL": () => (/* binding */ BREADTH_ALL),\n/* harmony export */ "TikTokProfileBehavior": () => (/* binding */ TikTokProfileBehavior),\n/* harmony export */ "TikTokSharedBehavior": () => (/* binding */ TikTokSharedBehavior),\n/* harmony export */ "TikTokVideoBehavior": () => (/* binding */ TikTokVideoBehavior)\n/* harmony export */ });\nconst Q = {\n commentList: "//div[contains(@class, \'CommentListContainer\')]",\n commentItem: "div[contains(@class, \'CommentItemContainer\')]",\n viewMoreReplies: ".//p[contains(@class, \'ReplyActionText\')]",\n viewMoreThread: ".//p[starts-with(@data-e2e, \'view-more\') and string-length(text()) > 0]",\n profileVideoList: "//div[starts-with(@data-e2e, \'user-post-item-list\')]",\n profileVideoItem: "div[contains(@class, \'DivItemContainerV2\')]",\n backButton: "button[contains(@class, \'StyledCloseIconContainer\')]",\n pageLoadWaitUntil: "//*[@role=\'dialog\']",\n};\nconst BREADTH_ALL = Symbol("BREADTH_ALL");\nclass TikTokSharedBehavior {\n async awaitPageLoad(ctx) {\n const { assertContentValid, waitUntilNode } = ctx.Lib;\n await waitUntilNode(Q.pageLoadWaitUntil, document, null, 10000);\n assertContentValid(() => !!document.querySelector("*[aria-label=\'Messages\']"), "not_logged_in");\n }\n}\nclass TikTokVideoBehavior extends TikTokSharedBehavior {\n static id = "TikTokVideo";\n static init() {\n return {\n state: { comments: 0 },\n opts: { breadth: BREADTH_ALL },\n };\n }\n static isMatch() {\n const pathRegex = /https:\\/\\/(www\\.)?tiktok\\.com\\/@.+\\/video\\/\\d+\\/?.*/;\n return !!window.location.href.match(pathRegex);\n }\n breadthComplete({ opts: { breadth } }, iter) {\n return breadth !== BREADTH_ALL && breadth <= iter;\n }\n async *crawlThread(ctx, parentNode, prev = null, iter = 0) {\n const { waitUntilNode, scrollAndClick, getState } = ctx.Lib;\n const next = await waitUntilNode(Q.viewMoreThread, parentNode, prev);\n if (!next || this.breadthComplete(ctx, iter))\n return;\n await scrollAndClick(next, 500);\n yield getState(ctx, "View more replies", "comments");\n yield* this.crawlThread(ctx, parentNode, next, iter + 1);\n }\n async *expandThread(ctx, item) {\n const { xpathNode, scrollAndClick, getState } = ctx.Lib;\n const viewMore = xpathNode(Q.viewMoreReplies, item);\n if (!viewMore)\n return;\n await scrollAndClick(viewMore, 500);\n yield getState(ctx, "View comment", "comments");\n yield* this.crawlThread(ctx, item, null, 1);\n }\n async *run(ctx) {\n const { xpathNode, iterChildMatches, scrollIntoView, getState, assertContentValid, } = ctx.Lib;\n setInterval(() => {\n assertContentValid(() => !document.querySelector("div[class*=captcha]"), "captcha_found");\n }, 500);\n const commentList = xpathNode(Q.commentList);\n const commentItems = iterChildMatches(Q.commentItem, commentList);\n for await (const item of commentItems) {\n scrollIntoView(item);\n yield getState(ctx, "View comment", "comments");\n if (this.breadthComplete(ctx, 0))\n continue;\n yield* this.expandThread(ctx, item);\n }\n yield getState(ctx, "TikTok Video Behavior Complete");\n }\n}\nclass TikTokProfileBehavior extends TikTokSharedBehavior {\n static id = "TikTokProfile";\n static isMatch() {\n const pathRegex = /https:\\/\\/(www\\.)?tiktok\\.com\\/@[a-zA-Z0-9]+(\\/?$|\\/\\?.*)/;\n return !!window.location.href.match(pathRegex);\n }\n static init() {\n return {\n state: { videos: 0, comments: 0 },\n opts: { breadth: BREADTH_ALL },\n };\n }\n async *openVideo(ctx, item) {\n const { HistoryState, xpathNode, sleep } = ctx.Lib;\n const link = xpathNode(".//a", item);\n if (!link)\n return;\n const viewState = new HistoryState(() => link.click());\n await sleep(500);\n if (viewState.changed) {\n const videoBehavior = new TikTokVideoBehavior();\n yield* videoBehavior.run(ctx);\n await sleep(500);\n await viewState.goBack(Q.backButton);\n }\n }\n async *run(ctx) {\n const { xpathNode, iterChildMatches, scrollIntoView, getState, sleep } = ctx.Lib;\n const profileVideoList = xpathNode(Q.profileVideoList);\n const profileVideos = iterChildMatches(Q.profileVideoItem, profileVideoList);\n for await (const item of profileVideos) {\n scrollIntoView(item);\n yield getState(ctx, "View video", "videos");\n yield* this.openVideo(ctx, item);\n await sleep(500);\n }\n yield getState(ctx, "TikTok Profile Behavior Complete");\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/site/tiktok.ts?')},"./src/site/twitter.ts": +/*!*****************************!*\ + !*** ./src/site/twitter.ts ***! + \*****************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "TwitterTimelineBehavior": () => (/* binding */ TwitterTimelineBehavior)\n/* harmony export */ });\nconst Q = {\n rootPath: "//h1[@role=\'heading\' and @aria-level=\'1\']/following-sibling::div[@aria-label]//div[@style]",\n anchor: ".//article",\n childMatchSelect: "string(.//article//a[starts-with(@href, \'/\') and @aria-label]/@href)",\n childMatch: "child::div[.//a[@href=\'$1\']]",\n expand: ".//div[@role=\'button\' and not(@aria-haspopup) and not(@data-testid)]",\n quote: ".//div[@role=\'blockquote\' and @aria-haspopup=\'false\']",\n image: ".//a[@role=\'link\' and starts-with(@href, \'/\') and contains(@href, \'/photo/\')]",\n imageFirstNext: "//div[@aria-roledescription=\'carousel\']/div[2]/div[1]//div[@role=\'button\']",\n imageNext: "//div[@aria-roledescription=\'carousel\']/div[2]/div[2]//div[@role=\'button\']",\n imageClose: "//div[@role=\'presentation\']/div[@role=\'button\' and @aria-label]",\n backButton: "//div[@data-testid=\'titleContainer\']//div[@role=\'button\']",\n viewSensitive: ".//a[@href=\'/settings/content_you_see\']/parent::div/parent::div/parent::div//div[@role=\'button\']",\n progress: ".//*[@role=\'progressbar\']",\n promoted: ".//div[data-testid=\'placementTracking\']",\n};\nclass TwitterTimelineBehavior {\n seenTweets;\n seenMediaTweets;\n static id = "Twitter";\n static isMatch() {\n return !!window.location.href.match(/https:\\/\\/(www\\.)?(x|twitter)\\.com\\//);\n }\n static init() {\n return {\n state: {\n tweets: 0,\n images: 0,\n videos: 0,\n },\n opts: {\n maxDepth: 0,\n },\n };\n }\n constructor() {\n this.seenTweets = new Set();\n this.seenMediaTweets = new Set();\n }\n showingProgressBar(ctx, root) {\n const { xpathNode } = ctx.Lib;\n const node = xpathNode(Q.progress, root);\n if (!node) {\n return false;\n }\n return node.clientHeight > 10;\n }\n async waitForNext(ctx, child) {\n const { sleep, waitUnit } = ctx.Lib;\n if (!child) {\n return null;\n }\n await sleep(waitUnit * 2);\n if (!child.nextElementSibling) {\n return null;\n }\n while (this.showingProgressBar(ctx, child.nextElementSibling)) {\n await sleep(waitUnit);\n }\n return child.nextElementSibling;\n }\n async expandMore(ctx, child) {\n const { sleep, waitUnit, xpathNode } = ctx.Lib;\n const expandElem = xpathNode(Q.expand, child);\n if (!expandElem) {\n return child;\n }\n const prev = child.previousElementSibling;\n expandElem.click();\n await sleep(waitUnit);\n while (this.showingProgressBar(ctx, prev.nextElementSibling)) {\n await sleep(waitUnit);\n }\n child = prev.nextElementSibling;\n return child;\n }\n async *infScroll(ctx) {\n const { scrollIntoView, RestoreState, sleep, waitUnit, xpathNode } = ctx.Lib;\n const root = xpathNode(Q.rootPath);\n if (!root) {\n return;\n }\n let child = root.firstElementChild;\n if (!child) {\n return;\n }\n while (child) {\n let anchorElem = xpathNode(Q.anchor, child);\n if (!anchorElem && Q.expand) {\n child = await this.expandMore(ctx, child);\n anchorElem = xpathNode(Q.anchor, child);\n }\n if (child?.innerText) {\n scrollIntoView(child);\n }\n if (child && anchorElem) {\n await sleep(waitUnit);\n const restorer = new RestoreState(Q.childMatchSelect, child);\n yield anchorElem;\n if (restorer.matchValue) {\n child = await restorer.restore(Q.rootPath, Q.childMatch);\n }\n }\n child = await this.waitForNext(ctx, child);\n }\n }\n async *mediaPlaying(ctx, tweet) {\n const { getState, sleep, xpathNode, xpathString } = ctx.Lib;\n const media = xpathNode("(.//video | .//audio)", tweet);\n if (!media || media.paused) {\n return;\n }\n let mediaTweetUrl = null;\n try {\n mediaTweetUrl = new URL(xpathString(Q.childMatchSelect, tweet.parentElement), window.location.origin).href;\n }\n catch (e) {\n console.warn(e);\n }\n if (media.src.startsWith("https://") && media.src.indexOf(".mp4") > 0) {\n yield getState(ctx, `Loading video for ${mediaTweetUrl || "unknown"}`, "videos");\n return;\n }\n let msg;\n if (mediaTweetUrl) {\n if (this.seenMediaTweets.has(mediaTweetUrl)) {\n return;\n }\n msg = `Waiting for media playback for ${mediaTweetUrl} to finish`;\n this.seenMediaTweets.add(mediaTweetUrl);\n }\n else {\n msg = "Loading video";\n }\n yield getState(ctx, msg, "videos");\n const p = new Promise((resolve) => {\n media.addEventListener("ended", () => resolve(null));\n media.addEventListener("abort", () => resolve(null));\n media.addEventListener("error", () => resolve(null));\n media.addEventListener("pause", () => resolve(null));\n });\n await Promise.race([p, sleep(60000)]);\n }\n async *clickImages(ctx, tweet) {\n const { getState, HistoryState, sleep, waitUnit, xpathNode } = ctx.Lib;\n const imagePopup = xpathNode(Q.image, tweet);\n if (imagePopup) {\n const imageState = new HistoryState(() => imagePopup.click());\n yield getState(ctx, "Loading Image: " + window.location.href, "images");\n await sleep(waitUnit * 5);\n let nextImage = xpathNode(Q.imageFirstNext);\n let prevLocation = window.location.href;\n while (nextImage) {\n nextImage.click();\n await sleep(waitUnit * 2);\n if (window.location.href === prevLocation) {\n await sleep(waitUnit * 5);\n break;\n }\n prevLocation = window.location.href;\n yield getState(ctx, "Loading Image: " + window.location.href, "images");\n await sleep(waitUnit * 5);\n nextImage = xpathNode(Q.imageNext);\n }\n await imageState.goBack(Q.imageClose);\n }\n }\n async *clickTweet(ctx, tweet, depth) {\n const { getState, HistoryState, sleep, waitUnit } = ctx.Lib;\n const tweetState = new HistoryState(() => tweet.click());\n await sleep(waitUnit);\n if (tweetState.changed) {\n yield getState(ctx, "Capturing Tweet: " + window.location.href, "tweets");\n const maxDepth = ctx.opts.maxDepth;\n if (depth < maxDepth && !this.seenTweets.has(window.location.href)) {\n yield* this.iterTimeline(ctx, depth + 1);\n }\n this.seenTweets.add(window.location.href);\n await sleep(waitUnit * 2);\n await tweetState.goBack(Q.backButton);\n await sleep(waitUnit);\n }\n }\n async *iterTimeline(ctx, depth = 0) {\n const { getState, sleep, waitUnit, xpathNode } = ctx.Lib;\n if (this.seenTweets.has(window.location.href)) {\n return;\n }\n yield getState(ctx, "Capturing thread: " + window.location.href, "threads");\n for await (const tweet of this.infScroll(ctx)) {\n if (xpathNode(Q.promoted, tweet)) {\n continue;\n }\n await sleep(waitUnit * 2.5);\n const viewButton = xpathNode(Q.viewSensitive, tweet);\n if (viewButton) {\n viewButton.click();\n await sleep(waitUnit * 2.5);\n }\n yield* this.clickImages(ctx, tweet);\n const quoteTweet = xpathNode(Q.quote, tweet);\n if (quoteTweet) {\n yield* this.clickTweet(ctx, quoteTweet, 1000);\n }\n yield* this.mediaPlaying(ctx, tweet);\n yield* this.clickTweet(ctx, tweet, depth);\n await sleep(waitUnit * 5);\n }\n }\n async *run(ctx) {\n yield* this.iterTimeline(ctx, 0);\n }\n async awaitPageLoad(ctx) {\n const { sleep, assertContentValid } = ctx.Lib;\n await sleep(5);\n assertContentValid(() => !document.documentElement.outerHTML.match(/Log In/i), "not_logged_in");\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/site/twitter.ts?')},"./src/site/youtube.ts": +/*!*****************************!*\ + !*** ./src/site/youtube.ts ***! + \*****************************/(__unused_webpack_module,__webpack_exports__,__webpack_require__)=>{"use strict";eval('__webpack_require__.r(__webpack_exports__);\n/* harmony export */ __webpack_require__.d(__webpack_exports__, {\n/* harmony export */ "YoutubeBehavior": () => (/* binding */ YoutubeBehavior)\n/* harmony export */ });\n/* harmony import */ var _autoscroll__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! ../autoscroll */ "./src/autoscroll.ts");\n\nclass YoutubeBehavior extends _autoscroll__WEBPACK_IMPORTED_MODULE_0__.AutoScroll {\n async awaitPageLoad(ctx) {\n const { sleep, assertContentValid } = ctx.Lib;\n await sleep(10);\n assertContentValid(() => {\n const video = document.querySelector("video");\n const paused = video && video.paused;\n if (paused) {\n return false;\n }\n return document.documentElement.outerHTML.indexOf("not a bot") === -1;\n }, "no_video_playing");\n }\n}\n\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/site/youtube.ts?')},"./src lazy recursive ^.*$": +/*!*****************************************!*\ + !*** ./src/ lazy ^.*$ namespace object ***! + \*****************************************/(module,__unused_webpack_exports,__webpack_require__)=>{eval('var map = {\n\t".": [\n\t\t"./src/index.ts"\n\t],\n\t"./": [\n\t\t"./src/index.ts"\n\t],\n\t"./autoclick": [\n\t\t"./src/autoclick.ts"\n\t],\n\t"./autoclick.ts": [\n\t\t"./src/autoclick.ts"\n\t],\n\t"./autofetcher": [\n\t\t"./src/autofetcher.ts"\n\t],\n\t"./autofetcher.ts": [\n\t\t"./src/autofetcher.ts"\n\t],\n\t"./autoplay": [\n\t\t"./src/autoplay.ts"\n\t],\n\t"./autoplay.ts": [\n\t\t"./src/autoplay.ts"\n\t],\n\t"./autoscroll": [\n\t\t"./src/autoscroll.ts"\n\t],\n\t"./autoscroll.ts": [\n\t\t"./src/autoscroll.ts"\n\t],\n\t"./index": [\n\t\t"./src/index.ts"\n\t],\n\t"./index.ts": [\n\t\t"./src/index.ts"\n\t],\n\t"./lib/behavior": [\n\t\t"./src/lib/behavior.ts"\n\t],\n\t"./lib/behavior.ts": [\n\t\t"./src/lib/behavior.ts"\n\t],\n\t"./lib/utils": [\n\t\t"./src/lib/utils.ts"\n\t],\n\t"./lib/utils.ts": [\n\t\t"./src/lib/utils.ts"\n\t],\n\t"./site": [\n\t\t"./src/site/index.ts"\n\t],\n\t"./site/": [\n\t\t"./src/site/index.ts"\n\t],\n\t"./site/facebook": [\n\t\t"./src/site/facebook.ts"\n\t],\n\t"./site/facebook.ts": [\n\t\t"./src/site/facebook.ts"\n\t],\n\t"./site/index": [\n\t\t"./src/site/index.ts"\n\t],\n\t"./site/index.ts": [\n\t\t"./src/site/index.ts"\n\t],\n\t"./site/instagram": [\n\t\t"./src/site/instagram.ts"\n\t],\n\t"./site/instagram.ts": [\n\t\t"./src/site/instagram.ts"\n\t],\n\t"./site/telegram": [\n\t\t"./src/site/telegram.ts"\n\t],\n\t"./site/telegram.ts": [\n\t\t"./src/site/telegram.ts"\n\t],\n\t"./site/tiktok": [\n\t\t"./src/site/tiktok.ts"\n\t],\n\t"./site/tiktok.ts": [\n\t\t"./src/site/tiktok.ts"\n\t],\n\t"./site/twitter": [\n\t\t"./src/site/twitter.ts"\n\t],\n\t"./site/twitter.ts": [\n\t\t"./src/site/twitter.ts"\n\t],\n\t"./site/youtube": [\n\t\t"./src/site/youtube.ts",\n\t\t"main"\n\t],\n\t"./site/youtube.ts": [\n\t\t"./src/site/youtube.ts",\n\t\t"main"\n\t]\n};\nfunction webpackAsyncContext(req) {\n\tif(!__webpack_require__.o(map, req)) {\n\t\treturn Promise.resolve().then(() => {\n\t\t\tvar e = new Error("Cannot find module \'" + req + "\'");\n\t\t\te.code = \'MODULE_NOT_FOUND\';\n\t\t\tthrow e;\n\t\t});\n\t}\n\n\tvar ids = map[req], id = ids[0];\n\treturn Promise.all(ids.slice(1).map(__webpack_require__.e)).then(() => {\n\t\treturn __webpack_require__(id);\n\t});\n}\nwebpackAsyncContext.keys = () => (Object.keys(map));\nwebpackAsyncContext.id = "./src lazy recursive ^.*$";\nmodule.exports = webpackAsyncContext;\n\n//# sourceURL=webpack://browsertrix-behaviors/./src/_lazy_^.*$_namespace_object?')}},__webpack_module_cache__={};function __webpack_require__(e){var n=__webpack_module_cache__[e];if(void 0!==n)return n.exports;var t=__webpack_module_cache__[e]={exports:{}};return __webpack_modules__[e](t,t.exports,__webpack_require__),t.exports}__webpack_require__.d=(e,n)=>{for(var t in n)__webpack_require__.o(n,t)&&!__webpack_require__.o(e,t)&&Object.defineProperty(e,t,{enumerable:!0,get:n[t]})},__webpack_require__.e=()=>Promise.resolve(),__webpack_require__.o=(e,n)=>Object.prototype.hasOwnProperty.call(e,n),__webpack_require__.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})};var __webpack_exports__=__webpack_require__("./index.ts")})(); \ No newline at end of file