diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e3bdb701..4cd5fe2f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -2,17 +2,53 @@ name: Publish Docker image on: release: types: [published] + jobs: push_to_registries: - name: Push Docker image to Dockerhub + name: Build x86 and ARM Images and push to Dockerhub runs-on: ubuntu-latest steps: - - name: Check out the repo + - + name: Check out the repo uses: actions/checkout@v2 - - name: Push to Docker Hub - uses: docker/build-push-action@v1 + + - + name: Prepare + id: prep + run: | + DOCKER_IMAGE=webrecorder/browsertrix-crawler + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + TAGS="${DOCKER_IMAGE}:${VERSION}" + echo ::set-output name=tags::${TAGS} + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - repository: webrecorder/browsertrix-crawler - tag_with_ref: true + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ steps.prep.outputs.tags }} + platforms: "linux/amd64,linux/arm64" + - + name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} + diff --git a/CHANGES.md b/CHANGES.md index b15d8b2a..bd0a53f1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,14 @@ ## CHANGES +v0.4.1 +- BlockRules Optimizations: don't intercept requests if no blockRules +- Profile Creation: Support extending existing profile by passing a --profile param to load on startup +- Profile Creation: Set default window size to 1600x900, add --windowSize param for setting custom size +- Behavior Timeouts: Add --behaviorTimeout to specify custom timeout for behaviors, in seconds (defaulting to 90 seconds) +- Load Wait Default: Switch to 'load,networkidle2' to speed-up waiting for initial load +- Multi-platform build: Support building for amd64 and Arm using oldwebtoday/chrome:91 images (check for google-chrome and chromium-browser automatically) +- CI: Build a multi-platform (amd64 and arm64) image on each release + v0.4.0 - YAML based config, specifyable via --config property or via stdin (with '--config stdin') - Support for different scope types ('page', 'prefix', 'host', 'any', 'none') + crawl depth at crawl level diff --git a/Dockerfile b/Dockerfile index edfd7d3e..6d792a00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -ARG BROWSER_VERSION=90 +ARG BROWSER_VERSION=91 ARG BROWSER_IMAGE_BASE=oldwebtoday/chrome ARG BROWSER_BIN=google-chrome -FROM ${BROWSER_IMAGE_BASE}:${BROWSER_VERSION} as chrome +FROM ${BROWSER_IMAGE_BASE}:${BROWSER_VERSION} AS browser FROM ubuntu:bionic @@ -36,8 +36,8 @@ ENV PROXY_HOST=localhost \ BROWSER_VERSION=${BROWSER_VERSION} \ BROWSER_BIN=${BROWSER_BIN} -COPY --from=chrome /tmp/*.deb /deb/ -COPY --from=chrome /app/libpepflashplayer.so /app/libpepflashplayer.so +COPY --from=browser /tmp/*.deb /deb/ +COPY --from=browser /app/libpepflashplayer.so /app/libpepflashplayer.so RUN dpkg -i /deb/*.deb; apt-get update; apt-get install -fqqy && \ rm -rf /var/lib/opts/lists/* diff --git a/README.md b/README.md index 7bf18d0a..8fb00d2d 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Browsertrix Crawler includes a number of additional command-line options, explai --waitUntil Puppeteer page.goto() condition to wait for before continuing, can be multiple separate by ',' - [default: "load,networkidle0"] + [default: "load,networkidle2"] --depth The depth of the crawl for all seeds [number] [default: -1] --limit Limit crawl to this number of pages @@ -138,6 +138,10 @@ Browsertrix Crawler includes a number of additional command-line options, explai --behaviors Which background behaviors to enable on each page [string] [default: "autoplay,autofetch,siteSpecific"] + --behaviorTimeout If >0, timeout (in seconds) for + in-page behavior will run on each + page. If 0, a behavior can run until + finish. [number] [default: 90] --profile Path to tar.gz file which will be extracted and used as the browser profile [string] @@ -152,10 +156,14 @@ Browsertrix Crawler includes a number of additional command-line options, explai -For the `--waitUntil` flag, see [page.goto waitUntil options](https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagegotourl-options). +### Waiting for Page Load + +One of the key nuances of browser-based crawling is determining when a page is finished loading. This can be configured with the `--waitUntil` flag. + +The default is `load,networkidle2`, which waits until page load and <=2 requests remain, but for static sites, `--wait-until domcontentloaded` may be used to speed up the crawl (to avoid waiting for ads to load for example). The `--waitUntil networkidle0` may make sense for sites, where absolutely all requests must be waited until before proceeding. + +See [page.goto waitUntil options](https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagegotourl-options) for more info on the options that can be used with this flag from the Puppeteer docs. -The default is `load`, but for static sites, `--wait-until domcontentloaded` may be used to speed up the crawl (to avoid waiting for ads to load for example), -while `--waitUntil networkidle0` may make sense for dynamic sites. ### YAML Crawl Config @@ -288,6 +296,8 @@ and auto-fetch content that is not loaded by default, and also run custom behavi Behaviors to run can be specified via a comma-separated list passed to the `--behaviors` option. By default, the auto-scroll behavior is not enabled by default, as it may slow down crawling. To enable this behaviors, you can add `--behaviors autoscroll` or to enable all behaviors, add `--behaviors autoscroll,autoplay,autofetch,siteSpecific`. +The site-specific behavior (or autoscroll) will start running after the page is finished its initial load (as defined by the `--waitUntil` settings). The behavior will then run until finished or until the behavior timeout is exceeded. This timeout can be set (in seconds) via the `--behaviorTimeout` flag (90 seconds by default). Setting the timeout to 0 will allow the behavior to run until it is finished. + See [Browsertrix Behaviors](https://github.com/webrecorder/browsertrix-behaviors) for more info on all of the currently available behaviors. @@ -336,6 +346,12 @@ The script will then prompt you for login credentials, attempt to login and crea - To specify headless mode, add the `--headless` flag. Note that for crawls run with `--headless` flag, it is recommended to also create the profile with `--headless` to ensure the profile is compatible. +- To specify the window size for the profile creation embedded browser, specify `--windowSize WIDTH,HEIGHT`. (The default is 1600x900) + + +The current profile creation script is still experimental and the script attempts to detect the username and password fields on a site as generically as possible, but may not work for all sites. Additional profile functionality, such as support for custom profile creation scripts, may be added in the future. + + ### Interactive Profile Creation For creating profiles of more complex sites, or logging in to multiple sites at once, the interactive profile creation mode can be used. @@ -351,14 +367,19 @@ Browsertrix Crawler will then create a profile as before using the current state For example, to start in interactive profile creation mode, run: ``` -docker run -p 9222:9222 -p 9223:9223 -v $PWD/crawls/profiles:/output/ -it webrecorder/browsertrix-crawler:0.4.0-beta.3 create-login-profile --interactive --url "https://example.com/" +docker run -p 9222:9222 -p 9223:9223 -v $PWD/profiles:/output/ -it webrecorder/browsertrix-crawler:[VERSION] create-login-profile --interactive --url "https://example.com/" ``` Then, open a browser pointing to `http://localhost:9223/` and use the embedded browser to log in to any sites or configure any settings as needed. Click 'Create Profile at the top when done. The profile will then be created in `./crawls/profiles/profile.tar.gz` containing the settings of this browsing session. +It is also possible to extend an existing profiles by also passing in an existing profile via the `--profile` flag. In this way, it is possible to build new profiles by extending previous browsing sessions as needed. -### Using Browser Profile +``` +docker run -p 9222:9222 -p 9223:9223 -v $PWD/profiles:/profiles --filename /profiles/newProfile.tar.gz -it webrecorder/browsertrix-crawler:[VERSION] create-login-profile --interactive --url "https://example.com/ --profile /profiles/oldProfile.tar.gz" +``` + +### Using Browser Profile with a Crawl To use a previously created profile with a crawl, use the `--profile` flag or `profile` option. The `--profile` flag can then be used to specify any Chrome profile stored as a tarball. Using profiles created with same or older version of Browsertrix Crawler is recommended to ensure compatibility. This option allows running a crawl with the browser already pre-configured, logged in to certain sites, language settings configured, etc... @@ -369,9 +390,6 @@ After running the above command, you can now run a crawl with the profile, as fo docker run -v $PWD/crawls:/crawls/ -it webrecorder/browsertrix-crawler crawl --profile /crawls/profiles/profile.tar.gz --url https://twitter.com/--generateWACZ --collection test-with-profile ``` -The current profile creation script is still experimental and the script attempts to detect the usename and password fields on a site as generically as possible, but may not work for all sites. Additional profile functionality, such as support for custom profile creation scripts, may be added in the future. - - ## Architecture The Docker container provided here packages up several components used in Browsertrix. @@ -386,20 +404,6 @@ The crawl produces a single pywb collection, at `/crawls/collections/ argument_list.push(path.join(archiveDir, val))); // eslint-disable-line no-unused-vars // Run the wacz create command - child_process.spawnSync("wacz" , argument_list); - console.log(`WACZ successfully generated and saved to: ${waczPath}`); + child_process.spawnSync("wacz" , argument_list, {stdio: "inherit"}); + this.debugLog(`WACZ successfully generated and saved to: ${waczPath}`); } } @@ -400,7 +420,7 @@ class Crawler { try { await page.goto(url, this.gotoOpts); } catch (e) { - console.log(`Load timeout for ${url}`, e); + console.warn(`Load timeout for ${url}`, e); } if (selector) { @@ -408,7 +428,7 @@ class Crawler { } } - async extractLinks(page, seedId, depth, selector = "a[href]") { + async extractLinks(page, seedId, depth, selector = "a[href]", prop = "href", isAttribute = false) { const results = []; const seed = this.params.scopedSeeds[seedId]; @@ -418,11 +438,18 @@ class Crawler { return; } + const loadProp = (selector, prop) => { + return [...document.querySelectorAll(selector)].map(elem => elem[prop]); + }; + + const loadAttr = (selector, attr) => { + return [...document.querySelectorAll(selector)].map(elem => elem.getAttribute(attr)); + }; + + const loadFunc = isAttribute ? loadAttr : loadProp; + try { - const linkResults = await Promise.allSettled(page.frames().map(frame => frame.evaluate((selector) => { - /* eslint-disable-next-line no-undef */ - return [...document.querySelectorAll(selector)].map(elem => elem.href); - }, selector))); + const linkResults = await Promise.allSettled(page.frames().map(frame => frame.evaluate(loadFunc, selector, prop))); if (linkResults) { for (const linkResult of linkResults) { @@ -452,7 +479,7 @@ class Crawler { } } } catch (e) { - console.log("Queuing Error: ", e); + console.error("Queuing Error: ", e); } } @@ -486,19 +513,18 @@ class Crawler { if (createNew) { const header = {"format": "json-pages-1.0", "id": "pages", "title": "All Pages"}; if (this.params.text) { - console.log("creating pages with full text"); header["hasText"] = true; - } - else{ - console.log("creating pages without full text"); + this.statusLog("Text Extraction: Enabled"); + } else { header["hasText"] = false; + this.statusLog("Text Extraction: Disabled"); } const header_formatted = JSON.stringify(header).concat("\n"); await this.pagesFH.writeFile(header_formatted); } } catch(err) { - console.log("pages/pages.jsonl creation failed", err); + console.error("pages/pages.jsonl creation failed", err); } } @@ -531,7 +557,7 @@ class Crawler { agent: this.resolveAgent }); if (resp.status >= 400) { - console.log(`Skipping HEAD check ${url}, invalid status ${resp.status}`); + this.debugLog(`Skipping HEAD check ${url}, invalid status ${resp.status}`); return true; } @@ -564,7 +590,7 @@ class Crawler { } async awaitPendingClear() { - console.log("Waiting to ensure pending data is written to WARC..."); + this.statusLog("Waiting to ensure pending data is written to WARCs..."); const redis = new Redis("redis://localhost/0"); @@ -574,7 +600,7 @@ class Crawler { break; } - console.log(`Still waiting for ${res} pending requests to finish...`); + this.debugLog(`Still waiting for ${res} pending requests to finish...`); await this.sleep(1000); } @@ -595,17 +621,17 @@ class Crawler { const { sites } = await sitemapper.fetch(); this.queueUrls(seedId, sites, 0); } catch(e) { - console.log(e); + console.warn(e); } } async combineWARC() { - console.log("Combining the WARCs"); + this.statusLog("Generating Combined WARCs"); // Get the list of created Warcs const warcLists = await fsp.readdir(path.join(this.collDir, "archive")); - console.log(`Combining ${warcLists.length} WARCs...`); + this.debugLog(`Combining ${warcLists.length} WARCs...`); const fileSizeObjects = []; // Used to sort the created warc by fileSize @@ -674,7 +700,7 @@ class Crawler { fh.write(warcBuffer); } - console.log(`Appending WARC ${fileSizeObjects[j].fileName}`); + this.debugLog(`Appending WARC ${fileSizeObjects[j].fileName}`); const reader = fs.createReadStream(fileSizeObjects[j].fileName); @@ -691,7 +717,7 @@ class Crawler { await fh.end(); } - console.log(`Combined WARCs saved as: ${generatedCombinedWarcs}`); + this.debugLog(`Combined WARCs saved as: ${generatedCombinedWarcs}`); } } diff --git a/create-login-profile.js b/create-login-profile.js index 596f4d44..477c4f1b 100755 --- a/create-login-profile.js +++ b/create-login-profile.js @@ -6,12 +6,11 @@ const child_process = require("child_process"); const puppeteer = require("puppeteer-core"); const yargs = require("yargs"); -const { BROWSER_BIN } = require("./util/constants"); +const { getBrowserExe, loadProfile, saveProfile } = require("./util/browser"); const fs = require("fs"); const path = require("path"); const http = require("http"); -const url = require("url"); const profileHTML = fs.readFileSync(path.join(__dirname, "screencast", "createProfile.html"), {encoding: "utf8"}); function cliOpts() { @@ -49,6 +48,17 @@ function cliOpts() { describe: "Start in interactive mode!", type: "boolean", default: false, + }, + + "profile": { + describe: "Path to tar.gz file which will be extracted and used as the browser profile", + type: "string", + }, + + "windowSize": { + type: "string", + describe: "Browser window dimensions, specified as: width,height", + default: "1600,900" } }; } @@ -77,10 +87,11 @@ async function main() { } //await new Promise(resolve => setTimeout(resolve, 2000)); + const profileDir = loadProfile(params.profile); const args = { headless: !!params.headless, - executablePath: BROWSER_BIN, + executablePath: getBrowserExe(), ignoreHTTPSErrors: true, args: [ "--no-xshm", @@ -88,9 +99,11 @@ async function main() { "--disable-background-media-suspend", "--autoplay-policy=no-user-gesture-required", "--disable-features=IsolateOrigins,site-per-process", - "--user-data-dir=/tmp/profile", "--remote-debugging-port=9221", - ] + `--window-size=${params.windowSize}` + ], + userDataDir: profileDir, + defaultViewport: null, }; if (!params.user && !params.interactive) { @@ -163,7 +176,8 @@ async function createProfile(params, browser, page) { const profileFilename = params.filename || "/output/profile.tar.gz"; - child_process.execFileSync("tar", ["cvfz", profileFilename, "./"], {cwd: "/tmp/profile"}); + saveProfile(profileFilename); + console.log("done"); } @@ -199,16 +213,17 @@ function promptInput(msg, hidden = false) { async function handleInteractive(params, browser, page) { const target = page.target(); - const targetUrl = `http://localhost:9222/devtools/inspector.html?ws=localhost:9222/devtools/page/${target._targetId}`; + const targetUrl = `http://$HOST:9222/devtools/inspector.html?ws=localhost:9222/devtools/page/${target._targetId}&panel=resources`; console.log("Creating Profile Interactively..."); child_process.spawn("socat", ["tcp-listen:9222,fork", "tcp:localhost:9221"]); const httpServer = http.createServer(async (req, res) => { - const pathname = url.parse(req.url).pathname; + const parsedUrl = new URL(req.url, `http://${req.headers.host}`); + const pathname = parsedUrl.pathname; if (pathname === "/") { res.writeHead(200, {"Content-Type": "text/html"}); - res.end(profileHTML.replace("$DEVTOOLS_SRC", targetUrl)); + res.end(profileHTML.replace("$DEVTOOLS_SRC", targetUrl.replace("$HOST", parsedUrl.hostname))); } else if (pathname === "/createProfile" && req.method === "POST") { @@ -234,7 +249,7 @@ async function handleInteractive(params, browser, page) { const port = 9223; httpServer.listen(port); - console.log(`Browser Profile UI Server started. Load http://localhost:${port}/ to interact with the browser, click 'Create Profile' when done.`); + console.log(`Browser Profile UI Server started. Load http://localhost:${port}/ to interact with a Chromium-based browser, click 'Create Profile' when done.`); } main(); diff --git a/docker-compose.yml b/docker-compose.yml index 253add1f..5b21e9c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.5' services: crawler: - image: webrecorder/browsertrix-crawler:0.4.0 + image: webrecorder/browsertrix-crawler:0.4.1 build: context: ./ diff --git a/package.json b/package.json index 3f879826..82c03e76 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "browsertrix-crawler", - "version": "0.4.0", + "version": "0.4.1", "main": "browsertrix-crawler", "repository": "https://github.com/webrecorder/browsertrix-crawler", "author": "Ilya Kreymer , Webrecorder Software", diff --git a/screencast/createProfile.html b/screencast/createProfile.html index 572197ca..ad712ed8 100644 --- a/screencast/createProfile.html +++ b/screencast/createProfile.html @@ -7,10 +7,16 @@ html, body, iframe { height: 100%; margin: 0; padding: 0; + border: 0; overflow: hidden; + font-family: sans-serif; +} +body { + display: flex; + flex-direction: column; } iframe#main { - height: calc(100% - 36px); + height: calc(100% - 36px); } div#info { margin: 8px; @@ -18,11 +24,15 @@ div#info { form { display: inline; } +button { + font-weight: bold; + font-size: 15px; +}
-Create Profile Interactively -- Load any pages that you want to be part of the profile. When Done, Click
+Log in to any site(s) that you want to be part of the crawl profile using the embedded browser below. When done, click
diff --git a/tests/scopes.test.js b/tests/scopes.test.js index 266fc142..8d2f1ecd 100644 --- a/tests/scopes.test.js +++ b/tests/scopes.test.js @@ -12,7 +12,7 @@ function getSeeds(config) { return orig(name, ...args); }; - return parseArgs(null, ["node", "crawler", "--config", "configtest"]).scopedSeeds; + return parseArgs(["node", "crawler", "--config", "configtest"]).scopedSeeds; } test("default scope", async () => { diff --git a/util/argParser.js b/util/argParser.js index d81cf9ce..7396293a 100644 --- a/util/argParser.js +++ b/util/argParser.js @@ -1,6 +1,5 @@ const path = require("path"); const fs = require("fs"); -const child_process = require("child_process"); const yaml = require("js-yaml"); const puppeteer = require("puppeteer-core"); @@ -16,10 +15,6 @@ const { ScopedSeed } = require("./seeds"); // ============================================================================ class ArgParser { - constructor(profileDir) { - this.profileDir = profileDir; - } - get cliOpts() { return { "seeds": { @@ -50,7 +45,7 @@ class ArgParser { "waitUntil": { describe: "Puppeteer page.goto() condition to wait for before continuing, can be multiple separate by ','", - default: "load,networkidle0", + default: "load,networkidle2", }, "depth": { @@ -195,6 +190,12 @@ class ArgParser { type: "string", }, + "behaviorTimeout": { + describe: "If >0, timeout (in seconds) for in-page behavior will run on each page. If 0, a behavior can run until finish.", + default: 90, + type: "number", + }, + "profile": { describe: "Path to tar.gz file which will be extracted and used as the browser profile", type: "string", @@ -261,6 +262,9 @@ class ArgParser { argv.behaviors = argv.behaviors.split(","); } argv.behaviors.forEach((x) => behaviorOpts[x] = true); + if (argv.behaviorTimeout) { + behaviorOpts.timeout = argv.behaviorTimeout *= 1000; + } if (argv.logging.includes("behaviors")) { behaviorOpts.log = BEHAVIOR_LOG_FUNC; } else if (argv.logging.includes("behaviors-debug")) { @@ -277,7 +281,8 @@ class ArgParser { case "page": argv.newContext = Cluster.CONCURRENCY_PAGE; if (argv.screencastPort && argv.workers > 1) { - console.warn("Note: Screencast with >1 workers and default page context may only show one page at a time. To fix, add '--newContext window' to open each page in a new window"); + console.log("Note: to support screencasting with >1 workers, newContext set to 'window' instead of 'page'"); + argv.newContext = NewWindowPage; } break; @@ -348,15 +353,10 @@ class ArgParser { argv.statsFilename = path.resolve(argv.cwd, argv.statsFilename); } - if (argv.profile) { - child_process.execSync("tar xvfz " + argv.profile, {cwd: this.profileDir}); - } - return true; } } - -module.exports.parseArgs = function(profileDir, argv) { - return new ArgParser(profileDir).parseArgs(argv); +module.exports.parseArgs = function(argv) { + return new ArgParser().parseArgs(argv); }; diff --git a/util/blockrules.js b/util/blockrules.js index 75dc2507..77f356c9 100644 --- a/util/blockrules.js +++ b/util/blockrules.js @@ -56,9 +56,19 @@ class BlockRules } async initPage(page) { + if (!this.rules.length) { + return; + } + await page.setRequestInterception(true); - page.on("request", (request) => this.handleRequest(request)); + page.on("request", async (request) => { + try { + await this.handleRequest(request); + } catch (e) { + console.warn(e); + } + }); } async handleRequest(request) { diff --git a/util/browser.js b/util/browser.js new file mode 100644 index 00000000..792a7499 --- /dev/null +++ b/util/browser.js @@ -0,0 +1,33 @@ +const child_process = require("child_process"); +const fs = require("fs"); +const path = require("path"); +const os = require("os"); + +const profileDir = fs.mkdtempSync(path.join(os.tmpdir(), "profile-")); + +module.exports.loadProfile = function(profileFilename) { + if (profileFilename) { + child_process.execSync("tar xvfz " + profileFilename, {cwd: profileDir}); + } + + return profileDir; +}; + +module.exports.saveProfile = function(profileFilename) { + child_process.execFileSync("tar", ["cvfz", profileFilename, "./"], {cwd: profileDir}); +}; + +module.exports.getBrowserExe = function() { + const files = [process.env.BROWSER_BIN, "/usr/bin/google-chrome", "/usr/bin/chromium-browser"]; + for (const file of files) { + if (file && fs.existsSync(file)) { + return file; + } + } + + return null; +}; + + + + diff --git a/util/constants.js b/util/constants.js index 34f57c5a..db081549 100644 --- a/util/constants.js +++ b/util/constants.js @@ -2,5 +2,4 @@ module.exports.HTML_TYPES = ["text/html", "application/xhtml", "application/xhtml+xml"]; module.exports.WAIT_UNTIL_OPTS = ["load", "domcontentloaded", "networkidle0", "networkidle2"]; module.exports.BEHAVIOR_LOG_FUNC = "__bx_log"; -module.exports.BROWSER_BIN = process.env.BROWSER_BIN || "google-chrome"; diff --git a/util/screencaster.js b/util/screencaster.js index 54922538..90d9cb61 100644 --- a/util/screencaster.js +++ b/util/screencaster.js @@ -47,7 +47,6 @@ class ScreenCaster }); this.httpServer.listen(port); - console.log(`Screencast Server started on: ${port}`); } initWebSocket(ws) { @@ -65,7 +64,7 @@ class ScreenCaster } ws.on("close", () => { - console.log("Screencast WebSocket Disconnected"); + //console.log("Screencast WebSocket Disconnected"); this.allWS.delete(ws); if (this.allWS.size === 0) { @@ -100,7 +99,7 @@ class ScreenCaster try { await cdp.send("Page.screencastFrameAck", {sessionId}); } catch(e) { - console.log("Ack Failed, probably window/tab already closed", e); + //console.log("Ack Failed, probably window/tab already closed", e); } }); diff --git a/util/seeds.js b/util/seeds.js index 0506b422..c4834697 100644 --- a/util/seeds.js +++ b/util/seeds.js @@ -33,7 +33,7 @@ class ScopedSeed parseUrl(url) { let parsedUrl = null; try { - parsedUrl = new URL(url); + parsedUrl = new URL(url.trim()); } catch (e) { throw new Error(`Invalid Seed "${url}" - not a valid URL`); }