mirror of
https://github.com/webrecorder/browsertrix-crawler.git
synced 2025-10-19 06:23:16 +00:00
e585b6d194
- set crawl id from collection, not other way around, to ensure unique redis keyspace for different collections - by default, set crawl id to unique value based on host and collection, eg. '@hostname-@id' - don't include '@id' in collection interpolation, can only used hostname or timestamp - fixes issue mentioned / workaround provided in #784 - ci: add docker login + cacheing to work around rate limits - tests: fix sitemap tests
90 lines
2.3 KiB
YAML
90 lines
2.3 KiB
YAML
name: Node.js CI
|
|
|
|
on:
|
|
push:
|
|
pull_request:
|
|
|
|
# Cancel in progress workflows on pull_requests.
|
|
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
lint:
|
|
runs-on: ubuntu-latest
|
|
|
|
strategy:
|
|
matrix:
|
|
node-version: [20.x]
|
|
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- name: Use Node.js ${{ matrix.node-version }}
|
|
uses: actions/setup-node@v3
|
|
with:
|
|
node-version: ${{ matrix.node-version }}
|
|
- name: install requirements
|
|
run: yarn install
|
|
- name: run linter
|
|
run: yarn lint && yarn format
|
|
|
|
build:
|
|
runs-on: ubuntu-latest
|
|
|
|
strategy:
|
|
matrix:
|
|
node-version: [20.x]
|
|
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Use Node.js ${{ matrix.node-version }}
|
|
uses: actions/setup-node@v3
|
|
with:
|
|
node-version: ${{ matrix.node-version }}
|
|
|
|
- uses: actions/setup-python@v4
|
|
with:
|
|
python-version: 3.x
|
|
|
|
- name: install requirements
|
|
run: yarn install
|
|
|
|
- name: build js
|
|
run: yarn run tsc
|
|
|
|
- name: Cache Docker Images
|
|
uses: ScribeMD/docker-cache@0.5.0
|
|
with:
|
|
key: docker-${{ runner.os }}-${{ hashFiles('Dockerfile') }}
|
|
|
|
- name: Login to DockerHub
|
|
uses: docker/login-action@v3
|
|
with:
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
|
|
- name: build docker
|
|
run: docker compose build
|
|
|
|
- name: install python deps for docs
|
|
run: pip install mkdocs-material
|
|
|
|
- name: build docs for crawl test
|
|
run: cd docs/ && mkdocs build
|
|
|
|
- name: add http-server for tests
|
|
run: yarn add -D http-server
|
|
|
|
- name: install py-wacz as root for tests
|
|
run: sudo pip install wacz --ignore-installed
|
|
|
|
- name: run all tests as root
|
|
run: sudo DOCKER_HOST_NAME=172.17.0.1 CI=true yarn test -validate
|
|
|
|
- name: run saved state + qa compare test as non-root - with volume owned by current user
|
|
run: |
|
|
sudo rm -rf ./test-crawls
|
|
mkdir test-crawls
|
|
sudo CI=true yarn test ./tests/saved-state.test.js ./tests/qa_compare.test.js
|