add ci and linting (#21)

* linting with eslint
* ci: validate linting and check basic single-page crawl with wacz creation

Co-authored-by: Emma Dickson <emmadickson@Emmas-MacBook-Pro.local>

.eslintignore

@@ -0,0 +1 @@
+.*

.eslintrc.js

@@ -1,30 +0,0 @@
-module.exports = {
-    "env": {
-        "es6": true,
-        "node": true
-    },
-    "extends": "eslint:recommended",
-    "parserOptions": {
-        "ecmaVersion": 2018
-    },
-    "rules": {
-        "indent": [
-            "error",
-            2
-        ],
-        "linebreak-style": [
-            "error",
-            "unix"
-        ],
-        "quotes": [
-            "error",
-            "double"
-        ],
-        "semi": [
-            "error",
-            "always"
-        ],
-        "no-console": "off",
-        "no-unused-vars": "off",
-    }
-};

.eslintrc.yml

@@ -0,0 +1,14 @@
+env:
+  browser: true
+  node: true
+  es2021: true
+extends:
+  - 'plugin:react/recommended'
+parserOptions:
+  ecmaFeatures:
+    jsx: true
+  ecmaVersion: 12
+  sourceType: module
+plugins:
+  - react
+rules: {}

.github/workflows/ci.yaml

@@ -0,0 +1,46 @@
+name: Node.js CI
+
+on: [push]
+
+jobs:
+  lint:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [14.x]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: install requirements
+      run: npm install
+    - name: run linter
+      run: yarn run eslint .
+  
+  build:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [14.x]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: build docker
+      run: docker-compose build
+    - name: run crawl
+      run: docker-compose run crawler crawl --url http://www.example.com/ --generateWACZ  --collection wr-net --workers 2              
+    - name: validate existing wacz
+      run: docker-compose run crawler wacz validate --file collections/wr-net/wr-net.wacz
+
+

autofetcher.js

@@ -71,7 +71,7 @@ class AutoFetcher
           const resp = await fetch(url);
           await resp.blob();
         } catch (e) {
-
+          console.log(e)
         }
         this.numPending--;
       }

crawler.js

@@ -628,7 +628,7 @@ class Crawler {
     //console.log(`Direct capture: ${this.capturePrefix}${url}`);
     const abort = new AbortController();
     const signal = abort.signal;
-    const resp2 = await fetch(this.capturePrefix + url, {signal, headers: this.headers});
+    await fetch(this.capturePrefix + url, {signal, headers: this.headers});
     abort.abort();
   }
 

package.json

@@ -15,6 +15,7 @@
     "uuid": "8.3.2"
 },
   "devDependencies": {
-    "eslint": "^7.12.1"
+    "eslint": "^7.19.0",
+    "eslint-plugin-react": "^7.22.0"
   }
 }