serialization fixes:

- serialize original POST request to warc to avoid confusion, partial fix for webrecorder/replayweb.page#61
- don't serialize empty revisit records with just statusline (fixed via warcio.js 1.4.7)
optimization: don't convert response body to string unless rewriting!
also: don't convert post-to-get on replay as now supported in chrome! (via wombat 3.3.0)
fidelity improvements from wabac.js 2.8.0-beta.0 + wombat 3.3.0
recorder: when getting text nodes, timeout after 10s in case getDOM() call does not return (can happen in case of PDFs)
dependencies: update to wabac.js 2.8.0-beta.0 + wombat 3.3.0 + warcio.js
signing: update to new format, store signed data in signedData field in datapackage-digest.json
bump to 0.6.12

Author: ikreymer

package.json

@@ -1,14 +1,14 @@
 {
   "name": "archiveweb.page",
-  "version": "0.6.11",
+  "version": "0.6.12",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": "https://github.com/webrecorder/archiveweb.page",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.13.0",
-    "@webrecorder/wabac": "^2.7.11",
+    "@webrecorder/wabac": "^2.8.0-beta.0",
     "browsertrix-behaviors": "^0.2.3",
     "bulma": "^0.9.2",
     "flexsearch": "^0.6.32",
@@ -23,7 +23,7 @@
     "pretty-bytes": "^5.3.0",
     "replaywebpage": "^1.4.6",
     "uuid": "^8.3.2",
-    "warcio": "^1.4.5"
+    "warcio": "^1.4.7"
   },
   "devDependencies": {
     "copy-webpack-plugin": "^6.4.0",

src/downloader.js

@@ -453,9 +453,7 @@ class Downloader
 
     if (this.signer) {
       try {
-        const {signature, publicKey} = await this.signer.sign(hash);
-        data.signature = signature;
-        data.publicKey = publicKey;
+        data.signedData = await this.signer.sign(hash, this.createdDate);
 
         this.signer.close();
         this.signer = null;
@@ -600,10 +598,10 @@ class Downloader
   }
 
   async createWARCRecord(resource) {
-    const url = resource.url;
+    let url = resource.url;
     const date = new Date(resource.ts).toISOString();
     resource.timestamp = getTSMillis(date);
-    const httpHeaders = resource.respHeaders;
+    const httpHeaders = resource.respHeaders || {};
     const warcVersion = "WARC/1.1";
 
     // remove aas never preserved in browser-based capture
@@ -616,12 +614,26 @@ class Downloader
 
     let refersToUrl, refersToDate;
 
+    let method = "GET";
+    let requestBody;
+
+    // non-GET request/response:
+    // if original request body + original requestURL is preserved, write that with original method
+    // otherwise, just serialize the converted-to-GET form
+    if (resource.method && resource.method !== "GET" && resource.requestBody && resource.requestUrl) {
+      requestBody = resource.requestBody;
+      method = resource.method;
+      url = resource.requestUrl;
+    } else {
+      requestBody = new Uint8Array([]);
+    }
+
     const digestOriginal = this.digestsVisted[resource.digest];
 
     if (resource.digest && digestOriginal) {
 
       // if exact resource in a row, and same page, then just skip instead of writing revisit
-      if (url === this.lastUrl && pageId === this.lastPageId) {
+      if (url === this.lastUrl && pageId === this.lastPageId && method === "GET") {
         //console.log("Skip Dupe: " + url);
         return null;
       }
@@ -657,7 +669,9 @@ class Downloader
         return null;
       }
 
-      this.digestsVisted[resource.digest] = {url, date};
+      if (method === "GET") {
+        this.digestsVisted[resource.digest] = {url, date};
+      }
     }
 
     const status = resource.status || 200;
@@ -700,7 +714,6 @@ class Downloader
         "WARC-Concurrent-To": record.warcHeader("WARC-Record-ID"),
       };
 
-      const method = resource.method || "GET";
       const urlParsed = new URL(url);
       const statusline = method + " " + url.slice(urlParsed.origin.length);
 
@@ -709,7 +722,7 @@ class Downloader
         warcHeaders: reqWarcHeaders,
         httpHeaders: resource.reqHeaders,
         statusline,
-      }, getPayload(new Uint8Array([])));
+      }, getPayload(requestBody));
 
       records.push(await WARCSerializer.serialize(reqRecord, {gzip: true}));
     }

src/keystore.js

@@ -74,7 +74,7 @@ export class Signer
     }
   }
 
-  async sign(string) {
+  async sign(string, created) {
     let keyPair;
     let keys = await this.loadKeys();
 
@@ -119,7 +119,14 @@ export class Signer
 
     signature = encodeBase64(new Uint8Array(signature));
 
-    return { signature, publicKey: keys.public };
+    return {
+      hash: string,
+      signature,
+      publicKey: keys.public,
+      created,
+      // eslint-disable-next-line no-undef
+      software: `ArchiveWeb.page ${__VERSION__}`
+    };
   }
 
   async saveKeys(keys, id = "_userkey") {

src/recorder.js

@@ -627,9 +627,11 @@ class Recorder {
     }
 
     try {
-      //const startTime = new Date().getTime();
-      return await this.send("DOM.getDocument", {"depth": -1, "pierce": true});
-      //console.log(`Time getting text for ${this.pageInfo.id}: ${(new Date().getTime() - startTime)}`);
+      // wait upto 5s for getDocument, otherwise proceed
+      return await Promise.race([
+        this.send("DOM.getDocument", {"depth": -1, "pierce": true}),
+        sleep(10000)
+      ]);
     } catch(e) {
       console.log(e);
       return null;
@@ -962,10 +964,10 @@ class Recorder {
     case "text/javascript":
     case "application/javascript":
     case "application/x-javascript": {
-      string = payload.toString("utf-8");
       const rw = baseDSRules.getRewriter(params.request.url);
 
       if (rw !== baseDSRules.defaultRewriter) {
+        string = payload.toString("utf-8");
         newString = rw.rewrite(string, {live: true});
       }
       break;

src/requestresponseinfo.js

@@ -9,6 +9,8 @@ const MAX_URL_LENGTH = 4096;
 
 const EXCLUDE_HEADERS = ["content-encoding", "transfer-encoding"];
 
+const encoder = new TextEncoder();
+
 
 // ===========================================================================
 class RequestResponseInfo
@@ -154,6 +156,8 @@ class RequestResponseInfo
       respHeaders.headersDict["x-wabac-preset-cookie"] = cookie;
     }
 
+    const reqUrl = this.url;
+
     if (this.postData) {
       const convData = {
         url: this.url,
@@ -168,7 +172,8 @@ class RequestResponseInfo
       }
     }
 
-    const data = {url: this.url,
+    const data = {
+      url: this.url,
       ts: this.ts,
       status: this.status,
       statusText:this.statusText,
@@ -182,6 +187,14 @@ class RequestResponseInfo
 
     if (this.method !== "GET") {
       data.method = this.method;
+      if (this.postData) {
+        if (typeof(this.postData) === "string") {
+          data.requestBody = encoder.encode(this.postData);
+        } else {
+          data.requestBody = this.postData;
+        }
+        data.requestUrl = reqUrl;
+      }
     }
 
     return data;

wr-ext/bg.js

@@ -1,7 +1,7 @@
 {
   "name": "Webrecorder ArchiveWeb.page",
   "description": "Create high-fidelity web archives directly in your browser",
-  "version": "0.6.11",
+  "version": "0.6.12",
   "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'",
   "permissions": [
     "debugger",

wr-ext/manifest.json

@@ -876,12 +876,12 @@
     warcio "^1.4.6"
     wbn "^0.0.3"
 
-"@webrecorder/wabac@^2.7.11":
-  version "2.7.11"
-  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.7.11.tgz#6ffaada9bcf702245e8bd948d387ca0a675d9e37"
-  integrity sha512-uZz3b2kJRuGzE8wP0tJ848oIaqJHPrXivnDHd6wew3bPDhLr5s9twO1ozn9P6hJSkL3H6f5Yf8gw3HTKbNtygA==
+"@webrecorder/wabac@^2.8.0-beta.0":
+  version "2.8.0-beta.0"
+  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.8.0-beta.0.tgz#0bdc8b95cea811d0eef9af11602f85cde28cdd87"
+  integrity sha512-JWZ0/VtuJ0ax1GmQwpcqYnU6rWjvgLTgoHRTO0pV343gsMlBPXfBopEkPTmN0eUhgN7iLPTJownBNboRLGyECA==
   dependencies:
-    "@webrecorder/wombat" "^3.2.2"
+    "@webrecorder/wombat" "^3.3.0"
     brotli "github:foliojs/brotli.js"
     fast-xml-parser "^3.15.1"
     format-link-header "^3.1.1"
@@ -903,6 +903,13 @@
   resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.2.2.tgz#e803d6929f9c55d0e68bf0f4b363114c27695924"
   integrity sha512-oe+EMpPZUo5YoPjvrfxrf8+z/atBh9yzr7S2xFOeXXpeBt/s62rFY84n/tuE6poIKtYHSD9Radjap7JSuE9F6w==
 
+"@webrecorder/wombat@^3.3.0":
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.3.0.tgz#9f14cb75ce722683a047c8687d1ef6353a3e58ed"
+  integrity sha512-BpyRndvu6RaOVfIY4rdjcN8PM4ezcRItQp94/GVeksXahMlkf9CHsr/ATtPYUv1g7UIqq+O540WG9p3BmjwNqQ==
+  dependencies:
+    warcio "^1.4.6"
+
 "@xtuc/ieee754@^1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790"
@@ -10863,10 +10870,10 @@ vm-browserify@^1.0.1:
   resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0"
   integrity sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==
 
-warcio@^1.4.5:
-  version "1.4.5"
-  resolved "https://registry.yarnpkg.com/warcio/-/warcio-1.4.5.tgz#24ca61f799185c5d88cdd0a65d279f376b4f9a63"
-  integrity sha512-VwFBdmEQhWHmxsdyiLM0INHD1KZ2+EGYzslZXFe6JdbuTfSF/dYRQ/wEdvp+m28mydphROF6D32KfkIMRU1NZw==
+warcio@^1.4.6:
+  version "1.4.6"
+  resolved "https://registry.yarnpkg.com/warcio/-/warcio-1.4.6.tgz#56ea9a118b89b7b9c71eec493d140876f65d94ee"
+  integrity sha512-uncb/Xokc9XL4/1UMHPteAWpiLToOkcu2qhT2ivhVN1t8HwApHGAmLY4y9gqewgtO3s5aRq6T7lJ6WSP5IlA+A==
   dependencies:
     "@peculiar/webcrypto" "^1.1.1"
     esm "^3.2.25"
@@ -10876,10 +10883,10 @@ warcio@^1.4.5:
     uuid-random "^1.3.0"
     yargs "^15.3.1"
 
-warcio@^1.4.6:
-  version "1.4.6"
-  resolved "https://registry.yarnpkg.com/warcio/-/warcio-1.4.6.tgz#56ea9a118b89b7b9c71eec493d140876f65d94ee"
-  integrity sha512-uncb/Xokc9XL4/1UMHPteAWpiLToOkcu2qhT2ivhVN1t8HwApHGAmLY4y9gqewgtO3s5aRq6T7lJ6WSP5IlA+A==
+warcio@^1.4.7:
+  version "1.4.7"
+  resolved "https://registry.yarnpkg.com/warcio/-/warcio-1.4.7.tgz#16c2b02946926f80e346ef924178d9d8b558b2cf"
+  integrity sha512-0Yu5qVkV09PhvBVWXNidaH/OqUEKlnzZnNraCsvNJiYnoqxAlqeyBT3u6cbT36CIgxqHQSZlrkklr3fICzFoHg==
   dependencies:
     "@peculiar/webcrypto" "^1.1.1"
     esm "^3.2.25"