indexing: generate wacz indexes consistent with warcio.js, including surt, and post-request query in requestBody field and method field

app: fix wacz download/ipfs sharing in app
dependencies: update to wabac.js 2.6.1, warcio.js 1.4.1 and replayweb.page 1.3.9
bump to 0.5.9

Author: ikreymer

package.json

@@ -1,14 +1,14 @@
 {
   "name": "ArchiveWeb.page",
-  "version": "0.5.8",
+  "version": "0.5.9",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": "https://github.com/webrecorder/archiveweb.page",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.13.0",
-    "@webrecorder/wabac": "github:webrecorder/wabac.js",
+    "@webrecorder/wabac": "^2.6.1",
     "bulma": "^0.9.1",
     "flexsearch": "^0.6.32",
     "hash-wasm": "^4.4.1",
@@ -21,7 +21,7 @@
     "node-fetch": "^2.6.1",
     "pretty-bytes": "^5.3.0",
     "replaywebpage": "github:webrecorder/replayweb.page",
-    "warcio": "^1.3.3"
+    "warcio": "^1.4.1"
   },
   "devDependencies": {
     "copy-webpack-plugin": "^6.4.0",

src/downloader.js

@@ -6,7 +6,7 @@ import { Deflate } from 'pako';
 
 import { createMD5 } from 'hash-wasm';
 
-import { WARCRecord, WARCSerializer } from 'warcio';
+import { WARCRecord, WARCSerializer, getSurt } from 'warcio';
 
 import { getTSMillis, getStatusText } from '@webrecorder/wabac/src/utils';
 
@@ -19,6 +19,8 @@ const encoder = new TextEncoder();
 
 const EMPTY = new Uint8Array([]);
 
+const SPLIT_REQUEST_Q_RX = /(.*?)[?&](?:__wb_method=|__wb_post=)[^&]+&(.*)/;
+
 async function* getPayload(payload) {
   yield payload;
 }
@@ -294,6 +296,7 @@ class Downloader
     function getCDX(resource, filename, raw) {
 
       const data = {
+        url: resource.url,
         digest: resource.digest,
         mime: resource.mime,
         offset: resource.offset,
@@ -302,7 +305,18 @@ class Downloader
         status: resource.status
       }
 
-      const cdx = `${resource.url} ${resource.timestamp} ${JSON.stringify(data)}\n`;
+      if (resource.method && resource.method !== "GET") {
+        const m = resource.url.match(SPLIT_REQUEST_Q_RX);
+        if (m) {
+          data.url = m[1];
+          data.requestBody = m[2];
+        }
+        data.method = resource.method;
+      }
+
+      const surt = getSurt(resource.url);
+
+      const cdx = `${surt} ${resource.timestamp} ${JSON.stringify(data)}\n`;
 
       if (!raw) {
         return cdx;
@@ -362,7 +376,7 @@ class Downloader
       }
 
       if (!key) {
-        key = resource.url + " " + resource.timestamp;
+        key = cdx.split(" {", 1)[0];
       }
 
       if (++count === this.linesPerBlock) {

src/recorder.js

@@ -193,7 +193,7 @@ class Recorder {
       await this.send("Network.setCacheDisabled", {cacheDisabled: true}, sessions);
       await this.send("Network.setBypassServiceWorker", {bypass: true}, sessions);
       // another option: clear cache, but don't disable
-      //await this.send("Network.clearBrowserCache", null, sessions);
+      await this.send("Network.clearBrowserCache", null, sessions);
     } catch (e) {
       console.warn("Session Init Error: ");
       console.log(e);
@@ -935,7 +935,12 @@ class Recorder {
         opts.headers.delete("range");
       }
 
-      const resp = await fetch(request.url, opts);
+      let resp = await fetch(request.url, opts);
+      if (resp.status !== 200) {
+        console.warn(`async fetch error ${resp.status}, retrying without headers`);
+        resp = await fetch(request.url);
+      }
+
       const payload = await resp.arrayBuffer();
 
       const reqresp = new RequestResponseInfo(fetchId);

src/requestresponseinfo.js

@@ -1,6 +1,8 @@
 "use strict";
 
-import { postToGetUrl, getStatusText } from '@webrecorder/wabac/src/utils';
+import { getStatusText } from '@webrecorder/wabac/src/utils';
+
+import { postToGetUrl } from 'warcio';
 
 
 // ===========================================================================
@@ -155,8 +157,8 @@ class RequestResponseInfo
         postData: this.postData,
       }
       if (postToGetUrl(convData)) {
+        this.requestBody = convData.requestBody;
         this.url = convData.url;
-        this.method = "GET";
       }
     }
 

webpack.config.js

@@ -15,6 +15,14 @@ const BANNER = '[name].js is part of the Webrecorder Extension (https://replaywe
 
 const manifest = require("./src/ext/manifest.json");
 
+const defaultDefines = {
+  __VERSION__: JSON.stringify(PACKAGE.version),
+  __WARCIO_VERSION__: JSON.stringify(WARCIO_PACKAGE.version),
+  __SW_NAME__: JSON.stringify("sw.js"),
+  __IPFS_CORE_URL__: JSON.stringify(""),
+  __IPFS_HTTP_CLIENT_URL__: JSON.stringify("")
+};
+
 
 const moduleSettings =  {
   rules: [
@@ -56,10 +64,7 @@ const electronMainConfig = (env, argv) => {
       __filename: false,
     },
     plugins: [
-      new webpack.DefinePlugin({
-        __IPFS_CORE_URL__: JSON.stringify(""),
-        __APP_FILE_SERVE_PREFIX__ : JSON.stringify(APP_FILE_SERVE_PREFIX),
-      }),
+      new webpack.DefinePlugin(defaultDefines),
       new webpack.BannerPlugin(BANNER),
       new CopyPlugin({
         patterns: [
@@ -91,12 +96,7 @@ const electronPreloadConfig = (env, argv) => {
       filename: '[name].js'
     },
     plugins: [
-      // this needs to be defined, but not actually used, as electron app uses
-      // ipfs-core from node
-      new webpack.DefinePlugin({
-        __IPFS_CORE_URL__: JSON.stringify(""),
-        __APP_FILE_SERVE_PREFIX__ : JSON.stringify(APP_FILE_SERVE_PREFIX),
-      }),
+      new webpack.DefinePlugin(defaultDefines),
     ]
   }
 };
@@ -124,10 +124,7 @@ const electronRendererConfig = (env, argv) => {
           { from: 'src/electron/rec-window.html', to: '' },
         ]
       }),
-      new webpack.DefinePlugin({
-        __IPFS_CORE_URL__: JSON.stringify(""),
-        __APP_FILE_SERVE_PREFIX__ : JSON.stringify(APP_FILE_SERVE_PREFIX),
-      })
+      new webpack.DefinePlugin(defaultDefines),
     ],
 
     module: moduleSettings,
@@ -172,12 +169,8 @@ const extensionConfig = (env, argv) => {
       new MiniCssExtractPlugin(),
       new webpack.BannerPlugin(BANNER),
       new GenerateJsonPlugin('manifest.json', manifest, generateManifest, 2),
-      new webpack.DefinePlugin({
-        __VERSION__: JSON.stringify(PACKAGE.version),
-        __WARCIO_VERSION__: JSON.stringify(WARCIO_PACKAGE.version),
-        __SW_NAME__: JSON.stringify("sw.js"),
+      new webpack.DefinePlugin({...defaultDefines,
         __IPFS_CORE_URL__: JSON.stringify(IPFS_CORE_URL),
-        __IPFS_HTTP_CLIENT_URL__: JSON.stringify("")
       }),
       new CopyPlugin({
         patterns: [

wr-ext/bg.js

@@ -1,7 +1,7 @@
 {
   "name": "Webrecorder ArchiveWeb.page",
   "description": "Create high-fidelity web archives directly in your browser",
-  "version": "0.5.8",
+  "version": "0.5.9",
   "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'",
   "permissions": [
     "debugger",

wr-ext/manifest.json

@@ -808,11 +808,12 @@
     "@webassemblyjs/wast-parser" "1.9.0"
     "@xtuc/long" "4.2.2"
 
-"@webrecorder/wabac@github:webrecorder/wabac.js":
-  version "2.6.0-beta.1"
-  resolved "https://codeload.github.com/webrecorder/wabac.js/tar.gz/cf5f5fa51a8700cbcf3d0a04b44831366558d91e"
+"@webrecorder/wabac@^2.6.1":
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.6.1.tgz#e1e1cd5a2cd43043fbbdaa0fb49386f96bba4ccb"
+  integrity sha512-+7sfiRQ15/Xmw4ONZXC5UKXf7v1Vr3JhjHCvdpyuAdYNcS7zPeoqj7jK2CIywzuBkeKcRDkgvyKYbXGy2FCGfQ==
   dependencies:
-    "@webrecorder/wombat" "^3.1.0"
+    "@webrecorder/wombat" "^3.1.1"
     brotli "github:foliojs/brotli.js"
     fast-xml-parser "^3.15.1"
     format-link-header "^3.1.1"
@@ -827,13 +828,13 @@
     parse5-sax-parser "^6.0.1"
     path-parser "^6.1.0"
     unescape-js "^1.1.4"
-    warcio "^1.3.3"
+    warcio "^1.4.1"
     wbn "^0.0.3"
 
-"@webrecorder/wombat@^3.1.0":
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.1.0.tgz#6df2e84577e2cbcf4a577ed91526723537495666"
-  integrity sha512-4fYNahANNcWwXc8+JHZ7xtmMnESY1kZGwzgK3snRuO5/CMosNX1rMQ0TBbTGMjv0izFs3Pl4V4FGIDBkrAEHVA==
+"@webrecorder/wombat@^3.1.1":
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.1.1.tgz#8a62e1b260e4c8e6be52a0cb70f6d8689777ed1d"
+  integrity sha512-33LCH7L165CPx+Q/RqkQbTeD5SPI/d1OKnI9lzDo0nJu9e+zNdz+apAiCOMLXvzoFAkYzooITfvzHlHWaf+LKA==
 
 "@xtuc/ieee754@^1.2.0":
   version "1.2.0"
@@ -8361,7 +8362,7 @@ repeating@^2.0.0:
 
 "replaywebpage@github:webrecorder/replayweb.page":
   version "1.3.9"
-  resolved "https://codeload.github.com/webrecorder/replayweb.page/tar.gz/007da4b52cae4dea300663d15e5c45c9bae3bd4d"
+  resolved "https://codeload.github.com/webrecorder/replayweb.page/tar.gz/746d0e9a79fde6eaa090270c001b3641ea768716"
   dependencies:
     "@fortawesome/fontawesome-free" "^5.13.0"
     "@webrecorder/wabac" "github:webrecorder/wabac.js"
@@ -10013,10 +10014,10 @@ vm-browserify@^1.0.1:
   resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0"
   integrity sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==
 
-warcio@^1.3.3:
-  version "1.3.3"
-  resolved "https://registry.yarnpkg.com/warcio/-/warcio-1.3.3.tgz#d215223e6ea2cf1fbaea645f8dd34b9430c258a8"
-  integrity sha512-Qr0Uw/tqZXzDBwT3BmfHuFu90jMT0Mc/JzZMhCdXyqdihVtgOZqtvwj+JcDBBmBDrpIccabi/XRXVbAGbE9U+Q==
+warcio@^1.4.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/warcio/-/warcio-1.4.1.tgz#6d289f37044eda58e369db2bed18cdaf6f9f7beb"
+  integrity sha512-4Kue2T+Zt93L/CyBSFh3sHF7NgR+DER5FQHoU0uGBR6abcqBqQ+mE22OE6fpjSReZMx2jIeiAQLD8XQzPq3P3g==
   dependencies:
     "@peculiar/webcrypto" "^1.1.1"
     esm "^3.2.25"