optimizations:

- don't update page, size with every resource update, instead update periodically (every 10 secs) and when stopping
- extension: close db instance when recording stopped
- fixes slow replay after recording, ensuring less indexeddb connections and transactions total
fixes:
- update to warcio 1.4.4/wabac 2.7.0-beta.3: fix for json-to-query to include dupe params (fixes yahoo answers capture)
- bump max url length to 4k to avoid truncating long URLs (especially from social media)
- add autofocus to popup main button
bump to 0.6.3

Author: ikreymer

package.json

@@ -1,14 +1,14 @@
 {
   "name": "ArchiveWeb.page",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": "https://github.com/webrecorder/archiveweb.page",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.13.0",
-    "@webrecorder/wabac": "2.7.0-beta.2",
+    "@webrecorder/wabac": "2.7.0-beta.3",
     "browsertrix-behaviors": "^0.2.0",
     "bulma": "^0.9.2",
     "flexsearch": "^0.6.32",
@@ -21,8 +21,8 @@
     "lodash": "^4.17.20",
     "node-fetch": "^2.6.1",
     "pretty-bytes": "^5.3.0",
-    "replaywebpage": "^1.4.0-beta.0",
-    "warcio": "^1.4.3"
+    "replaywebpage": "1.4.0-beta.0",
+    "warcio": "^1.4.4"
   },
   "devDependencies": {
     "copy-webpack-plugin": "^6.4.0",

src/electron/electron-rec-preload.js

@@ -26,26 +26,26 @@ contextBridge.exposeInMainWorld('archivewebpage', {
 
 
 // ===========================================================================
-ipcRenderer.on('add-resource', async (event, data, pageInfo, collId) => {
+ipcRenderer.on('add-resource', async (event, data, collId) => {
   const db = await getDB(collId);
 
-  let writtenSize = 0;
   const payloadSize = data.payload.length;
+  let writtenSize = 0;
 
   try {
     if (await db.addResource(data)) {
-      writtenSize = payloadSize;
+      writtenSize += payloadSize;
     }
   } catch (e) {
     console.warn(`Commit error for ${data.url} @ ${data.ts} ${data.mime}`);
     console.warn(e);
     return;
   }
 
-  loader.updateSize(collId, payloadSize, writtenSize);
+  //loader.updateSize(collId, payloadSize, writtenSize);
 
   // increment page size
-  db.addPage(pageInfo);
+  //db.addPage(pageInfo);
 
   if (writtenSize) {
     ipcRenderer.send("inc-size", writtenSize);
@@ -62,6 +62,14 @@ ipcRenderer.on('add-page', async (event, pageInfo, collId) => {
 });
 
 
+// ===========================================================================
+ipcRenderer.on('inc-sizes', async (event, totalSize, writtenSize, collId) => {
+  if (totalSize > 0) {
+    loader.updateSize(collId, totalSize, writtenSize);
+  }
+});
+
+
 // ===========================================================================
 async function handleIpfsPin(collId) {
   const reqId = "pin-" + collId + (100 * Math.random());

src/electron/electron-recorder.js

@@ -37,9 +37,10 @@ class ElectronRecorder extends Recorder
       this.didNavigateInitPage(url);
     });
 
-    this.appWC.on("ipc-message", (event, channel, ...args) => {
+    this.appWC.on("ipc-message", (event, channel, size) => {
       if (channel === "inc-size") {
-        this.sizeNew += args[0];
+        this.sizeNew += size;
+        this._cacheSessionNew += size;
       }
     });
 
@@ -217,15 +218,19 @@ class ElectronRecorder extends Recorder
 
     //console.log("res", data.url);
 
-    // size incremented asynchronously
-    this.appWC.send("add-resource", data, this.pageInfo, this.collId);
+    // size updated asynchronously via 'inc-size' event
+    this.appWC.send("add-resource", data, this.collId);
     return 0;
   }
 
   _doAddPage(pageInfo) {
     this.appWC.send("add-page", this.pageInfo, this.collId);
   }
 
+  _doIncSizes(totalSize, writtenSize) {
+    this.appWC.send("inc-sizes", totalSize, writtenSize, this.collId);
+  }
+
   _doSendCommand(method, params, promise) {
     if (DEBUG) {
       console.log(" => ", method, params);

src/ext/browser-recorder.js

@@ -64,7 +64,7 @@ class BrowserRecorder extends Recorder {
   }
 
   setCollId(collId) {
-    if (collId !== this.collId) {
+    if (collId !== this.collId || !this.db) {
       this.collId = collId;
       this.db = null;
       this._initDB = this.collLoader.loadColl(this.collId);
@@ -88,6 +88,12 @@ class BrowserRecorder extends Recorder {
     chrome.debugger.onDetach.removeListener(this._onDetached);
     chrome.debugger.onEvent.removeListener(this._onEvent);
 
+    if (this.db) {
+      this.db.close();
+      this.db = null;
+      this._initDB = null;
+    }
+
     if (!this.tabId) {
       return;
     }
@@ -246,10 +252,10 @@ class BrowserRecorder extends Recorder {
     // increment size counter only if committed
     //incrArchiveSize('dedup', writtenSize);
     //incrArchiveSize('total', payloadSize);
-    this.collLoader.updateSize(this.collId, payloadSize, writtenSize);
+    // this.collLoader.updateSize(this.collId, payloadSize, writtenSize);
 
     // increment page size
-    await this._doAddPage(this.pageInfo);
+    // await this._doAddPage(this.pageInfo);
 
     return writtenSize;
   }
@@ -259,7 +265,13 @@ class BrowserRecorder extends Recorder {
       console.warn("Empty Page, Skipping");
       return;
     }
-    return this.db.addPage(pageInfo);
+    if (this.db) {
+      return this.db.addPage(pageInfo);
+    }
+  }
+
+  _doIncSizes(totalSize, writtenSize) {
+    this.collLoader.updateSize(this.collId, totalSize, writtenSize);
   }
 
   _doSendCommand(method, params, promise) {

src/popup.js

@@ -488,7 +488,7 @@ class RecPopup extends LitElement
         <div class="view-row">
           ${this.canRecord ? html`
           ${this.renderCollDropdown()}
-          <button
+          <button autofocus
            ?disabled=${this.actionButtonDisabled}
            @click="${!this.recording ? this.onStart : this.onStop}" class="button">
             <span class="icon">

src/recorder.js

@@ -65,6 +65,10 @@ class Recorder {
     this.id = 1;
     this.sessionSet = new Set();
 
+    this._cachePageInfo = null;
+    this._cacheSessionNew = 0;
+    this._cacheSessionTotal = 0;
+
     this.behaviorInitStr = JSON.stringify({
       autofetch: true,
       autoplay: true,
@@ -147,18 +151,24 @@ class Recorder {
     await this._stop(domNodes);
   }
 
-  _stop(domNodes = null) {
-    clearInterval(this._updateId);
-    clearInterval(this._cleanupId);
+  async _stop(domNodes = null) {
+    clearInterval(this._updateStatusId);
+    clearInterval(this._loopId);
 
     this.flushPending();
     this.running = false;
     this.pendingRequests = {};
     this.numPending = 0;
 
-    this._doStop();
+    await this.commitPage(this.pageInfo, domNodes, true);
+
+    if (this._cleaningUp) {
+      await this._cleanupStaleWait;
+    } else {
+      await this.doUpdateLoop();
+    }
 
-    return this.commitPage(this.pageInfo, domNodes, true);
+    this._doStop();
   }
 
   async attach() {
@@ -172,24 +182,54 @@ class Recorder {
     this.running = true;
     this.stopping = false;
 
-    this._updateId = setInterval(() => this.updateStatus(), 1000);
+    this._cachePageInfo = null;
+    this._cacheSessionNew = 0;
+    this._cacheSessionTotal = 0;
+    this._cleaningUp = false;
+    this._cleanupStaleWait = null;
 
-    this._cleanupId = setInterval(() => this.cleanupStale(), 10000);
+    this._updateStatusId = setInterval(() => this.updateStatus(), 1000);
+
+    this._loopId = setInterval(() => this.updateLoop(), 10000);
   };
 
-  cleanupStale() {
-    for (const key of Object.keys(this.pendingRequests)) {
-      const reqresp = this.pendingRequests[key];
+  updateLoop() {
+    if (!this._cleaningUp) {
+      this._cleanupStaleWait = this.doUpdateLoop();
+    }
+  }
 
-      if ((new Date() - reqresp._created) > 20000) {
-        if (this.noResponseForStatus(reqresp.status)) {
-          console.log("Dropping stale: " + key);
-        } else {
-          console.log(`Committing stale ${reqresp.status} ${reqresp.url}`);
-          this.fullCommit(reqresp, []);
+  async doUpdateLoop() {
+    this._cleaningUp = true;
+
+    try {
+      for (const key of Object.keys(this.pendingRequests)) {
+        const reqresp = this.pendingRequests[key];
+
+        if ((new Date() - reqresp._created) > 20000) {
+          if (this.noResponseForStatus(reqresp.status)) {
+            console.log("Dropping stale: " + key);
+          } else {
+            console.log(`Committing stale ${reqresp.status} ${reqresp.url}`);
+            await this.fullCommit(reqresp, []);
+          }
+          delete this.pendingRequests[key];
         }
-        delete this.pendingRequests[key];
       }
+
+      if (this._cachePageInfo) {
+        await this._doAddPage(this._cachePageInfo);
+        this._cachePageInfo = null;
+      }
+
+      if (this._cacheSessionTotal > 0) {
+        await this._doIncSizes(this._cacheSessionTotal, this._cacheSessionNew);
+        this._cacheSessionTotal = 0;
+        this._cacheSessionNew = 0;
+      }
+
+    } finally {
+      this._cleaningUp = false;
     }
   }
 
@@ -605,18 +645,28 @@ class Recorder {
 
     currPage.finished = finished;
 
-    return this._doAddPage(currPage);
+    const res = this._doAddPage(currPage);
+    if (currPage === this._cachePageInfo) {
+      this._cachePageInfo = null;
+    }
+    return res;
   }
 
-  commitResource(data, pageInfo) {
+  async commitResource(data, pageInfo) {
     const payloadSize = data.payload.length;
     pageInfo = pageInfo || this.pageInfo;
     pageInfo.size += payloadSize;
 
     this.sizeTotal += payloadSize;
     this.numUrls++;
 
-    this._doAddResource(data).then((writtenSize) => this.sizeNew += writtenSize);
+    const writtenSize = await this._doAddResource(data);
+
+    this.sizeNew += writtenSize;
+
+    this._cachePageInfo = pageInfo;
+    this._cacheSessionTotal += payloadSize;
+    this._cacheSessionNew += writtenSize;
   }
 
   receiveMessageFromTarget(params, sessions) {
@@ -965,7 +1015,7 @@ class Recorder {
   }
 
   async fullCommit(reqresp, sessions) {
-    const requestId = reqresp.requestId;
+    //const requestId = reqresp.requestId;
 
     // let doneResolve;
 

src/requestresponseinfo.js

@@ -4,6 +4,9 @@ import { getStatusText } from '@webrecorder/wabac/src/utils';
 
 import { postToGetUrl } from 'warcio';
 
+// max URL length for post/put payload-converted URLs
+const MAX_URL_LENGTH = 4096;
+
 
 // ===========================================================================
 class RequestResponseInfo
@@ -158,8 +161,8 @@ class RequestResponseInfo
       }
       if (postToGetUrl(convData)) {
         this.requestBody = convData.requestBody;
-        // truncate to first 2048 to avoid extra long URLs
-        this.url = convData.url.slice(0, 2048);
+        // truncate to avoid extra long URLs
+        this.url = convData.url.slice(0, MAX_URL_LENGTH);
       }
     }
 

wr-ext/bg.js

@@ -1,7 +1,7 @@
 {
   "name": "Webrecorder ArchiveWeb.page",
   "description": "Create high-fidelity web archives directly in your browser",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'",
   "permissions": [
     "debugger",