behaviors: only waiting for pending network requests, not internal async fetches (to lower wait time)

behaviors: fix start/pause button in electron app
ui: fix page selection, fixes #25
ui: add message when finishing recording, wait for fetch requests but not more than 20 sec
ui: replay: fix for location bar not always loading entered URL (from replayweb.page)
ipfs: use trickle dag, ui default to replayweb.page link as gateway links not reliable for partial loading
bump to browsertrix-behaviors 0.2.0 - experimental facebook behavior
bump to latest electron (12.0.5), latest ruffle
bump to 0.6.2

Author: ikreymer

package.json

@@ -1,16 +1,16 @@
 {
   "name": "ArchiveWeb.page",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "main": "index.js",
   "description": "Create Web Archives directly in your browser",
   "repository": "https://github.com/webrecorder/archiveweb.page",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.13.0",
-    "@webrecorder/wabac": "github:webrecorder/wabac.js#multi-wacz",
-    "browsertrix-behaviors": "0.1.1",
-    "bulma": "^0.9.1",
+    "@webrecorder/wabac": "2.7.0-beta.2",
+    "browsertrix-behaviors": "^0.2.0",
+    "bulma": "^0.9.2",
     "flexsearch": "^0.6.32",
     "hash-wasm": "^4.4.1",
     "http-status-codes": "^1.4.0",
@@ -21,13 +21,13 @@
     "lodash": "^4.17.20",
     "node-fetch": "^2.6.1",
     "pretty-bytes": "^5.3.0",
-    "replaywebpage": "github:webrecorder/replayweb.page#1.4.0-work",
+    "replaywebpage": "^1.4.0-beta.0",
     "warcio": "^1.4.3"
   },
   "devDependencies": {
     "copy-webpack-plugin": "^6.4.0",
     "css-loader": "^3.5.3",
-    "electron": "12.0.2",
+    "electron": "12.0.5",
     "electron-builder": "^22.9.1",
     "electron-notarize": "^1.0.0",
     "generate-json-webpack-plugin": "1.0.0",

src/electron/electron-recorder-app.js

@@ -181,6 +181,10 @@ class ElectronRecorderApp extends ElectronReplayApp
         case "stopRecording":
           await recorder.detach();
           break;
+
+        case "toggleBehaviors":
+          await recorder.toggleBehaviors();
+          break;
       }
     });
 

src/popup.js

@@ -56,6 +56,7 @@ class RecPopup extends LitElement
     this.allowCreate = true;
 
     this.waitingForStart = false;
+    this.waitingForStop = false;
     this.behaviorState = BEHAVIOR_WAIT_LOAD;
     this.behaviorMsg = "";
     this.autorun = localStorage.getItem("autorunBehaviors") === "1";
@@ -87,7 +88,7 @@ class RecPopup extends LitElement
   }
 
   firstUpdated() {
-    document.addEventListener("click", (event) => {
+    document.addEventListener("click", () => {
       if (this.collDrop === "show") {
         this.collDrop = "";
       }
@@ -127,6 +128,9 @@ class RecPopup extends LitElement
         if (this.waitingForStart && message.firstPageStarted) {
           this.waitingForStart = false;
         }
+        if (this.waitingForStop && !message.recording && !message.stopping) {
+          this.waitingForStop = false;
+        }
         this.status = message;
         this.behaviorState = message.behaviorState;
         this.behaviorMsg = message.behaviorData && message.behaviorData.msg || "Starting...";
@@ -350,9 +354,9 @@ class RecPopup extends LitElement
       return html`<span class="status-autopilot">Auto Recording, ${this.behaviorMsg}</span>`;
     }
 
-    if (this.recording) {    
-      return html`<b>Recording:&nbsp;</b>${this.status && this.status.numPending ? html`
-      <span class="status-pending">${this.status.numPending} URLs pending, please wait before loading a new page.</span>
+    if (this.recording) {
+      return html`<b>${this.waitingForStop ? "Finishing " : ""} Recording:&nbsp;</b>${this.status && this.status.numPending ? html`
+      <span class="status-pending">${this.status.numPending} URLs pending${this.waitingForStop ? '.' : ', please wait before loading a new page.'}</span>
      ` :
      html`
       <span class="status-ready">Idle, Continue Browsing</span>`}`;
@@ -485,7 +489,7 @@ class RecPopup extends LitElement
           ${this.canRecord ? html`
           ${this.renderCollDropdown()}
           <button
-           ?disabled=${this.collDrop === "create" || (!this.recording && this.waitingForStart)}
+           ?disabled=${this.actionButtonDisabled}
            @click="${!this.recording ? this.onStart : this.onStop}" class="button">
             <span class="icon">
               ${!this.recording ? html`
@@ -542,6 +546,14 @@ class RecPopup extends LitElement
     `;
   }
 
+  get actionButtonDisabled() {
+    if (this.collDrop === "create") {
+      return true;
+    }
+
+    return !this.recording ? this.waitingForStart : this.waitingForStop;
+  }
+
   get behaviorsButtonLabel() {
     switch (this.behaviorState) {
       case BEHAVIOR_WAIT_LOAD:
@@ -578,6 +590,7 @@ class RecPopup extends LitElement
   onStop() {
     this.sendMessage({type: "stopRecording"});
     this.waitingForStart = false;
+    this.waitingForStop = true;
   }
 
   onSelectColl(event) {

src/recorder.js

@@ -36,6 +36,7 @@ class Recorder {
     this.numPending = 0;
 
     this.running = false;
+    this.stopping = false;
 
     this.frameId = null;
     this.pageInfo = {size: 0};
@@ -122,14 +123,19 @@ class Recorder {
   }
 
   async detach() {
+    this.stopping = true;
+
     const domNodes = await this.getFullText();
 
     if (this.behaviorState === BEHAVIOR_RUNNING) {
       this.toggleBehaviors();
     }
 
     try {
-      await Promise.all(this._fetchPending.values());
+      await Promise.race([
+        Promise.all(this._fetchPending.values()),
+        sleep(15000)
+      ]);
     } catch(e) {
       console.log(e);
     }
@@ -164,6 +170,7 @@ class Recorder {
     await this._doAttach();
 
     this.running = true;
+    this.stopping = false;
 
     this._updateId = setInterval(() => this.updateStatus(), 1000);
 
@@ -187,9 +194,10 @@ class Recorder {
   }
 
   updateStatus() {
-    this.numPending = Object.keys(this.pendingRequests).length + this._fetchPending.size;
+    const networkPending = Object.keys(this.pendingRequests).length;
+    this.numPending = networkPending + this._fetchPending.size;
 
-    if (this.numPending === 0 && this._loadedDoneResolve) {
+    if (networkPending === 0 && this._loadedDoneResolve) {
       this._loadedDoneResolve();
     }
 
@@ -212,6 +220,7 @@ class Recorder {
       pageTs: this.pageInfo.ts,
       failureMsg: this.failureMsg,
       collId: this.collId,
+      stopping: this.stopping,
       type: "status"
     }
   }
@@ -1091,7 +1100,7 @@ class Recorder {
   }
 
   async doBackgroundFetch() {
-    if (!this._fetchQueue.length || this._fetchPending.size >= MAX_CONCURRENT_FETCH) {
+    if (!this._fetchQueue.length || this._fetchPending.size >= MAX_CONCURRENT_FETCH || this.stopping) {
       return;
     }
 
@@ -1157,10 +1166,10 @@ class Recorder {
     } catch(e) {
       console.log(e);
       this._fetchUrls.delete(request.url);
+    } finally {
+      doneResolve();
+      this._fetchPending.delete(fetchId);
     }
-
-    doneResolve();
-    this._fetchPending.delete(fetchId);
   }
 
   async fetchPayloads(params, reqresp, sessions, method) {

src/ui.js

@@ -668,18 +668,12 @@ class WrRecCollInfo extends LitElement
                         </span>
                         Copy IPFS URL
                       </a>
-                      <a @click="${this.onCopyRWPLink}" class="dropdown-item">
+                      <a @click="${this.onCopyRWPLink}" class="has-text-weight-bold dropdown-item">
                         <span class="icon is-small">
                           <fa-icon size="0.8em" .svg="${fasShare}"></fa-icon>
                         </span>
                         Copy Shareable ReplayWeb.page Link
                       </a>
-                      <a @click="${this.onCopyGatewayLink}" class="has-text-weight-bold dropdown-item">
-                      <span class="icon is-small">
-                        <fa-icon size="0.8em" .svg="${fasShare}"></fa-icon>
-                      </span>
-                      Copy Shareable IPFS Gateway Link
-                    </a>
                     </div>
                   </div>
                 </div>

src/utils.js

@@ -87,7 +87,10 @@ async function checkPins(ipfsClient, validPins) {
 
 // ===========================================================================
 async function ipfsAddPin(ipfsClient, path, content) {
-  const resp = await ipfsClient.ipfs.add({path, content}, {wrapWithDirectory: true});
+  const resp = await ipfsClient.ipfs.add({path, content}, {
+    wrapWithDirectory: true,
+    trickle: true
+  });
 
   const hash = resp.cid.toString();
 
@@ -133,7 +136,10 @@ async function ipfsAddWithReplay(ipfsClient, waczPath, waczContent, swContent, u
 
   fileData.push({path: waczPath, content: waczContent});
 
-  const resp = await ipfsClient.ipfs.addAll(fileData, {wrapWithDirectory: true});
+  const resp = await ipfsClient.ipfs.addAll(fileData, {
+    wrapWithDirectory: true,
+    trickle: true
+  });
 
   let hash;
   let size = 0;

wr-ext/bg.js

@@ -1,7 +1,7 @@
 {
   "name": "Webrecorder ArchiveWeb.page",
   "description": "Create high-fidelity web archives directly in your browser",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'",
   "permissions": [
     "debugger",