[pull] main from microsoft:main

packages/isomorphic/base64.ts

@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export function base64ByteLength(data: string): number {
+  if (!data)
+    return 0;
+  const padding = (data[data.length - 2] === '=') ? 2 : ((data[data.length - 1] === '=') ? 1 : 0);
+  return Math.max(0, Math.floor(data.length * 3 / 4) - padding);
+}

packages/isomorphic/index.ts

@@ -17,6 +17,7 @@
 export * from './ariaSnapshot';
 export * from './expectUtils';
 export * from './assert';
+export * from './base64';
 export * from './colors';
 export * from './headers';
 export * from './imageUtils';

packages/playwright-core/src/server/chromium/crNetworkManager.ts

@@ -54,6 +54,7 @@ export class CRNetworkManager {
   private _requestIdToRequestPausedEvent = new Map<string, { sessionInfo: SessionInfo, event: Protocol.Fetch.requestPausedPayload }>();
   private _responseExtraInfoTracker = new ResponseExtraInfoTracker();
   private _sessions = new Map<CRSession, SessionInfo>();
+  private _timestampBaselineForWebSocket = new Map<string, number>();
 
   constructor(page: Page | null, serviceWorker: CRServiceWorker | null) {
     this._page = page;
@@ -76,11 +77,11 @@ export class CRNetworkManager {
     if (this._page) {
       sessionInfo.eventListeners.push(...[
         eventsHelper.addEventListener(session, 'Network.webSocketCreated', e => this._page!.frameManager.onWebSocketCreated(e.requestId, e.url)),
-        eventsHelper.addEventListener(session, 'Network.webSocketWillSendHandshakeRequest', e => this._page!.frameManager.onWebSocketRequest(e.requestId, headersObjectToArray(e.request.headers, '\n'), e.wallTime * 1000, e.timestamp)),
+        eventsHelper.addEventListener(session, 'Network.webSocketWillSendHandshakeRequest', event => this._onWebSocketWillSendHandshakeRequest(event)),
         eventsHelper.addEventListener(session, 'Network.webSocketHandshakeResponseReceived', e => this._page!.frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText, headersObjectToArray(e.response.headers, '\n'))),
-        eventsHelper.addEventListener(session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page!.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
-        eventsHelper.addEventListener(session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page!.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData, e.timestamp)),
-        eventsHelper.addEventListener(session, 'Network.webSocketClosed', e => this._page!.frameManager.webSocketClosed(e.requestId)),
+        eventsHelper.addEventListener(session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page!.frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData, this._timestampToWallTimeMsForWebSocket(e.requestId, e.timestamp))),
+        eventsHelper.addEventListener(session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page!.frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData, this._timestampToWallTimeMsForWebSocket(e.requestId, e.timestamp))),
+        eventsHelper.addEventListener(session, 'Network.webSocketClosed', event => this._onWebSocketClosed(event)),
         eventsHelper.addEventListener(session, 'Network.webSocketFrameError', e => this._page!.frameManager.webSocketError(e.requestId, e.errorMessage)),
       ]);
     }
@@ -546,6 +547,21 @@ export class CRNetworkManager {
     (this._page?.frameManager || this._serviceWorker)!.requestFailed(request.request, !!event.canceled);
   }
 
+  _onWebSocketWillSendHandshakeRequest(event: Protocol.Network.webSocketWillSendHandshakeRequestPayload) {
+    const wallTimeMs = event.wallTime * 1000;
+    this._timestampBaselineForWebSocket.set(event.requestId, wallTimeMs - event.timestamp);
+    this._page!.frameManager.onWebSocketRequest(event.requestId, headersObjectToArray(event.request.headers, '\n'), wallTimeMs);
+  }
+
+  _onWebSocketClosed(event: Protocol.Network.webSocketClosedPayload) {
+    this._timestampBaselineForWebSocket.delete(event.requestId);
+    this._page!.frameManager.webSocketClosed(event.requestId);
+  }
+
+  _timestampToWallTimeMsForWebSocket(requestId: string, timestamp: number): number {
+    return this._timestampBaselineForWebSocket.get(requestId)! + timestamp;
+  }
+
   private _maybeUpdateRequestSession(sessionInfo: SessionInfo, request: InterceptableRequest) {
     // OOPIF has a main request that starts in the parent session but finishes in the child session.
     // We check for the main request by matching loaderId and requestId, and if it now belongs to

packages/playwright-core/src/server/frames.ts

@@ -403,12 +403,12 @@ export class FrameManager {
     this._webSockets.set(requestId, ws);
   }
 
-  onWebSocketRequest(requestId: string, headers: types.HeadersArray, wallTime?: number, timestamp?: number) {
+  onWebSocketRequest(requestId: string, headers: types.HeadersArray, wallTimeMs?: number) {
     const ws = this._webSockets.get(requestId);
     if (!ws)
       return;
 
-    ws.setRequestTiming(wallTime, timestamp);
+    ws.setWallTimeMs(wallTimeMs);
 
     if (ws.markAsNotified())
       this._page.emit(Page.Events.WebSocket, ws);
@@ -426,16 +426,16 @@ export class FrameManager {
       ws.error(`${statusText}: ${status}`);
   }
 
-  onWebSocketFrameSent(requestId: string, opcode: number, data: string, timestamp: number) {
+  onWebSocketFrameSent(requestId: string, opcode: number, data: string, wallTimeMs: number) {
     const ws = this._webSockets.get(requestId);
     if (ws)
-      ws.frameSent(opcode, data, timestamp);
+      ws.frameSent(opcode, data, wallTimeMs);
   }
 
-  webSocketFrameReceived(requestId: string, opcode: number, data: string, timestamp: number) {
+  webSocketFrameReceived(requestId: string, opcode: number, data: string, wallTimeMs: number) {
     const ws = this._webSockets.get(requestId);
     if (ws)
-      ws.frameReceived(opcode, data, timestamp);
+      ws.frameReceived(opcode, data, wallTimeMs);
   }
 
   webSocketClosed(requestId: string) {

packages/playwright-core/src/server/har/harTracer.ts

@@ -15,6 +15,7 @@
  */
 
 import mime from 'mime';
+import { base64ByteLength } from '@isomorphic/base64';
 import { ManualPromise } from '@isomorphic/manualPromise';
 import { eventsHelper } from '@utils/eventsHelper';
 import { assert } from '@isomorphic/assert';
@@ -440,25 +441,68 @@ export class HarTracer {
     if (!url)
       return;
 
+    const method = 'GET';
     const pageEntry = this._createPageEntryIfNeeded(page);
-    const harEntry = createHarEntry(pageEntry?.id, 'GET', url, page.mainFrame().guid, this._options, webSocket.wallTimeMs());
+    const harEntry = createHarEntry(pageEntry?.id, method, url, page.mainFrame().guid, this._options, webSocket.wallTimeMs());
     harEntry._resourceType = 'websocket';
     harEntry._webSocketMessages = [];
 
+    let oldestWallTimeMs = Infinity;
+    let newestWallTimeMs = -Infinity;
+    const updateTime = (wallTimeMs: number) => {
+      if (this._options.omitTiming)
+        return;
+
+      if (wallTimeMs >= oldestWallTimeMs && wallTimeMs <= newestWallTimeMs)
+        return;
+
+      if (wallTimeMs < oldestWallTimeMs)
+        oldestWallTimeMs = wallTimeMs;
+      if (wallTimeMs > newestWallTimeMs)
+        newestWallTimeMs = wallTimeMs;
+      if (oldestWallTimeMs === newestWallTimeMs)
+        return;
+
+      harEntry.time = newestWallTimeMs - oldestWallTimeMs;
+    };
+
     const eventListeners = [
       eventsHelper.addEventListener(webSocket, network.WebSocket.Events.Request, ({ headers }: { headers: HeadersArray }) => {
         this._recordRequestHeadersAndCookies(harEntry, headers);
+        if (!this._options.omitSizes)
+          harEntry.request.headersSize = network.requestHeadersSize(headers, webSocket.url(), method);
       }),
       eventsHelper.addEventListener(webSocket, network.WebSocket.Events.Response, ({ status, statusText, headers }: { status: number, statusText: string, headers: HeadersArray }) => {
         harEntry.response.status = status;
         harEntry.response.statusText = statusText;
         this._recordResponseHeaders(harEntry, headers);
+        if (!this._options.omitSizes) {
+          harEntry.response.headersSize = network.responseHeadersSize(headers, statusText);
+          harEntry.response._transferSize = Math.max(0, harEntry.response._transferSize!) + harEntry.response.headersSize;
+        }
       }),
-      eventsHelper.addEventListener(webSocket, network.WebSocket.Events.FrameSent, ({ opcode, data, timestamp }: { opcode: number, data: string, timestamp: number }) => {
-        harEntry._webSocketMessages!.push({ type: 'send', time: timestamp, opcode, data });
+      eventsHelper.addEventListener(webSocket, network.WebSocket.Events.FrameSent, ({ opcode, data, wallTimeMs }: { opcode: number, data: string, wallTimeMs: number }) => {
+        harEntry._webSocketMessages!.push({ type: 'send', time: this._options.omitTiming ? -1 : wallTimeMs, opcode, data });
+        updateTime(wallTimeMs);
       }),
-      eventsHelper.addEventListener(webSocket, network.WebSocket.Events.FrameReceived, ({ opcode, data, timestamp }: { opcode: number, data: string, timestamp: number }) => {
-        harEntry._webSocketMessages!.push({ type: 'receive', time: timestamp, opcode, data });
+      eventsHelper.addEventListener(webSocket, network.WebSocket.Events.FrameReceived, ({ opcode, data, wallTimeMs }: { opcode: number, data: string, wallTimeMs: number }) => {
+        harEntry._webSocketMessages!.push({ type: 'receive', time: this._options.omitTiming ? -1 : wallTimeMs, opcode, data });
+        updateTime(wallTimeMs);
+        if (!this._options.omitSizes) {
+          const length = (opcode === 1) ? Buffer.byteLength(data, 'utf8') : base64ByteLength(data);
+
+          // According to <https://www.rfc-editor.org/info/rfc6455/#section-5.2>:
+          // - there are always 16 bits at the beginning of every frame: FIN RSV1 RSV2 RSV3 opcode(4) mask length(7)
+          // - there are always 4 bytes for the masking key (see <https://www.rfc-editor.org/info/rfc6455/#section-5.1>)
+          // - there may be an additional 16 or 64 bits for payload length if it's too long to fit in the above 7 bits (or if it also can't fit in 16 bits)
+          let headerSize = 6;
+          if (length > 2 ** 16)
+            headerSize += 8;
+          else if (length > 125)
+            headerSize += 2;
+
+          harEntry.response._transferSize = Math.max(0, harEntry.response._transferSize!) + headerSize + length;
+        }
       }),
       eventsHelper.addEventListener(webSocket, network.WebSocket.Events.SocketError, (errorMessage: string) => {
         harEntry.response._failureText = errorMessage;

packages/playwright-core/src/server/network.ts

@@ -336,14 +336,7 @@ export class Request extends SdkObject {
   }
 
   async _requestHeadersSize(): Promise<number> {
-    let headersSize = 4; // 4 = 2 spaces + 2 line breaks (GET /path \r\n)
-    headersSize += this.method().length;
-    headersSize += (new URL(this.url())).pathname.length;
-    headersSize += 8; // httpVersion
-    const headers = await this._rawRequestHeaders();
-    for (const header of headers)
-      headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n'
-    return headersSize;
+    return requestHeadersSize(await this._rawRequestHeaders(), this.url(), this.method());
   }
 }
 
@@ -689,15 +682,7 @@ export class Response extends SdkObject {
       return availableSize;
 
     // Fallback to calculating it manually.
-    let headersSize = 4; // 4 = 2 spaces + 2 line breaks (HTTP/1.1 200 Ok\r\n)
-    headersSize += 8; // httpVersion;
-    headersSize += 3; // statusCode;
-    headersSize += this.statusText().length;
-    const headers = await this._rawResponseHeadersPromise;
-    for (const header of headers)
-      headersSize += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n'
-    headersSize += 2; // '\r\n'
-    return headersSize;
+    return responseHeadersSize(await this._rawResponseHeadersPromise, this.statusText());
   }
 
   private async _sizes(): Promise<ResourceSizes> {
@@ -731,8 +716,7 @@ export class Response extends SdkObject {
 export class WebSocket extends SdkObject {
   private _url: string;
   private _notified = false;
-  private _requestWallTimeMs: number | undefined;
-  private _requestTimestamp: number | undefined;
+  private _wallTimeMs: number | undefined;
   private _status: number | undefined;
   private _statusText: string | undefined;
   private _requestHeaders: HeadersArray | undefined;
@@ -749,7 +733,7 @@ export class WebSocket extends SdkObject {
 
   constructor(parent: SdkObject, url: string) {
     super(parent, 'ws');
-    this._url = url;
+    this._url = stripFragmentFromUrl(url);
   }
 
   markAsNotified() {
@@ -767,12 +751,11 @@ export class WebSocket extends SdkObject {
   }
 
   wallTimeMs(): number | undefined {
-    return this._requestWallTimeMs;
+    return this._wallTimeMs;
   }
 
-  setRequestTiming(wallTimeMs: number | undefined, timestamp: number | undefined) {
-    this._requestWallTimeMs = wallTimeMs;
-    this._requestTimestamp = timestamp;
+  setWallTimeMs(wallTimeMs: number | undefined) {
+    this._wallTimeMs = wallTimeMs;
   }
 
   requestSent(headers: HeadersArray) {
@@ -783,29 +766,12 @@ export class WebSocket extends SdkObject {
     this.emit(WebSocket.Events.Response, { status, statusText, headers });
   }
 
-  private _toWallTime(timestamp: number): number {
-    // The timestamp of each frame is relative to the timestamp (and walltime) of the initial request in Chromium and WebKit.
-    if (this._requestWallTimeMs !== undefined && this._requestTimestamp !== undefined)
-      return this._requestWallTimeMs + (timestamp - this._requestTimestamp);
-
-    // The timestamp is already a walltime in Firefox.
-    return timestamp;
+  frameSent(opcode: number, data: string, wallTimeMs: number) {
+    this.emit(WebSocket.Events.FrameSent, { opcode, data, wallTimeMs });
   }
 
-  frameSent(opcode: number, data: string, timestamp: number) {
-    this.emit(WebSocket.Events.FrameSent, {
-      opcode,
-      data,
-      timestamp: this._toWallTime(timestamp),
-    });
-  }
-
-  frameReceived(opcode: number, data: string, timestamp: number) {
-    this.emit(WebSocket.Events.FrameReceived, {
-      opcode,
-      data,
-      timestamp: this._toWallTime(timestamp),
-    });
+  frameReceived(opcode: number, data: string, wallTimeMs: number) {
+    this.emit(WebSocket.Events.FrameReceived, { opcode, data, wallTimeMs });
   }
 
   error(errorMessage: string) {
@@ -915,3 +881,29 @@ export function mergeHeaders(headers: (HeadersArray | undefined | null)[]): Head
     result.push({ name: lowerCaseToOriginalCase.get(lower)!, value });
   return result;
 }
+
+function headersSize(headers: HeadersArray): number {
+  let result = 0;
+  for (const header of headers)
+    result += header.name.length + header.value.length + 4; // 4 = ': ' + '\r\n'
+  return result;
+}
+
+export function requestHeadersSize(headers: HeadersArray, url: string, method: string): number {
+  let result = 4; // 4 = 2 spaces + 2 line breaks (GET /path \r\n)
+  result += method.length;
+  result += (new URL(url)).pathname.length;
+  result += 8; // httpVersion
+  result += headersSize(headers);
+  return result;
+}
+
+export function responseHeadersSize(headers: HeadersArray, statusText: string): number {
+  let result = 4; // 4 = 2 spaces + 2 line breaks (HTTP/1.1 200 Ok\r\n)
+  result += 8; // httpVersion;
+  result += 3; // statusCode;
+  result += statusText.length;
+  result += headersSize(headers);
+  result += 2; // '\r\n'
+  return result;
+}

packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts

@@ -160,9 +160,11 @@ class SocksProxyConnection {
     // the protocol on the first package and attach appropriate listeners.
     if (!this._firstPackageReceived) {
       this._firstPackageReceived = true;
-      // 0x16 is SSLv3/TLS "handshake" content type: https://en.wikipedia.org/wiki/Transport_Layer_Security#TLS_record
-      if (data[0] === 0x16)
-        this._establishTlsTunnel(this._browserEncrypted, data);
+      // 0x16 is the TLS "handshake" content type. Only intercept it when the origin has a client
+      // certificate; otherwise pass the connection through so the browser talks TLS to the server directly.
+      const secureContext = data[0] === 0x16 ? this.socksProxy.secureContextMap.get(normalizeOrigin(`https://${this.host}:${this.port}`)) : undefined;
+      if (secureContext)
+        this._establishTlsTunnel(this._browserEncrypted, data, secureContext);
       else
         this._establishPlaintextTunnel(this._browserEncrypted);
     }
@@ -176,14 +178,11 @@ class SocksProxyConnection {
     this._serverEncrypted.pipe(browserEncrypted);
   }
 
-  private _establishTlsTunnel(browserEncrypted: stream.Duplex, clientHello: Buffer) {
+  private _establishTlsTunnel(browserEncrypted: stream.Duplex, clientHello: Buffer, secureContext: tls.SecureContext) {
     const browserALPNProtocols = parseALPNFromClientHello(clientHello) || ['http/1.1'];
     debugLogger.log('client-certificates', `Browser->Proxy ${this.host}:${this.port} offers ALPN ${browserALPNProtocols}`);
 
-    const secureContext = this.socksProxy.secureContextMap.get(normalizeOrigin(`https://${this.host}:${this.port}`));
-    // Without a matching client certificate the proxy is a transparent pass-through, so let the
-    // browser validate the server cert instead of failing here on e.g. self-signed certs.
-    const rejectUnauthorized = !!secureContext && !this.socksProxy.ignoreHTTPSErrors;
+    const rejectUnauthorized = !this.socksProxy.ignoreHTTPSErrors;
 
     const serverDecrypted = tls.connect({
       socket: this._serverEncrypted,