diff --git a/.github/workflows/tests_components.yml b/.github/workflows/tests_components.yml index 585796c280070..ec611443bb3e3 100644 --- a/.github/workflows/tests_components.yml +++ b/.github/workflows/tests_components.yml @@ -6,15 +6,26 @@ on: - main - release-* pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - - 'packages/extension/**' - - 'packages/playwright-core/src/server/bidi/**' - - 'packages/playwright-core/src/tools/**' - - 'tests/bidi/**' - - 'tests/extension/**' - - 'tests/mcp/**' + paths: + - 'packages/playwright/**' + - 'packages/playwright-core/**' + - '!packages/playwright-core/src/server/bidi/**' + - '!packages/playwright-core/src/tools/**' + - 'packages/playwright-test/**' + - 'packages/playwright-ct-core/**' + - 'packages/playwright-ct-react/**' + - 'packages/playwright-ct-react17/**' + - 'packages/playwright-ct-solid/**' + - 'packages/playwright-ct-vue/**' + - 'packages/injected/**' + - 'packages/isomorphic/**' + - 'packages/protocol/**' + - 'packages/utils/**' + - 'tests/components/**' + - 'utils/**' + - 'package.json' + - 'package-lock.json' + - '.github/workflows/tests_components.yml' branches: - main - release-* diff --git a/.github/workflows/tests_mcp.yml b/.github/workflows/tests_mcp.yml index 1ea51a00138fd..1ecb4072f5178 100644 --- a/.github/workflows/tests_mcp.yml +++ b/.github/workflows/tests_mcp.yml @@ -6,11 +6,24 @@ on: - main - release-* pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - - 'packages/playwright-core/src/server/bidi/**' - - 'tests/bidi/**' + paths: + - 'packages/playwright/**' + - 'packages/playwright-core/**' + - '!packages/playwright-core/src/server/bidi/**' + - '!packages/playwright-core/src/tools/dashboard/**' + - '!packages/playwright-core/src/tools/trace/**' + - 'packages/playwright-test/**' + - 'packages/injected/**' + - 'packages/isomorphic/**' + - 'packages/protocol/**' + - 'packages/utils/**' + - 'tests/config/**' + - 'tests/mcp/**' + - 'utils/**' + - 'package.json' + - 'package-lock.json' + - '.github/actions/**' + - '.github/workflows/tests_mcp.yml' branches: - main - release-* diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index 46ee8c283ec18..9513f3e599c47 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -6,15 +6,20 @@ on: - main - release-* pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - - 'packages/extension/**' - - 'packages/playwright-core/src/server/bidi/**' - - 'packages/playwright-core/src/tools/**' - - 'tests/bidi/**' - - 'tests/extension/**' - - 'tests/mcp/**' + paths: + - 'packages/**' + - '!packages/extension/**' + - '!packages/playwright-core/src/server/bidi/**' + - '!packages/playwright-core/src/tools/**' + - 'tests/**' + - '!tests/bidi/**' + - '!tests/extension/**' + - '!tests/mcp/**' + - 'utils/**' + - 'package.json' + - 'package-lock.json' + - '.github/actions/**' + - '.github/workflows/tests_primary.yml' branches: - main - release-* diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index b53f852d15730..3396d37ddc0c8 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -6,13 +6,6 @@ on: - main - release-* pull_request: - paths-ignore: - - 'browser_patches/**' - - 'docs/**' - - 'packages/extension/**' - - 'packages/playwright-core/src/server/bidi/**' - - 'tests/bidi/**' - - 'tests/extension/**' types: [ labeled ] branches: - main diff --git a/docs/src/api/class-locator.md b/docs/src/api/class-locator.md index ad37165471462..5d8a763480206 100644 --- a/docs/src/api/class-locator.md +++ b/docs/src/api/class-locator.md @@ -244,6 +244,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption. Defaults to When specified, limits the depth of the snapshot. +### option: Locator.ariaSnapshot.boxes +* since: v1.60 +- `boxes` <[boolean]> + +When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + ## async method: Locator.blur * since: v1.28 diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 47e5626b725d8..b20c766651a90 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -4253,6 +4253,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption: including e When specified, limits the depth of the snapshot. +### option: Page.ariaSnapshot.boxes +* since: v1.60 +- `boxes` <[boolean]> + +When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + ## async method: Page.tap * since: v1.8 * discouraged: Use locator-based [`method: Locator.tap`] instead. Read more about [locators](../locators.md). diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsx index 1613205d19827..40c6ec572f9f0 100644 --- a/packages/dashboard/src/dashboard.tsx +++ b/packages/dashboard/src/dashboard.tsx @@ -16,7 +16,6 @@ import React from 'react'; import './dashboard.css'; -import { DashboardClientContext } from './dashboardContext'; import { ChevronLeftIcon, ChevronRightIcon, DownloadIcon, LockIcon, LockOpenIcon, ReloadIcon, ScreenshotRegionIcon } from './icons'; import { Annotations, getImageLayout, clientToViewport } from './annotations'; @@ -24,184 +23,9 @@ import type { Annotation, AnnotationsHandle } from './annotations'; import { ToolbarButton, ToolbarSeparator } from '@web/components/toolbarButton'; import { useMeasureForRef } from '@web/uiUtils'; -import type { Tab, DashboardChannelEvents } from './dashboardChannel'; +import type { DashboardModel } from './dashboardModel'; const BUTTONS = ['left', 'middle', 'right'] as const; -type Mode = 'readonly' | 'interactive' | 'annotate'; - -// Recording mode is a separate, mode-replacing overlay similar to annotate. -// While null, the normal toolbar is shown. When non-null, the recording -// toolbar replaces it. -type RecordingState = - | { phase: 'recording' } - | { phase: 'stopped'; blob: Blob; url: string }; - -type DashboardState = { - // Server-driven session state. - tabs: Tab[] | null; - url: string; - // The latest frame received from the server. Cleared on page swap; - // the server is responsible for emitting a fresh frame for the new - // page (see _startScreencast). - liveFrame: DashboardChannelEvents['frame'] | undefined; - // Snapshot of the frame at the moment we entered annotate mode. The - // overlay draws on this frozen image so the canvas does - // not jump around as new live frames arrive. - annotateFrame: DashboardChannelEvents['frame'] | undefined; - // Server requested annotate but we have no fresh frame for the active - // page. The next FRAME action will enter annotate mode. - cliAnnotatePending: boolean; - // Whether the current annotate session was initiated by CLI or by the - // user. Determines whether submit goes to the server or saves locally. - annotateInitiator: 'cli' | 'user' | null; - // Interaction mode and ephemeral UI flags. - mode: Mode; - recording: RecordingState | null; -}; - -type DashboardAction = - // Server events - | { type: 'tabs'; tabs: Tab[] } - | { type: 'frame'; frame: DashboardChannelEvents['frame'] } - | { type: 'cliAnnotate' } - | { type: 'cliCancelAnnotate' } - // User events - | { type: 'toggleInteractive' } - | { type: 'toggleAnnotate' } - | { type: 'startRecording' } - | { type: 'stoppedRecording'; blob: Blob; url: string } - | { type: 'exitRecording' } - | { type: 'submitAnnotation' } - | { type: 'setUrl'; url: string }; - -const initialDashboardState: DashboardState = { - tabs: null, - url: '', - liveFrame: undefined, - annotateFrame: undefined, - cliAnnotatePending: false, - annotateInitiator: null, - mode: 'readonly', - recording: null, -}; - -function dashboardReducer(state: DashboardState, action: DashboardAction): DashboardState { - switch (action.type) { - case 'tabs': { - const newSelected = action.tabs.find(t => t.selected); - const oldSelected = state.tabs?.find(t => t.selected); - const url = newSelected ? newSelected.url : state.url; - // Page change = had a selected tab, now have a different selected - // tab. Initial selection (none -> some) is not a "change". - const pageChanged = !!oldSelected && !!newSelected && newSelected.page !== oldSelected.page; - if (!pageChanged) - return { ...state, tabs: action.tabs, url }; - // Page swap. Clear frames and rely on the server to emit a fresh - // frame for the new page (see _startScreencast which awaits a - // screenshot to force one). If we were annotating, mark pending so - // the next FRAME re-enters annotate with the new page's image. - const wasAnnotateActive = state.mode === 'annotate' || state.cliAnnotatePending; - const newTabIsBrandNew = !!state.tabs && !state.tabs.some(t => t.page === newSelected.page); - let mode: Mode = state.mode; - if (!wasAnnotateActive && newTabIsBrandNew && state.tabs) - mode = 'interactive'; - return { - ...state, - tabs: action.tabs, - url, - mode, - recording: null, - liveFrame: undefined, - annotateFrame: undefined, - cliAnnotatePending: wasAnnotateActive, - }; - } - case 'frame': { - const liveFrame = action.frame; - if (state.cliAnnotatePending) { - return { - ...state, - liveFrame, - annotateFrame: liveFrame, - mode: 'annotate', - cliAnnotatePending: false, - annotateInitiator: state.annotateInitiator ?? 'cli', - }; - } - return { ...state, liveFrame }; - } - case 'cliAnnotate': { - if (state.mode === 'annotate') { - // Already annotating (user-initiated). Mark CLI as the source so - // submit goes to the server. - return { ...state, annotateInitiator: 'cli' }; - } - if (state.liveFrame) { - return { - ...state, - mode: 'annotate', - annotateFrame: state.liveFrame, - cliAnnotatePending: false, - annotateInitiator: 'cli', - }; - } - return { ...state, cliAnnotatePending: true, annotateInitiator: 'cli' }; - } - case 'cliCancelAnnotate': { - const exitingAnnotate = state.mode === 'annotate'; - return { - ...state, - mode: exitingAnnotate ? 'readonly' : state.mode, - annotateFrame: undefined, - cliAnnotatePending: false, - annotateInitiator: null, - }; - } - case 'toggleInteractive': { - const next: Mode = state.mode === 'interactive' ? 'readonly' : 'interactive'; - return { ...state, mode: next }; - } - case 'toggleAnnotate': { - if (state.mode === 'annotate') { - return { - ...state, - mode: 'readonly', - annotateFrame: undefined, - annotateInitiator: null, - }; - } - if (!state.liveFrame) - return state; - // Preserve a CLI initiator across mode toggles so that CLI annotate - // sessions stay engaged when the user switches to interactive and - // back. Only set 'user' if there was no prior CLI engagement. - const initiator: 'cli' | 'user' | null = state.annotateInitiator ?? 'user'; - return { - ...state, - mode: 'annotate', - annotateFrame: state.liveFrame, - cliAnnotatePending: false, - annotateInitiator: initiator, - }; - } - case 'startRecording': - return { ...state, recording: { phase: 'recording' } }; - case 'stoppedRecording': - return { ...state, recording: { phase: 'stopped', blob: action.blob, url: action.url } }; - case 'exitRecording': - return { ...state, recording: null }; - case 'submitAnnotation': - return { - ...state, - mode: 'readonly', - annotateFrame: undefined, - cliAnnotatePending: false, - annotateInitiator: null, - }; - case 'setUrl': - return { ...state, url: action.url }; - } -} async function pickSaveWritable(suggestedName: string, description: string, mime: string, extension: string): Promise { try { @@ -215,10 +39,6 @@ async function pickSaveWritable(suggestedName: string, description: string, mime } } -function base64ToBlob(base64: string, mime: string): Blob { - return new Blob([(Uint8Array as any).fromBase64(base64)], { type: mime }); -} - function smartUrl(input: string): string { const value = input.trim(); if (!value) @@ -237,10 +57,15 @@ function smartUrl(input: string): string { return 'https://' + host + '.com' + value.slice(host.length); } -export const Dashboard: React.FC = () => { - const client = React.useContext(DashboardClientContext); - const [state, dispatch] = React.useReducer(dashboardReducer, initialDashboardState); - const { tabs, url, mode, recording, liveFrame, annotateFrame, annotateInitiator } = state; +type DashboardProps = { + model: DashboardModel; +}; + +export const Dashboard: React.FC = ({ model }) => { + const [, setRevision] = React.useState(0); + React.useEffect(() => model.subscribe(() => setRevision(r => r + 1)), [model]); + + const { tabs, mode, recording, liveFrame, annotateFrame, annotateInitiator, pendingAnnotate } = model.state; const interactive = mode === 'interactive'; const annotating = mode === 'annotate'; @@ -309,8 +134,6 @@ export const Dashboard: React.FC = () => { }, [interactive]); const onSubmitAnnotations = React.useCallback(async (blob: Blob, annotations: Annotation[]) => { - if (!client) - return; const dataUrl = await new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(reader.result as string); @@ -318,103 +141,25 @@ export const Dashboard: React.FC = () => { reader.readAsDataURL(blob); }); const data = dataUrl.slice(dataUrl.indexOf(',') + 1); - await client.submitAnnotation({ - data, - annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })), - }); - dispatch({ type: 'submitAnnotation' }); - }, [client]); + await model.submitAnnotation(data, annotations); + }, [model]); function flashInteractiveHint() { setFlashTick(tick => tick + 1); } - const onStartRecording = React.useCallback(async () => { - if (!client) - return; - await client.startRecording(); - dispatch({ type: 'startRecording' }); - }, [client]); - - const onStopRecording = React.useCallback(async () => { - if (!client) - return; - const { streamId } = await client.stopRecording(); - const chunks: Blob[] = []; - while (true) { - const { data, eof } = await client.readStream({ streamId }); - if (data) - chunks.push(base64ToBlob(data, 'video/webm')); - if (eof) - break; - } - const blob = new Blob(chunks, { type: 'video/webm' }); - const url = URL.createObjectURL(blob); - dispatch({ type: 'stoppedRecording', blob, url }); - }, [client]); - const onSaveRecording = React.useCallback(async () => { - if (state.recording?.phase !== 'stopped') + if (recording?.phase !== 'stopped') return; const writable = await pickSaveWritable(`playwright-recording-${Date.now()}.webm`, 'WebM Video', 'video/webm', '.webm'); if (!writable) return; - await writable.write(state.recording.blob); + await writable.write(recording.blob); await writable.close(); - }, [state.recording]); - - const onExitRecording = React.useCallback(async () => { - if (state.recording?.phase === 'recording' && client) { - // Drain the stream without persisting; keeps the server clean. - const { streamId } = await client.stopRecording(); - while (true) { - const { eof } = await client.readStream({ streamId }); - if (eof) - break; - } - } - if (state.recording?.phase === 'stopped') - URL.revokeObjectURL(state.recording.url); - dispatch({ type: 'exitRecording' }); - }, [state.recording, client]); - - - React.useEffect(() => { - if (!client) - return; - let resized = false; - const onTabs = (params: DashboardChannelEvents['tabs']) => { - dispatch({ type: 'tabs', tabs: params.tabs }); - }; - const onFrame = (params: DashboardChannelEvents['frame']) => { - dispatch({ type: 'frame', frame: params }); - const toolbar = toolbarRef.current; - if (!resized && toolbar && params.viewportWidth && params.viewportHeight) { - resized = true; - const chromeHeight = toolbar.offsetHeight; - const extraW = window.outerWidth - window.innerWidth; - const extraH = window.outerHeight - window.innerHeight; - const targetW = Math.min(params.viewportWidth + extraW, screen.availWidth); - const targetH = Math.min(params.viewportHeight + chromeHeight + extraH, screen.availHeight); - window.resizeTo(targetW, targetH); - } - }; - const onAnnotate = () => dispatch({ type: 'cliAnnotate' }); - const onCancelAnnotate = () => dispatch({ type: 'cliCancelAnnotate' }); - client.on('tabs', onTabs); - client.on('frame', onFrame); - client.on('annotate', onAnnotate); - client.on('cancelAnnotate', onCancelAnnotate); - return () => { - client.off('tabs', onTabs); - client.off('frame', onFrame); - client.off('annotate', onAnnotate); - client.off('cancelAnnotate', onCancelAnnotate); - }; - }, [client]); + }, [recording]); const selectedTab = tabs?.find(t => t.selected); - const ready = !!client && !!selectedTab; + const ready = !!selectedTab; function imgCoords(e: React.MouseEvent): { x: number; y: number } { const vw = liveFrame?.viewportWidth ?? 0; @@ -428,10 +173,8 @@ export const Dashboard: React.FC = () => { } function sendMouseEvent(method: 'mousedown' | 'mouseup', e: React.MouseEvent) { - if (!client) - return; const { x, y } = imgCoords(e); - client[method]({ x, y, button: BUTTONS[e.button] || 'left' }); + model[method](x, y, BUTTONS[e.button] || 'left'); } function onScreenMouseDown(e: React.MouseEvent) { @@ -456,53 +199,46 @@ export const Dashboard: React.FC = () => { } function onScreenMouseMove(e: React.MouseEvent) { - if (annotating || !interactive || !client) + if (annotating || !interactive) return; const now = Date.now(); if (now - moveThrottleRef.current < 32) return; moveThrottleRef.current = now; const { x, y } = imgCoords(e); - client.mousemove({ x, y }); + model.mousemove(x, y); } function onScreenWheel(e: React.WheelEvent) { - if (annotating || !interactive || !client) + if (annotating || !interactive) return; e.preventDefault(); - client.wheel({ deltaX: e.deltaX, deltaY: e.deltaY }); + model.wheel(e.deltaX, e.deltaY); } function onScreenKeyDown(e: React.KeyboardEvent) { - if (annotating) - return; - if (!interactive || !client) + if (annotating || !interactive) return; e.preventDefault(); - client.keydown({ key: e.key }); + model.keydown(e.key); } function onScreenKeyUp(e: React.KeyboardEvent) { - if (annotating || !interactive || !client) + if (annotating || !interactive) return; e.preventDefault(); - client.keyup({ key: e.key }); + model.keyup(e.key); } function onOmniboxKeyDown(e: React.KeyboardEvent) { if (e.key === 'Enter') { const value = smartUrl((e.target as HTMLInputElement).value); - dispatch({ type: 'setUrl', url: value }); - client?.navigate({ url: value }); + model.navigate(value); e.currentTarget.blur(); } } - let overlayText: string | undefined; - if (!client) - overlayText = 'Disconnected'; - else if (!selectedTab) - overlayText = 'Select a session'; + const overlayText = selectedTab ? undefined : 'Select a session'; return (
@@ -541,7 +277,7 @@ export const Dashboard: React.FC = () => { className='annotate-toolbar-btn' title='Close annotation mode' icon='close' - onClick={() => dispatch({ type: 'toggleAnnotate' })} + onClick={() => model.completeAnnotation()} />
@@ -553,12 +289,12 @@ export const Dashboard: React.FC = () => { icon='record' toggled={recording.phase === 'recording'} style={{ color: recording.phase === 'recording' ? 'var(--color-scale-red-5)' : undefined }} - onClick={async () => { + onClick={() => { if (recording.phase === 'recording') { - await onStopRecording(); + model.stopRecording(); } else { - URL.revokeObjectURL(recording.url); - await onStartRecording(); + URL.revokeObjectURL(recording.blobUrl); + model.startRecording(); } }}> {recording.phase === 'recording' && Recording...} @@ -574,7 +310,7 @@ export const Dashboard: React.FC = () => { model.discardRecording()} /> @@ -586,9 +322,7 @@ export const Dashboard: React.FC = () => { title={interactive ? 'Disable interactive mode' : 'Enable interactive mode'} toggled={interactive} disabled={!ready} - onClick={() => { - dispatch({ type: 'toggleInteractive' }); - }} + onClick={() => model.toggleInteractive()} > {interactive ? : } @@ -596,9 +330,12 @@ export const Dashboard: React.FC = () => { { - dispatch({ type: 'toggleAnnotate' }); + if (pendingAnnotate) + model.cancelAnnotate(); + else + model.enterAnnotate('user'); }} > @@ -607,7 +344,7 @@ export const Dashboard: React.FC = () => { title='Record video' icon='record' disabled={!ready} - onClick={onStartRecording} + onClick={() => model.startRecording()} /> )} @@ -623,7 +360,7 @@ export const Dashboard: React.FC = () => { id='display' className='annotate-image' alt='annotation' - src={'data:image/jpeg;base64,' + annotateFrame.data} + src={'data:image/png;base64,' + annotateFrame.data} /> {
- {model.loading &&
Loading sessions...
} - {!model.loading && openSessions.length === 0 &&
No open sessions.
} + {loadingSessions &&
Loading sessions...
} + {!loadingSessions && openSessions.length === 0 &&
No open sessions.
} {workspaceGroups.map(([workspace, entries]) => { const workspacePath = normalizeWorkspacePath(workspace, clientInfo?.homeDir); return
@@ -149,7 +135,7 @@ export const SessionSidebar: React.FC = ({ model, onSelectT className='session-browser-close' icon='close' title='Close session' - onClick={() => void model.closeSession(session)} + onClick={() => model.closeSession(session)} />
{session.title} @@ -158,7 +144,7 @@ export const SessionSidebar: React.FC = ({ model, onSelectT className='sidebar-session-new-tab' icon='add' title='New tab' - onClick={() => onNewTab(guid, row.contextGuid!)} + onClick={() => model.newTab(guid, row.contextGuid!)} />} @@ -176,11 +162,11 @@ export const SessionSidebar: React.FC = ({ model, onSelectT tabIndex={0} aria-current={tab.context === activeContext && tab.selected ? 'page' : undefined} title={tab.url || tab.title} - onClick={() => onSelectTab(tab)} + onClick={() => model.selectTab(tab)} onKeyDown={e => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); - onSelectTab(tab); + model.selectTab(tab); } }} > @@ -198,7 +184,7 @@ export const SessionSidebar: React.FC = ({ model, onSelectT title='Close tab' onClick={e => { e.stopPropagation(); - onCloseTab(tab); + model.closeTab(tab); }} /> )} diff --git a/packages/extension/src/connectedTabGroup.ts b/packages/extension/src/connectedTabGroup.ts index 9806116c426a0..90ed6973deaee 100644 --- a/packages/extension/src/connectedTabGroup.ts +++ b/packages/extension/src/connectedTabGroup.ts @@ -50,9 +50,6 @@ export class ConnectedTabGroup { private _connection: RelayConnection; private _groupId: number | null = null; private _groupTabIds: Set = new Set(); - // Subset of `_groupTabIds` the debugger is actually attached to; drives the - // badge. A chrome:// tab can sit in the group without being attached. - private _attachedTabIds: Set = new Set(); private _onTabUpdatedListener: (tabId: number, changeInfo: chrome.tabs.TabChangeInfo, tab: chrome.tabs.Tab) => void; private _onTabRemovedListener: (tabId: number) => void; @@ -82,43 +79,41 @@ export class ConnectedTabGroup { private _onTabUpdated(tabId: number, changeInfo: chrome.tabs.TabChangeInfo, tab: chrome.tabs.Tab): void { if (changeInfo.groupId !== undefined) - this._onTabGroupChanged(tabId, changeInfo.groupId, tab.url); + this._onTabGroupChanged(tabId, tab); if (changeInfo.url === undefined) return; // Chrome resets per-tab badge state on navigation, so re-apply it. - if (this._attachedTabIds.has(tabId)) + if (this._connection.attachedTabs.has(tabId)) void this._updateBadge(tabId, CONNECTED_BADGE); else if (this._groupTabIds.has(tabId) && !isNonDebuggableUrl(changeInfo.url)) - void this._connection.attachTab(tabId); + this._connection.attachTab(tab); } // Single entry point for group membership changes, whether the user dragged // or we grouped the tab ourselves. Attaches on entry (if debuggable) and // detaches on exit; a chrome:// tab stays in the group until it navigates // (handled in _onTabUpdated). - private _onTabGroupChanged(tabId: number, newGroupId: number, url: string | undefined): void { - const inOurGroup = this._groupId !== null && newGroupId === this._groupId; + private _onTabGroupChanged(tabId: number, tab: chrome.tabs.Tab): void { + const inOurGroup = this._groupId !== null && tab.groupId === this._groupId; const wasInGroup = this._groupTabIds.has(tabId); if (inOurGroup === wasInGroup) return; if (inOurGroup) { this._groupTabIds.add(tabId); - if (!isNonDebuggableUrl(url)) - void this._connection.attachTab(tabId); + if (!isNonDebuggableUrl(tab.url)) + this._connection.attachTab(tab); } else { this._groupTabIds.delete(tabId); - if (this._attachedTabIds.has(tabId)) - void this._connection.detachTab(tabId); + if (this._connection.attachedTabs.has(tabId)) + this._connection.detachTab(tabId); } } private _onTabRemoved(tabId: number): void { this._groupTabIds.delete(tabId); - this._attachedTabIds.delete(tabId); } private _onTabAttached(tabId: number): void { - this._attachedTabIds.add(tabId); void this._updateBadge(tabId, CONNECTED_BADGE); void this._addTabToGroup(tabId); } @@ -127,18 +122,14 @@ export class ConnectedTabGroup { // badge but leave the tab in the group — the user's intent is still there, // and a subsequent navigation will re-attach via _onTabUpdated. private _onTabDetached(tabId: number): void { - this._attachedTabIds.delete(tabId); void this._updateBadge(tabId, { text: '' }); } private _onConnectionClose(): void { chrome.tabs.onUpdated.removeListener(this._onTabUpdatedListener); chrome.tabs.onRemoved.removeListener(this._onTabRemovedListener); - const attachedIds = [...this._attachedTabIds]; const groupTabs = [...this._groupTabIds]; - this._attachedTabIds.clear(); this._groupTabIds.clear(); - attachedIds.forEach(id => void this._updateBadge(id, { text: '' })); if (groupTabs.length) { this._retryOnDrag(() => chrome.tabs.ungroup(groupTabs)).catch(error => { debugLog('Error ungrouping tabs on close:', error); diff --git a/packages/extension/src/protocolHandlers.ts b/packages/extension/src/protocolHandlers.ts index a533b0216375b..b68c30f966d74 100644 --- a/packages/extension/src/protocolHandlers.ts +++ b/packages/extension/src/protocolHandlers.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import { debugLog } from './relayConnection'; - export type ProtocolCommand = { id: number; method: string; @@ -42,7 +40,7 @@ export interface ProtocolHandler { forwardChromeEvent(fullMethod: string, args: any[]): void; // The UI added a tab to the Playwright group. Handler tells the relay the // tab is now available; the relay attaches via the usual command path. - onUserAttachRequest(tabId: number): Promise; + onUserAttachRequest(tab: chrome.tabs.Tab): void; // The UI removed a tab. RelayConnection has already detached the debugger // and called notifyTabDetached; the handler only sends the wire-level // detach notification (if the protocol has one). @@ -96,7 +94,7 @@ export class ProtocolV1Handler implements ProtocolHandler { }); } - async onUserAttachRequest(_tabId: number): Promise { + onUserAttachRequest(_tab: chrome.tabs.Tab): void { // v1 is single-tab by design; dragging extra tabs into the group is a no-op. } @@ -148,15 +146,10 @@ export class ProtocolV2Handler implements ProtocolHandler { this._context.sendMessage({ method: fullMethod, params: args }); } - async onUserAttachRequest(tabId: number): Promise { + onUserAttachRequest(tab: chrome.tabs.Tab): void { // Simulate a "new tab opened" event; the relay responds by calling // chrome.debugger.attach, which flows through handleCommand. - try { - const tab = await chrome.tabs.get(tabId); - this._context.sendMessage({ method: 'chrome.tabs.onCreated', params: [tab] }); - } catch (error: any) { - debugLog('Error requesting attach for tab:', error); - } + this._context.sendMessage({ method: 'chrome.tabs.onCreated', params: [tab] }); } onUserDetachRequest(tabId: number): void { diff --git a/packages/extension/src/relayConnection.ts b/packages/extension/src/relayConnection.ts index 4988731733e91..ec04faa3fdb6d 100644 --- a/packages/extension/src/relayConnection.ts +++ b/packages/extension/src/relayConnection.ts @@ -59,6 +59,10 @@ export class RelayConnection { ontabattached?: (tabId: number) => void; ontabdetached?: (tabId: number) => void; + get attachedTabs(): ReadonlySet { + return this._attachedTabs; + } + constructor(ws: WebSocket, protocolVersion: number) { this._ws = ws; this._selectedTabPromise = new Promise(resolve => this._selectedTabResolve = resolve); @@ -91,23 +95,21 @@ export class RelayConnection { // Called when the UI adds a tab to the Playwright group. The handler asks // the relay to attach; the normal command path fires ontabattached. - async attachTab(tabId: number): Promise { - if (this._closed || this._attachedTabs.has(tabId)) + attachTab(tab: chrome.tabs.Tab): void { + if (this._closed || this._attachedTabs.has(tab.id!)) return; - await this._handler.onUserAttachRequest(tabId); + this._handler.onUserAttachRequest(tab); } // Called when the UI removes a tab from the Playwright group. We detach the // debugger and update bookkeeping; the handler emits the wire-level detach // notification for protocols that have one. - async detachTab(tabId: number): Promise { + detachTab(tabId: number): void { if (this._closed || !this._attachedTabs.has(tabId)) return; - try { - await chrome.debugger.detach({ tabId }); - } catch (error: any) { + chrome.debugger.detach({ tabId }).catch(error => { debugLog('Error detaching tab:', error); - } + }); this._notifyTabDetached(tabId); this._handler.onUserDetachRequest(tabId); this._checkLastTabDetached(); @@ -142,9 +144,10 @@ export class RelayConnection { for (const l of this._eventListeners) l.remove(); this._eventListeners = []; - for (const tabId of this._attachedTabs) + for (const tabId of [...this._attachedTabs]) { chrome.debugger.detach({ tabId }).catch(() => {}); - this._attachedTabs.clear(); + this._notifyTabDetached(tabId); + } this.onclose?.(); } diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts index 08e23d585ef6a..17b438dc13616 100644 --- a/packages/injected/src/ariaSnapshot.ts +++ b/packages/injected/src/ariaSnapshot.ts @@ -41,6 +41,7 @@ export type AriaTreeOptions = { refPrefix?: string; doNotRenderActive?: boolean; depth?: number; + boxes?: boolean; }; type InternalOptions = { @@ -51,9 +52,11 @@ type InternalOptions = { renderCursorPointer?: boolean, renderActive?: boolean, renderStringsAsRegex?: boolean, + renderBoxes?: boolean, }; function toInternalOptions(options: AriaTreeOptions): InternalOptions { + const renderBoxes = options.boxes; if (options.mode === 'ai') { // For AI consumption. return { @@ -63,18 +66,19 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions { includeGenericRole: true, renderActive: !options.doNotRenderActive, renderCursorPointer: true, + renderBoxes, }; } if (options.mode === 'autoexpect') { // To auto-generate assertions on visible elements. - return { visibility: 'ariaAndVisible', refs: 'none' }; + return { visibility: 'ariaAndVisible', refs: 'none', renderBoxes }; } if (options.mode === 'codegen') { // To generate aria assertion with regex heurisitcs. - return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true }; + return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true, renderBoxes }; } // To match aria snapshot. - return { visibility: 'aria', refs: 'none' }; + return { visibility: 'aria', refs: 'none', renderBoxes }; } export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOptions): AriaSnapshot { @@ -624,6 +628,13 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr if (renderCursorPointer && aria.hasPointerCursor(ariaNode)) key += ' [cursor=pointer]'; } + if (options.renderBoxes) { + const element = ariaNodeElement(ariaNode); + if (element) { + const r = element.getBoundingClientRect(); + key += ` [box=${Math.round(r.x)},${Math.round(r.y)},${Math.round(r.width)},${Math.round(r.height)}]`; + } + } return key; }; diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 30e475fcdea33..452fbe353a4cf 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -2051,6 +2051,11 @@ export interface Page { * @param options */ ariaSnapshot(options?: { + /** + * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + */ + boxes?: boolean; + /** * When specified, limits the depth of the snapshot. */ @@ -13057,6 +13062,11 @@ export interface Locator { * @param options */ ariaSnapshot(options?: { + /** + * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + */ + boxes?: boolean; + /** * When specified, limits the depth of the snapshot. */ diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.ts index 9de0f2b29528e..8dda1580c156e 100644 --- a/packages/playwright-core/src/client/locator.ts +++ b/packages/playwright-core/src/client/locator.ts @@ -329,8 +329,8 @@ export class Locator implements api.Locator { return ref ?? null; } - async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number } = {}): Promise { - const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth }); + async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean } = {}): Promise { + const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth, boxes: options.boxes }); return result.snapshot; } diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index 2d519d8095c62..1dc9817d36204 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -866,8 +866,8 @@ export class Page extends ChannelOwner implements api.Page return result.pdf; } - async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, _track?: string } = {}): Promise { - const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth }); + async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean, _track?: string } = {}): Promise { + const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth, boxes: options.boxes }); return result.snapshot; } diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 3c24734e63a40..d9eaec067b8da 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -1660,6 +1660,7 @@ scheme.FrameAriaSnapshotParams = tObject({ track: tOptional(tString), selector: tOptional(tString), depth: tOptional(tInt), + boxes: tOptional(tBoolean), timeout: tFloat, }); scheme.FrameAriaSnapshotResult = tObject({ diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts index 90fafe6282f98..a298a0df92773 100644 --- a/packages/playwright-core/src/server/frames.ts +++ b/packages/playwright-core/src/server/frames.ts @@ -1774,14 +1774,14 @@ export class Frame extends SdkObject { return { ref }; } - async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number } = {}): Promise<{ snapshot: string }> { + async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number, boxes?: boolean } = {}): Promise<{ snapshot: string }> { if (options.selector && options.track) throw new Error('Cannot specify both selector and track options'); if (options.selector && options.mode !== 'ai') { // Non-ai locator snapshot is auto-waiting and does not include iframes. const snapshot = await this._retryWithProgressIfNotConnected(progress, options.selector, { strict: true, performActionPreChecks: true }, async (progress, handle) => { - return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth })); + return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth, boxes: options.boxes })); }); return { snapshot }; } diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 257b657ba3d00..281ad0a6a1922 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -1061,7 +1061,7 @@ export class InitScript extends DisposableObject { } } -export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number } = {}): Promise<{ full: string[], incremental?: string[] }> { +export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number, boxes?: boolean } = {}): Promise<{ full: string[], incremental?: string[] }> { // Only await the topmost navigations, inner frames will be empty when racing. const snapshot = await frame.retryWithProgressAndTimeouts(progress, [1000, 2000, 4000, 8000], async (progress, continuePolling) => { try { @@ -1085,6 +1085,7 @@ export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Fra doNotRenderActive: options.doNotRenderActive, info: options.info, depth: options.depth, + boxes: options.boxes, })); if (snapshotOrRetry === true) return continuePolling; diff --git a/packages/playwright-core/src/tools/backend/response.ts b/packages/playwright-core/src/tools/backend/response.ts index 6b83adc25ae6b..eda678a8468e2 100644 --- a/packages/playwright-core/src/tools/backend/response.ts +++ b/packages/playwright-core/src/tools/backend/response.ts @@ -50,6 +50,7 @@ export class Response { private _includeSnapshotFileName: string | undefined; private _includeSnapshotRoot: playwright.Locator | undefined; private _includeSnapshotDepth: number | undefined; + private _includeSnapshotBoxes: boolean | undefined; private _isClose: boolean = false; readonly toolName: string; @@ -142,10 +143,11 @@ export class Response { this._includeSnapshot = this._context.config.snapshot?.mode ?? 'full'; } - setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number) { + setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number, boxes?: boolean) { this._includeSnapshot = 'explicit'; this._includeSnapshotFileName = includeSnapshotFileName; this._includeSnapshotDepth = depth; + this._includeSnapshotBoxes = boxes; this._includeSnapshotRoot = root; } @@ -242,7 +244,7 @@ export class Response { addSection('Ran Playwright code', this._code, 'js'); // Render tab titles upon changes or when more than one tab. - const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._clientWorkspace) : undefined; + const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._includeSnapshotBoxes, this._clientWorkspace) : undefined; const tabHeaders = await Promise.all(this._context.tabs().map(tab => tab.headerSnapshot())); if (this._includeSnapshot !== 'none' || tabHeaders.some(header => header.changed)) { if (tabHeaders.length !== 1) diff --git a/packages/playwright-core/src/tools/backend/snapshot.ts b/packages/playwright-core/src/tools/backend/snapshot.ts index 30730b33b4f00..516b17da43921 100644 --- a/packages/playwright-core/src/tools/backend/snapshot.ts +++ b/packages/playwright-core/src/tools/backend/snapshot.ts @@ -42,6 +42,7 @@ const snapshot = defineTabTool({ target: z.string().optional().describe(elementTargetDescription), filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'), depth: z.number().optional().describe('Limit the depth of the snapshot tree'), + boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot'), }), type: 'readOnly', }, @@ -50,7 +51,7 @@ const snapshot = defineTabTool({ let resolved: { locator: playwright.Locator | undefined, resolved: string } = { locator: undefined, resolved: '' }; if (params.target) resolved = await tab.targetLocator({ target: params.target }); - response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth); + response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth, params.boxes); }, }); diff --git a/packages/playwright-core/src/tools/backend/tab.ts b/packages/playwright-core/src/tools/backend/tab.ts index 0284509303422..e409fa344d07c 100644 --- a/packages/playwright-core/src/tools/backend/tab.ts +++ b/packages/playwright-core/src/tools/backend/tab.ts @@ -383,13 +383,13 @@ export class Tab extends EventEmitter { this._requests.length = 0; } - async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, relativeTo: string | undefined): Promise { + async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, boxes: boolean | undefined, relativeTo: string | undefined): Promise { await this._initializedPromise; let tabSnapshot: TabSnapshot | undefined; const modalStates = await this._raceAgainstModalStates(async () => { const ariaSnapshot = root - ? await root.ariaSnapshot({ mode: 'ai', depth }) - : await this.page.ariaSnapshot({ mode: 'ai', depth }); + ? await root.ariaSnapshot({ mode: 'ai', depth, boxes }) + : await this.page.ariaSnapshot({ mode: 'ai', depth, boxes }); tabSnapshot = { ariaSnapshot, modalStates: [], diff --git a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md index 66b5ad4acd108..9e70ca9322b40 100644 --- a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md +++ b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md @@ -256,6 +256,9 @@ playwright-cli snapshot "#main" # limit snapshot depth for efficiency, take a partial snapshot afterwards playwright-cli snapshot --depth=4 playwright-cli snapshot e34 + +# include each element's bounding box as [box=x,y,width,height] +playwright-cli snapshot --boxes ``` ## Targeting elements diff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.ts index d703773e874b5..594778e874b0f 100644 --- a/packages/playwright-core/src/tools/cli-daemon/commands.ts +++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts @@ -367,9 +367,10 @@ const snapshot = declareCommand({ options: z.object({ filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'), depth: numberArg.optional().describe('Limit snapshot depth, unlimited by default.'), + boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot.'), }), toolName: 'browser_snapshot', - toolParams: ({ filename, target, depth }) => ({ filename, target, depth }), + toolParams: ({ filename, target, depth, boxes }) => ({ filename, target, depth, boxes }), }); const generateLocator = declareCommand({ diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts index 43f486126a215..b1374ce9831c8 100644 --- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts +++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts @@ -53,7 +53,7 @@ async function startDashboardServer(options: DashboardOptions): Promise(); - const submitAnnotation = (base64Png: string, annotations: AnnotationData[]) => { + const submitAnnotation = (base64Png: string | undefined, annotations: AnnotationData[]) => { if (waitingSockets.size === 0) return; const payload = JSON.stringify({ png: base64Png, annotations }); @@ -407,14 +407,16 @@ async function runAnnotateClient(options: DashboardOptions): Promise { if (!text) return; const { png, annotations } = JSON.parse(text) as { png: string; annotations: AnnotationData[] }; - const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64')); for (const a of annotations) { // eslint-disable-next-line no-console console.log(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`); } - // eslint-disable-next-line no-console - console.log(`image: ${path.relative(process.cwd(), filePath)}`); + if (png) { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64')); + // eslint-disable-next-line no-console + console.log(`image: ${path.relative(process.cwd(), filePath)}`); + } } function selfDestructOnParentGone() { diff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.ts index 38b9cf7d5ad44..e6029bdfcf083 100644 --- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts +++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts @@ -110,7 +110,7 @@ export class DashboardConnection implements Transport { private _attachedPage: AttachedPage | undefined; private _onclose: () => void; private _onconnected?: () => void; - private _onAnnotationSubmit?: (base64Png: string, annotations: AnnotationData[]) => void; + private _onAnnotationSubmit?: (base64Png: string | undefined, annotations: AnnotationData[]) => void; private _serverRegistryDispose?: () => void; private _pushSessionsScheduled = false; private _pushTabsScheduled = false; @@ -120,7 +120,7 @@ export class DashboardConnection implements Transport { _recordingDir: string; _streams = new Map(); - constructor(onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string, annotations: AnnotationData[]) => void) { + constructor(onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string | undefined, annotations: AnnotationData[]) => void) { this._onclose = onclose; this._onconnected = onconnected; this._onAnnotationSubmit = onAnnotationSubmit; @@ -236,7 +236,7 @@ export class DashboardConnection implements Transport { this._pushTabs(); } - async submitAnnotation(params: { data: string; annotations: AnnotationData[] }) { + async submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[] }) { this._onAnnotationSubmit?.(params.data, params.annotations); } @@ -558,9 +558,14 @@ class AttachedPage { return { streamId }; } - async screenshot(): Promise { + async screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number }> { const buffer = await this._page.screenshot({ type: 'png' }); - return buffer.toString('base64'); + const vp = this._page.viewportSize(); + return { + data: buffer.toString('base64'), + viewportWidth: vp?.width ?? 0, + viewportHeight: vp?.height ?? 0, + }; } private async _startScreencast(page: api.Page) { diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 30e475fcdea33..452fbe353a4cf 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -2051,6 +2051,11 @@ export interface Page { * @param options */ ariaSnapshot(options?: { + /** + * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + */ + boxes?: boolean; + /** * When specified, limits the depth of the snapshot. */ @@ -13057,6 +13062,11 @@ export interface Locator { * @param options */ ariaSnapshot(options?: { + /** + * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + */ + boxes?: boolean; + /** * When specified, limits the depth of the snapshot. */ diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index d03bbcdd712ec..8125a83d924c8 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -2927,6 +2927,7 @@ export type FrameAriaSnapshotParams = { track?: string, selector?: string, depth?: number, + boxes?: boolean, timeout: number, }; export type FrameAriaSnapshotOptions = { @@ -2934,6 +2935,7 @@ export type FrameAriaSnapshotOptions = { track?: string, selector?: string, depth?: number, + boxes?: boolean, }; export type FrameAriaSnapshotResult = { snapshot: string, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index 45970c3cc14d6..4ec62cb7f1c84 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -2324,6 +2324,7 @@ Frame: track: string? selector: string? depth: int? + boxes: boolean? timeout: float returns: snapshot: string diff --git a/tests/electron/electronTest.ts b/tests/electron/electronTest.ts index 4d7319fd32314..07393bd8654ae 100644 --- a/tests/electron/electronTest.ts +++ b/tests/electron/electronTest.ts @@ -15,6 +15,7 @@ */ import { baseTest } from '../config/baseTest'; +import { execFileSync } from 'child_process'; import path from 'path'; import fs from 'fs'; import os from 'os'; @@ -33,8 +34,26 @@ const { removeFolders } = utils; type LocalFixtures = PageTestFixtures & { launchElectronApp: (appFile: string, args?: string[], options?: Parameters[0]) => Promise; createUserDataDir: () => Promise; + _electronDiag: void; }; +let diagSeq = 0; + +function countElectronProcesses(): number { + try { + if (process.platform === 'win32') { + const out = execFileSync('tasklist', ['/FI', 'IMAGENAME eq electron*', '/NH'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); + return out.split('\n').filter(l => /electron/i.test(l)).length; + } + // BSD pgrep (macOS) has no -c, so use ps+grep. `|| echo 0` keeps grep's + // no-match exit code from tripping execFileSync. + const out = execFileSync('sh', ['-c', 'ps -A -o comm= | grep -ic electron || echo 0'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); + return parseInt(out.trim(), 10) || 0; + } catch { + return -1; + } +} + export const electronTest = mergeTests(baseTest, electronBaseTest) .extend(traceViewerFixtures) .extend({ @@ -86,4 +105,20 @@ export const electronTest = mergeTests(baseTest, electronBaseTest) for (const app of apps) await app.close(); }, + + _electronDiag: [async ({}, use, testInfo) => { + const seq = ++diagSeq; + const startedAt = Date.now(); + const procsBefore = process.env.CI ? countElectronProcesses() : -1; + await use(); + if (!process.env.CI) + return; + const durMs = Date.now() - startedAt; + const rssMb = Math.round(process.memoryUsage().rss / 1024 / 1024); + const freeMb = Math.round(os.freemem() / 1024 / 1024); + const totalMb = Math.round(os.totalmem() / 1024 / 1024); + const procsAfter = countElectronProcesses(); + const title = testInfo.titlePath.slice(-2).join(' > '); + process.stderr.write(`[ediag] seq=${seq} dur=${durMs}ms rss=${rssMb}M free=${freeMb}/${totalMb}M eproc=${procsBefore}->${procsAfter} :: ${title}\n`); + }, { auto: true }], }); diff --git a/tests/library/browser.spec.ts b/tests/library/browser.spec.ts index a3db493051925..42750dad1c5c0 100644 --- a/tests/library/browser.spec.ts +++ b/tests/library/browser.spec.ts @@ -68,6 +68,7 @@ test('should fire context event on newContext', async ({ browser }) => { browser.on('context', ctx => events.push(ctx)); const context = await browser.newContext(); expect(events).toEqual([context]); + await context.close(); }); test('newContext should not leave a context upon failure', async ({ browser, toImpl }) => { diff --git a/tests/library/inspector/cli-codegen-1.spec.ts b/tests/library/inspector/cli-codegen-1.spec.ts index dcfe89c72ae5d..97929c01ae78e 100644 --- a/tests/library/inspector/cli-codegen-1.spec.ts +++ b/tests/library/inspector/cli-codegen-1.spec.ts @@ -1078,8 +1078,9 @@ await page.GetByRole(AriaRole.Button, new() { Name = "Submit" }).ClickAsync();`) }); test('should not throw csp directive violation errors', async ({ openRecorder, server }) => { + server.setCSP('/empty.html', `default-src 'self'`); const { page } = await openRecorder(); - await page.goto(server.PREFIX + '/csp.html'); + await page.goto(server.EMPTY_PAGE); const predicate = (msg: ConsoleMessage) => msg.type() === 'error' && /Content[\- ]Security[\- ]Policy/i.test(msg.text()); await expect(page.waitForEvent('console', { predicate, timeout: 1000 })).rejects.toThrow(); }); diff --git a/tests/library/locator-highlight.spec.ts b/tests/library/locator-highlight.spec.ts index 237b22c72875e..6e4fd5bf7ab0f 100644 --- a/tests/library/locator-highlight.spec.ts +++ b/tests/library/locator-highlight.spec.ts @@ -18,7 +18,9 @@ import { expect, browserTest as test } from '../config/browserTest'; test.skip(({ mode }) => mode !== 'default', 'Highlight overlay uses an open shadow root only in default mode'); -test('highlight should accept a CSS string style', async ({ browser, server, browserName }) => { +test('highlight should accept a CSS string style', async ({ browser, server, browserName, isFrozenWebkit }) => { + test.skip(isFrozenWebkit); + const context = await browser.newContext(); const page = await context.newPage(); await page.goto(server.PREFIX + '/input/button.html'); @@ -40,7 +42,9 @@ test('highlight should accept a CSS string style', async ({ browser, server, bro await context.close(); }); -test('highlight should accept an object style (JS only)', async ({ browser, server, browserName }) => { +test('highlight should accept an object style (JS only)', async ({ browser, server, browserName, isFrozenWebkit }) => { + test.skip(isFrozenWebkit); + const context = await browser.newContext(); const page = await context.newPage(); await page.goto(server.PREFIX + '/input/button.html'); diff --git a/tests/mcp/cli-core.spec.ts b/tests/mcp/cli-core.spec.ts index 89c472b887a09..24ef136ca5059 100644 --- a/tests/mcp/cli-core.spec.ts +++ b/tests/mcp/cli-core.spec.ts @@ -310,6 +310,20 @@ test('snapshot depth', async ({ cli, server }) => { - button "Cancel" [ref=e6]`); }); +test('snapshot --boxes', async ({ cli, server }) => { + server.setContent('/', ` + + + `, 'text/html'); + await cli('open', server.PREFIX); + + const { inlineSnapshot } = await cli('snapshot', '--boxes'); + expect(inlineSnapshot).toContain(`- button "click" [ref=e1] [box=100,50,80,40]`); + + const { inlineSnapshot: plain } = await cli('snapshot'); + expect(plain).not.toMatch(/\[box=/); +}); + test('eval --raw', async ({ cli, server }) => { await cli('open', server.HELLO_WORLD); const { output } = await cli('eval', '--raw', '() => document.title'); diff --git a/tests/mcp/core.spec.ts b/tests/mcp/core.spec.ts index 42490d8fd88c6..06423357b550d 100644 --- a/tests/mcp/core.spec.ts +++ b/tests/mcp/core.spec.ts @@ -276,6 +276,31 @@ test('snapshot depth', async ({ client, server }) => { }); }); +test('snapshot with boxes', async ({ client, server }) => { + server.setContent('/', ` + + + `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_snapshot', + arguments: { boxes: true }, + })).toHaveResponse({ + inlineSnapshot: expect.stringContaining(`- button "click" [ref=e1] [box=100,50,80,40]`), + }); + + expect(await client.callTool({ + name: 'browser_snapshot', + })).toHaveResponse({ + inlineSnapshot: expect.not.stringMatching(/\[box=/), + }); +}); + test('snapshot by ref', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright-cli/issues/347' } }, async ({ client, server }) => { server.setContent('/', `
    diff --git a/tests/page/page-aria-snapshot.spec.ts b/tests/page/page-aria-snapshot.spec.ts index 8b6bcf19e1e05..da9708d56c4dd 100644 --- a/tests/page/page-aria-snapshot.spec.ts +++ b/tests/page/page-aria-snapshot.spec.ts @@ -748,3 +748,29 @@ it('should snapshot a locator inside an iframe', async ({ page }) => { - listitem: Item 2 `); }); + +it('should snapshot with box from page', async ({ page }) => { + await page.setContent(` + + `); + + const snapshot = await page.ariaSnapshot({ boxes: true }); + expect(snapshot).toBe(`- button "click" [box=100,50,80,40]`); +}); + +it('should snapshot with box from locator', async ({ page }) => { + await page.setContent(` +
    + +
    + `); + + const snapshot = await page.locator('div').ariaSnapshot({ boxes: true }); + expect(snapshot).toBe(`- button "ok" [box=15,25,60,30]`); +}); + +it('should not include box when option is omitted', async ({ page }) => { + await page.setContent(``); + const snapshot = await page.ariaSnapshot(); + expect(snapshot).not.toMatch(/\[box=/); +});