diff --git a/.github/workflows/tests_docker.yml b/.github/workflows/tests_docker.yml new file mode 100644 index 0000000000000..e78d43c473050 --- /dev/null +++ b/.github/workflows/tests_docker.yml @@ -0,0 +1,131 @@ +name: tests docker + +on: + workflow_call: + +env: + FORCE_COLOR: 1 + +jobs: + test_linux_docker: + name: "Docker ${{ matrix.docker_tag }} ${{ matrix.docker_arch }}" + environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} + strategy: + fail-fast: false + matrix: + include: + - docker_tag: jammy + docker_arch: amd64 + host_os: ubuntu-22.04 + - docker_tag: jammy + docker_arch: arm64 + host_os: ubuntu-22.04-arm + - docker_tag: noble + docker_arch: amd64 + host_os: ubuntu-22.04 + - docker_tag: noble + docker_arch: arm64 + host_os: ubuntu-22.04-arm + runs-on: ${{ matrix.host_os }} + permissions: + id-token: write # This is required for OIDC login (azure/login) to succeed + contents: read # This is required for actions/checkout to succeed + steps: + - name: Create ~/.azure directory + run: mkdir -p ~/.azure + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: 22 + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + - run: npm ci + - run: npm run build + # Used as a Pull-through cache for the Docker images. Only available on the + # main repo, where the secrets are present; forks fall back to Docker Hub. + - name: Azure Login + if: ${{ github.event_name == 'push' && github.repository == 'microsoft/playwright' }} + uses: azure/login@v3 + with: + client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} + - name: Login to ACR via OIDC + if: ${{ github.event_name == 'push' && github.repository == 'microsoft/playwright' }} + run: az acr login --name playwright + - name: Add Dockerfile cache + if: ${{ github.event_name == 'push' && github.repository == 'microsoft/playwright' }} + run: sed -i 's/FROM ubuntu/FROM playwright.azurecr.io\/cached\/ubuntu/' utils/docker/Dockerfile.${{matrix.docker_tag}} + - run: ./utils/docker/build.sh --${{ matrix.docker_arch }} ${{ matrix.docker_tag }} playwright:localbuild + shell: bash + - name: Launch container + run: | + docker run \ + --rm \ + --name docker-tests \ + --platform linux/${{ matrix.docker_arch }} \ + --user=pwuser \ + --workdir /home/pwuser \ + --env CI \ + --env INSIDE_DOCKER=1 \ + -v ~/.azure:/root/.azure \ + -d \ + -t \ + playwright:localbuild /bin/bash + - name: Copy repository inside docker container + run: | + # Note: we cannot do explicit `git clone` inside container because it requires some smarts to decide if github.sha or + # github.ref should be used for checkout. + # The actions/checkout action already handled this complexity for us, so we'll just copy checkout. + docker cp . docker-tests:/home/pwuser/playwright + docker exec --user root docker-tests chown -R pwuser /home/pwuser/playwright + + # GIT is now picky on directory ownership + # See https://github.blog/2022-04-12-git-security-vulnerability-announced/ + docker exec \ + --user root \ + --workdir /home/pwuser/playwright docker-tests /bin/bash -c ' + git config --global --add safe.directory /home/pwuser/playwright + ' + - name: Run "npm ci" inside docker + run: docker exec --workdir /home/pwuser/playwright docker-tests npm ci + + - name: Run "npm run build" inside docker + run: docker exec --workdir /home/pwuser/playwright docker-tests npm run build + + - name: "Run @smoke tests inside docker" + run: docker exec --workdir /home/pwuser/playwright docker-tests xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" npm run test -- --grep "@smoke" + + - name: Azure Login + if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }} + uses: azure/login@v3 + with: + client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} + - name: Upload Flakiness dashboard + if: ${{ !cancelled() && github.event_name == 'push' && github.repository == 'microsoft/playwright' }} + run: | + # Flakiness dashboard has to be uploaded from-inside docker container so that it can collect meta information, e.g. arch + # See https://github.com/microsoft/playwright/blob/13b1e52d95df416fdfa7846c8a840aad8df263af/utils/upload_flakiness_dashboard.sh#L71-L86 + docker exec \ + --env GITHUB_REPOSITORY \ + --env GITHUB_REF \ + --env GITHUB_RUN_ID \ + --user root \ + --workdir /home/pwuser/playwright docker-tests /bin/bash -c ' + set -e + # Azure-CLI is needed for upload_flakiness_dashboard.sh script. + curl -sL https://aka.ms/InstallAzureCLIDeb | bash + ./utils/upload_flakiness_dashboard.sh "./test-results/report.json" + ' + - name: Copy blob report from container + if: ${{ !cancelled() && (github.event_name == 'pull_request' || failure()) }} + run: docker cp "docker-tests:/home/pwuser/playwright/blob-report" "./blob-report" + + - name: Upload blob report + if: ${{ !cancelled() && (github.event_name == 'pull_request' || failure()) }} + uses: ./.github/actions/upload-blob-report + with: + report_dir: blob-report + job_name: docker-${{ matrix.docker_tag }}-${{ matrix.docker_arch }} diff --git a/.github/workflows/tests_docker_changes.yml b/.github/workflows/tests_docker_changes.yml new file mode 100644 index 0000000000000..8807d32ae6788 --- /dev/null +++ b/.github/workflows/tests_docker_changes.yml @@ -0,0 +1,24 @@ +name: tests docker (changes) + +on: + push: + branches: + - main + paths: + - 'utils/docker/**' + - '.github/workflows/tests_docker*.yml' + pull_request: + branches: + - main + - release-* + paths: + - 'utils/docker/**' + - '.github/workflows/tests_docker*.yml' + +jobs: + test_linux_docker: + uses: ./.github/workflows/tests_docker.yml + permissions: + id-token: write + contents: read + secrets: inherit diff --git a/.github/workflows/tests_docker_release.yml b/.github/workflows/tests_docker_release.yml new file mode 100644 index 0000000000000..d9e22f16550a2 --- /dev/null +++ b/.github/workflows/tests_docker_release.yml @@ -0,0 +1,14 @@ +name: tests docker (release) + +on: + push: + branches: + - release-* + +jobs: + test_linux_docker: + uses: ./.github/workflows/tests_docker.yml + permissions: + id-token: write + contents: read + secrets: inherit diff --git a/.github/workflows/trigger_tests.yml b/.github/workflows/trigger_tests.yml deleted file mode 100644 index 761c11f395dcb..0000000000000 --- a/.github/workflows/trigger_tests.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: "Internal Tests" - -on: - push: - branches: - - main - - release-* - -jobs: - trigger: - name: "trigger" - runs-on: ubuntu-24.04 - steps: - - uses: actions/create-github-app-token@v3 - id: app-token - with: - app-id: ${{ vars.PLAYWRIGHT_APP_ID }} - private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} - repositories: playwright-browsers - - run: | - curl -X POST \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token ${GH_TOKEN}" \ - --data "{\"event_type\": \"playwright_tests\", \"client_payload\": {\"ref\": \"${GITHUB_SHA}\"}}" \ - https://api.github.com/repos/microsoft/playwright-browsers/dispatches - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 62425a56c0ff7..fdf956d6a6645 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -634,8 +634,8 @@ export default defineConfig({ ## property: TestOptions.video * since: v1.10 -- type: <[Object]|[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> - - `mode` <[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">> Video recording mode. +- type: <[Object]|[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries"|"retain-all-failures">> + - `mode` <[VideoMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries"|"retain-all-failures">> Video recording mode. - `size` ?<[Object]> Size of the recorded video. Optional. - `width` <[int]> - `height` <[int]> @@ -652,8 +652,12 @@ export default defineConfig({ Whether to record video for each test. Defaults to `'off'`. * `'off'`: Do not record video. * `'on'`: Record video for each test. -* `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs. * `'on-first-retry'`: Record video only when retrying a test for the first time. +* `'on-all-retries'`: Record video only when retrying a test. +* `'retain-on-failure'`: Record video for each test. When test run passes, remove the recorded video. +* `'retain-on-first-failure'`: Record video for the first run of each test, but not for retries. When test run passes, remove the recorded video. +* `'retain-on-failure-and-retries'`: Record video for each test run. Retains all videos when an attempt fails. +* `'retain-all-failures'`: Record video for each test run. Retains the video only for attempts that failed, regardless of the final test outcome. To control video size, pass an object with `mode` and `size` properties. If video size is not specified, it will be equal to [`property: TestOptions.viewport`] scaled down to fit into 800x800. If `viewport` is not configured explicitly the video size defaults to 800x450. Actual picture of each page will be scaled down if necessary to fit the specified size. diff --git a/packages/playwright-core/src/inprocess.ts b/packages/playwright-core/src/inprocess.ts index fba4fb5ad7155..49be0eb9d775d 100644 --- a/packages/playwright-core/src/inprocess.ts +++ b/packages/playwright-core/src/inprocess.ts @@ -25,10 +25,10 @@ import type { Playwright as PlaywrightAPI } from './client/playwright'; import type { Language } from '@isomorphic/locatorGenerators'; export function createInProcessPlaywright(): PlaywrightAPI { - const playwright = createPlaywright({ sdkLanguage: (process.env.PW_LANG_NAME as Language | undefined) || 'javascript' }); + const playwright = createPlaywright({ sdkLanguage: (process.env.PW_LANG_NAME as Language | undefined) || 'javascript', isClientCollocatedWithServer: true }); const clientConnection = new Connection(nodePlatform(packageRoot)); clientConnection.useRawBuffers(); - const dispatcherConnection = new DispatcherConnection(true /* local */); + const dispatcherConnection = new DispatcherConnection(true /* in process */); // Dispatch synchronously at first. dispatcherConnection.onmessage = message => clientConnection.dispatch(message); diff --git a/packages/playwright-core/src/server/artifact.ts b/packages/playwright-core/src/server/artifact.ts index 1a08e7e133737..562d2d9b9ae37 100644 --- a/packages/playwright-core/src/server/artifact.ts +++ b/packages/playwright-core/src/server/artifact.ts @@ -29,6 +29,7 @@ type CancelCallback = () => Promise; export class Artifact extends SdkObject { private _localPath: string; private _unaccessibleErrorMessage: string | undefined; + private _missingFileErrorMessage: string | undefined; private _cancelCallback: CancelCallback | undefined; private _finishedPromise = new ManualPromise(); private _saveCallbacks: SaveCallback[] = []; @@ -43,6 +44,14 @@ export class Artifact extends SdkObject { this._cancelCallback = cancelCallback; } + markMissingFileErrorMessage(errorMessage: string) { + this._missingFileErrorMessage = errorMessage; + } + + missingFileErrorMessage(): string | undefined { + return this._missingFileErrorMessage; + } + async localPathAfterFinished(progress: Progress): Promise { return await progress.race(this._localPathAfterFinished()); } diff --git a/packages/playwright-core/src/server/browser.ts b/packages/playwright-core/src/server/browser.ts index 812d0f23df6ac..edbc72a480ea9 100644 --- a/packages/playwright-core/src/server/browser.ts +++ b/packages/playwright-core/src/server/browser.ts @@ -80,7 +80,7 @@ export abstract class Browser extends SdkObject { private _startedClosing = false; private _contextForReuse: { context: BrowserContext, hash: string } | undefined; _closeReason: string | undefined; - _isCollocatedWithServer: boolean = true; + _isBrowserCollocatedWithServer: boolean = true; private _server: BrowserServer; constructor(parent: SdkObject, options: BrowserOptions) { diff --git a/packages/playwright-core/src/server/chromium/chromium.ts b/packages/playwright-core/src/server/chromium/chromium.ts index 7766548d19c62..d9f1c9da72602 100644 --- a/packages/playwright-core/src/server/chromium/chromium.ts +++ b/packages/playwright-core/src/server/chromium/chromium.ts @@ -159,7 +159,7 @@ export class Chromium extends BrowserType { validateBrowserContextOptions(persistent, browserOptions); const browser = await progress.race(CRBrowser.connect(this.attribution.playwright, transport, browserOptions)); if (!options.isLocal) - browser._isCollocatedWithServer = false; + browser._isBrowserCollocatedWithServer = false; browser.on(Browser.Events.Disconnected, doCleanup); return browser; } catch (error) { diff --git a/packages/playwright-core/src/server/chromium/crBrowser.ts b/packages/playwright-core/src/server/chromium/crBrowser.ts index b91df727c30f1..ec96fb94966da 100644 --- a/packages/playwright-core/src/server/chromium/crBrowser.ts +++ b/packages/playwright-core/src/server/chromium/crBrowser.ts @@ -63,7 +63,7 @@ export class CRBrowser extends Browser { const browser = new CRBrowser(parent, connection, options); browser._devtools = devtools; if (browser.isClank()) - browser._isCollocatedWithServer = false; + browser._isBrowserCollocatedWithServer = false; const session = connection.rootSession; if ((options as any).__testHookOnConnectToBrowser) await (options as any).__testHookOnConnectToBrowser(); diff --git a/packages/playwright-core/src/server/dispatchers/artifactDispatcher.ts b/packages/playwright-core/src/server/dispatchers/artifactDispatcher.ts index ff4e0f868bd60..2ff6dff8c23b8 100644 --- a/packages/playwright-core/src/server/dispatchers/artifactDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/artifactDispatcher.ts @@ -46,11 +46,13 @@ export class ArtifactDispatcher extends Dispatcher { + this._assertFileAccessible(); const path = await this._object.localPathAfterFinished(progress); return { value: path }; } async saveAs(params: channels.ArtifactSaveAsParams, progress: Progress): Promise { + this._assertFileAccessible(); return await progress.race(new Promise((resolve, reject) => { this._object.saveAs(progress, async (localPath, error) => { if (error) { @@ -68,7 +70,14 @@ export class ArtifactDispatcher extends Dispatcher { + this._assertFileAccessible(); return await progress.race(new Promise((resolve, reject) => { this._object.saveAs(progress, async (localPath, error) => { if (error) { @@ -94,6 +103,7 @@ export class ArtifactDispatcher extends Dispatcher { + this._assertFileAccessible(); const fileName = await this._object.localPathAfterFinished(progress); const readable = fs.createReadStream(fileName, { highWaterMark: 1024 * 1024 }); return { stream: new StreamDispatcher(this, readable) }; diff --git a/packages/playwright-core/src/server/dispatchers/dispatcher.ts b/packages/playwright-core/src/server/dispatchers/dispatcher.ts index df40e458af6f5..2a75e1d8d3dc8 100644 --- a/packages/playwright-core/src/server/dispatchers/dispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/dispatcher.ts @@ -195,10 +195,10 @@ export class DispatcherConnection { readonly _dispatchersByBucket = new Map>(); onmessage = (message: object) => {}; private _waitOperations = new Map(); - private _isLocal: boolean; + private _isInProcess: boolean; - constructor(isLocal?: boolean) { - this._isLocal = !!isLocal; + constructor(isInProcess?: boolean) { + this._isInProcess = !!isInProcess; } sendEvent(dispatcher: DispatcherScope, event: string, params: any) { @@ -224,7 +224,7 @@ export class DispatcherConnection { private _validatorToWireContext(): ValidatorContext { return { tChannelImpl: this._tChannelImplToWire.bind(this), - binary: this._isLocal ? 'buffer' : 'toBase64', + binary: this._isInProcess ? 'buffer' : 'toBase64', isUnderTest, }; } @@ -232,7 +232,7 @@ export class DispatcherConnection { private _validatorFromWireContext(): ValidatorContext { return { tChannelImpl: this._tChannelImplFromWire.bind(this), - binary: this._isLocal ? 'buffer' : 'fromBase64', + binary: this._isInProcess ? 'buffer' : 'fromBase64', isUnderTest, }; } diff --git a/packages/playwright-core/src/server/download.ts b/packages/playwright-core/src/server/download.ts index 0c8725c35b2f6..91958c3617ffa 100644 --- a/packages/playwright-core/src/server/download.ts +++ b/packages/playwright-core/src/server/download.ts @@ -32,6 +32,11 @@ export class Download { if (!downloadPath) throw new Error(`Download filename '${downloadFilename}' escapes download directory`); this.artifact = new Artifact(page, downloadPath, unaccessibleErrorMessage, () => this.cancel()); + if (!page.browserContext._browser._isBrowserCollocatedWithServer) { + this.artifact.markMissingFileErrorMessage( + `Downloaded file is not accessible from the Playwright server because the browser is running on a different host ` + + `(e.g. connected over CDP to a remote browser). Saving downloads requires the browser and the Playwright server to share a filesystem.`); + } this._page = page; this.url = url; this._uuid = uuid; diff --git a/packages/playwright-core/src/server/fileUploadUtils.ts b/packages/playwright-core/src/server/fileUploadUtils.ts index 94921e1740daf..033c0bed040d4 100644 --- a/packages/playwright-core/src/server/fileUploadUtils.ts +++ b/packages/playwright-core/src/server/fileUploadUtils.ts @@ -36,6 +36,8 @@ async function filesExceedUploadLimit(files: string[]) { export async function prepareFilesForUpload(frame: Frame, params: Omit): Promise { const { payloads, streams, directoryStream } = params; let { localPaths, localDirectory } = params; + if (localPaths && !frame.attribution.playwright.options.isClientCollocatedWithServer) + throw new Error('localPaths are not allowed when the client is not local'); if ([payloads, localPaths, localDirectory, streams, directoryStream].filter(Boolean).length !== 1) throw new Error('Exactly one of payloads, localPaths and streams must be provided'); @@ -57,7 +59,7 @@ export async function prepareFilesForUpload(frame: Frame, params: Omit p.video()).filter(video => !!video); @@ -521,7 +520,31 @@ function normalizeVideoMode(video: VideoMode | 'retry-with-video' | { mode: Vide } function shouldCaptureVideo(videoMode: VideoMode, testInfo: TestInfo) { - return (videoMode === 'on' || videoMode === 'retain-on-failure' || (videoMode === 'on-first-retry' && testInfo.retry === 1)); + return videoMode === 'on' + || videoMode === 'retain-on-failure' + || videoMode === 'retain-on-failure-and-retries' + || videoMode === 'retain-all-failures' + || (videoMode === 'on-first-retry' && testInfo.retry === 1) + || (videoMode === 'on-all-retries' && testInfo.retry > 0) + || (videoMode === 'retain-on-first-failure' && testInfo.retry === 0); +} + +function shouldPreserveVideo(videoMode: VideoMode, testInfo: TestInfo) { + const testFailed = testInfo.status !== testInfo.expectedStatus; + switch (videoMode) { + case 'on': + case 'on-first-retry': + case 'on-all-retries': + return true; + case 'retain-on-failure': + case 'retain-on-first-failure': + case 'retain-all-failures': + return testFailed; + case 'retain-on-failure-and-retries': + return testFailed || testInfo.retry > 0; + default: + return false; + } } function normalizeScreenshotMode(screenshot: ScreenshotOption): ScreenshotMode { diff --git a/packages/playwright/src/plugins/webServerPlugin.ts b/packages/playwright/src/plugins/webServerPlugin.ts index 777abd25dc1d1..a08d20bda2c13 100644 --- a/packages/playwright/src/plugins/webServerPlugin.ts +++ b/packages/playwright/src/plugins/webServerPlugin.ts @@ -15,6 +15,7 @@ */ import net from 'net'; import path from 'path'; +import { stripVTControlCharacters } from 'util'; import colors from 'colors/safe'; import debug from 'debug'; @@ -172,7 +173,7 @@ export class WebServerPlugin implements TestRunnerPlugin { launchedProcess[stdio]!.on('data', data => { if (!this._options.wait?.[stdio] || stdioWaitCollectors[stdio] === undefined) return; - stdioWaitCollectors[stdio] += data.toString(); + stdioWaitCollectors[stdio] += stripVTControlCharacters(data.toString()); this._options.wait[stdio].lastIndex = 0; const result = this._options.wait[stdio].exec(stdioWaitCollectors[stdio]); if (result) { diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index f916e685b434f..b42fc1477adc8 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -7024,8 +7024,14 @@ export interface PlaywrightWorkerOptions { * Whether to record video for each test. Defaults to `'off'`. * - `'off'`: Do not record video. * - `'on'`: Record video for each test. - * - `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs. * - `'on-first-retry'`: Record video only when retrying a test for the first time. + * - `'on-all-retries'`: Record video only when retrying a test. + * - `'retain-on-failure'`: Record video for each test. When test run passes, remove the recorded video. + * - `'retain-on-first-failure'`: Record video for the first run of each test, but not for retries. When test run + * passes, remove the recorded video. + * - `'retain-on-failure-and-retries'`: Record video for each test run. Retains all videos when an attempt fails. + * - `'retain-all-failures'`: Record video for each test run. Retains the video only for attempts that failed, + * regardless of the final test outcome. * * To control video size, pass an object with `mode` and `size` properties. If video size is not specified, it will be * equal to [testOptions.viewport](https://playwright.dev/docs/api/class-testoptions#test-options-viewport) scaled @@ -7056,7 +7062,7 @@ export interface PlaywrightWorkerOptions { export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure'; export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures'; -export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; +export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures'; /** * Playwright Test provides many options to configure test environment, * [Browser](https://playwright.dev/docs/api/class-browser), diff --git a/tests/library/chromium/connect-over-cdp.spec.ts b/tests/library/chromium/connect-over-cdp.spec.ts index 29c6e99f5d87c..5ebca4c902078 100644 --- a/tests/library/chromium/connect-over-cdp.spec.ts +++ b/tests/library/chromium/connect-over-cdp.spec.ts @@ -113,6 +113,7 @@ test('should connectOverCDP and manage downloads in default context', async ({ b try { const browser = await browserType.connectOverCDP({ endpointURL: `http://127.0.0.1:${port}/`, + isLocal: true, }); const page = await browser.contexts()[0].newPage(); await page.setContent(`download`); @@ -134,6 +135,40 @@ test('should connectOverCDP and manage downloads in default context', async ({ b } }); +test('should give a clear error for downloads when browser is not co-located with the server', async ({ browserType, server }, testInfo) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/41060' }); + server.setRoute('/downloadWithFilename', (req, res) => { + res.setHeader('Content-Type', 'application/octet-stream'); + res.setHeader('Content-Disposition', 'attachment; filename=file.txt'); + res.end(`Hello world`); + }); + + const port = 9339 + testInfo.workerIndex; + const browserServer = await browserType.launch({ + args: ['--remote-debugging-port=' + port] + }); + + try { + const browser = await browserType.connectOverCDP({ + endpointURL: `http://127.0.0.1:${port}/`, + }); + const page = await browser.contexts()[0].newPage(); + await page.setContent(`download`); + + const [download] = await Promise.all([ + page.waitForEvent('download'), + page.click('a') + ]); + + const saveError = await download.saveAs(testInfo.outputPath('download.txt')).catch(e => e); + expect(saveError.message).toContain('the browser is running on a different host'); + const pathError = await download.path().catch(e => e); + expect(pathError.message).toContain('the browser is running on a different host'); + } finally { + await browserServer.close(); + } +}); + test('should connect to an existing cdp session twice', async ({ browserType, mode, server }, testInfo) => { const port = 9339 + testInfo.workerIndex; const browserServer = await browserType.launch({ @@ -609,10 +644,10 @@ test('setInputFiles should use local path when isLocal is set', async ({ browser }); try { const cdpBrowser1 = await browserType.connectOverCDP(`http://127.0.0.1:${port}/`); - expect(toImpl(cdpBrowser1)._isCollocatedWithServer).toBe(false); + expect(toImpl(cdpBrowser1)._isBrowserCollocatedWithServer).toBe(false); const cdpBrowser2 = await browserType.connectOverCDP(`http://127.0.0.1:${port}/`, { isLocal: true }); - expect(toImpl(cdpBrowser2)._isCollocatedWithServer).toBe(true); + expect(toImpl(cdpBrowser2)._isBrowserCollocatedWithServer).toBe(true); } finally { await browserServer.close(); } diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.ts index 89339d93f4959..fa3b4e37e5ec6 100644 --- a/tests/playwright-test/playwright.spec.ts +++ b/tests/playwright-test/playwright.spec.ts @@ -525,6 +525,126 @@ test('should work with video: on-first-retry', async ({ runInlineTest }) => { }, errorPrompt]); }); +test('should work with video: on-all-retries', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { use: { video: 'on-all-retries' }, retries: 2, name: 'chromium' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({ page }) => { + await page.setContent('
FAIL
'); + await page.waitForTimeout(1000); + test.expect(1 + 1).toBe(1); + }); + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + + const dirFail = test.info().outputPath('test-results', 'a-fail-chromium'); + expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeFalsy(); + + const dirRetry1 = test.info().outputPath('test-results', 'a-fail-chromium-retry1'); + expect(fs.readdirSync(dirRetry1).find(file => file.endsWith('webm'))).toBeTruthy(); + + const dirRetry2 = test.info().outputPath('test-results', 'a-fail-chromium-retry2'); + expect(fs.readdirSync(dirRetry2).find(file => file.endsWith('webm'))).toBeTruthy(); +}); + +test('should work with video: retain-on-first-failure', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { use: { video: 'retain-on-first-failure' }, retries: 1, name: 'chromium' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.setContent('
PASS
'); + await page.waitForTimeout(1000); + test.expect(1 + 1).toBe(2); + }); + test('fail', async ({ page }) => { + await page.setContent('
FAIL
'); + await page.waitForTimeout(1000); + test.expect(1 + 1).toBe(1); + }); + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(1); + expect(result.failed).toBe(1); + + const dirPass = test.info().outputPath('test-results', 'a-pass-chromium'); + const videoPass = fs.existsSync(dirPass) ? fs.readdirSync(dirPass).find(file => file.endsWith('webm')) : undefined; + expect(videoPass).toBeFalsy(); + + // First run failed, so the video is retained. + const dirFail = test.info().outputPath('test-results', 'a-fail-chromium'); + expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeTruthy(); + + // No video is captured on retries. + const dirRetry = test.info().outputPath('test-results', 'a-fail-chromium-retry1'); + expect(fs.readdirSync(dirRetry).find(file => file.endsWith('webm'))).toBeFalsy(); +}); + +test('should work with video: retain-on-failure-and-retries', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { use: { video: 'retain-on-failure-and-retries' }, retries: 1, name: 'chromium' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('flaky', async ({ page }) => { + await page.setContent('
FLAKY
'); + await page.waitForTimeout(1000); + test.expect(test.info().retry).toBe(1); + }); + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + + // First attempt failed, video retained. + const dirFail = test.info().outputPath('test-results', 'a-flaky-chromium'); + expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeTruthy(); + + // Retry passed, but all videos are retained once the test was retried. + const dirRetry = test.info().outputPath('test-results', 'a-flaky-chromium-retry1'); + expect(fs.readdirSync(dirRetry).find(file => file.endsWith('webm'))).toBeTruthy(); +}); + +test('should work with video: retain-all-failures', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { use: { video: 'retain-all-failures' }, retries: 1, name: 'chromium' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('flaky', async ({ page }) => { + await page.setContent('
FLAKY
'); + await page.waitForTimeout(1000); + test.expect(test.info().retry).toBe(1); + }); + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + + // First attempt failed, video retained. + const dirFail = test.info().outputPath('test-results', 'a-flaky-chromium'); + expect(fs.readdirSync(dirFail).find(file => file.endsWith('webm'))).toBeTruthy(); + + // Retry passed, so its video is removed. + const dirRetry = test.info().outputPath('test-results', 'a-flaky-chromium-retry1'); + const videoRetry = fs.existsSync(dirRetry) ? fs.readdirSync(dirRetry).find(file => file.endsWith('webm')) : undefined; + expect(videoRetry).toBeFalsy(); +}); + test('should work with video size', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.js': ` diff --git a/tests/playwright-test/web-server.spec.ts b/tests/playwright-test/web-server.spec.ts index e8f491d415b38..364bc242a0a52 100644 --- a/tests/playwright-test/web-server.spec.ts +++ b/tests/playwright-test/web-server.spec.ts @@ -961,6 +961,33 @@ for (const stdio of ['stdout', 'stderr']) { expect(result.output).toContain('server started'); }); + test(`should wait for ${stdio} with ANSI escape codes`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'test.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({}) => {}); + `, + 'server.js': ` + // Emit text wrapped in ANSI color escape codes. + setTimeout(() => { console.${stdio === 'stdout' ? 'log' : 'error'}('\\u001b[32mserver started\\u001b[39m'); }, 1000); + setTimeout(() => {}, 100000); + `, + 'playwright.config.ts': ` + module.exports = { + webServer: [ + { + command: 'node server.js', + stdout: 'pipe', + wait: { ${stdio}: /^server started$/m }, + } + ], + }; + `, + }, undefined); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('server started'); + }); + test(`should wait for ${stdio} w/group`, async ({ runInlineTest }) => { const result = await runInlineTest({ 'test.spec.ts': ` diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 09c36aa081bf8..0a6c79c18776a 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -268,7 +268,7 @@ export interface PlaywrightWorkerOptions { export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure'; export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures'; -export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; +export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries' | 'retain-all-failures'; export interface PlaywrightTestOptions { acceptDownloads: boolean; bypassCSP: boolean;