From 19f6e89470729e5e1f4c76dab7a2ec7531403ebe Mon Sep 17 00:00:00 2001 From: Vladimir Date: Fri, 29 May 2026 13:05:07 +0200 Subject: [PATCH 1/2] feat(benchmark)!: rewrite the public API (#10113) --- .gitignore | 1 + .vscode/settings.json | 2 +- docs/.vitepress/config.ts | 4 + docs/api/advanced/vitest.md | 30 +- docs/api/test.md | 233 +-- docs/config/benchmark.md | 44 +- docs/config/index.md | 2 +- docs/config/runner.md | 1 - docs/guide/advanced/index.md | 6 +- docs/guide/benchmarking.md | 480 ++++++ docs/guide/cli-generated.md | 2 +- docs/guide/migration.md | 30 + .../guide/snippets/benchmark-per-project.ansi | 7 + docs/guide/snippets/benchmark-table.ansi | 9 + docs/guide/test-context.md | 23 + docs/guide/test-tags.md | 2 +- eslint.config.js | 1 + packages/browser/src/client/tester/runner.ts | 39 +- packages/browser/src/node/plugin.ts | 24 +- packages/browser/src/node/rpc.ts | 17 + packages/browser/src/types.ts | 6 +- packages/runner/package.json | 3 +- packages/runner/src/collect.ts | 4 +- packages/runner/src/fixture.ts | 1 + packages/runner/src/run.ts | 20 +- packages/runner/src/suite.ts | 1 + packages/runner/src/types.ts | 5 + packages/runner/src/types/runner.ts | 17 +- packages/runner/src/types/tasks.ts | 38 + packages/runner/tsconfig.json | 1 + packages/vitest/package.json | 2 +- packages/vitest/src/defaults.ts | 12 +- .../vitest/src/integrations/chai/bench.ts | 83 + .../vitest/src/integrations/chai/index.ts | 2 + packages/vitest/src/node/ast-collect.ts | 1 + packages/vitest/src/node/benchmark.ts | 42 + packages/vitest/src/node/cli/cac.ts | 38 +- packages/vitest/src/node/cli/cli-api.ts | 82 +- packages/vitest/src/node/cli/cli-config.ts | 19 +- .../vitest/src/node/config/resolveConfig.ts | 105 +- .../vitest/src/node/config/serializeConfig.ts | 8 +- packages/vitest/src/node/core.ts | 59 +- packages/vitest/src/node/create.ts | 36 +- packages/vitest/src/node/errors.ts | 4 +- packages/vitest/src/node/logger.ts | 8 +- packages/vitest/src/node/plugins/index.ts | 7 +- .../vitest/src/node/plugins/publicConfig.ts | 3 +- packages/vitest/src/node/plugins/workspace.ts | 12 +- packages/vitest/src/node/pools/poolRunner.ts | 2 +- packages/vitest/src/node/pools/rpc.ts | 9 + packages/vitest/src/node/project.ts | 5 +- .../src/node/projects/resolveProjects.ts | 80 +- packages/vitest/src/node/reporters/base.ts | 132 +- .../src/node/reporters/benchmark/index.ts | 17 - .../reporters/benchmark/json-formatter.ts | 69 - .../src/node/reporters/benchmark/reporter.ts | 114 -- .../node/reporters/benchmark/tableRender.ts | 221 --- .../src/node/reporters/benchmark/verbose.ts | 5 - packages/vitest/src/node/reporters/index.ts | 6 - packages/vitest/src/node/reporters/json.ts | 4 +- packages/vitest/src/node/reporters/junit.ts | 28 +- .../reporters/renderers/benchmark-table.ts | 80 + .../src/node/reporters/reported-tasks.ts | 10 + packages/vitest/src/node/reporters/summary.ts | 54 +- packages/vitest/src/node/reporters/utils.ts | 34 +- packages/vitest/src/node/reporters/verbose.ts | 6 + packages/vitest/src/node/test-run.ts | 37 +- packages/vitest/src/node/types/benchmark.ts | 46 +- packages/vitest/src/node/types/config.ts | 22 +- packages/vitest/src/node/types/reporter.ts | 8 +- packages/vitest/src/public/index.ts | 24 +- packages/vitest/src/public/node.ts | 4 - packages/vitest/src/runtime/benchmark.ts | 565 ++++++- packages/vitest/src/runtime/config.ts | 33 +- packages/vitest/src/runtime/getter-tracker.ts | 37 + .../runtime/moduleRunner/moduleEvaluator.ts | 28 +- .../moduleRunner/startVitestModuleRunner.ts | 1 + .../vitest/src/runtime/runners/benchmark.ts | 180 -- packages/vitest/src/runtime/runners/index.ts | 7 +- packages/vitest/src/runtime/runners/test.ts | 40 +- .../vitest/src/runtime/types/benchmark.ts | 35 - packages/vitest/src/runtime/worker.ts | 4 + packages/vitest/src/types/global.ts | 45 +- packages/vitest/src/types/rpc.ts | 6 +- packages/vitest/src/types/worker.ts | 2 + packages/vitest/src/utils/config-helpers.ts | 3 +- pnpm-lock.yaml | 17 +- pnpm-workspace.yaml | 1 + .../browser/fixtures/benchmark/basic.bench.ts | 32 - .../fixtures/benchmark/vitest.config.ts | 15 - test/browser/package.json | 1 + test/browser/specs/benchmark.test.ts | 21 - test/browser/specs/runner.test.ts | 73 +- test/browser/specs/utils.ts | 6 +- test/browser/test/browser.bench.ts | 43 + test/browser/vitest.config.mts | 3 + .../fixtures/benchmarking/basic/base.bench.ts | 67 - .../fixtures/benchmarking/basic/mode.bench.ts | 15 - .../fixtures/benchmarking/basic/only.bench.ts | 75 - .../basic/should-not-run.test-d.ts | 7 - .../benchmarking/basic/vitest.config.ts | 3 - .../benchmarking/compare/basic.bench.ts | 13 - .../benchmarking/compare/vitest.config.ts | 3 - .../benchmarking/reporter/multiple.bench.ts | 12 - .../benchmarking/reporter/summary.bench.ts | 32 - .../benchmarking/reporter/vitest.config.ts | 3 - .../benchmarking/sequential/f1.bench.ts | 26 - .../benchmarking/sequential/f2.bench.ts | 9 - .../benchmarking/sequential/helper.ts | 19 - .../fixtures/benchmarking/sequential/setup.ts | 6 - .../benchmarking/sequential/vitest.config.ts | 10 - .../fixtures/custom-pool/pool/custom-pool.ts | 1 + test/e2e/fixtures/mode/example.benchmark.ts | 8 - test/e2e/fixtures/mode/example.test.ts | 5 - .../fixtures/mode/vitest.benchmark.config.ts | 10 - test/e2e/fixtures/mode/vitest.test.config.ts | 10 - .../reporters/function-as-name.bench.ts | 14 +- .../__snapshots__/benchmarking.test.ts.snap | 33 - test/e2e/test/benchmarking.test-d.ts | 103 ++ test/e2e/test/benchmarking.test.ts | 1474 +++++++++++++++-- test/e2e/test/mode.test.ts | 45 - test/e2e/test/public.test.ts | 2 +- .../reporters/__snapshots__/json.test.ts.snap | 1 + .../__snapshots__/reporters.test.ts.snap | 54 + .../test/reporters/function-as-name.test.ts | 8 +- test/e2e/test/reporters/merge-reports.test.ts | 6 + test/e2e/test/reporters/utils.ts | 10 + test/e2e/vitest.config.ts | 3 +- test/node-runner/test/cli.test.js | 2 +- test/test-utils/cli.ts | 2 +- test/test-utils/index.ts | 8 +- test/ui/test/helper.ts | 4 +- test/unit/package.json | 1 + test/unit/test/cli-test.test.ts | 18 - test/unit/test/exports.test.ts | 5 - test/unit/vitest-environment-custom/index.ts | 1 + 136 files changed, 3793 insertions(+), 2126 deletions(-) create mode 100644 docs/guide/benchmarking.md create mode 100644 docs/guide/snippets/benchmark-per-project.ansi create mode 100644 docs/guide/snippets/benchmark-table.ansi create mode 100644 packages/vitest/src/integrations/chai/bench.ts create mode 100644 packages/vitest/src/node/benchmark.ts delete mode 100644 packages/vitest/src/node/reporters/benchmark/index.ts delete mode 100644 packages/vitest/src/node/reporters/benchmark/json-formatter.ts delete mode 100644 packages/vitest/src/node/reporters/benchmark/reporter.ts delete mode 100644 packages/vitest/src/node/reporters/benchmark/tableRender.ts delete mode 100644 packages/vitest/src/node/reporters/benchmark/verbose.ts create mode 100644 packages/vitest/src/node/reporters/renderers/benchmark-table.ts create mode 100644 packages/vitest/src/runtime/getter-tracker.ts delete mode 100644 packages/vitest/src/runtime/runners/benchmark.ts delete mode 100644 packages/vitest/src/runtime/types/benchmark.ts delete mode 100644 test/browser/fixtures/benchmark/basic.bench.ts delete mode 100644 test/browser/fixtures/benchmark/vitest.config.ts delete mode 100644 test/browser/specs/benchmark.test.ts create mode 100644 test/browser/test/browser.bench.ts delete mode 100644 test/e2e/fixtures/benchmarking/basic/base.bench.ts delete mode 100644 test/e2e/fixtures/benchmarking/basic/mode.bench.ts delete mode 100644 test/e2e/fixtures/benchmarking/basic/only.bench.ts delete mode 100644 test/e2e/fixtures/benchmarking/basic/should-not-run.test-d.ts delete mode 100644 test/e2e/fixtures/benchmarking/basic/vitest.config.ts delete mode 100644 test/e2e/fixtures/benchmarking/compare/basic.bench.ts delete mode 100644 test/e2e/fixtures/benchmarking/compare/vitest.config.ts delete mode 100644 test/e2e/fixtures/benchmarking/reporter/multiple.bench.ts delete mode 100644 test/e2e/fixtures/benchmarking/reporter/summary.bench.ts delete mode 100644 test/e2e/fixtures/benchmarking/reporter/vitest.config.ts delete mode 100644 test/e2e/fixtures/benchmarking/sequential/f1.bench.ts delete mode 100644 test/e2e/fixtures/benchmarking/sequential/f2.bench.ts delete mode 100644 test/e2e/fixtures/benchmarking/sequential/helper.ts delete mode 100644 test/e2e/fixtures/benchmarking/sequential/setup.ts delete mode 100644 test/e2e/fixtures/benchmarking/sequential/vitest.config.ts delete mode 100644 test/e2e/fixtures/mode/example.benchmark.ts delete mode 100644 test/e2e/fixtures/mode/example.test.ts delete mode 100644 test/e2e/fixtures/mode/vitest.benchmark.config.ts delete mode 100644 test/e2e/fixtures/mode/vitest.test.config.ts delete mode 100644 test/e2e/test/__snapshots__/benchmarking.test.ts.snap create mode 100644 test/e2e/test/benchmarking.test-d.ts delete mode 100644 test/e2e/test/mode.test.ts diff --git a/.gitignore b/.gitignore index f1b344b6a60c..0d71760bd25e 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ docs/.vitepress/cache/ !test/e2e/fixtures/dotted-files/**/.cache test/**/__screenshots__/**/* test/**/__traces__/**/* +test/browser/**/__benchmarks__ test/browser/fixtures/update-snapshot/basic.test.ts test/e2e/fixtures/browser-multiple/basic-* *.tsbuildinfo diff --git a/.vscode/settings.json b/.vscode/settings.json index 87ca5aaa89eb..2ec80fa65ee7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,7 +23,7 @@ // ], "vitest.ignoreWorkspace": true, - "vitest.configSearchPatternInclude": "test/{core,cli,config,browser,reporters}/{vitest,vite}.{config.ts,config.unit.mts}", + "vitest.configSearchPatternInclude": "test/{unit,e2e,config,browser,reporters}/{vitest,vite}.{config.ts,config.unit.mts}", "testing.automaticallyOpenTestResults": "neverOpen", // Enable eslint for all supported languages diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 0f324f9b121a..6a57443d8011 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -901,6 +901,10 @@ export default ({ mode }: { mode: string }) => { text: 'Testing Types', link: '/guide/testing-types', }, + { + text: 'Benchmarking', + link: '/guide/benchmarking', + }, { text: 'In-Source Testing', link: '/guide/in-source', diff --git a/docs/api/advanced/vitest.md b/docs/api/advanced/vitest.md index 37e335f28ecd..8ae0e43dccfb 100644 --- a/docs/api/advanced/vitest.md +++ b/docs/api/advanced/vitest.md @@ -5,35 +5,9 @@ title: Vitest API # Vitest -Vitest instance requires the current test mode. It can be either: - -- `test` when running runtime tests -- `benchmark` when running benchmarks - -::: details New in Vitest 4 -Vitest 4 added several new APIs (they are marked with a "4.0.0+" badge) and removed deprecated APIs: - -- `invalidates` -- `changedTests` (use [`onFilterWatchedSpecification`](#onfilterwatchedspecification) instead) -- `server` (use [`vite`](#vite) instead) -- `getProjectsByTestFile` (use [`getModuleSpecifications`](#getmodulespecifications) instead) -- `getFileWorkspaceSpecs` (use [`getModuleSpecifications`](#getmodulespecifications) instead) -- `getModuleProjects` (filter by [`this.projects`](#projects) yourself) -- `updateLastChanged` (renamed to [`invalidateFile`](#invalidatefile)) -- `globTestSpecs` (use [`globTestSpecifications`](#globtestspecifications) instead) -- `globTestFiles` (use [`globTestSpecifications`](#globtestspecifications) instead) -- `listFile` (use [`getRelevantTestSpecifications`](#getrelevanttestspecifications) instead) -::: - -## mode - -### test - -Test mode will only call functions inside `test` or `it`, and throws an error when `bench` is encountered. This mode uses `include` and `exclude` options in the config to find test files. - -### benchmark {#benchmark} +## mode {#mode} -Benchmark mode calls `bench` functions and throws an error, when it encounters `test` or `it`. This mode uses `benchmark.include` and `benchmark.exclude` options in the config to find benchmark files. +Since Vitest 5, this property is always `'test'`. ## config diff --git a/docs/api/test.md b/docs/api/test.md index 670f40067eae..e6d074f14c8a 100644 --- a/docs/api/test.md +++ b/docs/api/test.md @@ -670,235 +670,8 @@ Scoped `aroundAll` hook that inherits types from [`test.extend`](#test-extend). ## bench {#bench} -- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void` +::: warning Updated in Vitest 5 +The benchmarking API has been rewritten. `bench` is no longer a top-level import from `vitest`, and the `bench.skip` / `bench.only` / `bench.todo` helpers have been removed. `bench` is now a [test-context fixture](/guide/test-context#bench) accessed from inside a `test()`. -::: danger -Benchmarking is experimental and does not follow SemVer. +See the [Benchmarking guide](/guide/benchmarking) for the new API. ::: - -`bench` defines a benchmark. In Vitest terms, benchmark is a function that defines a series of operations. Vitest runs this function multiple times to display different performance results. - -Vitest uses the [`tinybench`](https://github.com/tinylibs/tinybench) library under the hood, inheriting all its options that can be used as a third argument. - -```ts -import { bench } from 'vitest' - -bench('normal sorting', () => { - const x = [1, 5, 4, 2, 3] - x.sort((a, b) => { - return a - b - }) -}, { time: 1000 }) -``` - -```ts -export interface Options { - /** - * time needed for running a benchmark task (milliseconds) - * @default 500 - */ - time?: number - - /** - * number of times that a task should run if even the time option is finished - * @default 10 - */ - iterations?: number - - /** - * function to get the current timestamp in milliseconds - */ - now?: () => number - - /** - * An AbortSignal for aborting the benchmark - */ - signal?: AbortSignal - - /** - * Throw if a task fails (events will not work if true) - */ - throws?: boolean - - /** - * warmup time (milliseconds) - * @default 100ms - */ - warmupTime?: number - - /** - * warmup iterations - * @default 5 - */ - warmupIterations?: number - - /** - * setup function to run before each benchmark task (cycle) - */ - setup?: Hook - - /** - * teardown function to run after each benchmark task (cycle) - */ - teardown?: Hook -} -``` -After the test case is run, the output structure information is as follows: - -``` - name hz min max mean p75 p99 p995 p999 rme samples -· normal sorting 6,526,368.12 0.0001 0.3638 0.0002 0.0002 0.0002 0.0002 0.0004 ±1.41% 652638 -``` -```ts -export interface TaskResult { - /* - * the last error that was thrown while running the task - */ - error?: unknown - - /** - * The amount of time in milliseconds to run the benchmark task (cycle). - */ - totalTime: number - - /** - * the minimum value in the samples - */ - min: number - /** - * the maximum value in the samples - */ - max: number - - /** - * the number of operations per second - */ - hz: number - - /** - * how long each operation takes (ms) - */ - period: number - - /** - * task samples of each task iteration time (ms) - */ - samples: number[] - - /** - * samples mean/average (estimate of the population mean) - */ - mean: number - - /** - * samples variance (estimate of the population variance) - */ - variance: number - - /** - * samples standard deviation (estimate of the population standard deviation) - */ - sd: number - - /** - * standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean) - */ - sem: number - - /** - * degrees of freedom - */ - df: number - - /** - * critical value of the samples - */ - critical: number - - /** - * margin of error - */ - moe: number - - /** - * relative margin of error - */ - rme: number - - /** - * median absolute deviation - */ - mad: number - - /** - * p50/median percentile - */ - p50: number - - /** - * p75 percentile - */ - p75: number - - /** - * p99 percentile - */ - p99: number - - /** - * p995 percentile - */ - p995: number - - /** - * p999 percentile - */ - p999: number -} -``` - -### bench.skip - -- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void` - -You can use `bench.skip` syntax to skip running certain benchmarks. - -```ts -import { bench } from 'vitest' - -bench.skip('normal sorting', () => { - const x = [1, 5, 4, 2, 3] - x.sort((a, b) => { - return a - b - }) -}) -``` - -### bench.only - -- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void` - -Use `bench.only` to only run certain benchmarks in a given suite. This is useful when debugging. - -```ts -import { bench } from 'vitest' - -bench.only('normal sorting', () => { - const x = [1, 5, 4, 2, 3] - x.sort((a, b) => { - return a - b - }) -}) -``` - -### bench.todo - -- **Type:** `(name: string | Function) => void` - -Use `bench.todo` to stub benchmarks to be implemented later. - -```ts -import { bench } from 'vitest' - -bench.todo('unimplemented test') -``` diff --git a/docs/config/benchmark.md b/docs/config/benchmark.md index 8958912ff280..9b63dc40a92c 100644 --- a/docs/config/benchmark.md +++ b/docs/config/benchmark.md @@ -9,6 +9,13 @@ outline: deep Options used when running `vitest bench`. +## benchmark.enabled + +- **Type:** `boolean` +- **Default:** `false` + +Enables the benchmark project. When set, Vitest creates a dedicated benchmark project alongside your regular test project, runs files matching [`benchmark.include`](#benchmark-include) in it, and exposes the [`bench` fixture](/guide/test-context#bench) to those files. Running `vitest bench` enables this automatically. + ## benchmark.include - **Type:** `string[]` @@ -32,39 +39,18 @@ Include globs for in-source benchmark test files. This option is similar to [`in When defined, Vitest will run all matched files with `import.meta.vitest` inside. -## benchmark.reporters - -- **Type:** `Arrayable` -- **Default:** `'default'` - -Custom reporter for output. Can contain one or more built-in report names, reporter instances, and/or paths to custom reporters. - -## benchmark.outputFile - -Deprecated in favor of `benchmark.outputJson`. - -## benchmark.outputJson {#benchmark-outputJson} - -- **Type:** `string | undefined` -- **Default:** `undefined` +## benchmark.retainSamples -A file path to store the benchmark result, which can be used for `--compare` option later. +- **Type:** `boolean` +- **Default:** `false` -For example: +Include the `samples` array of per-iteration timings on every benchmark result. Disabled by default to reduce memory usage; enable when a custom reporter or API consumer needs the raw samples. -```sh -# save main branch's result -git checkout main -vitest bench --outputJson main.json -# change a branch and compare against main -git checkout feature -vitest bench --compare main.json -``` +## benchmark.suppressExportGetterWarnings -## benchmark.compare {#benchmark-compare} +- **Type:** `boolean` +- **Default:** `false` -- **Type:** `string | undefined` -- **Default:** `undefined` +Suppress the warning printed when a benchmark accesses module export getters too many times. Vitest tracks getter access during benchmark runs because Vite's module runner wraps every export in a getter, and excessive access can dominate the measurement (see [Module Runner Overhead](/guide/benchmarking#module-runner-overhead)). Enable this when you've intentionally accepted the overhead, or when the warning is noisy for benchmarks where the getter cost is negligible. -A file path to a previous benchmark result to compare against current runs. diff --git a/docs/config/index.md b/docs/config/index.md index e7182476064b..3a4778238644 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -8,7 +8,7 @@ If you are using Vite and have a `vite.config` file, Vitest will read it to matc - Create `vitest.config.ts`, which will have the higher priority and will **override** the configuration from `vite.config.ts` (Vitest supports all conventional JS and TS extensions, but doesn't support `json`) - it means all options in your `vite.config` will be **ignored** - Pass `--config` option to CLI, e.g. `vitest --config ./path/to/vitest.config.ts` -- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test`/`benchmark` if not overridden with `--mode`) to conditionally apply different configuration in `vite.config.ts`. Note that like any other environment variable, `VITEST` is also exposed on `import.meta.env` in your tests +- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test` if not overridden with `--mode`) to conditionally apply different configuration in `vite.config.ts`. Note that like any other environment variable, `VITEST` is also exposed on `import.meta.env` in your tests To configure `vitest` itself, add `test` property in your Vite config. You'll also need to add a reference to Vitest types using a [triple slash command](https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html#-reference-types-) at the top of your config file, if you are importing `defineConfig` from `vite` itself. diff --git a/docs/config/runner.md b/docs/config/runner.md index 1d47b9537dd7..cdcc3100d7c6 100644 --- a/docs/config/runner.md +++ b/docs/config/runner.md @@ -6,6 +6,5 @@ outline: deep # runner - **Type:** `VitestRunnerConstructor` -- **Default:** `node`, when running tests, or `benchmark`, when running benchmarks Path to a custom test runner. This is an advanced feature and should be used with custom library runners. You can read more about it in [the documentation](/api/advanced/runner). diff --git a/docs/guide/advanced/index.md b/docs/guide/advanced/index.md index 01bca471a9d2..f796eb5e185c 100644 --- a/docs/guide/advanced/index.md +++ b/docs/guide/advanced/index.md @@ -14,7 +14,6 @@ You can import any method from the `vitest/node` entry-point. ```ts function startVitest( - mode: VitestRunMode, cliFilters: string[] = [], options: CliOptions = {}, viteOverrides?: ViteUserConfig, @@ -27,7 +26,7 @@ You can start running Vitest tests using its Node API: ```js import { startVitest } from 'vitest/node' -const vitest = await startVitest('test') +const vitest = await startVitest() await vitest.close() ``` @@ -47,7 +46,7 @@ After running the tests, you can get the results from the [`state.getTestModules ```ts import type { TestModule } from 'vitest/node' -const vitest = await startVitest('test') +const vitest = await startVitest() console.log(vitest.state.getTestModules()) // [TestModule] ``` @@ -60,7 +59,6 @@ The ["Running Tests"](/guide/advanced/tests#startvitest) guide has a usage examp ```ts function createVitest( - mode: VitestRunMode, options: CliOptions, viteOverrides: ViteUserConfig = {}, vitestOptions: VitestOptions = {}, diff --git a/docs/guide/benchmarking.md b/docs/guide/benchmarking.md new file mode 100644 index 000000000000..bfa5df8798a3 --- /dev/null +++ b/docs/guide/benchmarking.md @@ -0,0 +1,480 @@ +--- +title: Benchmarking | Guide +--- + +# Benchmarking + +Vitest lets you write benchmarks alongside your tests using the `bench` fixture from the [test context](/guide/test-context). Benchmarks are powered by [Tinybench](https://github.com/tinylibs/tinybench) and are defined inside regular `test()` calls, giving you access to the full power of Vitest's test runner: retries, lifecycle hooks, filtering, and assertions. + +## Defining a Benchmark + +Use the `bench` fixture to define a benchmark. Call `.run()` to execute it: + +```ts +import { expect, test } from 'vitest' + +test('parsing performance', async ({ bench }) => { + const result = await bench('parse', () => { + JSON.parse('{"key":"value"}') + }).run() +}) +``` + +The `bench()` function registers a benchmark without executing it. Calling `.run()` runs the benchmark and returns the result. After the test completes, Vitest prints a single-row version of the [comparison table](#comparing-benchmarks) (ops/sec, mean time, percentiles, etc.), so you get the same output for a one-off benchmark as you do for `bench.compare()`. + +::: warning +The `bench` fixture is only available in files matched by [`benchmark.include`](/config/#benchmark-include) (default: `**/*.{bench,benchmark}.?(c|m)[jt]s?(x)`). Using `{ bench }` inside a regular test file will throw an error. + +Whether a file participates in the benchmark run is decided by the filename, not by whether the test uses the `bench` fixture. Renaming `parser.test.ts` to `parser.bench.ts` (or adjusting `benchmark.include`) is what moves it into the benchmark project. +::: + +## Running Benchmarks + +Benchmark files are matched by [`benchmark.include`](/config/#benchmark-include) (default: `**/*.{bench,benchmark}.?(c|m)[jt]s?(x)`) and run in their own project, separate from your regular tests. There are three ways to run them, depending on whether you want to skip them, run them alongside tests, or run them on their own. + +### `vitest` (default) + +Without [`benchmark.enabled`](/config/#benchmark-enabled), the `vitest` command only runs regular tests. Benchmark files are ignored entirely. This is the default and the right choice for day-to-day development, since benchmarks are slow and noisy and shouldn't run on every save. + +### `vitest` with `benchmark.enabled` + +Set `benchmark.enabled: true` in your config to run benchmarks together with regular tests: + +```ts [vitest.config.ts] +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + benchmark: { + enabled: true, + }, + }, +}) +``` + +With this config, `vitest` runs your regular tests first, then runs the benchmarks in a separate isolated group (so benchmark execution never overlaps with test execution and adds noise to results). Useful in CI when you want a single command to validate correctness and performance. + +### `vitest bench` + +The `bench` subcommand runs only benchmarks and skips regular tests: + +```bash +vitest bench +``` + +This implicitly enables `benchmark.enabled` for the run, so you don't need to set it in the config. Like the `vitest` command, it accepts filename filters and `-t`/`--testNamePattern` to narrow the run: + +```bash +# only benchmarks in files matching "parser" +vitest bench parser + +# only benchmarks whose test name matches "JSON" +vitest bench -t JSON +``` + +## Comparing Benchmarks + +Use `bench.compare()` to compare multiple benchmarks against each other: + +```ts +import { expect, test } from 'vitest' + +test('compare JSON libraries', async ({ bench }) => { + const input = '{"key":"value","nested":{"a":1}}' + + const result = await bench.compare( + bench('JSON.parse', () => { + JSON.parse(input) + }), + bench('custom parser', () => { + customParse(input) + }), + ) +}) +``` + +When comparing benchmarks, Vitest runs them using interleaved iterations to reduce environmental bias (CPU throttling, GC pressure, etc.) and prints a comparison table after the test completes: + +<<< ./snippets/benchmark-table.ansi + +### Options + +You can pass [options](https://tinylibs.github.io/tinybench/interfaces/BenchOptions.html) as the last argument to `bench.compare()`: + +```ts +test('compare with options', async ({ bench }) => { + const result = await bench.compare( + bench('lib1', () => { lib1() }), + bench('lib2', () => { lib2() }), + { + iterations: 100, + time: 1000, + }, + ) +}) +``` + +You can also pass per-benchmark [options](https://tinylibs.github.io/tinybench/interfaces/FnOptions.html) as the second argument, matching how `test()` accepts options: + +```ts +test('benchmarks with setup', async ({ bench }) => { + const result = await bench.compare( + bench('with-cache', () => { + readFromCache() + }), + bench( + 'without-cache', + { beforeEach: () => clearCache() }, + () => { readFromDisk() }, + ), + ) +}) +``` + +## Comparing Across Projects + +When your workspace defines multiple projects (e.g., different browsers or runtimes), pass `perProject: true` in the bench options to compare how the same benchmark performs across all of them. Vitest still prints the result inline for the current project, and additionally collects per-project results into a single comparison table at the end of the test run. + +```ts +import { test } from 'vitest' + +test('simple example', async ({ bench }) => { + await bench('1 + 1', { perProject: true }, () => { + 1 + 1 + }).run() +}) +``` + +The same test file runs in each project (chromium, firefox, webkit, etc.), and Vitest groups the results: + +<<< ./snippets/benchmark-per-project.ansi + +You can also mix `perProject` benchmarks with regular ones inside `bench.compare()`: + +```ts +test('compare implementations across browsers', async ({ bench }) => { + await bench.compare( + bench('JSON.parse', { perProject: true }, () => { + JSON.parse('{"key":"value"}') + }), + bench('custom parser', () => { + customParse('{"key":"value"}') + }), + ) +}) +``` + +In this case, `custom parser` appears in the normal inline comparison table per project, while `JSON.parse` is additionally collected into the cross-project comparison table at the end. + +## Asserting Performance + +Use `toBeFasterThan()` and `toBeSlowerThan()` matchers to assert relative performance between benchmarks: + +```ts +import { expect, test } from 'vitest' + +test('lib1 is faster than lib2', async ({ bench }) => { + const result = await bench.compare( + bench('lib1', () => { lib1() }), + bench('lib2', () => { lib2() }), + ) + + expect(result.get('lib1')).toBeFasterThan(result.get('lib2')) +}) +``` + +The `delta` option specifies the minimum relative difference required for the assertion to pass. This helps avoid flaky tests caused by benchmark noise: + +```ts +// lib1 must be at least 10% faster than lib2 +expect(result.get('lib1')).toBeFasterThan(result.get('lib2'), { + delta: 0.1, +}) + +// lib2 must be at least 20% slower than lib1 +expect(result.get('lib2')).toBeSlowerThan(result.get('lib1'), { + delta: 0.2, +}) +``` + +You can also assert absolute performance using standard matchers: + +```ts +test('parsing is fast enough', async ({ bench }) => { + const result = await bench('parse', () => { + parse(largeInput) + }).run() + + expect(result.throughput.mean).toBeGreaterThan(10_000) +}) +``` + +## Retries + +Since benchmarks can be noisy, use the `retry` option to automatically retry failing benchmark tests: + +```ts +test('performance comparison', { retry: 3 }, async ({ bench }) => { + const result = await bench.compare( + bench('lib1', () => { lib1() }), + bench('lib2', () => { lib2() }), + ) + + expect(result.get('lib1')).toBeFasterThan(result.get('lib2')) +}) +``` + +## Storing and Replaying Results + +Two primitives let you persist benchmark results to disk and compare against them in future runs: the `writeResult` option saves a result, and `bench.from()` reads one back. + +### `writeResult` + +Pass `writeResult` as a per-bench option to write the result to a JSON file every time the benchmark runs. The path is resolved against the project root: + +```ts +test('parse', async ({ bench }) => { + await bench( + 'parse', + { writeResult: './benchmarks/parse.json' }, + () => parse(largeInput), + ).run() +}) +``` + +- The benchmark always runs. There is no skip-when-cached behaviour and no CLI flag, the file is overwritten on every successful run. +- If the function throws, the file is not written. +- Commit these files alongside your code so reviewers and CI share the same reference points. + +::: warning +If you commit these files, keep in mind that benchmark results vary significantly between environments (developer machines, CI runners, different OSes). Designate a single environment (typically CI) to generate the file, and avoid regenerating it locally. +::: + +### `bench.from()` + +`bench.from(name, source)` is a registration that doesn't execute a function. It reads a previously stored result and feeds it into `bench.compare()` (or returns it directly when you call `.run()`). + +The source can be a path (relative to the project root) or a function that returns the result data, including a Promise: + +```ts +test('compare against the stored baseline', async ({ bench }) => { + const result = await bench.compare( + bench( + 'current', + { writeResult: './benchmarks/parse.json' }, + () => parse(largeInput), + ), + bench.from('previous', './benchmarks/parse.json'), + bench.from('remote', () => fetch('https://path/to/external/file.json').then(r => r.json())), + ) + + expect(result.get('current')).toBeFasterThan(result.get('previous')) +}) +``` + +You can keep historical artifacts for older versions and compare them against the current implementation. Because `bench.from()` never invokes the function that produced the file, the original benchmark code can be deleted once the artifact is committed: + +```ts +test('compare parser versions', async ({ bench }) => { + const input = '{"key":"value"}' + + await bench.compare( + bench.from('v1', './benchmarks/parse.v1.json'), + bench.from('v2', './benchmarks/parse.v2.json'), + bench( + 'current', + { writeResult: './benchmarks/parse.current.json' }, + () => customParser(input), + ), + ) +}) +``` + +To produce a new historical artifact, point a fresh `bench()` at that version's implementation, set `writeResult` to a versioned path (`./benchmarks/parse.v3.json`), run it once, then replace the call with `bench.from('v3', './benchmarks/parse.v3.json')`. + +To regenerate the baseline on demand, gate the write behind an environment variable so the same test either refreshes the artifact or compares against it: + +```ts +test('compare parser versions', async ({ bench }) => { + if (import.meta.env.VITE_WRITE_BENCH) { + const baseline = bench('baseline', { writeResult: './my-bench.json' }, () => fn()) + await baseline.run() + } + else { + const baseline = bench.from('baseline', './my-bench.json') + await bench.compare(bench('current', () => fn()), baseline) + } +}) +``` + +Run `VITE_WRITE_BENCH=1 vitest bench` to refresh the stored result, and `vitest bench` to compare the current implementation against it. + +### Per-project artifacts + +In a multi-project workspace (different browsers, different runtimes), share one benchmark file across projects by including `${projectName}` in the path. The placeholder is substituted with the current project name at write time: + +```ts +test('cross-project baseline', async ({ bench }) => { + await bench( + 'parse', + // eslint-disable-next-line no-template-curly-in-string + { perProject: true, writeResult: './benchmarks/parse.${projectName}.json' }, + () => parse(largeInput), + ).run() +}) +``` + +Use the same template in `bench.from()` so each project reads its own artifact. + +## Stability + +Benchmarks are inherently flaky: CPU load, thermal throttling, GC pressure, and background processes all affect results. Vitest takes several steps to minimize this noise: + +- **Separate project**: Benchmark files are grouped into their own project based on the [`benchmark.include`](/config/#benchmark-include) pattern. The `bench` fixture is only exposed in files matched by that pattern. Using it inside a regular test file will throw an error. +- **No concurrency**: Tests within a benchmark file always run sequentially. Benchmark files themselves also run one at a time, never in parallel. This prevents benchmarks from interfering with each other. + +To further improve stability: + +- Use the [`retry`](#retries) option to automatically rerun flaky benchmark assertions. +- Use the [`delta`](#asserting-performance) option in `toBeFasterThan` / `toBeSlowerThan` to allow for acceptable variance. +- Avoid running benchmarks alongside CPU-intensive processes. +- Close browsers, IDEs, and other applications that compete for CPU time. + +### Dead Code Elimination + +JavaScript engines can optimize away code that has no observable side effects. If your benchmark function doesn't use its result, the engine may skip the computation entirely, producing misleadingly fast numbers: + +```ts +test('parsing', async ({ bench }) => { + // BAD: the engine may eliminate the work + await bench('parse', () => { + JSON.parse(input) + }).run() + + // GOOD: the result is consumed + await bench('parse', () => { + const result = JSON.parse(input) + doSomething(result) + }).run() +}) +``` + +This applies to all engines (V8, JavaScriptCore, SpiderMonkey) but is especially aggressive in V8's TurboFan and JavaScriptCore's FTL compiler tiers. + +### Module Runner Overhead + +By default, Vitest runs tests in Node.js using Vite's module runner (configured by [`experimental.viteModuleRunner`](/config/experimental#experimental-vitemodulerunner)). This transforms all module exports into getters, so every access to an imported binding goes through something like `__vite_ssr_module__.value`. In regular tests this overhead is negligible, but in benchmarks where a function is called millions of times, the getter call itself can dominate the measurement. + +Vitest will print a warning if it detects excessive getter calls (which you can silence via [`benchmark.suppressExportGetterWarnings`](/config/benchmark#benchmark-suppressexportgetterwarnings)), but you should be aware of this when benchmarking imported functions: + +```ts +import { parse } from './parser.js' + +const _parse = parse + +test('parsing', async ({ bench }) => { + // BAD: every call to `parse` goes through a getter + await bench('parse', () => { + parse(input) + }).run() + + // GOOD: store the reference locally to bypass the getter + await bench('parse', () => { + _parse(input) + }).run() +}) +``` + +If you are the library author, the same overhead applies inside the library you are benchmarking: every cross-module call within its source goes through the same getter wrapper. If you are benchmarking your own library, you have two ways to remove this: + +**Benchmark the pre-built artifact.** Import the library through its package name (which resolves to its built output) instead of reaching into its source. The built file has already collapsed internal imports into direct references, so Vite's module runner sees a single module with no internal getters: + +```ts +// BAD: every internal call inside the library goes through a getter +import { parse } from '../src/index.ts' + +// GOOD: the published entry has no internal getters +import { parse } from 'my-library' +``` + +If you compare your library against other packages, benchmark the same kind of artifact for every implementation. For workspace packages, make sure the package name resolves to the built output instead of source, for example by externalizing the package in Vite or by importing from `dist`. + +**Disable the module runner for the benchmark.** If the benchmark does not need Vite transforms, mocks, or Vitest module interception, disable [`experimental.viteModuleRunner`](/config/experimental#experimental-vitemodulerunner) for the benchmark project so Node runs native ESM directly. + +This only affects Node.js mode. Browser mode uses native ESM imports and does not have this overhead. + +### Engine-Specific Considerations + +#### V8 (Node.js, Chrome) + +- **JIT tiering**: V8 compiles functions through multiple optimization tiers (Sparkplug → Maglev → TurboFan). A function may run at different speeds during warmup vs. steady-state. Tinybench handles warmup automatically, but very short benchmark runs may not reach the highest optimization tier. +- **Deoptimization**: V8 can "bail out" of optimized code mid-benchmark if it encounters unexpected types or shapes. Keep the types consistent in your benchmark function: + + ```ts + test('process items', async ({ bench }) => { + // BAD: mixed shapes cause deoptimization + await bench('process', () => { + for (const item of items) { + // some items have { name: string }, others have { name: string, id: number } + process(item) + } + }).run() + + // GOOD: consistent object shapes + await bench('process', () => { + for (const item of items) { + // all items have the same shape { name: string, id: number } + process(item) + } + }).run() + }) + ``` + +- **Garbage collection**: Large allocations inside the benchmark loop add GC noise. If you're measuring computation, pre-allocate data in a `setup` hook rather than inside the benchmarked function: + + ```ts + test('sorting', async ({ bench }) => { + const original = Array.from({ length: 10000 }, () => Math.random()) + let data: number[] + + // BAD: allocates a new array every iteration, GC adds noise + await bench('sort', () => { + const data = Array.from({ length: 10000 }, () => Math.random()) + data.sort() + }).run() + + // GOOD: pre-allocate, copy in beforeEach + await bench( + 'sort', + () => { data.sort() }, + { + beforeEach() { + data = [...original] + }, + }, + ).run() + }) + ``` + +#### JavaScriptCore (Bun, Safari) + +- **Different optimization thresholds**: JSC uses its own JIT tiers (LLInt → Baseline → DFG → FTL) with different inlining and optimization heuristics. A benchmark that is fast on V8 may behave very differently on JSC. +- **Async benchmarks**: Bun's event loop implementation differs from Node.js. If your benchmark involves async operations or timers, results may not be directly comparable across runtimes. + +#### Browser + +- **Timer resolution**: Browsers may reduce `performance.now()` precision (e.g., to 100μs or even 1ms) as a security mechanism. This makes very fast operations difficult to measure accurately, so increase iterations to compensate: + + ```ts + test('fast operations', async ({ bench }) => { + await bench.compare( + bench('fast-op', () => { fastOp() }), + bench('other-op', () => { otherOp() }), + { + // more iterations help overcome low timer resolution + iterations: 1000, + }, + ) + }) + ``` +- **Cross-browser differences**: V8 (Chrome), SpiderMonkey (Firefox), and JSC (Safari) optimize different patterns differently. A benchmark that shows one library winning in Chrome may show the opposite in Firefox. diff --git a/docs/guide/cli-generated.md b/docs/guide/cli-generated.md index 32484b6b3995..7538e4f2e72f 100644 --- a/docs/guide/cli-generated.md +++ b/docs/guide/cli-generated.md @@ -311,7 +311,7 @@ Track coverage of the `node:child_process` and `node:worker_threads` spawned dur - **CLI:** `--mode ` - **Config:** [mode](/config/mode) -Override Vite mode (default: `test` or `benchmark`) +Override Vite mode (default: `test`) ### isolate diff --git a/docs/guide/migration.md b/docs/guide/migration.md index 2943981a15ab..23e81f9bfb9c 100644 --- a/docs/guide/migration.md +++ b/docs/guide/migration.md @@ -13,6 +13,36 @@ outline: deep Vitest 5.0 is currently in beta. This section tracks breaking changes as they are merged and may change before the stable release. ::: +### Benchmarking API Rewrite + +The benchmarking API has been rewritten. `bench` is no longer a top-level import from `vitest`; it is a [test-context fixture](/guide/test-context#bench) accessed from inside a regular `test()`. See the [Benchmarking guide](/guide/benchmarking) for the new API. + +Removed, with replacements where applicable: + +- **`bench(name, fn)` at module scope**: destructure `bench` from the test context instead. + +```ts +// v4 +import { bench } from 'vitest' // [!code --] + +bench('sort', () => { // [!code --] + [3, 1, 2].sort() // [!code --] +}) // [!code --] + +// v5 +import { test } from 'vitest' // [!code ++] + +test('sort', async ({ bench }) => { // [!code ++] + await bench('sort', () => { [3, 1, 2].sort() }).run() // [!code ++] +}) // [!code ++] +``` + +- **`bench.skip`, `bench.only`, `bench.todo`** are removed. Use the regular `test.skip`, `test.only`, `test.todo` on the surrounding `test()` instead. +- **`benchmark.reporters` / `benchmark.outputFile`** are removed. Benchmark output is now part of the default reporter and the `json` reporter; configure those at the top level via `test.reporters` instead. +- **`benchmark.compare` config and the `--compare` CLI flag** are removed. Pass [`writeResult`](/guide/benchmarking#storing-and-replaying-results) as a per-bench option to persist a result, and read it back with [`bench.from()`](/guide/benchmarking#bench-from) inside `bench.compare()`. +- **`benchmark.outputJson` config and the `--outputJson` CLI flag** are removed. Use `--reporter=json --outputFile=` to capture benchmark results; the JSON reporter now includes a `benchmarks` field on each test case. +- **`Vitest` instance `mode` property** is now always `'test'`. The previous `'benchmark'` value is no longer used; benchmarks run inside a dedicated project of the same `Vitest` instance. + ### Removed `test.sequential`, `describe.sequential`, and `sequential` Options Vitest 5.0 removes the deprecated `test.sequential`, `describe.sequential`, and `sequential` test options. Use `concurrent: false` when you need a test or suite to opt out of inherited or globally configured concurrency. diff --git a/docs/guide/snippets/benchmark-per-project.ansi b/docs/guide/snippets/benchmark-per-project.ansi new file mode 100644 index 000000000000..2eb283f35af9 --- /dev/null +++ b/docs/guide/snippets/benchmark-per-project.ansi @@ -0,0 +1,7 @@ + Cross-Project Benchmark Comparison + + test/parser.bench.ts > regular parsing > parsing + name   hz  min  max  mean  p75  p99  p995  p999  rme samples + webkit (bench) 6,730,808.15 0.0000 1.0000 0.0001 0.0000 0.0000 0.0000 0.0000 ±6.20% 6731808 fastest + chromium (bench) 3,841,599.95 0.0000 0.2000 0.0003 0.0000 0.0000 0.0000 0.1000 ±1.96% 3851571 + firefox (bench) 3,546,066.28 0.0000 5.0000 0.0003 0.0000 0.0000 0.0000 0.0000 ±6.26% 3547062 slowest diff --git a/docs/guide/snippets/benchmark-table.ansi b/docs/guide/snippets/benchmark-table.ansi new file mode 100644 index 000000000000..8afa07702764 --- /dev/null +++ b/docs/guide/snippets/benchmark-table.ansi @@ -0,0 +1,9 @@ + ✓  bench  test/basic.bench.ts (1 test) 3446ms + ✓ different libraries 3445ms + name   hz  min  max  mean  p75  p99  p995  p999  rme samples + lib 1 40,678,348.31 0.0000 0.0002 0.0000 0.0000 0.0000 0.0000 0.0000 ±0.95%  53989 fastest + lib 3 39,152,132.23 0.0000 0.0002 0.0000 0.0000 0.0000 0.0000 0.0000 ±0.93%  52080 + lib 2 38,088,138.97 0.0000 0.0066 0.0000 0.0000 0.0000 0.0000 0.0000 ±1.58%  50618 slowest + + Test Files  1 passed (1) + Tests  1 passed (1) diff --git a/docs/guide/test-context.md b/docs/guide/test-context.md index f1d775ee4f79..3bc41c23745b 100644 --- a/docs/guide/test-context.md +++ b/docs/guide/test-context.md @@ -117,6 +117,29 @@ it('stop request when test times out', async ({ signal }) => { }, 2000) ``` +### `bench` 5.0.0 {#bench} + +The `bench` fixture lets you define and run benchmarks inside regular tests. You can measure throughput, compare implementations, and assert relative performance: + +```ts +import { expect, test } from 'vitest' + +test('compare parsers', async ({ bench }) => { + const result = await bench.compare( + bench('JSON.parse', () => { + JSON.parse('{"key":"value"}') + }), + bench('custom parser', () => { + customParse('{"key":"value"}') + }), + ) + + expect(result.get('JSON.parse')).toBeFasterThan(result.get('custom parser')) +}) +``` + +See the [Benchmarks guide](/guide/benchmarking) for full documentation on comparisons, baselines, and assertion matchers. + ### `onTestFailed` The [`onTestFailed`](/api/hooks#ontestfailed) hook bound to the current test. This API is useful if you are running tests concurrently and need to have a special handling only for this specific test. diff --git a/docs/guide/test-tags.md b/docs/guide/test-tags.md index eea36179c006..5ec18b0a566c 100644 --- a/docs/guide/test-tags.md +++ b/docs/guide/test-tags.md @@ -233,7 +233,7 @@ If you are using a programmatic API, you can pass down a `tagsFilter` option to ```ts import { startVitest } from 'vitest/node' -await startVitest('test', [], { +await startVitest([], { tagsFilter: ['frontend and backend'], }) ``` diff --git a/eslint.config.js b/eslint.config.js index 2ccb501c8045..e8d27b650ab2 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -122,6 +122,7 @@ export default antfu( 'no-self-compare': 'off', 'import/no-mutable-exports': 'off', 'no-throw-literal': 'off', + 'import/no-duplicates': 'off', }, }, { diff --git a/packages/browser/src/client/tester/runner.ts b/packages/browser/src/client/tester/runner.ts index f6fc36feb8b0..c3f08d1c6858 100644 --- a/packages/browser/src/client/tester/runner.ts +++ b/packages/browser/src/client/tester/runner.ts @@ -8,15 +8,15 @@ import type { Test, TestAnnotation, TestArtifact, + TestTryOptions, VitestRunner, } from '@vitest/runner' import type { SerializedConfig, TestExecutionMethod, WorkerGlobalState } from 'vitest' -import type { Traces } from 'vitest/internal/traces' import type { VitestBrowserClientMocker } from './mocker' import type { CommandsManager } from './tester-utils' import { globalChannel, onCancel } from '@vitest/browser/client' import { getTestName } from '@vitest/runner/utils' -import { BenchmarkRunner, recordArtifact, TestRunner } from 'vitest' +import { recordArtifact, TestRunner } from 'vitest' import { page, userEvent } from 'vitest/browser' import { DecodedMap, @@ -48,18 +48,16 @@ interface BrowserVitestRunner extends VitestRunner { } export function createBrowserRunner( - runnerClass: { new (config: SerializedConfig): VitestRunner }, mocker: VitestBrowserClientMocker, state: WorkerGlobalState, - coverageModule: CoverageHandler | null, + coverageModule: CoverageHandler, ): { new (options: BrowserRunnerOptions): BrowserVitestRunner } { - return class BrowserTestRunner extends runnerClass implements VitestRunner { + return class BrowserTestRunner extends TestRunner implements VitestRunner { public config: SerializedConfig - hashMap = browserHashMap + public hashMap = browserHashMap public sourceMapCache = new Map() public method = 'run' as TestExecutionMethod private commands: CommandsManager - private _otel!: Traces constructor(options: BrowserRunnerOptions) { super(options.config) @@ -75,12 +73,11 @@ export function createBrowserRunner( private traces = new Map() - onBeforeTryTask: VitestRunner['onBeforeTryTask'] = async (...args) => { + async onBeforeTryTask(test: Test, options: TestTryOptions) { await userEvent.cleanup() - await super.onBeforeTryTask?.(...args) + super.onBeforeTryTask?.(test, options) const trace = this.config.browser.trace - const test = args[0] - const { retry, repeats } = args[1] + const { retry, repeats } = options const shouldTrace = trace !== 'off' && !(trace === 'on-all-retries' && retry === 0) && !(trace === 'on-first-retry' && retry !== 1) @@ -152,7 +149,7 @@ export function createBrowserRunner( } onAfterRunTask = async (task: Test) => { - await super.onAfterRunTask?.(task) + super.onAfterRunTask?.(task) const trace = this.config.browser.trace const traces = this.traces.get(task.id) || [] if (traces.length) { @@ -230,10 +227,11 @@ export function createBrowserRunner( } onAfterRunFiles = async (files: File[]) => { + super.onAfterRunFiles(files) + const [coverage] = await Promise.all([ - coverageModule?.takeCoverage?.(), + coverageModule.takeCoverage(), mocker.invalidate(), - super.onAfterRunFiles?.(files), ]) if (coverage) { @@ -357,10 +355,7 @@ export async function initiateRunner( if (cachedRunner) { return cachedRunner } - const runnerClass - = config.mode === 'test' ? TestRunner : BenchmarkRunner - - const BrowserRunner = createBrowserRunner(runnerClass, mocker, state, { + const BrowserRunner = createBrowserRunner(mocker, state, { takeCoverage: () => takeCoverageInsideWorker(config.coverage, moduleRunner), }) @@ -377,10 +372,10 @@ export async function initiateRunner( }) const [diffOptions] = await Promise.all([ - loadDiffConfig(config, moduleRunner as any), - loadSnapshotSerializers(config, moduleRunner as any), + loadDiffConfig(config, moduleRunner), + loadSnapshotSerializers(config, moduleRunner), ]) - runner.config.diffOptions = diffOptions + runner.config._diffOptions = diffOptions getWorkerState().onFilterStackTrace = (stack: string) => { const stacks = parseStacktrace(stack, { getSourceMap(file) { @@ -400,7 +395,7 @@ async function getTraceMap(file: string, sourceMaps: Map) { if (!result) { return null } - return new DecodedMap(result as any, file) + return new DecodedMap(result, file) } async function updateTestFilesLocations(files: File[], sourceMaps: Map) { diff --git a/packages/browser/src/node/plugin.ts b/packages/browser/src/node/plugin.ts index b6fdd547d55c..ff89517703b2 100644 --- a/packages/browser/src/node/plugin.ts +++ b/packages/browser/src/node/plugin.ts @@ -168,7 +168,27 @@ export default (parentServer: ParentBrowserProject, base = '/'): Plugin[] => { // this plugin can be used in different projects, but all of them // have the same `include` pattern, so it doesn't matter which project we use const project = parentServer.project - const { testFiles: browserTestFiles } = await project.globTestFiles() + // only glob benchmarks when a browser-enabled bench project exists in + // the workspace — keeps the optimizeDeps entries symmetrical across + // the test/bench project clones so the optimizer doesn't re-scan when + // the user switches modes + const hasBrowserBenchProject = parentServer.vitest.projects.some(p => + p.config.browser.enabled && p.config.benchmark.enabled, + ) + const benchInclude = hasBrowserBenchProject + ? project.config.benchmark.include + : [] + const dir = project.config.dir || project.config.root + const [{ testFiles: browserTestFiles }, browserBenchFiles] = await Promise.all([ + project.globTestFiles(), + benchInclude.length > 0 + ? project.globFiles( + benchInclude, + project.config.benchmark.exclude ?? project.config.exclude, + dir, + ) + : [], + ]) const setupFiles = toArray(project.config.setupFiles) // replace env values - cannot be reassign at runtime @@ -179,7 +199,7 @@ export default (parentServer: ParentBrowserProject, base = '/'): Plugin[] => { } const entries: string[] = [ - ...browserTestFiles, + ...new Set([...browserTestFiles, ...browserBenchFiles]), ...setupFiles, resolve(vitestDist, 'index.js'), resolve(vitestDist, 'browser.js'), diff --git a/packages/browser/src/node/rpc.ts b/packages/browser/src/node/rpc.ts index eadf29d8701f..b3498920fe92 100644 --- a/packages/browser/src/node/rpc.ts +++ b/packages/browser/src/node/rpc.ts @@ -202,6 +202,23 @@ export function setupBrowserRpc(globalServer: ParentBrowserProject, defaultMocke return vitest._testRun.recordArtifact(id, artifact) }, + async onTestBenchmark(testId, benchmark) { + return vitest._testRun.recordBenchmark(testId, benchmark) + }, + async readBenchmarkResult(relativePath) { + checkFileAccess(project.benchmark.resolve(relativePath)) + return project.benchmark.readResult(relativePath) + }, + async writeBenchmarkResult(relativePath, data) { + if (!canWrite(project)) { + vitest.logger.error( + `[vitest] Cannot write benchmark artifact "${relativePath}" because file writing is disabled. See https://vitest.dev/config/browser/api.`, + ) + return + } + checkFileAccess(project.benchmark.resolve(relativePath)) + return project.benchmark.writeResult(relativePath, data) + }, async onTaskUpdate(method, packs, events) { if (method === 'collect') { vitest.state.updateTasks(packs) diff --git a/packages/browser/src/types.ts b/packages/browser/src/types.ts index 5098cb5f03b2..4e26dfe0b3c7 100644 --- a/packages/browser/src/types.ts +++ b/packages/browser/src/types.ts @@ -1,5 +1,5 @@ import type { MockedModuleSerialized, ServerIdResolution, ServerMockResolution } from '@vitest/mocker' -import type { TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner' +import type { BaselineData, TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner' import type { BirpcReturn } from 'birpc' import type { AfterSuiteRunMeta, @@ -8,6 +8,7 @@ import type { RunnerTestFile, SerializedTestSpecification, SnapshotResult, + TestBenchmark, TestExecutionMethod, UserConsoleLog, } from 'vitest' @@ -21,6 +22,9 @@ export interface WebSocketBrowserHandlers { onCollected: (method: TestExecutionMethod, files: RunnerTestFile[]) => Promise onTaskArtifactRecord: (testId: string, artifact: Artifact) => Promise onTaskUpdate: (method: TestExecutionMethod, packs: TaskResultPack[], events: TaskEventPack[]) => void + onTestBenchmark: (testId: string, benchmark: TestBenchmark) => void + readBenchmarkResult: (relativePath: string) => Promise + writeBenchmarkResult: (relativePath: string, data: BaselineData) => Promise onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void cancelCurrentRun: (reason: CancelReason) => void getCountOfFailedTests: () => number diff --git a/packages/runner/package.json b/packages/runner/package.json index 1f3a91a97b29..e7666b79ffeb 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -48,6 +48,7 @@ }, "dependencies": { "@vitest/utils": "workspace:*", - "pathe": "catalog:" + "pathe": "catalog:", + "tinybench": "catalog:" } } diff --git a/packages/runner/src/collect.ts b/packages/runner/src/collect.ts index 7d89da6a2915..98425e85a97f 100644 --- a/packages/runner/src/collect.ts +++ b/packages/runner/src/collect.ts @@ -112,8 +112,8 @@ export async function collectTests( } catch (e) { const errors = e instanceof AggregateError - ? e.errors.map(e => processError(e, runner.config.diffOptions)) - : [processError(e, runner.config.diffOptions)] + ? e.errors.map(e => processError(e, runner.config._diffOptions)) + : [processError(e, runner.config._diffOptions)] file.result = { state: 'fail', errors, diff --git a/packages/runner/src/fixture.ts b/packages/runner/src/fixture.ts index d260612872ac..e0f8dd0b262e 100644 --- a/packages/runner/src/fixture.ts +++ b/packages/runner/src/fixture.ts @@ -32,6 +32,7 @@ export class TestFixtures { 'onTestFinished', 'skip', 'annotate', + 'bench', ] satisfies (keyof TestContext)[] private static _fixtureOptionKeys: string[] = ['auto', 'injected', 'scope'] diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 4faf5475c1d3..7494d5fadf5d 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -146,7 +146,7 @@ async function callTestHooks( await Promise.all(hooks.map(fn => limitMaxConcurrency(() => fn(test.context)))) } catch (e) { - failTask(test.result!, e, runner.config.diffOptions) + failTask(test.result!, e, runner.config._diffOptions) } } else { @@ -155,7 +155,7 @@ async function callTestHooks( await limitMaxConcurrency(() => fn(test.context)) } catch (e) { - failTask(test.result!, e, runner.config.diffOptions) + failTask(test.result!, e, runner.config._diffOptions) } } } @@ -662,14 +662,14 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { } } catch (e) { - failTask(test.result!, e, runner.config.diffOptions) + failTask(test.result!, e, runner.config._diffOptions) } try { await runner.onTaskFinished?.(test) } catch (e) { - failTask(test.result!, e, runner.config.diffOptions) + failTask(test.result!, e, runner.config._diffOptions) } try { @@ -685,7 +685,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { await callFixtureCleanupFrom(test.context, fixtureCheckpoint) } catch (e) { - failTask(test.result!, e, runner.config.diffOptions) + failTask(test.result!, e, runner.config._diffOptions) } if (test.onFinished?.length) { @@ -709,7 +709,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { repeats: repeatCount, }) }).catch((error) => { - failTask(test.result!, error, runner.config.diffOptions) + failTask(test.result!, error, runner.config._diffOptions) }) // Clean up fixtures that were created for aroundEach (before the checkpoint) @@ -718,7 +718,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { await callFixtureCleanup(test.context) } catch (e) { - failTask(test.result!, e, runner.config.diffOptions) + failTask(test.result!, e, runner.config._diffOptions) } // skipped with new PendingError @@ -886,7 +886,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise) +export interface TestTryOptions { + retry: number + repeats: number +} + export interface VitestRunner { /** * First thing that's getting called before actually collecting and running tests. @@ -123,7 +132,7 @@ export interface VitestRunner { */ onBeforeTryTask?: ( test: Test, - options: { retry: number; repeats: number }, + options: TestTryOptions, ) => unknown /** * When the task has finished running, but before cleanup hooks are called @@ -138,7 +147,7 @@ export interface VitestRunner { */ onAfterTryTask?: ( test: Test, - options: { retry: number; repeats: number }, + options: TestTryOptions, ) => unknown /** * Called after the retry resolution happened. Unlike `onAfterTryTask`, the test now has a new state. @@ -146,7 +155,7 @@ export interface VitestRunner { */ onAfterRetryTask?: ( test: Test, - options: { retry: number; repeats: number }, + options: TestTryOptions, ) => unknown /** diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index dbf07d9caf06..a7dbf7a1c65b 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -1,4 +1,5 @@ import type { Awaitable, TestError } from '@vitest/utils' +import type { Statistics } from 'tinybench' import type { TestFixtures } from '../fixture' import type { afterAll, afterEach, aroundAll, aroundEach, beforeAll, beforeEach } from '../hooks' import type { kChainableContext, TypedChainableFunction } from '../utils/chain' @@ -340,6 +341,43 @@ export interface Test extends TaskPopulated { */ artifacts: TestArtifact[] fullTestName: string + /** + * An array of benchmark results generated by `context.bench` function. + * The benchmark is added only after `bench().run` or `bench.compare` is resolved. + * + * @experimental + */ + benchmarks: TestBenchmark[] +} + +export interface TestBenchmark { + name: string + tasks: TestBenchmarkTask[] +} + +export type TestBenchmarkStatistics = Statistics + +export interface BaselineData { + latency: TestBenchmarkStatistics + throughput: TestBenchmarkStatistics + period: number + totalTime: number +} + +export interface TestBenchmarkTask { + name: string + latency: TestBenchmarkStatistics + throughput: TestBenchmarkStatistics + period: number + totalTime: number + rank: number + perProject?: boolean + /** + * `true` when the task was produced by `bench.from()` rather than by + * executing a function. Reporters can use this to render the row as a + * static reference (no margin of error, no samples). + */ + fromStore?: boolean } export type Task = Test | Suite | File diff --git a/packages/runner/tsconfig.json b/packages/runner/tsconfig.json index 1f52dab27ae6..83a275cdf5f8 100644 --- a/packages/runner/tsconfig.json +++ b/packages/runner/tsconfig.json @@ -4,6 +4,7 @@ "target": "ESNext", "module": "ESNext", "moduleResolution": "Bundler", + "types": ["node"], "isolatedDeclarations": true }, "include": ["./src/**/*.ts"], diff --git a/packages/vitest/package.json b/packages/vitest/package.json index fd2a35d50565..d3dadbb8c581 100644 --- a/packages/vitest/package.json +++ b/packages/vitest/package.json @@ -180,7 +180,7 @@ "pathe": "catalog:", "picomatch": "^4.0.3", "std-env": "catalog:", - "tinybench": "^2.9.0", + "tinybench": "catalog:", "tinyexec": "^1.0.2", "tinyglobby": "catalog:", "tinyrainbow": "catalog:", diff --git a/packages/vitest/src/defaults.ts b/packages/vitest/src/defaults.ts index 560638eb1fa7..0ded1230acde 100644 --- a/packages/vitest/src/defaults.ts +++ b/packages/vitest/src/defaults.ts @@ -14,14 +14,16 @@ export const defaultExclude: string[] = [ '**/node_modules/**', '**/.git/**', ] -export const benchmarkConfigDefaults: Required< - Omit -> = { +export const benchmarkConfigDefaults: Required = { + enabled: false, include: ['**/*.{bench,benchmark}.?(c|m)[jt]s?(x)'], exclude: defaultExclude, includeSource: [], - reporters: ['default'], - includeSamples: false, + retainSamples: false, + suppressExportGetterWarnings: false, + // Populated automatically when Vitest clones the parent project; the default + // here applies to the (unused) raw config that's never run as a benchmark. + projectName: '', } // These are the generic defaults for coverage. Providers may also set some provider specific defaults. diff --git a/packages/vitest/src/integrations/chai/bench.ts b/packages/vitest/src/integrations/chai/bench.ts new file mode 100644 index 000000000000..1da427e79d88 --- /dev/null +++ b/packages/vitest/src/integrations/chai/bench.ts @@ -0,0 +1,83 @@ +import type { MatchersObject } from '@vitest/expect' +import type { BenchResult } from '../../runtime/benchmark' + +function isBenchResult(value: unknown): value is BenchResult { + return ( + typeof value === 'object' + && value !== null + && 'latency' in value + && typeof (value as any).latency?.mean === 'number' + ) +} + +function formatOps(ops: number): string { + return ops.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }) +} + +export const benchMatchers: MatchersObject = { + toBeFasterThan(actual: unknown, expected: unknown, options?: { delta?: number }) { + const { matcherHint, RECEIVED_COLOR, EXPECTED_COLOR } = this.utils + const delta = options?.delta ?? 0 + + if (!isBenchResult(actual)) { + throw new TypeError( + `${matcherHint('.toBeFasterThan')} expects the actual value to be a benchmark result.`, + ) + } + if (!isBenchResult(expected)) { + throw new TypeError( + `${matcherHint('.toBeFasterThan')} expects the expected value to be a benchmark result.`, + ) + } + + const threshold = expected.latency.mean * (1 - delta) + const pass = actual.latency.mean < threshold + + return { + pass, + message: () => { + const relation = ((actual.latency.mean - expected.latency.mean) / expected.latency.mean * 100).toFixed(2) + return pass + ? `${matcherHint('.not.toBeFasterThan')}\n\nExpected to not be faster, but was ${Math.abs(Number(relation))}% faster.\n\n` + + `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n` + + `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n` + : `${matcherHint('.toBeFasterThan')}\n\nExpected to be faster${delta > 0 ? ` by at least ${(delta * 100).toFixed(0)}%` : ''}, but was ${Number(relation) > 0 ? `${relation}% slower` : `only ${Math.abs(Number(relation))}% faster`}.\n\n` + + `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n` + + `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n` + }, + } + }, + + toBeSlowerThan(actual: unknown, expected: unknown, options?: { delta?: number }) { + const { matcherHint, RECEIVED_COLOR, EXPECTED_COLOR } = this.utils + const delta = options?.delta ?? 0 + + if (!isBenchResult(actual)) { + throw new TypeError( + `${matcherHint('.toBeSlowerThan')} expects the actual value to be a benchmark result.`, + ) + } + if (!isBenchResult(expected)) { + throw new TypeError( + `${matcherHint('.toBeSlowerThan')} expects the expected value to be a benchmark result.`, + ) + } + + const threshold = expected.latency.mean * (1 + delta) + const pass = actual.latency.mean > threshold + + return { + pass, + message: () => { + const relation = ((actual.latency.mean - expected.latency.mean) / expected.latency.mean * 100).toFixed(2) + return pass + ? `${matcherHint('.not.toBeSlowerThan')}\n\nExpected to not be slower, but was ${relation}% slower.\n\n` + + `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n` + + `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n` + : `${matcherHint('.toBeSlowerThan')}\n\nExpected to be slower${delta > 0 ? ` by at least ${(delta * 100).toFixed(0)}%` : ''}, but was ${Number(relation) < 0 ? `${Math.abs(Number(relation))}% faster` : `only ${relation}% slower`}.\n\n` + + `Received: ${RECEIVED_COLOR(formatOps(actual.throughput.mean))} ops/sec\n` + + `Expected: ${EXPECTED_COLOR(formatOps(expected.throughput.mean))} ops/sec\n` + }, + } + }, +} diff --git a/packages/vitest/src/integrations/chai/index.ts b/packages/vitest/src/integrations/chai/index.ts index 7508a933c3cf..c516fa8f83f8 100644 --- a/packages/vitest/src/integrations/chai/index.ts +++ b/packages/vitest/src/integrations/chai/index.ts @@ -11,6 +11,7 @@ import { } from '@vitest/expect' import { getCurrentTest } from '@vitest/runner' import { getWorkerState } from '../../runtime/utils' +import { benchMatchers } from './bench' import { createExpectPoll } from './poll' import './setup' @@ -108,6 +109,7 @@ export function createExpect(test?: Test | TaskPopulated): ExpectStatic { chai.util.addMethod(expect, 'hasAssertions', hasAssertions) expect.extend(customMatchers) + expect.extend(benchMatchers) return expect } diff --git a/packages/vitest/src/node/ast-collect.ts b/packages/vitest/src/node/ast-collect.ts index 25696900b505..76089450c106 100644 --- a/packages/vitest/src/node/ast-collect.ts +++ b/packages/vitest/src/node/ast-collect.ts @@ -487,6 +487,7 @@ function createFileTask( timeout: 0, annotations: [], artifacts: [], + benchmarks: [], tags: taskTags, } definition.task = task diff --git a/packages/vitest/src/node/benchmark.ts b/packages/vitest/src/node/benchmark.ts new file mode 100644 index 000000000000..61c31cb15e49 --- /dev/null +++ b/packages/vitest/src/node/benchmark.ts @@ -0,0 +1,42 @@ +import type { BaselineData } from '@vitest/runner' +import type { TestProject } from './project' +import { existsSync } from 'node:fs' +import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, isAbsolute, resolve } from 'pathe' + +export class BenchmarkManager { + constructor(private project: TestProject) {} + + // Resolve a user-supplied path against the project root. Reject paths that + // escape the project root: `bench.from()` accepts arbitrary input, and we + // never want a benchmark file to be able to read or clobber files outside + // the workspace. + public resolve(relativePath: string): string { + const root = this.project.config.root + const absolute = isAbsolute(relativePath) + ? resolve(relativePath) + : resolve(root, relativePath) + const rootWithSep = root.endsWith('/') ? root : `${root}/` + if (absolute !== root && !absolute.startsWith(rootWithSep)) { + throw new Error( + `Benchmark artifact path "${relativePath}" resolves outside the project root (${root}). ` + + `Paths passed to \`writeResult\` and \`bench.from()\` must point inside the project.`, + ) + } + return absolute + } + + async readResult(relativePath: string): Promise { + const path = this.resolve(relativePath) + if (!existsSync(path)) { + return null + } + return JSON.parse(await readFile(path, 'utf-8')) as BaselineData + } + + async writeResult(relativePath: string, data: BaselineData): Promise { + const absolute = this.resolve(relativePath) + await mkdir(dirname(absolute), { recursive: true }) + await writeFile(absolute, `${JSON.stringify(data, null, 2)}\n`, 'utf-8') + } +} diff --git a/packages/vitest/src/node/cli/cac.ts b/packages/vitest/src/node/cli/cac.ts index 29f94b6b653d..3d082ffb8bed 100644 --- a/packages/vitest/src/node/cli/cac.ts +++ b/packages/vitest/src/node/cli/cac.ts @@ -1,14 +1,13 @@ import type { CAC, Command } from 'cac' -import type { VitestRunMode } from '../types/config' import type { CliOptions } from './cli-api' import type { CLIOption, CLIOptions as CLIOptionsConfig } from './cli-config' import { toArray } from '@vitest/utils/helpers' import cac from 'cac' import { normalize } from 'pathe' -import c, { disableDefaultColors } from 'tinyrainbow' +import { disableDefaultColors } from 'tinyrainbow' import { version } from '../../../package.json' with { type: 'json' } import { isAgent, isForceColor } from '../../utils/env' -import { benchCliOptionsConfig, cliOptionsConfig, collectCliOptionsConfig } from './cli-config' +import { cliOptionsConfig, collectCliOptionsConfig } from './cli-config' import { setupTabCompletions } from './completions' function addCommand(cli: CAC | Command, name: string, option: CLIOption) { @@ -179,12 +178,9 @@ export function createCLI(options: CliParseOptions = {}): CAC { .command('dev [...filters]', undefined, options) .action(watch) - addCliOptions( - cli - .command('bench [...filters]', undefined, options) - .action(benchmark), - benchCliOptionsConfig, - ) + cli + .command('bench [...filters]', undefined, options) + .action(benchmark) cli .command('init ', undefined, options) @@ -193,13 +189,13 @@ export function createCLI(options: CliParseOptions = {}): CAC { addCliOptions( cli .command('list [...filters]', undefined, options) - .action((filters, options) => collect('test', filters, options)), + .action((filters, options) => collect(filters, options)), collectCliOptionsConfig, ) cli .command('[...filters]', undefined, options) - .action((filters, options) => start('test', filters, options)) + .action((filters, options) => start(filters, options)) setupTabCompletions(cli) return cli @@ -263,24 +259,26 @@ export function parseCLI(argv: string | string[], config: CliParseOptions = {}): async function runRelated(relatedFiles: string[] | string, argv: CliOptions): Promise { argv.related = relatedFiles argv.passWithNoTests ??= true - await start('test', [], argv) + await start([], argv) } async function watch(cliFilters: string[], options: CliOptions): Promise { options.watch = true - await start('test', cliFilters, options) + await start(cliFilters, options) } async function run(cliFilters: string[], options: CliOptions): Promise { // "vitest run --watch" should still be watch mode options.run = !options.watch - await start('test', cliFilters, options) + await start(cliFilters, options) } async function benchmark(cliFilters: string[], options: CliOptions): Promise { - console.warn(c.yellow('Benchmarking is an experimental feature.\nBreaking changes might not follow SemVer, please pin Vitest\'s version when using it.')) - await start('benchmark', cliFilters, options) + options.benchmarkOnly = true + options.coverage ??= {} + options.coverage.enabled = false + await start(cliFilters, options) } function normalizeCliOptions(cliFilters: string[], argv: CliOptions): CliOptions { @@ -303,10 +301,10 @@ function normalizeCliOptions(cliFilters: string[], argv: CliOptions): CliOptions return argv } -async function start(mode: VitestRunMode, cliFilters: string[], options: CliOptions): Promise { +async function start(cliFilters: string[], options: CliOptions): Promise { try { const { startVitest } = await import('./cli-api') - const ctx = await startVitest(mode, cliFilters.map(normalize), normalizeCliOptions(cliFilters, options)) + const ctx = await startVitest(cliFilters.map(normalize), normalizeCliOptions(cliFilters, options)) if (!ctx.shouldKeepServer()) { await ctx.exit() } @@ -335,10 +333,10 @@ async function init(project: string) { await create() } -async function collect(mode: VitestRunMode, cliFilters: string[], options: CliOptions): Promise { +async function collect(cliFilters: string[], options: CliOptions): Promise { try { const { prepareVitest, processCollected, outputFileList } = await import('./cli-api') - const ctx = await prepareVitest(mode, { + const ctx = await prepareVitest({ ...normalizeCliOptions(cliFilters, options), watch: false, run: true, diff --git a/packages/vitest/src/node/cli/cli-api.ts b/packages/vitest/src/node/cli/cli-api.ts index 0ec33f732fd6..17f4190de1a3 100644 --- a/packages/vitest/src/node/cli/cli-api.ts +++ b/packages/vitest/src/node/cli/cli-api.ts @@ -46,6 +46,13 @@ export interface CliOptions extends UserConfig { * @experimental */ configLoader?: ViteInlineConfig extends { configLoader?: infer T } ? T : never + + /** + * Only run benchmark projects, filtering out all other projects. + * Set automatically by `vitest bench`. + * @internal + */ + benchmarkOnly?: boolean } /** @@ -53,24 +60,55 @@ export interface CliOptions extends UserConfig { * * Returns a Vitest instance if initialized successfully. */ +export async function startVitest( + cliFilters?: string[], + options?: CliOptions, + viteOverrides?: ViteUserConfig, + vitestOptions?: VitestOptions, +): Promise +/** + * @deprecated The `mode` argument is no longer used. Use `startVitest(cliFilters?, options?, viteOverrides?, vitestOptions?)` instead. + */ export async function startVitest( mode: VitestRunMode, - cliFilters: string[] = [], - options: CliOptions = {}, + cliFilters?: string[], + options?: CliOptions, viteOverrides?: ViteUserConfig, vitestOptions?: VitestOptions, +): Promise +export async function startVitest( + modeOrCliFilters?: VitestRunMode | string[], + cliFiltersOrOptions?: string[] | CliOptions, + optionsOrViteOverrides?: CliOptions | ViteUserConfig, + viteOverridesOrVitestOptions?: ViteUserConfig | VitestOptions, + maybeVitestOptions?: VitestOptions, ): Promise { + let cliFilters: string[] + let options: CliOptions + let viteOverrides: ViteUserConfig | undefined + let vitestOptions: VitestOptions | undefined + if (typeof modeOrCliFilters === 'string') { + cliFilters = (cliFiltersOrOptions as string[] | undefined) ?? [] + options = (optionsOrViteOverrides as CliOptions | undefined) ?? {} + viteOverrides = viteOverridesOrVitestOptions as ViteUserConfig | undefined + vitestOptions = maybeVitestOptions + } + else { + cliFilters = modeOrCliFilters ?? [] + options = (cliFiltersOrOptions as CliOptions | undefined) ?? {} + viteOverrides = optionsOrViteOverrides as ViteUserConfig | undefined + vitestOptions = viteOverridesOrVitestOptions as VitestOptions | undefined + } const root = resolve(options.root || process.cwd()) const ctx = await prepareVitest( - mode, options, viteOverrides, vitestOptions, cliFilters, ) - if (mode === 'test' && ctx._coverageOptions.enabled) { + if (ctx._coverageOptions.enabled) { const provider = ctx._coverageOptions.provider || 'v8' const requiredPackages = CoverageProviderMap[provider] @@ -150,13 +188,45 @@ export async function startVitest( } } +export async function prepareVitest( + options?: CliOptions, + viteOverrides?: ViteUserConfig, + vitestOptions?: VitestOptions, + cliFilters?: string[], +): Promise +/** + * @deprecated The `mode` argument is no longer used. Use `prepareVitest(options?, viteOverrides?, vitestOptions?, cliFilters?)` instead. + */ export async function prepareVitest( mode: VitestRunMode, - options: CliOptions = {}, + options?: CliOptions, viteOverrides?: ViteUserConfig, vitestOptions?: VitestOptions, cliFilters?: string[], +): Promise +export async function prepareVitest( + modeOrOptions?: VitestRunMode | CliOptions, + optionsOrViteOverrides?: CliOptions | ViteUserConfig, + viteOverridesOrVitestOptions?: ViteUserConfig | VitestOptions, + vitestOptionsOrCliFilters?: VitestOptions | string[], + maybeCliFilters?: string[], ): Promise { + let options: CliOptions + let viteOverrides: ViteUserConfig | undefined + let vitestOptions: VitestOptions | undefined + let cliFilters: string[] | undefined + if (typeof modeOrOptions === 'string') { + options = (optionsOrViteOverrides as CliOptions | undefined) ?? {} + viteOverrides = viteOverridesOrVitestOptions as ViteUserConfig | undefined + vitestOptions = vitestOptionsOrCliFilters as VitestOptions | undefined + cliFilters = maybeCliFilters + } + else { + options = modeOrOptions ?? {} + viteOverrides = optionsOrViteOverrides as ViteUserConfig | undefined + vitestOptions = viteOverridesOrVitestOptions as VitestOptions | undefined + cliFilters = vitestOptionsOrCliFilters as string[] | undefined + } process.env.TEST = 'true' process.env.VITEST = 'true' process.env.NODE_ENV ??= 'test' @@ -172,7 +242,7 @@ export async function prepareVitest( // this shouldn't affect _application root_ that can be changed inside config const root = resolve(options.root || process.cwd()) - const ctx = await createVitest(mode, options, viteOverrides, vitestOptions) + const ctx = await createVitest(options, viteOverrides, vitestOptions) const environmentPackage = getEnvPackageName(ctx.config.environment) diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 5f2becbbeafe..e41c744d274a 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -332,7 +332,7 @@ export const cliOptionsConfig: VitestCLIOptions = { }, }, mode: { - description: 'Override Vite mode (default: `test` or `benchmark`)', + description: 'Override Vite mode (default: `test`)', argument: '', }, isolate: { @@ -976,8 +976,6 @@ export const cliOptionsConfig: VitestCLIOptions = { deps: null, name: null, snapshotEnvironment: null, - compare: null, - outputJson: null, json: null, provide: null, filesOnly: null, @@ -986,23 +984,10 @@ export const cliOptionsConfig: VitestCLIOptions = { projects: null, watchTriggerPatterns: null, tags: null, + benchmarkOnly: null, taskTitleValueFormatTruncate: null, } -export const benchCliOptionsConfig: Pick< - VitestCLIOptions, - 'compare' | 'outputJson' -> = { - compare: { - description: 'Benchmark output file to compare against', - argument: '', - }, - outputJson: { - description: 'Benchmark output file', - argument: '', - }, -} - export const collectCliOptionsConfig: VitestCLIOptions = { ...cliOptionsConfig, json: { diff --git a/packages/vitest/src/node/config/resolveConfig.ts b/packages/vitest/src/node/config/resolveConfig.ts index 4a7af583ffef..1d2b43bd5d02 100644 --- a/packages/vitest/src/node/config/resolveConfig.ts +++ b/packages/vitest/src/node/config/resolveConfig.ts @@ -1,7 +1,6 @@ import type { ResolvedConfig as ResolvedViteConfig } from 'vite' import type { Vitest } from '../core' import type { Logger } from '../logger' -import type { BenchmarkBuiltinReporters } from '../reporters' import type { ResolvedBrowserOptions } from '../types/browser' import type { ApiConfig, @@ -155,7 +154,6 @@ export function resolveConfig( options: UserConfig, viteConfig: ResolvedViteConfig, ): ResolvedConfig { - const mode = vitest.mode const logger = vitest.logger if (options.dom) { if ( @@ -178,9 +176,10 @@ export function resolveConfig( ...configDefaults, ...options, root: viteConfig.root, - mode, } as any as ResolvedConfig + resolved.mode ??= viteConfig.mode ?? 'test' + if (resolved.retry && typeof resolved.retry === 'object' && typeof resolved.retry.condition === 'function') { logger.console.warn( c.yellow('Warning: retry.condition function cannot be used inside a config file. ' @@ -245,6 +244,11 @@ export function resolveConfig( throw new Error(`Looks like you set "test.environment" to "browser". To enable Browser Mode, use "test.browser.enabled" instead.`) } + resolved.benchmark = { + ...benchmarkConfigDefaults, + ...resolved.benchmark, + } + const inspector = resolved.inspect || resolved.inspectBrk resolved.inspector = { @@ -295,8 +299,7 @@ export function resolveConfig( resolved.maxWorkers = resolveInlineWorkerOption(resolved.maxWorkers) } - // run benchmark sequentially by default - const fileParallelism = options.fileParallelism ?? mode !== 'benchmark' + const fileParallelism = options.fileParallelism ?? true if (!fileParallelism) { // ignore user config, parallelism cannot be implemented without limiting workers @@ -634,44 +637,6 @@ export function resolveConfig( resolved.maxWorkers = Number.parseInt(process.env.VITEST_MAX_WORKERS) } - if (mode === 'benchmark') { - resolved.benchmark = { - ...benchmarkConfigDefaults, - ...resolved.benchmark, - } - // override test config - resolved.coverage.enabled = false - resolved.typecheck.enabled = false - resolved.include = resolved.benchmark.include - resolved.exclude = resolved.benchmark.exclude - resolved.includeSource = resolved.benchmark.includeSource - const reporters = Array.from( - new Set([ - ...toArray(resolved.benchmark.reporters), - // @ts-expect-error reporter is CLI flag - ...toArray(options.reporter), - ]), - ).filter(Boolean) - if (reporters.length) { - resolved.benchmark.reporters = reporters - } - else { - resolved.benchmark.reporters = ['default'] - } - - if (options.outputFile) { - resolved.benchmark.outputFile = options.outputFile - } - - // --compare from cli - if (options.compare) { - resolved.benchmark.compare = options.compare - } - if (options.outputJson) { - resolved.benchmark.outputJson = options.outputJson - } - } - if (typeof resolved.diff === 'string') { resolved.diff = resolvePath(resolved.diff, resolved.root) resolved.forceRerunTriggers.push(resolved.diff) @@ -729,39 +694,37 @@ export function resolveConfig( } } - if (mode !== 'benchmark') { - // @ts-expect-error "reporter" is from CLI, should be absolute to the running directory - // it is passed down as "vitest --reporter ../reporter.js" - const reportersFromCLI = resolved.reporter + // @ts-expect-error "reporter" is from CLI, should be absolute to the running directory + // it is passed down as "vitest --reporter ../reporter.js" + const reportersFromCLI = resolved.reporter - const cliReporters = toArray(reportersFromCLI || []).map( - (reporter: string) => { - // ./reporter.js || ../reporter.js, but not .reporters/reporter.js - if (/^\.\.?\//.test(reporter)) { - return resolve(process.cwd(), reporter) - } - return reporter - }, - ) + const cliReporters = toArray(reportersFromCLI || []).map( + (reporter: string) => { + // ./reporter.js || ../reporter.js, but not .reporters/reporter.js + if (/^\.\.?\//.test(reporter)) { + return resolve(process.cwd(), reporter) + } + return reporter + }, + ) - if (cliReporters.length) { - // When CLI reporters are specified, preserve options from config file - const configReportersMap = new Map>() + if (cliReporters.length) { + // When CLI reporters are specified, preserve options from config file + const configReportersMap = new Map>() - // Build a map of reporter names to their options from the config - for (const reporter of resolved.reporters) { - if (Array.isArray(reporter)) { - const [reporterName, reporterOptions] = reporter - if (typeof reporterName === 'string') { - configReportersMap.set(reporterName, reporterOptions as Record) - } + // Build a map of reporter names to their options from the config + for (const reporter of resolved.reporters) { + if (Array.isArray(reporter)) { + const [reporterName, reporterOptions] = reporter + if (typeof reporterName === 'string') { + configReportersMap.set(reporterName, reporterOptions as Record) } } - - resolved.reporters = Array.from(new Set(toArray(cliReporters))) - .filter(Boolean) - .map(reporter => [reporter, configReportersMap.get(reporter) || {}]) } + + resolved.reporters = Array.from(new Set(toArray(cliReporters))) + .filter(Boolean) + .map(reporter => [reporter, configReportersMap.get(reporter) || {}]) } resolved.mergeReportsLabel = process.env.VITEST_BLOB_LABEL @@ -842,7 +805,7 @@ export function resolveConfig( } resolved.browser.isolate ??= resolved.isolate ?? true resolved.browser.fileParallelism - ??= options.fileParallelism ?? mode !== 'benchmark' + ??= options.fileParallelism ?? true // disable in headless mode by default, and if CI is detected resolved.browser.ui ??= resolved.browser.headless === true ? false : !isCI resolved.browser.commands ??= {} diff --git a/packages/vitest/src/node/config/serializeConfig.ts b/packages/vitest/src/node/config/serializeConfig.ts index d3f303886588..0ce56e9ada6e 100644 --- a/packages/vitest/src/node/config/serializeConfig.ts +++ b/packages/vitest/src/node/config/serializeConfig.ts @@ -13,7 +13,6 @@ export function serializeConfig(project: TestProject): SerializedConfig { return { // TODO: remove functions from environmentOptions environmentOptions: config.environmentOptions, - mode: config.mode, isolate: config.isolate, maxWorkers: config.maxWorkers, base: config.base, @@ -134,8 +133,11 @@ export function serializeConfig(project: TestProject): SerializedConfig { standalone: config.standalone, printConsoleTrace: config.printConsoleTrace ?? globalConfig.printConsoleTrace, - benchmark: config.benchmark && { - includeSamples: config.benchmark.includeSamples, + benchmark: { + enabled: config.benchmark.enabled, + retainSamples: config.benchmark.retainSamples, + suppressExportGetterWarnings: config.benchmark.suppressExportGetterWarnings, + projectName: config.benchmark.projectName, }, // the browser initialized them via `@vite/env` import serializedDefines: config.browser.enabled diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index efa95c6082e9..21c68ad0dd5a 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -44,11 +44,11 @@ import { collectModuleDurationsDiagnostic, collectSourceModulesLocations } from import { VitestPackageInstaller } from './packageInstaller' import { createPool } from './pool' import { TestProject } from './project' -import { getDefaultTestProject, resolveBrowserProjects, resolveProjects } from './projects/resolveProjects' +import { getDefaultTestProject, resolveDefaultProjects, resolveProjects } from './projects/resolveProjects' import { BlobReporter, readBlobs } from './reporters/blob' import { HangingProcessReporter } from './reporters/hanging-process' import { createReport } from './reporters/report' -import { createBenchmarkReporters, createReporters } from './reporters/utils' +import { createReporters } from './reporters/utils' import { VitestResolver } from './resolver' import { VitestSpecifications } from './specifications' import { StateManager } from './state' @@ -77,7 +77,7 @@ export class Vitest { * The logger instance used to log messages. It's recommended to use this logger instead of `console`. * It's possible to override stdout and stderr streams when initiating Vitest. * @example - * new Vitest('test', { + * new Vitest({ * stdout: new Writable(), * }) */ @@ -143,11 +143,38 @@ export class Vitest { private _snapshot?: SnapshotManager private _coverageProvider?: CoverageProvider | null | undefined + /** + * @deprecated Do not rely on this property, it's always `test`. Scheduled to be removed in the next major. + */ + public readonly mode = 'test' + + constructor( + cliOptions: UserConfig, + options?: VitestOptions, + ) + /** + * @deprecated The `mode` argument is no longer used. Use `new Vitest(cliOptions, options)` instead. + */ constructor( - public readonly mode: VitestRunMode, + mode: VitestRunMode, cliOptions: UserConfig, - options: VitestOptions = {}, + options?: VitestOptions, + ) + constructor( + modeOrCliOptions: VitestRunMode | UserConfig, + cliOptionsOrOptions?: UserConfig | VitestOptions, + maybeOptions?: VitestOptions, ) { + let cliOptions: UserConfig + let options: VitestOptions + if (typeof modeOrCliOptions === 'string') { + cliOptions = cliOptionsOrOptions as UserConfig + options = maybeOptions ?? {} + } + else { + cliOptions = modeOrCliOptions + options = (cliOptionsOrOptions as VitestOptions) ?? {} + } this._cliOptions = cliOptions this.logger = new Logger(this, options.stdout, options.stderr) this.packageInstaller = options.packageInstaller || new VitestPackageInstaller() @@ -306,10 +333,12 @@ export class Vitest { } catch { } - const projects = await this.resolveProjects(this._cliOptions) - this.projects = projects + this.projects = await this.resolveProjects(this._cliOptions) + if (this._cliOptions.benchmarkOnly) { + this.projects = this.projects.filter(c => c.config.benchmark.enabled) + } - await Promise.all(projects.flatMap((project) => { + await Promise.all(this.projects.flatMap((project) => { const hooks = project.vite.config.getSortedPluginHooks('configureVitest') return hooks.map(hook => hook({ project, @@ -356,9 +385,7 @@ export class Vitest { populateProjectsTags(this.coreWorkspaceProject, this.projects) } - this.reporters = resolved.mode === 'benchmark' - ? await createBenchmarkReporters(toArray(resolved.benchmark?.reporters), this.runner) - : await createReporters(resolved.reporters, this) + this.reporters = await createReporters(resolved.reporters, this) await this._fsCache.ensureCacheIntegrity() @@ -567,7 +594,7 @@ export class Vitest { if (!project) { return [] } - return resolveBrowserProjects(this, new Set([project.name]), [project]) + return resolveDefaultProjects(this, new Set([project.name]), [project]) } /** @@ -797,7 +824,7 @@ export class Vitest { }) if (!this.config.watch || !(this.config.changed || this.config.related?.length)) { - throw new FilesNotFoundError(this.mode) + throw new FilesNotFoundError() } } @@ -1042,9 +1069,6 @@ export class Vitest { /** @default os.availableParallelism() */ concurrency?: number }): Promise { - if (this.mode !== 'test') { - throw new Error(`The \`experimental_parseSpecifications\` does not support "${this.mode}" mode.`) - } const concurrency = options?.concurrency ?? (typeof os.availableParallelism === 'function' ? os.availableParallelism() : os.cpus().length) @@ -1084,9 +1108,6 @@ export class Vitest { } public async experimental_parseSpecification(specification: TestSpecification): Promise { - if (this.mode !== 'test') { - throw new Error(`The \`experimental_parseSpecification\` does not support "${this.mode}" mode.`) - } const file = await astCollectTests(specification.project, specification.moduleId).catch((error) => { return createFailedFileTask(specification.project, specification.moduleId, error) }) diff --git a/packages/vitest/src/node/create.ts b/packages/vitest/src/node/create.ts index 02498f366da8..e102e934cceb 100644 --- a/packages/vitest/src/node/create.ts +++ b/packages/vitest/src/node/create.ts @@ -15,13 +15,40 @@ import { Vitest } from './core' import { VitestPlugin } from './plugins' import { createViteServer } from './vite' +export async function createVitest( + options: CliOptions, + viteOverrides?: ViteUserConfig, + vitestOptions?: VitestOptions, +): Promise +/** + * @deprecated The `mode` argument is no longer used. Use `createVitest(options, viteOverrides?, vitestOptions?)` instead. + */ export async function createVitest( mode: VitestRunMode, options: CliOptions, - viteOverrides: ViteUserConfig = {}, - vitestOptions: VitestOptions = {}, + viteOverrides?: ViteUserConfig, + vitestOptions?: VitestOptions, +): Promise +export async function createVitest( + modeOrOptions: VitestRunMode | CliOptions, + optionsOrViteOverrides: CliOptions | ViteUserConfig = {}, + viteOverridesOrVitestOptions: ViteUserConfig | VitestOptions = {}, + maybeVitestOptions: VitestOptions = {}, ): Promise { - const ctx = new Vitest(mode, deepClone(options), vitestOptions) + let options: CliOptions + let viteOverrides: ViteUserConfig + let vitestOptions: VitestOptions + if (typeof modeOrOptions === 'string') { + options = optionsOrViteOverrides as CliOptions + viteOverrides = viteOverridesOrVitestOptions as ViteUserConfig + vitestOptions = maybeVitestOptions + } + else { + options = modeOrOptions + viteOverrides = optionsOrViteOverrides as ViteUserConfig + vitestOptions = viteOverridesOrVitestOptions as VitestOptions + } + const ctx = new Vitest(deepClone(options), vitestOptions) const root = slash(resolve(options.root || process.cwd())) const configPath @@ -38,8 +65,7 @@ export async function createVitest( const config: ViteInlineConfig = { configFile: configPath, configLoader: options.configLoader, - // this will make "mode": "test" | "benchmark" inside defineConfig - mode: options.mode || mode, + mode: options.mode || 'test', plugins: await VitestPlugin(restOptions, ctx), } diff --git a/packages/vitest/src/node/errors.ts b/packages/vitest/src/node/errors.ts index 00db3293cf1c..5d030767eff0 100644 --- a/packages/vitest/src/node/errors.ts +++ b/packages/vitest/src/node/errors.ts @@ -1,8 +1,8 @@ export class FilesNotFoundError extends Error { code = 'VITEST_FILES_NOT_FOUND' - constructor(mode: 'test' | 'benchmark') { - super(`No ${mode} files found`) + constructor() { + super(`No test files found`) } } diff --git a/packages/vitest/src/node/logger.ts b/packages/vitest/src/node/logger.ts index fcae0950fad2..db4da36a35ef 100644 --- a/packages/vitest/src/node/logger.ts +++ b/packages/vitest/src/node/logger.ts @@ -169,20 +169,20 @@ export class Logger { const config = this.ctx.config if (config.watch && (config.changed || config.related?.length)) { - this.log(`No affected ${config.mode} files found\n`) + this.log(`No affected test files found\n`) } else if (config.watch) { this.log( - c.red(`No ${config.mode} files found. You can change the file name pattern by pressing "p"\n`), + c.red(`No test files found. You can change the file name pattern by pressing "p"\n`), ) } else { if (config.passWithNoTests) { - this.log(`No ${config.mode} files found, exiting with code 0\n`) + this.log(`No test files found, exiting with code 0\n`) } else { this.error( - c.red(`No ${config.mode} files found, exiting with code 1\n`), + c.red(`No test files found, exiting with code 1\n`), ) } } diff --git a/packages/vitest/src/node/plugins/index.ts b/packages/vitest/src/node/plugins/index.ts index 3b648c72c408..ddcbb6784a3b 100644 --- a/packages/vitest/src/node/plugins/index.ts +++ b/packages/vitest/src/node/plugins/index.ts @@ -25,7 +25,7 @@ import { VitestCoreResolver } from './vitestResolver' export async function VitestPlugin( options: UserConfig = {}, - vitest: Vitest = new Vitest('test', deepClone(options)), + vitest: Vitest = new Vitest(deepClone(options)), ): Promise { const userConfig = deepMerge({}, options) as UserConfig @@ -150,6 +150,11 @@ export async function VitestPlugin( } } + if (vitest._cliOptions.benchmarkOnly) { + config.test!.benchmark ??= {} + config.test!.benchmark.enabled = true + } + // inherit so it's available in VitestOptimizer // I cannot wait to rewrite all of this in Vitest 4 if (options.cache != null) { diff --git a/packages/vitest/src/node/plugins/publicConfig.ts b/packages/vitest/src/node/plugins/publicConfig.ts index 28a58fb385ae..e7c4201761a4 100644 --- a/packages/vitest/src/node/plugins/publicConfig.ts +++ b/packages/vitest/src/node/plugins/publicConfig.ts @@ -27,12 +27,11 @@ export async function resolveConfig( : find.any(configFiles, { cwd: root }) options.config = configPath - const vitest = new Vitest('test', deepClone(options)) + const vitest = new Vitest(deepClone(options)) const config = await resolveViteConfig( mergeConfig( { configFile: configPath, - // this will make "mode": "test" | "benchmark" inside defineConfig mode: options.mode || 'test', plugins: [ await VitestPlugin(options, vitest), diff --git a/packages/vitest/src/node/plugins/workspace.ts b/packages/vitest/src/node/plugins/workspace.ts index 3338ab253787..cf9265fba87e 100644 --- a/packages/vitest/src/node/plugins/workspace.ts +++ b/packages/vitest/src/node/plugins/workspace.ts @@ -37,7 +37,9 @@ export function WorkspaceVitestPlugin( name: 'vitest:project:name', enforce: 'post', config(viteConfig) { - const testConfig = viteConfig.test || {} + viteConfig.test ??= {} + + const testConfig = viteConfig.test let { label: name, color } = typeof testConfig.name === 'string' ? { label: testConfig.name } @@ -62,6 +64,11 @@ export function WorkspaceVitestPlugin( } } + if (project.vitest._cliOptions.benchmarkOnly) { + viteConfig.test.benchmark ??= {} + viteConfig.test.benchmark.enabled = true + } + const isUserBrowserEnabled = viteConfig.test?.browser?.enabled const isBrowserEnabled = isUserBrowserEnabled ?? (viteConfig.test?.browser && project.vitest._cliOptions.browser?.enabled) // keep project names to potentially filter it out @@ -79,6 +86,9 @@ export function WorkspaceVitestPlugin( workspaceNames.push(instance.name) } }) + if (viteConfig.test?.benchmark?.enabled) { + workspaceNames.push(name ? `${name} (bench)` : 'bench') + } const filters = project.vitest.config.project // if there is `--project=...` filter, check if any of the potential projects match diff --git a/packages/vitest/src/node/pools/poolRunner.ts b/packages/vitest/src/node/pools/poolRunner.ts index 26613932da1e..5eaf22235439 100644 --- a/packages/vitest/src/node/pools/poolRunner.ts +++ b/packages/vitest/src/node/pools/poolRunner.ts @@ -367,7 +367,7 @@ export class PoolRunner { } private emitUnexpectedExit = (): void => { - const error = new Error('Worker exited unexpectedly') + const error = new Error(`Worker exited unexpectedly during ${this._state} state`) this._eventEmitter.emit('error', error) } diff --git a/packages/vitest/src/node/pools/rpc.ts b/packages/vitest/src/node/pools/rpc.ts index 1d09a45b87c5..bb5bd4150442 100644 --- a/packages/vitest/src/node/pools/rpc.ts +++ b/packages/vitest/src/node/pools/rpc.ts @@ -121,6 +121,15 @@ export function createMethodsRPC(project: TestProject, methodsOptions: MethodsOp onAfterSuiteRun(meta) { vitest.coverageProvider?.onAfterSuiteRun(meta) }, + async onTestBenchmark(testId, benchmark) { + return vitest._testRun.recordBenchmark(testId, benchmark) + }, + async readBenchmarkResult(relativePath) { + return project.benchmark.readResult(relativePath) + }, + async writeBenchmarkResult(relativePath, data) { + return project.benchmark.writeResult(relativePath, data) + }, async onTaskArtifactRecord(testId, artifact) { return vitest._testRun.recordArtifact(testId, artifact) }, diff --git a/packages/vitest/src/node/project.ts b/packages/vitest/src/node/project.ts index 234bc56a4e00..76c2002f6303 100644 --- a/packages/vitest/src/node/project.ts +++ b/packages/vitest/src/node/project.ts @@ -28,6 +28,7 @@ import { isRunnableDevEnvironment } from 'vite' import { setup } from '../api/setup' import { createDefinesScript } from '../utils/config-helpers' import { NativeModuleRunner } from '../utils/nativeModuleRunner' +import { BenchmarkManager } from './benchmark' import { isBrowserEnabled, resolveConfig } from './config/resolveConfig' import { serializeConfig } from './config/serializeConfig' import { createFetchModuleFunction } from './environments/fetchModule' @@ -63,6 +64,8 @@ export class TestProject { */ public readonly tmpDir: string + public readonly benchmark: BenchmarkManager = new BenchmarkManager(this) + /** @internal */ typechecker?: Typechecker /** @internal */ _config?: ResolvedConfig /** @internal */ _vite?: ViteDevServer @@ -731,7 +734,7 @@ export class TestProject { } /** @internal */ - static _cloneBrowserProject(parent: TestProject, config: ResolvedConfig): TestProject { + static _cloneTestProject(parent: TestProject, config: ResolvedConfig): TestProject { const clone = new TestProject(parent.vitest, undefined, parent.tmpDir) clone.runner = parent.runner clone._vite = parent._vite diff --git a/packages/vitest/src/node/projects/resolveProjects.ts b/packages/vitest/src/node/projects/resolveProjects.ts index 0ce5691b5beb..74f222b930a1 100644 --- a/packages/vitest/src/node/projects/resolveProjects.ts +++ b/packages/vitest/src/node/projects/resolveProjects.ts @@ -197,10 +197,80 @@ export async function resolveProjects( names.add(name) } - return resolveBrowserProjects(vitest, names, resolvedProjects) + return resolveDefaultProjects(vitest, names, resolvedProjects) } -export async function resolveBrowserProjects( +export async function resolveDefaultProjects( + vitest: Vitest, + names: Set, + resolvedProjects: TestProject[], +): Promise { + const newProjects = await resolveBrowserProjects(vitest, names, resolvedProjects) + + let lastGroupOrder = Math.max(0, ...newProjects.map(p => p.config.sequence.groupOrder)) + + newProjects.forEach((project) => { + const benchmark = project.config.benchmark + if (!benchmark.enabled) { + return + } + + if (vitest.isExcludedByProjectFilter(project.config.name)) { + benchmark.enabled = false + return + } + + const name = project.config.name ? `${project.config.name} (bench)` : 'bench' + if (!vitest.matchesProjectFilter(name)) { + benchmark.enabled = false + return + } + + if (names.has(name)) { + throw new Error(`Cannot create a benchmark project because the name "${name}" is already in use.`) + } + names.add(name) + + const benchmarkProject = TestProject._cloneTestProject(project, { + ...project.config, + name, + include: benchmark.include, + exclude: benchmark.exclude, + includeSource: benchmark.includeSource, + coverage: { + ...project.config.coverage, + enabled: false, + }, + maxWorkers: 1, + maxConcurrency: 1, + testTimeout: project.config.testTimeout < 60_000 ? 60_000 : project.config.testTimeout, + hookTimeout: project.config.hookTimeout < 120_000 ? 120_000 : project.config.hookTimeout, + // Spread because we disable it in the original project. `projectName` + // carries the parent's name so the runtime can substitute it into + // `${projectName}` placeholders inside `writeResult` / `bench.from()` + // paths — `project.config.name` already excludes the ` (bench)` suffix + // we add to the cloned project's own name above. + benchmark: { ...benchmark, projectName: project.config.name ?? '' }, + sequence: { + ...project.config.sequence, + concurrent: false, + // benchmarks should always run in a separate isolated group + groupOrder: ++lastGroupOrder, + }, + typecheck: { + ...project.config.typecheck, + enabled: false, + }, + // TODO: mark if benchmark project? + }) + // disable benchmark in the original project + benchmark.enabled = false + newProjects.push(benchmarkProject) + }) + return newProjects +} + +async function resolveBrowserProjects( vitest: Vitest, names: Set, resolvedProjects: TestProject[], @@ -260,7 +330,7 @@ export async function resolveBrowserProjects( names.add(name) const clonedConfig = cloneConfig(project, config) clonedConfig.name = name - const clone = TestProject._cloneBrowserProject(project, clonedConfig) + const clone = TestProject._cloneTestProject(project, clonedConfig) resolvedProjects.push(clone) }) @@ -481,11 +551,15 @@ export function getDefaultTestProject(vitest: Vitest): TestProject | null { function getPotentialProjectNames(project: TestProject) { const names = [project.name] + // TODO: include benchmarks in browsers if (project.config.browser.instances) { names.push(...project.config.browser.instances.map(i => i.name!)) } else if (project.config.browser.name) { names.push(project.config.browser.name) } + if (project.config.benchmark.enabled) { + names.push(project.name ? `${project.name} (bench)` : 'bench') + } return names } diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index b5442314c44f..7cb26ca68244 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -1,4 +1,4 @@ -import type { File, Task, TestAnnotation } from '@vitest/runner' +import type { File, Task, TestAnnotation, TestBenchmark, TestBenchmarkTask } from '@vitest/runner' import type { ParsedStack, SerializedError } from '@vitest/utils' import type { AsyncLeak, TestError, UserConsoleLog } from '../../types/general' import type { Vitest } from '../core' @@ -16,6 +16,7 @@ import { groupBy } from '../../utils/base' import { isTTY } from '../../utils/env' import { hasFailedSnapshot } from '../../utils/tasks' import { generateCodeFrame, printStack } from '../printError' +import { BENCH_TABLE_HEAD, computeBenchColumnWidths, padBenchRow, renderBenchmarkRow } from './renderers/benchmark-table' import { F_CHECK, F_DOWN_RIGHT, F_POINTER } from './renderers/figures' import { countTestErrors, @@ -54,6 +55,7 @@ export abstract class BaseReporter implements Reporter { private _filesInWatchMode = new Map() private _timeStart = formatTimeString(new Date()) + private _perProjectBenchmarks = new Map>() constructor(options: BaseOptions = {}) { this.isTTY = options.isTTY ?? isTTY @@ -82,6 +84,7 @@ export abstract class BaseReporter implements Reporter { onTestRunStart(_specifications: ReadonlyArray): void { this.start = performance.now() this._timeStart = formatTimeString(new Date()) + this._perProjectBenchmarks.clear() } onTestRunEnd( @@ -97,6 +100,7 @@ export abstract class BaseReporter implements Reporter { this.ctx.logger.printNoTestFound(this.ctx.filenamePattern) } else { + this.printPerProjectBenchmarks() this.reportSummary(files, errors) } } @@ -107,6 +111,22 @@ export abstract class BaseReporter implements Reporter { } } + onTestCaseBenchmark(testCase: TestCase, benchmark: TestBenchmark): void { + const projectName = testCase.project.name || '' + for (const task of benchmark.tasks) { + if (!task.perProject) { + continue + } + const benchKey = `${testCase.module.relativeModuleId} > ${testCase.fullName} > ${task.name}` + let projectMap = this._perProjectBenchmarks.get(benchKey) + if (!projectMap) { + projectMap = new Map() + this._perProjectBenchmarks.set(benchKey, projectMap) + } + projectMap.set(projectName, task) + } + } + onTestSuiteResult(testSuite: TestSuite): void { if (testSuite.state() === 'failed') { this.logFailedTask(testSuite.task) @@ -206,6 +226,10 @@ export abstract class BaseReporter implements Reporter { const { duration = 0 } = test.diagnostic() || {} const padding = this.getTestIndentation(test.task) const suffix = this.getTestCaseSuffix(test) + const benchmarks = test.benchmarks() + // perProject tasks still appear in the inline table — they're additionally + // aggregated in the cross-project section at the end of the run + const inlineBenchmarks: TestBenchmark[] = benchmarks.filter(b => b.tasks.length > 0) if (testResult.state === 'failed') { this.log(c.red(` ${padding}${taskFail} ${this.getTestName(test.task, separator)}`) + suffix) @@ -213,16 +237,20 @@ export abstract class BaseReporter implements Reporter { // also print slow tests else if (duration > this.ctx.config.slowTestThreshold) { - this.log(` ${padding}${c.yellow(c.dim(F_CHECK))} ${this.getTestName(test.task, separator)} ${suffix}`) + this.log(` ${padding}${c.yellow(c.dim(F_CHECK))} ${this.getTestName(test.task, separator)}${suffix}`) } else if (this.ctx.config.hideSkippedTests && testResult.state === 'skipped' && test.options.mode !== 'todo') { // Skipped tests are hidden when --hideSkippedTests } - else if (this.renderSucceed || moduleState === 'failed') { + else if (this.renderSucceed || moduleState === 'failed' || inlineBenchmarks.length) { this.log(` ${padding}${this.getStateSymbol(test)} ${this.getTestName(test.task, separator)}${suffix}`) } + + if (inlineBenchmarks.length > 0) { + this.printBenchmarkTable(inlineBenchmarks, padding) + } } private getModuleLog(testModule: TestModule, counts: { @@ -545,12 +573,7 @@ export abstract class BaseReporter implements Reporter { const leakCount = this.printLeaksSummary() - if (this.ctx.config.mode === 'benchmark') { - this.reportBenchmarkSummary(files) - } - else { - this.reportTestSummary(files, errors, leakCount) - } + this.reportTestSummary(files, errors, leakCount) } reportTestSummary(files: File[], errors: unknown[], leakCount: number): void { @@ -873,34 +896,91 @@ export abstract class BaseReporter implements Reporter { return leakWithStacks.size } - reportBenchmarkSummary(files: File[]): void { - const benches = getTests(files) - const topBenches = benches.filter(i => i.result?.benchmark?.rank === 1) + protected printPerProjectBenchmarks(): void { + if (this._perProjectBenchmarks.size === 0) { + return + } + + let hasComparable = false + for (const projectMap of this._perProjectBenchmarks.values()) { + if (projectMap.size > 1) { + hasComparable = true + break + } + } + if (!hasComparable) { + return + } - this.log(`\n${withLabel('cyan', 'BENCH', 'Summary\n')}`) + this.log('') + this.log(divider(c.bold(c.bgBlue(` Cross-Project Benchmark Comparison `)), null, null, c.blue)) - for (const bench of topBenches) { - const group = bench.suite || bench.file + for (const [benchName, projectMap] of this._perProjectBenchmarks) { + const tasks = [...projectMap.entries()] + .sort((a, b) => a[1].latency.mean - b[1].latency.mean) + .map(([projectName, task], index) => ({ ...task, name: projectName, rank: index + 1 })) - if (!group) { + if (tasks.length <= 1) { continue } - const groupName = this.getFullName(group, separator) - const project = this.ctx.projects.find(p => p.name === bench.file.projectName) + this.log('') + this.log(` ${c.dim(benchName)}`) + this.printBenchmarkTable([{ name: benchName, tasks }], '', 'project') + } - this.log(` ${formatProjectName(project)}${bench.name}${c.dim(` - ${groupName}`)}`) + this.log('') + } - const siblings = group.tasks - .filter(i => i.meta.benchmark && i.result?.benchmark && i !== bench) - .sort((a, b) => a.result!.benchmark!.rank - b.result!.benchmark!.rank) + protected printBenchmarkTable(benchmarks: readonly TestBenchmark[], basePadding: string, columnName = 'name'): void { + let printedCount = 0 + for (const benchmark of benchmarks) { + const { tasks } = benchmark + if (tasks.length === 0) { + continue + } - for (const sibling of siblings) { - const number = (sibling.result!.benchmark!.mean / bench.result!.benchmark!.mean).toFixed(2) - this.log(c.green(` ${number}x `) + c.gray('faster than ') + sibling.name) + if (printedCount > 0) { + this.log('') } - this.log('') + const rows = tasks.map(t => renderBenchmarkRow(t)) + const tableHead = [ + columnName, + ...BENCH_TABLE_HEAD, + ] + const widths = computeBenchColumnWidths(tableHead, rows) + const indent = ` ${basePadding} ` + + this.log(`${indent}${padBenchRow(tableHead, widths).map(c.bold).join(' ')}`) + printedCount++ + + for (const task of tasks) { + const padded = padBenchRow(renderBenchmarkRow(task), widths) + let row = [ + padded[0], + c.blue(padded[1]), + c.cyan(padded[2]), + c.cyan(padded[3]), + c.cyan(padded[4]), + c.cyan(padded[5]), + c.cyan(padded[6]), + c.cyan(padded[7]), + c.cyan(padded[8]), + c.dim(padded[9]), + c.dim(padded[10]), + ].join(' ') + + if (task.rank === 1 && tasks.length > 1) { + row += c.bold(c.green(' fastest')) + } + + if (task.rank === tasks.length && tasks.length > 2) { + row += c.bold(c.gray(' slowest')) + } + + this.log(`${indent}${row}`) + } } } diff --git a/packages/vitest/src/node/reporters/benchmark/index.ts b/packages/vitest/src/node/reporters/benchmark/index.ts deleted file mode 100644 index 3d1db37c0b9b..000000000000 --- a/packages/vitest/src/node/reporters/benchmark/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { BenchmarkReporter } from './reporter' -import { VerboseBenchmarkReporter } from './verbose' - -export { - BenchmarkReporter, - VerboseBenchmarkReporter, -} - -export const BenchmarkReportsMap: { - default: typeof BenchmarkReporter - verbose: typeof VerboseBenchmarkReporter -} = { - default: BenchmarkReporter, - verbose: VerboseBenchmarkReporter, -} - -export type BenchmarkBuiltinReporters = keyof typeof BenchmarkReportsMap diff --git a/packages/vitest/src/node/reporters/benchmark/json-formatter.ts b/packages/vitest/src/node/reporters/benchmark/json-formatter.ts deleted file mode 100644 index 074d68b0713e..000000000000 --- a/packages/vitest/src/node/reporters/benchmark/json-formatter.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { File } from '@vitest/runner' -import type { BenchmarkResult } from '../../../runtime/types/benchmark' -import { getFullName, getTasks } from '@vitest/runner/utils' - -interface Report { - files: { - filepath: string - groups: Group[] - }[] -} - -interface Group { - fullName: string - benchmarks: FormattedBenchmarkResult[] -} - -export type FormattedBenchmarkResult = BenchmarkResult & { - id: string -} - -export function createBenchmarkJsonReport(files: File[]): Report { - const report: Report = { files: [] } - - for (const file of files) { - const groups: Group[] = [] - - for (const task of getTasks(file)) { - if (task?.type === 'suite') { - const benchmarks: FormattedBenchmarkResult[] = [] - - for (const t of task.tasks) { - const benchmark = t.meta.benchmark && t.result?.benchmark - - if (benchmark) { - benchmarks.push({ id: t.id, ...benchmark, samples: [] }) - } - } - - if (benchmarks.length) { - groups.push({ - fullName: getFullName(task, ' > '), - benchmarks, - }) - } - } - } - - report.files.push({ - filepath: file.filepath, - groups, - }) - } - - return report -} - -export function flattenFormattedBenchmarkReport(report: Report): Record { - const flat: Record = {} - - for (const file of report.files) { - for (const group of file.groups) { - for (const t of group.benchmarks) { - flat[t.id] = t - } - } - } - - return flat -} diff --git a/packages/vitest/src/node/reporters/benchmark/reporter.ts b/packages/vitest/src/node/reporters/benchmark/reporter.ts deleted file mode 100644 index 1abc7f376a4b..000000000000 --- a/packages/vitest/src/node/reporters/benchmark/reporter.ts +++ /dev/null @@ -1,114 +0,0 @@ -import type { TaskResultPack } from '@vitest/runner' -import type { SerializedError } from '@vitest/utils' -import type { Vitest } from '../../core' -import type { TestRunEndReason } from '../../types/reporter' -import type { TestModule, TestSuite } from '../reported-tasks' -import fs from 'node:fs' -import { getFullName } from '@vitest/runner/utils' -import * as pathe from 'pathe' -import c from 'tinyrainbow' -import { DefaultReporter } from '../default' -import { formatProjectName, getStateSymbol, separator } from '../renderers/utils' -import { createBenchmarkJsonReport, flattenFormattedBenchmarkReport } from './json-formatter' -import { renderTable } from './tableRender' - -export class BenchmarkReporter extends DefaultReporter { - compare?: Parameters[0]['compare'] - - async onInit(ctx: Vitest): Promise { - super.onInit(ctx) - - if (this.ctx.config.benchmark?.compare) { - const compareFile = pathe.resolve( - this.ctx.config.root, - this.ctx.config.benchmark?.compare, - ) - try { - this.compare = flattenFormattedBenchmarkReport( - JSON.parse(await fs.promises.readFile(compareFile, 'utf-8')), - ) - } - catch (e) { - this.error(`Failed to read '${compareFile}'`, e) - } - } - } - - onTaskUpdate(packs: TaskResultPack[]): void { - for (const pack of packs) { - const task = this.ctx.state.idMap.get(pack[0]) - - if (task?.type === 'suite' && task.result?.state !== 'run') { - task.tasks.filter(task => task.result?.benchmark) - .sort((benchA, benchB) => benchA.result!.benchmark!.mean - benchB.result!.benchmark!.mean) - .forEach((bench, idx) => { - bench.result!.benchmark!.rank = Number(idx) + 1 - }) - } - } - } - - onTestSuiteResult(testSuite: TestSuite): void { - super.onTestSuiteResult(testSuite) - this.printSuiteTable(testSuite) - } - - protected printTestModule(testModule: TestModule): void { - this.printSuiteTable(testModule) - } - - private printSuiteTable(testTask: TestModule | TestSuite): void { - const state = testTask.state() - if (state === 'pending' || state === 'queued') { - return - } - - const benches = testTask.task.tasks.filter(t => t.meta.benchmark) - const duration = testTask.task.result?.duration || 0 - - if (benches.length > 0 && benches.every(t => t.result?.state !== 'run' && t.result?.state !== 'queued')) { - let title = `\n ${getStateSymbol(testTask.task)} ${formatProjectName(testTask.project)}${getFullName(testTask.task, separator)}` - - if (duration != null && duration > this.ctx.config.slowTestThreshold) { - title += c.yellow(` ${Math.round(duration)}${c.dim('ms')}`) - } - - this.log(title) - this.log(renderTable({ - tasks: benches, - level: 1, - shallow: true, - columns: this.ctx.logger.getColumns(), - compare: this.compare, - showHeap: this.ctx.config.logHeapUsage, - slowTestThreshold: this.ctx.config.slowTestThreshold, - })) - } - } - - async onTestRunEnd( - testModules: ReadonlyArray, - unhandledErrors: ReadonlyArray, - reason: TestRunEndReason, - ): Promise { - super.onTestRunEnd(testModules, unhandledErrors, reason) - - // write output for future comparison - let outputFile = this.ctx.config.benchmark?.outputJson - - if (outputFile) { - outputFile = pathe.resolve(this.ctx.config.root, outputFile) - const outputDirectory = pathe.dirname(outputFile) - - if (!fs.existsSync(outputDirectory)) { - await fs.promises.mkdir(outputDirectory, { recursive: true }) - } - - const files = testModules.map(t => t.task.file) - const output = createBenchmarkJsonReport(files) - - await fs.promises.writeFile(outputFile, JSON.stringify(output, null, 2)) - this.log(`Benchmark report written to ${outputFile}`) - } - } -} diff --git a/packages/vitest/src/node/reporters/benchmark/tableRender.ts b/packages/vitest/src/node/reporters/benchmark/tableRender.ts deleted file mode 100644 index a613ac909546..000000000000 --- a/packages/vitest/src/node/reporters/benchmark/tableRender.ts +++ /dev/null @@ -1,221 +0,0 @@ -import type { Task } from '@vitest/runner' -import type { BenchmarkResult } from '../../../runtime/types/benchmark' -import type { FormattedBenchmarkResult } from './json-formatter' -import { stripVTControlCharacters } from 'node:util' -import { getTests } from '@vitest/runner/utils' -import { notNullish } from '@vitest/utils/helpers' -import c from 'tinyrainbow' -import { F_RIGHT } from '../renderers/figures' -import { getStateSymbol, truncateString } from '../renderers/utils' - -const outputMap = new WeakMap() - -function formatNumber(number: number) { - const res = String(number.toFixed(number < 100 ? 4 : 2)).split('.') - - return res[0].replace(/(?=(?:\d{3})+$)\B/g, ',') + (res[1] ? `.${res[1]}` : '') -} - -const tableHead = [ - 'name', - 'hz', - 'min', - 'max', - 'mean', - 'p75', - 'p99', - 'p995', - 'p999', - 'rme', - 'samples', -] - -function renderBenchmarkItems(result: BenchmarkResult) { - return [ - result.name, - formatNumber(result.hz || 0), - formatNumber(result.min || 0), - formatNumber(result.max || 0), - formatNumber(result.mean || 0), - formatNumber(result.p75 || 0), - formatNumber(result.p99 || 0), - formatNumber(result.p995 || 0), - formatNumber(result.p999 || 0), - `±${(result.rme || 0).toFixed(2)}%`, - (result.sampleCount || 0).toString(), - ] -} - -function computeColumnWidths(results: BenchmarkResult[]): number[] { - const rows = [tableHead, ...results.map(v => renderBenchmarkItems(v))] - - return Array.from(tableHead, (_, i) => - Math.max(...rows.map(row => stripVTControlCharacters(row[i]).length))) -} - -function padRow(row: string[], widths: number[]) { - return row.map( - (v, i) => (i ? v.padStart(widths[i], ' ') : v.padEnd(widths[i], ' ')), // name - ) -} - -function renderTableHead(widths: number[]) { - return ' '.repeat(3) + padRow(tableHead, widths).map(c.bold).join(' ') -} - -function renderBenchmark(result: BenchmarkResult, widths: number[]) { - const padded = padRow(renderBenchmarkItems(result), widths) - return [ - padded[0], // name - c.blue(padded[1]), // hz - c.cyan(padded[2]), // min - c.cyan(padded[3]), // max - c.cyan(padded[4]), // mean - c.cyan(padded[5]), // p75 - c.cyan(padded[6]), // p99 - c.cyan(padded[7]), // p995 - c.cyan(padded[8]), // p999 - c.dim(padded[9]), // rem - c.dim(padded[10]), // sample - ].join(' ') -} - -export function renderTable( - options: { - tasks: Task[] - level: number - shallow?: boolean - showHeap: boolean - columns: number - slowTestThreshold: number - compare?: Record - }, -): string { - const output: string[] = [] - - const benchMap: Record = {} - - for (const task of options.tasks) { - if (task.meta.benchmark && task.result?.benchmark) { - benchMap[task.id] = { - current: task.result.benchmark, - baseline: options.compare?.[task.id], - } - } - } - - const benchCount = Object.entries(benchMap).length - - const columnWidths = computeColumnWidths( - Object.values(benchMap) - .flatMap(v => [v.current, v.baseline]) - .filter(notNullish), - ) - - let idx = 0 - const padding = ' '.repeat(options.level ? 1 : 0) - - for (const task of options.tasks) { - const duration = task.result?.duration - const bench = benchMap[task.id] - - let prefix = '' - - if (idx === 0 && task.meta?.benchmark) { - prefix += `${renderTableHead(columnWidths)}\n${padding}` - } - - prefix += ` ${getStateSymbol(task)} ` - - let suffix = '' - - if (task.type === 'suite') { - suffix += c.dim(` (${getTests(task).length})`) - } - - if (task.mode === 'skip' || task.mode === 'todo') { - suffix += c.dim(c.gray(' [skipped]')) - } - - if (duration != null) { - const color = duration > options.slowTestThreshold ? c.yellow : c.green - - suffix += color(` ${Math.round(duration)}${c.dim('ms')}`) - } - - if (options.showHeap && task.result?.heap != null) { - suffix += c.magenta(` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`) - } - - if (bench) { - let body = renderBenchmark(bench.current, columnWidths) - - if (options.compare && bench.baseline) { - if (bench.current.hz) { - const diff = bench.current.hz / bench.baseline.hz - const diffFixed = diff.toFixed(2) - - if (diffFixed === '1.0.0') { - body += c.gray(` [${diffFixed}x]`) - } - - if (diff > 1) { - body += c.blue(` [${diffFixed}x] ⇑`) - } - else { - body += c.red(` [${diffFixed}x] ⇓`) - } - } - output.push(padding + prefix + body + suffix) - - const bodyBaseline = renderBenchmark(bench.baseline, columnWidths) - output.push(`${padding} ${bodyBaseline} ${c.dim('(baseline)')}`) - } - - else { - if (bench.current.rank === 1 && benchCount > 1) { - body += c.bold(c.green(' fastest')) - } - - if (bench.current.rank === benchCount && benchCount > 2) { - body += c.bold(c.gray(' slowest')) - } - - output.push(padding + prefix + body + suffix) - } - } - else { - output.push(padding + prefix + task.name + suffix) - } - - if (task.result?.state !== 'pass' && outputMap.get(task) != null) { - let data: string | undefined = outputMap.get(task) - - if (typeof data === 'string') { - data = stripVTControlCharacters(data.trim().split('\n').filter(Boolean).pop()!) - if (data === '') { - data = undefined - } - } - - if (data != null) { - const out = ` ${' '.repeat(options.level)}${F_RIGHT} ${data}` - output.push(c.gray(truncateString(out, options.columns))) - } - } - - if (!options.shallow && task.type === 'suite' && task.tasks.length > 0) { - if (task.result?.state) { - output.push(renderTable({ - ...options, - tasks: task.tasks, - level: options.level + 1, - shallow: false, - })) - } - } - idx++ - } - - return output.filter(Boolean).join('\n') -} diff --git a/packages/vitest/src/node/reporters/benchmark/verbose.ts b/packages/vitest/src/node/reporters/benchmark/verbose.ts deleted file mode 100644 index b3b20c91ccfa..000000000000 --- a/packages/vitest/src/node/reporters/benchmark/verbose.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { BenchmarkReporter } from './reporter' - -export class VerboseBenchmarkReporter extends BenchmarkReporter { - protected verbose = true -} diff --git a/packages/vitest/src/node/reporters/index.ts b/packages/vitest/src/node/reporters/index.ts index b9c46c7aecf8..e058597e305d 100644 --- a/packages/vitest/src/node/reporters/index.ts +++ b/packages/vitest/src/node/reporters/index.ts @@ -35,12 +35,6 @@ export { } export type { BaseReporter, Reporter, TestRunEndReason } -export type { BenchmarkBuiltinReporters } from './benchmark' -export { - BenchmarkReporter, - BenchmarkReportsMap, - VerboseBenchmarkReporter, -} from './benchmark' export type { JsonAssertionResult, JsonTestResult, diff --git a/packages/vitest/src/node/reporters/json.ts b/packages/vitest/src/node/reporters/json.ts index 1b63f17cc672..4091d23002a4 100644 --- a/packages/vitest/src/node/reporters/json.ts +++ b/packages/vitest/src/node/reporters/json.ts @@ -1,4 +1,4 @@ -import type { Suite, TaskMeta, TaskState } from '@vitest/runner' +import type { Suite, TaskMeta, TaskState, TestBenchmark } from '@vitest/runner' import type { SnapshotSummary } from '@vitest/snapshot' import type { CoverageMap } from 'istanbul-lib-coverage' import type { Vitest } from '../core' @@ -40,6 +40,7 @@ export interface JsonAssertionResult { failureMessages: Array | null location?: Callsite | null tags: string[] + benchmarks: TestBenchmark[] } export interface JsonTestResult { @@ -177,6 +178,7 @@ export class JsonReporter implements Reporter { })() : t.meta, tags: t.tags || [], + benchmarks: t.benchmarks, } satisfies JsonAssertionResult }) diff --git a/packages/vitest/src/node/reporters/junit.ts b/packages/vitest/src/node/reporters/junit.ts index 1b724307b5ca..35d92f81db59 100644 --- a/packages/vitest/src/node/reporters/junit.ts +++ b/packages/vitest/src/node/reporters/junit.ts @@ -11,6 +11,7 @@ import { stripVTControlCharacters } from 'node:util' import { getSuites } from '@vitest/runner/utils' import { basename, dirname, relative, resolve } from 'pathe' import { getOutputFile } from '../../utils/config-helpers' +import { renderBenchmarkTableText } from './renderers/benchmark-table' import { IndentedLogger } from './renderers/indented-logger' export interface ClassnameTemplateVariables { @@ -319,6 +320,30 @@ export class JUnitReporter implements Reporter { }) } + async writeSystemOut(task: Task): Promise { + const logs + = this.options.includeConsoleOutput && task.logs + ? task.logs.filter(log => log.type === 'stdout') + : [] + const benchmarks = task.type === 'test' ? task.benchmarks : [] + + if (logs.length === 0 && benchmarks.length === 0) { + return + } + + await this.writeElement('system-out', {}, async () => { + for (const log of logs) { + await this.baseLog(escapeXML(log.content)) + } + if (benchmarks.length > 0) { + if (logs.length > 0) { + await this.baseLog('') + } + await this.baseLog(escapeXML(renderBenchmarkTableText(benchmarks))) + } + }) + } + private applyTemplate( template: string | ((vars: ClassnameTemplateVariables) => string), vars: ClassnameTemplateVariables, @@ -368,8 +393,8 @@ export class JUnitReporter implements Reporter { time: getDuration(task), }, async () => { + await this.writeSystemOut(task) if (this.options.includeConsoleOutput) { - await this.writeLogs(task, 'out') await this.writeLogs(task, 'err') } @@ -569,6 +594,7 @@ export class JUnitReporter implements Reporter { file: null as any, annotations: [], artifacts: [], + benchmarks: [], } satisfies Task) } diff --git a/packages/vitest/src/node/reporters/renderers/benchmark-table.ts b/packages/vitest/src/node/reporters/renderers/benchmark-table.ts new file mode 100644 index 000000000000..4d15d23467c5 --- /dev/null +++ b/packages/vitest/src/node/reporters/renderers/benchmark-table.ts @@ -0,0 +1,80 @@ +import type { TestBenchmark, TestBenchmarkTask } from '@vitest/runner' +import { stripVTControlCharacters } from 'node:util' + +export const BENCH_TABLE_HEAD: string[] = [ + 'hz', + 'min', + 'max', + 'mean', + 'p75', + 'p99', + 'p995', + 'p999', + 'rme', + 'samples', +] + +function formatBenchNumber(number: number): string { + const res = String(number.toFixed(number < 100 ? 4 : 2)).split('.') + return res[0].replace(/(?=(?:\d{3})+$)\B/g, ',') + (res[1] ? `.${res[1]}` : '') +} + +// Plain-text rendering of the benchmark table (no ANSI colors, no indent). +// Used by the junit reporter to embed benchmark data in . +export function renderBenchmarkTableText( + benchmarks: readonly TestBenchmark[], + columnName = 'name', +): string { + const lines: string[] = [] + for (const benchmark of benchmarks) { + const { tasks } = benchmark + if (tasks.length === 0) { + continue + } + if (lines.length > 0) { + lines.push('') + } + const rows = tasks.map(renderBenchmarkRow) + const head = [columnName, ...BENCH_TABLE_HEAD] + const widths = computeBenchColumnWidths(head, rows) + lines.push(padBenchRow(head, widths).join(' ')) + for (const task of tasks) { + let row = padBenchRow(renderBenchmarkRow(task), widths).join(' ') + if (task.rank === 1 && tasks.length > 1) { + row += ' fastest' + } + if (task.rank === tasks.length && tasks.length > 2) { + row += ' slowest' + } + lines.push(row) + } + } + return lines.join('\n') +} + +export function renderBenchmarkRow(task: TestBenchmarkTask): string[] { + return [ + task.name, + formatBenchNumber(task.throughput.mean || 0), + formatBenchNumber(task.latency.min || 0), + formatBenchNumber(task.latency.max || 0), + formatBenchNumber(task.latency.mean || 0), + formatBenchNumber(task.latency.p75 || 0), + formatBenchNumber(task.latency.p99 || 0), + formatBenchNumber(task.latency.p995 || 0), + formatBenchNumber(task.latency.p999 || 0), + `\u00B1${(task.latency.rme || 0).toFixed(2)}%`, + String(task.latency.samplesCount || 0), + ] +} + +export function computeBenchColumnWidths(header: string[], rows: string[][]): number[] { + const allRows = [header, ...rows] + return Array.from(header, (_, i) => Math.max(...allRows.map(row => stripVTControlCharacters(row[i]).length))) +} + +export function padBenchRow(row: string[], widths: number[]): string[] { + return row.map( + (v, i) => (i === 0 ? v.padEnd(widths[i]) : v.padStart(widths[i])), + ) +} diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts index 39547748f0e3..97a77e42a7f7 100644 --- a/packages/vitest/src/node/reporters/reported-tasks.ts +++ b/packages/vitest/src/node/reporters/reported-tasks.ts @@ -8,6 +8,7 @@ import type { TaskMeta, TestAnnotation, TestArtifact, + TestBenchmark, } from '@vitest/runner' import type { SerializedError, TestError } from '@vitest/utils' import type { DevEnvironment } from 'vite' @@ -213,6 +214,15 @@ export class TestCase extends ReportedTaskImplementation { return [...this.task.artifacts] } + /** + * @experimental + * + * A list of benchmarks performed during the test. + */ + public benchmarks(): ReadonlyArray { + return [...this.task.benchmarks] + } + /** * Useful information about the test like duration, memory usage, etc. * Diagnostic is only available after the test has finished. diff --git a/packages/vitest/src/node/reporters/summary.ts b/packages/vitest/src/node/reporters/summary.ts index 1c9dcadc45f0..6e910cb90b32 100644 --- a/packages/vitest/src/node/reporters/summary.ts +++ b/packages/vitest/src/node/reporters/summary.ts @@ -2,7 +2,7 @@ import type { Vitest } from '../core' import type { TestSpecification } from '../test-specification' import type { Reporter } from '../types/reporter' import type { Options as WindowRendererOptions } from './renderers/windowedRenderer' -import type { ReportedHookContext, TestCase, TestModule } from './reported-tasks' +import type { ReportedHookContext, TestCase, TestModule, TestSuite } from './reported-tasks' import c from 'tinyrainbow' import { F_POINTER, F_TREE_NODE_END, F_TREE_NODE_MIDDLE } from './renderers/figures' import { formatProjectName, formatTime, formatTimeString, padSummaryTitle } from './renderers/utils' @@ -36,14 +36,14 @@ interface SlowTask { visible: boolean startTime: number onFinish: () => void - hook?: Omit + step?: Omit } interface RunningModule extends Pick { filename: TestModule['task']['name'] projectName: TestModule['project']['name'] projectColor: TestModule['project']['color'] - hook?: Omit + step?: Omit tests: Map meta: TestModule['task']['meta'] } @@ -139,42 +139,44 @@ export class SummaryReporter implements Reporter { this.renderer.schedule() } - onHookStart(options: ReportedHookContext): void { - const stats = this.getHookStats(options) - - if (!stats) { - return - } - - const hook = { - name: options.name, + private startStep(stats: RunningModule | SlowTask, name: string) { + const step = { + name, visible: false, startTime: performance.now(), onFinish: () => {}, } - stats.hook?.onFinish?.() - stats.hook = hook + stats.step?.onFinish?.() + stats.step = step if (!Number.isFinite(this.ctx.config.slowTestThreshold)) { return } const timeout = setTimeout(() => { - hook.visible = true + step.visible = true }, this.ctx.config.slowTestThreshold).unref() - hook.onFinish = () => clearTimeout(timeout) + step.onFinish = () => clearTimeout(timeout) + } + + onHookStart(options: ReportedHookContext): void { + const stats = this.getStepStats(options.entity) + + if (stats) { + this.startStep(stats, options.name) + } } onHookEnd(options: ReportedHookContext): void { - const stats = this.getHookStats(options) + const stats = this.getStepStats(options.entity) - if (stats?.hook?.name !== options.name) { + if (stats?.step?.name !== options.name) { return } - stats.hook.onFinish() - stats.hook.visible = false + stats.step.onFinish() + stats.step.visible = false } onTestCaseReady(test: TestCase): void { @@ -203,7 +205,7 @@ export class SummaryReporter implements Reporter { : undefined slowTest.onFinish = () => { - slowTest.hook?.onFinish() + slowTest.step?.onFinish() clearTimeout(timeout) } @@ -283,7 +285,7 @@ export class SummaryReporter implements Reporter { this.renderer.schedule() } - private getHookStats({ entity }: ReportedHookContext) { + private getStepStats(entity: TestSuite | TestModule | TestCase) { // Track slow running hooks only on verbose mode if (!this.options.verbose) { return @@ -317,7 +319,7 @@ export class SummaryReporter implements Reporter { ) const slowTasks = [ - testFile.hook, + testFile.step, ...testFile.tests.values(), ].filter((t): t is SlowTask => t != null && t.visible) @@ -331,8 +333,8 @@ export class SummaryReporter implements Reporter { + c.bold(c.yellow(` ${formatTime(Math.max(0, elapsed))}`)), ) - if (task.hook?.visible) { - summary.push(c.bold(c.yellow(` ${F_TREE_NODE_END} `)) + task.hook.name) + if (task.step?.visible) { + summary.push(c.bold(c.yellow(` ${F_TREE_NODE_END} `)) + task.step.name) } } } @@ -367,7 +369,7 @@ export class SummaryReporter implements Reporter { } const testFile = this.runningModules.get(id) - testFile?.hook?.onFinish() + testFile?.step?.onFinish() testFile?.tests?.forEach(test => test.onFinish()) this.runningModules.delete(id) diff --git a/packages/vitest/src/node/reporters/utils.ts b/packages/vitest/src/node/reporters/utils.ts index cf1067f0d27d..086fc910cc77 100644 --- a/packages/vitest/src/node/reporters/utils.ts +++ b/packages/vitest/src/node/reporters/utils.ts @@ -3,8 +3,8 @@ import type { Vitest } from '../core' import type { ResolvedConfig } from '../types/config' import type { Reporter } from '../types/reporter' import type { BlobReporter } from './blob' -import type { BenchmarkBuiltinReporters, BenchmarkReporter, BuiltinReporters, DefaultReporter, DotReporter, GithubActionsReporter, HangingProcessReporter, JsonReporter, JUnitReporter, TapReporter } from './index' -import { BenchmarkReportsMap, ReportersMap } from './index' +import type { BuiltinReporters, DefaultReporter, DotReporter, GithubActionsReporter, HangingProcessReporter, JsonReporter, JUnitReporter, TapReporter } from './index' +import { ReportersMap } from './index' async function loadCustomReporterModule( path: string, @@ -70,32 +70,4 @@ function createReporters( return Promise.all(promisedReporters) } -function createBenchmarkReporters( - reporterReferences: Array, - runner: ModuleRunner, -): Promise<(Reporter | BenchmarkReporter)[]> { - const promisedReporters = reporterReferences.map( - async (referenceOrInstance) => { - if (typeof referenceOrInstance === 'string') { - if (referenceOrInstance in BenchmarkReportsMap) { - const BuiltinReporter - = BenchmarkReportsMap[ - referenceOrInstance as BenchmarkBuiltinReporters - ] - return new BuiltinReporter() - } - else { - const CustomReporter = await loadCustomReporterModule( - referenceOrInstance, - runner, - ) - return new CustomReporter() - } - } - return referenceOrInstance - }, - ) - return Promise.all(promisedReporters) -} - -export { createBenchmarkReporters, createReporters } +export { createReporters } diff --git a/packages/vitest/src/node/reporters/verbose.ts b/packages/vitest/src/node/reporters/verbose.ts index 36a62f86271b..934236c9f160 100644 --- a/packages/vitest/src/node/reporters/verbose.ts +++ b/packages/vitest/src/node/reporters/verbose.ts @@ -44,5 +44,11 @@ export class VerboseReporter extends DefaultReporter { this.printAnnotations(test, 'log', 3) this.log() } + + const benchmarks = test.benchmarks() + const inlineBenchmarks = benchmarks.filter(b => b.tasks.length > 0) + if (inlineBenchmarks.length > 0) { + this.printBenchmarkTable(inlineBenchmarks, '') + } } } diff --git a/packages/vitest/src/node/test-run.ts b/packages/vitest/src/node/test-run.ts index 6493627585dd..9bd54f99e5df 100644 --- a/packages/vitest/src/node/test-run.ts +++ b/packages/vitest/src/node/test-run.ts @@ -1,11 +1,13 @@ import type { File as RunnerTestFile, + TaskEventData, TaskEventPack, TaskResultPack, TaskUpdateEvent, + TestArtifact, TestAttachment, + TestBenchmark, } from '@vitest/runner' -import type { TaskEventData, TestArtifact } from '@vitest/runner/types/tasks' import type { SerializedError } from '@vitest/utils' import type { UserConsoleLog } from '../types/general' import type { Vitest } from './core' @@ -55,33 +57,35 @@ export class TestRun { await this.vitest.report('onUserConsoleLog', log) } - async recordArtifact(testId: string, artifact: Artifact): Promise { - const task = this.vitest.state.idMap.get(testId) - const entity = task && this.vitest.state.getReportedEntity(task) + async recordBenchmark(testId: string, benchmark: TestBenchmark): Promise { + const testCase = this.getTestCaseById(testId, 'Benchmark') + testCase.task.benchmarks.push(benchmark) + await this.vitest.report('onTestCaseBenchmark', testCase, benchmark) + } - assert(task && entity, `Entity must be found for task ${task?.name || testId}`) - assert(entity.type === 'test', `Artifacts can only be recorded on a test, instead got ${entity.type}`) + async recordArtifact(testId: string, artifact: Artifact): Promise { + const testCase = this.getTestCaseById(testId, 'Artifact') // annotations won't resolve as artifacts for backwards compatibility until next major if (artifact.type === 'internal:annotation') { - await this.resolveTestAttachment(entity, artifact.annotation.attachment, artifact.annotation.message) + await this.resolveTestAttachment(testCase, artifact.annotation.attachment, artifact.annotation.message) - entity.task.annotations.push(artifact.annotation) + testCase.task.annotations.push(artifact.annotation) - await this.vitest.report('onTestCaseAnnotate', entity, artifact.annotation) + await this.vitest.report('onTestCaseAnnotate', testCase, artifact.annotation) return artifact } if (Array.isArray(artifact.attachments)) { await Promise.all( - artifact.attachments.map(attachment => this.resolveTestAttachment(entity, attachment)), + artifact.attachments.map(attachment => this.resolveTestAttachment(testCase, attachment)), ) } - entity.task.artifacts.push(artifact) + testCase.task.artifacts.push(artifact) - await this.vitest.report('onTestCaseArtifactRecord', entity, artifact) + await this.vitest.report('onTestCaseArtifactRecord', testCase, artifact) return artifact } @@ -102,6 +106,15 @@ export class TestRun { await this.vitest.report('onTaskUpdate', update, events) } + private getTestCaseById(testId: string, recordType: string) { + const task = this.vitest.state.idMap.get(testId) + const entity = task && this.vitest.state.getReportedEntity(task) + + assert(task && entity, `Entity must be found for task ${task?.name || testId}`) + assert(entity.type === 'test', `${recordType} can only be recorded on a test, instead got ${entity.type}`) + return entity + } + async end(specifications: TestSpecification[], errors: unknown[], coverage?: unknown): Promise { if (coverage) { await this.vitest.report('onCoverage', coverage) diff --git a/packages/vitest/src/node/types/benchmark.ts b/packages/vitest/src/node/types/benchmark.ts index 567f85e0025a..6423a38d9777 100644 --- a/packages/vitest/src/node/types/benchmark.ts +++ b/packages/vitest/src/node/types/benchmark.ts @@ -1,9 +1,6 @@ -import type { Arrayable } from '@vitest/utils' - -import type { BenchmarkBuiltinReporters } from '../reporters' -import type { Reporter } from './reporter' - export interface BenchmarkUserOptions { + enabled?: boolean + /** * Include globs for benchmark test files * @@ -13,7 +10,7 @@ export interface BenchmarkUserOptions { /** * Exclude globs for benchmark test files - * @default ['**\/node_modules/**', '**\/dist/**', '**\/cypress/**', '**\/.{idea,git,cache,output,temp}/**', '**\/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*'] + * @default [] */ exclude?: string[] @@ -25,35 +22,24 @@ export interface BenchmarkUserOptions { includeSource?: string[] /** - * Custom reporters to use for output. Can contain one or more built-in reporter names, reporter instances, - * and/or paths to custom reporter files to import. - * - * @default ['default'] - */ - reporters?: Arrayable - - /** - * @deprecated Use `benchmark.outputJson` instead - */ - outputFile?: - | string - | (Partial> - & Record) - - /** - * benchmark output file to compare against + * Include `samples` array of benchmark results for API or custom reporter usages. + * This is disabled by default to reduce memory usage. + * @default false */ - compare?: string + retainSamples?: boolean /** - * benchmark output file + * Disable warnings when a benchmark accesses module export getters too many times. + * @default false */ - outputJson?: string + suppressExportGetterWarnings?: boolean /** - * Include `samples` array of benchmark results for API or custom reporter usages. - * This is disabled by default to reduce memory usage. - * @default false + * The name of the parent project that this benchmark project was cloned + * from. Populated automatically when Vitest creates the dedicated benchmark + * project for a parent project. Used by the runtime as the value for the + * `${projectName}` placeholder in `writeResult` / `bench.from()` paths. + * @internal */ - includeSamples?: boolean + projectName?: string } diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index 53a2e37d9596..a27ad7bdd80f 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -73,7 +73,10 @@ export interface EnvironmentOptions { export type { HappyDOMOptions, JSDOMOptions } -export type VitestRunMode = 'test' | 'benchmark' +/** + * @deprecated + */ +export type VitestRunMode = 'test' export interface ProjectName { label: string @@ -1093,16 +1096,6 @@ export interface UserConfig extends InlineConfig { */ clearScreen?: boolean - /** - * benchmark.compare option exposed at the top level for cli - */ - compare?: string - - /** - * benchmark.outputJson option exposed at the top level for cli - */ - outputJson?: string - /** * Directory of blob reports to merge * @default '.vitest/blob' @@ -1159,8 +1152,6 @@ export interface ResolvedConfig | 'fileParallelism' | 'tagsFilter' > { - mode: VitestRunMode - name: ProjectName['label'] color?: ProjectName['color'] base?: string @@ -1191,10 +1182,7 @@ export interface ResolvedConfig cliExclude?: string[] project: string[] - benchmark?: Required< - Omit - > - & Pick + benchmark: Required shard?: { index: number count: number diff --git a/packages/vitest/src/node/types/reporter.ts b/packages/vitest/src/node/types/reporter.ts index cb43e475c97e..62ec71a0695c 100644 --- a/packages/vitest/src/node/types/reporter.ts +++ b/packages/vitest/src/node/types/reporter.ts @@ -1,4 +1,4 @@ -import type { File, TaskEventPack, TaskResultPack, TestAnnotation, TestArtifact } from '@vitest/runner' +import type { File, TaskEventPack, TaskResultPack, TestAnnotation, TestArtifact, TestBenchmark } from '@vitest/runner' import type { Awaitable, SerializedError } from '@vitest/utils' import type { UserConsoleLog } from '../../types/general' import type { Vitest } from '../core' @@ -96,4 +96,10 @@ export interface Reporter { onHookEnd?: (hook: ReportedHookContext) => Awaitable onCoverage?: (coverage: unknown) => Awaitable + + /** + * @experimental + * Called after the benchmark is finished. + */ + onTestCaseBenchmark?: (testCase: TestCase, benchmark: TestBenchmark) => Awaitable } diff --git a/packages/vitest/src/public/index.ts b/packages/vitest/src/public/index.ts index a7414198e989..453baa134ef4 100644 --- a/packages/vitest/src/public/index.ts +++ b/packages/vitest/src/public/index.ts @@ -34,7 +34,15 @@ export { Snapshots } from '../integrations/snapshot/chai' export { vi, vitest } from '../integrations/vi' export type { VitestUtils } from '../integrations/vi' -export { bench } from '../runtime/benchmark' +export type { + Bench, + BenchCompareOptions, + BenchFnOptions, + BenchFromSource, + BenchRegistration, + BenchResult, + BenchStorage, +} from '../runtime/benchmark' export type { RuntimeConfig, @@ -45,18 +53,7 @@ export type { export { VitestEvaluatedModules as EvaluatedModules } from '../runtime/moduleRunner/evaluatedModules' -export { NodeBenchmarkRunner as BenchmarkRunner } from '../runtime/runners/benchmark' export { TestRunner } from '../runtime/runners/test' -export type { - BenchFactory, - BenchFunction, - Benchmark, - BenchmarkAPI, - BenchmarkResult, - BenchOptions, - BenchTask, - BenchTaskResult, -} from '../runtime/types/benchmark' export { assertType } from '../typecheck/assertType' export type { AssertType } from '../typecheck/assertType' @@ -117,6 +114,7 @@ export { test, } from '@vitest/runner' export type { + BaselineData, ImportDuration, OnTestFailedHandler, OnTestFinishedHandler, @@ -144,6 +142,8 @@ export type { TestArtifactLocation, TestArtifactRegistry, TestAttachment, + TestBenchmark, + TestBenchmarkTask, TestContext, TestFunction, TestOptions, diff --git a/packages/vitest/src/public/node.ts b/packages/vitest/src/public/node.ts index db229f85e82a..5da63fb42a37 100644 --- a/packages/vitest/src/public/node.ts +++ b/packages/vitest/src/public/node.ts @@ -44,8 +44,6 @@ export type { SerializedTestProject, TestProject } from '../node/project' export { AgentReporter, - BenchmarkReporter, - BenchmarkReportsMap, DefaultReporter, DotReporter, GithubActionsReporter, @@ -56,12 +54,10 @@ export { ReportersMap, TapFlatReporter, TapReporter, - VerboseBenchmarkReporter, VerboseReporter, } from '../node/reporters' export type { BaseReporter, - BenchmarkBuiltinReporters, BuiltinReporterOptions, BuiltinReporters, JsonAssertionResult, diff --git a/packages/vitest/src/runtime/benchmark.ts b/packages/vitest/src/runtime/benchmark.ts index f78178ea85db..db8823489413 100644 --- a/packages/vitest/src/runtime/benchmark.ts +++ b/packages/vitest/src/runtime/benchmark.ts @@ -1,70 +1,517 @@ -import type { Test } from '@vitest/runner' -import type { BenchFunction, BenchmarkAPI, BenchOptions } from './types/benchmark' -import { getCurrentSuite } from '@vitest/runner' -import { createChainable } from '@vitest/runner/utils' -import { noop } from '@vitest/utils/helpers' +import type { BaselineData, Test, TestBenchmark, TestBenchmarkTask } from '@vitest/runner' +import type { + BenchOptions as BenchCompareOptions, + Fn, + FnOptions, + TaskResultCompleted, + TaskResultRuntimeInfo, + TaskResultTimestampProviderInfo, + Task as TinybenchTask, +} from 'tinybench' +import type { SerializedConfig } from './config' +import { isAbsolute, relative } from 'pathe' +import { Bench as Tinybench } from 'tinybench' +import c from 'tinyrainbow' +import { rpc } from './rpc' +import { TestRunner } from './runners/test' import { getWorkerState } from './utils' -const benchFns = new WeakMap() -const benchOptsMap = new WeakMap() +const now = globalThis.performance + ? globalThis.performance.now.bind(globalThis.performance) + : Date.now -export function getBenchOptions(key: Test): BenchOptions { - return benchOptsMap.get(key) +const kRegistration: unique symbol = Symbol('registration') +const kFromSource: unique symbol = Symbol('fromSource') +const kPerProject: unique symbol = Symbol('perProject') +const kWriteResult: unique symbol = Symbol('writeResult') +export const kFinalize: unique symbol = Symbol('finalize') + +type ExtractBenchNames[]> = Exclude<{ + [K in keyof T]: T[K] extends BenchRegistration ? N : never +}[number], never> + +// We throw an error if benchmark did not complete, so it will always be TaskResultCompleted +export type BenchResult = TaskResultCompleted & TaskResultRuntimeInfo & TaskResultTimestampProviderInfo + +export interface BenchStorage { + get: (name: T) => BenchResult +} + +export type { BenchOptions as BenchCompareOptions } from 'tinybench' + +/** + * Options accepted by `bench(name, options, fn)`. Extends tinybench's + * `FnOptions` with Vitest-specific fields. + */ +export interface BenchFnOptions extends FnOptions { + /** + * Path (relative to the project root) where the benchmark result is written + * after a successful run. The string `${projectName}` is substituted with + * the current project name. Absolute paths are accepted as long as they + * resolve inside the project root. + */ + writeResult?: string + /** + * Mark this benchmark as a per-project entry. Per-project tasks still appear + * in the inline comparison table for the current run, and Vitest additionally + * collects them across projects and prints a single cross-project table at + * the end of the run. + */ + perProject?: boolean +} + +export interface BenchRegistration { + name: Name + /** + * The benchmark function. Absent for registrations created via `bench.from()`. + */ + fn?: Fn + /** + * Per-benchmark options (`beforeEach`, `beforeAll`, etc.). Absent for + * registrations created via `bench.from()`. + */ + fnOpts?: FnOptions + /** + * @internal + */ + [kRegistration]: true + run: (options?: BenchCompareOptions) => Promise +} + +interface BenchCompare { + []>(...args: Args): Promise>> + []>(...args: [...Args, BenchCompareOptions]): Promise>> } -export function getBenchFn(key: Test): BenchFunction { - return benchFns.get(key)! +interface BenchFactory { + (name: Name | Function, fn: Fn): BenchRegistration + (name: Name | Function, options: BenchFnOptions, fn: Fn): BenchRegistration } -export const bench: BenchmarkAPI = createBenchmark(function ( - name, - fn: BenchFunction = noop, - options: BenchOptions = {}, -) { - if (getWorkerState().config.mode !== 'benchmark') { - throw new Error('`bench()` is only available in benchmark mode.') +export interface BenchFromSource { + (): BaselineData | Promise +} + +interface BenchFrom { + (name: Name | Function, source: string | BenchFromSource): BenchRegistration +} + +export interface Bench extends BenchFactory { + compare: BenchCompare + from: BenchFrom + /** @internal */ + [kFinalize]: () => void +} + +interface RunnableRegistration extends BenchRegistration { + fn: Fn + fnOpts?: FnOptions + [kWriteResult]?: string + [kPerProject]?: true +} + +interface FromRegistration extends BenchRegistration { + [kFromSource]: string | BenchFromSource +} + +function isFromRegistration(reg: BenchRegistration): reg is FromRegistration { + return kFromSource in reg +} + +function substitutePath(template: string, projectName: string | undefined): string { + return template.replace(/\$\{projectName\}/g, projectName ?? '') +} + +export function createBench(test: Test, config: SerializedConfig): Bench { + let benchIdx = 0 + const pending = new Set>() + const createTinybench = (options?: BenchCompareOptions) => { + const currentIndex = ++benchIdx + return new Tinybench({ + signal: test.context.signal, + name: `${test.fullTestName} ${currentIndex}`, + retainSamples: config.benchmark.retainSamples, + ...options, + now, + }) } - const task = getCurrentSuite().task(formatName(name), { - ...this, - meta: { - benchmark: true, - }, + const resolveTemplate = (template: string) => substitutePath(template, config.benchmark.projectName) + + const resolveFromSource = async (source: string | BenchFromSource): Promise => { + if (typeof source === 'function') { + return source() + } + const resolved = resolveTemplate(source) + const data = await rpc().readBenchmarkResult(resolved) + if (data == null) { + throw new Error(`\`bench.from()\` could not find a result file at "${resolved}". Run the source benchmark first to create it.`) + } + return data + } + + const taskFromBaseline = ( + name: string, + data: BaselineData, + ): TestBenchmarkTask => ({ + name, + latency: data.latency, + throughput: data.throughput, + period: data.period, + totalTime: data.totalTime, + rank: 0, + fromStore: true, }) - benchFns.set(task, fn) - benchOptsMap.set(task, options) - // vitest runner sets mode to `todo` if handler is not passed down - // but we store handler separately - if (!this.todo && task.mode === 'todo') { - task.mode = 'run' - } -}) - -function createBenchmark( - fn: ( - this: Record<'skip' | 'only' | 'todo', boolean | undefined>, - name: string | Function, - fn?: BenchFunction, - options?: BenchOptions, - ) => void, -) { - const benchmark = createChainable( - ['skip', 'only', 'todo'], - fn, - ) as BenchmarkAPI - - benchmark.skipIf = (condition: any) => - (condition ? benchmark.skip : benchmark) as BenchmarkAPI - benchmark.runIf = (condition: any) => - (condition ? benchmark : benchmark.skip) as BenchmarkAPI - - return benchmark as BenchmarkAPI -} - -function formatName(name: string | Function) { - return typeof name === 'string' - ? name - : typeof name === 'function' - ? name.name || '' - : String(name) + + const createCompareStorage = ( + bench: Tinybench, + fromResults?: Map, + ): BenchStorage => { + return { + get(name: T) { + const stored = fromResults?.get(name) + if (stored) { + return stored as BenchResult + } + const task = bench.getTask(name) + if (!task) { + throw new Error(`task "${name}" was not defined`) + } + return task.result as BenchResult + }, + } + } + + interface TaskMeta { perProject?: true } + + const serializeBenchmark = ( + tinybenchTasks: TinybenchTask[], + name: string | undefined, + taskMeta?: Map, + fromTasks?: TestBenchmarkTask[], + ): TestBenchmark => { + const tasks: TestBenchmarkTask[] = tinybenchTasks.map((t) => { + const result = t.result + if (result.state === 'errored') { + throw result.error + } + if (result.state !== 'completed') { + throw new Error(`task "${t.name}" did not complete: received "${result.state}"`) + } + return { + name: t.name, + latency: result.latency, + throughput: result.throughput, + period: result.period, + totalTime: result.totalTime, + rank: 0, + ...taskMeta?.get(t.name), + } + }) + if (fromTasks) { + tasks.push(...fromTasks) + } + tasks.sort((a, b) => a.latency.mean - b.latency.mean) + tasks.forEach((task, idx) => { + task.rank = idx + 1 + }) + return { + name: name || test.fullTestName, + tasks, + } + } + + const recordBenchmark = async ( + tinybenchTasks: TinybenchTask[], + name: string | undefined, + taskMeta?: Map, + fromTasks?: TestBenchmarkTask[], + ) => { + const serializedBenchmark = serializeBenchmark(tinybenchTasks, name, taskMeta, fromTasks) + test.benchmarks.push(serializedBenchmark) + await rpc().onTestBenchmark(test.id, serializedBenchmark) + } + + const writeResultArtifact = async (template: string, result: BenchResult) => { + const resolved = resolveTemplate(template) + const data: BaselineData = { + latency: result.latency, + throughput: result.throughput, + period: result.period, + totalTime: result.totalTime, + } + await rpc().writeBenchmarkResult(resolved, data) + } + + const runBenchmarks = async (tinybench: Tinybench) => { + const workerState = getWorkerState() + const getterTracker = workerState.getterTracker + getterTracker?.resetInvocations() + try { + return await TestRunner.runBenchmarks(tinybench) + } + finally { + const excessiveInvocations = config.benchmark.suppressExportGetterWarnings + ? undefined + : getterTracker?.getExcessiveInvocations() + if (excessiveInvocations?.length) { + const entries = excessiveInvocations + .map(({ moduleId, exportName }) => ` - ${formatModuleId(moduleId, workerState.config.root)} > ${exportName}`) + .join('\n') + console.warn( + [ + c.yellow(c.bold('Benchmark Warning')), + `Benchmark ${c.bold(`"${tinybench.name}"`)} accessed module export getters too many times.`, + '', + 'This can make results unreliable because export getters add overhead.', + 'See https://vitest.dev/guide/benchmarking#module-runner-overhead', + '', + 'Tracked exports:', + entries, + ].join('\n'), + ) + } + } + } + + const runSingle = async ( + name: string, + fn: Fn, + fnOpts: FnOptions | undefined, + options: BenchCompareOptions | undefined, + meta: TaskMeta | undefined, + writeResult: string | undefined, + ): Promise => { + const tinybench = createTinybench(options).add(name, fn, fnOpts) + const tasks = await runBenchmarks(tinybench) + const task = tinybench.getTask(name)! + if (task.result.state === 'errored') { + throw task.result.error + } + await recordBenchmark(tasks, tinybench.name, meta ? new Map([[name, meta]]) : undefined) + if (writeResult) { + await writeResultArtifact(writeResult, task.result as BenchResult) + } + return task.result as BenchResult + } + + const runFrom = async ( + name: string, + source: string | BenchFromSource, + ): Promise => { + const data = await resolveFromSource(source) + const benchmark: TestBenchmark = { + name: test.fullTestName, + tasks: [{ ...taskFromBaseline(name, data), rank: 1 }], + } + test.benchmarks.push(benchmark) + await rpc().onTestBenchmark(test.id, benchmark) + return data as BenchResult + } + + const bench: Bench = (nameOrFunction: string | Function, a: Fn | BenchFnOptions, b?: Fn | BenchFnOptions) => { + validateBenchmarkProject(config) + const { fn, fnOpts, writeResult, perProject } = normalizeBenchArgs(a, b) + const name = typeof nameOrFunction === 'function' ? nameOrFunction.name || '' : nameOrFunction + const meta: TaskMeta | undefined = perProject ? { perProject: true } : undefined + const registration: RunnableRegistration = { + [kRegistration]: true, + name, + fn, + fnOpts, + run: (options?: BenchCompareOptions) => { + pending.delete(registration) + return runSingle(name, fn, fnOpts, options, meta, writeResult) + }, + } + if (perProject) { + registration[kPerProject] = true + } + if (writeResult) { + registration[kWriteResult] = writeResult + } + pending.add(registration) + return registration + } + + bench.from = (nameOrFunction: Name | Function, source: string | BenchFromSource): BenchRegistration => { + validateBenchmarkProject(config) + if (typeof nameOrFunction !== 'string' && typeof nameOrFunction !== 'function') { + throw new TypeError('`bench.from()` requires a name (string or named function) as its first argument.') + } + if (typeof source !== 'string' && typeof source !== 'function') { + throw new TypeError('`bench.from()` expects a string path or a function returning the result data as its second argument.') + } + const name = (typeof nameOrFunction === 'function' ? nameOrFunction.name || '' : nameOrFunction) as Name + const registration: FromRegistration = { + [kRegistration]: true, + [kFromSource]: source, + name, + run: () => { + pending.delete(registration) + return runFrom(name, source) + }, + } + pending.add(registration) + return registration + } + + bench.compare = async (...args) => { + validateBenchmarkProject(config) + + // extract optional trailing BenchCompareOptions argument + const lastArg = args[args.length - 1] + const isOptions = lastArg != null && typeof lastArg === 'object' && !(kRegistration in lastArg) + const benchOptions = isOptions ? args.pop() as BenchCompareOptions : undefined + const registrations = args as BenchRegistration[] + + // Mark every passed-in registration as consumed before validation so a + // throwing `bench.compare()` (wrong arity, wrong shape) doesn't also + // trigger the unrun-bench warning — the user's intent was to consume them. + for (const reg of registrations) { + if (reg != null && typeof reg === 'object' && kRegistration in reg) { + pending.delete(reg) + } + } + + if (registrations.length < 2) { + throw new SyntaxError(`\`bench.compare()\` requires at least 2 benchmarks, received ${registrations.length} instead. ${registrations.length === 1 ? 'Consider calling `bench().run()`. ' : 'Define benchmarks by calling `bench()`. '}See https://vitest.dev/guide/benchmarking#comparing-benchmarks`) + } + for (const reg of registrations) { + if (reg == null || typeof reg !== 'object' || !(kRegistration in reg)) { + throw new SyntaxError('`bench.compare()` expects every argument to be the return value of `bench` or `bench.from`.') + } + } + + const runnable: RunnableRegistration[] = [] + const fromEntries: FromRegistration[] = [] + for (const reg of registrations) { + if (isFromRegistration(reg)) { + fromEntries.push(reg) + } + else { + runnable.push(reg as RunnableRegistration) + } + } + + const taskMeta = new Map() + for (const reg of runnable) { + if (reg[kPerProject]) { + taskMeta.set(reg.name, { perProject: true }) + } + } + + const fromResults = new Map() + const fromTasks: TestBenchmarkTask[] = [] + if (fromEntries.length > 0) { + const resolved = await Promise.all( + fromEntries.map(async (reg) => { + const data = await resolveFromSource(reg[kFromSource]) + return { reg, data } + }), + ) + for (const { reg, data } of resolved) { + fromResults.set(reg.name, data) + fromTasks.push(taskFromBaseline(reg.name, data)) + } + } + + const tinybench = createTinybench(benchOptions) + runnable.forEach((reg) => { + tinybench.add(reg.name, reg.fn, reg.fnOpts) + }) + + let tasks: TinybenchTask[] = [] + if (runnable.length > 0) { + tasks = await runBenchmarks(tinybench) + const errors = tinybench.tasks + .filter(task => task.result.state === 'errored') + .map(task => (task.result as any).error) + if (errors.length > 0) { + throw new AggregateError(errors, 'Some benchmarks failed') + } + } + + await recordBenchmark(tasks, tinybench.name, taskMeta, fromTasks) + + // write artifacts for every runnable registration that requested it. We + // do this after recording so a write failure can't be confused with a + // benchmark failure in the reporter output. + await Promise.all( + runnable + .filter(reg => reg[kWriteResult] != null) + .map((reg) => { + const task = tinybench.getTask(reg.name)! + return writeResultArtifact(reg[kWriteResult]!, task.result as BenchResult) + }), + ) + + return createCompareStorage(tinybench, fromResults) + } + + bench[kFinalize] = () => { + if (pending.size === 0) { + return + } + const names = [...pending].map(reg => `"${reg.name}"`).join(', ') + pending.clear() + console.warn( + [ + c.yellow(c.bold('Benchmark Warning')), + `Test ${c.bold(`"${test.fullTestName}"`)} registered benchmarks that never ran: ${names}.`, + '', + 'Call `.run()` on the registration, or pass it to `bench.compare()`.', + 'See https://vitest.dev/guide/benchmarking#defining-a-benchmark', + ].join('\n'), + ) + } + + return bench +} + +function formatModuleId(moduleId: string, root: string): string { + if (!root || !isAbsolute(moduleId)) { + return moduleId + } + return relative(root, moduleId) +} + +function normalizeBenchArgs( + a: Fn | BenchFnOptions, + b: Fn | BenchFnOptions | undefined, +): { fn: Fn; fnOpts: FnOptions | undefined; writeResult: string | undefined; perProject: boolean } { + if (typeof a === 'function') { + if (b !== undefined) { + throw new TypeError('`bench()` does not accept options as the third argument. Pass options as the second argument instead: `bench(name, options, fn)`.') + } + return { fn: a, fnOpts: undefined, writeResult: undefined, perProject: false } + } + if (typeof b !== 'function') { + throw new TypeError('`bench()` expects a benchmark function. Call `bench(name, fn)` or `bench(name, options, fn)`.') + } + // Strip vitest-specific fields only when present so we don't allocate a new + // object — preserving referential identity matters: users inspect + // `registration.fnOpts` and tinybench's `add` sees the same object the + // caller passed in. + if (a.writeResult === undefined && a.perProject === undefined) { + return { fn: b, fnOpts: a as FnOptions, writeResult: undefined, perProject: false } + } + const { writeResult, perProject, ...fnOpts } = a + return { + fn: b, + fnOpts: Object.keys(fnOpts).length > 0 ? fnOpts as FnOptions : undefined, + writeResult, + perProject: perProject ?? false, + } +} + +function validateBenchmarkProject(config: SerializedConfig) { + if (!config.benchmark.enabled) { + throw new Error( + `Cannot use the \`bench\` test-context fixture within a regular test run. ` + + `Benchmarks are inherently flaky, so Vitest runs them in a dedicated project based on the \`benchmark.include\` pattern (default \`**/*.{bench,benchmark}.?(c|m)[jt]s?(x)\`). ` + + `Move this code to a file matched by \`benchmark.include\`, and make sure \`bench\` is destructured from the test context (\`test('...', async ({ bench }) => { ... })\`) — it is not a top-level export of \`vitest\`. ` + + `See https://vitest.dev/guide/benchmarking#stability`, + ) + } } diff --git a/packages/vitest/src/runtime/config.ts b/packages/vitest/src/runtime/config.ts index a8e2e298bc57..d784162cc3bd 100644 --- a/packages/vitest/src/runtime/config.ts +++ b/packages/vitest/src/runtime/config.ts @@ -1,6 +1,6 @@ import type { Config as FakeTimersConfig } from '@sinonjs/fake-timers' import type { PrettyFormatOptions } from '@vitest/pretty-format' -import type { SequenceHooks, SequenceSetupFiles, SerializableRetry, TestTagDefinition } from '@vitest/runner' +import type { SequenceHooks, VitestRunnerConfig } from '@vitest/runner' import type { SnapshotEnvironment, SnapshotUpdateState } from '@vitest/snapshot' import type { SerializedDiffOptions } from '@vitest/utils/diff' import type { LabelColor } from '../types/general' @@ -8,8 +8,7 @@ import type { LabelColor } from '../types/general' /** * Config that tests have access to. */ -export interface SerializedConfig { - name: string | undefined +export interface SerializedConfig extends VitestRunnerConfig { color?: LabelColor globals: boolean base: string | undefined @@ -18,16 +17,8 @@ export interface SerializedConfig { runner: string | undefined isolate: boolean maxWorkers: number - mode: 'test' | 'benchmark' bail: number | undefined environmentOptions?: Record - root: string - setupFiles: string[] - passWithNoTests: boolean - testNamePattern: RegExp | undefined - allowOnly: boolean - testTimeout: number - hookTimeout: number clearMocks: boolean mockReset: boolean restoreMocks: boolean @@ -35,7 +26,6 @@ export interface SerializedConfig { unstubEnvs: boolean // TODO: make optional fakeTimers: FakeTimersConfig - maxConcurrency: number defines: Record expect: { requireAssertions?: boolean @@ -45,13 +35,6 @@ export interface SerializedConfig { } } printConsoleTrace: boolean | undefined - sequence: { - shuffle?: boolean - concurrent?: boolean - seed: number - hooks: SequenceHooks - setupFiles: SequenceSetupFiles - } deps: { web: { transformAssets?: boolean @@ -84,8 +67,6 @@ export interface SerializedConfig { allowWrite: boolean | undefined } diff: string | SerializedDiffOptions | undefined - retry: SerializableRetry - includeTaskLocation: boolean | undefined inspect: boolean | string | undefined inspectBrk: boolean | string | undefined inspector: { @@ -130,8 +111,11 @@ export interface SerializedConfig { detectAsyncLeaks: boolean coverage: SerializedCoverageConfig benchmark: { - includeSamples: boolean - } | undefined + enabled: boolean + retainSamples: boolean + suppressExportGetterWarnings: boolean + projectName: string + } serializedDefines: string experimental: { fsModuleCache: boolean @@ -152,9 +136,6 @@ export interface SerializedConfig { browserSdkPath?: string } | undefined } - tags: TestTagDefinition[] - tagsFilter: string[] | undefined - strictTags: boolean mergeReportsLabel: string | undefined slowTestThreshold: number | undefined disableColors: boolean diff --git a/packages/vitest/src/runtime/getter-tracker.ts b/packages/vitest/src/runtime/getter-tracker.ts new file mode 100644 index 000000000000..d4bd30858ea3 --- /dev/null +++ b/packages/vitest/src/runtime/getter-tracker.ts @@ -0,0 +1,37 @@ +export class GetterTracker { + static EXPORTS_MAX_INVOCATIONS = 1_000_000 + + private invocations = new Map() + private excessiveInvocations = new Map() + + public createTracker( + moduleId: string, + defineExport: (name: string, getter: () => unknown) => void, + ): (name: string, getter: () => unknown) => void { + return (name, getter) => { + const key = `${moduleId}:${name}` + defineExport(name, () => { + const count = (this.invocations.get(key) || 0) + 1 + this.invocations.set(key, count) + if (count > GetterTracker.EXPORTS_MAX_INVOCATIONS && !this.excessiveInvocations.has(key)) { + this.excessiveInvocations.set(key, { moduleId, exportName: name }) + } + return getter() + }) + } + } + + public resetInvocations(): void { + this.invocations.clear() + this.excessiveInvocations.clear() + } + + public getExcessiveInvocations(): GetterTrackerExport[] { + return [...this.excessiveInvocations.values()] + } +} + +export interface GetterTrackerExport { + moduleId: string + exportName: string +} diff --git a/packages/vitest/src/runtime/moduleRunner/moduleEvaluator.ts b/packages/vitest/src/runtime/moduleRunner/moduleEvaluator.ts index 9fc24fd352f7..f5caccd70630 100644 --- a/packages/vitest/src/runtime/moduleRunner/moduleEvaluator.ts +++ b/packages/vitest/src/runtime/moduleRunner/moduleEvaluator.ts @@ -5,6 +5,7 @@ import type { ModuleRunnerContext, ModuleRunnerImportMeta, } from 'vite/module-runner' +import type { GetterTracker } from '../getter-tracker' import type { VitestEvaluatedModules } from './evaluatedModules' import type { ModuleExecutionInfo } from './moduleDebug' import type { VitestVmOptions } from './moduleRunner' @@ -30,6 +31,7 @@ export interface VitestModuleEvaluatorOptions { getCurrentTestFilepath?: () => string | undefined compiledFunctionArgumentsNames?: string[] compiledFunctionArgumentsValues?: unknown[] + getterTracker?: GetterTracker /** * @internal */ @@ -43,6 +45,9 @@ export class VitestModuleEvaluator implements ModuleEvaluator { private compiledFunctionArgumentsNames?: string[] private compiledFunctionArgumentsValues: unknown[] = [] + private getterTracker: GetterTracker | undefined + + static EXPORTS_MAX_INVOCATIONS = 1_000_000 private primitives: { Object: typeof Object @@ -81,6 +86,7 @@ export class VitestModuleEvaluator implements ModuleEvaluator { Reflect, } } + this.getterTracker = options.getterTracker } private convertIdToImportUrl(id: string) { @@ -296,6 +302,7 @@ export class VitestModuleEvaluator implements ModuleEvaluator { argumentsList.push( // TODO@discuss deprecate in Vitest 5, remove in Vitest 6(?) // backwards compat for vite-node + // https://github.com/vitest-dev/vitest/issues/10292 '__filename', '__dirname', 'module', @@ -333,19 +340,26 @@ export class VitestModuleEvaluator implements ModuleEvaluator { ? vm.runInContext(wrappedCode, this.vm.context, options) : vm.runInThisContext(wrappedCode, options) + const __vite_ssr_exportName__ = context.__vite_ssr_exportName__ + || ((name: string, getter: () => unknown) => Object.defineProperty(context[ssrModuleExportsKey], name, { + enumerable: true, + configurable: true, + get: getter, + })) + + let __vite_track_exportName__: ((name: string, getter: () => unknown) => void) | undefined + const getterTracker = this.getterTracker + if (getterTracker) { + __vite_track_exportName__ = getterTracker.createTracker(module.id, __vite_ssr_exportName__) + } + await initModule( context[ssrModuleExportsKey], context[ssrImportMetaKey], context[ssrImportKey], context[ssrDynamicImportKey], context[ssrExportAllKey], - // vite 7 support, remove when vite 7+ is supported - context.__vite_ssr_exportName__ - || ((name: string, getter: () => unknown) => Object.defineProperty(context[ssrModuleExportsKey], name, { - enumerable: true, - configurable: true, - get: getter, - })), + __vite_track_exportName__ || __vite_ssr_exportName__, cjsGlobals.__filename, cjsGlobals.__dirname, diff --git a/packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts b/packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts index 22d7eb6c9988..ea777b21fed1 100644 --- a/packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts +++ b/packages/vitest/src/runtime/moduleRunner/startVitestModuleRunner.ts @@ -66,6 +66,7 @@ export function startVitestModuleRunner(options: ContextModuleRunnerOptions): Vi return state().config.deps.interopDefault }, getCurrentTestFilepath: () => state().filepath, + getterTracker: state().getterTracker, }, ) diff --git a/packages/vitest/src/runtime/runners/benchmark.ts b/packages/vitest/src/runtime/runners/benchmark.ts deleted file mode 100644 index 0d773910fdd9..000000000000 --- a/packages/vitest/src/runtime/runners/benchmark.ts +++ /dev/null @@ -1,180 +0,0 @@ -import type { - Suite, - Task, - TaskUpdateEvent, - VitestRunner, - VitestRunnerImportSource, -} from '@vitest/runner' -import type { ModuleRunner } from 'vite/module-runner' -import type { SerializedConfig } from '../config' -// import type { VitestExecutor } from '../execute' -import type { - Benchmark, - BenchmarkResult, - BenchTask, -} from '../types/benchmark' -import { updateTask as updateRunnerTask } from '@vitest/runner' -import { createDefer } from '@vitest/utils/helpers' -import { getSafeTimers } from '@vitest/utils/timers' -import { getBenchFn, getBenchOptions } from '../benchmark' -import { getWorkerState } from '../utils' - -function createBenchmarkResult(name: string): BenchmarkResult { - return { - name, - rank: 0, - rme: 0, - samples: [] as number[], - } as BenchmarkResult -} - -const benchmarkTasks = new WeakMap() - -async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) { - const { Task, Bench } = await runner.importTinybench() - - const start = performance.now() - - const benchmarkGroup: Benchmark[] = [] - const benchmarkSuiteGroup = [] - for (const task of suite.tasks) { - if (task.mode !== 'run' && task.mode !== 'queued') { - continue - } - - if (task.meta?.benchmark) { - benchmarkGroup.push(task as Benchmark) - } - else if (task.type === 'suite') { - benchmarkSuiteGroup.push(task) - } - } - - // run sub suites sequentially - for (const subSuite of benchmarkSuiteGroup) { - await runBenchmarkSuite(subSuite, runner) - } - - if (benchmarkGroup.length) { - const defer = createDefer() - suite.result = { - state: 'run', - startTime: start, - benchmark: createBenchmarkResult(suite.name), - } - updateTask('suite-prepare', suite) - - const addBenchTaskListener = ( - task: InstanceType, - benchmark: Benchmark, - ) => { - task.addEventListener( - 'complete', - (e) => { - const task = e.task - const taskRes = task.result! - const result = benchmark.result!.benchmark! - benchmark.result!.state = 'pass' - Object.assign(result, taskRes) - // compute extra stats and free raw samples as early as possible - const samples = result.samples - result.sampleCount = samples.length - result.median = samples.length % 2 - ? samples[Math.floor(samples.length / 2)] - : (samples[samples.length / 2] + samples[samples.length / 2 - 1]) / 2 - if (!runner.config.benchmark?.includeSamples) { - result.samples.length = 0 - } - updateTask('test-finished', benchmark) - }, - { - once: true, - }, - ) - task.addEventListener( - 'error', - (e) => { - const task = e.task - defer.reject(benchmark ? task.result!.error : e) - }, - { - once: true, - }, - ) - } - - benchmarkGroup.forEach((benchmark) => { - const options = getBenchOptions(benchmark) - const benchmarkInstance = new Bench(options) - - const benchmarkFn = getBenchFn(benchmark) - - benchmark.result = { - state: 'run', - startTime: start, - benchmark: createBenchmarkResult(benchmark.name), - } - - const task = new Task(benchmarkInstance, benchmark.name, benchmarkFn) - benchmarkTasks.set(benchmark, task) - addBenchTaskListener(task, benchmark) - }) - - const { setTimeout } = getSafeTimers() - const tasks: [BenchTask, Benchmark][] = [] - - for (const benchmark of benchmarkGroup) { - const task = benchmarkTasks.get(benchmark)! - updateTask('test-prepare', benchmark) - await task.warmup() - tasks.push([ - await new Promise(resolve => - setTimeout(async () => { - resolve(await task.run()) - }), - ), - benchmark, - ]) - } - - suite.result!.duration = performance.now() - start - suite.result!.state = 'pass' - - updateTask('suite-finished', suite) - defer.resolve(null) - - await defer - } - - function updateTask(event: TaskUpdateEvent, task: Task) { - updateRunnerTask(event, task, runner) - } -} - -export class NodeBenchmarkRunner implements VitestRunner { - private moduleRunner!: ModuleRunner - - constructor(public config: SerializedConfig) {} - - async importTinybench(): Promise { - return await import('tinybench') - } - - importFile(filepath: string, source: VitestRunnerImportSource): unknown { - if (source === 'setup') { - const moduleNode = getWorkerState().evaluatedModules.getModuleById(filepath) - if (moduleNode) { - getWorkerState().evaluatedModules.invalidateModule(moduleNode) - } - } - return this.moduleRunner.import(filepath) - } - - async runSuite(suite: Suite): Promise { - await runBenchmarkSuite(suite, this) - } - - async runTask(): Promise { - throw new Error('`test()` and `it()` is only available in test mode.') - } -} diff --git a/packages/vitest/src/runtime/runners/index.ts b/packages/vitest/src/runtime/runners/index.ts index fc232ea4447f..52a5b190fc09 100644 --- a/packages/vitest/src/runtime/runners/index.ts +++ b/packages/vitest/src/runtime/runners/index.ts @@ -6,7 +6,6 @@ import { takeCoverageInsideWorker } from '../../integrations/coverage' import { rpc } from '../rpc' import { loadDiffConfig, loadSnapshotSerializers } from '../setup-common' import { getWorkerState } from '../utils' -import { NodeBenchmarkRunner } from './benchmark' import { TestRunner } from './test' async function getTestRunnerConstructor( @@ -14,9 +13,7 @@ async function getTestRunnerConstructor( moduleRunner: TestModuleRunner, ): Promise { if (!config.runner) { - return ( - config.mode === 'test' ? TestRunner : NodeBenchmarkRunner - ) as any as VitestRunnerConstructor + return TestRunner as any as VitestRunnerConstructor } const mod = await moduleRunner.import(config.runner) if (!mod.default && typeof mod.default !== 'function') { @@ -60,7 +57,7 @@ export async function resolveTestRunner( loadDiffConfig(config, moduleRunner), loadSnapshotSerializers(config, moduleRunner), ]) - testRunner.config.diffOptions = diffOptions + testRunner.config._diffOptions = diffOptions // patch some methods, so custom runners don't need to call RPC const originalOnTaskUpdate = testRunner.onTaskUpdate diff --git a/packages/vitest/src/runtime/runners/test.ts b/packages/vitest/src/runtime/runners/test.ts index e0986b83ddf1..0c2d8348428d 100644 --- a/packages/vitest/src/runtime/runners/test.ts +++ b/packages/vitest/src/runtime/runners/test.ts @@ -8,11 +8,14 @@ import type { Task, Test, TestContext, + TestTryOptions, VitestRunnerImportSource, VitestRunner as VitestTestRunner, } from '@vitest/runner' +import type { Bench as Tinybench, Task as TinybenchTask } from 'tinybench' import type { ModuleRunner } from 'vite/module-runner' import type { Traces } from '../../utils/traces' +import type { Bench } from '../benchmark' import type { SerializedConfig } from '../config' import { getState, GLOBAL_EXPECT, setState } from '@vitest/expect' import { @@ -29,7 +32,7 @@ import { createExpect } from '../../integrations/chai/index' import { inject } from '../../integrations/inject' import { getSnapshotClient } from '../../integrations/snapshot/chai' import { vi } from '../../integrations/vi' -import { getBenchFn, getBenchOptions } from '../benchmark' +import { createBench, kFinalize } from '../benchmark' import { rpc } from '../rpc' import { getWorkerState } from '../utils' @@ -40,9 +43,13 @@ export class TestRunner implements VitestTestRunner { private cancelRun = false private assertionsErrors = new WeakMap, Error>() + private benchInstances = new WeakMap, Bench>() public pool: string = this.workerState.ctx.pool - private _otel!: Traces + /** + * @internal + */ + public _otel!: Traces public viteEnvironment: string private viteModuleRunner: boolean @@ -83,7 +90,7 @@ export class TestRunner implements VitestTestRunner { this.workerState.onCleanup(listener) } - onAfterRunFiles(): void { + onAfterRunFiles(_files: File[]): void { this.snapshotClient.clear() this.workerState.current = undefined } @@ -167,7 +174,7 @@ export class TestRunner implements VitestTestRunner { this.workerState.current = suite } - onBeforeTryTask(test: Task): void { + onBeforeTryTask(test: Task, _options: TestTryOptions): void { clearModuleMocks(this.config) this.snapshotClient.clearTest(test.file.filepath, test.id) setState( @@ -185,6 +192,7 @@ export class TestRunner implements VitestTestRunner { } onAfterTryTask(test: Test): void { + this.benchInstances.get(test)?.[kFinalize]() const { assertionCalls, expectedAssertionsNumber, @@ -231,6 +239,18 @@ export class TestRunner implements VitestTestRunner { return _expect != null }, }) + let _bench: Bench | undefined + const runnerConfig = this.config + const benchInstances = this.benchInstances + Object.defineProperty(context, 'bench', { + get() { + if (!_bench) { + _bench = createBench(context.task, runnerConfig) + benchInstances.set(context.task, _bench) + } + return _bench + }, + }) return context } @@ -281,13 +301,13 @@ export class TestRunner implements VitestTestRunner { static matchesTags: typeof matchesTags = matchesTags /** - * @deprecated - */ - static getBenchFn: typeof getBenchFn = getBenchFn - /** - * @deprecated + * @experimental + * A function that runs tinybench tasks. + * Can be overriden to run tasks in a special environment. */ - static getBenchOptions: typeof getBenchOptions = getBenchOptions + static async runBenchmarks(tinybench: Tinybench): Promise { + return await tinybench.run() + } } function clearModuleMocks(config: SerializedConfig) { diff --git a/packages/vitest/src/runtime/types/benchmark.ts b/packages/vitest/src/runtime/types/benchmark.ts deleted file mode 100644 index 9ffd62c4f83b..000000000000 --- a/packages/vitest/src/runtime/types/benchmark.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { Test } from '@vitest/runner' -import type { ChainableFunction } from '@vitest/runner/utils' -import type { - Bench as BenchFactory, - Options as BenchOptions, - Task as BenchTask, - TaskResult as BenchTaskResult, - TaskResult as TinybenchResult, -} from 'tinybench' - -export interface Benchmark extends Test { - meta: { - benchmark: true - result?: BenchTaskResult - } -} - -export interface BenchmarkResult extends TinybenchResult { - name: string - rank: number - sampleCount: number - median: number -} - -export type BenchFunction = (this: BenchFactory) => Promise | void -type ChainableBenchmarkAPI = ChainableFunction< - 'skip' | 'only' | 'todo', - (name: string | Function, fn?: BenchFunction, options?: BenchOptions) => void -> -export type BenchmarkAPI = ChainableBenchmarkAPI & { - skipIf: (condition: any) => ChainableBenchmarkAPI - runIf: (condition: any) => ChainableBenchmarkAPI -} - -export { BenchFactory, BenchOptions, BenchTask, BenchTaskResult } diff --git a/packages/vitest/src/runtime/worker.ts b/packages/vitest/src/runtime/worker.ts index b08b8e6dc20a..c6199f479d76 100644 --- a/packages/vitest/src/runtime/worker.ts +++ b/packages/vitest/src/runtime/worker.ts @@ -2,6 +2,7 @@ import type { ContextRPC, WorkerGlobalState } from '../types/worker' import type { Traces } from '../utils/traces' import type { VitestWorker } from './workers/types' import { createStackString, parseStacktrace } from '@vitest/utils/source-map' +import { GetterTracker } from './getter-tracker' import { setupInspect } from './inspector' import * as listeners from './listeners' import { VitestEvaluatedModules } from './moduleRunner/evaluatedModules' @@ -47,6 +48,9 @@ async function execute(method: 'run' | 'collect', ctx: ContextRPC, worker: Vites return createStackString(parseStacktrace(stack)) }, metaEnv: createImportMetaEnvProxy(), + getterTracker: ctx.config.benchmark.enabled && !ctx.config.benchmark.suppressExportGetterWarnings + ? new GetterTracker() + : undefined, } satisfies WorkerGlobalState const methodName = method === 'collect' ? 'collectTests' : 'runTests' diff --git a/packages/vitest/src/types/global.ts b/packages/vitest/src/types/global.ts index 9ac1fa9a7df1..f9dc6d170a13 100644 --- a/packages/vitest/src/types/global.ts +++ b/packages/vitest/src/types/global.ts @@ -2,7 +2,7 @@ import type { ExpectStatic, PromisifyAssertion, Tester } from '@vitest/expect' import type { Plugin as PrettyFormatPlugin } from '@vitest/pretty-format' import type { Test } from '@vitest/runner' import type { SnapshotState } from '@vitest/snapshot' -import type { BenchmarkResult } from '../runtime/types/benchmark' +import type { Bench, BenchResult } from '../runtime/benchmark' import type { UserConsoleLog } from './general' interface SnapshotMatcher { @@ -92,6 +92,40 @@ declare module 'vitest' { * await expect(largeData).toMatchFileSnapshot('path/to/snapshot.json'); */ toMatchFileSnapshot: (filepath: string, hint?: string) => Promise + + /** + * Asserts that a benchmark result is faster than another benchmark result. + * Compares mean latency — lower is faster. + * + * @example + * const result = await bench.compare( + * bench('lib1', () => { lib1() }), + * bench('lib2', () => { lib2() }), + * ) + * expect(result.get('lib1')).toBeFasterThan(result.get('lib2')) + * expect(result.get('lib1')).toBeFasterThan(result.get('lib2'), { delta: 0.1 }) + */ + toBeFasterThan: ( + expected: BenchResult, + options?: { delta?: number }, + ) => void + + /** + * Asserts that a benchmark result is slower than another benchmark result. + * Compares mean latency — higher is slower. + * + * @example + * const result = await bench.compare( + * bench('lib1', () => { lib1() }), + * bench('lib2', () => { lib2() }), + * ) + * expect(result.get('lib2')).toBeSlowerThan(result.get('lib1')) + * expect(result.get('lib2')).toBeSlowerThan(result.get('lib1'), { delta: 0.2 }) + */ + toBeSlowerThan: ( + expected: BenchResult, + options?: { delta?: number }, + ) => void } } @@ -103,6 +137,11 @@ declare module '@vitest/runner' { * This API is useful for running snapshot tests concurrently because global expect cannot track them. */ readonly expect: ExpectStatic + /** + * Create a benchmark to run. It will be reported after the test is finished. + * @see {@link https://vitest.dev/guide/benchmarking} + */ + readonly bench: Bench /** @internal */ _local: boolean } @@ -121,8 +160,4 @@ declare module '@vitest/runner' { interface TaskBase { logs?: UserConsoleLog[] } - - interface TaskResult { - benchmark?: BenchmarkResult - } } diff --git a/packages/vitest/src/types/rpc.ts b/packages/vitest/src/types/rpc.ts index e0d0ae9f3410..fa6733f34a78 100644 --- a/packages/vitest/src/types/rpc.ts +++ b/packages/vitest/src/types/rpc.ts @@ -1,4 +1,4 @@ -import type { CancelReason, File, TaskEventPack, TaskResultPack, TestArtifact } from '@vitest/runner' +import type { BaselineData, CancelReason, File, TaskEventPack, TaskResultPack, TestArtifact, TestBenchmark } from '@vitest/runner' import type { SnapshotResult } from '@vitest/snapshot' import type { FetchFunctionOptions, FetchResult } from 'vite/module-runner' import type { OTELCarrier } from '../utils/traces' @@ -21,6 +21,7 @@ export interface RuntimeRPC { onQueued: (file: File) => void onCollected: (files: File[]) => Promise onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void + onTestBenchmark: (testId: string, bench: TestBenchmark) => void onTaskArtifactRecord: (testId: string, artifact: Artifact) => Promise onTaskUpdate: (pack: TaskResultPack[], events: TaskEventPack[]) => Promise onCancel: (reason: CancelReason) => void @@ -29,6 +30,9 @@ export interface RuntimeRPC { snapshotSaved: (snapshot: SnapshotResult) => void resolveSnapshotPath: (testPath: string) => string + readBenchmarkResult: (relativePath: string) => Promise + writeBenchmarkResult: (relativePath: string, data: BaselineData) => Promise + ensureModuleGraphEntry: (id: string, importer: string) => void } diff --git a/packages/vitest/src/types/worker.ts b/packages/vitest/src/types/worker.ts index d96da85a49f4..2824d6fc441e 100644 --- a/packages/vitest/src/types/worker.ts +++ b/packages/vitest/src/types/worker.ts @@ -2,6 +2,7 @@ import type { CancelReason, FileSpecification, Task } from '@vitest/runner' import type { BirpcReturn } from 'birpc' import type { EvaluatedModules } from 'vite/module-runner' import type { SerializedConfig } from '../runtime/config' +import type { GetterTracker } from '../runtime/getter-tracker' import type { Traces } from '../utils/traces' import type { Environment } from './environment' import type { RunnerRPC, RuntimeRPC } from './rpc' @@ -74,6 +75,7 @@ export interface WorkerGlobalState { evaluatedModules: EvaluatedModules resolvingModules: Set moduleExecutionInfo: Map + getterTracker?: GetterTracker onCancel: (listener: (reason: CancelReason) => unknown) => void onCleanup: (listener: () => unknown) => void providedContext: Record diff --git a/packages/vitest/src/utils/config-helpers.ts b/packages/vitest/src/utils/config-helpers.ts index a04f02ee16d3..cf96cdc28e2b 100644 --- a/packages/vitest/src/utils/config-helpers.ts +++ b/packages/vitest/src/utils/config-helpers.ts @@ -1,5 +1,4 @@ import type { - BenchmarkBuiltinReporters, BuiltinReporters, } from '../node/reporters' @@ -9,7 +8,7 @@ interface PotentialConfig { export function getOutputFile( config: PotentialConfig | undefined, - reporter: BuiltinReporters | BenchmarkBuiltinReporters | 'html', + reporter: BuiltinReporters | 'html', ): string | undefined { if (!config?.outputFile) { return diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 27f216d7fedb..387f9bb69aaa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -132,6 +132,9 @@ catalogs: strip-literal: specifier: ^3.1.0 version: 3.1.0 + tinybench: + specifier: ^6.0.1 + version: 6.0.1 tinyexec: specifier: ^1.0.2 version: 1.0.2 @@ -835,6 +838,9 @@ importers: pathe: specifier: 'catalog:' version: 2.0.3 + tinybench: + specifier: 'catalog:' + version: 6.0.1 packages/snapshot: dependencies: @@ -1072,8 +1078,8 @@ importers: specifier: 'catalog:' version: 4.0.0-rc.1 tinybench: - specifier: ^2.9.0 - version: 2.9.0 + specifier: 'catalog:' + version: 6.0.1 tinyexec: specifier: ^1.0.2 version: 1.0.2 @@ -9709,8 +9715,9 @@ packages: thread-stream@3.1.0: resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} - tinybench@2.9.0: - resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + tinybench@6.0.1: + resolution: {integrity: sha512-cMdWsxmysdg8mNWf1pujiWl3TW0cU6m8QuNw55QlnP3I6N96Grb0wnu5N0syHIu3LbiVZCNqlfWzWDq84HZphA==} + engines: {node: '>=20.0.0'} tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -19042,7 +19049,7 @@ snapshots: dependencies: real-require: 0.2.0 - tinybench@2.9.0: {} + tinybench@6.0.1: {} tinyexec@0.3.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e669a1ad555c..d99024ec33af 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -89,6 +89,7 @@ catalog: sirv: ^3.0.2 std-env: ^4.0.0-rc.1 strip-literal: ^3.1.0 + tinybench: ^6.0.1 tinyexec: ^1.0.2 tinyglobby: ^0.2.15 tinyhighlight: ^0.3.2 diff --git a/test/browser/fixtures/benchmark/basic.bench.ts b/test/browser/fixtures/benchmark/basic.bench.ts deleted file mode 100644 index d61e094276bd..000000000000 --- a/test/browser/fixtures/benchmark/basic.bench.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { bench, describe } from 'vitest' - -describe('suite-a', () => { - bench('good', async () => { - await sleep(10) - }, options) - - bench('bad', async () => { - await sleep(300) - }, options) -}) - -describe('suite-b', () => { - bench('good', async () => { - await sleep(25) - }, options) - - describe('suite-b-nested', () => { - bench('good', async () => { - await sleep(50) - }, options) - }) -}) - -const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); - -const options = { - time: 0, - iterations: 2, - warmupIterations: 0, - warmupTime: 0, -} diff --git a/test/browser/fixtures/benchmark/vitest.config.ts b/test/browser/fixtures/benchmark/vitest.config.ts deleted file mode 100644 index c57a43ee6168..000000000000 --- a/test/browser/fixtures/benchmark/vitest.config.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { fileURLToPath } from 'node:url' -import { defineConfig } from 'vitest/config' -import { instances, provider } from '../../settings' - -export default defineConfig({ - test: { - browser: { - enabled: true, - headless: true, - provider, - instances, - }, - }, - cacheDir: fileURLToPath(new URL("./node_modules/.vite", import.meta.url)), -}) diff --git a/test/browser/package.json b/test/browser/package.json index 11b67b81ca3e..fedee2be6988 100644 --- a/test/browser/package.json +++ b/test/browser/package.json @@ -9,6 +9,7 @@ "test:playwright": "PROVIDER=playwright pnpm run test:unit", "test:safaridriver": "PROVIDER=webdriverio BROWSER=safari pnpm run test:unit", "test-fixtures": "vitest", + "bench-fixtures": "vitest bench", "test-expect-dom": "vitest --root ./fixtures/expect-dom", "test-mocking": "vitest --root ./fixtures/mocking", "test-logs": "vitest --root ./fixtures/print-logs", diff --git a/test/browser/specs/benchmark.test.ts b/test/browser/specs/benchmark.test.ts deleted file mode 100644 index e7e5eb608124..000000000000 --- a/test/browser/specs/benchmark.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { expect, test } from 'vitest' -import { runVitest } from '../../test-utils' - -const IS_PLAYWRIGHT = process.env.PROVIDER === 'playwright' - -test('benchmark', async () => { - const result = await runVitest({ root: 'fixtures/benchmark' }, [], { mode: 'benchmark' }) - expect(result.stderr).toReportNoErrors() - - if (IS_PLAYWRIGHT) { - expect(result.stdout).toContain('✓ |chromium| basic.bench.ts > suite-a') - expect(result.stdout).toContain('✓ |firefox| basic.bench.ts > suite-a') - expect(result.stdout).toContain('✓ |webkit| basic.bench.ts > suite-a') - } - else { - expect(result.stdout).toContain('✓ |chrome| basic.bench.ts > suite-a') - expect(result.stdout).toContain('✓ |firefox| basic.bench.ts > suite-a') - } - - expect(result.exitCode).toBe(0) -}) diff --git a/test/browser/specs/runner.test.ts b/test/browser/specs/runner.test.ts index b17d96366bf4..aa1e989cd766 100644 --- a/test/browser/specs/runner.test.ts +++ b/test/browser/specs/runner.test.ts @@ -1,4 +1,5 @@ -import type { JsonTestResults, Vitest } from 'vitest/node' +import type { TestBenchmark } from 'vitest' +import type { JsonTestResult, JsonTestResults, Vitest } from 'vitest/node' import { readdirSync } from 'node:fs' import { readFile } from 'node:fs/promises' import { beforeAll, describe, expect, onTestFailed, test } from 'vitest' @@ -12,10 +13,11 @@ describe('running browser tests', async () => { let stderr: string let stdout: string let browserResultJson: JsonTestResults - let passedTests: any[] - let failedTests: any[] + let passedTests: JsonTestResult[] + let failedTests: JsonTestResult[] let vitest: Vitest const events: string[] = [] + const emittedBenchmarks: Array<{ projectName: string; testName: string; benchmark: TestBenchmark }> = [] beforeAll(async () => { ({ @@ -29,6 +31,13 @@ describe('running browser tests', async () => { onBrowserInit(project) { events.push(`onBrowserInit ${project.name}`) }, + onTestCaseBenchmark(testCase, benchmark) { + emittedBenchmarks.push({ + projectName: testCase.project.name || '', + testName: testCase.fullName, + benchmark, + }) + }, }, 'json', { @@ -47,9 +56,9 @@ describe('running browser tests', async () => { })) const browserResult = await readFile('./browser.json', 'utf-8') - browserResultJson = JSON.parse(browserResult) - const getPassed = results => results.filter(result => result.status === 'passed' && !result.message) - const getFailed = results => results.filter(result => result.status === 'failed') + browserResultJson = JSON.parse(browserResult) as JsonTestResults + const getPassed = (results: JsonTestResult[]) => results.filter(result => result.status === 'passed' && !result.message) + const getFailed = (results: JsonTestResult[]) => results.filter(result => result.status === 'failed') passedTests = getPassed(browserResultJson.testResults) failedTests = getFailed(browserResultJson.testResults) }) @@ -70,7 +79,7 @@ describe('running browser tests', async () => { .toEqual(vitest.projects.map(() => expect.arrayContaining(runtimeTestFiles))) const testFilesCount = readdirSync('./test') - .filter(n => n.includes('.test.') || n.includes('.test-d.')) + .filter(n => n.includes('.test.') || n.includes('.test-d.') || n.includes('.bench.')) .length + 1 // 1 is in-source-test expect(browserResultJson.testResults).toHaveLength(testFilesCount * instances.length) @@ -78,6 +87,50 @@ describe('running browser tests', async () => { expect(failedTests).toHaveLength(0) }) + test('benchmarks run in a dedicated `(bench)` project per browser instance', () => { + const benchProjects = vitest.projects.filter(p => p.name.endsWith('(bench)')) + expect(benchProjects.map(p => p.name).sort()).toEqual( + instances.map(({ browser }) => `${browser} (bench)`).sort(), + ) + }) + + test('perProject benchmarks emit tasks with the perProject flag in every browser', () => { + const records = emittedBenchmarks.filter(e => + e.testName === 'perProject registrations flow through the browser RPC (onTestBenchmark)', + ) + // the test calls `.run()` twice, so each browser produces 2 benchmark records + expect(records.length, `perProject emitted: ${records.length}`).toBe(2 * instances.length) + for (const record of records) { + expect(record.benchmark.tasks, `empty tasks for ${record.projectName}`).toHaveLength(1) + const [task] = record.benchmark.tasks + expect(task.perProject, `missing perProject flag on ${record.projectName}/${task.name}`).toBe(true) + expect(task.fromStore).toBeUndefined() + } + }) + + test('bench.compare emits one benchmark with both registrations ranked', () => { + const records = emittedBenchmarks.filter(e => + e.testName === 'bench.compare resolves a BenchStorage in the browser', + ) + expect(records.length).toBe(instances.length) + for (const record of records) { + expect(record.benchmark.tasks.map(t => t.name).sort(), `unexpected tasks for ${record.projectName}`).toEqual(['a', 'b']) + expect(record.benchmark.tasks.map(t => t.rank).sort()).toEqual([1, 2]) + } + }) + + test('writeResult flows through the write-artifact RPC in every browser', () => { + const records = emittedBenchmarks.filter(e => + e.testName === 'writeResult exercises the writeBenchmarkResult RPC round-trip', + ) + expect(records.length).toBe(instances.length) + for (const record of records) { + expect(record.benchmark.tasks, `empty tasks for ${record.projectName}`).toHaveLength(1) + const [task] = record.benchmark.tasks + expect(task.name).toBe('with-write') + } + }) + test('tags are collected', () => { expect(vitest.config.tags).toEqual([ { name: 'e2e', priority: 10 }, @@ -110,7 +163,7 @@ describe('running browser tests', async () => { test('runs in-source tests', () => { expect(stdout).toContain('src/actions.ts') const actionsTest = passedTests.find(t => t.name.includes('/actions.ts')) - expect(actionsTest).toBeDefined() + expect.assert(actionsTest) expect(actionsTest.assertionResults).toHaveLength(1) }) @@ -231,7 +284,7 @@ test(`stack trace points to correct file in every browser when failed`, async () return } if (testCase.project.name === 'chromium' || testCase.project.name === 'chrome') { - expect(testCase.result().errors[0].stacks).toEqual([ + expect(testCase.result().errors?.[0].stacks).toEqual([ { line: 11, column: 12, @@ -380,7 +433,7 @@ test.runIf(provider.name === 'playwright')('timeout hooks', async ({ onTestFaile }) const lines = stderr.split('\n') - const timeoutErrorsIndexes = [] + const timeoutErrorsIndexes: number[] = [] lines.forEach((line, index) => { if (line.includes('TimeoutError:')) { timeoutErrorsIndexes.push(index) diff --git a/test/browser/specs/utils.ts b/test/browser/specs/utils.ts index c243a33af557..ae49e2becb9e 100644 --- a/test/browser/specs/utils.ts +++ b/test/browser/specs/utils.ts @@ -43,5 +43,9 @@ export async function runBrowserTests( $viteConfig: viteOverrides, }, include, runnerOptions) - return { ...result, stderr: result.stderr.replace('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow SemVer, please pin Vitest\'s version when using it.\n', '') } + return { + ...result, + ctx: result.ctx!, + stderr: result.stderr.replace('Testing types with tsc and vue-tsc is an experimental feature.\nBreaking changes might not follow SemVer, please pin Vitest\'s version when using it.\n', ''), + } } diff --git a/test/browser/test/browser.bench.ts b/test/browser/test/browser.bench.ts new file mode 100644 index 000000000000..325c6ee86350 --- /dev/null +++ b/test/browser/test/browser.bench.ts @@ -0,0 +1,43 @@ +import { expect, test } from 'vitest' + +// Keep runs tiny — this file smoke-tests the browser RPC path +// (onTestBenchmark, readBenchmarkResult / writeBenchmarkResult). +// It is not a measurement harness. +const fastBenchOptions = { + time: 0, + iterations: 2, + warmupTime: 0, + warmupIterations: 0, +} + +test('perProject registrations flow through the browser RPC (onTestBenchmark)', async ({ bench }) => { + await bench('1 + 1', { perProject: true, ...fastBenchOptions }, () => { + const result = 1 + 1 + expect.assert(result === 2) + }).run() + await bench('1 + 2', { perProject: true, ...fastBenchOptions }, () => { + const result = 1 + 2 + expect.assert(result === 3) + }).run() +}) + +test('bench.compare resolves a BenchStorage in the browser', async ({ bench }) => { + const storage = await bench.compare( + bench('a', () => { const _ = 1 + 1 }), + bench('b', () => { const _ = 1 + 2 }), + fastBenchOptions, + ) + // runtime smoke — every registration is accessible with a valid BenchResult + expect.assert(typeof storage.get('a').latency.mean === 'number') + expect.assert(typeof storage.get('b').latency.mean === 'number') +}) + +test('writeResult exercises the writeBenchmarkResult RPC round-trip', async ({ bench }) => { + // The browser worker forwards writeResult through the WebSocket RPC to the + // node side. We don't assert on the file contents here (the spec layer can + // do that), just that the round-trip completes without throwing. + const result = await bench('with-write', { writeResult: './out/with-write.json', ...fastBenchOptions }, () => { + const _ = 1 + 1 + }).run() + expect.assert(typeof result.latency.mean === 'number') +}) diff --git a/test/browser/vitest.config.mts b/test/browser/vitest.config.mts index 161a6143281e..461606e6a275 100644 --- a/test/browser/vitest.config.mts +++ b/test/browser/vitest.config.mts @@ -72,6 +72,9 @@ export default defineConfig({ { name: 'test', priority: 5 }, { name: 'browser', priority: 1 }, ], + benchmark: { + enabled: true, + }, alias: { '#src': resolve(dir, './src'), }, diff --git a/test/e2e/fixtures/benchmarking/basic/base.bench.ts b/test/e2e/fixtures/benchmarking/basic/base.bench.ts deleted file mode 100644 index 872f214c7f79..000000000000 --- a/test/e2e/fixtures/benchmarking/basic/base.bench.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { bench, describe } from 'vitest' - -describe('sort', () => { - bench('normal', () => { - const x = [1, 5, 4, 2, 3] - x.sort((a, b) => { - return a - b - }) - }, { iterations: 5, time: 0 }) - - bench('reverse', () => { - const x = [1, 5, 4, 2, 3] - x.reverse().sort((a, b) => { - return a - b - }) - }, { iterations: 5, time: 0 }) - - // TODO: move to failed tests - // should not be collected - // it('test', () => { - // expect(1 + 1).toBe(3) - // }) -}) - -function timeout(time: number) { - return new Promise((resolve) => { - setTimeout(resolve, time) - }) -} - -describe('timeout', () => { - bench('timeout100', async () => { - await timeout(100) - }, { - setup() { - - }, - teardown() { - - }, - ...benchOptions - }) - - bench('timeout75', async () => { - await timeout(75) - }, benchOptions) - - bench('timeout50', async () => { - await timeout(50) - }, benchOptions) - - bench('timeout25', async () => { - await timeout(25) - }, benchOptions) - - // TODO: move to failed tests - // test('reduce', () => { - // expect(1 - 1).toBe(2) - // }) -}) - -const benchOptions = { - time: 0, - iterations: 3, - warmupIterations: 0, - warmupTime: 0, -} diff --git a/test/e2e/fixtures/benchmarking/basic/mode.bench.ts b/test/e2e/fixtures/benchmarking/basic/mode.bench.ts deleted file mode 100644 index 0967332de750..000000000000 --- a/test/e2e/fixtures/benchmarking/basic/mode.bench.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { bench, describe } from 'vitest' - -describe.todo('unimplemented suite') - -describe.skip('skipped', () => { - bench('skipped', () => { - throw new Error('should be skipped') - }) - - bench.todo('unimplemented test') -}) - -bench.skip('skipped', () => { - throw new Error('should be skipped') -}) diff --git a/test/e2e/fixtures/benchmarking/basic/only.bench.ts b/test/e2e/fixtures/benchmarking/basic/only.bench.ts deleted file mode 100644 index b0647b319292..000000000000 --- a/test/e2e/fixtures/benchmarking/basic/only.bench.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { bench, describe, expect, assert } from 'vitest' - -const run = [false, false, false, false, false] - -describe('a0', () => { - bench.only('0', () => { - run[0] = true - }, { iterations: 1, time: 0 }) - bench('s0', () => { - expect(true).toBe(false) - }) -}) - -describe('a1', () => { - describe('b1', () => { - describe('c1', () => { - bench.only('1', () => { - run[1] = true - }, { iterations: 1, time: 0 }) - }) - bench('s1', () => { - expect(true).toBe(false) - }) - }) -}) - -describe.only('a2', () => { - bench('2', () => { - run[2] = true - }, { iterations: 1, time: 0 }) -}) - -bench('s2', () => { - expect(true).toBe(false) -}) - -describe.only('a3', () => { - describe('b3', () => { - bench('3', () => { - run[3] = true - }, { iterations: 1, time: 0 }) - }) - bench.skip('s3', () => { - expect(true).toBe(false) - }) -}) - -describe('a4', () => { - describe.only('b4', () => { - bench('4', () => { - run[4] = true - }, { iterations: 1, time: 0 }) - }) - describe('sb4', () => { - bench('s4', () => { - expect(true).toBe(false) - }) - }) -}) - -bench.only( - 'visited', - () => { - assert.deepEqual(run, [true, true, true, true, true]) - }, - { iterations: 1, time: 0 }, -) - -bench.only( - 'visited2', - () => { - assert.deepEqual(run, [true, true, true, true, true]) - }, - { iterations: 1, time: 0 }, -) diff --git a/test/e2e/fixtures/benchmarking/basic/should-not-run.test-d.ts b/test/e2e/fixtures/benchmarking/basic/should-not-run.test-d.ts deleted file mode 100644 index eb35b7540de5..000000000000 --- a/test/e2e/fixtures/benchmarking/basic/should-not-run.test-d.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { describe, expectTypeOf, test } from 'vitest' - -describe('test', () => { - test('some-test', () => { - expectTypeOf({ a: 1 }).toEqualTypeOf({ a: "should not match" }) - }) -}) diff --git a/test/e2e/fixtures/benchmarking/basic/vitest.config.ts b/test/e2e/fixtures/benchmarking/basic/vitest.config.ts deleted file mode 100644 index abed6b2116e1..000000000000 --- a/test/e2e/fixtures/benchmarking/basic/vitest.config.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({}) diff --git a/test/e2e/fixtures/benchmarking/compare/basic.bench.ts b/test/e2e/fixtures/benchmarking/compare/basic.bench.ts deleted file mode 100644 index 0fa89fef4ce1..000000000000 --- a/test/e2e/fixtures/benchmarking/compare/basic.bench.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { bench, describe } from 'vitest' - -const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); - -describe('suite', () => { - bench('sleep10', async () => { - await sleep(10) - }, { time: 20, iterations: 0 }) - - bench('sleep100', async () => { - await sleep(100); - }, { time: 200, iterations: 0 }) -}) diff --git a/test/e2e/fixtures/benchmarking/compare/vitest.config.ts b/test/e2e/fixtures/benchmarking/compare/vitest.config.ts deleted file mode 100644 index abed6b2116e1..000000000000 --- a/test/e2e/fixtures/benchmarking/compare/vitest.config.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({}) diff --git a/test/e2e/fixtures/benchmarking/reporter/multiple.bench.ts b/test/e2e/fixtures/benchmarking/reporter/multiple.bench.ts deleted file mode 100644 index 1b204ffa7a96..000000000000 --- a/test/e2e/fixtures/benchmarking/reporter/multiple.bench.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { bench, describe } from 'vitest' -import { setTimeout } from 'node:timers/promises' - -const options = { iterations: 1, warmupIterations: 1 } - -bench('first', async () => { - await setTimeout(500) -}, options) - -bench('second', async () => { - await setTimeout(500) -}, options) diff --git a/test/e2e/fixtures/benchmarking/reporter/summary.bench.ts b/test/e2e/fixtures/benchmarking/reporter/summary.bench.ts deleted file mode 100644 index d61e094276bd..000000000000 --- a/test/e2e/fixtures/benchmarking/reporter/summary.bench.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { bench, describe } from 'vitest' - -describe('suite-a', () => { - bench('good', async () => { - await sleep(10) - }, options) - - bench('bad', async () => { - await sleep(300) - }, options) -}) - -describe('suite-b', () => { - bench('good', async () => { - await sleep(25) - }, options) - - describe('suite-b-nested', () => { - bench('good', async () => { - await sleep(50) - }, options) - }) -}) - -const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); - -const options = { - time: 0, - iterations: 2, - warmupIterations: 0, - warmupTime: 0, -} diff --git a/test/e2e/fixtures/benchmarking/reporter/vitest.config.ts b/test/e2e/fixtures/benchmarking/reporter/vitest.config.ts deleted file mode 100644 index abed6b2116e1..000000000000 --- a/test/e2e/fixtures/benchmarking/reporter/vitest.config.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({}) diff --git a/test/e2e/fixtures/benchmarking/sequential/f1.bench.ts b/test/e2e/fixtures/benchmarking/sequential/f1.bench.ts deleted file mode 100644 index acf940fae47b..000000000000 --- a/test/e2e/fixtures/benchmarking/sequential/f1.bench.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { bench, describe } from "vitest" -import { appendLog, benchOptions, sleepBench } from "./helper"; - -bench("B1", async () => { - await appendLog("F1 / B1") - await sleepBench(); -}, benchOptions) - -describe("S1", () => { - bench("B1", async () => { - await appendLog("F1 / S1 / B1") - await sleepBench(); - }, benchOptions) - - bench("B2", async () => { - await appendLog("F1 / S1 / B2") - await sleepBench(); - }, benchOptions) -}) - -describe("S2", () => { - bench("B1", async () => { - await appendLog("F1 / S2 / B1") - await sleepBench(); - }, benchOptions) -}) diff --git a/test/e2e/fixtures/benchmarking/sequential/f2.bench.ts b/test/e2e/fixtures/benchmarking/sequential/f2.bench.ts deleted file mode 100644 index 3968708f7a8b..000000000000 --- a/test/e2e/fixtures/benchmarking/sequential/f2.bench.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { bench, describe } from "vitest" -import { appendLog, benchOptions, sleepBench } from "./helper"; - -describe("S1", () => { - bench("B1", async () => { - await appendLog("F2 / S1 / B1") - await sleepBench(); - }, benchOptions) -}) diff --git a/test/e2e/fixtures/benchmarking/sequential/helper.ts b/test/e2e/fixtures/benchmarking/sequential/helper.ts deleted file mode 100644 index b609d7c9375c..000000000000 --- a/test/e2e/fixtures/benchmarking/sequential/helper.ts +++ /dev/null @@ -1,19 +0,0 @@ -import fs from "node:fs"; - -const SLEEP_BENCH_MS = Number(process.env["SLEEP_BENCH_MS"] || 10); -const BENCH_ITERATIONS = Number(process.env["BENCH_ITERATIONS"] || 3); - -export const sleepBench = () => new Promise(resolve => setTimeout(resolve, SLEEP_BENCH_MS)) - -export const testLogFile = new URL("./test.log", import.meta.url); - -export async function appendLog(data: string) { - await fs.promises.appendFile(testLogFile, data + "\n"); -} - -export const benchOptions = { - time: 0, - iterations: BENCH_ITERATIONS, - warmupIterations: 0, - warmupTime: 0, -} diff --git a/test/e2e/fixtures/benchmarking/sequential/setup.ts b/test/e2e/fixtures/benchmarking/sequential/setup.ts deleted file mode 100644 index 9a929fa54cad..000000000000 --- a/test/e2e/fixtures/benchmarking/sequential/setup.ts +++ /dev/null @@ -1,6 +0,0 @@ -import fs from "node:fs"; -import { testLogFile } from "./helper"; - -export default async function setup() { - await fs.promises.rm(testLogFile, { force: true }); -} diff --git a/test/e2e/fixtures/benchmarking/sequential/vitest.config.ts b/test/e2e/fixtures/benchmarking/sequential/vitest.config.ts deleted file mode 100644 index fe63b73cf264..000000000000 --- a/test/e2e/fixtures/benchmarking/sequential/vitest.config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { defineConfig } from "vitest/config" - -// to see the difference better, increase sleep time and iterations e.g. by -// SLEEP_BENCH_MS=100 pnpm -C test/benchmark test bench -- --root fixtures/sequential --fileParallelism - -export default defineConfig({ - test: { - globalSetup: ["./setup.ts"] - } -}); diff --git a/test/e2e/fixtures/custom-pool/pool/custom-pool.ts b/test/e2e/fixtures/custom-pool/pool/custom-pool.ts index 7ab07a26e8e4..283bbec5921c 100644 --- a/test/e2e/fixtures/custom-pool/pool/custom-pool.ts +++ b/test/e2e/fixtures/custom-pool/pool/custom-pool.ts @@ -98,6 +98,7 @@ async function onMessage(message: WorkerRequest, project: TestProject, options: artifacts: [], timeout: 0, file: taskFile, + benchmarks: [], result: { state: 'pass', }, diff --git a/test/e2e/fixtures/mode/example.benchmark.ts b/test/e2e/fixtures/mode/example.benchmark.ts deleted file mode 100644 index a216fdfc4165..000000000000 --- a/test/e2e/fixtures/mode/example.benchmark.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { bench, describe } from 'vitest' - -describe('example', () => { - bench('simple', () => { - let _ = 0 - _ += 1 - }, { iterations: 1, time: 1, warmupIterations: 0, warmupTime: 0 }) -}) diff --git a/test/e2e/fixtures/mode/example.test.ts b/test/e2e/fixtures/mode/example.test.ts deleted file mode 100644 index 5528955569f1..000000000000 --- a/test/e2e/fixtures/mode/example.test.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { expect, test } from 'vitest' - -test('should pass', () => { - expect(1).toBe(1) -}) diff --git a/test/e2e/fixtures/mode/vitest.benchmark.config.ts b/test/e2e/fixtures/mode/vitest.benchmark.config.ts deleted file mode 100644 index 61158da65a58..000000000000 --- a/test/e2e/fixtures/mode/vitest.benchmark.config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig((env) => { - if (env.mode !== 'benchmark') { - console.error('env.mode: ', env.mode) - throw new Error('env.mode should be equal to "benchmark"') - } - - return ({}) -}) diff --git a/test/e2e/fixtures/mode/vitest.test.config.ts b/test/e2e/fixtures/mode/vitest.test.config.ts deleted file mode 100644 index 556dcbc890e1..000000000000 --- a/test/e2e/fixtures/mode/vitest.test.config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig((env) => { - if (env.mode !== 'test') { - console.error('env.mode: ', env.mode) - throw new Error('env.mode should be equal to "test"') - } - - return ({}) -}) diff --git a/test/e2e/fixtures/reporters/function-as-name.bench.ts b/test/e2e/fixtures/reporters/function-as-name.bench.ts index cf93b565c472..3ac472d9576e 100644 --- a/test/e2e/fixtures/reporters/function-as-name.bench.ts +++ b/test/e2e/fixtures/reporters/function-as-name.bench.ts @@ -1,4 +1,4 @@ -import { bench } from 'vitest' +import { test } from 'vitest' const options = { time: 0, @@ -10,6 +10,12 @@ const options = { function foo() {} class Bar {} -bench(foo, () => {}, options) -bench(Bar, () => {}, options) -bench(() => {}, () => {}, options) +test('benches', async ({ bench }) => { + await bench.compare( + bench(foo, () => {}), + bench(Bar, () => {}), + bench(() => {}, () => {}), + options, + ) +}) + diff --git a/test/e2e/test/__snapshots__/benchmarking.test.ts.snap b/test/e2e/test/__snapshots__/benchmarking.test.ts.snap deleted file mode 100644 index 3ebbc8a64d42..000000000000 --- a/test/e2e/test/__snapshots__/benchmarking.test.ts.snap +++ /dev/null @@ -1,33 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`sequential 1`] = ` -"F1 / S1 / B1 -F1 / S1 / B1 -F1 / S1 / B1 -F1 / S1 / B2 -F1 / S1 / B2 -F1 / S1 / B2 -F1 / S2 / B1 -F1 / S2 / B1 -F1 / S2 / B1 -F1 / B1 -F1 / B1 -F1 / B1 -F2 / S1 / B1 -F2 / S1 / B1 -F2 / S1 / B1 -" -`; - -exports[`summary 1`] = ` -" - - good - summary.bench.ts > suite-a - (?) faster than bad - - good - summary.bench.ts > suite-b - - good - summary.bench.ts > suite-b > suite-b-nested - -" -`; diff --git a/test/e2e/test/benchmarking.test-d.ts b/test/e2e/test/benchmarking.test-d.ts new file mode 100644 index 000000000000..f7ee8d897ca7 --- /dev/null +++ b/test/e2e/test/benchmarking.test-d.ts @@ -0,0 +1,103 @@ +import type { + BaselineData, + Bench, + BenchFnOptions, + BenchFromSource, + BenchRegistration, + BenchResult, + BenchStorage, + TestContext, +} from 'vitest' +import { assertType, expect, expectTypeOf, test } from 'vitest' + +test('plain `bench()` returns BenchRegistration with the name literal narrowed', ({ bench }) => { + const reg = bench('plain', () => {}) + expectTypeOf(reg).toEqualTypeOf>() + expectTypeOf(reg.name).toEqualTypeOf<'plain'>() +}) + +test('`bench(name, options, fn)` is the preferred options-second overload', ({ bench }) => { + const reg = bench('x', { beforeEach: () => {} }, () => {}) + expectTypeOf(reg).toEqualTypeOf>() +}) + +test('`bench(name, fn, options)` is a type error — options as third arg is not accepted', ({ bench }) => { + // @ts-expect-error legacy (name, fn, options) form is no longer accepted + bench('x', () => {}, { beforeEach: () => {} } satisfies BenchFnOptions) +}) + +test('`bench.compare(...regs)` returns a BenchStorage keyed by the UNION of registration names', async ({ bench }) => { + const storage = await bench.compare(bench('a', () => {}), bench('b', () => {})) + expectTypeOf(storage).toEqualTypeOf>() + expectTypeOf(storage.get('a')).toEqualTypeOf() + expectTypeOf(storage.get('b')).toEqualTypeOf() +}) + +test('`bench.compare(...).get(unknownName)` is a type error', async ({ bench }) => { + const storage = await bench.compare(bench('a', () => {}), bench('b', () => {})) + // @ts-expect-error 'missing' is not one of 'a' | 'b' + storage.get('missing') +}) + +test('`bench.compare` accepts BenchCompareOptions as the trailing argument', async ({ bench }) => { + const storage = await bench.compare( + bench('a', () => {}), + bench('b', () => {}), + { time: 10, iterations: 5 }, + ) + expectTypeOf(storage).toEqualTypeOf>() +}) + +test('`expect(result).toBeFasterThan` and `.toBeSlowerThan` are callable on BenchResult', () => { + const a = {} as BenchResult + const b = {} as BenchResult + // calling both matchers must type-check; runtime behaviour is in Bucket F + assertType(expect(a).toBeFasterThan(b)) + assertType(expect(a).toBeFasterThan(b, { delta: 0.1 })) + assertType(expect(a).toBeSlowerThan(b)) + assertType(expect(a).toBeSlowerThan(b, { delta: 0.1 })) +}) + +test('`TestContext.bench` is typed as the `Bench` factory', () => { + type CtxBench = TestContext['bench'] + expectTypeOf().toEqualTypeOf() +}) + +test('`bench.from(name, path)` returns BenchRegistration with the name literal narrowed', ({ bench }) => { + const reg = bench.from('baseline', 'results/baseline.json') + expectTypeOf(reg).toEqualTypeOf>() + expectTypeOf(reg.name).toEqualTypeOf<'baseline'>() +}) + +test('`bench.from(name, source)` accepts a function returning BaselineData', ({ bench }) => { + const sync: BenchFromSource = () => ({} as BaselineData) + const async: BenchFromSource = async () => ({} as BaselineData) + expectTypeOf(bench.from('s', sync)).toEqualTypeOf>() + expectTypeOf(bench.from('a', async)).toEqualTypeOf>() +}) + +test('`bench.from(...).fn` is optional — `bench.from` registrations carry no benchmark function', ({ bench }) => { + const reg = bench.from('baseline', 'results.json') + expectTypeOf(reg.fn).toEqualTypeOf['fn']>() + expectTypeOf(reg.fn).toBeNullable() +}) + +test('`bench.from` registrations contribute to the name union in `bench.compare`', async ({ bench }) => { + const storage = await bench.compare( + bench('live', () => {}), + bench.from('baseline', 'results.json'), + ) + expectTypeOf(storage).toEqualTypeOf>() + expectTypeOf(storage.get('baseline')).toEqualTypeOf() +}) + +test('`bench.from(fn, source)` accepts a function and uses its name as the benchmark name', ({ bench }) => { + function myBench() {} + const reg = bench.from(myBench, 'results.json') + expectTypeOf(reg).toEqualTypeOf>() +}) + +test('`bench.from(name)` without a source is a type error', ({ bench }) => { + // @ts-expect-error second argument (source) is required + bench.from('baseline') +}) diff --git a/test/e2e/test/benchmarking.test.ts b/test/e2e/test/benchmarking.test.ts index acd0e8457d16..92af67411018 100644 --- a/test/e2e/test/benchmarking.test.ts +++ b/test/e2e/test/benchmarking.test.ts @@ -1,185 +1,1325 @@ -import type { createBenchmarkJsonReport } from 'vitest/src/node/reporters/benchmark/json-formatter.js' -import fs from 'node:fs' -import * as pathe from 'pathe' -import { assert, expect, it } from 'vitest' -import { runVitest } from '../../test-utils' - -it('sequential', async () => { - const root = pathe.join(import.meta.dirname, '../fixtures/benchmarking/sequential') - await runVitest({ root }, [], { mode: 'benchmark' }) - const testLog = await fs.promises.readFile(pathe.join(root, 'test.log'), 'utf-8') - expect(testLog).toMatchSnapshot() -}) - -it('summary', async () => { - const root = pathe.join(import.meta.dirname, '../fixtures/benchmarking/reporter') - const result = await runVitest({ root }, ['summary.bench.ts'], { mode: 'benchmark' }) - expect(result.stderr).toBe('') - expect(result.stdout).not.toContain('NaNx') - expect(result.stdout.split('BENCH Summary')[1].replaceAll(/[0-9.]+x/g, '(?)')).toMatchSnapshot() -}) - -it('non-tty', async () => { - const root = pathe.join(import.meta.dirname, '../fixtures/benchmarking/basic') - const result = await runVitest({ root }, ['base.bench.ts'], { mode: 'benchmark' }) - const lines = result.stdout.split('\n').slice(4).slice(0, 11) - const expected = `\ - ✓ base.bench.ts > sort - name - · normal - · reverse - - ✓ base.bench.ts > timeout - name - · timeout100 - · timeout75 - · timeout50 - · timeout25 -` - - for (const [index, line] of expected.trim().split('\n').entries()) { - expect(lines[index]).toMatch(line) +import type { BaselineData, BenchResult, TestBenchmark, TestBenchmarkTask } from 'vitest' +import type { JsonTestResults } from 'vitest/node' +import { expect, test } from 'vitest' +import { runInlineTests } from '../../test-utils' + +// Synthetic stats — just enough to satisfy BenchResult shape for the matchers +// and BaselineData shape for on-disk round-trip tests. All fields beyond +// `mean` are filled with deterministic numbers so snapshots stay stable. +function fakeStats(mean: number) { + return { + aad: 0, + critical: 0, + df: 0, + mad: 0, + max: mean, + samples: undefined, + mean, + min: mean, + moe: 0, + p50: mean, + p75: mean, + p99: mean, + p995: mean, + p999: mean, + rme: 0, + samplesCount: 2, + sd: 0, + sem: 0, + variance: 0, + } as const +} + +function fakeBaseline(mean: number): BaselineData { + return { + latency: fakeStats(mean), + throughput: fakeStats(mean > 0 ? 1 / mean : 0), + period: mean, + totalTime: mean * 10, } +} + +function fakeResult(mean: number): BenchResult { + return { + ...fakeBaseline(mean), + name: 'fake', + } as unknown as BenchResult +} + +// keep runs tiny — this suite asserts wiring, not measurement accuracy +const fastBenchOptions = { + time: 0, + iterations: 2, + warmupTime: 0, + warmupIterations: 0, +} + +test('bench.compare records benchmark results for each registration', async () => { + const benchmarks: TestBenchmark[] = [] + + const { stderr } = await runInlineTests( + { + 'basic.bench.ts': /* ts */` + import { test, inject } from 'vitest' + + test('compare loops', async ({ bench }) => { + await bench.compare( + bench('for', () => { let x = 0; for (let i = 0; i < 10; i++) x += i }), + bench('while', () => { let x = 0, i = 0; while (i < 10) { x += i; i++ } }), + inject('options'), + ) + }) +`, + }, + { + benchmark: { enabled: true }, + reporters: [ + { + onTestCaseBenchmark(_testCase, benchmark) { + benchmarks.push(benchmark) + }, + }, + ], + provide: { + options: fastBenchOptions, + }, + }, + ) + + expect(stderr).toBe('') + expect(benchmarks).toHaveLength(1) + const [{ tasks }] = benchmarks + expect(tasks.map(t => t.name).sort()).toEqual(['for', 'while']) + expect(tasks.every(t => typeof t.latency.mean === 'number')).toBe(true) + expect(tasks.map(t => t.rank).sort()).toEqual([1, 2]) }) -it.for([true, false])('includeSamples %s', async (includeSamples) => { - const result = await runVitest( +test('bench accepts options as second argument and rejects them as third', async () => { + const { stderr, results } = await runInlineTests( { - root: pathe.join(import.meta.dirname, '../fixtures/benchmarking/reporter'), - benchmark: { includeSamples }, + 'sig.bench.ts': /* ts */` + import { test, expect, inject } from 'vitest' + + test('bench signatures', async ({ bench }) => { + const fn = () => 1 + const opts = { async: false } + + // options as the 2nd argument (preferred form, matches test()) + const withOpts = bench('with-opts', opts, fn) + expect(withOpts.name).toBe('with-opts') + expect(withOpts.fn).toBe(fn) + expect(withOpts.fnOpts).toBe(opts) + + // simplest form — no options + const noOpts = bench('no-opts', fn) + expect(noOpts.fn).toBe(fn) + expect(noOpts.fnOpts).toBeUndefined() + + // legacy (fn, options) form must throw + expect(() => bench('legacy', fn, opts)).toThrow(/third argument/) + + // consume the registrations so the unrun-bench warning stays silent + await bench.compare(withOpts, noOpts, inject('options')) + }) +`, }, - ['summary.bench.ts'], - { mode: 'benchmark' }, + { benchmark: { enabled: true }, provide: { options: fastBenchOptions } }, ) - assert(result.ctx) - const allSamples = [...result.ctx.state.idMap.values()] - .filter(t => t.meta.benchmark) - .map(t => t.result?.benchmark?.samples) - if (includeSamples) { - expect(allSamples[0]).not.toEqual([]) - } - else { - expect(allSamples[0]).toEqual([]) - } + + expect(stderr).toBe('') + const testCases = [...(results[0]?.children.allTests() ?? [])] + expect(testCases).toHaveLength(1) + expect(testCases[0].result()?.state).toBe('passed') }) -it('compare', async () => { - await fs.promises.rm('./fixtures/benchmarking/compare/bench.json', { force: true }) +test('bench exposes plain and perProject compositions and prints a table', async () => { + const tasks: TestBenchmarkTask[] = [] - // --outputJson - { - const result = await runVitest({ - root: './fixtures/benchmarking/compare', - outputJson: './bench.json', - reporters: ['default'], - }, [], { mode: 'benchmark' }) - expect(result.exitCode).toBe(0) - expect(fs.existsSync('./fixtures/benchmarking/compare/bench.json')).toBe(true) - } + const { stderr, stdout } = await runInlineTests( + { + 'compositions.bench.ts': /* ts */` + import { test, inject } from 'vitest' - // --compare - { - const result = await runVitest({ - root: './fixtures/benchmarking/compare', - compare: './bench.json', - reporters: ['default'], - }, [], { mode: 'benchmark' }) - expect(result.exitCode).toBe(0) - const lines = result.stdout.split('\n').slice(4).slice(0, 6) - const expected = ` -✓ basic.bench.ts > suite - name - · sleep10 - (baseline) - · sleep100 - (baseline) - ` - - for (const [index, line] of expected.trim().split('\n').entries()) { - expect(lines[index]).toMatch(line.trim()) + test('all compositions', async ({ bench }) => { + await bench.compare( + bench('plain', () => {}), + bench('perProject', { perProject: true }, () => {}), + inject('options'), + ) + }) +`, + }, + { + benchmark: { enabled: true }, + reporters: [ + 'default', + { + onTestCaseBenchmark(_testCase, benchmark) { + tasks.push(...benchmark.tasks) + }, + }, + ], + provide: { options: fastBenchOptions }, + }, + ) + + expect(stderr).toBe('') + + // every factory shape produces a registration with the right flags + const byName = Object.fromEntries( + tasks.map(t => [t.name, { perProject: !!t.perProject }]), + ) + expect(byName).toEqual({ + plain: { perProject: false }, + perProject: { perProject: true }, + }) + + // snapshot the rendered inline benchmark table. Rows are sorted by name so + // measurement-driven rank ordering doesn't reshuffle them, and the + // rank-dependent fastest/slowest suffix is stripped. + const lines = stdout.split('\n') + const headerIdx = lines.findIndex(l => /^\s*name\s+hz\s+min/.test(l)) + expect(headerIdx, `inline table header not found in stdout:\n${stdout}`).toBeGreaterThanOrEqual(0) + const [header, ...rows] = lines.slice(headerIdx, headerIdx + 3) + const normalized = formatBenchTable([ + header, + ...rows.map(r => r.replace(/\s+(?:fastest|slowest)\s*$/, '')).sort(), + ]) + + expect(normalized).toMatchInlineSnapshot(` + " name hz min max mean p75 p99 p995 p999 rme samples + perProject d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+ + plain d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+" + `) + + // Only one project ran, so the cross-project section is skipped — a + // single-row comparison has nothing to compare against. + expect(stdout).not.toContain('Cross-Project Benchmark Comparison') +}) + +// Rebuilds a benchmark table with every numeric cell replaced by `d+`, padded +// with spaces so each column keeps its natural alignment (first column +// left-aligned, numeric columns right-aligned — same rules the reporter uses). +// Column widths come from the normalized content so measurement noise at +// `time: 0` can't shift them between runs. The negative lookbehind on `\d+` +// keeps column labels like `p75` / `p995` intact. +function formatBenchTable(tableLines: string[]): string { + const indent = tableLines[0].match(/^\s*/)![0] + const rows = tableLines.map(line => + line.slice(indent.length).trimEnd().split(/\s{2,}/).map(cell => + cell.trim().replace(/(? + Math.max(...rows.map(r => (r[i] ?? '').length)), + ) + return rows + .map(row => indent + row.map((cell, i) => + i === 0 ? cell.padEnd(widths[i]) : cell.padStart(widths[i]), + ).join(' ')) + .join('\n') +} + +// Runs a single `bench(...).run()` through runInlineTests and pulls apart +// the reporter output so each test below can snapshot its table in isolation. +async function runComposition(benchCall: string): Promise<{ + tasks: TestBenchmarkTask[] + inlineTable: string + crossProjectSection: string | null +}> { + const tasks: TestBenchmarkTask[] = [] + const { stderr, stdout } = await runInlineTests( + { + 'composition.bench.ts': /* ts */` + import { test, inject } from 'vitest' + + test('composition', async ({ bench }) => { + await ${benchCall}.run(inject('options')) + }) +`, + }, + { + benchmark: { enabled: true }, + reporters: [ + 'default', + { + onTestCaseBenchmark(_testCase, benchmark) { + tasks.push(...benchmark.tasks) + }, + }, + ], + provide: { options: fastBenchOptions }, + }, + ) + + expect(stderr).toBe('') + + const lines = stdout.split('\n') + const headerIdx = lines.findIndex(l => /^\s*name\s+hz\s+min/.test(l)) + expect(headerIdx, `inline table header not found in stdout:\n${stdout}`).toBeGreaterThanOrEqual(0) + const inlineTable = formatBenchTable([ + lines[headerIdx], + lines[headerIdx + 1].replace(/\s+(?:fastest|slowest)\s*$/, ''), + ]) + + // the cross-project section is a divider + a series of titled sub-tables + // (each 2 lines: `project …` header + data row). Reformat each sub-table + // through formatBenchTable while leaving divider and title lines alone. + let crossProjectSection: string | null = null + const xpIdx = lines.findIndex(l => /Cross-Project Benchmark Comparison/.test(l)) + if (xpIdx >= 0) { + const summaryIdx = lines.findIndex((l, i) => i > xpIdx && /^\s*Test Files\s/.test(l)) + const xpLines = lines.slice(xpIdx, summaryIdx < 0 ? undefined : summaryIdx) + const out: string[] = [] + for (let i = 0; i < xpLines.length; i++) { + const line = xpLines[i] + if (/^\s*project\s+hz\s+min/.test(line) && i + 1 < xpLines.length) { + out.push(formatBenchTable([ + line, + xpLines[i + 1].replace(/\s+(?:fastest|slowest)\s*$/, ''), + ])) + i++ + } + else { + out.push(line) + } } + crossProjectSection = out.join('\n').trim() } + + return { tasks, inlineTable, crossProjectSection } +} + +function assertFlags(task: TestBenchmarkTask, name: string, flags: { perProject?: true }) { + expect(task.name).toBe(name) + expect(task.perProject).toBe(flags.perProject) +} + +test('plain `bench()` records a task with no flags', async () => { + const { tasks, inlineTable, crossProjectSection } = await runComposition( + `bench('plain', () => {})`, + ) + expect(tasks).toHaveLength(1) + assertFlags(tasks[0], 'plain', {}) + expect(crossProjectSection).toBeNull() + expect(inlineTable).toMatchInlineSnapshot(` + " name hz min max mean p75 p99 p995 p999 rme samples + plain d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+" + `) +}) + +test('`bench(..., { perProject: true }, fn)` records a perProject task in the inline table; cross-project section is omitted with only one project', async () => { + const { tasks, inlineTable, crossProjectSection } = await runComposition( + `bench('perProject', { perProject: true }, () => {})`, + ) + expect(tasks).toHaveLength(1) + assertFlags(tasks[0], 'perProject', { perProject: true }) + expect(inlineTable).toMatchInlineSnapshot(` + " name hz min max mean p75 p99 p995 p999 rme samples + perProject d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+" + `) + expect(crossProjectSection).toBeNull() +}) + +test('junit reporter embeds the benchmark table inside ', async () => { + const { stdout } = await runInlineTests( + { + 'junit.bench.ts': /* ts */` + import { test, inject } from 'vitest' + + test('junit benches', async ({ bench }) => { + await bench.compare( + bench('a', () => {}), + bench('b', () => {}), + inject('options'), + ) + }) +`, + }, + { + benchmark: { enabled: true }, + reporters: 'junit', + provide: { options: fastBenchOptions }, + }, + ) + + // extract the block from the rendered XML + // eslint-disable-next-line regexp/no-super-linear-backtracking + const systemOut = stdout.match(/\s*\n([\s\S]*?)<\/system-out>/)?.[1] + expect(systemOut, stdout).toBeDefined() + + // a header + 2 data rows — reformat through the shared helper so digits + // collapse to `d+` and widths become measurement-independent + const tableLines = systemOut!.split('\n').filter(l => l.trim()) + expect(tableLines).toHaveLength(3) + const [header, ...rows] = tableLines + const formatted = formatBenchTable([ + header, + ...rows.map(r => r.replace(/\s+(?:fastest|slowest)\s*$/, '')).sort(), + ]) + + expect(formatted).toMatchInlineSnapshot(` + "name hz min max mean p75 p99 p995 p999 rme samples + a d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+ + b d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+" + `) +}) + +// Runs the given inner-test source as a virtual bench file and asserts the +// inner test case finished in `passed` state (i.e. every `expect` inside the +// bench passed). Lets each outer test push its assertions INTO the inner +// test file and just verify the aggregate outcome. +async function runPassingBench( + filename: string, + source: string, + config: Parameters[1] = {}, +) { + const { stderr, results } = await runInlineTests( + { [filename]: source }, + { benchmark: { enabled: true }, provide: { options: fastBenchOptions }, ...config }, + ) + expect(stderr, `stderr should be empty:\n${stderr}`).toBe('') + const testCases = [...(results[0]?.children.allTests() ?? [])] + expect(testCases).toHaveLength(1) + expect( + testCases[0].result()?.state, + JSON.stringify(testCases[0].result()?.errors?.map(e => e.message), null, 2), + ).toBe('passed') +} + +test('`bench()` inside a non-benchmark project throws a helpful error', async () => { + const { stderr, results } = await runInlineTests( + { + 'regular.test.ts': /* ts */` + import { test, expect } from 'vitest' + test('misuse', async ({ bench }) => { + expect(() => bench('x', () => {})).toThrow( + /Cannot use the \`bench\` test-context fixture within a regular test run/, + ) + }) +`, + }, + { /* benchmark.enabled defaults to false */ }, + ) + expect(stderr).toBe('') + const testCases = [...(results[0]?.children.allTests() ?? [])] + expect(testCases).toHaveLength(1) + expect(testCases[0].result()?.state).toBe('passed') +}) + +test('`bench.compare()` with zero registrations throws "requires at least 2"', async () => { + await runPassingBench('zero.bench.ts', /* ts */` + import { test, expect } from 'vitest' + test('zero', async ({ bench }) => { + await expect(bench.compare()).rejects.toThrow( + /requires at least 2 benchmarks, received 0/, + ) + }) + `) +}) + +test('`bench.compare(regA)` with one registration throws and suggests `.run()`', async () => { + await runPassingBench('one.bench.ts', /* ts */` + import { test, expect } from 'vitest' + test('one', async ({ bench }) => { + await expect(bench.compare(bench('a', () => {}))).rejects.toThrow( + /received 1.+Consider calling .+bench\\(\\)\\.run\\(\\)/s, + ) + }) + `) +}) + +test('`bench.compare(reg, non-reg, reg)` throws the shape error', async () => { + await runPassingBench('shape.bench.ts', /* ts */` + import { test, expect } from 'vitest' + test('shape', async ({ bench }) => { + await expect(bench.compare( + bench('a', () => {}), + { name: 'fake', fn: () => {} }, + bench('b', () => {}), + )).rejects.toThrow( + /expects every argument to be the return value of/, + ) + }) + `) +}) + +test('`bench(name, { writeResult }, fn)` writes a result file at the given path', async () => { + const { stderr, fs } = await runInlineTests( + { + 'foo.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('write', async ({ bench }) => { + await bench('x', { writeResult: './out/x.json' }, () => {}).run(inject('options')) + }) +`, + }, + { + benchmark: { enabled: true }, + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + const content = JSON.parse(fs.readFile('out/x.json')) + expect(typeof content.latency.mean).toBe('number') + expect(typeof content.throughput.mean).toBe('number') + expect(typeof content.period).toBe('number') + expect(typeof content.totalTime).toBe('number') }) -it('basic', { timeout: 60_000 }, async () => { - const root = pathe.join(import.meta.dirname, '../fixtures/benchmarking/basic') - const benchFile = pathe.join(root, 'bench.json') - fs.rmSync(benchFile, { force: true }) - - const result = await runVitest({ - root, - allowOnly: true, - outputJson: 'bench.json', - - // Verify that type testing cannot be used with benchmark - typecheck: { enabled: true }, - }, [], { mode: 'benchmark' }) - expect(result.stderr).toBe('') - expect(result.exitCode).toBe(0) - - const benchResult = await fs.promises.readFile(benchFile, 'utf-8') - const resultJson: ReturnType = JSON.parse(benchResult) - const names = resultJson.files.map(f => f.groups.map(g => [g.fullName, g.benchmarks.map(b => b.name)])) - expect(names).toMatchInlineSnapshot(` - [ - [ - [ - "base.bench.ts > sort", - [ - "normal", - "reverse", - ], - ], - [ - "base.bench.ts > timeout", - [ - "timeout100", - "timeout75", - "timeout50", - "timeout25", - ], - ], +test('`writeResult` is overwritten on every successful run', async () => { + // seed a file with a sentinel value, then run the benchmark — the run + // must replace the sentinel with fresh measurements + const sentinel = fakeBaseline(99999) + const { stderr, fs } = await runInlineTests( + { + 'upd.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('upd', async ({ bench }) => { + await bench('x', { writeResult: './out/upd.json' }, () => {}).run(inject('options')) + }) +`, + 'out/upd.json': JSON.stringify(sentinel), + }, + { benchmark: { enabled: true }, provide: { options: fastBenchOptions } }, + ) + expect(stderr).toBe('') + const after = JSON.parse(fs.readFile('out/upd.json')) + expect(after.latency.mean).not.toBe(99999) +}) + +// eslint-disable-next-line no-template-curly-in-string +test('`writeResult` substitutes `${projectName}` so multi-project runs do not collide', async () => { + const { stderr, fs } = await runInlineTests( + { + 'shared.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('t', async ({ bench }) => { + await bench( + 'x', + { writeResult: './out/x.\${projectName}.json' }, + () => {}, + ).run(inject('options')) + }) +`, + }, + { + projects: [ + { test: { name: 'one', benchmark: { enabled: true } } }, + { test: { name: 'two', benchmark: { enabled: true } } }, ], - [], - [ - [ - "only.bench.ts", - [ - "visited", - "visited2", - ], - ], - [ - "only.bench.ts > a0", - [ - "0", - ], - ], - [ - "only.bench.ts > a1 > b1 > c1", - [ - "1", - ], - ], - [ - "only.bench.ts > a2", - [ - "2", - ], - ], - [ - "only.bench.ts > a3 > b3", - [ - "3", - ], - ], - [ - "only.bench.ts > a4 > b4", - [ - "4", - ], - ], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + // ${projectName} substitutes to the parent project name (not the cloned + // bench project name) so users never see the internal " (bench)" suffix + // bleed into their paths. + expect(typeof JSON.parse(fs.readFile('out/x.one.json')).latency.mean).toBe('number') + expect(typeof JSON.parse(fs.readFile('out/x.two.json')).latency.mean).toBe('number') +}) + +test('`writeResult` does NOT write a file when the benchmark throws', async () => { + const { fs } = await runInlineTests( + { + 'throw.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('throws', async ({ bench }) => { + try { + await bench( + 'x', + { writeResult: './out/should-not-exist.json' }, + () => { throw new Error('boom') }, + ).run(inject('options')) + } catch {} + }) +`, + }, + { benchmark: { enabled: true }, provide: { options: fastBenchOptions } }, + ) + expect(() => fs.readFile('out/should-not-exist.json')).toThrow() +}) + +test('`bench.from(name, path)` reads a stored result without invoking any function', async () => { + const seed = fakeBaseline(0.5) + const { stderr, results } = await runInlineTests( + { + 'read.bench.ts': /* ts */` + import { test, expect } from 'vitest' + test('read', async ({ bench }) => { + const r = await bench.from('previous', './out/seed.json').run() + expect(r.latency.mean).toBe(0.5) + }) +`, + 'out/seed.json': JSON.stringify(seed), + }, + { benchmark: { enabled: true } }, + ) + expect(stderr).toBe('') + expect([...(results[0]?.children.allTests() ?? [])][0]?.result()?.state).toBe('passed') +}) + +test('`bench.from(name, fn)` awaits the function and treats its return value as the result', async () => { + await runPassingBench('readfn.bench.ts', /* ts */` + import { test, expect } from 'vitest' + test('read via function', async ({ bench }) => { + const data = { + latency: { mean: 0.42, min: 0.42, max: 0.42, samplesCount: 1 }, + throughput: { mean: 1, min: 1, max: 1, samplesCount: 1 }, + period: 0.42, + totalTime: 0.42, + } + const r = await bench.from('previous', () => Promise.resolve(data)).run() + expect(r.latency.mean).toBe(0.42) + }) + `) +}) + +test('`bench.from()` raises a helpful error when the file is missing', async () => { + await runPassingBench('missingfile.bench.ts', /* ts */` + import { test, expect } from 'vitest' + test('missing file', async ({ bench }) => { + await expect(bench.from('x', './does-not-exist.json').run()).rejects.toThrow( + /could not find a result file at/, + ) + }) + `) +}) + +test('`bench.from()` rejects a path that escapes the project root', async () => { + await runPassingBench('escape.bench.ts', /* ts */` + import { test, expect } from 'vitest' + test('escape', async ({ bench }) => { + await expect(bench.from('x', '../package.json').run()).rejects.toThrow( + /resolves outside the project root/, + ) + }) + `) +}) + +test('`bench.compare` with a live `writeResult` AND a `bench.from()` records both tasks', async () => { + const seed = fakeBaseline(0.5) + const tasks: TestBenchmarkTask[] = [] + const { stderr, results } = await runInlineTests( + { + 'mixed.bench.ts': /* ts */` + import { test, expect, inject } from 'vitest' + test('mixed', async ({ bench }) => { + const storage = await bench.compare( + bench('current', { writeResult: './out/current.json' }, () => {}), + bench.from('previous', './out/previous.json'), + inject('options'), + ) + expect(storage.get('previous').latency.mean).toBe(0.5) + expect(typeof storage.get('current').latency.mean).toBe('number') + }) +`, + 'out/previous.json': JSON.stringify(seed), + }, + { + benchmark: { enabled: true }, + reporters: [{ + onTestCaseBenchmark(_tc, benchmark) { + tasks.push(...benchmark.tasks) + }, + }], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + expect([...(results[0]?.children.allTests() ?? [])][0]?.result()?.state).toBe('passed') + // both rows appear in the same TestBenchmark, with the from() row marked + expect(tasks.map(t => ({ name: t.name, fromStore: !!t.fromStore })).sort((a, b) => a.name.localeCompare(b.name))) + .toEqual([ + { name: 'current', fromStore: false }, + { name: 'previous', fromStore: true }, + ]) +}) + +test('`bench.compare` with only `bench.from()` registrations skips tinybench entirely', async () => { + const a = fakeBaseline(0.5) + const b = fakeBaseline(1) + const tasks: TestBenchmarkTask[] = [] + const { stderr, results } = await runInlineTests( + { + 'only-from.bench.ts': /* ts */` + import { test, expect } from 'vitest' + test('only from', async ({ bench }) => { + const storage = await bench.compare( + bench.from('a', './out/a.json'), + bench.from('b', './out/b.json'), + ) + expect(storage.get('a').latency.mean).toBe(0.5) + expect(storage.get('b').latency.mean).toBe(1) + }) +`, + 'out/a.json': JSON.stringify(a), + 'out/b.json': JSON.stringify(b), + }, + { + benchmark: { enabled: true }, + reporters: [{ + onTestCaseBenchmark(_tc, benchmark) { + tasks.push(...benchmark.tasks) + }, + }], + }, + ) + expect(stderr).toBe('') + expect([...(results[0]?.children.allTests() ?? [])][0]?.result()?.state).toBe('passed') + expect(tasks.every(t => t.fromStore)).toBe(true) +}) + +test('`bench.from()` rows render rme and samples columns from the stored data', async () => { + // The on-disk BaselineData includes the full `latency` Statistics — including + // `rme` and `samplesCount` — so a stored row should display real numbers in + // those columns, not placeholder dashes. + const seed = { ...fakeBaseline(0.5), latency: { ...fakeStats(0.5), rme: 1.23, samplesCount: 7 } } + const { stderr, stdout } = await runInlineTests( + { + 'render.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('render', async ({ bench }) => { + await bench.compare( + bench('live', () => {}), + bench.from('stored', './out/from.json'), + inject('options'), + ) + }) +`, + 'out/from.json': JSON.stringify(seed), + }, + { + benchmark: { enabled: true }, + reporters: ['default'], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + const lines = stdout.split('\n') + const headerIdx = lines.findIndex(l => /^\s*name\s+hz\s+min/.test(l)) + expect(headerIdx, `inline table header not found in stdout:\n${stdout}`).toBeGreaterThanOrEqual(0) + // The header columns are: name, hz, min, max, mean, p75, p99, p995, p999, rme, samples. + // Find the row for "stored" and inspect its last two cells. + const storedRow = lines.slice(headerIdx + 1, headerIdx + 3).find(l => /^\s*stored\b/.test(l))! + expect(storedRow, `stored row not found in:\n${stdout}`).toBeDefined() + const cells = storedRow.trim().replace(/\s+(?:fastest|slowest)\s*$/, '').split(/\s+/) + // rme + samples must be real values, not the `-` placeholder + expect(cells[cells.length - 2]).toBe('±1.23%') + expect(cells[cells.length - 1]).toBe('7') +}) + +test('`benchmark.include` overrides the default `*.bench.ts` pattern', async () => { + const tasks: TestBenchmarkTask[] = [] + const { stderr } = await runInlineTests( + { + 'a.perf.ts': /* ts */` + import { test, inject } from 'vitest' + test('custom include', async ({ bench }) => { + await bench('x', () => {}).run(inject('options')) + }) +`, + // this .bench.ts would run under the default pattern — but we override + 'b.bench.ts': /* ts */` + import { test } from 'vitest' + test('should not run', async ({ bench }) => { + await bench('never', () => { throw new Error('not discovered') }).run() + }) +`, + }, + { + benchmark: { enabled: true, include: ['**/*.perf.ts'] }, + reporters: [{ + onTestCaseBenchmark(_tc, benchmark) { + tasks.push(...benchmark.tasks) + }, + }], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + expect(tasks.map(t => t.name)).toEqual(['x']) +}) + +test('`benchmark.exclude` filters matching files out of the bench project', async () => { + const tasks: TestBenchmarkTask[] = [] + const { stderr } = await runInlineTests( + { + 'wanted.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('wanted', async ({ bench }) => { + await bench('x', () => {}).run(inject('options')) + }) +`, + 'skipped.bench.ts': /* ts */` + import { test } from 'vitest' + test('should not run', async ({ bench }) => { + await bench('never', () => { throw new Error('excluded') }).run() + }) +`, + }, + { + benchmark: { + enabled: true, + exclude: ['**/skipped.bench.ts', '**/node_modules/**'], + }, + reporters: [{ + onTestCaseBenchmark(_tc, benchmark) { + tasks.push(...benchmark.tasks) + }, + }], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + expect(tasks.map(t => t.name)).toEqual(['x']) +}) + +test('`benchmark.includeSource` runs in-source benchmarks via `import.meta.vitest`', async () => { + const tasks: TestBenchmarkTask[] = [] + const { stderr } = await runInlineTests( + { + 'lib.ts': /* ts */` + export function add(a: number, b: number) { return a + b } + if (import.meta.vitest) { + const { test } = import.meta.vitest + test('in-source', async ({ bench }) => { + await bench('add', () => add(1, 2)).run({ + time: 0, iterations: 2, warmupTime: 0, warmupIterations: 0, + }) + }) + } +`, + }, + { + benchmark: { enabled: true, includeSource: ['**/*.ts'] }, + reporters: [{ + onTestCaseBenchmark(_tc, benchmark) { + tasks.push(...benchmark.tasks) + }, + }], + }, + ) + expect(stderr).toBe('') + expect(tasks.map(t => t.name)).toEqual(['add']) +}) + +test('`benchmark.retainSamples: true` preserves the raw samples array', async () => { + const tasks: TestBenchmarkTask[] = [] + const { stderr } = await runInlineTests( + { + 'samples.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('samples', async ({ bench }) => { + await bench('x', () => {}).run(inject('options')) + }) +`, + }, + { + benchmark: { enabled: true, retainSamples: true }, + reporters: [{ + onTestCaseBenchmark(_tc, benchmark) { + tasks.push(...benchmark.tasks) + }, + }], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + expect(tasks).toHaveLength(1) + const samples = tasks[0].latency.samples + expect(Array.isArray(samples)).toBe(true) + expect(samples!.length).toBeGreaterThanOrEqual(1) +}) + +test('`benchmark.retainSamples: false` (the default) omits the samples array', async () => { + const tasks: TestBenchmarkTask[] = [] + const { stderr } = await runInlineTests( + { + 'nosamples.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('nosamples', async ({ bench }) => { + await bench('x', () => {}).run(inject('options')) + }) +`, + }, + { + benchmark: { enabled: true }, + reporters: [{ + onTestCaseBenchmark(_tc, benchmark) { + tasks.push(...benchmark.tasks) + }, + }], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + expect(tasks).toHaveLength(1) + expect(tasks[0].latency.samples).toBeUndefined() +}) + +test('`benchmark.enabled: false` skips .bench.ts files entirely', async () => { + const names: string[] = [] + const { stderr } = await runInlineTests( + { + 'ignored.bench.ts': /* ts */` + import { test } from 'vitest' + test('should-never-run', () => { + throw new Error('bench project should not have been created') + }) +`, + 'regular.test.ts': /* ts */` + import { test } from 'vitest' + test('runs', () => {}) +`, + }, + { + // benchmark.enabled defaults to false → no bench project cloned + reporters: [{ + onTestCaseReady(testCase) { + names.push(testCase.name) + }, + }], + }, + ) + expect(stderr).toBe('') + expect(names).toEqual(['runs']) +}) + +test('`vitest bench` CLI invocation filters to the cloned benchmark project', async () => { + const { stderr, ctx } = await runInlineTests( + { + 'x.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('x', async ({ bench }) => { + await bench('a', () => {}).run(inject('options')) + }) +`, + }, + { + $cliOptions: { benchmarkOnly: true }, + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + // --benchmarkOnly narrows ctx.projects to only bench-enabled projects + const projectNames = ctx?.projects.map(p => p.name) ?? [] + expect(projectNames).toEqual(['bench']) +}) + +test('json reporter surfaces benchmarks on each assertion result', async () => { + const { stderr, stdout } = await runInlineTests( + { + 'foo.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('smoke', async ({ bench }) => { + await bench.compare( + bench('a', () => {}), + bench('b', () => {}), + inject('options'), + ) + }) +`, + }, + { + benchmark: { enabled: true }, + reporters: 'json', + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + const parsed = JSON.parse(stdout) as JsonTestResults + const assertionResults = parsed.testResults.flatMap(tr => tr.assertionResults) + const smoke = assertionResults.find(a => a.title === 'smoke')! + expect( + smoke, + `no assertion result titled "smoke" in json output:\n${JSON.stringify(parsed, null, 2)}`, + ).toBeDefined() + expect(Array.isArray(smoke.benchmarks)).toBe(true) + expect(smoke.benchmarks).toHaveLength(1) + const [benchmark] = smoke.benchmarks + expect(benchmark.tasks.map(t => t.name).sort()).toEqual(['a', 'b']) + // tasks are ranked 1..n and carry the full statistics surface + expect(benchmark.tasks.map(t => t.rank).sort()).toEqual([1, 2]) + expect(typeof benchmark.tasks[0].latency.mean).toBe('number') + expect(typeof benchmark.tasks[0].throughput.mean).toBe('number') +}) + +test('multi-project run aggregates perProject tasks into a single cross-project sub-table', async () => { + const { stderr, stdout } = await runInlineTests( + { + 'shared.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('cross', async ({ bench }) => { + await bench('x', { perProject: true }, () => {}).run(inject('options')) + }) +`, + }, + { + projects: [ + { test: { name: 'one', benchmark: { enabled: true } } }, + { test: { name: 'two', benchmark: { enabled: true } } }, ], - ] + reporters: ['default'], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + expect(stdout).toContain('Cross-Project Benchmark Comparison') + + // find the `project ... hz ... min` header of the `x` sub-table + 2 data rows + const lines = stdout.split('\n') + const headerIdx = lines.findIndex(l => /^\s*project\s+hz\s+min/.test(l)) + expect(headerIdx, `cross-project sub-table header not found in stdout:\n${stdout}`).toBeGreaterThan(-1) + const [header, ...rows] = lines.slice(headerIdx, headerIdx + 3) + const normalized = formatBenchTable([ + header, + ...rows.map(r => r.replace(/\s+(?:fastest|slowest)\s*$/, '')).sort(), + ]) + expect(normalized).toMatchInlineSnapshot(` + " project hz min max mean p75 p99 p995 p999 rme samples + one (bench) d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+ + two (bench) d+ d+ d+ d+ d+ d+ d+ d+ ±d+% d+" `) }) + +test('cross-project section is skipped when every perProject benchmark ran in only one project', async () => { + // Even when several perProject benchmarks are recorded, the cross-project + // table is useless if each one ran in exactly one project — every sub-table + // would be a single row with nothing to compare against. + const { stderr, stdout } = await runInlineTests( + { + 'solo.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('solo', async ({ bench }) => { + await bench.compare( + bench('a', { perProject: true }, () => {}), + bench('b', { perProject: true }, () => {}), + inject('options'), + ) + }) +`, + }, + { + benchmark: { enabled: true }, + reporters: ['default'], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + expect(stdout).not.toContain('Cross-Project Benchmark Comparison') +}) + +test('cross-project section is absent when no benchmark is perProject', async () => { + const { stderr, stdout } = await runInlineTests( + { + 'nox.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('no perProject', async ({ bench }) => { + await bench('only', () => {}).run(inject('options')) + }) +`, + }, + { + benchmark: { enabled: true }, + reporters: ['default'], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + expect(stdout).not.toContain('Cross-Project Benchmark Comparison') +}) + +test('`bench.compare` wraps multiple failed benchmarks in an AggregateError', async () => { + await runPassingBench('aggregate.bench.ts', /* ts */` + import { test, expect, inject } from 'vitest' + test('aggregate errors', async ({ bench }) => { + const err = await bench.compare( + bench('a', () => { throw new Error('A failed') }), + bench('b', () => { throw new Error('B failed') }), + inject('options'), + ).catch(e => e) + expect(err).toBeInstanceOf(AggregateError) + expect(err.message).toBe('Some benchmarks failed') + const messages = err.errors.map((e) => e.message).sort() + expect(messages).toEqual(['A failed', 'B failed']) + }) + `) +}) + +test('`BenchStorage.get` returns a valid BenchResult shape for every registration', async () => { + await runPassingBench('storage.bench.ts', /* ts */` + import { test, expect, inject } from 'vitest' + test('storage shape', async ({ bench }) => { + const storage = await bench.compare( + bench('a', () => {}), + bench('b', () => {}), + inject('options'), + ) + for (const name of ['a', 'b'] as const) { + const result = storage.get(name) + expect(typeof result.latency.mean).toBe('number') + expect(typeof result.throughput.mean).toBe('number') + expect(typeof result.period).toBe('number') + expect(typeof result.totalTime).toBe('number') + } + }) + `) +}) + +test('`bench.compare` trailing options propagate through to the underlying Tinybench', async () => { + const benchmarks: TestBenchmark[] = [] + const { stderr } = await runInlineTests( + { + 'options.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('options', async ({ bench }) => { + await bench.compare( + bench('a', () => {}), + bench('b', () => {}), + { ...inject('options'), name: 'custom-bench-name' }, + ) + }) +`, + }, + { + benchmark: { enabled: true }, + reporters: [{ + onTestCaseBenchmark(_tc, benchmark) { + benchmarks.push(benchmark) + }, + }], + provide: { options: fastBenchOptions }, + }, + ) + expect(stderr).toBe('') + expect(benchmarks).toHaveLength(1) + // user-supplied `name` overrides the default ` ` label; + // the serialized benchmark emitted to reporters carries it verbatim + expect(benchmarks[0].name).toBe('custom-bench-name') +}) + +test('benchmark warns when module export getters are accessed too many times', async () => { + const { stderr } = await runInlineTests( + { + 'fixture.ts': /* ts */` + export const value = 1 +`, + 'getter-warning.bench.ts': /* ts */` + import { test, inject } from 'vitest' + import * as fixture from './fixture' + + test('getter warning', async ({ bench }) => { + await bench('read getter', () => { + for (let i = 0; i < 1_000_001; i++) { + void fixture.value + } + }).run(inject('options')) + }) +`, + }, + { + benchmark: { enabled: true }, + provide: { options: { ...fastBenchOptions, iterations: 1 } }, + }, + ) + + expect(stderr).toMatchInlineSnapshot(` + "stderr | getter-warning.bench.ts > getter warning + Benchmark Warning + Benchmark "getter warning 1" accessed module export getters too many times. + + This can make results unreliable because export getters add overhead. + See https://vitest.dev/guide/benchmarking#module-runner-overhead + + Tracked exports: + - fixture.ts > value + + " + `) +}) + +test('benchmark export getter warning can be suppressed', async () => { + const { stderr } = await runInlineTests( + { + 'fixture.ts': /* ts */` + export const value = 1 +`, + 'getter-warning.bench.ts': /* ts */` + import { test, inject } from 'vitest' + import * as fixture from './fixture' + + test('getter warning', async ({ bench }) => { + await bench('read getter', () => { + for (let i = 0; i < 1_000_001; i++) { + void fixture.value + } + }).run(inject('options')) + }) +`, + }, + { + benchmark: { enabled: true, suppressExportGetterWarnings: true }, + provide: { options: { ...fastBenchOptions, iterations: 1 } }, + }, + ) + + expect(stderr).toBe('') +}) + +test('warns when `bench()` is registered but never run', async () => { + const { stderr } = await runInlineTests( + { + 'unrun.bench.ts': /* ts */` + import { test } from 'vitest' + test('forgot to run', ({ bench }) => { + bench('a', () => {}) + }) +`, + }, + { benchmark: { enabled: true } }, + ) + expect(stderr).toContain('Benchmark Warning') + expect(stderr).toContain('forgot to run') + expect(stderr).toContain('"a"') +}) + +test('warns about every unrun registration in the message, including `bench.from()`', async () => { + const { stderr } = await runInlineTests( + { + 'out/seed.json': JSON.stringify(fakeBaseline(0.5)), + 'multi.bench.ts': /* ts */` + import { test } from 'vitest' + test('multi unrun', ({ bench }) => { + bench('a', () => {}) + bench('b', () => {}) + bench.from('seed', './out/seed.json') + }) +`, + }, + { benchmark: { enabled: true } }, + ) + expect(stderr).toContain('Benchmark Warning') + // every name appears in a single warning line + expect(stderr).toMatch(/"a".+"b".+"seed"/) +}) + +test('does NOT warn when every registration is consumed by `.run()` or `bench.compare()`', async () => { + const { stderr } = await runInlineTests( + { + 'consumed.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('all consumed', async ({ bench }) => { + await bench('lone', () => {}).run(inject('options')) + await bench.compare( + bench('x', () => {}), + bench('y', () => {}), + inject('options'), + ) + }) +`, + }, + { benchmark: { enabled: true }, provide: { options: fastBenchOptions } }, + ) + expect(stderr).toBe('') +}) + +test('warns only about the unrun registration when others are consumed', async () => { + const { stderr } = await runInlineTests( + { + 'partial.bench.ts': /* ts */` + import { test, inject } from 'vitest' + test('partial', async ({ bench }) => { + await bench('used', () => {}).run(inject('options')) + bench('forgotten', () => {}) + }) +`, + }, + { benchmark: { enabled: true }, provide: { options: fastBenchOptions } }, + ) + expect(stderr).toContain('Benchmark Warning') + expect(stderr).toContain('"forgotten"') + expect(stderr).not.toContain('"used"') +}) + +test('`BenchStorage.get("missing")` throws a descriptive error', async () => { + await runPassingBench('missing.bench.ts', /* ts */` + import { test, expect, inject } from 'vitest' + test('missing', async ({ bench }) => { + const storage = await bench.compare( + bench('a', () => {}), + bench('b', () => {}), + inject('options'), + ) + expect(() => storage.get('missing')).toThrow( + /task "missing" was not defined/, + ) + }) + `) +}) + +test('`toBeFasterThan` passes when actual.latency.mean is strictly smaller', () => { + expect(fakeResult(0.5)).toBeFasterThan(fakeResult(1.0)) +}) + +test('`toBeFasterThan` fails with a percent-slower message when actual is slower', () => { + expect(() => expect(fakeResult(1.0)).toBeFasterThan(fakeResult(0.5))) + .toThrowErrorMatchingInlineSnapshot(` + [Error: expect(received).toBeFasterThan(expected) + + Expected to be faster, but was 100.00% slower. + + Received: 1.00 ops/sec + Expected: 2.00 ops/sec + ] + `) +}) + +test('`toBeFasterThan` honours the `delta` threshold', () => { + const fast = fakeResult(0.8) // 20% faster than 1.0 + const slow = fakeResult(1.0) + // 20% faster is not enough when delta demands 30% + expect(() => expect(fast).toBeFasterThan(slow, { delta: 0.3 })) + .toThrow(/faster by at least 30%/) + // passes when the demanded margin is only 10% + expect(fast).toBeFasterThan(slow, { delta: 0.1 }) +}) + +test('`toBeSlowerThan` passes when actual.latency.mean is strictly larger', () => { + expect(fakeResult(1.0)).toBeSlowerThan(fakeResult(0.5)) +}) + +test('`toBeSlowerThan` fails with a percent-faster message when actual is faster', () => { + expect(() => expect(fakeResult(0.5)).toBeSlowerThan(fakeResult(1.0))) + .toThrowErrorMatchingInlineSnapshot(` + [Error: expect(received).toBeSlowerThan(expected) + + Expected to be slower, but was 50% faster. + + Received: 2.00 ops/sec + Expected: 1.00 ops/sec + ] + `) +}) + +test('`toBeSlowerThan` honours the `delta` threshold', () => { + // slow = 2x fast → 100% slower. delta 0.5 → threshold 150% → passes at 100% + expect(fakeResult(1.0)).toBeSlowerThan(fakeResult(0.5), { delta: 0.5 }) + expect(() => expect(fakeResult(1.0)).toBeSlowerThan(fakeResult(0.5), { delta: 1.5 })) + .toThrow(/slower by at least 150%/) +}) + +test('bench matchers reject non-benchmark-result values with a TypeError', () => { + expect(() => expect({ foo: 'bar' }).toBeFasterThan(fakeResult(1.0))) + .toThrow(TypeError) + expect(() => expect(fakeResult(1.0)).toBeFasterThan({ foo: 'bar' } as any)) + .toThrow(TypeError) + expect(() => expect({ foo: 'bar' }).toBeSlowerThan(fakeResult(1.0))) + .toThrow(TypeError) +}) + +declare module 'vitest' { + interface ProvidedContext { + options: typeof fastBenchOptions + } +} diff --git a/test/e2e/test/mode.test.ts b/test/e2e/test/mode.test.ts deleted file mode 100644 index e3e64862c006..000000000000 --- a/test/e2e/test/mode.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { expect, test } from 'vitest' -import * as testUtils from '../../test-utils' - -test.each([ - { expectedMode: 'test', command: ['run'] }, - { expectedMode: 'benchmark', command: ['bench', '--run'] }, -])(`env.mode should have the $expectedMode value when running in $name mode`, async ({ command, expectedMode }) => { - const { stdout } = await testUtils.runVitestCli(...(command), 'fixtures/mode', '-c', `fixtures/mode/vitest.${expectedMode}.config.ts`) - - expect(stdout).toContain(`✓ fixtures/mode/example.${expectedMode}.ts`) -}) - -test.each([ - { expectedMode: 'test', command: ['bench', '--run'], actualMode: 'benchmark' }, - { expectedMode: 'benchmark', command: ['run'], actualMode: 'test' }, -])(`should return error if actual mode $actualMode is different than expected mode $expectedMode`, async ({ command, expectedMode, actualMode }) => { - const { stdout, stderr } = await testUtils.runVitestCli(...(command), 'fixtures/mode', '-c', `fixtures/mode/vitest.${expectedMode}.config.ts`) - - expect(stderr).toContain(`env.mode: ${actualMode}`) - expect(stderr).toContain('Startup Error') - expect(stderr).toContain(`Error: env.mode should be equal to "${expectedMode}"`) - expect(stdout).toBe('') -}) - -test.each([ - { options: ['run'], expected: 'run' }, - { options: ['run', '--watch'], expected: 'watch' }, - { options: ['watch'], expected: 'watch' }, -] as const)(`vitest $options.0 $options.1 resolves to $expected-mode`, async ({ options, expected }) => { - const { vitest } = await testUtils.runVitestCli(...options, '--root', 'fixtures/run-mode') - - if (expected === 'watch') { - await vitest.waitForStdout('Test Files 1 passed (1)') - - expect(vitest.stdout).not.toContain('RUN') - expect(vitest.stdout).toContain('DEV') - expect(vitest.stdout).toContain('Waiting for file changes') - } - - if (expected === 'run') { - expect(vitest.stdout).toContain('RUN') - expect(vitest.stdout).not.toContain('DEV') - expect(vitest.stdout).not.toContain('Waiting for file changes') - } -}) diff --git a/test/e2e/test/public.test.ts b/test/e2e/test/public.test.ts index d82d3e857974..f35421edd89e 100644 --- a/test/e2e/test/public.test.ts +++ b/test/e2e/test/public.test.ts @@ -20,7 +20,7 @@ test('applies custom options', async () => { setupFiles: ['/test/setup.ts'], }) expect(viteConfig.mode).toBe('development') - expect(vitestConfig.mode).toBe('test') // vitest mode is "test" or "benchmark" + expect(vitestConfig.mode).toBe('development') expect(vitestConfig.setupFiles).toEqual(['/test/setup.ts']) expect(viteConfig.plugins.find(p => p.name === 'vitest')).toBeDefined() }) diff --git a/test/e2e/test/reporters/__snapshots__/json.test.ts.snap b/test/e2e/test/reporters/__snapshots__/json.test.ts.snap index 81268367765e..51c29af9cc49 100644 --- a/test/e2e/test/reporters/__snapshots__/json.test.ts.snap +++ b/test/e2e/test/reporters/__snapshots__/json.test.ts.snap @@ -3,6 +3,7 @@ exports[`json reporter > generates correct report 1`] = ` { "ancestorTitles": [], + "benchmarks": [], "failureMessages": [ "AssertionError: expected 2 to deeply equal 1 at /test/e2e/fixtures/reporters/json-fail.test.ts:8:13", diff --git a/test/e2e/test/reporters/__snapshots__/reporters.test.ts.snap b/test/e2e/test/reporters/__snapshots__/reporters.test.ts.snap index 6d5a1f7c5b2e..dd8e363a2132 100644 --- a/test/e2e/test/reporters/__snapshots__/reporters.test.ts.snap +++ b/test/e2e/test/reporters/__snapshots__/reporters.test.ts.snap @@ -130,6 +130,7 @@ exports[`json reporter (no outputFile entry) 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.4422860145568848, "failureMessages": [ "AssertionError: expected 2.23606797749979 to equal 2 @@ -158,6 +159,7 @@ exports[`json reporter (no outputFile entry) 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.0237109661102295, "failureMessages": [], "fullName": "suite JSON", @@ -170,6 +172,7 @@ exports[`json reporter (no outputFile entry) 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite async with timeout", "meta": {}, @@ -181,6 +184,7 @@ exports[`json reporter (no outputFile entry) 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 100.50598406791687, "failureMessages": [], "fullName": "suite timeout", @@ -193,6 +197,7 @@ exports[`json reporter (no outputFile entry) 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 20.184875011444092, "failureMessages": [], "fullName": "suite callback setup success ", @@ -205,6 +210,7 @@ exports[`json reporter (no outputFile entry) 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.33245420455932617, "failureMessages": [], "fullName": "suite callback test success ", @@ -217,6 +223,7 @@ exports[`json reporter (no outputFile entry) 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 19.738605976104736, "failureMessages": [], "fullName": "suite callback setup success done(false)", @@ -229,6 +236,7 @@ exports[`json reporter (no outputFile entry) 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.1923508644104004, "failureMessages": [], "fullName": "suite callback test success done(false)", @@ -241,6 +249,7 @@ exports[`json reporter (no outputFile entry) 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite todo test", "meta": {}, @@ -283,6 +292,7 @@ exports[`json reporter 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.4422860145568848, "failureMessages": [ "AssertionError: expected 2.23606797749979 to equal 2 @@ -311,6 +321,7 @@ exports[`json reporter 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.0237109661102295, "failureMessages": [], "fullName": "suite JSON", @@ -323,6 +334,7 @@ exports[`json reporter 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite async with timeout", "meta": {}, @@ -334,6 +346,7 @@ exports[`json reporter 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 100.50598406791687, "failureMessages": [], "fullName": "suite timeout", @@ -346,6 +359,7 @@ exports[`json reporter 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 20.184875011444092, "failureMessages": [], "fullName": "suite callback setup success ", @@ -358,6 +372,7 @@ exports[`json reporter 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.33245420455932617, "failureMessages": [], "fullName": "suite callback test success ", @@ -370,6 +385,7 @@ exports[`json reporter 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 19.738605976104736, "failureMessages": [], "fullName": "suite callback setup success done(false)", @@ -382,6 +398,7 @@ exports[`json reporter 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.1923508644104004, "failureMessages": [], "fullName": "suite callback test success done(false)", @@ -394,6 +411,7 @@ exports[`json reporter 1`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite todo test", "meta": {}, @@ -441,6 +459,7 @@ exports[`json reporter with outputFile 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.4422860145568848, "failureMessages": [ "AssertionError: expected 2.23606797749979 to equal 2 @@ -469,6 +488,7 @@ exports[`json reporter with outputFile 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.0237109661102295, "failureMessages": [], "fullName": "suite JSON", @@ -481,6 +501,7 @@ exports[`json reporter with outputFile 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite async with timeout", "meta": {}, @@ -492,6 +513,7 @@ exports[`json reporter with outputFile 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 100.50598406791687, "failureMessages": [], "fullName": "suite timeout", @@ -504,6 +526,7 @@ exports[`json reporter with outputFile 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 20.184875011444092, "failureMessages": [], "fullName": "suite callback setup success ", @@ -516,6 +539,7 @@ exports[`json reporter with outputFile 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.33245420455932617, "failureMessages": [], "fullName": "suite callback test success ", @@ -528,6 +552,7 @@ exports[`json reporter with outputFile 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 19.738605976104736, "failureMessages": [], "fullName": "suite callback setup success done(false)", @@ -540,6 +565,7 @@ exports[`json reporter with outputFile 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.1923508644104004, "failureMessages": [], "fullName": "suite callback test success done(false)", @@ -552,6 +578,7 @@ exports[`json reporter with outputFile 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite todo test", "meta": {}, @@ -599,6 +626,7 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.4422860145568848, "failureMessages": [ "AssertionError: expected 2.23606797749979 to equal 2 @@ -627,6 +655,7 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.0237109661102295, "failureMessages": [], "fullName": "suite JSON", @@ -639,6 +668,7 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite async with timeout", "meta": {}, @@ -650,6 +680,7 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 100.50598406791687, "failureMessages": [], "fullName": "suite timeout", @@ -662,6 +693,7 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 20.184875011444092, "failureMessages": [], "fullName": "suite callback setup success ", @@ -674,6 +706,7 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.33245420455932617, "failureMessages": [], "fullName": "suite callback test success ", @@ -686,6 +719,7 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 19.738605976104736, "failureMessages": [], "fullName": "suite callback setup success done(false)", @@ -698,6 +732,7 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.1923508644104004, "failureMessages": [], "fullName": "suite callback test success done(false)", @@ -710,6 +745,7 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite todo test", "meta": {}, @@ -757,6 +793,7 @@ exports[`json reporter with outputFile object 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.4422860145568848, "failureMessages": [ "AssertionError: expected 2.23606797749979 to equal 2 @@ -785,6 +822,7 @@ exports[`json reporter with outputFile object 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.0237109661102295, "failureMessages": [], "fullName": "suite JSON", @@ -797,6 +835,7 @@ exports[`json reporter with outputFile object 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite async with timeout", "meta": {}, @@ -808,6 +847,7 @@ exports[`json reporter with outputFile object 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 100.50598406791687, "failureMessages": [], "fullName": "suite timeout", @@ -820,6 +860,7 @@ exports[`json reporter with outputFile object 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 20.184875011444092, "failureMessages": [], "fullName": "suite callback setup success ", @@ -832,6 +873,7 @@ exports[`json reporter with outputFile object 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.33245420455932617, "failureMessages": [], "fullName": "suite callback test success ", @@ -844,6 +886,7 @@ exports[`json reporter with outputFile object 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 19.738605976104736, "failureMessages": [], "fullName": "suite callback setup success done(false)", @@ -856,6 +899,7 @@ exports[`json reporter with outputFile object 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.1923508644104004, "failureMessages": [], "fullName": "suite callback test success done(false)", @@ -868,6 +912,7 @@ exports[`json reporter with outputFile object 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite todo test", "meta": {}, @@ -915,6 +960,7 @@ exports[`json reporter with outputFile object in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.4422860145568848, "failureMessages": [ "AssertionError: expected 2.23606797749979 to equal 2 @@ -943,6 +989,7 @@ exports[`json reporter with outputFile object in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 1.0237109661102295, "failureMessages": [], "fullName": "suite JSON", @@ -955,6 +1002,7 @@ exports[`json reporter with outputFile object in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite async with timeout", "meta": {}, @@ -966,6 +1014,7 @@ exports[`json reporter with outputFile object in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 100.50598406791687, "failureMessages": [], "fullName": "suite timeout", @@ -978,6 +1027,7 @@ exports[`json reporter with outputFile object in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 20.184875011444092, "failureMessages": [], "fullName": "suite callback setup success ", @@ -990,6 +1040,7 @@ exports[`json reporter with outputFile object in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.33245420455932617, "failureMessages": [], "fullName": "suite callback test success ", @@ -1002,6 +1053,7 @@ exports[`json reporter with outputFile object in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 19.738605976104736, "failureMessages": [], "fullName": "suite callback setup success done(false)", @@ -1014,6 +1066,7 @@ exports[`json reporter with outputFile object in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "duration": 0.1923508644104004, "failureMessages": [], "fullName": "suite callback test success done(false)", @@ -1026,6 +1079,7 @@ exports[`json reporter with outputFile object in non-existing directory 2`] = ` "ancestorTitles": [ "suite", ], + "benchmarks": [], "failureMessages": [], "fullName": "suite todo test", "meta": {}, diff --git a/test/e2e/test/reporters/function-as-name.test.ts b/test/e2e/test/reporters/function-as-name.test.ts index c683f9dda4dc..810eccae8070 100644 --- a/test/e2e/test/reporters/function-as-name.test.ts +++ b/test/e2e/test/reporters/function-as-name.test.ts @@ -15,9 +15,13 @@ test('should print function name', async () => { expect(stdout).toContain('function-as-name.test.ts > Bar > Bar') }) -test('should print function name in benchmark', async () => { +test.for(['default', 'verbose'])('should print function name in benchmark in %s reporter', async (reporters) => { const filename = resolve('./fixtures/reporters/function-as-name.bench.ts') - const { stdout } = await runVitest({ root: './fixtures/reporters' }, [filename], { mode: 'benchmark' }) + const { stdout } = await runVitest({ + root: './fixtures/reporters', + reporters, + benchmark: { enabled: true }, + }, [filename]) expect(stdout).toBeTruthy() expect(stdout).toContain('Bar') diff --git a/test/e2e/test/reporters/merge-reports.test.ts b/test/e2e/test/reporters/merge-reports.test.ts index 961955bc4a77..be4fb1ebb0de 100644 --- a/test/e2e/test/reporters/merge-reports.test.ts +++ b/test/e2e/test/reporters/merge-reports.test.ts @@ -176,6 +176,7 @@ test('merge reports', async () => { "assertionResults": [ { "ancestorTitles": [], + "benchmarks": [], "failureMessages": [], "fullName": "test 1-1", "meta": {}, @@ -185,6 +186,7 @@ test('merge reports', async () => { }, { "ancestorTitles": [], + "benchmarks": [], "failureMessages": [ "AssertionError: expected 1 to be 2 // Object.is equality at /fixtures/reporters/merge-reports/first.test.ts:15:13", @@ -206,6 +208,7 @@ test('merge reports', async () => { "assertionResults": [ { "ancestorTitles": [], + "benchmarks": [], "failureMessages": [ "AssertionError: expected 1 to be 2 // Object.is equality at /fixtures/reporters/merge-reports/second.test.ts:5:13", @@ -220,6 +223,7 @@ test('merge reports', async () => { "ancestorTitles": [ "group", ], + "benchmarks": [], "failureMessages": [], "fullName": "group test 2-2", "meta": {}, @@ -231,6 +235,7 @@ test('merge reports', async () => { "ancestorTitles": [ "group", ], + "benchmarks": [], "failureMessages": [], "fullName": "group test 2-3", "meta": {}, @@ -544,6 +549,7 @@ function createTest(name: string, file: File): Test { result: { state: 'pass' }, meta: {}, context: {} as any, + benchmarks: [], } } diff --git a/test/e2e/test/reporters/utils.ts b/test/e2e/test/reporters/utils.ts index ef8a9a5253a6..fe8be7f42c1e 100644 --- a/test/e2e/test/reporters/utils.ts +++ b/test/e2e/test/reporters/utils.ts @@ -117,6 +117,7 @@ passedFile.tasks.push({ duration: 1.4422860145568848, }, context: null as any, + benchmarks: [], }) const error: TestError = { @@ -165,6 +166,7 @@ const tasks: RunnerTestCase[] = [ }, timeout: 0, context: null as any, + benchmarks: [], }, { id: `${suite.id}_1`, @@ -182,6 +184,7 @@ const tasks: RunnerTestCase[] = [ file, result: { state: 'pass', duration: 1.0237109661102295 }, context: null as any, + benchmarks: [], }, { id: `${suite.id}_3`, @@ -199,6 +202,7 @@ const tasks: RunnerTestCase[] = [ artifacts: [], result: undefined, context: null as any, + benchmarks: [], }, { id: `${suite.id}_4`, @@ -216,6 +220,7 @@ const tasks: RunnerTestCase[] = [ file, result: { state: 'pass', duration: 100.50598406791687 }, context: null as any, + benchmarks: [], }, { id: `${suite.id}_5`, @@ -233,6 +238,7 @@ const tasks: RunnerTestCase[] = [ file, result: { state: 'pass', duration: 20.184875011444092 }, context: null as any, + benchmarks: [], }, { id: `${suite.id}_6`, @@ -250,6 +256,7 @@ const tasks: RunnerTestCase[] = [ file, result: { state: 'pass', duration: 0.33245420455932617 }, context: null as any, + benchmarks: [], }, { id: `${suite.id}_7`, @@ -267,6 +274,7 @@ const tasks: RunnerTestCase[] = [ file, result: { state: 'pass', duration: 19.738605976104736 }, context: null as any, + benchmarks: [], }, { id: `${suite.id}_8`, @@ -284,6 +292,7 @@ const tasks: RunnerTestCase[] = [ file, result: { state: 'pass', duration: 0.1923508644104004 }, context: null as any, + benchmarks: [], logs: [ { content: 'error', @@ -309,6 +318,7 @@ const tasks: RunnerTestCase[] = [ file, result: undefined, context: null as any, + benchmarks: [], }, ] diff --git a/test/e2e/vitest.config.ts b/test/e2e/vitest.config.ts index b790877c48e6..5f94d0c8d8a8 100644 --- a/test/e2e/vitest.config.ts +++ b/test/e2e/vitest.config.ts @@ -45,8 +45,9 @@ export default defineConfig({ typecheck: { enabled: true, include: [ - './test/config-types.test-d.ts', './test/reporters/configuration-options.test-d.ts', + './test/benchmarking.test-d.ts', + './test/config-types.test-d.ts', ], }, sequence: { diff --git a/test/node-runner/test/cli.test.js b/test/node-runner/test/cli.test.js index bcf82a82a1d4..4926df9041fd 100644 --- a/test/node-runner/test/cli.test.js +++ b/test/node-runner/test/cli.test.js @@ -2,7 +2,7 @@ import test from 'node:test' import { startVitest } from 'vitest/node' await test('importing vitest in the global setup is reported as an error', async (t) => { - const vitest = await startVitest('test', [], { + const vitest = await startVitest([], { root: './fixtures/globalSetup', globalSetup: [ './failing.ts', diff --git a/test/test-utils/cli.ts b/test/test-utils/cli.ts index aa64df694750..eb12d3ec71f7 100644 --- a/test/test-utils/cli.ts +++ b/test/test-utils/cli.ts @@ -86,7 +86,7 @@ export class Cli { const timeoutId = setTimeout(() => { error.message = `Timeout when waiting for error "${expected}".\nReceived:\nstdout: ${this.stdout}\nstderr: ${this.stderr}` reject(error) - }, timeout ?? process.env.CI ? 20_000 : 4_000) + }, timeout ?? (process.env.CI ? 20_000 : 4_000)) const listener = () => { if (this[source].includes(expected)) { diff --git a/test/test-utils/index.ts b/test/test-utils/index.ts index 9be8365ae01a..fbb2927e82bc 100644 --- a/test/test-utils/index.ts +++ b/test/test-utils/index.ts @@ -33,7 +33,6 @@ export interface VitestRunnerCLIOptions { printExitCode?: boolean preserveAnsi?: boolean tty?: boolean - mode?: 'test' | 'benchmark' } export interface RunVitestConfig extends TestUserConfig { @@ -128,8 +127,6 @@ export async function runVitest( project, cliExclude, clearScreen, - compare, - outputJson, mergeReports, clearCache, // #endregion @@ -145,7 +142,7 @@ export async function runVitest( ;(viteConfig as any).test = rest try { - ctx = await startVitest(runnerOptions.mode || 'test', cliFilters, { + ctx = await startVitest(cliFilters, { root, config: configFile, standalone, @@ -157,8 +154,6 @@ export async function runVitest( project, cliExclude, clearScreen, - compare, - outputJson, mergeReports, clearCache, cache: 'cache' in config ? config.cache : false, @@ -172,6 +167,7 @@ export async function runVitest( ...cliOptions, env: { NO_COLOR: 'true', + FORCE_COLOR: undefined, AI_AGENT: '', ...rest.env, ...cliOptions?.env, diff --git a/test/ui/test/helper.ts b/test/ui/test/helper.ts index 3e31fd4fef51..3e5e43468534 100644 --- a/test/ui/test/helper.ts +++ b/test/ui/test/helper.ts @@ -12,7 +12,7 @@ import { startVitest } from 'vitest/node' export async function startVitestSimple(cliOptions: CliOptions): Promise { const stdout = new Writable({ write: (_, __, callback) => callback() }) const stderr = new Writable({ write: (_, __, callback) => callback() }) - const vitest = await startVitest('test', undefined, cliOptions, {}, { stdout, stderr }) + const vitest = await startVitest(undefined, cliOptions, {}, { stdout, stderr }) await vitest.close() return vitest } @@ -24,7 +24,7 @@ export async function startVitestUi( // silence Vitest logs const stdout = new Writable({ write: (_, __, callback) => callback() }) const stderr = new Writable({ write: (_, __, callback) => callback() }) - const vitest = await startVitest('test', undefined, cliOptions, viteOverrides, { stdout, stderr }) + const vitest = await startVitest(undefined, cliOptions, viteOverrides, { stdout, stderr }) const address = vitest.vite.httpServer?.address() assert(address && typeof address === 'object', 'Invalid server address') diff --git a/test/unit/package.json b/test/unit/package.json index 56836c58826c..a7d3841bb9d7 100644 --- a/test/unit/package.json +++ b/test/unit/package.json @@ -4,6 +4,7 @@ "private": true, "scripts": { "test": "vitest", + "bench": "vitest bench", "test:html": "vitest --reporter=html", "test:threads": "vitest --project threads", "test:forks": "vitest --project forks", diff --git a/test/unit/test/cli-test.test.ts b/test/unit/test/cli-test.test.ts index c53863f701ac..b45e89cc4a50 100644 --- a/test/unit/test/cli-test.test.ts +++ b/test/unit/test/cli-test.test.ts @@ -133,24 +133,6 @@ test('coverage autoUpdate accepts boolean values from CLI', async () => { expect(getCLIOptions('--coverage.thresholds.autoUpdate no').coverage.thresholds.autoUpdate).toBe(false) }) -test('bench only options', async () => { - expect(() => - parseArguments('--compare file.json').matchedCommand?.checkUnknownOptions(), - ).toThrowErrorMatchingInlineSnapshot( - `[CACError: Unknown option \`--compare\`]`, - ) - - expect(() => - parseArguments( - 'bench --compare file.json', - ).matchedCommand?.checkUnknownOptions(), - ).not.toThrow() - - expect(parseArguments('bench --compare file.json').options).toEqual({ - compare: 'file.json', - }) -}) - test('even if coverage is boolean, don\'t fail', () => { expect(getCLIOptions('--coverage --coverage.provider v8').coverage).toEqual({ enabled: true, diff --git a/test/unit/test/exports.test.ts b/test/unit/test/exports.test.ts index ed1949bd9840..2c8bf7dd0012 100644 --- a/test/unit/test/exports.test.ts +++ b/test/unit/test/exports.test.ts @@ -17,7 +17,6 @@ it('exports snapshot', async ({ skip, task }) => { expect(manifest.exports).toMatchInlineSnapshot(` { ".": { - "BenchmarkRunner": "function", "EvaluatedModules": "function", "Snapshots": "object", "TestRunner": "function", @@ -29,7 +28,6 @@ it('exports snapshot', async ({ skip, task }) => { "assertType": "function", "beforeAll": "function", "beforeEach": "function", - "bench": "function", "chai": "object", "createExpect": "function", "describe": "function", @@ -84,8 +82,6 @@ it('exports snapshot', async ({ skip, task }) => { "AgentReporter": "function", "BaseCoverageProvider": "function", "BaseSequencer": "function", - "BenchmarkReporter": "function", - "BenchmarkReportsMap": "object", "DefaultReporter": "function", "DotReporter": "function", "ForksPoolWorker": "function", @@ -101,7 +97,6 @@ it('exports snapshot', async ({ skip, task }) => { "TestsNotFoundError": "function", "ThreadsPoolWorker": "function", "TypecheckPoolWorker": "function", - "VerboseBenchmarkReporter": "function", "VerboseReporter": "function", "VitestPackageInstaller": "function", "VitestPlugin": "function", diff --git a/test/unit/vitest-environment-custom/index.ts b/test/unit/vitest-environment-custom/index.ts index 72694817b8b7..c81f6b704d11 100644 --- a/test/unit/vitest-environment-custom/index.ts +++ b/test/unit/vitest-environment-custom/index.ts @@ -21,6 +21,7 @@ export default { Event, TextDecoder, TextEncoder, + performance, }) return { getVmContext() { From aab60c0be40f4d1ac2d8e10ae9f2a9c11546581a Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Fri, 29 May 2026 22:10:20 +0900 Subject: [PATCH 2/2] chore(deps): update playwright 1.60.0 (#10426) Co-authored-by: Codex --- examples/lit/package.json | 2 +- pnpm-lock.yaml | 42 +++++++-------- pnpm-workspace.yaml | 4 +- test/browser/docker-compose.yaml | 8 +-- test/browser/specs/playwright-connect.test.ts | 51 +++++++++++-------- 5 files changed, 59 insertions(+), 48 deletions(-) diff --git a/examples/lit/package.json b/examples/lit/package.json index 3641127b636b..431d0e4b6926 100644 --- a/examples/lit/package.json +++ b/examples/lit/package.json @@ -19,7 +19,7 @@ "devDependencies": { "@vitest/browser-playwright": "latest", "jsdom": "latest", - "playwright": "^1.59.0", + "playwright": "^1.60.0", "vite": "latest", "vitest": "latest" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 387f9bb69aaa..a1726009c047 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,8 +28,8 @@ catalogs: specifier: 0.3.31 version: 0.3.31 '@playwright/test': - specifier: ^1.59.0 - version: 1.59.0 + specifier: ^1.60.0 + version: 1.60.0 '@rolldown/plugin-babel': specifier: ^0.2.1 version: 0.2.1 @@ -115,8 +115,8 @@ catalogs: specifier: ^2.0.3 version: 2.0.3 playwright: - specifier: ^1.59.0 - version: 1.59.0 + specifier: ^1.60.0 + version: 1.60.0 sinon: specifier: ^21.0.3 version: 21.0.3 @@ -202,7 +202,7 @@ importers: version: 28.3.0 '@playwright/test': specifier: 'catalog:' - version: 1.59.0 + version: 1.60.0 '@rollup/plugin-commonjs': specifier: ^29.0.2 version: 29.0.2(rollup@4.59.0) @@ -410,8 +410,8 @@ importers: specifier: latest version: 29.1.1(@noble/hashes@1.8.0) playwright: - specifier: ^1.59.0 - version: 1.59.0 + specifier: ^1.60.0 + version: 1.60.0 vite: specifier: 8.0.11 version: 8.0.11(@types/node@24.12.0)(esbuild@0.27.3)(jiti@2.6.1)(sass-embedded@1.97.3)(sass@1.97.3)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) @@ -599,7 +599,7 @@ importers: devDependencies: playwright: specifier: 'catalog:' - version: 1.59.0 + version: 1.60.0 vitest: specifier: workspace:* version: link:../vitest @@ -1231,7 +1231,7 @@ importers: version: link:cjs-lib playwright: specifier: 'catalog:' - version: 1.59.0 + version: 1.60.0 react: specifier: ^19.2.4 version: 19.2.4 @@ -1405,7 +1405,7 @@ importers: version: file:test/e2e/deps/vite-ssr-resolve/other-dep playwright: specifier: 'catalog:' - version: 1.59.0 + version: 1.60.0 ssr-no-external-dep: specifier: file:./deps/vite-ssr-resolve/ssr-no-external-dep version: file:test/e2e/deps/vite-ssr-resolve/ssr-no-external-dep @@ -4366,8 +4366,8 @@ packages: resolution: {integrity: sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} - '@playwright/test@1.59.0': - resolution: {integrity: sha512-TOA5sTLd49rTDaZpYpvCQ9hGefHQq/OYOyCVnGqS2mjMfX+lGZv2iddIJd0I48cfxqSPttS9S3OuLKyylHcO1w==} + '@playwright/test@1.60.0': + resolution: {integrity: sha512-O71yZIbAh/PxDMNGns37GHBIfrVkEVyn+AXyIa5dOTfb4/xNvRWV+Vv/NMbNCtODB/pO7vLlF2OTmMVLhmr7Ag==} engines: {node: '>=18'} hasBin: true @@ -8727,13 +8727,13 @@ packages: pkg-types@2.3.0: resolution: {integrity: sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==} - playwright-core@1.59.0: - resolution: {integrity: sha512-PW/X/IoZ6BMUUy8rpwHEZ8Kc0IiLIkgKYGNFaMs5KmQhcfLILNx9yCQD0rnWeWfz1PNeqcFP1BsihQhDOBCwZw==} + playwright-core@1.60.0: + resolution: {integrity: sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==} engines: {node: '>=18'} hasBin: true - playwright@1.59.0: - resolution: {integrity: sha512-wihGScriusvATUxmhfENxg0tj1vHEFeIwxlnPFKQTOQVd7aG08mUfvvniRP/PtQOC+2Bs52kBOC/Up1jTXeIbw==} + playwright@1.60.0: + resolution: {integrity: sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA==} engines: {node: '>=18'} hasBin: true @@ -12863,9 +12863,9 @@ snapshots: '@pkgr/core@0.2.9': {} - '@playwright/test@1.59.0': + '@playwright/test@1.60.0': dependencies: - playwright: 1.59.0 + playwright: 1.60.0 '@polka/url@1.0.0-next.24': {} @@ -17888,11 +17888,11 @@ snapshots: exsolve: 1.0.7 pathe: 2.0.3 - playwright-core@1.59.0: {} + playwright-core@1.60.0: {} - playwright@1.59.0: + playwright@1.60.0: dependencies: - playwright-core: 1.59.0 + playwright-core: 1.60.0 optionalDependencies: fsevents: 2.3.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index d99024ec33af..7964e4a16c4b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -53,7 +53,7 @@ catalog: '@iconify/vue': ^5.0.0 '@jridgewell/remapping': ^2.3.5 '@jridgewell/trace-mapping': 0.3.31 - '@playwright/test': ^1.59.0 + '@playwright/test': ^1.60.0 '@rolldown/plugin-babel': ^0.2.1 '@types/chai': ^5.2.2 '@types/estree': ^1.0.8 @@ -83,7 +83,7 @@ catalog: msw: ^2.12.10 obug: ^2.1.1 pathe: ^2.0.3 - playwright: ^1.59.0 + playwright: ^1.60.0 sinon: ^21.0.3 sinon-chai: ^4.0.1 sirv: ^3.0.2 diff --git a/test/browser/docker-compose.yaml b/test/browser/docker-compose.yaml index f10812153ecc..7ae4a2c49f1a 100644 --- a/test/browser/docker-compose.yaml +++ b/test/browser/docker-compose.yaml @@ -1,7 +1,7 @@ services: playwright: - image: mcr.microsoft.com/playwright:v1.59.0-noble - command: /bin/sh -c "npx -y playwright@1.59.0 run-server --port 6677 --host 0.0.0.0" + image: mcr.microsoft.com/playwright:v1.60.0-noble + command: /bin/sh -c "npx -y playwright@1.60.0 run-server --port 6677 --host 0.0.0.0" init: true ipc: host user: pwuser @@ -15,8 +15,8 @@ services: # Run it by: # pnpm run docker up playwright-host playwright-host: - image: mcr.microsoft.com/playwright:v1.59.0-noble - command: /bin/sh -c "npx -y playwright@1.59.0 run-server --port 6677" + image: mcr.microsoft.com/playwright:v1.60.0-noble + command: /bin/sh -c "npx -y playwright@1.60.0 run-server --port 6677" init: true ipc: host user: pwuser diff --git a/test/browser/specs/playwright-connect.test.ts b/test/browser/specs/playwright-connect.test.ts index 95c3ce4fca2a..13a80d7517c5 100644 --- a/test/browser/specs/playwright-connect.test.ts +++ b/test/browser/specs/playwright-connect.test.ts @@ -8,7 +8,15 @@ import { runBrowserTests } from './utils' test.runIf(provider.name === 'playwright')('[playwright] runs in connect mode', async ({ onTestFinished }) => { const cliPath = fileURLToPath(new URL('./cli.js', import.meta.resolve('@playwright/test'))) - const subprocess = x(process.execPath, [cliPath, 'run-server', '--port', '9898']).process + const subprocess = x(process.execPath, [ + cliPath, + 'run-server', + '--port', + '9898', + '--host', + '127.0.0.1', + '--unsafe', + ]).process const cli = new Cli({ stdin: subprocess.stdin, stdout: subprocess.stdout, @@ -22,27 +30,30 @@ test.runIf(provider.name === 'playwright')('[playwright] runs in connect mode', await isDone }) - await cli.waitForStdout('Listening on ws://localhost:9898') + await cli.waitForStdout('Listening on ws://127.0.0.1:9898') - const result = await runBrowserTests({ - root: './fixtures/playwright-connect', - browser: { - instances: [ - { - browser: 'chromium', - name: 'chromium', - provider: playwright({ - connectOptions: { - wsEndpoint: 'ws://localhost:9898', - }, - launchOptions: { - args: [`--user-agent=VitestLaunchOptionsTester`], - }, - }), - }, - ], + const result = await runBrowserTests( + { + root: './fixtures/playwright-connect', + browser: { + instances: [ + { + browser: 'chromium', + name: 'chromium', + provider: playwright({ + connectOptions: { + wsEndpoint: 'ws://127.0.0.1:9898', + }, + launchOptions: { + args: [`--user-agent=VitestLaunchOptionsTester`], + }, + }), + }, + ], + }, }, - }, ['basic.test.js']) + ['basic.test.js'], + ) expect(result.stderr).toMatchInlineSnapshot(`""`) expect(result.errorTree()).toMatchInlineSnapshot(`