[pull] main from vitest-dev:main

.gitignore

@@ -26,6 +26,7 @@ docs/.vitepress/cache/
 !test/e2e/fixtures/dotted-files/**/.cache
 test/**/__screenshots__/**/*
 test/**/__traces__/**/*
+test/browser/**/__benchmarks__
 test/browser/fixtures/update-snapshot/basic.test.ts
 test/e2e/fixtures/browser-multiple/basic-*
 *.tsbuildinfo

.vscode/settings.json

@@ -23,7 +23,7 @@
   // ],
 
   "vitest.ignoreWorkspace": true,
-  "vitest.configSearchPatternInclude": "test/{core,cli,config,browser,reporters}/{vitest,vite}.{config.ts,config.unit.mts}",
+  "vitest.configSearchPatternInclude": "test/{unit,e2e,config,browser,reporters}/{vitest,vite}.{config.ts,config.unit.mts}",
   "testing.automaticallyOpenTestResults": "neverOpen",
 
   // Enable eslint for all supported languages

docs/.vitepress/config.ts

@@ -901,6 +901,10 @@ export default ({ mode }: { mode: string }) => {
                 text: 'Testing Types',
                 link: '/guide/testing-types',
               },
+              {
+                text: 'Benchmarking',
+                link: '/guide/benchmarking',
+              },
               {
                 text: 'In-Source Testing',
                 link: '/guide/in-source',

docs/api/advanced/vitest.md

@@ -5,35 +5,9 @@ title: Vitest API
 
 # Vitest
 
-Vitest instance requires the current test mode. It can be either:
-
-- `test` when running runtime tests
-- `benchmark` when running benchmarks <Experimental />
-
-::: details New in Vitest 4
-Vitest 4 added several new APIs (they are marked with a "4.0.0+" badge) and removed deprecated APIs:
-
-- `invalidates`
-- `changedTests` (use [`onFilterWatchedSpecification`](#onfilterwatchedspecification) instead)
-- `server` (use [`vite`](#vite) instead)
-- `getProjectsByTestFile` (use [`getModuleSpecifications`](#getmodulespecifications) instead)
-- `getFileWorkspaceSpecs` (use [`getModuleSpecifications`](#getmodulespecifications) instead)
-- `getModuleProjects` (filter by [`this.projects`](#projects) yourself)
-- `updateLastChanged` (renamed to [`invalidateFile`](#invalidatefile))
-- `globTestSpecs` (use [`globTestSpecifications`](#globtestspecifications) instead)
-- `globTestFiles` (use [`globTestSpecifications`](#globtestspecifications) instead)
-- `listFile` (use [`getRelevantTestSpecifications`](#getrelevanttestspecifications) instead)
-:::
-
-## mode
-
-### test
-
-Test mode will only call functions inside `test` or `it`, and throws an error when `bench` is encountered. This mode uses `include` and `exclude` options in the config to find test files.
-
-### benchmark <Experimental /> {#benchmark}
+## mode <Deprecated /> {#mode}
 
-Benchmark mode calls `bench` functions and throws an error, when it encounters `test` or `it`. This mode uses `benchmark.include` and `benchmark.exclude` options in the config to find benchmark files.
+Since Vitest 5, this property is always `'test'`.
 
 ## config
 

docs/api/test.md

@@ -670,235 +670,8 @@ Scoped `aroundAll` hook that inherits types from [`test.extend`](#test-extend).
 
 ## bench <Experimental /> {#bench}
 
-- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void`
+::: warning Updated in Vitest 5
+The benchmarking API has been rewritten. `bench` is no longer a top-level import from `vitest`, and the `bench.skip` / `bench.only` / `bench.todo` helpers have been removed. `bench` is now a [test-context fixture](/guide/test-context#bench) accessed from inside a `test()`.
 
-::: danger
-Benchmarking is experimental and does not follow SemVer.
+See the [Benchmarking guide](/guide/benchmarking) for the new API.
 :::
-
-`bench` defines a benchmark. In Vitest terms, benchmark is a function that defines a series of operations. Vitest runs this function multiple times to display different performance results.
-
-Vitest uses the [`tinybench`](https://github.com/tinylibs/tinybench) library under the hood, inheriting all its options that can be used as a third argument.
-
-```ts
-import { bench } from 'vitest'
-
-bench('normal sorting', () => {
-  const x = [1, 5, 4, 2, 3]
-  x.sort((a, b) => {
-    return a - b
-  })
-}, { time: 1000 })
-```
-
-```ts
-export interface Options {
-  /**
-   * time needed for running a benchmark task (milliseconds)
-   * @default 500
-   */
-  time?: number
-
-  /**
-   * number of times that a task should run if even the time option is finished
-   * @default 10
-   */
-  iterations?: number
-
-  /**
-   * function to get the current timestamp in milliseconds
-   */
-  now?: () => number
-
-  /**
-   * An AbortSignal for aborting the benchmark
-   */
-  signal?: AbortSignal
-
-  /**
-   * Throw if a task fails (events will not work if true)
-   */
-  throws?: boolean
-
-  /**
-   * warmup time (milliseconds)
-   * @default 100ms
-   */
-  warmupTime?: number
-
-  /**
-   * warmup iterations
-   * @default 5
-   */
-  warmupIterations?: number
-
-  /**
-   * setup function to run before each benchmark task (cycle)
-   */
-  setup?: Hook
-
-  /**
-   * teardown function to run after each benchmark task (cycle)
-   */
-  teardown?: Hook
-}
-```
-After the test case is run, the output structure information is as follows:
-
-```
-  name                      hz     min     max    mean     p75     p99    p995    p999     rme  samples
-· normal sorting  6,526,368.12  0.0001  0.3638  0.0002  0.0002  0.0002  0.0002  0.0004  ±1.41%   652638
-```
-```ts
-export interface TaskResult {
-  /*
-   * the last error that was thrown while running the task
-   */
-  error?: unknown
-
-  /**
-   * The amount of time in milliseconds to run the benchmark task (cycle).
-   */
-  totalTime: number
-
-  /**
-   * the minimum value in the samples
-   */
-  min: number
-  /**
-   * the maximum value in the samples
-   */
-  max: number
-
-  /**
-   * the number of operations per second
-   */
-  hz: number
-
-  /**
-   * how long each operation takes (ms)
-   */
-  period: number
-
-  /**
-   * task samples of each task iteration time (ms)
-   */
-  samples: number[]
-
-  /**
-   * samples mean/average (estimate of the population mean)
-   */
-  mean: number
-
-  /**
-   * samples variance (estimate of the population variance)
-   */
-  variance: number
-
-  /**
-   * samples standard deviation (estimate of the population standard deviation)
-   */
-  sd: number
-
-  /**
-   * standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean)
-   */
-  sem: number
-
-  /**
-   * degrees of freedom
-   */
-  df: number
-
-  /**
-   * critical value of the samples
-   */
-  critical: number
-
-  /**
-   * margin of error
-   */
-  moe: number
-
-  /**
-   * relative margin of error
-   */
-  rme: number
-
-  /**
-   * median absolute deviation
-   */
-  mad: number
-
-  /**
-   * p50/median percentile
-   */
-  p50: number
-
-  /**
-   * p75 percentile
-   */
-  p75: number
-
-  /**
-   * p99 percentile
-   */
-  p99: number
-
-  /**
-   * p995 percentile
-   */
-  p995: number
-
-  /**
-   * p999 percentile
-   */
-  p999: number
-}
-```
-
-### bench.skip
-
-- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void`
-
-You can use `bench.skip` syntax to skip running certain benchmarks.
-
-```ts
-import { bench } from 'vitest'
-
-bench.skip('normal sorting', () => {
-  const x = [1, 5, 4, 2, 3]
-  x.sort((a, b) => {
-    return a - b
-  })
-})
-```
-
-### bench.only
-
-- **Type:** `(name: string | Function, fn: BenchFunction, options?: BenchOptions) => void`
-
-Use `bench.only` to only run certain benchmarks in a given suite. This is useful when debugging.
-
-```ts
-import { bench } from 'vitest'
-
-bench.only('normal sorting', () => {
-  const x = [1, 5, 4, 2, 3]
-  x.sort((a, b) => {
-    return a - b
-  })
-})
-```
-
-### bench.todo
-
-- **Type:** `(name: string | Function) => void`
-
-Use `bench.todo` to stub benchmarks to be implemented later.
-
-```ts
-import { bench } from 'vitest'
-
-bench.todo('unimplemented test')
-```

docs/config/benchmark.md

@@ -9,6 +9,13 @@ outline: deep
 
 Options used when running `vitest bench`.
 
+## benchmark.enabled
+
+- **Type:** `boolean`
+- **Default:** `false`
+
+Enables the benchmark project. When set, Vitest creates a dedicated benchmark project alongside your regular test project, runs files matching [`benchmark.include`](#benchmark-include) in it, and exposes the [`bench` fixture](/guide/test-context#bench) to those files. Running `vitest bench` enables this automatically.
+
 ## benchmark.include
 
 - **Type:** `string[]`
@@ -32,39 +39,18 @@ Include globs for in-source benchmark test files. This option is similar to [`in
 
 When defined, Vitest will run all matched files with `import.meta.vitest` inside.
 
-## benchmark.reporters
-
-- **Type:** `Arrayable<BenchmarkBuiltinReporters | Reporter>`
-- **Default:** `'default'`
-
-Custom reporter for output. Can contain one or more built-in report names, reporter instances, and/or paths to custom reporters.
-
-## benchmark.outputFile
-
-Deprecated in favor of `benchmark.outputJson`.
-
-## benchmark.outputJson {#benchmark-outputJson}
-
-- **Type:** `string | undefined`
-- **Default:** `undefined`
+## benchmark.retainSamples
 
-A file path to store the benchmark result, which can be used for `--compare` option later.
+- **Type:** `boolean`
+- **Default:** `false`
 
-For example:
+Include the `samples` array of per-iteration timings on every benchmark result. Disabled by default to reduce memory usage; enable when a custom reporter or API consumer needs the raw samples.
 
-```sh
-# save main branch's result
-git checkout main
-vitest bench --outputJson main.json
 
-# change a branch and compare against main
-git checkout feature
-vitest bench --compare main.json
-```
+## benchmark.suppressExportGetterWarnings
 
-## benchmark.compare {#benchmark-compare}
+- **Type:** `boolean`
+- **Default:** `false`
 
-- **Type:** `string | undefined`
-- **Default:** `undefined`
+Suppress the warning printed when a benchmark accesses module export getters too many times. Vitest tracks getter access during benchmark runs because Vite's module runner wraps every export in a getter, and excessive access can dominate the measurement (see [Module Runner Overhead](/guide/benchmarking#module-runner-overhead)). Enable this when you've intentionally accepted the overhead, or when the warning is noisy for benchmarks where the getter cost is negligible.
 
-A file path to a previous benchmark result to compare against current runs.

docs/config/index.md

@@ -8,7 +8,7 @@ If you are using Vite and have a `vite.config` file, Vitest will read it to matc
 
 - Create `vitest.config.ts`, which will have the higher priority and will **override** the configuration from `vite.config.ts` (Vitest supports all conventional JS and TS extensions, but doesn't support `json`) - it means all options in your `vite.config` will be **ignored**
 - Pass `--config` option to CLI, e.g. `vitest --config ./path/to/vitest.config.ts`
-- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test`/`benchmark` if not overridden with `--mode`) to conditionally apply different configuration in `vite.config.ts`. Note that like any other environment variable, `VITEST` is also exposed on `import.meta.env` in your tests
+- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test` if not overridden with `--mode`) to conditionally apply different configuration in `vite.config.ts`. Note that like any other environment variable, `VITEST` is also exposed on `import.meta.env` in your tests
 
 To configure `vitest` itself, add `test` property in your Vite config. You'll also need to add a reference to Vitest types using a [triple slash command](https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html#-reference-types-) at the top of your config file, if you are importing `defineConfig` from `vite` itself.
 

docs/config/runner.md

@@ -6,6 +6,5 @@ outline: deep
 # runner
 
 - **Type:** `VitestRunnerConstructor`
-- **Default:** `node`, when running tests, or `benchmark`, when running benchmarks
 
 Path to a custom test runner. This is an advanced feature and should be used with custom library runners. You can read more about it in [the documentation](/api/advanced/runner).