Add perf profiling to tinybench

.gitignore

@@ -7,6 +7,9 @@ yarn-error.log*
 lerna-debug.log*
 .pnpm-debug.log*
 
+# JIT dumps
+jit-*.dump
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 

packages/core/binding.gyp

@@ -9,6 +9,7 @@
         "-fno-exceptions"
       ],
       "cflags": [
+        "-g",
         "-Wno-maybe-uninitialized",
         "-Wno-unused-variable",
         "-Wno-unused-parameter",

packages/core/src/native_core/index.ts

@@ -40,12 +40,17 @@ try {
       stopBenchmark: () => {
         return 0;
       },
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
       setExecutedBenchmark: (_pid: number, _uri: string) => {
         return 0;
       },
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
       setIntegration: (_name: string, _version: string) => {
         return 0;
       },
+      __codspeed_root_frame__: <T>(callback: () => T): T => {
+        return callback();
+      },
     },
     isBound: false,
   };

packages/core/src/native_core/instruments/hooks.ts

@@ -31,4 +31,10 @@ export interface InstrumentHooks {
    * @returns 0 on success, non-zero on error
    */
   setIntegration(name: string, version: string): number;
+
+  /**
+   * Execute a callback function with __codspeed_root_frame__ in its stack trace
+   * @param callback Function to execute
+   */
+  __codspeed_root_frame__<T>(callback: () => T): T;
 }

packages/core/src/native_core/instruments/hooks_wrapper.cc

@@ -81,6 +81,27 @@ Napi::Number SetIntegration(const Napi::CallbackInfo &info) {
   return Napi::Number::New(env, result);
 }
 
+Napi::Value __attribute__ ((noinline)) __codspeed_root_frame__(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+
+  if (info.Length() != 1) {
+    Napi::TypeError::New(env, "Expected 1 argument: callback function")
+        .ThrowAsJavaScriptException();
+    return env.Undefined();
+  }
+
+  if (!info[0].IsFunction()) {
+    Napi::TypeError::New(env, "Expected function argument")
+        .ThrowAsJavaScriptException();
+    return env.Undefined();
+  }
+
+  Napi::Function callback = info[0].As<Napi::Function>();
+  Napi::Value result = callback.Call(env.Global(), {});
+
+  return result;
+}
+
 Napi::Object Initialize(Napi::Env env, Napi::Object exports) {
   Napi::Object instrumentHooksObj = Napi::Object::New(env);
 
@@ -96,6 +117,8 @@ Napi::Object Initialize(Napi::Env env, Napi::Object exports) {
                          Napi::Function::New(env, SetExecutedBenchmark));
   instrumentHooksObj.Set(Napi::String::New(env, "setIntegration"),
                          Napi::Function::New(env, SetIntegration));
+  instrumentHooksObj.Set(Napi::String::New(env, "__codspeed_root_frame__"),
+                         Napi::Function::New(env, __codspeed_root_frame__));
 
   exports.Set(Napi::String::New(env, "InstrumentHooks"), instrumentHooksObj);
 

packages/core/src/native_core/linux_perf/linux_perf.cc

@@ -39,4 +39,4 @@ Napi::Value LinuxPerf::Stop(const Napi::CallbackInfo &info) {
   return Napi::Boolean::New(info.Env(), false);
 }
 
-} // namespace codspeed_native
+} // namespace codspeed_native

packages/core/src/walltime/index.ts

@@ -8,19 +8,34 @@ export function getProfileFolder(): string | null {
   return process.env.CODSPEED_PROFILE_FOLDER || null;
 }
 
-export function writeWalltimeResults(benchmarks: Benchmark[]) {
+export function writeWalltimeResults(
+  benchmarks: Benchmark[],
+  asyncWarning = false
+): void {
   const profileFolder = getProfileFolder();
-  let resultPath: string;
-
-  if (profileFolder) {
-    const resultsDir = path.join(profileFolder, "results");
-    fs.mkdirSync(resultsDir, { recursive: true });
-    resultPath = path.join(resultsDir, `${process.pid}.json`);
-  } else {
-    // Fallback: write to .codspeed in current working directory
-    const codspeedDir = path.join(process.cwd(), ".codspeed");
-    fs.mkdirSync(codspeedDir, { recursive: true });
-    resultPath = path.join(codspeedDir, `results_${Date.now()}.json`);
+
+  const resultDir = (() => {
+    if (profileFolder) {
+      return path.join(profileFolder, "results");
+    } else {
+      // Fallback: write to .codspeed in current working directory
+      return path.join(process.cwd(), ".codspeed");
+    }
+  })();
+  fs.mkdirSync(resultDir, { recursive: true });
+  const resultPath = path.join(resultDir, `${process.pid}.json`);
+
+  // Check if file already exists and merge benchmarks
+  let existingBenchmarks: Benchmark[] = [];
+  if (fs.existsSync(resultPath)) {
+    try {
+      const existingData = JSON.parse(
+        fs.readFileSync(resultPath, "utf-8")
+      ) as ResultData;
+      existingBenchmarks = existingData.benchmarks || [];
+    } catch (error) {
+      console.warn(`[CodSpeed] Failed to read existing results file: ${error}`);
+    }
   }
 
   const data: ResultData = {
@@ -30,11 +45,18 @@ export function writeWalltimeResults(benchmarks: Benchmark[]) {
       pid: process.pid,
     },
     instrument: { type: "walltime" },
-    benchmarks: benchmarks,
+    benchmarks: [...existingBenchmarks, ...benchmarks],
+    metadata: asyncWarning
+      ? {
+          async_warning: "Profiling is inaccurate due to async operations",
+        }
+      : undefined,
   };
 
   fs.writeFileSync(resultPath, JSON.stringify(data, null, 2));
-  console.log(`[CodSpeed] Results written to ${resultPath}`);
+  console.log(
+    `[CodSpeed] Results written to ${resultPath} (${data.benchmarks.length} total benchmarks)`
+  );
 }
 
 export * from "./interfaces";

packages/core/src/walltime/interfaces.ts

@@ -40,4 +40,5 @@ export interface ResultData {
   };
   instrument: { type: "walltime" };
   benchmarks: Benchmark[];
+  metadata?: Record<string, unknown>;
 }

packages/tinybench-plugin/benches/sample.ts

@@ -35,7 +35,7 @@ bench
   });
 
 (async () => {
-  await bench.run();
+  bench.runSync();
   console.table(bench.table());
 
   const timingBench = withCodSpeed(
@@ -44,6 +44,6 @@ bench
 
   registerTimingBenchmarks(timingBench);
 
-  await timingBench.run();
+  timingBench.runSync();
   console.table(timingBench.table());
 })();

packages/tinybench-plugin/benches/timing.ts

@@ -8,15 +8,15 @@ const busySleep = (ms: number): void => {
 };
 
 export function registerTimingBenchmarks(bench: Bench) {
-  bench.add("wait 1ms", async () => {
+  bench.add("wait 1ms", () => {
     busySleep(1);
   });
 
-  bench.add("wait 500ms", async () => {
+  bench.add("wait 500ms", () => {
     busySleep(500);
   });
 
-  bench.add("wait 1sec", async () => {
+  bench.add("wait 1sec", () => {
     busySleep(1000);
   });
 }

packages/tinybench-plugin/src/index.ts

@@ -11,9 +11,9 @@ import path from "path";
 import { get as getStackTrace } from "stack-trace";
 import { Bench } from "tinybench";
 import { fileURLToPath } from "url";
-import { runInstrumentedBench } from "./instrumented";
+import { setupCodspeedInstrumentedBench } from "./instrumented";
 import { getOrCreateUriMap } from "./uri";
-import { runWalltimeBench } from "./walltime";
+import { setupCodspeedWalltimeBench } from "./walltime";
 
 tryIntrospect();
 
@@ -40,9 +40,9 @@ export function withCodSpeed(bench: Bench): Bench {
   };
 
   if (codspeedRunnerMode === "instrumented") {
-    runInstrumentedBench(bench, rootCallingFile);
+    setupCodspeedInstrumentedBench(bench, rootCallingFile);
   } else if (codspeedRunnerMode === "walltime") {
-    runWalltimeBench(bench, rootCallingFile);
+    setupCodspeedWalltimeBench(bench, rootCallingFile);
   }
 
   return bench;

packages/tinybench-plugin/src/index.unit.test.ts

@@ -3,15 +3,15 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
 import { withCodSpeed } from ".";
 
 const mockInstrumented = vi.hoisted(() => ({
-  runInstrumentedBench: vi.fn(),
+  setupCodspeedInstrumentedBench: vi.fn(),
 }));
 
 vi.mock("./instrumented", () => ({
   ...mockInstrumented,
 }));
 
 const mockWalltime = vi.hoisted(() => ({
-  runWalltimeBench: vi.fn(),
+  setupCodspeedWalltimeBench: vi.fn(),
 }));
 
 vi.mock("./walltime", () => ({
@@ -44,8 +44,8 @@ describe("withCodSpeed behavior without different codspeed modes", () => {
 
     withCodSpeed(new Bench());
 
-    expect(mockInstrumented.runInstrumentedBench).toHaveBeenCalled();
-    expect(mockWalltime.runWalltimeBench).not.toHaveBeenCalled();
+    expect(mockInstrumented.setupCodspeedInstrumentedBench).toHaveBeenCalled();
+    expect(mockWalltime.setupCodspeedWalltimeBench).not.toHaveBeenCalled();
   });
 
   it("should run in walltime mode when CODSPEED_RUNNER_MODE=walltime", async () => {
@@ -54,7 +54,9 @@ describe("withCodSpeed behavior without different codspeed modes", () => {
 
     withCodSpeed(new Bench());
 
-    expect(mockInstrumented.runInstrumentedBench).not.toHaveBeenCalled();
-    expect(mockWalltime.runWalltimeBench).toHaveBeenCalled();
+    expect(
+      mockInstrumented.setupCodspeedInstrumentedBench
+    ).not.toHaveBeenCalled();
+    expect(mockWalltime.setupCodspeedWalltimeBench).toHaveBeenCalled();
   });
 });

packages/tinybench-plugin/src/instrumented.ts

@@ -2,67 +2,83 @@ import {
   InstrumentHooks,
   mongoMeasurement,
   optimizeFunction,
-  setupCore,
-  teardownCore,
 } from "@codspeed/core";
-import { Bench, Fn, FnOptions } from "tinybench";
-import { getTaskUri } from "./uri";
+import { Bench, Fn, FnOptions, Task } from "tinybench";
+import { BaseBenchRunner } from "./shared";
 
-declare const __VERSION__: string;
-
-export function runInstrumentedBench(
+export function setupCodspeedInstrumentedBench(
   bench: Bench,
   rootCallingFile: string
 ): void {
-  bench.run = async () => {
-    console.log(
-      `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)`
-    );
-    setupCore();
-
-    for (const task of bench.tasks) {
-      const uri = getTaskUri(bench, task.name, rootCallingFile);
+  const runner = new InstrumentedBenchRunner(bench, rootCallingFile);
+  runner.setupBenchMethods();
+}
 
-      // Access private fields
-      const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn };
+class InstrumentedBenchRunner extends BaseBenchRunner {
+  protected getModeName(): string {
+    return "instrumented mode";
+  }
 
-      // Call beforeAll hook if it exists
-      await fnOpts?.beforeAll?.call(task, "run");
+  private taskCompletionMessage() {
+    return InstrumentHooks.isInstrumented() ? "Measured" : "Checked";
+  }
 
-      // run optimizations
-      await optimizeFunction(async () => {
-        await fnOpts?.beforeEach?.call(task, "run");
+  private wrapFunctionWithFrame(fn: Fn, isAsync: boolean): Fn {
+    if (isAsync) {
+      return async function __codspeed_root_frame__() {
         await fn();
-        await fnOpts?.afterEach?.call(task, "run");
-      });
+      };
+    } else {
+      return function __codspeed_root_frame__() {
+        fn();
+      };
+    }
+  }
+
+  protected async runTaskAsync(task: Task, uri: string): Promise<void> {
+    const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn };
 
-      // run instrumented benchmark
+    await fnOpts?.beforeAll?.call(task, "run");
+    await optimizeFunction(async () => {
       await fnOpts?.beforeEach?.call(task, "run");
+      await fn();
+      await fnOpts?.afterEach?.call(task, "run");
+    });
+    await fnOpts?.beforeEach?.call(task, "run");
+    await mongoMeasurement.start(uri);
 
-      await mongoMeasurement.start(uri);
-      global.gc?.();
-      await (async function __codspeed_root_frame__() {
-        InstrumentHooks.startBenchmark();
-        await fn();
-        InstrumentHooks.stopBenchmark();
-        InstrumentHooks.setExecutedBenchmark(process.pid, uri);
-      })();
-      await mongoMeasurement.stop(uri);
+    global.gc?.();
+    await this.wrapWithInstrumentHooksAsync(
+      this.wrapFunctionWithFrame(fn, true),
+      uri
+    );
 
-      await fnOpts?.afterEach?.call(task, "run");
+    await mongoMeasurement.stop(uri);
+    await fnOpts?.afterEach?.call(task, "run");
+    await fnOpts?.afterAll?.call(task, "run");
 
-      await fnOpts?.afterAll?.call(task, "run");
+    this.logTaskCompletion(uri, this.taskCompletionMessage());
+  }
 
-      // print results
-      console.log(
-        `    ✔ ${
-          InstrumentHooks.isInstrumented() ? "Measured" : "Checked"
-        } ${uri}`
-      );
-    }
+  protected runTaskSync(task: Task, uri: string): void {
+    const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn };
+
+    fnOpts?.beforeAll?.call(task, "run");
+    fnOpts?.beforeEach?.call(task, "run");
+
+    this.wrapWithInstrumentHooks(this.wrapFunctionWithFrame(fn, false), uri);
+
+    fnOpts?.afterEach?.call(task, "run");
+    fnOpts?.afterAll?.call(task, "run");
+
+    this.logTaskCompletion(uri, this.taskCompletionMessage());
+  }
+
+  protected finalizeAsyncRun(): Task[] {
+    return this.finalizeBenchRun();
+  }
 
-    teardownCore();
-    console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`);
-    return bench.tasks;
-  };
+  protected finalizeSyncRun(): Task[] {
+    return this.finalizeBenchRun();
+  }
 }