From a8f38a1b521c15efd999579744890e8062761755 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 12:32:17 +0100 Subject: [PATCH 01/14] ci: move macro jobs to the x64 macro runner Switch codspeed-walltime and electron-e2e from codspeed-macro to codspeed-macro-x64-ryzen-9950x-ubuntu-24-04. --- .github/workflows/codspeed.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 4fb45d50..863b1a3c 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -69,7 +69,7 @@ jobs: codspeed-walltime: name: Run CodSpeed walltime - runs-on: "codspeed-macro" + runs-on: codspeed-macro-x64-ryzen-9950x-ubuntu-24-04 steps: - uses: "actions/checkout@v4" with: @@ -83,7 +83,7 @@ jobs: - name: Restore turbo cache uses: ./.github/actions/turbo-cache # The native addon prebuild links against the host's libstdc++, which - # Turbo does not hash; keep GitHub's arm64 runners' builds out. + # Turbo does not hash; keep GitHub-hosted x64 runners' builds out. with: key-suffix: codspeed-macro - run: pnpm install --frozen-lockfile --prefer-offline @@ -136,7 +136,7 @@ jobs: electron-e2e: name: Run electron inbox e2e - runs-on: codspeed-macro + runs-on: codspeed-macro-x64-ryzen-9950x-ubuntu-24-04 steps: - name: Checkout uses: actions/checkout@v6 From 6b3f9f77c71794e3ef420569ca69781d6815769e Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 12:37:17 +0100 Subject: [PATCH 02/14] ci: run electron e2e on the latest runner Drop the v4.17.7-alpha.2 pin. Current runner releases still support the samply walltime profiler. --- .github/workflows/codspeed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 863b1a3c..52487ae3 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -190,4 +190,4 @@ jobs: working-directory: examples/with-electron-and-walltime run: xvfb-run -a pnpm bench:electron mode: walltime - runner-version: v4.17.7-alpha.2 + runner-version: latest From d24bcbc6b0a7fabfd55873e1d56ecc5d25c25e25 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 5 Nov 2026 14:13:42 +0100 Subject: [PATCH 03/14] ci: keep Electron shared memory out of /dev/shm in the e2e example --- .../with-electron-and-walltime/apps/electron/test/inbox.e2e.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/with-electron-and-walltime/apps/electron/test/inbox.e2e.ts b/examples/with-electron-and-walltime/apps/electron/test/inbox.e2e.ts index 0ec20d59..9101407a 100644 --- a/examples/with-electron-and-walltime/apps/electron/test/inbox.e2e.ts +++ b/examples/with-electron-and-walltime/apps/electron/test/inbox.e2e.ts @@ -30,6 +30,8 @@ await bench( kind: "electron", appPath: path.join(appRoot, "out/main/index.js"), cwd: appRoot, + // Keep Chromium's shared memory in /tmp: CI runners may lack /dev/shm. + electronArgs: ["--disable-dev-shm-usage"], }, beforeRound: async ({ page }) => { page.setDefaultTimeout(180_000); From f6255caf1802b223bdd6fc6c64c7e42105b19adb Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 29 Oct 2026 12:09:22 +0100 Subject: [PATCH 04/14] feat(core): run walltime benchmarks in forks Benchmarks in one process share allocator and JIT state, and rounds within one process are correlated, so a single process gives a biased view of a benchmark. Add the building blocks for integrations to run each walltime benchmark in several fresh processes (forks) and pool their rounds: - `walltimeIsolation(forks?)` picks the mode from the env: parent, fork (`CODSPEED_ISOLATED_BENCHMARK`), or one shared process when `CODSPEED_PROCESS_ISOLATION=false` - the fork count is `CODSPEED_WALLTIME_FORKS`, else the integration's `forks` option, else 3 - `runForks`/`runForksSync` re-run the benchmark command (keeping the V8 flags) once per fork and collect the samples each fork writes with `writeForkSamples`; a failing fork's exit code is passed on - `samplesToResult` builds tinybench-style statistics over the pooled samples Refs COD-3960 --- packages/core/src/index.ts | 1 + packages/core/src/isolation.ts | 213 ++++++++++++++++++ packages/core/src/walltime/utils.ts | 46 ++++ packages/core/tests/fixtures/isolation.cjs | 36 +++ packages/core/tests/isolation.integ.test.ts | 187 +++++++++++++++ .../core/tests/walltimeUtils.integ.test.ts | 31 +++ 6 files changed, 514 insertions(+) create mode 100644 packages/core/src/isolation.ts create mode 100644 packages/core/tests/fixtures/isolation.cjs create mode 100644 packages/core/tests/isolation.integ.test.ts create mode 100644 packages/core/tests/walltimeUtils.integ.test.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8dc74641..8ceb582f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -53,6 +53,7 @@ export type { SetupInstrumentsResponse, } from "./generated/openapi"; export { getV8Flags, tryIntrospect } from "./introspection"; +export * from "./isolation"; export { getUnsupportedNodeVersionWarning, SUPPORTED_NODE_MAJORS, diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts new file mode 100644 index 00000000..6f0f1d36 --- /dev/null +++ b/packages/core/src/isolation.ts @@ -0,0 +1,213 @@ +/** + * Process isolation for walltime runs: every benchmark runs in fresh copies + * of the benchmark process (forks), so the allocator state, JIT state, and + * handles that other benchmarks or setup code leave behind can't change its + * measurement. Rounds within one process are correlated (JIT tier-up, heap + * sizing), so the rounds of several forks are pooled into one result. + */ +import { spawn, spawnSync } from "child_process"; +import { randomUUID } from "crypto"; +import { readFileSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { deserialize, serialize } from "v8"; +import { constants, gunzipSync, gzipSync } from "zlib"; + +/** Accepts `0`, `false`, `1`, or `true`; defaults to enabled. */ +export const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +/** Forks per walltime benchmark. Overrides the integration's `forks` option. */ +export const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +/** Forks per walltime benchmark when neither the env nor an option sets it. */ +export const DEFAULT_WALLTIME_FORKS = 3; +/** Set on each fork: the URI of the only benchmark it runs. */ +export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; +/** Set on each fork: the file where it writes its round samples. */ +export const FORK_SAMPLES_FILE_ENV = "CODSPEED_FORK_SAMPLES_FILE"; + +/** Walltime settings that every integration exposes next to its own options. */ +export interface WalltimeOptions { + /** + * Fresh processes that each walltime benchmark runs in; their rounds are + * reported together. {@link WALLTIME_FORKS_ENV} overrides it; defaults to + * {@link DEFAULT_WALLTIME_FORKS}. `CODSPEED_PROCESS_ISOLATION=false` runs + * everything in the benchmark process instead. + */ + forks?: number; +} + +export type IsolationMode = + /** All benchmarks are measured in this process. */ + | { kind: "SharedProcess" } + /** Every benchmark is measured in `forks` fresh processes, one after the other. */ + | { kind: "Forks"; forks: number } + /** A fork: measures only `uri` and writes its samples to `samplesFile`. */ + | { kind: "Fork"; uri: string; samplesFile: string }; + +/** + * The isolation of a walltime run, from the environment. `forks` is the + * integration's option, which {@link WALLTIME_FORKS_ENV} overrides. + */ +export function walltimeIsolation(forks?: number): IsolationMode { + const uri = process.env[ISOLATED_BENCHMARK_ENV]; + if (uri !== undefined) { + const samplesFile = process.env[FORK_SAMPLES_FILE_ENV]; + if (samplesFile === undefined) { + throw new Error( + `[CodSpeed] ${ISOLATED_BENCHMARK_ENV} is set without ${FORK_SAMPLES_FILE_ENV}`, + ); + } + return { kind: "Fork", uri, samplesFile }; + } + const isolation = process.env[PROCESS_ISOLATION_ENV]; + if (isolation === "0" || isolation === "false") { + return { kind: "SharedProcess" }; + } + if (isolation !== undefined && isolation !== "1" && isolation !== "true") { + throw new Error( + `[CodSpeed] ${PROCESS_ISOLATION_ENV} must be 0, false, 1, or true, got \`${isolation}\``, + ); + } + return { kind: "Forks", forks: resolveWalltimeForks(forks) }; +} + +/** + * The number of forks per walltime benchmark: {@link WALLTIME_FORKS_ENV}, + * else `option`, else {@link DEFAULT_WALLTIME_FORKS}. + */ +export function resolveWalltimeForks(option?: number): number { + const env = process.env[WALLTIME_FORKS_ENV]; + if (env !== undefined && env !== "") { + return validForks(env, WALLTIME_FORKS_ENV); + } + if (option !== undefined) { + return validForks(option, "forks"); + } + return DEFAULT_WALLTIME_FORKS; +} + +function validForks(raw: string | number, source: string): number { + const forks = Number(raw); + if (!Number.isInteger(forks) || forks < 1) { + throw new Error( + `[CodSpeed] ${source} must be a positive integer, got \`${raw}\``, + ); + } + return forks; +} + +/** The command of this process, which each fork re-runs. */ +function currentCommand(): string[] { + // execArgv keeps the V8 flags (e.g. --perf-prof) that the runner set. + return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; +} + +/** + * Runs the benchmark `uri` in `forks` fresh processes, one after the other, + * and returns the round samples of all of them, in milliseconds. Throws when + * a fork fails, without running the next one. + * + * Each fork re-runs this process's command, not fork(): Node can't fork, and a + * fork would only copy the calling thread. Call it from the process whose + * command runs the benchmarks: Vitest's main process, not a worker. + */ +export async function runForks(uri: string, forks: number): Promise { + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + // Not push(...): spreading a time-based run's samples overflows the stack. + samples = samples.concat(await runFork(uri)); + } + return samples; +} + +/** {@link runForks} for synchronous runs, which block anyway. */ +export function runForksSync(uri: string, forks: number): number[] { + let samples: number[] = []; + for (let fork = 0; fork < forks; fork++) { + samples = samples.concat(runForkSync(uri)); + } + return samples; +} + +/** A fresh file per fork: overlapping runForks() calls must not share one. */ +function newSamplesFile(): string { + return join( + tmpdir(), + `codspeed-samples-${process.pid}-${randomUUID()}.v8.gz`, + ); +} + +/** Runs one fork of `uri` and returns the samples it wrote. */ +async function runFork(uri: string): Promise { + const samplesFile = newSamplesFile(); + const [executable, ...args] = currentCommand(); + const child = spawn(executable, args, { + stdio: "inherit", + env: forkEnv(uri, samplesFile), + }); + const { promise, resolve, reject } = + Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); + child.once("error", (error) => reject(startError(uri, error))); + child.once("close", (status, signal) => resolve([status, signal])); + const [status, signal] = await promise; + return forkSamples(uri, samplesFile, status, signal); +} + +function runForkSync(uri: string): number[] { + const samplesFile = newSamplesFile(); + const [executable, ...args] = currentCommand(); + const result = spawnSync(executable, args, { + stdio: "inherit", + env: forkEnv(uri, samplesFile), + }); + if (result.error !== undefined) { + throw startError(uri, result.error); + } + return forkSamples(uri, samplesFile, result.status, result.signal); +} + +function forkEnv(uri: string, samplesFile: string): NodeJS.ProcessEnv { + return { + ...process.env, + [ISOLATED_BENCHMARK_ENV]: uri, + [FORK_SAMPLES_FILE_ENV]: samplesFile, + }; +} + +function startError(uri: string, error: Error): Error { + return new Error( + `[CodSpeed] failed to start the process for ${uri}: ${error.message}`, + ); +} + +/** The samples a finished fork wrote, deleting the file; throws if it failed. */ +function forkSamples( + uri: string, + samplesFile: string, + status: number | null, + signal: NodeJS.Signals | null, +): number[] { + try { + if (status !== 0) { + throw new Error( + `[CodSpeed] the process for ${uri} failed with ${signal ?? `exit code ${status}`}`, + ); + } + try { + return deserialize(gunzipSync(readFileSync(samplesFile))) as number[]; + } catch (error) { + throw new Error( + `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, + ); + } + } finally { + rmSync(samplesFile, { force: true }); + } +} + +/** Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. */ +export function writeForkSamples(samplesFile: string, samples: number[]) { + writeFileSync( + samplesFile, + gzipSync(serialize(samples), { level: constants.Z_BEST_SPEED }), + ); +} diff --git a/packages/core/src/walltime/utils.ts b/packages/core/src/walltime/utils.ts index fafa9e9e..baf97181 100644 --- a/packages/core/src/walltime/utils.ts +++ b/packages/core/src/walltime/utils.ts @@ -18,3 +18,49 @@ export const msToNs = (ms: number) => ms * 1e6; * @returns the seconds */ export const msToS = (ms: number) => ms / 1e3; + +/** tinybench-style statistics of round samples, in milliseconds. */ +export interface RoundsResult { + totalTime: number; + latency: { + min: number; + max: number; + mean: number; + sd: number; + samples: number[]; + }; +} + +/** + * The result of nonempty round samples in milliseconds, e.g. the samples of + * several forks of one benchmark. Throws if no samples were recorded. + */ +export function samplesToResult(samples: number[]): RoundsResult { + const n = samples.length; + if (n === 0) { + throw new Error("[CodSpeed] Cannot compute statistics from empty samples"); + } + let totalTime = 0; + let min = Infinity; + let max = -Infinity; + for (const ms of samples) { + totalTime += ms; + if (ms < min) min = ms; + if (ms > max) max = ms; + } + const mean = totalTime / n; + const variance = + n > 1 + ? samples.reduce((sum, ms) => sum + (ms - mean) ** 2, 0) / (n - 1) + : 0; + return { + totalTime, + latency: { + min, + max, + mean, + sd: Math.sqrt(variance), + samples, + }, + }; +} diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs new file mode 100644 index 00000000..bd2d6d2e --- /dev/null +++ b/packages/core/tests/fixtures/isolation.cjs @@ -0,0 +1,36 @@ +// Run by `isolation.integ.test.ts`. Measures benchmark "a" in forks. +const { + runForks, + runForksSync, + walltimeIsolation, + writeForkSamples, +} = require("../../dist/index.cjs.js"); + +const mode = walltimeIsolation(); + +function measure() { + console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`); + switch (process.env.ISOLATION_FIXTURE_ACTION) { + case "exit": + process.exit(3); + case "no-samples": + process.exit(0); + } + writeForkSamples(mode.samplesFile, [Number(process.pid), 1]); +} + +async function main() { + if (mode.kind === "Fork") { + measure(); + return; + } + const forks = Number(process.env.ISOLATION_FIXTURE_FORKS); + const samples = + process.env.ISOLATION_FIXTURE_SYNC !== undefined + ? runForksSync("a", forks) + : await runForks("a", forks); + console.log(`samples ${samples.length}`); +} + +// A rejection ends the process with exit code 1. +main(); diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts new file mode 100644 index 00000000..c61d3f45 --- /dev/null +++ b/packages/core/tests/isolation.integ.test.ts @@ -0,0 +1,187 @@ +import { spawnSync } from "child_process"; +import { join } from "path"; +import { + FORK_SAMPLES_FILE_ENV, + ISOLATED_BENCHMARK_ENV, + PROCESS_ISOLATION_ENV, + resolveWalltimeForks, + WALLTIME_FORKS_ENV, + walltimeIsolation, +} from "../src/isolation"; + +const FIXTURE = join(__dirname, "fixtures", "isolation.cjs"); +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const CHILD_EXIT_CODE = 3; +const RAN_LINE = /^ran (\d+) (.+)$/; +const SAMPLES_LINE = /^samples (\d+)$/; +/** Passed to Node to check that forks keep the flags of their parent. */ +const NODE_FLAG = "--no-warnings"; +/** Each fork of the fixture reports two samples. */ +const SAMPLES_PER_FORK = 2; + +const ENV_KEYS = [ + PROCESS_ISOLATION_ENV, + ISOLATED_BENCHMARK_ENV, + FORK_SAMPLES_FILE_ENV, + WALLTIME_FORKS_ENV, +]; +const savedEnv = Object.fromEntries( + ENV_KEYS.map((key) => [key, process.env[key]]), +); + +function setEnv(env: Record): void { + for (const key of ENV_KEYS) { + delete process.env[key]; + } + for (const [key, value] of Object.entries(env)) { + if (value !== undefined) { + process.env[key] = value; + } + } +} + +afterEach(() => setEnv(savedEnv)); + +describe("resolveWalltimeForks", () => { + it("defaults to 3 forks", () => { + setEnv({}); + expect(resolveWalltimeForks()).toBe(3); + }); + + it("takes the option over the default", () => { + setEnv({}); + expect(resolveWalltimeForks(5)).toBe(5); + }); + + it("takes the env over the option", () => { + setEnv({ [WALLTIME_FORKS_ENV]: "2" }); + expect(resolveWalltimeForks(5)).toBe(2); + }); + + it("ignores an empty env", () => { + setEnv({ [WALLTIME_FORKS_ENV]: "" }); + expect(resolveWalltimeForks(5)).toBe(5); + }); + + it.each(["0", "-1", "1.5", "abc"])("rejects the env %p", (value) => { + setEnv({ [WALLTIME_FORKS_ENV]: value }); + expect(() => resolveWalltimeForks()).toThrow(WALLTIME_FORKS_ENV); + }); + + it.each([0, -1, 1.5, NaN])("rejects the option %p", (value) => { + setEnv({}); + expect(() => resolveWalltimeForks(value)).toThrow("forks"); + }); +}); + +describe("walltimeIsolation", () => { + it.each([ + [undefined, "Forks"], + ["1", "Forks"], + ["true", "Forks"], + ["0", "SharedProcess"], + ["false", "SharedProcess"], + ])("selects %s as %s", (value, kind) => { + setEnv({ [PROCESS_ISOLATION_ENV]: value }); + expect(walltimeIsolation().kind).toBe(kind); + }); + + it("runs everything in this process when isolation is off, whatever the fork count", () => { + setEnv({ [PROCESS_ISOLATION_ENV]: "false", [WALLTIME_FORKS_ENV]: "4" }); + expect(walltimeIsolation(2).kind).toBe("SharedProcess"); + }); + + it("restricts a fork to its benchmark, whatever the isolation setting", () => { + setEnv({ + [PROCESS_ISOLATION_ENV]: "false", + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + }); + expect(walltimeIsolation()).toEqual({ + kind: "Fork", + uri: "file::bench", + samplesFile: "samples.json", + }); + }); +}); + +interface FixtureOptions { + forks: number; + action?: "exit" | "no-samples"; + sync?: boolean; +} + +function runFixture({ forks, action, sync }: FixtureOptions) { + const env: NodeJS.ProcessEnv = { + ...process.env, + ISOLATION_FIXTURE_ACTION: action, + ISOLATION_FIXTURE_FORKS: String(forks), + }; + for (const key of ENV_KEYS) { + delete env[key]; + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + const result = spawnSync(process.execPath, [NODE_FLAG, FIXTURE], { + env, + encoding: "utf8", + timeout: RUN_TIMEOUT_MS, + }); + const lines = result.stdout.split("\n"); + const ran = lines + .map((line) => RAN_LINE.exec(line)) + .filter((match): match is RegExpExecArray => match !== null); + const samples = lines + .map((line) => SAMPLES_LINE.exec(line)) + .find((match) => match !== null); + return { + status: result.status, + stderr: result.stderr, + ran, + samples: samples ? Number(samples[1]) : undefined, + }; +} + +describe.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, +])("runForks ($name run)", ({ sync }) => { + it("runs every fork in a fresh process with the flags of its parent and pools their samples", () => { + const forks = 3; + const { status, ran, samples } = runFixture({ forks, sync }); + + expect(status).toBe(0); + expect(ran).toHaveLength(forks); + expect(new Set(ran.map((match) => match[1])).size).toBe(forks); + for (const match of ran) { + expect(JSON.parse(match[2])).toEqual([NODE_FLAG]); + } + expect(samples).toBe(forks * SAMPLES_PER_FORK); + }); + + it("fails on a failing fork and runs no further fork", () => { + const { status, stderr, ran, samples } = runFixture({ + forks: 3, + sync, + action: "exit", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain(`failed with exit code ${CHILD_EXIT_CODE}`); + expect(ran).toHaveLength(1); + expect(samples).toBeUndefined(); + }); + + it("fails when a fork exits without reporting its samples", () => { + const { status, stderr } = runFixture({ + forks: 2, + sync, + action: "no-samples", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain("did not report its samples"); + }); +}); diff --git a/packages/core/tests/walltimeUtils.integ.test.ts b/packages/core/tests/walltimeUtils.integ.test.ts new file mode 100644 index 00000000..132b7559 --- /dev/null +++ b/packages/core/tests/walltimeUtils.integ.test.ts @@ -0,0 +1,31 @@ +import { samplesToResult } from "../src/walltime/utils"; + +describe("samplesToResult", () => { + it("computes pooled statistics", () => { + const samples = [2, 4, 4, 4, 5, 5, 7, 9]; + const { totalTime, latency } = samplesToResult(samples); + expect(totalTime).toBe(40); + expect(latency.min).toBe(2); + expect(latency.max).toBe(9); + expect(latency.mean).toBe(5); + // sample standard deviation: sqrt(32 / 7) + expect(latency.sd).toBeCloseTo(Math.sqrt(32 / 7), 10); + }); + + it("has zero deviation for a single sample", () => { + const { totalTime, latency } = samplesToResult([3]); + expect(totalTime).toBe(3); + expect(latency).toMatchObject({ min: 3, max: 3, mean: 3, sd: 0 }); + }); + + it("handles very large sample counts without overflowing the stack", () => { + const samples = new Array(500_000).fill(1); + expect(samplesToResult(samples).totalTime).toBe(500_000); + }); + + it("rejects empty samples", () => { + expect(() => samplesToResult([])).toThrow( + "[CodSpeed] Cannot compute statistics from empty samples", + ); + }); +}); From 68883f3758cd752071aeabbe8cd8b170b0a24a57 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 29 Oct 2026 12:09:50 +0100 Subject: [PATCH 05/14] feat(tinybench): run walltime benchmarks in forks `withCodSpeed(bench, { forks })` sets the fork count of walltime runs, and `CODSPEED_WALLTIME_FORKS` overrides it. The parent runs each task in its forks and reports their pooled rounds; a fork runs only its task, hands its samples over and prints no banner or summary. Simulation runs are unchanged. Refs COD-3960 --- packages/tinybench-plugin/src/analysis.ts | 4 +- packages/tinybench-plugin/src/index.ts | 8 +- packages/tinybench-plugin/src/shared.ts | 54 ++++- packages/tinybench-plugin/src/walltime.ts | 69 ++++++- .../tests/fixtures/isolation.mjs | 32 +++ .../tests/isolation.integ.test.ts | 194 ++++++++++++++++++ 6 files changed, 341 insertions(+), 20 deletions(-) create mode 100644 packages/tinybench-plugin/tests/fixtures/isolation.mjs create mode 100644 packages/tinybench-plugin/tests/isolation.integ.test.ts diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 65bd6183..54c8d975 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -14,7 +14,9 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile); + const runner = new AnalysisBenchRunner(bench, rootCallingFile, { + kind: "SharedProcess", + }); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index 298f97b9..bb903ae0 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -8,6 +8,7 @@ import { SetupInstrumentsResponse, tryIntrospect, wrapWithRootFrameSync, + type WalltimeOptions, } from "@codspeed/core"; import { Bench } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; @@ -17,7 +18,10 @@ import { setupCodspeedWalltimeBench } from "./walltime"; tryIntrospect(); -export function withCodSpeed(bench: Bench): Bench { +export function withCodSpeed( + bench: Bench, + options: WalltimeOptions = {}, +): Bench { const codspeedRunnerMode = getCodspeedRunnerMode(); if (codspeedRunnerMode === "disabled") { return bench; @@ -52,7 +56,7 @@ export function withCodSpeed(bench: Bench): Bench { if (instrumentMode === "analysis") { setupCodspeedAnalysisBench(bench, rootCallingFile); } else if (instrumentMode === "walltime") { - setupCodspeedWalltimeBench(bench, rootCallingFile); + setupCodspeedWalltimeBench(bench, rootCallingFile, options); } return bench; diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 0fb811c5..1e10b4bf 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,4 @@ -import { setupCore, teardownCore } from "@codspeed/core"; +import { setupCore, teardownCore, type IsolationMode } from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -8,10 +8,21 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - - constructor(bench: Bench, rootCallingFile: string) { + /** Walltime runners pass the isolation of this process; analysis shares it. */ + protected readonly isolationMode: IsolationMode; + + constructor( + bench: Bench, + rootCallingFile: string, + isolationMode: IsolationMode, + ) { this.bench = bench; this.rootCallingFile = rootCallingFile; + this.isolationMode = isolationMode; + } + + protected get isChild(): boolean { + return this.isolationMode.kind === "Fork"; } private setupBenchRun(): void { @@ -20,6 +31,10 @@ export abstract class BaseBenchRunner { } private logStart(): void { + // The parent already printed the banner, children stay quiet. + if (this.isChild) { + return; + } console.log( `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})`, ); @@ -45,7 +60,12 @@ export abstract class BaseBenchRunner { protected finalizeBenchRun(): Task[] { teardownCore(); - console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); + // The parent already printed the summary line, children stay quiet. + if (!this.isChild) { + console.log( + `[CodSpeed] Done running ${this.bench.tasks.length} benches.`, + ); + } return this.bench.tasks; } @@ -57,10 +77,15 @@ export abstract class BaseBenchRunner { public setupBenchMethods(): void { this.bench.run = async () => { + const tasks = this.selectTasks(); + // A child that owns none of this bench's tasks must not touch the + // instruments, the bench that owns its task does. + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); + for (const { task, uri } of tasks) { await this.runTaskAsync(task, uri); } @@ -68,14 +93,27 @@ export abstract class BaseBenchRunner { }; this.bench.runSync = () => { + const tasks = this.selectTasks(); + if (this.isChild && tasks.length === 0) { + return this.bench.tasks; + } this.setupBenchRun(); - for (const task of this.bench.tasks) { - const uri = this.getTaskUri(task); + for (const { task, uri } of tasks) { this.runTaskSync(task, uri); } return this.finalizeSyncRun(); }; } + + /** The tasks this process measures: all of them, or the isolated one. */ + private selectTasks(): { task: Task; uri: string }[] { + return this.bench.tasks + .map((task) => ({ task, uri: this.getTaskUri(task) })) + .filter( + ({ uri }) => + this.isolationMode.kind !== "Fork" || this.isolationMode.uri === uri, + ); + } } diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 90e6b700..7168cb0c 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -6,19 +6,28 @@ import { mongoMeasurement, msToNs, msToS, + runForks, + runForksSync, + samplesToResult, + walltimeIsolation, + writeForkSamples, writeWalltimeResults, type BenchmarkStats, type Benchmark as CodspeedBenchmark, + type RoundsResult, + type WalltimeOptions, } from "@codspeed/core"; -import { Bench, Task, TaskResult } from "tinybench"; +import { Bench, Task } from "tinybench"; import { getBenchOptions } from "./benchOptions"; import { BaseBenchRunner } from "./shared"; export function setupCodspeedWalltimeBench( bench: Bench, rootCallingFile: string, + options: WalltimeOptions, ): void { - const runner = new WalltimeBenchRunner(bench, rootCallingFile); + const isolationMode = walltimeIsolation(options.forks); + const runner = new WalltimeBenchRunner(bench, rootCallingFile, isolationMode); runner.installInstrumentHooks(); runner.setupBenchMethods(); } @@ -91,6 +100,13 @@ class WalltimeBenchRunner extends BaseBenchRunner { } protected async runTaskAsync(task: Task, uri: string): Promise { + const { isolationMode } = this; + if (isolationMode.kind === "Forks") { + const samples = await runForks(uri, isolationMode.forks); + setTaskResult(task, samples); + return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + } + // run the warmup of the task right before its actual run if (getBenchOptions(this.bench).warmup) { await task.warmup(); @@ -103,8 +119,14 @@ class WalltimeBenchRunner extends BaseBenchRunner { this.registerCodspeedBenchmarkFromTask(task); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - protected runTaskSync(task: Task, _uri: string): void { + protected runTaskSync(task: Task, uri: string): void { + const { isolationMode } = this; + if (isolationMode.kind === "Forks") { + const samples = runForksSync(uri, isolationMode.forks); + setTaskResult(task, samples); + return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + } + if (getBenchOptions(this.bench).warmup) { task.warmupSync(); } @@ -123,19 +145,30 @@ class WalltimeBenchRunner extends BaseBenchRunner { } private registerCodspeedBenchmarkFromTask(task: Task): void { - const uri = this.getTaskUri(task); - if (!task.result) { - console.warn(` ⚠ No result data available for ${uri}`); + console.warn( + ` ⚠ No result data available for ${this.getTaskUri(task)}`, + ); + return; + } + // A fork hands its samples to the parent, which reports the benchmark. + if (this.isolationMode.kind === "Fork") { + writeForkSamples(this.isolationMode.samplesFile, [ + ...task.result.latency.samples, + ]); return; } + this.registerCodspeedBenchmark(task, task.result); + } + private registerCodspeedBenchmark(task: Task, result: RoundsResult): void { + const uri = this.getTaskUri(task); const opts = getBenchOptions(this.bench); const warmupIterations = opts.warmup ? (opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT) : 0; const stats = convertTinybenchResultToBenchmarkStats( - task.result, + result, warmupIterations, ); @@ -156,6 +189,11 @@ class WalltimeBenchRunner extends BaseBenchRunner { } private finalizeWalltimeRun(isAsync: boolean): Task[] { + // A fork only hands its samples to the parent, which reports the results. + if (this.isChild) { + return this.bench.tasks; + } + // Write results to JSON file using core function if (this.codspeedBenchmarks.length > 0) { writeWalltimeResults(this.codspeedBenchmarks, isAsync); @@ -171,7 +209,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { const TINYBENCH_WARMUP_DEFAULT = 16; function convertTinybenchResultToBenchmarkStats( - result: TaskResult, + result: RoundsResult, warmupIterations: number, ): BenchmarkStats { const { min, max, mean, sd, samples } = result.latency; @@ -200,3 +238,16 @@ function convertTinybenchResultToBenchmarkStats( warmup_iters: warmupIterations, }; } + +/** + * Fills `task.result` from the pooled fork samples so `bench.table()` and + * `bench.results` work. Only tinybench 4/5 expose a way in; v6 keeps the + * result `#private`. + */ +function setTaskResult(task: Task, samples: number[]): void { + const { processRunResult } = task as unknown as { + processRunResult?: (run: { latencySamples: number[] }) => void; + }; + // tinybench sorts the samples in place. + processRunResult?.call(task, { latencySamples: [...samples] }); +} diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs new file mode 100644 index 00000000..e347110d --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -0,0 +1,32 @@ +// Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a +// benchmark, and the sample count of every `task.result` the parent ends with. +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +const options = process.env.ISOLATION_FIXTURE_FORKS_OPTION; +const bench = withCodSpeed( + new Bench({ iterations: 4, time: 1, warmupTime: 0 }), + options === undefined ? {} : { forks: Number(options) }, +); + +for (const name of ["first", "second"]) { + // tinybench calls the function once when it is added, in every process. + let calls = 0; + bench.add(name, () => { + if (++calls === 2) { + console.log(`ran ${process.pid} ${name}`); + } + }); +} + +if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { + bench.runSync(); +} else { + await bench.run(); +} + +if (process.env.CODSPEED_ISOLATED_BENCHMARK === undefined) { + for (const task of bench.tasks) { + console.log(`result ${task.name} ${task.result?.samples.length ?? 0}`); + } +} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts new file mode 100644 index 00000000..b78a5db4 --- /dev/null +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -0,0 +1,194 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawn } from "child_process"; +import { once } from "events"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/isolation.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const BENCHMARKS = ["first", "second"]; +const DEFAULT_FORKS = 3; +/** The fixture's `iterations`: every fork measures at least this many rounds. */ +const MIN_ROUNDS_PER_FORK = 4; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 60_000; +const RAN_LINE = /^ran (\d+) (.+)$/; +const RESULT_LINE = /^result (.+) (\d+)$/; + +interface FixtureRun { + status: number | null; + /** PIDs that ran each benchmark. */ + pidsByName: Map>; + /** PID of the process the test started. */ + parentPid: number; + /** Rounds of each benchmark in the results file. */ + roundsByName: Map; + /** Samples in the parent's `task.result` of each benchmark. */ + parentSamplesByName: Map; +} + +interface FixtureOptions { + /** `CODSPEED_PROCESS_ISOLATION=false`: every benchmark shares our process. */ + shared?: boolean; + /** `forks` option of `withCodSpeed`. */ + forksOption?: number; + /** `CODSPEED_WALLTIME_FORKS`. */ + forksEnv?: number; + sync?: boolean; +} + +async function runFixture({ + shared, + forksOption, + forksEnv, + sync, +}: FixtureOptions = {}): Promise { + const profileFolder = mkdtempSync(join(tmpdir(), "codspeed-tinybench-")); + try { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[WALLTIME_FORKS_ENV]; + if (shared) { + env[PROCESS_ISOLATION_ENV] = "false"; + } + if (forksOption !== undefined) { + env.ISOLATION_FIXTURE_FORKS_OPTION = String(forksOption); + } + if (forksEnv !== undefined) { + env[WALLTIME_FORKS_ENV] = String(forksEnv); + } + if (sync) { + env.ISOLATION_FIXTURE_SYNC = "1"; + } + + // The runner starts Node with these flags, and forks must keep them. Not + // spawnSync: blocking the test worker times out its calls to Vitest. + const child = spawn(process.execPath, [...getV8Flags(), FIXTURE], { + env, + timeout: RUN_TIMEOUT_MS, + }); + let stdout = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + const [status] = (await once(child, "close")) as [number | null]; + + const pidsByName = new Map>(); + const parentSamplesByName = new Map(); + for (const line of stdout.split("\n")) { + const ran = RAN_LINE.exec(line); + if (ran) { + const pids = pidsByName.get(ran[2]) ?? new Set(); + pids.add(Number(ran[1])); + pidsByName.set(ran[2], pids); + continue; + } + const result = RESULT_LINE.exec(line); + if (result) { + parentSamplesByName.set(result[1], Number(result[2])); + } + } + + const roundsByName = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of readdirSync(resultsDir)) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); + for (const benchmark of data.benchmarks) { + roundsByName.set(benchmark.name, benchmark.stats.rounds); + } + } + return { + status, + pidsByName, + parentPid: child.pid!, + roundsByName, + parentSamplesByName, + }; + } finally { + rmSync(profileFolder, { recursive: true, force: true }); + } +} + +/** PIDs that ran a benchmark besides the one we started. */ +function forkPids(run: FixtureRun, name: string): Set { + const pids = new Set(run.pidsByName.get(name)); + pids.delete(run.parentPid); + return pids; +} + +describe("walltime forks", () => { + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "runs every benchmark in 3 forks by default and pools their rounds ($name run)", + async ({ sync }) => { + const run = await runFixture({ sync }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(DEFAULT_FORKS); + const rounds = run.roundsByName.get(name); + expect(rounds).toBeGreaterThanOrEqual( + DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, + ); + // The parent reports the pooled rounds as its own `task.result`. + expect(run.parentSamplesByName.get(name)).toBe(rounds); + } + // Forks of different benchmarks are different processes too. + const all = new Set( + BENCHMARKS.flatMap((name) => [...forkPids(run, name)]), + ); + expect(all.size).toBe(DEFAULT_FORKS * BENCHMARKS.length); + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the fork count from the withCodSpeed option", + async () => { + const run = await runFixture({ forksOption: 2 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(2); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the fork count from the environment over the option", + async () => { + const run = await runFixture({ forksOption: 2, forksEnv: 1 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(forkPids(run, name).size).toBe(1); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "runs every benchmark in the parent process when isolation is disabled", + async () => { + const run = await runFixture({ shared: true, forksEnv: 4 }); + + expect(run.status).toBe(0); + for (const name of BENCHMARKS) { + expect(run.pidsByName.get(name)).toEqual(new Set([run.parentPid])); + } + }, + RUN_TIMEOUT_MS, + ); +}); From fe13d0c892ec4646a99f94d66c890b75506e4d10 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 29 Oct 2026 12:09:50 +0100 Subject: [PATCH 06/14] feat(vitest): run walltime benchmarks in forks With Vitest 5, each walltime benchmark runs in fresh `vitest bench` processes restricted to its file. The fork count resolves as `CODSPEED_WALLTIME_FORKS`, the benchmark's or run's `forks` option, `codspeedPlugin({ forks })`, then 3. The test timeout is scaled by the env or plugin fork count, since every fork of a test runs within it. Vitest's own report shows zeroed statistics for forked benchmarks; the CodSpeed results hold the pooled rounds. Vitest 3/4 don't fork. Closes COD-3960 --- .../benches/exclusive-resource.bench.ts | 28 +++ .../vitest-plugin/src/__tests__/index.test.ts | 13 ++ .../src/__tests__/isolation.test.ts | 159 ++++++++++++++++++ packages/vitest-plugin/src/index.ts | 28 ++- packages/vitest-plugin/src/instrument.ts | 27 +++ packages/vitest-plugin/src/v5/forkContext.ts | 49 ++++++ packages/vitest-plugin/src/v5/forkReporter.ts | 57 +++++++ packages/vitest-plugin/src/v5/provider.ts | 74 +++++++- packages/vitest-plugin/src/vitestBackend.ts | 78 ++++++++- .../tests/fixtures/isolation/globalSetup.ts | 34 ++++ .../tests/fixtures/isolation/pids.bench.ts | 47 ++++++ .../tests/fixtures/isolation/vitest.config.ts | 15 ++ packages/vitest-plugin/vitest.config.ts | 3 + 13 files changed, 598 insertions(+), 14 deletions(-) create mode 100644 packages/vitest-plugin/benches/exclusive-resource.bench.ts create mode 100644 packages/vitest-plugin/src/__tests__/isolation.test.ts create mode 100644 packages/vitest-plugin/src/v5/forkContext.ts create mode 100644 packages/vitest-plugin/src/v5/forkReporter.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/globalSetup.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts create mode 100644 packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts diff --git a/packages/vitest-plugin/benches/exclusive-resource.bench.ts b/packages/vitest-plugin/benches/exclusive-resource.bench.ts new file mode 100644 index 00000000..a6d45b55 --- /dev/null +++ b/packages/vitest-plugin/benches/exclusive-resource.bench.ts @@ -0,0 +1,28 @@ +import { connect, createServer, type Server } from "node:net"; +import { afterAll, beforeAll, test } from "vitest"; + +// Fixed on purpose: a second process holding it must fail with EADDRINUSE. +const PORT = 48_123; +let server: Server; + +beforeAll(() => { + const { promise, resolve, reject } = Promise.withResolvers(); + server = createServer((socket) => socket.end()) + .once("error", reject) + .listen(PORT, resolve); + return promise; +}); + +afterAll(() => { + const { promise, resolve } = Promise.withResolvers(); + server.close(() => resolve()); + return promise; +}); + +test("exclusive port", async ({ bench }) => { + await bench("connect", async () => { + const { promise, resolve } = Promise.withResolvers(); + connect(PORT).once("close", resolve); + return promise; + }).run(); +}); diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index d7459fea..f697d49a 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -2,6 +2,7 @@ import { getV8Flags } from "@codspeed/core"; import { fromPartial } from "@total-typescript/shoehorn"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { type ViteUserConfig } from "vitest/config"; +import { type Vitest } from "vitest/node"; import codspeedPlugin from "../index"; const coreMocks = vi.hoisted(() => { @@ -162,5 +163,17 @@ describe("codSpeedPlugin", () => { expect(result.test?.benchmark?.suppressExportGetterWarnings).toBe(false); }); + + it("should register the fork reporter once across projects", () => { + vi.stubEnv("CODSPEED_RUNNER_MODE", "walltime"); + if (typeof plugin.configureVitest !== "function") + throw new Error("configureVitest is not a function"); + const vitest = fromPartial({ config: { reporters: [] } }); + for (const name of ["a (bench)", "b (bench)"]) { + plugin.configureVitest(fromPartial({ vitest, project: { name } })); + } + + expect(vitest.config.reporters).toHaveLength(1); + }); }); }); diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts new file mode 100644 index 00000000..0169373e --- /dev/null +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -0,0 +1,159 @@ +import { spawn } from "child_process"; +import { once } from "events"; +import { + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "fs"; +import { createRequire } from "module"; +import { tmpdir } from "os"; +import { dirname, join } from "path"; +import { fileURLToPath } from "url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const FIXTURE_DIR = fileURLToPath( + new URL("../../tests/fixtures/isolation", import.meta.url), +); +const VITEST_BIN = join( + dirname(createRequire(import.meta.url).resolve("vitest/package.json")), + "vitest.mjs", +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const DEFAULT_FORKS = 3; +/** The fixture's `iterations`: every fork measures at least this many rounds. */ +const MIN_ROUNDS_PER_FORK = 4; +const RUN_TIMEOUT_MS = 180_000; +const RAN_LINE = /^ran (\d+) (.+)$/; + +let tmpDir: string; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "codspeed-isolation-")); +}); + +afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); +}); + +interface FixtureOptions { + isolation?: string; + forksEnv?: number; + pluginForks?: number; + firstForks?: number; + failing?: boolean; +} + +async function runFixture(options: FixtureOptions = {}) { + const logFile = join(tmpDir, "ran.log"); + writeFileSync(logFile, ""); + const profileFolder = join(tmpDir, "profile"); + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + ISOLATION_LOG_FILE: logFile, + [PROCESS_ISOLATION_ENV]: options.isolation, + [WALLTIME_FORKS_ENV]: options.forksEnv?.toString(), + ISOLATION_FIXTURE_PLUGIN_FORKS: options.pluginForks?.toString(), + ISOLATION_FIXTURE_FIRST_FORKS: options.firstForks?.toString(), + ISOLATION_FIXTURE_FAIL: options.failing ? "1" : undefined, + }; + + // Not spawnSync: blocking this worker times out its calls to Vitest. + const child = spawn(process.execPath, [VITEST_BIN, "bench", "--run"], { + cwd: FIXTURE_DIR, + env, + timeout: RUN_TIMEOUT_MS, + stdio: "ignore", + }); + const [status] = (await once(child, "close")) as [number | null]; + + const pidsByName = new Map>(); + for (const line of readFileSync(logFile, "utf8").split("\n")) { + const match = RAN_LINE.exec(line); + if (!match) continue; + const pids = pidsByName.get(match[2]) ?? new Set(); + pids.add(Number(match[1])); + pidsByName.set(match[2], pids); + } + + const roundsByName = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of status === 0 ? readdirSync(resultsDir) : []) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")); + for (const benchmark of data.benchmarks) { + roundsByName.set(benchmark.name, benchmark.stats.rounds); + } + } + return { status, pidsByName, roundsByName }; +} + +describe("walltime forks", () => { + it( + "runs every benchmark in 3 forks by default and pools their rounds", + async () => { + const run = await runFixture(); + + expect(run.status).toBe(0); + for (const name of ["first", "second"]) { + expect(run.pidsByName.get(name)?.size).toBe(DEFAULT_FORKS); + expect(run.roundsByName.get(name)).toBeGreaterThanOrEqual( + DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, + ); + } + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the benchmark's option over the plugin's", + async () => { + const run = await runFixture({ firstForks: 2, pluginForks: 1 }); + + expect(run.status).toBe(0); + expect(run.pidsByName.get("first")?.size).toBe(2); + expect(run.pidsByName.get("second")?.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "takes the environment over the benchmark's option", + async () => { + const run = await runFixture({ firstForks: 2, forksEnv: 1 }); + + expect(run.status).toBe(0); + expect(run.pidsByName.get("first")?.size).toBe(1); + expect(run.pidsByName.get("second")?.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "runs every benchmark in one process when isolation is disabled", + async () => { + const run = await runFixture({ isolation: "false" }); + + expect(run.status).toBe(0); + const pids = new Set( + [...run.pidsByName.values()].flatMap((set) => [...set]), + ); + expect(pids.size).toBe(1); + }, + RUN_TIMEOUT_MS, + ); + + it( + "fails the run when a benchmark throws in its fork", + async () => { + const run = await runFixture({ failing: true, forksEnv: 1 }); + + expect(run.status).not.toBe(0); + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index cb3af1ae..0b7d7854 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -5,12 +5,18 @@ import { mongoMeasurement, SetupInstrumentsRequestBody, SetupInstrumentsResponse, + type WalltimeOptions, } from "@codspeed/core"; import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; +import { type Vitest } from "vitest/node"; import { resolveVitestBackend } from "./vitestBackend"; +// `configureVitest` runs once per project, but one reporter sees every +// project's modules: a second one would fork every benchmark twice. +const vitestsWithForkReporter = new WeakSet(); + // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; const isFileInTs = import.meta.url.endsWith(".ts"); @@ -21,7 +27,8 @@ function resolveFile(name: string): string { return join(__dirname, `${name}.${fileExtension}`); } -export default function codspeedPlugin(): Plugin { +/** `options.forks`: see {@link WalltimeOptions}; isolation requires Vitest 5. */ +export default function codspeedPlugin(options: WalltimeOptions = {}): Plugin { return { name: "codspeed:vitest", apply(_, { mode }) { @@ -37,19 +44,32 @@ export default function codspeedPlugin(): Plugin { return true; }, enforce: "post", - config(userConfig): ViteUserConfig { + config(userConfig: ViteUserConfig): ViteUserConfig { + const backend = resolveVitestBackend(); + backend.configureIsolation(userConfig); return { test: { pool: "forks", globalSetup: [resolveFile("globalSetup")], - ...resolveVitestBackend().getBenchmarkTestConfig( + ...backend.getBenchmarkTestConfig( getV8Flags(), resolveFile, - (userConfig as ViteUserConfig).test, + userConfig.test, ), }, }; }, + configureVitest({ project, vitest }) { + const backend = resolveVitestBackend(); + backend.configureProject(project); + if (vitestsWithForkReporter.has(vitest)) return; + vitestsWithForkReporter.add(vitest); + const reporter = backend.forkReporter(options); + // Vitest builds its reporters from the config after this hook runs. + if (reporter) { + vitest.config.reporters.push(reporter); + } + }, }; } diff --git a/packages/vitest-plugin/src/instrument.ts b/packages/vitest-plugin/src/instrument.ts index 24039088..e9a7d7fb 100644 --- a/packages/vitest-plugin/src/instrument.ts +++ b/packages/vitest-plugin/src/instrument.ts @@ -10,6 +10,7 @@ import { type BenchmarkStats, } from "@codspeed/core"; import type * as tinybench from "tinybench"; +import type { TaskMeta } from "vitest"; export type Tinybench = typeof tinybench; @@ -51,6 +52,32 @@ export interface TinybenchOptions { iterations?: number; } +/** A benchmark the main process measures in forks once the run is over. */ +export interface ForkedBenchmark { + uri: string; + name: string; + /** The benchmark's own `forks` option; the plugin option applies otherwise. */ + forks: number | undefined; + options: TinybenchOptions; +} + +// Vitest ships a test's meta to the main process with its result, so the +// worker hands its queued benchmarks to the reporter through it. +const FORKED_BENCHMARKS_META = "codspeedForkedBenchmarks"; + +type ForkMeta = TaskMeta & { [FORKED_BENCHMARKS_META]?: ForkedBenchmark[] }; + +export function setForkedBenchmarks( + meta: TaskMeta, + benchmarks: ForkedBenchmark[], +): void { + (meta as ForkMeta)[FORKED_BENCHMARKS_META] = benchmarks; +} + +export function forkedBenchmarks(meta: TaskMeta): ForkedBenchmark[] { + return (meta as ForkMeta)[FORKED_BENCHMARKS_META] ?? []; +} + /** * Drive the instrumentation window from each bench's run-mode setup/teardown * hooks so it brackets only tinybench's measured loop, excluding the warmup diff --git a/packages/vitest-plugin/src/v5/forkContext.ts b/packages/vitest-plugin/src/v5/forkContext.ts new file mode 100644 index 00000000..0da54bc1 --- /dev/null +++ b/packages/vitest-plugin/src/v5/forkContext.ts @@ -0,0 +1,49 @@ +import { randomUUID } from "crypto"; +import { readFileSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { deserialize, serialize } from "v8"; +import type { TestModule, TestProject } from "vitest/node"; + +const FORK_CONTEXT_FILE_ENV = "CODSPEED_VITEST_FORK_CONTEXT_FILE"; + +/** What each project's global setup provided, by project name. */ +type ProvidedContexts = Record>; + +/** + * Hands the modules' provided context to the forks `run` starts, through a + * file the forks inherit by its environment variable (not the environment + * itself: a large provided fixture would exceed its size limit). + */ +export async function withForkContext( + modules: readonly TestModule[], + run: () => Promise, +): Promise { + const contexts: ProvidedContexts = {}; + for (const { project } of modules) { + contexts[project.name] = { ...project.getProvidedContext() }; + } + const file = join(tmpdir(), `codspeed-vitest-context-${randomUUID()}`); + writeFileSync(file, serialize(contexts)); + process.env[FORK_CONTEXT_FILE_ENV] = file; + try { + return await run(); + } finally { + delete process.env[FORK_CONTEXT_FILE_ENV]; + rmSync(file, { force: true }); + } +} + +/** In a fork: provides `project` what the parent's global setup provided it. */ +export function provideParentContext(project: TestProject): void { + const file = process.env[FORK_CONTEXT_FILE_ENV]; + if (file === undefined) { + throw new Error("[CodSpeed] fork started without its parent's context"); + } + const contexts: ProvidedContexts = deserialize(readFileSync(file)); + // `provide` is typed by the user's `ProvidedContext` augmentation, empty here. + const provide = project.provide as (key: string, value: unknown) => void; + for (const [key, value] of Object.entries(contexts[project.name] ?? {})) { + provide(key, value); + } +} diff --git a/packages/vitest-plugin/src/v5/forkReporter.ts b/packages/vitest-plugin/src/v5/forkReporter.ts new file mode 100644 index 00000000..7055eb7b --- /dev/null +++ b/packages/vitest-plugin/src/v5/forkReporter.ts @@ -0,0 +1,57 @@ +import { + resolveWalltimeForks, + runForks, + samplesToResult, + type Benchmark, + type WalltimeOptions, +} from "@codspeed/core"; +import type { Reporter, TestModule } from "vitest/node"; +import { + forkedBenchmarks, + tinybenchTaskToBenchmark, + writeAndLogWalltimeResults, +} from "../instrument"; +import { withForkContext } from "./forkContext"; + +/** + * Measures the benchmarks the workers queued (see `provider.ts`) in fresh + * processes and reports their pooled rounds. Runs in the main process once the + * run is over: every file has torn down, so resources its hooks held (ports, + * databases, ...) are free for the forks. The global setup is still up (it + * tears down on close), so the forks skip it and inherit what it provided. + */ +export function forkReporter(options: WalltimeOptions): Reporter { + return { + async onTestRunEnd(modules, _errors, reason) { + if (reason === "interrupted") return; + const benchmarks = await withForkContext(modules, () => + measureInForks(modules, options), + ); + writeAndLogWalltimeResults(benchmarks); + }, + }; +} + +async function measureInForks( + modules: readonly TestModule[], + options: WalltimeOptions, +): Promise { + const benchmarks: Benchmark[] = []; + for (const module of modules) { + for (const test of module.children.allTests()) { + for (const queued of forkedBenchmarks(test.meta())) { + const forks = resolveWalltimeForks(queued.forks ?? options.forks); + const samples = await runForks(queued.uri, forks); + const benchmark = tinybenchTaskToBenchmark( + { name: queued.name, result: samplesToResult(samples) }, + queued.uri, + queued.options, + ); + if (benchmark) { + benchmarks.push(benchmark); + } + } + } + } + return benchmarks; +} diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index ef38fe0e..bd15ae80 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -2,13 +2,16 @@ import { getGitDir, getInstrumentMode, InstrumentHooks, + walltimeIsolation, optimizeFunction, optimizeFunctionSync, setupCore, teardownCore, wrapWithRootFrame, wrapWithRootFrameSync, + writeForkSamples, type Benchmark, + type WalltimeOptions, } from "@codspeed/core"; import { createRequire } from "module"; import path from "path"; @@ -21,6 +24,7 @@ import type { } from "vitest"; import { installInstrumentHooks, + setForkedBenchmarks, tinybenchTaskToBenchmark, writeAndLogWalltimeResults, type TinybenchBench, @@ -221,6 +225,16 @@ async function runWalltime( getUri: (name: string) => string, test: BenchmarkGroup["test"], ): Promise { + const mode = walltimeIsolation(); + if (mode.kind === "Forks") { + await queueForks(registrations, options, getUri, test); + return registrations.map(({ name }) => analysisResult(name)); + } + // A fork measures only its benchmark; Vitest still gets every result. + const measured = registrations.filter( + ({ name }) => mode.kind !== "Fork" || mode.uri === getUri(name), + ); + const { Bench } = await importHostTinybench(); const bench = new Bench({ signal: test.context.signal, @@ -229,7 +243,7 @@ async function runWalltime( retainSamples: true, }); - for (const { name, fn, fnOpts } of registrations) { + for (const { name, fn, fnOpts } of measured) { // tinybench v6 keeps `fn` private, so the root frame is baked in here bench.add( name, @@ -242,9 +256,48 @@ async function runWalltime( const tasks = await bench.run(); throwOnErroredTasks(tasks); - collectWalltimeResults(bench, getUri); + if (mode.kind === "Fork") { + for (const task of tasks) { + writeForkSamples(mode.samplesFile, [ + ...(task.result?.latency.samples ?? []), + ]); + } + } else { + collectWalltimeResults(bench, getUri); + } - return tasks.map(tinybenchTaskToBenchResult); + const results = new Map( + tasks.map((task) => [task.name, tinybenchTaskToBenchResult(task)]), + ); + return registrations.map( + ({ name }) => results.get(name) ?? analysisResult(name), + ); +} + +/** + * Queues every benchmark for measurement in fresh processes once the run is + * over (see `forkReporter.ts`). Vitest gets zeroed results, like in analysis mode. + */ +async function queueForks( + registrations: BenchRegistrationInput[], + options: BenchRunOptions | undefined, + getUri: (name: string) => string, + test: BenchmarkGroup["test"], +): Promise { + const { Bench } = await importHostTinybench(); + // Never run: it resolves tinybench's defaults for the reported config. + const bench = new Bench(options); + setForkedBenchmarks( + test.meta, + registrations.map(({ name, fnOpts }) => ({ + uri: getUri(name), + name, + forks: + (fnOpts as WalltimeOptions | undefined)?.forks ?? + (options as WalltimeOptions | undefined)?.forks, + options: resolvedOptions(bench), + })), + ); } /** Fail the test like Vitest's own provider does, instead of returning empty results. */ @@ -260,11 +313,9 @@ function throwOnErroredTasks(tasks: TinybenchTask[]): void { } } -function collectWalltimeResults( - bench: TinybenchWithTasks, - getUri: (name: string) => string, -): void { - const options: TinybenchOptions = { +/** The options the bench runs with, as reported to CodSpeed. */ +function resolvedOptions(bench: TinybenchWithTasks): TinybenchOptions { + return { time: bench.time, iterations: bench.iterations, // A bench with warmup turned off keeps its warmup timings, so they'd @@ -272,6 +323,13 @@ function collectWalltimeResults( warmupTime: bench.warmup ? bench.warmupTime : 0, warmupIterations: bench.warmup ? bench.warmupIterations : 0, }; +} + +function collectWalltimeResults( + bench: TinybenchWithTasks, + getUri: (name: string) => string, +): void { + const options = resolvedOptions(bench); const benchmarks: Benchmark[] = []; for (const task of bench.tasks) { if (task.result?.state !== "completed") continue; diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index 46c171d7..7a5cf3e1 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -1,11 +1,29 @@ -import { getInstrumentMode } from "@codspeed/core"; +import { + getGitDir, + getInstrumentMode, + walltimeIsolation, + type WalltimeOptions, +} from "@codspeed/core"; import { join } from "path"; import { type ViteUserConfig } from "vitest/config"; -import { version as vitestVersion } from "vitest/node"; +import { + type Reporter, + type TestProject, + version as vitestVersion, +} from "vitest/node"; +import { provideParentContext } from "./v5/forkContext"; +import { forkReporter } from "./v5/forkReporter"; /** The parts of the Vitest integration that depend on the installed major. */ export interface VitestBackend { isActive(mode: string): boolean; + configureIsolation(userConfig: ViteUserConfig): void; + + /** Runs per project, before Vitest initializes its global setup. */ + configureProject(project: TestProject): void; + + /** The reporter that measures the queued benchmarks in forks, if any. */ + forkReporter(options: WalltimeOptions): Reporter | undefined; getBenchmarkTestConfig( v8Flags: string[], @@ -34,6 +52,39 @@ class V5Backend implements VitestBackend { return getInstrumentMode() !== "disabled"; } + configureIsolation(userConfig: ViteUserConfig): void { + if (getInstrumentMode() !== "walltime") return; + + const isolationMode = walltimeIsolation(); + if (isolationMode.kind !== "Fork") return; + restrictToBenchmarkFile(userConfig, isolationMode.uri); + // The parent's global setup is still up while its forks run: forks skip + // it and inherit what it provided (see `configureProject`). + userConfig.test!.globalSetup = []; + } + + configureProject(project: TestProject): void { + if (getInstrumentMode() !== "walltime") return; + if (walltimeIsolation().kind !== "Fork") return; + provideParentContext(project); + } + + /** + * The parent of a forked walltime run measures nothing in its workers: they + * queue the benchmarks, and the main process forks them once the run is over. + */ + forkReporter(options: WalltimeOptions): Reporter | undefined { + if (getInstrumentMode() !== "walltime") return undefined; + const isolationMode = walltimeIsolation(); + switch (isolationMode.kind) { + case "Forks": + return forkReporter(options); + case "SharedProcess": + case "Fork": + return undefined; + } + } + getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, @@ -53,6 +104,24 @@ class V5Backend implements VitestBackend { } } +function restrictToBenchmarkFile( + userConfig: ViteUserConfig, + uri: string, +): void { + const gitDir = getGitDir(userConfig.root ?? process.cwd()); + if (gitDir === undefined) { + throw new Error("Could not find a git repository"); + } + // Replace rather than merge include globs: isolated children must not + // transform and import unrelated benchmark files. + const file = join(gitDir, uri.split("::")[0]) + .replace(/\\/g, "/") + .replace(/[\\*?{}()[\]!+@]/g, "\\$&"); + userConfig.test ??= {}; + userConfig.test.benchmark ??= {}; + userConfig.test.benchmark.include = [file]; +} + /** * Vitest 3/4: `vitest bench` runs in the `"benchmark"` Vite mode, and each * instrument mode has its own `test.runner` (`legacy/analysis`, `legacy/walltime`). @@ -64,6 +133,11 @@ class LegacyBackend implements VitestBackend { return mode === "benchmark"; } + // Vitest 3/4 do not support process isolation. + configureIsolation(): void {} + configureProject(): void {} + forkReporter(): undefined {} + getBenchmarkTestConfig( v8Flags: string[], resolveFile: (name: string) => string, diff --git a/packages/vitest-plugin/tests/fixtures/isolation/globalSetup.ts b/packages/vitest-plugin/tests/fixtures/isolation/globalSetup.ts new file mode 100644 index 00000000..7aedb03c --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/globalSetup.ts @@ -0,0 +1,34 @@ +import { createServer, type Server } from "net"; +import type { TestProject } from "vitest/node"; + +// Fixed on purpose: a fork that ran this setup again would fail with EADDRINUSE. +const PORT = 48_124; + +declare module "vitest" { + export interface ProvidedContext { + port: number; + } +} + +// Vitest 5 runs this for the base project and for the `(bench)` project it +// clones from it, in the same process: bind once, and provide to both. +let server: Server | undefined; + +export default async function setup(project: TestProject) { + if (server === undefined) { + const listening = Promise.withResolvers(); + server = createServer((socket) => socket.end()) + .once("error", listening.reject) + .listen(PORT, listening.resolve); + await listening.promise; + } + project.provide("port", PORT); + + return () => { + const closed = Promise.withResolvers(); + if (server === undefined) return closed.resolve(); + server.close(() => closed.resolve()); + server = undefined; + return closed.promise; + }; +} diff --git a/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts new file mode 100644 index 00000000..834da854 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/pids.bench.ts @@ -0,0 +1,47 @@ +import { appendFileSync } from "fs"; +import { connect } from "net"; +import { describe, inject, test, type BenchRegistration } from "vitest"; + +// Vitest swallows what workers print, so the PIDs that run each benchmark go +// to a file. `beforeAll` only runs when tinybench measures the benchmark. +// It also connects to the global setup's server: a fork without the provided +// port, or one that re-ran the setup, fails here. +function logPid(name: string) { + return async () => { + const closed = Promise.withResolvers(); + connect(inject("port")) + .once("error", closed.reject) + .once("close", () => closed.resolve()); + await closed.promise; + appendFileSync( + process.env.ISOLATION_LOG_FILE!, + `ran ${process.pid} ${name}\n`, + ); + }; +} + +const firstForks = process.env.ISOLATION_FIXTURE_FIRST_FORKS; + +describe("isolation", () => { + test("benchmarks", async ({ bench }) => { + const benchmarks: BenchRegistration[] = [ + bench( + "first", + { + beforeAll: logPid("first"), + ...(firstForks && { forks: Number(firstForks) }), + }, + () => {}, + ), + bench("second", { beforeAll: logPid("second") }, () => {}), + ]; + if (process.env.ISOLATION_FIXTURE_FAIL !== undefined) { + benchmarks.push( + bench("failing", () => { + throw new Error("benchmark failed"); + }), + ); + } + await bench.compare(...benchmarks, { iterations: 4, time: 1 }); + }); +}); diff --git a/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts new file mode 100644 index 00000000..2adcb374 --- /dev/null +++ b/packages/vitest-plugin/tests/fixtures/isolation/vitest.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vitest/config"; +import codspeedPlugin from "../../../dist/index.mjs"; + +const pluginForks = process.env.ISOLATION_FIXTURE_PLUGIN_FORKS; + +export default defineConfig({ + plugins: [codspeedPlugin(pluginForks ? { forks: Number(pluginForks) } : {})], + define: { + __VERSION__: JSON.stringify("1.0.0"), + }, + test: { + include: ["**/*.bench.ts"], + globalSetup: ["./globalSetup.ts"], + }, +}); diff --git a/packages/vitest-plugin/vitest.config.ts b/packages/vitest-plugin/vitest.config.ts index 672e3a27..e874589b 100644 --- a/packages/vitest-plugin/vitest.config.ts +++ b/packages/vitest-plugin/vitest.config.ts @@ -10,5 +10,8 @@ export default defineConfig({ test: { exclude: ["**/tests/**/*", "**/.rollup.cache/**/*"], mockReset: true, + benchmark: { + exclude: ["**/node_modules/**", "**/tests/**/*", "**/.rollup.cache/**/*"], + }, }, }); From eae555ce9c7eedc35b2b8bec08b14af4ea7406f3 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:46:53 +0100 Subject: [PATCH 07/14] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/walltime.ts | 22 ++++--------------- .../tests/fixtures/isolation.mjs | 8 +------ .../tests/isolation.integ.test.ts | 21 ++++-------------- 3 files changed, 9 insertions(+), 42 deletions(-) diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 7168cb0c..acf8837b 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -103,7 +103,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = await runForks(uri, isolationMode.forks); - setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -123,7 +122,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = runForksSync(uri, isolationMode.forks); - setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -153,9 +151,10 @@ class WalltimeBenchRunner extends BaseBenchRunner { } // A fork hands its samples to the parent, which reports the benchmark. if (this.isolationMode.kind === "Fork") { - writeForkSamples(this.isolationMode.samplesFile, [ - ...task.result.latency.samples, - ]); + writeForkSamples( + this.isolationMode.samplesFile, + task.result.latency.samples, + ); return; } this.registerCodspeedBenchmark(task, task.result); @@ -238,16 +237,3 @@ function convertTinybenchResultToBenchmarkStats( warmup_iters: warmupIterations, }; } - -/** - * Fills `task.result` from the pooled fork samples so `bench.table()` and - * `bench.results` work. Only tinybench 4/5 expose a way in; v6 keeps the - * result `#private`. - */ -function setTaskResult(task: Task, samples: number[]): void { - const { processRunResult } = task as unknown as { - processRunResult?: (run: { latencySamples: number[] }) => void; - }; - // tinybench sorts the samples in place. - processRunResult?.call(task, { latencySamples: [...samples] }); -} diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs index e347110d..118c5b07 100644 --- a/packages/tinybench-plugin/tests/fixtures/isolation.mjs +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -1,5 +1,5 @@ // Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a -// benchmark, and the sample count of every `task.result` the parent ends with. +// benchmark. import { Bench } from "tinybench"; import { withCodSpeed } from "../../dist/index.es5.js"; @@ -24,9 +24,3 @@ if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { } else { await bench.run(); } - -if (process.env.CODSPEED_ISOLATED_BENCHMARK === undefined) { - for (const task of bench.tasks) { - console.log(`result ${task.name} ${task.result?.samples.length ?? 0}`); - } -} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index b78a5db4..2f2a2962 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -19,7 +19,6 @@ const MIN_ROUNDS_PER_FORK = 4; /** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 60_000; const RAN_LINE = /^ran (\d+) (.+)$/; -const RESULT_LINE = /^result (.+) (\d+)$/; interface FixtureRun { status: number | null; @@ -29,8 +28,6 @@ interface FixtureRun { parentPid: number; /** Rounds of each benchmark in the results file. */ roundsByName: Map; - /** Samples in the parent's `task.result` of each benchmark. */ - parentSamplesByName: Map; } interface FixtureOptions { @@ -83,19 +80,12 @@ async function runFixture({ const [status] = (await once(child, "close")) as [number | null]; const pidsByName = new Map>(); - const parentSamplesByName = new Map(); for (const line of stdout.split("\n")) { const ran = RAN_LINE.exec(line); - if (ran) { - const pids = pidsByName.get(ran[2]) ?? new Set(); - pids.add(Number(ran[1])); - pidsByName.set(ran[2], pids); - continue; - } - const result = RESULT_LINE.exec(line); - if (result) { - parentSamplesByName.set(result[1], Number(result[2])); - } + if (!ran) continue; + const pids = pidsByName.get(ran[2]) ?? new Set(); + pids.add(Number(ran[1])); + pidsByName.set(ran[2], pids); } const roundsByName = new Map(); @@ -111,7 +101,6 @@ async function runFixture({ pidsByName, parentPid: child.pid!, roundsByName, - parentSamplesByName, }; } finally { rmSync(profileFolder, { recursive: true, force: true }); @@ -141,8 +130,6 @@ describe("walltime forks", () => { expect(rounds).toBeGreaterThanOrEqual( DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, ); - // The parent reports the pooled rounds as its own `task.result`. - expect(run.parentSamplesByName.get(name)).toBe(rounds); } // Forks of different benchmarks are different processes too. const all = new Set( From 6c9d38d033c81bdf7869cb9e221a93906527dc03 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:47:39 +0100 Subject: [PATCH 08/14] fixup! feat(vitest): run walltime benchmarks in forks --- packages/vitest-plugin/src/index.ts | 15 +------ packages/vitest-plugin/src/vitestBackend.ts | 46 +++++++++++---------- 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index 0b7d7854..fdfbc7c1 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -10,13 +10,8 @@ import { import { join } from "path"; import { Plugin } from "vite"; import { type ViteUserConfig } from "vitest/config"; -import { type Vitest } from "vitest/node"; import { resolveVitestBackend } from "./vitestBackend"; -// `configureVitest` runs once per project, but one reporter sees every -// project's modules: a second one would fork every benchmark twice. -const vitestsWithForkReporter = new WeakSet(); - // get this file's directory path from import.meta.url const __dirname = new URL(".", import.meta.url).pathname; const isFileInTs = import.meta.url.endsWith(".ts"); @@ -60,15 +55,7 @@ export default function codspeedPlugin(options: WalltimeOptions = {}): Plugin { }; }, configureVitest({ project, vitest }) { - const backend = resolveVitestBackend(); - backend.configureProject(project); - if (vitestsWithForkReporter.has(vitest)) return; - vitestsWithForkReporter.add(vitest); - const reporter = backend.forkReporter(options); - // Vitest builds its reporters from the config after this hook runs. - if (reporter) { - vitest.config.reporters.push(reporter); - } + resolveVitestBackend().configureVitest(project, vitest, options); }, }; } diff --git a/packages/vitest-plugin/src/vitestBackend.ts b/packages/vitest-plugin/src/vitestBackend.ts index 7a5cf3e1..d1867bfa 100644 --- a/packages/vitest-plugin/src/vitestBackend.ts +++ b/packages/vitest-plugin/src/vitestBackend.ts @@ -7,8 +7,8 @@ import { import { join } from "path"; import { type ViteUserConfig } from "vitest/config"; import { - type Reporter, type TestProject, + type Vitest, version as vitestVersion, } from "vitest/node"; import { provideParentContext } from "./v5/forkContext"; @@ -20,10 +20,11 @@ export interface VitestBackend { configureIsolation(userConfig: ViteUserConfig): void; /** Runs per project, before Vitest initializes its global setup. */ - configureProject(project: TestProject): void; - - /** The reporter that measures the queued benchmarks in forks, if any. */ - forkReporter(options: WalltimeOptions): Reporter | undefined; + configureVitest( + project: TestProject, + vitest: Vitest, + options: WalltimeOptions, + ): void; getBenchmarkTestConfig( v8Flags: string[], @@ -41,6 +42,10 @@ export function resolveVitestBackend(): VitestBackend { return major >= 5 ? new V5Backend() : new LegacyBackend(major); } +// `configureVitest` runs once per project, but one reporter sees every +// project's modules: a second one would fork every benchmark twice. +const vitestsWithForkReporter = new WeakSet(); + /** Vitest 5+: benchmarks run through a `benchmark.provider` (`v5/provider.ts`). */ class V5Backend implements VitestBackend { /** @@ -63,26 +68,26 @@ class V5Backend implements VitestBackend { userConfig.test!.globalSetup = []; } - configureProject(project: TestProject): void { - if (getInstrumentMode() !== "walltime") return; - if (walltimeIsolation().kind !== "Fork") return; - provideParentContext(project); - } - /** * The parent of a forked walltime run measures nothing in its workers: they * queue the benchmarks, and the main process forks them once the run is over. */ - forkReporter(options: WalltimeOptions): Reporter | undefined { - if (getInstrumentMode() !== "walltime") return undefined; + configureVitest( + project: TestProject, + vitest: Vitest, + options: WalltimeOptions, + ): void { + if (getInstrumentMode() !== "walltime") return; const isolationMode = walltimeIsolation(); - switch (isolationMode.kind) { - case "Forks": - return forkReporter(options); - case "SharedProcess": - case "Fork": - return undefined; + if (isolationMode.kind === "Fork") { + provideParentContext(project); + return; } + if (isolationMode.kind !== "Forks") return; + if (vitestsWithForkReporter.has(vitest)) return; + vitestsWithForkReporter.add(vitest); + // Vitest builds its reporters from the config after this hook runs. + vitest.config.reporters.push(forkReporter(options)); } getBenchmarkTestConfig( @@ -135,8 +140,7 @@ class LegacyBackend implements VitestBackend { // Vitest 3/4 do not support process isolation. configureIsolation(): void {} - configureProject(): void {} - forkReporter(): undefined {} + configureVitest(): void {} getBenchmarkTestConfig( v8Flags: string[], From 409385ce5c0e9bbafbc5c21decc394a5b6324871 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:47:55 +0100 Subject: [PATCH 09/14] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/analysis.ts | 4 +--- packages/tinybench-plugin/src/shared.ts | 25 +++++++++++------------ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 54c8d975..65bd6183 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -14,9 +14,7 @@ export function setupCodspeedAnalysisBench( bench: Bench, rootCallingFile: string, ): void { - const runner = new AnalysisBenchRunner(bench, rootCallingFile, { - kind: "SharedProcess", - }); + const runner = new AnalysisBenchRunner(bench, rootCallingFile); runner.setupBenchMethods(); } diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 1e10b4bf..2d86eced 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -8,17 +8,15 @@ declare const __VERSION__: string; export abstract class BaseBenchRunner { protected bench: Bench; protected rootCallingFile: string; - /** Walltime runners pass the isolation of this process; analysis shares it. */ - protected readonly isolationMode: IsolationMode; constructor( bench: Bench, rootCallingFile: string, - isolationMode: IsolationMode, + /** Only walltime runs are isolated. */ + protected readonly isolationMode: IsolationMode = { kind: "SharedProcess" }, ) { this.bench = bench; this.rootCallingFile = rootCallingFile; - this.isolationMode = isolationMode; } protected get isChild(): boolean { @@ -85,7 +83,8 @@ export abstract class BaseBenchRunner { } this.setupBenchRun(); - for (const { task, uri } of tasks) { + for (const task of tasks) { + const uri = this.getTaskUri(task); await this.runTaskAsync(task, uri); } @@ -99,7 +98,8 @@ export abstract class BaseBenchRunner { } this.setupBenchRun(); - for (const { task, uri } of tasks) { + for (const task of tasks) { + const uri = this.getTaskUri(task); this.runTaskSync(task, uri); } @@ -108,12 +108,11 @@ export abstract class BaseBenchRunner { } /** The tasks this process measures: all of them, or the isolated one. */ - private selectTasks(): { task: Task; uri: string }[] { - return this.bench.tasks - .map((task) => ({ task, uri: this.getTaskUri(task) })) - .filter( - ({ uri }) => - this.isolationMode.kind !== "Fork" || this.isolationMode.uri === uri, - ); + private selectTasks(): Task[] { + const { isolationMode } = this; + if (isolationMode.kind !== "Fork") return this.bench.tasks; + return this.bench.tasks.filter( + (task) => this.getTaskUri(task) === isolationMode.uri, + ); } } From ff3bce0f5eae1ef1555d2960acbe8ccc8ddb2668 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:48:04 +0100 Subject: [PATCH 10/14] fixup! feat(vitest): run walltime benchmarks in forks --- .../tinybench-plugin/tests/isolation.integ.test.ts | 13 ------------- .../vitest-plugin/src/__tests__/isolation.test.ts | 12 ------------ 2 files changed, 25 deletions(-) diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index 2f2a2962..f7ec64cd 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -153,19 +153,6 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); - it( - "takes the fork count from the environment over the option", - async () => { - const run = await runFixture({ forksOption: 2, forksEnv: 1 }); - - expect(run.status).toBe(0); - for (const name of BENCHMARKS) { - expect(forkPids(run, name).size).toBe(1); - } - }, - RUN_TIMEOUT_MS, - ); - it( "runs every benchmark in the parent process when isolation is disabled", async () => { diff --git a/packages/vitest-plugin/src/__tests__/isolation.test.ts b/packages/vitest-plugin/src/__tests__/isolation.test.ts index 0169373e..e5b0f92a 100644 --- a/packages/vitest-plugin/src/__tests__/isolation.test.ts +++ b/packages/vitest-plugin/src/__tests__/isolation.test.ts @@ -121,18 +121,6 @@ describe("walltime forks", () => { RUN_TIMEOUT_MS, ); - it( - "takes the environment over the benchmark's option", - async () => { - const run = await runFixture({ firstForks: 2, forksEnv: 1 }); - - expect(run.status).toBe(0); - expect(run.pidsByName.get("first")?.size).toBe(1); - expect(run.pidsByName.get("second")?.size).toBe(1); - }, - RUN_TIMEOUT_MS, - ); - it( "runs every benchmark in one process when isolation is disabled", async () => { From 979fac59bedde665822c44e0e5aaba5ba2f90e7d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 17:48:05 +0100 Subject: [PATCH 11/14] fixup! feat(vitest): run walltime benchmarks in forks --- .../benches/exclusive-resource.bench.ts | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 packages/vitest-plugin/benches/exclusive-resource.bench.ts diff --git a/packages/vitest-plugin/benches/exclusive-resource.bench.ts b/packages/vitest-plugin/benches/exclusive-resource.bench.ts deleted file mode 100644 index a6d45b55..00000000 --- a/packages/vitest-plugin/benches/exclusive-resource.bench.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { connect, createServer, type Server } from "node:net"; -import { afterAll, beforeAll, test } from "vitest"; - -// Fixed on purpose: a second process holding it must fail with EADDRINUSE. -const PORT = 48_123; -let server: Server; - -beforeAll(() => { - const { promise, resolve, reject } = Promise.withResolvers(); - server = createServer((socket) => socket.end()) - .once("error", reject) - .listen(PORT, resolve); - return promise; -}); - -afterAll(() => { - const { promise, resolve } = Promise.withResolvers(); - server.close(() => resolve()); - return promise; -}); - -test("exclusive port", async ({ bench }) => { - await bench("connect", async () => { - const { promise, resolve } = Promise.withResolvers(); - connect(PORT).once("close", resolve); - return promise; - }).run(); -}); From a591f57bcf4302ddb4a46438f58b2eabca1f994a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 18:28:11 +0100 Subject: [PATCH 12/14] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/walltime.ts | 22 +++++++++++++++---- .../tests/fixtures/isolation.mjs | 8 ++++++- .../tests/isolation.integ.test.ts | 21 ++++++++++++++---- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index acf8837b..7168cb0c 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -103,6 +103,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = await runForks(uri, isolationMode.forks); + setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -122,6 +123,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = runForksSync(uri, isolationMode.forks); + setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -151,10 +153,9 @@ class WalltimeBenchRunner extends BaseBenchRunner { } // A fork hands its samples to the parent, which reports the benchmark. if (this.isolationMode.kind === "Fork") { - writeForkSamples( - this.isolationMode.samplesFile, - task.result.latency.samples, - ); + writeForkSamples(this.isolationMode.samplesFile, [ + ...task.result.latency.samples, + ]); return; } this.registerCodspeedBenchmark(task, task.result); @@ -237,3 +238,16 @@ function convertTinybenchResultToBenchmarkStats( warmup_iters: warmupIterations, }; } + +/** + * Fills `task.result` from the pooled fork samples so `bench.table()` and + * `bench.results` work. Only tinybench 4/5 expose a way in; v6 keeps the + * result `#private`. + */ +function setTaskResult(task: Task, samples: number[]): void { + const { processRunResult } = task as unknown as { + processRunResult?: (run: { latencySamples: number[] }) => void; + }; + // tinybench sorts the samples in place. + processRunResult?.call(task, { latencySamples: [...samples] }); +} diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs index 118c5b07..e347110d 100644 --- a/packages/tinybench-plugin/tests/fixtures/isolation.mjs +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -1,5 +1,5 @@ // Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a -// benchmark. +// benchmark, and the sample count of every `task.result` the parent ends with. import { Bench } from "tinybench"; import { withCodSpeed } from "../../dist/index.es5.js"; @@ -24,3 +24,9 @@ if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { } else { await bench.run(); } + +if (process.env.CODSPEED_ISOLATED_BENCHMARK === undefined) { + for (const task of bench.tasks) { + console.log(`result ${task.name} ${task.result?.samples.length ?? 0}`); + } +} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index f7ec64cd..5977a192 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -19,6 +19,7 @@ const MIN_ROUNDS_PER_FORK = 4; /** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 60_000; const RAN_LINE = /^ran (\d+) (.+)$/; +const RESULT_LINE = /^result (.+) (\d+)$/; interface FixtureRun { status: number | null; @@ -28,6 +29,8 @@ interface FixtureRun { parentPid: number; /** Rounds of each benchmark in the results file. */ roundsByName: Map; + /** Samples in the parent's `task.result` of each benchmark. */ + parentSamplesByName: Map; } interface FixtureOptions { @@ -80,12 +83,19 @@ async function runFixture({ const [status] = (await once(child, "close")) as [number | null]; const pidsByName = new Map>(); + const parentSamplesByName = new Map(); for (const line of stdout.split("\n")) { const ran = RAN_LINE.exec(line); - if (!ran) continue; - const pids = pidsByName.get(ran[2]) ?? new Set(); - pids.add(Number(ran[1])); - pidsByName.set(ran[2], pids); + if (ran) { + const pids = pidsByName.get(ran[2]) ?? new Set(); + pids.add(Number(ran[1])); + pidsByName.set(ran[2], pids); + continue; + } + const result = RESULT_LINE.exec(line); + if (result) { + parentSamplesByName.set(result[1], Number(result[2])); + } } const roundsByName = new Map(); @@ -101,6 +111,7 @@ async function runFixture({ pidsByName, parentPid: child.pid!, roundsByName, + parentSamplesByName, }; } finally { rmSync(profileFolder, { recursive: true, force: true }); @@ -130,6 +141,8 @@ describe("walltime forks", () => { expect(rounds).toBeGreaterThanOrEqual( DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, ); + // The parent reports the pooled rounds as its own `task.result`. + expect(run.parentSamplesByName.get(name)).toBe(rounds); } // Forks of different benchmarks are different processes too. const all = new Set( From 2e8744995c9b5fd47f988da3cb75146f73a3512f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 4 Nov 2026 19:28:45 +0100 Subject: [PATCH 13/14] fixup! feat(tinybench): run walltime benchmarks in forks --- packages/tinybench-plugin/src/walltime.ts | 22 ++++--------------- .../tests/fixtures/isolation.mjs | 8 +------ .../tests/isolation.integ.test.ts | 21 ++++-------------- 3 files changed, 9 insertions(+), 42 deletions(-) diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 7168cb0c..acf8837b 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -103,7 +103,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = await runForks(uri, isolationMode.forks); - setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -123,7 +122,6 @@ class WalltimeBenchRunner extends BaseBenchRunner { const { isolationMode } = this; if (isolationMode.kind === "Forks") { const samples = runForksSync(uri, isolationMode.forks); - setTaskResult(task, samples); return this.registerCodspeedBenchmark(task, samplesToResult(samples)); } @@ -153,9 +151,10 @@ class WalltimeBenchRunner extends BaseBenchRunner { } // A fork hands its samples to the parent, which reports the benchmark. if (this.isolationMode.kind === "Fork") { - writeForkSamples(this.isolationMode.samplesFile, [ - ...task.result.latency.samples, - ]); + writeForkSamples( + this.isolationMode.samplesFile, + task.result.latency.samples, + ); return; } this.registerCodspeedBenchmark(task, task.result); @@ -238,16 +237,3 @@ function convertTinybenchResultToBenchmarkStats( warmup_iters: warmupIterations, }; } - -/** - * Fills `task.result` from the pooled fork samples so `bench.table()` and - * `bench.results` work. Only tinybench 4/5 expose a way in; v6 keeps the - * result `#private`. - */ -function setTaskResult(task: Task, samples: number[]): void { - const { processRunResult } = task as unknown as { - processRunResult?: (run: { latencySamples: number[] }) => void; - }; - // tinybench sorts the samples in place. - processRunResult?.call(task, { latencySamples: [...samples] }); -} diff --git a/packages/tinybench-plugin/tests/fixtures/isolation.mjs b/packages/tinybench-plugin/tests/fixtures/isolation.mjs index e347110d..118c5b07 100644 --- a/packages/tinybench-plugin/tests/fixtures/isolation.mjs +++ b/packages/tinybench-plugin/tests/fixtures/isolation.mjs @@ -1,5 +1,5 @@ // Run by `isolation.integ.test.ts`. Prints the PID of every process that runs a -// benchmark, and the sample count of every `task.result` the parent ends with. +// benchmark. import { Bench } from "tinybench"; import { withCodSpeed } from "../../dist/index.es5.js"; @@ -24,9 +24,3 @@ if (process.env.ISOLATION_FIXTURE_SYNC !== undefined) { } else { await bench.run(); } - -if (process.env.CODSPEED_ISOLATED_BENCHMARK === undefined) { - for (const task of bench.tasks) { - console.log(`result ${task.name} ${task.result?.samples.length ?? 0}`); - } -} diff --git a/packages/tinybench-plugin/tests/isolation.integ.test.ts b/packages/tinybench-plugin/tests/isolation.integ.test.ts index 5977a192..f7ec64cd 100644 --- a/packages/tinybench-plugin/tests/isolation.integ.test.ts +++ b/packages/tinybench-plugin/tests/isolation.integ.test.ts @@ -19,7 +19,6 @@ const MIN_ROUNDS_PER_FORK = 4; /** Fails a run whose benchmark process got stuck instead of blocking the suite. */ const RUN_TIMEOUT_MS = 60_000; const RAN_LINE = /^ran (\d+) (.+)$/; -const RESULT_LINE = /^result (.+) (\d+)$/; interface FixtureRun { status: number | null; @@ -29,8 +28,6 @@ interface FixtureRun { parentPid: number; /** Rounds of each benchmark in the results file. */ roundsByName: Map; - /** Samples in the parent's `task.result` of each benchmark. */ - parentSamplesByName: Map; } interface FixtureOptions { @@ -83,19 +80,12 @@ async function runFixture({ const [status] = (await once(child, "close")) as [number | null]; const pidsByName = new Map>(); - const parentSamplesByName = new Map(); for (const line of stdout.split("\n")) { const ran = RAN_LINE.exec(line); - if (ran) { - const pids = pidsByName.get(ran[2]) ?? new Set(); - pids.add(Number(ran[1])); - pidsByName.set(ran[2], pids); - continue; - } - const result = RESULT_LINE.exec(line); - if (result) { - parentSamplesByName.set(result[1], Number(result[2])); - } + if (!ran) continue; + const pids = pidsByName.get(ran[2]) ?? new Set(); + pids.add(Number(ran[1])); + pidsByName.set(ran[2], pids); } const roundsByName = new Map(); @@ -111,7 +101,6 @@ async function runFixture({ pidsByName, parentPid: child.pid!, roundsByName, - parentSamplesByName, }; } finally { rmSync(profileFolder, { recursive: true, force: true }); @@ -141,8 +130,6 @@ describe("walltime forks", () => { expect(rounds).toBeGreaterThanOrEqual( DEFAULT_FORKS * MIN_ROUNDS_PER_FORK, ); - // The parent reports the pooled rounds as its own `task.result`. - expect(run.parentSamplesByName.get(name)).toBe(rounds); } // Forks of different benchmarks are different processes too. const all = new Set( From 38afc2aac4aeb0f306c051d44b1c3362c74769e6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 3 Nov 2026 18:22:14 +0100 Subject: [PATCH 14/14] perf(tinybench): batch fast walltime invocations into rounds Calibrate approximately 1 ms rounds from warmup and share the iteration count across isolated forks. Normalize latency statistics per invocation while retaining raw total time. Keep per-invocation hooks and concurrent tasks unbatched, and cover shared, forked, synchronous, and asynchronous execution. --- packages/core/src/isolation.ts | 128 +++++++--- packages/core/tests/fixtures/isolation.cjs | 23 +- packages/core/tests/isolation.integ.test.ts | 87 ++++++- packages/tinybench-plugin/src/batching.ts | 107 +++++++++ packages/tinybench-plugin/src/benchOptions.ts | 1 + packages/tinybench-plugin/src/index.ts | 35 ++- packages/tinybench-plugin/src/taskData.ts | 1 + packages/tinybench-plugin/src/walltime.ts | 52 +++-- .../tests/batching.integ.test.ts | 218 ++++++++++++++++++ .../tests/fixtures/batching.mjs | 106 +++++++++ packages/vitest-plugin/src/v5/forkReporter.ts | 2 +- packages/vitest-plugin/src/v5/provider.ts | 2 +- 12 files changed, 705 insertions(+), 57 deletions(-) create mode 100644 packages/tinybench-plugin/src/batching.ts create mode 100644 packages/tinybench-plugin/tests/batching.integ.test.ts create mode 100644 packages/tinybench-plugin/tests/fixtures/batching.mjs diff --git a/packages/core/src/isolation.ts b/packages/core/src/isolation.ts index 6f0f1d36..fd833b5c 100644 --- a/packages/core/src/isolation.ts +++ b/packages/core/src/isolation.ts @@ -23,6 +23,13 @@ export const DEFAULT_WALLTIME_FORKS = 3; export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK"; /** Set on each fork: the file where it writes its round samples. */ export const FORK_SAMPLES_FILE_ENV = "CODSPEED_FORK_SAMPLES_FILE"; +/** + * Set on the forks after the first one by {@link runForks}: the number of + * iterations that the first fork measured per round. Every fork of a benchmark + * MUST batch the same way, because the samples of all forks are pooled. + */ +export const FORK_ITERATIONS_PER_ROUND_ENV = + "CODSPEED_FORK_ITERATIONS_PER_ROUND"; /** Walltime settings that every integration exposes next to its own options. */ export interface WalltimeOptions { @@ -40,8 +47,16 @@ export type IsolationMode = | { kind: "SharedProcess" } /** Every benchmark is measured in `forks` fresh processes, one after the other. */ | { kind: "Forks"; forks: number } - /** A fork: measures only `uri` and writes its samples to `samplesFile`. */ - | { kind: "Fork"; uri: string; samplesFile: string }; + /** + * A fork: measures only `uri` and writes its samples to `samplesFile`. The + * first fork picks `iterationsPerRound` itself; the ones after it inherit it. + */ + | { + kind: "Fork"; + uri: string; + samplesFile: string; + iterationsPerRound?: number; + }; /** * The isolation of a walltime run, from the environment. `forks` is the @@ -56,7 +71,16 @@ export function walltimeIsolation(forks?: number): IsolationMode { `[CodSpeed] ${ISOLATED_BENCHMARK_ENV} is set without ${FORK_SAMPLES_FILE_ENV}`, ); } - return { kind: "Fork", uri, samplesFile }; + const iterationsPerRound = process.env[FORK_ITERATIONS_PER_ROUND_ENV]; + return { + kind: "Fork", + uri, + samplesFile, + iterationsPerRound: + iterationsPerRound === undefined || iterationsPerRound === "" + ? undefined + : validPositiveInt(iterationsPerRound, FORK_ITERATIONS_PER_ROUND_ENV), + }; } const isolation = process.env[PROCESS_ISOLATION_ENV]; if (isolation === "0" || isolation === "false") { @@ -77,22 +101,22 @@ export function walltimeIsolation(forks?: number): IsolationMode { export function resolveWalltimeForks(option?: number): number { const env = process.env[WALLTIME_FORKS_ENV]; if (env !== undefined && env !== "") { - return validForks(env, WALLTIME_FORKS_ENV); + return validPositiveInt(env, WALLTIME_FORKS_ENV); } if (option !== undefined) { - return validForks(option, "forks"); + return validPositiveInt(option, "forks"); } return DEFAULT_WALLTIME_FORKS; } -function validForks(raw: string | number, source: string): number { - const forks = Number(raw); - if (!Number.isInteger(forks) || forks < 1) { +function validPositiveInt(raw: string | number, source: string): number { + const value = Number(raw); + if (!Number.isInteger(value) || value < 1) { throw new Error( `[CodSpeed] ${source} must be a positive integer, got \`${raw}\``, ); } - return forks; + return value; } /** The command of this process, which each fork re-runs. */ @@ -101,31 +125,62 @@ function currentCommand(): string[] { return [process.execPath, ...process.execArgv, ...process.argv.slice(1)]; } +/** The pooled samples of the forks of one benchmark. */ +export interface ForkSamples { + /** Round samples (milliseconds), one per measured round. */ + samples: number[]; + /** Benchmark iterations one round measured; every fork MUST agree. */ + iterationsPerRound: number; +} + /** * Runs the benchmark `uri` in `forks` fresh processes, one after the other, * and returns the round samples of all of them, in milliseconds. Throws when * a fork fails, without running the next one. * + * The first fork picks how many iterations one round batches; the ones after + * it inherit that count, so the pooled samples mean the same thing. + * * Each fork re-runs this process's command, not fork(): Node can't fork, and a * fork would only copy the calling thread. Call it from the process whose * command runs the benchmarks: Vitest's main process, not a worker. */ -export async function runForks(uri: string, forks: number): Promise { - let samples: number[] = []; +export async function runForks( + uri: string, + forks: number, +): Promise { + let result: ForkSamples | undefined; for (let fork = 0; fork < forks; fork++) { - // Not push(...): spreading a time-based run's samples overflows the stack. - samples = samples.concat(await runFork(uri)); + const next = await runFork(uri, result?.iterationsPerRound); + result = poolForkSamples(uri, result, next); } - return samples; + return result ?? { samples: [], iterationsPerRound: 1 }; } /** {@link runForks} for synchronous runs, which block anyway. */ -export function runForksSync(uri: string, forks: number): number[] { - let samples: number[] = []; +export function runForksSync(uri: string, forks: number): ForkSamples { + let result: ForkSamples | undefined; for (let fork = 0; fork < forks; fork++) { - samples = samples.concat(runForkSync(uri)); + const next = runForkSync(uri, result?.iterationsPerRound); + result = poolForkSamples(uri, result, next); + } + return result ?? { samples: [], iterationsPerRound: 1 }; +} + +function poolForkSamples( + uri: string, + pooled: ForkSamples | undefined, + next: ForkSamples, +): ForkSamples { + if (!pooled) return next; + if (pooled.iterationsPerRound !== next.iterationsPerRound) { + throw new Error( + `[CodSpeed] the forks of ${uri} measured a different number of iterations per round (${pooled.iterationsPerRound} and ${next.iterationsPerRound})`, + ); } - return samples; + // Not push(...): spreading a time-based run's samples overflows the stack. + for (const sample of next.samples) pooled.samples.push(sample); + return pooled; } /** A fresh file per fork: overlapping runForks() calls must not share one. */ @@ -137,12 +192,15 @@ function newSamplesFile(): string { } /** Runs one fork of `uri` and returns the samples it wrote. */ -async function runFork(uri: string): Promise { +async function runFork( + uri: string, + iterationsPerRound?: number, +): Promise { const samplesFile = newSamplesFile(); const [executable, ...args] = currentCommand(); const child = spawn(executable, args, { stdio: "inherit", - env: forkEnv(uri, samplesFile), + env: forkEnv(uri, samplesFile, iterationsPerRound), }); const { promise, resolve, reject } = Promise.withResolvers<[number | null, NodeJS.Signals | null]>(); @@ -152,12 +210,12 @@ async function runFork(uri: string): Promise { return forkSamples(uri, samplesFile, status, signal); } -function runForkSync(uri: string): number[] { +function runForkSync(uri: string, iterationsPerRound?: number): ForkSamples { const samplesFile = newSamplesFile(); const [executable, ...args] = currentCommand(); const result = spawnSync(executable, args, { stdio: "inherit", - env: forkEnv(uri, samplesFile), + env: forkEnv(uri, samplesFile, iterationsPerRound), }); if (result.error !== undefined) { throw startError(uri, result.error); @@ -165,11 +223,16 @@ function runForkSync(uri: string): number[] { return forkSamples(uri, samplesFile, result.status, result.signal); } -function forkEnv(uri: string, samplesFile: string): NodeJS.ProcessEnv { +function forkEnv( + uri: string, + samplesFile: string, + iterationsPerRound?: number, +): NodeJS.ProcessEnv { return { ...process.env, [ISOLATED_BENCHMARK_ENV]: uri, [FORK_SAMPLES_FILE_ENV]: samplesFile, + [FORK_ITERATIONS_PER_ROUND_ENV]: iterationsPerRound?.toString(), }; } @@ -185,7 +248,7 @@ function forkSamples( samplesFile: string, status: number | null, signal: NodeJS.Signals | null, -): number[] { +): ForkSamples { try { if (status !== 0) { throw new Error( @@ -193,7 +256,7 @@ function forkSamples( ); } try { - return deserialize(gunzipSync(readFileSync(samplesFile))) as number[]; + return deserialize(gunzipSync(readFileSync(samplesFile))) as ForkSamples; } catch (error) { throw new Error( `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`, @@ -204,10 +267,19 @@ function forkSamples( } } -/** Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. */ -export function writeForkSamples(samplesFile: string, samples: number[]) { +/** + * Hands round samples (milliseconds) to the parent as gzip-compressed V8 data, + * along with the number of benchmark iterations one round measured. + */ +export function writeForkSamples( + samplesFile: string, + samples: number[], + iterationsPerRound = 1, +) { writeFileSync( samplesFile, - gzipSync(serialize(samples), { level: constants.Z_BEST_SPEED }), + gzipSync(serialize({ samples, iterationsPerRound }), { + level: constants.Z_BEST_SPEED, + }), ); } diff --git a/packages/core/tests/fixtures/isolation.cjs b/packages/core/tests/fixtures/isolation.cjs index bd2d6d2e..f6bf8500 100644 --- a/packages/core/tests/fixtures/isolation.cjs +++ b/packages/core/tests/fixtures/isolation.cjs @@ -7,6 +7,8 @@ const { } = require("../../dist/index.cjs.js"); const mode = walltimeIsolation(); +/** The count a first fork picks; the forks after it inherit it from the parent. */ +const CALIBRATED = Number(process.env.ISOLATION_FIXTURE_ITERATIONS ?? 1); function measure() { console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`); @@ -16,7 +18,21 @@ function measure() { case "no-samples": process.exit(0); } - writeForkSamples(mode.samplesFile, [Number(process.pid), 1]); + const inherited = mode.iterationsPerRound; + // Tells the test whether this fork picked the count or reused the first one. + console.log( + `${inherited === undefined ? "calibrated" : "reused"} ${inherited ?? CALIBRATED}`, + ); + const iterationsPerRound = + inherited !== undefined && + process.env.ISOLATION_FIXTURE_ACTION === "mismatch" + ? inherited + 1 // batches differently than the first fork + : (inherited ?? CALIBRATED); + writeForkSamples( + mode.samplesFile, + [Number(process.pid), 1], + iterationsPerRound, + ); } async function main() { @@ -25,11 +41,12 @@ async function main() { return; } const forks = Number(process.env.ISOLATION_FIXTURE_FORKS); - const samples = + const result = process.env.ISOLATION_FIXTURE_SYNC !== undefined ? runForksSync("a", forks) : await runForks("a", forks); - console.log(`samples ${samples.length}`); + console.log(`samples ${result.samples.length}`); + console.log(`iterations ${result.iterationsPerRound}`); } // A rejection ends the process with exit code 1. diff --git a/packages/core/tests/isolation.integ.test.ts b/packages/core/tests/isolation.integ.test.ts index c61d3f45..fc5866c1 100644 --- a/packages/core/tests/isolation.integ.test.ts +++ b/packages/core/tests/isolation.integ.test.ts @@ -1,6 +1,7 @@ import { spawnSync } from "child_process"; import { join } from "path"; import { + FORK_ITERATIONS_PER_ROUND_ENV, FORK_SAMPLES_FILE_ENV, ISOLATED_BENCHMARK_ENV, PROCESS_ISOLATION_ENV, @@ -15,6 +16,10 @@ const RUN_TIMEOUT_MS = 60_000; const CHILD_EXIT_CODE = 3; const RAN_LINE = /^ran (\d+) (.+)$/; const SAMPLES_LINE = /^samples (\d+)$/; +/** A fork tells whether it picked the iterations per round or reused them. */ +const CALIBRATED_LINE = /^calibrated (\d+)$/; +const REUSED_LINE = /^reused (\d+)$/; +const ITERATIONS_LINE = /^iterations (\d+)$/; /** Passed to Node to check that forks keep the flags of their parent. */ const NODE_FLAG = "--no-warnings"; /** Each fork of the fixture reports two samples. */ @@ -24,6 +29,7 @@ const ENV_KEYS = [ PROCESS_ISOLATION_ENV, ISOLATED_BENCHMARK_ENV, FORK_SAMPLES_FILE_ENV, + FORK_ITERATIONS_PER_ROUND_ENV, WALLTIME_FORKS_ENV, ]; const savedEnv = Object.fromEntries( @@ -104,19 +110,48 @@ describe("walltimeIsolation", () => { samplesFile: "samples.json", }); }); + + it("tells a fork the iterations per round of the first fork", () => { + setEnv({ + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + [FORK_ITERATIONS_PER_ROUND_ENV]: "4", + }); + expect(walltimeIsolation()).toEqual({ + kind: "Fork", + uri: "file::bench", + samplesFile: "samples.json", + iterationsPerRound: 4, + }); + }); + + it.each(["0", "-1", "1.5", "abc"])( + "rejects the iterations per round %p", + (value) => { + setEnv({ + [ISOLATED_BENCHMARK_ENV]: "file::bench", + [FORK_SAMPLES_FILE_ENV]: "samples.json", + [FORK_ITERATIONS_PER_ROUND_ENV]: value, + }); + expect(() => walltimeIsolation()).toThrow(FORK_ITERATIONS_PER_ROUND_ENV); + }, + ); }); interface FixtureOptions { forks: number; - action?: "exit" | "no-samples"; + /** The iterations per round the first fork reports. Defaults to 1. */ + iterations?: number; + action?: "exit" | "no-samples" | "mismatch"; sync?: boolean; } -function runFixture({ forks, action, sync }: FixtureOptions) { +function runFixture({ forks, iterations, action, sync }: FixtureOptions) { const env: NodeJS.ProcessEnv = { ...process.env, ISOLATION_FIXTURE_ACTION: action, ISOLATION_FIXTURE_FORKS: String(forks), + ISOLATION_FIXTURE_ITERATIONS: iterations?.toString(), }; for (const key of ENV_KEYS) { delete env[key]; @@ -141,9 +176,19 @@ function runFixture({ forks, action, sync }: FixtureOptions) { stderr: result.stderr, ran, samples: samples ? Number(samples[1]) : undefined, + calibrated: counts(CALIBRATED_LINE, lines), + reused: counts(REUSED_LINE, lines), + iterations: counts(ITERATIONS_LINE, lines)[0], }; } +function counts(pattern: RegExp, lines: string[]): number[] { + return lines + .map((line) => pattern.exec(line)) + .filter((match): match is RegExpExecArray => match !== null) + .map((match) => Number(match[1])); +} + describe.each([ { name: "async", sync: false }, { name: "sync", sync: true }, @@ -161,6 +206,44 @@ describe.each([ expect(samples).toBe(forks * SAMPLES_PER_FORK); }); + it("reports one iteration per round when nothing batches", () => { + const { status, calibrated, reused, iterations } = runFixture({ + forks: 2, + sync, + }); + + expect(status).toBe(0); + expect(calibrated).toEqual([1]); + expect(reused).toEqual([1]); + expect(iterations).toBe(1); + }); + + it("runs the forks after the first one with its iterations per round", () => { + const forks = 3; + const { status, calibrated, reused, iterations } = runFixture({ + forks, + sync, + iterations: 4, + }); + + expect(status).toBe(0); + expect(calibrated).toEqual([4]); + expect(reused).toEqual(Array(forks - 1).fill(4)); + expect(iterations).toBe(4); + }); + + it("fails when a fork measures a different number of iterations per round", () => { + const { status, stderr } = runFixture({ + forks: 2, + sync, + iterations: 2, + action: "mismatch", + }); + + expect(status).not.toBe(0); + expect(stderr).toContain("a different number of iterations per round"); + }); + it("fails on a failing fork and runs no further fork", () => { const { status, stderr, ran, samples } = runFixture({ forks: 3, diff --git a/packages/tinybench-plugin/src/batching.ts b/packages/tinybench-plugin/src/batching.ts new file mode 100644 index 00000000..b58804a2 --- /dev/null +++ b/packages/tinybench-plugin/src/batching.ts @@ -0,0 +1,107 @@ +import { msToNs } from "@codspeed/core"; +import { Bench, Fn, FnOptions, Task } from "tinybench"; +import { getBenchOptions } from "./benchOptions"; + +const MIN_ROUND_TIME_MS = 1; +export const MIN_ROUND_TIME_NS = msToNs(MIN_ROUND_TIME_MS); + +/** Round samples are batch averages, not individual-call latency distributions. */ +export function batchTask( + bench: Bench, + fn: Fn, + options: FnOptions | undefined, + inheritedIterations?: number, +) { + const state = { iterationsPerRound: inheritedIterations ?? 1 }; + if ( + options?.beforeEach || + options?.afterEach || + bench.concurrency === "task" + ) { + return { iterationsPerRound: 1, fn, options }; + } + + const { now } = getBenchOptions(bench); + let active = false; + let warmingUp = false; + let calibrated = inheritedIterations !== undefined; + let elapsed = 0; + let calls = 0; + + function finishCalibration() { + if (elapsed > 0) { + state.iterationsPerRound = Math.max( + 1, + Math.ceil((MIN_ROUND_TIME_MS * calls) / elapsed), + ); + } + calibrated = true; + } + function runBatch() { + const result = fn(); + if (result instanceof Promise) { + return (async () => { + await result; + for (let i = 1; i < state.iterationsPerRound; i++) await fn(); + })(); + } + for (let i = 1; i < state.iterationsPerRound; i++) fn(); + return result; + } + + function measure(): void | Promise { + const iterations = state.iterationsPerRound; + const start = now(); + const result = runBatch(); + function record() { + elapsed += now() - start; + calls += iterations; + if (!calibrated && elapsed >= MIN_ROUND_TIME_MS) finishCalibration(); + } + if (result instanceof Promise) return result.then(record); + record(); + } + + function prepare(mode?: "warmup" | "run"): void | Promise { + active = true; + warmingUp = mode === "warmup"; + if (warmingUp || inheritedIterations !== undefined) return; + if (calls > 0) { + finishCalibration(); + return; + } + // With warmup disabled, calibrate after setup/beforeAll, outside timed rounds. + // Wall-clock bounds also terminate calibration with a coarse/custom timer. + const deadline = performance.now() + MIN_ROUND_TIME_MS; + function calibrate(): void | Promise { + do { + const result = measure(); + if (result instanceof Promise) { + return result.then(() => { + if (performance.now() < deadline) return calibrate(); + finishCalibration(); + }); + } + } while (performance.now() < deadline); + finishCalibration(); + } + return calibrate(); + } + + return Object.assign(state, { + fn: () => { + // Tinybench may call the function at registration to detect promises. + if (!active) return fn(); + if (warmingUp) return measure(); + return runBatch(); + }, + options: { + ...options, + beforeAll(this: Task, mode?: "warmup" | "run") { + const result = options?.beforeAll?.call(this, mode); + if (result instanceof Promise) return result.then(() => prepare(mode)); + return prepare(mode); + }, + }, + }); +} diff --git a/packages/tinybench-plugin/src/benchOptions.ts b/packages/tinybench-plugin/src/benchOptions.ts index a1ffef42..a95b6ae0 100644 --- a/packages/tinybench-plugin/src/benchOptions.ts +++ b/packages/tinybench-plugin/src/benchOptions.ts @@ -4,6 +4,7 @@ import { Bench, Hook } from "tinybench"; // setup/teardown hooks. tinybench guarantees `setup`/`teardown` are populated // (at least with no-op defaults), so they are non-optional here. export interface ResolvedBenchOptions { + now: () => number; setup: Hook; teardown: Hook; warmup?: boolean; diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index bb903ae0..fbba5cda 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -7,17 +7,25 @@ import { SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, + walltimeIsolation, wrapWithRootFrameSync, type WalltimeOptions, } from "@codspeed/core"; import { Bench } from "tinybench"; import { setupCodspeedAnalysisBench } from "./analysis"; +import { batchTask } from "./batching"; import { getOrCreateTaskDataMap } from "./taskData"; import { getOrCreateUriMap } from "./uri"; import { setupCodspeedWalltimeBench } from "./walltime"; tryIntrospect(); +/** + * Walltime rounds batch sequential invocations toward a 1 ms target, estimated + * from warmup (or a short calibration when disabled). Forks share one batch size. + * Per-invocation hooks and task concurrency stay unbatched. CodSpeed reports + * per-invocation batch averages; total time is the sum of measured round durations. + */ export function withCodSpeed( bench: Bench, options: WalltimeOptions = {}, @@ -42,15 +50,24 @@ export function withCodSpeed( } uri += `::${name}`; uriMap.set(name, uri); - taskDataMap.set(name, { fn, fnOpts: opts }); - // In walltime mode the task is driven by tinybench's own measured loop, so - // the root frame must be baked into the function tinybench stores rather - // than injected later (the function is an inaccessible private field on the - // task from tinybench v6 onwards). The sync wrapper is transparent to the - // function's return value, so it preserves tinybench's sync/async handling. - const registeredFn = - instrumentMode === "walltime" ? wrapWithRootFrameSync(fn) : fn; - return rawAdd.bind(bench)(name, registeredFn, opts); + if (instrumentMode !== "walltime") { + taskDataMap.set(name, { fn, fnOpts: opts }); + return rawAdd.call(bench, name, fn, opts); + } + const isolation = walltimeIsolation(options.forks); + let inheritedIterations: number | undefined; + if (isolation.kind === "Fork") { + inheritedIterations = isolation.iterationsPerRound; + } + const batching = batchTask(bench, fn, opts, inheritedIterations); + taskDataMap.set(name, { fn, fnOpts: opts, batching }); + // Tinybench v6 stores the function in a private field, so wrap at registration. + return rawAdd.call( + bench, + name, + wrapWithRootFrameSync(batching.fn), + batching.options, + ); }; if (instrumentMode === "analysis") { diff --git a/packages/tinybench-plugin/src/taskData.ts b/packages/tinybench-plugin/src/taskData.ts index a5ecb2ab..58eedd82 100644 --- a/packages/tinybench-plugin/src/taskData.ts +++ b/packages/tinybench-plugin/src/taskData.ts @@ -7,6 +7,7 @@ import { Bench, Fn, FnOptions } from "tinybench"; export interface CapturedTaskData { fn: Fn; fnOpts?: FnOptions; + batching?: { iterationsPerRound: number }; } const taskDataMap = new WeakMap>(); diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index acf8837b..8d0ac694 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -18,6 +18,7 @@ import { type WalltimeOptions, } from "@codspeed/core"; import { Bench, Task } from "tinybench"; +import { MIN_ROUND_TIME_NS } from "./batching"; import { getBenchOptions } from "./benchOptions"; import { BaseBenchRunner } from "./shared"; @@ -102,8 +103,15 @@ class WalltimeBenchRunner extends BaseBenchRunner { protected async runTaskAsync(task: Task, uri: string): Promise { const { isolationMode } = this; if (isolationMode.kind === "Forks") { - const samples = await runForks(uri, isolationMode.forks); - return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + const { samples, iterationsPerRound } = await runForks( + uri, + isolationMode.forks, + ); + return this.registerCodspeedBenchmark( + task, + samplesToResult(samples), + iterationsPerRound, + ); } // run the warmup of the task right before its actual run @@ -121,8 +129,15 @@ class WalltimeBenchRunner extends BaseBenchRunner { protected runTaskSync(task: Task, uri: string): void { const { isolationMode } = this; if (isolationMode.kind === "Forks") { - const samples = runForksSync(uri, isolationMode.forks); - return this.registerCodspeedBenchmark(task, samplesToResult(samples)); + const { samples, iterationsPerRound } = runForksSync( + uri, + isolationMode.forks, + ); + return this.registerCodspeedBenchmark( + task, + samplesToResult(samples), + iterationsPerRound, + ); } if (getBenchOptions(this.bench).warmup) { @@ -149,18 +164,25 @@ class WalltimeBenchRunner extends BaseBenchRunner { ); return; } + const iterationsPerRound = + this.getTaskData(task).batching?.iterationsPerRound ?? 1; // A fork hands its samples to the parent, which reports the benchmark. if (this.isolationMode.kind === "Fork") { writeForkSamples( this.isolationMode.samplesFile, task.result.latency.samples, + iterationsPerRound, ); return; } - this.registerCodspeedBenchmark(task, task.result); + this.registerCodspeedBenchmark(task, task.result, iterationsPerRound); } - private registerCodspeedBenchmark(task: Task, result: RoundsResult): void { + private registerCodspeedBenchmark( + task: Task, + result: RoundsResult, + iterationsPerRound: number, + ): void { const uri = this.getTaskUri(task); const opts = getBenchOptions(this.bench); const warmupIterations = opts.warmup @@ -169,6 +191,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { const stats = convertTinybenchResultToBenchmarkStats( result, warmupIterations, + iterationsPerRound, ); this.codspeedBenchmarks.push({ @@ -177,7 +200,7 @@ class WalltimeBenchRunner extends BaseBenchRunner { config: { max_rounds: opts.iterations ?? null, max_time_ns: opts.time ? msToNs(opts.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this + min_round_time_ns: iterationsPerRound > 1 ? MIN_ROUND_TIME_NS : null, warmup_time_ns: opts.warmup && opts.warmupTime ? msToNs(opts.warmupTime) : null, }, @@ -210,27 +233,30 @@ const TINYBENCH_WARMUP_DEFAULT = 16; function convertTinybenchResultToBenchmarkStats( result: RoundsResult, warmupIterations: number, + iterationsPerRound: number, ): BenchmarkStats { const { min, max, mean, sd, samples } = result.latency; // Get individual sample times in nanoseconds and sort them - const sortedTimesNs = samples.map(msToNs).sort((a, b) => a - b); - const meanNs = msToNs(mean); - const stdevNs = msToNs(sd); + const sortedTimesNs = samples + .map((sample) => msToNs(sample) / iterationsPerRound) + .sort((a, b) => a - b); + const meanNs = msToNs(mean) / iterationsPerRound; + const stdevNs = msToNs(sd) / iterationsPerRound; const { q1_ns, q3_ns, median_ns, iqr_outlier_rounds, stdev_outlier_rounds } = calculateQuantiles({ meanNs, stdevNs, sortedTimesNs }); return { - min_ns: msToNs(min), - max_ns: msToNs(max), + min_ns: msToNs(min) / iterationsPerRound, + max_ns: msToNs(max) / iterationsPerRound, mean_ns: meanNs, stdev_ns: stdevNs, q1_ns, median_ns, q3_ns, total_time: msToS(result.totalTime), - iter_per_round: 1, // as there is only one round in tinybench, we define that there were n rounds of 1 iteration + iter_per_round: iterationsPerRound, rounds: sortedTimesNs.length, iqr_outlier_rounds, stdev_outlier_rounds, diff --git a/packages/tinybench-plugin/tests/batching.integ.test.ts b/packages/tinybench-plugin/tests/batching.integ.test.ts new file mode 100644 index 00000000..e1f4f307 --- /dev/null +++ b/packages/tinybench-plugin/tests/batching.integ.test.ts @@ -0,0 +1,218 @@ +import { getV8Flags } from "@codspeed/core"; +import { spawn } from "child_process"; +import { once } from "events"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { fileURLToPath } from "url"; +import { describe, expect, it } from "vitest"; + +const FIXTURE = fileURLToPath( + new URL("./fixtures/batching.mjs", import.meta.url), +); +const PROCESS_ISOLATION_ENV = "CODSPEED_PROCESS_ISOLATION"; +const WALLTIME_FORKS_ENV = "CODSPEED_WALLTIME_FORKS"; +const FIXTURE_SYNC_ENV = "BATCHING_FIXTURE_SYNC"; +/** Fails a run whose benchmark process got stuck instead of blocking the suite. */ +const RUN_TIMEOUT_MS = 120_000; +const FORKS = 2; +/** Rounds the fixture's benchmarks measure per process once a round costs 1ms. */ +const ROUNDS_PER_FORK = 4; +/** The fixture's iteration cost: 0.01ms on the fake clock. */ +const BATCHED_ITERATION_NS = 10_000; +/** The lifecycle benchmark halves its iteration cost in the suite setup hook. */ +const LIFECYCLE_ITERATION_NS = 5_000; +/** The 1ms round target in units of the fixture's iteration cost. */ +const BATCHED_ITER_PER_ROUND = 100; +const LIFECYCLE_ITER_PER_ROUND = 200; +/** Allows for the fake clock's rounding, both ways, around the target count. */ +const ITERATIONS_TOLERANCE = 3; + +interface BenchmarkStats { + min_ns: number; + max_ns: number; + mean_ns: number; + rounds: number; + total_time: number; + iter_per_round: number; + warmup_iters: number; +} + +interface FixtureOptions { + /** `CODSPEED_PROCESS_ISOLATION=false`: every benchmark shares our process. */ + shared?: boolean; + /** `CODSPEED_WALLTIME_FORKS`. */ + forks?: number; + sync?: boolean; +} + +async function runFixture({ + shared, + forks, + sync, +}: FixtureOptions = {}): Promise> { + const profileFolder = mkdtempSync(join(tmpdir(), "codspeed-batching-")); + try { + const env: NodeJS.ProcessEnv = { + ...process.env, + CODSPEED_ENV: "true", + CODSPEED_RUNNER_MODE: "walltime", + CODSPEED_PROFILE_FOLDER: profileFolder, + }; + delete env[PROCESS_ISOLATION_ENV]; + delete env[WALLTIME_FORKS_ENV]; + if (shared) { + env[PROCESS_ISOLATION_ENV] = "false"; + } + if (forks !== undefined) { + env[WALLTIME_FORKS_ENV] = String(forks); + } + if (sync) { + env[FIXTURE_SYNC_ENV] = "1"; + } + + // The runner starts Node with these flags, and forks must keep them. Not + // spawnSync: blocking the test worker times out its calls to Vitest. + const child = spawn(process.execPath, [...getV8Flags(), FIXTURE], { + env, + timeout: RUN_TIMEOUT_MS, + }); + let stdout = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + const [status] = (await once(child, "close")) as [number | null]; + expect(status, stdout).toBe(0); + + const results = new Map(); + const resultsDir = join(profileFolder, "results"); + for (const file of readdirSync(resultsDir)) { + const data = JSON.parse(readFileSync(join(resultsDir, file), "utf8")) as { + benchmarks: { name: string; stats: BenchmarkStats }[]; + }; + for (const { name, stats } of data.benchmarks) { + results.set(name, stats); + } + } + return results; + } finally { + rmSync(profileFolder, { recursive: true, force: true }); + } +} + +function getStats( + results: Map, + name: string, +): BenchmarkStats { + const stats = results.get(name); + if (!stats) { + throw new Error(`no results for ${name}`); + } + return stats; +} + +/** Every sample is one iteration, not one batch of iterations. */ +function expectPerInvocationStats( + stats: BenchmarkStats, + perInvocationNs: number, +): void { + expect(Math.round(stats.mean_ns)).toBe(perInvocationNs); + expect(Math.round(stats.min_ns)).toBe(perInvocationNs); + expect(Math.round(stats.max_ns)).toBe(perInvocationNs); + // The total stays the raw batched time: rounds x iterations x per-iteration. + expect(stats.total_time).toBeCloseTo( + (stats.rounds * stats.iter_per_round * stats.mean_ns) / 1e9, + 9, + ); +} + +function expectBatchedStats( + stats: BenchmarkStats, + { + rounds, + iterationsPerRound = BATCHED_ITER_PER_ROUND, + perInvocationNs = BATCHED_ITERATION_NS, + }: { + rounds: number; + iterationsPerRound?: number; + perInvocationNs?: number; + }, +): void { + expect(stats.iter_per_round).toBeGreaterThanOrEqual( + iterationsPerRound - ITERATIONS_TOLERANCE, + ); + expect(stats.iter_per_round).toBeLessThanOrEqual( + iterationsPerRound + ITERATIONS_TOLERANCE, + ); + expect(stats.rounds).toBe(rounds); + expectPerInvocationStats(stats, perInvocationNs); +} + +function expectHookedStats( + stats: BenchmarkStats, + { forks }: { forks: number }, +): void { + expect(stats.iter_per_round).toBe(1); + // Each 0.01ms call is timed on its own, so tinybench's 1ms budget is spent. + expect(stats.rounds).toBeGreaterThanOrEqual(98 * forks); + expect(stats.rounds).toBeLessThanOrEqual(102 * forks); + expectPerInvocationStats(stats, BATCHED_ITERATION_NS); +} + +describe("walltime batching", () => { + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "batches fast benchmarks and normalizes their stats in the shared process ($name run)", + async ({ sync }) => { + const stats = await runFixture({ shared: true, sync }); + + expectBatchedStats(getStats(stats, "batched"), { + rounds: ROUNDS_PER_FORK, + }); + // The suite setup hook halves the iteration cost before calibration. + expectBatchedStats(getStats(stats, "lifecycle"), { + rounds: ROUNDS_PER_FORK, + iterationsPerRound: LIFECYCLE_ITER_PER_ROUND, + perInvocationNs: LIFECYCLE_ITERATION_NS, + }); + expect(getStats(stats, "lifecycle").warmup_iters).toBe(0); + expectHookedStats(getStats(stats, "hooked"), { forks: 1 }); + + if (!sync) { + // An overlapping async round would have failed the fixture run. + expectBatchedStats(getStats(stats, "async"), { + rounds: ROUNDS_PER_FORK, + }); + } + }, + RUN_TIMEOUT_MS, + ); + + it.each([ + { name: "async", sync: false }, + { name: "sync", sync: true }, + ])( + "batches and normalizes the pooled samples of several forks ($name run)", + async ({ sync }) => { + const stats = await runFixture({ forks: FORKS, sync }); + + expectBatchedStats(getStats(stats, "batched"), { + rounds: FORKS * ROUNDS_PER_FORK, + }); + expectBatchedStats(getStats(stats, "lifecycle"), { + rounds: FORKS * ROUNDS_PER_FORK, + iterationsPerRound: LIFECYCLE_ITER_PER_ROUND, + perInvocationNs: LIFECYCLE_ITERATION_NS, + }); + // Batched and unbatched benchmarks pool very different sample counts. + expectHookedStats(getStats(stats, "hooked"), { forks: FORKS }); + + if (!sync) { + expectBatchedStats(getStats(stats, "async"), { + rounds: FORKS * ROUNDS_PER_FORK, + }); + } + }, + RUN_TIMEOUT_MS, + ); +}); diff --git a/packages/tinybench-plugin/tests/fixtures/batching.mjs b/packages/tinybench-plugin/tests/fixtures/batching.mjs new file mode 100644 index 00000000..4e149b2a --- /dev/null +++ b/packages/tinybench-plugin/tests/fixtures/batching.mjs @@ -0,0 +1,106 @@ +// Run by `batching.integ.test.ts`. A deterministic clock makes batching +// observable: every benchmark iteration costs exactly one tick, wherever +// tinybench or the plugin reads the clock. +import { Bench } from "tinybench"; +import { withCodSpeed } from "../../dist/index.es5.js"; + +/** Milliseconds one benchmark iteration costs on the fake clock. */ +const ITERATION_MS = 0.01; +/** tinybench stops after this many rounds once a round costs 1ms. */ +const ROUNDS = 4; +const sync = process.env.BATCHING_FIXTURE_SYNC !== undefined; + +// Warmup enabled: the plugin learns the per-iteration cost from the warmup +// rounds and then batches ~1ms worth of iterations per measured round. +let tickA = 0; +const costA = ITERATION_MS; +const benchA = withCodSpeed( + new Bench({ iterations: ROUNDS, time: 1, warmupTime: 0, now: () => tickA }), +); + +benchA.add("batched", () => { + tickA += costA; +}); + +// A batched async benchmark must await every iteration of a round: overlapping +// iterations are a bug, and they fail the run instead of skewing the stats. +let inFlight = 0; +if (!sync) { + benchA.add("async", async () => { + if (inFlight++ > 0) { + throw new Error("async iterations overlapped"); + } + tickA += costA; + await Promise.resolve(); + inFlight--; + }); +} + +// beforeEach/afterEach are only correct when tinybench times each call alone. +let beforeEachCalls = 0; +let afterEachCalls = 0; +let hookedCalls = 0; +benchA.add( + "hooked", + () => { + if (beforeEachCalls > 0 && beforeEachCalls !== hookedCalls + 1) { + throw new Error("beforeEach did not run for this invocation"); + } + hookedCalls++; + tickA += costA; + }, + { + beforeAll: () => { + beforeEachCalls = 0; + afterEachCalls = 0; + hookedCalls = 0; + }, + beforeEach: () => { + beforeEachCalls++; + }, + afterEach: () => { + afterEachCalls++; + }, + afterAll: () => { + if (beforeEachCalls !== hookedCalls || afterEachCalls !== hookedCalls) { + throw new Error("afterEach did not run for every invocation"); + } + }, + }, +); + +// Warmup disabled: the plugin calibrates inside the run lifecycle, after the +// suite setup hook has run (which is what decides the iteration cost here). +let tickB = 0; +let costB = ITERATION_MS; +let lifecycleRuns = 0; +const benchB = withCodSpeed( + new Bench({ + iterations: ROUNDS, + time: 1, + warmup: false, + now: () => tickB, + setup: (_task, mode) => { + if (mode === "run") { + lifecycleRuns++; + costB = ITERATION_MS / 2; + } + }, + }), +); +benchB.add("lifecycle", () => { + tickB += costB; +}); + +if (sync) { + benchA.runSync(); + benchB.runSync(); +} else { + await benchA.run(); + await benchB.run(); +} + +// A fork that owns another benchmark's URI does not run this suite at all. +if (lifecycleRuns > 1) { + throw new Error(`ran the lifecycle ${lifecycleRuns} times`); +} diff --git a/packages/vitest-plugin/src/v5/forkReporter.ts b/packages/vitest-plugin/src/v5/forkReporter.ts index 7055eb7b..89731ab0 100644 --- a/packages/vitest-plugin/src/v5/forkReporter.ts +++ b/packages/vitest-plugin/src/v5/forkReporter.ts @@ -41,7 +41,7 @@ async function measureInForks( for (const test of module.children.allTests()) { for (const queued of forkedBenchmarks(test.meta())) { const forks = resolveWalltimeForks(queued.forks ?? options.forks); - const samples = await runForks(queued.uri, forks); + const { samples } = await runForks(queued.uri, forks); const benchmark = tinybenchTaskToBenchmark( { name: queued.name, result: samplesToResult(samples) }, queued.uri, diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index bd15ae80..d047af70 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -2,11 +2,11 @@ import { getGitDir, getInstrumentMode, InstrumentHooks, - walltimeIsolation, optimizeFunction, optimizeFunctionSync, setupCore, teardownCore, + walltimeIsolation, wrapWithRootFrame, wrapWithRootFrameSync, writeForkSamples,