From 339d73999df09b317788bd80359f88fcb60cb617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vin=C3=ADcius=20Louren=C3=A7o?= Date: Mon, 31 Aug 2026 23:20:52 -0300 Subject: [PATCH] feat(node:bench): use node:bench as base benchmark --- .github/workflows/benchmark-comparison.yml | 2 +- .github/workflows/test.yml | 2 +- README.md | 60 ++++- doc/Plugins.md | 50 +++-- index.d.ts | 142 +++++++----- lib/clock.js | 246 +------------------- lib/index.js | 149 ++++++------ lib/lifecycle.js | 240 -------------------- lib/native-runner.js | 250 +++++++++++++++++++++ lib/plugin-runner.js | 126 +++++++++++ lib/plugins.js | 2 + lib/plugins/v8-opt.js | 12 +- lib/worker-runner.js | 55 +---- package.json | 8 +- test/fixtures/native-cli.js | 26 +++ test/native-runner.js | 97 ++++++++ test/worker.js | 38 ++++ types/types.test-d.ts | 33 +-- 18 files changed, 820 insertions(+), 718 deletions(-) delete mode 100644 lib/lifecycle.js create mode 100644 lib/native-runner.js create mode 100644 lib/plugin-runner.js create mode 100644 test/fixtures/native-cli.js create mode 100644 test/native-runner.js diff --git a/.github/workflows/benchmark-comparison.yml b/.github/workflows/benchmark-comparison.yml index 3e0a5bd..d909b7d 100644 --- a/.github/workflows/benchmark-comparison.yml +++ b/.github/workflows/benchmark-comparison.yml @@ -37,7 +37,7 @@ jobs: strategy: matrix: - node-version: [20, 22, 24] + node-version: [27] steps: - name: Checkout code diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 695838d..672f1ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: strategy: matrix: - node-version: [18, 20, 22, 24, 25] + node-version: [27] env: CI: true diff --git a/README.md b/README.md index 1b466c9..87e8085 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ [![Downloads][downloads-img]][downloads-url] [![Issues][issues-img]][issues-url] -The `bench-node` module allows you to measure operations per second of Node.js code blocks. +The `bench-node` module allows you to measure operations per second of Node.js code blocks. Its benchmark execution and sample lifecycle are powered by the built-in `node:bench` module. ## Install @@ -30,6 +30,8 @@ The `bench-node` module allows you to measure operations per second of Node.js c $ npm install bench-node ``` +`bench-node` requires Node.js 27 or a development build that provides `node:bench`. + ## Usage ```cjs @@ -49,12 +51,30 @@ suite.add('Using delete property', () => { suite.run() ``` +Run the file directly to use the suite's configured reporter: + +```bash +node --allow-natives-syntax my-benchmark.js +``` + +The same file can be run by Node's native benchmark CLI. In this mode, Node's +reporter options control output and the `Suite` reporter is not invoked: + +```bash +node --allow-natives-syntax --bench --bench-reporter=spec my-benchmark.js +``` + +When using `--bench`, call `suite.run()` while the benchmark module is being +evaluated; do not block module evaluation with top-level `await`. If +`useWorkers` is enabled, the native CLI's process isolation takes precedence +and the benchmarks execute in its managed child process. + ```bash $ node --allow-natives-syntax my-benchmark.js Using delete property x 3,326,913 ops/sec (11 runs sampled) v8-never-optimize=true min..max=(0ns ... 0ns) p75=0ns p99=0ns ``` -This module uses V8 deoptimization to help ensure that the code block is not optimized away, producing accurate benchmarks -- but not realistic. +This module uses `node:bench` for measurement and V8 deoptimization to help ensure that the code block is not optimized away, producing accurate benchmarks -- but not realistic. See the [Writing JavaScript Microbenchmark Mistakes](#writing-javascript-mistakes) section for more details. The [`bench-node-cli`](https://github.com/RafaelGSS/bench-node-cli) tool allows you to execute a `bench-node` benchmark @@ -121,9 +141,10 @@ A `Suite` manages and executes benchmark functions. It provides two methods: `ad * `reporter` {Function} Callback function for reporting results. Receives two arguments: * `results` {Object[]} Array of benchmark results: * `name` {string} Benchmark name. - * `opsSec` {string} Operations per second. + * `opsSec` {number} Operations per second. * `iterations` {Number} Number of iterations. - * `histogram` {Histogram} Histogram instance. + * `histogram` {Object} Normalized histogram data with `samples`, `min`, + `max`, and `sampleData` properties. * `ttest` {boolean} Enable Welch's t-test for statistical significance testing. Automatically sets `repeatSuite=30`. **Default:** `false`. * `reporterOptions` {Object} Reporter-specific options. * `printHeader` {boolean} Whether to print system information header. **Default:** `true`. @@ -133,10 +154,10 @@ A `Suite` manages and executes benchmark functions. It provides two methods: `ad * `'ops'` - Measures operations per second (traditional benchmarking). * `'time'` - Measures actual execution time for a single run. * `useWorkers` {boolean} Whether to run benchmarks in worker threads. **Default:** `false`. - * `plugins` {Array} Array of plugin instances to use. * `repeatSuite` {number} Number of times to repeat each benchmark. Automatically set to `30` when `ttest: true`. **Default:** `1`. * `plugins` {Array} Array of plugin instances to use. **Default:** `[V8NeverOptimizePlugin]`. * `minSamples` {number} Minimum number of samples per round for all benchmarks in the suite. Can be overridden per benchmark. **Default:** `10` samples. + * `pretty` {boolean} Use the pretty reporter when no explicit reporter is provided. **Default:** `false`. * `detectDeadCodeElimination` {boolean} Enable dead code elimination detection. When enabled, default plugins are disabled to allow V8 optimizations. **Default:** `false`. * `dceThreshold` {number} Threshold multiplier for DCE detection. Benchmarks faster than baseline × threshold will trigger warnings. **Default:** `10`. @@ -180,9 +201,9 @@ Using delete property x 5,853,505 ops/sec (10 runs sampled) min..max=(169ns ... * `opsSecPerRun` {Array} Array of operations per second (useful when repeatSuite > 1). * `totalTime` {number} Mean execution time in seconds per sample (only in `'time'` mode). * `iterations` {number} Number of executions of `fn`. - * `histogram` {Histogram} Histogram of benchmark iterations. + * `histogram` {Object} Normalized histogram of nanoseconds per operation. * `name` {string} Benchmark name. - * `plugins` {Object} Object with plugin results if any plugins are active. + * `plugins` {Object[]} Plugin results if any plugins are active. Runs all added benchmarks and returns the results. @@ -254,7 +275,7 @@ suite.add('computation', () => { }); ``` -**Note:** DCE detection only works in `'ops'` benchmark mode and when not using worker threads. It is automatically disabled for `'time'` mode and worker-based benchmarks. +**Note:** DCE detection only works in direct `'ops'` mode. It is automatically disabled for `'time'` mode, worker-based benchmarks, and the native `--bench` CLI. See [examples/dce-detection/](./examples/dce-detection/) for more examples. @@ -284,18 +305,26 @@ See [Plugins](./doc/Plugins.md) for details. class V8OptimizeOnNextCallPlugin { isSupported() { try { - new Function(`%OptimizeFunctionOnNextCall(() => {})`)(); + new Function(` + const fn = () => {}; + %PrepareFunctionForOptimization(fn); + fn(); + fn(); + %OptimizeFunctionOnNextCall(fn); + fn(); + `)(); return true; } catch (e) { return false; } } - beforeClockTemplate({ awaitOrEmpty, bench }) { + beforeClockTemplate({ awaitOrEmpty, bench, timer }) { let code = ''; + code += `%PrepareFunctionForOptimization(${bench}.fn);\n`; + code += `${awaitOrEmpty}${bench}.fn(${timer});\n`; + code += `${awaitOrEmpty}${bench}.fn(${timer});\n`; code += `%OptimizeFunctionOnNextCall(${bench}.fn);\n`; - code += `${awaitOrEmpty}${bench}.fn();\n`; - code += `${awaitOrEmpty}${bench}.fn();\n`; return [code]; } @@ -662,6 +691,13 @@ const suite = new Suite({ }); ``` +When the file is launched with `node --bench`, Node's benchmark-file isolation +takes precedence and `useWorkers` does not create a nested worker. Use the +native CLI's `--bench-concurrency` option to run benchmark files concurrently. +For direct runs, worker benchmark functions are serialized and therefore cannot +close over variables from the declaring module. Plugin sample context must be +structured-cloneable so it can be returned to the parent thread. + ## Benchmark Modes `bench-node` supports multiple benchmarking modes to measure code performance in different ways. diff --git a/doc/Plugins.md b/doc/Plugins.md index 6900e3b..76ea92e 100644 --- a/doc/Plugins.md +++ b/doc/Plugins.md @@ -7,6 +7,11 @@ plugins within the benchmarking framework. [V8NeverOptimizePlugin](#class-v8neveroptimizeplugin) is enabled by default. +Plugin templates are compiled around the callback executed by `node:bench`. +For unmanaged benchmarks, setup runs before `BenchContext.start()` and teardown +runs after `BenchContext.end()`. Managed benchmarks report their explicit timer +through `BenchContext.record()`. + To observe how a plugin is used, see the `plugin-api-doc.js` file in tests and explore its results. ## Structure @@ -42,10 +47,11 @@ this method ensures the environment supports them. * `context` {string} - Name for the context variable. * `timer` {string} - Name for the timer variable. * `awaitOrEmpty` {string} - A string with `await` or empty string (`''`). + * `managed` {boolean} - Whether the benchmark uses the explicit timer API. -Some plugins need to modify or prepare the code before the benchmark starts. -The `beforeClockTemplate()` method allows you to inject code before the timing -process begins. +Some plugins need to modify or prepare the code before a benchmark sample +starts. The `beforeClockTemplate()` method allows you to inject code before the +timed region of each native warmup and measurement callback. This method must return an array where: @@ -90,9 +96,10 @@ These two protections address different parts of the generated code: * `context` {string} - Name for the context variable. * `timer` {string} - Name for the timer variable. * `awaitOrEmpty` {string} - A string with `await` or empty string (`''`). + * `managed` {boolean} - Whether the benchmark uses the explicit timer API. -After the benchmark runs, this method can inject code to gather performance data -or reset configurations. It must return an array where: +After each benchmark sample runs, this method can inject code to gather +performance data or reset configurations. It must return an array where: * The first element is a string containing the JavaScript code to be executed after the benchmark finishes. @@ -101,15 +108,18 @@ Unlike `beforeClockTemplate`, `afterClockTemplate` does not support a second element in the returned array, as it only runs cleanup or data collection code after the benchmark is executed. -### `onCompleteBenchmark(result)` +### `onCompleteBenchmark(result, benchmark)` -* `result` {Object} - * `duration` {number} - Benchmark duration - * `count` {number} - Number of iterations - * `context` {Object} - A object used to store results after the benchmark clock +* `result` {Array} + * `result[0]` {number} - Sample duration in nanoseconds. + * `result[1]` {number} - Number of operations in the sample. + * `result[2]` {Object} - Context populated by plugin templates. +* `benchmark` {Object} Benchmark metadata. -This method is called when the benchmark completes. Plugins can collect and -process data from the benchmark results in this step. +This method is called after each native warmup or measurement sample. Plugins +can collect and process data from the sample in this step. In worker mode, the +context must be structured-cloneable because the hook is replayed in the parent +thread. ### `toString()` (required) @@ -124,19 +134,27 @@ Here are examples of plugins that follow the required structure and functionalit class V8OptimizeOnNextCallPlugin { isSupported() { try { - new Function(`%OptimizeFunctionOnNextCall(() => {})`)(); + new Function(` + const fn = () => {}; + %PrepareFunctionForOptimization(fn); + fn(); + fn(); + %OptimizeFunctionOnNextCall(fn); + fn(); + `)(); return true; } catch (e) { return false; } } - beforeClockTemplate({ awaitOrEmpty, bench }) { + beforeClockTemplate({ awaitOrEmpty, bench, timer }) { let code = ''; + code += `%PrepareFunctionForOptimization(${ bench }.fn);\n`; + code += `${ awaitOrEmpty }${ bench }.fn(${ timer });\n`; + code += `${ awaitOrEmpty }${ bench }.fn(${ timer });\n`; code += `%OptimizeFunctionOnNextCall(${ bench }.fn);\n`; - code += `${ awaitOrEmpty }${ bench }.fn();\n`; - code += `${ awaitOrEmpty }${ bench }.fn();\n`; return [code]; } diff --git a/index.d.ts b/index.d.ts index 29e27d1..c07e26e 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,22 +1,62 @@ // Type definitions for bench-node -/// -import type { Histogram } from "node:perf_hooks"; - export declare namespace BenchNode { - class Benchmark { + interface PluginHookVarNames { + awaitOrEmpty: string; + bench: string; + context: string; + timer: string; + managed: boolean; + } + + interface BenchmarkHistogram { + samples: number; + min: number; + max: number; + sampleData: number[]; + } + + interface BenchmarkPluginResult { name: string; - fn: any; + result: any; + report: string; + } + + interface PluginResult { + type: string; + [key: string]: any; + } + + interface BenchmarkMetadata { + name: string; + fn: BenchmarkFunction; + fnStr: string; + minTime: number; + maxTime: number; + plugins: Plugin[]; + repeatSuite: number; + minSamples: number; + baseline: boolean; + hasArg: boolean; + isAsync: boolean; + } + + class Benchmark implements BenchmarkMetadata { + name: string; + fn: BenchmarkFunction; + fnStr: string; minTime: number; maxTime: number; plugins: Plugin[]; repeatSuite: number; minSamples: number; baseline: boolean; + hasArg: boolean; + isAsync: boolean; constructor( name: string, - fn: any, + fn: BenchmarkFunction, minTime: number, maxTime: number, plugins: Plugin[], @@ -25,29 +65,20 @@ export declare namespace BenchNode { baseline?: boolean, ); - serializeBenchmark(): void; - } - - interface PluginHookVarNames { - awaitOrEmpty: string; - bench: string; - context: string; - timer: string; - managed: boolean; + serializeBenchmark(): Record; } interface BenchmarkResult { name: string; opsSec?: number; // Only in 'ops' mode opsSecPerRun?: number[]; // Useful when repeatSuite > 1 - totalTime?: number; // Total execution time in seconds (Only in 'time' mode) + totalTime?: number; // Mean execution time in seconds per sample in 'time' mode iterations: number; - histogram: Histogram; - plugins?: Record; // Object with plugin results + histogram: BenchmarkHistogram; + plugins: BenchmarkPluginResult[]; + baseline: boolean; } - type ReporterFunction = (results: BenchmarkResult[]) => void; - interface ReporterOptions { printHeader?: boolean; labelWidth?: number; @@ -55,6 +86,11 @@ export declare namespace BenchNode { alpha?: number; // Significance level for t-test (default: 0.05) } + type ReporterFunction = ( + results: BenchmarkResult[], + options?: ReporterOptions, + ) => void; + interface SuiteOptions { reporter?: ReporterFunction | false | null; benchmarkMode?: "ops" | "time"; @@ -63,6 +99,7 @@ export declare namespace BenchNode { minSamples?: number; // Minimum number of samples per round for all benchmarks repeatSuite?: number; // Number of times to repeat each benchmark (default: 1, or 30 when ttest is enabled) ttest?: boolean; // Enable t-test mode for statistical significance (auto-sets repeatSuite=30) + pretty?: boolean; reporterOptions?: ReporterOptions; detectDeadCodeElimination?: boolean; // Enable DCE detection, default: false dceThreshold?: number; // DCE detection threshold multiplier, default: 10 @@ -72,7 +109,8 @@ export declare namespace BenchNode { minTime?: number; // Minimum duration in seconds maxTime?: number; // Maximum duration in seconds repeatSuite?: number; // Number of times to repeat benchmark - minSamples?: number; // Minimum number of timed samples collected per round (the benchmark fn runs at least this many times per round) + minSamples?: number; // Minimum number of timed samples collected per round + baseline?: boolean; } type BenchmarkFunction = (timer?: { @@ -83,26 +121,22 @@ export declare namespace BenchNode { type OnCompleteBenchmarkResult = [ duration: number, - count: number, + iterations: number, context: Record, ]; - type PluginResult = { - type: string; - [key: string]: any; - }; interface Plugin { - isSupported?(): boolean; + isSupported(): boolean; beforeClockTemplate?(varNames: PluginHookVarNames): string[]; afterClockTemplate?(varNames: PluginHookVarNames): string[]; onCompleteBenchmark?( result: OnCompleteBenchmarkResult, - bench: Benchmark, + benchmark: BenchmarkMetadata, ): void; - toString?(): string; getReport?(benchmarkName: string): string; - getResult?(benchmarkName: string): PluginResult; + getResult?(benchmarkName: string): any; reset?(): void; + toString(): string; } class Suite { @@ -115,24 +149,25 @@ export declare namespace BenchNode { class V8NeverOptimizePlugin implements Plugin { isSupported(): boolean; beforeClockTemplate(varNames: PluginHookVarNames): string[]; - toString(): string; getReport(benchmarkName: string): string; + toString(): string; } class V8GetOptimizationStatus implements Plugin { isSupported(): boolean; afterClockTemplate(varNames: PluginHookVarNames): string[]; onCompleteBenchmark(result: OnCompleteBenchmarkResult): void; - toString(): string; getReport(benchmarkName: string): string; - getResult?(benchmarkName: string): PluginResult; + getResult(benchmarkName: string): PluginResult; + reset(): void; + toString(): string; } class V8OptimizeOnNextCallPlugin implements Plugin { isSupported(): boolean; beforeClockTemplate(varNames: PluginHookVarNames): string[]; - toString(): string; getReport(): string; + toString(): string; } class MemoryPlugin implements Plugin { @@ -142,30 +177,30 @@ export declare namespace BenchNode { onCompleteBenchmark(result: OnCompleteBenchmarkResult): void; getReport(benchmarkName: string): string; getResult(benchmarkName: string): PluginResult; + reset(): void; toString(): string; } + interface DceWarning { + timePerOp: number; + baselineTime: number; + ratio: number; + } + class DeadCodeEliminationDetectionPlugin implements Plugin { constructor(options?: { threshold?: number }); isSupported(): boolean; setBaseline(timePerOp: number): void; onCompleteBenchmark( result: OnCompleteBenchmarkResult, - bench: Benchmark, + benchmark: BenchmarkMetadata, ): void; - getWarning( - benchmarkName: string, - ): { timePerOp: number; baselineTime: number; ratio: number } | undefined; - getAllWarnings(): Array<{ - name: string; - timePerOp: number; - baselineTime: number; - ratio: number; - }>; + getWarning(benchmarkName: string): DceWarning | undefined; + getAllWarnings(): Array; hasWarning(benchmarkName: string): boolean; emitWarnings(): void; - toString(): string; reset(): void; + toString(): string; } } @@ -181,8 +216,8 @@ export declare class V8NeverOptimizePlugin extends BenchNode.V8NeverOptimizePlug export declare class V8GetOptimizationStatus extends BenchNode.V8GetOptimizationStatus {} export declare class V8OptimizeOnNextCallPlugin extends BenchNode.V8OptimizeOnNextCallPlugin {} export declare class MemoryPlugin extends BenchNode.MemoryPlugin {} +export declare class DeadCodeEliminationDetectionPlugin extends BenchNode.DeadCodeEliminationDetectionPlugin {} -// Statistical T-Test utilities export declare namespace TTest { interface WelchTTestResult { tStatistic: number; @@ -208,8 +243,6 @@ export declare namespace TTest { /** * Returns significance stars based on p-value thresholds. - * @param pValue - The p-value from statistical test - * @returns Stars indicating significance level ('***', '**', '*', or '') */ export declare function getSignificanceStars( pValue: number, @@ -217,10 +250,6 @@ export declare function getSignificanceStars( /** * Performs Welch's t-test for two independent samples. - * Does not assume equal variances between the samples. - * @param sample1 - First sample array - * @param sample2 - Second sample array - * @returns Test results including t-statistic, degrees of freedom, p-value, and significance */ export declare function welchTTest( sample1: number[], @@ -228,17 +257,10 @@ export declare function welchTTest( ): TTest.WelchTTestResult; /** - * Determines if two benchmark results are statistically different - * using Welch's t-test at a given significance level. - * @param sample1 - Sample data from first benchmark - * @param sample2 - Sample data from second benchmark - * @param alpha - Significance level (default 0.05 for 95% confidence) - * @returns Comparison result with significance info + * Determines if two benchmark results are statistically different. */ export declare function compareBenchmarks( sample1: number[], sample2: number[], alpha?: number, ): TTest.CompareBenchmarksResult; - -export declare class DeadCodeEliminationDetectionPlugin extends BenchNode.DeadCodeEliminationDetectionPlugin {} diff --git a/lib/clock.js b/lib/clock.js index 2d70016..af022d1 100644 --- a/lib/clock.js +++ b/lib/clock.js @@ -3,12 +3,6 @@ const { validateNumber } = require("./validators"); const debugBench = debuglog("benchmark"); -const kUnmanagedTimerResult = Symbol("kUnmanagedTimerResult"); - -// If the smallest time measurement is 1ns -// the minimum resolution of this timer is 0.5 -const MIN_RESOLUTION = 0.5; - class Timer { constructor() { this.now = process.hrtime.bigint; @@ -22,249 +16,19 @@ class Timer { return 1 / 1e9; } - /** - * @param {number} timeInNs - * @returns {string} - */ format(timeInNs) { validateNumber(timeInNs, "timeInNs", 0); - if (timeInNs > 1e9) { - return `${(timeInNs / 1e9).toFixed(2)}s`; - } - - if (timeInNs > 1e6) { - return `${(timeInNs / 1e6).toFixed(2)}ms`; - } - - if (timeInNs > 1e3) { - return `${(timeInNs / 1e3).toFixed(2)}us`; - } - - return `${(timeInNs).toFixed(2)}ns`; + if (timeInNs > 1e9) return `${(timeInNs / 1e9).toFixed(2)}s`; + if (timeInNs > 1e6) return `${(timeInNs / 1e6).toFixed(2)}ms`; + if (timeInNs > 1e3) return `${(timeInNs / 1e3).toFixed(2)}us`; + return `${timeInNs.toFixed(2)}ns`; } } const timer = new Timer(); -class ManagedTimer { - startTime; - endTime; - iterations; - recommendedCount; - - /** - * @param {number} recommendedCount - */ - constructor(recommendedCount) { - this.recommendedCount = recommendedCount; - } - - /** - * Returns the recommended value to be used to benchmark your code - * @returns {number} - */ - get count() { - return this.recommendedCount; - } - - /** - * Starts the timer - */ - start() { - this.startTime = timer.now(); - } - - /** - * Stops the timer - * @param {number} [iterations=1] The amount of iterations that run - */ - end(iterations = 1) { - this.endTime = timer.now(); - validateNumber(iterations, "iterations", 1); - this.iterations = iterations; - } - - [kUnmanagedTimerResult](context) { - if (this.startTime === undefined) - throw new Error("You forgot to call .start()"); - - if (this.endTime === undefined) - throw new Error("You forgot to call .end(count)"); - - return [Number(this.endTime - this.startTime), this.iterations, context]; - } -} - -function createRunUnmanagedBenchmark(bench, awaitOrEmpty) { - const varNames = { - awaitOrEmpty, - timer: "timer", - context: "context", - bench: "bench", - managed: false, - }; - - let code = ` -let i = 0; -let ${varNames.context} = {}; -`; - - let benchFnCall = `${awaitOrEmpty}${varNames.bench}.fn()`; - const wrapFunctions = []; - for (const p of bench.plugins) { - if (typeof p.beforeClockTemplate === "function") { - const [newCode, functionToCall] = p.beforeClockTemplate(varNames); - code += newCode; - if (functionToCall) { - wrapFunctions.push(functionToCall); - } - } - } - benchFnCall = wrapFunctions.reduce((prev, n) => { - return `${n}(${prev})`; - }, benchFnCall); - - code += ` -const startedAt = ${varNames.timer}.now(); - -for (; i < count; i++) - ${benchFnCall}; - -const duration = Number(${varNames.timer}.now() - startedAt); -`; - - for (const p of bench.plugins) { - if (typeof p.afterClockTemplate === "function") { - const [newCode] = p.afterClockTemplate(varNames); - code += newCode; - } - } - - code += `return [duration, count, ${varNames.context}];`; - return code; -} - -function createRunManagedBenchmark(bench, awaitOrEmpty) { - const varNames = { - awaitOrEmpty, - timer: "timer", - context: "context", - bench: "bench", - managed: true, - }; - - let code = ` -let i = 0; -let ${varNames.context} = {}; -`; - - let benchFnCall = `${awaitOrEmpty}${varNames.bench}.fn(${varNames.timer})`; - const wrapFunctions = []; - for (const p of bench.plugins) { - if (typeof p.beforeClockTemplate === "function") { - const [newCode, functionToCall] = p.beforeClockTemplate(varNames); - code += newCode; - if (functionToCall) { - wrapFunctions.push(functionToCall); - } - } - } - benchFnCall = wrapFunctions.reduce((prev, n) => { - return `${n}(${prev})`; - }, benchFnCall); - - code += ` -${benchFnCall}; -const result = ${varNames.timer}[kUnmanagedTimerResult](${varNames.context}); -`; - for (const p of bench.plugins) { - if (typeof p.afterClockTemplate === "function") { - const [newCode] = p.afterClockTemplate(varNames); - code += newCode; - } - } - - code += "return result;"; - return code; -} - -const AsyncFunction = (async () => {}).constructor; -const SyncFunction = (() => {}).constructor; - -function createFnString(bench) { - const { isAsync, hasArg } = bench; - - const compiledFnStringFactory = hasArg - ? createRunManagedBenchmark - : createRunUnmanagedBenchmark; - const compiledFnString = compiledFnStringFactory( - bench, - isAsync ? "await " : "", - ); - return compiledFnString; -} - -function createRunner(bench, recommendedCount) { - const { isAsync, hasArg } = bench; - const compiledFnString = bench.fnStr; - - const createFnPrototype = isAsync ? AsyncFunction : SyncFunction; - const compiledFn = createFnPrototype( - "bench", - "timer", - "count", - "kUnmanagedTimerResult", - compiledFnString, - ); - const selectedTimer = hasArg ? new ManagedTimer(recommendedCount) : timer; - const runner = compiledFn.bind( - globalThis, - bench, - selectedTimer, - recommendedCount, - kUnmanagedTimerResult, - ); - debugBench(`Compiled Code: ${compiledFnString}`); - debugBench( - `Created compiled benchmark, hasArg=${hasArg}, isAsync=${isAsync}, recommendedCount=${recommendedCount}`, - ); - - return runner; -} - -/** - * Executes a benchmark and returns the time taken and number of iterations - * @param {import('./index').Benchmark} bench - The benchmark to execute - * @param {number} recommendedCount - The recommended number of iterations - * @param {Object} [options] - Additional options - * @param {boolean} [options.timeMode=false] - If true, runs the benchmark exactly once - * @returns {Promise<[number, number]>} - Returns [duration, iterations] - */ -async function clockBenchmark(bench, recommendedCount, options = {}) { - const runner = createRunner(bench, recommendedCount); - const result = await runner(); - - // Just to avoid issues with empty fn - result[0] = Math.max(MIN_RESOLUTION, result[0]); - - for (const p of bench.plugins) { - if (typeof p.onCompleteBenchmark === "function") { - // TODO: this won't work when useWorkers=true - p.onCompleteBenchmark(result, bench); - } - } - - debugBench( - `Took ${timer.format(result[0])} to execute ${result[1]} iterations${options.timeMode ? " (time mode)" : ""}`, - ); - return result; -} - module.exports = { - clockBenchmark, - createFnString, - timer, - MIN_RESOLUTION, debugBench, + timer, }; diff --git a/lib/index.js b/lib/index.js index 9bd1455..06af608 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,4 +1,4 @@ -const { Worker } = require("node:worker_threads"); +const workerThreads = require("node:worker_threads"); const { types } = require("node:util"); const path = require("node:path"); @@ -10,12 +10,13 @@ const { csvReport, prettyReport, } = require("./report"); +const { debugBench, timer } = require("./clock"); +const { isBenchCli, runNativeBenchmark } = require("./native-runner"); const { - getInitialIterations, - runBenchmark, - runWarmup, -} = require("./lifecycle"); -const { debugBench, timer, createFnString } = require("./clock"); + createNativeFnString, + parsePluginsResult, + resetPlugins, +} = require("./plugin-runner"); const { validatePlugins, V8NeverOptimizePlugin, @@ -39,9 +40,6 @@ const { getSignificanceStars, } = require("./utils/ttest"); -const getFunctionBody = (string) => - string.substring(string.indexOf("{") + 1, string.lastIndexOf("}")); - class Benchmark { name = "Benchmark"; fn; @@ -80,14 +78,24 @@ class Benchmark { this.isAsync = types.isAsyncFunction(this.fn); - this.fnStr = createFnString(this); + this.fnStr = createNativeFnString(this); } serializeBenchmark() { - // Regular functions can't be passed to worker.postMessage - // So we pass the string and deserialize fnStr into a new Function - // on worker - this.fn = getFunctionBody(this.fn.toString()); + return { + baseline: this.baseline, + capturePluginSamples: true, + fnSource: this.fn.toString(), + fnStr: this.fnStr, + hasArg: this.hasArg, + isAsync: this.isAsync, + maxTime: this.maxTime, + minSamples: this.minSamples, + minTime: this.minTime, + name: this.name, + plugins: [], + repeatSuite: this.repeatSuite, + }; } } @@ -248,76 +256,59 @@ class Suite { async run() { throwIfNoNativesSyntax(); const results = new Array(this.#benchmarks.length); - - // Measure baseline for DCE detection (only in ops mode, not in worker mode) - if ( + const nativeCli = isBenchCli(); + const dceEnabled = this.#dceDetector && !this.#useWorkers && - this.#benchmarkMode === "ops" - ) { + !nativeCli && + this.#benchmarkMode === "ops"; + + if (dceEnabled) { await this.#measureBaseline(); + for (const benchmark of this.#benchmarks) { + if (!benchmark.plugins.includes(this.#dceDetector)) { + benchmark.plugins = [...benchmark.plugins, this.#dceDetector]; + benchmark.fnStr = createNativeFnString(benchmark); + } + } } - // It doesn't make sense to warmup a fresh new instance of Worker. - // TODO: Should this be folded into the main loop? - if (!this.#useWorkers) { - // This is required to avoid variance on first benchmark run + if (nativeCli) { + // The native CLI already runs benchmark files in isolated child processes. + // Workers inherit --bench, where explicit runners are intentionally invalid, + // so declarations must stay in the CLI-managed process in this mode. + const pending = this.#benchmarks.map((benchmark) => + runNativeBenchmark(benchmark, this.#benchmarkMode, true), + ); + const nativeResults = await Promise.all(pending); + for (let i = 0; i < nativeResults.length; i++) { + results[i] = nativeResults[i]; + } + } else if (this.#useWorkers) { + for (let i = 0; i < this.#benchmarks.length; ++i) { + const benchmark = this.#benchmarks[i]; + results[i] = await this.runWorkerBenchmark(benchmark); + } + } else { for (let i = 0; i < this.#benchmarks.length; ++i) { const benchmark = this.#benchmarks[i]; debugBench( - `Warmup ${benchmark.name} with minTime=${benchmark.minTime}, maxTime=${benchmark.maxTime}`, + `Starting ${benchmark.name} with node:bench, mode=${this.#benchmarkMode}, minTime=${benchmark.minTime}, maxTime=${benchmark.maxTime}, repeatSuite=${benchmark.repeatSuite}, minSamples=${benchmark.minSamples}`, ); - const initialIteration = await getInitialIterations(benchmark); - await runWarmup(benchmark, initialIteration, { - minTime: 0.005, - maxTime: 0.05, - }); - } - } - - for (let i = 0; i < this.#benchmarks.length; ++i) { - const benchmark = this.#benchmarks[i]; - - // Add DCE detector to benchmark plugins if enabled - if (this.#dceDetector && this.#benchmarkMode === "ops") { - const originalPlugins = benchmark.plugins; - benchmark.plugins = [...benchmark.plugins, this.#dceDetector]; - // Regenerate function string with new plugins - benchmark.fnStr = createFnString(benchmark); - } - - // Warmup is calculated to reduce noise/bias on the results - const initialIterations = await getInitialIterations(benchmark); - debugBench( - `Starting ${benchmark.name} with mode=${this.#benchmarkMode}, minTime=${benchmark.minTime}, maxTime=${benchmark.maxTime}, repeatSuite=${benchmark.repeatSuite}, minSamples=${benchmark.minSamples}`, - ); - - let result; - if (this.#useWorkers) { - if (this.#benchmarkMode === "time") { - console.warn( - "Warning: Worker mode currently doesn't fully support 'time' benchmarkMode.", - ); - } - result = await this.runWorkerBenchmark(benchmark, initialIterations); - } else { - result = await runBenchmark( + results[i] = await runNativeBenchmark( benchmark, - initialIterations, this.#benchmarkMode, - benchmark.repeatSuite, - benchmark.minSamples, + false, ); } - results[i] = result; } - if (this.#reporter) { + if (this.#reporter && !nativeCli) { this.#reporter(results, this.#reporterOptions); } // Emit DCE warnings after reporting - if (this.#dceDetector) { + if (dceEnabled) { this.#dceDetector.emitWarnings(); } @@ -338,14 +329,7 @@ class Suite { 10, // minSamples ); - const initialIterations = await getInitialIterations(baselineBench); - const result = await runBenchmark( - baselineBench, - initialIterations, - "ops", - 1, - 10, - ); + const result = await runNativeBenchmark(baselineBench, "ops", false); const baselineTimePerOp = (1 / result.opsSec) * 1e9; // Convert to ns debugBench(`DCE baseline: ${timer.format(baselineTimePerOp)}/iter`); @@ -353,21 +337,24 @@ class Suite { this.#dceDetector.setBaseline(baselineTimePerOp); } - async runWorkerBenchmark(benchmark, initialIterations) { + async runWorkerBenchmark(benchmark) { return new Promise((resolve, reject) => { const workerPath = path.resolve(__dirname, "./worker-runner.js"); - const worker = new Worker(workerPath); + const worker = new workerThreads.Worker(workerPath); - benchmark.serializeBenchmark(); worker.postMessage({ - benchmark, - initialIterations, + benchmark: benchmark.serializeBenchmark(), benchmarkMode: this.#benchmarkMode, // Pass suite mode - repeatSuite: benchmark.repeatSuite, - minSamples: benchmark.minSamples, }); - worker.on("message", (result) => { + worker.on("message", ({ pluginSamples, result }) => { + for (const pluginResult of pluginSamples) { + for (const plugin of benchmark.plugins) { + plugin.onCompleteBenchmark?.(pluginResult, benchmark); + } + } + result.plugins = parsePluginsResult(benchmark.plugins, benchmark.name); + resetPlugins(benchmark.plugins); resolve(result); worker.terminate(); }); diff --git a/lib/lifecycle.js b/lib/lifecycle.js deleted file mode 100644 index f732ad6..0000000 --- a/lib/lifecycle.js +++ /dev/null @@ -1,240 +0,0 @@ -const { - clockBenchmark, - debugBench, - MIN_RESOLUTION, - timer, -} = require("./clock"); -const { StatisticalHistogram } = require("./histogram"); - -/** - * @param {number} durationPerOp The amount of time each operation takes, in timer.scale - * @param {number} targetTime The amount of time we want the benchmark to execute, in seconds - * @return {number} - a suggested iteration count >= 1 - */ -function getItersForOpDuration(durationPerOp, targetTime) { - const secondsPerOp = durationPerOp / timer.scale; - const opsForTargetTime = Math.round(targetTime / secondsPerOp); - - return Math.min(Number.MAX_SAFE_INTEGER, Math.max(1, opsForTargetTime)); -} - -function parsePluginsResult(plugins, name) { - const result = []; - for (const p of plugins) { - result.push({ - name: p.toString(), - result: p.getResult?.(name) ?? "enabled", - report: p.getReport?.(name) ?? "", - }); - } - return result; -} - -/** - * Calculates and returns the initial number of iterations for a benchmark - * @param {import('./index').Benchmark} bench - The benchmark object to be executed - * @returns {Promise} The calculated number of initial iterations - */ -async function getInitialIterations(bench) { - const { 0: duration, 1: realIterations } = await clockBenchmark(bench, 30); - - // Just to avoid issues with empty fn - const durationPerOp = Math.max(MIN_RESOLUTION, duration / realIterations); - debugBench( - `Duration per operation on initial count: ${timer.format(durationPerOp)}`, - ); - - // TODO: is this a correct assumpion? - if (durationPerOp > bench.maxTime * timer.scale) - process.emitWarning( - `The benchmark "${bench.name}" has a duration per operation greater than the maxTime.`, - ); - - return getItersForOpDuration(durationPerOp, bench.minTime); -} - -/** - * Executes the warmup phase of a benchmark - * @param {import('./index').Benchmark} bench - The benchmark object to be executed - * @param {number} initialIterations - The initial number of iterations to run - * @param {Object} options - Warmup options - * @param {number} [options.minTime] - Minimum time for warmup, in seconds. Defaults to bench.minTime - * @param {number} [options.maxTime] - Maximum time for warmup, in seconds. Defaults to bench.minTime - * @returns {Promise} - */ -async function runWarmup(bench, initialIterations, { minTime, maxTime }) { - minTime = minTime ?? bench.minTime; - maxTime = maxTime ?? bench.minTime; - - const maxDuration = maxTime * timer.scale; - const minSamples = 10; - - let iterations = 0; - let timeSpent = 0; - let samples = 0; - - while (timeSpent < maxDuration || samples <= minSamples) { - const { 0: duration, 1: realIterations } = await clockBenchmark( - bench, - initialIterations, - ); - timeSpent += duration; - - iterations += realIterations; - iterations = Math.min(Number.MAX_SAFE_INTEGER, iterations); - - // Just to avoid issues with empty fn - const durationPerOp = Math.max(MIN_RESOLUTION, duration / realIterations); - const remainingTime = Math.max(0, (maxDuration - timeSpent) / timer.scale); - const targetTime = Math.min(remainingTime, minTime); - - initialIterations = getItersForOpDuration(durationPerOp, targetTime); - samples++; - } -} - -async function collectSamplesOfTimeMode(bench, histogram, minSamples) { - let samples = 0; - let iterations = 0; - let timeSpent = 0; - while (samples < minSamples) { - const { 0: duration, 1: realIterations } = await clockBenchmark(bench, 1); - timeSpent += duration; - iterations += realIterations; - - // Record the duration in the histogram - histogram.record(duration); - samples++; - } - - return { iterations, timeSpent }; -} - -async function runBenchmarkOnce( - bench, - histogram, - { initialIterations, maxDuration, minSamples }, - benchmarkMode = "ops", -) { - let iterations = 0; - let timeSpent = 0; - - // For time mode, collect minSamples measurements, each timing a single - // execution. A local counter is used (rather than histogram.samples.length) - // because the histogram is shared across repeatSuite iterations. - if (benchmarkMode === "time") { - return collectSamplesOfTimeMode(bench, histogram, minSamples); - } - - // Ops mode - run the sampling loop - while (timeSpent < maxDuration || histogram.samples.length <= minSamples) { - const { 0: duration, 1: realIterations } = await clockBenchmark( - bench, - initialIterations, - ); - timeSpent += duration; - - iterations = Math.min(Number.MAX_SAFE_INTEGER, iterations + realIterations); - - // Just to avoid issues with empty fn - const durationPerOp = Math.max(MIN_RESOLUTION, duration / realIterations); - - histogram.record(durationPerOp); - - const remainingTime = Math.max(0, (maxDuration - timeSpent) / timer.scale); - const targetTime = Math.min(remainingTime, bench.minTime); - initialIterations = getItersForOpDuration(durationPerOp, targetTime); - } - - return { iterations, timeSpent }; -} - -/** - * Executes a benchmark with the specified parameters - * @param {import('./index').Benchmark} bench - The benchmark object to be executed - * @param {number} initialIterations - The initial number of iterations to run - * @param {string} benchmarkMode - The benchmark mode ('ops' or 'time') - * @param {number} repeatSuite - Number of times to repeat the benchmark suite - * @param {number} minSamples - Minimum number of samples to collect - * @returns {Promise} The benchmark results containing operations per second or total time, iterations, histogram data and plugin results - */ -async function runBenchmark( - bench, - initialIterations, - benchmarkMode, - repeatSuite, - minSamples, -) { - const histogram = new StatisticalHistogram(); - const maxDuration = bench.maxTime * timer.scale; - - let totalIterations = 0; - let totalTimeSpent = 0; - const opsSecPerRun = []; - - for (let i = 0; i < repeatSuite; ++i) { - const { iterations, timeSpent } = await runBenchmarkOnce( - bench, - histogram, - { - initialIterations, - maxDuration, - minSamples, - }, - benchmarkMode, - ); - - const runOpsSec = iterations / (timeSpent / timer.scale); - opsSecPerRun.push(runOpsSec); - - totalTimeSpent += timeSpent; - totalIterations += iterations; - } - histogram.finish(); - - const totalTime = totalTimeSpent / timer.scale; // Convert ns to seconds - const opsSec = totalIterations / totalTime; - - const sampleData = histogram.samples; - - const result = { - iterations: totalIterations, - // StatisticalHistogram is not a serializable object, keep raw ns for min/max - histogram: { - samples: sampleData.length, - min: histogram.min, - max: histogram.max, - sampleData, - }, - name: bench.name, - plugins: parsePluginsResult(bench.plugins, bench.name), - baseline: bench.baseline, - }; - - // Add the appropriate metric based on the benchmark mode - if (benchmarkMode === "time") { - result.totalTime = totalTime / totalIterations; // Mean time per execution - debugBench( - `${bench.name} completed ${repeatSuite} repeats with average time ${result.totalTime.toFixed(6)} seconds`, - ); - } else { - result.opsSec = opsSec; - result.opsSecPerRun = opsSecPerRun; - debugBench( - `${bench.name} completed ${sampleData.length} samples with ${opsSec.toFixed(2)} ops/sec`, - ); - } - - // since the instance is shared across benchmarks, reset it after use - for (const plugin of bench.plugins) { - plugin.reset?.(); - } - - return result; -} - -module.exports = { - getInitialIterations, - runBenchmark, - runWarmup, -}; diff --git a/lib/native-runner.js b/lib/native-runner.js new file mode 100644 index 0000000..bc90601 --- /dev/null +++ b/lib/native-runner.js @@ -0,0 +1,250 @@ +const { bench: declareBench, createRunner } = require("node:bench"); + +const { debugBench, timer: clock } = require("./clock"); +const { StatisticalHistogram } = require("./histogram"); +const { + ManagedTimer, + completePluginSample, + createPluginInvoker, + parsePluginsResult, + resetPlugins, +} = require("./plugin-runner"); + +const MAX_NATIVE_SAMPLES = 0xffffffff; +const WARMUP_SAMPLES = 2; + +const cliNameCounts = new Map(); + +function isBenchCli() { + return process.execArgv.some( + (arg) => arg === "--bench" || arg.startsWith("--bench="), + ); +} + +function getIterations(durationPerOperation, targetDuration) { + if (targetDuration <= 0) return 1; + + return Math.min( + Number.MAX_SAFE_INTEGER, + Math.max(1, Math.round(targetDuration / durationPerOperation)), + ); +} + +function createExecution(benchmark, benchmarkMode) { + const repeatCount = Math.ceil(benchmark.repeatSuite); + const samplesPerRun = Math.ceil(benchmark.minSamples); + const state = { + count: benchmarkMode === "time" ? 1 : 30, + currentRun: 0, + initialMeasurementCount: undefined, + measurementSamples: 0, + pluginSamples: [], + runIndexes: [], + samplesPerRun, + runs: Array.from({ length: repeatCount }, () => ({ + duration: 0, + operations: 0, + samples: 0, + })), + }; + + const invoke = createPluginInvoker(benchmark); + const callback = benchmark.isAsync + ? async (context) => { + const timer = benchmark.hasArg ? new ManagedTimer(state.count) : clock; + const [sample, pluginContext] = await invoke( + benchmark, + context, + timer, + state.count, + ); + recordPluginSample(benchmark, state, sample, pluginContext); + afterSample(benchmark, benchmarkMode, state, context, sample); + } + : (context) => { + const timer = benchmark.hasArg ? new ManagedTimer(state.count) : clock; + const [sample, pluginContext] = invoke( + benchmark, + context, + timer, + state.count, + ); + recordPluginSample(benchmark, state, sample, pluginContext); + afterSample(benchmark, benchmarkMode, state, context, sample); + }; + + return { callback, repeatCount, state }; +} + +function recordPluginSample(benchmark, state, sample, pluginContext) { + if (benchmark.capturePluginSamples) { + state.pluginSamples.push([ + Number(sample.duration_ns), + sample.operations, + pluginContext, + ]); + return; + } + + completePluginSample(benchmark, sample, pluginContext); +} + +function afterSample(benchmark, benchmarkMode, state, context, sample) { + if (benchmarkMode === "time") { + if (context.phase === "measurement") { + state.runIndexes.push( + Math.floor(state.measurementSamples / state.samplesPerRun), + ); + state.measurementSamples++; + } + return; + } + + const duration = Number(sample.duration_ns); + const durationPerOperation = duration / sample.operations; + + if (context.phase === "warmup") { + state.count = getIterations(durationPerOperation, benchmark.minTime * 1e9); + return; + } + + state.initialMeasurementCount ??= state.count; + const run = state.runs[state.currentRun]; + run.duration += duration; + run.operations += sample.operations; + run.samples++; + state.runIndexes.push(state.currentRun); + + const runComplete = + run.duration >= benchmark.maxTime * 1e9 && + run.samples > benchmark.minSamples; + + if (runComplete) { + if (state.currentRun + 1 === state.runs.length) { + context.done(); + return; + } + + state.currentRun++; + state.count = state.initialMeasurementCount; + return; + } + + const remainingDuration = Math.max( + 0, + Math.min(benchmark.maxTime * 1e9 - run.duration, benchmark.minTime * 1e9), + ); + state.count = getIterations(durationPerOperation, remainingDuration); +} + +function normalizeResult(benchmark, benchmarkMode, nativeResult, state) { + if (nativeResult.error !== undefined) throw nativeResult.error; + + const histogram = new StatisticalHistogram(); + let totalDuration = 0; + let totalIterations = 0; + + for (let i = 0; i < nativeResult.samples.length; i++) { + const sample = nativeResult.samples[i]; + const duration = Number(sample.duration_ns); + const run = state.runs[state.runIndexes[i]]; + run.duration += benchmarkMode === "time" ? duration : 0; + run.operations += benchmarkMode === "time" ? sample.operations : 0; + run.samples += benchmarkMode === "time" ? 1 : 0; + totalDuration += duration; + totalIterations += sample.operations; + histogram.record(duration / sample.operations); + } + histogram.finish(); + + const plugins = parsePluginsResult(benchmark.plugins, benchmark.name); + resetPlugins(benchmark.plugins); + + const result = { + iterations: totalIterations, + histogram: { + samples: histogram.samples.length, + min: histogram.min, + max: histogram.max, + sampleData: histogram.samples, + }, + name: benchmark.name, + plugins, + baseline: benchmark.baseline, + }; + + if (benchmarkMode === "time") { + result.totalTime = totalDuration / 1e9 / totalIterations; + } else { + result.opsSec = totalIterations / (totalDuration / 1e9); + result.opsSecPerRun = state.runs.map( + (run) => run.operations / (run.duration / 1e9), + ); + } + + debugBench( + `${benchmark.name} completed ${nativeResult.samples.length} native samples`, + ); + return result; +} + +function getNativeOptions(benchmark, benchmarkMode, cli) { + const options = { + samples: + benchmarkMode === "time" + ? Math.ceil(benchmark.repeatSuite) * Math.ceil(benchmark.minSamples) + : MAX_NATIVE_SAMPLES, + warmup: WARMUP_SAMPLES, + }; + + if (cli) { + const count = cliNameCounts.get(benchmark.name) ?? 0; + cliNameCounts.set(benchmark.name, count + 1); + if (count > 0) options.params = { __benchNodeDeclaration: count }; + } + + return options; +} + +async function runNativeBenchmark( + benchmark, + benchmarkMode, + cli = isBenchCli(), + includeState = false, +) { + const execution = createExecution(benchmark, benchmarkMode); + const options = getNativeOptions(benchmark, benchmarkMode, cli); + let nativeResult; + + if (cli) { + nativeResult = await declareBench( + benchmark.name, + options, + execution.callback, + ); + } else { + const runner = createRunner({ yieldBetweenSamples: false }); + const completion = runner.bench( + benchmark.name, + options, + execution.callback, + ); + for await (const _record of runner.run()); + nativeResult = await completion; + } + + const result = normalizeResult( + benchmark, + benchmarkMode, + nativeResult, + execution.state, + ); + return includeState + ? { pluginSamples: execution.state.pluginSamples, result } + : result; +} + +module.exports = { + isBenchCli, + runNativeBenchmark, +}; diff --git a/lib/plugin-runner.js b/lib/plugin-runner.js new file mode 100644 index 0000000..96b9661 --- /dev/null +++ b/lib/plugin-runner.js @@ -0,0 +1,126 @@ +const { validateNumber } = require("./validators"); + +const AsyncFunction = (async () => {}).constructor; +const SyncFunction = (() => {}).constructor; + +class ManagedTimer { + #endTime; + #iterations; + #startTime; + + constructor(count) { + this.count = count; + } + + start() { + this.#startTime = process.hrtime.bigint(); + } + + end(iterations = 1) { + this.#endTime = process.hrtime.bigint(); + validateNumber(iterations, "iterations", 1); + this.#iterations = iterations; + } + + record(context) { + if (this.#startTime === undefined) { + throw new Error("You forgot to call .start()"); + } + if (this.#endTime === undefined) { + throw new Error("You forgot to call .end(count)"); + } + + const duration = this.#endTime - this.#startTime; + return context.record({ + duration_ns: duration > 0n ? duration : 1n, + operations: this.#iterations, + }); + } +} + +function createNativeFnString(benchmark) { + const varNames = { + awaitOrEmpty: benchmark.isAsync ? "await " : "", + bench: "bench", + context: "context", + timer: "timer", + managed: benchmark.hasArg, + }; + + let code = "let context = {};\n"; + let benchmarkCall = benchmark.hasArg + ? `${varNames.awaitOrEmpty}${varNames.bench}.fn(${varNames.timer})` + : `${varNames.awaitOrEmpty}${varNames.bench}.fn()`; + const wrappers = []; + + for (const plugin of benchmark.plugins) { + if (typeof plugin.beforeClockTemplate !== "function") continue; + const [injectedCode, wrapper] = plugin.beforeClockTemplate(varNames); + code += injectedCode; + if (wrapper) wrappers.push(wrapper); + } + + benchmarkCall = wrappers.reduce( + (previous, wrapper) => `${wrapper}(${previous})`, + benchmarkCall, + ); + + if (benchmark.hasArg) { + code += `${benchmarkCall};\n`; + code += "const sample = timer.record(nodeContext);\n"; + } else { + code += "nodeContext.start();\n"; + code += `for (let i = 0; i < count; i++) ${benchmarkCall};\n`; + code += "const sample = nodeContext.end(count);\n"; + } + + for (const plugin of benchmark.plugins) { + if (typeof plugin.afterClockTemplate !== "function") continue; + const [injectedCode] = plugin.afterClockTemplate(varNames); + code += injectedCode; + } + + code += "return [sample, context];"; + return code; +} + +function createPluginInvoker(benchmark) { + const FunctionConstructor = benchmark.isAsync ? AsyncFunction : SyncFunction; + return FunctionConstructor( + "bench", + "nodeContext", + "timer", + "count", + benchmark.fnStr, + ); +} + +function completePluginSample(benchmark, sample, context) { + const result = [Number(sample.duration_ns), sample.operations, context]; + for (const plugin of benchmark.plugins) { + plugin.onCompleteBenchmark?.(result, benchmark); + } +} + +function parsePluginsResult(plugins, name) { + return plugins.map((plugin) => ({ + name: plugin.toString(), + result: plugin.getResult?.(name) ?? "enabled", + report: plugin.getReport?.(name) ?? "", + })); +} + +function resetPlugins(plugins) { + for (const plugin of plugins) { + plugin.reset?.(); + } +} + +module.exports = { + ManagedTimer, + completePluginSample, + createNativeFnString, + createPluginInvoker, + parsePluginsResult, + resetPlugins, +}; diff --git a/lib/plugins.js b/lib/plugins.js index 643deaf..26b0bab 100644 --- a/lib/plugins.js +++ b/lib/plugins.js @@ -30,6 +30,7 @@ function validatePlugins(plugins) { awaitOrEmpty: "", context: "", timer: "", + managed: false, }); validateArray(result, `${p.toString()}.beforeClockTemplate()`); } @@ -40,6 +41,7 @@ function validatePlugins(plugins) { awaitOrEmpty: "", context: "", timer: "", + managed: false, }); validateArray(result, `${p.toString()}.afterClockTemplate()`); } diff --git a/lib/plugins/v8-opt.js b/lib/plugins/v8-opt.js index 5f150af..60fd3dd 100644 --- a/lib/plugins/v8-opt.js +++ b/lib/plugins/v8-opt.js @@ -1,7 +1,14 @@ class V8OptimizeOnNextCallPlugin { isSupported() { try { - new Function("%OptimizeFunctionOnNextCall(() => {})")(); + new Function(` + const fn = () => {}; + %PrepareFunctionForOptimization(fn); + fn(); + fn(); + %OptimizeFunctionOnNextCall(fn); + fn(); + `)(); return true; } catch (e) { @@ -12,9 +19,10 @@ class V8OptimizeOnNextCallPlugin { beforeClockTemplate({ awaitOrEmpty, bench, timer }) { let code = ""; - code += `%OptimizeFunctionOnNextCall(${bench}.fn);\n`; + code += `%PrepareFunctionForOptimization(${bench}.fn);\n`; code += `${awaitOrEmpty}${bench}.fn(${timer});\n`; code += `${awaitOrEmpty}${bench}.fn(${timer});\n`; + code += `%OptimizeFunctionOnNextCall(${bench}.fn);\n`; return [code]; } diff --git a/lib/worker-runner.js b/lib/worker-runner.js index b24f0e7..99c6e43 100644 --- a/lib/worker-runner.js +++ b/lib/worker-runner.js @@ -1,51 +1,18 @@ const { parentPort } = require("node:worker_threads"); -const { - runBenchmark, - getInitialIterations, - runWarmup, -} = require("./lifecycle"); -const { debugBench } = require("./clock"); -const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor; +const { runNativeBenchmark } = require("./native-runner"); -// Deserialize the benchmark function function deserializeBenchmark(benchmark) { - const { isAsync, hasArg } = benchmark; - const fnPrototype = isAsync ? AsyncFunction : Function; - - if (hasArg) { - benchmark.fn = new fnPrototype("timer", benchmark.fn); - } else { - benchmark.fn = new fnPrototype(benchmark.fn); - } + benchmark.fn = new Function(`return (${benchmark.fnSource})`)(); + benchmark.fnSource = undefined; } -parentPort.on( - "message", - async ({ +parentPort.on("message", async ({ benchmark, benchmarkMode }) => { + deserializeBenchmark(benchmark); + const output = await runNativeBenchmark( benchmark, - initialIterations, benchmarkMode, - repeatSuite, - minSamples, - }) => { - deserializeBenchmark(benchmark); - - debugBench( - `Warmup ${benchmark.name} with minTime=${benchmark.minTime}, maxTime=${benchmark.maxTime}`, - ); - const initialIteration = await getInitialIterations(benchmark); - await runWarmup(benchmark, initialIteration, { - minTime: 0.005, - maxTime: 0.05, - }); - - const result = await runBenchmark( - benchmark, - initialIterations, - benchmarkMode, - repeatSuite, - minSamples, - ); - parentPort.postMessage(result); - }, -); + false, + true, + ); + parentPort.postMessage(output); +}); diff --git a/package.json b/package.json index fc13457..77cb9d0 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,11 @@ "description": "", "main": "lib/index.js", "types": "index.d.ts", + "engines": { + "node": ">=27.0.0-0" + }, "scripts": { - "test": "c8 node --test --allow-natives-syntax --expose-gc && npm run lint:ci", + "test": "npm run lint:ci && c8 node --test --test-isolation=none --test-concurrency=1 --allow-natives-syntax --expose-gc", "test:types": "tsd -f types/types.test-d.ts", "lint": "biome lint .", "lint:ci": "biome ci .", @@ -33,9 +36,6 @@ "url": "https://github.com/RafaelGSS/bench-node/issues" }, "homepage": "https://github.com/RafaelGSS/bench-node#readme", - "dependencies": { - "piscina": "^4.8.0" - }, "devDependencies": { "@biomejs/biome": "1.9.4", "@types/node": "^20", diff --git a/test/fixtures/native-cli.js b/test/fixtures/native-cli.js new file mode 100644 index 0000000..ea5cb0a --- /dev/null +++ b/test/fixtures/native-cli.js @@ -0,0 +1,26 @@ +const { Suite } = require("../../lib"); + +new Suite({ + reporter: () => { + if (process.execArgv.includes("--bench")) { + console.log("legacy reporter should not run under --bench"); + } + }, + plugins: [], + useWorkers: process.execArgv.includes("--bench"), +}) + .add( + "suite cli", + { minTime: 0.00001, maxTime: 0.00002, minSamples: 2 }, + () => { + Math.sqrt(42); + }, + ) + .add( + "suite cli", + { minTime: 0.00001, maxTime: 0.00002, minSamples: 2 }, + () => { + Math.sqrt(84); + }, + ) + .run(); diff --git a/test/native-runner.js b/test/native-runner.js new file mode 100644 index 0000000..35cc4ff --- /dev/null +++ b/test/native-runner.js @@ -0,0 +1,97 @@ +const assert = require("node:assert"); +const { spawnSync } = require("node:child_process"); +const path = require("node:path"); +const { describe, it } = require("node:test"); + +const { Suite } = require("../lib"); + +describe("node:bench adapter", () => { + it("normalizes native operation samples into legacy results", async () => { + let calls = 0; + const suite = new Suite({ reporter: false, plugins: [] }); + suite.add( + "native ops", + { minTime: 0.00001, maxTime: 0.00002, minSamples: 2 }, + () => { + calls++; + }, + ); + + const [result] = await suite.run(); + + assert.strictEqual(result.name, "native ops"); + assert.ok(result.opsSec > 0); + assert.deepStrictEqual(result.opsSecPerRun.length, 1); + assert.ok(result.iterations >= result.histogram.samples); + assert.ok(result.histogram.samples >= 2); + assert.strictEqual( + result.histogram.sampleData.length, + result.histogram.samples, + ); + assert.ok(calls >= result.iterations); + }); + + it("supports managed timing through the native context", async () => { + const suite = new Suite({ reporter: false, plugins: [] }); + suite.add( + "native managed", + { minTime: 0.00001, maxTime: 0.00002, minSamples: 2 }, + (timer) => { + timer.start(); + for (let i = 0; i < timer.count; i++); + timer.end(timer.count); + }, + ); + + const [result] = await suite.run(); + + assert.ok(result.opsSec > 0); + assert.ok(result.iterations > 0); + }); + + it("preserves time mode repeats", async () => { + const suite = new Suite({ + reporter: false, + plugins: [], + benchmarkMode: "time", + }); + suite.add("native time", { repeatSuite: 3, minSamples: 1 }, () => {}); + + const [result] = await suite.run(); + + assert.strictEqual(result.opsSec, undefined); + assert.ok(result.totalTime > 0); + assert.strictEqual(result.histogram.samples, 3); + assert.strictEqual(result.iterations, 3); + }); + + it("declares Suite benchmarks when launched with --bench", () => { + const fixture = path.join(__dirname, "fixtures", "native-cli.js"); + const child = spawnSync( + process.execPath, + [ + "--no-warnings", + "--allow-natives-syntax", + "--bench", + "--bench-reporter=json", + fixture, + ], + { encoding: "utf8" }, + ); + + assert.strictEqual(child.status, 0, child.stderr || child.stdout); + const records = child.stdout + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + const complete = records.filter( + (record) => + record.type === "bench:complete" && record.data.name === "suite cli", + ); + + assert.strictEqual(complete.length, 2, child.stdout); + assert.ok(complete.every((record) => record.data.samples.length > 0)); + assert.doesNotMatch(child.stdout, /legacy reporter should not run/); + }); +}); diff --git a/test/worker.js b/test/worker.js index 95c3396..804c32a 100644 --- a/test/worker.js +++ b/test/worker.js @@ -1,6 +1,7 @@ const workerThreads = require("node:worker_threads"); const { describe, it, before, after, mock } = require("node:test"); const assert = require("node:assert"); +const { Suite, V8GetOptimizationStatus } = require("../lib"); function noop() {} @@ -45,3 +46,40 @@ describe("Using worker_threads", () => { assert.strictEqual(workerThreads.Worker.mock.calls.length, 4); }); }); + +describe("native worker results", () => { + it("normalizes samples and replays plugin results", async () => { + const suite = new Suite({ + reporter: false, + useWorkers: true, + plugins: [new V8GetOptimizationStatus()], + }); + suite.add( + "worker plugin", + { minTime: 0.00001, maxTime: 0.00002, minSamples: 2 }, + () => Math.sqrt(42), + ); + + const [result] = await suite.run(); + + assert.ok(result.opsSec > 0); + assert.strictEqual(result.plugins[0].name, "V8GetOptimizationStatus"); + assert.match(result.plugins[0].report, /v8-opt-status/); + }); + + it("supports time mode repeats", async () => { + const suite = new Suite({ + reporter: false, + plugins: [], + useWorkers: true, + benchmarkMode: "time", + }); + suite.add("worker time", { repeatSuite: 3, minSamples: 1 }, () => {}); + + const [result] = await suite.run(); + + assert.ok(result.totalTime > 0); + assert.strictEqual(result.histogram.samples, 3); + assert.strictEqual(result.iterations, 3); + }); +}); diff --git a/types/types.test-d.ts b/types/types.test-d.ts index ea8d63f..c7eddb2 100644 --- a/types/types.test-d.ts +++ b/types/types.test-d.ts @@ -1,4 +1,3 @@ -import type { Histogram } from "node:perf_hooks"; import { expectAssignable, expectNotAssignable, expectType } from "tsd"; import { @@ -24,6 +23,8 @@ expectType( benchmarkMode: "ops", useWorkers: true, plugins: [new V8NeverOptimizePlugin()], + pretty: false, + reporterOptions: { printHeader: true }, }), ); expectType(new Suite({ reporter: false })); @@ -47,6 +48,7 @@ expectAssignable({ minTime: 0.1 }); expectAssignable({ maxTime: 1 }); expectAssignable({ repeatSuite: 2 }); expectAssignable({ minSamples: 5 }); +expectAssignable({ baseline: true }); expectNotAssignable({ minTime: "not-a-number" }); // Test Suite.add method @@ -99,14 +101,11 @@ suite.run().then((results) => { expectType(result.opsSecPerRun); expectType(result.totalTime); expectType(result.iterations); - expectType(result.histogram); - expectType | undefined>(result.plugins); + expectType(result.histogram); + expectType(result.plugins); + expectType(result.baseline); - if (result.plugins?.V8GetOptimizationStatus) { - expectType( - result.plugins.V8GetOptimizationStatus.optimizationStatuses, - ); - } + expectType(result.plugins[0].name); } }); @@ -115,11 +114,12 @@ const sampleResults: BenchNode.BenchmarkResult[] = [ { name: "sample", iterations: 100, - histogram: {} as Histogram, // Cast for simplicity in type test + histogram: {} as BenchNode.BenchmarkHistogram, opsSec: 10000, opsSecPerRun: [10000], totalTime: 0.1, - plugins: { MyPlugin: { data: "value" } }, + plugins: [{ name: "MyPlugin", result: { data: "value" }, report: "" }], + baseline: false, }, ]; expectType(textReport(sampleResults)); @@ -136,7 +136,7 @@ if (plugin1.isSupported?.()) { expectType(plugin1.isSupported()); const varNames: BenchNode.PluginHookVarNames = { awaitOrEmpty: "", - bench: "fn", + bench: "bench", context: "context", timer: "timer", managed: false, @@ -151,14 +151,15 @@ if (plugin2.isSupported?.()) { expectType(plugin2.isSupported()); const varNames: BenchNode.PluginHookVarNames = { awaitOrEmpty: "", - bench: "fn", + bench: "bench", context: "context", timer: "timer", managed: false, }; - const benchmarkResult: BenchNode.OnCompleteBenchmarkResult = [0, 0, {}]; expectType(plugin2.afterClockTemplate(varNames)); - expectType(plugin2.onCompleteBenchmark(benchmarkResult)); + expectAssignable( + plugin2.onCompleteBenchmark, + ); expectType(plugin2.toString()); } @@ -168,7 +169,7 @@ if (plugin3.isSupported?.()) { expectType(plugin3.isSupported()); const varNames: BenchNode.PluginHookVarNames = { awaitOrEmpty: "", - bench: "fn", + bench: "bench", context: "context", timer: "timer", managed: false, @@ -183,7 +184,7 @@ if (plugin4.isSupported?.()) { expectType(plugin3.isSupported()); const varNames: BenchNode.PluginHookVarNames = { awaitOrEmpty: "", - bench: "fn", + bench: "bench", context: "context", timer: "timer", managed: false,