import { spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { classifyBoundedUnsignedDecimal } from "./arg-utils.mts"; type BenchmarkCliOptions = { help: boolean; json: boolean; output?: string; }; export type BenchmarkTimingSummary = { count: number; min: number; p50: number; max: number; p95?: number; p99?: number; }; export type BenchmarkWorkerProcessResult = { status: number | null; stdout: string; stderr: string; error?: Error & { code?: string }; }; export type BenchmarkWorkerSpawner = ( command: string, args: string[], options: { cwd: string; encoding: "utf8"; env: NodeJS.ProcessEnv; timeout: number; killSignal: NodeJS.Signals; maxBuffer: number; }, ) => BenchmarkWorkerProcessResult; export function parseBenchmarkInteger(raw: string, flag: string, min: number, max: number) { const result = classifyBoundedUnsignedDecimal(raw, min, max); if (result.kind === "syntax") { throw new Error(`${flag} must be an integer`); } if (result.kind === "below") { throw new Error(`${flag} must be at least ${min}`); } if (result.kind === "above") { throw new Error(`${flag} must be at most ${max}`); } return result.value; } export function parseBenchmarkIntegerList(raw: string, flag: string, max: number) { if (!raw || raw.split(",").some((value) => value.length === 0)) { throw new Error(`${flag} requires a comma-separated integer list`); } const values = raw.split(",").map((value) => parseBenchmarkInteger(value, flag, 1, max)); if (new Set(values).size !== values.length) { throw new Error(`${flag} contains duplicate values`); } return values; } export function parseBenchmarkOptions( argv: string[], defaults: T, valueFlags: Record void>, ) { const options = { ...defaults }; const seen = new Set(); for (let index = 0; index < argv.length; index += 1) { const flag = argv[index]; if (!flag?.startsWith("--")) { throw new Error(`Unknown argument: ${flag}`); } if (seen.has(flag)) { throw new Error(`${flag} was provided more than once`); } seen.add(flag); if (flag === "--json" || flag === "--help") { options[flag === "--json" ? "json" : "help"] = true; continue; } const applyValue = valueFlags[flag]; if (!applyValue) { throw new Error(`Unknown argument: ${flag}`); } const value = argv[index + 1]; if (!value || value.startsWith("--")) { throw new Error(`${flag} requires a value`); } index += 1; applyValue(options, value); } return options; } function percentile(sorted: number[], ratio: number) { return sorted[Math.max(0, Math.ceil(sorted.length * ratio) - 1)] ?? 0; } export function summarizeBenchmarkTimings(values: number[]): BenchmarkTimingSummary { if (values.length === 0) { throw new Error("cannot summarize an empty timing set"); } const sorted = values.toSorted((left, right) => left - right); const summary: BenchmarkTimingSummary = { count: sorted.length, min: sorted[0] ?? 0, p50: percentile(sorted, 0.5), max: sorted.at(-1) ?? 0, }; if (sorted.length >= 20) { summary.p95 = percentile(sorted, 0.95); summary.p99 = percentile(sorted, 0.99); } return summary; } export function parseBenchmarkWorkerResult(params: { label: string; result: BenchmarkWorkerProcessResult; sentinel: string; timeoutMs: number; validate(value: unknown): T; }) { if (params.result.error) { const detail = params.result.error.code === "ETIMEDOUT" ? `timed out after ${params.timeoutMs}ms` : params.result.error.message; throw new Error(`worker ${params.label} failed: ${detail}`); } if (params.result.status !== 0) { throw new Error( `worker ${params.label} failed (${params.result.status ?? "signal"}): ${params.result.stderr.trim() || params.result.stdout.trim()}`, ); } const payloads = params.result.stdout .split(/\r?\n/u) .filter((line) => line.startsWith(params.sentinel)); if (payloads.length !== 1) { throw new Error(`worker ${params.label} returned ${payloads.length} result payloads`); } let parsed: unknown; try { parsed = JSON.parse(payloads[0]!.slice(params.sentinel.length)); } catch { throw new Error(`worker ${params.label} returned invalid JSON`); } return params.validate(parsed); } export function runBenchmarkWorker(params: { args: string[]; label: string; sentinel: string; spawnWorker?: BenchmarkWorkerSpawner; timeoutMs: number; validate(value: unknown): T; }) { const spawnWorker: BenchmarkWorkerSpawner = params.spawnWorker ?? spawnSync; const result = spawnWorker(process.execPath, params.args, { cwd: process.cwd(), encoding: "utf8", env: { ...process.env, NODE_NO_WARNINGS: "1" }, timeout: params.timeoutMs, killSignal: "SIGTERM", maxBuffer: 16 * 1024 * 1024, }); return parseBenchmarkWorkerResult({ label: params.label, result, sentinel: params.sentinel, timeoutMs: params.timeoutMs, validate: (value) => params.validate(value), }); } export function runBenchmarkJobs( jobs: TJob[], options: { describe(job: TJob): string; now?: () => number; prefix: string; run(job: TJob): TResult; writeProgress?: (line: string) => void; }, ) { const now = options.now ?? Date.now; const writeProgress = options.writeProgress ?? ((line: string) => process.stderr.write(`${line}\n`)); return jobs.map((job, index) => { const ordinal = index + 1; const description = options.describe(job); writeProgress(`[${options.prefix}] worker ${ordinal}/${jobs.length} start ${description}`); const startedAt = now(); try { const result = options.run(job); const elapsedMs = Math.max(0, now() - startedAt); writeProgress( `[${options.prefix}] worker ${ordinal}/${jobs.length} complete ${description} elapsed=${(elapsedMs / 1_000).toFixed(3)}s`, ); return result; } catch (error) { const elapsedMs = Math.max(0, now() - startedAt); writeProgress( `[${options.prefix}] worker ${ordinal}/${jobs.length} failed ${description} elapsed=${(elapsedMs / 1_000).toFixed(3)}s`, ); throw error; } }); } export function emitBenchmarkReport( report: T, options: BenchmarkCliOptions, renderLines: (report: T) => string[], ) { const json = `${JSON.stringify(report, null, 2)}\n`; if (options.output) { fs.mkdirSync(path.dirname(path.resolve(options.output)), { recursive: true }); fs.writeFileSync(options.output, json); } if (options.json) { process.stdout.write(json); return; } const lines = renderLines(report); if (lines.length > 0) { process.stdout.write(`${lines.join("\n")}\n`); } } export async function runBenchmarkEntrypoint(name: string, run: () => void | Promise) { try { await run(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; } finally { if (process.exitCode && process.exitCode !== 0) { console.error(`[${name}] FAILED (exit ${process.exitCode})`); } } }