From 29b4d0597ea4fb45cac82b8ffe83dd18266604db Mon Sep 17 00:00:00 2001 From: Jono Prest Date: Mon, 9 Nov 2026 14:22:58 +0000 Subject: [PATCH 1/2] Count the build's allocations on generated projects in the benchmarks Why: The benchmarks compile runtime and Belt files one at a time, whose dependency graphs are small, and build tests/tests, whose modules use a few small modules. Costs that grow with how many modules a build has or loads barely show: a change that avoids copying the signatures of large dependencies that a module hardly reads moved them by 0.4%, and opening the map of a namespaced package, which is quadratic in its modules per build, doesn't run at all. With `--project`, `measure` now also builds two projects that the script generates, clean and after an edit, on one worker domain and counting allocated words like tests/tests: - WideDeps: 240 modules that each use one or two values of four modules with 600 types and 1,200 values each. The edit adds a value to one of them, which compiles its users again. - Namespace: a package with `"namespace": true` and 1,000 one-line modules. The edit changes a module that nothing uses. Their counts are larger than all the others together, so the report adds them up on a line of their own, `Generated projects`, and `In total` covers the other benchmarks as before. Each has its row and threshold like any other benchmark, and its inputs are a hash of the project's own files. Locally, the signature copy pull requests take WideDeps clean from 505M to 333M words and its edit from 232M to 146M, and building the namespace map on demand takes Namespace clean from 611M to 203M; the other benchmarks' counts are unchanged. The builds of tests/tests now share countBuild and editModule with the generated projects. Signed-off-by: Jono Prest --- .../generated-project-benchmarks.internal.md | 1 + scripts/compilerBenchmarks.js | 244 ++++++++++++++++-- scripts/compilerBenchmarks.test.js | 57 ++++ 3 files changed, 277 insertions(+), 25 deletions(-) create mode 100644 changes/generated-project-benchmarks.internal.md diff --git a/changes/generated-project-benchmarks.internal.md b/changes/generated-project-benchmarks.internal.md new file mode 100644 index 0000000000..b7cee3cb79 --- /dev/null +++ b/changes/generated-project-benchmarks.internal.md @@ -0,0 +1 @@ +- Count the words the build allocates on generated projects in the CI benchmarks: many modules using little of a few large dependencies, and a namespaced package of 1,000 modules. diff --git a/scripts/compilerBenchmarks.js b/scripts/compilerBenchmarks.js index eb94d1d618..c566f0c56a 100644 --- a/scripts/compilerBenchmarks.js +++ b/scripts/compilerBenchmarks.js @@ -41,7 +41,9 @@ // of this script, so both measurements use the same ones. With `--project`, // it also builds tests/tests of the given checkout with the build system, // clean, with nothing to do and after two edits (see measureProjectBuild), -// counting allocated words only. +// and a few generated projects, clean and after an edit, for costs that grow +// with the number of modules of a build (see generatedProjects), counting +// allocated words only. // With `--playground`, it measures the size of the playground's compiler. // // `compare` reports the changes between two measurements as Markdown, @@ -329,6 +331,94 @@ export const stressInputs = { }, }; +/** + * A generated project: its files by path relative to its root, rescript.json + * among them, and the source file that its edit changes. + * + * @typedef {{ files: Record, edit: string }} GeneratedProject + */ + +/** + * The rescript.json of a generated project, with JavaScript written to lib/ + * so that removing lib/ makes the next build a clean one. + * + * @param {string} name + * @param {Record} [extra] more settings + */ +function projectConfig(name, extra = {}) { + const config = { + name, + sources: [{ dir: "src" }], + "package-specs": { module: "esmodule", "in-source": false }, + suffix: ".mjs", + ...extra, + }; + return `${JSON.stringify(config, null, 2)}\n`; +} + +/** + * Generated projects for the build benchmarks, one for each cost of a build + * that grows with the number of modules rather than with their size, which + * compiling one file at a time doesn't show. Their modules are small, so + * each builds in a few seconds at most. + * + * @type {Record GeneratedProject>} + */ +export const generatedProjects = { + // Many modules that each use one or two values of a few large + // dependencies: the cost of loading a dependency's interface, which the + // type checker copies when it reads it, against the little of it that is + // used. + WideDeps: () => { + const dependencies = 4; + const values = 600; + const users = 240; + /** @type {Record} */ + const files = { "rescript.json": projectConfig("bench-wide-deps") }; + for (let d = 0; d < dependencies; d++) { + const lines = []; + for (let v = 0; v < values; v++) { + lines.push(`type t${v} = {a${v}: int, b${v}: string}`); + lines.push( + `let make${v} = (x: int): t${v} => {a${v}: x, b${v}: "${v}"}`, + ); + lines.push(`let get${v} = (r: t${v}) => r.a${v} + ${v}`); + } + files[`src/Big${d}.res`] = `${lines.join("\n")}\n`; + } + for (let u = 0; u < users; u++) { + const first = u % dependencies; + const second = (u * 7 + 1) % dependencies; + const v = (u * 37) % values; + files[`src/User${u}.res`] = [ + `let a = Big${first}.get${v}(Big${first}.make${v}(${u}))`, + `let b = Big${second}.make${(v + 1) % values}(a).b${(v + 1) % values}`, + "", + ].join("\n"); + } + // A new value in a dependency changes its interface, so every module + // that uses it is compiled again. + return { files, edit: "src/Big0.res" }; + }, + // A namespaced package of trivial modules. Each module opens the + // namespace's map of module aliases, one for every module of the package, + // so a cost per alias of opening it is quadratic in the package's size. + Namespace: () => { + const modules = 1000; + /** @type {Record} */ + const files = { + "rescript.json": projectConfig("bench-namespace", { namespace: true }), + }; + for (let i = 0; i < modules; i++) { + // Each module but the first uses another one, through the map. + const use = i === 0 ? "0" : `Mod${Math.floor((i - 1) / 2)}.value`; + files[`src/Mod${i}.res`] = `let value = ${use} + ${i}\n`; + } + // The last module, which nothing uses. + return { files, edit: `src/Mod${modules - 1}.res` }; + }, +}; + /** * The runtime and Belt sources of a checkout, by package. * @@ -519,17 +609,7 @@ function measureProjectBuild(checkout, projectCheckout, results) { throw new Error(`The modules to edit are missing from ${projectDir}.`); } - const build = () => ({ - ...measureRun(rescript, ["build"], workDir, false, { - env: { - REWATCH_COMPILER_DOMAINS: "1", - REWATCH_REPORT_WORKER_ALLOCATIONS: "1", - }, - }), - inputs, - }); - /** @param {string} file */ - const edit = file => fs.appendFileSync(file, "\nlet benchmarkEdit = 1\n"); + const build = () => countBuild(rescript, workDir, inputs); results["build/clean"] = build(); const outputs = outputSizes(workDir); @@ -537,15 +617,98 @@ function measureProjectBuild(checkout, projectCheckout, results) { results[`output/tests/*${ext}`] = { bytes: outputs[ext], inputs }; } results["build/no-op"] = build(); - edit(leafModule); + editModule(leafModule); results["build/leaf-edit"] = build(); - edit(path.join(workDir, sharedModule)); + editModule(path.join(workDir, sharedModule)); results["build/shared-edit"] = build(); } finally { fs.rmSync(workDir, { recursive: true, force: true }); } } +/** + * Builds a project with a build system on one worker domain, so that the + * counts don't depend on how the work is shared, and counts the words it + * and its workers allocated. + * + * @param {string} rescript + * @param {string} dir + * @param {string} inputs + * @returns {Measurement} + */ +function countBuild(rescript, dir, inputs) { + return { + ...measureRun(rescript, ["build"], dir, false, { + env: { + REWATCH_COMPILER_DOMAINS: "1", + REWATCH_REPORT_WORKER_ALLOCATIONS: "1", + }, + }), + inputs, + }; +} + +/** + * Adds a value to a module, which changes its interface when it has no + * interface file. + * + * @param {string} file + */ +function editModule(file) { + fs.appendFileSync(file, "\nlet benchmarkEdit = 1\n"); +} + +/** + * Writes a generated project to an empty directory, with the runtime of + * `checkout` as its only dependency. Returns a hash of its files. + * + * @param {GeneratedProject} project + * @param {string} dir + * @param {string} checkout + */ +function writeProject(project, dir, checkout) { + const hash = createHash("sha256"); + for (const [file, contents] of Object.entries(project.files)) { + fs.mkdirSync(path.join(dir, path.dirname(file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), contents); + hash.update(file).update("\0").update(contents).update("\0"); + } + const scope = path.join(dir, "node_modules", "@rescript"); + fs.mkdirSync(scope, { recursive: true }); + fs.symlinkSync( + path.join(checkout, "packages", "@rescript", "runtime"), + path.join(scope, "runtime"), + "junction", + ); + return hash.digest("hex").slice(0, 16); +} + +/** + * Builds each generated project with the build system of `checkout`, clean + * and after its edit, counting allocated words like measureProjectBuild. + * The projects come from the checkout of this script, and are built in a + * directory with the same name in every measurement. + * + * @param {string} checkout + * @param {Results} results + */ +function measureGeneratedProjects(checkout, results) { + const rescript = path.join(checkout, relativeBinDir, "rescript.exe"); + const workDir = path.join(os.tmpdir(), "compiler-bench-generated"); + for (const [name, generate] of Object.entries(generatedProjects)) { + fs.rmSync(workDir, { recursive: true, force: true }); + try { + const project = generate(); + const inputs = writeProject(project, workDir, checkout); + results[`build/${name}/clean`] = countBuild(rescript, workDir, inputs); + editModule(path.join(workDir, project.edit)); + results[`build/${name}/edit`] = countBuild(rescript, workDir, inputs); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + } +} + /** * @param {string} checkout * @param {{ instructions: boolean, project?: string, playground?: boolean }} options @@ -666,6 +829,7 @@ export function measure(checkout, { instructions, project, playground }) { if (project) { measureProjectBuild(checkout, project, results); + measureGeneratedProjects(checkout, results); } return results; @@ -681,10 +845,23 @@ export function measure(checkout, { instructions, project, playground }) { * }} Change */ +/** + * Whether a benchmark builds one of the generated projects. Their counts + * are larger than all the others together, so they are added up apart. + * + * @param {string} name + */ +const isGeneratedProject = name => + Object.keys(generatedProjects).some(project => + name.startsWith(`build/${project}/`), + ); + /** * Lists the counts of one metric that differ between two measurements, * largest relative change first, and the benchmarks that only one of them - * has. Benchmarks without the metric on either side are left out. + * has. Benchmarks without the metric on either side are left out. The + * total leaves out the generated projects, which have a subtotal of their + * own, `generated`, when there are any. * * @param {Results} base * @param {Results} head @@ -715,9 +892,14 @@ export function compareResults(base, head, threshold, metric = "allocated") { } changes.sort((x, y) => Math.abs(y.percent) - Math.abs(x.percent)); - /** @param {Results} results */ - const total = results => - common.reduce((sum, name) => sum + (results[name][metric] ?? 0), 0); + /** + * @param {Results} results + * @param {string[]} names + */ + const total = (results, names) => + names.reduce((sum, name) => sum + (results[name][metric] ?? 0), 0); + const counted = common.filter(name => !isGeneratedProject(name)); + const generated = common.filter(isGeneratedProject); return { metric, @@ -729,7 +911,11 @@ export function compareResults(base, head, threshold, metric = "allocated") { added: [...headNames].filter(name => !baseNames.has(name)), removed: [...baseNames].filter(name => !headNames.has(name)), compared: common.length, - total: { base: total(base), head: total(head) }, + total: { base: total(base, counted), head: total(head, counted) }, + generated: + generated.length > 0 + ? { base: total(base, generated), head: total(head, generated) } + : undefined, }; } @@ -810,19 +996,27 @@ function renderSection(comparison, threshold, visibleRows) { removed, compared, total, + generated, } = comparison; const words = metricWords[metric]; - const totalPercent = - total.base === 0 ? 0 : ((total.head - total.base) / total.base) * 100; - // A total that rounds to 0.00% would read as "+0.00%" or "-0.00%". - const totalChange = - Math.abs(totalPercent) < 0.005 ? "±0.00%" : formatPercent(totalPercent); + /** @param {{ base: number, head: number }} sum */ + const sumChange = sum => { + const percent = + sum.base === 0 ? 0 : ((sum.head - sum.base) / sum.base) * 100; + // A total that rounds to 0.00% would read as "+0.00%" or "-0.00%". + const change = + Math.abs(percent) < 0.005 ? "±0.00%" : formatPercent(percent); + return `${formatCount(sum.base)} → ${formatCount(sum.head)} (${change})`; + }; const lines = [ `### ${words.title}`, "", - `In total: ${formatCount(total.base)} → ${formatCount(total.head)} (${totalChange}).`, + `In total: ${sumChange(total)}.`, "", ]; + if (generated) { + lines.push(`Generated projects: ${sumChange(generated)}.`, ""); + } if (regressions.length > 0) { lines.push( `**⚠️ Regression: ${words.more(regressions.length, threshold)}.**`, diff --git a/scripts/compilerBenchmarks.test.js b/scripts/compilerBenchmarks.test.js index 1e9325455c..e64de4a51e 100644 --- a/scripts/compilerBenchmarks.test.js +++ b/scripts/compilerBenchmarks.test.js @@ -3,6 +3,7 @@ import test from "node:test"; import { compareAll, compareResults, + generatedProjects, parseAllocatedWords, parseInstructions, renderSummary, @@ -348,3 +349,59 @@ test("generates the same stress inputs every time", () => { assert.equal(generate(), generate()); } }); + +test("adds up the generated projects apart from the total", () => { + const base = { + a: { allocated: 1000, inputs: "x" }, + "build/clean": { allocated: 2000, inputs: "x" }, + "build/Namespace/clean": { allocated: 100000, inputs: "x" }, + "build/WideDeps/edit": { allocated: 50000, inputs: "x" }, + }; + const head = { + a: { allocated: 1000, inputs: "x" }, + "build/clean": { allocated: 2000, inputs: "x" }, + "build/Namespace/clean": { allocated: 40000, inputs: "x" }, + "build/WideDeps/edit": { allocated: 50000, inputs: "x" }, + }; + const comparison = compareResults(base, head, 1); + assert.deepEqual(comparison.total, { base: 3000, head: 3000 }); + assert.deepEqual(comparison.generated, { base: 150000, head: 90000 }); + // The generated projects still have rows and thresholds of their own. + assert.deepEqual( + comparison.improvements.map(c => c.name), + ["build/Namespace/clean"], + ); + const summary = renderSummary([comparison], 1); + assert.match(summary, /^In total: 3,000 → 3,000 \(±0\.00%\)\.$/m); + assert.match( + summary, + /^Generated projects: 150,000 → 90,000 \(-40\.00%\)\.$/m, + ); + // Without generated projects, there is no subtotal. + assert.doesNotMatch( + renderSummary([compareResults({ a: base.a }, { a: head.a }, 1)], 1), + /Generated projects/, + ); +}); + +test("generates the same projects every time", () => { + for (const generate of Object.values(generatedProjects)) { + const project = generate(); + assert.deepEqual(generate(), project); + // The edit changes one of the project's sources. + assert.ok(project.edit.endsWith(".res")); + assert.ok(project.edit in project.files); + const config = JSON.parse(project.files["rescript.json"]); + // JavaScript goes to lib/, which a clean build removes. + assert.equal(config["package-specs"]["in-source"], false); + } +}); + +test("namespaces only the namespace benchmark's project", () => { + const { files } = generatedProjects.Namespace(); + assert.equal(JSON.parse(files["rescript.json"]).namespace, true); + // Modules use others through the namespace's map. + assert.match(files["src/Mod10.res"], /^let value = Mod4\.value \+ 10$/m); + const wide = generatedProjects.WideDeps(); + assert.equal(JSON.parse(wide.files["rescript.json"]).namespace, undefined); +}); From 6cae0e534ec75665b7ac26f11009da30a7bb9b50 Mon Sep 17 00:00:00 2001 From: Jono Prest Date: Mon, 9 Nov 2026 14:27:03 +0000 Subject: [PATCH 2/2] Time a chain of modules with the base's and the head's build systems Why: A change to the build's scheduling, such as starting a module once the interfaces it reads are final, does the same work sooner. Counting allocated words and instructions can only show what such a change costs, never what it saves. `time ` builds a generated project of 150 modules in a chain, each using the one before it, and a module beside each, clean with the two build systems in turn on the default number of worker domains: one build each that isn't counted, then 10 rounds, the order alternating each round, each build stopped after 2 minutes. It writes the times to the JSON file and prints the base's and the head's median times and the median of the head's time over the base's in the same round, with a 95% confidence interval from a percentile bootstrap of the rounds with a fixed seed. The command is run by hand, so that a pull request that changes the scheduling can quote its numbers; CI doesn't run it. Locally, with load around 5 to 9 on 8 cores, the ratio was 0.695 (0.646 to 0.825) for the pull request that starts dependents once a module's interface is final, 0.991 (0.948 to 1.055) for one that only changes namespaced packages, and 0.965 (0.921 to 1.078) with the same build system on both sides. The command took about 45 seconds. Signed-off-by: Jono Prest --- .../generated-project-benchmarks.internal.md | 1 + scripts/compilerBenchmarks.js | 249 +++++++++++++++++- scripts/compilerBenchmarks.test.js | 61 ++++- 3 files changed, 309 insertions(+), 2 deletions(-) diff --git a/changes/generated-project-benchmarks.internal.md b/changes/generated-project-benchmarks.internal.md index b7cee3cb79..31aad5b456 100644 --- a/changes/generated-project-benchmarks.internal.md +++ b/changes/generated-project-benchmarks.internal.md @@ -1 +1,2 @@ - Count the words the build allocates on generated projects in the CI benchmarks: many modules using little of a few large dependencies, and a namespaced package of 1,000 modules. +- Add a `time` command to the compiler benchmarks script for local use: it times clean builds of a generated chain of modules with two checkouts' build systems in turn and prints the median ratio with a 95% confidence interval. diff --git a/scripts/compilerBenchmarks.js b/scripts/compilerBenchmarks.js index c566f0c56a..fe0edf6010 100644 --- a/scripts/compilerBenchmarks.js +++ b/scripts/compilerBenchmarks.js @@ -26,6 +26,8 @@ // node scripts/compilerBenchmarks.js prepare // node scripts/compilerBenchmarks.js measure [--instructions] // [--project ] [--playground] +// node scripts/compilerBenchmarks.js time +// [--rounds ] // node scripts/compilerBenchmarks.js compare // [--threshold ] [--summary ] // @@ -46,6 +48,13 @@ // allocated words only. // With `--playground`, it measures the size of the playground's compiler. // +// `time` builds a generated project with the build systems of two prepared +// checkouts in turn, writes the wall-clock times to and prints +// their medians and the median ratio of the head's time to the base's with +// a 95% confidence interval (see timeBuilds). It is the one benchmark that +// measures time, run by hand to indicate the effect of a change that does +// the same work sooner; CI doesn't run it. +// // `compare` reports the changes between two measurements as Markdown, // improvements and regressions alike, one section per count, and exits with // 1 when a count grew by more than the threshold. Runtime and Belt files, @@ -419,6 +428,50 @@ export const generatedProjects = { }, }; +/** + * The project the wall-time benchmark builds: a long chain of modules, each + * using the one before it, and a module beside each that uses it. A build + * spends most of its time on the chain with most of its worker domains + * idle, which is what a change to the build's scheduling would improve. + * + * @returns {GeneratedProject} + */ +export function chainProject() { + const length = 150; + /** @type {Record} */ + const files = { "rescript.json": projectConfig("bench-chain") }; + for (let i = 0; i < length; i++) { + const lines = + i === 0 + ? [ + "type t = {depth: int, items: array}", + "let base = {depth: 0, items: []}", + ] + : [ + `type t = {depth: int, items: array, prev: Chain${i - 1}.t}`, + `let base = {depth: ${i}, items: [${i}], prev: Chain${i - 1}.base}`, + ]; + // Bodies with some work for the type checker and code generation. + for (let f = 0; f < 20; f++) { + lines.push( + `let f${f} = (r: t, x: int) =>`, + " switch mod(x, 4) {", + ` | 0 => r.items->Array.map(y => y + x + ${f})->Array.length`, + ` | 1 => r.depth * ${f + 1} + x`, + ` | 2 => r.items->Array.reduce(0, (acc, y) => acc + y * ${f})`, + " | _ => -1", + " }", + ); + } + files[`src/Chain${i}.res`] = `${lines.join("\n")}\n`; + files[`src/Side${i}.res`] = [ + `let total = Chain${i}.f0(Chain${i}.base, ${i}) + Chain${i}.f1(Chain${i}.base, 1)`, + "", + ].join("\n"); + } + return { files, edit: `src/Chain${length - 1}.res` }; +} + /** * The runtime and Belt sources of a checkout, by package. * @@ -835,6 +888,175 @@ export function measure(checkout, { instructions, project, playground }) { return results; } +/** + * How many times the wall-time benchmark builds its project with each + * compiler, after one build with each that isn't counted. + */ +const timingRounds = 10; + +/** + * How long one of the wall-time benchmark's builds may take, in + * milliseconds, many times as long as a build takes, so that a build that + * hangs, as it could with a deadlock in the scheduling, ends the benchmark. + */ +const timingBuildTimeout = 2 * 60 * 1000; + +/** + * The wall-clock times of the wall-time benchmark's builds, in + * milliseconds, by benchmark: round i built with the base's compiler and + * the head's, one right after the other. + * + * @typedef {Record} Timings + */ + +/** + * Builds the wall-time benchmark's project clean with the build systems of + * two checkouts in turn, on as many worker domains as the build system + * chooses, and returns how long each build took. The two compilers take + * turns going first, so that a machine that gets slower or faster during + * the run changes both alike. + * + * Wall-clock time on a loaded machine varies by several percent from one + * build to the next, and more over minutes, so this only indicates the + * effect of a change that counting doesn't show, such as starting a module + * earlier: building the same modules with idle domains doing less is + * faster, but allocates and executes as much or more. + * + * Each checkout builds a copy of the project of its own, with its own + * runtime, both under paths of the same length. + * + * @param {string} baseCheckout + * @param {string} headCheckout + * @param {number} rounds + * @returns {Timings} + */ +export function timeBuilds(baseCheckout, headCheckout, rounds) { + const workDir = path.join(os.tmpdir(), "compiler-bench-time"); + fs.rmSync(workDir, { recursive: true, force: true }); + try { + const project = chainProject(); + const sides = [ + { name: "base", checkout: baseCheckout }, + { name: "head", checkout: headCheckout }, + ].map(side => { + const dir = path.join(workDir, side.name); + writeProject(project, dir, side.checkout); + return { + ...side, + dir, + rescript: path.join(side.checkout, relativeBinDir, "rescript.exe"), + /** @type {number[]} */ + times: [], + }; + }); + /** @param {(typeof sides)[number]} side */ + const build = side => { + fs.rmSync(path.join(side.dir, "lib"), { recursive: true, force: true }); + const start = process.hrtime.bigint(); + execFileSync(side.rescript, ["build"], { + cwd: side.dir, + stdio: "pipe", + timeout: timingBuildTimeout, + killSignal: "SIGKILL", + }); + return Number(process.hrtime.bigint() - start) / 1e6; + }; + for (const side of sides) build(side); + for (let round = 0; round < rounds; round++) { + const order = round % 2 === 0 ? sides : [...sides].reverse(); + for (const side of order) side.times.push(build(side)); + } + return { + "time/Chain/clean": { base: sides[0].times, head: sides[1].times }, + }; + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } +} + +/** + * The median of numbers. + * + * @param {number[]} values + */ +export function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +} + +/** + * A pseudo-random number generator (mulberry32), so that the confidence + * interval of the same times is the same in every run. + * + * @param {number} seed + * @returns {() => number} numbers in [0, 1) + */ +function randomNumbers(seed) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * The median of the head's time divided by the base's time in the same + * round, with a 95% confidence interval from a percentile bootstrap: the + * medians of `resamples` samples of the rounds drawn with replacement, of + * which 2.5% are below `low` and 2.5% above `high`. Pairing the rounds + * leaves out a drift that slows both builds of a round alike. + * + * @param {number[]} base + * @param {number[]} head + * @param {number} [resamples] + * @returns {{ ratio: number, low: number, high: number, rounds: number }} + */ +export function timeRatio(base, head, resamples = 2000) { + if (base.length !== head.length || base.length === 0) { + throw new Error("The base and the head need the same rounds."); + } + const ratios = base.map((time, i) => head[i] / time); + const random = randomNumbers(1); + const medians = Array.from({ length: resamples }, () => + median(ratios.map(() => ratios[Math.floor(random() * ratios.length)])), + ).sort((a, b) => a - b); + /** @param {number} q */ + const quantile = q => + medians[Math.min(resamples - 1, Math.floor(q * resamples))]; + return { + ratio: median(ratios), + low: quantile(0.025), + high: quantile(0.975), + rounds: ratios.length, + }; +} + +/** + * Summarizes the wall-time benchmarks as lines of text: for each, the + * base's and the head's median times, and the median of each round's head + * time over its base time, which isn't the ratio of the two medians, with + * its 95% confidence interval. + * + * @param {Timings} timings + * @returns {string[]} + */ +export function renderTimings(timings) { + return Object.entries(timings).map(([name, { base, head }]) => { + const { ratio, low, high, rounds } = timeRatio(base, head); + return [ + `${name} (${rounds} ${rounds === 1 ? "round" : "rounds"}):`, + `base ${median(base).toFixed(0)} ms, head ${median(head).toFixed(0)} ms (medians),`, + `head / base ${ratio.toFixed(3)} (95% CI ${low.toFixed(3)} to ${high.toFixed(3)})`, + ].join(" "); + }); +} + /** * @typedef {{ * name: string, @@ -1105,7 +1327,11 @@ export function compareAll(base, head, threshold) { ); } -/** @param {string[]} args */ +/** + * Runs a command and returns its exit status. + * + * @param {string[]} args + */ function main(args) { const [command, ...rest] = args; if (command === "prepare" && rest.length === 1) { @@ -1133,6 +1359,25 @@ function main(args) { console.log(`Measured ${Object.keys(results).length} benchmarks.`); return 0; } + if (command === "time" && rest.length >= 3) { + const [baseCheckout, headCheckout, out, ...options] = rest; + let rounds = timingRounds; + for (let i = 0; i < options.length; i += 2) { + if (options[i] === "--rounds") rounds = Number(options[i + 1]); + else throw new Error(`Unknown option ${options[i]}`); + } + if (!Number.isInteger(rounds) || rounds < 1) { + throw new Error("--rounds takes a positive integer."); + } + const timings = timeBuilds( + path.resolve(baseCheckout), + path.resolve(headCheckout), + rounds, + ); + fs.writeFileSync(out, `${JSON.stringify(timings, null, 2)}\n`); + for (const line of renderTimings(timings)) console.log(line); + return 0; + } if (command === "compare" && rest.length >= 2) { const [basePath, headPath, ...options] = rest; let threshold = 1; @@ -1154,6 +1399,8 @@ function main(args) { "Usage: compilerBenchmarks.js prepare \n" + " compilerBenchmarks.js measure [--instructions]\n" + " [--project ] [--playground]\n" + + " compilerBenchmarks.js time \n" + + " [--rounds ]\n" + " compilerBenchmarks.js compare [--threshold ]\n" + " [--summary ]", ); diff --git a/scripts/compilerBenchmarks.test.js b/scripts/compilerBenchmarks.test.js index e64de4a51e..9cf0aa4e43 100644 --- a/scripts/compilerBenchmarks.test.js +++ b/scripts/compilerBenchmarks.test.js @@ -1,13 +1,17 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + chainProject, compareAll, compareResults, generatedProjects, + median, parseAllocatedWords, parseInstructions, renderSummary, + renderTimings, stressInputs, + timeRatio, } from "./compilerBenchmarks.js"; const stats = `allocated_words: 628461 @@ -385,7 +389,7 @@ test("adds up the generated projects apart from the total", () => { }); test("generates the same projects every time", () => { - for (const generate of Object.values(generatedProjects)) { + for (const generate of [...Object.values(generatedProjects), chainProject]) { const project = generate(); assert.deepEqual(generate(), project); // The edit changes one of the project's sources. @@ -405,3 +409,58 @@ test("namespaces only the namespace benchmark's project", () => { const wide = generatedProjects.WideDeps(); assert.equal(JSON.parse(wide.files["rescript.json"]).namespace, undefined); }); + +test("chains the modules of the wall-time benchmark's project", () => { + const { files } = chainProject(); + assert.match(files["src/Chain1.res"], /prev: Chain0\.t/); + assert.match(files["src/Chain149.res"], /prev: Chain148\.t/); + assert.equal(files["src/Chain150.res"], undefined); + assert.match(files["src/Side7.res"], /Chain7\.f0/); +}); + +test("takes the median of an odd and an even number of values", () => { + assert.equal(median([3, 1, 2]), 2); + assert.equal(median([4, 1, 3, 2]), 2.5); + // The values are left as they were. + const values = [2, 1]; + median(values); + assert.deepEqual(values, [2, 1]); +}); + +test("pairs the rounds of the times to compare", () => { + // The machine gets slower during the run, but the head is always 10% + // faster in the same round. + const base = [100, 120, 140, 160, 180, 200, 220, 240, 260, 280]; + const head = base.map(time => time * 0.9); + const { ratio, low, high, rounds } = timeRatio(base, head); + assert.equal(rounds, 10); + assert.ok(Math.abs(ratio - 0.9) < 1e-9); + assert.ok(Math.abs(low - 0.9) < 1e-9); + assert.ok(Math.abs(high - 0.9) < 1e-9); +}); + +test("gives a confidence interval around the median ratio", () => { + const base = [100, 100, 100, 100, 100, 100, 100, 100, 100, 100]; + const head = [85, 95, 90, 88, 92, 99, 80, 91, 89, 93]; + const result = timeRatio(base, head); + assert.equal(result.ratio, 0.905); + assert.ok(result.low <= result.ratio && result.ratio <= result.high); + assert.ok(result.low >= 0.8 && result.high <= 0.99); + // The resamples are drawn the same way every time. + assert.deepEqual(timeRatio(base, head), result); + assert.throws(() => timeRatio([1, 2], [1])); + assert.throws(() => timeRatio([], [])); +}); + +test("summarizes the wall times", () => { + assert.deepEqual( + renderTimings({ + "time/Chain/clean": { base: [100, 100, 100], head: [150, 160, 155] }, + one: { base: [100], head: [90] }, + }), + [ + "time/Chain/clean (3 rounds): base 100 ms, head 155 ms (medians), head / base 1.550 (95% CI 1.500 to 1.600)", + "one (1 round): base 100 ms, head 90 ms (medians), head / base 0.900 (95% CI 0.900 to 0.900)", + ], + ); +});