nessar/vac-optimizer
1
0 Fork 0
Code Issues 3 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

docs(benchmark): compare engines by hand before each frozen-engine rebuild

Browse Source 
A standing "no worse than the previous engine" check is not added to
CI: the differential check already fails on unintended changes, and a
fixed acceptance threshold would need renegotiating with every
deliberate change. Instead, the rebuild procedure now runs the
comparison while the frozen engine is still the previous one, and
records it with the change. The runner compares against the frozen
engine by default and takes `calibration` to reproduce the #302 record.

Part of #302.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
nessarandClaude Opus 5.5 committed 2026-10-18 22:12:50 +02:00
1 parent fe705fe618
commit 75320da685
3 files changed
+36 -14
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/app/services/frozen-engine/compare-engines.ts
+16 -7
View File
@@ -1,6 +1,7 @@
// Opt-in before/after record, outside the functional suite and CI: compares the
// current engine with the calibration engine (a871f8f) over seeded requests.
// bun src/app/services/frozen-engine/compare-engines.ts [small|loaded-year] [first seed] [count]
// current engine with the frozen engine (before its rebuild) or the calibration
// engine (a871f8f) over seeded requests.
// bun src/app/services/frozen-engine/compare-engines.ts [small|loaded-year] [first seed] [count] [frozen|calibration]
import { isDeepStrictEqual } from 'node:util';
import type { PlanningArtifacts, PlanningObjective, PlanningOutcome } from '../leave-planner.core';
import { compareObjectiveValues } from '../planning-objectives';
@@ -9,9 +10,15 @@ import { compareObjectiveValues } from '../planning-objectives';
await import('@angular/compiler');
const { generateLoadedYearRequest, generateRequest } = await import('./random-planning-requests');
const current = await import('../leave-planner.core');
const calibration = await import('./leave-planner.core.calibration.mjs');
const baselines = {
frozen: await import('./leave-planner.core.frozen.mjs'),
calibration: await import('./leave-planner.core.calibration.mjs'),
};
const [kind = 'small', first = '280000', count = '5000'] = process.argv.slice(2);
const [kind = 'small', first = '280000', count = '5000', baselineName = 'frozen'] =
process.argv.slice(2);
const baseline = baselines[baselineName as keyof typeof baselines];
if (!baseline) throw new Error(`Unknown baseline engine: ${baselineName}`);
const generate = kind === 'loaded-year' ? generateLoadedYearRequest : generateRequest;
type Engine = (
request: unknown,
@@ -45,7 +52,7 @@ const counts = new Map<string, number>();
const time = { before: 0, after: 0 };
for (let seed = Number(first); seed < Number(first) + Number(count); seed++) {
const request = generate(seed);
const before = run(calibration.calculatePlanningInternal, request);
const before = run(baseline.calculatePlanningInternal, request);
const after = run(current.calculatePlanningInternal, request);
time.before += before.ms;
time.after += after.ms;
@@ -73,8 +80,10 @@ for (let seed = Number(first); seed < Number(first) + Number(count); seed++) {
if (category !== 'identical') console.log(`${seed}\t${category}${detail}`);
counts.set(category, (counts.get(category) ?? 0) + 1);
}
console.log(`${kind} requests, seeds ${first}..${Number(first) + Number(count) - 1}`);
console.log(
`${kind} requests, seeds ${first}..${Number(first) + Number(count) - 1}, against the ${baselineName} engine`,
);
for (const [category, total] of [...counts].sort()) console.log(`${category}\t${total}`);
console.log(
`calibration engine ${(time.before / 1000).toFixed(1)} s, current engine ${(time.after / 1000).toFixed(1)} s`,
`${baselineName} engine ${(time.before / 1000).toFixed(1)} s, current engine ${(time.after / 1000).toFixed(1)} s`,
);