nessarandClaude Opus 5.5
75320da685
A standing "no worse than the previous engine" check is not added to CI: the differential check already fails on unintended changes, and a fixed acceptance threshold would need renegotiating with every deliberate change. Instead, the rebuild procedure now runs the comparison while the frozen engine is still the previous one, and records it with the change. The runner compares against the frozen engine by default and takes `calibration` to reproduce the #302 record. Part of #302. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
90 lines
4.0 KiB
TypeScript
90 lines
4.0 KiB
TypeScript
// Opt-in before/after record, outside the functional suite and CI: compares the
|
|
// current engine with the frozen engine (before its rebuild) or the calibration
|
|
// engine (a871f8f) over seeded requests.
|
|
// bun src/app/services/frozen-engine/compare-engines.ts [small|loaded-year] [first seed] [count] [frozen|calibration]
|
|
import { isDeepStrictEqual } from 'node:util';
|
|
import type { PlanningArtifacts, PlanningObjective, PlanningOutcome } from '../leave-planner.core';
|
|
import { compareObjectiveValues } from '../planning-objectives';
|
|
|
|
// The preset rules come from an Angular service file, which needs the JIT compiler outside tests.
|
|
await import('@angular/compiler');
|
|
const { generateLoadedYearRequest, generateRequest } = await import('./random-planning-requests');
|
|
const current = await import('../leave-planner.core');
|
|
const baselines = {
|
|
frozen: await import('./leave-planner.core.frozen.mjs'),
|
|
calibration: await import('./leave-planner.core.calibration.mjs'),
|
|
};
|
|
|
|
const [kind = 'small', first = '280000', count = '5000', baselineName = 'frozen'] =
|
|
process.argv.slice(2);
|
|
const baseline = baselines[baselineName as keyof typeof baselines];
|
|
if (!baseline) throw new Error(`Unknown baseline engine: ${baselineName}`);
|
|
const generate = kind === 'loaded-year' ? generateLoadedYearRequest : generateRequest;
|
|
type Engine = (
|
|
request: unknown,
|
|
checkpoint?: undefined,
|
|
artifacts?: (artifacts: PlanningArtifacts) => void,
|
|
) => PlanningOutcome;
|
|
|
|
function run(calculate: Engine, request: unknown) {
|
|
let artifacts: PlanningArtifacts | undefined;
|
|
const started = performance.now();
|
|
const outcome = calculate(structuredClone(request), undefined, (value) => (artifacts = value));
|
|
return {
|
|
ms: performance.now() - started,
|
|
result: {
|
|
outcome,
|
|
explanation: artifacts?.explanation,
|
|
finalState: artifacts?.finalState,
|
|
explanationError: artifacts?.explanationError,
|
|
},
|
|
};
|
|
}
|
|
|
|
/** The final plan's objective vector, which every placement explanation repeats. */
|
|
function objectives(result: ReturnType<typeof run>['result']): readonly PlanningObjective[] {
|
|
return result.explanation?.kind === 'PlanExplanation'
|
|
? (result.explanation.placements[0]?.objectiveVector.objectives ?? [])
|
|
: [];
|
|
}
|
|
|
|
const counts = new Map<string, number>();
|
|
const time = { before: 0, after: 0 };
|
|
for (let seed = Number(first); seed < Number(first) + Number(count); seed++) {
|
|
const request = generate(seed);
|
|
const before = run(baseline.calculatePlanningInternal, request);
|
|
const after = run(current.calculatePlanningInternal, request);
|
|
time.before += before.ms;
|
|
time.after += after.ms;
|
|
let category: string;
|
|
let detail = '';
|
|
if (isDeepStrictEqual(before.result, after.result)) category = 'identical';
|
|
else if (before.result.outcome.kind !== after.result.outcome.kind)
|
|
category = 'changed outcome kind';
|
|
else if (after.result.outcome.kind !== 'Plan') category = 'different non-plan outcome';
|
|
else {
|
|
const left = objectives(before.result);
|
|
const right = objectives(after.result);
|
|
const index = right.findIndex((objective, i) => objective.value !== left[i]?.value);
|
|
if (index < 0 && left.length === right.length) category = 'same objective values';
|
|
else if (index < 0 || left[index]?.id !== right[index].id) category = 'not comparable';
|
|
else {
|
|
const objective = right[index];
|
|
const better =
|
|
compareObjectiveValues(objective.direction, objective.value, left[index].value) < 0;
|
|
const tieBreak = ['balance-expiry', 'placement-date'].includes(objective.id);
|
|
category = `${better ? 'better' : 'worse'}${tieBreak ? ' (tie-break only)' : ''}`;
|
|
detail = `\t${objective.id}\t${left[index].value} -> ${objective.value}`;
|
|
}
|
|
}
|
|
if (category !== 'identical') console.log(`${seed}\t${category}${detail}`);
|
|
counts.set(category, (counts.get(category) ?? 0) + 1);
|
|
}
|
|
console.log(
|
|
`${kind} requests, seeds ${first}..${Number(first) + Number(count) - 1}, against the ${baselineName} engine`,
|
|
);
|
|
for (const [category, total] of [...counts].sort()) console.log(`${category}\t${total}`);
|
|
console.log(
|
|
`${baselineName} engine ${(time.before / 1000).toFixed(1)} s, current engine ${(time.after / 1000).toFixed(1)} s`,
|
|
);
|