perf(planning): plan from the best-ranked obligation solution
With several obligation solutions, the planner ran the whole preference chain on candidate solutions as previews: every solution after a complete enumeration, or the eight best-ranked after a truncated window search. It now orders all solutions by the existing pre-preference ranking (active objectives, then tie-breaks, of each obligation-only plan; equal ranks keep enumeration order) and runs the preference stages once, on the first. If that run ends with earned leave available only after its final merged time-off period, the stages restart on the next solution, so a request that planned before never becomes a conflict. The loaded year drops from 43 to 5.5 seconds and typed obligations from 3.3 to 0.1 seconds, with byte-identical benchmark plans. Over the 5,000 seeded requests, 23 plans become worse (9 on a real objective) and 1 better; over 50 loaded-year-sized requests, 4 become worse on a real objective. ADR 0010 records the decision; docs/benchmark/issue-302.md records the comparison and the opt-in runner that reproduces it. The frozen-engine check fails until the frozen engine is rebuilt from this commit in the next one. Part of #302. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aa6a0c6f1b
commit
8168137cd5
13 files changed
+579
-278
No files matched your search
@@ -0,0 +1,80 @@
|
||||
// Opt-in before/after record, outside the functional suite and CI: compares the
|
||||
// current engine with the calibration engine (a871f8f) over seeded requests.
|
||||
// bun src/app/services/frozen-engine/compare-engines.ts [small|loaded-year] [first seed] [count]
|
||||
import { isDeepStrictEqual } from 'node:util';
|
||||
import type { PlanningArtifacts, PlanningObjective, PlanningOutcome } from '../leave-planner.core';
|
||||
import { compareObjectiveValues } from '../planning-objectives';
|
||||
|
||||
// The preset rules come from an Angular service file, which needs the JIT compiler outside tests.
|
||||
await import('@angular/compiler');
|
||||
const { generateLoadedYearRequest, generateRequest } = await import('./random-planning-requests');
|
||||
const current = await import('../leave-planner.core');
|
||||
const calibration = await import('./leave-planner.core.calibration.mjs');
|
||||
|
||||
const [kind = 'small', first = '280000', count = '5000'] = process.argv.slice(2);
|
||||
const generate = kind === 'loaded-year' ? generateLoadedYearRequest : generateRequest;
|
||||
type Engine = (
|
||||
request: unknown,
|
||||
checkpoint?: undefined,
|
||||
artifacts?: (artifacts: PlanningArtifacts) => void,
|
||||
) => PlanningOutcome;
|
||||
|
||||
function run(calculate: Engine, request: unknown) {
|
||||
let artifacts: PlanningArtifacts | undefined;
|
||||
const started = performance.now();
|
||||
const outcome = calculate(structuredClone(request), undefined, (value) => (artifacts = value));
|
||||
return {
|
||||
ms: performance.now() - started,
|
||||
result: {
|
||||
outcome,
|
||||
explanation: artifacts?.explanation,
|
||||
finalState: artifacts?.finalState,
|
||||
explanationError: artifacts?.explanationError,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** The final plan's objective vector, which every placement explanation repeats. */
|
||||
function objectives(result: ReturnType<typeof run>['result']): readonly PlanningObjective[] {
|
||||
return result.explanation?.kind === 'PlanExplanation'
|
||||
? (result.explanation.placements[0]?.objectiveVector.objectives ?? [])
|
||||
: [];
|
||||
}
|
||||
|
||||
const counts = new Map<string, number>();
|
||||
const time = { before: 0, after: 0 };
|
||||
for (let seed = Number(first); seed < Number(first) + Number(count); seed++) {
|
||||
const request = generate(seed);
|
||||
const before = run(calibration.calculatePlanningInternal, request);
|
||||
const after = run(current.calculatePlanningInternal, request);
|
||||
time.before += before.ms;
|
||||
time.after += after.ms;
|
||||
let category: string;
|
||||
let detail = '';
|
||||
if (isDeepStrictEqual(before.result, after.result)) category = 'identical';
|
||||
else if (before.result.outcome.kind !== after.result.outcome.kind)
|
||||
category = 'changed outcome kind';
|
||||
else if (after.result.outcome.kind !== 'Plan') category = 'different non-plan outcome';
|
||||
else {
|
||||
const left = objectives(before.result);
|
||||
const right = objectives(after.result);
|
||||
const index = right.findIndex((objective, i) => objective.value !== left[i]?.value);
|
||||
if (index < 0 && left.length === right.length) category = 'same objective values';
|
||||
else if (index < 0 || left[index]?.id !== right[index].id) category = 'not comparable';
|
||||
else {
|
||||
const objective = right[index];
|
||||
const better =
|
||||
compareObjectiveValues(objective.direction, objective.value, left[index].value) < 0;
|
||||
const tieBreak = ['balance-expiry', 'placement-date'].includes(objective.id);
|
||||
category = `${better ? 'better' : 'worse'}${tieBreak ? ' (tie-break only)' : ''}`;
|
||||
detail = `\t${objective.id}\t${left[index].value} -> ${objective.value}`;
|
||||
}
|
||||
}
|
||||
if (category !== 'identical') console.log(`${seed}\t${category}${detail}`);
|
||||
counts.set(category, (counts.get(category) ?? 0) + 1);
|
||||
}
|
||||
console.log(`${kind} requests, seeds ${first}..${Number(first) + Number(count) - 1}`);
|
||||
for (const [category, total] of [...counts].sort()) console.log(`${category}\t${total}`);
|
||||
console.log(
|
||||
`calibration engine ${(time.before / 1000).toFixed(1)} s, current engine ${(time.after / 1000).toFixed(1)} s`,
|
||||
);
|
||||
Reference in new issue
Block a user