ci(benchmark): gate scenario speed relative to the runner

Before the protocol, time the frozen engine on the two-year scenario, best
of three, and fail any scenario whose mean planning time exceeds its
multiplier times that calibration. Multipliers come from development-machine
ratios with about 1.5x headroom until #286 tightens them. The 300-second
timeout stays only as a hang guard. The benchmark keeps running on push and
pull_request events because the push run gates image publication.

This replaces the absolute 300-second acceptance protocol named in #78.

Refs #285

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
nessarandClaude Opus 5.5 committed 2026-10-17 00:45:28 +02:00
1 parent 68a9e33de6
commit acdb19352a
7 files changed
+144 -14

No files matched your search

@@ -307,3 +307,40 @@ export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
? [BENCHMARK_REPORT_HEADER, benchmarkReportRows(measurements)].join('\n')
: BENCHMARK_REPORT_HEADER;
}
// Measured on the development machine (ratio to the frozen two-years run), tightened by #286.
export const BENCHMARK_BUDGET_MULTIPLIERS: Record<string, number> = {
'representative year': 0.25,
'typed obligations': 3,
'chained earned entitlements': 0.05,
'deliberate conflict': 0.05,
'loaded year': 25,
'two years': 1,
'three years': 3.5,
};
/** One message per scenario whose mean planning time exceeds multiplier x calibration, or that has no multiplier. */
export function budgetViolations(
measurements: BenchmarkMeasurement[],
calibrationMs: number,
multipliers: Record<string, number> = BENCHMARK_BUDGET_MULTIPLIERS,
): string[] {
const byScenario = new Map<string, number[]>();
for (const { scenario, planningMs } of measurements)
byScenario.set(scenario, [...(byScenario.get(scenario) ?? []), planningMs]);
const violations: string[] = [];
for (const [scenario, times] of byScenario) {
const mean = times.reduce((sum, value) => sum + value, 0) / times.length;
const multiplier = multipliers[scenario];
if (multiplier === undefined) {
violations.push(`${scenario}: no budget multiplier defined`);
continue;
}
const budget = multiplier * calibrationMs;
if (mean > budget)
violations.push(
`${scenario}: measured ${mean.toFixed(1)} ms > budget ${budget.toFixed(1)} ms (multiplier ${multiplier} x calibration ${calibrationMs.toFixed(1)} ms)`,
);
}
return violations;
}