ci(benchmark): gate scenario speed relative to the runner
Before the protocol, time the frozen engine on the two-year scenario, best of three, and fail any scenario whose mean planning time exceeds its multiplier times that calibration. Multipliers come from development-machine ratios with about 1.5x headroom until #286 tightens them. The 300-second timeout stays only as a hang guard. The benchmark keeps running on push and pull_request events because the push run gates image publication. This replaces the absolute 300-second acceptance protocol named in #78. Refs #285 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
68a9e33de6
commit
acdb19352a
7 files changed
+144
-14
No files matched your search
@@ -307,3 +307,40 @@ export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
|
||||
? [BENCHMARK_REPORT_HEADER, benchmarkReportRows(measurements)].join('\n')
|
||||
: BENCHMARK_REPORT_HEADER;
|
||||
}
|
||||
|
||||
// Measured on the development machine (ratio to the frozen two-years run), tightened by #286.
|
||||
export const BENCHMARK_BUDGET_MULTIPLIERS: Record<string, number> = {
|
||||
'representative year': 0.25,
|
||||
'typed obligations': 3,
|
||||
'chained earned entitlements': 0.05,
|
||||
'deliberate conflict': 0.05,
|
||||
'loaded year': 25,
|
||||
'two years': 1,
|
||||
'three years': 3.5,
|
||||
};
|
||||
|
||||
/** One message per scenario whose mean planning time exceeds multiplier x calibration, or that has no multiplier. */
|
||||
export function budgetViolations(
|
||||
measurements: BenchmarkMeasurement[],
|
||||
calibrationMs: number,
|
||||
multipliers: Record<string, number> = BENCHMARK_BUDGET_MULTIPLIERS,
|
||||
): string[] {
|
||||
const byScenario = new Map<string, number[]>();
|
||||
for (const { scenario, planningMs } of measurements)
|
||||
byScenario.set(scenario, [...(byScenario.get(scenario) ?? []), planningMs]);
|
||||
const violations: string[] = [];
|
||||
for (const [scenario, times] of byScenario) {
|
||||
const mean = times.reduce((sum, value) => sum + value, 0) / times.length;
|
||||
const multiplier = multipliers[scenario];
|
||||
if (multiplier === undefined) {
|
||||
violations.push(`${scenario}: no budget multiplier defined`);
|
||||
continue;
|
||||
}
|
||||
const budget = multiplier * calibrationMs;
|
||||
if (mean > budget)
|
||||
violations.push(
|
||||
`${scenario}: measured ${mean.toFixed(1)} ms > budget ${budget.toFixed(1)} ms (multiplier ${multiplier} x calibration ${calibrationMs.toFixed(1)} ms)`,
|
||||
);
|
||||
}
|
||||
return violations;
|
||||
}
|
||||
Reference in new issue
Block a user