ci(benchmark): gate scenario speed relative to the runner

Before the protocol, time the frozen engine on the two-year scenario, best
of three, and fail any scenario whose mean planning time exceeds its
multiplier times that calibration. Multipliers come from development-machine
ratios with about 1.5x headroom until #286 tightens them. The 300-second
timeout stays only as a hang guard. The benchmark keeps running on push and
pull_request events because the push run gates image publication.

This replaces the absolute 300-second acceptance protocol named in #78.

Refs #285

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
nessarandClaude Opus 5.5 committed 2026-10-17 00:45:28 +02:00
1 parent 68a9e33de6
commit acdb19352a
7 files changed
+144 -14

No files matched your search

@@ -0,0 +1,30 @@
import { BenchmarkMeasurement, budgetViolations } from './leave-planner-benchmark';
const measure = (scenario: string, planningMs: number) =>
({ scenario, planningMs }) as BenchmarkMeasurement;
describe('benchmark budget gate', () => {
const multipliers = { a: 2 };
it('accepts a mean within multiplier x calibration', () => {
expect(budgetViolations([measure('a', 150), measure('a', 250)], 100, multipliers)).toEqual(
[],
);
});
it('reports measured time, budget, multiplier and calibration when over budget', () => {
const [message] = budgetViolations(
[measure('a', 300), measure('a', 320)],
100,
multipliers,
);
expect(message).toContain('a:');
expect(message).toContain('measured 310.0 ms');
expect(message).toContain('budget 200.0 ms');
expect(message).toContain('multiplier 2');
expect(message).toContain('calibration 100.0 ms');
});
it('fails a scenario without a multiplier', () => {
expect(budgetViolations([measure('new', 1)], 100, multipliers)).toEqual([
'new: no budget multiplier defined',
]);
});
});