ci(benchmark): gate scenario speed relative to the runner

Before the protocol, time the frozen engine on the two-year scenario, best
of three, and fail any scenario whose mean planning time exceeds its
multiplier times that calibration. Multipliers come from development-machine
ratios with about 1.5x headroom until #286 tightens them. The 300-second
timeout stays only as a hang guard. The benchmark keeps running on push and
pull_request events because the push run gates image publication.

This replaces the absolute 300-second acceptance protocol named in #78.

Refs #285

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
nessarandClaude Opus 5.5 committed 2026-10-17 00:45:28 +02:00
1 parent 68a9e33de6
commit acdb19352a
7 files changed
+144 -14

No files matched your search

@@ -1,10 +1,13 @@
import { vi } from 'vitest';
import { calculatePlanningAsyncInternal } from '../services/frozen-engine/leave-planner.core.frozen.mjs';
import { createPlanningCalculation } from '../services/leave-planner.service';
import {
BENCHMARK_BUDGET_MULTIPLIERS,
BENCHMARK_REPORT_HEADER,
benchmarkReport,
benchmarkReportRows,
benchmarkScenarios,
budgetViolations,
runLeavePlannerBenchmark,
} from './leave-planner-benchmark';
@@ -66,6 +69,16 @@ describe('leave planner benchmark protocol', () => {
),
).toHaveLength(2);
}
// Frozen engine (a871f8f) on 'two years', best of three: the runner-speed yardstick.
const calibrationRequest = scenarios.find((item) => item.name === 'two years')!.request;
let calibrationMs = Infinity;
for (let run = 0; run < 3; run++) {
const request = structuredClone(calibrationRequest);
const start = performance.now();
await calculatePlanningAsyncInternal({ ...request, calculationDate: '2024-01-01' });
calibrationMs = Math.min(calibrationMs, performance.now() - start);
}
console.info(`calibration\tfrozen two years best of 3\t${calibrationMs.toFixed(1)}`);
console.info(BENCHMARK_REPORT_HEADER);
const measurements = await runLeavePlannerBenchmark(2, (rows) =>
console.info(benchmarkReportRows(rows)),
@@ -106,6 +119,13 @@ describe('leave planner benchmark protocol', () => {
expect(report).toContain('planning_ms\twall_ms');
expect(report).toContain('planning_work_ms\tpeak_main_thread_heap_bytes');
expect(report).toContain('\tbaseline\t');
const violations = budgetViolations(
measurements,
calibrationMs,
BENCHMARK_BUDGET_MULTIPLIERS,
);
if (violations.length)
throw new Error(`Benchmark over budget:\n${violations.join('\n')}`);
} finally {
vi.stubGlobal('window', browserWindow);
}