ci(benchmark): gate scenario speed relative to the runner
Before the protocol, time the frozen engine on the two-year scenario, best of three, and fail any scenario whose mean planning time exceeds its multiplier times that calibration. Multipliers come from development-machine ratios with about 1.5x headroom until #286 tightens them. The 300-second timeout stays only as a hang guard. The benchmark keeps running on push and pull_request events because the push run gates image publication. This replaces the absolute 300-second acceptance protocol named in #78. Refs #285 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
68a9e33de6
commit
acdb19352a
7 files changed
+144
-14
No files matched your search
@@ -1,10 +1,13 @@
|
||||
import { vi } from 'vitest';
|
||||
import { calculatePlanningAsyncInternal } from '../services/frozen-engine/leave-planner.core.frozen.mjs';
|
||||
import { createPlanningCalculation } from '../services/leave-planner.service';
|
||||
import {
|
||||
BENCHMARK_BUDGET_MULTIPLIERS,
|
||||
BENCHMARK_REPORT_HEADER,
|
||||
benchmarkReport,
|
||||
benchmarkReportRows,
|
||||
benchmarkScenarios,
|
||||
budgetViolations,
|
||||
runLeavePlannerBenchmark,
|
||||
} from './leave-planner-benchmark';
|
||||
|
||||
@@ -66,6 +69,16 @@ describe('leave planner benchmark protocol', () => {
|
||||
),
|
||||
).toHaveLength(2);
|
||||
}
|
||||
// Frozen engine (a871f8f) on 'two years', best of three: the runner-speed yardstick.
|
||||
const calibrationRequest = scenarios.find((item) => item.name === 'two years')!.request;
|
||||
let calibrationMs = Infinity;
|
||||
for (let run = 0; run < 3; run++) {
|
||||
const request = structuredClone(calibrationRequest);
|
||||
const start = performance.now();
|
||||
await calculatePlanningAsyncInternal({ ...request, calculationDate: '2024-01-01' });
|
||||
calibrationMs = Math.min(calibrationMs, performance.now() - start);
|
||||
}
|
||||
console.info(`calibration\tfrozen two years best of 3\t${calibrationMs.toFixed(1)}`);
|
||||
console.info(BENCHMARK_REPORT_HEADER);
|
||||
const measurements = await runLeavePlannerBenchmark(2, (rows) =>
|
||||
console.info(benchmarkReportRows(rows)),
|
||||
@@ -106,6 +119,13 @@ describe('leave planner benchmark protocol', () => {
|
||||
expect(report).toContain('planning_ms\twall_ms');
|
||||
expect(report).toContain('planning_work_ms\tpeak_main_thread_heap_bytes');
|
||||
expect(report).toContain('\tbaseline\t');
|
||||
const violations = budgetViolations(
|
||||
measurements,
|
||||
calibrationMs,
|
||||
BENCHMARK_BUDGET_MULTIPLIERS,
|
||||
);
|
||||
if (violations.length)
|
||||
throw new Error(`Benchmark over budget:\n${violations.join('\n')}`);
|
||||
} finally {
|
||||
vi.stubGlobal('window', browserWindow);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user