import { vi } from '../../testing/vitest-compat'; import { calculatePlanningAsyncInternal } from '../services/frozen-engine/leave-planner.core.calibration.mjs'; import { createPlanningCalculation } from '../services/leave-planner.service'; import { BENCHMARK_BUDGET_MULTIPLIERS, type BenchmarkMeasurement, benchmarkReport, benchmarkScenarios, benchmarkSummaryHeader, benchmarkSummaryRow, budgetViolations, runLeavePlannerBenchmark, } from './leave-planner-benchmark'; describe('leave planner benchmark protocol', () => { beforeEach(() => vi.setSystemTime(new Date('2024-01-01T12:00:00Z'))); afterEach(() => vi.useRealTimers()); // Warm-up and two repetitions include explanation delivery after planning. it('keeps seven deterministic scenarios including two and three years and a later earned consumer', async () => { const scenarios = benchmarkScenarios(); expect(scenarios).toHaveLength(7); for (const [name, end] of [ ['two years', '2026-01-01'], ['three years', '2027-01-01'], ]) { const scenario = scenarios.find((item) => item.name === name)!; expect(scenario.request.horizon).toEqual({ start: '2024-01-01', end }); expect( scenario.request.balances.some( (item) => String(item.availableFrom) >= '2025-01-01', ), ).toBe(true); expect(scenario.request.deadlineMs).toBeUndefined(); } const typedObligations = scenarios.find( (scenario) => scenario.name === 'typed obligations', ); const cpRule = typedObligations?.request.rules?.find((rule) => rule.id === 'cp-window'); const rttRule = typedObligations?.request.rules?.find((rule) => rule.id === 'rtt-window'); expect(cpRule).toMatchObject({ placeType: 'cp', placeDays: 2 }); expect(rttRule).toMatchObject({ placeType: 'rtt', placeDays: 2 }); expect(cpRule?.type).toBe('time_window'); expect(rttRule?.type).toBe('time_window'); if (cpRule && 'startDate' in cpRule && rttRule && 'startDate' in rttRule) expect( String(cpRule.startDate) <= String(rttRule.endDate) && String(rttRule.startDate) <= String(cpRule.endDate), ).toBe(true); const chain = scenarios.find((scenario) => scenario.name === 'chained earned entitlements'); expect( chain?.request.rules ?.filter(({ type }) => type !== 'optimization') .map((rule) => rule.id), ).toEqual(['wednesday-holidays', 'thursday-consumer']); const browserWindow = globalThis.window; vi.stubGlobal('window', undefined); try { const typedOutcome = await createPlanningCalculation(typedObligations!.request).outcome; expect(typedOutcome.kind).toBe('Plan'); if (typedOutcome.kind === 'Plan') { expect( typedOutcome.placements.filter((placement) => placement.ruleIds.includes('cp-window'), ), ).toHaveLength(2); expect( typedOutcome.placements.filter((placement) => placement.ruleIds.includes('rtt-window'), ), ).toHaveLength(2); } // Calibration engine (frozen at a871f8f) on 'two years', best of three: the runner-speed yardstick. const calibrationRequest = scenarios.find((item) => item.name === 'two years')!.request; const calibrate = async () => { let best = Infinity; for (let run = 0; run < 3; run++) { const request = structuredClone(calibrationRequest); const start = performance.now(); await calculatePlanningAsyncInternal({ ...request, calculationDate: '2024-01-01', }); best = Math.min(best, performance.now() - start); } return best; }; const calibrationBefore = await calibrate(); // Direct stdout writes show each row as soon as its scenario finishes. const print = (text: string) => process.stdout.write(`${text}\n`); print( `\nCalibration: frozen engine on "two years", best of 3 = ${calibrationBefore.toFixed(1)} ms` + '\nBudget = limit x calibration; ratio = mean / calibration.\n', ); print(benchmarkSummaryHeader()); const scenarioRows: BenchmarkMeasurement[][] = []; const measurements = await runLeavePlannerBenchmark(2, (rows) => { scenarioRows.push(rows); print(benchmarkSummaryRow(rows, calibrationBefore)); }); // Load that starts after the first calibration slows the scenarios but not the yardstick; // calibrating again and judging against the slower value absorbs it. Never tighter than before. const calibrationAfter = await calibrate(); const calibrationMs = Math.max(calibrationBefore, calibrationAfter); if (calibrationMs !== calibrationBefore) { print( `\nCalibration after the protocol = ${calibrationAfter.toFixed(1)} ms; budgets use the slower: ${calibrationMs.toFixed(1)} ms\n`, ); print(benchmarkSummaryHeader()); for (const rows of scenarioRows) print(benchmarkSummaryRow(rows, calibrationMs)); } for (const [name, placements] of [ ['two years', 60], ['three years', 90], ] as const) { const longer = measurements.filter((measurement) => measurement.scenario === name); expect(longer).toHaveLength(2); expect(longer.map((item) => item.outcome)).toEqual(['Plan', 'Plan']); expect(longer.map((item) => item.lastCompletedStage)).toEqual(['dates', 'dates']); expect(longer.map((item) => item.placements)).toEqual([placements, placements]); expect(longer.map((item) => item.deterministic)).toEqual(['baseline', 'yes']); } const chained = measurements.find( (measurement) => measurement.scenario === 'chained earned entitlements', ); const loaded = measurements.filter( (measurement) => measurement.scenario === 'loaded year', ); expect(chained?.outcome).toBe('Plan'); expect(chained?.placements).toBeGreaterThanOrEqual(3); expect(loaded).toHaveLength(2); expect(loaded[0].outcome).toBe('Plan'); expect(loaded[0].deterministic).toBe('baseline'); expect(loaded[1].deterministic).toBe('yes'); expect(loaded[0].planningMs).toBeGreaterThanOrEqual(0); expect(loaded[0].planningWorkMs).toBeGreaterThanOrEqual(0); expect(loaded[0].planningWorkMs).toBeLessThanOrEqual(loaded[0].planningMs); expect(loaded[0].maxUiDelayMs).toBeGreaterThanOrEqual(0); expect(loaded[0].wallMs).toBeGreaterThanOrEqual( loaded[0].preparationMs + loaded[0].planningMs, ); const report = benchmarkReport(measurements); print(`\nRaw report (TSV):\n${report}\n`); expect(report.split('\n')).toHaveLength(15); expect(report).toContain('planning_ms\twall_ms'); expect(report).toContain('planning_work_ms\tpeak_main_thread_heap_bytes'); expect(report).toContain('\tbaseline\t'); const violations = budgetViolations( measurements, calibrationMs, BENCHMARK_BUDGET_MULTIPLIERS, ); if (violations.length) throw new Error(`Benchmark over budget:\n${violations.join('\n')}`); } finally { vi.stubGlobal('window', browserWindow); } }, 210_000); });