vac-optimizer/src/app/benchmark/leave-planner-benchmark.spec.ts

import { vi } from '../../testing/vitest-compat';
import { calculatePlanningAsyncInternal } from '../services/frozen-engine/leave-planner.core.calibration.mjs';
import { createPlanningCalculation } from '../services/leave-planner.service';
import {
	BENCHMARK_BUDGET_MULTIPLIERS,
	type BenchmarkMeasurement,
	benchmarkReport,
	benchmarkScenarios,
	benchmarkSummaryHeader,
	benchmarkSummaryRow,
	budgetViolations,
	runLeavePlannerBenchmark,
} from './leave-planner-benchmark';

describe('leave planner benchmark protocol', () => {
	beforeEach(() => vi.setSystemTime(new Date('2024-01-01T12:00:00Z')));
	afterEach(() => vi.useRealTimers());
	// Warm-up and two repetitions include explanation delivery after planning.
	it('keeps seven deterministic scenarios including two and three years and a later earned consumer', async () => {
		const scenarios = benchmarkScenarios();
		expect(scenarios).toHaveLength(7);
		for (const [name, end] of [
			['two years', '2026-01-01'],
			['three years', '2027-01-01'],
		]) {
			const scenario = scenarios.find((item) => item.name === name)!;
			expect(scenario.request.horizon).toEqual({ start: '2024-01-01', end });
			expect(
				scenario.request.balances.some(
					(item) => String(item.availableFrom) >= '2025-01-01',
				),
			).toBe(true);
			expect(scenario.request.deadlineMs).toBeUndefined();
		}
		const typedObligations = scenarios.find(
			(scenario) => scenario.name === 'typed obligations',
		);
		const cpRule = typedObligations?.request.rules?.find((rule) => rule.id === 'cp-window');
		const rttRule = typedObligations?.request.rules?.find((rule) => rule.id === 'rtt-window');
		expect(cpRule).toMatchObject({ placeType: 'cp', placeDays: 2 });
		expect(rttRule).toMatchObject({ placeType: 'rtt', placeDays: 2 });
		expect(cpRule?.type).toBe('time_window');
		expect(rttRule?.type).toBe('time_window');
		if (cpRule && 'startDate' in cpRule && rttRule && 'startDate' in rttRule)
			expect(
				String(cpRule.startDate) <= String(rttRule.endDate) &&
					String(rttRule.startDate) <= String(cpRule.endDate),
			).toBe(true);
		const chain = scenarios.find((scenario) => scenario.name === 'chained earned entitlements');
		expect(
			chain?.request.rules
				?.filter(({ type }) => type !== 'optimization')
				.map((rule) => rule.id),
		).toEqual(['wednesday-holidays', 'thursday-consumer']);

		const browserWindow = globalThis.window;
		vi.stubGlobal('window', undefined);
		try {
			const typedOutcome = await createPlanningCalculation(typedObligations!.request).outcome;
			expect(typedOutcome.kind).toBe('Plan');
			if (typedOutcome.kind === 'Plan') {
				expect(
					typedOutcome.placements.filter((placement) =>
						placement.ruleIds.includes('cp-window'),
					),
				).toHaveLength(2);
				expect(
					typedOutcome.placements.filter((placement) =>
						placement.ruleIds.includes('rtt-window'),
					),
				).toHaveLength(2);
			}
			// Calibration engine (frozen at a871f8f) on 'two years', best of three: the runner-speed yardstick.
			const calibrationRequest = scenarios.find((item) => item.name === 'two years')!.request;
			const calibrate = async () => {
				let best = Infinity;
				for (let run = 0; run < 3; run++) {
					const request = structuredClone(calibrationRequest);
					const start = performance.now();
					await calculatePlanningAsyncInternal({
						...request,
						calculationDate: '2024-01-01',
					});
					best = Math.min(best, performance.now() - start);
				}
				return best;
			};
			const calibrationBefore = await calibrate();
			// Direct stdout writes show each row as soon as its scenario finishes.
			const print = (text: string) => process.stdout.write(`${text}\n`);
			print(
				`\nCalibration: frozen engine on "two years", best of 3 = ${calibrationBefore.toFixed(1)} ms` +
					'\nBudget = limit x calibration; ratio = mean / calibration.\n',
			);
			print(benchmarkSummaryHeader());
			const scenarioRows: BenchmarkMeasurement[][] = [];
			const measurements = await runLeavePlannerBenchmark(2, (rows) => {
				scenarioRows.push(rows);
				print(benchmarkSummaryRow(rows, calibrationBefore));
			});
			// Load that starts after the first calibration slows the scenarios but not the yardstick;
			// calibrating again and judging against the slower value absorbs it. Never tighter than before.
			const calibrationAfter = await calibrate();
			const calibrationMs = Math.max(calibrationBefore, calibrationAfter);
			if (calibrationMs !== calibrationBefore) {
				print(
					`\nCalibration after the protocol = ${calibrationAfter.toFixed(1)} ms; budgets use the slower: ${calibrationMs.toFixed(1)} ms\n`,
				);
				print(benchmarkSummaryHeader());
				for (const rows of scenarioRows) print(benchmarkSummaryRow(rows, calibrationMs));
			}
			for (const [name, placements] of [
				['two years', 60],
				['three years', 90],
			] as const) {
				const longer = measurements.filter((measurement) => measurement.scenario === name);
				expect(longer).toHaveLength(2);
				expect(longer.map((item) => item.outcome)).toEqual(['Plan', 'Plan']);
				expect(longer.map((item) => item.lastCompletedStage)).toEqual(['dates', 'dates']);
				expect(longer.map((item) => item.placements)).toEqual([placements, placements]);
				expect(longer.map((item) => item.deterministic)).toEqual(['baseline', 'yes']);
			}
			const chained = measurements.find(
				(measurement) => measurement.scenario === 'chained earned entitlements',
			);
			const loaded = measurements.filter(
				(measurement) => measurement.scenario === 'loaded year',
			);
			expect(chained?.outcome).toBe('Plan');
			expect(chained?.placements).toBeGreaterThanOrEqual(3);
			expect(loaded).toHaveLength(2);
			expect(loaded[0].outcome).toBe('Plan');
			expect(loaded[0].deterministic).toBe('baseline');
			expect(loaded[1].deterministic).toBe('yes');
			expect(loaded[0].planningMs).toBeGreaterThanOrEqual(0);
			expect(loaded[0].planningWorkMs).toBeGreaterThanOrEqual(0);
			expect(loaded[0].planningWorkMs).toBeLessThanOrEqual(loaded[0].planningMs);
			expect(loaded[0].maxUiDelayMs).toBeGreaterThanOrEqual(0);
			expect(loaded[0].wallMs).toBeGreaterThanOrEqual(
				loaded[0].preparationMs + loaded[0].planningMs,
			);
			const report = benchmarkReport(measurements);
			print(`\nRaw report (TSV):\n${report}\n`);
			expect(report.split('\n')).toHaveLength(15);
			expect(report).toContain('planning_ms\twall_ms');
			expect(report).toContain('planning_work_ms\tpeak_main_thread_heap_bytes');
			expect(report).toContain('\tbaseline\t');
			const violations = budgetViolations(
				measurements,
				calibrationMs,
				BENCHMARK_BUDGET_MULTIPLIERS,
			);
			if (violations.length)
				throw new Error(`Benchmark over budget:\n${violations.join('\n')}`);
		} finally {
			vi.stubGlobal('window', browserWindow);
		}
	}, 210_000);
});