vac-optimizer/src/app/benchmark/leave-planner-benchmark.ts

import {
	createPlanningCalculation,
	PlanningOutcome,
	PlanningRequest,
} from '../services/leave-planner.service';
import { createOptimizationPresetRules } from '../services/rule-service';

export interface BenchmarkScenario {
	name: string;
	request: PlanningRequest;
}

export interface BenchmarkMeasurement {
	scenario: string;
	repetition: number;
	preparationMs: number;
	planningMs: number;
	maxUiDelayMs: number;
	/** Chromium's main-thread heap only; worker memory needs a separate browser profile. */
	peakMainThreadHeapBytes?: number;
	planningWorkMs?: number;
	wallMs: number;
	outcome: PlanningOutcome['kind'];
	lastCompletedStage?: string;
	placements: number;
	deterministic: 'baseline' | 'yes' | 'no';
}

const balance = (
	id: string,
	type: 'cp' | 'rtt' | 'other',
	quantity: number,
	expiryDate = '2024-12-31',
) => ({
	id,
	type,
	quantity,
	availableFrom: '2024-01-01',
	expiryDate,
});

export function benchmarkScenarios(): BenchmarkScenario[] {
	const horizon = { start: '2024-01-01', end: '2025-01-01' };
	const presets = createOptimizationPresetRules();
	const scenarios: BenchmarkScenario[] = [
		{
			name: 'representative year',
			request: {
				horizon,
				balances: [balance('cp', 'cp', 20), balance('rtt', 'rtt', 10)],
				rules: [],
			},
		},
		{
			name: 'typed obligations',
			request: {
				horizon,
				balances: [balance('cp', 'cp', 5), balance('rtt', 'rtt', 5)],
				rules: [
					{
						id: 'cp-window',
						type: 'time_window',
						force: 'obligation',
						startDate: '2024-04-08',
						endDate: '2024-04-12',
						placeDays: 2,
						placeType: 'cp',
					},
					{
						id: 'rtt-window',
						type: 'time_window',
						force: 'obligation',
						startDate: '2024-04-08',
						endDate: '2024-04-12',
						placeDays: 2,
						placeType: 'rtt',
					},
				],
			},
		},
		{
			name: 'chained earned entitlements',
			request: {
				horizon: { start: '2024-04-25', end: '2024-05-15' },
				balances: [balance('cp', 'cp', 2), balance('rtt', 'rtt', 0, '2026-01-01')],
				rules: [
					{
						id: 'wednesday-holidays',
						type: 'if_then',
						force: 'obligation',
						dayOfWeek: 3,
						placeDays: 1,
						placeType: 'cp',
						placement: 'after',
						rewardDays: 1,
						rewardType: 'rtt',
					},
					{
						id: 'thursday-consumer',
						type: 'if_then',
						force: 'obligation',
						dayOfWeek: 4,
						placeDays: 1,
						placeType: 'rtt',
						placement: 'before',
						rewardDays: 0,
					},
				],
			},
		},
		{
			name: 'deliberate conflict',
			request: {
				horizon,
				balances: [balance('cp', 'cp', 1)],
				rules: [
					{
						id: 'rtt-required',
						type: 'time_window',
						force: 'obligation',
						startDate: '2024-04-01',
						endDate: '2024-04-05',
						placeDays: 1,
						placeType: 'rtt',
					},
				],
			},
		},
		{
			name: 'loaded year',
			request: {
				horizon,
				balances: Array.from({ length: 6 }, (_, index) =>
					balance(`cp-${index}`, index % 2 ? 'rtt' : 'cp', 12),
				),
				rules: Array.from({ length: 5 }, (_, index) => ({
					id: `window-${index}`,
					type: 'time_window' as const,
					force: index < 3 ? ('obligation' as const) : ('preference' as const),
					startDate: `2024-${String(index + 1).padStart(2, '0')}-01`,
					endDate: `2024-${String(index + 1).padStart(2, '0')}-28`,
					placeDays: 3,
					placeType: index % 2 ? ('rtt' as const) : ('any' as const),
				})),
			},
		},
		...[2, 3].map(
			(years): BenchmarkScenario => ({
				name: years === 2 ? 'two years' : 'three years',
				request: {
					horizon: { start: '2024-01-01', end: `${2024 + years}-01-01` },
					balances: Array.from({ length: years }, (_, index) => 2024 + index).flatMap(
						(year) =>
							(['cp', 'rtt'] as const).map((type) => ({
								id: `${type}-${year}`,
								type,
								quantity: type === 'cp' ? 20 : 10,
								availableFrom: `${year}-01-01`,
								expiryDate: `${year}-12-31`,
							})),
					),
					rules: [],
				},
			}),
		),
	];
	return scenarios.map((scenario) => ({
		...scenario,
		request: {
			...scenario.request,
			rules: [presets[0], ...scenario.request.rules, ...presets.slice(1)],
		},
	}));
}

function resultSummary(outcome: PlanningOutcome): {
	outcome: PlanningOutcome['kind'];
	lastCompletedStage?: string;
	placements: number;
} {
	if (outcome.kind === 'Plan')
		return {
			outcome: outcome.kind,
			lastCompletedStage: outcome.lastCompletedStage,
			placements: outcome.placements.length,
		};
	if (outcome.kind === 'PlanningIncomplete')
		return {
			outcome: outcome.kind,
			lastCompletedStage: outcome.lastCompletedStage,
			placements: outcome.incumbent?.placements.length ?? 0,
		};
	return { outcome: outcome.kind, placements: 0 };
}

export async function runLeavePlannerBenchmark(
	repetitions = 1,
	onScenario?: (measurements: BenchmarkMeasurement[]) => void,
): Promise<BenchmarkMeasurement[]> {
	const scenarios = benchmarkScenarios();
	for (const scenario of scenarios) await warmUpScenario(scenario);

	const measurements: BenchmarkMeasurement[] = [];
	for (const scenario of scenarios) {
		const scenarioMeasurements: BenchmarkMeasurement[] = [];
		let baseline: string | undefined;
		for (let repetition = 1; repetition <= repetitions; repetition++) {
			const result = await measureRepetition(scenario, repetition, baseline);
			scenarioMeasurements.push(result.measurement);
			baseline ??= result.stable;
		}
		measurements.push(...scenarioMeasurements);
		onScenario?.(scenarioMeasurements);
	}
	return measurements;
}

async function warmUpScenario(scenario: BenchmarkScenario): Promise<void> {
	const calculation = createPlanningCalculation(structuredClone(scenario.request));
	await calculation.outcome;
	await calculation.explanation;
	await calculation.timings;
}

async function measureRepetition(
	scenario: BenchmarkScenario,
	repetition: number,
	baseline: string | undefined,
): Promise<{ measurement: BenchmarkMeasurement; stable: string }> {
	const wallStart = performance.now();
	let lastTick = wallStart;
	let maxUiDelayMs = 0;
	let peakMainThreadHeapBytes: number | undefined;
	const measureDelay = () => {
		const current = performance.now();
		maxUiDelayMs = Math.max(maxUiDelayMs, current - lastTick - 50);
		lastTick = current;
		const heap = (performance as Performance & { memory?: { usedJSHeapSize: number } }).memory
			?.usedJSHeapSize;
		if (heap !== undefined)
			peakMainThreadHeapBytes = Math.max(peakMainThreadHeapBytes ?? 0, heap);
	};
	measureDelay();
	const heartbeat = setInterval(measureDelay, 50);
	try {
		const preparationStart = performance.now();
		const request = structuredClone(scenario.request);
		const preparationMs = performance.now() - preparationStart;
		const calculationStart = performance.now();
		const calculation = createPlanningCalculation(request);
		const outcome = await calculation.outcome;
		const planningMs = performance.now() - calculationStart;
		const timings = await calculation.timings;
		const summary = resultSummary(outcome);
		const stable = JSON.stringify(outcome);
		const deterministic =
			baseline === undefined ? 'baseline' : stable === baseline ? 'yes' : 'no';
		const wallMs = performance.now() - wallStart;
		measureDelay();
		return {
			measurement: {
				scenario: scenario.name,
				repetition,
				preparationMs,
				planningMs,
				maxUiDelayMs,
				peakMainThreadHeapBytes,
				...timings,
				wallMs,
				...summary,
				deterministic,
			},
			stable,
		};
	} finally {
		clearInterval(heartbeat);
	}
}

export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
	const header =
		'scenario\trepetition\tpreparation_ms\tplanning_ms\twall_ms\toutcome\tlast_stage\tplacements\tdeterministic\tmax_ui_delay_ms\tplanning_work_ms\tpeak_main_thread_heap_bytes';
	return [
		header,
		...measurements.map((item) =>
			[
				item.scenario,
				item.repetition,
				item.preparationMs.toFixed(1),
				item.planningMs.toFixed(1),
				item.wallMs.toFixed(1),
				item.outcome,
				item.lastCompletedStage ?? '',
				item.placements,
				item.deterministic,
				item.maxUiDelayMs.toFixed(1),
				item.planningWorkMs?.toFixed(1) ?? '',
				item.peakMainThreadHeapBytes ?? '',
			].join('\t'),
		),
	].join('\n');
}

// About 1.5x the development machine's ratios to the calibration run after #302 (#286).
export const BENCHMARK_BUDGET_MULTIPLIERS: Record<string, number> = {
	'representative year': 0.15,
	'typed obligations': 0.1,
	'chained earned entitlements': 0.05,
	'deliberate conflict': 0.05,
	'loaded year': 2.5,
	'two years': 0.5,
	'three years': 1.5,
};

/** One message per scenario whose mean planning time exceeds multiplier x calibration, or that has no multiplier. */
export function budgetViolations(
	measurements: BenchmarkMeasurement[],
	calibrationMs: number,
	multipliers: Record<string, number> = BENCHMARK_BUDGET_MULTIPLIERS,
): string[] {
	const byScenario = new Map<string, number[]>();
	for (const { scenario, planningMs } of measurements)
		byScenario.set(scenario, [...(byScenario.get(scenario) ?? []), planningMs]);
	const violations: string[] = [];
	for (const [scenario, times] of byScenario) {
		const mean = average(times);
		const multiplier = multipliers[scenario];
		if (multiplier === undefined) {
			violations.push(`${scenario}: no budget multiplier defined`);
			continue;
		}
		const budget = multiplier * calibrationMs;
		if (mean > budget)
			violations.push(
				`${scenario}: measured ${mean.toFixed(1)} ms > budget ${budget.toFixed(1)} ms (multiplier ${multiplier} x calibration ${calibrationMs.toFixed(1)} ms)`,
			);
	}
	return violations;
}

function average(values: number[]): number {
	return values.reduce((sum, value) => sum + value, 0) / values.length;
}

// Fixed widths keep rows printed as each scenario finishes aligned with the header.
const SUMMARY_COLUMNS: [title: string, width: number, alignRight: boolean][] = [
	['Scenario', 27, false],
	['Planning (runs)', 17, true],
	['Mean', 9, true],
	['Budget', 9, true],
	['Ratio', 6, true],
	['Limit', 6, true],
	['Outcome', 16, false],
	['Placements', 10, true],
	['Deterministic', 13, false],
	['Status', 9, false],
];

function summaryLine(cells: string[]): string {
	return `| ${cells
		.map((cell, index) => {
			const [, width, alignRight] = SUMMARY_COLUMNS[index];
			return alignRight ? cell.padStart(width) : cell.padEnd(width);
		})
		.join(' | ')} |`;
}

/** Markdown table header for {@link benchmarkSummaryRow}. */
export function benchmarkSummaryHeader(): string {
	return [
		summaryLine(SUMMARY_COLUMNS.map(([title]) => title)),
		`|${SUMMARY_COLUMNS.map(([, width, alignRight]) => (alignRight ? `${'-'.repeat(width + 1)}:` : `:${'-'.repeat(width + 1)}`)).join('|')}|`,
	].join('\n');
}

/** One aligned table row for one scenario's measured repetitions, judged like {@link budgetViolations}. */
export function benchmarkSummaryRow(
	scenarioMeasurements: BenchmarkMeasurement[],
	calibrationMs: number,
	multipliers: Record<string, number> = BENCHMARK_BUDGET_MULTIPLIERS,
): string {
	const [first] = scenarioMeasurements;
	const seconds = (ms: number) => (ms / 1000).toFixed(2);
	const mean = average(scenarioMeasurements.map(({ planningMs }) => planningMs));
	const multiplier = multipliers[first.scenario];
	const budget = multiplier === undefined ? undefined : multiplier * calibrationMs;
	return summaryLine([
		first.scenario,
		`${scenarioMeasurements.map(({ planningMs }) => seconds(planningMs)).join(' / ')} s`,
		`${seconds(mean)} s`,
		budget === undefined ? '-' : `${seconds(budget)} s`,
		(mean / calibrationMs).toFixed(2),
		multiplier === undefined ? '-' : String(multiplier),
		first.lastCompletedStage ? `${first.outcome} (${first.lastCompletedStage})` : first.outcome,
		String(first.placements),
		scenarioMeasurements.slice(1).every(({ deterministic }) => deterministic === 'yes')
			? 'yes'
			: 'no',
		budget === undefined ? 'no budget' : mean > budget ? 'OVER' : 'ok',
	]);
}