vac-optimizer/src/app/benchmark/leave-planner-benchmark.ts

import {
	createPlanningCalculation,
	PlanningOutcome,
	PlanningRequest,
	PlanningTrace,
} from '../services/leave-planner.service';

export interface BenchmarkScenario {
	name: string;
	request: PlanningRequest;
}

export interface BenchmarkMeasurement {
	scenario: string;
	repetition: number;
	preparationMs: number;
	planningMs: number;
	traceDeliveryMs: number;
	wallMs: number;
	outcome: PlanningOutcome['kind'];
	lastCompletedStage?: string;
	placements: number;
	deterministic: 'baseline' | 'yes' | 'no';
	traceStatus: PlanningTrace['status'] | 'not-produced';
	traceCandidates: number;
}

const balance = (
	id: string,
	type: 'cp' | 'rtt' | 'other',
	quantity: number,
	expiryDate = '2024-12-31',
) => ({
	id,
	type,
	quantity,
	availableFrom: '2024-01-01',
	expiryDate,
});

export function benchmarkScenarios(): BenchmarkScenario[] {
	const horizon = { start: '2024-01-01', end: '2025-01-01' };
	return [
		{
			name: 'representative year',
			request: {
				horizon,
				balances: [balance('cp', 'cp', 20), balance('rtt', 'rtt', 10)],
				rules: [],
			},
		},
		{
			name: 'typed obligations',
			request: {
				horizon,
				balances: [balance('cp', 'cp', 5), balance('rtt', 'rtt', 5)],
				rules: [
					{
						id: 'cp-window',
						type: 'time_window',
						force: 'obligation',
						startDate: '2024-04-08',
						endDate: '2024-04-12',
						placeDays: 2,
						placeType: 'cp',
					},
					{
						id: 'rtt-window',
						type: 'time_window',
						force: 'obligation',
						startDate: '2024-04-08',
						endDate: '2024-04-12',
						placeDays: 2,
						placeType: 'rtt',
					},
				],
			},
		},
		{
			name: 'chained earned entitlements',
			request: {
				horizon: { start: '2024-04-25', end: '2024-05-15' },
				balances: [balance('cp', 'cp', 2), balance('rtt', 'rtt', 0, '2026-01-01')],
				rules: [
					{
						id: 'wednesday-holidays',
						type: 'if_then',
						force: 'obligation',
						dayOfWeek: 3,
						placeDays: 1,
						placeType: 'cp',
						placement: 'after',
						rewardDays: 1,
						rewardType: 'rtt',
					},
					{
						id: 'thursday-consumer',
						type: 'if_then',
						force: 'obligation',
						dayOfWeek: 4,
						placeDays: 1,
						placeType: 'rtt',
						placement: 'before',
						rewardDays: 0,
					},
				],
			},
		},
		{
			name: 'deliberate conflict',
			request: {
				horizon,
				balances: [balance('cp', 'cp', 1)],
				rules: [
					{
						id: 'rtt-required',
						type: 'time_window',
						force: 'obligation',
						startDate: '2024-04-01',
						endDate: '2024-04-05',
						placeDays: 1,
						placeType: 'rtt',
					},
				],
			},
		},
		{
			name: 'loaded year',
			request: {
				horizon,
				balances: Array.from({ length: 12 }, (_, index) =>
					balance(`cp-${index}`, index % 2 ? 'rtt' : 'cp', 12),
				),
				rules: Array.from({ length: 8 }, (_, index) => ({
					id: `window-${index}`,
					type: 'time_window' as const,
					force: index < 3 ? ('obligation' as const) : ('preference' as const),
					startDate: `2024-${String(index + 1).padStart(2, '0')}-01`,
					endDate: `2024-${String(index + 1).padStart(2, '0')}-28`,
					placeDays: 3,
					placeType: index % 2 ? ('rtt' as const) : ('any' as const),
				})),
			},
		},
	];
}

function resultSummary(outcome: PlanningOutcome): {
	outcome: PlanningOutcome['kind'];
	lastCompletedStage?: string;
	placements: number;
} {
	if (outcome.kind === 'Plan')
		return {
			outcome: outcome.kind,
			lastCompletedStage: outcome.lastCompletedStage,
			placements: outcome.placements.length,
		};
	if (outcome.kind === 'PlanningIncomplete')
		return {
			outcome: outcome.kind,
			lastCompletedStage: outcome.lastCompletedStage,
			placements: outcome.incumbent?.placements.length ?? 0,
		};
	return { outcome: outcome.kind, placements: 0 };
}

export async function runLeavePlannerBenchmark(repetitions = 1): Promise<BenchmarkMeasurement[]> {
	const scenarios = benchmarkScenarios();
	for (const scenario of scenarios) {
		const calculation = createPlanningCalculation(structuredClone(scenario.request));
		await calculation.outcome;
		await calculation.trace;
	}

	const measurements: BenchmarkMeasurement[] = [];
	for (const scenario of scenarios) {
		let baseline: string | undefined;
		for (let repetition = 1; repetition <= repetitions; repetition++) {
			const wallStart = performance.now();
			const preparationStart = performance.now();
			const request = structuredClone(scenario.request);
			const preparationMs = performance.now() - preparationStart;
			const calculationStart = performance.now();
			const calculation = createPlanningCalculation(request);
			const outcome = await calculation.outcome;
			const planningMs = performance.now() - calculationStart;
			const traceStart = performance.now();
			const trace = await calculation.trace;
			const traceDeliveryMs = performance.now() - traceStart;
			const summary = resultSummary(outcome);
			const stable = JSON.stringify(outcome);
			const deterministic =
				baseline === undefined ? 'baseline' : stable === baseline ? 'yes' : 'no';
			const traceCandidates =
				trace?.phases.reduce(
					(total, phase) =>
						total +
						phase.decisions.reduce(
							(decisionTotal, decision) => decisionTotal + decision.candidates.length,
							0,
						),
					0,
				) ?? 0;
			const wallMs = performance.now() - wallStart;
			measurements.push({
				scenario: scenario.name,
				repetition,
				preparationMs,
				planningMs,
				traceDeliveryMs,
				wallMs,
				...summary,
				deterministic,
				traceStatus: trace?.status ?? 'not-produced',
				traceCandidates,
			});
			baseline ??= stable;
		}
	}
	return measurements;
}

export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
	const header =
		'scenario\trepetition\tpreparation_ms\tplanning_ms\ttrace_delivery_ms\twall_ms\toutcome\tlast_stage\tplacements\tdeterministic\ttrace_status\ttrace_candidates';
	return [
		header,
		...measurements.map((item) =>
			[
				item.scenario,
				item.repetition,
				item.preparationMs.toFixed(1),
				item.planningMs.toFixed(1),
				item.traceDeliveryMs.toFixed(1),
				item.wallMs.toFixed(1),
				item.outcome,
				item.lastCompletedStage ?? '',
				item.placements,
				item.deterministic,
				item.traceStatus,
				item.traceCandidates,
			].join('\t'),
		),
	].join('\n');
}