import { createPlanningCalculation, PlanningOutcome, PlanningRequest, } from '../services/leave-planner.service'; import { createOptimizationPresetRules } from '../services/rule-service'; export interface BenchmarkScenario { name: string; request: PlanningRequest; } export interface BenchmarkMeasurement { scenario: string; repetition: number; preparationMs: number; planningMs: number; maxUiDelayMs: number; /** Chromium's main-thread heap only; worker memory needs a separate browser profile. */ peakMainThreadHeapBytes?: number; planningWorkMs?: number; wallMs: number; outcome: PlanningOutcome['kind']; lastCompletedStage?: string; placements: number; deterministic: 'baseline' | 'yes' | 'no'; } const balance = ( id: string, type: 'cp' | 'rtt' | 'other', quantity: number, expiryDate = '2024-12-31', ) => ({ id, type, quantity, availableFrom: '2024-01-01', expiryDate, }); export function benchmarkScenarios(): BenchmarkScenario[] { const horizon = { start: '2024-01-01', end: '2025-01-01' }; const presets = createOptimizationPresetRules(); const scenarios: BenchmarkScenario[] = [ { name: 'representative year', request: { horizon, balances: [balance('cp', 'cp', 20), balance('rtt', 'rtt', 10)], rules: [], }, }, { name: 'typed obligations', request: { horizon, balances: [balance('cp', 'cp', 5), balance('rtt', 'rtt', 5)], rules: [ { id: 'cp-window', type: 'time_window', force: 'obligation', startDate: '2024-04-08', endDate: '2024-04-12', placeDays: 2, placeType: 'cp', }, { id: 'rtt-window', type: 'time_window', force: 'obligation', startDate: '2024-04-08', endDate: '2024-04-12', placeDays: 2, placeType: 'rtt', }, ], }, }, { name: 'chained earned entitlements', request: { horizon: { start: '2024-04-25', end: '2024-05-15' }, balances: [balance('cp', 'cp', 2), balance('rtt', 'rtt', 0, '2026-01-01')], rules: [ { id: 'wednesday-holidays', type: 'if_then', force: 'obligation', dayOfWeek: 3, placeDays: 1, placeType: 'cp', placement: 'after', rewardDays: 1, rewardType: 'rtt', }, { id: 'thursday-consumer', type: 'if_then', force: 'obligation', dayOfWeek: 4, placeDays: 1, placeType: 'rtt', placement: 'before', rewardDays: 0, }, ], }, }, { name: 'deliberate conflict', request: { horizon, balances: [balance('cp', 'cp', 1)], rules: [ { id: 'rtt-required', type: 'time_window', force: 'obligation', startDate: '2024-04-01', endDate: '2024-04-05', placeDays: 1, placeType: 'rtt', }, ], }, }, { name: 'loaded year', request: { horizon, balances: Array.from({ length: 6 }, (_, index) => balance(`cp-${index}`, index % 2 ? 'rtt' : 'cp', 12), ), rules: Array.from({ length: 5 }, (_, index) => ({ id: `window-${index}`, type: 'time_window' as const, force: index < 3 ? ('obligation' as const) : ('preference' as const), startDate: `2024-${String(index + 1).padStart(2, '0')}-01`, endDate: `2024-${String(index + 1).padStart(2, '0')}-28`, placeDays: 3, placeType: index % 2 ? ('rtt' as const) : ('any' as const), })), }, }, ...[2, 3].map( (years): BenchmarkScenario => ({ name: years === 2 ? 'two years' : 'three years', request: { horizon: { start: '2024-01-01', end: `${2024 + years}-01-01` }, balances: Array.from({ length: years }, (_, index) => 2024 + index).flatMap( (year) => (['cp', 'rtt'] as const).map((type) => ({ id: `${type}-${year}`, type, quantity: type === 'cp' ? 20 : 10, availableFrom: `${year}-01-01`, expiryDate: `${year}-12-31`, })), ), rules: [], }, }), ), ]; return scenarios.map((scenario) => ({ ...scenario, request: { ...scenario.request, rules: [presets[0], ...scenario.request.rules, ...presets.slice(1)], }, })); } function resultSummary(outcome: PlanningOutcome): { outcome: PlanningOutcome['kind']; lastCompletedStage?: string; placements: number; } { if (outcome.kind === 'Plan') return { outcome: outcome.kind, lastCompletedStage: outcome.lastCompletedStage, placements: outcome.placements.length, }; if (outcome.kind === 'PlanningIncomplete') return { outcome: outcome.kind, lastCompletedStage: outcome.lastCompletedStage, placements: outcome.incumbent?.placements.length ?? 0, }; return { outcome: outcome.kind, placements: 0 }; } export async function runLeavePlannerBenchmark( repetitions = 1, onScenario?: (measurements: BenchmarkMeasurement[]) => void, ): Promise { const scenarios = benchmarkScenarios(); for (const scenario of scenarios) await warmUpScenario(scenario); const measurements: BenchmarkMeasurement[] = []; for (const scenario of scenarios) { const scenarioMeasurements: BenchmarkMeasurement[] = []; let baseline: string | undefined; for (let repetition = 1; repetition <= repetitions; repetition++) { const result = await measureRepetition(scenario, repetition, baseline); scenarioMeasurements.push(result.measurement); baseline ??= result.stable; } measurements.push(...scenarioMeasurements); onScenario?.(scenarioMeasurements); } return measurements; } async function warmUpScenario(scenario: BenchmarkScenario): Promise { const calculation = createPlanningCalculation(structuredClone(scenario.request)); await calculation.outcome; await calculation.explanation; await calculation.timings; } async function measureRepetition( scenario: BenchmarkScenario, repetition: number, baseline: string | undefined, ): Promise<{ measurement: BenchmarkMeasurement; stable: string }> { const wallStart = performance.now(); let lastTick = wallStart; let maxUiDelayMs = 0; let peakMainThreadHeapBytes: number | undefined; const measureDelay = () => { const current = performance.now(); maxUiDelayMs = Math.max(maxUiDelayMs, current - lastTick - 50); lastTick = current; const heap = (performance as Performance & { memory?: { usedJSHeapSize: number } }).memory ?.usedJSHeapSize; if (heap !== undefined) peakMainThreadHeapBytes = Math.max(peakMainThreadHeapBytes ?? 0, heap); }; measureDelay(); const heartbeat = setInterval(measureDelay, 50); try { const preparationStart = performance.now(); const request = structuredClone(scenario.request); const preparationMs = performance.now() - preparationStart; const calculationStart = performance.now(); const calculation = createPlanningCalculation(request); const outcome = await calculation.outcome; const planningMs = performance.now() - calculationStart; const timings = await calculation.timings; const summary = resultSummary(outcome); const stable = JSON.stringify(outcome); const deterministic = baseline === undefined ? 'baseline' : stable === baseline ? 'yes' : 'no'; const wallMs = performance.now() - wallStart; measureDelay(); return { measurement: { scenario: scenario.name, repetition, preparationMs, planningMs, maxUiDelayMs, peakMainThreadHeapBytes, ...timings, wallMs, ...summary, deterministic, }, stable, }; } finally { clearInterval(heartbeat); } } export function benchmarkReport(measurements: BenchmarkMeasurement[]): string { const header = 'scenario\trepetition\tpreparation_ms\tplanning_ms\twall_ms\toutcome\tlast_stage\tplacements\tdeterministic\tmax_ui_delay_ms\tplanning_work_ms\tpeak_main_thread_heap_bytes'; return [ header, ...measurements.map((item) => [ item.scenario, item.repetition, item.preparationMs.toFixed(1), item.planningMs.toFixed(1), item.wallMs.toFixed(1), item.outcome, item.lastCompletedStage ?? '', item.placements, item.deterministic, item.maxUiDelayMs.toFixed(1), item.planningWorkMs?.toFixed(1) ?? '', item.peakMainThreadHeapBytes ?? '', ].join('\t'), ), ].join('\n'); } // About 1.5x the development machine's ratios to the calibration run after #302 (#286). export const BENCHMARK_BUDGET_MULTIPLIERS: Record = { 'representative year': 0.15, 'typed obligations': 0.1, 'chained earned entitlements': 0.05, 'deliberate conflict': 0.05, 'loaded year': 2.5, 'two years': 0.5, 'three years': 1.5, }; /** One message per scenario whose mean planning time exceeds multiplier x calibration, or that has no multiplier. */ export function budgetViolations( measurements: BenchmarkMeasurement[], calibrationMs: number, multipliers: Record = BENCHMARK_BUDGET_MULTIPLIERS, ): string[] { const byScenario = new Map(); for (const { scenario, planningMs } of measurements) byScenario.set(scenario, [...(byScenario.get(scenario) ?? []), planningMs]); const violations: string[] = []; for (const [scenario, times] of byScenario) { const mean = average(times); const multiplier = multipliers[scenario]; if (multiplier === undefined) { violations.push(`${scenario}: no budget multiplier defined`); continue; } const budget = multiplier * calibrationMs; if (mean > budget) violations.push( `${scenario}: measured ${mean.toFixed(1)} ms > budget ${budget.toFixed(1)} ms (multiplier ${multiplier} x calibration ${calibrationMs.toFixed(1)} ms)`, ); } return violations; } function average(values: number[]): number { return values.reduce((sum, value) => sum + value, 0) / values.length; } // Fixed widths keep rows printed as each scenario finishes aligned with the header. const SUMMARY_COLUMNS: [title: string, width: number, alignRight: boolean][] = [ ['Scenario', 27, false], ['Planning (runs)', 17, true], ['Mean', 9, true], ['Budget', 9, true], ['Ratio', 6, true], ['Limit', 6, true], ['Outcome', 16, false], ['Placements', 10, true], ['Deterministic', 13, false], ['Status', 9, false], ]; function summaryLine(cells: string[]): string { return `| ${cells .map((cell, index) => { const [, width, alignRight] = SUMMARY_COLUMNS[index]; return alignRight ? cell.padStart(width) : cell.padEnd(width); }) .join(' | ')} |`; } /** Markdown table header for {@link benchmarkSummaryRow}. */ export function benchmarkSummaryHeader(): string { return [ summaryLine(SUMMARY_COLUMNS.map(([title]) => title)), `|${SUMMARY_COLUMNS.map(([, width, alignRight]) => (alignRight ? `${'-'.repeat(width + 1)}:` : `:${'-'.repeat(width + 1)}`)).join('|')}|`, ].join('\n'); } /** One aligned table row for one scenario's measured repetitions, judged like {@link budgetViolations}. */ export function benchmarkSummaryRow( scenarioMeasurements: BenchmarkMeasurement[], calibrationMs: number, multipliers: Record = BENCHMARK_BUDGET_MULTIPLIERS, ): string { const [first] = scenarioMeasurements; const seconds = (ms: number) => (ms / 1000).toFixed(2); const mean = average(scenarioMeasurements.map(({ planningMs }) => planningMs)); const multiplier = multipliers[first.scenario]; const budget = multiplier === undefined ? undefined : multiplier * calibrationMs; return summaryLine([ first.scenario, `${scenarioMeasurements.map(({ planningMs }) => seconds(planningMs)).join(' / ')} s`, `${seconds(mean)} s`, budget === undefined ? '-' : `${seconds(budget)} s`, (mean / calibrationMs).toFixed(2), multiplier === undefined ? '-' : String(multiplier), first.lastCompletedStage ? `${first.outcome} (${first.lastCompletedStage})` : first.outcome, String(first.placements), scenarioMeasurements.slice(1).every(({ deterministic }) => deterministic === 'yes') ? 'yes' : 'no', budget === undefined ? 'no budget' : mean > budget ? 'OVER' : 'ok', ]); }