From f1f08356dfd0278129ecf42979be466c85e10d0b Mon Sep 17 00:00:00 2001 From: Nessar Date: Fri, 25 Sep 2026 14:07:00 +0200 Subject: [PATCH] feat(benchmark): enhance benchmark documentation and add emulated run details fix(benchmark): update deterministic checks and improve measurement accuracy fix(service): refine candidate recording and enhance balance assignment logic test(service): add test for balance recording with preference assignment --- docs/benchmark/README.md | 53 ++++- .../benchmark/leave-planner-benchmark.spec.ts | 53 ++++- src/app/benchmark/leave-planner-benchmark.ts | 51 +++-- src/app/services/leave-planner.core.ts | 194 ++++++++++++------ .../services/leave-planner.service.spec.ts | 47 +++++ 5 files changed, 300 insertions(+), 98 deletions(-) diff --git a/docs/benchmark/README.md b/docs/benchmark/README.md index c68b6c9..c0d02ea 100644 --- a/docs/benchmark/README.md +++ b/docs/benchmark/README.md @@ -16,11 +16,14 @@ stage and incumbent are retained. battery level. - Use a cool phone, unplugged, with other applications closed. Record whether the device became warm or throttled during the run. -- Run the benchmark three times after one warm-up run. Do not interact with - the page while a run is active. -- Representative and loaded scenarios should normally complete below five - seconds; every scenario must return within ten seconds. These are release - measurements, not CI timing assertions. +- Tap **Run benchmark** once. It first runs every scenario as an unreported + warm-up, then records three measured repetitions per scenario. Do not interact + with the page while a run is active. The first measured result is the + determinism baseline; later repetitions report whether they match it. +- The original target was below five seconds for representative and loaded + scenarios, with a ten-second planning ceiling for every scenario. The owner + accepted six to seven seconds for the loaded scenario on 2026-09-25. These + are release measurements, not CI timing assertions. - Keep the copied report unchanged, including any incomplete outcomes or hardware limitations. No mobile-device measurement is claimed by the source tree until a human records it here. @@ -29,3 +32,43 @@ The browser page uses the same `createPlanningCalculation(request)` context as the calendar and adds no solver or backend dependency. The ten-second release ceiling applies to planning latency; trace delivery is reported separately and cannot replace a valid planning outcome. + +## Hardware status + +No reference-phone run has been recorded. The project owner elected to close +the release tickets on 2026-09-25 using the emulated run below, accepting its +six-to-seven-second loaded planning time and the observed post-result trace UI +pause. This is an explicit release exception, not a reference-phone measurement +or proof of physical-device responsiveness. + +## Emulated browser run — 2026-09-25 + +Chromium 149.0.7827.55 on an AMD Ryzen 7 7800X3D (x86_64), headless at +412×915 pixels, device scale 2.625, touch enabled, and 4× CPU throttling. +One full warm-up was excluded, followed by three measured repetitions. All +five scenarios returned within ten seconds of planning time and complete +results matched across repetitions. The loaded year took 5.74–6.17 seconds, +above the five-second target. A 100 ms main-thread timer observed a maximum +extra delay of 5,950.9 ms. Worker event timestamps place the four loaded-year +pauses of 4.0–6.0 seconds after their planning results, around delivery of +full trace artifacts. This desktop emulation does not establish reference-phone +performance or responsiveness. + +```tsv +scenario repetition preparation_ms planning_ms trace_delivery_ms wall_ms outcome last_stage placements deterministic trace_status trace_candidates +representative year 1 0.0 443.7 624.7 1069.1 Plan dates 30 baseline ready 29451 +representative year 2 0.1 490.0 609.3 1099.4 Plan dates 30 yes ready 29451 +representative year 3 0.1 465.5 614.2 1079.8 Plan dates 30 yes ready 29451 +typed obligations 1 0.0 157.0 261.2 418.2 Plan dates 10 baseline ready 13103 +typed obligations 2 0.0 162.6 255.6 418.2 Plan dates 10 yes ready 13103 +typed obligations 3 0.0 163.4 258.2 421.6 Plan dates 10 yes ready 13103 +chained earned entitlements 1 0.0 45.3 1.0 47.0 Plan dates 3 baseline ready 27 +chained earned entitlements 2 0.1 44.5 0.9 45.5 Plan dates 3 yes ready 27 +chained earned entitlements 3 0.1 44.4 0.8 45.4 Plan dates 3 yes ready 27 +deliberate conflict 1 0.0 92.8 0.9 93.8 PlanningConflict 0 baseline ready 366 +deliberate conflict 2 0.0 44.7 0.2 44.9 PlanningConflict 0 yes ready 366 +deliberate conflict 3 0.0 43.5 0.6 44.1 PlanningConflict 0 yes ready 366 +loaded year 1 0.3 5901.5 5677.7 11581.0 Plan dates 144 baseline ready 865117 +loaded year 2 0.8 6167.5 7582.4 13752.4 Plan dates 144 yes ready 865117 +loaded year 3 0.6 5738.3 6219.5 11962.5 Plan dates 144 yes ready 865117 +``` diff --git a/src/app/benchmark/leave-planner-benchmark.spec.ts b/src/app/benchmark/leave-planner-benchmark.spec.ts index f525709..3f8bcf8 100644 --- a/src/app/benchmark/leave-planner-benchmark.spec.ts +++ b/src/app/benchmark/leave-planner-benchmark.spec.ts @@ -1,4 +1,5 @@ import { vi } from 'vitest'; +import { createPlanningCalculation } from '../services/leave-planner.service'; import { benchmarkReport, benchmarkScenarios, @@ -9,6 +10,20 @@ describe('leave planner benchmark protocol', () => { it('keeps five deterministic scenarios and exercises a later earned consumer', async () => { const scenarios = benchmarkScenarios(); expect(scenarios).toHaveLength(5); + const typedObligations = scenarios.find( + (scenario) => scenario.name === 'typed obligations', + ); + const cpRule = typedObligations?.request.rules?.find((rule) => rule.id === 'cp-window'); + const rttRule = typedObligations?.request.rules?.find((rule) => rule.id === 'rtt-window'); + expect(cpRule).toMatchObject({ placeType: 'cp', placeDays: 2 }); + expect(rttRule).toMatchObject({ placeType: 'rtt', placeDays: 2 }); + expect(cpRule?.type).toBe('time_window'); + expect(rttRule?.type).toBe('time_window'); + if (cpRule && 'startDate' in cpRule && rttRule && 'startDate' in rttRule) + expect( + String(cpRule.startDate) <= String(rttRule.endDate) && + String(rttRule.startDate) <= String(cpRule.endDate), + ).toBe(true); const chain = scenarios.find((scenario) => scenario.name === 'chained earned entitlements'); expect(chain?.request.rules?.map((rule) => rule.id)).toEqual([ 'wednesday-holidays', @@ -18,24 +33,44 @@ describe('leave planner benchmark protocol', () => { const browserWindow = globalThis.window; vi.stubGlobal('window', undefined); try { - const measurements = await runLeavePlannerBenchmark(1); + const typedOutcome = await createPlanningCalculation(typedObligations!.request).outcome; + expect(typedOutcome.kind).toBe('Plan'); + if (typedOutcome.kind === 'Plan') { + expect( + typedOutcome.placements.filter((placement) => + placement.ruleIds.includes('cp-window'), + ), + ).toHaveLength(2); + expect( + typedOutcome.placements.filter((placement) => + placement.ruleIds.includes('rtt-window'), + ), + ).toHaveLength(2); + } + const measurements = await runLeavePlannerBenchmark(2); const chained = measurements.find( (measurement) => measurement.scenario === 'chained earned entitlements', ); - const loaded = measurements.find( + const loaded = measurements.filter( (measurement) => measurement.scenario === 'loaded year', ); expect(chained?.outcome).toBe('Plan'); expect(chained?.placements).toBeGreaterThanOrEqual(3); - expect(loaded?.outcome).toBe('Plan'); - expect(loaded?.deterministic).toBe(true); - expect(loaded?.planningMs).toBeGreaterThanOrEqual(0); - expect(loaded?.traceDeliveryMs).toBeGreaterThanOrEqual(0); - expect(loaded?.traceStatus).toBe('ready'); - expect(loaded?.traceCandidates).toBeGreaterThan(0); + expect(loaded).toHaveLength(2); + expect(loaded[0].outcome).toBe('Plan'); + expect(loaded[0].deterministic).toBe('baseline'); + expect(loaded[1].deterministic).toBe('yes'); + expect(loaded[0].planningMs).toBeGreaterThanOrEqual(0); + expect(loaded[0].traceDeliveryMs).toBeGreaterThanOrEqual(0); + expect(loaded[0].wallMs).toBeGreaterThanOrEqual( + loaded[0].preparationMs + loaded[0].planningMs + loaded[0].traceDeliveryMs, + ); + expect(loaded[0].traceStatus).toBe('ready'); + expect(loaded[0].traceCandidates).toBeGreaterThan(0); const report = benchmarkReport(measurements); - expect(report.split('\n')).toHaveLength(6); + expect(report.split('\n')).toHaveLength(11); expect(report).toContain('planning_ms\ttrace_delivery_ms'); + expect(report).toContain('\tbaseline\t'); } finally { vi.stubGlobal('window', browserWindow); } diff --git a/src/app/benchmark/leave-planner-benchmark.ts b/src/app/benchmark/leave-planner-benchmark.ts index b71b336..b1c60ab 100644 --- a/src/app/benchmark/leave-planner-benchmark.ts +++ b/src/app/benchmark/leave-planner-benchmark.ts @@ -20,7 +20,7 @@ export interface BenchmarkMeasurement { outcome: PlanningOutcome['kind']; lastCompletedStage?: string; placements: number; - deterministic: boolean; + deterministic: 'baseline' | 'yes' | 'no'; traceStatus: PlanningTrace['status'] | 'not-produced'; traceCandidates: number; } @@ -59,7 +59,7 @@ export function benchmarkScenarios(): BenchmarkScenario[] { id: 'cp-window', type: 'time_window', force: 'obligation', - startDate: '2024-04-01', + startDate: '2024-04-08', endDate: '2024-04-12', placeDays: 2, placeType: 'cp', @@ -68,8 +68,8 @@ export function benchmarkScenarios(): BenchmarkScenario[] { id: 'rtt-window', type: 'time_window', force: 'obligation', - startDate: '2024-05-01', - endDate: '2024-05-12', + startDate: '2024-04-08', + endDate: '2024-04-12', placeDays: 2, placeType: 'rtt', }, @@ -166,10 +166,18 @@ function resultSummary(outcome: PlanningOutcome): { } export async function runLeavePlannerBenchmark(repetitions = 1): Promise { + const scenarios = benchmarkScenarios(); + for (const scenario of scenarios) { + const calculation = createPlanningCalculation(structuredClone(scenario.request)); + await calculation.outcome; + await calculation.trace; + } + const measurements: BenchmarkMeasurement[] = []; - for (const scenario of benchmarkScenarios()) { - let previous: string | undefined; + for (const scenario of scenarios) { + let baseline: string | undefined; for (let repetition = 1; repetition <= repetitions; repetition++) { + const wallStart = performance.now(); const preparationStart = performance.now(); const request = structuredClone(scenario.request); const preparationMs = performance.now() - preparationStart; @@ -182,29 +190,32 @@ export async function runLeavePlannerBenchmark(repetitions = 1): Promise + total + + phase.decisions.reduce( + (decisionTotal, decision) => decisionTotal + decision.candidates.length, + 0, + ), + 0, + ) ?? 0; + const wallMs = performance.now() - wallStart; measurements.push({ scenario: scenario.name, repetition, preparationMs, planningMs, traceDeliveryMs, - wallMs: preparationMs + planningMs + traceDeliveryMs, + wallMs, ...summary, - deterministic: previous === undefined || previous === stable, + deterministic, traceStatus: trace?.status ?? 'not-produced', - traceCandidates: - trace?.phases.reduce( - (total, phase) => - total + - phase.decisions.reduce( - (decisionTotal, decision) => - decisionTotal + decision.candidates.length, - 0, - ), - 0, - ) ?? 0, + traceCandidates, }); - previous = stable; + baseline ??= stable; } } return measurements; diff --git a/src/app/services/leave-planner.core.ts b/src/app/services/leave-planner.core.ts index 1bb70a3..8b5c9ff 100644 --- a/src/app/services/leave-planner.core.ts +++ b/src/app/services/leave-planner.core.ts @@ -582,15 +582,19 @@ class TraceRecorder { return undefined; } - recordCandidate(candidate: PlanningTraceCandidate): void { - this.latestDecision().candidates.push({ - ...candidate, - dates: candidate.dates ? [...candidate.dates] : undefined, - reason: candidate.reason ? { ...candidate.reason } : undefined, - placement: candidate.placement - ? { ...candidate.placement, ruleIds: [...candidate.placement.ruleIds] } - : undefined, - }); + recordCandidate(candidate: PlanningTraceCandidate, copy = true): void { + this.latestDecision().candidates.push( + copy + ? { + ...candidate, + dates: candidate.dates ? [...candidate.dates] : undefined, + reason: candidate.reason ? { ...candidate.reason } : undefined, + placement: candidate.placement + ? { ...candidate.placement, ruleIds: [...candidate.placement.ruleIds] } + : undefined, + } + : candidate, + ); } recordDecision( @@ -891,7 +895,8 @@ function observeTrace(context: SearchContext, observe: (trace: TraceRecorder) => } function recordTraceCandidate(context: SearchContext, candidate: PlanningTraceCandidate): void { - observeTrace(context, (trace) => trace.recordCandidate(candidate)); + // These internal callsites pass fresh literals; retain the full snapshot copy for plan data. + observeTrace(context, (trace) => trace.recordCandidate(candidate, false)); } function recordTraceIneligible( @@ -1633,7 +1638,7 @@ function assignDate( ruleId: string, context: SearchContext, ): State | null { - return assignDateVariants(state, date, selector, ruleId, context)[0] ?? null; + return assignDateVariants(state, date, selector, ruleId, context, true)[0] ?? null; } function assignDateVariants( @@ -1642,6 +1647,7 @@ function assignDateVariants( selector: LeaveSelector, ruleId: string, context: SearchContext, + firstOnly = false, ): State[] { context.check(); const existing = state.placements.get(date); @@ -1724,6 +1730,40 @@ function assignDateVariants( 'No compatible leave balance can fund this candidate date.', ), }); + if (firstOnly && balances.length) { + const first = balances[0]; + const result = cloneState(state); + result.balances.find((item) => item.id === first.id)!.remaining--; + result.placements.set(date, { + date, + balanceId: first.id, + type: first.type, + ruleIds: ruleId ? new Set([ruleId]) : new Set(), + supporting: false, + }); + for (const [index, balance] of balances.entries()) { + recordTraceCandidate(context, { + status: index === 0 ? 'evaluated-rejected' : 'not-examined', + date, + ...(index === 0 + ? {} + : { + reason: reason( + 'first-compatible-balance', + 'This balance was not examined after the first compatible balance was selected.', + ), + }), + placement: { + date, + balanceId: balance.id, + type: balance.type, + ruleIds: ruleId ? [ruleId] : [], + supporting: false, + }, + }); + } + return [result]; + } return balances.map((balance) => { const result = cloneState(state); const target = result.balances.find((item) => item.id === balance.id)!; @@ -1806,23 +1846,40 @@ function assignDateToBalanceInPlace( date: string, balanceId: string, context: SearchContext, + ruleIds = new Set(), ): boolean { context.check(); - if (state.placements.has(date)) return false; + if (state.placements.has(date)) { + recordTraceCandidate(context, { + status: 'ineligible', + date, + reason: reason('occupied', 'Another placement already uses this date.'), + }); + return false; + } const balance = state.balances.find((item) => item.id === balanceId); if ( !balance || balance.remaining <= 0 || compareDate(balance.availableFrom, date) > 0 || compareDate(date, balance.expiryDate) > 0 - ) + ) { + recordTraceCandidate(context, { + status: 'evaluated-rejected', + date, + reason: reason( + 'balance-unavailable', + 'The selected leave balance cannot fund this date.', + ), + }); return false; + } balance.remaining--; state.placements.set(date, { date, balanceId, type: balance.type, - ruleIds: new Set(), + ruleIds: new Set(ruleIds), supporting: false, }); recordTraceCandidate(context, { @@ -1830,7 +1887,7 @@ function assignDateToBalanceInPlace( date, placement: { ...state.placements.get(date)!, - ruleIds: [], + ruleIds: [...state.placements.get(date)!.ruleIds], }, }); return true; @@ -2637,60 +2694,69 @@ function optimizeMovablePlacements( ); continue; } - const candidate = assignDateToBalance( - without, - date, - current.balanceId, - context, - current.ruleIds, - ); - if (!candidate) continue; - const feasibleCandidate = finalizeEarnedState(candidate, context); - if (!feasibleCandidate) continue; - feasibleCandidate.placements.get(date)!.supporting = current.supporting; - const metrics = qualityMetrics(feasibleCandidate, context, preferences); - recordTraceEvaluation( - context, - result, - feasibleCandidate, - preferences, - baseline, - metrics, - [date], - current.date, - ); - if (metrics.unplacedAtRisk !== baseline.unplacedAtRisk) continue; - if (metrics.holidayWeekValue < baseline.holidayWeekValue) continue; if ( - metrics.preferenceSatisfaction.length !== baseline.preferenceSatisfaction.length || - metrics.preferenceSatisfaction.some( - (value, index) => value !== baseline.preferenceSatisfaction[index], + !assignDateToBalanceInPlace( + without, + date, + current.balanceId, + context, + current.ruleIds, ) ) continue; - const better = - metrics.holidayWeekValue > bestMetrics.holidayWeekValue || - (metrics.holidayWeekValue === bestMetrics.holidayWeekValue && - (stage === 'non-working-day-gain' - ? metrics.nonWorkingDayGain > bestMetrics.nonWorkingDayGain - : stage === 'longest-work-sequence' - ? metrics.nonWorkingDayGain === bestMetrics.nonWorkingDayGain && - metrics.longestWorkSequence < bestMetrics.longestWorkSequence - : stage === 'short-sequence-deficit' + try { + const feasibleCandidate = finalizeEarnedState(without, context); + if (!feasibleCandidate) continue; + feasibleCandidate.placements.get(date)!.supporting = current.supporting; + const metrics = qualityMetrics(feasibleCandidate, context, preferences); + recordTraceEvaluation( + context, + result, + feasibleCandidate, + preferences, + baseline, + metrics, + [date], + current.date, + ); + if (metrics.unplacedAtRisk !== baseline.unplacedAtRisk) continue; + if (metrics.holidayWeekValue < baseline.holidayWeekValue) continue; + if ( + metrics.preferenceSatisfaction.length !== + baseline.preferenceSatisfaction.length || + metrics.preferenceSatisfaction.some( + (value, index) => value !== baseline.preferenceSatisfaction[index], + ) + ) + continue; + const better = + metrics.holidayWeekValue > bestMetrics.holidayWeekValue || + (metrics.holidayWeekValue === bestMetrics.holidayWeekValue && + (stage === 'non-working-day-gain' + ? metrics.nonWorkingDayGain > bestMetrics.nonWorkingDayGain + : stage === 'longest-work-sequence' ? metrics.nonWorkingDayGain === bestMetrics.nonWorkingDayGain && - metrics.longestWorkSequence === - bestMetrics.longestWorkSequence && - metrics.shortSequenceDeficit < bestMetrics.shortSequenceDeficit - : metrics.nonWorkingDayGain === bestMetrics.nonWorkingDayGain && - metrics.longestWorkSequence === - bestMetrics.longestWorkSequence && - metrics.shortSequenceDeficit === - bestMetrics.shortSequenceDeficit && - date < bestDate)); - if (better) { - best = feasibleCandidate; - bestMetrics = metrics; - bestDate = date; + metrics.longestWorkSequence < bestMetrics.longestWorkSequence + : stage === 'short-sequence-deficit' + ? metrics.nonWorkingDayGain === bestMetrics.nonWorkingDayGain && + metrics.longestWorkSequence === + bestMetrics.longestWorkSequence && + metrics.shortSequenceDeficit < + bestMetrics.shortSequenceDeficit + : metrics.nonWorkingDayGain === bestMetrics.nonWorkingDayGain && + metrics.longestWorkSequence === + bestMetrics.longestWorkSequence && + metrics.shortSequenceDeficit === + bestMetrics.shortSequenceDeficit && + date < bestDate)); + if (better) { + best = cloneState(feasibleCandidate); + bestMetrics = metrics; + bestDate = date; + } + } finally { + without.placements.delete(date); + without.balances.find((item) => item.id === current.balanceId)!.remaining++; } } if (best !== result) recordTraceState(context, best); diff --git a/src/app/services/leave-planner.service.spec.ts b/src/app/services/leave-planner.service.spec.ts index 5182802..073ec45 100644 --- a/src/app/services/leave-planner.service.spec.ts +++ b/src/app/services/leave-planner.service.spec.ts @@ -1660,6 +1660,53 @@ describe('planning calculation context', () => { } }); + it('records every eligible balance when a preference keeps the first assignment', async () => { + const browserWindow = globalThis.window; + vi.stubGlobal('window', undefined); + try { + const calculation = createPlanningCalculation({ + horizon: { start: '2024-01-01', end: '2024-01-10' }, + balances: [ + balance('later', 'cp', 1, '2026-01-01'), + balance('earlier', 'cp', 1, '2025-01-01'), + ], + rules: [ + { + id: 'window', + type: 'time_window', + force: 'preference', + startDate: '2024-01-02', + endDate: '2024-01-02', + placeDays: 1, + placeType: 'cp', + }, + ], + }); + const outcome = await calculation.outcome; + expect(outcome.kind).toBe('Plan'); + if (outcome.kind !== 'Plan') return; + expect(outcome.placements[0].balanceId).toBe('earlier'); + + const trace = await calculation.trace; + const candidates = + trace?.phases + .find((phase) => phase.name === 'preference:window') + ?.decisions.flatMap((decision) => decision.candidates) + .filter((candidate) => candidate.date === '2024-01-02' && candidate.placement) + .map((candidate) => ({ + balanceId: candidate.placement!.balanceId, + status: candidate.status, + reason: candidate.reason?.code, + })) ?? []; + expect(candidates.slice(0, 2)).toEqual([ + { balanceId: 'earlier', status: 'evaluated-rejected', reason: undefined }, + { balanceId: 'later', status: 'not-examined', reason: 'first-compatible-balance' }, + ]); + } finally { + vi.stubGlobal('window', browserWindow); + } + }); + it('offers no explanation trace for invalid input and explains conflicts', async () => { const browserWindow = globalThis.window; vi.stubGlobal('window', undefined);