test(planning): compare the engine against a frozen engine on random requests

Bundle the planning core at a871f8f (develop after PR #278) as a frozen
fixture and compare it with the current engine on 5,000 seeded random
requests. Outcomes, stable explanations, explanation errors and final-state
snapshots must be strictly deep-equal. Requests cover obligations,
preferences, IF_THEN and TIME_WINDOW rules, earned entitlements, composed
optimization rules, declared days, year boundaries and public holidays.

Refs #280

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/benchmark/README.md

@@ -44,6 +44,49 @@ The measurements below describe earlier implementations and protocols. Their
 trace counters and timings are historical; current evidence is in
 [the #78 report](issue-78.md).
 
+## Frozen-engine differential check
+
+`src/app/services/frozen-engine.spec.ts` runs 5,000 seeded random planning
+requests (seeds 280000 to 284999, one test per 200 seeds) through the current
+engine and through a frozen copy of the engine at `a871f8f`, develop after
+PR #278. For each request, the planning outcome, stable plan explanation,
+explanation error and final-state snapshot must be strictly deep-equal,
+including `PlanningConflict` and `InvalidInput` outcomes. Timings are not
+compared.
+
+Start dates span 2025 to 2027. About 15% of requests start in late December and
+cross a year boundary, and about 15% start shortly before a weekday French
+public holiday. Most horizons last 7 to 28 days; about 10% last 29 to 70 days
+with optimization presets and at most one calendar rule. Requests cover
+obligation and preference rules, `IF_THEN` rules with earned leave
+entitlements, `TIME_WINDOW` rules, optimization presets in random order and
+subsets, random composed optimization configurations, disabled rules, declared
+non-working days and a few deliberately invalid requests. The last test fails
+if an outcome kind, rule type, year-crossing horizon or public-holiday horizon
+stops appearing. Requests stay small because candidate search grows quickly
+with horizon length and balances; the slowest case takes about one second.
+
+The fixture is `src/app/services/frozen-engine/leave-planner.core.frozen.mjs`,
+a generated bundle with `date-holidays` left external, and its hand-written
+`leave-planner.core.frozen.d.mts`. Rebuild it only for an intended behavior
+change, in its own reviewed commit, from the commit that introduces the change:
+
+```sh
+bun build src/app/services/leave-planner.core.ts --target=node --external date-holidays --outfile src/app/services/frozen-engine/leave-planner.core.frozen.mjs
+```
+
+A failure prints the seed and the full request as JSON. To run only that seed:
+
+```sh
+FROZEN_ENGINE_SEED=<seed> bunx vitest run src/app/services/frozen-engine.spec.ts
+```
+
+The check runs in the functional suite. On the development laptop it takes
+about 32 seconds without coverage and 38 seconds with coverage
+(`NODE_OPTIONS=--max-old-space-size=2048 bunx vitest run src/app/services/frozen-engine.spec.ts --maxWorkers=1`).
+A flipped objective direction in `compareObjectiveValues` made 26 of its tests
+fail, starting at seed 280002; the change was reverted.
+
 ## Reference-phone protocol
 
 Open the deployed application at `/benchmark` on the reference phone. Tap

src/app/services/frozen-engine.spec.ts

@@ -0,0 +1,257 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { createOptimizationPresetRules } from './rule-service';
+import { frenchPublicHolidaysForYear, isWorkingPlanningDate } from './declared-non-working-days';
+import { OPTIMIZATION_CATALOG } from './planning-objectives';
+import {
+	calculatePlanningInternal,
+	type PlanningArtifacts,
+	type PlanningOutcome,
+} from './leave-planner.core';
+import { calculatePlanningInternal as frozenCalculate } from './frozen-engine/leave-planner.core.frozen.mjs';
+
+/** Rebuild instructions for the frozen engine: docs/benchmark/README.md. */
+const BASE_SEED = 280_000;
+const CASES = 5000;
+const CHUNK = 200;
+
+function mulberry32(seed: number): () => number {
+	let a = seed >>> 0;
+	return () => {
+		a = (a + 0x6d2b79f5) >>> 0;
+		let t = a;
+		t = Math.imul(t ^ (t >>> 15), t | 1);
+		t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+		return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+	};
+}
+
+const DAY = 86_400_000;
+const weekday = (ms: number) => ![0, 6].includes(new Date(ms).getUTCDay());
+const iso = (ms: number) => new Date(ms).toISOString().slice(0, 10);
+
+type Req = Record<string, unknown> & { rules: Record<string, unknown>[] };
+
+function generateRequest(seed: number): Req {
+	const rnd = mulberry32(seed);
+	const int = (min: number, max: number) => min + Math.floor(rnd() * (max - min + 1));
+	const pick = <T>(items: readonly T[]): T => items[int(0, items.length - 1)];
+	const chance = (p: number) => rnd() < p;
+	const shuffle = <T>(items: T[]): T[] => {
+		for (let i = items.length - 1; i > 0; i--) {
+			const j = int(0, i);
+			[items[i], items[j]] = [items[j], items[i]];
+		}
+		return items;
+	};
+
+	const long = chance(0.1);
+	const mode = rnd();
+	let startMs = Date.UTC(2025, 0, 1) + int(0, 1094) * DAY;
+	if (mode < 0.15) startMs = Date.UTC(pick([2025, 2026]), 11, int(8, 31));
+	else if (mode < 0.3) {
+		const year = pick([2025, 2026, 2027]);
+		const holidays = [...frenchPublicHolidaysForYear(year)].filter((d) =>
+			weekday(Date.parse(d)),
+		);
+		startMs = Date.parse(pick(holidays)) - int(1, 10) * DAY;
+	}
+	const days = long ? int(29, 70) : int(7, 28);
+	const start = iso(startMs);
+	const date = (offset: number) => iso(startMs + offset * DAY);
+	const types = ['cp', 'rtt', 'other'] as const;
+
+	const balances = Array.from({ length: int(1, 3) }, (_, i) => ({
+		id: `b${i}`,
+		type: pick(types),
+		quantity: int(0, long ? 4 : 3),
+		...(chance(0.3) ? { availableFrom: date(int(1, days)) } : {}),
+		expiryDate: date(int(-3, days + 40)),
+	})).map((b) =>
+		b.availableFrom && b.availableFrom > b.expiryDate
+			? { ...b, expiryDate: b.availableFrom }
+			: b,
+	);
+
+	const declaredDays: { date: string; type: string }[] = [];
+	for (let n = int(0, 4); n > 0; n--) {
+		const ms = startMs + int(0, days) * DAY;
+		const d = iso(ms);
+		if (isWorkingPlanningDate(d) && !declaredDays.some((x) => x.date === d))
+			declaredDays.push({ date: d, type: pick(types) });
+	}
+
+	const selectors = ['cp', 'rtt', 'other', 'any'] as const;
+	const rules: Record<string, unknown>[] = [];
+	for (let n = long ? int(0, 1) : int(0, 2); n > 0; n--) {
+		const from = int(0, days - 1);
+		rules.push({
+			id: `tw${rules.length}`,
+			type: chance(0.5) ? 'time_window' : 'TIME_WINDOW',
+			force: pick(['obligation', 'preference']),
+			startDate: date(from),
+			endDate: date(from + int(0, 10)),
+			placeDays: int(1, 4),
+			placeType: pick(selectors),
+		});
+	}
+	for (let n = long && rules.length ? 0 : long ? int(0, 1) : int(0, 2); n > 0; n--) {
+		rules.push({
+			id: `it${rules.length}`,
+			type: chance(0.5) ? 'if_then' : 'IF_THEN',
+			force: pick(['obligation', 'preference']),
+			dayOfWeek: int(1, 5),
+			placeDays: int(1, 3),
+			placeType: pick(selectors),
+			placement: pick(['before', 'after', 'around', 'auto']),
+			rewardDays: int(0, 2),
+			rewardType: pick(types),
+		});
+	}
+	const presets: Record<string, unknown>[] = shuffle(
+		createOptimizationPresetRules() as unknown as Record<string, unknown>[],
+	).slice(0, long ? int(1, 4) : int(0, 3));
+	rules.push(...presets);
+	if (!long && chance(0.3)) {
+		const row = pick(OPTIMIZATION_CATALOG);
+		const conditions = (row.conditions as readonly string[]).filter(() => chance(0.5));
+		rules.push({
+			id: 'composed',
+			type: 'optimization',
+			force: 'preference',
+			configuration: {
+				elements: row.elements,
+				conditions,
+				measure: row.measure,
+				aggregation: pick(row.aggregations),
+				direction: pick(['minimize', 'maximize']),
+				...(row.measure === 'work-sequence-deficit' ? { thresholdDays: int(1, 8) } : {}),
+			},
+		});
+	}
+	for (const rule of shuffle(rules)) if (chance(0.08)) rule['enabled'] = false;
+
+	const request: Req = {
+		calculationDate: start,
+		horizon: { start, end: date(days) },
+		balances,
+		declaredDays,
+		rules,
+	};
+	if (chance(0.04)) {
+		const breakIt = int(0, 3);
+		if (breakIt === 0) balances[0].quantity = -1;
+		else if (breakIt === 1) request['horizon'] = { start: date(days), end: start };
+		else if (breakIt === 2) rules.push({ id: 'bad', type: 'unknown_type' });
+		else declaredDays.push({ date: start, type: 'nope' });
+	}
+	return request;
+}
+
+interface Run {
+	outcome: PlanningOutcome;
+	artifacts?: Pick<PlanningArtifacts, 'explanation' | 'finalState' | 'explanationError'>;
+}
+
+function run(
+	calculate: (
+		request: unknown,
+		c?: undefined,
+		a?: (artifacts: PlanningArtifacts) => void,
+	) => PlanningOutcome,
+	request: Req,
+): Run {
+	let artifacts: Run['artifacts'];
+	const outcome = calculate(structuredClone(request), undefined, (a) => {
+		artifacts = {
+			explanation: a.explanation,
+			finalState: a.finalState,
+			explanationError: a.explanationError,
+		};
+	});
+	return { outcome, artifacts };
+}
+
+const kinds = new Map<string, number>();
+const ruleTypes = new Set<string>();
+const count = (key: string) => kinds.set(key, (kinds.get(key) ?? 0) + 1);
+
+let slowest = { seed: 0, ms: 0 };
+
+function crossesYearAndHoliday(request: Req): void {
+	const { start, end } = request['horizon'] as { start: string; end: string };
+	if (start.slice(0, 4) !== end.slice(0, 4)) count('crosses-year');
+	for (let ms = Date.parse(start); ms <= Date.parse(end); ms += DAY)
+		if (
+			weekday(ms) &&
+			frenchPublicHolidaysForYear(new Date(ms).getUTCFullYear()).has(iso(ms))
+		) {
+			count('holiday-week');
+			return;
+		}
+}
+
+function assertSameAsFrozen(seed: number): void {
+	const request = generateRequest(seed);
+	const t0 = performance.now();
+	const current = run(calculatePlanningInternal, request);
+	const frozen = run(frozenCalculate, request);
+	const ms = performance.now() - t0;
+	if (ms > slowest.ms) slowest = { seed, ms: Math.round(ms) };
+	crossesYearAndHoliday(request);
+	count(current.outcome.kind);
+	for (const rule of request.rules) ruleTypes.add(String(rule['type']).toLowerCase());
+	if ((request['declaredDays'] as unknown[]).length) count('declared');
+	if (request.rules.some((r) => r['rewardDays'])) count('reward');
+	try {
+		expect(current).toStrictEqual(frozen);
+	} catch (error) {
+		throw new Error(
+			`Engine differs from frozen engine. seed=${seed}\nrequest=${JSON.stringify(request)}\n${
+				(error as Error).message
+			}`,
+			{ cause: error },
+		);
+	}
+}
+
+const ONLY_SEED = process.env['FROZEN_ENGINE_SEED'];
+
+describe('frozen engine differential check', () => {
+	if (ONLY_SEED) {
+		it(`matches the frozen engine for seed ${ONLY_SEED}`, () => {
+			assertSameAsFrozen(Number(ONLY_SEED));
+		});
+		return;
+	}
+	for (let from = 0; from < CASES; from += CHUNK) {
+		it(`matches the frozen engine for seeds ${BASE_SEED + from}..${BASE_SEED + from + CHUNK - 1}`, () => {
+			for (let i = from; i < from + CHUNK; i++) assertSameAsFrozen(BASE_SEED + i);
+		}, 120_000);
+	}
+
+	afterAll(() => {
+		console.info(
+			'frozen-engine coverage of generated cases',
+			Object.fromEntries(kinds),
+			[...ruleTypes],
+			'slowest case',
+			slowest,
+		);
+	});
+
+	it('generates every outcome kind and rule type', () => {
+		for (const kind of [
+			'Plan',
+			'PlanningConflict',
+			'InvalidInput',
+			'declared',
+			'reward',
+			'crosses-year',
+			'holiday-week',
+		])
+			expect(kinds.get(kind) ?? 0, kind).toBeGreaterThan(20);
+		expect(['if_then', 'optimization', 'time_window'].filter((t) => !ruleTypes.has(t))).toEqual(
+			[],
+		);
+	});
+});

src/app/services/frozen-engine/leave-planner.core.frozen.d.mts

@@ -0,0 +1,7 @@
+import type { PlanningArtifactsCallback, PlanningOutcome } from '../leave-planner.core';
+
+export function calculatePlanningInternal(
+	request: unknown,
+	checkpointCallback?: undefined,
+	artifactsCallback?: PlanningArtifactsCallback,
+): PlanningOutcome;

vite.config.mts

@@ -3,6 +3,7 @@
 import angular from '@analogjs/vite-plugin-angular';
 
 import { defineConfig } from 'vite';
+import { coverageConfigDefaults } from 'vitest/config';
 
 // https://vitejs.dev/config/
 export default defineConfig(({ mode }) => {
@@ -18,6 +19,7 @@ export default defineConfig(({ mode }) => {
 			coverage: {
 				provider: 'istanbul',
 				reporter: ['text', 'html', 'cobertura'],
+				exclude: [...coverageConfigDefaults.exclude, '**/frozen-engine/**'],
 			},
 		},
 		define: {