feat(benchmark): enhance benchmark documentation and add emulated run details
fix(benchmark): update deterministic checks and improve measurement accuracy fix(service): refine candidate recording and enhance balance assignment logic test(service): add test for balance recording with preference assignment
This commit is contained in:
1 parent
ff86cf48b4
commit
f1f08356df
5 files changed
+300
-98
No files matched your search
@@ -20,7 +20,7 @@ export interface BenchmarkMeasurement {
|
||||
outcome: PlanningOutcome['kind'];
|
||||
lastCompletedStage?: string;
|
||||
placements: number;
|
||||
deterministic: boolean;
|
||||
deterministic: 'baseline' | 'yes' | 'no';
|
||||
traceStatus: PlanningTrace['status'] | 'not-produced';
|
||||
traceCandidates: number;
|
||||
}
|
||||
@@ -59,7 +59,7 @@ export function benchmarkScenarios(): BenchmarkScenario[] {
|
||||
id: 'cp-window',
|
||||
type: 'time_window',
|
||||
force: 'obligation',
|
||||
startDate: '2024-04-01',
|
||||
startDate: '2024-04-08',
|
||||
endDate: '2024-04-12',
|
||||
placeDays: 2,
|
||||
placeType: 'cp',
|
||||
@@ -68,8 +68,8 @@ export function benchmarkScenarios(): BenchmarkScenario[] {
|
||||
id: 'rtt-window',
|
||||
type: 'time_window',
|
||||
force: 'obligation',
|
||||
startDate: '2024-05-01',
|
||||
endDate: '2024-05-12',
|
||||
startDate: '2024-04-08',
|
||||
endDate: '2024-04-12',
|
||||
placeDays: 2,
|
||||
placeType: 'rtt',
|
||||
},
|
||||
@@ -166,10 +166,18 @@ function resultSummary(outcome: PlanningOutcome): {
|
||||
}
|
||||
|
||||
export async function runLeavePlannerBenchmark(repetitions = 1): Promise<BenchmarkMeasurement[]> {
|
||||
const scenarios = benchmarkScenarios();
|
||||
for (const scenario of scenarios) {
|
||||
const calculation = createPlanningCalculation(structuredClone(scenario.request));
|
||||
await calculation.outcome;
|
||||
await calculation.trace;
|
||||
}
|
||||
|
||||
const measurements: BenchmarkMeasurement[] = [];
|
||||
for (const scenario of benchmarkScenarios()) {
|
||||
let previous: string | undefined;
|
||||
for (const scenario of scenarios) {
|
||||
let baseline: string | undefined;
|
||||
for (let repetition = 1; repetition <= repetitions; repetition++) {
|
||||
const wallStart = performance.now();
|
||||
const preparationStart = performance.now();
|
||||
const request = structuredClone(scenario.request);
|
||||
const preparationMs = performance.now() - preparationStart;
|
||||
@@ -182,29 +190,32 @@ export async function runLeavePlannerBenchmark(repetitions = 1): Promise<Benchma
|
||||
const traceDeliveryMs = performance.now() - traceStart;
|
||||
const summary = resultSummary(outcome);
|
||||
const stable = JSON.stringify(outcome);
|
||||
const deterministic =
|
||||
baseline === undefined ? 'baseline' : stable === baseline ? 'yes' : 'no';
|
||||
const traceCandidates =
|
||||
trace?.phases.reduce(
|
||||
(total, phase) =>
|
||||
total +
|
||||
phase.decisions.reduce(
|
||||
(decisionTotal, decision) => decisionTotal + decision.candidates.length,
|
||||
0,
|
||||
),
|
||||
0,
|
||||
) ?? 0;
|
||||
const wallMs = performance.now() - wallStart;
|
||||
measurements.push({
|
||||
scenario: scenario.name,
|
||||
repetition,
|
||||
preparationMs,
|
||||
planningMs,
|
||||
traceDeliveryMs,
|
||||
wallMs: preparationMs + planningMs + traceDeliveryMs,
|
||||
wallMs,
|
||||
...summary,
|
||||
deterministic: previous === undefined || previous === stable,
|
||||
deterministic,
|
||||
traceStatus: trace?.status ?? 'not-produced',
|
||||
traceCandidates:
|
||||
trace?.phases.reduce(
|
||||
(total, phase) =>
|
||||
total +
|
||||
phase.decisions.reduce(
|
||||
(decisionTotal, decision) =>
|
||||
decisionTotal + decision.candidates.length,
|
||||
0,
|
||||
),
|
||||
0,
|
||||
) ?? 0,
|
||||
traceCandidates,
|
||||
});
|
||||
previous = stable;
|
||||
baseline ??= stable;
|
||||
}
|
||||
}
|
||||
return measurements;
|
||||
|
||||
Reference in new issue
Block a user