test(benchmark): report each scenario as it finishes
Print the report header before the measured runs and each scenario's rows as soon as its repetitions complete, so a timed-out job still shows the finished scenarios. The final report, scenarios, warm-ups, repetitions, assertions, heap limit and timeout are unchanged. Refs #281 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4d26e65672
commit
24c35542a8
2 files changed
+29
-11
No files matched your search
@@ -1,7 +1,9 @@
|
||||
import { vi } from 'vitest';
|
||||
import { createPlanningCalculation } from '../services/leave-planner.service';
|
||||
import {
|
||||
BENCHMARK_REPORT_HEADER,
|
||||
benchmarkReport,
|
||||
benchmarkReportRows,
|
||||
benchmarkScenarios,
|
||||
runLeavePlannerBenchmark,
|
||||
} from './leave-planner-benchmark';
|
||||
@@ -64,7 +66,10 @@ describe('leave planner benchmark protocol', () => {
|
||||
),
|
||||
).toHaveLength(2);
|
||||
}
|
||||
const measurements = await runLeavePlannerBenchmark(2);
|
||||
console.info(BENCHMARK_REPORT_HEADER);
|
||||
const measurements = await runLeavePlannerBenchmark(2, (rows) =>
|
||||
console.info(benchmarkReportRows(rows)),
|
||||
);
|
||||
for (const [name, placements] of [
|
||||
['two years', 60],
|
||||
['three years', 90],
|
||||
|
||||
@@ -193,18 +193,24 @@ function resultSummary(outcome: PlanningOutcome): {
|
||||
return { outcome: outcome.kind, placements: 0 };
|
||||
}
|
||||
|
||||
export async function runLeavePlannerBenchmark(repetitions = 1): Promise<BenchmarkMeasurement[]> {
|
||||
export async function runLeavePlannerBenchmark(
|
||||
repetitions = 1,
|
||||
onScenario?: (measurements: BenchmarkMeasurement[]) => void,
|
||||
): Promise<BenchmarkMeasurement[]> {
|
||||
const scenarios = benchmarkScenarios();
|
||||
for (const scenario of scenarios) await warmUpScenario(scenario);
|
||||
|
||||
const measurements: BenchmarkMeasurement[] = [];
|
||||
for (const scenario of scenarios) {
|
||||
const scenarioMeasurements: BenchmarkMeasurement[] = [];
|
||||
let baseline: string | undefined;
|
||||
for (let repetition = 1; repetition <= repetitions; repetition++) {
|
||||
const result = await measureRepetition(scenario, repetition, baseline);
|
||||
measurements.push(result.measurement);
|
||||
scenarioMeasurements.push(result.measurement);
|
||||
baseline ??= result.stable;
|
||||
}
|
||||
measurements.push(...scenarioMeasurements);
|
||||
onScenario?.(scenarioMeasurements);
|
||||
}
|
||||
return measurements;
|
||||
}
|
||||
@@ -271,12 +277,12 @@ async function measureRepetition(
|
||||
}
|
||||
}
|
||||
|
||||
export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
|
||||
const header =
|
||||
'scenario\trepetition\tpreparation_ms\tplanning_ms\twall_ms\toutcome\tlast_stage\tplacements\tdeterministic\tmax_ui_delay_ms\tplanning_work_ms\tpeak_main_thread_heap_bytes';
|
||||
return [
|
||||
header,
|
||||
...measurements.map((item) =>
|
||||
export const BENCHMARK_REPORT_HEADER =
|
||||
'scenario\trepetition\tpreparation_ms\tplanning_ms\twall_ms\toutcome\tlast_stage\tplacements\tdeterministic\tmax_ui_delay_ms\tplanning_work_ms\tpeak_main_thread_heap_bytes';
|
||||
|
||||
export function benchmarkReportRows(measurements: BenchmarkMeasurement[]): string {
|
||||
return measurements
|
||||
.map((item) =>
|
||||
[
|
||||
item.scenario,
|
||||
item.repetition,
|
||||
@@ -291,6 +297,13 @@ export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
|
||||
item.planningWorkMs?.toFixed(1) ?? '',
|
||||
item.peakMainThreadHeapBytes ?? '',
|
||||
].join('\t'),
|
||||
),
|
||||
].join('\n');
|
||||
)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
|
||||
// Empty input yields the header alone, with no trailing newline.
|
||||
return measurements.length
|
||||
? [BENCHMARK_REPORT_HEADER, benchmarkReportRows(measurements)].join('\n')
|
||||
: BENCHMARK_REPORT_HEADER;
|
||||
}
|
||||
Reference in new issue
Block a user