test(benchmark): print a readable summary table
CI / policy (push) Successful in 7s
CI / validate (push) Successful in 1m5s
CI / build (push) Successful in 1m38s
CI / policy (pull_request) Successful in 10s
CI / security (push) Successful in 1m19s
CI / test (push) Successful in 3m29s
CI / validate (pull_request) Successful in 1m42s
CI / benchmark (push) Successful in 4m13s
CI / security (pull_request) Successful in 56s
CI / build (pull_request) Successful in 1m42s
CI / test (pull_request) Successful in 3m8s
CI / images (push) Successful in 1m15s
CI / required (push) Successful in 1s
CI / benchmark (pull_request) Successful in 3m37s
CI / images (pull_request) Successful in 1m11s
CI / required (pull_request) Successful in 1s

Print the calibration and a Markdown table with one aligned row per
scenario as soon as it finishes: both planning times, mean, budget, ratio,
multiplier, outcome, placements, determinism and budget status. Write
directly to stdout so Vitest adds no per-line headers. The complete TSV
report still follows the table.

Refs #281, #285

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
nessarandClaude Opus 5.5 committed 2026-10-17 02:09:54 +02:00
1 parent cfdb28c90a
commit 3bbe989eca
4 files changed
+127 -28

No files matched your search

+71 -16
View File
@@ -277,12 +277,12 @@ async function measureRepetition(
}
}
export const BENCHMARK_REPORT_HEADER =
'scenario\trepetition\tpreparation_ms\tplanning_ms\twall_ms\toutcome\tlast_stage\tplacements\tdeterministic\tmax_ui_delay_ms\tplanning_work_ms\tpeak_main_thread_heap_bytes';
export function benchmarkReportRows(measurements: BenchmarkMeasurement[]): string {
return measurements
.map((item) =>
export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
const header =
'scenario\trepetition\tpreparation_ms\tplanning_ms\twall_ms\toutcome\tlast_stage\tplacements\tdeterministic\tmax_ui_delay_ms\tplanning_work_ms\tpeak_main_thread_heap_bytes';
return [
header,
...measurements.map((item) =>
[
item.scenario,
item.repetition,
@@ -297,15 +297,8 @@ export function benchmarkReportRows(measurements: BenchmarkMeasurement[]): strin
item.planningWorkMs?.toFixed(1) ?? '',
item.peakMainThreadHeapBytes ?? '',
].join('\t'),
)
.join('\n');
}
export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
// Empty input yields the header alone, with no trailing newline.
return measurements.length
? [BENCHMARK_REPORT_HEADER, benchmarkReportRows(measurements)].join('\n')
: BENCHMARK_REPORT_HEADER;
),
].join('\n');
}
// Measured on the development machine (ratio to the frozen two-years run), tightened by #286.
@@ -330,7 +323,7 @@ export function budgetViolations(
byScenario.set(scenario, [...(byScenario.get(scenario) ?? []), planningMs]);
const violations: string[] = [];
for (const [scenario, times] of byScenario) {
const mean = times.reduce((sum, value) => sum + value, 0) / times.length;
const mean = average(times);
const multiplier = multipliers[scenario];
if (multiplier === undefined) {
violations.push(`${scenario}: no budget multiplier defined`);
@@ -344,3 +337,65 @@ export function budgetViolations(
}
return violations;
}
function average(values: number[]): number {
return values.reduce((sum, value) => sum + value, 0) / values.length;
}
// Fixed widths keep rows printed as each scenario finishes aligned with the header.
const SUMMARY_COLUMNS: [title: string, width: number, alignRight: boolean][] = [
['Scenario', 27, false],
['Planning (runs)', 17, true],
['Mean', 9, true],
['Budget', 9, true],
['Ratio', 6, true],
['Limit', 6, true],
['Outcome', 16, false],
['Placements', 10, true],
['Deterministic', 13, false],
['Status', 9, false],
];
function summaryLine(cells: string[]): string {
return `| ${cells
.map((cell, index) => {
const [, width, alignRight] = SUMMARY_COLUMNS[index];
return alignRight ? cell.padStart(width) : cell.padEnd(width);
})
.join(' | ')} |`;
}
/** Markdown table header for {@link benchmarkSummaryRow}. */
export function benchmarkSummaryHeader(): string {
return [
summaryLine(SUMMARY_COLUMNS.map(([title]) => title)),
`|${SUMMARY_COLUMNS.map(([, width, alignRight]) => (alignRight ? `${'-'.repeat(width + 1)}:` : `:${'-'.repeat(width + 1)}`)).join('|')}|`,
].join('\n');
}
/** One aligned table row for one scenario's measured repetitions, judged like {@link budgetViolations}. */
export function benchmarkSummaryRow(
scenarioMeasurements: BenchmarkMeasurement[],
calibrationMs: number,
multipliers: Record<string, number> = BENCHMARK_BUDGET_MULTIPLIERS,
): string {
const [first] = scenarioMeasurements;
const seconds = (ms: number) => (ms / 1000).toFixed(2);
const mean = average(scenarioMeasurements.map(({ planningMs }) => planningMs));
const multiplier = multipliers[first.scenario];
const budget = multiplier === undefined ? undefined : multiplier * calibrationMs;
return summaryLine([
first.scenario,
`${scenarioMeasurements.map(({ planningMs }) => seconds(planningMs)).join(' / ')} s`,
`${seconds(mean)} s`,
budget === undefined ? '-' : `${seconds(budget)} s`,
(mean / calibrationMs).toFixed(2),
multiplier === undefined ? '-' : String(multiplier),
first.lastCompletedStage ? `${first.outcome} (${first.lastCompletedStage})` : first.outcome,
String(first.placements),
scenarioMeasurements.slice(1).every(({ deterministic }) => deterministic === 'yes')
? 'yes'
: 'no',
budget === undefined ? 'no budget' : mean > budget ? 'OVER' : 'ok',
]);
}