test(benchmark): print a readable summary table
CI / policy (push) Successful in 7s
CI / validate (push) Successful in 1m5s
CI / build (push) Successful in 1m38s
CI / policy (pull_request) Successful in 10s
CI / security (push) Successful in 1m19s
CI / test (push) Successful in 3m29s
CI / validate (pull_request) Successful in 1m42s
CI / benchmark (push) Successful in 4m13s
CI / security (pull_request) Successful in 56s
CI / build (pull_request) Successful in 1m42s
CI / test (pull_request) Successful in 3m8s
CI / images (push) Successful in 1m15s
CI / required (push) Successful in 1s
CI / benchmark (pull_request) Successful in 3m37s
CI / images (pull_request) Successful in 1m11s
CI / required (pull_request) Successful in 1s
CI / policy (push) Successful in 7s
CI / validate (push) Successful in 1m5s
CI / build (push) Successful in 1m38s
CI / policy (pull_request) Successful in 10s
CI / security (push) Successful in 1m19s
CI / test (push) Successful in 3m29s
CI / validate (pull_request) Successful in 1m42s
CI / benchmark (push) Successful in 4m13s
CI / security (pull_request) Successful in 56s
CI / build (pull_request) Successful in 1m42s
CI / test (pull_request) Successful in 3m8s
CI / images (push) Successful in 1m15s
CI / required (push) Successful in 1s
CI / benchmark (pull_request) Successful in 3m37s
CI / images (pull_request) Successful in 1m11s
CI / required (pull_request) Successful in 1s
Print the calibration and a Markdown table with one aligned row per scenario as soon as it finishes: both planning times, mean, budget, ratio, multiplier, outcome, placements, determinism and budget status. Write directly to stdout so Vitest adds no per-line headers. The complete TSV report still follows the table. Refs #281, #285 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cfdb28c90a
commit
3bbe989eca
4 files changed
+127
-28
No files matched your search
@@ -277,12 +277,12 @@ async function measureRepetition(
|
||||
}
|
||||
}
|
||||
|
||||
export const BENCHMARK_REPORT_HEADER =
|
||||
'scenario\trepetition\tpreparation_ms\tplanning_ms\twall_ms\toutcome\tlast_stage\tplacements\tdeterministic\tmax_ui_delay_ms\tplanning_work_ms\tpeak_main_thread_heap_bytes';
|
||||
|
||||
export function benchmarkReportRows(measurements: BenchmarkMeasurement[]): string {
|
||||
return measurements
|
||||
.map((item) =>
|
||||
export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
|
||||
const header =
|
||||
'scenario\trepetition\tpreparation_ms\tplanning_ms\twall_ms\toutcome\tlast_stage\tplacements\tdeterministic\tmax_ui_delay_ms\tplanning_work_ms\tpeak_main_thread_heap_bytes';
|
||||
return [
|
||||
header,
|
||||
...measurements.map((item) =>
|
||||
[
|
||||
item.scenario,
|
||||
item.repetition,
|
||||
@@ -297,15 +297,8 @@ export function benchmarkReportRows(measurements: BenchmarkMeasurement[]): strin
|
||||
item.planningWorkMs?.toFixed(1) ?? '',
|
||||
item.peakMainThreadHeapBytes ?? '',
|
||||
].join('\t'),
|
||||
)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
export function benchmarkReport(measurements: BenchmarkMeasurement[]): string {
|
||||
// Empty input yields the header alone, with no trailing newline.
|
||||
return measurements.length
|
||||
? [BENCHMARK_REPORT_HEADER, benchmarkReportRows(measurements)].join('\n')
|
||||
: BENCHMARK_REPORT_HEADER;
|
||||
),
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
// Measured on the development machine (ratio to the frozen two-years run), tightened by #286.
|
||||
@@ -330,7 +323,7 @@ export function budgetViolations(
|
||||
byScenario.set(scenario, [...(byScenario.get(scenario) ?? []), planningMs]);
|
||||
const violations: string[] = [];
|
||||
for (const [scenario, times] of byScenario) {
|
||||
const mean = times.reduce((sum, value) => sum + value, 0) / times.length;
|
||||
const mean = average(times);
|
||||
const multiplier = multipliers[scenario];
|
||||
if (multiplier === undefined) {
|
||||
violations.push(`${scenario}: no budget multiplier defined`);
|
||||
@@ -344,3 +337,65 @@ export function budgetViolations(
|
||||
}
|
||||
return violations;
|
||||
}
|
||||
|
||||
function average(values: number[]): number {
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
// Fixed widths keep rows printed as each scenario finishes aligned with the header.
|
||||
const SUMMARY_COLUMNS: [title: string, width: number, alignRight: boolean][] = [
|
||||
['Scenario', 27, false],
|
||||
['Planning (runs)', 17, true],
|
||||
['Mean', 9, true],
|
||||
['Budget', 9, true],
|
||||
['Ratio', 6, true],
|
||||
['Limit', 6, true],
|
||||
['Outcome', 16, false],
|
||||
['Placements', 10, true],
|
||||
['Deterministic', 13, false],
|
||||
['Status', 9, false],
|
||||
];
|
||||
|
||||
function summaryLine(cells: string[]): string {
|
||||
return `| ${cells
|
||||
.map((cell, index) => {
|
||||
const [, width, alignRight] = SUMMARY_COLUMNS[index];
|
||||
return alignRight ? cell.padStart(width) : cell.padEnd(width);
|
||||
})
|
||||
.join(' | ')} |`;
|
||||
}
|
||||
|
||||
/** Markdown table header for {@link benchmarkSummaryRow}. */
|
||||
export function benchmarkSummaryHeader(): string {
|
||||
return [
|
||||
summaryLine(SUMMARY_COLUMNS.map(([title]) => title)),
|
||||
`|${SUMMARY_COLUMNS.map(([, width, alignRight]) => (alignRight ? `${'-'.repeat(width + 1)}:` : `:${'-'.repeat(width + 1)}`)).join('|')}|`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/** One aligned table row for one scenario's measured repetitions, judged like {@link budgetViolations}. */
|
||||
export function benchmarkSummaryRow(
|
||||
scenarioMeasurements: BenchmarkMeasurement[],
|
||||
calibrationMs: number,
|
||||
multipliers: Record<string, number> = BENCHMARK_BUDGET_MULTIPLIERS,
|
||||
): string {
|
||||
const [first] = scenarioMeasurements;
|
||||
const seconds = (ms: number) => (ms / 1000).toFixed(2);
|
||||
const mean = average(scenarioMeasurements.map(({ planningMs }) => planningMs));
|
||||
const multiplier = multipliers[first.scenario];
|
||||
const budget = multiplier === undefined ? undefined : multiplier * calibrationMs;
|
||||
return summaryLine([
|
||||
first.scenario,
|
||||
`${scenarioMeasurements.map(({ planningMs }) => seconds(planningMs)).join(' / ')} s`,
|
||||
`${seconds(mean)} s`,
|
||||
budget === undefined ? '-' : `${seconds(budget)} s`,
|
||||
(mean / calibrationMs).toFixed(2),
|
||||
multiplier === undefined ? '-' : String(multiplier),
|
||||
first.lastCompletedStage ? `${first.outcome} (${first.lastCompletedStage})` : first.outcome,
|
||||
String(first.placements),
|
||||
scenarioMeasurements.slice(1).every(({ deterministic }) => deterministic === 'yes')
|
||||
? 'yes'
|
||||
: 'no',
|
||||
budget === undefined ? 'no budget' : mean > budget ? 'OVER' : 'ok',
|
||||
]);
|
||||
}
|
||||
Reference in new issue
Block a user