From 1405140c5e42594ba812a6b8338a5cbe2ca18c6c Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 13 Nov 2026 14:57:27 +0000 Subject: [PATCH] Reorder bench groups and SQL file size charts - Groups: Compression and Compression Size, then SQL File Size, then Random Access and the query groups as before. String Encoding and String Encoding Size move to the end, after the groups sorted by name. - SQL File Size charts: clickbench, then tpch, then tpcds, then the other benchmarks by name, each by numeric scale factor. - TPC-H, TPC-DS and nested TPC-H charts always name their scale factor, so `tpch sf=1.0` sits beside `tpch sf=10.0`. - Bump the group payload cache version, since the cached order and names change. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniB9quRiv5fCx8AhzH9s Signed-off-by: Joe Isaacs --- src/bench/lib/data-cache.test.ts | 4 +-- src/bench/lib/data-cache.ts | 2 +- src/bench/lib/group-charts.test.ts | 2 +- src/bench/lib/groups.test.ts | 8 ++++- src/bench/lib/queries.test.ts | 23 +++++++++++++++ src/bench/lib/queries.ts | 47 +++++++++++++++++++++++------- 6 files changed, 70 insertions(+), 16 deletions(-) diff --git a/src/bench/lib/data-cache.test.ts b/src/bench/lib/data-cache.test.ts index e1a8b18b..884865fd 100644 --- a/src/bench/lib/data-cache.test.ts +++ b/src/bench/lib/data-cache.test.ts @@ -69,9 +69,9 @@ describe('data-cache wrappers', () => { expect(BENCH_DATA_TAG).toBe('bench-data'); expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400); expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([ - 'data-cache:group-charts:v13:n100', + 'data-cache:group-charts:v14:n100', ]); - expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v13']); + expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v14']); for (const call of cacheCalls) { expect(call.options.tags).toEqual([BENCH_DATA_TAG]); expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS); diff --git a/src/bench/lib/data-cache.ts b/src/bench/lib/data-cache.ts index baf9da9e..f5fa5ebc 100644 --- a/src/bench/lib/data-cache.ts +++ b/src/bench/lib/data-cache.ts @@ -57,7 +57,7 @@ const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_ * code can serve payloads the old code computed until the next ingest. * It is independent of the producer-facing benchmark schema version. */ -const GROUP_PAYLOAD_CACHE_VERSION = 'v13'; +const GROUP_PAYLOAD_CACHE_VERSION = 'v14'; // The default last-100 group bundle, keyed by group slug. The slug is the cache // key (an `unstable_cache` argument), so one wrapper covers every group. A diff --git a/src/bench/lib/group-charts.test.ts b/src/bench/lib/group-charts.test.ts index dc4f5b2d..519c21eb 100644 --- a/src/bench/lib/group-charts.test.ts +++ b/src/bench/lib/group-charts.test.ts @@ -648,7 +648,7 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', () ...defined(OTHER_FAMILIES[3]), parseCommitWindow('all'), ); - expect(sf1?.display_name).toBe('tpch'); + expect(sf1?.display_name).toBe('tpch sf=1.0'); expect(sf1?.unit_kind).toBe('bytes'); expect(sf1?.series).toEqual({ 'vortex-compact': [null, null, null, null, 88, 86], diff --git a/src/bench/lib/groups.test.ts b/src/bench/lib/groups.test.ts index f4fb364f..7d3784cd 100644 --- a/src/bench/lib/groups.test.ts +++ b/src/bench/lib/groups.test.ts @@ -642,6 +642,8 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg [develop, 'tpch', '10.0', 'vortex-compact', 'lineitem.vortex', 1000], [nightly, 'tpch', '100', 'vortex-file-compressed', 'lineitem.vortex', 40000], [develop, 'clickbench', '1.0', 'vortex-file-compressed', 'hits_0.vortex', 1000], + [develop, 'tpcds', '1.0', 'vortex-file-compressed', 'store_sales.vortex', 500], + [develop, 'appian', '1.0', 'vortex-file-compressed', 'data.vortex', 70], ] as const) { await getPool().query( `INSERT INTO file_sizes @@ -655,11 +657,15 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(groups.map((group) => group.name)).toEqual(['SQL File Size']); const [group] = groups; expect(groupKeyFromSlug(defined(group).slug)).toEqual({ k: 'FileSizeGroup' }); + // Clickbench, TPC-H and TPC-DS lead; the rest follow by name. TPC-H always names its scale + // factor, while Clickbench and Appian have none. expect(defined(group).charts.map((chart) => chart.name)).toEqual([ 'clickbench', - 'tpch', + 'tpch sf=1.0', 'tpch sf=10.0', 'tpch sf=100', + 'tpcds sf=1.0', + 'appian', ]); expect(chartKeyFromSlug(defined(defined(group).charts[3]).slug)).toEqual({ k: 'FileSize', diff --git a/src/bench/lib/queries.test.ts b/src/bench/lib/queries.test.ts index fe0b16f4..777d7dad 100644 --- a/src/bench/lib/queries.test.ts +++ b/src/bench/lib/queries.test.ts @@ -371,6 +371,29 @@ describe('compareGroupSortKey canonical group ordering (no DB)', () => { ]); }); + it('puts compression and SQL file size before random access and string encoding last', () => { + const sorted = [ + 'String Encoding Size', + 'Clickbench', + 'cohere-large-10m / partitioned', + 'String Encoding', + 'Random Access (NVMe)', + 'SQL File Size', + 'Compression Size', + 'Compression', + ].sort(compareGroupSortKey); + expect(sorted).toEqual([ + 'Compression', + 'Compression Size', + 'SQL File Size', + 'Random Access (NVMe)', + 'Clickbench', + 'cohere-large-10m / partitioned', + 'String Encoding', + 'String Encoding Size', + ]); + }); + it('places the appian group directly after both fineweb groups', () => { const sorted = [ 'fineweb [s3]', diff --git a/src/bench/lib/queries.ts b/src/bench/lib/queries.ts index cc67bf42..de6d0a3c 100644 --- a/src/bench/lib/queries.ts +++ b/src/bench/lib/queries.ts @@ -438,12 +438,18 @@ function chartDisplayName(key: ChartKey): string { } } +/** Benchmarks run at several scale factors, so their file-size charts always name it. */ +const SCALED_FILE_SIZE_BENCHMARKS: ReadonlySet = new Set(['tpch', 'tpcds', 'tpch-nested']); + /** - * A file-size chart's name: the benchmark, with its scale factor unless it is the default `1.0`. - * The monorepo's capture script records `1.0` for a benchmark with no scale-factor directory. + * A file-size chart's name: the benchmark, with its scale factor unless it is the default `1.0` + * of a benchmark with no scale factors. The monorepo's capture script records `1.0` for a + * benchmark with no scale-factor directory. */ function fileSizeChartName(benchmark: string, scaleFactor: string): string { - return scaleFactor === '1.0' ? benchmark : `${benchmark} sf=${scaleFactor}`; + return scaleFactor === '1.0' && !SCALED_FILE_SIZE_BENCHMARKS.has(benchmark) + ? benchmark + : `${benchmark} sf=${scaleFactor}`; } /** A chart's base `unit_kind`: sizes are bytes, everything else is a time. */ @@ -1236,12 +1242,13 @@ export interface GroupChartsResponse { charts: NamedChartResponse[]; } -/** Canonical group order for the v4 site. Unknown groups sort last by name. */ +/** + * Canonical group order for the v4 site. Unknown groups follow by name, then + * [`GROUP_ORDER_LAST`]. + */ const GROUP_ORDER: readonly string[] = [ 'Compression', 'Compression Size', - 'String Encoding', - 'String Encoding Size', 'SQL File Size', 'Random Access (NVMe)', 'Random Access (S3)', @@ -1264,14 +1271,21 @@ const GROUP_ORDER: readonly string[] = [ 'appian [nvme]', ]; +/** Groups that sort after every other group, in this order: the string-bench groups. */ +const GROUP_ORDER_LAST: readonly string[] = ['String Encoding', 'String Encoding Size']; + /** - * Sort key for a group name against [`GROUP_ORDER`]: names in the list sort by - * position; names not in the list sort after, with an alphabetical tiebreak. - * Mirrors the Rust `group_sort_key` `(usize, &str)` tuple. + * Sort key for a group name: names in [`GROUP_ORDER`] sort by position, then + * names in neither list alphabetically, then names in [`GROUP_ORDER_LAST`] by + * position. Mirrors the Rust `group_sort_key` `(usize, &str)` tuple. */ function groupSortKey(name: string): [number, string] { const pos = GROUP_ORDER.indexOf(name); - return [pos === -1 ? GROUP_ORDER.length : pos, name]; + if (pos !== -1) { + return [pos, name]; + } + const last = GROUP_ORDER_LAST.indexOf(name); + return [last === -1 ? GROUP_ORDER.length : GROUP_ORDER.length + 1 + last, name]; } /** Comparator over [`groupSortKey`] applying the canonical group ordering. */ @@ -1659,9 +1673,19 @@ async function collectVectorSearchGroups(): Promise { type FileSizeScopeRow = { benchmark: string; scale_factor: string }; +/** Benchmarks whose file-size charts lead the group, in this order; the rest follow by name. */ +const FILE_SIZE_CHART_ORDER: readonly string[] = ['clickbench', 'tpch', 'tpcds']; + +/** A benchmark's position in the file-size group: its `FILE_SIZE_CHART_ORDER` index, else after. */ +function fileSizeChartRank(benchmark: string): number { + const pos = FILE_SIZE_CHART_ORDER.indexOf(benchmark); + return pos === -1 ? FILE_SIZE_CHART_ORDER.length : pos; +} + /** * The `SQL File Size` group: one chart per benchmark and scale factor, or `[]` if the table is - * empty. Charts order by benchmark, then by numeric scale factor, so `10.0` follows `1.0`. + * empty. Charts order by [`FILE_SIZE_CHART_ORDER`], then by benchmark name, then by numeric + * scale factor, so `10.0` follows `1.0`. */ async function collectFileSizeGroup(): Promise { // A recursive skip scan of `idx_file_sizes_baseline`: one descent per scope instead of a scan @@ -1691,6 +1715,7 @@ async function collectFileSizeGroup(): Promise { } rows.sort( (a, b) => + fileSizeChartRank(a.benchmark) - fileSizeChartRank(b.benchmark) || compareCodeUnits(a.benchmark, b.benchmark) || Number(a.scale_factor) - Number(b.scale_factor) || compareCodeUnits(a.scale_factor, b.scale_factor),