Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/bench/lib/data-cache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,9 @@ describe('data-cache wrappers', () => {
expect(BENCH_DATA_TAG).toBe('bench-data');
expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400);
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([
'data-cache:group-charts:v13:n100',
'data-cache:group-charts:v14:n100',
]);
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v13']);
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v14']);
for (const call of cacheCalls) {
expect(call.options.tags).toEqual([BENCH_DATA_TAG]);
expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS);
Expand Down
2 changes: 1 addition & 1 deletion src/bench/lib/data-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_
* code can serve payloads the old code computed until the next ingest.
* It is independent of the producer-facing benchmark schema version.
*/
const GROUP_PAYLOAD_CACHE_VERSION = 'v13';
const GROUP_PAYLOAD_CACHE_VERSION = 'v14';

// The default last-100 group bundle, keyed by group slug. The slug is the cache
// key (an `unstable_cache` argument), so one wrapper covers every group. A
Expand Down
2 changes: 1 addition & 1 deletion src/bench/lib/group-charts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -648,7 +648,7 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', ()
...defined(OTHER_FAMILIES[3]),
parseCommitWindow('all'),
);
expect(sf1?.display_name).toBe('tpch');
expect(sf1?.display_name).toBe('tpch sf=1.0');
expect(sf1?.unit_kind).toBe('bytes');
expect(sf1?.series).toEqual({
'vortex-compact': [null, null, null, null, 88, 86],
Expand Down
8 changes: 7 additions & 1 deletion src/bench/lib/groups.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,8 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg
[develop, 'tpch', '10.0', 'vortex-compact', 'lineitem.vortex', 1000],
[nightly, 'tpch', '100', 'vortex-file-compressed', 'lineitem.vortex', 40000],
[develop, 'clickbench', '1.0', 'vortex-file-compressed', 'hits_0.vortex', 1000],
[develop, 'tpcds', '1.0', 'vortex-file-compressed', 'store_sales.vortex', 500],
[develop, 'appian', '1.0', 'vortex-file-compressed', 'data.vortex', 70],
] as const) {
await getPool().query(
`INSERT INTO file_sizes
Expand All @@ -655,11 +657,15 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg
expect(groups.map((group) => group.name)).toEqual(['SQL File Size']);
const [group] = groups;
expect(groupKeyFromSlug(defined(group).slug)).toEqual({ k: 'FileSizeGroup' });
// Clickbench, TPC-H and TPC-DS lead; the rest follow by name. TPC-H always names its scale
// factor, while Clickbench and Appian have none.
expect(defined(group).charts.map((chart) => chart.name)).toEqual([
'clickbench',
'tpch',
'tpch sf=1.0',
'tpch sf=10.0',
'tpch sf=100',
'tpcds sf=1.0',
'appian',
]);
expect(chartKeyFromSlug(defined(defined(group).charts[3]).slug)).toEqual({
k: 'FileSize',
Expand Down
23 changes: 23 additions & 0 deletions src/bench/lib/queries.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,29 @@ describe('compareGroupSortKey canonical group ordering (no DB)', () => {
]);
});

it('puts compression and SQL file size before random access and string encoding last', () => {
const sorted = [
'String Encoding Size',
'Clickbench',
'cohere-large-10m / partitioned',
'String Encoding',
'Random Access (NVMe)',
'SQL File Size',
'Compression Size',
'Compression',
].sort(compareGroupSortKey);
expect(sorted).toEqual([
'Compression',
'Compression Size',
'SQL File Size',
'Random Access (NVMe)',
'Clickbench',
'cohere-large-10m / partitioned',
'String Encoding',
'String Encoding Size',
]);
});

it('places the appian group directly after both fineweb groups', () => {
const sorted = [
'fineweb [s3]',
Expand Down
47 changes: 36 additions & 11 deletions src/bench/lib/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -438,12 +438,18 @@ function chartDisplayName(key: ChartKey): string {
}
}

/** Benchmarks run at several scale factors, so their file-size charts always name it. */
const SCALED_FILE_SIZE_BENCHMARKS: ReadonlySet<string> = new Set(['tpch', 'tpcds', 'tpch-nested']);

/**
* A file-size chart's name: the benchmark, with its scale factor unless it is the default `1.0`.
* The monorepo's capture script records `1.0` for a benchmark with no scale-factor directory.
* A file-size chart's name: the benchmark, with its scale factor unless it is the default `1.0`
* of a benchmark with no scale factors. The monorepo's capture script records `1.0` for a
* benchmark with no scale-factor directory.
*/
function fileSizeChartName(benchmark: string, scaleFactor: string): string {
return scaleFactor === '1.0' ? benchmark : `${benchmark} sf=${scaleFactor}`;
return scaleFactor === '1.0' && !SCALED_FILE_SIZE_BENCHMARKS.has(benchmark)
? benchmark
: `${benchmark} sf=${scaleFactor}`;
}

/** A chart's base `unit_kind`: sizes are bytes, everything else is a time. */
Expand Down Expand Up @@ -1236,12 +1242,13 @@ export interface GroupChartsResponse {
charts: NamedChartResponse[];
}

/** Canonical group order for the v4 site. Unknown groups sort last by name. */
/**
* Canonical group order for the v4 site. Unknown groups follow by name, then
* [`GROUP_ORDER_LAST`].
*/
const GROUP_ORDER: readonly string[] = [
'Compression',
'Compression Size',
'String Encoding',
'String Encoding Size',
'SQL File Size',
'Random Access (NVMe)',
'Random Access (S3)',
Expand All @@ -1264,14 +1271,21 @@ const GROUP_ORDER: readonly string[] = [
'appian [nvme]',
];

/** Groups that sort after every other group, in this order: the string-bench groups. */
const GROUP_ORDER_LAST: readonly string[] = ['String Encoding', 'String Encoding Size'];

/**
* Sort key for a group name against [`GROUP_ORDER`]: names in the list sort by
* position; names not in the list sort after, with an alphabetical tiebreak.
* Mirrors the Rust `group_sort_key` `(usize, &str)` tuple.
* Sort key for a group name: names in [`GROUP_ORDER`] sort by position, then
* names in neither list alphabetically, then names in [`GROUP_ORDER_LAST`] by
* position. Mirrors the Rust `group_sort_key` `(usize, &str)` tuple.
*/
function groupSortKey(name: string): [number, string] {
const pos = GROUP_ORDER.indexOf(name);
return [pos === -1 ? GROUP_ORDER.length : pos, name];
if (pos !== -1) {
return [pos, name];
}
const last = GROUP_ORDER_LAST.indexOf(name);
return [last === -1 ? GROUP_ORDER.length : GROUP_ORDER.length + 1 + last, name];
}

/** Comparator over [`groupSortKey`] applying the canonical group ordering. */
Expand Down Expand Up @@ -1659,9 +1673,19 @@ async function collectVectorSearchGroups(): Promise<Group[]> {

type FileSizeScopeRow = { benchmark: string; scale_factor: string };

/** Benchmarks whose file-size charts lead the group, in this order; the rest follow by name. */
const FILE_SIZE_CHART_ORDER: readonly string[] = ['clickbench', 'tpch', 'tpcds'];

/** A benchmark's position in the file-size group: its `FILE_SIZE_CHART_ORDER` index, else after. */
function fileSizeChartRank(benchmark: string): number {
const pos = FILE_SIZE_CHART_ORDER.indexOf(benchmark);
return pos === -1 ? FILE_SIZE_CHART_ORDER.length : pos;
}

/**
* The `SQL File Size` group: one chart per benchmark and scale factor, or `[]` if the table is
* empty. Charts order by benchmark, then by numeric scale factor, so `10.0` follows `1.0`.
* empty. Charts order by [`FILE_SIZE_CHART_ORDER`], then by benchmark name, then by numeric
* scale factor, so `10.0` follows `1.0`.
*/
async function collectFileSizeGroup(): Promise<Group[]> {
// A recursive skip scan of `idx_file_sizes_baseline`: one descent per scope instead of a scan
Expand Down Expand Up @@ -1691,6 +1715,7 @@ async function collectFileSizeGroup(): Promise<Group[]> {
}
rows.sort(
(a, b) =>
fileSizeChartRank(a.benchmark) - fileSizeChartRank(b.benchmark) ||
compareCodeUnits(a.benchmark, b.benchmark) ||
Number(a.scale_factor) - Number(b.scale_factor) ||
compareCodeUnits(a.scale_factor, b.scale_factor),
Expand Down
Loading