datafusion-arrowmetal 0.4.1

A DataFusion 55.1 physical optimizer rule that runs full sorts, and count(*), DISTINCT and integer MIN/MAX group-bys over tables of 50M rows or more in the measured group-count ranges, on Apple silicon GPUs through ArrowMetal; hash joins, top-k and filters are available and off by default
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

//! The measured join table. Generated by scripts/join_table.py from
//! results/datafusion_join_2026-10-01.csv
//! Do not edit by hand (`--check` fails when this file and those disagree).
//!
//! One row per (join type of DataFusion's `HashJoinExec`, key class, input class, build-side
//! bucket) measured: `min_probe_rows` is the fewest probe-side rows from which every case, in
//! both layouts, was at least 1.5x faster than DataFusion alone warm and at least
//! 1.0x on the first run after 500 ms and 5000 ms of idle against DataFusion alone's
//! first run after the same idle, at that size and every larger measured size (None: not
//! taken). Every row ends with the reason for its threshold.

pub(crate) struct JoinRow {
    pub how: &'static str,
    pub key_class: &'static str,
    pub source: &'static str,
    pub build: &'static str,
    pub min_probe_rows: Option<u64>,
    pub reason: &'static str,
}

pub(crate) const SOURCE: &str = "results/datafusion_join_2026-10-01.csv";
pub(crate) const BUILD_BUCKETS: &[(&str, u64, u64)] = &[("10k", 3163, 31622), ("1M", 316228, 3162277), ("10M", 3162278, 31622776)];

pub(crate) const TABLE: &[JoinRow] = &[
    JoinRow { how: "inner", key_class: "i32", source: "memory", build: "10M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.44x, idle 500 ms 1.02x, 5000 ms 0.80x [jg32_inner_10M b8192: idle vs idle 0.80x after 5000 ms]; 50M: warm 1.30x, idle 500 ms 1.08x, 5000 ms 1.01x [jg32_inner_10M part1: warm 1.30x]" },
    JoinRow { how: "inner", key_class: "i32", source: "memory", build: "10k", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.25x, idle 500 ms 0.71x, 5000 ms 0.47x [jg32_inner_10k part1: idle vs idle 0.47x after 5000 ms]; 50M: warm 1.11x, idle 500 ms 0.70x, 5000 ms 0.71x [jg32_inner_10k b8192: idle vs idle 0.70x after 500 ms]" },
    JoinRow { how: "inner", key_class: "i32", source: "memory", build: "1M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.40x, idle 500 ms 0.91x, 5000 ms 0.77x [jg32_inner_1M b8192: idle vs idle 0.77x after 5000 ms]; 50M: warm 1.58x, idle 500 ms 0.99x, 5000 ms 0.87x [jg32_inner_1M b8192: idle vs idle 0.87x after 5000 ms]" },
    JoinRow { how: "inner", key_class: "i64", source: "memory", build: "10M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.46x, idle 500 ms 0.88x, 5000 ms 0.81x [jg_inner_10M b8192: idle vs idle 0.81x after 5000 ms]; 50M: warm 1.28x, idle 500 ms 0.97x, 5000 ms 0.95x [jg_inner_10M part1: warm 1.28x]" },
    JoinRow { how: "inner", key_class: "i64", source: "memory", build: "10k", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.25x, idle 500 ms 0.54x, 5000 ms 0.59x [jg_inner_10k b8192: idle vs idle 0.54x after 500 ms]; 50M: warm 1.03x, idle 500 ms 0.86x, 5000 ms 0.58x [jg_inner_10k b8192: idle vs idle 0.58x after 5000 ms]" },
    JoinRow { how: "inner", key_class: "i64", source: "memory", build: "1M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.35x, idle 500 ms 0.81x, 5000 ms 0.69x [j_inner_1M b8192: idle vs idle 0.69x after 5000 ms]; 50M: warm 1.49x, idle 500 ms 0.93x, 5000 ms 0.95x [j_inner_1M b8192: idle vs idle 0.93x after 500 ms]" },
    JoinRow { how: "inner", key_class: "string", source: "memory", build: "10M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 0.79x, idle 500 ms 0.56x, 5000 ms 0.54x [js_inner_10M part1: warm 0.79x]; 50M: warm 0.81x, idle 500 ms 0.64x, 5000 ms 0.60x [js_inner_10M b8192: warm 0.81x]" },
    JoinRow { how: "inner", key_class: "string", source: "memory", build: "10k", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 0.95x, idle 500 ms 0.39x, 5000 ms 0.47x [js_inner_10k b8192: idle vs idle 0.39x after 500 ms]; 50M: warm 0.90x, idle 500 ms 0.60x, 5000 ms 0.56x [js_inner_10k b8192: idle vs idle 0.56x after 5000 ms]" },
    JoinRow { how: "inner", key_class: "string", source: "memory", build: "1M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 0.92x, idle 500 ms 0.62x, 5000 ms 0.49x [jgs_inner_1M part1: idle vs idle 0.49x after 5000 ms]; 50M: warm 0.98x, idle 500 ms 0.64x, 5000 ms 0.59x [js_inner_1M b8192: idle vs idle 0.59x after 5000 ms]" },
    JoinRow { how: "right", key_class: "i32", source: "memory", build: "10M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.33x, idle 500 ms 0.93x, 5000 ms 0.88x [jg32_left_10M b8192: idle vs idle 0.88x after 5000 ms]; 50M: warm 1.15x, idle 500 ms 0.99x, 5000 ms 1.00x [jg32_left_10M b8192: warm 1.15x]" },
    JoinRow { how: "right", key_class: "i32", source: "memory", build: "10k", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.33x, idle 500 ms 0.78x, 5000 ms 0.81x [jg32_left_10k b8192: idle vs idle 0.78x after 500 ms]; 50M: warm 1.19x, idle 500 ms 1.04x, 5000 ms 0.85x [jg32_left_10k b8192: warm 1.19x]" },
    JoinRow { how: "right", key_class: "i32", source: "memory", build: "1M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.22x, idle 500 ms 0.73x, 5000 ms 0.63x [j32_left_1M b8192: idle vs idle 0.63x after 5000 ms]; 50M: warm 1.37x, idle 500 ms 0.94x, 5000 ms 0.98x [jg32_left_1M part1: warm 1.37x]" },
    JoinRow { how: "right", key_class: "i64", source: "memory", build: "10M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.26x, idle 500 ms 0.85x, 5000 ms 0.86x [jg_left_10M part1: warm 1.26x]; 50M: warm 1.13x, idle 500 ms 0.94x, 5000 ms 0.93x [jg_left_10M b8192: warm 1.13x]" },
    JoinRow { how: "right", key_class: "i64", source: "memory", build: "10k", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.16x, idle 500 ms 0.93x, 5000 ms 0.70x [jg_left_10k part1: idle vs idle 0.70x after 5000 ms]; 50M: warm 1.16x, idle 500 ms 0.69x, 5000 ms 0.63x [jg_left_10k b8192: idle vs idle 0.63x after 5000 ms]" },
    JoinRow { how: "right", key_class: "i64", source: "memory", build: "1M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 1.22x, idle 500 ms 0.93x, 5000 ms 0.67x [j_left_1M part1: idle vs idle 0.67x after 5000 ms]; 50M: warm 1.26x, idle 500 ms 0.99x, 5000 ms 0.78x [j_left_1M b8192: idle vs idle 0.78x after 5000 ms]" },
    JoinRow { how: "right", key_class: "string", source: "memory", build: "10M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 0.51x, idle 500 ms 0.38x, 5000 ms 0.39x [js_left_10M part1: warm 0.51x]; 50M: warm 0.41x, idle 500 ms 0.36x, 5000 ms 0.36x [js_left_10M part1: warm 0.41x]" },
    JoinRow { how: "right", key_class: "string", source: "memory", build: "10k", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 0.28x, idle 500 ms 0.32x, 5000 ms 0.29x [js_left_10k b8192: warm 0.28x]; 50M: warm 0.22x, idle 500 ms 0.22x, 5000 ms 0.24x [js_left_10k b8192: warm 0.22x]" },
    JoinRow { how: "right", key_class: "string", source: "memory", build: "1M", min_probe_rows: None, reason: "not taken at any measured probe size; 10M: warm 0.35x, idle 500 ms 0.37x, 5000 ms 0.26x [js_left_1M part1: warm 0.35x]; 50M: warm 0.33x, idle 500 ms 0.31x, 5000 ms 0.30x [js_left_1M b8192: warm 0.33x]" },
];