mod group_key_cmp;
use super::super::select_ast::AggregateType;
use super::super::select_optimizer::{AggregateComputation, AggregationPlan};
use super::numeric_acc::NumericAcc;
use super::value_ops::compare_values_ordering;
use crate::schema::CqlType;
use crate::{
query::result::QueryRow,
types::{RowKey, Value},
};
use group_key_cmp::{group_key_eq, hash_group_key};
use rustc_hash::FxHashMap;
use std::collections::HashMap;
#[cfg(test)]
thread_local! {
pub(crate) static GROUP_KEY_COMPARISONS: std::cell::Cell<usize> =
const { std::cell::Cell::new(0) };
}
#[derive(Debug)]
pub(super) struct AggregationState {
pub(super) groups: Vec<(Vec<Value>, Vec<AggregateValue>)>,
pub(super) group_index: FxHashMap<u64, Vec<usize>>,
pub(super) memory_usage_bytes: usize,
pub(super) memory_limit_bytes: usize,
}
#[derive(Debug, Clone)]
pub(super) enum AggregateValue {
Count(u64),
Sum(NumericAcc),
Avg { acc: NumericAcc, count: u64 },
Min(Value),
Max(Value),
}
pub(super) fn init_aggregate_accumulators(
aggregates: &[AggregateComputation],
result_types: &[Option<CqlType>],
) -> Vec<AggregateValue> {
aggregates
.iter()
.enumerate()
.map(|(i, c)| {
let result_type = result_types.get(i).and_then(|t| t.as_ref());
match c.function {
AggregateType::Count => AggregateValue::Count(0),
AggregateType::Sum => AggregateValue::Sum(NumericAcc::init(result_type)),
AggregateType::Avg => AggregateValue::Avg {
acc: NumericAcc::init(result_type),
count: 0,
},
AggregateType::Min => AggregateValue::Min(Value::Null),
AggregateType::Max => AggregateValue::Max(Value::Null),
}
})
.collect()
}
pub(super) fn build_group_key(row: &QueryRow, group_by_columns: &[String]) -> Vec<Value> {
if group_by_columns.is_empty() {
return vec![Value::Null];
}
group_by_columns
.iter()
.map(|col| row.values.get(col.as_str()).cloned().unwrap_or(Value::Null))
.collect()
}
pub(super) fn find_or_init_group(
state: &mut AggregationState,
key: Vec<Value>,
aggregates: &[AggregateComputation],
result_types: &[Option<CqlType>],
) -> usize {
let hash = hash_group_key(&key);
if let Some(candidates) = state.group_index.get(&hash) {
for &idx in candidates {
#[cfg(test)]
GROUP_KEY_COMPARISONS.with(|c| c.set(c.get() + 1));
if group_key_eq(&state.groups[idx].0, &key) {
return idx;
}
}
}
let initial = init_aggregate_accumulators(aggregates, result_types);
let new_index = state.groups.len();
state.groups.push((key, initial));
state.group_index.entry(hash).or_default().push(new_index);
new_index
}
pub(super) fn update_aggregate(
state: &mut AggregateValue,
agg_comp: &AggregateComputation,
row: &QueryRow,
) {
let is_star = agg_comp.column == "*";
let value: Option<&Value> = if is_star {
None
} else {
row.values.get(agg_comp.column.as_str())
};
let is_null = !is_star && value.is_none_or(Value::is_null);
match state {
AggregateValue::Count(count) => {
if is_star || !is_null {
*count += 1;
}
}
AggregateValue::Sum(acc) => {
if let Some(v) = value {
if !v.is_null() {
acc.add(v);
}
}
}
AggregateValue::Avg { acc, count } => {
if let Some(v) = value {
if !v.is_null() && acc.add(v) {
*count += 1;
}
}
}
AggregateValue::Min(min_val) => {
if let Some(v) = value {
if !v.is_null()
&& (min_val.is_null() || compare_values_ordering(v, min_val).is_lt())
{
*min_val = v.clone();
}
}
}
AggregateValue::Max(max_val) => {
if let Some(v) = value {
if !v.is_null()
&& (max_val.is_null() || compare_values_ordering(v, max_val).is_gt())
{
*max_val = v.clone();
}
}
}
}
}
pub(super) fn finalize_group(
group_key: Vec<Value>,
group_aggregates: Vec<AggregateValue>,
agg_plan: &AggregationPlan,
) -> QueryRow {
let mut row_values: HashMap<std::sync::Arc<str>, Value> =
HashMap::with_capacity(agg_plan.group_by_columns.len() + agg_plan.aggregates.len());
for (i, out_name) in agg_plan.group_by_output_names.iter().enumerate() {
if let Some(v) = group_key.get(i) {
row_values.insert(out_name.as_str().into(), v.clone());
}
}
for (i, agg_comp) in agg_plan.aggregates.iter().enumerate() {
let result_value = match &group_aggregates[i] {
AggregateValue::Count(count) => Value::BigInt(*count as i64),
AggregateValue::Sum(acc) => acc.finalize_sum(),
AggregateValue::Avg { acc, count } => {
if *count > 0 {
acc.finalize_avg(*count)
} else {
Value::Null
}
}
AggregateValue::Min(val) | AggregateValue::Max(val) => val.clone(),
};
row_values.insert(agg_comp.alias.as_str().into(), result_value);
}
QueryRow {
values: row_values,
key: RowKey::new(vec![]),
metadata: Default::default(),
cell_metadata: None,
}
}
pub(super) fn finalize_empty_global_aggregate(agg_plan: &AggregationPlan) -> QueryRow {
let mut row_values: HashMap<std::sync::Arc<str>, Value> =
HashMap::with_capacity(agg_plan.aggregates.len());
for agg_comp in &agg_plan.aggregates {
let result_value = match agg_comp.function {
AggregateType::Count => Value::BigInt(0),
AggregateType::Sum | AggregateType::Avg | AggregateType::Min | AggregateType::Max => {
Value::Null
}
};
row_values.insert(agg_comp.alias.as_str().into(), result_value);
}
QueryRow {
values: row_values,
key: RowKey::new(vec![]),
metadata: Default::default(),
cell_metadata: None,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::query::select_ast::AggregateType;
fn count_star_plan() -> AggregationPlan {
AggregationPlan {
group_by_columns: vec!["g".to_string()],
group_by_output_names: vec!["g".to_string()],
aggregates: vec![AggregateComputation {
function: AggregateType::Count,
column: "*".to_string(),
alias: "c".to_string(),
distinct: false,
}],
}
}
fn new_state() -> AggregationState {
AggregationState {
groups: Vec::new(),
group_index: FxHashMap::default(),
memory_usage_bytes: 0,
memory_limit_bytes: usize::MAX,
}
}
#[test]
fn find_or_init_group_is_subquadratic() {
let plan = count_star_plan();
let groups_count = 200usize;
let repeats = 3usize; let rows = groups_count * repeats;
let mut state = new_state();
GROUP_KEY_COMPARISONS.with(|c| c.set(0));
for r in 0..repeats {
for g in 0..groups_count {
let key = vec![Value::Integer(g as i32)];
let idx = find_or_init_group(&mut state, key, &plan.aggregates, &[]);
assert_eq!(idx, g, "row (repeat {r}, group {g}) maps to a stable index");
}
}
let comparisons = GROUP_KEY_COMPARISONS.with(|c| c.get());
assert_eq!(
state.groups.len(),
groups_count,
"exactly one group per key"
);
assert!(
comparisons <= rows * 2,
"issue #1587: group lookup must be sub-quadratic — {comparisons} comparisons \
for {rows} rows / {groups_count} groups (linear scan would be ~{})",
rows * groups_count / 2
);
}
#[test]
fn finalize_empty_global_aggregate_count_zero_others_null() {
let agg = |function, alias: &str| AggregateComputation {
function,
column: "x".to_string(),
alias: alias.to_string(),
distinct: false,
};
let plan = AggregationPlan {
group_by_columns: Vec::new(),
group_by_output_names: Vec::new(),
aggregates: vec![
agg(AggregateType::Count, "c"),
agg(AggregateType::Sum, "s"),
agg(AggregateType::Avg, "a"),
agg(AggregateType::Min, "mn"),
agg(AggregateType::Max, "mx"),
],
};
let row = finalize_empty_global_aggregate(&plan);
assert_eq!(
row.values.get("c"),
Some(&Value::BigInt(0)),
"COUNT of empty input is 0"
);
for (alias, kind) in [("s", "SUM"), ("a", "AVG"), ("mn", "MIN"), ("mx", "MAX")] {
assert_eq!(
row.values.get(alias),
Some(&Value::Null),
"{kind} of empty input is NULL (not 0)"
);
}
}
#[test]
fn find_or_init_group_uses_cassandra_float_comparator() {
let plan = count_star_plan();
let mut state = new_state();
let mut group =
|f: f64| find_or_init_group(&mut state, vec![Value::Float(f)], &plan.aggregates, &[]);
let i0 = group(0.0);
let i1 = group(-0.0);
assert_ne!(i0, i1, "-0.0 and +0.0 must land in DISTINCT groups (#2074)");
let other_nan = f64::from_bits(0xFFF8_0000_0000_0001);
assert!(other_nan.is_nan());
let n0 = group(f64::NAN);
let n1 = group(f64::NAN);
let n2 = group(other_nan);
assert_eq!(n0, n1, "all NaN keys share ONE group (#2074)");
assert_eq!(
n0, n2,
"a DIFFERENT NaN bit pattern also joins the single NaN group (#2074)"
);
let f32key = |f: f32| vec![Value::Float32(f)];
let f0 = find_or_init_group(&mut state, f32key(0.0), &plan.aggregates, &[]);
let f1 = find_or_init_group(&mut state, f32key(-0.0), &plan.aggregates, &[]);
assert_ne!(f0, f1, "f32 -0.0 and +0.0 must be DISTINCT groups (#2074)");
let other_nan32 = f32::from_bits(0xFFC0_0001);
assert!(other_nan32.is_nan());
let g0 = find_or_init_group(&mut state, f32key(f32::NAN), &plan.aggregates, &[]);
let g1 = find_or_init_group(&mut state, f32key(other_nan32), &plan.aggregates, &[]);
assert_eq!(g0, g1, "f32 NaN bit patterns share ONE group (#2074)");
let a = find_or_init_group(&mut state, vec![Value::Integer(1)], &plan.aggregates, &[]);
let b = find_or_init_group(&mut state, vec![Value::BigInt(1)], &plan.aggregates, &[]);
assert_ne!(a, b, "Integer(1) and BigInt(1) are distinct groups");
}
#[test]
fn find_or_init_group_nested_float_uses_cassandra_comparator() {
let plan = count_star_plan();
let mut state = new_state();
let other_nan = f64::from_bits(0xFFF8_0000_0000_0001);
assert!(other_nan.is_nan());
let mut group = |k: Vec<Value>| find_or_init_group(&mut state, k, &plan.aggregates, &[]);
let t0 = group(vec![Value::Tuple(vec![Value::Float(0.0)])]);
let t1 = group(vec![Value::Tuple(vec![Value::Float(-0.0)])]);
assert_ne!(t0, t1, "nested Tuple -0.0/+0.0 must be DISTINCT (#2074)");
let n0 = group(vec![Value::Tuple(vec![Value::Float(f64::NAN)])]);
let n1 = group(vec![Value::Tuple(vec![Value::Float(other_nan)])]);
assert_eq!(
n0, n1,
"nested Tuple differing-NaN must be ONE group (#2074)"
);
let l0 = group(vec![Value::List(vec![Value::Float(0.0)])]);
let l1 = group(vec![Value::List(vec![Value::Float(-0.0)])]);
assert_ne!(l0, l1, "nested List -0.0/+0.0 must be DISTINCT (#2074)");
}
fn scalar_agg_plan(function: AggregateType, col: &str) -> AggregationPlan {
AggregationPlan {
group_by_columns: Vec::new(),
group_by_output_names: Vec::new(),
aggregates: vec![AggregateComputation {
function,
column: col.to_string(),
alias: "r".to_string(),
distinct: false,
}],
}
}
fn run_scalar_agg(
function: AggregateType,
result_type: Option<CqlType>,
values: &[Value],
) -> Value {
let plan = scalar_agg_plan(function, "v");
let result_types = vec![result_type];
let mut accs = init_aggregate_accumulators(&plan.aggregates, &result_types);
for v in values {
let row = QueryRow {
values: [(std::sync::Arc::<str>::from("v"), v.clone())]
.into_iter()
.collect(),
key: RowKey::new(vec![]),
metadata: Default::default(),
cell_metadata: None,
};
update_aggregate(&mut accs[0], &plan.aggregates[0], &row);
}
let row = finalize_group(vec![Value::Null], accs, &plan);
row.values.get("r").cloned().expect("result present")
}
#[test]
fn sum_integral_inputs_preserve_integral_result() {
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::Int),
&[Value::Integer(1), Value::Integer(2), Value::Integer(3)],
),
Value::Integer(6),
"SUM(int) emits Value::Integer, not a float"
);
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::BigInt),
&[Value::BigInt(10), Value::BigInt(20)],
),
Value::BigInt(30),
"SUM(bigint) emits Value::BigInt"
);
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::SmallInt),
&[Value::SmallInt(100), Value::SmallInt(50)],
),
Value::SmallInt(150),
"SUM(smallint) stays Value::SmallInt — Cassandra does not promote to int"
);
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::TinyInt),
&[Value::TinyInt(40), Value::TinyInt(2)],
),
Value::TinyInt(42),
"SUM(tinyint) stays Value::TinyInt — Cassandra does not promote to int"
);
}
#[test]
fn sum_float_inputs_stay_double() {
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::Double),
&[Value::Float(1.5), Value::Float(2.25)],
),
Value::Float(3.75),
);
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::Float),
&[Value::Float32(1.0), Value::Float32(2.0)],
),
Value::Float(3.0),
"SUM(float) stays double (Value::Float)"
);
}
#[test]
fn avg_integral_uses_integer_division_and_preserves_type() {
assert_eq!(
run_scalar_agg(
AggregateType::Avg,
Some(CqlType::Int),
&[Value::Integer(1), Value::Integer(2)],
),
Value::Integer(1),
);
assert_eq!(
run_scalar_agg(
AggregateType::Avg,
Some(CqlType::Int),
&[Value::Integer(0), Value::Integer(0), Value::Integer(3)],
),
Value::Integer(1),
"AVG counts zeros: 3/3 = 1"
);
assert_eq!(
run_scalar_agg(
AggregateType::Avg,
Some(CqlType::BigInt),
&[Value::BigInt(10), Value::BigInt(21)],
),
Value::BigInt(15),
"AVG(bigint) integer division stays bigint"
);
assert_eq!(
run_scalar_agg(
AggregateType::Avg,
Some(CqlType::Double),
&[Value::Float(1.0), Value::Float(2.0)],
),
Value::Float(1.5),
"AVG(double) divides in f64",
);
assert_eq!(
run_scalar_agg(
AggregateType::Avg,
Some(CqlType::SmallInt),
&[Value::SmallInt(10), Value::SmallInt(21)],
),
Value::SmallInt(15),
"AVG(smallint) integer division stays smallint"
);
assert_eq!(
run_scalar_agg(
AggregateType::Avg,
Some(CqlType::TinyInt),
&[Value::TinyInt(10), Value::TinyInt(21)],
),
Value::TinyInt(15),
"AVG(tinyint) integer division stays tinyint"
);
}
#[test]
fn sum_integral_overflow_wraps() {
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::TinyInt),
&[Value::TinyInt(i8::MAX), Value::TinyInt(1)],
),
Value::TinyInt(i8::MIN),
"SUM(tinyint) wraps at the i8 boundary"
);
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::SmallInt),
&[Value::SmallInt(i16::MAX), Value::SmallInt(1)],
),
Value::SmallInt(i16::MIN),
"SUM(smallint) wraps at the i16 boundary"
);
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::Int),
&[Value::Integer(i32::MAX), Value::Integer(1)],
),
Value::Integer(i32::MIN),
"SUM(int) wraps at the i32 boundary"
);
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::BigInt),
&[Value::BigInt(i64::MAX), Value::BigInt(1)],
),
Value::BigInt(i64::MIN),
"SUM(bigint) wraps at the i64 boundary"
);
}
#[test]
fn sum_avg_ignore_nulls_typed_zero_and_null() {
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
Some(CqlType::Int),
&[Value::Null, Value::Integer(5), Value::Null],
),
Value::Integer(5),
"SUM ignores NULLs"
);
assert_eq!(
run_scalar_agg(AggregateType::Sum, Some(CqlType::Int), &[Value::Null]),
Value::Integer(0),
"all-null int SUM group is a typed integer zero (not a float)"
);
assert_eq!(
run_scalar_agg(AggregateType::Avg, Some(CqlType::Int), &[Value::Null]),
Value::Null,
"AVG over no counted inputs is NULL"
);
}
#[test]
fn sum_avg_unknown_type_falls_back_to_double() {
assert_eq!(
run_scalar_agg(
AggregateType::Sum,
None,
&[Value::Integer(1), Value::Integer(2)],
),
Value::Float(3.0),
);
}
}