1use vortex_error::VortexExpect;
5use vortex_error::VortexResult;
6use vortex_error::vortex_bail;
7use vortex_session::VortexSession;
8use vortex_session::registry::CachedId;
9
10use crate::ArrayRef;
11use crate::ExecutionCtx;
12use crate::IntoArray;
13use crate::aggregate_fn::Accumulator;
14use crate::aggregate_fn::AggregateArgs;
15use crate::aggregate_fn::AggregateFnId;
16use crate::aggregate_fn::DynAccumulator;
17use crate::aggregate_fn::NumericalAggregateOpts;
18use crate::aggregate_fn::combined::BinaryCombined;
19use crate::aggregate_fn::combined::Combined;
20use crate::aggregate_fn::combined::CombinedOptions;
21use crate::aggregate_fn::combined::PairOptions;
22use crate::aggregate_fn::fns::count::Count;
23use crate::aggregate_fn::fns::sum::Sum;
24use crate::aggregate_fn::fns::sum::sum_decimal_dtype;
25use crate::arrays::ConstantArray;
26use crate::builtins::ArrayBuiltins;
27use crate::dtype::DType;
28use crate::dtype::DecimalDType;
29use crate::dtype::MAX_PRECISION;
30use crate::dtype::MAX_SCALE;
31use crate::dtype::Nullability;
32use crate::dtype::PType;
33use crate::dtype::i256;
34use crate::scalar::DecimalValue;
35use crate::scalar::Scalar;
36use crate::scalar_fn::fns::operators::Operator;
37
38pub fn mean(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Scalar> {
42 let mut acc = Accumulator::try_new(
43 Mean::combined(),
44 PairOptions(
45 NumericalAggregateOpts::default(),
46 NumericalAggregateOpts::default(),
47 ),
48 array.dtype().clone(),
49 )?;
50 acc.accumulate(array, ctx)?;
51 acc.finish()
52}
53
54#[derive(Clone, Debug)]
61pub struct Mean;
62
63impl Mean {
64 pub fn combined() -> Combined<Self> {
65 Combined(Mean)
66 }
67}
68
69impl BinaryCombined for Mean {
70 type Left = Sum;
71 type Right = Count;
72
73 fn id(&self) -> AggregateFnId {
74 static ID: CachedId = CachedId::new("vortex.mean");
75 *ID
76 }
77
78 fn left(&self) -> Sum {
79 Sum
80 }
81
82 fn right(&self) -> Count {
83 Count
84 }
85
86 fn left_name(&self) -> &'static str {
87 "sum"
88 }
89
90 fn right_name(&self) -> &'static str {
91 "count"
92 }
93
94 fn return_dtype(&self, _options: &CombinedOptions<Self>, input_dtype: &DType) -> Option<DType> {
95 Some(mean_output_dtype(input_dtype)?.with_nullability(Nullability::Nullable))
96 }
97
98 fn finalize(
99 &self,
100 args: AggregateArgs<'_, CombinedOptions<Self>>,
101 sum: ArrayRef,
102 count: ArrayRef,
103 ) -> VortexResult<ArrayRef> {
104 if let DType::Decimal(..) = sum.dtype() {
105 vortex_bail!("grouped mean over decimals is not yet supported");
106 }
107 let target = args.return_dtype.clone();
108 let sum = sum.cast(target.clone())?;
109 let count = count.cast(target.clone())?;
110
111 let non_zero = count
112 .binary(
113 ConstantArray::new(Scalar::zero_value(&target), count.len()).into_array(),
114 Operator::NotEq,
115 )?
116 .fill_null(false)?;
117 let count = count.mask(non_zero)?;
121
122 sum.binary(count, Operator::Div)
123 }
124
125 fn finalize_scalar(
126 &self,
127 args: AggregateArgs<'_, CombinedOptions<Self>>,
128 left_scalar: Scalar,
129 right_scalar: Scalar,
130 ) -> VortexResult<Scalar> {
131 if let DType::Decimal(decimal_dtype, _) = *left_scalar.dtype() {
132 return finalize_decimal_scalar(
133 &left_scalar,
134 &right_scalar,
135 decimal_dtype,
136 args.return_dtype,
137 );
138 }
139
140 let target = args.return_dtype.clone();
141 let sum_cast = left_scalar.cast(&target)?;
142 let count_cast = right_scalar.cast(&target)?;
143
144 let sum = sum_cast.as_primitive().typed_value::<f64>();
145 let count = count_cast.as_primitive().typed_value::<f64>();
146 let value = match (sum, count) {
147 (None, _) | (_, None) | (_, Some(0.0)) => return Ok(Scalar::null(target)),
149 (Some(s), Some(c)) => s / c,
150 };
151 Ok(Scalar::primitive(value, Nullability::Nullable))
152 }
153
154 fn serialize(&self, options: &CombinedOptions<Self>) -> VortexResult<Option<Vec<u8>>> {
155 Ok(Some(vec![
156 u8::from(options.0.skip_nans),
157 u8::from(options.1.skip_nans),
158 ]))
159 }
160
161 fn deserialize(
162 &self,
163 metadata: &[u8],
164 _session: &VortexSession,
165 ) -> VortexResult<CombinedOptions<Self>> {
166 match metadata {
167 &[sum @ 0..=1, count @ 0..=1] => Ok(PairOptions(
168 NumericalAggregateOpts {
169 skip_nans: sum != 0,
170 },
171 NumericalAggregateOpts {
172 skip_nans: count != 0,
173 },
174 )),
175 _ => vortex_bail!("Invalid Mean options: expected two boolean bytes"),
176 }
177 }
178}
179
180fn mean_output_dtype(input_dtype: &DType) -> Option<DType> {
181 match input_dtype {
182 DType::Bool(_) | DType::Primitive(..) => {
183 Some(DType::Primitive(PType::F64, Nullability::Nullable))
184 }
185 DType::Decimal(decimal_dtype, _) => Some(DType::Decimal(
186 mean_decimal_dtype(&sum_decimal_dtype(decimal_dtype)),
187 Nullability::Nullable,
188 )),
189 _ => None,
190 }
191}
192
193fn mean_decimal_dtype(sum: &DecimalDType) -> DecimalDType {
195 DecimalDType::new(
196 u8::min(MAX_PRECISION, sum.precision().saturating_sub(6)),
197 i8::min(MAX_SCALE, sum.scale() + 4),
198 )
199}
200
201fn finalize_decimal_scalar(
202 sum: &Scalar,
203 count: &Scalar,
204 sum_decimal: DecimalDType,
205 target_dtype: &DType,
206) -> VortexResult<Scalar> {
207 let target_decimal_dtype = *target_dtype
208 .as_decimal_opt()
209 .vortex_expect("decimal mean result dtype");
210
211 let Some(sum_value) = sum.as_decimal().decimal_value() else {
213 return Ok(Scalar::null(target_dtype.clone()));
214 };
215 let count = count.as_primitive().typed_value::<u64>().unwrap_or(0);
217 if count == 0 {
218 return Ok(Scalar::null(target_dtype.clone()));
219 }
220
221 let Ok(sum) = DecimalValue::rescale_i256(
222 sum_value.as_i256(),
223 sum_decimal.scale(),
224 target_decimal_dtype.scale(),
225 ) else {
226 return Ok(Scalar::null(target_dtype.clone()));
227 };
228 let mean = sum / i256::from_i128(i128::from(count));
229
230 let Ok(mean) = DecimalValue::try_from_i256(mean, target_decimal_dtype) else {
231 return Ok(Scalar::null(target_dtype.clone()));
232 };
233 Ok(Scalar::decimal(
234 mean,
235 target_decimal_dtype,
236 Nullability::Nullable,
237 ))
238}
239
240#[cfg(test)]
241mod tests {
242 use std::sync::LazyLock;
243
244 use vortex_buffer::buffer;
245 use vortex_error::VortexResult;
246 use vortex_session::VortexSession;
247
248 use super::*;
249 use crate::VortexSessionExecute;
250 use crate::aggregate_fn::DynGroupedAccumulator;
251 use crate::aggregate_fn::GroupedAccumulator;
252 use crate::arrays::BoolArray;
253 use crate::arrays::ChunkedArray;
254 use crate::arrays::DecimalArray;
255 use crate::arrays::FixedSizeListArray;
256 use crate::arrays::PrimitiveArray;
257 use crate::assert_arrays_eq;
258 use crate::dtype::DecimalDType;
259 use crate::validity::Validity;
260
261 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
262
263 #[test]
264 fn mean_all_valid() -> VortexResult<()> {
265 let array = PrimitiveArray::new(buffer![1.0f64, 2.0, 3.0, 4.0, 5.0], Validity::NonNullable)
266 .into_array();
267 let mut ctx = SESSION.create_execution_ctx();
268 let result = mean(&array, &mut ctx)?;
269 assert_eq!(result.as_primitive().as_::<f64>(), Some(3.0));
270 Ok(())
271 }
272
273 #[test]
274 fn mean_with_nulls() -> VortexResult<()> {
275 let array = PrimitiveArray::from_option_iter([Some(2.0f64), None, Some(4.0)]).into_array();
276 let mut ctx = SESSION.create_execution_ctx();
277 let result = mean(&array, &mut ctx)?;
278 assert_eq!(result.as_primitive().as_::<f64>(), Some(3.0));
279 Ok(())
280 }
281
282 #[test]
283 fn mean_integers() -> VortexResult<()> {
284 let array = PrimitiveArray::new(buffer![10i32, 20, 30], Validity::NonNullable).into_array();
285 let mut ctx = SESSION.create_execution_ctx();
286 let result = mean(&array, &mut ctx)?;
287 assert_eq!(result.as_primitive().as_::<f64>(), Some(20.0));
288 Ok(())
289 }
290
291 #[test]
292 fn mean_bool() -> VortexResult<()> {
293 let array: BoolArray = [true, false, true, true].into_iter().collect();
294 let mut ctx = SESSION.create_execution_ctx();
295 let result = mean(&array.into_array(), &mut ctx)?;
296 assert_eq!(result.as_primitive().as_::<f64>(), Some(0.75));
297 Ok(())
298 }
299
300 #[test]
301 fn mean_constant_non_null() -> VortexResult<()> {
302 let array = ConstantArray::new(5.0f64, 4);
303 let mut ctx = SESSION.create_execution_ctx();
304 let result = mean(&array.into_array(), &mut ctx)?;
305 assert_eq!(result.as_primitive().as_::<f64>(), Some(5.0));
306 Ok(())
307 }
308
309 #[test]
310 fn mean_chunked() -> VortexResult<()> {
311 let chunk1 = PrimitiveArray::from_option_iter([Some(1.0f64), None, Some(3.0)]);
312 let chunk2 = PrimitiveArray::from_option_iter([Some(5.0f64), None]);
313 let dtype = chunk1.dtype().clone();
314 let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
315 let mut ctx = SESSION.create_execution_ctx();
316 let result = mean(&chunked.into_array(), &mut ctx)?;
317 assert_eq!(result.as_primitive().as_::<f64>(), Some(3.0));
318 Ok(())
319 }
320
321 #[test]
322 fn mean_skips_nans_by_default() -> VortexResult<()> {
323 let array =
325 PrimitiveArray::new(buffer![1.0f64, f64::NAN, 3.0], Validity::NonNullable).into_array();
326 let mut ctx = SESSION.create_execution_ctx();
327 let result = mean(&array, &mut ctx)?;
328 assert_eq!(result.as_primitive().as_::<f64>(), Some(2.0));
329 Ok(())
330 }
331
332 #[test]
333 fn mean_with_nan_not_skipping() -> VortexResult<()> {
334 let array =
335 PrimitiveArray::new(buffer![1.0f64, f64::NAN, 3.0], Validity::NonNullable).into_array();
336 let mut ctx = SESSION.create_execution_ctx();
337 let keep_nans = NumericalAggregateOpts::include_nans();
338 let mut acc = Accumulator::try_new(
339 Mean::combined(),
340 PairOptions(keep_nans, keep_nans),
341 array.dtype().clone(),
342 )?;
343 acc.accumulate(&array, &mut ctx)?;
344 let result = acc.finish()?;
345 assert!(result.as_primitive().as_::<f64>().is_some_and(f64::is_nan));
346 Ok(())
347 }
348
349 #[test]
350 fn mean_all_null_returns_null() -> VortexResult<()> {
351 let array = PrimitiveArray::from_option_iter::<f64, _>([None, None, None]).into_array();
352 let mut ctx = SESSION.create_execution_ctx();
353 let result = mean(&array, &mut ctx)?;
354 assert_eq!(result.as_primitive().as_::<f64>(), None);
355 Ok(())
356 }
357
358 #[test]
359 fn mean_decimal() -> VortexResult<()> {
360 let dtype = DecimalDType::new(6, 2);
361 let array =
362 DecimalArray::new(buffer![100i32, 200, 300], dtype, Validity::NonNullable).into_array();
363 let mut ctx = SESSION.create_execution_ctx();
364 let result = mean(&array, &mut ctx)?;
365 assert_eq!(
366 result.dtype(),
367 &DType::Decimal(DecimalDType::new(10, 6), Nullability::Nullable)
368 );
369 assert_eq!(
371 result.as_decimal().decimal_value(),
372 Some(DecimalValue::I256(i256::from_i128(2_000_000)))
373 );
374 Ok(())
375 }
376
377 #[test]
378 fn mean_decimal_null() -> VortexResult<()> {
379 let dtype = DecimalDType::new(6, 2);
380 let validity = Validity::from_iter([true, false, true]);
381 let array = DecimalArray::new(buffer![150i32, 0, 450], dtype, validity).into_array();
382 let mut ctx = SESSION.create_execution_ctx();
383 let result = mean(&array, &mut ctx)?;
384 assert_eq!(
386 result.as_decimal().decimal_value(),
387 Some(DecimalValue::I256(i256::from_i128(3_000_000)))
388 );
389 assert_eq!(
390 result.dtype(),
391 &DType::Decimal(DecimalDType::new(10, 6), Nullability::Nullable)
392 );
393 Ok(())
394 }
395
396 #[test]
397 fn mean_decimal_chunked() -> VortexResult<()> {
398 let dtype = DecimalDType::new(6, 2);
399 let validity = Validity::NonNullable;
400 let chunk1 = DecimalArray::new(buffer![100i32, 200], dtype, validity.clone()).into_array();
401 let chunk2 = DecimalArray::new(buffer![300i32, 400, 500], dtype, validity).into_array();
402 let dtype = chunk1.dtype().clone();
403 let chunked = ChunkedArray::try_new(vec![chunk1, chunk2], dtype)?;
404 let mut ctx = SESSION.create_execution_ctx();
405 let result = mean(&chunked.into_array(), &mut ctx)?;
406 assert_eq!(
408 result.as_decimal().decimal_value(),
409 Some(DecimalValue::I256(i256::from_i128(3_000_000)))
410 );
411 assert_eq!(
412 result.dtype(),
413 &DType::Decimal(DecimalDType::new(10, 6), Nullability::Nullable)
414 );
415 Ok(())
416 }
417
418 #[test]
419 fn mean_decimal_33() -> VortexResult<()> {
420 let dtype = DecimalDType::new(6, 2);
421 let buf = buffer![100i32, 0, 0];
422 let array = DecimalArray::new(buf, dtype, Validity::NonNullable).into_array();
423 let mut ctx = SESSION.create_execution_ctx();
424 let result = mean(&array, &mut ctx)?;
425 assert_eq!(
427 result.as_decimal().decimal_value(),
428 Some(DecimalValue::I256(i256::from_i128(333_333)))
429 );
430 assert_eq!(
431 result.dtype(),
432 &DType::Decimal(DecimalDType::new(10, 6), Nullability::Nullable)
433 );
434 Ok(())
435 }
436
437 #[test]
438 fn mean_multi_batch() -> VortexResult<()> {
439 let mut ctx = SESSION.create_execution_ctx();
440 let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
441 let mut acc = Accumulator::try_new(
442 Mean::combined(),
443 PairOptions(
444 NumericalAggregateOpts::default(),
445 NumericalAggregateOpts::default(),
446 ),
447 dtype,
448 )?;
449
450 let batch1 =
451 PrimitiveArray::new(buffer![1.0f64, 2.0, 3.0], Validity::NonNullable).into_array();
452 acc.accumulate(&batch1, &mut ctx)?;
453
454 let batch2 = PrimitiveArray::new(buffer![4.0f64, 5.0], Validity::NonNullable).into_array();
455 acc.accumulate(&batch2, &mut ctx)?;
456
457 let result = acc.finish()?;
458 assert_eq!(result.as_primitive().as_::<f64>(), Some(3.0));
459 Ok(())
460 }
461
462 fn mean_nan_null() -> Vec<(Vec<Option<f64>>, Option<f64>)> {
463 vec![
464 (vec![Some(f64::NAN), Some(1.0), None], Some(1.0)),
465 (vec![Some(f64::NAN), Some(1.0), Some(3.0)], Some(2.0)),
466 (vec![None, None, Some(f64::NAN)], None),
467 (vec![None, None, None], None),
468 (vec![Some(1.0), Some(2.0), Some(3.0)], Some(2.0)),
469 ]
470 }
471
472 #[test]
473 fn mean_combined_partials() -> VortexResult<()> {
474 let mut ctx = SESSION.create_execution_ctx();
475 for (case, (group, expected)) in mean_nan_null().into_iter().enumerate() {
476 let mut acc = Accumulator::try_new(
477 Mean::combined(),
478 PairOptions(
479 NumericalAggregateOpts::default(),
480 NumericalAggregateOpts::default(),
481 ),
482 DType::Primitive(PType::F64, Nullability::Nullable),
483 )?;
484 let (head, tail) = group.split_at(2);
485 let head = PrimitiveArray::from_option_iter(head.iter().copied()).into_array();
486 let tail = PrimitiveArray::from_option_iter(tail.iter().copied()).into_array();
487 acc.accumulate(&head, &mut ctx)?;
488 acc.accumulate(&tail, &mut ctx)?;
489 let result = acc.finish()?;
490 assert_eq!(result.as_primitive().as_::<f64>(), expected, "case {case}");
491 }
492 Ok(())
493 }
494
495 #[test]
496 fn mean_grouped_finalize() -> VortexResult<()> {
497 let cases = mean_nan_null();
498 let elements = PrimitiveArray::from_option_iter(
499 cases.iter().flat_map(|(group, _)| group.iter().copied()),
500 )
501 .into_array();
502 let groups = FixedSizeListArray::try_new(elements, 3, Validity::NonNullable, cases.len())?;
503
504 let mut acc = GroupedAccumulator::try_new(
505 Mean::combined(),
506 PairOptions(
507 NumericalAggregateOpts::default(),
508 NumericalAggregateOpts::default(),
509 ),
510 DType::Primitive(PType::F64, Nullability::Nullable),
511 )?;
512 let mut ctx = SESSION.create_execution_ctx();
513 acc.accumulate_list(&groups.into_array(), &mut ctx)?;
514 let result = acc.finish()?;
515
516 for (case, (_, expected)) in cases.into_iter().enumerate() {
517 let actual = result.execute_scalar(case, &mut ctx)?;
518 assert_eq!(actual.as_primitive().as_::<f64>(), expected, "case {case}");
519 }
520 let expected =
521 PrimitiveArray::from_option_iter([Some(1.0f64), Some(2.0), None, None, Some(2.0)])
522 .into_array();
523 assert_arrays_eq!(&result, &expected, &mut ctx);
524 Ok(())
525 }
526}