Skip to main content

vortex_array/aggregate_fn/fns/mean/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use vortex_error::VortexExpect;
5use vortex_error::VortexResult;
6use vortex_error::vortex_bail;
7use vortex_session::VortexSession;
8use vortex_session::registry::CachedId;
9
10use crate::ArrayRef;
11use crate::ExecutionCtx;
12use crate::IntoArray;
13use crate::aggregate_fn::Accumulator;
14use crate::aggregate_fn::AggregateArgs;
15use crate::aggregate_fn::AggregateFnId;
16use crate::aggregate_fn::DynAccumulator;
17use crate::aggregate_fn::NumericalAggregateOpts;
18use crate::aggregate_fn::combined::BinaryCombined;
19use crate::aggregate_fn::combined::Combined;
20use crate::aggregate_fn::combined::CombinedOptions;
21use crate::aggregate_fn::combined::PairOptions;
22use crate::aggregate_fn::fns::count::Count;
23use crate::aggregate_fn::fns::sum::Sum;
24use crate::aggregate_fn::fns::sum::sum_decimal_dtype;
25use crate::arrays::ConstantArray;
26use crate::builtins::ArrayBuiltins;
27use crate::dtype::DType;
28use crate::dtype::DecimalDType;
29use crate::dtype::MAX_PRECISION;
30use crate::dtype::MAX_SCALE;
31use crate::dtype::Nullability;
32use crate::dtype::PType;
33use crate::dtype::i256;
34use crate::scalar::DecimalValue;
35use crate::scalar::Scalar;
36use crate::scalar_fn::fns::operators::Operator;
37
38/// Compute the arithmetic mean of an array.
39///
40/// See [`Mean`] for details.
41pub fn mean(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Scalar> {
42    let mut acc = Accumulator::try_new(
43        Mean::combined(),
44        PairOptions(
45            NumericalAggregateOpts::default(),
46            NumericalAggregateOpts::default(),
47        ),
48        array.dtype().clone(),
49    )?;
50    acc.accumulate(array, ctx)?;
51    acc.finish()
52}
53
54/// Compute the arithmetic mean of an array.
55///
56/// Implemented as `Sum / Count` via [`BinaryCombined`].
57///
58/// Booleans and primitive numeric types produce nullable `f64` results.
59/// Decimals produce a nullable decimal result.
60#[derive(Clone, Debug)]
61pub struct Mean;
62
63impl Mean {
64    pub fn combined() -> Combined<Self> {
65        Combined(Mean)
66    }
67}
68
69impl BinaryCombined for Mean {
70    type Left = Sum;
71    type Right = Count;
72
73    fn id(&self) -> AggregateFnId {
74        static ID: CachedId = CachedId::new("vortex.mean");
75        *ID
76    }
77
78    fn left(&self) -> Sum {
79        Sum
80    }
81
82    fn right(&self) -> Count {
83        Count
84    }
85
86    fn left_name(&self) -> &'static str {
87        "sum"
88    }
89
90    fn right_name(&self) -> &'static str {
91        "count"
92    }
93
94    fn return_dtype(&self, _options: &CombinedOptions<Self>, input_dtype: &DType) -> Option<DType> {
95        Some(mean_output_dtype(input_dtype)?.with_nullability(Nullability::Nullable))
96    }
97
98    fn finalize(
99        &self,
100        args: AggregateArgs<'_, CombinedOptions<Self>>,
101        sum: ArrayRef,
102        count: ArrayRef,
103    ) -> VortexResult<ArrayRef> {
104        if let DType::Decimal(..) = sum.dtype() {
105            vortex_bail!("grouped mean over decimals is not yet supported");
106        }
107        let target = args.return_dtype.clone();
108        let sum = sum.cast(target.clone())?;
109        let count = count.cast(target.clone())?;
110
111        let non_zero = count
112            .binary(
113                ConstantArray::new(Scalar::zero_value(&target), count.len()).into_array(),
114                Operator::NotEq,
115            )?
116            .fill_null(false)?;
117        // if count is 0, dividing by 0 below produces NaN, and we need Null.
118        // mask values to skip 0 so on 0 count turns into Null, dividing by
119        // Null is always Null
120        let count = count.mask(non_zero)?;
121
122        sum.binary(count, Operator::Div)
123    }
124
125    fn finalize_scalar(
126        &self,
127        args: AggregateArgs<'_, CombinedOptions<Self>>,
128        left_scalar: Scalar,
129        right_scalar: Scalar,
130    ) -> VortexResult<Scalar> {
131        if let DType::Decimal(decimal_dtype, _) = *left_scalar.dtype() {
132            return finalize_decimal_scalar(
133                &left_scalar,
134                &right_scalar,
135                decimal_dtype,
136                args.return_dtype,
137            );
138        }
139
140        let target = args.return_dtype.clone();
141        let sum_cast = left_scalar.cast(&target)?;
142        let count_cast = right_scalar.cast(&target)?;
143
144        let sum = sum_cast.as_primitive().typed_value::<f64>();
145        let count = count_cast.as_primitive().typed_value::<f64>();
146        let value = match (sum, count) {
147            // None sum means sum overflowed, 0 count means empty input
148            (None, _) | (_, None) | (_, Some(0.0)) => return Ok(Scalar::null(target)),
149            (Some(s), Some(c)) => s / c,
150        };
151        Ok(Scalar::primitive(value, Nullability::Nullable))
152    }
153
154    fn serialize(&self, options: &CombinedOptions<Self>) -> VortexResult<Option<Vec<u8>>> {
155        Ok(Some(vec![
156            u8::from(options.0.skip_nans),
157            u8::from(options.1.skip_nans),
158        ]))
159    }
160
161    fn deserialize(
162        &self,
163        metadata: &[u8],
164        _session: &VortexSession,
165    ) -> VortexResult<CombinedOptions<Self>> {
166        match metadata {
167            &[sum @ 0..=1, count @ 0..=1] => Ok(PairOptions(
168                NumericalAggregateOpts {
169                    skip_nans: sum != 0,
170                },
171                NumericalAggregateOpts {
172                    skip_nans: count != 0,
173                },
174            )),
175            _ => vortex_bail!("Invalid Mean options: expected two boolean bytes"),
176        }
177    }
178}
179
180fn mean_output_dtype(input_dtype: &DType) -> Option<DType> {
181    match input_dtype {
182        DType::Bool(_) | DType::Primitive(..) => {
183            Some(DType::Primitive(PType::F64, Nullability::Nullable))
184        }
185        DType::Decimal(decimal_dtype, _) => Some(DType::Decimal(
186            mean_decimal_dtype(&sum_decimal_dtype(decimal_dtype)),
187            Nullability::Nullable,
188        )),
189        _ => None,
190    }
191}
192
193/// mean() output decimal type mimicking Spark/DataFusion/MySQL: decimal(p+4, s+4)
194fn mean_decimal_dtype(sum: &DecimalDType) -> DecimalDType {
195    DecimalDType::new(
196        u8::min(MAX_PRECISION, sum.precision().saturating_sub(6)),
197        i8::min(MAX_SCALE, sum.scale() + 4),
198    )
199}
200
201fn finalize_decimal_scalar(
202    sum: &Scalar,
203    count: &Scalar,
204    sum_decimal: DecimalDType,
205    target_dtype: &DType,
206) -> VortexResult<Scalar> {
207    let target_decimal_dtype = *target_dtype
208        .as_decimal_opt()
209        .vortex_expect("decimal mean result dtype");
210
211    // overflow
212    let Some(sum_value) = sum.as_decimal().decimal_value() else {
213        return Ok(Scalar::null(target_dtype.clone()));
214    };
215    // empty input
216    let count = count.as_primitive().typed_value::<u64>().unwrap_or(0);
217    if count == 0 {
218        return Ok(Scalar::null(target_dtype.clone()));
219    }
220
221    let Ok(sum) = DecimalValue::rescale_i256(
222        sum_value.as_i256(),
223        sum_decimal.scale(),
224        target_decimal_dtype.scale(),
225    ) else {
226        return Ok(Scalar::null(target_dtype.clone()));
227    };
228    let mean = sum / i256::from_i128(i128::from(count));
229
230    let Ok(mean) = DecimalValue::try_from_i256(mean, target_decimal_dtype) else {
231        return Ok(Scalar::null(target_dtype.clone()));
232    };
233    Ok(Scalar::decimal(
234        mean,
235        target_decimal_dtype,
236        Nullability::Nullable,
237    ))
238}
239
240#[cfg(test)]
241mod tests {
242    use std::sync::LazyLock;
243
244    use vortex_buffer::buffer;
245    use vortex_error::VortexResult;
246    use vortex_session::VortexSession;
247
248    use super::*;
249    use crate::VortexSessionExecute;
250    use crate::aggregate_fn::DynGroupedAccumulator;
251    use crate::aggregate_fn::GroupedAccumulator;
252    use crate::arrays::BoolArray;
253    use crate::arrays::ChunkedArray;
254    use crate::arrays::DecimalArray;
255    use crate::arrays::FixedSizeListArray;
256    use crate::arrays::PrimitiveArray;
257    use crate::assert_arrays_eq;
258    use crate::dtype::DecimalDType;
259    use crate::validity::Validity;
260
261    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
262
263    #[test]
264    fn mean_all_valid() -> VortexResult<()> {
265        let array = PrimitiveArray::new(buffer![1.0f64, 2.0, 3.0, 4.0, 5.0], Validity::NonNullable)
266            .into_array();
267        let mut ctx = SESSION.create_execution_ctx();
268        let result = mean(&array, &mut ctx)?;
269        assert_eq!(result.as_primitive().as_::<f64>(), Some(3.0));
270        Ok(())
271    }
272
273    #[test]
274    fn mean_with_nulls() -> VortexResult<()> {
275        let array = PrimitiveArray::from_option_iter([Some(2.0f64), None, Some(4.0)]).into_array();
276        let mut ctx = SESSION.create_execution_ctx();
277        let result = mean(&array, &mut ctx)?;
278        assert_eq!(result.as_primitive().as_::<f64>(), Some(3.0));
279        Ok(())
280    }
281
282    #[test]
283    fn mean_integers() -> VortexResult<()> {
284        let array = PrimitiveArray::new(buffer![10i32, 20, 30], Validity::NonNullable).into_array();
285        let mut ctx = SESSION.create_execution_ctx();
286        let result = mean(&array, &mut ctx)?;
287        assert_eq!(result.as_primitive().as_::<f64>(), Some(20.0));
288        Ok(())
289    }
290
291    #[test]
292    fn mean_bool() -> VortexResult<()> {
293        let array: BoolArray = [true, false, true, true].into_iter().collect();
294        let mut ctx = SESSION.create_execution_ctx();
295        let result = mean(&array.into_array(), &mut ctx)?;
296        assert_eq!(result.as_primitive().as_::<f64>(), Some(0.75));
297        Ok(())
298    }
299
300    #[test]
301    fn mean_constant_non_null() -> VortexResult<()> {
302        let array = ConstantArray::new(5.0f64, 4);
303        let mut ctx = SESSION.create_execution_ctx();
304        let result = mean(&array.into_array(), &mut ctx)?;
305        assert_eq!(result.as_primitive().as_::<f64>(), Some(5.0));
306        Ok(())
307    }
308
309    #[test]
310    fn mean_chunked() -> VortexResult<()> {
311        let chunk1 = PrimitiveArray::from_option_iter([Some(1.0f64), None, Some(3.0)]);
312        let chunk2 = PrimitiveArray::from_option_iter([Some(5.0f64), None]);
313        let dtype = chunk1.dtype().clone();
314        let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
315        let mut ctx = SESSION.create_execution_ctx();
316        let result = mean(&chunked.into_array(), &mut ctx)?;
317        assert_eq!(result.as_primitive().as_::<f64>(), Some(3.0));
318        Ok(())
319    }
320
321    #[test]
322    fn mean_skips_nans_by_default() -> VortexResult<()> {
323        // NaNs are excluded from both the sum and the count.
324        let array =
325            PrimitiveArray::new(buffer![1.0f64, f64::NAN, 3.0], Validity::NonNullable).into_array();
326        let mut ctx = SESSION.create_execution_ctx();
327        let result = mean(&array, &mut ctx)?;
328        assert_eq!(result.as_primitive().as_::<f64>(), Some(2.0));
329        Ok(())
330    }
331
332    #[test]
333    fn mean_with_nan_not_skipping() -> VortexResult<()> {
334        let array =
335            PrimitiveArray::new(buffer![1.0f64, f64::NAN, 3.0], Validity::NonNullable).into_array();
336        let mut ctx = SESSION.create_execution_ctx();
337        let keep_nans = NumericalAggregateOpts::include_nans();
338        let mut acc = Accumulator::try_new(
339            Mean::combined(),
340            PairOptions(keep_nans, keep_nans),
341            array.dtype().clone(),
342        )?;
343        acc.accumulate(&array, &mut ctx)?;
344        let result = acc.finish()?;
345        assert!(result.as_primitive().as_::<f64>().is_some_and(f64::is_nan));
346        Ok(())
347    }
348
349    #[test]
350    fn mean_all_null_returns_null() -> VortexResult<()> {
351        let array = PrimitiveArray::from_option_iter::<f64, _>([None, None, None]).into_array();
352        let mut ctx = SESSION.create_execution_ctx();
353        let result = mean(&array, &mut ctx)?;
354        assert_eq!(result.as_primitive().as_::<f64>(), None);
355        Ok(())
356    }
357
358    #[test]
359    fn mean_decimal() -> VortexResult<()> {
360        let dtype = DecimalDType::new(6, 2);
361        let array =
362            DecimalArray::new(buffer![100i32, 200, 300], dtype, Validity::NonNullable).into_array();
363        let mut ctx = SESSION.create_execution_ctx();
364        let result = mean(&array, &mut ctx)?;
365        assert_eq!(
366            result.dtype(),
367            &DType::Decimal(DecimalDType::new(10, 6), Nullability::Nullable)
368        );
369        // mean(1.00, 2.00, 3.00) = 2.000000
370        assert_eq!(
371            result.as_decimal().decimal_value(),
372            Some(DecimalValue::I256(i256::from_i128(2_000_000)))
373        );
374        Ok(())
375    }
376
377    #[test]
378    fn mean_decimal_null() -> VortexResult<()> {
379        let dtype = DecimalDType::new(6, 2);
380        let validity = Validity::from_iter([true, false, true]);
381        let array = DecimalArray::new(buffer![150i32, 0, 450], dtype, validity).into_array();
382        let mut ctx = SESSION.create_execution_ctx();
383        let result = mean(&array, &mut ctx)?;
384        // mean(1.50, 4.50) = 3.000000
385        assert_eq!(
386            result.as_decimal().decimal_value(),
387            Some(DecimalValue::I256(i256::from_i128(3_000_000)))
388        );
389        assert_eq!(
390            result.dtype(),
391            &DType::Decimal(DecimalDType::new(10, 6), Nullability::Nullable)
392        );
393        Ok(())
394    }
395
396    #[test]
397    fn mean_decimal_chunked() -> VortexResult<()> {
398        let dtype = DecimalDType::new(6, 2);
399        let validity = Validity::NonNullable;
400        let chunk1 = DecimalArray::new(buffer![100i32, 200], dtype, validity.clone()).into_array();
401        let chunk2 = DecimalArray::new(buffer![300i32, 400, 500], dtype, validity).into_array();
402        let dtype = chunk1.dtype().clone();
403        let chunked = ChunkedArray::try_new(vec![chunk1, chunk2], dtype)?;
404        let mut ctx = SESSION.create_execution_ctx();
405        let result = mean(&chunked.into_array(), &mut ctx)?;
406        // mean(1.00, 2.00, 3.00, 4.00, 5.00) = 3.000000
407        assert_eq!(
408            result.as_decimal().decimal_value(),
409            Some(DecimalValue::I256(i256::from_i128(3_000_000)))
410        );
411        assert_eq!(
412            result.dtype(),
413            &DType::Decimal(DecimalDType::new(10, 6), Nullability::Nullable)
414        );
415        Ok(())
416    }
417
418    #[test]
419    fn mean_decimal_33() -> VortexResult<()> {
420        let dtype = DecimalDType::new(6, 2);
421        let buf = buffer![100i32, 0, 0];
422        let array = DecimalArray::new(buf, dtype, Validity::NonNullable).into_array();
423        let mut ctx = SESSION.create_execution_ctx();
424        let result = mean(&array, &mut ctx)?;
425        // mean(1.00, 0.00, 0.00) = 1/3 => 0.333333
426        assert_eq!(
427            result.as_decimal().decimal_value(),
428            Some(DecimalValue::I256(i256::from_i128(333_333)))
429        );
430        assert_eq!(
431            result.dtype(),
432            &DType::Decimal(DecimalDType::new(10, 6), Nullability::Nullable)
433        );
434        Ok(())
435    }
436
437    #[test]
438    fn mean_multi_batch() -> VortexResult<()> {
439        let mut ctx = SESSION.create_execution_ctx();
440        let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
441        let mut acc = Accumulator::try_new(
442            Mean::combined(),
443            PairOptions(
444                NumericalAggregateOpts::default(),
445                NumericalAggregateOpts::default(),
446            ),
447            dtype,
448        )?;
449
450        let batch1 =
451            PrimitiveArray::new(buffer![1.0f64, 2.0, 3.0], Validity::NonNullable).into_array();
452        acc.accumulate(&batch1, &mut ctx)?;
453
454        let batch2 = PrimitiveArray::new(buffer![4.0f64, 5.0], Validity::NonNullable).into_array();
455        acc.accumulate(&batch2, &mut ctx)?;
456
457        let result = acc.finish()?;
458        assert_eq!(result.as_primitive().as_::<f64>(), Some(3.0));
459        Ok(())
460    }
461
462    fn mean_nan_null() -> Vec<(Vec<Option<f64>>, Option<f64>)> {
463        vec![
464            (vec![Some(f64::NAN), Some(1.0), None], Some(1.0)),
465            (vec![Some(f64::NAN), Some(1.0), Some(3.0)], Some(2.0)),
466            (vec![None, None, Some(f64::NAN)], None),
467            (vec![None, None, None], None),
468            (vec![Some(1.0), Some(2.0), Some(3.0)], Some(2.0)),
469        ]
470    }
471
472    #[test]
473    fn mean_combined_partials() -> VortexResult<()> {
474        let mut ctx = SESSION.create_execution_ctx();
475        for (case, (group, expected)) in mean_nan_null().into_iter().enumerate() {
476            let mut acc = Accumulator::try_new(
477                Mean::combined(),
478                PairOptions(
479                    NumericalAggregateOpts::default(),
480                    NumericalAggregateOpts::default(),
481                ),
482                DType::Primitive(PType::F64, Nullability::Nullable),
483            )?;
484            let (head, tail) = group.split_at(2);
485            let head = PrimitiveArray::from_option_iter(head.iter().copied()).into_array();
486            let tail = PrimitiveArray::from_option_iter(tail.iter().copied()).into_array();
487            acc.accumulate(&head, &mut ctx)?;
488            acc.accumulate(&tail, &mut ctx)?;
489            let result = acc.finish()?;
490            assert_eq!(result.as_primitive().as_::<f64>(), expected, "case {case}");
491        }
492        Ok(())
493    }
494
495    #[test]
496    fn mean_grouped_finalize() -> VortexResult<()> {
497        let cases = mean_nan_null();
498        let elements = PrimitiveArray::from_option_iter(
499            cases.iter().flat_map(|(group, _)| group.iter().copied()),
500        )
501        .into_array();
502        let groups = FixedSizeListArray::try_new(elements, 3, Validity::NonNullable, cases.len())?;
503
504        let mut acc = GroupedAccumulator::try_new(
505            Mean::combined(),
506            PairOptions(
507                NumericalAggregateOpts::default(),
508                NumericalAggregateOpts::default(),
509            ),
510            DType::Primitive(PType::F64, Nullability::Nullable),
511        )?;
512        let mut ctx = SESSION.create_execution_ctx();
513        acc.accumulate_list(&groups.into_array(), &mut ctx)?;
514        let result = acc.finish()?;
515
516        for (case, (_, expected)) in cases.into_iter().enumerate() {
517            let actual = result.execute_scalar(case, &mut ctx)?;
518            assert_eq!(actual.as_primitive().as_::<f64>(), expected, "case {case}");
519        }
520        let expected =
521            PrimitiveArray::from_option_iter([Some(1.0f64), Some(2.0), None, None, Some(2.0)])
522                .into_array();
523        assert_arrays_eq!(&result, &expected, &mut ctx);
524        Ok(())
525    }
526}