Skip to main content

vortex_array/aggregate_fn/fns/sum/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4mod bool;
5mod constant;
6mod decimal;
7mod grouped;
8mod primitive;
9pub(crate) use grouped::PrimitiveGroupedSumEncodingKernel;
10use vortex_error::VortexExpect;
11use vortex_error::VortexResult;
12use vortex_error::vortex_bail;
13use vortex_error::vortex_ensure;
14use vortex_error::vortex_panic;
15use vortex_session::VortexSession;
16use vortex_session::registry::CachedId;
17
18pub(crate) use self::bool::accumulate_bool;
19pub(crate) use self::constant::multiply_constant;
20pub(crate) use self::decimal::accumulate_decimal;
21pub(crate) use self::primitive::accumulate_primitive;
22pub(crate) use self::primitive::sum_float_all;
23pub(crate) use self::primitive::sum_signed_all;
24pub(crate) use self::primitive::sum_unsigned_all;
25use crate::ArrayRef;
26use crate::Canonical;
27use crate::Columnar;
28use crate::ExecutionCtx;
29use crate::aggregate_fn::Accumulator;
30use crate::aggregate_fn::AggregateArgs;
31use crate::aggregate_fn::AggregateFnId;
32use crate::aggregate_fn::AggregateFnVTable;
33use crate::aggregate_fn::DynAccumulator;
34use crate::aggregate_fn::NumericalAggregateOpts;
35use crate::dtype::DType;
36use crate::dtype::DecimalDType;
37use crate::dtype::MAX_PRECISION;
38use crate::dtype::Nullability;
39use crate::dtype::PType;
40use crate::expr::stats::Precision;
41use crate::expr::stats::Stat;
42use crate::expr::stats::StatsProvider;
43use crate::expr::stats::StatsProviderExt;
44use crate::scalar::DecimalValue;
45use crate::scalar::Scalar;
46
47/// Return the sum of an array.
48///
49/// See [`Sum`] for details.
50pub fn sum(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Scalar> {
51    // Short-circuit using cached array statistics.
52    if let Precision::Exact(sum_scalar) = array.statistics().get(Stat::Sum) {
53        return Ok(sum_scalar);
54    }
55
56    // Compute using Accumulator<Sum>.
57    // TODO(ngates): we may want to wrap this three-step dance up into an extension crate maybe.
58    let mut acc = Accumulator::try_new(
59        Sum,
60        NumericalAggregateOpts::default(),
61        array.dtype().clone(),
62    )?;
63    acc.accumulate(array, ctx)?;
64    // The accumulator caches the sum on `array` as a statistic.
65    acc.finish()
66}
67
68/// Sum an array, starting from zero.
69///
70/// If the sum overflows, a null scalar will be returned.
71/// If the array is all-invalid, the sum will be zero.
72///
73/// NaN handling for float inputs is controlled by [`NumericalAggregateOpts`]: with `skip_nans` (the
74/// default) NaN values contribute nothing, otherwise any NaN value poisons the sum to NaN.
75#[derive(Clone, Debug)]
76pub struct Sum;
77
78// Both Spark and DataFusion use this heuristic.
79// - https://github.com/apache/spark/blob/fcf636d9eb8d645c24be3db2d599aba2d7e2955a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala#L66
80// - https://github.com/apache/datafusion/blob/4153adf2c0f6e317ef476febfdc834208bd46622/datafusion/functions-aggregate/src/sum.rs#L188
81pub(crate) fn sum_decimal_dtype(input: &DecimalDType) -> DecimalDType {
82    DecimalDType::new(
83        u8::min(MAX_PRECISION, input.precision() + 10),
84        input.scale(),
85    )
86}
87
88impl AggregateFnVTable for Sum {
89    type Options = NumericalAggregateOpts;
90    type Partial = SumPartial;
91
92    fn id(&self) -> AggregateFnId {
93        static ID: CachedId = CachedId::new("vortex.sum");
94        *ID
95    }
96
97    fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
98        Ok(Some(options.serialize()))
99    }
100
101    fn deserialize(
102        &self,
103        metadata: &[u8],
104        _session: &VortexSession,
105    ) -> VortexResult<Self::Options> {
106        NumericalAggregateOpts::deserialize(metadata)
107    }
108
109    fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {
110        // When a sum overflows, we return a sum _value_ of null. Therefore, we all return dtypes
111        // are nullable.
112        use Nullability::Nullable;
113
114        Some(match input_dtype {
115            DType::Bool(_) => DType::Primitive(PType::U64, Nullable),
116            DType::Primitive(ptype, _) => match ptype {
117                PType::U8 | PType::U16 | PType::U32 | PType::U64 => {
118                    DType::Primitive(PType::U64, Nullable)
119                }
120                PType::I8 | PType::I16 | PType::I32 | PType::I64 => {
121                    DType::Primitive(PType::I64, Nullable)
122                }
123                PType::F16 | PType::F32 | PType::F64 => {
124                    // Float sums cannot overflow, but all null floats still end up as null
125                    DType::Primitive(PType::F64, Nullable)
126                }
127            },
128            DType::Decimal(decimal_dtype, _) => {
129                DType::Decimal(sum_decimal_dtype(decimal_dtype), Nullable)
130            }
131            // Unsupported types
132            _ => return None,
133        })
134    }
135
136    fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
137        self.return_dtype(options, input_dtype)
138    }
139
140    fn empty_partial(&self, args: AggregateArgs<'_, Self::Options>) -> VortexResult<Self::Partial> {
141        Ok(SumPartial {
142            current: Some(make_zero_state(args.return_dtype)),
143        })
144    }
145
146    fn partial_from_scalar(
147        &self,
148        args: AggregateArgs<'_, Self::Options>,
149        scalar: &Scalar,
150    ) -> VortexResult<Self::Partial> {
151        vortex_ensure!(
152            scalar.dtype().eq_ignore_nullability(args.return_dtype),
153            "Sum partial has dtype {}, expected {}",
154            scalar.dtype(),
155            args.return_dtype
156        );
157        // A null partial means the producing accumulator saturated (overflow).
158        let current = if scalar.is_null() {
159            None
160        } else {
161            Some(sum_state_from_scalar(scalar, args.return_dtype)?)
162        };
163        Ok(SumPartial { current })
164    }
165
166    fn merge_partials(
167        &self,
168        args: AggregateArgs<'_, Self::Options>,
169        mut first: Self::Partial,
170        second: Self::Partial,
171    ) -> VortexResult<Self::Partial> {
172        let overflow = match (first.current.as_mut(), second.current) {
173            // A saturated sum stays saturated.
174            (None, _) => false,
175            // A saturated (overflowed) partial poisons the merge.
176            (Some(_), None) => true,
177            (Some(acc), Some(state)) => checked_add_sum_states(acc, args.return_dtype, &state)?,
178        };
179        if overflow {
180            first.current = None;
181        }
182        Ok(first)
183    }
184
185    fn to_scalar(
186        &self,
187        args: AggregateArgs<'_, Self::Options>,
188        partial: &Self::Partial,
189    ) -> VortexResult<Scalar> {
190        Ok(match &partial.current {
191            None => Scalar::null(args.return_dtype.as_nullable()),
192            Some(SumState::Unsigned(v)) => Scalar::primitive(*v, Nullability::Nullable),
193            Some(SumState::Signed(v)) => Scalar::primitive(*v, Nullability::Nullable),
194            Some(SumState::Float(v)) => Scalar::primitive(*v, Nullability::Nullable),
195            Some(SumState::Decimal(value)) => {
196                let decimal_dtype = *args
197                    .return_dtype
198                    .as_decimal_opt()
199                    .vortex_expect("return dtype must be decimal");
200                Scalar::decimal(*value, decimal_dtype, Nullability::Nullable)
201            }
202        })
203    }
204
205    #[inline]
206    fn is_saturated(
207        &self,
208        _args: AggregateArgs<'_, Self::Options>,
209        partial: &Self::Partial,
210    ) -> bool {
211        match partial.current.as_ref() {
212            None => true,
213            Some(SumState::Float(v)) => v.is_nan(),
214            Some(_) => false,
215        }
216    }
217
218    fn try_accumulate(
219        &self,
220        args: AggregateArgs<'_, Self::Options>,
221        partial: &mut Self::Partial,
222        batch: &ArrayRef,
223        _ctx: &mut ExecutionCtx,
224    ) -> VortexResult<bool> {
225        // NaN-aware shortcircuits only apply to NaN-including float sums; everything else takes
226        // the default dispatch path.
227        if args.options.skip_nans || !matches!(partial.current, Some(SumState::Float(_))) {
228            return Ok(false);
229        }
230        match batch.statistics().get_as::<u64>(Stat::NaNCount) {
231            Precision::Exact(0) => {
232                // NaN-free batch: the cached NaN-skipping sum (if any) equals the
233                // NaN-including sum.
234                if let Precision::Exact(sum) = batch.statistics().get(Stat::Sum) {
235                    let sum = if sum.dtype() == args.return_dtype {
236                        sum
237                    } else {
238                        sum.cast(args.return_dtype)?
239                    };
240                    merge_sum_result(partial, args.return_dtype, sum)?;
241                    return Ok(true);
242                }
243                Ok(false)
244            }
245            Precision::Exact(_) => {
246                // At least one NaN value: the sum is NaN without scanning the batch.
247                if let Some(SumState::Float(acc)) = partial.current.as_mut() {
248                    *acc = f64::NAN;
249                }
250                Ok(true)
251            }
252            _ => Ok(false),
253        }
254    }
255
256    fn accumulate(
257        &self,
258        args: AggregateArgs<'_, Self::Options>,
259        partial: &mut Self::Partial,
260        batch: &Columnar,
261        ctx: &mut ExecutionCtx,
262    ) -> VortexResult<()> {
263        if let Columnar::Constant(c) = batch {
264            // NaN constants are treated as missing when skipping NaNs.
265            if args.options.skip_nans && c.scalar().as_primitive_opt().is_some_and(|p| p.is_nan()) {
266                return Ok(());
267            }
268            if let Some(product) = multiply_constant(c.scalar(), c.len(), args.return_dtype)? {
269                merge_sum_result(partial, args.return_dtype, product)?;
270            }
271            return Ok(());
272        }
273
274        let mut inner = match partial.current.take() {
275            Some(inner) => inner,
276            None => return Ok(()),
277        };
278
279        let result = match batch {
280            Columnar::Canonical(c) => match c {
281                Canonical::Primitive(p) => {
282                    accumulate_primitive(&mut inner, p, ctx, args.options.skip_nans)
283                }
284                Canonical::Bool(b) => accumulate_bool(&mut inner, b, ctx),
285                Canonical::Decimal(d) => accumulate_decimal(&mut inner, args.return_dtype, d, ctx),
286                _ => {
287                    vortex_bail!("Unsupported canonical type for sum: {}", batch.dtype())
288                }
289            },
290            Columnar::Constant(_) => unreachable!(),
291        };
292
293        match result {
294            Ok(false) => partial.current = Some(inner),
295            Ok(true) => {} // saturated: current stays None
296            Err(e) => {
297                partial.current = Some(inner);
298                return Err(e);
299            }
300        }
301        Ok(())
302    }
303
304    fn finalize(
305        &self,
306        _args: AggregateArgs<'_, Self::Options>,
307        partials: ArrayRef,
308    ) -> VortexResult<ArrayRef> {
309        Ok(partials)
310    }
311
312    fn finalize_scalar(
313        &self,
314        args: AggregateArgs<'_, Self::Options>,
315        partial: &Self::Partial,
316    ) -> VortexResult<Scalar> {
317        self.to_scalar(args, partial)
318    }
319}
320
321/// The accumulated sum, or the overflow state.
322pub struct SumPartial {
323    /// The current accumulated state, or `None` if saturated (checked overflow).
324    current: Option<SumState>,
325}
326
327/// The accumulated sum value.
328///
329/// Decimal sums do not carry their dtype: the result dtype of the aggregate provides the
330/// precision and scale whenever they are needed.
331// TODO(ngates): instead of an enum, we should use a Box<dyn State> to avoid dispatcher over the
332//  input type every time? Perhaps?
333pub enum SumState {
334    Unsigned(u64),
335    Signed(i64),
336    Float(f64),
337    Decimal(DecimalValue),
338}
339
340pub(crate) fn make_zero_state(return_dtype: &DType) -> SumState {
341    match return_dtype {
342        DType::Primitive(ptype, _) => match ptype {
343            PType::U8 | PType::U16 | PType::U32 | PType::U64 => SumState::Unsigned(0),
344            PType::I8 | PType::I16 | PType::I32 | PType::I64 => SumState::Signed(0),
345            PType::F16 | PType::F32 | PType::F64 => SumState::Float(0.0),
346        },
347        DType::Decimal(decimal, _) => SumState::Decimal(DecimalValue::zero(decimal)),
348        _ => vortex_panic!("Unsupported sum type"),
349    }
350}
351
352/// Parse a non-null sum value of `return_dtype` into its accumulated state.
353fn sum_state_from_scalar(scalar: &Scalar, return_dtype: &DType) -> VortexResult<SumState> {
354    Ok(match return_dtype {
355        DType::Primitive(ptype, _) if ptype.is_unsigned_int() => {
356            SumState::Unsigned(u64::try_from(scalar)?)
357        }
358        DType::Primitive(ptype, _) if ptype.is_signed_int() => {
359            SumState::Signed(i64::try_from(scalar)?)
360        }
361        DType::Primitive(..) => SumState::Float(f64::try_from(scalar)?),
362        DType::Decimal(..) => SumState::Decimal(DecimalValue::try_from(scalar)?),
363        _ => vortex_bail!("Unsupported sum type {}", return_dtype),
364    })
365}
366
367/// Checked add of one sum state into another, returning true if overflow occurred.
368///
369/// A decimal sum that no longer fits the precision of `return_dtype` counts as an overflow.
370pub(crate) fn checked_add_sum_states(
371    state: &mut SumState,
372    return_dtype: &DType,
373    other: &SumState,
374) -> VortexResult<bool> {
375    Ok(match (state, other) {
376        (SumState::Unsigned(acc), SumState::Unsigned(other)) => checked_add_u64(acc, *other),
377        (SumState::Signed(acc), SumState::Signed(other)) => checked_add_i64(acc, *other),
378        (SumState::Float(acc), SumState::Float(other)) => {
379            *acc += *other;
380            false
381        }
382        (SumState::Decimal(value), SumState::Decimal(other)) => {
383            let dtype = return_dtype
384                .as_decimal_opt()
385                .vortex_expect("decimal sum result dtype");
386            match value.checked_add(other) {
387                Some(result) if result.fits_in_precision(*dtype) => {
388                    *value = result;
389                    false
390                }
391                Some(_) | None => true,
392            }
393        }
394        _ => vortex_bail!("Mismatched sum partial states"),
395    })
396}
397
398/// Merge a finalized sum result (nullable; null means overflow) into the partial.
399fn merge_sum_result(
400    partial: &mut SumPartial,
401    return_dtype: &DType,
402    result: Scalar,
403) -> VortexResult<()> {
404    let overflow = match partial.current.as_mut() {
405        None => return Ok(()),
406        Some(_) if result.is_null() => true,
407        Some(acc) => checked_add_sum_states(
408            acc,
409            return_dtype,
410            &sum_state_from_scalar(&result, return_dtype)?,
411        )?,
412    };
413    if overflow {
414        partial.current = None;
415    }
416    Ok(())
417}
418
419/// Checked add for u64, returning true if overflow occurred.
420#[allow(clippy::inline_always)]
421#[inline(always)]
422fn checked_add_u64(acc: &mut u64, val: u64) -> bool {
423    match acc.checked_add(val) {
424        Some(r) => {
425            *acc = r;
426            false
427        }
428        None => true,
429    }
430}
431
432/// Checked add for i64, returning true if overflow occurred.
433#[allow(clippy::inline_always)]
434#[inline(always)]
435fn checked_add_i64(acc: &mut i64, val: i64) -> bool {
436    match acc.checked_add(val) {
437        Some(r) => {
438            *acc = r;
439            false
440        }
441        None => true,
442    }
443}
444
445#[cfg(test)]
446mod tests {
447    use std::sync::LazyLock;
448
449    use vortex_buffer::buffer;
450    use vortex_error::VortexExpect;
451    use vortex_error::VortexResult;
452    use vortex_session::VortexSession;
453
454    use crate::ArrayRef;
455    use crate::IntoArray;
456    use crate::VortexSessionExecute;
457    use crate::aggregate_fn::Accumulator;
458    use crate::aggregate_fn::AggregateDTypes;
459    use crate::aggregate_fn::AggregateFnVTable;
460    use crate::aggregate_fn::DynAccumulator;
461    use crate::aggregate_fn::DynGroupedAccumulator;
462    use crate::aggregate_fn::GroupedAccumulator;
463    use crate::aggregate_fn::NumericalAggregateOpts;
464    use crate::aggregate_fn::fns::sum::Sum;
465    use crate::aggregate_fn::fns::sum::SumPartial;
466    use crate::aggregate_fn::fns::sum::SumState;
467    use crate::aggregate_fn::fns::sum::sum;
468    use crate::arrays::BoolArray;
469    use crate::arrays::ChunkedArray;
470    use crate::arrays::ConstantArray;
471    use crate::arrays::DecimalArray;
472    use crate::arrays::FixedSizeListArray;
473    use crate::arrays::ListViewArray;
474    use crate::arrays::PrimitiveArray;
475    use crate::assert_arrays_eq;
476    use crate::dtype::DType;
477    use crate::dtype::DecimalDType;
478    use crate::dtype::Nullability;
479    use crate::dtype::Nullability::Nullable;
480    use crate::dtype::PType;
481    use crate::dtype::i256;
482    use crate::expr::stats::Precision;
483    use crate::expr::stats::Stat;
484    use crate::expr::stats::StatsProvider;
485    use crate::scalar::DecimalValue;
486    use crate::scalar::Scalar;
487    use crate::validity::Validity;
488
489    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
490
491    // Multi-batch and reset tests
492
493    #[test]
494    fn sum_multi_batch() -> VortexResult<()> {
495        let mut ctx = SESSION.create_execution_ctx();
496        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
497        let mut acc = Accumulator::try_new(Sum, NumericalAggregateOpts::default(), dtype)?;
498
499        let batch1 = PrimitiveArray::new(buffer![10i32, 20], Validity::NonNullable).into_array();
500        acc.accumulate(&batch1, &mut ctx)?;
501
502        let batch2 = PrimitiveArray::new(buffer![3i32, 6, 9], Validity::NonNullable).into_array();
503        acc.accumulate(&batch2, &mut ctx)?;
504
505        let result = acc.finish()?;
506        assert_eq!(result.as_primitive().typed_value::<i64>(), Some(48));
507        Ok(())
508    }
509
510    #[test]
511    fn sum_finish_resets_state() -> VortexResult<()> {
512        let mut ctx = SESSION.create_execution_ctx();
513        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
514        let mut acc = Accumulator::try_new(Sum, NumericalAggregateOpts::default(), dtype)?;
515
516        let batch1 = PrimitiveArray::new(buffer![10i32, 20], Validity::NonNullable).into_array();
517        acc.accumulate(&batch1, &mut ctx)?;
518        let result1 = acc.finish()?;
519        assert_eq!(result1.as_primitive().typed_value::<i64>(), Some(30));
520
521        let batch2 = PrimitiveArray::new(buffer![3i32, 6, 9], Validity::NonNullable).into_array();
522        acc.accumulate(&batch2, &mut ctx)?;
523        let result2 = acc.finish()?;
524        assert_eq!(result2.as_primitive().typed_value::<i64>(), Some(18));
525        Ok(())
526    }
527
528    // State merge tests (vtable-level)
529
530    #[test]
531    fn sum_state_merge() -> VortexResult<()> {
532        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
533        let options = NumericalAggregateOpts::default();
534        let owned = AggregateDTypes::try_new(&Sum, &options, dtype)?;
535        let args = owned.args(&options);
536
537        let partial_of = |value: i64| SumPartial {
538            current: Some(SumState::Signed(value)),
539        };
540        let state = Sum.merge_partials(args, partial_of(100), partial_of(50))?;
541
542        let result = Sum.to_scalar(args, &state)?;
543        assert_eq!(result.as_primitive().typed_value::<i64>(), Some(150));
544        Ok(())
545    }
546
547    #[test]
548    fn sum_overflowed_partial_poisons_reduction() -> VortexResult<()> {
549        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
550        let options = NumericalAggregateOpts::default();
551        let owned = AggregateDTypes::try_new(&Sum, &options, dtype)?;
552        let args = owned.args(&options);
553
554        let overflowed =
555            Sum.partial_from_scalar(args, &Scalar::null(DType::Primitive(PType::I64, Nullable)))?;
556        let five = Sum.partial_from_scalar(args, &Scalar::primitive(5i64, Nullable))?;
557        let state = Sum.merge_partials(args, five, overflowed)?;
558
559        assert!(Sum.is_saturated(args, &state));
560        assert!(Sum.to_scalar(args, &state)?.is_null());
561        Ok(())
562    }
563
564    // Stats caching test
565
566    #[test]
567    fn sum_stats() -> VortexResult<()> {
568        let array = ChunkedArray::try_new(
569            vec![
570                PrimitiveArray::from_iter([1, 1, 1]).into_array(),
571                PrimitiveArray::from_iter([2, 2, 2]).into_array(),
572            ],
573            DType::Primitive(PType::I32, Nullability::NonNullable),
574        )
575        .vortex_expect("operation should succeed in test");
576        let array = array.into_array();
577        let mut ctx = SESSION.create_execution_ctx();
578        let mut acc = Accumulator::try_new(
579            Sum,
580            NumericalAggregateOpts::default(),
581            array.dtype().clone(),
582        )?;
583        acc.combine_partials(&Scalar::primitive(2i64, Nullable))?;
584        acc.accumulate(&array, &mut ctx)?;
585        assert_eq!(acc.finish()?, Scalar::primitive(11i64, Nullable));
586        assert_eq!(array.statistics().get(Stat::Sum), Precision::exact(9i64));
587
588        let sum_without_acc = sum(&array, &mut SESSION.create_execution_ctx())?;
589        assert_eq!(sum_without_acc, Scalar::primitive(9i64, Nullable));
590        Ok(())
591    }
592
593    // Constant float non-multiply test
594
595    #[test]
596    fn sum_constant_float_non_multiply() -> VortexResult<()> {
597        let acc = -2048669276050936500000000000f64;
598        let array = ConstantArray::new(6.1811675e16f64, 25);
599        let array = array.into_array();
600        let mut accumulator = Accumulator::try_new(
601            Sum,
602            NumericalAggregateOpts::default(),
603            array.dtype().clone(),
604        )?;
605        accumulator.combine_partials(&Scalar::primitive(acc, Nullable))?;
606        accumulator.accumulate(&array, &mut SESSION.create_execution_ctx())?;
607        let result = accumulator.finish()?;
608        assert_eq!(
609            f64::try_from(&result).vortex_expect("operation should succeed in test"),
610            -2048669274505644600000000000f64
611        );
612        Ok(())
613    }
614
615    // Grouped sum tests
616
617    fn run_grouped_sum(groups: &ArrayRef, elem_dtype: &DType) -> VortexResult<ArrayRef> {
618        let mut acc = GroupedAccumulator::try_new(
619            Sum,
620            NumericalAggregateOpts::default(),
621            elem_dtype.clone(),
622        )?;
623        acc.accumulate_list(groups, &mut SESSION.create_execution_ctx())?;
624        acc.finish()
625    }
626
627    #[test]
628    fn grouped_sum_fixed_size_list() -> VortexResult<()> {
629        let mut ctx = SESSION.create_execution_ctx();
630        let elements =
631            PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5, 6], Validity::NonNullable).into_array();
632        let groups = FixedSizeListArray::try_new(elements, 3, Validity::NonNullable, 2)?;
633
634        let elem_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
635        let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
636
637        let expected = PrimitiveArray::from_option_iter([Some(6i64), Some(15i64)]).into_array();
638        assert_arrays_eq!(&result, &expected, &mut ctx);
639        Ok(())
640    }
641
642    #[test]
643    fn grouped_sum_with_null_elements() -> VortexResult<()> {
644        let mut ctx = SESSION.create_execution_ctx();
645        let elements =
646            PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5), Some(6)])
647                .into_array();
648        let groups = FixedSizeListArray::try_new(elements, 3, Validity::NonNullable, 2)?;
649
650        let elem_dtype = DType::Primitive(PType::I32, Nullable);
651        let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
652
653        let expected = PrimitiveArray::from_option_iter([Some(4i64), Some(11i64)]).into_array();
654        assert_arrays_eq!(&result, &expected, &mut ctx);
655        Ok(())
656    }
657
658    #[test]
659    fn grouped_sum_with_null_group() -> VortexResult<()> {
660        let mut ctx = SESSION.create_execution_ctx();
661        let elements =
662            PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5, 6, 7, 8, 9], Validity::NonNullable)
663                .into_array();
664        let validity = Validity::from_iter([true, false, true]);
665        let groups = FixedSizeListArray::try_new(elements, 3, validity, 3)?;
666
667        let elem_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
668        let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
669
670        let expected =
671            PrimitiveArray::from_option_iter([Some(6i64), None, Some(24i64)]).into_array();
672        assert_arrays_eq!(&result, &expected, &mut ctx);
673        Ok(())
674    }
675
676    #[test]
677    fn grouped_sum_all_null_elements_in_group() -> VortexResult<()> {
678        let mut ctx = SESSION.create_execution_ctx();
679        let elements =
680            PrimitiveArray::from_option_iter([None::<i32>, None, Some(3), Some(4)]).into_array();
681        let groups = FixedSizeListArray::try_new(elements, 2, Validity::NonNullable, 2)?;
682
683        let elem_dtype = DType::Primitive(PType::I32, Nullable);
684        let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
685
686        let expected = PrimitiveArray::from_option_iter([Some(0i64), Some(7i64)]).into_array();
687        assert_arrays_eq!(&result, &expected, &mut ctx);
688        Ok(())
689    }
690
691    #[test]
692    fn grouped_sum_bool() -> VortexResult<()> {
693        let mut ctx = SESSION.create_execution_ctx();
694        let elements: BoolArray = [true, false, true, true, true, true].into_iter().collect();
695        let groups =
696            FixedSizeListArray::try_new(elements.into_array(), 3, Validity::NonNullable, 2)?;
697
698        let elem_dtype = DType::Bool(Nullability::NonNullable);
699        let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
700
701        let expected = PrimitiveArray::from_option_iter([Some(2u64), Some(3u64)]).into_array();
702        assert_arrays_eq!(&result, &expected, &mut ctx);
703        Ok(())
704    }
705
706    #[test]
707    fn grouped_sum_finish_resets() -> VortexResult<()> {
708        let mut ctx = SESSION.create_execution_ctx();
709        let elem_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
710        let mut acc =
711            GroupedAccumulator::try_new(Sum, NumericalAggregateOpts::default(), elem_dtype)?;
712
713        let elements1 =
714            PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable).into_array();
715        let groups1 = FixedSizeListArray::try_new(elements1, 2, Validity::NonNullable, 2)?;
716        acc.accumulate_list(&groups1.into_array(), &mut ctx)?;
717        let result1 = acc.finish()?;
718
719        let expected1 = PrimitiveArray::from_option_iter([Some(3i64), Some(7i64)]).into_array();
720        assert_arrays_eq!(&result1, &expected1, &mut ctx);
721
722        let elements2 = PrimitiveArray::new(buffer![10i32, 20], Validity::NonNullable).into_array();
723        let groups2 = FixedSizeListArray::try_new(elements2, 2, Validity::NonNullable, 1)?;
724        acc.accumulate_list(&groups2.into_array(), &mut ctx)?;
725        let result2 = acc.finish()?;
726
727        let expected2 = PrimitiveArray::from_option_iter([Some(30i64)]).into_array();
728        assert_arrays_eq!(&result2, &expected2, &mut ctx);
729        Ok(())
730    }
731
732    #[test]
733    fn grouped_sum_listview_out_of_order_offsets_with_null_group() -> VortexResult<()> {
734        let mut ctx = SESSION.create_execution_ctx();
735        let elements =
736            PrimitiveArray::new(buffer![100i32, 200, 300], Validity::NonNullable).into_array();
737        let offsets = PrimitiveArray::new(buffer![2i32, 0, 1], Validity::NonNullable).into_array();
738        let sizes = PrimitiveArray::new(buffer![1i32, 1, 1], Validity::NonNullable).into_array();
739        let validity = Validity::from_iter([true, false, true]);
740        let groups =
741            ListViewArray::try_new(elements, offsets, sizes, validity, &mut ctx)?.into_array();
742
743        let elem_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
744        let result = run_grouped_sum(&groups, &elem_dtype)?;
745
746        // group 0 -> elements[2..3] = 300; group 1 -> null; group 2 -> elements[1..2] = 200.
747        let expected =
748            PrimitiveArray::from_option_iter([Some(300i64), None, Some(200i64)]).into_array();
749        assert_arrays_eq!(&result, &expected, &mut ctx);
750        Ok(())
751    }
752
753    // Chunked array tests
754
755    #[test]
756    fn sum_chunked_floats_with_nulls() -> VortexResult<()> {
757        let chunk1 =
758            PrimitiveArray::from_option_iter(vec![Some(1.5f64), None, Some(3.2), Some(4.8)]);
759        let chunk2 = PrimitiveArray::from_option_iter(vec![Some(2.1f64), Some(5.7), None]);
760        let chunk3 = PrimitiveArray::from_option_iter(vec![None, Some(1.0f64), Some(2.5), None]);
761        let dtype = chunk1.dtype().clone();
762        let chunked = ChunkedArray::try_new(
763            vec![
764                chunk1.into_array(),
765                chunk2.into_array(),
766                chunk3.into_array(),
767            ],
768            dtype,
769        )?;
770
771        let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
772        assert_eq!(result.as_primitive().as_::<f64>(), Some(20.8));
773        Ok(())
774    }
775
776    #[test]
777    fn sum_chunked_floats_all_nulls_is_zero() -> VortexResult<()> {
778        let chunk1 = PrimitiveArray::from_option_iter::<f32, _>(vec![None, None, None]);
779        let chunk2 = PrimitiveArray::from_option_iter::<f32, _>(vec![None, None]);
780        let dtype = chunk1.dtype().clone();
781        let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
782        let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
783        assert_eq!(result, Scalar::primitive(0f64, Nullable));
784        Ok(())
785    }
786
787    #[test]
788    fn sum_chunked_floats_empty_chunks() -> VortexResult<()> {
789        let chunk1 = PrimitiveArray::from_option_iter(vec![Some(10.5f64), Some(20.3)]);
790        let chunk2 = ConstantArray::new(Scalar::primitive(0f64, Nullable), 0);
791        let chunk3 = PrimitiveArray::from_option_iter(vec![Some(5.2f64)]);
792        let dtype = chunk1.dtype().clone();
793        let chunked = ChunkedArray::try_new(
794            vec![
795                chunk1.into_array(),
796                chunk2.into_array(),
797                chunk3.into_array(),
798            ],
799            dtype,
800        )?;
801
802        let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
803        assert_eq!(result.as_primitive().as_::<f64>(), Some(36.0));
804        Ok(())
805    }
806
807    #[test]
808    fn sum_chunked_int_almost_all_null() -> VortexResult<()> {
809        let chunk1 = PrimitiveArray::from_option_iter::<u32, _>(vec![Some(1)]);
810        let chunk2 = PrimitiveArray::from_option_iter::<u32, _>(vec![None]);
811        let dtype = chunk1.dtype().clone();
812        let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
813
814        let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
815        assert_eq!(result.as_primitive().as_::<u64>(), Some(1));
816        Ok(())
817    }
818
819    #[test]
820    fn sum_chunked_decimals() -> VortexResult<()> {
821        let decimal_dtype = DecimalDType::new(10, 2);
822        let chunk1 = DecimalArray::new(
823            buffer![100i32, 100i32, 100i32, 100i32, 100i32],
824            decimal_dtype,
825            Validity::AllValid,
826        );
827        let chunk2 = DecimalArray::new(
828            buffer![200i32, 200i32, 200i32],
829            decimal_dtype,
830            Validity::AllValid,
831        );
832        let chunk3 = DecimalArray::new(buffer![300i32, 300i32], decimal_dtype, Validity::AllValid);
833        let dtype = chunk1.dtype().clone();
834        let chunked = ChunkedArray::try_new(
835            vec![
836                chunk1.into_array(),
837                chunk2.into_array(),
838                chunk3.into_array(),
839            ],
840            dtype,
841        )?;
842
843        let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
844        let decimal_result = result.as_decimal();
845        assert_eq!(
846            decimal_result.decimal_value(),
847            Some(DecimalValue::I256(i256::from_i128(1700)))
848        );
849        Ok(())
850    }
851
852    #[test]
853    fn sum_chunked_decimals_with_nulls() -> VortexResult<()> {
854        let decimal_dtype = DecimalDType::new(10, 2);
855        let chunk1 = DecimalArray::new(
856            buffer![100i32, 100i32, 100i32],
857            decimal_dtype,
858            Validity::AllValid,
859        );
860        let chunk2 = DecimalArray::new(
861            buffer![0i32, 0i32],
862            decimal_dtype,
863            Validity::from_iter([false, false]),
864        );
865        let chunk3 = DecimalArray::new(buffer![200i32, 200i32], decimal_dtype, Validity::AllValid);
866        let dtype = chunk1.dtype().clone();
867        let chunked = ChunkedArray::try_new(
868            vec![
869                chunk1.into_array(),
870                chunk2.into_array(),
871                chunk3.into_array(),
872            ],
873            dtype,
874        )?;
875
876        let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
877        let decimal_result = result.as_decimal();
878        assert_eq!(
879            decimal_result.decimal_value(),
880            Some(DecimalValue::I256(i256::from_i128(700)))
881        );
882        Ok(())
883    }
884
885    #[test]
886    fn sum_chunked_decimals_large() -> VortexResult<()> {
887        let decimal_dtype = DecimalDType::new(3, 0);
888        let chunk1 = ConstantArray::new(
889            Scalar::decimal(
890                DecimalValue::I16(500),
891                decimal_dtype,
892                Nullability::NonNullable,
893            ),
894            1,
895        );
896        let chunk2 = ConstantArray::new(
897            Scalar::decimal(
898                DecimalValue::I16(600),
899                decimal_dtype,
900                Nullability::NonNullable,
901            ),
902            1,
903        );
904        let dtype = chunk1.dtype().clone();
905        let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
906
907        let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
908        let decimal_result = result.as_decimal();
909        assert_eq!(
910            decimal_result.decimal_value(),
911            Some(DecimalValue::I256(i256::from_i128(1100)))
912        );
913        assert_eq!(
914            result.dtype(),
915            &DType::Decimal(DecimalDType::new(13, 0), Nullable)
916        );
917        Ok(())
918    }
919}