1mod bool;
5mod constant;
6mod decimal;
7mod grouped;
8mod primitive;
9pub(crate) use grouped::PrimitiveGroupedSumEncodingKernel;
10use vortex_error::VortexExpect;
11use vortex_error::VortexResult;
12use vortex_error::vortex_bail;
13use vortex_error::vortex_ensure;
14use vortex_error::vortex_panic;
15use vortex_session::VortexSession;
16use vortex_session::registry::CachedId;
17
18pub(crate) use self::bool::accumulate_bool;
19pub(crate) use self::constant::multiply_constant;
20pub(crate) use self::decimal::accumulate_decimal;
21pub(crate) use self::primitive::accumulate_primitive;
22pub(crate) use self::primitive::sum_float_all;
23pub(crate) use self::primitive::sum_signed_all;
24pub(crate) use self::primitive::sum_unsigned_all;
25use crate::ArrayRef;
26use crate::Canonical;
27use crate::Columnar;
28use crate::ExecutionCtx;
29use crate::aggregate_fn::Accumulator;
30use crate::aggregate_fn::AggregateArgs;
31use crate::aggregate_fn::AggregateFnId;
32use crate::aggregate_fn::AggregateFnVTable;
33use crate::aggregate_fn::DynAccumulator;
34use crate::aggregate_fn::NumericalAggregateOpts;
35use crate::dtype::DType;
36use crate::dtype::DecimalDType;
37use crate::dtype::MAX_PRECISION;
38use crate::dtype::Nullability;
39use crate::dtype::PType;
40use crate::expr::stats::Precision;
41use crate::expr::stats::Stat;
42use crate::expr::stats::StatsProvider;
43use crate::expr::stats::StatsProviderExt;
44use crate::scalar::DecimalValue;
45use crate::scalar::Scalar;
46
47pub fn sum(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Scalar> {
51 if let Precision::Exact(sum_scalar) = array.statistics().get(Stat::Sum) {
53 return Ok(sum_scalar);
54 }
55
56 let mut acc = Accumulator::try_new(
59 Sum,
60 NumericalAggregateOpts::default(),
61 array.dtype().clone(),
62 )?;
63 acc.accumulate(array, ctx)?;
64 acc.finish()
66}
67
68#[derive(Clone, Debug)]
76pub struct Sum;
77
78pub(crate) fn sum_decimal_dtype(input: &DecimalDType) -> DecimalDType {
82 DecimalDType::new(
83 u8::min(MAX_PRECISION, input.precision() + 10),
84 input.scale(),
85 )
86}
87
88impl AggregateFnVTable for Sum {
89 type Options = NumericalAggregateOpts;
90 type Partial = SumPartial;
91
92 fn id(&self) -> AggregateFnId {
93 static ID: CachedId = CachedId::new("vortex.sum");
94 *ID
95 }
96
97 fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
98 Ok(Some(options.serialize()))
99 }
100
101 fn deserialize(
102 &self,
103 metadata: &[u8],
104 _session: &VortexSession,
105 ) -> VortexResult<Self::Options> {
106 NumericalAggregateOpts::deserialize(metadata)
107 }
108
109 fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {
110 use Nullability::Nullable;
113
114 Some(match input_dtype {
115 DType::Bool(_) => DType::Primitive(PType::U64, Nullable),
116 DType::Primitive(ptype, _) => match ptype {
117 PType::U8 | PType::U16 | PType::U32 | PType::U64 => {
118 DType::Primitive(PType::U64, Nullable)
119 }
120 PType::I8 | PType::I16 | PType::I32 | PType::I64 => {
121 DType::Primitive(PType::I64, Nullable)
122 }
123 PType::F16 | PType::F32 | PType::F64 => {
124 DType::Primitive(PType::F64, Nullable)
126 }
127 },
128 DType::Decimal(decimal_dtype, _) => {
129 DType::Decimal(sum_decimal_dtype(decimal_dtype), Nullable)
130 }
131 _ => return None,
133 })
134 }
135
136 fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
137 self.return_dtype(options, input_dtype)
138 }
139
140 fn empty_partial(&self, args: AggregateArgs<'_, Self::Options>) -> VortexResult<Self::Partial> {
141 Ok(SumPartial {
142 current: Some(make_zero_state(args.return_dtype)),
143 })
144 }
145
146 fn partial_from_scalar(
147 &self,
148 args: AggregateArgs<'_, Self::Options>,
149 scalar: &Scalar,
150 ) -> VortexResult<Self::Partial> {
151 vortex_ensure!(
152 scalar.dtype().eq_ignore_nullability(args.return_dtype),
153 "Sum partial has dtype {}, expected {}",
154 scalar.dtype(),
155 args.return_dtype
156 );
157 let current = if scalar.is_null() {
159 None
160 } else {
161 Some(sum_state_from_scalar(scalar, args.return_dtype)?)
162 };
163 Ok(SumPartial { current })
164 }
165
166 fn merge_partials(
167 &self,
168 args: AggregateArgs<'_, Self::Options>,
169 mut first: Self::Partial,
170 second: Self::Partial,
171 ) -> VortexResult<Self::Partial> {
172 let overflow = match (first.current.as_mut(), second.current) {
173 (None, _) => false,
175 (Some(_), None) => true,
177 (Some(acc), Some(state)) => checked_add_sum_states(acc, args.return_dtype, &state)?,
178 };
179 if overflow {
180 first.current = None;
181 }
182 Ok(first)
183 }
184
185 fn to_scalar(
186 &self,
187 args: AggregateArgs<'_, Self::Options>,
188 partial: &Self::Partial,
189 ) -> VortexResult<Scalar> {
190 Ok(match &partial.current {
191 None => Scalar::null(args.return_dtype.as_nullable()),
192 Some(SumState::Unsigned(v)) => Scalar::primitive(*v, Nullability::Nullable),
193 Some(SumState::Signed(v)) => Scalar::primitive(*v, Nullability::Nullable),
194 Some(SumState::Float(v)) => Scalar::primitive(*v, Nullability::Nullable),
195 Some(SumState::Decimal(value)) => {
196 let decimal_dtype = *args
197 .return_dtype
198 .as_decimal_opt()
199 .vortex_expect("return dtype must be decimal");
200 Scalar::decimal(*value, decimal_dtype, Nullability::Nullable)
201 }
202 })
203 }
204
205 #[inline]
206 fn is_saturated(
207 &self,
208 _args: AggregateArgs<'_, Self::Options>,
209 partial: &Self::Partial,
210 ) -> bool {
211 match partial.current.as_ref() {
212 None => true,
213 Some(SumState::Float(v)) => v.is_nan(),
214 Some(_) => false,
215 }
216 }
217
218 fn try_accumulate(
219 &self,
220 args: AggregateArgs<'_, Self::Options>,
221 partial: &mut Self::Partial,
222 batch: &ArrayRef,
223 _ctx: &mut ExecutionCtx,
224 ) -> VortexResult<bool> {
225 if args.options.skip_nans || !matches!(partial.current, Some(SumState::Float(_))) {
228 return Ok(false);
229 }
230 match batch.statistics().get_as::<u64>(Stat::NaNCount) {
231 Precision::Exact(0) => {
232 if let Precision::Exact(sum) = batch.statistics().get(Stat::Sum) {
235 let sum = if sum.dtype() == args.return_dtype {
236 sum
237 } else {
238 sum.cast(args.return_dtype)?
239 };
240 merge_sum_result(partial, args.return_dtype, sum)?;
241 return Ok(true);
242 }
243 Ok(false)
244 }
245 Precision::Exact(_) => {
246 if let Some(SumState::Float(acc)) = partial.current.as_mut() {
248 *acc = f64::NAN;
249 }
250 Ok(true)
251 }
252 _ => Ok(false),
253 }
254 }
255
256 fn accumulate(
257 &self,
258 args: AggregateArgs<'_, Self::Options>,
259 partial: &mut Self::Partial,
260 batch: &Columnar,
261 ctx: &mut ExecutionCtx,
262 ) -> VortexResult<()> {
263 if let Columnar::Constant(c) = batch {
264 if args.options.skip_nans && c.scalar().as_primitive_opt().is_some_and(|p| p.is_nan()) {
266 return Ok(());
267 }
268 if let Some(product) = multiply_constant(c.scalar(), c.len(), args.return_dtype)? {
269 merge_sum_result(partial, args.return_dtype, product)?;
270 }
271 return Ok(());
272 }
273
274 let mut inner = match partial.current.take() {
275 Some(inner) => inner,
276 None => return Ok(()),
277 };
278
279 let result = match batch {
280 Columnar::Canonical(c) => match c {
281 Canonical::Primitive(p) => {
282 accumulate_primitive(&mut inner, p, ctx, args.options.skip_nans)
283 }
284 Canonical::Bool(b) => accumulate_bool(&mut inner, b, ctx),
285 Canonical::Decimal(d) => accumulate_decimal(&mut inner, args.return_dtype, d, ctx),
286 _ => {
287 vortex_bail!("Unsupported canonical type for sum: {}", batch.dtype())
288 }
289 },
290 Columnar::Constant(_) => unreachable!(),
291 };
292
293 match result {
294 Ok(false) => partial.current = Some(inner),
295 Ok(true) => {} Err(e) => {
297 partial.current = Some(inner);
298 return Err(e);
299 }
300 }
301 Ok(())
302 }
303
304 fn finalize(
305 &self,
306 _args: AggregateArgs<'_, Self::Options>,
307 partials: ArrayRef,
308 ) -> VortexResult<ArrayRef> {
309 Ok(partials)
310 }
311
312 fn finalize_scalar(
313 &self,
314 args: AggregateArgs<'_, Self::Options>,
315 partial: &Self::Partial,
316 ) -> VortexResult<Scalar> {
317 self.to_scalar(args, partial)
318 }
319}
320
321pub struct SumPartial {
323 current: Option<SumState>,
325}
326
327pub enum SumState {
334 Unsigned(u64),
335 Signed(i64),
336 Float(f64),
337 Decimal(DecimalValue),
338}
339
340pub(crate) fn make_zero_state(return_dtype: &DType) -> SumState {
341 match return_dtype {
342 DType::Primitive(ptype, _) => match ptype {
343 PType::U8 | PType::U16 | PType::U32 | PType::U64 => SumState::Unsigned(0),
344 PType::I8 | PType::I16 | PType::I32 | PType::I64 => SumState::Signed(0),
345 PType::F16 | PType::F32 | PType::F64 => SumState::Float(0.0),
346 },
347 DType::Decimal(decimal, _) => SumState::Decimal(DecimalValue::zero(decimal)),
348 _ => vortex_panic!("Unsupported sum type"),
349 }
350}
351
352fn sum_state_from_scalar(scalar: &Scalar, return_dtype: &DType) -> VortexResult<SumState> {
354 Ok(match return_dtype {
355 DType::Primitive(ptype, _) if ptype.is_unsigned_int() => {
356 SumState::Unsigned(u64::try_from(scalar)?)
357 }
358 DType::Primitive(ptype, _) if ptype.is_signed_int() => {
359 SumState::Signed(i64::try_from(scalar)?)
360 }
361 DType::Primitive(..) => SumState::Float(f64::try_from(scalar)?),
362 DType::Decimal(..) => SumState::Decimal(DecimalValue::try_from(scalar)?),
363 _ => vortex_bail!("Unsupported sum type {}", return_dtype),
364 })
365}
366
367pub(crate) fn checked_add_sum_states(
371 state: &mut SumState,
372 return_dtype: &DType,
373 other: &SumState,
374) -> VortexResult<bool> {
375 Ok(match (state, other) {
376 (SumState::Unsigned(acc), SumState::Unsigned(other)) => checked_add_u64(acc, *other),
377 (SumState::Signed(acc), SumState::Signed(other)) => checked_add_i64(acc, *other),
378 (SumState::Float(acc), SumState::Float(other)) => {
379 *acc += *other;
380 false
381 }
382 (SumState::Decimal(value), SumState::Decimal(other)) => {
383 let dtype = return_dtype
384 .as_decimal_opt()
385 .vortex_expect("decimal sum result dtype");
386 match value.checked_add(other) {
387 Some(result) if result.fits_in_precision(*dtype) => {
388 *value = result;
389 false
390 }
391 Some(_) | None => true,
392 }
393 }
394 _ => vortex_bail!("Mismatched sum partial states"),
395 })
396}
397
398fn merge_sum_result(
400 partial: &mut SumPartial,
401 return_dtype: &DType,
402 result: Scalar,
403) -> VortexResult<()> {
404 let overflow = match partial.current.as_mut() {
405 None => return Ok(()),
406 Some(_) if result.is_null() => true,
407 Some(acc) => checked_add_sum_states(
408 acc,
409 return_dtype,
410 &sum_state_from_scalar(&result, return_dtype)?,
411 )?,
412 };
413 if overflow {
414 partial.current = None;
415 }
416 Ok(())
417}
418
419#[allow(clippy::inline_always)]
421#[inline(always)]
422fn checked_add_u64(acc: &mut u64, val: u64) -> bool {
423 match acc.checked_add(val) {
424 Some(r) => {
425 *acc = r;
426 false
427 }
428 None => true,
429 }
430}
431
432#[allow(clippy::inline_always)]
434#[inline(always)]
435fn checked_add_i64(acc: &mut i64, val: i64) -> bool {
436 match acc.checked_add(val) {
437 Some(r) => {
438 *acc = r;
439 false
440 }
441 None => true,
442 }
443}
444
445#[cfg(test)]
446mod tests {
447 use std::sync::LazyLock;
448
449 use vortex_buffer::buffer;
450 use vortex_error::VortexExpect;
451 use vortex_error::VortexResult;
452 use vortex_session::VortexSession;
453
454 use crate::ArrayRef;
455 use crate::IntoArray;
456 use crate::VortexSessionExecute;
457 use crate::aggregate_fn::Accumulator;
458 use crate::aggregate_fn::AggregateDTypes;
459 use crate::aggregate_fn::AggregateFnVTable;
460 use crate::aggregate_fn::DynAccumulator;
461 use crate::aggregate_fn::DynGroupedAccumulator;
462 use crate::aggregate_fn::GroupedAccumulator;
463 use crate::aggregate_fn::NumericalAggregateOpts;
464 use crate::aggregate_fn::fns::sum::Sum;
465 use crate::aggregate_fn::fns::sum::SumPartial;
466 use crate::aggregate_fn::fns::sum::SumState;
467 use crate::aggregate_fn::fns::sum::sum;
468 use crate::arrays::BoolArray;
469 use crate::arrays::ChunkedArray;
470 use crate::arrays::ConstantArray;
471 use crate::arrays::DecimalArray;
472 use crate::arrays::FixedSizeListArray;
473 use crate::arrays::ListViewArray;
474 use crate::arrays::PrimitiveArray;
475 use crate::assert_arrays_eq;
476 use crate::dtype::DType;
477 use crate::dtype::DecimalDType;
478 use crate::dtype::Nullability;
479 use crate::dtype::Nullability::Nullable;
480 use crate::dtype::PType;
481 use crate::dtype::i256;
482 use crate::expr::stats::Precision;
483 use crate::expr::stats::Stat;
484 use crate::expr::stats::StatsProvider;
485 use crate::scalar::DecimalValue;
486 use crate::scalar::Scalar;
487 use crate::validity::Validity;
488
489 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
490
491 #[test]
494 fn sum_multi_batch() -> VortexResult<()> {
495 let mut ctx = SESSION.create_execution_ctx();
496 let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
497 let mut acc = Accumulator::try_new(Sum, NumericalAggregateOpts::default(), dtype)?;
498
499 let batch1 = PrimitiveArray::new(buffer![10i32, 20], Validity::NonNullable).into_array();
500 acc.accumulate(&batch1, &mut ctx)?;
501
502 let batch2 = PrimitiveArray::new(buffer![3i32, 6, 9], Validity::NonNullable).into_array();
503 acc.accumulate(&batch2, &mut ctx)?;
504
505 let result = acc.finish()?;
506 assert_eq!(result.as_primitive().typed_value::<i64>(), Some(48));
507 Ok(())
508 }
509
510 #[test]
511 fn sum_finish_resets_state() -> VortexResult<()> {
512 let mut ctx = SESSION.create_execution_ctx();
513 let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
514 let mut acc = Accumulator::try_new(Sum, NumericalAggregateOpts::default(), dtype)?;
515
516 let batch1 = PrimitiveArray::new(buffer![10i32, 20], Validity::NonNullable).into_array();
517 acc.accumulate(&batch1, &mut ctx)?;
518 let result1 = acc.finish()?;
519 assert_eq!(result1.as_primitive().typed_value::<i64>(), Some(30));
520
521 let batch2 = PrimitiveArray::new(buffer![3i32, 6, 9], Validity::NonNullable).into_array();
522 acc.accumulate(&batch2, &mut ctx)?;
523 let result2 = acc.finish()?;
524 assert_eq!(result2.as_primitive().typed_value::<i64>(), Some(18));
525 Ok(())
526 }
527
528 #[test]
531 fn sum_state_merge() -> VortexResult<()> {
532 let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
533 let options = NumericalAggregateOpts::default();
534 let owned = AggregateDTypes::try_new(&Sum, &options, dtype)?;
535 let args = owned.args(&options);
536
537 let partial_of = |value: i64| SumPartial {
538 current: Some(SumState::Signed(value)),
539 };
540 let state = Sum.merge_partials(args, partial_of(100), partial_of(50))?;
541
542 let result = Sum.to_scalar(args, &state)?;
543 assert_eq!(result.as_primitive().typed_value::<i64>(), Some(150));
544 Ok(())
545 }
546
547 #[test]
548 fn sum_overflowed_partial_poisons_reduction() -> VortexResult<()> {
549 let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
550 let options = NumericalAggregateOpts::default();
551 let owned = AggregateDTypes::try_new(&Sum, &options, dtype)?;
552 let args = owned.args(&options);
553
554 let overflowed =
555 Sum.partial_from_scalar(args, &Scalar::null(DType::Primitive(PType::I64, Nullable)))?;
556 let five = Sum.partial_from_scalar(args, &Scalar::primitive(5i64, Nullable))?;
557 let state = Sum.merge_partials(args, five, overflowed)?;
558
559 assert!(Sum.is_saturated(args, &state));
560 assert!(Sum.to_scalar(args, &state)?.is_null());
561 Ok(())
562 }
563
564 #[test]
567 fn sum_stats() -> VortexResult<()> {
568 let array = ChunkedArray::try_new(
569 vec![
570 PrimitiveArray::from_iter([1, 1, 1]).into_array(),
571 PrimitiveArray::from_iter([2, 2, 2]).into_array(),
572 ],
573 DType::Primitive(PType::I32, Nullability::NonNullable),
574 )
575 .vortex_expect("operation should succeed in test");
576 let array = array.into_array();
577 let mut ctx = SESSION.create_execution_ctx();
578 let mut acc = Accumulator::try_new(
579 Sum,
580 NumericalAggregateOpts::default(),
581 array.dtype().clone(),
582 )?;
583 acc.combine_partials(&Scalar::primitive(2i64, Nullable))?;
584 acc.accumulate(&array, &mut ctx)?;
585 assert_eq!(acc.finish()?, Scalar::primitive(11i64, Nullable));
586 assert_eq!(array.statistics().get(Stat::Sum), Precision::exact(9i64));
587
588 let sum_without_acc = sum(&array, &mut SESSION.create_execution_ctx())?;
589 assert_eq!(sum_without_acc, Scalar::primitive(9i64, Nullable));
590 Ok(())
591 }
592
593 #[test]
596 fn sum_constant_float_non_multiply() -> VortexResult<()> {
597 let acc = -2048669276050936500000000000f64;
598 let array = ConstantArray::new(6.1811675e16f64, 25);
599 let array = array.into_array();
600 let mut accumulator = Accumulator::try_new(
601 Sum,
602 NumericalAggregateOpts::default(),
603 array.dtype().clone(),
604 )?;
605 accumulator.combine_partials(&Scalar::primitive(acc, Nullable))?;
606 accumulator.accumulate(&array, &mut SESSION.create_execution_ctx())?;
607 let result = accumulator.finish()?;
608 assert_eq!(
609 f64::try_from(&result).vortex_expect("operation should succeed in test"),
610 -2048669274505644600000000000f64
611 );
612 Ok(())
613 }
614
615 fn run_grouped_sum(groups: &ArrayRef, elem_dtype: &DType) -> VortexResult<ArrayRef> {
618 let mut acc = GroupedAccumulator::try_new(
619 Sum,
620 NumericalAggregateOpts::default(),
621 elem_dtype.clone(),
622 )?;
623 acc.accumulate_list(groups, &mut SESSION.create_execution_ctx())?;
624 acc.finish()
625 }
626
627 #[test]
628 fn grouped_sum_fixed_size_list() -> VortexResult<()> {
629 let mut ctx = SESSION.create_execution_ctx();
630 let elements =
631 PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5, 6], Validity::NonNullable).into_array();
632 let groups = FixedSizeListArray::try_new(elements, 3, Validity::NonNullable, 2)?;
633
634 let elem_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
635 let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
636
637 let expected = PrimitiveArray::from_option_iter([Some(6i64), Some(15i64)]).into_array();
638 assert_arrays_eq!(&result, &expected, &mut ctx);
639 Ok(())
640 }
641
642 #[test]
643 fn grouped_sum_with_null_elements() -> VortexResult<()> {
644 let mut ctx = SESSION.create_execution_ctx();
645 let elements =
646 PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5), Some(6)])
647 .into_array();
648 let groups = FixedSizeListArray::try_new(elements, 3, Validity::NonNullable, 2)?;
649
650 let elem_dtype = DType::Primitive(PType::I32, Nullable);
651 let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
652
653 let expected = PrimitiveArray::from_option_iter([Some(4i64), Some(11i64)]).into_array();
654 assert_arrays_eq!(&result, &expected, &mut ctx);
655 Ok(())
656 }
657
658 #[test]
659 fn grouped_sum_with_null_group() -> VortexResult<()> {
660 let mut ctx = SESSION.create_execution_ctx();
661 let elements =
662 PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5, 6, 7, 8, 9], Validity::NonNullable)
663 .into_array();
664 let validity = Validity::from_iter([true, false, true]);
665 let groups = FixedSizeListArray::try_new(elements, 3, validity, 3)?;
666
667 let elem_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
668 let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
669
670 let expected =
671 PrimitiveArray::from_option_iter([Some(6i64), None, Some(24i64)]).into_array();
672 assert_arrays_eq!(&result, &expected, &mut ctx);
673 Ok(())
674 }
675
676 #[test]
677 fn grouped_sum_all_null_elements_in_group() -> VortexResult<()> {
678 let mut ctx = SESSION.create_execution_ctx();
679 let elements =
680 PrimitiveArray::from_option_iter([None::<i32>, None, Some(3), Some(4)]).into_array();
681 let groups = FixedSizeListArray::try_new(elements, 2, Validity::NonNullable, 2)?;
682
683 let elem_dtype = DType::Primitive(PType::I32, Nullable);
684 let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
685
686 let expected = PrimitiveArray::from_option_iter([Some(0i64), Some(7i64)]).into_array();
687 assert_arrays_eq!(&result, &expected, &mut ctx);
688 Ok(())
689 }
690
691 #[test]
692 fn grouped_sum_bool() -> VortexResult<()> {
693 let mut ctx = SESSION.create_execution_ctx();
694 let elements: BoolArray = [true, false, true, true, true, true].into_iter().collect();
695 let groups =
696 FixedSizeListArray::try_new(elements.into_array(), 3, Validity::NonNullable, 2)?;
697
698 let elem_dtype = DType::Bool(Nullability::NonNullable);
699 let result = run_grouped_sum(&groups.into_array(), &elem_dtype)?;
700
701 let expected = PrimitiveArray::from_option_iter([Some(2u64), Some(3u64)]).into_array();
702 assert_arrays_eq!(&result, &expected, &mut ctx);
703 Ok(())
704 }
705
706 #[test]
707 fn grouped_sum_finish_resets() -> VortexResult<()> {
708 let mut ctx = SESSION.create_execution_ctx();
709 let elem_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
710 let mut acc =
711 GroupedAccumulator::try_new(Sum, NumericalAggregateOpts::default(), elem_dtype)?;
712
713 let elements1 =
714 PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable).into_array();
715 let groups1 = FixedSizeListArray::try_new(elements1, 2, Validity::NonNullable, 2)?;
716 acc.accumulate_list(&groups1.into_array(), &mut ctx)?;
717 let result1 = acc.finish()?;
718
719 let expected1 = PrimitiveArray::from_option_iter([Some(3i64), Some(7i64)]).into_array();
720 assert_arrays_eq!(&result1, &expected1, &mut ctx);
721
722 let elements2 = PrimitiveArray::new(buffer![10i32, 20], Validity::NonNullable).into_array();
723 let groups2 = FixedSizeListArray::try_new(elements2, 2, Validity::NonNullable, 1)?;
724 acc.accumulate_list(&groups2.into_array(), &mut ctx)?;
725 let result2 = acc.finish()?;
726
727 let expected2 = PrimitiveArray::from_option_iter([Some(30i64)]).into_array();
728 assert_arrays_eq!(&result2, &expected2, &mut ctx);
729 Ok(())
730 }
731
732 #[test]
733 fn grouped_sum_listview_out_of_order_offsets_with_null_group() -> VortexResult<()> {
734 let mut ctx = SESSION.create_execution_ctx();
735 let elements =
736 PrimitiveArray::new(buffer![100i32, 200, 300], Validity::NonNullable).into_array();
737 let offsets = PrimitiveArray::new(buffer![2i32, 0, 1], Validity::NonNullable).into_array();
738 let sizes = PrimitiveArray::new(buffer![1i32, 1, 1], Validity::NonNullable).into_array();
739 let validity = Validity::from_iter([true, false, true]);
740 let groups =
741 ListViewArray::try_new(elements, offsets, sizes, validity, &mut ctx)?.into_array();
742
743 let elem_dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
744 let result = run_grouped_sum(&groups, &elem_dtype)?;
745
746 let expected =
748 PrimitiveArray::from_option_iter([Some(300i64), None, Some(200i64)]).into_array();
749 assert_arrays_eq!(&result, &expected, &mut ctx);
750 Ok(())
751 }
752
753 #[test]
756 fn sum_chunked_floats_with_nulls() -> VortexResult<()> {
757 let chunk1 =
758 PrimitiveArray::from_option_iter(vec![Some(1.5f64), None, Some(3.2), Some(4.8)]);
759 let chunk2 = PrimitiveArray::from_option_iter(vec![Some(2.1f64), Some(5.7), None]);
760 let chunk3 = PrimitiveArray::from_option_iter(vec![None, Some(1.0f64), Some(2.5), None]);
761 let dtype = chunk1.dtype().clone();
762 let chunked = ChunkedArray::try_new(
763 vec![
764 chunk1.into_array(),
765 chunk2.into_array(),
766 chunk3.into_array(),
767 ],
768 dtype,
769 )?;
770
771 let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
772 assert_eq!(result.as_primitive().as_::<f64>(), Some(20.8));
773 Ok(())
774 }
775
776 #[test]
777 fn sum_chunked_floats_all_nulls_is_zero() -> VortexResult<()> {
778 let chunk1 = PrimitiveArray::from_option_iter::<f32, _>(vec![None, None, None]);
779 let chunk2 = PrimitiveArray::from_option_iter::<f32, _>(vec![None, None]);
780 let dtype = chunk1.dtype().clone();
781 let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
782 let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
783 assert_eq!(result, Scalar::primitive(0f64, Nullable));
784 Ok(())
785 }
786
787 #[test]
788 fn sum_chunked_floats_empty_chunks() -> VortexResult<()> {
789 let chunk1 = PrimitiveArray::from_option_iter(vec![Some(10.5f64), Some(20.3)]);
790 let chunk2 = ConstantArray::new(Scalar::primitive(0f64, Nullable), 0);
791 let chunk3 = PrimitiveArray::from_option_iter(vec![Some(5.2f64)]);
792 let dtype = chunk1.dtype().clone();
793 let chunked = ChunkedArray::try_new(
794 vec![
795 chunk1.into_array(),
796 chunk2.into_array(),
797 chunk3.into_array(),
798 ],
799 dtype,
800 )?;
801
802 let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
803 assert_eq!(result.as_primitive().as_::<f64>(), Some(36.0));
804 Ok(())
805 }
806
807 #[test]
808 fn sum_chunked_int_almost_all_null() -> VortexResult<()> {
809 let chunk1 = PrimitiveArray::from_option_iter::<u32, _>(vec![Some(1)]);
810 let chunk2 = PrimitiveArray::from_option_iter::<u32, _>(vec![None]);
811 let dtype = chunk1.dtype().clone();
812 let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
813
814 let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
815 assert_eq!(result.as_primitive().as_::<u64>(), Some(1));
816 Ok(())
817 }
818
819 #[test]
820 fn sum_chunked_decimals() -> VortexResult<()> {
821 let decimal_dtype = DecimalDType::new(10, 2);
822 let chunk1 = DecimalArray::new(
823 buffer![100i32, 100i32, 100i32, 100i32, 100i32],
824 decimal_dtype,
825 Validity::AllValid,
826 );
827 let chunk2 = DecimalArray::new(
828 buffer![200i32, 200i32, 200i32],
829 decimal_dtype,
830 Validity::AllValid,
831 );
832 let chunk3 = DecimalArray::new(buffer![300i32, 300i32], decimal_dtype, Validity::AllValid);
833 let dtype = chunk1.dtype().clone();
834 let chunked = ChunkedArray::try_new(
835 vec![
836 chunk1.into_array(),
837 chunk2.into_array(),
838 chunk3.into_array(),
839 ],
840 dtype,
841 )?;
842
843 let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
844 let decimal_result = result.as_decimal();
845 assert_eq!(
846 decimal_result.decimal_value(),
847 Some(DecimalValue::I256(i256::from_i128(1700)))
848 );
849 Ok(())
850 }
851
852 #[test]
853 fn sum_chunked_decimals_with_nulls() -> VortexResult<()> {
854 let decimal_dtype = DecimalDType::new(10, 2);
855 let chunk1 = DecimalArray::new(
856 buffer![100i32, 100i32, 100i32],
857 decimal_dtype,
858 Validity::AllValid,
859 );
860 let chunk2 = DecimalArray::new(
861 buffer![0i32, 0i32],
862 decimal_dtype,
863 Validity::from_iter([false, false]),
864 );
865 let chunk3 = DecimalArray::new(buffer![200i32, 200i32], decimal_dtype, Validity::AllValid);
866 let dtype = chunk1.dtype().clone();
867 let chunked = ChunkedArray::try_new(
868 vec![
869 chunk1.into_array(),
870 chunk2.into_array(),
871 chunk3.into_array(),
872 ],
873 dtype,
874 )?;
875
876 let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
877 let decimal_result = result.as_decimal();
878 assert_eq!(
879 decimal_result.decimal_value(),
880 Some(DecimalValue::I256(i256::from_i128(700)))
881 );
882 Ok(())
883 }
884
885 #[test]
886 fn sum_chunked_decimals_large() -> VortexResult<()> {
887 let decimal_dtype = DecimalDType::new(3, 0);
888 let chunk1 = ConstantArray::new(
889 Scalar::decimal(
890 DecimalValue::I16(500),
891 decimal_dtype,
892 Nullability::NonNullable,
893 ),
894 1,
895 );
896 let chunk2 = ConstantArray::new(
897 Scalar::decimal(
898 DecimalValue::I16(600),
899 decimal_dtype,
900 Nullability::NonNullable,
901 ),
902 1,
903 );
904 let dtype = chunk1.dtype().clone();
905 let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
906
907 let result = sum(&chunked.into_array(), &mut SESSION.create_execution_ctx())?;
908 let decimal_result = result.as_decimal();
909 assert_eq!(
910 decimal_result.decimal_value(),
911 Some(DecimalValue::I256(i256::from_i128(1100)))
912 );
913 assert_eq!(
914 result.dtype(),
915 &DType::Decimal(DecimalDType::new(13, 0), Nullable)
916 );
917 Ok(())
918 }
919}