Skip to main content

vortex_array/scalar_fn/fns/
list_sum.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use vortex_error::VortexResult;
5use vortex_error::vortex_bail;
6use vortex_error::vortex_err;
7use vortex_session::VortexSession;
8use vortex_session::registry::CachedId;
9
10use crate::ArrayRef;
11use crate::Canonical;
12use crate::Columnar;
13use crate::ExecutionCtx;
14use crate::IntoArray;
15use crate::aggregate_fn::AggregateFnVTable;
16use crate::aggregate_fn::DynGroupedAccumulator;
17use crate::aggregate_fn::GroupedAccumulator;
18use crate::aggregate_fn::NumericalAggregateOpts;
19use crate::aggregate_fn::fns::sum_v2::SumV2;
20use crate::arrays::ConstantArray;
21use crate::dtype::DType;
22use crate::scalar_fn::Arity;
23use crate::scalar_fn::ChildName;
24use crate::scalar_fn::ExecutionArgs;
25use crate::scalar_fn::ScalarFnId;
26use crate::scalar_fn::ScalarFnVTable;
27
28/// Sum of the elements in each list of a `List` or `FixedSizeList` typed array.
29///
30/// Follows SQL `SUM` semantics per list, matching DuckDB's `list_sum`: null lists, empty
31/// lists, and lists whose elements are all null yield a null sum; null elements are skipped.
32/// Integer and decimal overflow yields a null sum value, matching [`SumV2`]. The result dtype
33/// follows [`SumV2`]'s widening rules and is always nullable.
34///
35/// NaN handling for float elements is controlled by [`NumericalAggregateOpts`]: with
36/// `skip_nans` (the default) NaN values contribute nothing, otherwise any NaN poisons the
37/// list's sum to NaN.
38#[derive(Clone)]
39pub struct ListSum;
40
41impl ScalarFnVTable for ListSum {
42    type Options = NumericalAggregateOpts;
43
44    fn id(&self) -> ScalarFnId {
45        static ID: CachedId = CachedId::new("vortex.list.sum");
46        *ID
47    }
48
49    fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
50        Ok(Some(options.serialize()))
51    }
52
53    fn deserialize(
54        &self,
55        metadata: &[u8],
56        _session: &VortexSession,
57    ) -> VortexResult<Self::Options> {
58        NumericalAggregateOpts::deserialize(metadata)
59    }
60
61    fn arity(&self, _options: &Self::Options) -> Arity {
62        Arity::Exact(1)
63    }
64
65    fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName {
66        match child_idx {
67            0 => ChildName::from("input"),
68            _ => unreachable!("Invalid child index {child_idx} for list_sum()"),
69        }
70    }
71
72    fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
73        let elem_dtype = match &arg_dtypes[0] {
74            DType::List(elem, _) | DType::FixedSizeList(elem, ..) => elem.as_ref(),
75            other => {
76                vortex_bail!("list_sum() requires List or FixedSizeList, got {other}")
77            }
78        };
79        SumV2
80            .return_dtype(options, elem_dtype)
81            .ok_or_else(|| vortex_err!("list_sum() cannot sum elements of type {elem_dtype}"))
82    }
83
84    fn execute(
85        &self,
86        options: &Self::Options,
87        args: &dyn ExecutionArgs,
88        ctx: &mut ExecutionCtx,
89    ) -> VortexResult<ArrayRef> {
90        let input = args.get(0)?;
91
92        let elem_dtype = match input.dtype() {
93            DType::List(elem, _) | DType::FixedSizeList(elem, ..) => elem.as_ref().clone(),
94            other => {
95                vortex_bail!("list_sum() requires List or FixedSizeList, got {other}")
96            }
97        };
98
99        let columnar = input.execute::<Columnar>(ctx)?;
100
101        match columnar {
102            Columnar::Constant(constant) => {
103                // Canonicalize one row of the constant and broadcast its sum.
104                let one_row = ConstantArray::new(constant.scalar().clone(), 1)
105                    .into_array()
106                    .execute::<Canonical>(ctx)?
107                    .into_array();
108                let sum =
109                    list_sum_impl(one_row, elem_dtype, options, ctx)?.execute_scalar(0, ctx)?;
110                Ok(ConstantArray::new(sum, constant.len()).into_array())
111            }
112            Columnar::Canonical(canonical) => {
113                list_sum_impl(canonical.into_array(), elem_dtype, options, ctx)
114            }
115        }
116    }
117
118    fn is_strict(&self, _options: &Self::Options) -> bool {
119        // A null list sums to null, which is all that strictness requires. Element nulls are part
120        // of the list value rather than row-level nulls, and a valid list may still sum to null
121        // (empty, all-null, or overflow) since strictness is only one-directional.
122        true
123    }
124
125    fn is_infallible(&self, _options: &Self::Options) -> bool {
126        true
127    }
128}
129
130/// Sum each list of a canonical `array` into one value per list.
131fn list_sum_impl(
132    canonical: ArrayRef,
133    elem_dtype: DType,
134    options: &NumericalAggregateOpts,
135    ctx: &mut ExecutionCtx,
136) -> VortexResult<ArrayRef> {
137    let mut acc = GroupedAccumulator::try_new(SumV2, *options, elem_dtype)?;
138    acc.accumulate_list(&canonical, ctx)?;
139    acc.finish()
140}
141
142#[cfg(test)]
143mod tests {
144    use std::sync::Arc;
145    use std::sync::LazyLock;
146
147    use prost::Message;
148    use rstest::rstest;
149    use vortex_buffer::buffer;
150    use vortex_error::VortexResult;
151    use vortex_session::VortexSession;
152
153    use crate::ArrayRef;
154    use crate::IntoArray;
155    use crate::VortexSessionExecute;
156    use crate::aggregate_fn::NumericalAggregateOpts;
157    use crate::arrays::BoolArray;
158    use crate::arrays::ConstantArray;
159    use crate::arrays::FixedSizeListArray;
160    use crate::arrays::ListArray;
161    use crate::arrays::ListViewArray;
162    use crate::arrays::PrimitiveArray;
163    use crate::assert_arrays_eq;
164    use crate::dtype::DType;
165    use crate::dtype::Nullability;
166    use crate::dtype::PType;
167    use crate::expr::Expression;
168    use crate::expr::list_sum;
169    use crate::expr::list_sum_opts;
170    use crate::expr::proto::ExprSerializeProtoExt;
171    use crate::expr::root;
172    use crate::proto::expr as pb;
173    use crate::scalar::Scalar;
174    use crate::scalar_fn::ScalarFnVTable;
175    use crate::scalar_fn::fns::list_sum::ListSum;
176    use crate::validity::Validity;
177
178    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
179
180    fn create_list_elements() -> ArrayRef {
181        PrimitiveArray::from_option_iter::<i32, _>([
182            Some(1),
183            Some(2),
184            Some(3),
185            Some(4),
186            Some(5),
187            Some(6),
188            None,
189        ])
190        .into_array()
191    }
192
193    #[rstest]
194    #[case(buffer![0u32, 2, 5, 5, 7].into_array())]
195    #[case(buffer![0u64, 2, 5, 5, 7].into_array())]
196    fn test_list_sum(#[case] offsets: ArrayRef) -> VortexResult<()> {
197        let mut ctx = SESSION.create_execution_ctx();
198        let elements = create_list_elements();
199        let list =
200            ListArray::try_new(elements, offsets, Validity::NonNullable, &mut ctx)?.into_array();
201        let result = list.apply(&list_sum(root()))?;
202        let mut ctx = SESSION.create_execution_ctx();
203        // [1, 2] = 3; [3, 4, 5] = 12; [] = null; [6, null] = 6 (nulls skipped).
204        let expected =
205            PrimitiveArray::from_option_iter::<i64, _>([Some(3), Some(12), None, Some(6)]);
206        assert_arrays_eq!(result, expected, &mut ctx);
207        Ok(())
208    }
209
210    #[test]
211    fn test_nullable_list_sum() -> VortexResult<()> {
212        let mut ctx = SESSION.create_execution_ctx();
213        let elements = create_list_elements();
214        let list = ListArray::try_new(
215            elements,
216            buffer![0u32, 2, 5, 5, 7].into_array(),
217            Validity::Array(BoolArray::from_iter([true, false, true, false]).into_array()),
218            &mut ctx,
219        )?
220        .into_array();
221        let result = list.apply(&list_sum(root()))?;
222
223        let mut ctx = SESSION.create_execution_ctx();
224        // Row 1 and 3 are null lists; row 2 is a valid but empty list.
225        let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(3), None, None, None]);
226        assert_arrays_eq!(result, expected, &mut ctx);
227        Ok(())
228    }
229
230    #[test]
231    fn test_all_null_elements_sum_to_null() -> VortexResult<()> {
232        let mut ctx = SESSION.create_execution_ctx();
233        let elements = PrimitiveArray::from_option_iter::<i32, _>([None, None, Some(1)]);
234        let list = ListArray::try_new(
235            elements.into_array(),
236            buffer![0u32, 2, 3].into_array(),
237            Validity::NonNullable,
238            &mut ctx,
239        )?
240        .into_array();
241        let result = list.apply(&list_sum(root()))?;
242
243        let mut ctx = SESSION.create_execution_ctx();
244        let expected = PrimitiveArray::from_option_iter::<i64, _>([None, Some(1)]);
245        assert_arrays_eq!(result, expected, &mut ctx);
246        Ok(())
247    }
248
249    #[test]
250    fn test_overflow_sums_to_null() -> VortexResult<()> {
251        let mut ctx = SESSION.create_execution_ctx();
252        let elements = PrimitiveArray::from_iter([i64::MAX, 1, 1, 2]);
253        let list = ListArray::try_new(
254            elements.into_array(),
255            buffer![0u32, 2, 4].into_array(),
256            Validity::NonNullable,
257            &mut ctx,
258        )?
259        .into_array();
260        let result = list.apply(&list_sum(root()))?;
261
262        let mut ctx = SESSION.create_execution_ctx();
263        let expected = PrimitiveArray::from_option_iter::<i64, _>([None, Some(3)]);
264        assert_arrays_eq!(result, expected, &mut ctx);
265        Ok(())
266    }
267
268    #[test]
269    fn test_unsigned_widening() -> VortexResult<()> {
270        let mut ctx = SESSION.create_execution_ctx();
271        let elements = PrimitiveArray::from_iter([1u8, 2, 3]);
272        let list = ListArray::try_new(
273            elements.into_array(),
274            buffer![0u32, 3].into_array(),
275            Validity::NonNullable,
276            &mut ctx,
277        )?
278        .into_array();
279        let result = list.apply(&list_sum(root()))?;
280
281        let mut ctx = SESSION.create_execution_ctx();
282        let expected = PrimitiveArray::from_option_iter::<u64, _>([Some(6)]);
283        assert_arrays_eq!(result, expected, &mut ctx);
284        Ok(())
285    }
286
287    #[test]
288    fn test_bool_elements() -> VortexResult<()> {
289        let mut ctx = SESSION.create_execution_ctx();
290        let elements = BoolArray::from_iter([true, true, false, true]);
291        let list = ListArray::try_new(
292            elements.into_array(),
293            buffer![0u32, 3, 4].into_array(),
294            Validity::NonNullable,
295            &mut ctx,
296        )?
297        .into_array();
298        let result = list.apply(&list_sum(root()))?;
299
300        let mut ctx = SESSION.create_execution_ctx();
301        let expected = PrimitiveArray::from_option_iter::<u64, _>([Some(2), Some(1)]);
302        assert_arrays_eq!(result, expected, &mut ctx);
303        Ok(())
304    }
305
306    #[test]
307    fn test_nan_skipped_by_default() -> VortexResult<()> {
308        let mut ctx = SESSION.create_execution_ctx();
309        let elements = PrimitiveArray::from_iter([1.0f64, f64::NAN, 2.0]);
310        let list = ListArray::try_new(
311            elements.into_array(),
312            buffer![0u32, 3].into_array(),
313            Validity::NonNullable,
314            &mut ctx,
315        )?
316        .into_array();
317        let result = list.apply(&list_sum(root()))?;
318
319        let mut ctx = SESSION.create_execution_ctx();
320        let expected = PrimitiveArray::from_option_iter::<f64, _>([Some(3.0)]);
321        assert_arrays_eq!(result, expected, &mut ctx);
322        Ok(())
323    }
324
325    #[test]
326    fn test_all_nan_list_sums_to_zero() -> VortexResult<()> {
327        let mut ctx = SESSION.create_execution_ctx();
328        // NaN elements are *valid*: with the default `skip_nans` they contribute nothing, but
329        // the list still has valid elements, so the sum is `0.0` rather than null (unlike an
330        // all-null list).
331        let elements = PrimitiveArray::from_iter([f64::NAN, f64::NAN]);
332        let list = ListArray::try_new(
333            elements.into_array(),
334            buffer![0u32, 2].into_array(),
335            Validity::NonNullable,
336            &mut ctx,
337        )?
338        .into_array();
339        let result = list.apply(&list_sum(root()))?;
340
341        let mut ctx = SESSION.create_execution_ctx();
342        let expected = PrimitiveArray::from_option_iter::<f64, _>([Some(0.0)]);
343        assert_arrays_eq!(result, expected, &mut ctx);
344        Ok(())
345    }
346
347    #[test]
348    fn test_nan_poisons_with_include_nans() -> VortexResult<()> {
349        let mut ctx = SESSION.create_execution_ctx();
350        let elements = PrimitiveArray::from_iter([1.0f64, f64::NAN, 2.0]);
351        let list = ListArray::try_new(
352            elements.into_array(),
353            buffer![0u32, 3].into_array(),
354            Validity::NonNullable,
355            &mut ctx,
356        )?
357        .into_array();
358        let result = list.apply(&list_sum_opts(
359            root(),
360            NumericalAggregateOpts::include_nans(),
361        ))?;
362
363        let mut ctx = SESSION.create_execution_ctx();
364        assert!(result.is_valid(0, &mut ctx)?);
365        let result = result.execute::<PrimitiveArray>(&mut ctx)?;
366        assert!(result.as_slice::<f64>()[0].is_nan());
367        Ok(())
368    }
369
370    #[test]
371    fn test_listview_sum() -> VortexResult<()> {
372        let mut ctx = SESSION.create_execution_ctx();
373        let elements = create_list_elements();
374        // Overlapping and out-of-order views over the shared elements.
375        let lv = ListViewArray::new(
376            elements,
377            buffer![5u32, 0, 4, 1].into_array(),
378            buffer![2u32, 3, 0, 2].into_array(),
379            Validity::NonNullable,
380            &mut ctx,
381        )
382        .into_array();
383        let result = lv.apply(&list_sum(root()))?;
384
385        let mut ctx = SESSION.create_execution_ctx();
386        // [6, null] = 6; [1, 2, 3] = 6; [] = null; [2, 3] = 5.
387        let expected =
388            PrimitiveArray::from_option_iter::<i64, _>([Some(6), Some(6), None, Some(5)]);
389        assert_arrays_eq!(result, expected, &mut ctx);
390        Ok(())
391    }
392
393    fn create_fixed_size_list(validity: Validity) -> ArrayRef {
394        // 4 lists of size 2 over 8 primitive elements.
395        let elements = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5, 6, 7, 8]).into_array();
396        FixedSizeListArray::new(elements, 2, validity, 4).into_array()
397    }
398
399    #[test]
400    fn test_fixed_size_list_sum() -> VortexResult<()> {
401        let fsl = create_fixed_size_list(Validity::NonNullable);
402        let result = fsl.apply(&list_sum(root()))?;
403
404        let mut ctx = SESSION.create_execution_ctx();
405        let expected =
406            PrimitiveArray::from_option_iter::<i64, _>([Some(3), Some(7), Some(11), Some(15)]);
407        assert_arrays_eq!(result, expected, &mut ctx);
408        Ok(())
409    }
410
411    #[test]
412    fn test_fixed_size_list_sum_nullable() -> VortexResult<()> {
413        let fsl = create_fixed_size_list(Validity::Array(
414            BoolArray::from_iter([true, false, true, false]).into_array(),
415        ));
416        let result = fsl.apply(&list_sum(root()))?;
417
418        let mut ctx = SESSION.create_execution_ctx();
419        let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(3), None, Some(11), None]);
420        assert_arrays_eq!(result, expected, &mut ctx);
421        Ok(())
422    }
423
424    #[test]
425    fn test_list_sum_take() -> VortexResult<()> {
426        let mut ctx = SESSION.create_execution_ctx();
427        let elements = create_list_elements();
428        let list = ListArray::try_new(
429            elements,
430            buffer![0u32, 2, 5, 5, 7].into_array(),
431            Validity::NonNullable,
432            &mut ctx,
433        )?
434        .into_array();
435        let taken = list.take(buffer![3u64, 0, 2].into_array())?;
436
437        let result = taken.apply(&list_sum(root()))?;
438        let mut ctx = SESSION.create_execution_ctx();
439        let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(6), Some(3), None]);
440        assert_arrays_eq!(result, expected, &mut ctx);
441        Ok(())
442    }
443
444    #[test]
445    fn test_list_sum_slice() -> VortexResult<()> {
446        let mut ctx = SESSION.create_execution_ctx();
447        let elements = create_list_elements();
448        let list = ListArray::try_new(
449            elements,
450            buffer![0u32, 2, 5, 5, 7].into_array(),
451            Validity::NonNullable,
452            &mut ctx,
453        )?
454        .into_array();
455        let sliced = list.slice(1..4)?;
456
457        let result = sliced.apply(&list_sum(root()))?;
458        let mut ctx = SESSION.create_execution_ctx();
459        let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(12), None, Some(6)]);
460        assert_arrays_eq!(result, expected, &mut ctx);
461        Ok(())
462    }
463
464    #[test]
465    fn test_empty_array() -> VortexResult<()> {
466        let mut ctx = SESSION.create_execution_ctx();
467        let elements = PrimitiveArray::from_iter([0i32; 0]);
468        let list = ListArray::try_new(
469            elements.into_array(),
470            buffer![0u32].into_array(),
471            Validity::NonNullable,
472            &mut ctx,
473        )?
474        .into_array();
475        let result = list.apply(&list_sum(root()))?;
476
477        let mut ctx = SESSION.create_execution_ctx();
478        assert_arrays_eq!(
479            result,
480            PrimitiveArray::from_option_iter::<i64, _>([]),
481            &mut ctx
482        );
483        Ok(())
484    }
485
486    #[test]
487    fn test_constant_list_sum() -> VortexResult<()> {
488        let mut ctx = SESSION.create_execution_ctx();
489        let elements = create_list_elements();
490        let list = ListArray::try_new(
491            elements,
492            buffer![0u32, 2, 5, 5, 7].into_array(),
493            Validity::NonNullable,
494            &mut ctx,
495        )?
496        .into_array();
497        let mut ctx = SESSION.create_execution_ctx();
498        let scalar = list.execute_scalar(1, &mut ctx)?;
499
500        let constant = ConstantArray::new(scalar, 3).into_array();
501        let result = constant.apply(&list_sum(root()))?;
502        let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(12), Some(12), Some(12)]);
503        assert_arrays_eq!(result, expected, &mut ctx);
504        Ok(())
505    }
506
507    #[test]
508    fn test_null_scalar_list_sum() -> VortexResult<()> {
509        let null_scalar = Scalar::null(DType::List(
510            Arc::new(DType::Primitive(PType::I32, Nullability::NonNullable)),
511            Nullability::Nullable,
512        ));
513        let array = ConstantArray::new(null_scalar, 2).into_array();
514        let result = array.apply(&list_sum(root()))?;
515
516        let mut ctx = SESSION.create_execution_ctx();
517        assert!(!result.is_valid(0, &mut ctx)?);
518        assert!(!result.is_valid(1, &mut ctx)?);
519        assert_arrays_eq!(
520            result,
521            PrimitiveArray::from_option_iter([None::<i64>, None]),
522            &mut ctx
523        );
524        Ok(())
525    }
526
527    #[test]
528    fn test_unsupported_dtypes() {
529        let opts = NumericalAggregateOpts::default();
530        // Non-list inputs are rejected.
531        assert!(
532            ListSum
533                .return_dtype(
534                    &opts,
535                    &[DType::Primitive(PType::I32, Nullability::NonNullable)]
536                )
537                .is_err()
538        );
539        // Non-numeric element types are rejected.
540        assert!(
541            ListSum
542                .return_dtype(
543                    &opts,
544                    &[DType::List(
545                        Arc::new(DType::Utf8(Nullability::NonNullable)),
546                        Nullability::NonNullable,
547                    )],
548                )
549                .is_err()
550        );
551    }
552
553    #[test]
554    fn test_display() {
555        assert_eq!(list_sum(root()).to_string(), "vortex.list.sum($)");
556        assert_eq!(
557            list_sum_opts(root(), NumericalAggregateOpts::include_nans()).to_string(),
558            "vortex.list.sum($, opts=skip_nans=false)"
559        );
560    }
561
562    #[test]
563    fn test_proto_round_trip() -> VortexResult<()> {
564        for expr in [
565            list_sum(root()),
566            list_sum_opts(root(), NumericalAggregateOpts::include_nans()),
567        ] {
568            let proto = expr.serialize_proto()?;
569            let buf = proto.encode_to_vec();
570            let decoded = pb::Expr::decode(buf.as_slice())?;
571            let deser = Expression::from_proto(&decoded, &SESSION)?;
572            assert_eq!(expr, deser);
573        }
574        Ok(())
575    }
576}