1use vortex_error::VortexResult;
5use vortex_error::vortex_bail;
6use vortex_error::vortex_err;
7use vortex_session::VortexSession;
8use vortex_session::registry::CachedId;
9
10use crate::ArrayRef;
11use crate::Canonical;
12use crate::Columnar;
13use crate::ExecutionCtx;
14use crate::IntoArray;
15use crate::aggregate_fn::AggregateFnVTable;
16use crate::aggregate_fn::DynGroupedAccumulator;
17use crate::aggregate_fn::GroupedAccumulator;
18use crate::aggregate_fn::NumericalAggregateOpts;
19use crate::aggregate_fn::fns::sum_v2::SumV2;
20use crate::arrays::ConstantArray;
21use crate::dtype::DType;
22use crate::scalar_fn::Arity;
23use crate::scalar_fn::ChildName;
24use crate::scalar_fn::ExecutionArgs;
25use crate::scalar_fn::ScalarFnId;
26use crate::scalar_fn::ScalarFnVTable;
27
28#[derive(Clone)]
39pub struct ListSum;
40
41impl ScalarFnVTable for ListSum {
42 type Options = NumericalAggregateOpts;
43
44 fn id(&self) -> ScalarFnId {
45 static ID: CachedId = CachedId::new("vortex.list.sum");
46 *ID
47 }
48
49 fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
50 Ok(Some(options.serialize()))
51 }
52
53 fn deserialize(
54 &self,
55 metadata: &[u8],
56 _session: &VortexSession,
57 ) -> VortexResult<Self::Options> {
58 NumericalAggregateOpts::deserialize(metadata)
59 }
60
61 fn arity(&self, _options: &Self::Options) -> Arity {
62 Arity::Exact(1)
63 }
64
65 fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName {
66 match child_idx {
67 0 => ChildName::from("input"),
68 _ => unreachable!("Invalid child index {child_idx} for list_sum()"),
69 }
70 }
71
72 fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult<DType> {
73 let elem_dtype = match &arg_dtypes[0] {
74 DType::List(elem, _) | DType::FixedSizeList(elem, ..) => elem.as_ref(),
75 other => {
76 vortex_bail!("list_sum() requires List or FixedSizeList, got {other}")
77 }
78 };
79 SumV2
80 .return_dtype(options, elem_dtype)
81 .ok_or_else(|| vortex_err!("list_sum() cannot sum elements of type {elem_dtype}"))
82 }
83
84 fn execute(
85 &self,
86 options: &Self::Options,
87 args: &dyn ExecutionArgs,
88 ctx: &mut ExecutionCtx,
89 ) -> VortexResult<ArrayRef> {
90 let input = args.get(0)?;
91
92 let elem_dtype = match input.dtype() {
93 DType::List(elem, _) | DType::FixedSizeList(elem, ..) => elem.as_ref().clone(),
94 other => {
95 vortex_bail!("list_sum() requires List or FixedSizeList, got {other}")
96 }
97 };
98
99 let columnar = input.execute::<Columnar>(ctx)?;
100
101 match columnar {
102 Columnar::Constant(constant) => {
103 let one_row = ConstantArray::new(constant.scalar().clone(), 1)
105 .into_array()
106 .execute::<Canonical>(ctx)?
107 .into_array();
108 let sum =
109 list_sum_impl(one_row, elem_dtype, options, ctx)?.execute_scalar(0, ctx)?;
110 Ok(ConstantArray::new(sum, constant.len()).into_array())
111 }
112 Columnar::Canonical(canonical) => {
113 list_sum_impl(canonical.into_array(), elem_dtype, options, ctx)
114 }
115 }
116 }
117
118 fn is_strict(&self, _options: &Self::Options) -> bool {
119 true
123 }
124
125 fn is_infallible(&self, _options: &Self::Options) -> bool {
126 true
127 }
128}
129
130fn list_sum_impl(
132 canonical: ArrayRef,
133 elem_dtype: DType,
134 options: &NumericalAggregateOpts,
135 ctx: &mut ExecutionCtx,
136) -> VortexResult<ArrayRef> {
137 let mut acc = GroupedAccumulator::try_new(SumV2, *options, elem_dtype)?;
138 acc.accumulate_list(&canonical, ctx)?;
139 acc.finish()
140}
141
142#[cfg(test)]
143mod tests {
144 use std::sync::Arc;
145 use std::sync::LazyLock;
146
147 use prost::Message;
148 use rstest::rstest;
149 use vortex_buffer::buffer;
150 use vortex_error::VortexResult;
151 use vortex_session::VortexSession;
152
153 use crate::ArrayRef;
154 use crate::IntoArray;
155 use crate::VortexSessionExecute;
156 use crate::aggregate_fn::NumericalAggregateOpts;
157 use crate::arrays::BoolArray;
158 use crate::arrays::ConstantArray;
159 use crate::arrays::FixedSizeListArray;
160 use crate::arrays::ListArray;
161 use crate::arrays::ListViewArray;
162 use crate::arrays::PrimitiveArray;
163 use crate::assert_arrays_eq;
164 use crate::dtype::DType;
165 use crate::dtype::Nullability;
166 use crate::dtype::PType;
167 use crate::expr::Expression;
168 use crate::expr::list_sum;
169 use crate::expr::list_sum_opts;
170 use crate::expr::proto::ExprSerializeProtoExt;
171 use crate::expr::root;
172 use crate::proto::expr as pb;
173 use crate::scalar::Scalar;
174 use crate::scalar_fn::ScalarFnVTable;
175 use crate::scalar_fn::fns::list_sum::ListSum;
176 use crate::validity::Validity;
177
178 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
179
180 fn create_list_elements() -> ArrayRef {
181 PrimitiveArray::from_option_iter::<i32, _>([
182 Some(1),
183 Some(2),
184 Some(3),
185 Some(4),
186 Some(5),
187 Some(6),
188 None,
189 ])
190 .into_array()
191 }
192
193 #[rstest]
194 #[case(buffer![0u32, 2, 5, 5, 7].into_array())]
195 #[case(buffer![0u64, 2, 5, 5, 7].into_array())]
196 fn test_list_sum(#[case] offsets: ArrayRef) -> VortexResult<()> {
197 let mut ctx = SESSION.create_execution_ctx();
198 let elements = create_list_elements();
199 let list =
200 ListArray::try_new(elements, offsets, Validity::NonNullable, &mut ctx)?.into_array();
201 let result = list.apply(&list_sum(root()))?;
202 let mut ctx = SESSION.create_execution_ctx();
203 let expected =
205 PrimitiveArray::from_option_iter::<i64, _>([Some(3), Some(12), None, Some(6)]);
206 assert_arrays_eq!(result, expected, &mut ctx);
207 Ok(())
208 }
209
210 #[test]
211 fn test_nullable_list_sum() -> VortexResult<()> {
212 let mut ctx = SESSION.create_execution_ctx();
213 let elements = create_list_elements();
214 let list = ListArray::try_new(
215 elements,
216 buffer![0u32, 2, 5, 5, 7].into_array(),
217 Validity::Array(BoolArray::from_iter([true, false, true, false]).into_array()),
218 &mut ctx,
219 )?
220 .into_array();
221 let result = list.apply(&list_sum(root()))?;
222
223 let mut ctx = SESSION.create_execution_ctx();
224 let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(3), None, None, None]);
226 assert_arrays_eq!(result, expected, &mut ctx);
227 Ok(())
228 }
229
230 #[test]
231 fn test_all_null_elements_sum_to_null() -> VortexResult<()> {
232 let mut ctx = SESSION.create_execution_ctx();
233 let elements = PrimitiveArray::from_option_iter::<i32, _>([None, None, Some(1)]);
234 let list = ListArray::try_new(
235 elements.into_array(),
236 buffer![0u32, 2, 3].into_array(),
237 Validity::NonNullable,
238 &mut ctx,
239 )?
240 .into_array();
241 let result = list.apply(&list_sum(root()))?;
242
243 let mut ctx = SESSION.create_execution_ctx();
244 let expected = PrimitiveArray::from_option_iter::<i64, _>([None, Some(1)]);
245 assert_arrays_eq!(result, expected, &mut ctx);
246 Ok(())
247 }
248
249 #[test]
250 fn test_overflow_sums_to_null() -> VortexResult<()> {
251 let mut ctx = SESSION.create_execution_ctx();
252 let elements = PrimitiveArray::from_iter([i64::MAX, 1, 1, 2]);
253 let list = ListArray::try_new(
254 elements.into_array(),
255 buffer![0u32, 2, 4].into_array(),
256 Validity::NonNullable,
257 &mut ctx,
258 )?
259 .into_array();
260 let result = list.apply(&list_sum(root()))?;
261
262 let mut ctx = SESSION.create_execution_ctx();
263 let expected = PrimitiveArray::from_option_iter::<i64, _>([None, Some(3)]);
264 assert_arrays_eq!(result, expected, &mut ctx);
265 Ok(())
266 }
267
268 #[test]
269 fn test_unsigned_widening() -> VortexResult<()> {
270 let mut ctx = SESSION.create_execution_ctx();
271 let elements = PrimitiveArray::from_iter([1u8, 2, 3]);
272 let list = ListArray::try_new(
273 elements.into_array(),
274 buffer![0u32, 3].into_array(),
275 Validity::NonNullable,
276 &mut ctx,
277 )?
278 .into_array();
279 let result = list.apply(&list_sum(root()))?;
280
281 let mut ctx = SESSION.create_execution_ctx();
282 let expected = PrimitiveArray::from_option_iter::<u64, _>([Some(6)]);
283 assert_arrays_eq!(result, expected, &mut ctx);
284 Ok(())
285 }
286
287 #[test]
288 fn test_bool_elements() -> VortexResult<()> {
289 let mut ctx = SESSION.create_execution_ctx();
290 let elements = BoolArray::from_iter([true, true, false, true]);
291 let list = ListArray::try_new(
292 elements.into_array(),
293 buffer![0u32, 3, 4].into_array(),
294 Validity::NonNullable,
295 &mut ctx,
296 )?
297 .into_array();
298 let result = list.apply(&list_sum(root()))?;
299
300 let mut ctx = SESSION.create_execution_ctx();
301 let expected = PrimitiveArray::from_option_iter::<u64, _>([Some(2), Some(1)]);
302 assert_arrays_eq!(result, expected, &mut ctx);
303 Ok(())
304 }
305
306 #[test]
307 fn test_nan_skipped_by_default() -> VortexResult<()> {
308 let mut ctx = SESSION.create_execution_ctx();
309 let elements = PrimitiveArray::from_iter([1.0f64, f64::NAN, 2.0]);
310 let list = ListArray::try_new(
311 elements.into_array(),
312 buffer![0u32, 3].into_array(),
313 Validity::NonNullable,
314 &mut ctx,
315 )?
316 .into_array();
317 let result = list.apply(&list_sum(root()))?;
318
319 let mut ctx = SESSION.create_execution_ctx();
320 let expected = PrimitiveArray::from_option_iter::<f64, _>([Some(3.0)]);
321 assert_arrays_eq!(result, expected, &mut ctx);
322 Ok(())
323 }
324
325 #[test]
326 fn test_all_nan_list_sums_to_zero() -> VortexResult<()> {
327 let mut ctx = SESSION.create_execution_ctx();
328 let elements = PrimitiveArray::from_iter([f64::NAN, f64::NAN]);
332 let list = ListArray::try_new(
333 elements.into_array(),
334 buffer![0u32, 2].into_array(),
335 Validity::NonNullable,
336 &mut ctx,
337 )?
338 .into_array();
339 let result = list.apply(&list_sum(root()))?;
340
341 let mut ctx = SESSION.create_execution_ctx();
342 let expected = PrimitiveArray::from_option_iter::<f64, _>([Some(0.0)]);
343 assert_arrays_eq!(result, expected, &mut ctx);
344 Ok(())
345 }
346
347 #[test]
348 fn test_nan_poisons_with_include_nans() -> VortexResult<()> {
349 let mut ctx = SESSION.create_execution_ctx();
350 let elements = PrimitiveArray::from_iter([1.0f64, f64::NAN, 2.0]);
351 let list = ListArray::try_new(
352 elements.into_array(),
353 buffer![0u32, 3].into_array(),
354 Validity::NonNullable,
355 &mut ctx,
356 )?
357 .into_array();
358 let result = list.apply(&list_sum_opts(
359 root(),
360 NumericalAggregateOpts::include_nans(),
361 ))?;
362
363 let mut ctx = SESSION.create_execution_ctx();
364 assert!(result.is_valid(0, &mut ctx)?);
365 let result = result.execute::<PrimitiveArray>(&mut ctx)?;
366 assert!(result.as_slice::<f64>()[0].is_nan());
367 Ok(())
368 }
369
370 #[test]
371 fn test_listview_sum() -> VortexResult<()> {
372 let mut ctx = SESSION.create_execution_ctx();
373 let elements = create_list_elements();
374 let lv = ListViewArray::new(
376 elements,
377 buffer![5u32, 0, 4, 1].into_array(),
378 buffer![2u32, 3, 0, 2].into_array(),
379 Validity::NonNullable,
380 &mut ctx,
381 )
382 .into_array();
383 let result = lv.apply(&list_sum(root()))?;
384
385 let mut ctx = SESSION.create_execution_ctx();
386 let expected =
388 PrimitiveArray::from_option_iter::<i64, _>([Some(6), Some(6), None, Some(5)]);
389 assert_arrays_eq!(result, expected, &mut ctx);
390 Ok(())
391 }
392
393 fn create_fixed_size_list(validity: Validity) -> ArrayRef {
394 let elements = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5, 6, 7, 8]).into_array();
396 FixedSizeListArray::new(elements, 2, validity, 4).into_array()
397 }
398
399 #[test]
400 fn test_fixed_size_list_sum() -> VortexResult<()> {
401 let fsl = create_fixed_size_list(Validity::NonNullable);
402 let result = fsl.apply(&list_sum(root()))?;
403
404 let mut ctx = SESSION.create_execution_ctx();
405 let expected =
406 PrimitiveArray::from_option_iter::<i64, _>([Some(3), Some(7), Some(11), Some(15)]);
407 assert_arrays_eq!(result, expected, &mut ctx);
408 Ok(())
409 }
410
411 #[test]
412 fn test_fixed_size_list_sum_nullable() -> VortexResult<()> {
413 let fsl = create_fixed_size_list(Validity::Array(
414 BoolArray::from_iter([true, false, true, false]).into_array(),
415 ));
416 let result = fsl.apply(&list_sum(root()))?;
417
418 let mut ctx = SESSION.create_execution_ctx();
419 let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(3), None, Some(11), None]);
420 assert_arrays_eq!(result, expected, &mut ctx);
421 Ok(())
422 }
423
424 #[test]
425 fn test_list_sum_take() -> VortexResult<()> {
426 let mut ctx = SESSION.create_execution_ctx();
427 let elements = create_list_elements();
428 let list = ListArray::try_new(
429 elements,
430 buffer![0u32, 2, 5, 5, 7].into_array(),
431 Validity::NonNullable,
432 &mut ctx,
433 )?
434 .into_array();
435 let taken = list.take(buffer![3u64, 0, 2].into_array())?;
436
437 let result = taken.apply(&list_sum(root()))?;
438 let mut ctx = SESSION.create_execution_ctx();
439 let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(6), Some(3), None]);
440 assert_arrays_eq!(result, expected, &mut ctx);
441 Ok(())
442 }
443
444 #[test]
445 fn test_list_sum_slice() -> VortexResult<()> {
446 let mut ctx = SESSION.create_execution_ctx();
447 let elements = create_list_elements();
448 let list = ListArray::try_new(
449 elements,
450 buffer![0u32, 2, 5, 5, 7].into_array(),
451 Validity::NonNullable,
452 &mut ctx,
453 )?
454 .into_array();
455 let sliced = list.slice(1..4)?;
456
457 let result = sliced.apply(&list_sum(root()))?;
458 let mut ctx = SESSION.create_execution_ctx();
459 let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(12), None, Some(6)]);
460 assert_arrays_eq!(result, expected, &mut ctx);
461 Ok(())
462 }
463
464 #[test]
465 fn test_empty_array() -> VortexResult<()> {
466 let mut ctx = SESSION.create_execution_ctx();
467 let elements = PrimitiveArray::from_iter([0i32; 0]);
468 let list = ListArray::try_new(
469 elements.into_array(),
470 buffer![0u32].into_array(),
471 Validity::NonNullable,
472 &mut ctx,
473 )?
474 .into_array();
475 let result = list.apply(&list_sum(root()))?;
476
477 let mut ctx = SESSION.create_execution_ctx();
478 assert_arrays_eq!(
479 result,
480 PrimitiveArray::from_option_iter::<i64, _>([]),
481 &mut ctx
482 );
483 Ok(())
484 }
485
486 #[test]
487 fn test_constant_list_sum() -> VortexResult<()> {
488 let mut ctx = SESSION.create_execution_ctx();
489 let elements = create_list_elements();
490 let list = ListArray::try_new(
491 elements,
492 buffer![0u32, 2, 5, 5, 7].into_array(),
493 Validity::NonNullable,
494 &mut ctx,
495 )?
496 .into_array();
497 let mut ctx = SESSION.create_execution_ctx();
498 let scalar = list.execute_scalar(1, &mut ctx)?;
499
500 let constant = ConstantArray::new(scalar, 3).into_array();
501 let result = constant.apply(&list_sum(root()))?;
502 let expected = PrimitiveArray::from_option_iter::<i64, _>([Some(12), Some(12), Some(12)]);
503 assert_arrays_eq!(result, expected, &mut ctx);
504 Ok(())
505 }
506
507 #[test]
508 fn test_null_scalar_list_sum() -> VortexResult<()> {
509 let null_scalar = Scalar::null(DType::List(
510 Arc::new(DType::Primitive(PType::I32, Nullability::NonNullable)),
511 Nullability::Nullable,
512 ));
513 let array = ConstantArray::new(null_scalar, 2).into_array();
514 let result = array.apply(&list_sum(root()))?;
515
516 let mut ctx = SESSION.create_execution_ctx();
517 assert!(!result.is_valid(0, &mut ctx)?);
518 assert!(!result.is_valid(1, &mut ctx)?);
519 assert_arrays_eq!(
520 result,
521 PrimitiveArray::from_option_iter([None::<i64>, None]),
522 &mut ctx
523 );
524 Ok(())
525 }
526
527 #[test]
528 fn test_unsupported_dtypes() {
529 let opts = NumericalAggregateOpts::default();
530 assert!(
532 ListSum
533 .return_dtype(
534 &opts,
535 &[DType::Primitive(PType::I32, Nullability::NonNullable)]
536 )
537 .is_err()
538 );
539 assert!(
541 ListSum
542 .return_dtype(
543 &opts,
544 &[DType::List(
545 Arc::new(DType::Utf8(Nullability::NonNullable)),
546 Nullability::NonNullable,
547 )],
548 )
549 .is_err()
550 );
551 }
552
553 #[test]
554 fn test_display() {
555 assert_eq!(list_sum(root()).to_string(), "vortex.list.sum($)");
556 assert_eq!(
557 list_sum_opts(root(), NumericalAggregateOpts::include_nans()).to_string(),
558 "vortex.list.sum($, opts=skip_nans=false)"
559 );
560 }
561
562 #[test]
563 fn test_proto_round_trip() -> VortexResult<()> {
564 for expr in [
565 list_sum(root()),
566 list_sum_opts(root(), NumericalAggregateOpts::include_nans()),
567 ] {
568 let proto = expr.serialize_proto()?;
569 let buf = proto.encode_to_vec();
570 let decoded = pb::Expr::decode(buf.as_slice())?;
571 let deser = Expression::from_proto(&decoded, &SESSION)?;
572 assert_eq!(expr, deser);
573 }
574 Ok(())
575 }
576}