Skip to main content

vortex_array/aggregate_fn/fns/count/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4mod grouped;
5pub(crate) use grouped::CountGroupedKernel;
6use vortex_error::VortexExpect;
7use vortex_error::VortexResult;
8use vortex_session::VortexSession;
9use vortex_session::registry::CachedId;
10
11use crate::ArrayRef;
12use crate::Columnar;
13use crate::ExecutionCtx;
14use crate::aggregate_fn::AggregateArgs;
15use crate::aggregate_fn::AggregateFnId;
16use crate::aggregate_fn::AggregateFnVTable;
17use crate::aggregate_fn::NumericalAggregateOpts;
18use crate::aggregate_fn::fns::nan_count::nan_count;
19use crate::dtype::DType;
20use crate::dtype::Nullability;
21use crate::dtype::PType;
22use crate::scalar::Scalar;
23
24/// Count the number of non-null elements in an array.
25///
26/// Applies to all types. Returns a `u64` count.
27/// The identity value is zero.
28///
29/// For float inputs, NaN handling is controlled by [`NumericalAggregateOpts`]: with `skip_nans` (the
30/// default) NaN values are treated as missing and excluded from the count, otherwise they are
31/// counted like any other non-null value.
32#[derive(Clone, Debug)]
33pub struct Count;
34
35impl AggregateFnVTable for Count {
36    type Options = NumericalAggregateOpts;
37    type Partial = u64;
38
39    fn id(&self) -> AggregateFnId {
40        static ID: CachedId = CachedId::new("vortex.count");
41        *ID
42    }
43
44    fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
45        Ok(Some(options.serialize()))
46    }
47
48    fn deserialize(
49        &self,
50        metadata: &[u8],
51        _session: &VortexSession,
52    ) -> VortexResult<Self::Options> {
53        NumericalAggregateOpts::deserialize(metadata)
54    }
55
56    fn return_dtype(&self, _options: &Self::Options, _input_dtype: &DType) -> Option<DType> {
57        Some(DType::Primitive(PType::U64, Nullability::NonNullable))
58    }
59
60    fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
61        self.return_dtype(options, input_dtype)
62    }
63
64    fn empty_partial(
65        &self,
66        _args: AggregateArgs<'_, Self::Options>,
67    ) -> VortexResult<Self::Partial> {
68        Ok(0)
69    }
70
71    fn partial_from_scalar(
72        &self,
73        _args: AggregateArgs<'_, Self::Options>,
74        scalar: &Scalar,
75    ) -> VortexResult<Self::Partial> {
76        Ok(scalar
77            .as_primitive()
78            .typed_value::<u64>()
79            .vortex_expect("count partial should not be null"))
80    }
81
82    fn merge_partials(
83        &self,
84        _args: AggregateArgs<'_, Self::Options>,
85        first: Self::Partial,
86        second: Self::Partial,
87    ) -> VortexResult<Self::Partial> {
88        Ok(first + second)
89    }
90
91    fn to_scalar(
92        &self,
93        _args: AggregateArgs<'_, Self::Options>,
94        partial: &Self::Partial,
95    ) -> VortexResult<Scalar> {
96        Ok(Scalar::primitive(*partial, Nullability::NonNullable))
97    }
98
99    #[inline]
100    fn is_saturated(
101        &self,
102        _args: AggregateArgs<'_, Self::Options>,
103        _partial: &Self::Partial,
104    ) -> bool {
105        false
106    }
107
108    fn try_accumulate(
109        &self,
110        args: AggregateArgs<'_, Self::Options>,
111        state: &mut Self::Partial,
112        batch: &ArrayRef,
113        ctx: &mut ExecutionCtx,
114    ) -> VortexResult<bool> {
115        let mut count = batch.valid_count(ctx)? as u64;
116        // NaN values are excluded from the count of a float input when they are skipped.
117        if args.options.skip_nans && args.dtype.is_float() {
118            // `nan_count` shortcircuits on an exact `Stat::NaNCount` before scanning the batch.
119            count = count.saturating_sub(nan_count(batch, ctx)? as u64);
120        }
121        *state += count;
122        Ok(true)
123    }
124
125    fn accumulate(
126        &self,
127        _args: AggregateArgs<'_, Self::Options>,
128        _partial: &mut Self::Partial,
129        _batch: &Columnar,
130        _ctx: &mut ExecutionCtx,
131    ) -> VortexResult<()> {
132        unreachable!("Count::try_accumulate handles all arrays")
133    }
134
135    fn finalize(
136        &self,
137        _args: AggregateArgs<'_, Self::Options>,
138        partials: ArrayRef,
139    ) -> VortexResult<ArrayRef> {
140        Ok(partials)
141    }
142
143    fn finalize_scalar(
144        &self,
145        args: AggregateArgs<'_, Self::Options>,
146        partial: &Self::Partial,
147    ) -> VortexResult<Scalar> {
148        self.to_scalar(args, partial)
149    }
150}
151
152#[cfg(test)]
153mod tests {
154    use std::sync::LazyLock;
155
156    use vortex_buffer::buffer;
157    use vortex_error::VortexExpect;
158    use vortex_error::VortexResult;
159    use vortex_session::VortexSession;
160
161    use crate::ArrayRef;
162    use crate::ExecutionCtx;
163    use crate::IntoArray;
164    use crate::VortexSessionExecute;
165    use crate::aggregate_fn::Accumulator;
166    use crate::aggregate_fn::AggregateDTypes;
167    use crate::aggregate_fn::AggregateFnVTable;
168    use crate::aggregate_fn::DynAccumulator;
169    use crate::aggregate_fn::NumericalAggregateOpts;
170    use crate::aggregate_fn::fns::count::Count;
171    use crate::arrays::ChunkedArray;
172    use crate::arrays::ConstantArray;
173    use crate::arrays::PrimitiveArray;
174    use crate::dtype::DType;
175    use crate::dtype::Nullability;
176    use crate::dtype::PType;
177    use crate::expr::stats::Precision;
178    use crate::expr::stats::Stat;
179    use crate::scalar::Scalar;
180    use crate::scalar::ScalarValue;
181    use crate::validity::Validity;
182
183    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
184
185    pub fn count(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
186        let mut acc = Accumulator::try_new(
187            Count,
188            NumericalAggregateOpts::default(),
189            array.dtype().clone(),
190        )?;
191        acc.accumulate(array, ctx)?;
192        let result = acc.finish()?;
193
194        Ok(usize::try_from(
195            result
196                .as_primitive()
197                .typed_value::<u64>()
198                .vortex_expect("count result should not be null"),
199        )?)
200    }
201
202    #[test]
203    fn count_all_valid() -> VortexResult<()> {
204        let array =
205            PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5], Validity::NonNullable).into_array();
206        let mut ctx = SESSION.create_execution_ctx();
207        assert_eq!(count(&array, &mut ctx)?, 5);
208        Ok(())
209    }
210
211    #[test]
212    fn count_with_nulls() -> VortexResult<()> {
213        let array = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5)])
214            .into_array();
215        let mut ctx = SESSION.create_execution_ctx();
216        assert_eq!(count(&array, &mut ctx)?, 3);
217        Ok(())
218    }
219
220    #[test]
221    fn count_all_null() -> VortexResult<()> {
222        let array = PrimitiveArray::from_option_iter::<i32, _>([None, None, None]).into_array();
223        let mut ctx = SESSION.create_execution_ctx();
224        assert_eq!(count(&array, &mut ctx)?, 0);
225        Ok(())
226    }
227
228    #[test]
229    fn count_empty() -> VortexResult<()> {
230        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
231        let mut acc = Accumulator::try_new(Count, NumericalAggregateOpts::default(), dtype)?;
232        let result = acc.finish()?;
233        assert_eq!(result.as_primitive().typed_value::<u64>(), Some(0));
234        Ok(())
235    }
236
237    #[test]
238    fn count_multi_batch() -> VortexResult<()> {
239        let mut ctx = SESSION.create_execution_ctx();
240        let dtype = DType::Primitive(PType::I32, Nullability::Nullable);
241        let mut acc = Accumulator::try_new(Count, NumericalAggregateOpts::default(), dtype)?;
242
243        let batch1 = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array();
244        acc.accumulate(&batch1, &mut ctx)?;
245
246        let batch2 = PrimitiveArray::from_option_iter([None, Some(5i32)]).into_array();
247        acc.accumulate(&batch2, &mut ctx)?;
248
249        let result = acc.finish()?;
250        assert_eq!(result.as_primitive().typed_value::<u64>(), Some(3));
251        Ok(())
252    }
253
254    #[test]
255    fn count_finish_resets_state() -> VortexResult<()> {
256        let mut ctx = SESSION.create_execution_ctx();
257        let dtype = DType::Primitive(PType::I32, Nullability::Nullable);
258        let mut acc = Accumulator::try_new(Count, NumericalAggregateOpts::default(), dtype)?;
259
260        let batch1 = PrimitiveArray::from_option_iter([Some(1i32), None]).into_array();
261        acc.accumulate(&batch1, &mut ctx)?;
262        let result1 = acc.finish()?;
263        assert_eq!(result1.as_primitive().typed_value::<u64>(), Some(1));
264
265        let batch2 = PrimitiveArray::from_option_iter([Some(2i32), Some(3), None]).into_array();
266        acc.accumulate(&batch2, &mut ctx)?;
267        let result2 = acc.finish()?;
268        assert_eq!(result2.as_primitive().typed_value::<u64>(), Some(2));
269        Ok(())
270    }
271
272    #[test]
273    fn count_state_merge() -> VortexResult<()> {
274        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
275        let options = NumericalAggregateOpts::default();
276        let dtypes = AggregateDTypes::try_new(&Count, &options, dtype)?;
277
278        let state = Count.merge_partials(dtypes.args(&options), 5, 3)?;
279
280        let result = Count.to_scalar(dtypes.args(&options), &state)?;
281        assert_eq!(result.as_primitive().typed_value::<u64>(), Some(8));
282        Ok(())
283    }
284
285    fn count_with_options(
286        array: &ArrayRef,
287        ctx: &mut ExecutionCtx,
288        options: NumericalAggregateOpts,
289    ) -> VortexResult<u64> {
290        let mut acc = Accumulator::try_new(Count, options, array.dtype().clone())?;
291        acc.accumulate(array, ctx)?;
292        Ok(acc
293            .finish()?
294            .as_primitive()
295            .typed_value::<u64>()
296            .vortex_expect("count result should not be null"))
297    }
298
299    #[test]
300    fn count_float_excludes_nans_by_default() -> VortexResult<()> {
301        let array =
302            PrimitiveArray::from_option_iter([Some(1.0f64), Some(f64::NAN), None, Some(3.0)])
303                .into_array();
304        let mut ctx = SESSION.create_execution_ctx();
305        assert_eq!(count(&array, &mut ctx)?, 2);
306        Ok(())
307    }
308
309    #[test]
310    fn count_float_includes_nans_when_not_skipping() -> VortexResult<()> {
311        let array =
312            PrimitiveArray::from_option_iter([Some(1.0f64), Some(f64::NAN), None, Some(3.0)])
313                .into_array();
314        let mut ctx = SESSION.create_execution_ctx();
315        assert_eq!(
316            count_with_options(&array, &mut ctx, NumericalAggregateOpts::include_nans())?,
317            3
318        );
319        Ok(())
320    }
321
322    #[test]
323    fn count_float_with_exact_nan_count_stat() -> VortexResult<()> {
324        // The exact NaN count agrees with the batch; the four finite values remain counted.
325        let array = PrimitiveArray::new(
326            buffer![1.0f64, 2.0, 3.0, 4.0, f64::NAN, f64::NAN, f64::NAN],
327            Validity::NonNullable,
328        )
329        .into_array();
330        array
331            .statistics()
332            .set(Stat::NaNCount, Precision::Exact(ScalarValue::from(3u64)));
333        let mut ctx = SESSION.create_execution_ctx();
334        assert_eq!(count(&array, &mut ctx)?, 4);
335        Ok(())
336    }
337
338    #[test]
339    fn count_constant_nan() -> VortexResult<()> {
340        let array = ConstantArray::new(f64::NAN, 5).into_array();
341        let mut ctx = SESSION.create_execution_ctx();
342        assert_eq!(count(&array, &mut ctx)?, 0);
343        assert_eq!(
344            count_with_options(&array, &mut ctx, NumericalAggregateOpts::include_nans())?,
345            5
346        );
347        Ok(())
348    }
349
350    #[test]
351    fn count_constant_non_null() -> VortexResult<()> {
352        let array = ConstantArray::new(42i32, 10);
353        let mut ctx = SESSION.create_execution_ctx();
354        assert_eq!(count(&array.into_array(), &mut ctx)?, 10);
355        Ok(())
356    }
357
358    #[test]
359    fn count_constant_null() -> VortexResult<()> {
360        let array = ConstantArray::new(
361            Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)),
362            10,
363        );
364        let mut ctx = SESSION.create_execution_ctx();
365        assert_eq!(count(&array.into_array(), &mut ctx)?, 0);
366        Ok(())
367    }
368
369    #[test]
370    fn count_chunked() -> VortexResult<()> {
371        let chunk1 = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]);
372        let chunk2 = PrimitiveArray::from_option_iter([None, Some(5i32), None]);
373        let dtype = chunk1.dtype().clone();
374        let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
375        let mut ctx = SESSION.create_execution_ctx();
376        assert_eq!(count(&chunked.into_array(), &mut ctx)?, 3);
377        Ok(())
378    }
379}