vortex_array/aggregate_fn/fns/count/
mod.rs1mod grouped;
5pub(crate) use grouped::CountGroupedKernel;
6use vortex_error::VortexExpect;
7use vortex_error::VortexResult;
8use vortex_session::VortexSession;
9use vortex_session::registry::CachedId;
10
11use crate::ArrayRef;
12use crate::Columnar;
13use crate::ExecutionCtx;
14use crate::aggregate_fn::AggregateArgs;
15use crate::aggregate_fn::AggregateFnId;
16use crate::aggregate_fn::AggregateFnVTable;
17use crate::aggregate_fn::NumericalAggregateOpts;
18use crate::aggregate_fn::fns::nan_count::nan_count;
19use crate::dtype::DType;
20use crate::dtype::Nullability;
21use crate::dtype::PType;
22use crate::scalar::Scalar;
23
24#[derive(Clone, Debug)]
33pub struct Count;
34
35impl AggregateFnVTable for Count {
36 type Options = NumericalAggregateOpts;
37 type Partial = u64;
38
39 fn id(&self) -> AggregateFnId {
40 static ID: CachedId = CachedId::new("vortex.count");
41 *ID
42 }
43
44 fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
45 Ok(Some(options.serialize()))
46 }
47
48 fn deserialize(
49 &self,
50 metadata: &[u8],
51 _session: &VortexSession,
52 ) -> VortexResult<Self::Options> {
53 NumericalAggregateOpts::deserialize(metadata)
54 }
55
56 fn return_dtype(&self, _options: &Self::Options, _input_dtype: &DType) -> Option<DType> {
57 Some(DType::Primitive(PType::U64, Nullability::NonNullable))
58 }
59
60 fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
61 self.return_dtype(options, input_dtype)
62 }
63
64 fn empty_partial(
65 &self,
66 _args: AggregateArgs<'_, Self::Options>,
67 ) -> VortexResult<Self::Partial> {
68 Ok(0)
69 }
70
71 fn partial_from_scalar(
72 &self,
73 _args: AggregateArgs<'_, Self::Options>,
74 scalar: &Scalar,
75 ) -> VortexResult<Self::Partial> {
76 Ok(scalar
77 .as_primitive()
78 .typed_value::<u64>()
79 .vortex_expect("count partial should not be null"))
80 }
81
82 fn merge_partials(
83 &self,
84 _args: AggregateArgs<'_, Self::Options>,
85 first: Self::Partial,
86 second: Self::Partial,
87 ) -> VortexResult<Self::Partial> {
88 Ok(first + second)
89 }
90
91 fn to_scalar(
92 &self,
93 _args: AggregateArgs<'_, Self::Options>,
94 partial: &Self::Partial,
95 ) -> VortexResult<Scalar> {
96 Ok(Scalar::primitive(*partial, Nullability::NonNullable))
97 }
98
99 #[inline]
100 fn is_saturated(
101 &self,
102 _args: AggregateArgs<'_, Self::Options>,
103 _partial: &Self::Partial,
104 ) -> bool {
105 false
106 }
107
108 fn try_accumulate(
109 &self,
110 args: AggregateArgs<'_, Self::Options>,
111 state: &mut Self::Partial,
112 batch: &ArrayRef,
113 ctx: &mut ExecutionCtx,
114 ) -> VortexResult<bool> {
115 let mut count = batch.valid_count(ctx)? as u64;
116 if args.options.skip_nans && args.dtype.is_float() {
118 count = count.saturating_sub(nan_count(batch, ctx)? as u64);
120 }
121 *state += count;
122 Ok(true)
123 }
124
125 fn accumulate(
126 &self,
127 _args: AggregateArgs<'_, Self::Options>,
128 _partial: &mut Self::Partial,
129 _batch: &Columnar,
130 _ctx: &mut ExecutionCtx,
131 ) -> VortexResult<()> {
132 unreachable!("Count::try_accumulate handles all arrays")
133 }
134
135 fn finalize(
136 &self,
137 _args: AggregateArgs<'_, Self::Options>,
138 partials: ArrayRef,
139 ) -> VortexResult<ArrayRef> {
140 Ok(partials)
141 }
142
143 fn finalize_scalar(
144 &self,
145 args: AggregateArgs<'_, Self::Options>,
146 partial: &Self::Partial,
147 ) -> VortexResult<Scalar> {
148 self.to_scalar(args, partial)
149 }
150}
151
152#[cfg(test)]
153mod tests {
154 use std::sync::LazyLock;
155
156 use vortex_buffer::buffer;
157 use vortex_error::VortexExpect;
158 use vortex_error::VortexResult;
159 use vortex_session::VortexSession;
160
161 use crate::ArrayRef;
162 use crate::ExecutionCtx;
163 use crate::IntoArray;
164 use crate::VortexSessionExecute;
165 use crate::aggregate_fn::Accumulator;
166 use crate::aggregate_fn::AggregateDTypes;
167 use crate::aggregate_fn::AggregateFnVTable;
168 use crate::aggregate_fn::DynAccumulator;
169 use crate::aggregate_fn::NumericalAggregateOpts;
170 use crate::aggregate_fn::fns::count::Count;
171 use crate::arrays::ChunkedArray;
172 use crate::arrays::ConstantArray;
173 use crate::arrays::PrimitiveArray;
174 use crate::dtype::DType;
175 use crate::dtype::Nullability;
176 use crate::dtype::PType;
177 use crate::expr::stats::Precision;
178 use crate::expr::stats::Stat;
179 use crate::scalar::Scalar;
180 use crate::scalar::ScalarValue;
181 use crate::validity::Validity;
182
183 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
184
185 pub fn count(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
186 let mut acc = Accumulator::try_new(
187 Count,
188 NumericalAggregateOpts::default(),
189 array.dtype().clone(),
190 )?;
191 acc.accumulate(array, ctx)?;
192 let result = acc.finish()?;
193
194 Ok(usize::try_from(
195 result
196 .as_primitive()
197 .typed_value::<u64>()
198 .vortex_expect("count result should not be null"),
199 )?)
200 }
201
202 #[test]
203 fn count_all_valid() -> VortexResult<()> {
204 let array =
205 PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5], Validity::NonNullable).into_array();
206 let mut ctx = SESSION.create_execution_ctx();
207 assert_eq!(count(&array, &mut ctx)?, 5);
208 Ok(())
209 }
210
211 #[test]
212 fn count_with_nulls() -> VortexResult<()> {
213 let array = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5)])
214 .into_array();
215 let mut ctx = SESSION.create_execution_ctx();
216 assert_eq!(count(&array, &mut ctx)?, 3);
217 Ok(())
218 }
219
220 #[test]
221 fn count_all_null() -> VortexResult<()> {
222 let array = PrimitiveArray::from_option_iter::<i32, _>([None, None, None]).into_array();
223 let mut ctx = SESSION.create_execution_ctx();
224 assert_eq!(count(&array, &mut ctx)?, 0);
225 Ok(())
226 }
227
228 #[test]
229 fn count_empty() -> VortexResult<()> {
230 let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
231 let mut acc = Accumulator::try_new(Count, NumericalAggregateOpts::default(), dtype)?;
232 let result = acc.finish()?;
233 assert_eq!(result.as_primitive().typed_value::<u64>(), Some(0));
234 Ok(())
235 }
236
237 #[test]
238 fn count_multi_batch() -> VortexResult<()> {
239 let mut ctx = SESSION.create_execution_ctx();
240 let dtype = DType::Primitive(PType::I32, Nullability::Nullable);
241 let mut acc = Accumulator::try_new(Count, NumericalAggregateOpts::default(), dtype)?;
242
243 let batch1 = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array();
244 acc.accumulate(&batch1, &mut ctx)?;
245
246 let batch2 = PrimitiveArray::from_option_iter([None, Some(5i32)]).into_array();
247 acc.accumulate(&batch2, &mut ctx)?;
248
249 let result = acc.finish()?;
250 assert_eq!(result.as_primitive().typed_value::<u64>(), Some(3));
251 Ok(())
252 }
253
254 #[test]
255 fn count_finish_resets_state() -> VortexResult<()> {
256 let mut ctx = SESSION.create_execution_ctx();
257 let dtype = DType::Primitive(PType::I32, Nullability::Nullable);
258 let mut acc = Accumulator::try_new(Count, NumericalAggregateOpts::default(), dtype)?;
259
260 let batch1 = PrimitiveArray::from_option_iter([Some(1i32), None]).into_array();
261 acc.accumulate(&batch1, &mut ctx)?;
262 let result1 = acc.finish()?;
263 assert_eq!(result1.as_primitive().typed_value::<u64>(), Some(1));
264
265 let batch2 = PrimitiveArray::from_option_iter([Some(2i32), Some(3), None]).into_array();
266 acc.accumulate(&batch2, &mut ctx)?;
267 let result2 = acc.finish()?;
268 assert_eq!(result2.as_primitive().typed_value::<u64>(), Some(2));
269 Ok(())
270 }
271
272 #[test]
273 fn count_state_merge() -> VortexResult<()> {
274 let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
275 let options = NumericalAggregateOpts::default();
276 let dtypes = AggregateDTypes::try_new(&Count, &options, dtype)?;
277
278 let state = Count.merge_partials(dtypes.args(&options), 5, 3)?;
279
280 let result = Count.to_scalar(dtypes.args(&options), &state)?;
281 assert_eq!(result.as_primitive().typed_value::<u64>(), Some(8));
282 Ok(())
283 }
284
285 fn count_with_options(
286 array: &ArrayRef,
287 ctx: &mut ExecutionCtx,
288 options: NumericalAggregateOpts,
289 ) -> VortexResult<u64> {
290 let mut acc = Accumulator::try_new(Count, options, array.dtype().clone())?;
291 acc.accumulate(array, ctx)?;
292 Ok(acc
293 .finish()?
294 .as_primitive()
295 .typed_value::<u64>()
296 .vortex_expect("count result should not be null"))
297 }
298
299 #[test]
300 fn count_float_excludes_nans_by_default() -> VortexResult<()> {
301 let array =
302 PrimitiveArray::from_option_iter([Some(1.0f64), Some(f64::NAN), None, Some(3.0)])
303 .into_array();
304 let mut ctx = SESSION.create_execution_ctx();
305 assert_eq!(count(&array, &mut ctx)?, 2);
306 Ok(())
307 }
308
309 #[test]
310 fn count_float_includes_nans_when_not_skipping() -> VortexResult<()> {
311 let array =
312 PrimitiveArray::from_option_iter([Some(1.0f64), Some(f64::NAN), None, Some(3.0)])
313 .into_array();
314 let mut ctx = SESSION.create_execution_ctx();
315 assert_eq!(
316 count_with_options(&array, &mut ctx, NumericalAggregateOpts::include_nans())?,
317 3
318 );
319 Ok(())
320 }
321
322 #[test]
323 fn count_float_with_exact_nan_count_stat() -> VortexResult<()> {
324 let array = PrimitiveArray::new(
326 buffer![1.0f64, 2.0, 3.0, 4.0, f64::NAN, f64::NAN, f64::NAN],
327 Validity::NonNullable,
328 )
329 .into_array();
330 array
331 .statistics()
332 .set(Stat::NaNCount, Precision::Exact(ScalarValue::from(3u64)));
333 let mut ctx = SESSION.create_execution_ctx();
334 assert_eq!(count(&array, &mut ctx)?, 4);
335 Ok(())
336 }
337
338 #[test]
339 fn count_constant_nan() -> VortexResult<()> {
340 let array = ConstantArray::new(f64::NAN, 5).into_array();
341 let mut ctx = SESSION.create_execution_ctx();
342 assert_eq!(count(&array, &mut ctx)?, 0);
343 assert_eq!(
344 count_with_options(&array, &mut ctx, NumericalAggregateOpts::include_nans())?,
345 5
346 );
347 Ok(())
348 }
349
350 #[test]
351 fn count_constant_non_null() -> VortexResult<()> {
352 let array = ConstantArray::new(42i32, 10);
353 let mut ctx = SESSION.create_execution_ctx();
354 assert_eq!(count(&array.into_array(), &mut ctx)?, 10);
355 Ok(())
356 }
357
358 #[test]
359 fn count_constant_null() -> VortexResult<()> {
360 let array = ConstantArray::new(
361 Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)),
362 10,
363 );
364 let mut ctx = SESSION.create_execution_ctx();
365 assert_eq!(count(&array.into_array(), &mut ctx)?, 0);
366 Ok(())
367 }
368
369 #[test]
370 fn count_chunked() -> VortexResult<()> {
371 let chunk1 = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]);
372 let chunk2 = PrimitiveArray::from_option_iter([None, Some(5i32), None]);
373 let dtype = chunk1.dtype().clone();
374 let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
375 let mut ctx = SESSION.create_execution_ctx();
376 assert_eq!(count(&chunked.into_array(), &mut ctx)?, 3);
377 Ok(())
378 }
379}