vortex_array/aggregate_fn/fns/nan_count/
mod.rs1mod primitive;
5
6use vortex_error::VortexExpect;
7use vortex_error::VortexResult;
8use vortex_error::vortex_bail;
9use vortex_session::VortexSession;
10use vortex_session::registry::CachedId;
11
12use self::primitive::accumulate_primitive;
13use crate::ArrayRef;
14use crate::Canonical;
15use crate::Columnar;
16use crate::ExecutionCtx;
17use crate::aggregate_fn::Accumulator;
18use crate::aggregate_fn::AggregateArgs;
19use crate::aggregate_fn::AggregateDTypes;
20use crate::aggregate_fn::AggregateFnId;
21use crate::aggregate_fn::AggregateFnVTable;
22use crate::aggregate_fn::DynAccumulator;
23use crate::aggregate_fn::EmptyOptions;
24use crate::dtype::DType;
25use crate::dtype::Nullability::NonNullable;
26use crate::dtype::PType;
27use crate::expr::stats::Precision;
28use crate::expr::stats::Stat;
29use crate::expr::stats::StatsProvider;
30use crate::scalar::Scalar;
31
32pub fn nan_count(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
38 if let Precision::Exact(nan_count_scalar) = array.statistics().get(Stat::NaNCount) {
40 return usize::try_from(&nan_count_scalar)
41 .map_err(|e| e.with_context("Failed to convert NaN count stat to usize"));
42 }
43
44 if NanCount
46 .return_dtype(&EmptyOptions, array.dtype())
47 .is_none()
48 {
49 return Ok(0);
50 }
51
52 if array.is_empty() || array.valid_count(ctx)? == 0 {
54 return Ok(0);
55 }
56
57 let mut acc = Accumulator::try_new(NanCount, EmptyOptions, array.dtype().clone())?;
59 acc.accumulate(array, ctx)?;
60 let result = acc.finish()?;
61
62 let count = result
63 .as_primitive()
64 .typed_value::<u64>()
65 .vortex_expect("nan_count result should not be null");
66 Ok(usize::try_from(count).vortex_expect("Cannot be more nans than usize::MAX"))
68}
69
70pub(crate) fn columnar_nan_count(batch: &Columnar, ctx: &mut ExecutionCtx) -> VortexResult<u64> {
76 if NanCount
78 .return_dtype(&EmptyOptions, batch.dtype())
79 .is_none()
80 {
81 return Ok(0);
82 }
83
84 let dtypes = AggregateDTypes::try_new(&NanCount, &EmptyOptions, batch.dtype().clone())?;
85 let args = dtypes.args(&EmptyOptions);
86 let mut count = NanCount.empty_partial(args)?;
87 NanCount.accumulate(args, &mut count, batch, ctx)?;
88 Ok(count)
89}
90
91#[derive(Clone, Debug)]
96pub struct NanCount;
97
98impl AggregateFnVTable for NanCount {
99 type Options = EmptyOptions;
100 type Partial = u64;
101
102 fn id(&self) -> AggregateFnId {
103 static ID: CachedId = CachedId::new("vortex.nan_count");
104 *ID
105 }
106
107 fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
108 Ok(Some(vec![]))
109 }
110
111 fn deserialize(
112 &self,
113 _metadata: &[u8],
114 _session: &VortexSession,
115 ) -> VortexResult<Self::Options> {
116 Ok(EmptyOptions)
117 }
118
119 fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {
120 if let DType::Primitive(ptype, ..) = input_dtype
121 && ptype.is_float()
122 {
123 Some(DType::Primitive(PType::U64, NonNullable))
124 } else {
125 None
126 }
127 }
128
129 fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
130 self.return_dtype(options, input_dtype)
131 }
132
133 fn empty_partial(
134 &self,
135 _args: AggregateArgs<'_, Self::Options>,
136 ) -> VortexResult<Self::Partial> {
137 Ok(0)
138 }
139
140 fn partial_from_scalar(
141 &self,
142 _args: AggregateArgs<'_, Self::Options>,
143 scalar: &Scalar,
144 ) -> VortexResult<Self::Partial> {
145 Ok(scalar
146 .as_primitive()
147 .typed_value::<u64>()
148 .vortex_expect("nan_count partial should not be null"))
149 }
150
151 fn merge_partials(
152 &self,
153 _args: AggregateArgs<'_, Self::Options>,
154 first: Self::Partial,
155 second: Self::Partial,
156 ) -> VortexResult<Self::Partial> {
157 Ok(first + second)
158 }
159
160 fn to_scalar(
161 &self,
162 _args: AggregateArgs<'_, Self::Options>,
163 partial: &Self::Partial,
164 ) -> VortexResult<Scalar> {
165 Ok(Scalar::primitive(*partial, NonNullable))
166 }
167
168 #[inline]
169 fn is_saturated(
170 &self,
171 _args: AggregateArgs<'_, Self::Options>,
172 _partial: &Self::Partial,
173 ) -> bool {
174 false
175 }
176
177 fn accumulate(
178 &self,
179 _args: AggregateArgs<'_, Self::Options>,
180 partial: &mut Self::Partial,
181 batch: &Columnar,
182 ctx: &mut ExecutionCtx,
183 ) -> VortexResult<()> {
184 match batch {
185 Columnar::Constant(c) => {
186 if c.scalar().is_null() {
187 return Ok(());
189 }
190 if c.scalar().as_primitive().is_nan() {
191 *partial += c.len() as u64;
192 }
193 Ok(())
194 }
195 Columnar::Canonical(c) => match c {
196 Canonical::Primitive(p) => accumulate_primitive(partial, p, ctx),
197 _ => vortex_bail!(
198 "Unsupported canonical type for nan_count: {}",
199 batch.dtype()
200 ),
201 },
202 }
203 }
204
205 fn finalize(
206 &self,
207 _args: AggregateArgs<'_, Self::Options>,
208 partials: ArrayRef,
209 ) -> VortexResult<ArrayRef> {
210 Ok(partials)
211 }
212
213 fn finalize_scalar(
214 &self,
215 args: AggregateArgs<'_, Self::Options>,
216 partial: &Self::Partial,
217 ) -> VortexResult<Scalar> {
218 self.to_scalar(args, partial)
219 }
220}
221
222#[cfg(test)]
223mod tests {
224 use std::sync::LazyLock;
225
226 use vortex_buffer::buffer;
227 use vortex_error::VortexResult;
228 use vortex_session::VortexSession;
229
230 use crate::IntoArray;
231 use crate::VortexSessionExecute;
232 use crate::aggregate_fn::Accumulator;
233 use crate::aggregate_fn::AggregateDTypes;
234 use crate::aggregate_fn::AggregateFnVTable;
235 use crate::aggregate_fn::DynAccumulator;
236 use crate::aggregate_fn::EmptyOptions;
237 use crate::aggregate_fn::fns::nan_count::NanCount;
238 use crate::aggregate_fn::fns::nan_count::nan_count;
239 use crate::arrays::ChunkedArray;
240 use crate::arrays::ConstantArray;
241 use crate::arrays::PrimitiveArray;
242 use crate::dtype::DType;
243 use crate::dtype::Nullability;
244 use crate::dtype::PType;
245 use crate::validity::Validity;
246
247 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
248
249 #[test]
250 fn nan_count_multi_batch() -> VortexResult<()> {
251 let mut ctx = SESSION.create_execution_ctx();
252 let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
253 let mut acc = Accumulator::try_new(NanCount, EmptyOptions, dtype)?;
254
255 let batch1 =
256 PrimitiveArray::new(buffer![f64::NAN, 1.0f64, f64::NAN], Validity::NonNullable)
257 .into_array();
258 acc.accumulate(&batch1, &mut ctx)?;
259
260 let batch2 =
261 PrimitiveArray::new(buffer![2.0f64, f64::NAN], Validity::NonNullable).into_array();
262 acc.accumulate(&batch2, &mut ctx)?;
263
264 let result = acc.finish()?;
265 assert_eq!(result.as_primitive().typed_value::<u64>(), Some(3));
266 Ok(())
267 }
268
269 #[test]
270 fn nan_count_finish_resets_state() -> VortexResult<()> {
271 let mut ctx = SESSION.create_execution_ctx();
272 let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
273 let mut acc = Accumulator::try_new(NanCount, EmptyOptions, dtype)?;
274
275 let batch1 =
276 PrimitiveArray::new(buffer![f64::NAN, 1.0f64], Validity::NonNullable).into_array();
277 acc.accumulate(&batch1, &mut ctx)?;
278 let result1 = acc.finish()?;
279 assert_eq!(result1.as_primitive().typed_value::<u64>(), Some(1));
280
281 let batch2 = PrimitiveArray::new(buffer![f64::NAN, f64::NAN, 2.0], Validity::NonNullable)
282 .into_array();
283 acc.accumulate(&batch2, &mut ctx)?;
284 let result2 = acc.finish()?;
285 assert_eq!(result2.as_primitive().typed_value::<u64>(), Some(2));
286 Ok(())
287 }
288
289 #[test]
290 fn nan_count_state_merge() -> VortexResult<()> {
291 let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
292 let dtypes = AggregateDTypes::try_new(&NanCount, &EmptyOptions, dtype)?;
293
294 let state = NanCount.merge_partials(dtypes.args(&EmptyOptions), 5, 3)?;
295
296 let result = NanCount.to_scalar(dtypes.args(&EmptyOptions), &state)?;
297 assert_eq!(result.as_primitive().typed_value::<u64>(), Some(8));
298 Ok(())
299 }
300
301 #[test]
302 fn nan_count_constant_nan() -> VortexResult<()> {
303 let array = ConstantArray::new(f64::NAN, 10);
304 let mut ctx = SESSION.create_execution_ctx();
305 assert_eq!(nan_count(&array.into_array(), &mut ctx)?, 10);
306 Ok(())
307 }
308
309 #[test]
310 fn nan_count_constant_non_nan() -> VortexResult<()> {
311 let array = ConstantArray::new(1.0f64, 10);
312 let mut ctx = SESSION.create_execution_ctx();
313 assert_eq!(nan_count(&array.into_array(), &mut ctx)?, 0);
314 Ok(())
315 }
316
317 #[test]
318 fn nan_count_empty() -> VortexResult<()> {
319 let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
320 let mut acc = Accumulator::try_new(NanCount, EmptyOptions, dtype)?;
321 let result = acc.finish()?;
322 assert_eq!(result.as_primitive().typed_value::<u64>(), Some(0));
323 Ok(())
324 }
325
326 #[test]
327 fn nan_count_chunked() -> VortexResult<()> {
328 let chunk1 = PrimitiveArray::from_option_iter([Some(f64::NAN), None, Some(1.0)]);
329 let chunk2 = PrimitiveArray::from_option_iter([Some(f64::NAN), Some(f64::NAN), None]);
330 let dtype = chunk1.dtype().clone();
331 let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
332 let mut ctx = SESSION.create_execution_ctx();
333 assert_eq!(nan_count(&chunked.into_array(), &mut ctx)?, 3);
334 Ok(())
335 }
336
337 #[test]
338 fn nan_count_all_null() -> VortexResult<()> {
339 let p = PrimitiveArray::new(buffer![f64::NAN, f64::NAN, f64::NAN], Validity::AllInvalid);
340 let mut ctx = SESSION.create_execution_ctx();
341 assert_eq!(nan_count(&p.into_array(), &mut ctx)?, 0);
342 Ok(())
343 }
344}