Skip to main content

vortex_array/aggregate_fn/fns/nan_count/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4mod primitive;
5
6use vortex_error::VortexExpect;
7use vortex_error::VortexResult;
8use vortex_error::vortex_bail;
9use vortex_session::VortexSession;
10use vortex_session::registry::CachedId;
11
12use self::primitive::accumulate_primitive;
13use crate::ArrayRef;
14use crate::Canonical;
15use crate::Columnar;
16use crate::ExecutionCtx;
17use crate::aggregate_fn::Accumulator;
18use crate::aggregate_fn::AggregateArgs;
19use crate::aggregate_fn::AggregateDTypes;
20use crate::aggregate_fn::AggregateFnId;
21use crate::aggregate_fn::AggregateFnVTable;
22use crate::aggregate_fn::DynAccumulator;
23use crate::aggregate_fn::EmptyOptions;
24use crate::dtype::DType;
25use crate::dtype::Nullability::NonNullable;
26use crate::dtype::PType;
27use crate::expr::stats::Precision;
28use crate::expr::stats::Stat;
29use crate::expr::stats::StatsProvider;
30use crate::scalar::Scalar;
31
32/// Return the number of NaN values in an array.
33///
34/// Null values are not NaN and are not counted.
35///
36/// See [`NanCount`] for details.
37pub fn nan_count(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
38    // Short-circuit using cached array statistics.
39    if let Precision::Exact(nan_count_scalar) = array.statistics().get(Stat::NaNCount) {
40        return usize::try_from(&nan_count_scalar)
41            .map_err(|e| e.with_context("Failed to convert NaN count stat to usize"));
42    }
43
44    // Short-circuit for non-float types.
45    if NanCount
46        .return_dtype(&EmptyOptions, array.dtype())
47        .is_none()
48    {
49        return Ok(0);
50    }
51
52    // Short-circuit for empty arrays or all-null arrays.
53    if array.is_empty() || array.valid_count(ctx)? == 0 {
54        return Ok(0);
55    }
56
57    // Compute using Accumulator<NanCount>.
58    let mut acc = Accumulator::try_new(NanCount, EmptyOptions, array.dtype().clone())?;
59    acc.accumulate(array, ctx)?;
60    let result = acc.finish()?;
61
62    let count = result
63        .as_primitive()
64        .typed_value::<u64>()
65        .vortex_expect("nan_count result should not be null");
66    // The accumulator caches the NaN count on `array` as a statistic.
67    Ok(usize::try_from(count).vortex_expect("Cannot be more nans than usize::MAX"))
68}
69
70/// Count the NaN values of an already-executed batch without touching any statistics.
71///
72/// Aggregates that delegate to [`NanCount`] use this on the columnar form of their input, so that
73/// results are only ever cached on the array the aggregate was requested for, never on a
74/// temporary canonical array.
75pub(crate) fn columnar_nan_count(batch: &Columnar, ctx: &mut ExecutionCtx) -> VortexResult<u64> {
76    // Non-float types have no NaN values.
77    if NanCount
78        .return_dtype(&EmptyOptions, batch.dtype())
79        .is_none()
80    {
81        return Ok(0);
82    }
83
84    let dtypes = AggregateDTypes::try_new(&NanCount, &EmptyOptions, batch.dtype().clone())?;
85    let args = dtypes.args(&EmptyOptions);
86    let mut count = NanCount.empty_partial(args)?;
87    NanCount.accumulate(args, &mut count, batch, ctx)?;
88    Ok(count)
89}
90
91/// Count the number of NaN values in an array.
92///
93/// Only applies to floating-point primitive types. Returns a `u64` count.
94/// Null values are not NaN, so if the array is all-invalid, the NaN count is zero.
95#[derive(Clone, Debug)]
96pub struct NanCount;
97
98impl AggregateFnVTable for NanCount {
99    type Options = EmptyOptions;
100    type Partial = u64;
101
102    fn id(&self) -> AggregateFnId {
103        static ID: CachedId = CachedId::new("vortex.nan_count");
104        *ID
105    }
106
107    fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
108        Ok(Some(vec![]))
109    }
110
111    fn deserialize(
112        &self,
113        _metadata: &[u8],
114        _session: &VortexSession,
115    ) -> VortexResult<Self::Options> {
116        Ok(EmptyOptions)
117    }
118
119    fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {
120        if let DType::Primitive(ptype, ..) = input_dtype
121            && ptype.is_float()
122        {
123            Some(DType::Primitive(PType::U64, NonNullable))
124        } else {
125            None
126        }
127    }
128
129    fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
130        self.return_dtype(options, input_dtype)
131    }
132
133    fn empty_partial(
134        &self,
135        _args: AggregateArgs<'_, Self::Options>,
136    ) -> VortexResult<Self::Partial> {
137        Ok(0)
138    }
139
140    fn partial_from_scalar(
141        &self,
142        _args: AggregateArgs<'_, Self::Options>,
143        scalar: &Scalar,
144    ) -> VortexResult<Self::Partial> {
145        Ok(scalar
146            .as_primitive()
147            .typed_value::<u64>()
148            .vortex_expect("nan_count partial should not be null"))
149    }
150
151    fn merge_partials(
152        &self,
153        _args: AggregateArgs<'_, Self::Options>,
154        first: Self::Partial,
155        second: Self::Partial,
156    ) -> VortexResult<Self::Partial> {
157        Ok(first + second)
158    }
159
160    fn to_scalar(
161        &self,
162        _args: AggregateArgs<'_, Self::Options>,
163        partial: &Self::Partial,
164    ) -> VortexResult<Scalar> {
165        Ok(Scalar::primitive(*partial, NonNullable))
166    }
167
168    #[inline]
169    fn is_saturated(
170        &self,
171        _args: AggregateArgs<'_, Self::Options>,
172        _partial: &Self::Partial,
173    ) -> bool {
174        false
175    }
176
177    fn accumulate(
178        &self,
179        _args: AggregateArgs<'_, Self::Options>,
180        partial: &mut Self::Partial,
181        batch: &Columnar,
182        ctx: &mut ExecutionCtx,
183    ) -> VortexResult<()> {
184        match batch {
185            Columnar::Constant(c) => {
186                if c.scalar().is_null() {
187                    // Null values are not NaN.
188                    return Ok(());
189                }
190                if c.scalar().as_primitive().is_nan() {
191                    *partial += c.len() as u64;
192                }
193                Ok(())
194            }
195            Columnar::Canonical(c) => match c {
196                Canonical::Primitive(p) => accumulate_primitive(partial, p, ctx),
197                _ => vortex_bail!(
198                    "Unsupported canonical type for nan_count: {}",
199                    batch.dtype()
200                ),
201            },
202        }
203    }
204
205    fn finalize(
206        &self,
207        _args: AggregateArgs<'_, Self::Options>,
208        partials: ArrayRef,
209    ) -> VortexResult<ArrayRef> {
210        Ok(partials)
211    }
212
213    fn finalize_scalar(
214        &self,
215        args: AggregateArgs<'_, Self::Options>,
216        partial: &Self::Partial,
217    ) -> VortexResult<Scalar> {
218        self.to_scalar(args, partial)
219    }
220}
221
222#[cfg(test)]
223mod tests {
224    use std::sync::LazyLock;
225
226    use vortex_buffer::buffer;
227    use vortex_error::VortexResult;
228    use vortex_session::VortexSession;
229
230    use crate::IntoArray;
231    use crate::VortexSessionExecute;
232    use crate::aggregate_fn::Accumulator;
233    use crate::aggregate_fn::AggregateDTypes;
234    use crate::aggregate_fn::AggregateFnVTable;
235    use crate::aggregate_fn::DynAccumulator;
236    use crate::aggregate_fn::EmptyOptions;
237    use crate::aggregate_fn::fns::nan_count::NanCount;
238    use crate::aggregate_fn::fns::nan_count::nan_count;
239    use crate::arrays::ChunkedArray;
240    use crate::arrays::ConstantArray;
241    use crate::arrays::PrimitiveArray;
242    use crate::dtype::DType;
243    use crate::dtype::Nullability;
244    use crate::dtype::PType;
245    use crate::validity::Validity;
246
247    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
248
249    #[test]
250    fn nan_count_multi_batch() -> VortexResult<()> {
251        let mut ctx = SESSION.create_execution_ctx();
252        let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
253        let mut acc = Accumulator::try_new(NanCount, EmptyOptions, dtype)?;
254
255        let batch1 =
256            PrimitiveArray::new(buffer![f64::NAN, 1.0f64, f64::NAN], Validity::NonNullable)
257                .into_array();
258        acc.accumulate(&batch1, &mut ctx)?;
259
260        let batch2 =
261            PrimitiveArray::new(buffer![2.0f64, f64::NAN], Validity::NonNullable).into_array();
262        acc.accumulate(&batch2, &mut ctx)?;
263
264        let result = acc.finish()?;
265        assert_eq!(result.as_primitive().typed_value::<u64>(), Some(3));
266        Ok(())
267    }
268
269    #[test]
270    fn nan_count_finish_resets_state() -> VortexResult<()> {
271        let mut ctx = SESSION.create_execution_ctx();
272        let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
273        let mut acc = Accumulator::try_new(NanCount, EmptyOptions, dtype)?;
274
275        let batch1 =
276            PrimitiveArray::new(buffer![f64::NAN, 1.0f64], Validity::NonNullable).into_array();
277        acc.accumulate(&batch1, &mut ctx)?;
278        let result1 = acc.finish()?;
279        assert_eq!(result1.as_primitive().typed_value::<u64>(), Some(1));
280
281        let batch2 = PrimitiveArray::new(buffer![f64::NAN, f64::NAN, 2.0], Validity::NonNullable)
282            .into_array();
283        acc.accumulate(&batch2, &mut ctx)?;
284        let result2 = acc.finish()?;
285        assert_eq!(result2.as_primitive().typed_value::<u64>(), Some(2));
286        Ok(())
287    }
288
289    #[test]
290    fn nan_count_state_merge() -> VortexResult<()> {
291        let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
292        let dtypes = AggregateDTypes::try_new(&NanCount, &EmptyOptions, dtype)?;
293
294        let state = NanCount.merge_partials(dtypes.args(&EmptyOptions), 5, 3)?;
295
296        let result = NanCount.to_scalar(dtypes.args(&EmptyOptions), &state)?;
297        assert_eq!(result.as_primitive().typed_value::<u64>(), Some(8));
298        Ok(())
299    }
300
301    #[test]
302    fn nan_count_constant_nan() -> VortexResult<()> {
303        let array = ConstantArray::new(f64::NAN, 10);
304        let mut ctx = SESSION.create_execution_ctx();
305        assert_eq!(nan_count(&array.into_array(), &mut ctx)?, 10);
306        Ok(())
307    }
308
309    #[test]
310    fn nan_count_constant_non_nan() -> VortexResult<()> {
311        let array = ConstantArray::new(1.0f64, 10);
312        let mut ctx = SESSION.create_execution_ctx();
313        assert_eq!(nan_count(&array.into_array(), &mut ctx)?, 0);
314        Ok(())
315    }
316
317    #[test]
318    fn nan_count_empty() -> VortexResult<()> {
319        let dtype = DType::Primitive(PType::F64, Nullability::NonNullable);
320        let mut acc = Accumulator::try_new(NanCount, EmptyOptions, dtype)?;
321        let result = acc.finish()?;
322        assert_eq!(result.as_primitive().typed_value::<u64>(), Some(0));
323        Ok(())
324    }
325
326    #[test]
327    fn nan_count_chunked() -> VortexResult<()> {
328        let chunk1 = PrimitiveArray::from_option_iter([Some(f64::NAN), None, Some(1.0)]);
329        let chunk2 = PrimitiveArray::from_option_iter([Some(f64::NAN), Some(f64::NAN), None]);
330        let dtype = chunk1.dtype().clone();
331        let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
332        let mut ctx = SESSION.create_execution_ctx();
333        assert_eq!(nan_count(&chunked.into_array(), &mut ctx)?, 3);
334        Ok(())
335    }
336
337    #[test]
338    fn nan_count_all_null() -> VortexResult<()> {
339        let p = PrimitiveArray::new(buffer![f64::NAN, f64::NAN, f64::NAN], Validity::AllInvalid);
340        let mut ctx = SESSION.create_execution_ctx();
341        assert_eq!(nan_count(&p.into_array(), &mut ctx)?, 0);
342        Ok(())
343    }
344}