Skip to main content

vortex_array/aggregate_fn/fns/null_count/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use vortex_error::VortexExpect;
5use vortex_error::VortexResult;
6use vortex_session::VortexSession;
7use vortex_session::registry::CachedId;
8
9use crate::ArrayRef;
10use crate::Columnar;
11use crate::ExecutionCtx;
12use crate::IntoArray;
13use crate::aggregate_fn::Accumulator;
14use crate::aggregate_fn::AggregateArgs;
15use crate::aggregate_fn::AggregateFnId;
16use crate::aggregate_fn::AggregateFnVTable;
17use crate::aggregate_fn::DynAccumulator;
18use crate::aggregate_fn::EmptyOptions;
19use crate::dtype::DType;
20use crate::dtype::Nullability::NonNullable;
21use crate::dtype::PType;
22use crate::expr::stats::Precision;
23use crate::expr::stats::Stat;
24use crate::expr::stats::StatsProvider;
25use crate::scalar::Scalar;
26
27/// Return the number of null values in an array.
28pub fn null_count(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
29    if let Precision::Exact(null_count_scalar) = array.statistics().get(Stat::NullCount) {
30        return usize::try_from(&null_count_scalar)
31            .map_err(|e| e.with_context("Failed to convert null count stat to usize"));
32    }
33
34    let mut acc = Accumulator::try_new(NullCount, EmptyOptions, array.dtype().clone())?;
35    acc.accumulate(array, ctx)?;
36    let result = acc.finish()?;
37
38    let count = result
39        .as_primitive()
40        .typed_value::<u64>()
41        .vortex_expect("null_count result should not be null");
42    // The accumulator caches the null count on `array` as a statistic.
43    Ok(usize::try_from(count).vortex_expect("Cannot be more nulls than usize::MAX"))
44}
45
46/// Count the number of null values in an array.
47///
48/// Applies to all types and returns a non-null `u64`.
49#[derive(Clone, Debug)]
50pub struct NullCount;
51
52impl AggregateFnVTable for NullCount {
53    type Options = EmptyOptions;
54    type Partial = u64;
55
56    fn id(&self) -> AggregateFnId {
57        static ID: CachedId = CachedId::new("vortex.null_count");
58        *ID
59    }
60
61    fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
62        Ok(Some(vec![]))
63    }
64
65    fn deserialize(
66        &self,
67        _metadata: &[u8],
68        _session: &VortexSession,
69    ) -> VortexResult<Self::Options> {
70        Ok(EmptyOptions)
71    }
72
73    fn return_dtype(&self, _options: &Self::Options, _input_dtype: &DType) -> Option<DType> {
74        Some(DType::Primitive(PType::U64, NonNullable))
75    }
76
77    fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
78        self.return_dtype(options, input_dtype)
79    }
80
81    fn empty_partial(
82        &self,
83        _args: AggregateArgs<'_, Self::Options>,
84    ) -> VortexResult<Self::Partial> {
85        Ok(0)
86    }
87
88    fn partial_from_scalar(
89        &self,
90        _args: AggregateArgs<'_, Self::Options>,
91        scalar: &Scalar,
92    ) -> VortexResult<Self::Partial> {
93        Ok(scalar
94            .as_primitive()
95            .typed_value::<u64>()
96            .vortex_expect("null_count partial should not be null"))
97    }
98
99    fn merge_partials(
100        &self,
101        _args: AggregateArgs<'_, Self::Options>,
102        first: Self::Partial,
103        second: Self::Partial,
104    ) -> VortexResult<Self::Partial> {
105        Ok(first + second)
106    }
107
108    fn to_scalar(
109        &self,
110        _args: AggregateArgs<'_, Self::Options>,
111        partial: &Self::Partial,
112    ) -> VortexResult<Scalar> {
113        Ok(Scalar::primitive(*partial, NonNullable))
114    }
115
116    #[inline]
117    fn is_saturated(
118        &self,
119        _args: AggregateArgs<'_, Self::Options>,
120        _partial: &Self::Partial,
121    ) -> bool {
122        false
123    }
124
125    fn try_accumulate(
126        &self,
127        _args: AggregateArgs<'_, Self::Options>,
128        state: &mut Self::Partial,
129        batch: &ArrayRef,
130        ctx: &mut ExecutionCtx,
131    ) -> VortexResult<bool> {
132        *state += batch.invalid_count(ctx)? as u64;
133        Ok(true)
134    }
135
136    fn accumulate(
137        &self,
138        _args: AggregateArgs<'_, Self::Options>,
139        partial: &mut Self::Partial,
140        batch: &Columnar,
141        ctx: &mut ExecutionCtx,
142    ) -> VortexResult<()> {
143        *partial += match batch {
144            Columnar::Constant(c) => {
145                if c.scalar().is_null() {
146                    c.len() as u64
147                } else {
148                    0
149                }
150            }
151            Columnar::Canonical(c) => c.clone().into_array().invalid_count(ctx)? as u64,
152        };
153        Ok(())
154    }
155
156    fn finalize(
157        &self,
158        _args: AggregateArgs<'_, Self::Options>,
159        partials: ArrayRef,
160    ) -> VortexResult<ArrayRef> {
161        Ok(partials)
162    }
163
164    fn finalize_scalar(
165        &self,
166        args: AggregateArgs<'_, Self::Options>,
167        partial: &Self::Partial,
168    ) -> VortexResult<Scalar> {
169        self.to_scalar(args, partial)
170    }
171}
172
173#[cfg(test)]
174mod tests {
175    use std::sync::LazyLock;
176
177    use vortex_error::VortexResult;
178    use vortex_session::VortexSession;
179
180    use crate::IntoArray;
181    use crate::VortexSessionExecute;
182    use crate::aggregate_fn::Accumulator;
183    use crate::aggregate_fn::DynAccumulator;
184    use crate::aggregate_fn::EmptyOptions;
185    use crate::aggregate_fn::fns::null_count::NullCount;
186    use crate::aggregate_fn::fns::null_count::null_count;
187    use crate::arrays::PrimitiveArray;
188    use crate::dtype::DType;
189    use crate::dtype::Nullability;
190    use crate::dtype::PType;
191    use crate::expr::stats::Precision;
192    use crate::expr::stats::Stat;
193    use crate::expr::stats::StatsProviderExt;
194
195    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
196
197    #[test]
198    fn null_count_with_nulls() -> VortexResult<()> {
199        let array =
200            PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None]).into_array();
201        let mut ctx = SESSION.create_execution_ctx();
202
203        assert_eq!(null_count(&array, &mut ctx)?, 2);
204        assert_eq!(
205            array.statistics().get_as::<u64>(Stat::NullCount),
206            Precision::exact(2u64)
207        );
208        Ok(())
209    }
210
211    #[test]
212    fn null_count_multi_batch() -> VortexResult<()> {
213        let mut ctx = SESSION.create_execution_ctx();
214        let dtype = DType::Primitive(PType::I32, Nullability::Nullable);
215        let mut acc = Accumulator::try_new(NullCount, EmptyOptions, dtype)?;
216
217        let batch1 = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array();
218        acc.accumulate(&batch1, &mut ctx)?;
219
220        let batch2 = PrimitiveArray::from_option_iter([None, Some(5i32), None]).into_array();
221        acc.accumulate(&batch2, &mut ctx)?;
222
223        let result = acc.finish()?;
224        assert_eq!(result.as_primitive().typed_value::<u64>(), Some(3));
225        Ok(())
226    }
227}