vortex_array/aggregate_fn/fns/null_count/
mod.rs1use vortex_error::VortexExpect;
5use vortex_error::VortexResult;
6use vortex_session::VortexSession;
7use vortex_session::registry::CachedId;
8
9use crate::ArrayRef;
10use crate::Columnar;
11use crate::ExecutionCtx;
12use crate::IntoArray;
13use crate::aggregate_fn::Accumulator;
14use crate::aggregate_fn::AggregateArgs;
15use crate::aggregate_fn::AggregateFnId;
16use crate::aggregate_fn::AggregateFnVTable;
17use crate::aggregate_fn::DynAccumulator;
18use crate::aggregate_fn::EmptyOptions;
19use crate::dtype::DType;
20use crate::dtype::Nullability::NonNullable;
21use crate::dtype::PType;
22use crate::expr::stats::Precision;
23use crate::expr::stats::Stat;
24use crate::expr::stats::StatsProvider;
25use crate::scalar::Scalar;
26
27pub fn null_count(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<usize> {
29 if let Precision::Exact(null_count_scalar) = array.statistics().get(Stat::NullCount) {
30 return usize::try_from(&null_count_scalar)
31 .map_err(|e| e.with_context("Failed to convert null count stat to usize"));
32 }
33
34 let mut acc = Accumulator::try_new(NullCount, EmptyOptions, array.dtype().clone())?;
35 acc.accumulate(array, ctx)?;
36 let result = acc.finish()?;
37
38 let count = result
39 .as_primitive()
40 .typed_value::<u64>()
41 .vortex_expect("null_count result should not be null");
42 Ok(usize::try_from(count).vortex_expect("Cannot be more nulls than usize::MAX"))
44}
45
46#[derive(Clone, Debug)]
50pub struct NullCount;
51
52impl AggregateFnVTable for NullCount {
53 type Options = EmptyOptions;
54 type Partial = u64;
55
56 fn id(&self) -> AggregateFnId {
57 static ID: CachedId = CachedId::new("vortex.null_count");
58 *ID
59 }
60
61 fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
62 Ok(Some(vec![]))
63 }
64
65 fn deserialize(
66 &self,
67 _metadata: &[u8],
68 _session: &VortexSession,
69 ) -> VortexResult<Self::Options> {
70 Ok(EmptyOptions)
71 }
72
73 fn return_dtype(&self, _options: &Self::Options, _input_dtype: &DType) -> Option<DType> {
74 Some(DType::Primitive(PType::U64, NonNullable))
75 }
76
77 fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
78 self.return_dtype(options, input_dtype)
79 }
80
81 fn empty_partial(
82 &self,
83 _args: AggregateArgs<'_, Self::Options>,
84 ) -> VortexResult<Self::Partial> {
85 Ok(0)
86 }
87
88 fn partial_from_scalar(
89 &self,
90 _args: AggregateArgs<'_, Self::Options>,
91 scalar: &Scalar,
92 ) -> VortexResult<Self::Partial> {
93 Ok(scalar
94 .as_primitive()
95 .typed_value::<u64>()
96 .vortex_expect("null_count partial should not be null"))
97 }
98
99 fn merge_partials(
100 &self,
101 _args: AggregateArgs<'_, Self::Options>,
102 first: Self::Partial,
103 second: Self::Partial,
104 ) -> VortexResult<Self::Partial> {
105 Ok(first + second)
106 }
107
108 fn to_scalar(
109 &self,
110 _args: AggregateArgs<'_, Self::Options>,
111 partial: &Self::Partial,
112 ) -> VortexResult<Scalar> {
113 Ok(Scalar::primitive(*partial, NonNullable))
114 }
115
116 #[inline]
117 fn is_saturated(
118 &self,
119 _args: AggregateArgs<'_, Self::Options>,
120 _partial: &Self::Partial,
121 ) -> bool {
122 false
123 }
124
125 fn try_accumulate(
126 &self,
127 _args: AggregateArgs<'_, Self::Options>,
128 state: &mut Self::Partial,
129 batch: &ArrayRef,
130 ctx: &mut ExecutionCtx,
131 ) -> VortexResult<bool> {
132 *state += batch.invalid_count(ctx)? as u64;
133 Ok(true)
134 }
135
136 fn accumulate(
137 &self,
138 _args: AggregateArgs<'_, Self::Options>,
139 partial: &mut Self::Partial,
140 batch: &Columnar,
141 ctx: &mut ExecutionCtx,
142 ) -> VortexResult<()> {
143 *partial += match batch {
144 Columnar::Constant(c) => {
145 if c.scalar().is_null() {
146 c.len() as u64
147 } else {
148 0
149 }
150 }
151 Columnar::Canonical(c) => c.clone().into_array().invalid_count(ctx)? as u64,
152 };
153 Ok(())
154 }
155
156 fn finalize(
157 &self,
158 _args: AggregateArgs<'_, Self::Options>,
159 partials: ArrayRef,
160 ) -> VortexResult<ArrayRef> {
161 Ok(partials)
162 }
163
164 fn finalize_scalar(
165 &self,
166 args: AggregateArgs<'_, Self::Options>,
167 partial: &Self::Partial,
168 ) -> VortexResult<Scalar> {
169 self.to_scalar(args, partial)
170 }
171}
172
173#[cfg(test)]
174mod tests {
175 use std::sync::LazyLock;
176
177 use vortex_error::VortexResult;
178 use vortex_session::VortexSession;
179
180 use crate::IntoArray;
181 use crate::VortexSessionExecute;
182 use crate::aggregate_fn::Accumulator;
183 use crate::aggregate_fn::DynAccumulator;
184 use crate::aggregate_fn::EmptyOptions;
185 use crate::aggregate_fn::fns::null_count::NullCount;
186 use crate::aggregate_fn::fns::null_count::null_count;
187 use crate::arrays::PrimitiveArray;
188 use crate::dtype::DType;
189 use crate::dtype::Nullability;
190 use crate::dtype::PType;
191 use crate::expr::stats::Precision;
192 use crate::expr::stats::Stat;
193 use crate::expr::stats::StatsProviderExt;
194
195 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
196
197 #[test]
198 fn null_count_with_nulls() -> VortexResult<()> {
199 let array =
200 PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None]).into_array();
201 let mut ctx = SESSION.create_execution_ctx();
202
203 assert_eq!(null_count(&array, &mut ctx)?, 2);
204 assert_eq!(
205 array.statistics().get_as::<u64>(Stat::NullCount),
206 Precision::exact(2u64)
207 );
208 Ok(())
209 }
210
211 #[test]
212 fn null_count_multi_batch() -> VortexResult<()> {
213 let mut ctx = SESSION.create_execution_ctx();
214 let dtype = DType::Primitive(PType::I32, Nullability::Nullable);
215 let mut acc = Accumulator::try_new(NullCount, EmptyOptions, dtype)?;
216
217 let batch1 = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]).into_array();
218 acc.accumulate(&batch1, &mut ctx)?;
219
220 let batch2 = PrimitiveArray::from_option_iter([None, Some(5i32), None]).into_array();
221 acc.accumulate(&batch2, &mut ctx)?;
222
223 let result = acc.finish()?;
224 assert_eq!(result.as_primitive().typed_value::<u64>(), Some(3));
225 Ok(())
226 }
227}