Skip to main content

vortex_array/aggregate_fn/fns/last/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use vortex_error::VortexResult;
5use vortex_session::VortexSession;
6use vortex_session::registry::CachedId;
7
8use crate::ArrayRef;
9use crate::Columnar;
10use crate::ExecutionCtx;
11use crate::aggregate_fn::Accumulator;
12use crate::aggregate_fn::AggregateArgs;
13use crate::aggregate_fn::AggregateFnId;
14use crate::aggregate_fn::AggregateFnVTable;
15use crate::aggregate_fn::DynAccumulator;
16use crate::aggregate_fn::EmptyOptions;
17use crate::dtype::DType;
18use crate::scalar::Scalar;
19
20/// Return the last non-null value of an array.
21///
22/// See [`Last`] for details.
23pub fn last(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Scalar> {
24    let mut acc = Accumulator::try_new(Last, EmptyOptions, array.dtype().clone())?;
25    acc.accumulate(array, ctx)?;
26    acc.finish()
27}
28
29/// Return the last non-null value seen across all batches.
30#[derive(Clone, Debug)]
31pub struct Last;
32
33/// Partial accumulator state for the [`Last`] aggregate.
34pub struct LastPartial {
35    /// The last non-null value seen so far, or `None` if no non-null value has been observed.
36    value: Option<Scalar>,
37}
38
39impl AggregateFnVTable for Last {
40    type Options = EmptyOptions;
41    type Partial = LastPartial;
42
43    fn id(&self) -> AggregateFnId {
44        static ID: CachedId = CachedId::new("vortex.last");
45        *ID
46    }
47
48    fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
49        Ok(Some(vec![]))
50    }
51
52    fn deserialize(
53        &self,
54        _metadata: &[u8],
55        _session: &VortexSession,
56    ) -> VortexResult<Self::Options> {
57        Ok(EmptyOptions)
58    }
59
60    fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {
61        Some(input_dtype.as_nullable())
62    }
63
64    fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
65        self.return_dtype(options, input_dtype)
66    }
67
68    fn empty_partial(
69        &self,
70        _args: AggregateArgs<'_, Self::Options>,
71    ) -> VortexResult<Self::Partial> {
72        Ok(LastPartial { value: None })
73    }
74
75    fn partial_from_scalar(
76        &self,
77        _args: AggregateArgs<'_, Self::Options>,
78        scalar: &Scalar,
79    ) -> VortexResult<Self::Partial> {
80        // A null partial means the producing accumulator saw nothing valid.
81        Ok(LastPartial {
82            value: (!scalar.is_null()).then(|| scalar.clone()),
83        })
84    }
85
86    fn merge_partials(
87        &self,
88        _args: AggregateArgs<'_, Self::Options>,
89        first: Self::Partial,
90        second: Self::Partial,
91    ) -> VortexResult<Self::Partial> {
92        // The later value wins.
93        Ok(LastPartial {
94            value: second.value.or(first.value),
95        })
96    }
97
98    fn to_scalar(
99        &self,
100        args: AggregateArgs<'_, Self::Options>,
101        partial: &Self::Partial,
102    ) -> VortexResult<Scalar> {
103        Ok(match &partial.value {
104            Some(v) => v.clone(),
105            None => Scalar::null(args.return_dtype.clone()),
106        })
107    }
108
109    #[inline]
110    fn is_saturated(
111        &self,
112        _args: AggregateArgs<'_, Self::Options>,
113        _partial: &Self::Partial,
114    ) -> bool {
115        // Last can never short-circuit: a later batch can always supersede the current value.
116        false
117    }
118
119    fn try_accumulate(
120        &self,
121        _args: AggregateArgs<'_, Self::Options>,
122        partial: &mut Self::Partial,
123        batch: &ArrayRef,
124        ctx: &mut ExecutionCtx,
125    ) -> VortexResult<bool> {
126        if let Some(idx) = batch.validity()?.execute_mask(batch.len(), ctx)?.last() {
127            let scalar = batch.execute_scalar(idx, ctx)?;
128            partial.value = Some(scalar.into_nullable());
129        }
130        Ok(true)
131    }
132
133    fn accumulate(
134        &self,
135        _args: AggregateArgs<'_, Self::Options>,
136        _partial: &mut Self::Partial,
137        _batch: &Columnar,
138        _ctx: &mut ExecutionCtx,
139    ) -> VortexResult<()> {
140        unreachable!("Last::try_accumulate handles all arrays")
141    }
142
143    fn finalize(
144        &self,
145        _args: AggregateArgs<'_, Self::Options>,
146        partials: ArrayRef,
147    ) -> VortexResult<ArrayRef> {
148        Ok(partials)
149    }
150
151    fn finalize_scalar(
152        &self,
153        args: AggregateArgs<'_, Self::Options>,
154        partial: &Self::Partial,
155    ) -> VortexResult<Scalar> {
156        self.to_scalar(args, partial)
157    }
158}
159
160#[cfg(test)]
161mod tests {
162    use std::sync::LazyLock;
163
164    use vortex_buffer::buffer;
165    use vortex_error::VortexResult;
166    use vortex_session::VortexSession;
167
168    use crate::IntoArray;
169    use crate::VortexSessionExecute;
170    use crate::aggregate_fn::Accumulator;
171    use crate::aggregate_fn::AggregateDTypes;
172    use crate::aggregate_fn::AggregateFnVTable;
173    use crate::aggregate_fn::DynAccumulator;
174    use crate::aggregate_fn::EmptyOptions;
175    use crate::aggregate_fn::fns::last::Last;
176    use crate::aggregate_fn::fns::last::last;
177    use crate::arrays::ChunkedArray;
178    use crate::arrays::ConstantArray;
179    use crate::arrays::PrimitiveArray;
180    use crate::arrays::VarBinArray;
181    use crate::dtype::DType;
182    use crate::dtype::Nullability;
183    use crate::dtype::Nullability::Nullable;
184    use crate::dtype::PType;
185    use crate::scalar::Scalar;
186    use crate::validity::Validity;
187
188    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
189
190    #[test]
191    fn last_non_null() -> VortexResult<()> {
192        let array = PrimitiveArray::new(buffer![10i32, 20, 30], Validity::NonNullable).into_array();
193        let mut ctx = SESSION.create_execution_ctx();
194        assert_eq!(last(&array, &mut ctx)?, Scalar::primitive(30i32, Nullable));
195        Ok(())
196    }
197
198    #[test]
199    fn last_skips_trailing_nulls() -> VortexResult<()> {
200        let array =
201            PrimitiveArray::from_option_iter([Some(7i32), Some(8), None, None]).into_array();
202        let mut ctx = SESSION.create_execution_ctx();
203        assert_eq!(last(&array, &mut ctx)?, Scalar::primitive(8i32, Nullable));
204        Ok(())
205    }
206
207    #[test]
208    fn last_all_null() -> VortexResult<()> {
209        let array = PrimitiveArray::from_option_iter::<i32, _>([None, None, None]).into_array();
210        let mut ctx = SESSION.create_execution_ctx();
211        let dtype = DType::Primitive(PType::I32, Nullable);
212        assert_eq!(last(&array, &mut ctx)?, Scalar::null(dtype));
213        Ok(())
214    }
215
216    #[test]
217    fn last_empty() -> VortexResult<()> {
218        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
219        let mut acc = Accumulator::try_new(Last, EmptyOptions, dtype)?;
220        let result = acc.finish()?;
221        assert_eq!(result, Scalar::null(DType::Primitive(PType::I32, Nullable)));
222        Ok(())
223    }
224
225    #[test]
226    fn last_constant() -> VortexResult<()> {
227        let array = ConstantArray::new(42i32, 10).into_array();
228        let mut ctx = SESSION.create_execution_ctx();
229        assert_eq!(last(&array, &mut ctx)?, Scalar::primitive(42i32, Nullable));
230        Ok(())
231    }
232
233    #[test]
234    fn last_constant_null() -> VortexResult<()> {
235        let dtype = DType::Primitive(PType::I32, Nullable);
236        let array = ConstantArray::new(Scalar::null(dtype.clone()), 10).into_array();
237        let mut ctx = SESSION.create_execution_ctx();
238        assert_eq!(last(&array, &mut ctx)?, Scalar::null(dtype));
239        Ok(())
240    }
241
242    #[test]
243    fn last_varbin() -> VortexResult<()> {
244        let array = VarBinArray::from_iter(
245            vec![Some("hello"), Some("world"), None],
246            DType::Utf8(Nullable),
247        )
248        .into_array();
249        let mut ctx = SESSION.create_execution_ctx();
250        assert_eq!(last(&array, &mut ctx)?, Scalar::utf8("world", Nullable));
251        Ok(())
252    }
253
254    #[test]
255    fn last_multi_batch_picks_latest_non_null() -> VortexResult<()> {
256        let mut ctx = SESSION.create_execution_ctx();
257        let dtype = DType::Primitive(PType::I32, Nullable);
258        let mut acc = Accumulator::try_new(Last, EmptyOptions, dtype)?;
259
260        let batch1 = PrimitiveArray::from_option_iter([Some(1i32), Some(2)]).into_array();
261        acc.accumulate(&batch1, &mut ctx)?;
262
263        // All-null batch must not clobber the previously-stored value.
264        let batch2 = PrimitiveArray::from_option_iter::<i32, _>([None, None]).into_array();
265        acc.accumulate(&batch2, &mut ctx)?;
266        assert_eq!(acc.final_scalar()?, Scalar::primitive(2i32, Nullable));
267
268        let batch3 = PrimitiveArray::from_option_iter([Some(99i32), None]).into_array();
269        acc.accumulate(&batch3, &mut ctx)?;
270
271        // Last is never saturated; later batches keep updating it.
272        assert!(!acc.is_saturated());
273
274        let result = acc.finish()?;
275        assert_eq!(result, Scalar::primitive(99i32, Nullable));
276        Ok(())
277    }
278
279    #[test]
280    fn last_finish_resets_state() -> VortexResult<()> {
281        let mut ctx = SESSION.create_execution_ctx();
282        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
283        let mut acc = Accumulator::try_new(Last, EmptyOptions, dtype)?;
284
285        let batch1 = PrimitiveArray::new(buffer![10i32, 20], Validity::NonNullable).into_array();
286        acc.accumulate(&batch1, &mut ctx)?;
287        assert_eq!(acc.finish()?, Scalar::primitive(20i32, Nullable));
288        assert_eq!(
289            acc.final_scalar()?,
290            Scalar::null(DType::Primitive(PType::I32, Nullable))
291        );
292
293        let batch2 = PrimitiveArray::new(buffer![3i32, 6, 9], Validity::NonNullable).into_array();
294        acc.accumulate(&batch2, &mut ctx)?;
295        assert_eq!(acc.finish()?, Scalar::primitive(9i32, Nullable));
296        Ok(())
297    }
298
299    #[test]
300    fn last_state_merge() -> VortexResult<()> {
301        let dtype = DType::Primitive(PType::I32, Nullability::NonNullable);
302        let owned = AggregateDTypes::try_new(&Last, &EmptyOptions, dtype)?;
303        let args = owned.args(&EmptyOptions);
304        let partial_of =
305            |value: i32| Last.partial_from_scalar(args, &Scalar::primitive(value, Nullable));
306
307        let five = partial_of(5)?;
308        let seven = partial_of(7)?;
309        let empty = Last.empty_partial(args)?;
310
311        // The last non-empty partial wins, and a trailing empty one changes nothing.
312        let merge = |first, second| Last.merge_partials(args, first, second);
313        let state = merge(merge(five, seven)?, empty)?;
314        assert_eq!(
315            Last.to_scalar(args, &state)?,
316            Scalar::primitive(7i32, Nullable)
317        );
318        Ok(())
319    }
320
321    #[test]
322    fn last_chunked() -> VortexResult<()> {
323        let chunk1 = PrimitiveArray::from_option_iter([Some(42i32), Some(100)]);
324        let chunk2 = PrimitiveArray::from_option_iter::<i32, _>([None, None]);
325        let dtype = chunk1.dtype().clone();
326        let chunked = ChunkedArray::try_new(vec![chunk1.into_array(), chunk2.into_array()], dtype)?;
327        let mut ctx = SESSION.create_execution_ctx();
328        assert_eq!(
329            last(&chunked.into_array(), &mut ctx)?,
330            Scalar::primitive(100i32, Nullable)
331        );
332        Ok(())
333    }
334}