Skip to main content

vortex_array/array/probe/
repeated.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::any::Any;
5
6use vortex_error::VortexResult;
7use vortex_error::vortex_err;
8
9use crate::ArrayRef;
10use crate::ExecutionCtx;
11use crate::IntoArray;
12use crate::array::probe::ArrayProbe;
13use crate::array::probe::array::check_bounds;
14use crate::array::probe::array::check_dtype;
15use crate::array::probe::array::child_of;
16use crate::arrays::BoolArray;
17use crate::arrays::ScalarFn;
18use crate::scalar::Scalar;
19use crate::validity::Validity;
20
21/// A row accessor that owns its array and keeps preparation between reads.
22///
23/// The encoding's state, its validity probe and the probes over child slots are created on
24/// first use and reused by every following read. Dropping the probe drops all of them. Array
25/// handles share their buffers, so the probe can outlive the handle it was built from. Probes
26/// are local to a thread.
27pub struct RepeatedArrayProbe {
28    array: ArrayRef,
29    /// The encoding's [`RepeatedState`], type-erased, created on the first read.
30    state: Option<Box<dyn Any>>,
31}
32
33impl RepeatedArrayProbe {
34    /// Own an array for repeated reads.
35    pub fn new(array: ArrayRef) -> Self {
36        Self { array, state: None }
37    }
38
39    /// The array this probe reads from.
40    pub fn array(&self) -> &ArrayRef {
41        &self.array
42    }
43
44    /// Borrow this probe as an [`ArrayProbe`].
45    #[inline]
46    pub fn as_probe(&mut self) -> ArrayProbe<'_> {
47        ArrayProbe::Repeated(self)
48    }
49
50    /// Read the scalar at `index`, including its nullness, reusing retained preparation.
51    pub fn execute_scalar(&mut self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<Scalar> {
52        check_bounds(&self.array, index)?;
53        let result =
54            self.array
55                .dyn_array()
56                .probe_scalar_retained(&self.array, index, &mut self.state, ctx);
57        check_dtype(&self.array, result)
58    }
59
60    /// Whether the row at `index` is valid, through the retained validity.
61    pub fn execute_is_valid(&mut self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
62        check_bounds(&self.array, index)?;
63        if !self.array.dtype().is_nullable() {
64            return Ok(true);
65        }
66        self.array
67            .dyn_array()
68            .probe_is_valid_retained(&self.array, index, &mut self.state, ctx)
69    }
70
71    /// Whether the row at `index` is null.
72    pub fn execute_is_invalid(
73        &mut self,
74        index: usize,
75        ctx: &mut ExecutionCtx,
76    ) -> VortexResult<bool> {
77        Ok(!self.execute_is_valid(index, ctx)?)
78    }
79}
80
81/// The validity of a [`RepeatedState`], resolved on first use and kept between reads.
82enum ResolvedValidity {
83    Uniform(bool),
84    Array(RepeatedArrayProbe),
85}
86
87impl ResolvedValidity {
88    /// Whether row `index` is valid.
89    #[inline]
90    fn is_valid(&mut self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
91        match self {
92            Self::Uniform(valid) => Ok(*valid),
93            Self::Array(probe) => is_valid_scalar(probe.execute_scalar(index, ctx)?, index),
94        }
95    }
96
97    /// The resolved validity of `array`.
98    fn resolve(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
99        if !array.dtype().is_nullable() {
100            return Ok(Self::Uniform(true));
101        }
102        Ok(match array.validity()? {
103            Validity::NonNullable | Validity::AllValid => Self::Uniform(true),
104            Validity::AllInvalid => Self::Uniform(false),
105            Validity::Array(validity) => {
106                // ScalarFn validity is lazy; materialize it once instead of once per row.
107                // TODO(myrrc): remove this once the probing API can specify validity.
108                let validity = if array.is::<ScalarFn>() {
109                    validity.execute::<BoolArray>(ctx)?.into_array()
110                } else {
111                    validity
112                };
113                Self::Array(RepeatedArrayProbe::new(validity))
114            }
115        })
116    }
117}
118
119/// The validity bit read out of a validity array's scalar.
120#[inline]
121pub(super) fn is_valid_scalar(scalar: Scalar, index: usize) -> VortexResult<bool> {
122    scalar
123        .as_bool()
124        .value()
125        .ok_or_else(|| vortex_err!("validity value at index {index} is null"))
126}
127
128/// Get or create the [`RepeatedState`] for encoding state `S` in a probe's erased slot.
129pub(crate) fn repeated_state<S: Default + 'static>(
130    slot: &mut Option<Box<dyn Any>>,
131) -> VortexResult<&mut RepeatedState<S>> {
132    slot.get_or_insert_with(|| Box::new(RepeatedState::<S>::default()))
133        .downcast_mut::<RepeatedState<S>>()
134        .ok_or_else(|| vortex_err!("Probe state type mismatch"))
135}
136
137/// What a [`RepeatedArrayProbe`] keeps for its encoding between reads.
138pub(crate) struct RepeatedState<S> {
139    state: S,
140    /// Probes over the source's child slots, allocated on first use. Slots that are never
141    /// requested stay empty.
142    slots: Vec<Option<RepeatedArrayProbe>>,
143    /// The source's validity, resolved on first use.
144    validity: Option<ResolvedValidity>,
145}
146
147impl<S: Default> Default for RepeatedState<S> {
148    fn default() -> Self {
149        Self {
150            state: S::default(),
151            slots: Vec::new(),
152            validity: None,
153        }
154    }
155}
156
157impl<S> RepeatedState<S> {
158    #[inline]
159    pub(super) fn state_mut(&mut self) -> &mut S {
160        &mut self.state
161    }
162
163    /// The encoding state and the child slot table as disjoint borrows.
164    #[inline]
165    pub(super) fn split_mut(&mut self) -> (&mut S, &mut Vec<Option<RepeatedArrayProbe>>) {
166        (&mut self.state, &mut self.slots)
167    }
168
169    /// Whether row `index` of `array` is valid, through the kept validity.
170    #[inline]
171    pub(crate) fn is_valid(
172        &mut self,
173        array: &ArrayRef,
174        index: usize,
175        ctx: &mut ExecutionCtx,
176    ) -> VortexResult<bool> {
177        let validity = match &mut self.validity {
178            Some(validity) => validity,
179            slot @ None => slot.insert(ResolvedValidity::resolve(array, ctx)?),
180        };
181        validity.is_valid(index, ctx)
182    }
183}
184
185/// Get or create the retained probe over `parent`'s child in `slot`.
186pub(super) fn child_probe<'s>(
187    slots: &'s mut Vec<Option<RepeatedArrayProbe>>,
188    parent: &ArrayRef,
189    slot: usize,
190) -> VortexResult<Option<&'s mut RepeatedArrayProbe>> {
191    let Some(child) = child_of(parent, slot)? else {
192        return Ok(None);
193    };
194    if slots.is_empty() {
195        slots.resize_with(parent.slots().len(), || None);
196    }
197    Ok(Some(slots[slot].get_or_insert_with(|| {
198        RepeatedArrayProbe::new(child.clone())
199    })))
200}
201
202#[cfg(test)]
203mod tests {
204    use std::sync::LazyLock;
205
206    use vortex_error::VortexResult;
207    use vortex_error::vortex_err;
208    use vortex_session::VortexSession;
209
210    use super::*;
211    use crate::VortexSessionExecute;
212    use crate::array::IntoArray;
213    use crate::array::probe::ProbeState;
214    use crate::arrays::PrimitiveArray;
215    use crate::arrays::Struct;
216    use crate::arrays::StructArray;
217
218    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
219
220    fn nullable_ints() -> ArrayRef {
221        PrimitiveArray::from_option_iter([Some(10i32), None, Some(30)]).into_array()
222    }
223
224    fn check_reads(mut probe: ArrayProbe<'_>, ctx: &mut ExecutionCtx) -> VortexResult<()> {
225        assert!(probe.execute_scalar(3, ctx).is_err());
226        assert!(probe.execute_is_valid(3, ctx).is_err());
227        assert!(probe.execute_scalar(1, ctx)?.is_null());
228        assert!(!probe.execute_is_valid(1, ctx)?);
229        assert!(probe.execute_is_invalid(1, ctx)?);
230        assert_eq!(probe.execute_scalar(2, ctx)?, Scalar::from(Some(30i32)));
231        assert_eq!(probe.execute_scalar(0, ctx)?, Scalar::from(Some(10i32)));
232        Ok(())
233    }
234
235    #[test]
236    fn repeated_probe_initializes_lazily_and_outlives_handle() -> VortexResult<()> {
237        let mut ctx = SESSION.create_execution_ctx();
238        let array = nullable_ints();
239        let mut probe = RepeatedArrayProbe::new(array.clone());
240        drop(array);
241        assert!(probe.state.is_none());
242
243        check_reads(probe.as_probe(), &mut ctx)?;
244
245        let state = repeated_state::<()>(&mut probe.state)?;
246        assert!(matches!(state.validity, Some(ResolvedValidity::Array(_))));
247        Ok(())
248    }
249
250    #[test]
251    fn repeated_probe_resolves_uniform_validity_once() -> VortexResult<()> {
252        let mut ctx = SESSION.create_execution_ctx();
253        let array = PrimitiveArray::from_option_iter([Some(1i32), Some(2)]).into_array();
254        let mut probe = array.repeated_probe();
255        assert!(probe.execute_is_valid(1, &mut ctx)?);
256        let state = repeated_state::<()>(&mut probe.state)?;
257        assert!(matches!(
258            state.validity,
259            Some(ResolvedValidity::Uniform(true))
260        ));
261        assert_eq!(probe.execute_scalar(1, &mut ctx)?, Scalar::from(Some(2i32)));
262        Ok(())
263    }
264
265    #[test]
266    fn validity_and_scalar_reads_share_the_validity_slot() -> VortexResult<()> {
267        let mut ctx = SESSION.create_execution_ctx();
268        let mut probe = nullable_ints().repeated_probe();
269        assert!(!probe.execute_is_valid(1, &mut ctx)?);
270        let Some(ResolvedValidity::Array(validity)) =
271            &repeated_state::<()>(&mut probe.state)?.validity
272        else {
273            return Err(vortex_err!("validity slot should hold a probe"));
274        };
275        assert!(validity.state.is_some());
276        assert!(probe.execute_scalar(1, &mut ctx)?.is_null());
277        assert_eq!(
278            probe.execute_scalar(2, &mut ctx)?,
279            Scalar::from(Some(30i32))
280        );
281        Ok(())
282    }
283
284    #[test]
285    fn repeated_probe_reads_null_rows_through_encoding() -> VortexResult<()> {
286        let mut ctx = SESSION.create_execution_ctx();
287        let array = PrimitiveArray::from_option_iter([Some(1i32), None]).into_array();
288        let mut probe = array.repeated_probe();
289        assert!(probe.execute_scalar(1, &mut ctx)?.is_null());
290        assert_eq!(probe.execute_scalar(0, &mut ctx)?, Scalar::from(Some(1i32)));
291        assert!(!probe.execute_is_valid(1, &mut ctx)?);
292        Ok(())
293    }
294
295    #[test]
296    fn repeated_state_creates_children_on_demand() -> VortexResult<()> {
297        let mut ctx = SESSION.create_execution_ctx();
298        let array = struct_of_two_fields()?;
299        let typed = array
300            .as_opt::<Struct>()
301            .ok_or_else(|| vortex_err!("expected a struct"))?;
302        let mut repeated = RepeatedState::<()>::default();
303        {
304            let mut state = ProbeState::repeated(typed, &mut repeated);
305            assert!(state.slot(5).is_err());
306            assert!(state.slot(0)?.is_none());
307            assert!(state.retained().is_some());
308            let mut field = state.slot(2)?.ok_or_else(|| vortex_err!("missing field"))?;
309            assert!(matches!(field, ArrayProbe::Repeated(_)));
310            assert_eq!(field.execute_scalar(1, &mut ctx)?, Scalar::from(4i64));
311            assert_eq!(field.execute_scalar(0, &mut ctx)?, Scalar::from(3i64));
312        }
313
314        assert_eq!(repeated.slots.len(), array.slots().len());
315        assert!(repeated.slots[1].is_none());
316        let child = repeated.slots[2]
317            .as_ref()
318            .ok_or_else(|| vortex_err!("missing child probe"))?;
319        assert_eq!(child.array().len(), 2);
320        Ok(())
321    }
322
323    fn struct_of_two_fields() -> VortexResult<ArrayRef> {
324        Ok(StructArray::from_fields(&[
325            ("a", PrimitiveArray::from_iter([1i32, 2]).into_array()),
326            ("b", PrimitiveArray::from_iter([3i64, 4]).into_array()),
327        ])?
328        .into_array())
329    }
330
331    #[test]
332    fn state_slot_rejects_mismatched_type() -> VortexResult<()> {
333        let mut slot: Option<Box<dyn Any>> = None;
334        repeated_state::<()>(&mut slot)?;
335        assert!(repeated_state::<u8>(&mut slot).is_err());
336        Ok(())
337    }
338}