Skip to main content

vortex_array/arrays/masked/vtable/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3mod operations;
4mod validity;
5
6use std::hash::Hasher;
7
8use smallvec::smallvec;
9use vortex_error::VortexExpect;
10use vortex_error::VortexResult;
11use vortex_error::vortex_bail;
12use vortex_error::vortex_ensure;
13use vortex_error::vortex_ensure_eq;
14use vortex_error::vortex_panic;
15use vortex_session::VortexSession;
16use vortex_session::registry::CachedId;
17
18use crate::AnyCanonical;
19use crate::ArrayEq;
20use crate::ArrayHash;
21use crate::ArrayParts;
22use crate::ArrayRef;
23use crate::Canonical;
24use crate::EqMode;
25use crate::IntoArray;
26use crate::VortexSessionExecute;
27use crate::array::Array;
28use crate::array::ArrayId;
29use crate::array::ArrayView;
30use crate::array::VTable;
31use crate::array::validity_to_child;
32use crate::array::with_empty_buffers;
33use crate::arrays::ConstantArray;
34use crate::arrays::masked::MaskedArrayExt;
35use crate::arrays::masked::MaskedArraySlotsExt;
36use crate::arrays::masked::MaskedData;
37use crate::arrays::masked::array::MaskedSlots;
38use crate::arrays::masked::compute::rules::PARENT_RULES;
39use crate::arrays::masked::mask_validity_canonical;
40use crate::buffer::BufferHandle;
41use crate::dtype::DType;
42use crate::executor::ExecutionCtx;
43use crate::executor::ExecutionResult;
44use crate::require_child;
45use crate::scalar::Scalar;
46use crate::serde::ArrayChildren;
47use crate::validity::Validity;
48/// A [`Masked`]-encoded Vortex array.
49pub type MaskedArray = Array<Masked>;
50
51#[derive(Clone, Debug)]
52pub struct Masked;
53
54impl ArrayHash for MaskedData {
55    fn array_hash<H: Hasher>(&self, _state: &mut H, _accuracy: EqMode) {}
56}
57
58impl ArrayEq for MaskedData {
59    fn array_eq(&self, _other: &Self, _accuracy: EqMode) -> bool {
60        true
61    }
62}
63
64impl VTable for Masked {
65    type TypedArrayData = MaskedData;
66
67    type OperationsVTable = Self;
68    type ValidityVTable = Self;
69
70    fn id(&self) -> ArrayId {
71        static ID: CachedId = CachedId::new("vortex.masked");
72        *ID
73    }
74
75    fn validate(
76        &self,
77        _data: &MaskedData,
78        dtype: &DType,
79        len: usize,
80        slots: &[Option<ArrayRef>],
81    ) -> VortexResult<()> {
82        vortex_ensure!(
83            slots[MaskedSlots::CHILD].is_some(),
84            "MaskedArray child slot must be present"
85        );
86        let child = slots[MaskedSlots::CHILD]
87            .as_ref()
88            .vortex_expect("validated child slot");
89        vortex_ensure_eq!(child.len(), len, "MaskedArray child length mismatch");
90        vortex_ensure_eq!(
91            child.dtype().as_nullable(),
92            *dtype,
93            "MaskedArray dtype does not match child and validity",
94        );
95        Ok(())
96    }
97
98    fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
99        0
100    }
101
102    fn buffer(_array: ArrayView<'_, Self>, _idx: usize) -> BufferHandle {
103        vortex_panic!("MaskedArray has no buffers")
104    }
105
106    fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option<String> {
107        None
108    }
109
110    fn with_buffers(
111        &self,
112        array: ArrayView<'_, Self>,
113        buffers: &[BufferHandle],
114    ) -> VortexResult<ArrayParts<Self>> {
115        with_empty_buffers(self, array, buffers)
116    }
117
118    fn serialize(
119        _array: ArrayView<'_, Self>,
120        _session: &VortexSession,
121    ) -> VortexResult<Option<Vec<u8>>> {
122        Ok(Some(vec![]))
123    }
124
125    fn deserialize(
126        &self,
127        dtype: &DType,
128        len: usize,
129        metadata: &[u8],
130
131        buffers: &[BufferHandle],
132        children: &dyn ArrayChildren,
133        session: &VortexSession,
134    ) -> VortexResult<ArrayParts<Self>> {
135        if !metadata.is_empty() {
136            vortex_bail!(
137                "MaskedArray expects empty metadata, got {} bytes",
138                metadata.len()
139            );
140        }
141        if !buffers.is_empty() {
142            vortex_bail!("Expected 0 buffer, got {}", buffers.len());
143        }
144
145        vortex_ensure!(
146            children.len() == 1 || children.len() == 2,
147            "`MaskedArray::build` expects 1 or 2 children, got {}",
148            children.len()
149        );
150
151        let child = children.get(0, &dtype.as_nonnullable(), len)?;
152
153        let validity = if children.len() == 2 {
154            let validity = children.get(1, &Validity::DTYPE, len)?;
155            Validity::Array(validity)
156        } else {
157            Validity::from(dtype.nullability())
158        };
159
160        let validity_slot = validity_to_child(&validity, len);
161        let data = MaskedData::try_new(
162            len,
163            child.all_valid(&mut session.create_execution_ctx())?,
164            validity,
165        )?;
166        Ok(ArrayParts::new(
167            self.clone(),
168            dtype.clone(),
169            len,
170            data,
171            smallvec![Some(child), validity_slot],
172        ))
173    }
174
175    fn execute(array: Array<Self>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
176        let array = require_child!(array, array.child(), MaskedSlots::CHILD => AnyCanonical);
177
178        let validity = array.masked_validity();
179
180        // Fast path: all masked means result is all nulls.
181        if validity.definitely_all_null() {
182            return Ok(ExecutionResult::done(
183                ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.len())
184                    .into_array(),
185            ));
186        }
187
188        // NB: We intentionally do NOT have a fast path for `validity_mask.all_true()`.
189        // `MaskedArray`'s dtype is always `Nullable`, but the child has `NonNullable` `DType` (by
190        // invariant). Simply returning the child's canonical would cause a dtype mismatch.
191        // While we could manually convert the dtype, `mask_validity_canonical` is already O(1) for
192        // `AllTrue` masks (no data copying), so there's no benefit.
193
194        let child = Canonical::from(array.child().as_::<AnyCanonical>());
195        Ok(ExecutionResult::done(
196            mask_validity_canonical(child, validity, ctx)?.into_array(),
197        ))
198    }
199
200    fn reduce_parent(
201        array: ArrayView<'_, Self>,
202        parent: &ArrayRef,
203        child_idx: usize,
204    ) -> VortexResult<Option<ArrayRef>> {
205        PARENT_RULES.evaluate(array, parent, child_idx)
206    }
207    fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
208        MaskedSlots::NAMES[idx].to_string()
209    }
210}
211
212#[cfg(test)]
213mod tests {
214    use std::sync::LazyLock;
215
216    use rstest::rstest;
217    use vortex_buffer::ByteBufferMut;
218    use vortex_error::VortexError;
219    use vortex_session::VortexSession;
220    use vortex_session::registry::ReadContext;
221
222    use crate::ArrayContext;
223    use crate::Canonical;
224    use crate::IntoArray;
225    use crate::VortexSessionExecute;
226    use crate::arrays::Masked;
227    use crate::arrays::MaskedArray;
228    use crate::arrays::PrimitiveArray;
229    use crate::assert_arrays_eq;
230    use crate::dtype::Nullability;
231    use crate::serde::SerializeOptions;
232    use crate::serde::SerializedArray;
233    use crate::validity::Validity;
234
235    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
236
237    #[rstest]
238    #[case(
239        MaskedArray::try_new(
240            PrimitiveArray::from_iter([1i32, 2, 3]).into_array(),
241            Validity::AllValid,
242            &mut SESSION.create_execution_ctx(),
243        ).unwrap()
244    )]
245    #[case(
246        MaskedArray::try_new(
247            PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]).into_array(),
248            Validity::from_iter([true, true, false, true, false]),
249            &mut SESSION.create_execution_ctx(),
250        ).unwrap()
251    )]
252    #[case(
253        MaskedArray::try_new(
254            PrimitiveArray::from_iter(0..100).into_array(),
255            Validity::from_iter((0..100).map(|i| i % 3 != 0)),
256            &mut SESSION.create_execution_ctx(),
257        ).unwrap()
258    )]
259    fn test_serde_roundtrip(#[case] array: MaskedArray) {
260        let dtype = array.dtype().clone();
261        let len = array.len();
262
263        let ctx = ArrayContext::empty();
264        let serialized = array
265            .clone()
266            .into_array()
267            .serialize(&ctx, &VortexSession::empty(), &SerializeOptions::default())
268            .unwrap();
269
270        // Concat into a single buffer.
271        let mut concat = ByteBufferMut::empty();
272        for buf in serialized {
273            concat.extend_from_slice(buf.as_ref());
274        }
275        let concat = concat.freeze();
276
277        let parts = SerializedArray::try_from(concat).unwrap();
278        let decoded = parts
279            .decode(
280                &dtype,
281                len,
282                &ReadContext::new(ctx.to_ids()),
283                &VortexSession::empty(),
284            )
285            .unwrap();
286
287        assert!(decoded.is::<Masked>());
288        assert_arrays_eq!(decoded, array, &mut SESSION.create_execution_ctx());
289    }
290
291    /// Regression test for issue #5989: execute_fast_path returns child with wrong dtype.
292    ///
293    /// When MaskedArray's validity mask is all true, returning the child's canonical form
294    /// directly would cause a dtype mismatch because the child has NonNullable dtype while
295    /// MaskedArray always has Nullable dtype.
296    #[test]
297    fn test_execute_with_all_valid_preserves_nullable_dtype() -> Result<(), VortexError> {
298        let mut ctx = SESSION.create_execution_ctx();
299        // Create a MaskedArray with AllValid validity.
300
301        // Child has NonNullable dtype, but MaskedArray's dtype is Nullable.
302        let child = PrimitiveArray::from_iter([1i32, 2, 3]).into_array();
303        assert_eq!(child.dtype().nullability(), Nullability::NonNullable);
304
305        let array = MaskedArray::try_new(child, Validity::AllValid, &mut ctx)?;
306        assert_eq!(array.dtype().nullability(), Nullability::Nullable);
307
308        // Execute the array. This should produce a Canonical with Nullable dtype.
309        let mut ctx = SESSION.create_execution_ctx();
310        let result: Canonical = array.into_array().execute(&mut ctx)?;
311
312        assert_eq!(
313            result.dtype().nullability(),
314            Nullability::Nullable,
315            "MaskedArray execute should produce Nullable dtype"
316        );
317
318        Ok(())
319    }
320}