vortex_array/arrays/masked/vtable/
mod.rs1mod operations;
4mod validity;
5
6use std::hash::Hasher;
7
8use smallvec::smallvec;
9use vortex_error::VortexExpect;
10use vortex_error::VortexResult;
11use vortex_error::vortex_bail;
12use vortex_error::vortex_ensure;
13use vortex_error::vortex_ensure_eq;
14use vortex_error::vortex_panic;
15use vortex_session::VortexSession;
16use vortex_session::registry::CachedId;
17
18use crate::AnyCanonical;
19use crate::ArrayEq;
20use crate::ArrayHash;
21use crate::ArrayParts;
22use crate::ArrayRef;
23use crate::Canonical;
24use crate::EqMode;
25use crate::IntoArray;
26use crate::VortexSessionExecute;
27use crate::array::Array;
28use crate::array::ArrayId;
29use crate::array::ArrayView;
30use crate::array::VTable;
31use crate::array::validity_to_child;
32use crate::array::with_empty_buffers;
33use crate::arrays::ConstantArray;
34use crate::arrays::masked::MaskedArrayExt;
35use crate::arrays::masked::MaskedArraySlotsExt;
36use crate::arrays::masked::MaskedData;
37use crate::arrays::masked::array::MaskedSlots;
38use crate::arrays::masked::compute::rules::PARENT_RULES;
39use crate::arrays::masked::mask_validity_canonical;
40use crate::buffer::BufferHandle;
41use crate::dtype::DType;
42use crate::executor::ExecutionCtx;
43use crate::executor::ExecutionResult;
44use crate::require_child;
45use crate::scalar::Scalar;
46use crate::serde::ArrayChildren;
47use crate::validity::Validity;
48pub type MaskedArray = Array<Masked>;
50
51#[derive(Clone, Debug)]
52pub struct Masked;
53
54impl ArrayHash for MaskedData {
55 fn array_hash<H: Hasher>(&self, _state: &mut H, _accuracy: EqMode) {}
56}
57
58impl ArrayEq for MaskedData {
59 fn array_eq(&self, _other: &Self, _accuracy: EqMode) -> bool {
60 true
61 }
62}
63
64impl VTable for Masked {
65 type TypedArrayData = MaskedData;
66
67 type OperationsVTable = Self;
68 type ValidityVTable = Self;
69
70 fn id(&self) -> ArrayId {
71 static ID: CachedId = CachedId::new("vortex.masked");
72 *ID
73 }
74
75 fn validate(
76 &self,
77 _data: &MaskedData,
78 dtype: &DType,
79 len: usize,
80 slots: &[Option<ArrayRef>],
81 ) -> VortexResult<()> {
82 vortex_ensure!(
83 slots[MaskedSlots::CHILD].is_some(),
84 "MaskedArray child slot must be present"
85 );
86 let child = slots[MaskedSlots::CHILD]
87 .as_ref()
88 .vortex_expect("validated child slot");
89 vortex_ensure_eq!(child.len(), len, "MaskedArray child length mismatch");
90 vortex_ensure_eq!(
91 child.dtype().as_nullable(),
92 *dtype,
93 "MaskedArray dtype does not match child and validity",
94 );
95 Ok(())
96 }
97
98 fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
99 0
100 }
101
102 fn buffer(_array: ArrayView<'_, Self>, _idx: usize) -> BufferHandle {
103 vortex_panic!("MaskedArray has no buffers")
104 }
105
106 fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option<String> {
107 None
108 }
109
110 fn with_buffers(
111 &self,
112 array: ArrayView<'_, Self>,
113 buffers: &[BufferHandle],
114 ) -> VortexResult<ArrayParts<Self>> {
115 with_empty_buffers(self, array, buffers)
116 }
117
118 fn serialize(
119 _array: ArrayView<'_, Self>,
120 _session: &VortexSession,
121 ) -> VortexResult<Option<Vec<u8>>> {
122 Ok(Some(vec![]))
123 }
124
125 fn deserialize(
126 &self,
127 dtype: &DType,
128 len: usize,
129 metadata: &[u8],
130
131 buffers: &[BufferHandle],
132 children: &dyn ArrayChildren,
133 session: &VortexSession,
134 ) -> VortexResult<ArrayParts<Self>> {
135 if !metadata.is_empty() {
136 vortex_bail!(
137 "MaskedArray expects empty metadata, got {} bytes",
138 metadata.len()
139 );
140 }
141 if !buffers.is_empty() {
142 vortex_bail!("Expected 0 buffer, got {}", buffers.len());
143 }
144
145 vortex_ensure!(
146 children.len() == 1 || children.len() == 2,
147 "`MaskedArray::build` expects 1 or 2 children, got {}",
148 children.len()
149 );
150
151 let child = children.get(0, &dtype.as_nonnullable(), len)?;
152
153 let validity = if children.len() == 2 {
154 let validity = children.get(1, &Validity::DTYPE, len)?;
155 Validity::Array(validity)
156 } else {
157 Validity::from(dtype.nullability())
158 };
159
160 let validity_slot = validity_to_child(&validity, len);
161 let data = MaskedData::try_new(
162 len,
163 child.all_valid(&mut session.create_execution_ctx())?,
164 validity,
165 )?;
166 Ok(ArrayParts::new(
167 self.clone(),
168 dtype.clone(),
169 len,
170 data,
171 smallvec![Some(child), validity_slot],
172 ))
173 }
174
175 fn execute(array: Array<Self>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
176 let array = require_child!(array, array.child(), MaskedSlots::CHILD => AnyCanonical);
177
178 let validity = array.masked_validity();
179
180 if validity.definitely_all_null() {
182 return Ok(ExecutionResult::done(
183 ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.len())
184 .into_array(),
185 ));
186 }
187
188 let child = Canonical::from(array.child().as_::<AnyCanonical>());
195 Ok(ExecutionResult::done(
196 mask_validity_canonical(child, validity, ctx)?.into_array(),
197 ))
198 }
199
200 fn reduce_parent(
201 array: ArrayView<'_, Self>,
202 parent: &ArrayRef,
203 child_idx: usize,
204 ) -> VortexResult<Option<ArrayRef>> {
205 PARENT_RULES.evaluate(array, parent, child_idx)
206 }
207 fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
208 MaskedSlots::NAMES[idx].to_string()
209 }
210}
211
212#[cfg(test)]
213mod tests {
214 use std::sync::LazyLock;
215
216 use rstest::rstest;
217 use vortex_buffer::ByteBufferMut;
218 use vortex_error::VortexError;
219 use vortex_session::VortexSession;
220 use vortex_session::registry::ReadContext;
221
222 use crate::ArrayContext;
223 use crate::Canonical;
224 use crate::IntoArray;
225 use crate::VortexSessionExecute;
226 use crate::arrays::Masked;
227 use crate::arrays::MaskedArray;
228 use crate::arrays::PrimitiveArray;
229 use crate::assert_arrays_eq;
230 use crate::dtype::Nullability;
231 use crate::serde::SerializeOptions;
232 use crate::serde::SerializedArray;
233 use crate::validity::Validity;
234
235 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
236
237 #[rstest]
238 #[case(
239 MaskedArray::try_new(
240 PrimitiveArray::from_iter([1i32, 2, 3]).into_array(),
241 Validity::AllValid,
242 &mut SESSION.create_execution_ctx(),
243 ).unwrap()
244 )]
245 #[case(
246 MaskedArray::try_new(
247 PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]).into_array(),
248 Validity::from_iter([true, true, false, true, false]),
249 &mut SESSION.create_execution_ctx(),
250 ).unwrap()
251 )]
252 #[case(
253 MaskedArray::try_new(
254 PrimitiveArray::from_iter(0..100).into_array(),
255 Validity::from_iter((0..100).map(|i| i % 3 != 0)),
256 &mut SESSION.create_execution_ctx(),
257 ).unwrap()
258 )]
259 fn test_serde_roundtrip(#[case] array: MaskedArray) {
260 let dtype = array.dtype().clone();
261 let len = array.len();
262
263 let ctx = ArrayContext::empty();
264 let serialized = array
265 .clone()
266 .into_array()
267 .serialize(&ctx, &VortexSession::empty(), &SerializeOptions::default())
268 .unwrap();
269
270 let mut concat = ByteBufferMut::empty();
272 for buf in serialized {
273 concat.extend_from_slice(buf.as_ref());
274 }
275 let concat = concat.freeze();
276
277 let parts = SerializedArray::try_from(concat).unwrap();
278 let decoded = parts
279 .decode(
280 &dtype,
281 len,
282 &ReadContext::new(ctx.to_ids()),
283 &VortexSession::empty(),
284 )
285 .unwrap();
286
287 assert!(decoded.is::<Masked>());
288 assert_arrays_eq!(decoded, array, &mut SESSION.create_execution_ctx());
289 }
290
291 #[test]
297 fn test_execute_with_all_valid_preserves_nullable_dtype() -> Result<(), VortexError> {
298 let mut ctx = SESSION.create_execution_ctx();
299 let child = PrimitiveArray::from_iter([1i32, 2, 3]).into_array();
303 assert_eq!(child.dtype().nullability(), Nullability::NonNullable);
304
305 let array = MaskedArray::try_new(child, Validity::AllValid, &mut ctx)?;
306 assert_eq!(array.dtype().nullability(), Nullability::Nullable);
307
308 let mut ctx = SESSION.create_execution_ctx();
310 let result: Canonical = array.into_array().execute(&mut ctx)?;
311
312 assert_eq!(
313 result.dtype().nullability(),
314 Nullability::Nullable,
315 "MaskedArray execute should produce Nullable dtype"
316 );
317
318 Ok(())
319 }
320}