1mod vtable;
5
6pub(crate) mod compute;
7
8use vortex_error::VortexResult;
9use vortex_error::vortex_ensure;
10
11pub use self::vtable::Variant;
12pub use self::vtable::VariantArray;
13
14pub(crate) fn initialize(session: &vortex_session::VortexSession) {
15 vtable::initialize(session);
16}
17
18use crate::ArrayRef;
19use crate::array::Array;
20use crate::array::ArrayParts;
21use crate::array::EmptyArrayData;
22use crate::array_slots;
23use crate::dtype::DType;
24
25#[array_slots(Variant)]
35pub struct VariantSlots {
36 #[slot(0)]
38 pub core_storage: ArrayRef,
39 #[slot(1)]
43 pub shredded: Option<ArrayRef>,
44}
45
46impl Array<Variant> {
47 pub fn try_new(core_storage: ArrayRef, shredded: Option<ArrayRef>) -> VortexResult<Self> {
55 let dtype = core_storage.dtype().clone();
56 vortex_ensure!(
57 matches!(dtype, DType::Variant(_)),
58 "VariantArray core_storage dtype must be Variant, found {dtype}"
59 );
60 let len = core_storage.len();
61 let stats = core_storage.statistics().to_owned();
62 Ok(Array::try_from_parts(ArrayParts::new(
63 Variant,
64 dtype,
65 len,
66 EmptyArrayData,
67 VariantSlots {
68 core_storage,
69 shredded,
70 }
71 .into_slots(),
72 ))?
73 .with_stats_set(stats))
74 }
75}
76
77#[cfg(test)]
78mod tests {
79 use std::sync::LazyLock;
80
81 use vortex_buffer::buffer;
82 use vortex_error::VortexResult;
83 use vortex_error::vortex_err;
84 use vortex_mask::Mask;
85 use vortex_session::VortexSession;
86
87 use crate::ArrayRef;
88 use crate::Canonical;
89 use crate::IntoArray;
90 use crate::VortexSessionExecute;
91 use crate::arrays::BoolArray;
92 use crate::arrays::ChunkedArray;
93 use crate::arrays::ConstantArray;
94 use crate::arrays::PrimitiveArray;
95 use crate::arrays::StructArray;
96 use crate::arrays::VariantArray;
97 use crate::arrays::variant::VariantArraySlotsExt;
98 use crate::assert_arrays_eq;
99 use crate::builtins::ArrayBuiltins;
100 use crate::dtype::DType;
101 use crate::dtype::Nullability;
102 use crate::dtype::PType;
103 use crate::expr::root;
104 use crate::expr::variant_get;
105 use crate::scalar::Scalar;
106 use crate::scalar_fn::fns::variant_get::VariantPath;
107
108 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
109
110 fn core_storage(len: usize) -> ArrayRef {
111 ConstantArray::new(
112 Scalar::variant(Scalar::primitive(1i32, Nullability::NonNullable)),
113 len,
114 )
115 .into_array()
116 }
117
118 fn row_storage(values: impl IntoIterator<Item = i32>) -> VortexResult<ArrayRef> {
119 Ok(ChunkedArray::try_new(
120 values.into_iter().map(|value| {
121 ConstantArray::new(
122 Scalar::variant(Scalar::primitive(value, Nullability::NonNullable)),
123 1,
124 )
125 .into_array()
126 }),
127 DType::Variant(Nullability::NonNullable),
128 )?
129 .into_array())
130 }
131
132 fn variant_with_shredded(
133 core_values: impl IntoIterator<Item = i32>,
134 shredded_values: impl IntoIterator<Item = i32>,
135 ) -> VortexResult<VariantArray> {
136 VariantArray::try_new(
137 row_storage(core_values)?,
138 Some(PrimitiveArray::from_iter(shredded_values).into_array()),
139 )
140 }
141
142 fn execute_variant(array: ArrayRef) -> VortexResult<VariantArray> {
143 let mut ctx = SESSION.create_execution_ctx();
144 let Canonical::Variant(variant) = array.execute::<Canonical>(&mut ctx)? else {
145 return Err(vortex_err!("expected canonical variant array"));
146 };
147 Ok(variant)
148 }
149
150 fn assert_variant_rows(
151 array: &VariantArray,
152 expected_core: &[Option<i32>],
153 expected_shredded: &[Option<i32>],
154 ) -> VortexResult<()> {
155 assert_variant_core_rows(array, expected_core)?;
156 assert_eq!(array.len(), expected_shredded.len());
157
158 let shredded = array
159 .shredded()
160 .ok_or_else(|| vortex_err!("expected shredded child"))?;
161 let mut ctx = SESSION.create_execution_ctx();
162 let expected_shredded_array = if let Some(values) = expected_shredded
163 .iter()
164 .copied()
165 .collect::<Option<Vec<_>>>()
166 {
167 PrimitiveArray::from_iter(values)
168 } else {
169 PrimitiveArray::from_option_iter(expected_shredded.iter().copied())
170 };
171 assert_arrays_eq!(shredded, expected_shredded_array, &mut ctx);
172
173 Ok(())
174 }
175
176 fn assert_variant_core_rows(
177 array: &VariantArray,
178 expected_core: &[Option<i32>],
179 ) -> VortexResult<()> {
180 assert_eq!(array.len(), expected_core.len());
181
182 let mut ctx = SESSION.create_execution_ctx();
183 for (idx, expected) in expected_core.iter().enumerate() {
184 let scalar = array.core_storage().execute_scalar(idx, &mut ctx)?;
185 let variant = scalar.as_variant();
186 match expected {
187 Some(expected) => {
188 let value = variant
189 .value()
190 .ok_or_else(|| vortex_err!("expected non-null variant row"))?;
191 assert_eq!(value.as_primitive().typed_value::<i32>(), Some(*expected));
192 }
193 None => assert!(variant.is_null()),
194 }
195 }
196
197 Ok(())
198 }
199
200 #[test]
201 fn try_new_exposes_core_storage_without_shredded() -> VortexResult<()> {
202 let core_storage = core_storage(2);
203
204 let variant = VariantArray::try_new(core_storage.clone(), None)?;
205
206 assert_eq!(variant.dtype(), core_storage.dtype());
207 assert_eq!(variant.len(), 2);
208 assert_eq!(variant.core_storage().dtype(), core_storage.dtype());
209 assert!(variant.shredded().is_none());
210 assert_variant_core_rows(&variant, &[Some(1), Some(1)])?;
211
212 Ok(())
213 }
214
215 #[test]
216 fn try_new_exposes_core_storage_and_shredded() -> VortexResult<()> {
217 let core_storage = core_storage(3);
218 let shredded = buffer![10i32, 20, 30].into_array();
219
220 let variant = VariantArray::try_new(core_storage.clone(), Some(shredded.clone()))?;
221
222 assert_eq!(variant.dtype(), &DType::Variant(Nullability::NonNullable));
223 assert_eq!(variant.len(), 3);
224 assert_eq!(variant.core_storage().dtype(), core_storage.dtype());
225 assert_eq!(variant.core_storage().len(), core_storage.len());
226 assert_eq!(
227 variant.shredded().map(|child| child.dtype()),
228 Some(shredded.dtype())
229 );
230 assert_eq!(
231 variant.shredded().map(|child| child.len()),
232 Some(shredded.len())
233 );
234 assert_eq!(variant.as_ref().slot_name(0), "core_storage");
235 assert_eq!(variant.as_ref().slot_name(1), "shredded");
236 assert_variant_rows(&variant, &[Some(1); 3], &[Some(10), Some(20), Some(30)])?;
237
238 Ok(())
239 }
240
241 #[test]
242 fn try_new_rejects_non_variant_core_storage() {
243 let core_storage = PrimitiveArray::from_iter([1i32, 2, 3]).into_array();
244
245 assert!(VariantArray::try_new(core_storage, None).is_err());
246 }
247
248 #[test]
249 fn try_new_rejects_shredded_length_mismatch() {
250 let core_storage = core_storage(3);
251 let shredded = buffer![10i32, 20].into_array();
252
253 assert!(VariantArray::try_new(core_storage, Some(shredded)).is_err());
254 }
255
256 #[test]
257 fn scalar_at_merges_shredded_with_core_storage() -> VortexResult<()> {
258 let dtype = DType::Variant(Nullability::Nullable);
259 let core_chunks = [Some(1i32), None, Some(3)]
260 .into_iter()
261 .map(|value| {
262 let scalar = match value {
263 Some(value) => {
264 Scalar::variant(Scalar::primitive(value, Nullability::NonNullable))
265 .cast(&dtype)?
266 }
267 None => Scalar::null(dtype.clone()),
268 };
269 Ok(ConstantArray::new(scalar, 1).into_array())
270 })
271 .collect::<VortexResult<Vec<_>>>()?;
272 let core_storage = ChunkedArray::try_new(core_chunks, dtype)?.into_array();
273 let shredded = PrimitiveArray::from_option_iter([Some(10i32), Some(20), None]).into_array();
274 let variant = VariantArray::try_new(core_storage, Some(shredded))?;
275
276 let mut ctx = SESSION.create_execution_ctx();
277 for (idx, expected) in [Some(10i32), None, Some(3)].into_iter().enumerate() {
278 let scalar = variant.execute_scalar(idx, &mut ctx)?;
279 let variant = scalar.as_variant();
280 match expected {
281 Some(expected) => {
282 let value = variant
283 .value()
284 .ok_or_else(|| vortex_err!("expected non-null variant row"))?;
285 assert_eq!(value.as_primitive().typed_value::<i32>(), Some(expected));
286 }
287 None => assert!(variant.is_null()),
288 }
289 }
290
291 Ok(())
292 }
293
294 #[test]
295 fn slice_preserves_core_storage_and_shredded_rows() -> VortexResult<()> {
296 let variant = variant_with_shredded(0..5, 10..15)?;
297
298 let sliced = execute_variant(variant.into_array().slice(1..4)?)?;
299
300 assert_variant_rows(
301 &sliced,
302 &[Some(1), Some(2), Some(3)],
303 &[Some(11), Some(12), Some(13)],
304 )
305 }
306
307 #[test]
308 fn filter_preserves_core_storage_and_shredded_rows() -> VortexResult<()> {
309 let variant = variant_with_shredded(0..5, 10..15)?;
310
311 let filtered = execute_variant(
312 variant
313 .into_array()
314 .filter(Mask::from_iter([true, false, true, false, true]))?,
315 )?;
316
317 assert_variant_rows(
318 &filtered,
319 &[Some(0), Some(2), Some(4)],
320 &[Some(10), Some(12), Some(14)],
321 )
322 }
323
324 #[test]
325 fn take_preserves_core_storage_and_shredded_rows() -> VortexResult<()> {
326 let variant = variant_with_shredded(0..5, 10..15)?;
327
328 let taken = execute_variant(
329 variant
330 .into_array()
331 .take(buffer![4u64, 1, 3].into_array())?,
332 )?;
333
334 assert_variant_rows(
335 &taken,
336 &[Some(4), Some(1), Some(3)],
337 &[Some(14), Some(11), Some(13)],
338 )
339 }
340
341 #[test]
342 fn mask_preserves_core_storage_and_shredded_rows() -> VortexResult<()> {
343 let variant = variant_with_shredded(0..5, 10..15)?;
344 let mask = BoolArray::from_iter([true, false, true, false, true]).into_array();
345
346 let masked = execute_variant(variant.into_array().mask(mask)?)?;
347
348 assert_variant_rows(
349 &masked,
350 &[Some(0), None, Some(2), None, Some(4)],
351 &[Some(10), None, Some(12), None, Some(14)],
352 )
353 }
354
355 #[test]
356 fn mask_preserves_chunked_core_storage_validity() -> VortexResult<()> {
357 let dtype = DType::Variant(Nullability::Nullable);
358 let core_chunks = [Some(1i32), None, Some(3), Some(4)]
359 .into_iter()
360 .map(|value| {
361 let scalar = match value {
362 Some(value) => {
363 Scalar::variant(Scalar::primitive(value, Nullability::NonNullable))
364 .cast(&dtype)?
365 }
366 None => Scalar::null(dtype.clone()),
367 };
368 Ok(ConstantArray::new(scalar, 1).into_array())
369 })
370 .collect::<VortexResult<Vec<_>>>()?;
371 let core_storage = ChunkedArray::try_new(core_chunks, dtype)?.into_array();
372 let variant = VariantArray::try_new(core_storage, None)?;
373 let mask = BoolArray::from_iter([true, true, false, true]).into_array();
374
375 let masked = execute_variant(variant.into_array().mask(mask)?)?;
376
377 assert_variant_core_rows(&masked, &[Some(1), None, None, Some(4)])
378 }
379
380 #[test]
381 fn variant_get_keeps_valid_shredded_rows_for_matching_dtype() -> VortexResult<()> {
382 let mut ctx = SESSION.create_execution_ctx();
383 let core_storage = row_storage([1, 2, 3])?;
384 let shredded = StructArray::try_from_iter([(
385 "a",
386 PrimitiveArray::from_iter([10i32, 20, 30]).into_array(),
387 )])?;
388 let variant = VariantArray::try_new(core_storage, Some(shredded.into_array()))?;
389 let expr = variant_get(
390 root(),
391 VariantPath::field("a"),
392 Some(DType::Primitive(PType::I32, Nullability::NonNullable)),
393 );
394
395 let result = variant.into_array().apply(&expr)?;
396
397 assert_arrays_eq!(
398 result,
399 PrimitiveArray::from_option_iter([Some(10i32), Some(20), Some(30)]),
400 &mut ctx
401 );
402 Ok(())
403 }
404
405 #[test]
406 fn variant_get_treats_value_and_typed_value_as_logical_field_names() -> VortexResult<()> {
407 let mut ctx = SESSION.create_execution_ctx();
408 let core_storage = row_storage([1, 2, 3])?;
409 let shredded = StructArray::try_from_iter([
410 (
411 "value",
412 PrimitiveArray::from_iter([10i32, 20, 30]).into_array(),
413 ),
414 (
415 "typed_value",
416 PrimitiveArray::from_iter([40i32, 50, 60]).into_array(),
417 ),
418 ])?;
419 let variant = VariantArray::try_new(core_storage, Some(shredded.into_array()))?;
420
421 let value_expr = variant_get(
422 root(),
423 VariantPath::field("value"),
424 Some(DType::Primitive(PType::I32, Nullability::NonNullable)),
425 );
426 let value_result = variant.clone().into_array().apply(&value_expr)?;
427 assert_arrays_eq!(
428 value_result,
429 PrimitiveArray::from_option_iter([Some(10i32), Some(20), Some(30)]),
430 &mut ctx
431 );
432
433 let typed_value_expr = variant_get(
434 root(),
435 VariantPath::field("typed_value"),
436 Some(DType::Primitive(PType::I32, Nullability::NonNullable)),
437 );
438 let typed_value_result = variant.into_array().apply(&typed_value_expr)?;
439 assert_arrays_eq!(
440 typed_value_result,
441 PrimitiveArray::from_option_iter([Some(40i32), Some(50), Some(60)]),
442 &mut ctx
443 );
444 Ok(())
445 }
446}