1mod array;
43mod execute;
44
45use vortex_error::VortexExpect;
46use vortex_error::VortexResult;
47use vortex_error::vortex_bail;
48use vortex_error::vortex_ensure;
49use vortex_error::vortex_ensure_eq;
50use vortex_error::vortex_panic;
51use vortex_session::VortexSession;
52use vortex_session::registry::CachedId;
53
54pub use self::array::InterleaveArray;
55pub use self::array::InterleaveArrayExt;
56pub use self::array::InterleaveData;
57use self::array::InterleaveSlots;
58use crate::ArrayRef;
59use crate::ExecutionCtx;
60use crate::IntoArray;
61use crate::array::Array;
62use crate::array::ArrayId;
63use crate::array::ArrayParts;
64use crate::array::ArraySlots;
65use crate::array::ArrayView;
66use crate::array::OperationsVTable;
67use crate::array::VTable;
68use crate::array::ValidityVTable;
69use crate::array::with_empty_buffers;
70use crate::arrays::ConstantArray;
71use crate::buffer::BufferHandle;
72use crate::dtype::DType;
73use crate::dtype::Nullability;
74use crate::executor::ExecutionResult;
75use crate::scalar::Scalar;
76use crate::serde::ArrayChildren;
77use crate::validity::Validity;
78
79#[derive(Clone, Debug)]
81pub struct Interleave;
82
83impl Interleave {
84 fn check(
90 values: &[ArrayRef],
91 array_indices: &ArrayRef,
92 row_indices: &ArrayRef,
93 ) -> VortexResult<DType> {
94 vortex_ensure!(
95 values.len() >= 2,
96 "interleave requires at least 2 values, got {}",
97 values.len()
98 );
99
100 for (name, selector) in [
103 ("array_indices", array_indices),
104 ("row_indices", row_indices),
105 ] {
106 match selector.dtype() {
107 DType::Primitive(ptype, nullability) if ptype.is_unsigned_int() => {
108 vortex_ensure!(
109 !nullability.is_nullable(),
110 "interleave {name} must be non-nullable, got {}",
111 selector.dtype()
112 );
113 }
114 other => vortex_bail!(
115 "interleave {name} must be a non-nullable unsigned integer, got {other}"
116 ),
117 }
118 }
119
120 vortex_ensure_eq!(
121 array_indices.len(),
122 row_indices.len(),
123 "interleave selectors must have equal length",
124 );
125
126 let base_dtype = values[0].dtype();
127 let mut nullability = Nullability::NonNullable;
128 for value in values {
129 vortex_ensure!(
130 value.dtype().eq_ignore_nullability(base_dtype),
131 "interleave values must share a dtype up to nullability: {} vs {}",
132 base_dtype,
133 value.dtype()
134 );
135 nullability |= value.dtype().nullability();
136 }
137
138 Ok(base_dtype.with_nullability(nullability))
139 }
140}
141
142impl VTable for Interleave {
143 type TypedArrayData = InterleaveData;
144 type OperationsVTable = Self;
145 type ValidityVTable = Self;
146
147 fn id(&self) -> ArrayId {
148 static ID: CachedId = CachedId::new("vortex.interleave");
149 *ID
150 }
151
152 fn validate(
153 &self,
154 data: &Self::TypedArrayData,
155 dtype: &DType,
156 len: usize,
157 slots: &[Option<ArrayRef>],
158 ) -> VortexResult<()> {
159 vortex_ensure_eq!(
160 slots.len(),
161 data.num_values + InterleaveSlots::FIXED_COUNT,
162 "InterleaveArray expects values + array_indices + row_indices slots",
163 );
164 vortex_ensure!(
165 slots.iter().all(|s| s.is_some()),
166 "InterleaveArray slots must all be present"
167 );
168
169 let array_indices = slots[InterleaveSlots::ARRAY_INDICES]
170 .clone()
171 .vortex_expect("validated array_indices slot");
172 let row_indices = slots[InterleaveSlots::ROW_INDICES]
173 .clone()
174 .vortex_expect("validated row_indices slot");
175 let values: Vec<ArrayRef> = slots[InterleaveSlots::VALUES_OFFSET..]
176 .iter()
177 .map(|s| s.clone().vortex_expect("validated value slot"))
178 .collect();
179
180 let expected_dtype = Interleave::check(&values, &array_indices, &row_indices)?;
183 vortex_ensure_eq!(
184 dtype,
185 &expected_dtype,
186 "InterleaveArray dtype does not match the dtype implied by its children",
187 );
188 vortex_ensure_eq!(
189 len,
190 array_indices.len(),
191 "InterleaveArray length does not match array_indices length",
192 );
193 Ok(())
194 }
195
196 fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
197 0
198 }
199
200 fn buffer(_array: ArrayView<'_, Self>, _idx: usize) -> BufferHandle {
201 vortex_panic!("InterleaveArray has no buffers")
202 }
203
204 fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option<String> {
205 None
206 }
207
208 fn with_buffers(
209 &self,
210 array: ArrayView<'_, Self>,
211 buffers: &[BufferHandle],
212 ) -> VortexResult<ArrayParts<Self>> {
213 with_empty_buffers(self, array, buffers)
214 }
215
216 fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
217 InterleaveSlots::slot_name(idx)
218 }
219
220 fn serialize(
221 _array: ArrayView<'_, Self>,
222 _session: &VortexSession,
223 ) -> VortexResult<Option<Vec<u8>>> {
224 vortex_bail!("Interleave array is not serializable")
225 }
226
227 fn deserialize(
228 &self,
229 _dtype: &DType,
230 _len: usize,
231 _metadata: &[u8],
232 _buffers: &[BufferHandle],
233 _children: &dyn ArrayChildren,
234 _session: &VortexSession,
235 ) -> VortexResult<ArrayParts<Self>> {
236 vortex_bail!("Interleave array is not serializable")
237 }
238
239 fn execute(array: Array<Self>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
240 execute::execute(array, ctx)
241 }
242}
243
244impl OperationsVTable<Interleave> for Interleave {
245 type ProbeState = ();
246
247 fn scalar_at(
248 array: ArrayView<'_, Interleave>,
249 index: usize,
250 ctx: &mut ExecutionCtx,
251 ) -> VortexResult<Scalar> {
252 let branch_idx = array
255 .array_indices()
256 .execute_scalar(index, ctx)?
257 .as_primitive()
258 .as_::<usize>()
259 .vortex_expect("interleave array_indices is non-nullable");
260 let row = array
261 .row_indices()
262 .execute_scalar(index, ctx)?
263 .as_primitive()
264 .as_::<usize>()
265 .vortex_expect("interleave row_indices is non-nullable");
266
267 let scalar = array.value(branch_idx).execute_scalar(row, ctx)?;
268 Ok(if array.as_ref().dtype().is_nullable() {
270 scalar.into_nullable()
271 } else {
272 scalar
273 })
274 }
275}
276
277impl ValidityVTable<Interleave> for Interleave {
278 fn validity(array: ArrayView<'_, Interleave>) -> VortexResult<Validity> {
279 if !array.as_ref().dtype().is_nullable() {
280 return Ok(Validity::NonNullable);
281 }
282 let num_values = array.num_values();
283 let mut slots: ArraySlots =
284 ArraySlots::with_capacity(num_values + InterleaveSlots::FIXED_COUNT);
285 slots.push(Some(array.array_indices().clone()));
286 slots.push(Some(array.row_indices().clone()));
287 for i in 0..num_values {
288 slots.push(Some(value_validity_array(array.value(i))?));
289 }
290 let interleaved = unsafe {
293 InterleaveArray::new_unchecked_slots(slots, DType::Bool(Nullability::NonNullable))
294 };
295 Ok(Validity::Array(interleaved.into_array()))
296 }
297}
298
299fn value_validity_array(value: &ArrayRef) -> VortexResult<ArrayRef> {
302 Ok(match value.validity()? {
303 Validity::NonNullable | Validity::AllValid => {
304 ConstantArray::new(true, value.len()).into_array()
305 }
306 Validity::AllInvalid => ConstantArray::new(false, value.len()).into_array(),
307 Validity::Array(array) => array,
308 })
309}
310
311#[cfg(test)]
312mod tests {
313 use rstest::rstest;
314 use vortex_error::VortexResult;
315
316 use super::*;
317 use crate::AnyColumnar;
318 use crate::Canonical;
319 use crate::VortexSessionExecute;
320 use crate::array_session;
321 use crate::arrays::Bool;
322 use crate::arrays::BoolArray;
323 use crate::arrays::Primitive;
324 use crate::arrays::PrimitiveArray;
325 use crate::arrays::SliceArray;
326 use crate::assert_arrays_eq;
327 use crate::dtype::PType;
328
329 #[derive(Clone, Copy)]
330 enum LazyChild {
331 ArraySelector,
332 RowSelector,
333 ThirdValue,
334 }
335
336 #[rstest]
337 #[case::array_selector(LazyChild::ArraySelector)]
338 #[case::row_selector(LazyChild::RowSelector)]
339 #[case::third_value(LazyChild::ThirdValue)]
340 fn executes_lazy_children(
341 #[case] lazy_child: LazyChild,
342 #[values(false, true)] boolean: bool,
343 ) -> VortexResult<()> {
344 let (mut values, expected) = if boolean {
345 (
346 vec![
347 BoolArray::from_iter([true, false]).into_array(),
348 BoolArray::from_iter([false, true]).into_array(),
349 BoolArray::from_iter([Some(false), None]).into_array(),
350 ],
351 BoolArray::from_iter([None, Some(true), Some(true), Some(false), Some(false)])
352 .into_array(),
353 )
354 } else {
355 (
356 vec![
357 PrimitiveArray::from_iter([10i32, 11]).into_array(),
358 PrimitiveArray::from_iter([20i32, 21]).into_array(),
359 PrimitiveArray::from_option_iter([Some(30i32), None]).into_array(),
360 ],
361 PrimitiveArray::from_option_iter([None, Some(10i32), Some(21), Some(30), Some(11)])
362 .into_array(),
363 )
364 };
365 let mut array_indices = PrimitiveArray::from_iter([2u8, 0, 1, 2, 0]).into_array();
366 let mut row_indices = PrimitiveArray::from_iter([1u32, 0, 1, 0, 1]).into_array();
367 let child = match lazy_child {
368 LazyChild::ArraySelector => &mut array_indices,
369 LazyChild::RowSelector => &mut row_indices,
370 LazyChild::ThirdValue => &mut values[2],
371 };
372 *child = SliceArray::try_new(child.clone(), 0..child.len())?.into_array();
373
374 assert_eq!(
375 array_indices.is::<Primitive>(),
376 !matches!(lazy_child, LazyChild::ArraySelector)
377 );
378 assert_eq!(
379 row_indices.is::<Primitive>(),
380 !matches!(lazy_child, LazyChild::RowSelector)
381 );
382 for (i, value) in values.iter().enumerate() {
383 let matches = if boolean {
384 value.is::<Bool>()
385 } else {
386 value.is::<AnyColumnar>()
387 };
388 assert_eq!(
389 matches,
390 !(matches!(lazy_child, LazyChild::ThirdValue) && i == 2)
391 );
392 }
393
394 let interleaved =
395 InterleaveArray::try_new(values, array_indices, row_indices)?.into_array();
396 let mut ctx = array_session().create_execution_ctx();
397 assert_arrays_eq!(interleaved, expected, &mut ctx);
398 Ok(())
399 }
400
401 fn interleave_reference(
408 values: &[ArrayRef],
409 array_indices: &ArrayRef,
410 row_indices: &ArrayRef,
411 ctx: &mut ExecutionCtx,
412 ) -> VortexResult<ArrayRef> {
413 let len = array_indices.len();
414 let nullable = values.iter().any(|v| v.dtype().is_nullable());
415 let mut out: Vec<Option<bool>> = Vec::with_capacity(len);
416
417 for i in 0..len {
418 let j = array_indices
419 .execute_scalar(i, ctx)?
420 .as_primitive()
421 .as_::<usize>()
422 .vortex_expect("array_indices is non-nullable");
423 let row = row_indices
424 .execute_scalar(i, ctx)?
425 .as_primitive()
426 .as_::<usize>()
427 .vortex_expect("row_indices is non-nullable");
428 out.push(values[j].execute_scalar(row, ctx)?.as_bool().value());
429 }
430
431 Ok(if nullable {
432 BoolArray::from_iter(out).into_array()
433 } else {
434 BoolArray::from_iter(
435 out.into_iter()
436 .map(|v| v.vortex_expect("non-nullable value produced a null")),
437 )
438 .into_array()
439 })
440 }
441
442 fn build(
445 branches: &[&[Option<bool>]],
446 indices: &[(usize, usize)],
447 ) -> (Vec<ArrayRef>, ArrayRef, ArrayRef) {
448 let nullable = branches.iter().flat_map(|b| b.iter()).any(Option::is_none);
449 let to_value = |vals: &[Option<bool>]| -> ArrayRef {
450 if nullable {
451 BoolArray::from_iter(vals.iter().copied()).into_array()
452 } else {
453 BoolArray::from_iter(
454 vals.iter()
455 .map(|v| v.vortex_expect("non-nullable value produced a null")),
456 )
457 .into_array()
458 }
459 };
460
461 let values = branches.iter().map(|b| to_value(b)).collect();
462 let array_indices = PrimitiveArray::from_iter(
463 indices
464 .iter()
465 .map(|&(a, _)| u32::try_from(a).vortex_expect("array index fits in u32")),
466 )
467 .into_array();
468 let row_indices = PrimitiveArray::from_iter(
469 indices
470 .iter()
471 .map(|&(_, r)| u32::try_from(r).vortex_expect("row index fits in u32")),
472 )
473 .into_array();
474 (values, array_indices, row_indices)
475 }
476
477 fn check(branches: &[&[Option<bool>]], indices: &[(usize, usize)]) -> VortexResult<()> {
481 let (values, array_indices, row_indices) = build(branches, indices);
482
483 let interleaved =
484 InterleaveArray::try_new(values.clone(), array_indices.clone(), row_indices.clone())?
485 .into_array();
486
487 let mut ctx = array_session().create_execution_ctx();
488 let reference = interleave_reference(&values, &array_indices, &row_indices, &mut ctx)?;
489
490 assert_arrays_eq!(interleaved, reference, &mut ctx);
491 Ok(())
492 }
493
494 #[test]
495 fn interleave_reorders_and_repeats() -> VortexResult<()> {
496 check(
498 &[&[Some(true), Some(false)], &[Some(false), Some(true)]],
499 &[(0, 1), (1, 0), (0, 0), (1, 1), (0, 0)],
500 )
501 }
502
503 #[test]
504 fn interleave_skips_rows() -> VortexResult<()> {
505 check(
507 &[
508 &[Some(true), Some(false), Some(true)],
509 &[Some(false), Some(true)],
510 ],
511 &[(0, 0), (1, 1), (0, 2)],
512 )
513 }
514
515 #[test]
516 fn interleave_binary_spans_word_boundary() -> VortexResult<()> {
517 let branch0: Vec<Option<bool>> = (0..100).map(|i| Some(i % 3 == 0)).collect();
520 let branch1: Vec<Option<bool>> = (0..100).map(|i| Some(i % 5 == 0)).collect();
521 let indices: Vec<(usize, usize)> = (0..200).map(|i| (i % 2, (i * 7) % 100)).collect();
522 check(&[&branch0, &branch1], &indices)
523 }
524
525 #[test]
526 fn interleave_binary_nullable_spans_word_boundary() -> VortexResult<()> {
527 let branch0: Vec<Option<bool>> = (0..70)
529 .map(|i| (i % 4 != 0).then_some(i % 2 == 0))
530 .collect();
531 let branch1: Vec<Option<bool>> = (0..70)
532 .map(|i| (i % 3 != 0).then_some(i % 2 == 1))
533 .collect();
534 let indices: Vec<(usize, usize)> = (0..150).map(|i| (i % 2, (i * 11) % 70)).collect();
535 check(&[&branch0, &branch1], &indices)
536 }
537
538 #[test]
539 fn interleave_three_values() -> VortexResult<()> {
540 check(
542 &[
543 &[Some(true), Some(false)],
544 &[Some(false)],
545 &[Some(true), Some(true), Some(false)],
546 ],
547 &[(2, 1), (0, 0), (1, 0), (2, 2), (0, 1), (2, 0)],
548 )
549 }
550
551 #[test]
552 fn interleave_only_one_branch() -> VortexResult<()> {
553 check(
554 &[&[Some(true), Some(false), Some(true)], &[Some(false)]],
555 &[(0, 2), (0, 0), (0, 1)],
556 )
557 }
558
559 #[test]
560 fn interleave_nullable_with_nulls_in_values() -> VortexResult<()> {
561 check(
562 &[&[None, Some(true), None], &[Some(false), None]],
563 &[(1, 1), (0, 0), (1, 0), (0, 2), (0, 1)],
564 )
565 }
566
567 #[test]
568 fn interleave_empty() -> VortexResult<()> {
569 check(&[&[Some(true)], &[Some(false)]], &[])
570 }
571
572 #[test]
573 fn rejects_boolean_array_indices() {
574 let value = BoolArray::from_iter([true, false]).into_array();
575 let array_indices = BoolArray::from_iter([true, false]).into_array();
576 let row_indices = PrimitiveArray::from_iter([0u32, 1]).into_array();
577 let err = InterleaveArray::try_new(vec![value.clone(), value], array_indices, row_indices)
578 .err()
579 .vortex_expect("expected interleave to reject a boolean array_indices");
580 assert!(err.to_string().contains("unsigned integer"), "{err}");
581 }
582
583 #[test]
584 fn rejects_signed_integer_array_indices() {
585 let value = BoolArray::from_iter([true]).into_array();
586 let array_indices = PrimitiveArray::from_iter([0i32, 1]).into_array();
587 let row_indices = PrimitiveArray::from_iter([0u32, 0]).into_array();
588 let err = InterleaveArray::try_new(vec![value.clone(), value], array_indices, row_indices)
589 .err()
590 .vortex_expect("expected interleave to reject a signed integer array_indices");
591 assert!(err.to_string().contains("unsigned integer"), "{err}");
592 }
593
594 #[test]
595 fn rejects_nullable_row_indices() {
596 let value = BoolArray::from_iter([true, false]).into_array();
597 let array_indices = PrimitiveArray::from_iter([0u32, 1]).into_array();
598 let row_indices = PrimitiveArray::from_option_iter([Some(0u32), Some(1)]).into_array();
599 let err = InterleaveArray::try_new(vec![value.clone(), value], array_indices, row_indices)
600 .err()
601 .vortex_expect("expected interleave to reject nullable row_indices");
602 assert!(err.to_string().contains("non-nullable"), "{err}");
603 }
604
605 #[test]
606 fn rejects_mismatched_selector_lengths() {
607 let value = BoolArray::from_iter([true, false]).into_array();
608 let array_indices = PrimitiveArray::from_iter([0u32, 1]).into_array();
609 let row_indices = PrimitiveArray::from_iter([0u32]).into_array();
610 let err = InterleaveArray::try_new(vec![value.clone(), value], array_indices, row_indices)
611 .err()
612 .vortex_expect("expected interleave to reject mismatched selector lengths");
613 assert!(err.to_string().contains("equal length"), "{err}");
614 }
615
616 #[test]
617 fn execute_rejects_out_of_bounds_array_index() -> VortexResult<()> {
618 let value = BoolArray::from_iter([true]).into_array();
619 let array_indices = PrimitiveArray::from_iter([2u32]).into_array();
620 let row_indices = PrimitiveArray::from_iter([0u32]).into_array();
621 let interleaved =
622 InterleaveArray::try_new(vec![value.clone(), value], array_indices, row_indices)?
623 .into_array();
624
625 let mut ctx = array_session().create_execution_ctx();
626 let err = interleaved
627 .execute::<Canonical>(&mut ctx)
628 .err()
629 .vortex_expect("expected execution to reject out-of-bounds array index");
630 assert!(
631 err.to_string().contains("array index out of bounds"),
632 "{err}"
633 );
634 Ok(())
635 }
636
637 #[test]
638 fn execute_rejects_out_of_bounds_row_index() -> VortexResult<()> {
639 let value = BoolArray::from_iter([true]).into_array();
640 let array_indices = PrimitiveArray::from_iter([0u32]).into_array();
641 let row_indices = PrimitiveArray::from_iter([1u32]).into_array();
642 let interleaved =
643 InterleaveArray::try_new(vec![value.clone(), value], array_indices, row_indices)?
644 .into_array();
645
646 let mut ctx = array_session().create_execution_ctx();
647 let err = interleaved
648 .execute::<Canonical>(&mut ctx)
649 .err()
650 .vortex_expect("expected execution to reject out-of-bounds row index");
651 assert!(err.to_string().contains("row index out of bounds"), "{err}");
652 Ok(())
653 }
654
655 #[test]
656 fn executes_primitive_values() -> VortexResult<()> {
657 let v0 = PrimitiveArray::from_iter([1.0f64, 2.0]).into_array();
658 let v1 = PrimitiveArray::from_option_iter([Some(10.0f64), None]).into_array();
659 let array_indices = PrimitiveArray::from_iter([0u8, 1, 0, 1]).into_array();
660 let row_indices = PrimitiveArray::from_iter([0u32, 0, 1, 1]).into_array();
661 let interleaved =
662 InterleaveArray::try_new(vec![v0, v1], array_indices, row_indices)?.into_array();
663 let expected =
664 PrimitiveArray::from_option_iter([Some(1.0f64), Some(10.0), Some(2.0), None])
665 .into_array();
666 let mut ctx = array_session().create_execution_ctx();
667 assert_arrays_eq!(interleaved, expected, &mut ctx);
668 Ok(())
669 }
670
671 #[test]
672 fn executes_primitive_constant_values() -> VortexResult<()> {
673 let constant = ConstantArray::new(1.0f64, 2).into_array();
674 let column = PrimitiveArray::from_iter([10.0f64, 20.0]).into_array();
675 let array_indices = PrimitiveArray::from_iter([0u8, 1, 0, 1]).into_array();
676 let row_indices = PrimitiveArray::from_iter([0u32, 0, 1, 1]).into_array();
677 let interleaved =
678 InterleaveArray::try_new(vec![constant, column], array_indices, row_indices)?
679 .into_array();
680 let expected = PrimitiveArray::from_iter([1.0f64, 10.0, 1.0, 20.0]).into_array();
681 let mut ctx = array_session().create_execution_ctx();
682
683 assert_arrays_eq!(interleaved, expected, &mut ctx);
684 Ok(())
685 }
686
687 #[test]
688 fn executes_null_primitive_constant_values() -> VortexResult<()> {
689 let constant = ConstantArray::new(
690 Scalar::null(DType::Primitive(PType::F64, Nullability::Nullable)),
691 2,
692 )
693 .into_array();
694 let column = PrimitiveArray::from_iter([10.0f64, 20.0]).into_array();
695 let array_indices = PrimitiveArray::from_iter([0u8, 1, 0, 1]).into_array();
696 let row_indices = PrimitiveArray::from_iter([0u32, 0, 1, 1]).into_array();
697 let interleaved =
698 InterleaveArray::try_new(vec![constant, column], array_indices, row_indices)?
699 .into_array();
700 let expected =
701 PrimitiveArray::from_option_iter([None, Some(10.0f64), None, Some(20.0)]).into_array();
702 let mut ctx = array_session().create_execution_ctx();
703
704 assert_arrays_eq!(interleaved, expected, &mut ctx);
705 Ok(())
706 }
707}