1use std::any::TypeId;
7use std::sync::Arc;
8
9use vortex_buffer::BitBuffer;
10use vortex_buffer::Buffer;
11use vortex_error::VortexExpect;
12use vortex_error::VortexResult;
13use vortex_error::vortex_ensure;
14use vortex_error::vortex_panic;
15
16use crate::ArrayRef;
17use crate::ArraySlots;
18use crate::Executable;
19use crate::ExecutionCtx;
20use crate::IntoArray;
21use crate::array::ArrayView;
22use crate::array::VTable;
23use crate::array::child_to_validity;
24use crate::arrays::Bool;
25use crate::arrays::BoolArray;
26use crate::arrays::Decimal;
27use crate::arrays::DecimalArray;
28use crate::arrays::Extension;
29use crate::arrays::ExtensionArray;
30use crate::arrays::FixedSizeList;
31use crate::arrays::FixedSizeListArray;
32use crate::arrays::ListView;
33use crate::arrays::ListViewArray;
34use crate::arrays::Map;
35use crate::arrays::MapArray;
36use crate::arrays::Null;
37use crate::arrays::NullArray;
38use crate::arrays::Primitive;
39use crate::arrays::PrimitiveArray;
40use crate::arrays::Struct;
41use crate::arrays::StructArray;
42use crate::arrays::Union;
43use crate::arrays::UnionArray;
44use crate::arrays::VarBinView;
45use crate::arrays::VarBinViewArray;
46use crate::arrays::Variant;
47use crate::arrays::VariantArray;
48use crate::arrays::bool::BoolDataParts;
49use crate::arrays::decimal::DecimalDataParts;
50use crate::arrays::extension::ExtensionArrayExt;
51use crate::arrays::fixed_size_list::FixedSizeListArrayExt;
52use crate::arrays::listview::ListViewDataParts;
53use crate::arrays::listview::ListViewRebuildMode;
54use crate::arrays::map::MapArrayExt;
55use crate::arrays::map::MapArraySlotsExt;
56use crate::arrays::primitive::PrimitiveDataParts;
57use crate::arrays::struct_::StructDataParts;
58use crate::arrays::union::UnionDataParts;
59use crate::arrays::varbinview::VarBinViewDataParts;
60use crate::arrays::variant::VariantArraySlotsExt;
61use crate::dtype::DType;
62use crate::dtype::NativePType;
63use crate::dtype::Nullability;
64use crate::dtype::PType;
65use crate::match_each_decimal_value_type;
66use crate::match_each_native_ptype;
67use crate::matcher::Matcher;
68use crate::validity::Validity;
69
70#[derive(Debug, Clone)]
134pub enum Canonical {
135 Null(NullArray),
136 Bool(BoolArray),
137 Primitive(PrimitiveArray),
138 Decimal(DecimalArray),
139 VarBinView(VarBinViewArray),
140 List(ListViewArray),
141 Map(MapArray),
142 FixedSizeList(FixedSizeListArray),
143 Struct(StructArray),
144 Union(UnionArray),
145 Extension(ExtensionArray),
147 Variant(VariantArray),
149}
150
151macro_rules! match_each_canonical {
153 ($self:expr, | $ident:ident | $eval:expr) => {{
154 match $self {
155 Canonical::Null($ident) => $eval,
156 Canonical::Bool($ident) => $eval,
157 Canonical::Primitive($ident) => $eval,
158 Canonical::Decimal($ident) => $eval,
159 Canonical::VarBinView($ident) => $eval,
160 Canonical::List($ident) => $eval,
161 Canonical::Map($ident) => $eval,
162 Canonical::FixedSizeList($ident) => $eval,
163 Canonical::Struct($ident) => $eval,
164 Canonical::Union($ident) => $eval,
165 Canonical::Variant($ident) => $eval,
166 Canonical::Extension($ident) => $eval,
167 }
168 }};
169}
170
171impl Canonical {
172 pub fn empty(dtype: &DType) -> Canonical {
174 match dtype {
175 DType::Null => Canonical::Null(NullArray::new(0)),
176 DType::Bool(n) => Canonical::Bool(unsafe {
177 BoolArray::new_unchecked(BitBuffer::empty(), Validity::from(n))
178 }),
179 DType::Primitive(ptype, n) => {
180 match_each_native_ptype!(ptype, |P| {
181 Canonical::Primitive(unsafe {
182 PrimitiveArray::new_unchecked(Buffer::<P>::empty(), Validity::from(n))
183 })
184 })
185 }
186 DType::Decimal(decimal_type, n) => {
187 match_each_decimal_value_type!(decimal_type.canonical_values_type(), |D| {
188 Canonical::Decimal(unsafe {
189 DecimalArray::new_unchecked::<D>(
190 Buffer::empty(),
191 *decimal_type,
192 Validity::from(n),
193 )
194 })
195 })
196 }
197 DType::Utf8(n) => Canonical::VarBinView(unsafe {
198 VarBinViewArray::new_unchecked(
199 Buffer::empty(),
200 Arc::new([]),
201 dtype.clone(),
202 Validity::from(n),
203 )
204 }),
205 DType::Binary(n) => Canonical::VarBinView(unsafe {
206 VarBinViewArray::new_unchecked(
207 Buffer::empty(),
208 Arc::new([]),
209 dtype.clone(),
210 Validity::from(n),
211 )
212 }),
213 DType::List(dtype, n) => Canonical::List(unsafe {
214 ListViewArray::new_unchecked(
215 Canonical::empty(dtype).into_array(),
216 Canonical::empty(&DType::Primitive(PType::U8, Nullability::NonNullable))
217 .into_array(),
218 Canonical::empty(&DType::Primitive(PType::U8, Nullability::NonNullable))
219 .into_array(),
220 Validity::from(n),
221 )
222 .with_zero_copy_to_list(true)
224 }),
225 DType::Map(map_dtype, nullability) => Canonical::Map(MapArray::new(
226 map_dtype.clone(),
227 Canonical::empty(&DType::List(
228 Arc::new(map_dtype.entries_dtype()),
229 *nullability,
230 ))
231 .into_listview(),
232 )),
233 DType::FixedSizeList(elem_dtype, list_size, null) => Canonical::FixedSizeList(unsafe {
234 FixedSizeListArray::new_unchecked(
235 Canonical::empty(elem_dtype).into_array(),
236 *list_size,
237 Validity::from(null),
238 0,
239 )
240 }),
241 DType::Struct(struct_dtype, n) => Canonical::Struct(unsafe {
242 StructArray::new_unchecked(
243 struct_dtype
244 .fields()
245 .map(|f| Canonical::empty(&f).into_array())
246 .collect::<Vec<_>>(),
247 struct_dtype.clone(),
248 0,
249 Validity::from(n),
250 )
251 }),
252 DType::Union(variants, nullability) => {
253 Canonical::Union(UnionArray::empty(variants.clone(), *nullability))
254 }
255 DType::Variant(_) => {
256 vortex_panic!("Canonical empty is not supported for Variant")
257 }
258 DType::Extension(ext_dtype) => Canonical::Extension(ExtensionArray::new(
259 ext_dtype.clone(),
260 Canonical::empty(ext_dtype.storage_dtype()).into_array(),
261 )),
262 }
263 }
264
265 pub fn len(&self) -> usize {
266 match_each_canonical!(self, |arr| arr.len())
267 }
268
269 pub fn dtype(&self) -> &DType {
270 match_each_canonical!(self, |arr| arr.dtype())
271 }
272
273 pub fn is_empty(&self) -> bool {
274 match_each_canonical!(self, |arr| arr.is_empty())
275 }
276}
277
278impl Canonical {
279 pub fn compact(&self, ctx: &mut ExecutionCtx) -> VortexResult<Canonical> {
287 match self {
288 Canonical::Bool(array) => Ok(Canonical::Bool(array.trim_bits()?)),
289 Canonical::VarBinView(array) => Ok(Canonical::VarBinView(array.compact_buffers(ctx)?)),
290 Canonical::List(array) => Ok(Canonical::List(
291 array.rebuild(ListViewRebuildMode::TrimElements, ctx)?,
292 )),
293 Canonical::Map(array) => Ok(Canonical::Map(MapArray::new(
294 array.map_dtype().clone(),
295 array
296 .entries()
297 .as_::<ListView>()
298 .into_owned()
299 .rebuild(ListViewRebuildMode::TrimElements, ctx)?,
300 ))),
301 _ => Ok(self.clone()),
302 }
303 }
304}
305
306impl Canonical {
308 pub fn as_null(&self) -> &NullArray {
309 if let Canonical::Null(a) = self {
310 a
311 } else {
312 vortex_panic!("Cannot get NullArray from {:?}", &self)
313 }
314 }
315
316 pub fn into_null(self) -> NullArray {
317 if let Canonical::Null(a) = self {
318 a
319 } else {
320 vortex_panic!("Cannot unwrap NullArray from {:?}", &self)
321 }
322 }
323
324 pub fn as_bool(&self) -> &BoolArray {
325 if let Canonical::Bool(a) = self {
326 a
327 } else {
328 vortex_panic!("Cannot get BoolArray from {:?}", &self)
329 }
330 }
331
332 pub fn into_bool(self) -> BoolArray {
333 if let Canonical::Bool(a) = self {
334 a
335 } else {
336 vortex_panic!("Cannot unwrap BoolArray from {:?}", &self)
337 }
338 }
339
340 pub fn as_primitive(&self) -> &PrimitiveArray {
341 if let Canonical::Primitive(a) = self {
342 a
343 } else {
344 vortex_panic!("Cannot get PrimitiveArray from {:?}", &self)
345 }
346 }
347
348 pub fn into_primitive(self) -> PrimitiveArray {
349 if let Canonical::Primitive(a) = self {
350 a
351 } else {
352 vortex_panic!("Cannot unwrap PrimitiveArray from {:?}", &self)
353 }
354 }
355
356 pub fn as_decimal(&self) -> &DecimalArray {
357 if let Canonical::Decimal(a) = self {
358 a
359 } else {
360 vortex_panic!("Cannot get DecimalArray from {:?}", &self)
361 }
362 }
363
364 pub fn into_decimal(self) -> DecimalArray {
365 if let Canonical::Decimal(a) = self {
366 a
367 } else {
368 vortex_panic!("Cannot unwrap DecimalArray from {:?}", &self)
369 }
370 }
371
372 pub fn as_varbinview(&self) -> &VarBinViewArray {
373 if let Canonical::VarBinView(a) = self {
374 a
375 } else {
376 vortex_panic!("Cannot get VarBinViewArray from {:?}", &self)
377 }
378 }
379
380 pub fn into_varbinview(self) -> VarBinViewArray {
381 if let Canonical::VarBinView(a) = self {
382 a
383 } else {
384 vortex_panic!("Cannot unwrap VarBinViewArray from {:?}", &self)
385 }
386 }
387
388 pub fn as_listview(&self) -> &ListViewArray {
389 if let Canonical::List(a) = self {
390 a
391 } else {
392 vortex_panic!("Cannot get ListArray from {:?}", &self)
393 }
394 }
395
396 pub fn into_listview(self) -> ListViewArray {
397 if let Canonical::List(a) = self {
398 a
399 } else {
400 vortex_panic!("Cannot unwrap ListArray from {:?}", &self)
401 }
402 }
403
404 pub fn as_map(&self) -> &MapArray {
405 if let Canonical::Map(a) = self {
406 a
407 } else {
408 vortex_panic!("Cannot get MapArray from {:?}", &self)
409 }
410 }
411
412 pub fn into_map(self) -> MapArray {
413 if let Canonical::Map(a) = self {
414 a
415 } else {
416 vortex_panic!("Cannot unwrap MapArray from {:?}", &self)
417 }
418 }
419
420 pub fn as_fixed_size_list(&self) -> &FixedSizeListArray {
421 if let Canonical::FixedSizeList(a) = self {
422 a
423 } else {
424 vortex_panic!("Cannot get FixedSizeListArray from {:?}", &self)
425 }
426 }
427
428 pub fn into_fixed_size_list(self) -> FixedSizeListArray {
429 if let Canonical::FixedSizeList(a) = self {
430 a
431 } else {
432 vortex_panic!("Cannot unwrap FixedSizeListArray from {:?}", &self)
433 }
434 }
435
436 pub fn as_struct(&self) -> &StructArray {
437 if let Canonical::Struct(a) = self {
438 a
439 } else {
440 vortex_panic!("Cannot get StructArray from {:?}", &self)
441 }
442 }
443
444 pub fn into_struct(self) -> StructArray {
445 if let Canonical::Struct(a) = self {
446 a
447 } else {
448 vortex_panic!("Cannot unwrap StructArray from {:?}", &self)
449 }
450 }
451
452 pub fn as_union(&self) -> &UnionArray {
454 if let Canonical::Union(a) = self {
455 a
456 } else {
457 vortex_panic!("Cannot get UnionArray from {:?}", &self)
458 }
459 }
460
461 pub fn into_union(self) -> UnionArray {
463 if let Canonical::Union(a) = self {
464 a
465 } else {
466 vortex_panic!("Cannot unwrap UnionArray from {:?}", &self)
467 }
468 }
469
470 pub fn as_extension(&self) -> &ExtensionArray {
471 if let Canonical::Extension(a) = self {
472 a
473 } else {
474 vortex_panic!("Cannot get ExtensionArray from {:?}", &self)
475 }
476 }
477
478 pub fn into_extension(self) -> ExtensionArray {
479 if let Canonical::Extension(a) = self {
480 a
481 } else {
482 vortex_panic!("Cannot unwrap ExtensionArray from {:?}", &self)
483 }
484 }
485}
486
487impl IntoArray for Canonical {
488 fn into_array(self) -> ArrayRef {
489 match_each_canonical!(self, |arr| arr.into_array())
490 }
491}
492
493impl From<Canonical> for ArrayRef {
494 fn from(value: Canonical) -> Self {
495 match_each_canonical!(value, |arr| arr.into_array())
496 }
497}
498
499impl Executable for Canonical {
505 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
506 let result = array.execute_until::<AnyCanonical>(ctx)?;
507 Ok(result
508 .as_opt::<AnyCanonical>()
509 .map(Canonical::from)
510 .vortex_expect("execute_until::<AnyCanonical> must return a canonical array"))
511 }
512}
513
514pub struct CanonicalValidity(pub Canonical);
519
520impl Executable for CanonicalValidity {
521 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
522 match array.execute::<Canonical>(ctx)? {
523 n @ Canonical::Null(_) => Ok(CanonicalValidity(n)),
524 Canonical::Bool(b) => {
525 let validity = child_to_validity(b.slots()[0].as_ref(), b.dtype().nullability());
526 let len = b.len();
527 let BoolDataParts { bits, meta } = b.into_data().into_parts(len);
528 Ok(CanonicalValidity(Canonical::Bool(
529 BoolArray::try_new_from_handle(
530 bits,
531 meta.offset(),
532 meta.len(),
533 validity.execute(ctx)?,
534 )?,
535 )))
536 }
537 Canonical::Primitive(p) => {
538 let PrimitiveDataParts {
539 ptype,
540 buffer,
541 validity,
542 } = p.into_data_parts();
543 Ok(CanonicalValidity(Canonical::Primitive(unsafe {
544 PrimitiveArray::new_unchecked_from_handle(buffer, ptype, validity.execute(ctx)?)
545 })))
546 }
547 Canonical::Decimal(d) => {
548 let DecimalDataParts {
549 decimal_dtype,
550 values,
551 values_type,
552 validity,
553 } = d.into_data_parts();
554 Ok(CanonicalValidity(Canonical::Decimal(unsafe {
555 DecimalArray::new_unchecked_handle(
556 values,
557 values_type,
558 decimal_dtype,
559 validity.execute(ctx)?,
560 )
561 })))
562 }
563 Canonical::VarBinView(vbv) => {
564 let VarBinViewDataParts {
565 dtype,
566 buffers,
567 views,
568 validity,
569 } = vbv.into_data_parts();
570 Ok(CanonicalValidity(Canonical::VarBinView(unsafe {
571 VarBinViewArray::new_handle_unchecked(
572 views,
573 buffers,
574 dtype,
575 validity.execute(ctx)?,
576 )
577 })))
578 }
579 Canonical::List(l) => {
580 let zctl = l.is_zero_copy_to_list();
581 let ListViewDataParts {
582 elements,
583 offsets,
584 sizes,
585 validity,
586 ..
587 } = l.into_data_parts();
588 Ok(CanonicalValidity(Canonical::List(unsafe {
589 ListViewArray::new_unchecked(elements, offsets, sizes, validity.execute(ctx)?)
590 .with_zero_copy_to_list(zctl)
591 })))
592 }
593 Canonical::Map(map) => {
594 let map_dtype = map.map_dtype().clone();
595 let entries = map.entries().clone();
596 Ok(CanonicalValidity(Canonical::Map(MapArray::new(
597 map_dtype,
598 entries.execute::<CanonicalValidity>(ctx)?.0.into_listview(),
599 ))))
600 }
601 Canonical::FixedSizeList(fsl) => {
602 let list_size = fsl.list_size();
603 let len = fsl.len();
604 let parts = fsl.into_data_parts();
605 let elements = parts.elements;
606 let validity = parts.validity;
607 Ok(CanonicalValidity(Canonical::FixedSizeList(
608 FixedSizeListArray::new(elements, list_size, validity.execute(ctx)?, len),
609 )))
610 }
611 Canonical::Struct(st) => {
612 let len = st.len();
613 let StructDataParts {
614 struct_fields,
615 fields,
616 validity,
617 } = st.into_data_parts();
618 Ok(CanonicalValidity(Canonical::Struct(unsafe {
619 StructArray::new_unchecked(fields, struct_fields, len, validity.execute(ctx)?)
620 })))
621 }
622 Canonical::Union(union) => {
623 let UnionDataParts {
624 variants,
625 type_ids,
626 children,
627 } = union.into_data_parts();
628 let type_ids = type_ids.execute::<CanonicalValidity>(ctx)?.0.into_array();
629
630 Ok(CanonicalValidity(Canonical::Union(unsafe {
631 UnionArray::new_unchecked(type_ids, variants, children.iter().cloned())
632 })))
633 }
634 Canonical::Extension(ext) => Ok(CanonicalValidity(Canonical::Extension(
635 ExtensionArray::new(
636 ext.ext_dtype().clone(),
637 ext.storage_array()
638 .clone()
639 .execute::<CanonicalValidity>(ctx)?
640 .0
641 .into_array(),
642 ),
643 ))),
644 Canonical::Variant(variant) => {
645 let core_storage = recursively_canonicalize_slots(variant.core_storage(), ctx)?;
646 let shredded = variant
647 .shredded()
648 .map(|shredded| {
649 if shredded.is::<Variant>() {
650 recursively_canonicalize_slots(shredded, ctx)
651 } else {
652 shredded
653 .clone()
654 .execute::<CanonicalValidity>(ctx)
655 .map(|canonical| canonical.0.into_array())
656 }
657 })
658 .transpose()?;
659 Ok(CanonicalValidity(Canonical::Variant(
660 VariantArray::try_new(core_storage, shredded)?,
661 )))
662 }
663 }
664 }
665}
666
667pub struct RecursiveCanonical(pub Canonical);
672
673fn recursively_canonicalize_slots(
676 array: &ArrayRef,
677 ctx: &mut ExecutionCtx,
678) -> VortexResult<ArrayRef> {
679 let slots = array
680 .slots()
681 .iter()
682 .map(|slot| {
683 slot.as_ref()
684 .map(|child| {
685 child
686 .clone()
687 .execute::<RecursiveCanonical>(ctx)
688 .map(|canonical| canonical.0.into_array())
689 })
690 .transpose()
691 })
692 .collect::<VortexResult<ArraySlots>>()?;
693 unsafe { array.clone().with_slots(slots) }
696}
697impl Executable for RecursiveCanonical {
698 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
699 match array.execute::<Canonical>(ctx)? {
700 n @ Canonical::Null(_) => Ok(RecursiveCanonical(n)),
701 Canonical::Bool(b) => {
702 let validity = child_to_validity(b.slots()[0].as_ref(), b.dtype().nullability());
703 let len = b.len();
704 let BoolDataParts { bits, meta } = b.into_data().into_parts(len);
705 Ok(RecursiveCanonical(Canonical::Bool(
706 BoolArray::try_new_from_handle(
707 bits,
708 meta.offset(),
709 meta.len(),
710 validity.execute(ctx)?,
711 )?,
712 )))
713 }
714 Canonical::Primitive(p) => {
715 let PrimitiveDataParts {
716 ptype,
717 buffer,
718 validity,
719 } = p.into_data_parts();
720 Ok(RecursiveCanonical(Canonical::Primitive(unsafe {
721 PrimitiveArray::new_unchecked_from_handle(buffer, ptype, validity.execute(ctx)?)
722 })))
723 }
724 Canonical::Decimal(d) => {
725 let DecimalDataParts {
726 decimal_dtype,
727 values,
728 values_type,
729 validity,
730 } = d.into_data_parts();
731 Ok(RecursiveCanonical(Canonical::Decimal(unsafe {
732 DecimalArray::new_unchecked_handle(
733 values,
734 values_type,
735 decimal_dtype,
736 validity.execute(ctx)?,
737 )
738 })))
739 }
740 Canonical::VarBinView(vbv) => {
741 let VarBinViewDataParts {
742 dtype,
743 buffers,
744 views,
745 validity,
746 } = vbv.into_data_parts();
747 Ok(RecursiveCanonical(Canonical::VarBinView(unsafe {
748 VarBinViewArray::new_handle_unchecked(
749 views,
750 buffers,
751 dtype,
752 validity.execute(ctx)?,
753 )
754 })))
755 }
756 Canonical::List(l) => {
757 let zctl = l.is_zero_copy_to_list();
758 let ListViewDataParts {
759 elements,
760 offsets,
761 sizes,
762 validity,
763 ..
764 } = l.into_data_parts();
765 Ok(RecursiveCanonical(Canonical::List(unsafe {
766 ListViewArray::new_unchecked(
767 elements.execute::<RecursiveCanonical>(ctx)?.0.into_array(),
768 offsets.execute::<RecursiveCanonical>(ctx)?.0.into_array(),
769 sizes.execute::<RecursiveCanonical>(ctx)?.0.into_array(),
770 validity.execute(ctx)?,
771 )
772 .with_zero_copy_to_list(zctl)
773 })))
774 }
775 Canonical::Map(map) => {
776 let map_dtype = map.map_dtype().clone();
777 let entries = map.entries().clone();
778 Ok(RecursiveCanonical(Canonical::Map(MapArray::new(
779 map_dtype,
780 entries
781 .execute::<RecursiveCanonical>(ctx)?
782 .0
783 .into_listview(),
784 ))))
785 }
786 Canonical::FixedSizeList(fsl) => {
787 let list_size = fsl.list_size();
788 let len = fsl.len();
789 let parts = fsl.into_data_parts();
790 let elements = parts.elements;
791 let validity = parts.validity;
792 Ok(RecursiveCanonical(Canonical::FixedSizeList(
793 FixedSizeListArray::new(
794 elements.execute::<RecursiveCanonical>(ctx)?.0.into_array(),
795 list_size,
796 validity.execute(ctx)?,
797 len,
798 ),
799 )))
800 }
801 Canonical::Struct(st) => {
802 let len = st.len();
803 let StructDataParts {
804 struct_fields,
805 fields,
806 validity,
807 } = st.into_data_parts();
808 let executed_fields = fields
809 .into_iter()
810 .map(|f| Ok(f.execute::<RecursiveCanonical>(ctx)?.0.into_array()))
811 .collect::<VortexResult<Vec<_>>>()?;
812
813 Ok(RecursiveCanonical(Canonical::Struct(unsafe {
814 StructArray::new_unchecked(
815 executed_fields,
816 struct_fields,
817 len,
818 validity.execute(ctx)?,
819 )
820 })))
821 }
822 Canonical::Union(union) => {
823 let UnionDataParts {
824 variants,
825 type_ids,
826 children,
827 } = union.into_data_parts();
828 let type_ids = type_ids.execute::<RecursiveCanonical>(ctx)?.0.into_array();
829 let children = children
830 .iter()
831 .cloned()
832 .map(|child| {
833 child
834 .execute::<RecursiveCanonical>(ctx)
835 .map(|canonical| canonical.0.into_array())
836 })
837 .collect::<VortexResult<Vec<_>>>()?;
838
839 Ok(RecursiveCanonical(Canonical::Union(unsafe {
840 UnionArray::new_unchecked(type_ids, variants, children)
841 })))
842 }
843 Canonical::Extension(ext) => Ok(RecursiveCanonical(Canonical::Extension(
844 ExtensionArray::new(
845 ext.ext_dtype().clone(),
846 ext.storage_array()
847 .clone()
848 .execute::<RecursiveCanonical>(ctx)?
849 .0
850 .into_array(),
851 ),
852 ))),
853 Canonical::Variant(variant) => {
854 let core_storage = recursively_canonicalize_slots(variant.core_storage(), ctx)?;
855 let shredded = variant
856 .shredded()
857 .map(|shredded| {
858 if shredded.is::<Variant>() {
859 recursively_canonicalize_slots(shredded, ctx)
860 } else {
861 shredded
862 .clone()
863 .execute::<RecursiveCanonical>(ctx)
864 .map(|canonical| canonical.0.into_array())
865 }
866 })
867 .transpose()?;
868 Ok(RecursiveCanonical(Canonical::Variant(
869 VariantArray::try_new(core_storage, shredded)?,
870 )))
871 }
872 }
873 }
874}
875
876impl<T: NativePType> Executable for Buffer<T> {
882 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
883 let array = PrimitiveArray::execute(array, ctx)?;
884 vortex_ensure!(
885 matches!(
886 array.validity()?,
887 Validity::NonNullable | Validity::AllValid
888 ),
889 "Cannot execute to native buffer: array is not all-valid."
890 );
891 Ok(array.into_buffer())
892 }
893}
894
895impl Executable for PrimitiveArray {
899 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
900 match array.try_downcast::<Primitive>() {
901 Ok(primitive) => Ok(primitive),
902 Err(array) => Ok(Canonical::execute(array, ctx)?.into_primitive()),
903 }
904 }
905}
906
907impl Executable for BoolArray {
911 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
912 match array.try_downcast::<Bool>() {
913 Ok(bool_array) => Ok(bool_array),
914 Err(array) => Ok(Canonical::execute(array, ctx)?.into_bool()),
915 }
916 }
917}
918
919impl Executable for BitBuffer {
923 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
924 let bool = BoolArray::execute(array, ctx)?;
925 assert!(
926 !bool.dtype().is_nullable(),
927 "bit buffer execute only works with non-nullable bool arrays"
928 );
929 Ok(bool.into_bit_buffer())
930 }
931}
932
933impl Executable for NullArray {
937 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
938 match array.try_downcast::<Null>() {
939 Ok(null_array) => Ok(null_array),
940 Err(array) => Ok(Canonical::execute(array, ctx)?.into_null()),
941 }
942 }
943}
944
945impl Executable for VarBinViewArray {
949 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
950 match array.try_downcast::<VarBinView>() {
951 Ok(varbinview) => Ok(varbinview),
952 Err(array) => Ok(Canonical::execute(array, ctx)?.into_varbinview()),
953 }
954 }
955}
956
957impl Executable for ExtensionArray {
961 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
962 match array.try_downcast::<Extension>() {
963 Ok(ext_array) => Ok(ext_array),
964 Err(array) => Ok(Canonical::execute(array, ctx)?.into_extension()),
965 }
966 }
967}
968
969impl Executable for DecimalArray {
973 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
974 match array.try_downcast::<Decimal>() {
975 Ok(decimal) => Ok(decimal),
976 Err(array) => Ok(Canonical::execute(array, ctx)?.into_decimal()),
977 }
978 }
979}
980
981impl Executable for ListViewArray {
985 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
986 match array.try_downcast::<ListView>() {
987 Ok(list) => Ok(list),
988 Err(array) => Ok(Canonical::execute(array, ctx)?.into_listview()),
989 }
990 }
991}
992
993impl Executable for MapArray {
997 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
998 match array.try_downcast::<Map>() {
999 Ok(map) => Ok(map),
1000 Err(array) => Ok(Canonical::execute(array, ctx)?.into_map()),
1001 }
1002 }
1003}
1004
1005impl Executable for FixedSizeListArray {
1009 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
1010 match array.try_downcast::<FixedSizeList>() {
1011 Ok(fsl) => Ok(fsl),
1012 Err(array) => Ok(Canonical::execute(array, ctx)?.into_fixed_size_list()),
1013 }
1014 }
1015}
1016
1017impl Executable for StructArray {
1021 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
1022 match array.try_downcast::<Struct>() {
1023 Ok(struct_array) => Ok(struct_array),
1024 Err(array) => Ok(Canonical::execute(array, ctx)?.into_struct()),
1025 }
1026 }
1027}
1028
1029impl Executable for UnionArray {
1033 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
1034 match array.try_downcast::<Union>() {
1035 Ok(union_array) => Ok(union_array),
1036 Err(array) => Ok(Canonical::execute(array, ctx)?.into_union()),
1037 }
1038 }
1039}
1040
1041impl Executable for VariantArray {
1045 fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
1046 match array.try_downcast::<Variant>() {
1047 Ok(variant_array) => Ok(variant_array),
1048 Err(array) => match Canonical::execute(array, ctx)? {
1049 Canonical::Variant(variant_array) => Ok(variant_array),
1050 canonical => {
1051 vortex_panic!("Cannot unwrap VariantArray from {:?}", canonical)
1052 }
1053 },
1054 }
1055 }
1056}
1057
1058#[derive(Debug, Clone, Copy)]
1063pub enum CanonicalView<'a> {
1064 Null(ArrayView<'a, Null>),
1065 Bool(ArrayView<'a, Bool>),
1066 Primitive(ArrayView<'a, Primitive>),
1067 Decimal(ArrayView<'a, Decimal>),
1068 VarBinView(ArrayView<'a, VarBinView>),
1069 List(ArrayView<'a, ListView>),
1070 Map(ArrayView<'a, Map>),
1071 FixedSizeList(ArrayView<'a, FixedSizeList>),
1072 Struct(ArrayView<'a, Struct>),
1073 Union(ArrayView<'a, Union>),
1074 Extension(ArrayView<'a, Extension>),
1075 Variant(ArrayView<'a, Variant>),
1076}
1077
1078impl From<CanonicalView<'_>> for Canonical {
1079 fn from(value: CanonicalView<'_>) -> Self {
1080 match value {
1081 CanonicalView::Null(a) => Canonical::Null(a.into_owned()),
1082 CanonicalView::Bool(a) => Canonical::Bool(a.into_owned()),
1083 CanonicalView::Primitive(a) => Canonical::Primitive(a.into_owned()),
1084 CanonicalView::Decimal(a) => Canonical::Decimal(a.into_owned()),
1085 CanonicalView::VarBinView(a) => Canonical::VarBinView(a.into_owned()),
1086 CanonicalView::List(a) => Canonical::List(a.into_owned()),
1087 CanonicalView::Map(a) => Canonical::Map(a.into_owned()),
1088 CanonicalView::FixedSizeList(a) => Canonical::FixedSizeList(a.into_owned()),
1089 CanonicalView::Struct(a) => Canonical::Struct(a.into_owned()),
1090 CanonicalView::Union(a) => Canonical::Union(a.into_owned()),
1091 CanonicalView::Extension(a) => Canonical::Extension(a.into_owned()),
1092 CanonicalView::Variant(a) => Canonical::Variant(a.into_owned()),
1093 }
1094 }
1095}
1096
1097impl CanonicalView<'_> {
1098 pub fn to_array_ref(&self) -> ArrayRef {
1100 match self {
1101 CanonicalView::Null(a) => a.array().clone(),
1102 CanonicalView::Bool(a) => a.array().clone(),
1103 CanonicalView::Primitive(a) => a.array().clone(),
1104 CanonicalView::Decimal(a) => a.array().clone(),
1105 CanonicalView::VarBinView(a) => a.array().clone(),
1106 CanonicalView::List(a) => a.array().clone(),
1107 CanonicalView::Map(a) => a.array().clone(),
1108 CanonicalView::FixedSizeList(a) => a.array().clone(),
1109 CanonicalView::Struct(a) => a.array().clone(),
1110 CanonicalView::Union(a) => a.array().clone(),
1111 CanonicalView::Extension(a) => a.array().clone(),
1112 CanonicalView::Variant(a) => a.array().clone(),
1113 }
1114 }
1115}
1116
1117macro_rules! canonical_kinds {
1119 ($($kind:ident => $vtable:ty),+ $(,)?) => {
1120 #[derive(Clone, Copy)]
1122 pub(crate) enum CanonicalKind {
1123 $($kind),+
1124 }
1125
1126 impl CanonicalKind {
1127 #[inline]
1129 pub(crate) fn of<V: VTable>() -> Option<Self> {
1130 $(if TypeId::of::<V>() == TypeId::of::<$vtable>() {
1131 return Some(Self::$kind);
1132 })+
1133 None
1134 }
1135
1136 #[inline]
1139 unsafe fn view(self, array: &ArrayRef) -> CanonicalView<'_> {
1140 match self {
1141 $(Self::$kind => CanonicalView::$kind(unsafe {
1143 array.as_typed_unchecked::<$vtable>()
1144 })),+
1145 }
1146 }
1147 }
1148 };
1149}
1150
1151canonical_kinds! {
1152 Null => Null,
1153 Bool => Bool,
1154 Primitive => Primitive,
1155 Decimal => Decimal,
1156 Struct => Struct,
1157 Union => Union,
1158 List => ListView,
1159 Map => Map,
1160 FixedSizeList => FixedSizeList,
1161 VarBinView => VarBinView,
1162 Variant => Variant,
1163 Extension => Extension,
1164}
1165
1166pub struct AnyCanonical;
1168impl Matcher for AnyCanonical {
1169 type Match<'a> = CanonicalView<'a>;
1170
1171 #[inline]
1172 fn matches(array: &ArrayRef) -> bool {
1173 array.dyn_array().canonical_kind().is_some()
1174 }
1175
1176 #[inline]
1177 fn try_match(array: &ArrayRef) -> Option<Self::Match<'_>> {
1178 let kind = array.dyn_array().canonical_kind()?;
1179 Some(unsafe { kind.view(array) })
1181 }
1182}
1183
1184#[cfg(test)]
1185mod tests {
1186 use std::sync::Arc;
1187 use std::sync::LazyLock;
1188
1189 use rstest::rstest;
1190 use vortex_error::VortexResult;
1191 use vortex_error::vortex_err;
1192 use vortex_session::VortexSession;
1193
1194 use crate::ArrayRef;
1195 use crate::Canonical;
1196 use crate::CanonicalValidity;
1197 use crate::IntoArray;
1198 use crate::VortexSessionExecute;
1199 use crate::array::new_foreign_array;
1200 use crate::arrays::Constant;
1201 use crate::arrays::ConstantArray;
1202 use crate::arrays::Primitive;
1203 use crate::arrays::PrimitiveArray;
1204 use crate::arrays::Struct;
1205 use crate::arrays::VarBinArray;
1206 use crate::arrays::Variant;
1207 use crate::arrays::VariantArray;
1208 use crate::arrays::struct_::StructArrayExt;
1209 use crate::arrays::variant::VariantArraySlotsExt;
1210 use crate::assert_arrays_eq;
1211 use crate::canonical::AnyCanonical;
1212 use crate::canonical::StructArray;
1213 use crate::dtype::DType;
1214 use crate::dtype::DecimalDType;
1215 use crate::dtype::MapDType;
1216 use crate::dtype::Nullability;
1217 use crate::dtype::PType;
1218 use crate::dtype::StructFields;
1219 use crate::dtype::UnionVariants;
1220 use crate::extension::datetime::TimeUnit;
1221 use crate::extension::datetime::Timestamp;
1222 use crate::scalar::Scalar;
1223
1224 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
1226
1227 fn variant_core_storage(len: usize) -> ArrayRef {
1228 ConstantArray::new(
1229 Scalar::variant(Scalar::primitive(1i32, Nullability::NonNullable)),
1230 len,
1231 )
1232 .into_array()
1233 }
1234
1235 fn canonical_arrays(nullability: Nullability) -> VortexResult<Vec<ArrayRef>> {
1236 let i32_dtype = DType::Primitive(PType::I32, nullability);
1237 let dtypes = [
1238 DType::Null,
1239 DType::Bool(nullability),
1240 i32_dtype.clone(),
1241 DType::Decimal(DecimalDType::try_new(10, 2)?, nullability),
1242 DType::Utf8(nullability),
1243 DType::Binary(nullability),
1244 DType::List(Arc::new(i32_dtype.clone()), nullability),
1245 DType::FixedSizeList(Arc::new(i32_dtype.clone()), 2, nullability),
1246 DType::Map(
1247 MapDType::try_new(
1248 DType::Utf8(Nullability::NonNullable),
1249 i32_dtype.clone(),
1250 false,
1251 )?,
1252 nullability,
1253 ),
1254 DType::Struct(
1255 StructFields::from_iter([("a", i32_dtype.clone())]),
1256 nullability,
1257 ),
1258 DType::Union(
1259 UnionVariants::new(["a"].into(), vec![i32_dtype])?,
1260 nullability,
1261 ),
1262 DType::Extension(Timestamp::new(TimeUnit::Microseconds, nullability).erased()),
1263 ];
1264 let mut arrays: Vec<_> = dtypes
1265 .iter()
1266 .map(|dtype| Canonical::empty(dtype).into_array())
1267 .collect();
1268 let value = Scalar::variant(Scalar::primitive(1i32, nullability))
1269 .cast(&DType::Variant(nullability))?;
1270 arrays.push(
1271 VariantArray::try_new(ConstantArray::new(value, 2).into_array(), None)?.into_array(),
1272 );
1273 Ok(arrays)
1274 }
1275
1276 #[rstest]
1277 fn any_canonical_matches_all_encodings(
1278 #[values(Nullability::NonNullable, Nullability::Nullable)] nullability: Nullability,
1279 ) -> VortexResult<()> {
1280 for array in canonical_arrays(nullability)? {
1281 assert!(array.is::<AnyCanonical>(), "{array}");
1282 let canonical = Canonical::from(array.as_::<AnyCanonical>()).into_array();
1283 assert!(ArrayRef::ptr_eq(&array, &canonical));
1284 }
1285 Ok(())
1286 }
1287
1288 #[test]
1289 fn any_canonical_rejects_noncanonical_encodings() {
1290 let arrays = [
1291 ConstantArray::new(1i32, 4).into_array(),
1292 ConstantArray::new(true, 4).into_array(),
1293 ConstantArray::new("value", 4).into_array(),
1294 VarBinArray::from_iter([Some("value")], DType::Utf8(Nullability::Nullable))
1295 .into_array(),
1296 ];
1297 for array in arrays {
1298 assert!(!array.is::<AnyCanonical>(), "{array}");
1299 assert!(array.as_opt::<AnyCanonical>().is_none(), "{array}");
1300 }
1301 }
1302
1303 #[rstest]
1304 fn any_canonical_rejects_foreign_arrays_with_canonical_ids(
1305 #[values(Nullability::NonNullable, Nullability::Nullable)] nullability: Nullability,
1306 ) -> VortexResult<()> {
1307 let mut ctx = SESSION.create_execution_ctx();
1308 for canonical in canonical_arrays(nullability)? {
1309 let foreign = new_foreign_array(
1310 canonical.encoding_id(),
1311 canonical.dtype().clone(),
1312 canonical.len(),
1313 vec![],
1314 vec![],
1315 Default::default(),
1316 )?;
1317 assert!(!foreign.is::<AnyCanonical>(), "{foreign}");
1318 assert!(foreign.as_opt::<AnyCanonical>().is_none(), "{foreign}");
1319 assert!(foreign.execute::<Canonical>(&mut ctx).is_err());
1320 }
1321 Ok(())
1322 }
1323
1324 #[test]
1325 fn canonical_validity_canonicalizes_variant_shredded_physical_slots() -> VortexResult<()> {
1326 let len = 2;
1327 let nested_shredded =
1328 StructArray::try_from_iter([("value", ConstantArray::new(10i32, len).into_array())])?;
1329 let inner_variant = VariantArray::try_new(
1330 variant_core_storage(len),
1331 Some(nested_shredded.into_array()),
1332 )?;
1333 let outer_variant =
1334 VariantArray::try_new(variant_core_storage(len), Some(inner_variant.into_array()))?;
1335
1336 let mut ctx = SESSION.create_execution_ctx();
1337 let Canonical::Variant(canonical) = outer_variant
1338 .into_array()
1339 .execute::<CanonicalValidity>(&mut ctx)?
1340 .0
1341 else {
1342 return Err(vortex_err!("expected canonical variant"));
1343 };
1344
1345 let nested_variant = canonical
1346 .shredded()
1347 .and_then(|shredded| shredded.as_opt::<Variant>())
1348 .ok_or_else(|| vortex_err!("expected nested variant shredded child"))?;
1349 let nested_struct = nested_variant
1350 .shredded()
1351 .and_then(|shredded| shredded.as_opt::<Struct>())
1352 .ok_or_else(|| vortex_err!("expected nested struct shredded child"))?;
1353 let value = nested_struct.unmasked_field_by_name("value")?;
1354
1355 assert!(value.is::<Primitive>());
1356 assert!(!value.is::<Constant>());
1357 assert_arrays_eq!(value, PrimitiveArray::from_iter([10i32, 10]), &mut ctx);
1358
1359 Ok(())
1360 }
1361}