Skip to main content

vortex_array/
canonical.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Encodings that enable zero-copy sharing of data with Arrow.
5
6use std::any::TypeId;
7use std::sync::Arc;
8
9use vortex_buffer::BitBuffer;
10use vortex_buffer::Buffer;
11use vortex_error::VortexExpect;
12use vortex_error::VortexResult;
13use vortex_error::vortex_ensure;
14use vortex_error::vortex_panic;
15
16use crate::ArrayRef;
17use crate::ArraySlots;
18use crate::Executable;
19use crate::ExecutionCtx;
20use crate::IntoArray;
21use crate::array::ArrayView;
22use crate::array::VTable;
23use crate::array::child_to_validity;
24use crate::arrays::Bool;
25use crate::arrays::BoolArray;
26use crate::arrays::Decimal;
27use crate::arrays::DecimalArray;
28use crate::arrays::Extension;
29use crate::arrays::ExtensionArray;
30use crate::arrays::FixedSizeList;
31use crate::arrays::FixedSizeListArray;
32use crate::arrays::ListView;
33use crate::arrays::ListViewArray;
34use crate::arrays::Map;
35use crate::arrays::MapArray;
36use crate::arrays::Null;
37use crate::arrays::NullArray;
38use crate::arrays::Primitive;
39use crate::arrays::PrimitiveArray;
40use crate::arrays::Struct;
41use crate::arrays::StructArray;
42use crate::arrays::Union;
43use crate::arrays::UnionArray;
44use crate::arrays::VarBinView;
45use crate::arrays::VarBinViewArray;
46use crate::arrays::Variant;
47use crate::arrays::VariantArray;
48use crate::arrays::bool::BoolDataParts;
49use crate::arrays::decimal::DecimalDataParts;
50use crate::arrays::extension::ExtensionArrayExt;
51use crate::arrays::fixed_size_list::FixedSizeListArrayExt;
52use crate::arrays::listview::ListViewDataParts;
53use crate::arrays::listview::ListViewRebuildMode;
54use crate::arrays::map::MapArrayExt;
55use crate::arrays::map::MapArraySlotsExt;
56use crate::arrays::primitive::PrimitiveDataParts;
57use crate::arrays::struct_::StructDataParts;
58use crate::arrays::union::UnionDataParts;
59use crate::arrays::varbinview::VarBinViewDataParts;
60use crate::arrays::variant::VariantArraySlotsExt;
61use crate::dtype::DType;
62use crate::dtype::NativePType;
63use crate::dtype::Nullability;
64use crate::dtype::PType;
65use crate::match_each_decimal_value_type;
66use crate::match_each_native_ptype;
67use crate::matcher::Matcher;
68use crate::validity::Validity;
69
70/// An enum capturing the default uncompressed encodings for each [Vortex type](DType).
71///
72/// Any array can be decoded into canonical form via the `to_canonical`
73/// trait method. This is the simplest encoding for a type, and will not be compressed but may
74/// contain compressed child arrays.
75///
76/// Canonical form is useful for doing type-specific compute where you need to know that all
77/// elements are laid out decompressed and contiguous in memory.
78///
79/// Each `Canonical` variant has a corresponding [`DType`] variant, with the notable exception of
80/// [`Canonical::VarBinView`], which is the canonical encoding for both [`DType::Utf8`] and
81/// [`DType::Binary`].
82///
83/// # Laziness
84///
85/// Canonical form is not recursive, so while a `StructArray` is the canonical format for any
86/// `Struct` type, individual column child arrays may still be compressed. This allows
87/// compute over Vortex arrays to push decoding as late as possible, and ideally many child arrays
88/// never need to be decoded into canonical form at all depending on the compute.
89///
90/// # Arrow interoperability
91///
92/// Most Vortex canonical encodings have an equivalent Arrow encoding that can be built zero-copy,
93/// and the corresponding Arrow array types can also be built directly. Map array Arrow transport is
94/// not implemented yet, and [`UnionArray`]'s independent top-level validity cannot be represented
95/// directly by an Arrow union.
96///
97/// The full list of canonical types and their equivalent Arrow array types are:
98///
99/// * `NullArray`: `arrow_array::NullArray`
100/// * `BoolArray`: `arrow_array::BooleanArray`
101/// * `PrimitiveArray`: `arrow_array::PrimitiveArray`
102/// * `DecimalArray`: `arrow_array::Decimal128Array` and `arrow_array::Decimal256Array`
103/// * `VarBinViewArray`: `arrow_array::GenericByteViewArray`
104/// * `ListViewArray`: `arrow_array::ListViewArray`
105/// * `MapArray`: Vortex `ListView<Struct<key, value>>` storage
106/// * `FixedSizeListArray`: `arrow_array::FixedSizeListArray`
107/// * `StructArray`: `arrow_array::StructArray`
108///
109/// Vortex uses a logical type system, unlike Arrow which uses physical encodings for its types.
110/// As an example, there are at least six valid physical encodings for a `Utf8` array. This can
111/// create ambiguity.
112/// Thus, if you receive an Arrow array, compress it using Vortex, and then
113/// decompress it later to pass to a compute kernel, there are multiple suitable Arrow array
114/// variants to hold the data.
115///
116/// To disambiguate, we choose a canonical physical encoding for every Vortex [`DType`], which
117/// will correspond to an arrow-rs `arrow_schema::DataType`.
118///
119/// # Views support
120///
121/// Binary and String views, also known as "German strings" are a better encoding format for
122/// nearly all use-cases. Variable-length binary views are part of the Apache Arrow spec, and are
123/// fully supported by the Datafusion query engine. We use them as our canonical string encoding
124/// for all `Utf8` and `Binary` typed arrays in Vortex. They provide considerably faster filter
125/// execution than the core `StringArray` and `BinaryArray` types, at the expense of potentially
126/// needing garbage collection (`arrow_array::GenericByteViewArray::gc`) to clear unreferenced items
127/// from memory.
128///
129/// # For Developers
130///
131/// If you add another variant to this enum, make sure to update `canonical_kinds!`,
132/// and the fuzzer in `fuzz/fuzz_targets/array_ops.rs`.
133#[derive(Debug, Clone)]
134pub enum Canonical {
135    Null(NullArray),
136    Bool(BoolArray),
137    Primitive(PrimitiveArray),
138    Decimal(DecimalArray),
139    VarBinView(VarBinViewArray),
140    List(ListViewArray),
141    Map(MapArray),
142    FixedSizeList(FixedSizeListArray),
143    Struct(StructArray),
144    Union(UnionArray),
145    /// Canonical storage for extension dtypes, wrapping the canonical form of the storage dtype.
146    Extension(ExtensionArray),
147    /// Canonical storage for dynamic variant values, optionally with typed shredded paths.
148    Variant(VariantArray),
149}
150
151/// Match on every canonical variant and evaluate a code block on all variants
152macro_rules! match_each_canonical {
153    ($self:expr, | $ident:ident | $eval:expr) => {{
154        match $self {
155            Canonical::Null($ident) => $eval,
156            Canonical::Bool($ident) => $eval,
157            Canonical::Primitive($ident) => $eval,
158            Canonical::Decimal($ident) => $eval,
159            Canonical::VarBinView($ident) => $eval,
160            Canonical::List($ident) => $eval,
161            Canonical::Map($ident) => $eval,
162            Canonical::FixedSizeList($ident) => $eval,
163            Canonical::Struct($ident) => $eval,
164            Canonical::Union($ident) => $eval,
165            Canonical::Variant($ident) => $eval,
166            Canonical::Extension($ident) => $eval,
167        }
168    }};
169}
170
171impl Canonical {
172    /// Create an empty canonical array of the given dtype.
173    pub fn empty(dtype: &DType) -> Canonical {
174        match dtype {
175            DType::Null => Canonical::Null(NullArray::new(0)),
176            DType::Bool(n) => Canonical::Bool(unsafe {
177                BoolArray::new_unchecked(BitBuffer::empty(), Validity::from(n))
178            }),
179            DType::Primitive(ptype, n) => {
180                match_each_native_ptype!(ptype, |P| {
181                    Canonical::Primitive(unsafe {
182                        PrimitiveArray::new_unchecked(Buffer::<P>::empty(), Validity::from(n))
183                    })
184                })
185            }
186            DType::Decimal(decimal_type, n) => {
187                match_each_decimal_value_type!(decimal_type.canonical_values_type(), |D| {
188                    Canonical::Decimal(unsafe {
189                        DecimalArray::new_unchecked::<D>(
190                            Buffer::empty(),
191                            *decimal_type,
192                            Validity::from(n),
193                        )
194                    })
195                })
196            }
197            DType::Utf8(n) => Canonical::VarBinView(unsafe {
198                VarBinViewArray::new_unchecked(
199                    Buffer::empty(),
200                    Arc::new([]),
201                    dtype.clone(),
202                    Validity::from(n),
203                )
204            }),
205            DType::Binary(n) => Canonical::VarBinView(unsafe {
206                VarBinViewArray::new_unchecked(
207                    Buffer::empty(),
208                    Arc::new([]),
209                    dtype.clone(),
210                    Validity::from(n),
211                )
212            }),
213            DType::List(dtype, n) => Canonical::List(unsafe {
214                ListViewArray::new_unchecked(
215                    Canonical::empty(dtype).into_array(),
216                    Canonical::empty(&DType::Primitive(PType::U8, Nullability::NonNullable))
217                        .into_array(),
218                    Canonical::empty(&DType::Primitive(PType::U8, Nullability::NonNullable))
219                        .into_array(),
220                    Validity::from(n),
221                )
222                // An empty list view is trivially copyable to a list.
223                .with_zero_copy_to_list(true)
224            }),
225            DType::Map(map_dtype, nullability) => Canonical::Map(MapArray::new(
226                map_dtype.clone(),
227                Canonical::empty(&DType::List(
228                    Arc::new(map_dtype.entries_dtype()),
229                    *nullability,
230                ))
231                .into_listview(),
232            )),
233            DType::FixedSizeList(elem_dtype, list_size, null) => Canonical::FixedSizeList(unsafe {
234                FixedSizeListArray::new_unchecked(
235                    Canonical::empty(elem_dtype).into_array(),
236                    *list_size,
237                    Validity::from(null),
238                    0,
239                )
240            }),
241            DType::Struct(struct_dtype, n) => Canonical::Struct(unsafe {
242                StructArray::new_unchecked(
243                    struct_dtype
244                        .fields()
245                        .map(|f| Canonical::empty(&f).into_array())
246                        .collect::<Vec<_>>(),
247                    struct_dtype.clone(),
248                    0,
249                    Validity::from(n),
250                )
251            }),
252            DType::Union(variants, nullability) => {
253                Canonical::Union(UnionArray::empty(variants.clone(), *nullability))
254            }
255            DType::Variant(_) => {
256                vortex_panic!("Canonical empty is not supported for Variant")
257            }
258            DType::Extension(ext_dtype) => Canonical::Extension(ExtensionArray::new(
259                ext_dtype.clone(),
260                Canonical::empty(ext_dtype.storage_dtype()).into_array(),
261            )),
262        }
263    }
264
265    pub fn len(&self) -> usize {
266        match_each_canonical!(self, |arr| arr.len())
267    }
268
269    pub fn dtype(&self) -> &DType {
270        match_each_canonical!(self, |arr| arr.dtype())
271    }
272
273    pub fn is_empty(&self) -> bool {
274        match_each_canonical!(self, |arr| arr.is_empty())
275    }
276}
277
278impl Canonical {
279    /// Performs a (potentially expensive) compaction operation on the array before it is complete.
280    ///
281    /// This is mostly relevant for the variable-length types such as Utf8, Binary or List where
282    /// they can accumulate wasted space after slicing and taking operations.
283    ///
284    /// This operation can be very expensive and can result in things like allocations, full-scans
285    /// and copy operations.
286    pub fn compact(&self, ctx: &mut ExecutionCtx) -> VortexResult<Canonical> {
287        match self {
288            Canonical::Bool(array) => Ok(Canonical::Bool(array.trim_bits()?)),
289            Canonical::VarBinView(array) => Ok(Canonical::VarBinView(array.compact_buffers(ctx)?)),
290            Canonical::List(array) => Ok(Canonical::List(
291                array.rebuild(ListViewRebuildMode::TrimElements, ctx)?,
292            )),
293            Canonical::Map(array) => Ok(Canonical::Map(MapArray::new(
294                array.map_dtype().clone(),
295                array
296                    .entries()
297                    .as_::<ListView>()
298                    .into_owned()
299                    .rebuild(ListViewRebuildMode::TrimElements, ctx)?,
300            ))),
301            _ => Ok(self.clone()),
302        }
303    }
304}
305
306// Unwrap canonical type back down to specialized type.
307impl Canonical {
308    pub fn as_null(&self) -> &NullArray {
309        if let Canonical::Null(a) = self {
310            a
311        } else {
312            vortex_panic!("Cannot get NullArray from {:?}", &self)
313        }
314    }
315
316    pub fn into_null(self) -> NullArray {
317        if let Canonical::Null(a) = self {
318            a
319        } else {
320            vortex_panic!("Cannot unwrap NullArray from {:?}", &self)
321        }
322    }
323
324    pub fn as_bool(&self) -> &BoolArray {
325        if let Canonical::Bool(a) = self {
326            a
327        } else {
328            vortex_panic!("Cannot get BoolArray from {:?}", &self)
329        }
330    }
331
332    pub fn into_bool(self) -> BoolArray {
333        if let Canonical::Bool(a) = self {
334            a
335        } else {
336            vortex_panic!("Cannot unwrap BoolArray from {:?}", &self)
337        }
338    }
339
340    pub fn as_primitive(&self) -> &PrimitiveArray {
341        if let Canonical::Primitive(a) = self {
342            a
343        } else {
344            vortex_panic!("Cannot get PrimitiveArray from {:?}", &self)
345        }
346    }
347
348    pub fn into_primitive(self) -> PrimitiveArray {
349        if let Canonical::Primitive(a) = self {
350            a
351        } else {
352            vortex_panic!("Cannot unwrap PrimitiveArray from {:?}", &self)
353        }
354    }
355
356    pub fn as_decimal(&self) -> &DecimalArray {
357        if let Canonical::Decimal(a) = self {
358            a
359        } else {
360            vortex_panic!("Cannot get DecimalArray from {:?}", &self)
361        }
362    }
363
364    pub fn into_decimal(self) -> DecimalArray {
365        if let Canonical::Decimal(a) = self {
366            a
367        } else {
368            vortex_panic!("Cannot unwrap DecimalArray from {:?}", &self)
369        }
370    }
371
372    pub fn as_varbinview(&self) -> &VarBinViewArray {
373        if let Canonical::VarBinView(a) = self {
374            a
375        } else {
376            vortex_panic!("Cannot get VarBinViewArray from {:?}", &self)
377        }
378    }
379
380    pub fn into_varbinview(self) -> VarBinViewArray {
381        if let Canonical::VarBinView(a) = self {
382            a
383        } else {
384            vortex_panic!("Cannot unwrap VarBinViewArray from {:?}", &self)
385        }
386    }
387
388    pub fn as_listview(&self) -> &ListViewArray {
389        if let Canonical::List(a) = self {
390            a
391        } else {
392            vortex_panic!("Cannot get ListArray from {:?}", &self)
393        }
394    }
395
396    pub fn into_listview(self) -> ListViewArray {
397        if let Canonical::List(a) = self {
398            a
399        } else {
400            vortex_panic!("Cannot unwrap ListArray from {:?}", &self)
401        }
402    }
403
404    pub fn as_map(&self) -> &MapArray {
405        if let Canonical::Map(a) = self {
406            a
407        } else {
408            vortex_panic!("Cannot get MapArray from {:?}", &self)
409        }
410    }
411
412    pub fn into_map(self) -> MapArray {
413        if let Canonical::Map(a) = self {
414            a
415        } else {
416            vortex_panic!("Cannot unwrap MapArray from {:?}", &self)
417        }
418    }
419
420    pub fn as_fixed_size_list(&self) -> &FixedSizeListArray {
421        if let Canonical::FixedSizeList(a) = self {
422            a
423        } else {
424            vortex_panic!("Cannot get FixedSizeListArray from {:?}", &self)
425        }
426    }
427
428    pub fn into_fixed_size_list(self) -> FixedSizeListArray {
429        if let Canonical::FixedSizeList(a) = self {
430            a
431        } else {
432            vortex_panic!("Cannot unwrap FixedSizeListArray from {:?}", &self)
433        }
434    }
435
436    pub fn as_struct(&self) -> &StructArray {
437        if let Canonical::Struct(a) = self {
438            a
439        } else {
440            vortex_panic!("Cannot get StructArray from {:?}", &self)
441        }
442    }
443
444    pub fn into_struct(self) -> StructArray {
445        if let Canonical::Struct(a) = self {
446            a
447        } else {
448            vortex_panic!("Cannot unwrap StructArray from {:?}", &self)
449        }
450    }
451
452    /// Return this canonical array as a sparse [`UnionArray`].
453    pub fn as_union(&self) -> &UnionArray {
454        if let Canonical::Union(a) = self {
455            a
456        } else {
457            vortex_panic!("Cannot get UnionArray from {:?}", &self)
458        }
459    }
460
461    /// Unwrap this canonical array as a sparse [`UnionArray`].
462    pub fn into_union(self) -> UnionArray {
463        if let Canonical::Union(a) = self {
464            a
465        } else {
466            vortex_panic!("Cannot unwrap UnionArray from {:?}", &self)
467        }
468    }
469
470    pub fn as_extension(&self) -> &ExtensionArray {
471        if let Canonical::Extension(a) = self {
472            a
473        } else {
474            vortex_panic!("Cannot get ExtensionArray from {:?}", &self)
475        }
476    }
477
478    pub fn into_extension(self) -> ExtensionArray {
479        if let Canonical::Extension(a) = self {
480            a
481        } else {
482            vortex_panic!("Cannot unwrap ExtensionArray from {:?}", &self)
483        }
484    }
485}
486
487impl IntoArray for Canonical {
488    fn into_array(self) -> ArrayRef {
489        match_each_canonical!(self, |arr| arr.into_array())
490    }
491}
492
493impl From<Canonical> for ArrayRef {
494    fn from(value: Canonical) -> Self {
495        match_each_canonical!(value, |arr| arr.into_array())
496    }
497}
498
499/// Execute into [`Canonical`] by running `execute_until` with the [`AnyCanonical`] matcher.
500///
501/// Unlike executing into [`crate::Columnar`], this will fully expand constant arrays into their
502/// canonical form. Callers should prefer to execute into `Columnar` if they are able to optimize
503/// their use for constant arrays.
504impl Executable for Canonical {
505    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
506        let result = array.execute_until::<AnyCanonical>(ctx)?;
507        Ok(result
508            .as_opt::<AnyCanonical>()
509            .map(Canonical::from)
510            .vortex_expect("execute_until::<AnyCanonical> must return a canonical array"))
511    }
512}
513
514/// Recursively execute the array until it reaches canonical form along with its validity.
515///
516/// Callers should prefer to execute into `Columnar` instead of this specific target.
517/// This target is useful when preparing arrays for writing.
518pub struct CanonicalValidity(pub Canonical);
519
520impl Executable for CanonicalValidity {
521    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
522        match array.execute::<Canonical>(ctx)? {
523            n @ Canonical::Null(_) => Ok(CanonicalValidity(n)),
524            Canonical::Bool(b) => {
525                let validity = child_to_validity(b.slots()[0].as_ref(), b.dtype().nullability());
526                let len = b.len();
527                let BoolDataParts { bits, meta } = b.into_data().into_parts(len);
528                Ok(CanonicalValidity(Canonical::Bool(
529                    BoolArray::try_new_from_handle(
530                        bits,
531                        meta.offset(),
532                        meta.len(),
533                        validity.execute(ctx)?,
534                    )?,
535                )))
536            }
537            Canonical::Primitive(p) => {
538                let PrimitiveDataParts {
539                    ptype,
540                    buffer,
541                    validity,
542                } = p.into_data_parts();
543                Ok(CanonicalValidity(Canonical::Primitive(unsafe {
544                    PrimitiveArray::new_unchecked_from_handle(buffer, ptype, validity.execute(ctx)?)
545                })))
546            }
547            Canonical::Decimal(d) => {
548                let DecimalDataParts {
549                    decimal_dtype,
550                    values,
551                    values_type,
552                    validity,
553                } = d.into_data_parts();
554                Ok(CanonicalValidity(Canonical::Decimal(unsafe {
555                    DecimalArray::new_unchecked_handle(
556                        values,
557                        values_type,
558                        decimal_dtype,
559                        validity.execute(ctx)?,
560                    )
561                })))
562            }
563            Canonical::VarBinView(vbv) => {
564                let VarBinViewDataParts {
565                    dtype,
566                    buffers,
567                    views,
568                    validity,
569                } = vbv.into_data_parts();
570                Ok(CanonicalValidity(Canonical::VarBinView(unsafe {
571                    VarBinViewArray::new_handle_unchecked(
572                        views,
573                        buffers,
574                        dtype,
575                        validity.execute(ctx)?,
576                    )
577                })))
578            }
579            Canonical::List(l) => {
580                let zctl = l.is_zero_copy_to_list();
581                let ListViewDataParts {
582                    elements,
583                    offsets,
584                    sizes,
585                    validity,
586                    ..
587                } = l.into_data_parts();
588                Ok(CanonicalValidity(Canonical::List(unsafe {
589                    ListViewArray::new_unchecked(elements, offsets, sizes, validity.execute(ctx)?)
590                        .with_zero_copy_to_list(zctl)
591                })))
592            }
593            Canonical::Map(map) => {
594                let map_dtype = map.map_dtype().clone();
595                let entries = map.entries().clone();
596                Ok(CanonicalValidity(Canonical::Map(MapArray::new(
597                    map_dtype,
598                    entries.execute::<CanonicalValidity>(ctx)?.0.into_listview(),
599                ))))
600            }
601            Canonical::FixedSizeList(fsl) => {
602                let list_size = fsl.list_size();
603                let len = fsl.len();
604                let parts = fsl.into_data_parts();
605                let elements = parts.elements;
606                let validity = parts.validity;
607                Ok(CanonicalValidity(Canonical::FixedSizeList(
608                    FixedSizeListArray::new(elements, list_size, validity.execute(ctx)?, len),
609                )))
610            }
611            Canonical::Struct(st) => {
612                let len = st.len();
613                let StructDataParts {
614                    struct_fields,
615                    fields,
616                    validity,
617                } = st.into_data_parts();
618                Ok(CanonicalValidity(Canonical::Struct(unsafe {
619                    StructArray::new_unchecked(fields, struct_fields, len, validity.execute(ctx)?)
620                })))
621            }
622            Canonical::Union(union) => {
623                let UnionDataParts {
624                    variants,
625                    type_ids,
626                    children,
627                } = union.into_data_parts();
628                let type_ids = type_ids.execute::<CanonicalValidity>(ctx)?.0.into_array();
629
630                Ok(CanonicalValidity(Canonical::Union(unsafe {
631                    UnionArray::new_unchecked(type_ids, variants, children.iter().cloned())
632                })))
633            }
634            Canonical::Extension(ext) => Ok(CanonicalValidity(Canonical::Extension(
635                ExtensionArray::new(
636                    ext.ext_dtype().clone(),
637                    ext.storage_array()
638                        .clone()
639                        .execute::<CanonicalValidity>(ctx)?
640                        .0
641                        .into_array(),
642                ),
643            ))),
644            Canonical::Variant(variant) => {
645                let core_storage = recursively_canonicalize_slots(variant.core_storage(), ctx)?;
646                let shredded = variant
647                    .shredded()
648                    .map(|shredded| {
649                        if shredded.is::<Variant>() {
650                            recursively_canonicalize_slots(shredded, ctx)
651                        } else {
652                            shredded
653                                .clone()
654                                .execute::<CanonicalValidity>(ctx)
655                                .map(|canonical| canonical.0.into_array())
656                        }
657                    })
658                    .transpose()?;
659                Ok(CanonicalValidity(Canonical::Variant(
660                    VariantArray::try_new(core_storage, shredded)?,
661                )))
662            }
663        }
664    }
665}
666
667/// Recursively execute the array until all of its children are canonical.
668///
669/// This method is useful to guarantee that all operators are fully executed,
670/// callers should prefer an execution target that's suitable for their use case instead of this one.
671pub struct RecursiveCanonical(pub Canonical);
672
673// TODO: Currently only used for Variant, in the future
674// can probably be used for more canonical types like Struct.
675fn recursively_canonicalize_slots(
676    array: &ArrayRef,
677    ctx: &mut ExecutionCtx,
678) -> VortexResult<ArrayRef> {
679    let slots = array
680        .slots()
681        .iter()
682        .map(|slot| {
683            slot.as_ref()
684                .map(|child| {
685                    child
686                        .clone()
687                        .execute::<RecursiveCanonical>(ctx)
688                        .map(|canonical| canonical.0.into_array())
689                })
690                .transpose()
691        })
692        .collect::<VortexResult<ArraySlots>>()?;
693    // SAFETY: recursive canonicalization rewrites child slots to equivalent canonical
694    // representations, preserving the parent array's logical values and statistics.
695    unsafe { array.clone().with_slots(slots) }
696}
697impl Executable for RecursiveCanonical {
698    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
699        match array.execute::<Canonical>(ctx)? {
700            n @ Canonical::Null(_) => Ok(RecursiveCanonical(n)),
701            Canonical::Bool(b) => {
702                let validity = child_to_validity(b.slots()[0].as_ref(), b.dtype().nullability());
703                let len = b.len();
704                let BoolDataParts { bits, meta } = b.into_data().into_parts(len);
705                Ok(RecursiveCanonical(Canonical::Bool(
706                    BoolArray::try_new_from_handle(
707                        bits,
708                        meta.offset(),
709                        meta.len(),
710                        validity.execute(ctx)?,
711                    )?,
712                )))
713            }
714            Canonical::Primitive(p) => {
715                let PrimitiveDataParts {
716                    ptype,
717                    buffer,
718                    validity,
719                } = p.into_data_parts();
720                Ok(RecursiveCanonical(Canonical::Primitive(unsafe {
721                    PrimitiveArray::new_unchecked_from_handle(buffer, ptype, validity.execute(ctx)?)
722                })))
723            }
724            Canonical::Decimal(d) => {
725                let DecimalDataParts {
726                    decimal_dtype,
727                    values,
728                    values_type,
729                    validity,
730                } = d.into_data_parts();
731                Ok(RecursiveCanonical(Canonical::Decimal(unsafe {
732                    DecimalArray::new_unchecked_handle(
733                        values,
734                        values_type,
735                        decimal_dtype,
736                        validity.execute(ctx)?,
737                    )
738                })))
739            }
740            Canonical::VarBinView(vbv) => {
741                let VarBinViewDataParts {
742                    dtype,
743                    buffers,
744                    views,
745                    validity,
746                } = vbv.into_data_parts();
747                Ok(RecursiveCanonical(Canonical::VarBinView(unsafe {
748                    VarBinViewArray::new_handle_unchecked(
749                        views,
750                        buffers,
751                        dtype,
752                        validity.execute(ctx)?,
753                    )
754                })))
755            }
756            Canonical::List(l) => {
757                let zctl = l.is_zero_copy_to_list();
758                let ListViewDataParts {
759                    elements,
760                    offsets,
761                    sizes,
762                    validity,
763                    ..
764                } = l.into_data_parts();
765                Ok(RecursiveCanonical(Canonical::List(unsafe {
766                    ListViewArray::new_unchecked(
767                        elements.execute::<RecursiveCanonical>(ctx)?.0.into_array(),
768                        offsets.execute::<RecursiveCanonical>(ctx)?.0.into_array(),
769                        sizes.execute::<RecursiveCanonical>(ctx)?.0.into_array(),
770                        validity.execute(ctx)?,
771                    )
772                    .with_zero_copy_to_list(zctl)
773                })))
774            }
775            Canonical::Map(map) => {
776                let map_dtype = map.map_dtype().clone();
777                let entries = map.entries().clone();
778                Ok(RecursiveCanonical(Canonical::Map(MapArray::new(
779                    map_dtype,
780                    entries
781                        .execute::<RecursiveCanonical>(ctx)?
782                        .0
783                        .into_listview(),
784                ))))
785            }
786            Canonical::FixedSizeList(fsl) => {
787                let list_size = fsl.list_size();
788                let len = fsl.len();
789                let parts = fsl.into_data_parts();
790                let elements = parts.elements;
791                let validity = parts.validity;
792                Ok(RecursiveCanonical(Canonical::FixedSizeList(
793                    FixedSizeListArray::new(
794                        elements.execute::<RecursiveCanonical>(ctx)?.0.into_array(),
795                        list_size,
796                        validity.execute(ctx)?,
797                        len,
798                    ),
799                )))
800            }
801            Canonical::Struct(st) => {
802                let len = st.len();
803                let StructDataParts {
804                    struct_fields,
805                    fields,
806                    validity,
807                } = st.into_data_parts();
808                let executed_fields = fields
809                    .into_iter()
810                    .map(|f| Ok(f.execute::<RecursiveCanonical>(ctx)?.0.into_array()))
811                    .collect::<VortexResult<Vec<_>>>()?;
812
813                Ok(RecursiveCanonical(Canonical::Struct(unsafe {
814                    StructArray::new_unchecked(
815                        executed_fields,
816                        struct_fields,
817                        len,
818                        validity.execute(ctx)?,
819                    )
820                })))
821            }
822            Canonical::Union(union) => {
823                let UnionDataParts {
824                    variants,
825                    type_ids,
826                    children,
827                } = union.into_data_parts();
828                let type_ids = type_ids.execute::<RecursiveCanonical>(ctx)?.0.into_array();
829                let children = children
830                    .iter()
831                    .cloned()
832                    .map(|child| {
833                        child
834                            .execute::<RecursiveCanonical>(ctx)
835                            .map(|canonical| canonical.0.into_array())
836                    })
837                    .collect::<VortexResult<Vec<_>>>()?;
838
839                Ok(RecursiveCanonical(Canonical::Union(unsafe {
840                    UnionArray::new_unchecked(type_ids, variants, children)
841                })))
842            }
843            Canonical::Extension(ext) => Ok(RecursiveCanonical(Canonical::Extension(
844                ExtensionArray::new(
845                    ext.ext_dtype().clone(),
846                    ext.storage_array()
847                        .clone()
848                        .execute::<RecursiveCanonical>(ctx)?
849                        .0
850                        .into_array(),
851                ),
852            ))),
853            Canonical::Variant(variant) => {
854                let core_storage = recursively_canonicalize_slots(variant.core_storage(), ctx)?;
855                let shredded = variant
856                    .shredded()
857                    .map(|shredded| {
858                        if shredded.is::<Variant>() {
859                            recursively_canonicalize_slots(shredded, ctx)
860                        } else {
861                            shredded
862                                .clone()
863                                .execute::<RecursiveCanonical>(ctx)
864                                .map(|canonical| canonical.0.into_array())
865                        }
866                    })
867                    .transpose()?;
868                Ok(RecursiveCanonical(Canonical::Variant(
869                    VariantArray::try_new(core_storage, shredded)?,
870                )))
871            }
872        }
873    }
874}
875
876/// Execute a primitive typed array into a buffer of native values, assuming all values are valid.
877///
878/// # Errors
879///
880/// Returns a `VortexError` if the array is not all-valid (has any nulls).
881impl<T: NativePType> Executable for Buffer<T> {
882    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
883        let array = PrimitiveArray::execute(array, ctx)?;
884        vortex_ensure!(
885            matches!(
886                array.validity()?,
887                Validity::NonNullable | Validity::AllValid
888            ),
889            "Cannot execute to native buffer: array is not all-valid."
890        );
891        Ok(array.into_buffer())
892    }
893}
894
895/// Execute the array to canonical form and unwrap as a [`PrimitiveArray`].
896///
897/// This will panic if the array's dtype is not primitive.
898impl Executable for PrimitiveArray {
899    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
900        match array.try_downcast::<Primitive>() {
901            Ok(primitive) => Ok(primitive),
902            Err(array) => Ok(Canonical::execute(array, ctx)?.into_primitive()),
903        }
904    }
905}
906
907/// Execute the array to canonical form and unwrap as a [`BoolArray`].
908///
909/// This will panic if the array's dtype is not bool.
910impl Executable for BoolArray {
911    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
912        match array.try_downcast::<Bool>() {
913            Ok(bool_array) => Ok(bool_array),
914            Err(array) => Ok(Canonical::execute(array, ctx)?.into_bool()),
915        }
916    }
917}
918
919/// Execute the array to a [`BitBuffer`], aka a non-nullable  [`BoolArray`].
920///
921/// This will panic if the array's dtype is not non-nullable bool.
922impl Executable for BitBuffer {
923    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
924        let bool = BoolArray::execute(array, ctx)?;
925        assert!(
926            !bool.dtype().is_nullable(),
927            "bit buffer execute only works with non-nullable bool arrays"
928        );
929        Ok(bool.into_bit_buffer())
930    }
931}
932
933/// Execute the array to canonical form and unwrap as a [`NullArray`].
934///
935/// This will panic if the array's dtype is not null.
936impl Executable for NullArray {
937    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
938        match array.try_downcast::<Null>() {
939            Ok(null_array) => Ok(null_array),
940            Err(array) => Ok(Canonical::execute(array, ctx)?.into_null()),
941        }
942    }
943}
944
945/// Execute the array to canonical form and unwrap as a [`VarBinViewArray`].
946///
947/// This will panic if the array's dtype is not utf8 or binary.
948impl Executable for VarBinViewArray {
949    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
950        match array.try_downcast::<VarBinView>() {
951            Ok(varbinview) => Ok(varbinview),
952            Err(array) => Ok(Canonical::execute(array, ctx)?.into_varbinview()),
953        }
954    }
955}
956
957/// Execute the array to canonical form and unwrap as an [`ExtensionArray`].
958///
959/// This will panic if the array's dtype is not an extension type.
960impl Executable for ExtensionArray {
961    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
962        match array.try_downcast::<Extension>() {
963            Ok(ext_array) => Ok(ext_array),
964            Err(array) => Ok(Canonical::execute(array, ctx)?.into_extension()),
965        }
966    }
967}
968
969/// Execute the array to canonical form and unwrap as a [`DecimalArray`].
970///
971/// This will panic if the array's dtype is not decimal.
972impl Executable for DecimalArray {
973    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
974        match array.try_downcast::<Decimal>() {
975            Ok(decimal) => Ok(decimal),
976            Err(array) => Ok(Canonical::execute(array, ctx)?.into_decimal()),
977        }
978    }
979}
980
981/// Execute the array to canonical form and unwrap as a [`ListViewArray`].
982///
983/// This will panic if the array's dtype is not list.
984impl Executable for ListViewArray {
985    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
986        match array.try_downcast::<ListView>() {
987            Ok(list) => Ok(list),
988            Err(array) => Ok(Canonical::execute(array, ctx)?.into_listview()),
989        }
990    }
991}
992
993/// Execute the array to canonical form and unwrap as a [`MapArray`].
994///
995/// This will panic if the array's dtype is not map.
996impl Executable for MapArray {
997    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
998        match array.try_downcast::<Map>() {
999            Ok(map) => Ok(map),
1000            Err(array) => Ok(Canonical::execute(array, ctx)?.into_map()),
1001        }
1002    }
1003}
1004
1005/// Execute the array to canonical form and unwrap as a [`FixedSizeListArray`].
1006///
1007/// This will panic if the array's dtype is not fixed size list.
1008impl Executable for FixedSizeListArray {
1009    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
1010        match array.try_downcast::<FixedSizeList>() {
1011            Ok(fsl) => Ok(fsl),
1012            Err(array) => Ok(Canonical::execute(array, ctx)?.into_fixed_size_list()),
1013        }
1014    }
1015}
1016
1017/// Execute the array to canonical form and unwrap as a [`StructArray`].
1018///
1019/// This will panic if the array's dtype is not struct.
1020impl Executable for StructArray {
1021    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
1022        match array.try_downcast::<Struct>() {
1023            Ok(struct_array) => Ok(struct_array),
1024            Err(array) => Ok(Canonical::execute(array, ctx)?.into_struct()),
1025        }
1026    }
1027}
1028
1029/// Execute the array to canonical form and unwrap as a [`UnionArray`].
1030///
1031/// This will panic if the array's dtype is not union.
1032impl Executable for UnionArray {
1033    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
1034        match array.try_downcast::<Union>() {
1035            Ok(union_array) => Ok(union_array),
1036            Err(array) => Ok(Canonical::execute(array, ctx)?.into_union()),
1037        }
1038    }
1039}
1040
1041/// Execute the array to canonical form and unwrap as a [`VariantArray`].
1042///
1043/// This will panic if the array's dtype is not variant.
1044impl Executable for VariantArray {
1045    fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
1046        match array.try_downcast::<Variant>() {
1047            Ok(variant_array) => Ok(variant_array),
1048            Err(array) => match Canonical::execute(array, ctx)? {
1049                Canonical::Variant(variant_array) => Ok(variant_array),
1050                canonical => {
1051                    vortex_panic!("Cannot unwrap VariantArray from {:?}", canonical)
1052                }
1053            },
1054        }
1055    }
1056}
1057
1058/// A view into a canonical array type.
1059///
1060/// Uses `ArrayView<V>` because these are obtained by
1061/// downcasting through the `Matcher` trait which returns `ArrayView<V>`.
1062#[derive(Debug, Clone, Copy)]
1063pub enum CanonicalView<'a> {
1064    Null(ArrayView<'a, Null>),
1065    Bool(ArrayView<'a, Bool>),
1066    Primitive(ArrayView<'a, Primitive>),
1067    Decimal(ArrayView<'a, Decimal>),
1068    VarBinView(ArrayView<'a, VarBinView>),
1069    List(ArrayView<'a, ListView>),
1070    Map(ArrayView<'a, Map>),
1071    FixedSizeList(ArrayView<'a, FixedSizeList>),
1072    Struct(ArrayView<'a, Struct>),
1073    Union(ArrayView<'a, Union>),
1074    Extension(ArrayView<'a, Extension>),
1075    Variant(ArrayView<'a, Variant>),
1076}
1077
1078impl From<CanonicalView<'_>> for Canonical {
1079    fn from(value: CanonicalView<'_>) -> Self {
1080        match value {
1081            CanonicalView::Null(a) => Canonical::Null(a.into_owned()),
1082            CanonicalView::Bool(a) => Canonical::Bool(a.into_owned()),
1083            CanonicalView::Primitive(a) => Canonical::Primitive(a.into_owned()),
1084            CanonicalView::Decimal(a) => Canonical::Decimal(a.into_owned()),
1085            CanonicalView::VarBinView(a) => Canonical::VarBinView(a.into_owned()),
1086            CanonicalView::List(a) => Canonical::List(a.into_owned()),
1087            CanonicalView::Map(a) => Canonical::Map(a.into_owned()),
1088            CanonicalView::FixedSizeList(a) => Canonical::FixedSizeList(a.into_owned()),
1089            CanonicalView::Struct(a) => Canonical::Struct(a.into_owned()),
1090            CanonicalView::Union(a) => Canonical::Union(a.into_owned()),
1091            CanonicalView::Extension(a) => Canonical::Extension(a.into_owned()),
1092            CanonicalView::Variant(a) => Canonical::Variant(a.into_owned()),
1093        }
1094    }
1095}
1096
1097impl CanonicalView<'_> {
1098    /// Convert to a type-erased [`ArrayRef`].
1099    pub fn to_array_ref(&self) -> ArrayRef {
1100        match self {
1101            CanonicalView::Null(a) => a.array().clone(),
1102            CanonicalView::Bool(a) => a.array().clone(),
1103            CanonicalView::Primitive(a) => a.array().clone(),
1104            CanonicalView::Decimal(a) => a.array().clone(),
1105            CanonicalView::VarBinView(a) => a.array().clone(),
1106            CanonicalView::List(a) => a.array().clone(),
1107            CanonicalView::Map(a) => a.array().clone(),
1108            CanonicalView::FixedSizeList(a) => a.array().clone(),
1109            CanonicalView::Struct(a) => a.array().clone(),
1110            CanonicalView::Union(a) => a.array().clone(),
1111            CanonicalView::Extension(a) => a.array().clone(),
1112            CanonicalView::Variant(a) => a.array().clone(),
1113        }
1114    }
1115}
1116
1117// Keep classification and checked conversion together so they use the same concrete types.
1118macro_rules! canonical_kinds {
1119    ($($kind:ident => $vtable:ty),+ $(,)?) => {
1120        /// Canonical encoding classification derived from the concrete array vtable.
1121        #[derive(Clone, Copy)]
1122        pub(crate) enum CanonicalKind {
1123            $($kind),+
1124        }
1125
1126        impl CanonicalKind {
1127            /// Classify a concrete vtable; monomorphization folds these type comparisons.
1128            #[inline]
1129            pub(crate) fn of<V: VTable>() -> Option<Self> {
1130                $(if TypeId::of::<V>() == TypeId::of::<$vtable>() {
1131                    return Some(Self::$kind);
1132                })+
1133                None
1134            }
1135
1136            /// # Safety
1137            /// `self` must be the kind returned by `of::<V>()` for the concrete vtable of `array`.
1138            #[inline]
1139            unsafe fn view(self, array: &ArrayRef) -> CanonicalView<'_> {
1140                match self {
1141                    // SAFETY: the caller guarantees `of::<V>()` matched `$vtable` by `TypeId`.
1142                    $(Self::$kind => CanonicalView::$kind(unsafe {
1143                        array.as_typed_unchecked::<$vtable>()
1144                    })),+
1145                }
1146            }
1147        }
1148    };
1149}
1150
1151canonical_kinds! {
1152    Null => Null,
1153    Bool => Bool,
1154    Primitive => Primitive,
1155    Decimal => Decimal,
1156    Struct => Struct,
1157    Union => Union,
1158    List => ListView,
1159    Map => Map,
1160    FixedSizeList => FixedSizeList,
1161    VarBinView => VarBinView,
1162    Variant => Variant,
1163    Extension => Extension,
1164}
1165
1166/// A matcher for any canonical array type.
1167pub struct AnyCanonical;
1168impl Matcher for AnyCanonical {
1169    type Match<'a> = CanonicalView<'a>;
1170
1171    #[inline]
1172    fn matches(array: &ArrayRef) -> bool {
1173        array.dyn_array().canonical_kind().is_some()
1174    }
1175
1176    #[inline]
1177    fn try_match(array: &ArrayRef) -> Option<Self::Match<'_>> {
1178        let kind = array.dyn_array().canonical_kind()?;
1179        // SAFETY: `kind` was computed from the concrete vtable of `array`.
1180        Some(unsafe { kind.view(array) })
1181    }
1182}
1183
1184#[cfg(test)]
1185mod tests {
1186    use std::sync::Arc;
1187    use std::sync::LazyLock;
1188
1189    use rstest::rstest;
1190    use vortex_error::VortexResult;
1191    use vortex_error::vortex_err;
1192    use vortex_session::VortexSession;
1193
1194    use crate::ArrayRef;
1195    use crate::Canonical;
1196    use crate::CanonicalValidity;
1197    use crate::IntoArray;
1198    use crate::VortexSessionExecute;
1199    use crate::array::new_foreign_array;
1200    use crate::arrays::Constant;
1201    use crate::arrays::ConstantArray;
1202    use crate::arrays::Primitive;
1203    use crate::arrays::PrimitiveArray;
1204    use crate::arrays::Struct;
1205    use crate::arrays::VarBinArray;
1206    use crate::arrays::Variant;
1207    use crate::arrays::VariantArray;
1208    use crate::arrays::struct_::StructArrayExt;
1209    use crate::arrays::variant::VariantArraySlotsExt;
1210    use crate::assert_arrays_eq;
1211    use crate::canonical::AnyCanonical;
1212    use crate::canonical::StructArray;
1213    use crate::dtype::DType;
1214    use crate::dtype::DecimalDType;
1215    use crate::dtype::MapDType;
1216    use crate::dtype::Nullability;
1217    use crate::dtype::PType;
1218    use crate::dtype::StructFields;
1219    use crate::dtype::UnionVariants;
1220    use crate::extension::datetime::TimeUnit;
1221    use crate::extension::datetime::Timestamp;
1222    use crate::scalar::Scalar;
1223
1224    /// A shared session for these canonical tests, used to create execution contexts.
1225    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
1226
1227    fn variant_core_storage(len: usize) -> ArrayRef {
1228        ConstantArray::new(
1229            Scalar::variant(Scalar::primitive(1i32, Nullability::NonNullable)),
1230            len,
1231        )
1232        .into_array()
1233    }
1234
1235    fn canonical_arrays(nullability: Nullability) -> VortexResult<Vec<ArrayRef>> {
1236        let i32_dtype = DType::Primitive(PType::I32, nullability);
1237        let dtypes = [
1238            DType::Null,
1239            DType::Bool(nullability),
1240            i32_dtype.clone(),
1241            DType::Decimal(DecimalDType::try_new(10, 2)?, nullability),
1242            DType::Utf8(nullability),
1243            DType::Binary(nullability),
1244            DType::List(Arc::new(i32_dtype.clone()), nullability),
1245            DType::FixedSizeList(Arc::new(i32_dtype.clone()), 2, nullability),
1246            DType::Map(
1247                MapDType::try_new(
1248                    DType::Utf8(Nullability::NonNullable),
1249                    i32_dtype.clone(),
1250                    false,
1251                )?,
1252                nullability,
1253            ),
1254            DType::Struct(
1255                StructFields::from_iter([("a", i32_dtype.clone())]),
1256                nullability,
1257            ),
1258            DType::Union(
1259                UnionVariants::new(["a"].into(), vec![i32_dtype])?,
1260                nullability,
1261            ),
1262            DType::Extension(Timestamp::new(TimeUnit::Microseconds, nullability).erased()),
1263        ];
1264        let mut arrays: Vec<_> = dtypes
1265            .iter()
1266            .map(|dtype| Canonical::empty(dtype).into_array())
1267            .collect();
1268        let value = Scalar::variant(Scalar::primitive(1i32, nullability))
1269            .cast(&DType::Variant(nullability))?;
1270        arrays.push(
1271            VariantArray::try_new(ConstantArray::new(value, 2).into_array(), None)?.into_array(),
1272        );
1273        Ok(arrays)
1274    }
1275
1276    #[rstest]
1277    fn any_canonical_matches_all_encodings(
1278        #[values(Nullability::NonNullable, Nullability::Nullable)] nullability: Nullability,
1279    ) -> VortexResult<()> {
1280        for array in canonical_arrays(nullability)? {
1281            assert!(array.is::<AnyCanonical>(), "{array}");
1282            let canonical = Canonical::from(array.as_::<AnyCanonical>()).into_array();
1283            assert!(ArrayRef::ptr_eq(&array, &canonical));
1284        }
1285        Ok(())
1286    }
1287
1288    #[test]
1289    fn any_canonical_rejects_noncanonical_encodings() {
1290        let arrays = [
1291            ConstantArray::new(1i32, 4).into_array(),
1292            ConstantArray::new(true, 4).into_array(),
1293            ConstantArray::new("value", 4).into_array(),
1294            VarBinArray::from_iter([Some("value")], DType::Utf8(Nullability::Nullable))
1295                .into_array(),
1296        ];
1297        for array in arrays {
1298            assert!(!array.is::<AnyCanonical>(), "{array}");
1299            assert!(array.as_opt::<AnyCanonical>().is_none(), "{array}");
1300        }
1301    }
1302
1303    #[rstest]
1304    fn any_canonical_rejects_foreign_arrays_with_canonical_ids(
1305        #[values(Nullability::NonNullable, Nullability::Nullable)] nullability: Nullability,
1306    ) -> VortexResult<()> {
1307        let mut ctx = SESSION.create_execution_ctx();
1308        for canonical in canonical_arrays(nullability)? {
1309            let foreign = new_foreign_array(
1310                canonical.encoding_id(),
1311                canonical.dtype().clone(),
1312                canonical.len(),
1313                vec![],
1314                vec![],
1315                Default::default(),
1316            )?;
1317            assert!(!foreign.is::<AnyCanonical>(), "{foreign}");
1318            assert!(foreign.as_opt::<AnyCanonical>().is_none(), "{foreign}");
1319            assert!(foreign.execute::<Canonical>(&mut ctx).is_err());
1320        }
1321        Ok(())
1322    }
1323
1324    #[test]
1325    fn canonical_validity_canonicalizes_variant_shredded_physical_slots() -> VortexResult<()> {
1326        let len = 2;
1327        let nested_shredded =
1328            StructArray::try_from_iter([("value", ConstantArray::new(10i32, len).into_array())])?;
1329        let inner_variant = VariantArray::try_new(
1330            variant_core_storage(len),
1331            Some(nested_shredded.into_array()),
1332        )?;
1333        let outer_variant =
1334            VariantArray::try_new(variant_core_storage(len), Some(inner_variant.into_array()))?;
1335
1336        let mut ctx = SESSION.create_execution_ctx();
1337        let Canonical::Variant(canonical) = outer_variant
1338            .into_array()
1339            .execute::<CanonicalValidity>(&mut ctx)?
1340            .0
1341        else {
1342            return Err(vortex_err!("expected canonical variant"));
1343        };
1344
1345        let nested_variant = canonical
1346            .shredded()
1347            .and_then(|shredded| shredded.as_opt::<Variant>())
1348            .ok_or_else(|| vortex_err!("expected nested variant shredded child"))?;
1349        let nested_struct = nested_variant
1350            .shredded()
1351            .and_then(|shredded| shredded.as_opt::<Struct>())
1352            .ok_or_else(|| vortex_err!("expected nested struct shredded child"))?;
1353        let value = nested_struct.unmasked_field_by_name("value")?;
1354
1355        assert!(value.is::<Primitive>());
1356        assert!(!value.is::<Constant>());
1357        assert_arrays_eq!(value, PrimitiveArray::from_iter([10i32, 10]), &mut ctx);
1358
1359        Ok(())
1360    }
1361}