vortex_array/arrow/
convert.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::sync::Arc;
5
6use arrow_array::cast::{AsArray, as_null_array};
7use arrow_array::types::{
8    ByteArrayType, ByteViewType, Date32Type, Date64Type, Decimal32Type, Decimal64Type,
9    Decimal128Type, Decimal256Type, Float16Type, Float32Type, Float64Type, Int8Type, Int16Type,
10    Int32Type, Int64Type, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
11    Time64NanosecondType, TimestampMicrosecondType, TimestampMillisecondType,
12    TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
13};
14use arrow_array::{
15    Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray,
16    FixedSizeListArray as ArrowFixedSizeListArray, GenericByteArray, GenericByteViewArray,
17    GenericListArray, GenericListViewArray, NullArray as ArrowNullArray, OffsetSizeTrait,
18    PrimitiveArray as ArrowPrimitiveArray, RecordBatch, StructArray as ArrowStructArray,
19    make_array,
20};
21use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
22use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
23use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
24use itertools::Itertools;
25use vortex_buffer::{Alignment, BitBuffer, Buffer, ByteBuffer};
26use vortex_dtype::datetime::TimeUnit;
27use vortex_dtype::{DType, DecimalDType, IntegerPType, NativePType, PType};
28use vortex_error::{VortexExpect as _, vortex_panic};
29use vortex_scalar::i256;
30
31use crate::arrays::{
32    BoolArray, DecimalArray, DictArray, FixedSizeListArray, ListArray, ListViewArray, NullArray,
33    PrimitiveArray, StructArray, TemporalArray, VarBinArray, VarBinViewArray,
34};
35use crate::arrow::FromArrowArray;
36use crate::validity::Validity;
37use crate::{ArrayRef, IntoArray};
38
39impl IntoArray for ArrowBuffer {
40    fn into_array(self) -> ArrayRef {
41        PrimitiveArray::from_byte_buffer(
42            ByteBuffer::from_arrow_buffer(self, Alignment::of::<u8>()),
43            PType::U8,
44            Validity::NonNullable,
45        )
46        .into_array()
47    }
48}
49
50impl IntoArray for BooleanBuffer {
51    fn into_array(self) -> ArrayRef {
52        BoolArray::from_bit_buffer(self.into(), Validity::NonNullable).into_array()
53    }
54}
55
56impl<T> IntoArray for ScalarBuffer<T>
57where
58    T: ArrowNativeType + NativePType,
59{
60    fn into_array(self) -> ArrayRef {
61        PrimitiveArray::new(
62            Buffer::<T>::from_arrow_scalar_buffer(self),
63            Validity::NonNullable,
64        )
65        .into_array()
66    }
67}
68
69impl<O> IntoArray for OffsetBuffer<O>
70where
71    O: IntegerPType + OffsetSizeTrait,
72{
73    fn into_array(self) -> ArrayRef {
74        let primitive = PrimitiveArray::new(
75            Buffer::from_arrow_scalar_buffer(self.into_inner()),
76            Validity::NonNullable,
77        );
78
79        primitive.into_array()
80    }
81}
82
83macro_rules! impl_from_arrow_primitive {
84    ($T:path) => {
85        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
86            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
87                let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone());
88                let validity = nulls(value.nulls(), nullable);
89                PrimitiveArray::new(buffer, validity).into_array()
90            }
91        }
92    };
93}
94
95impl_from_arrow_primitive!(Int8Type);
96impl_from_arrow_primitive!(Int16Type);
97impl_from_arrow_primitive!(Int32Type);
98impl_from_arrow_primitive!(Int64Type);
99impl_from_arrow_primitive!(UInt8Type);
100impl_from_arrow_primitive!(UInt16Type);
101impl_from_arrow_primitive!(UInt32Type);
102impl_from_arrow_primitive!(UInt64Type);
103impl_from_arrow_primitive!(Float16Type);
104impl_from_arrow_primitive!(Float32Type);
105impl_from_arrow_primitive!(Float64Type);
106
107impl FromArrowArray<&ArrowPrimitiveArray<Decimal32Type>> for ArrayRef {
108    fn from_arrow(array: &ArrowPrimitiveArray<Decimal32Type>, nullable: bool) -> Self {
109        let decimal_type = DecimalDType::new(array.precision(), array.scale());
110        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
111        let validity = nulls(array.nulls(), nullable);
112        DecimalArray::new(buffer, decimal_type, validity).into_array()
113    }
114}
115
116impl FromArrowArray<&ArrowPrimitiveArray<Decimal64Type>> for ArrayRef {
117    fn from_arrow(array: &ArrowPrimitiveArray<Decimal64Type>, nullable: bool) -> Self {
118        let decimal_type = DecimalDType::new(array.precision(), array.scale());
119        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
120        let validity = nulls(array.nulls(), nullable);
121        DecimalArray::new(buffer, decimal_type, validity).into_array()
122    }
123}
124
125impl FromArrowArray<&ArrowPrimitiveArray<Decimal128Type>> for ArrayRef {
126    fn from_arrow(array: &ArrowPrimitiveArray<Decimal128Type>, nullable: bool) -> Self {
127        let decimal_type = DecimalDType::new(array.precision(), array.scale());
128        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
129        let validity = nulls(array.nulls(), nullable);
130        DecimalArray::new(buffer, decimal_type, validity).into_array()
131    }
132}
133
134impl FromArrowArray<&ArrowPrimitiveArray<Decimal256Type>> for ArrayRef {
135    fn from_arrow(array: &ArrowPrimitiveArray<Decimal256Type>, nullable: bool) -> Self {
136        let decimal_type = DecimalDType::new(array.precision(), array.scale());
137        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
138        // SAFETY: Our i256 implementation has the same bit-pattern representation of the
139        //  arrow_buffer::i256 type. It is safe to treat values held inside the buffer as values
140        //  of either type.
141        let buffer =
142            unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(buffer) };
143        let validity = nulls(array.nulls(), nullable);
144        DecimalArray::new(buffer, decimal_type, validity).into_array()
145    }
146}
147
148macro_rules! impl_from_arrow_temporal {
149    ($T:path) => {
150        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
151            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
152                temporal_array(value, nullable)
153            }
154        }
155    };
156}
157
158// timestamp
159impl_from_arrow_temporal!(TimestampSecondType);
160impl_from_arrow_temporal!(TimestampMillisecondType);
161impl_from_arrow_temporal!(TimestampMicrosecondType);
162impl_from_arrow_temporal!(TimestampNanosecondType);
163
164// time
165impl_from_arrow_temporal!(Time32SecondType);
166impl_from_arrow_temporal!(Time32MillisecondType);
167impl_from_arrow_temporal!(Time64MicrosecondType);
168impl_from_arrow_temporal!(Time64NanosecondType);
169
170// date
171impl_from_arrow_temporal!(Date32Type);
172impl_from_arrow_temporal!(Date64Type);
173
174fn temporal_array<T: ArrowPrimitiveType>(value: &ArrowPrimitiveArray<T>, nullable: bool) -> ArrayRef
175where
176    T::Native: NativePType,
177{
178    let arr = PrimitiveArray::new(
179        Buffer::from_arrow_scalar_buffer(value.values().clone()),
180        nulls(value.nulls(), nullable),
181    )
182    .into_array();
183
184    match value.data_type() {
185        DataType::Timestamp(time_unit, tz) => {
186            let tz = tz.as_ref().map(|s| s.to_string());
187            TemporalArray::new_timestamp(arr, time_unit.into(), tz).into()
188        }
189        DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
190        DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
191        DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::Days).into(),
192        DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Milliseconds).into(),
193        DataType::Duration(_) => unimplemented!(),
194        DataType::Interval(_) => unimplemented!(),
195        _ => vortex_panic!("Invalid temporal type: {}", value.data_type()),
196    }
197}
198
199impl<T: ByteArrayType> FromArrowArray<&GenericByteArray<T>> for ArrayRef
200where
201    <T as ByteArrayType>::Offset: IntegerPType,
202{
203    fn from_arrow(value: &GenericByteArray<T>, nullable: bool) -> Self {
204        let dtype = match T::DATA_TYPE {
205            DataType::Binary | DataType::LargeBinary => DType::Binary(nullable.into()),
206            DataType::Utf8 | DataType::LargeUtf8 => DType::Utf8(nullable.into()),
207            dt => vortex_panic!("Invalid data type for ByteArray: {dt}"),
208        };
209        VarBinArray::try_new(
210            value.offsets().clone().into_array(),
211            ByteBuffer::from_arrow_buffer(value.values().clone(), Alignment::of::<u8>()),
212            dtype,
213            nulls(value.nulls(), nullable),
214        )
215        .vortex_expect("Failed to convert Arrow GenericByteArray to Vortex VarBinArray")
216        .into_array()
217    }
218}
219
220impl<T: ByteViewType> FromArrowArray<&GenericByteViewArray<T>> for ArrayRef {
221    fn from_arrow(value: &GenericByteViewArray<T>, nullable: bool) -> Self {
222        let dtype = match T::DATA_TYPE {
223            DataType::BinaryView => DType::Binary(nullable.into()),
224            DataType::Utf8View => DType::Utf8(nullable.into()),
225            dt => vortex_panic!("Invalid data type for ByteViewArray: {dt}"),
226        };
227
228        let views_buffer = Buffer::from_byte_buffer(
229            Buffer::from_arrow_scalar_buffer(value.views().clone()).into_byte_buffer(),
230        );
231
232        // SAFETY: arrow-rs ByteViewArray already checks the same invariants, we inherit those
233        //  guarantees by zero-copy constructing from one.
234        unsafe {
235            VarBinViewArray::new_unchecked(
236                views_buffer,
237                Arc::from(
238                    value
239                        .data_buffers()
240                        .iter()
241                        .map(|b| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::<u8>()))
242                        .collect::<Vec<_>>(),
243                ),
244                dtype,
245                nulls(value.nulls(), nullable),
246            )
247            .into_array()
248        }
249    }
250}
251
252impl FromArrowArray<&ArrowBooleanArray> for ArrayRef {
253    fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self {
254        BoolArray::from_bit_buffer(
255            value.values().clone().into(),
256            nulls(value.nulls(), nullable),
257        )
258        .into_array()
259    }
260}
261
262/// Strip out the nulls from this array and return a new array without nulls.
263fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData {
264    if data.null_count() == 0 {
265        // No nulls to remove, return the array as is
266        return data;
267    }
268
269    let children = match data.data_type() {
270        DataType::Struct(fields) => Some(
271            fields
272                .iter()
273                .zip(data.child_data().iter())
274                .map(|(field, child_data)| {
275                    if field.is_nullable() {
276                        child_data.clone()
277                    } else {
278                        remove_nulls(child_data.clone())
279                    }
280                })
281                .collect_vec(),
282        ),
283        DataType::List(f)
284        | DataType::LargeList(f)
285        | DataType::ListView(f)
286        | DataType::LargeListView(f)
287        | DataType::FixedSizeList(f, _)
288            if !f.is_nullable() =>
289        {
290            // All list types only have one child
291            assert_eq!(
292                data.child_data().len(),
293                1,
294                "List types should have one child"
295            );
296            Some(vec![remove_nulls(data.child_data()[0].clone())])
297        }
298        _ => None,
299    };
300
301    let mut builder = data.into_builder().nulls(None);
302    if let Some(children) = children {
303        builder = builder.child_data(children);
304    }
305    builder
306        .build()
307        .vortex_expect("reconstructing array without nulls")
308}
309
310impl FromArrowArray<&ArrowStructArray> for ArrayRef {
311    fn from_arrow(value: &ArrowStructArray, nullable: bool) -> Self {
312        StructArray::try_new(
313            value.column_names().iter().copied().collect(),
314            value
315                .columns()
316                .iter()
317                .zip(value.fields())
318                .map(|(c, field)| {
319                    // Arrow pushes down nulls, even into non-nullable fields. So we strip them
320                    // out here because Vortex is a little more strict.
321                    if c.null_count() > 0 && !field.is_nullable() {
322                        let stripped = make_array(remove_nulls(c.into_data()));
323                        Self::from_arrow(stripped.as_ref(), false)
324                    } else {
325                        Self::from_arrow(c.as_ref(), field.is_nullable())
326                    }
327                })
328                .collect::<Vec<_>>(),
329            value.len(),
330            nulls(value.nulls(), nullable),
331        )
332        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
333        .into_array()
334    }
335}
336
337impl<O: IntegerPType + OffsetSizeTrait> FromArrowArray<&GenericListArray<O>> for ArrayRef {
338    fn from_arrow(value: &GenericListArray<O>, nullable: bool) -> Self {
339        // Extract the validity of the underlying element array.
340        let elements_are_nullable = match value.data_type() {
341            DataType::List(field) => field.is_nullable(),
342            DataType::LargeList(field) => field.is_nullable(),
343            dt => vortex_panic!("Invalid data type for ListArray: {dt}"),
344        };
345
346        let elements = Self::from_arrow(value.values().as_ref(), elements_are_nullable);
347
348        // `offsets` are always non-nullable.
349        let offsets = value.offsets().clone().into_array();
350        let nulls = nulls(value.nulls(), nullable);
351
352        ListArray::try_new(elements, offsets, nulls)
353            .vortex_expect("Failed to convert Arrow ListArray to Vortex ListArray")
354            .into_array()
355    }
356}
357
358impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListViewArray<O>> for ArrayRef {
359    fn from_arrow(array: &GenericListViewArray<O>, nullable: bool) -> Self {
360        // Extract the validity of the underlying element array.
361        let elements_are_nullable = match array.data_type() {
362            DataType::ListView(field) => field.is_nullable(),
363            DataType::LargeListView(field) => field.is_nullable(),
364            dt => vortex_panic!("Invalid data type for ListViewArray: {dt}"),
365        };
366
367        let elements = Self::from_arrow(array.values().as_ref(), elements_are_nullable);
368
369        // `offsets` and `sizes` are always non-nullable.
370        let offsets = array.offsets().clone().into_array();
371        let sizes = array.sizes().clone().into_array();
372        let nulls = nulls(array.nulls(), nullable);
373
374        ListViewArray::try_new(elements, offsets, sizes, nulls)
375            .vortex_expect("Failed to convert Arrow ListViewArray to Vortex ListViewArray")
376            .into_array()
377    }
378}
379
380impl FromArrowArray<&ArrowFixedSizeListArray> for ArrayRef {
381    fn from_arrow(array: &ArrowFixedSizeListArray, nullable: bool) -> Self {
382        let DataType::FixedSizeList(field, list_size) = array.data_type() else {
383            vortex_panic!("Invalid data type for ListArray: {}", array.data_type());
384        };
385
386        FixedSizeListArray::try_new(
387            Self::from_arrow(array.values().as_ref(), field.is_nullable()),
388            *list_size as u32,
389            nulls(array.nulls(), nullable),
390            array.len(),
391        )
392        .vortex_expect("Failed to convert Arrow FixedSizeListArray to Vortex FixedSizeListArray")
393        .into_array()
394    }
395}
396
397impl FromArrowArray<&ArrowNullArray> for ArrayRef {
398    fn from_arrow(value: &ArrowNullArray, nullable: bool) -> Self {
399        assert!(nullable);
400        NullArray::new(value.len()).into_array()
401    }
402}
403
404fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity {
405    if nullable {
406        nulls
407            .map(|nulls| {
408                if nulls.null_count() == nulls.len() {
409                    Validity::AllInvalid
410                } else {
411                    Validity::from(BitBuffer::from(nulls.inner().clone()))
412                }
413            })
414            .unwrap_or_else(|| Validity::AllValid)
415    } else {
416        assert!(nulls.map(|x| x.null_count() == 0).unwrap_or(true));
417        Validity::NonNullable
418    }
419}
420
421impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
422    fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> Self {
423        match array.data_type() {
424            DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable),
425            DataType::UInt8 => Self::from_arrow(array.as_primitive::<UInt8Type>(), nullable),
426            DataType::UInt16 => Self::from_arrow(array.as_primitive::<UInt16Type>(), nullable),
427            DataType::UInt32 => Self::from_arrow(array.as_primitive::<UInt32Type>(), nullable),
428            DataType::UInt64 => Self::from_arrow(array.as_primitive::<UInt64Type>(), nullable),
429            DataType::Int8 => Self::from_arrow(array.as_primitive::<Int8Type>(), nullable),
430            DataType::Int16 => Self::from_arrow(array.as_primitive::<Int16Type>(), nullable),
431            DataType::Int32 => Self::from_arrow(array.as_primitive::<Int32Type>(), nullable),
432            DataType::Int64 => Self::from_arrow(array.as_primitive::<Int64Type>(), nullable),
433            DataType::Float16 => Self::from_arrow(array.as_primitive::<Float16Type>(), nullable),
434            DataType::Float32 => Self::from_arrow(array.as_primitive::<Float32Type>(), nullable),
435            DataType::Float64 => Self::from_arrow(array.as_primitive::<Float64Type>(), nullable),
436            DataType::Utf8 => Self::from_arrow(array.as_string::<i32>(), nullable),
437            DataType::LargeUtf8 => Self::from_arrow(array.as_string::<i64>(), nullable),
438            DataType::Binary => Self::from_arrow(array.as_binary::<i32>(), nullable),
439            DataType::LargeBinary => Self::from_arrow(array.as_binary::<i64>(), nullable),
440            DataType::BinaryView => Self::from_arrow(array.as_binary_view(), nullable),
441            DataType::Utf8View => Self::from_arrow(array.as_string_view(), nullable),
442            DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),
443            DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),
444            DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),
445            DataType::ListView(_) => Self::from_arrow(array.as_list_view::<i32>(), nullable),
446            DataType::LargeListView(_) => Self::from_arrow(array.as_list_view::<i64>(), nullable),
447            DataType::FixedSizeList(..) => Self::from_arrow(array.as_fixed_size_list(), nullable),
448            DataType::Null => Self::from_arrow(as_null_array(array), nullable),
449            DataType::Timestamp(u, _) => match u {
450                ArrowTimeUnit::Second => {
451                    Self::from_arrow(array.as_primitive::<TimestampSecondType>(), nullable)
452                }
453                ArrowTimeUnit::Millisecond => {
454                    Self::from_arrow(array.as_primitive::<TimestampMillisecondType>(), nullable)
455                }
456                ArrowTimeUnit::Microsecond => {
457                    Self::from_arrow(array.as_primitive::<TimestampMicrosecondType>(), nullable)
458                }
459                ArrowTimeUnit::Nanosecond => {
460                    Self::from_arrow(array.as_primitive::<TimestampNanosecondType>(), nullable)
461                }
462            },
463            DataType::Date32 => Self::from_arrow(array.as_primitive::<Date32Type>(), nullable),
464            DataType::Date64 => Self::from_arrow(array.as_primitive::<Date64Type>(), nullable),
465            DataType::Time32(u) => match u {
466                ArrowTimeUnit::Second => {
467                    Self::from_arrow(array.as_primitive::<Time32SecondType>(), nullable)
468                }
469                ArrowTimeUnit::Millisecond => {
470                    Self::from_arrow(array.as_primitive::<Time32MillisecondType>(), nullable)
471                }
472                ArrowTimeUnit::Microsecond | ArrowTimeUnit::Nanosecond => unreachable!(),
473            },
474            DataType::Time64(u) => match u {
475                ArrowTimeUnit::Microsecond => {
476                    Self::from_arrow(array.as_primitive::<Time64MicrosecondType>(), nullable)
477                }
478                ArrowTimeUnit::Nanosecond => {
479                    Self::from_arrow(array.as_primitive::<Time64NanosecondType>(), nullable)
480                }
481                ArrowTimeUnit::Second | ArrowTimeUnit::Millisecond => unreachable!(),
482            },
483            DataType::Decimal32(..) => {
484                Self::from_arrow(array.as_primitive::<Decimal32Type>(), nullable)
485            }
486            DataType::Decimal64(..) => {
487                Self::from_arrow(array.as_primitive::<Decimal64Type>(), nullable)
488            }
489            DataType::Decimal128(..) => {
490                Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)
491            }
492            DataType::Decimal256(..) => {
493                Self::from_arrow(array.as_primitive::<Decimal256Type>(), nullable)
494            }
495            DataType::Dictionary(key_type, _) => match key_type.as_ref() {
496                DataType::Int8 => {
497                    DictArray::from_arrow(array.as_dictionary::<Int8Type>(), nullable).into_array()
498                }
499                DataType::Int16 => {
500                    DictArray::from_arrow(array.as_dictionary::<Int16Type>(), nullable).into_array()
501                }
502                DataType::Int32 => {
503                    DictArray::from_arrow(array.as_dictionary::<Int32Type>(), nullable).into_array()
504                }
505                DataType::Int64 => {
506                    DictArray::from_arrow(array.as_dictionary::<Int64Type>(), nullable).into_array()
507                }
508                DataType::UInt8 => {
509                    DictArray::from_arrow(array.as_dictionary::<UInt8Type>(), nullable).into_array()
510                }
511                DataType::UInt16 => {
512                    DictArray::from_arrow(array.as_dictionary::<UInt16Type>(), nullable)
513                        .into_array()
514                }
515                DataType::UInt32 => {
516                    DictArray::from_arrow(array.as_dictionary::<UInt32Type>(), nullable)
517                        .into_array()
518                }
519                DataType::UInt64 => {
520                    DictArray::from_arrow(array.as_dictionary::<UInt64Type>(), nullable)
521                        .into_array()
522                }
523                key_dt => vortex_panic!("Unsupported dictionary key type: {key_dt}"),
524            },
525            dt => vortex_panic!("Array encoding not implemented for Arrow data type {dt}"),
526        }
527    }
528}
529
530impl FromArrowArray<RecordBatch> for ArrayRef {
531    fn from_arrow(array: RecordBatch, nullable: bool) -> Self {
532        ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable)
533    }
534}
535
536impl FromArrowArray<&RecordBatch> for ArrayRef {
537    fn from_arrow(array: &RecordBatch, nullable: bool) -> Self {
538        Self::from_arrow(array.clone(), nullable)
539    }
540}
541
542#[cfg(test)]
543mod tests {
544    use std::sync::Arc;
545
546    use arrow_array::builder::{
547        BinaryViewBuilder, Decimal128Builder, Decimal256Builder, Int32Builder, LargeListBuilder,
548        ListBuilder, StringViewBuilder,
549    };
550    use arrow_array::types::{ArrowPrimitiveType, Float16Type};
551    use arrow_array::{
552        Array as ArrowArray, BinaryArray, BooleanArray, Date32Array, Date64Array,
553        FixedSizeListArray as ArrowFixedSizeListArray, Float32Array, Float64Array,
554        GenericListViewArray, Int8Array, Int16Array, Int32Array, Int64Array, LargeBinaryArray,
555        LargeStringArray, NullArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray,
556        Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
557        TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
558        TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, new_null_array,
559    };
560    use arrow_buffer::{BooleanBuffer, Buffer as ArrowBuffer, OffsetBuffer, ScalarBuffer};
561    use arrow_schema::{DataType, Field, Fields, Schema};
562    use vortex_dtype::datetime::{TIMESTAMP_ID, TemporalMetadata, TimeUnit};
563    use vortex_dtype::{DType, ExtDType, Nullability, PType};
564
565    use crate::arrays::{
566        DecimalVTable, FixedSizeListVTable, ListVTable, ListViewVTable, PrimitiveVTable,
567        StructVTable, TemporalArray, VarBinVTable, VarBinViewVTable,
568    };
569    use crate::arrow::FromArrowArray as _;
570    use crate::{ArrayRef, IntoArray};
571
572    // Test primitive array conversions
573    #[test]
574    fn test_int8_array_conversion() {
575        let arrow_array = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]);
576        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
577
578        let arrow_array_non_null = Int8Array::from(vec![1, 2, 3, 4]);
579        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
580
581        assert_eq!(vortex_array.len(), 4);
582        assert_eq!(vortex_array_non_null.len(), 4);
583
584        // Verify metadata - should be PrimitiveArray with I8 ptype
585        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
586        assert_eq!(primitive_array.ptype(), PType::I8);
587
588        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
589        assert_eq!(primitive_array_non_null.ptype(), PType::I8);
590    }
591
592    #[test]
593    fn test_int16_array_conversion() {
594        let arrow_array = Int16Array::from(vec![Some(100), None, Some(300), Some(400)]);
595        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
596
597        let arrow_array_non_null = Int16Array::from(vec![100, 200, 300, 400]);
598        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
599
600        assert_eq!(vortex_array.len(), 4);
601        assert_eq!(vortex_array_non_null.len(), 4);
602
603        // Verify metadata - should be PrimitiveArray with I16 ptype
604        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
605        assert_eq!(primitive_array.ptype(), PType::I16);
606
607        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
608        assert_eq!(primitive_array_non_null.ptype(), PType::I16);
609    }
610
611    #[test]
612    fn test_int32_array_conversion() {
613        let arrow_array = Int32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
614        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
615
616        let arrow_array_non_null = Int32Array::from(vec![1000, 2000, 3000, 4000]);
617        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
618
619        assert_eq!(vortex_array.len(), 4);
620        assert_eq!(vortex_array_non_null.len(), 4);
621
622        // Verify metadata - should be PrimitiveArray with I32 ptype
623        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
624        assert_eq!(primitive_array.ptype(), PType::I32);
625
626        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
627        assert_eq!(primitive_array_non_null.ptype(), PType::I32);
628    }
629
630    #[test]
631    fn test_int64_array_conversion() {
632        let arrow_array = Int64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
633        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
634
635        let arrow_array_non_null = Int64Array::from(vec![10000_i64, 20000, 30000, 40000]);
636        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
637
638        assert_eq!(vortex_array.len(), 4);
639        assert_eq!(vortex_array_non_null.len(), 4);
640
641        // Verify metadata - should be PrimitiveArray with I64 ptype
642        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
643        assert_eq!(primitive_array.ptype(), PType::I64);
644
645        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
646        assert_eq!(primitive_array_non_null.ptype(), PType::I64);
647    }
648
649    #[test]
650    fn test_uint8_array_conversion() {
651        let arrow_array = UInt8Array::from(vec![Some(1), None, Some(3), Some(4)]);
652        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
653
654        let arrow_array_non_null = UInt8Array::from(vec![1_u8, 2, 3, 4]);
655        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
656
657        assert_eq!(vortex_array.len(), 4);
658        assert_eq!(vortex_array_non_null.len(), 4);
659
660        // Verify metadata - should be PrimitiveArray with U8 ptype
661        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
662        assert_eq!(primitive_array.ptype(), PType::U8);
663
664        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
665        assert_eq!(primitive_array_non_null.ptype(), PType::U8);
666    }
667
668    #[test]
669    fn test_uint16_array_conversion() {
670        let arrow_array = UInt16Array::from(vec![Some(100), None, Some(300), Some(400)]);
671        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
672
673        let arrow_array_non_null = UInt16Array::from(vec![100_u16, 200, 300, 400]);
674        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
675
676        assert_eq!(vortex_array.len(), 4);
677        assert_eq!(vortex_array_non_null.len(), 4);
678
679        // Verify metadata - should be PrimitiveArray with U16 ptype
680        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
681        assert_eq!(primitive_array.ptype(), PType::U16);
682
683        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
684        assert_eq!(primitive_array_non_null.ptype(), PType::U16);
685    }
686
687    #[test]
688    fn test_uint32_array_conversion() {
689        let arrow_array = UInt32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
690        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
691
692        let arrow_array_non_null = UInt32Array::from(vec![1000_u32, 2000, 3000, 4000]);
693        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
694
695        assert_eq!(vortex_array.len(), 4);
696        assert_eq!(vortex_array_non_null.len(), 4);
697
698        // Verify metadata - should be PrimitiveArray with U32 ptype
699        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
700        assert_eq!(primitive_array.ptype(), PType::U32);
701
702        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
703        assert_eq!(primitive_array_non_null.ptype(), PType::U32);
704    }
705
706    #[test]
707    fn test_uint64_array_conversion() {
708        let arrow_array = UInt64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
709        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
710
711        let arrow_array_non_null = UInt64Array::from(vec![10000_u64, 20000, 30000, 40000]);
712        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
713
714        assert_eq!(vortex_array.len(), 4);
715        assert_eq!(vortex_array_non_null.len(), 4);
716
717        // Verify metadata - should be PrimitiveArray with U64 ptype
718        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
719        assert_eq!(primitive_array.ptype(), PType::U64);
720
721        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
722        assert_eq!(primitive_array_non_null.ptype(), PType::U64);
723    }
724
725    #[test]
726    fn test_float16_array_conversion() {
727        let values = vec![
728            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5)),
729            None,
730            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(3.5)),
731        ];
732        let arrow_array = arrow_array::PrimitiveArray::<Float16Type>::from(values);
733        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
734
735        let non_null_values = vec![
736            <Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5),
737            <Float16Type as ArrowPrimitiveType>::Native::from_f32(2.5),
738        ];
739        let arrow_array_non_null =
740            arrow_array::PrimitiveArray::<Float16Type>::from(non_null_values);
741        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
742
743        assert_eq!(vortex_array.len(), 3);
744        assert_eq!(vortex_array_non_null.len(), 2);
745
746        // Verify metadata - should be PrimitiveArray with F16 ptype
747        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
748        assert_eq!(primitive_array.ptype(), PType::F16);
749
750        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
751        assert_eq!(primitive_array_non_null.ptype(), PType::F16);
752    }
753
754    #[test]
755    fn test_float32_array_conversion() {
756        let arrow_array = Float32Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
757        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
758
759        let arrow_array_non_null = Float32Array::from(vec![1.5_f32, 2.5, 3.5, 4.5]);
760        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
761
762        assert_eq!(vortex_array.len(), 4);
763        assert_eq!(vortex_array_non_null.len(), 4);
764
765        // Verify metadata - should be PrimitiveArray with F32 ptype
766        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
767        assert_eq!(primitive_array.ptype(), PType::F32);
768
769        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
770        assert_eq!(primitive_array_non_null.ptype(), PType::F32);
771    }
772
773    #[test]
774    fn test_float64_array_conversion() {
775        let arrow_array = Float64Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
776        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
777
778        let arrow_array_non_null = Float64Array::from(vec![1.5_f64, 2.5, 3.5, 4.5]);
779        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
780
781        assert_eq!(vortex_array.len(), 4);
782        assert_eq!(vortex_array_non_null.len(), 4);
783
784        // Verify metadata - should be PrimitiveArray with F64 ptype
785        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
786        assert_eq!(primitive_array.ptype(), PType::F64);
787
788        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
789        assert_eq!(primitive_array_non_null.ptype(), PType::F64);
790    }
791
792    // Test decimal array conversions
793    #[test]
794    fn test_decimal128_array_conversion() {
795        let mut builder = Decimal128Builder::with_capacity(4);
796        builder.append_value(12345);
797        builder.append_null();
798        builder.append_value(67890);
799        builder.append_value(11111);
800        let decimal_array = builder.finish().with_precision_and_scale(10, 2).unwrap();
801
802        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
803        assert_eq!(vortex_array.len(), 4);
804
805        let mut builder_non_null = Decimal128Builder::with_capacity(3);
806        builder_non_null.append_value(12345);
807        builder_non_null.append_value(67890);
808        builder_non_null.append_value(11111);
809        let decimal_array_non_null = builder_non_null
810            .finish()
811            .with_precision_and_scale(10, 2)
812            .unwrap();
813
814        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
815        assert_eq!(vortex_array_non_null.len(), 3);
816
817        // Verify metadata - should be DecimalArray with correct precision and scale
818        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
819        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 10);
820        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 2);
821
822        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
823        assert_eq!(
824            decimal_vortex_array_non_null.decimal_dtype().precision(),
825            10
826        );
827        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 2);
828    }
829
830    #[test]
831    fn test_decimal256_array_conversion() {
832        let mut builder = Decimal256Builder::with_capacity(4);
833        builder.append_value(arrow_buffer::i256::from_i128(12345));
834        builder.append_null();
835        builder.append_value(arrow_buffer::i256::from_i128(67890));
836        builder.append_value(arrow_buffer::i256::from_i128(11111));
837        let decimal_array = builder.finish().with_precision_and_scale(38, 10).unwrap();
838
839        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
840        assert_eq!(vortex_array.len(), 4);
841
842        let mut builder_non_null = Decimal256Builder::with_capacity(3);
843        builder_non_null.append_value(arrow_buffer::i256::from_i128(12345));
844        builder_non_null.append_value(arrow_buffer::i256::from_i128(67890));
845        builder_non_null.append_value(arrow_buffer::i256::from_i128(11111));
846        let decimal_array_non_null = builder_non_null
847            .finish()
848            .with_precision_and_scale(38, 10)
849            .unwrap();
850
851        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
852        assert_eq!(vortex_array_non_null.len(), 3);
853
854        // Verify metadata - should be DecimalArray with correct precision and scale
855        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
856        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 38);
857        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 10);
858
859        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
860        assert_eq!(
861            decimal_vortex_array_non_null.decimal_dtype().precision(),
862            38
863        );
864        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 10);
865    }
866
867    // Test temporal array conversions
868    #[test]
869    fn test_timestamp_second_array_conversion() {
870        let arrow_array =
871            TimestampSecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
872        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
873
874        let arrow_array_non_null = TimestampSecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
875        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
876
877        assert_eq!(vortex_array.len(), 4);
878        assert_eq!(vortex_array_non_null.len(), 4);
879
880        // Verify metadata - should be TemporalArray with Second time unit
881        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
882        assert_eq!(
883            temporal_array.temporal_metadata().time_unit(),
884            TimeUnit::Seconds
885        );
886
887        let temporal_array_non_null =
888            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
889        assert_eq!(
890            temporal_array_non_null.temporal_metadata().time_unit(),
891            TimeUnit::Seconds
892        );
893    }
894
895    #[test]
896    fn test_timestamp_millisecond_array_conversion() {
897        let arrow_array =
898            TimestampMillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
899        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
900
901        let arrow_array_non_null =
902            TimestampMillisecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
903        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
904
905        assert_eq!(vortex_array.len(), 4);
906        assert_eq!(vortex_array_non_null.len(), 4);
907    }
908
909    #[test]
910    fn test_timestamp_microsecond_array_conversion() {
911        let arrow_array =
912            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
913        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
914
915        let arrow_array_non_null =
916            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
917        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
918
919        assert_eq!(vortex_array.len(), 4);
920        assert_eq!(vortex_array_non_null.len(), 4);
921    }
922
923    #[test]
924    fn test_timestamp_timezone_microsecond_array_conversion() {
925        let arrow_array =
926            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)])
927                .with_timezone("UTC");
928        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
929
930        let arrow_array_non_null =
931            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]).with_timezone("UTC");
932        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
933
934        assert_eq!(vortex_array.len(), 4);
935        assert_eq!(
936            vortex_array.dtype(),
937            &DType::Extension(Arc::new(ExtDType::new(
938                TIMESTAMP_ID.clone(),
939                Arc::new(DType::Primitive(PType::I64, Nullability::Nullable)),
940                Some(
941                    TemporalMetadata::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string()))
942                        .into()
943                )
944            )))
945        );
946        assert_eq!(vortex_array_non_null.len(), 4);
947        assert_eq!(
948            vortex_array_non_null.dtype(),
949            &DType::Extension(Arc::new(ExtDType::new(
950                TIMESTAMP_ID.clone(),
951                Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)),
952                Some(
953                    TemporalMetadata::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string()))
954                        .into()
955                )
956            )))
957        );
958    }
959
960    #[test]
961    fn test_timestamp_nanosecond_array_conversion() {
962        let arrow_array =
963            TimestampNanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
964        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
965
966        let arrow_array_non_null = TimestampNanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
967        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
968
969        assert_eq!(vortex_array.len(), 4);
970        assert_eq!(vortex_array_non_null.len(), 4);
971    }
972
973    #[test]
974    fn test_time32_second_array_conversion() {
975        let arrow_array = Time32SecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
976        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
977
978        let arrow_array_non_null = Time32SecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
979        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
980
981        assert_eq!(vortex_array.len(), 4);
982        assert_eq!(vortex_array_non_null.len(), 4);
983
984        // Verify metadata - should be TemporalArray with Second time unit
985        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
986        assert_eq!(
987            temporal_array.temporal_metadata().time_unit(),
988            TimeUnit::Seconds
989        );
990
991        let temporal_array_non_null =
992            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
993        assert_eq!(
994            temporal_array_non_null.temporal_metadata().time_unit(),
995            TimeUnit::Seconds
996        );
997    }
998
999    #[test]
1000    fn test_time32_millisecond_array_conversion() {
1001        let arrow_array =
1002            Time32MillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
1003        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1004
1005        let arrow_array_non_null = Time32MillisecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
1006        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1007
1008        assert_eq!(vortex_array.len(), 4);
1009        assert_eq!(vortex_array_non_null.len(), 4);
1010    }
1011
1012    #[test]
1013    fn test_time64_microsecond_array_conversion() {
1014        let arrow_array =
1015            Time64MicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
1016        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1017
1018        let arrow_array_non_null = Time64MicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
1019        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1020
1021        assert_eq!(vortex_array.len(), 4);
1022        assert_eq!(vortex_array_non_null.len(), 4);
1023    }
1024
1025    #[test]
1026    fn test_time64_nanosecond_array_conversion() {
1027        let arrow_array =
1028            Time64NanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
1029        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1030
1031        let arrow_array_non_null = Time64NanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
1032        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1033
1034        assert_eq!(vortex_array.len(), 4);
1035        assert_eq!(vortex_array_non_null.len(), 4);
1036    }
1037
1038    #[test]
1039    fn test_date32_array_conversion() {
1040        let arrow_array = Date32Array::from(vec![Some(18000), None, Some(18002), Some(18003)]);
1041        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1042
1043        let arrow_array_non_null = Date32Array::from(vec![18000_i32, 18001, 18002, 18003]);
1044        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1045
1046        assert_eq!(vortex_array.len(), 4);
1047        assert_eq!(vortex_array_non_null.len(), 4);
1048    }
1049
1050    #[test]
1051    fn test_date64_array_conversion() {
1052        let arrow_array = Date64Array::from(vec![
1053            Some(1555200000000),
1054            None,
1055            Some(1555286400000),
1056            Some(1555372800000),
1057        ]);
1058        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1059
1060        let arrow_array_non_null = Date64Array::from(vec![
1061            1555200000000_i64,
1062            1555213600000,
1063            1555286400000,
1064            1555372800000,
1065        ]);
1066        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1067
1068        assert_eq!(vortex_array.len(), 4);
1069        assert_eq!(vortex_array_non_null.len(), 4);
1070    }
1071
1072    // Test string/binary array conversions
1073    #[test]
1074    fn test_utf8_array_conversion() {
1075        let arrow_array = StringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
1076        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1077
1078        let arrow_array_non_null = StringArray::from(vec!["hello", "world", "test", "vortex"]);
1079        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1080
1081        assert_eq!(vortex_array.len(), 4);
1082        assert_eq!(vortex_array_non_null.len(), 4);
1083
1084        // Verify metadata - should be VarBinArray with Utf8 dtype
1085        let varbin_array = vortex_array.as_::<VarBinVTable>();
1086        assert_eq!(varbin_array.dtype(), &DType::Utf8(true.into()));
1087
1088        let varbin_array_non_null = vortex_array_non_null.as_::<VarBinVTable>();
1089        assert_eq!(varbin_array_non_null.dtype(), &DType::Utf8(false.into()));
1090    }
1091
1092    #[test]
1093    fn test_large_utf8_array_conversion() {
1094        let arrow_array =
1095            LargeStringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
1096        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1097
1098        let arrow_array_non_null = LargeStringArray::from(vec!["hello", "world", "test", "vortex"]);
1099        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1100
1101        assert_eq!(vortex_array.len(), 4);
1102        assert_eq!(vortex_array_non_null.len(), 4);
1103    }
1104
1105    #[test]
1106    fn test_binary_array_conversion() {
1107        let arrow_array = BinaryArray::from(vec![
1108            Some("hello".as_bytes()),
1109            None,
1110            Some("world".as_bytes()),
1111            Some("test".as_bytes()),
1112        ]);
1113        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1114
1115        let arrow_array_non_null = BinaryArray::from(vec![
1116            "hello".as_bytes(),
1117            "world".as_bytes(),
1118            "test".as_bytes(),
1119            "vortex".as_bytes(),
1120        ]);
1121        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1122
1123        assert_eq!(vortex_array.len(), 4);
1124        assert_eq!(vortex_array_non_null.len(), 4);
1125    }
1126
1127    #[test]
1128    fn test_large_binary_array_conversion() {
1129        let arrow_array = LargeBinaryArray::from(vec![
1130            Some("hello".as_bytes()),
1131            None,
1132            Some("world".as_bytes()),
1133            Some("test".as_bytes()),
1134        ]);
1135        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1136
1137        let arrow_array_non_null = LargeBinaryArray::from(vec![
1138            "hello".as_bytes(),
1139            "world".as_bytes(),
1140            "test".as_bytes(),
1141            "vortex".as_bytes(),
1142        ]);
1143        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1144
1145        assert_eq!(vortex_array.len(), 4);
1146        assert_eq!(vortex_array_non_null.len(), 4);
1147    }
1148
1149    #[test]
1150    fn test_utf8_view_array_conversion() {
1151        let mut builder = StringViewBuilder::new();
1152        builder.append_value("hello");
1153        builder.append_null();
1154        builder.append_value("world");
1155        builder.append_value("test");
1156        let arrow_array = builder.finish();
1157        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1158
1159        let mut builder_non_null = StringViewBuilder::new();
1160        builder_non_null.append_value("hello");
1161        builder_non_null.append_value("world");
1162        builder_non_null.append_value("test");
1163        builder_non_null.append_value("vortex");
1164        let arrow_array_non_null = builder_non_null.finish();
1165        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1166
1167        assert_eq!(vortex_array.len(), 4);
1168        assert_eq!(vortex_array_non_null.len(), 4);
1169
1170        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1171        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1172        assert_eq!(
1173            varbin_view_array.buffers().len(),
1174            arrow_array.data_buffers().len()
1175        );
1176        assert_eq!(varbin_view_array.dtype(), &DType::Utf8(true.into()));
1177
1178        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1179        assert_eq!(
1180            varbin_view_array_non_null.buffers().len(),
1181            arrow_array_non_null.data_buffers().len()
1182        );
1183        assert_eq!(
1184            varbin_view_array_non_null.dtype(),
1185            &DType::Utf8(false.into())
1186        );
1187    }
1188
1189    #[test]
1190    fn test_binary_view_array_conversion() {
1191        let mut builder = BinaryViewBuilder::new();
1192        builder.append_value(b"hello");
1193        builder.append_null();
1194        builder.append_value(b"world");
1195        builder.append_value(b"test");
1196        let arrow_array = builder.finish();
1197        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1198
1199        let mut builder_non_null = BinaryViewBuilder::new();
1200        builder_non_null.append_value(b"hello");
1201        builder_non_null.append_value(b"world");
1202        builder_non_null.append_value(b"test");
1203        builder_non_null.append_value(b"vortex");
1204        let arrow_array_non_null = builder_non_null.finish();
1205        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1206
1207        assert_eq!(vortex_array.len(), 4);
1208        assert_eq!(vortex_array_non_null.len(), 4);
1209
1210        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1211        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1212        assert_eq!(
1213            varbin_view_array.buffers().len(),
1214            arrow_array.data_buffers().len()
1215        );
1216        assert_eq!(varbin_view_array.dtype(), &DType::Binary(true.into()));
1217
1218        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1219        assert_eq!(
1220            varbin_view_array_non_null.buffers().len(),
1221            arrow_array_non_null.data_buffers().len()
1222        );
1223        assert_eq!(
1224            varbin_view_array_non_null.dtype(),
1225            &DType::Binary(false.into())
1226        );
1227    }
1228
1229    // Test boolean array conversions
1230    #[test]
1231    fn test_boolean_array_conversion() {
1232        let arrow_array = BooleanArray::from(vec![Some(true), None, Some(false), Some(true)]);
1233        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1234
1235        let arrow_array_non_null = BooleanArray::from(vec![true, false, true, false]);
1236        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1237
1238        assert_eq!(vortex_array.len(), 4);
1239        assert_eq!(vortex_array_non_null.len(), 4);
1240    }
1241
1242    // Test struct array conversions
1243    #[test]
1244    fn test_struct_array_conversion() {
1245        let fields = vec![
1246            Field::new("field1", DataType::Int32, true),
1247            Field::new("field2", DataType::Utf8, false),
1248        ];
1249        let schema = Fields::from(fields);
1250
1251        let field1_data = Int32Array::from(vec![Some(1), None, Some(3)]);
1252        let field2_data = StringArray::from(vec!["a", "b", "c"]);
1253
1254        let arrow_array = StructArray::new(
1255            schema.clone(),
1256            vec![Arc::new(field1_data), Arc::new(field2_data)],
1257            None,
1258        );
1259
1260        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1261        assert_eq!(vortex_array.len(), 3);
1262
1263        // Verify metadata - should be StructArray with correct field names
1264        let struct_vortex_array = vortex_array.as_::<StructVTable>();
1265        assert_eq!(struct_vortex_array.names().len(), 2);
1266        assert_eq!(struct_vortex_array.names()[0], "field1");
1267        assert_eq!(struct_vortex_array.names()[1], "field2");
1268
1269        // Test nullable struct
1270        let nullable_array = StructArray::new(
1271            schema,
1272            vec![
1273                Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
1274                Arc::new(StringArray::from(vec!["a", "b", "c"])),
1275            ],
1276            Some(arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![
1277                true, false, true,
1278            ]))),
1279        );
1280
1281        let vortex_nullable_array = ArrayRef::from_arrow(&nullable_array, true);
1282        assert_eq!(vortex_nullable_array.len(), 3);
1283
1284        // Verify metadata for nullable struct
1285        let struct_vortex_nullable_array = vortex_nullable_array.as_::<StructVTable>();
1286        assert_eq!(struct_vortex_nullable_array.names().len(), 2);
1287        assert_eq!(struct_vortex_nullable_array.names()[0], "field1");
1288        assert_eq!(struct_vortex_nullable_array.names()[1], "field2");
1289    }
1290
1291    // Test list array conversions
1292    #[test]
1293    fn test_list_array_conversion() {
1294        let mut builder = ListBuilder::new(Int32Builder::new());
1295        builder.append_value([Some(1), None, Some(3)]);
1296        builder.append_null();
1297        builder.append_value([Some(4), Some(5)]);
1298        let arrow_array = builder.finish();
1299
1300        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1301        assert_eq!(vortex_array.len(), 3);
1302
1303        // Verify metadata - should be ListArray with correct offsets
1304        let list_vortex_array = vortex_array.as_::<ListVTable>();
1305        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1306        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1307        assert_eq!(offsets_array.ptype(), PType::I32);
1308
1309        // Test non-nullable list
1310        let mut builder_non_null = ListBuilder::new(Int32Builder::new());
1311        builder_non_null.append_value([Some(1), None, Some(3)]);
1312        builder_non_null.append_value([Some(4), Some(5)]);
1313        let arrow_array_non_null = builder_non_null.finish();
1314
1315        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1316        assert_eq!(vortex_array_non_null.len(), 2);
1317
1318        // Verify metadata for non-nullable list
1319        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1320        let offsets_array_non_null = list_vortex_array_non_null
1321            .offsets()
1322            .as_::<PrimitiveVTable>();
1323        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1324        assert_eq!(offsets_array_non_null.ptype(), PType::I32);
1325    }
1326
1327    #[test]
1328    fn test_large_list_array_conversion() {
1329        let mut builder = LargeListBuilder::new(Int32Builder::new());
1330        builder.append_value([Some(1), None, Some(3)]);
1331        builder.append_null();
1332        builder.append_value([Some(4), Some(5)]);
1333        let arrow_array = builder.finish();
1334
1335        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1336        assert_eq!(vortex_array.len(), 3);
1337
1338        // Verify metadata - should be ListArray with correct offsets (I64 for large lists)
1339        let list_vortex_array = vortex_array.as_::<ListVTable>();
1340        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1341        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1342        assert_eq!(offsets_array.ptype(), PType::I64); // Large lists use I64 offsets
1343
1344        // Test non-nullable large list
1345        let mut builder_non_null = LargeListBuilder::new(Int32Builder::new());
1346        builder_non_null.append_value([Some(1), None, Some(3)]);
1347        builder_non_null.append_value([Some(4), Some(5)]);
1348        let arrow_array_non_null = builder_non_null.finish();
1349
1350        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1351        assert_eq!(vortex_array_non_null.len(), 2);
1352
1353        // Verify metadata for non-nullable large list
1354        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1355        let offsets_array_non_null = list_vortex_array_non_null
1356            .offsets()
1357            .as_::<PrimitiveVTable>();
1358        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1359        assert_eq!(offsets_array_non_null.ptype(), PType::I64); // Large lists use I64 offsets
1360    }
1361
1362    #[test]
1363    fn test_fixed_size_list_array_conversion() {
1364        // Create elements for the fixed-size lists
1365        let values = Int32Array::from(vec![
1366            Some(1),
1367            Some(2),
1368            Some(3), // First list
1369            Some(4),
1370            None,
1371            Some(6), // Second list (with null element)
1372            Some(7),
1373            Some(8),
1374            Some(9), // Third list
1375            Some(10),
1376            Some(11),
1377            Some(12), // Fourth list
1378        ]);
1379
1380        // Create a FixedSizeListArray with list_size=3
1381        let field = Arc::new(Field::new("item", DataType::Int32, true));
1382        let arrow_array =
1383            ArrowFixedSizeListArray::try_new(field.clone(), 3, Arc::new(values), None).unwrap();
1384        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1385
1386        assert_eq!(vortex_array.len(), 4);
1387
1388        // Verify metadata - should be FixedSizeListArray with correct list size
1389        let fsl_vortex_array = vortex_array.as_::<FixedSizeListVTable>();
1390        assert_eq!(fsl_vortex_array.list_size(), 3);
1391        assert_eq!(fsl_vortex_array.elements().len(), 12); // 4 lists * 3 elements
1392
1393        // Test nullable fixed-size list
1394        let values_nullable = Int32Array::from(vec![
1395            Some(1),
1396            Some(2),
1397            Some(3), // First list
1398            Some(4),
1399            None,
1400            Some(6), // Second list (will be null)
1401            Some(7),
1402            Some(8),
1403            Some(9), // Third list
1404        ]);
1405
1406        // Create nulls buffer - second list is null
1407        let null_buffer =
1408            arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![true, false, true]));
1409
1410        let arrow_array_nullable = ArrowFixedSizeListArray::try_new(
1411            field,
1412            3,
1413            Arc::new(values_nullable),
1414            Some(null_buffer),
1415        )
1416        .unwrap();
1417        let vortex_array_nullable = ArrayRef::from_arrow(&arrow_array_nullable, true);
1418
1419        assert_eq!(vortex_array_nullable.len(), 3);
1420
1421        // Verify metadata for nullable array
1422        let fsl_vortex_array_nullable = vortex_array_nullable.as_::<FixedSizeListVTable>();
1423        assert_eq!(fsl_vortex_array_nullable.list_size(), 3);
1424        assert_eq!(fsl_vortex_array_nullable.elements().len(), 9); // 3 lists * 3 elements
1425    }
1426
1427    #[test]
1428    fn test_list_view_array_conversion() {
1429        // Create values array for the lists
1430        let values = Int32Array::from(vec![
1431            Some(1),
1432            Some(2),
1433            Some(3), // First list [1, 2, 3]
1434            Some(4),
1435            Some(5), // Second list [4, 5]
1436            Some(6), // Third list [6]
1437            Some(7),
1438            Some(8),
1439            Some(9),
1440            Some(10), // Fourth list [7, 8, 9, 10]
1441        ]);
1442
1443        // Create offsets and sizes for ListView
1444        let offsets = ScalarBuffer::from(vec![0i32, 3, 5, 6]);
1445        let sizes = ScalarBuffer::from(vec![3i32, 2, 1, 4]);
1446
1447        let field = Arc::new(Field::new("item", DataType::Int32, true));
1448        let arrow_array = GenericListViewArray::try_new(
1449            field.clone(),
1450            offsets.clone(),
1451            sizes.clone(),
1452            Arc::new(values.clone()),
1453            None,
1454        )
1455        .unwrap();
1456
1457        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1458        assert_eq!(vortex_array.len(), 4);
1459
1460        // Verify metadata - should be ListViewArray with correct offsets and sizes
1461        let list_view_vortex_array = vortex_array.as_::<ListViewVTable>();
1462        let offsets_array = list_view_vortex_array.offsets().as_::<PrimitiveVTable>();
1463        let sizes_array = list_view_vortex_array.sizes().as_::<PrimitiveVTable>();
1464
1465        assert_eq!(offsets_array.len(), 4);
1466        assert_eq!(offsets_array.ptype(), PType::I32);
1467        assert_eq!(sizes_array.len(), 4);
1468        assert_eq!(sizes_array.ptype(), PType::I32);
1469
1470        // Test nullable ListView
1471        let null_buffer =
1472            arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![true, false, true, true]));
1473
1474        let arrow_array_nullable = GenericListViewArray::try_new(
1475            field.clone(),
1476            offsets,
1477            sizes,
1478            Arc::new(values.clone()),
1479            Some(null_buffer),
1480        )
1481        .unwrap();
1482
1483        let vortex_array_nullable = ArrayRef::from_arrow(&arrow_array_nullable, true);
1484        assert_eq!(vortex_array_nullable.len(), 4);
1485
1486        // Test LargeListView (i64 offsets and sizes)
1487        let large_offsets = ScalarBuffer::from(vec![0i64, 3, 5, 6]);
1488        let large_sizes = ScalarBuffer::from(vec![3i64, 2, 1, 4]);
1489
1490        let large_arrow_array = GenericListViewArray::try_new(
1491            field,
1492            large_offsets,
1493            large_sizes,
1494            Arc::new(values),
1495            None,
1496        )
1497        .unwrap();
1498
1499        let large_vortex_array = ArrayRef::from_arrow(&large_arrow_array, false);
1500        assert_eq!(large_vortex_array.len(), 4);
1501
1502        // Verify metadata for large ListView
1503        let large_list_view_vortex_array = large_vortex_array.as_::<ListViewVTable>();
1504        let large_offsets_array = large_list_view_vortex_array
1505            .offsets()
1506            .as_::<PrimitiveVTable>();
1507        let large_sizes_array = large_list_view_vortex_array
1508            .sizes()
1509            .as_::<PrimitiveVTable>();
1510
1511        assert_eq!(large_offsets_array.len(), 4);
1512        assert_eq!(large_offsets_array.ptype(), PType::I64); // Large ListView uses I64 offsets
1513        assert_eq!(large_sizes_array.len(), 4);
1514        assert_eq!(large_sizes_array.ptype(), PType::I64); // Large ListView uses I64 sizes
1515    }
1516
1517    // Test null array conversions
1518    #[test]
1519    fn test_null_array_conversion() {
1520        let arrow_array = NullArray::new(5);
1521        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1522        assert_eq!(vortex_array.len(), 5);
1523    }
1524
1525    // Test buffer conversions
1526    #[test]
1527    fn test_arrow_buffer_conversion() {
1528        let data = vec![1u8, 2, 3, 4, 5];
1529        let arrow_buffer = ArrowBuffer::from_vec(data);
1530        let vortex_array = arrow_buffer.into_array();
1531        assert_eq!(vortex_array.len(), 5);
1532    }
1533
1534    #[test]
1535    fn test_boolean_buffer_conversion() {
1536        let data = vec![true, false, true, false, true];
1537        let boolean_buffer = BooleanBuffer::from(data);
1538        let vortex_array = boolean_buffer.into_array();
1539        assert_eq!(vortex_array.len(), 5);
1540    }
1541
1542    #[test]
1543    fn test_scalar_buffer_conversion() {
1544        let data = vec![1i32, 2, 3, 4, 5];
1545        let scalar_buffer = ScalarBuffer::from(data);
1546        let vortex_array = scalar_buffer.into_array();
1547        assert_eq!(vortex_array.len(), 5);
1548    }
1549
1550    #[test]
1551    fn test_offset_buffer_conversion() {
1552        let data = vec![0i32, 2, 5, 8, 10];
1553        let offset_buffer = OffsetBuffer::new(ScalarBuffer::from(data));
1554        let vortex_array = offset_buffer.into_array();
1555        assert_eq!(vortex_array.len(), 5);
1556    }
1557
1558    // Test RecordBatch conversions
1559    #[test]
1560    fn test_record_batch_conversion() {
1561        let schema = Arc::new(Schema::new(vec![
1562            Field::new("field1", DataType::Int32, false),
1563            Field::new("field2", DataType::Utf8, false),
1564        ]));
1565
1566        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1567        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1568
1569        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1570
1571        let vortex_array = ArrayRef::from_arrow(record_batch, false);
1572        assert_eq!(vortex_array.len(), 4);
1573
1574        // Test with reference
1575        let schema = Arc::new(Schema::new(vec![
1576            Field::new("field1", DataType::Int32, false),
1577            Field::new("field2", DataType::Utf8, false),
1578        ]));
1579
1580        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1581        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1582
1583        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1584
1585        let vortex_array = ArrayRef::from_arrow(&record_batch, false);
1586        assert_eq!(vortex_array.len(), 4);
1587    }
1588
1589    // Test dynamic dispatch conversion
1590    #[test]
1591    fn test_dyn_array_conversion() {
1592        let int_array = Int32Array::from(vec![1, 2, 3, 4]);
1593        let dyn_array: &dyn ArrowArray = &int_array;
1594        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1595        assert_eq!(vortex_array.len(), 4);
1596
1597        let string_array = StringArray::from(vec!["a", "b", "c"]);
1598        let dyn_array: &dyn ArrowArray = &string_array;
1599        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1600        assert_eq!(vortex_array.len(), 3);
1601
1602        let bool_array = BooleanArray::from(vec![true, false, true]);
1603        let dyn_array: &dyn ArrowArray = &bool_array;
1604        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1605        assert_eq!(vortex_array.len(), 3);
1606    }
1607
1608    // Existing tests
1609    #[test]
1610    pub fn nullable_may_contain_non_nullable() {
1611        let null_struct_array_with_non_nullable_field = new_null_array(
1612            &DataType::Struct(Fields::from(vec![Field::new(
1613                "non_nullable_inner",
1614                DataType::Int32,
1615                false,
1616            )])),
1617            1,
1618        );
1619        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1620    }
1621
1622    #[test]
1623    pub fn nullable_may_contain_deeply_nested_non_nullable() {
1624        let null_struct_array_with_non_nullable_field = new_null_array(
1625            &DataType::Struct(Fields::from(vec![Field::new(
1626                "non_nullable_inner",
1627                DataType::Struct(Fields::from(vec![Field::new(
1628                    "non_nullable_deeper_inner",
1629                    DataType::Int32,
1630                    false,
1631                )])),
1632                false,
1633            )])),
1634            1,
1635        );
1636        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1637    }
1638
1639    #[test]
1640    #[should_panic]
1641    pub fn cannot_handle_nullable_struct_containing_non_nullable_dictionary() {
1642        let null_struct_array_with_non_nullable_field = new_null_array(
1643            &DataType::Struct(Fields::from(vec![Field::new(
1644                "non_nullable_deeper_inner",
1645                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1646                false,
1647            )])),
1648            1,
1649        );
1650
1651        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1652    }
1653}
vortex_array/arrow/convert.rs

vortex_array/arrow/
convert.rs
