vortex_array/arrow/
convert.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::sync::Arc;
5
6use arrow_array::cast::{AsArray, as_null_array};
7use arrow_array::types::{
8    ByteArrayType, ByteViewType, Date32Type, Date64Type, Decimal32Type, Decimal64Type,
9    Decimal128Type, Decimal256Type, Float16Type, Float32Type, Float64Type, Int8Type, Int16Type,
10    Int32Type, Int64Type, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
11    Time64NanosecondType, TimestampMicrosecondType, TimestampMillisecondType,
12    TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
13};
14use arrow_array::{
15    Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray,
16    FixedSizeListArray as ArrowFixedSizeListArray, GenericByteArray, GenericByteViewArray,
17    GenericListArray, GenericListViewArray, NullArray as ArrowNullArray, OffsetSizeTrait,
18    PrimitiveArray as ArrowPrimitiveArray, RecordBatch, StructArray as ArrowStructArray,
19    make_array,
20};
21use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
22use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
23use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
24use itertools::Itertools;
25use vortex_buffer::{Alignment, Buffer, ByteBuffer};
26use vortex_dtype::datetime::TimeUnit;
27use vortex_dtype::{DType, DecimalDType, IntegerPType, NativePType, PType};
28use vortex_error::{VortexExpect as _, vortex_panic};
29use vortex_scalar::i256;
30
31use crate::arrays::{
32    BoolArray, DecimalArray, FixedSizeListArray, ListArray, ListViewArray, NullArray,
33    PrimitiveArray, StructArray, TemporalArray, VarBinArray, VarBinViewArray,
34};
35use crate::arrow::FromArrowArray;
36use crate::validity::Validity;
37use crate::{ArrayRef, IntoArray};
38
39impl IntoArray for ArrowBuffer {
40    fn into_array(self) -> ArrayRef {
41        PrimitiveArray::from_byte_buffer(
42            ByteBuffer::from_arrow_buffer(self, Alignment::of::<u8>()),
43            PType::U8,
44            Validity::NonNullable,
45        )
46        .into_array()
47    }
48}
49
50impl IntoArray for BooleanBuffer {
51    fn into_array(self) -> ArrayRef {
52        BoolArray::from_bool_buffer(self, Validity::NonNullable).into_array()
53    }
54}
55
56impl<T> IntoArray for ScalarBuffer<T>
57where
58    T: ArrowNativeType + NativePType,
59{
60    fn into_array(self) -> ArrayRef {
61        PrimitiveArray::new(
62            Buffer::<T>::from_arrow_scalar_buffer(self),
63            Validity::NonNullable,
64        )
65        .into_array()
66    }
67}
68
69impl<O> IntoArray for OffsetBuffer<O>
70where
71    O: IntegerPType + OffsetSizeTrait,
72{
73    fn into_array(self) -> ArrayRef {
74        let primitive = PrimitiveArray::new(
75            Buffer::from_arrow_scalar_buffer(self.into_inner()),
76            Validity::NonNullable,
77        );
78
79        primitive.into_array()
80    }
81}
82
83macro_rules! impl_from_arrow_primitive {
84    ($T:path) => {
85        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
86            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
87                let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone());
88                let validity = nulls(value.nulls(), nullable);
89                PrimitiveArray::new(buffer, validity).into_array()
90            }
91        }
92    };
93}
94
95impl_from_arrow_primitive!(Int8Type);
96impl_from_arrow_primitive!(Int16Type);
97impl_from_arrow_primitive!(Int32Type);
98impl_from_arrow_primitive!(Int64Type);
99impl_from_arrow_primitive!(UInt8Type);
100impl_from_arrow_primitive!(UInt16Type);
101impl_from_arrow_primitive!(UInt32Type);
102impl_from_arrow_primitive!(UInt64Type);
103impl_from_arrow_primitive!(Float16Type);
104impl_from_arrow_primitive!(Float32Type);
105impl_from_arrow_primitive!(Float64Type);
106
107impl FromArrowArray<&ArrowPrimitiveArray<Decimal32Type>> for ArrayRef {
108    fn from_arrow(array: &ArrowPrimitiveArray<Decimal32Type>, nullable: bool) -> Self {
109        let decimal_type = DecimalDType::new(array.precision(), array.scale());
110        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
111        let validity = nulls(array.nulls(), nullable);
112        DecimalArray::new(buffer, decimal_type, validity).into_array()
113    }
114}
115
116impl FromArrowArray<&ArrowPrimitiveArray<Decimal64Type>> for ArrayRef {
117    fn from_arrow(array: &ArrowPrimitiveArray<Decimal64Type>, nullable: bool) -> Self {
118        let decimal_type = DecimalDType::new(array.precision(), array.scale());
119        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
120        let validity = nulls(array.nulls(), nullable);
121        DecimalArray::new(buffer, decimal_type, validity).into_array()
122    }
123}
124
125impl FromArrowArray<&ArrowPrimitiveArray<Decimal128Type>> for ArrayRef {
126    fn from_arrow(array: &ArrowPrimitiveArray<Decimal128Type>, nullable: bool) -> Self {
127        let decimal_type = DecimalDType::new(array.precision(), array.scale());
128        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
129        let validity = nulls(array.nulls(), nullable);
130        DecimalArray::new(buffer, decimal_type, validity).into_array()
131    }
132}
133
134impl FromArrowArray<&ArrowPrimitiveArray<Decimal256Type>> for ArrayRef {
135    fn from_arrow(array: &ArrowPrimitiveArray<Decimal256Type>, nullable: bool) -> Self {
136        let decimal_type = DecimalDType::new(array.precision(), array.scale());
137        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
138        // SAFETY: Our i256 implementation has the same bit-pattern representation of the
139        //  arrow_buffer::i256 type. It is safe to treat values held inside the buffer as values
140        //  of either type.
141        let buffer =
142            unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(buffer) };
143        let validity = nulls(array.nulls(), nullable);
144        DecimalArray::new(buffer, decimal_type, validity).into_array()
145    }
146}
147
148macro_rules! impl_from_arrow_temporal {
149    ($T:path) => {
150        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
151            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
152                temporal_array(value, nullable)
153            }
154        }
155    };
156}
157
158// timestamp
159impl_from_arrow_temporal!(TimestampSecondType);
160impl_from_arrow_temporal!(TimestampMillisecondType);
161impl_from_arrow_temporal!(TimestampMicrosecondType);
162impl_from_arrow_temporal!(TimestampNanosecondType);
163
164// time
165impl_from_arrow_temporal!(Time32SecondType);
166impl_from_arrow_temporal!(Time32MillisecondType);
167impl_from_arrow_temporal!(Time64MicrosecondType);
168impl_from_arrow_temporal!(Time64NanosecondType);
169
170// date
171impl_from_arrow_temporal!(Date32Type);
172impl_from_arrow_temporal!(Date64Type);
173
174fn temporal_array<T: ArrowPrimitiveType>(value: &ArrowPrimitiveArray<T>, nullable: bool) -> ArrayRef
175where
176    T::Native: NativePType,
177{
178    let arr = PrimitiveArray::new(
179        Buffer::from_arrow_scalar_buffer(value.values().clone()),
180        nulls(value.nulls(), nullable),
181    )
182    .into_array();
183
184    match value.data_type() {
185        DataType::Timestamp(time_unit, tz) => {
186            let tz = tz.as_ref().map(|s| s.to_string());
187            TemporalArray::new_timestamp(arr, time_unit.into(), tz).into()
188        }
189        DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
190        DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
191        DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::Days).into(),
192        DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Milliseconds).into(),
193        DataType::Duration(_) => unimplemented!(),
194        DataType::Interval(_) => unimplemented!(),
195        _ => vortex_panic!("Invalid temporal type: {}", value.data_type()),
196    }
197}
198
199impl<T: ByteArrayType> FromArrowArray<&GenericByteArray<T>> for ArrayRef
200where
201    <T as ByteArrayType>::Offset: IntegerPType,
202{
203    fn from_arrow(value: &GenericByteArray<T>, nullable: bool) -> Self {
204        let dtype = match T::DATA_TYPE {
205            DataType::Binary | DataType::LargeBinary => DType::Binary(nullable.into()),
206            DataType::Utf8 | DataType::LargeUtf8 => DType::Utf8(nullable.into()),
207            dt => vortex_panic!("Invalid data type for ByteArray: {dt}"),
208        };
209        VarBinArray::try_new(
210            value.offsets().clone().into_array(),
211            ByteBuffer::from_arrow_buffer(value.values().clone(), Alignment::of::<u8>()),
212            dtype,
213            nulls(value.nulls(), nullable),
214        )
215        .vortex_expect("Failed to convert Arrow GenericByteArray to Vortex VarBinArray")
216        .into_array()
217    }
218}
219
220impl<T: ByteViewType> FromArrowArray<&GenericByteViewArray<T>> for ArrayRef {
221    fn from_arrow(value: &GenericByteViewArray<T>, nullable: bool) -> Self {
222        let dtype = match T::DATA_TYPE {
223            DataType::BinaryView => DType::Binary(nullable.into()),
224            DataType::Utf8View => DType::Utf8(nullable.into()),
225            dt => vortex_panic!("Invalid data type for ByteViewArray: {dt}"),
226        };
227
228        let views_buffer = Buffer::from_byte_buffer(
229            Buffer::from_arrow_scalar_buffer(value.views().clone()).into_byte_buffer(),
230        );
231
232        // SAFETY: arrow-rs ByteViewArray already checks the same invariants, we inherit those
233        //  guarantees by zero-copy constructing from one.
234        unsafe {
235            VarBinViewArray::new_unchecked(
236                views_buffer,
237                Arc::from(
238                    value
239                        .data_buffers()
240                        .iter()
241                        .map(|b| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::<u8>()))
242                        .collect::<Vec<_>>(),
243                ),
244                dtype,
245                nulls(value.nulls(), nullable),
246            )
247            .into_array()
248        }
249    }
250}
251
252impl FromArrowArray<&ArrowBooleanArray> for ArrayRef {
253    fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self {
254        BoolArray::from_bool_buffer(value.values().clone(), nulls(value.nulls(), nullable))
255            .into_array()
256    }
257}
258
259/// Strip out the nulls from this array and return a new array without nulls.
260fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData {
261    if data.null_count() == 0 {
262        // No nulls to remove, return the array as is
263        return data;
264    }
265
266    let children = match data.data_type() {
267        DataType::Struct(fields) => Some(
268            fields
269                .iter()
270                .zip(data.child_data().iter())
271                .map(|(field, child_data)| {
272                    if field.is_nullable() {
273                        child_data.clone()
274                    } else {
275                        remove_nulls(child_data.clone())
276                    }
277                })
278                .collect_vec(),
279        ),
280        DataType::List(f)
281        | DataType::LargeList(f)
282        | DataType::ListView(f)
283        | DataType::LargeListView(f)
284        | DataType::FixedSizeList(f, _)
285            if !f.is_nullable() =>
286        {
287            // All list types only have one child
288            assert_eq!(
289                data.child_data().len(),
290                1,
291                "List types should have one child"
292            );
293            Some(vec![remove_nulls(data.child_data()[0].clone())])
294        }
295        _ => None,
296    };
297
298    let mut builder = data.into_builder().nulls(None);
299    if let Some(children) = children {
300        builder = builder.child_data(children);
301    }
302    builder
303        .build()
304        .vortex_expect("reconstructing array without nulls")
305}
306
307impl FromArrowArray<&ArrowStructArray> for ArrayRef {
308    fn from_arrow(value: &ArrowStructArray, nullable: bool) -> Self {
309        StructArray::try_new(
310            value.column_names().iter().copied().collect(),
311            value
312                .columns()
313                .iter()
314                .zip(value.fields())
315                .map(|(c, field)| {
316                    // Arrow pushes down nulls, even into non-nullable fields. So we strip them
317                    // out here because Vortex is a little more strict.
318                    if c.null_count() > 0 && !field.is_nullable() {
319                        let stripped = make_array(remove_nulls(c.into_data()));
320                        Self::from_arrow(stripped.as_ref(), false)
321                    } else {
322                        Self::from_arrow(c.as_ref(), field.is_nullable())
323                    }
324                })
325                .collect::<Vec<_>>(),
326            value.len(),
327            nulls(value.nulls(), nullable),
328        )
329        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
330        .into_array()
331    }
332}
333
334impl<O: IntegerPType + OffsetSizeTrait> FromArrowArray<&GenericListArray<O>> for ArrayRef {
335    fn from_arrow(value: &GenericListArray<O>, nullable: bool) -> Self {
336        // Extract the validity of the underlying element array.
337        let elements_are_nullable = match value.data_type() {
338            DataType::List(field) => field.is_nullable(),
339            DataType::LargeList(field) => field.is_nullable(),
340            dt => vortex_panic!("Invalid data type for ListArray: {dt}"),
341        };
342
343        let elements = Self::from_arrow(value.values().as_ref(), elements_are_nullable);
344
345        // `offsets` are always non-nullable.
346        let offsets = value.offsets().clone().into_array();
347        let nulls = nulls(value.nulls(), nullable);
348
349        ListArray::try_new(elements, offsets, nulls)
350            .vortex_expect("Failed to convert Arrow ListArray to Vortex ListArray")
351            .into_array()
352    }
353}
354
355impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListViewArray<O>> for ArrayRef {
356    fn from_arrow(array: &GenericListViewArray<O>, nullable: bool) -> Self {
357        // Extract the validity of the underlying element array.
358        let elements_are_nullable = match array.data_type() {
359            DataType::ListView(field) => field.is_nullable(),
360            DataType::LargeListView(field) => field.is_nullable(),
361            dt => vortex_panic!("Invalid data type for ListViewArray: {dt}"),
362        };
363
364        let elements = Self::from_arrow(array.values().as_ref(), elements_are_nullable);
365
366        // `offsets` and `sizes` are always non-nullable.
367        let offsets = array.offsets().clone().into_array();
368        let sizes = array.sizes().clone().into_array();
369        let nulls = nulls(array.nulls(), nullable);
370
371        ListViewArray::try_new(elements, offsets, sizes, nulls)
372            .vortex_expect("Failed to convert Arrow ListViewArray to Vortex ListViewArray")
373            .into_array()
374    }
375}
376
377impl FromArrowArray<&ArrowFixedSizeListArray> for ArrayRef {
378    fn from_arrow(array: &ArrowFixedSizeListArray, nullable: bool) -> Self {
379        let DataType::FixedSizeList(field, list_size) = array.data_type() else {
380            vortex_panic!("Invalid data type for ListArray: {}", array.data_type());
381        };
382
383        FixedSizeListArray::try_new(
384            Self::from_arrow(array.values().as_ref(), field.is_nullable()),
385            *list_size as u32,
386            nulls(array.nulls(), nullable),
387            array.len(),
388        )
389        .vortex_expect("Failed to convert Arrow FixedSizeListArray to Vortex FixedSizeListArray")
390        .into_array()
391    }
392}
393
394impl FromArrowArray<&ArrowNullArray> for ArrayRef {
395    fn from_arrow(value: &ArrowNullArray, nullable: bool) -> Self {
396        assert!(nullable);
397        NullArray::new(value.len()).into_array()
398    }
399}
400
401fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity {
402    if nullable {
403        nulls
404            .map(|nulls| {
405                if nulls.null_count() == nulls.len() {
406                    Validity::AllInvalid
407                } else {
408                    Validity::from(nulls.inner().clone())
409                }
410            })
411            .unwrap_or_else(|| Validity::AllValid)
412    } else {
413        assert!(nulls.map(|x| x.null_count() == 0).unwrap_or(true));
414        Validity::NonNullable
415    }
416}
417
418impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
419    fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> Self {
420        match array.data_type() {
421            DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable),
422            DataType::UInt8 => Self::from_arrow(array.as_primitive::<UInt8Type>(), nullable),
423            DataType::UInt16 => Self::from_arrow(array.as_primitive::<UInt16Type>(), nullable),
424            DataType::UInt32 => Self::from_arrow(array.as_primitive::<UInt32Type>(), nullable),
425            DataType::UInt64 => Self::from_arrow(array.as_primitive::<UInt64Type>(), nullable),
426            DataType::Int8 => Self::from_arrow(array.as_primitive::<Int8Type>(), nullable),
427            DataType::Int16 => Self::from_arrow(array.as_primitive::<Int16Type>(), nullable),
428            DataType::Int32 => Self::from_arrow(array.as_primitive::<Int32Type>(), nullable),
429            DataType::Int64 => Self::from_arrow(array.as_primitive::<Int64Type>(), nullable),
430            DataType::Float16 => Self::from_arrow(array.as_primitive::<Float16Type>(), nullable),
431            DataType::Float32 => Self::from_arrow(array.as_primitive::<Float32Type>(), nullable),
432            DataType::Float64 => Self::from_arrow(array.as_primitive::<Float64Type>(), nullable),
433            DataType::Utf8 => Self::from_arrow(array.as_string::<i32>(), nullable),
434            DataType::LargeUtf8 => Self::from_arrow(array.as_string::<i64>(), nullable),
435            DataType::Binary => Self::from_arrow(array.as_binary::<i32>(), nullable),
436            DataType::LargeBinary => Self::from_arrow(array.as_binary::<i64>(), nullable),
437            DataType::BinaryView => Self::from_arrow(array.as_binary_view(), nullable),
438            DataType::Utf8View => Self::from_arrow(array.as_string_view(), nullable),
439            DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),
440            DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),
441            DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),
442            DataType::ListView(_) => Self::from_arrow(array.as_list_view::<i32>(), nullable),
443            DataType::LargeListView(_) => Self::from_arrow(array.as_list_view::<i64>(), nullable),
444            DataType::FixedSizeList(..) => Self::from_arrow(array.as_fixed_size_list(), nullable),
445            DataType::Null => Self::from_arrow(as_null_array(array), nullable),
446            DataType::Timestamp(u, _) => match u {
447                ArrowTimeUnit::Second => {
448                    Self::from_arrow(array.as_primitive::<TimestampSecondType>(), nullable)
449                }
450                ArrowTimeUnit::Millisecond => {
451                    Self::from_arrow(array.as_primitive::<TimestampMillisecondType>(), nullable)
452                }
453                ArrowTimeUnit::Microsecond => {
454                    Self::from_arrow(array.as_primitive::<TimestampMicrosecondType>(), nullable)
455                }
456                ArrowTimeUnit::Nanosecond => {
457                    Self::from_arrow(array.as_primitive::<TimestampNanosecondType>(), nullable)
458                }
459            },
460            DataType::Date32 => Self::from_arrow(array.as_primitive::<Date32Type>(), nullable),
461            DataType::Date64 => Self::from_arrow(array.as_primitive::<Date64Type>(), nullable),
462            DataType::Time32(u) => match u {
463                ArrowTimeUnit::Second => {
464                    Self::from_arrow(array.as_primitive::<Time32SecondType>(), nullable)
465                }
466                ArrowTimeUnit::Millisecond => {
467                    Self::from_arrow(array.as_primitive::<Time32MillisecondType>(), nullable)
468                }
469                ArrowTimeUnit::Microsecond | ArrowTimeUnit::Nanosecond => unreachable!(),
470            },
471            DataType::Time64(u) => match u {
472                ArrowTimeUnit::Microsecond => {
473                    Self::from_arrow(array.as_primitive::<Time64MicrosecondType>(), nullable)
474                }
475                ArrowTimeUnit::Nanosecond => {
476                    Self::from_arrow(array.as_primitive::<Time64NanosecondType>(), nullable)
477                }
478                ArrowTimeUnit::Second | ArrowTimeUnit::Millisecond => unreachable!(),
479            },
480            DataType::Decimal32(..) => {
481                Self::from_arrow(array.as_primitive::<Decimal32Type>(), nullable)
482            }
483            DataType::Decimal64(..) => {
484                Self::from_arrow(array.as_primitive::<Decimal64Type>(), nullable)
485            }
486            DataType::Decimal128(..) => {
487                Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)
488            }
489            DataType::Decimal256(..) => {
490                Self::from_arrow(array.as_primitive::<Decimal256Type>(), nullable)
491            }
492            dt => vortex_panic!("Array encoding not implemented for Arrow data type {dt}"),
493        }
494    }
495}
496
497impl FromArrowArray<RecordBatch> for ArrayRef {
498    fn from_arrow(array: RecordBatch, nullable: bool) -> Self {
499        ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable)
500    }
501}
502
503impl FromArrowArray<&RecordBatch> for ArrayRef {
504    fn from_arrow(array: &RecordBatch, nullable: bool) -> Self {
505        Self::from_arrow(array.clone(), nullable)
506    }
507}
508
509#[cfg(test)]
510mod tests {
511    use std::sync::Arc;
512
513    use arrow_array::builder::{
514        BinaryViewBuilder, Decimal128Builder, Decimal256Builder, Int32Builder, LargeListBuilder,
515        ListBuilder, StringViewBuilder,
516    };
517    use arrow_array::types::{ArrowPrimitiveType, Float16Type};
518    use arrow_array::{
519        Array as ArrowArray, BinaryArray, BooleanArray, Date32Array, Date64Array,
520        FixedSizeListArray as ArrowFixedSizeListArray, Float32Array, Float64Array,
521        GenericListViewArray, Int8Array, Int16Array, Int32Array, Int64Array, LargeBinaryArray,
522        LargeStringArray, NullArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray,
523        Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
524        TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
525        TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, new_null_array,
526    };
527    use arrow_buffer::{BooleanBuffer, Buffer as ArrowBuffer, OffsetBuffer, ScalarBuffer};
528    use arrow_schema::{DataType, Field, Fields, Schema};
529    use vortex_dtype::datetime::{TIMESTAMP_ID, TemporalMetadata, TimeUnit};
530    use vortex_dtype::{DType, ExtDType, Nullability, PType};
531
532    use crate::arrays::{
533        DecimalVTable, FixedSizeListVTable, ListVTable, ListViewVTable, PrimitiveVTable,
534        StructVTable, TemporalArray, VarBinVTable, VarBinViewVTable,
535    };
536    use crate::arrow::FromArrowArray as _;
537    use crate::{ArrayRef, IntoArray};
538
539    // Test primitive array conversions
540    #[test]
541    fn test_int8_array_conversion() {
542        let arrow_array = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]);
543        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
544
545        let arrow_array_non_null = Int8Array::from(vec![1, 2, 3, 4]);
546        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
547
548        assert_eq!(vortex_array.len(), 4);
549        assert_eq!(vortex_array_non_null.len(), 4);
550
551        // Verify metadata - should be PrimitiveArray with I8 ptype
552        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
553        assert_eq!(primitive_array.ptype(), PType::I8);
554
555        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
556        assert_eq!(primitive_array_non_null.ptype(), PType::I8);
557    }
558
559    #[test]
560    fn test_int16_array_conversion() {
561        let arrow_array = Int16Array::from(vec![Some(100), None, Some(300), Some(400)]);
562        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
563
564        let arrow_array_non_null = Int16Array::from(vec![100, 200, 300, 400]);
565        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
566
567        assert_eq!(vortex_array.len(), 4);
568        assert_eq!(vortex_array_non_null.len(), 4);
569
570        // Verify metadata - should be PrimitiveArray with I16 ptype
571        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
572        assert_eq!(primitive_array.ptype(), PType::I16);
573
574        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
575        assert_eq!(primitive_array_non_null.ptype(), PType::I16);
576    }
577
578    #[test]
579    fn test_int32_array_conversion() {
580        let arrow_array = Int32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
581        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
582
583        let arrow_array_non_null = Int32Array::from(vec![1000, 2000, 3000, 4000]);
584        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
585
586        assert_eq!(vortex_array.len(), 4);
587        assert_eq!(vortex_array_non_null.len(), 4);
588
589        // Verify metadata - should be PrimitiveArray with I32 ptype
590        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
591        assert_eq!(primitive_array.ptype(), PType::I32);
592
593        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
594        assert_eq!(primitive_array_non_null.ptype(), PType::I32);
595    }
596
597    #[test]
598    fn test_int64_array_conversion() {
599        let arrow_array = Int64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
600        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
601
602        let arrow_array_non_null = Int64Array::from(vec![10000_i64, 20000, 30000, 40000]);
603        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
604
605        assert_eq!(vortex_array.len(), 4);
606        assert_eq!(vortex_array_non_null.len(), 4);
607
608        // Verify metadata - should be PrimitiveArray with I64 ptype
609        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
610        assert_eq!(primitive_array.ptype(), PType::I64);
611
612        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
613        assert_eq!(primitive_array_non_null.ptype(), PType::I64);
614    }
615
616    #[test]
617    fn test_uint8_array_conversion() {
618        let arrow_array = UInt8Array::from(vec![Some(1), None, Some(3), Some(4)]);
619        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
620
621        let arrow_array_non_null = UInt8Array::from(vec![1_u8, 2, 3, 4]);
622        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
623
624        assert_eq!(vortex_array.len(), 4);
625        assert_eq!(vortex_array_non_null.len(), 4);
626
627        // Verify metadata - should be PrimitiveArray with U8 ptype
628        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
629        assert_eq!(primitive_array.ptype(), PType::U8);
630
631        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
632        assert_eq!(primitive_array_non_null.ptype(), PType::U8);
633    }
634
635    #[test]
636    fn test_uint16_array_conversion() {
637        let arrow_array = UInt16Array::from(vec![Some(100), None, Some(300), Some(400)]);
638        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
639
640        let arrow_array_non_null = UInt16Array::from(vec![100_u16, 200, 300, 400]);
641        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
642
643        assert_eq!(vortex_array.len(), 4);
644        assert_eq!(vortex_array_non_null.len(), 4);
645
646        // Verify metadata - should be PrimitiveArray with U16 ptype
647        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
648        assert_eq!(primitive_array.ptype(), PType::U16);
649
650        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
651        assert_eq!(primitive_array_non_null.ptype(), PType::U16);
652    }
653
654    #[test]
655    fn test_uint32_array_conversion() {
656        let arrow_array = UInt32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
657        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
658
659        let arrow_array_non_null = UInt32Array::from(vec![1000_u32, 2000, 3000, 4000]);
660        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
661
662        assert_eq!(vortex_array.len(), 4);
663        assert_eq!(vortex_array_non_null.len(), 4);
664
665        // Verify metadata - should be PrimitiveArray with U32 ptype
666        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
667        assert_eq!(primitive_array.ptype(), PType::U32);
668
669        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
670        assert_eq!(primitive_array_non_null.ptype(), PType::U32);
671    }
672
673    #[test]
674    fn test_uint64_array_conversion() {
675        let arrow_array = UInt64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
676        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
677
678        let arrow_array_non_null = UInt64Array::from(vec![10000_u64, 20000, 30000, 40000]);
679        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
680
681        assert_eq!(vortex_array.len(), 4);
682        assert_eq!(vortex_array_non_null.len(), 4);
683
684        // Verify metadata - should be PrimitiveArray with U64 ptype
685        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
686        assert_eq!(primitive_array.ptype(), PType::U64);
687
688        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
689        assert_eq!(primitive_array_non_null.ptype(), PType::U64);
690    }
691
692    #[test]
693    fn test_float16_array_conversion() {
694        let values = vec![
695            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5)),
696            None,
697            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(3.5)),
698        ];
699        let arrow_array = arrow_array::PrimitiveArray::<Float16Type>::from(values);
700        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
701
702        let non_null_values = vec![
703            <Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5),
704            <Float16Type as ArrowPrimitiveType>::Native::from_f32(2.5),
705        ];
706        let arrow_array_non_null =
707            arrow_array::PrimitiveArray::<Float16Type>::from(non_null_values);
708        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
709
710        assert_eq!(vortex_array.len(), 3);
711        assert_eq!(vortex_array_non_null.len(), 2);
712
713        // Verify metadata - should be PrimitiveArray with F16 ptype
714        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
715        assert_eq!(primitive_array.ptype(), PType::F16);
716
717        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
718        assert_eq!(primitive_array_non_null.ptype(), PType::F16);
719    }
720
721    #[test]
722    fn test_float32_array_conversion() {
723        let arrow_array = Float32Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
724        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
725
726        let arrow_array_non_null = Float32Array::from(vec![1.5_f32, 2.5, 3.5, 4.5]);
727        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
728
729        assert_eq!(vortex_array.len(), 4);
730        assert_eq!(vortex_array_non_null.len(), 4);
731
732        // Verify metadata - should be PrimitiveArray with F32 ptype
733        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
734        assert_eq!(primitive_array.ptype(), PType::F32);
735
736        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
737        assert_eq!(primitive_array_non_null.ptype(), PType::F32);
738    }
739
740    #[test]
741    fn test_float64_array_conversion() {
742        let arrow_array = Float64Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
743        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
744
745        let arrow_array_non_null = Float64Array::from(vec![1.5_f64, 2.5, 3.5, 4.5]);
746        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
747
748        assert_eq!(vortex_array.len(), 4);
749        assert_eq!(vortex_array_non_null.len(), 4);
750
751        // Verify metadata - should be PrimitiveArray with F64 ptype
752        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
753        assert_eq!(primitive_array.ptype(), PType::F64);
754
755        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
756        assert_eq!(primitive_array_non_null.ptype(), PType::F64);
757    }
758
759    // Test decimal array conversions
760    #[test]
761    fn test_decimal128_array_conversion() {
762        let mut builder = Decimal128Builder::with_capacity(4);
763        builder.append_value(12345);
764        builder.append_null();
765        builder.append_value(67890);
766        builder.append_value(11111);
767        let decimal_array = builder.finish().with_precision_and_scale(10, 2).unwrap();
768
769        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
770        assert_eq!(vortex_array.len(), 4);
771
772        let mut builder_non_null = Decimal128Builder::with_capacity(3);
773        builder_non_null.append_value(12345);
774        builder_non_null.append_value(67890);
775        builder_non_null.append_value(11111);
776        let decimal_array_non_null = builder_non_null
777            .finish()
778            .with_precision_and_scale(10, 2)
779            .unwrap();
780
781        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
782        assert_eq!(vortex_array_non_null.len(), 3);
783
784        // Verify metadata - should be DecimalArray with correct precision and scale
785        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
786        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 10);
787        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 2);
788
789        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
790        assert_eq!(
791            decimal_vortex_array_non_null.decimal_dtype().precision(),
792            10
793        );
794        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 2);
795    }
796
797    #[test]
798    fn test_decimal256_array_conversion() {
799        let mut builder = Decimal256Builder::with_capacity(4);
800        builder.append_value(arrow_buffer::i256::from_i128(12345));
801        builder.append_null();
802        builder.append_value(arrow_buffer::i256::from_i128(67890));
803        builder.append_value(arrow_buffer::i256::from_i128(11111));
804        let decimal_array = builder.finish().with_precision_and_scale(38, 10).unwrap();
805
806        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
807        assert_eq!(vortex_array.len(), 4);
808
809        let mut builder_non_null = Decimal256Builder::with_capacity(3);
810        builder_non_null.append_value(arrow_buffer::i256::from_i128(12345));
811        builder_non_null.append_value(arrow_buffer::i256::from_i128(67890));
812        builder_non_null.append_value(arrow_buffer::i256::from_i128(11111));
813        let decimal_array_non_null = builder_non_null
814            .finish()
815            .with_precision_and_scale(38, 10)
816            .unwrap();
817
818        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
819        assert_eq!(vortex_array_non_null.len(), 3);
820
821        // Verify metadata - should be DecimalArray with correct precision and scale
822        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
823        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 38);
824        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 10);
825
826        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
827        assert_eq!(
828            decimal_vortex_array_non_null.decimal_dtype().precision(),
829            38
830        );
831        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 10);
832    }
833
834    // Test temporal array conversions
835    #[test]
836    fn test_timestamp_second_array_conversion() {
837        let arrow_array =
838            TimestampSecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
839        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
840
841        let arrow_array_non_null = TimestampSecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
842        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
843
844        assert_eq!(vortex_array.len(), 4);
845        assert_eq!(vortex_array_non_null.len(), 4);
846
847        // Verify metadata - should be TemporalArray with Second time unit
848        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
849        assert_eq!(
850            temporal_array.temporal_metadata().time_unit(),
851            TimeUnit::Seconds
852        );
853
854        let temporal_array_non_null =
855            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
856        assert_eq!(
857            temporal_array_non_null.temporal_metadata().time_unit(),
858            TimeUnit::Seconds
859        );
860    }
861
862    #[test]
863    fn test_timestamp_millisecond_array_conversion() {
864        let arrow_array =
865            TimestampMillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
866        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
867
868        let arrow_array_non_null =
869            TimestampMillisecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
870        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
871
872        assert_eq!(vortex_array.len(), 4);
873        assert_eq!(vortex_array_non_null.len(), 4);
874    }
875
876    #[test]
877    fn test_timestamp_microsecond_array_conversion() {
878        let arrow_array =
879            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
880        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
881
882        let arrow_array_non_null =
883            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
884        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
885
886        assert_eq!(vortex_array.len(), 4);
887        assert_eq!(vortex_array_non_null.len(), 4);
888    }
889
890    #[test]
891    fn test_timestamp_timezone_microsecond_array_conversion() {
892        let arrow_array =
893            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)])
894                .with_timezone("UTC");
895        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
896
897        let arrow_array_non_null =
898            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]).with_timezone("UTC");
899        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
900
901        assert_eq!(vortex_array.len(), 4);
902        assert_eq!(
903            vortex_array.dtype(),
904            &DType::Extension(Arc::new(ExtDType::new(
905                TIMESTAMP_ID.clone(),
906                Arc::new(DType::Primitive(PType::I64, Nullability::Nullable)),
907                Some(
908                    TemporalMetadata::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string()))
909                        .into()
910                )
911            )))
912        );
913        assert_eq!(vortex_array_non_null.len(), 4);
914        assert_eq!(
915            vortex_array_non_null.dtype(),
916            &DType::Extension(Arc::new(ExtDType::new(
917                TIMESTAMP_ID.clone(),
918                Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)),
919                Some(
920                    TemporalMetadata::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string()))
921                        .into()
922                )
923            )))
924        );
925    }
926
927    #[test]
928    fn test_timestamp_nanosecond_array_conversion() {
929        let arrow_array =
930            TimestampNanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
931        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
932
933        let arrow_array_non_null = TimestampNanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
934        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
935
936        assert_eq!(vortex_array.len(), 4);
937        assert_eq!(vortex_array_non_null.len(), 4);
938    }
939
940    #[test]
941    fn test_time32_second_array_conversion() {
942        let arrow_array = Time32SecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
943        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
944
945        let arrow_array_non_null = Time32SecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
946        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
947
948        assert_eq!(vortex_array.len(), 4);
949        assert_eq!(vortex_array_non_null.len(), 4);
950
951        // Verify metadata - should be TemporalArray with Second time unit
952        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
953        assert_eq!(
954            temporal_array.temporal_metadata().time_unit(),
955            TimeUnit::Seconds
956        );
957
958        let temporal_array_non_null =
959            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
960        assert_eq!(
961            temporal_array_non_null.temporal_metadata().time_unit(),
962            TimeUnit::Seconds
963        );
964    }
965
966    #[test]
967    fn test_time32_millisecond_array_conversion() {
968        let arrow_array =
969            Time32MillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
970        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
971
972        let arrow_array_non_null = Time32MillisecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
973        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
974
975        assert_eq!(vortex_array.len(), 4);
976        assert_eq!(vortex_array_non_null.len(), 4);
977    }
978
979    #[test]
980    fn test_time64_microsecond_array_conversion() {
981        let arrow_array =
982            Time64MicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
983        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
984
985        let arrow_array_non_null = Time64MicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
986        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
987
988        assert_eq!(vortex_array.len(), 4);
989        assert_eq!(vortex_array_non_null.len(), 4);
990    }
991
992    #[test]
993    fn test_time64_nanosecond_array_conversion() {
994        let arrow_array =
995            Time64NanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
996        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
997
998        let arrow_array_non_null = Time64NanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
999        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1000
1001        assert_eq!(vortex_array.len(), 4);
1002        assert_eq!(vortex_array_non_null.len(), 4);
1003    }
1004
1005    #[test]
1006    fn test_date32_array_conversion() {
1007        let arrow_array = Date32Array::from(vec![Some(18000), None, Some(18002), Some(18003)]);
1008        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1009
1010        let arrow_array_non_null = Date32Array::from(vec![18000_i32, 18001, 18002, 18003]);
1011        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1012
1013        assert_eq!(vortex_array.len(), 4);
1014        assert_eq!(vortex_array_non_null.len(), 4);
1015    }
1016
1017    #[test]
1018    fn test_date64_array_conversion() {
1019        let arrow_array = Date64Array::from(vec![
1020            Some(1555200000000),
1021            None,
1022            Some(1555286400000),
1023            Some(1555372800000),
1024        ]);
1025        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1026
1027        let arrow_array_non_null = Date64Array::from(vec![
1028            1555200000000_i64,
1029            1555213600000,
1030            1555286400000,
1031            1555372800000,
1032        ]);
1033        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1034
1035        assert_eq!(vortex_array.len(), 4);
1036        assert_eq!(vortex_array_non_null.len(), 4);
1037    }
1038
1039    // Test string/binary array conversions
1040    #[test]
1041    fn test_utf8_array_conversion() {
1042        let arrow_array = StringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
1043        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1044
1045        let arrow_array_non_null = StringArray::from(vec!["hello", "world", "test", "vortex"]);
1046        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1047
1048        assert_eq!(vortex_array.len(), 4);
1049        assert_eq!(vortex_array_non_null.len(), 4);
1050
1051        // Verify metadata - should be VarBinArray with Utf8 dtype
1052        let varbin_array = vortex_array.as_::<VarBinVTable>();
1053        assert_eq!(varbin_array.dtype(), &DType::Utf8(true.into()));
1054
1055        let varbin_array_non_null = vortex_array_non_null.as_::<VarBinVTable>();
1056        assert_eq!(varbin_array_non_null.dtype(), &DType::Utf8(false.into()));
1057    }
1058
1059    #[test]
1060    fn test_large_utf8_array_conversion() {
1061        let arrow_array =
1062            LargeStringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
1063        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1064
1065        let arrow_array_non_null = LargeStringArray::from(vec!["hello", "world", "test", "vortex"]);
1066        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1067
1068        assert_eq!(vortex_array.len(), 4);
1069        assert_eq!(vortex_array_non_null.len(), 4);
1070    }
1071
1072    #[test]
1073    fn test_binary_array_conversion() {
1074        let arrow_array = BinaryArray::from(vec![
1075            Some("hello".as_bytes()),
1076            None,
1077            Some("world".as_bytes()),
1078            Some("test".as_bytes()),
1079        ]);
1080        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1081
1082        let arrow_array_non_null = BinaryArray::from(vec![
1083            "hello".as_bytes(),
1084            "world".as_bytes(),
1085            "test".as_bytes(),
1086            "vortex".as_bytes(),
1087        ]);
1088        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1089
1090        assert_eq!(vortex_array.len(), 4);
1091        assert_eq!(vortex_array_non_null.len(), 4);
1092    }
1093
1094    #[test]
1095    fn test_large_binary_array_conversion() {
1096        let arrow_array = LargeBinaryArray::from(vec![
1097            Some("hello".as_bytes()),
1098            None,
1099            Some("world".as_bytes()),
1100            Some("test".as_bytes()),
1101        ]);
1102        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1103
1104        let arrow_array_non_null = LargeBinaryArray::from(vec![
1105            "hello".as_bytes(),
1106            "world".as_bytes(),
1107            "test".as_bytes(),
1108            "vortex".as_bytes(),
1109        ]);
1110        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1111
1112        assert_eq!(vortex_array.len(), 4);
1113        assert_eq!(vortex_array_non_null.len(), 4);
1114    }
1115
1116    #[test]
1117    fn test_utf8_view_array_conversion() {
1118        let mut builder = StringViewBuilder::new();
1119        builder.append_value("hello");
1120        builder.append_null();
1121        builder.append_value("world");
1122        builder.append_value("test");
1123        let arrow_array = builder.finish();
1124        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1125
1126        let mut builder_non_null = StringViewBuilder::new();
1127        builder_non_null.append_value("hello");
1128        builder_non_null.append_value("world");
1129        builder_non_null.append_value("test");
1130        builder_non_null.append_value("vortex");
1131        let arrow_array_non_null = builder_non_null.finish();
1132        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1133
1134        assert_eq!(vortex_array.len(), 4);
1135        assert_eq!(vortex_array_non_null.len(), 4);
1136
1137        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1138        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1139        assert_eq!(
1140            varbin_view_array.buffers().len(),
1141            arrow_array.data_buffers().len()
1142        );
1143        assert_eq!(varbin_view_array.dtype(), &DType::Utf8(true.into()));
1144
1145        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1146        assert_eq!(
1147            varbin_view_array_non_null.buffers().len(),
1148            arrow_array_non_null.data_buffers().len()
1149        );
1150        assert_eq!(
1151            varbin_view_array_non_null.dtype(),
1152            &DType::Utf8(false.into())
1153        );
1154    }
1155
1156    #[test]
1157    fn test_binary_view_array_conversion() {
1158        let mut builder = BinaryViewBuilder::new();
1159        builder.append_value(b"hello");
1160        builder.append_null();
1161        builder.append_value(b"world");
1162        builder.append_value(b"test");
1163        let arrow_array = builder.finish();
1164        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1165
1166        let mut builder_non_null = BinaryViewBuilder::new();
1167        builder_non_null.append_value(b"hello");
1168        builder_non_null.append_value(b"world");
1169        builder_non_null.append_value(b"test");
1170        builder_non_null.append_value(b"vortex");
1171        let arrow_array_non_null = builder_non_null.finish();
1172        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1173
1174        assert_eq!(vortex_array.len(), 4);
1175        assert_eq!(vortex_array_non_null.len(), 4);
1176
1177        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1178        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1179        assert_eq!(
1180            varbin_view_array.buffers().len(),
1181            arrow_array.data_buffers().len()
1182        );
1183        assert_eq!(varbin_view_array.dtype(), &DType::Binary(true.into()));
1184
1185        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1186        assert_eq!(
1187            varbin_view_array_non_null.buffers().len(),
1188            arrow_array_non_null.data_buffers().len()
1189        );
1190        assert_eq!(
1191            varbin_view_array_non_null.dtype(),
1192            &DType::Binary(false.into())
1193        );
1194    }
1195
1196    // Test boolean array conversions
1197    #[test]
1198    fn test_boolean_array_conversion() {
1199        let arrow_array = BooleanArray::from(vec![Some(true), None, Some(false), Some(true)]);
1200        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1201
1202        let arrow_array_non_null = BooleanArray::from(vec![true, false, true, false]);
1203        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1204
1205        assert_eq!(vortex_array.len(), 4);
1206        assert_eq!(vortex_array_non_null.len(), 4);
1207    }
1208
1209    // Test struct array conversions
1210    #[test]
1211    fn test_struct_array_conversion() {
1212        let fields = vec![
1213            Field::new("field1", DataType::Int32, true),
1214            Field::new("field2", DataType::Utf8, false),
1215        ];
1216        let schema = Fields::from(fields);
1217
1218        let field1_data = Int32Array::from(vec![Some(1), None, Some(3)]);
1219        let field2_data = StringArray::from(vec!["a", "b", "c"]);
1220
1221        let arrow_array = StructArray::new(
1222            schema.clone(),
1223            vec![Arc::new(field1_data), Arc::new(field2_data)],
1224            None,
1225        );
1226
1227        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1228        assert_eq!(vortex_array.len(), 3);
1229
1230        // Verify metadata - should be StructArray with correct field names
1231        let struct_vortex_array = vortex_array.as_::<StructVTable>();
1232        assert_eq!(struct_vortex_array.names().len(), 2);
1233        assert_eq!(struct_vortex_array.names()[0], "field1");
1234        assert_eq!(struct_vortex_array.names()[1], "field2");
1235
1236        // Test nullable struct
1237        let nullable_array = StructArray::new(
1238            schema,
1239            vec![
1240                Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
1241                Arc::new(StringArray::from(vec!["a", "b", "c"])),
1242            ],
1243            Some(arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![
1244                true, false, true,
1245            ]))),
1246        );
1247
1248        let vortex_nullable_array = ArrayRef::from_arrow(&nullable_array, true);
1249        assert_eq!(vortex_nullable_array.len(), 3);
1250
1251        // Verify metadata for nullable struct
1252        let struct_vortex_nullable_array = vortex_nullable_array.as_::<StructVTable>();
1253        assert_eq!(struct_vortex_nullable_array.names().len(), 2);
1254        assert_eq!(struct_vortex_nullable_array.names()[0], "field1");
1255        assert_eq!(struct_vortex_nullable_array.names()[1], "field2");
1256    }
1257
1258    // Test list array conversions
1259    #[test]
1260    fn test_list_array_conversion() {
1261        let mut builder = ListBuilder::new(Int32Builder::new());
1262        builder.append_value([Some(1), None, Some(3)]);
1263        builder.append_null();
1264        builder.append_value([Some(4), Some(5)]);
1265        let arrow_array = builder.finish();
1266
1267        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1268        assert_eq!(vortex_array.len(), 3);
1269
1270        // Verify metadata - should be ListArray with correct offsets
1271        let list_vortex_array = vortex_array.as_::<ListVTable>();
1272        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1273        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1274        assert_eq!(offsets_array.ptype(), PType::I32);
1275
1276        // Test non-nullable list
1277        let mut builder_non_null = ListBuilder::new(Int32Builder::new());
1278        builder_non_null.append_value([Some(1), None, Some(3)]);
1279        builder_non_null.append_value([Some(4), Some(5)]);
1280        let arrow_array_non_null = builder_non_null.finish();
1281
1282        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1283        assert_eq!(vortex_array_non_null.len(), 2);
1284
1285        // Verify metadata for non-nullable list
1286        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1287        let offsets_array_non_null = list_vortex_array_non_null
1288            .offsets()
1289            .as_::<PrimitiveVTable>();
1290        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1291        assert_eq!(offsets_array_non_null.ptype(), PType::I32);
1292    }
1293
1294    #[test]
1295    fn test_large_list_array_conversion() {
1296        let mut builder = LargeListBuilder::new(Int32Builder::new());
1297        builder.append_value([Some(1), None, Some(3)]);
1298        builder.append_null();
1299        builder.append_value([Some(4), Some(5)]);
1300        let arrow_array = builder.finish();
1301
1302        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1303        assert_eq!(vortex_array.len(), 3);
1304
1305        // Verify metadata - should be ListArray with correct offsets (I64 for large lists)
1306        let list_vortex_array = vortex_array.as_::<ListVTable>();
1307        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1308        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1309        assert_eq!(offsets_array.ptype(), PType::I64); // Large lists use I64 offsets
1310
1311        // Test non-nullable large list
1312        let mut builder_non_null = LargeListBuilder::new(Int32Builder::new());
1313        builder_non_null.append_value([Some(1), None, Some(3)]);
1314        builder_non_null.append_value([Some(4), Some(5)]);
1315        let arrow_array_non_null = builder_non_null.finish();
1316
1317        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1318        assert_eq!(vortex_array_non_null.len(), 2);
1319
1320        // Verify metadata for non-nullable large list
1321        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1322        let offsets_array_non_null = list_vortex_array_non_null
1323            .offsets()
1324            .as_::<PrimitiveVTable>();
1325        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1326        assert_eq!(offsets_array_non_null.ptype(), PType::I64); // Large lists use I64 offsets
1327    }
1328
1329    #[test]
1330    fn test_fixed_size_list_array_conversion() {
1331        // Create elements for the fixed-size lists
1332        let values = Int32Array::from(vec![
1333            Some(1),
1334            Some(2),
1335            Some(3), // First list
1336            Some(4),
1337            None,
1338            Some(6), // Second list (with null element)
1339            Some(7),
1340            Some(8),
1341            Some(9), // Third list
1342            Some(10),
1343            Some(11),
1344            Some(12), // Fourth list
1345        ]);
1346
1347        // Create a FixedSizeListArray with list_size=3
1348        let field = Arc::new(Field::new("item", DataType::Int32, true));
1349        let arrow_array =
1350            ArrowFixedSizeListArray::try_new(field.clone(), 3, Arc::new(values), None).unwrap();
1351        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1352
1353        assert_eq!(vortex_array.len(), 4);
1354
1355        // Verify metadata - should be FixedSizeListArray with correct list size
1356        let fsl_vortex_array = vortex_array.as_::<FixedSizeListVTable>();
1357        assert_eq!(fsl_vortex_array.list_size(), 3);
1358        assert_eq!(fsl_vortex_array.elements().len(), 12); // 4 lists * 3 elements
1359
1360        // Test nullable fixed-size list
1361        let values_nullable = Int32Array::from(vec![
1362            Some(1),
1363            Some(2),
1364            Some(3), // First list
1365            Some(4),
1366            None,
1367            Some(6), // Second list (will be null)
1368            Some(7),
1369            Some(8),
1370            Some(9), // Third list
1371        ]);
1372
1373        // Create nulls buffer - second list is null
1374        let null_buffer =
1375            arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![true, false, true]));
1376
1377        let arrow_array_nullable = ArrowFixedSizeListArray::try_new(
1378            field,
1379            3,
1380            Arc::new(values_nullable),
1381            Some(null_buffer),
1382        )
1383        .unwrap();
1384        let vortex_array_nullable = ArrayRef::from_arrow(&arrow_array_nullable, true);
1385
1386        assert_eq!(vortex_array_nullable.len(), 3);
1387
1388        // Verify metadata for nullable array
1389        let fsl_vortex_array_nullable = vortex_array_nullable.as_::<FixedSizeListVTable>();
1390        assert_eq!(fsl_vortex_array_nullable.list_size(), 3);
1391        assert_eq!(fsl_vortex_array_nullable.elements().len(), 9); // 3 lists * 3 elements
1392    }
1393
1394    #[test]
1395    fn test_list_view_array_conversion() {
1396        // Create values array for the lists
1397        let values = Int32Array::from(vec![
1398            Some(1),
1399            Some(2),
1400            Some(3), // First list [1, 2, 3]
1401            Some(4),
1402            Some(5), // Second list [4, 5]
1403            Some(6), // Third list [6]
1404            Some(7),
1405            Some(8),
1406            Some(9),
1407            Some(10), // Fourth list [7, 8, 9, 10]
1408        ]);
1409
1410        // Create offsets and sizes for ListView
1411        let offsets = ScalarBuffer::from(vec![0i32, 3, 5, 6]);
1412        let sizes = ScalarBuffer::from(vec![3i32, 2, 1, 4]);
1413
1414        let field = Arc::new(Field::new("item", DataType::Int32, true));
1415        let arrow_array = GenericListViewArray::try_new(
1416            field.clone(),
1417            offsets.clone(),
1418            sizes.clone(),
1419            Arc::new(values.clone()),
1420            None,
1421        )
1422        .unwrap();
1423
1424        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1425        assert_eq!(vortex_array.len(), 4);
1426
1427        // Verify metadata - should be ListViewArray with correct offsets and sizes
1428        let list_view_vortex_array = vortex_array.as_::<ListViewVTable>();
1429        let offsets_array = list_view_vortex_array.offsets().as_::<PrimitiveVTable>();
1430        let sizes_array = list_view_vortex_array.sizes().as_::<PrimitiveVTable>();
1431
1432        assert_eq!(offsets_array.len(), 4);
1433        assert_eq!(offsets_array.ptype(), PType::I32);
1434        assert_eq!(sizes_array.len(), 4);
1435        assert_eq!(sizes_array.ptype(), PType::I32);
1436
1437        // Test nullable ListView
1438        let null_buffer =
1439            arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![true, false, true, true]));
1440
1441        let arrow_array_nullable = GenericListViewArray::try_new(
1442            field.clone(),
1443            offsets,
1444            sizes,
1445            Arc::new(values.clone()),
1446            Some(null_buffer),
1447        )
1448        .unwrap();
1449
1450        let vortex_array_nullable = ArrayRef::from_arrow(&arrow_array_nullable, true);
1451        assert_eq!(vortex_array_nullable.len(), 4);
1452
1453        // Test LargeListView (i64 offsets and sizes)
1454        let large_offsets = ScalarBuffer::from(vec![0i64, 3, 5, 6]);
1455        let large_sizes = ScalarBuffer::from(vec![3i64, 2, 1, 4]);
1456
1457        let large_arrow_array = GenericListViewArray::try_new(
1458            field,
1459            large_offsets,
1460            large_sizes,
1461            Arc::new(values),
1462            None,
1463        )
1464        .unwrap();
1465
1466        let large_vortex_array = ArrayRef::from_arrow(&large_arrow_array, false);
1467        assert_eq!(large_vortex_array.len(), 4);
1468
1469        // Verify metadata for large ListView
1470        let large_list_view_vortex_array = large_vortex_array.as_::<ListViewVTable>();
1471        let large_offsets_array = large_list_view_vortex_array
1472            .offsets()
1473            .as_::<PrimitiveVTable>();
1474        let large_sizes_array = large_list_view_vortex_array
1475            .sizes()
1476            .as_::<PrimitiveVTable>();
1477
1478        assert_eq!(large_offsets_array.len(), 4);
1479        assert_eq!(large_offsets_array.ptype(), PType::I64); // Large ListView uses I64 offsets
1480        assert_eq!(large_sizes_array.len(), 4);
1481        assert_eq!(large_sizes_array.ptype(), PType::I64); // Large ListView uses I64 sizes
1482    }
1483
1484    // Test null array conversions
1485    #[test]
1486    fn test_null_array_conversion() {
1487        let arrow_array = NullArray::new(5);
1488        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1489        assert_eq!(vortex_array.len(), 5);
1490    }
1491
1492    // Test buffer conversions
1493    #[test]
1494    fn test_arrow_buffer_conversion() {
1495        let data = vec![1u8, 2, 3, 4, 5];
1496        let arrow_buffer = ArrowBuffer::from_vec(data);
1497        let vortex_array = arrow_buffer.into_array();
1498        assert_eq!(vortex_array.len(), 5);
1499    }
1500
1501    #[test]
1502    fn test_boolean_buffer_conversion() {
1503        let data = vec![true, false, true, false, true];
1504        let boolean_buffer = BooleanBuffer::from(data);
1505        let vortex_array = boolean_buffer.into_array();
1506        assert_eq!(vortex_array.len(), 5);
1507    }
1508
1509    #[test]
1510    fn test_scalar_buffer_conversion() {
1511        let data = vec![1i32, 2, 3, 4, 5];
1512        let scalar_buffer = ScalarBuffer::from(data);
1513        let vortex_array = scalar_buffer.into_array();
1514        assert_eq!(vortex_array.len(), 5);
1515    }
1516
1517    #[test]
1518    fn test_offset_buffer_conversion() {
1519        let data = vec![0i32, 2, 5, 8, 10];
1520        let offset_buffer = OffsetBuffer::new(ScalarBuffer::from(data));
1521        let vortex_array = offset_buffer.into_array();
1522        assert_eq!(vortex_array.len(), 5);
1523    }
1524
1525    // Test RecordBatch conversions
1526    #[test]
1527    fn test_record_batch_conversion() {
1528        let schema = Arc::new(Schema::new(vec![
1529            Field::new("field1", DataType::Int32, false),
1530            Field::new("field2", DataType::Utf8, false),
1531        ]));
1532
1533        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1534        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1535
1536        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1537
1538        let vortex_array = ArrayRef::from_arrow(record_batch, false);
1539        assert_eq!(vortex_array.len(), 4);
1540
1541        // Test with reference
1542        let schema = Arc::new(Schema::new(vec![
1543            Field::new("field1", DataType::Int32, false),
1544            Field::new("field2", DataType::Utf8, false),
1545        ]));
1546
1547        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1548        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1549
1550        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1551
1552        let vortex_array = ArrayRef::from_arrow(&record_batch, false);
1553        assert_eq!(vortex_array.len(), 4);
1554    }
1555
1556    // Test dynamic dispatch conversion
1557    #[test]
1558    fn test_dyn_array_conversion() {
1559        let int_array = Int32Array::from(vec![1, 2, 3, 4]);
1560        let dyn_array: &dyn ArrowArray = &int_array;
1561        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1562        assert_eq!(vortex_array.len(), 4);
1563
1564        let string_array = StringArray::from(vec!["a", "b", "c"]);
1565        let dyn_array: &dyn ArrowArray = &string_array;
1566        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1567        assert_eq!(vortex_array.len(), 3);
1568
1569        let bool_array = BooleanArray::from(vec![true, false, true]);
1570        let dyn_array: &dyn ArrowArray = &bool_array;
1571        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1572        assert_eq!(vortex_array.len(), 3);
1573    }
1574
1575    // Existing tests
1576    #[test]
1577    pub fn nullable_may_contain_non_nullable() {
1578        let null_struct_array_with_non_nullable_field = new_null_array(
1579            &DataType::Struct(Fields::from(vec![Field::new(
1580                "non_nullable_inner",
1581                DataType::Int32,
1582                false,
1583            )])),
1584            1,
1585        );
1586        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1587    }
1588
1589    #[test]
1590    pub fn nullable_may_contain_deeply_nested_non_nullable() {
1591        let null_struct_array_with_non_nullable_field = new_null_array(
1592            &DataType::Struct(Fields::from(vec![Field::new(
1593                "non_nullable_inner",
1594                DataType::Struct(Fields::from(vec![Field::new(
1595                    "non_nullable_deeper_inner",
1596                    DataType::Int32,
1597                    false,
1598                )])),
1599                false,
1600            )])),
1601            1,
1602        );
1603        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1604    }
1605
1606    #[test]
1607    #[should_panic]
1608    pub fn cannot_handle_nullable_struct_containing_non_nullable_dictionary() {
1609        let null_struct_array_with_non_nullable_field = new_null_array(
1610            &DataType::Struct(Fields::from(vec![Field::new(
1611                "non_nullable_deeper_inner",
1612                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1613                false,
1614            )])),
1615            1,
1616        );
1617
1618        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1619    }
1620}
vortex_array/arrow/convert.rs

vortex_array/arrow/
convert.rs
