vortex_array/arrow/
convert.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::sync::Arc;
5
6use arrow_array::cast::{AsArray, as_null_array};
7use arrow_array::types::{
8    ByteArrayType, ByteViewType, Date32Type, Date64Type, Decimal128Type, Decimal256Type,
9    Float16Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
10    Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
11    TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
12    TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
13};
14use arrow_array::{
15    Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray, GenericByteArray,
16    GenericByteViewArray, GenericListArray, NullArray as ArrowNullArray, OffsetSizeTrait,
17    PrimitiveArray as ArrowPrimitiveArray, RecordBatch, StructArray as ArrowStructArray,
18    make_array,
19};
20use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
21use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
22use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
23use itertools::Itertools;
24use vortex_buffer::{Alignment, Buffer, ByteBuffer};
25use vortex_dtype::datetime::TimeUnit;
26use vortex_dtype::{DType, DecimalDType, NativePType, PType};
27use vortex_error::{VortexExpect as _, vortex_panic};
28use vortex_scalar::i256;
29
30use crate::arrays::{
31    BoolArray, DecimalArray, ListArray, NullArray, PrimitiveArray, StructArray, TemporalArray,
32    VarBinArray, VarBinViewArray,
33};
34use crate::arrow::FromArrowArray;
35use crate::validity::Validity;
36use crate::{ArrayRef, IntoArray};
37
38impl IntoArray for ArrowBuffer {
39    fn into_array(self) -> ArrayRef {
40        PrimitiveArray::from_byte_buffer(
41            ByteBuffer::from_arrow_buffer(self, Alignment::of::<u8>()),
42            PType::U8,
43            Validity::NonNullable,
44        )
45        .into_array()
46    }
47}
48
49impl IntoArray for BooleanBuffer {
50    fn into_array(self) -> ArrayRef {
51        BoolArray::new(self, Validity::NonNullable).into_array()
52    }
53}
54
55impl<T> IntoArray for ScalarBuffer<T>
56where
57    T: ArrowNativeType + NativePType,
58{
59    fn into_array(self) -> ArrayRef {
60        PrimitiveArray::new(
61            Buffer::<T>::from_arrow_scalar_buffer(self),
62            Validity::NonNullable,
63        )
64        .into_array()
65    }
66}
67
68impl<O> IntoArray for OffsetBuffer<O>
69where
70    O: NativePType + OffsetSizeTrait,
71{
72    fn into_array(self) -> ArrayRef {
73        let primitive = PrimitiveArray::new(
74            Buffer::from_arrow_scalar_buffer(self.into_inner()),
75            Validity::NonNullable,
76        );
77
78        primitive.into_array()
79    }
80}
81
82macro_rules! impl_from_arrow_primitive {
83    ($T:path) => {
84        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
85            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
86                let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone());
87                let validity = nulls(value.nulls(), nullable);
88                PrimitiveArray::new(buffer, validity).into_array()
89            }
90        }
91    };
92}
93
94impl_from_arrow_primitive!(Int8Type);
95impl_from_arrow_primitive!(Int16Type);
96impl_from_arrow_primitive!(Int32Type);
97impl_from_arrow_primitive!(Int64Type);
98impl_from_arrow_primitive!(UInt8Type);
99impl_from_arrow_primitive!(UInt16Type);
100impl_from_arrow_primitive!(UInt32Type);
101impl_from_arrow_primitive!(UInt64Type);
102impl_from_arrow_primitive!(Float16Type);
103impl_from_arrow_primitive!(Float32Type);
104impl_from_arrow_primitive!(Float64Type);
105
106impl FromArrowArray<&ArrowPrimitiveArray<Decimal128Type>> for ArrayRef {
107    fn from_arrow(array: &ArrowPrimitiveArray<Decimal128Type>, nullable: bool) -> Self {
108        let decimal_type = DecimalDType::new(array.precision(), array.scale());
109        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
110        let validity = nulls(array.nulls(), nullable);
111        DecimalArray::new(buffer, decimal_type, validity).into_array()
112    }
113}
114
115impl FromArrowArray<&ArrowPrimitiveArray<Decimal256Type>> for ArrayRef {
116    fn from_arrow(array: &ArrowPrimitiveArray<Decimal256Type>, nullable: bool) -> Self {
117        let decimal_type = DecimalDType::new(array.precision(), array.scale());
118        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
119        // SAFETY: Our i256 implementation has the same bit-pattern representation of the
120        //  arrow_buffer::i256 type. It is safe to treat values held inside the buffer as values
121        //  of either type.
122        let buffer =
123            unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(buffer) };
124        let validity = nulls(array.nulls(), nullable);
125        DecimalArray::new(buffer, decimal_type, validity).into_array()
126    }
127}
128
129macro_rules! impl_from_arrow_temporal {
130    ($T:path) => {
131        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
132            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
133                temporal_array(value, nullable)
134            }
135        }
136    };
137}
138
139// timestamp
140impl_from_arrow_temporal!(TimestampSecondType);
141impl_from_arrow_temporal!(TimestampMillisecondType);
142impl_from_arrow_temporal!(TimestampMicrosecondType);
143impl_from_arrow_temporal!(TimestampNanosecondType);
144
145// time
146impl_from_arrow_temporal!(Time32SecondType);
147impl_from_arrow_temporal!(Time32MillisecondType);
148impl_from_arrow_temporal!(Time64MicrosecondType);
149impl_from_arrow_temporal!(Time64NanosecondType);
150
151// date
152impl_from_arrow_temporal!(Date32Type);
153impl_from_arrow_temporal!(Date64Type);
154
155fn temporal_array<T: ArrowPrimitiveType>(value: &ArrowPrimitiveArray<T>, nullable: bool) -> ArrayRef
156where
157    T::Native: NativePType,
158{
159    let arr = PrimitiveArray::new(
160        Buffer::from_arrow_scalar_buffer(value.values().clone()),
161        nulls(value.nulls(), nullable),
162    )
163    .into_array();
164
165    match value.data_type() {
166        DataType::Timestamp(time_unit, tz) => {
167            let tz = tz.as_ref().map(|s| s.to_string());
168            TemporalArray::new_timestamp(arr, time_unit.into(), tz).into()
169        }
170        DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
171        DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
172        DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::Days).into(),
173        DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Milliseconds).into(),
174        DataType::Duration(_) => unimplemented!(),
175        DataType::Interval(_) => unimplemented!(),
176        _ => vortex_panic!("Invalid temporal type: {}", value.data_type()),
177    }
178}
179
180impl<T: ByteArrayType> FromArrowArray<&GenericByteArray<T>> for ArrayRef
181where
182    <T as ByteArrayType>::Offset: NativePType,
183{
184    fn from_arrow(value: &GenericByteArray<T>, nullable: bool) -> Self {
185        let dtype = match T::DATA_TYPE {
186            DataType::Binary | DataType::LargeBinary => DType::Binary(nullable.into()),
187            DataType::Utf8 | DataType::LargeUtf8 => DType::Utf8(nullable.into()),
188            dt => vortex_panic!("Invalid data type for ByteArray: {dt}"),
189        };
190        VarBinArray::try_new(
191            value.offsets().clone().into_array(),
192            ByteBuffer::from_arrow_buffer(value.values().clone(), Alignment::of::<u8>()),
193            dtype,
194            nulls(value.nulls(), nullable),
195        )
196        .vortex_expect("Failed to convert Arrow GenericByteArray to Vortex VarBinArray")
197        .into_array()
198    }
199}
200
201impl<T: ByteViewType> FromArrowArray<&GenericByteViewArray<T>> for ArrayRef {
202    fn from_arrow(value: &GenericByteViewArray<T>, nullable: bool) -> Self {
203        let dtype = match T::DATA_TYPE {
204            DataType::BinaryView => DType::Binary(nullable.into()),
205            DataType::Utf8View => DType::Utf8(nullable.into()),
206            dt => vortex_panic!("Invalid data type for ByteViewArray: {dt}"),
207        };
208
209        let views_buffer = Buffer::from_byte_buffer(
210            Buffer::from_arrow_scalar_buffer(value.views().clone()).into_byte_buffer(),
211        );
212
213        // SAFETY: arrow-rs ByteViewArray already checks the same invariants, we inherit those
214        //  guarantees by zero-copy constructing from one.
215        unsafe {
216            VarBinViewArray::new_unchecked(
217                views_buffer,
218                Arc::from(
219                    value
220                        .data_buffers()
221                        .iter()
222                        .map(|b| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::<u8>()))
223                        .collect::<Vec<_>>(),
224                ),
225                dtype,
226                nulls(value.nulls(), nullable),
227            )
228            .into_array()
229        }
230    }
231}
232
233impl FromArrowArray<&ArrowBooleanArray> for ArrayRef {
234    fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self {
235        BoolArray::new(value.values().clone(), nulls(value.nulls(), nullable)).into_array()
236    }
237}
238
239/// Strip out the nulls from this array and return a new array without nulls.
240fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData {
241    if data.null_count() == 0 {
242        // No nulls to remove, return the array as is
243        return data;
244    }
245
246    let children = match data.data_type() {
247        DataType::Struct(fields) => Some(
248            fields
249                .iter()
250                .zip(data.child_data().iter())
251                .map(|(field, child_data)| {
252                    if field.is_nullable() {
253                        child_data.clone()
254                    } else {
255                        remove_nulls(child_data.clone())
256                    }
257                })
258                .collect_vec(),
259        ),
260        DataType::List(f)
261        | DataType::LargeList(f)
262        | DataType::ListView(f)
263        | DataType::LargeListView(f)
264        | DataType::FixedSizeList(f, _)
265            if !f.is_nullable() =>
266        {
267            // All list types only have one child
268            assert_eq!(
269                data.child_data().len(),
270                1,
271                "List types should have one child"
272            );
273            Some(vec![remove_nulls(data.child_data()[0].clone())])
274        }
275        _ => None,
276    };
277
278    let mut builder = data.into_builder().nulls(None);
279    if let Some(children) = children {
280        builder = builder.child_data(children);
281    }
282    builder
283        .build()
284        .vortex_expect("reconstructing array without nulls")
285}
286
287impl FromArrowArray<&ArrowStructArray> for ArrayRef {
288    fn from_arrow(value: &ArrowStructArray, nullable: bool) -> Self {
289        StructArray::try_new(
290            value.column_names().iter().copied().collect(),
291            value
292                .columns()
293                .iter()
294                .zip(value.fields())
295                .map(|(c, field)| {
296                    // Arrow pushes down nulls, even into non-nullable fields. So we strip them
297                    // out here because Vortex is a little more strict.
298                    if c.null_count() > 0 && !field.is_nullable() {
299                        let stripped = make_array(remove_nulls(c.into_data()));
300                        Self::from_arrow(stripped.as_ref(), false)
301                    } else {
302                        Self::from_arrow(c.as_ref(), field.is_nullable())
303                    }
304                })
305                .collect(),
306            value.len(),
307            nulls(value.nulls(), nullable),
308        )
309        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
310        .into_array()
311    }
312}
313
314impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListArray<O>> for ArrayRef {
315    fn from_arrow(value: &GenericListArray<O>, nullable: bool) -> Self {
316        // Extract the validity of the underlying element array
317        let elem_nullable = match value.data_type() {
318            DataType::List(field) => field.is_nullable(),
319            DataType::LargeList(field) => field.is_nullable(),
320            dt => vortex_panic!("Invalid data type for ListArray: {dt}"),
321        };
322        ListArray::try_new(
323            Self::from_arrow(value.values().as_ref(), elem_nullable),
324            // offsets are always non-nullable
325            value.offsets().clone().into_array(),
326            nulls(value.nulls(), nullable),
327        )
328        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
329        .into_array()
330    }
331}
332
333impl FromArrowArray<&ArrowNullArray> for ArrayRef {
334    fn from_arrow(value: &ArrowNullArray, nullable: bool) -> Self {
335        assert!(nullable);
336        NullArray::new(value.len()).into_array()
337    }
338}
339
340fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity {
341    if nullable {
342        nulls
343            .map(|nulls| {
344                if nulls.null_count() == nulls.len() {
345                    Validity::AllInvalid
346                } else {
347                    Validity::from(nulls.inner().clone())
348                }
349            })
350            .unwrap_or_else(|| Validity::AllValid)
351    } else {
352        assert!(nulls.map(|x| x.null_count() == 0).unwrap_or(true));
353        Validity::NonNullable
354    }
355}
356
357impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
358    fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> Self {
359        match array.data_type() {
360            DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable),
361            DataType::UInt8 => Self::from_arrow(array.as_primitive::<UInt8Type>(), nullable),
362            DataType::UInt16 => Self::from_arrow(array.as_primitive::<UInt16Type>(), nullable),
363            DataType::UInt32 => Self::from_arrow(array.as_primitive::<UInt32Type>(), nullable),
364            DataType::UInt64 => Self::from_arrow(array.as_primitive::<UInt64Type>(), nullable),
365            DataType::Int8 => Self::from_arrow(array.as_primitive::<Int8Type>(), nullable),
366            DataType::Int16 => Self::from_arrow(array.as_primitive::<Int16Type>(), nullable),
367            DataType::Int32 => Self::from_arrow(array.as_primitive::<Int32Type>(), nullable),
368            DataType::Int64 => Self::from_arrow(array.as_primitive::<Int64Type>(), nullable),
369            DataType::Float16 => Self::from_arrow(array.as_primitive::<Float16Type>(), nullable),
370            DataType::Float32 => Self::from_arrow(array.as_primitive::<Float32Type>(), nullable),
371            DataType::Float64 => Self::from_arrow(array.as_primitive::<Float64Type>(), nullable),
372            DataType::Utf8 => Self::from_arrow(array.as_string::<i32>(), nullable),
373            DataType::LargeUtf8 => Self::from_arrow(array.as_string::<i64>(), nullable),
374            DataType::Binary => Self::from_arrow(array.as_binary::<i32>(), nullable),
375            DataType::LargeBinary => Self::from_arrow(array.as_binary::<i64>(), nullable),
376            DataType::BinaryView => Self::from_arrow(array.as_binary_view(), nullable),
377            DataType::Utf8View => Self::from_arrow(array.as_string_view(), nullable),
378            DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),
379            DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),
380            DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),
381            DataType::Null => Self::from_arrow(as_null_array(array), nullable),
382            DataType::Timestamp(u, _) => match u {
383                ArrowTimeUnit::Second => {
384                    Self::from_arrow(array.as_primitive::<TimestampSecondType>(), nullable)
385                }
386                ArrowTimeUnit::Millisecond => {
387                    Self::from_arrow(array.as_primitive::<TimestampMillisecondType>(), nullable)
388                }
389                ArrowTimeUnit::Microsecond => {
390                    Self::from_arrow(array.as_primitive::<TimestampMicrosecondType>(), nullable)
391                }
392                ArrowTimeUnit::Nanosecond => {
393                    Self::from_arrow(array.as_primitive::<TimestampNanosecondType>(), nullable)
394                }
395            },
396            DataType::Date32 => Self::from_arrow(array.as_primitive::<Date32Type>(), nullable),
397            DataType::Date64 => Self::from_arrow(array.as_primitive::<Date64Type>(), nullable),
398            DataType::Time32(u) => match u {
399                ArrowTimeUnit::Second => {
400                    Self::from_arrow(array.as_primitive::<Time32SecondType>(), nullable)
401                }
402                ArrowTimeUnit::Millisecond => {
403                    Self::from_arrow(array.as_primitive::<Time32MillisecondType>(), nullable)
404                }
405                ArrowTimeUnit::Microsecond | ArrowTimeUnit::Nanosecond => unreachable!(),
406            },
407            DataType::Time64(u) => match u {
408                ArrowTimeUnit::Microsecond => {
409                    Self::from_arrow(array.as_primitive::<Time64MicrosecondType>(), nullable)
410                }
411                ArrowTimeUnit::Nanosecond => {
412                    Self::from_arrow(array.as_primitive::<Time64NanosecondType>(), nullable)
413                }
414                ArrowTimeUnit::Second | ArrowTimeUnit::Millisecond => unreachable!(),
415            },
416            DataType::Decimal128(..) => {
417                Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)
418            }
419            DataType::Decimal256(..) => {
420                Self::from_arrow(array.as_primitive::<Decimal256Type>(), nullable)
421            }
422            dt => vortex_panic!("Array encoding not implemented for Arrow data type {dt}"),
423        }
424    }
425}
426
427impl FromArrowArray<RecordBatch> for ArrayRef {
428    fn from_arrow(array: RecordBatch, nullable: bool) -> Self {
429        ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable)
430    }
431}
432
433impl FromArrowArray<&RecordBatch> for ArrayRef {
434    fn from_arrow(array: &RecordBatch, nullable: bool) -> Self {
435        Self::from_arrow(array.clone(), nullable)
436    }
437}
438
439#[cfg(test)]
440mod tests {
441    use std::sync::Arc;
442
443    use arrow_array::builder::{
444        BinaryViewBuilder, Decimal128Builder, Decimal256Builder, Int32Builder, LargeListBuilder,
445        ListBuilder, StringViewBuilder,
446    };
447    use arrow_array::types::{ArrowPrimitiveType, Float16Type};
448    use arrow_array::{
449        Array as ArrowArray, BinaryArray, BooleanArray, Date32Array, Date64Array, Float32Array,
450        Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, LargeBinaryArray,
451        LargeStringArray, NullArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray,
452        Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
453        TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
454        TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, new_null_array,
455    };
456    use arrow_buffer::{BooleanBuffer, Buffer as ArrowBuffer, OffsetBuffer, ScalarBuffer};
457    use arrow_schema::{DataType, Field, Fields, Schema};
458    use vortex_dtype::datetime::{TIMESTAMP_ID, TemporalMetadata, TimeUnit};
459    use vortex_dtype::{DType, ExtDType, Nullability, PType};
460
461    use crate::arrays::{
462        DecimalVTable, ListVTable, PrimitiveVTable, StructVTable, TemporalArray, VarBinVTable,
463        VarBinViewVTable,
464    };
465    use crate::arrow::FromArrowArray as _;
466    use crate::{ArrayRef, IntoArray};
467
468    // Test primitive array conversions
469    #[test]
470    fn test_int8_array_conversion() {
471        let arrow_array = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]);
472        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
473
474        let arrow_array_non_null = Int8Array::from(vec![1, 2, 3, 4]);
475        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
476
477        assert_eq!(vortex_array.len(), 4);
478        assert_eq!(vortex_array_non_null.len(), 4);
479
480        // Verify metadata - should be PrimitiveArray with I8 ptype
481        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
482        assert_eq!(primitive_array.ptype(), PType::I8);
483
484        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
485        assert_eq!(primitive_array_non_null.ptype(), PType::I8);
486    }
487
488    #[test]
489    fn test_int16_array_conversion() {
490        let arrow_array = Int16Array::from(vec![Some(100), None, Some(300), Some(400)]);
491        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
492
493        let arrow_array_non_null = Int16Array::from(vec![100, 200, 300, 400]);
494        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
495
496        assert_eq!(vortex_array.len(), 4);
497        assert_eq!(vortex_array_non_null.len(), 4);
498
499        // Verify metadata - should be PrimitiveArray with I16 ptype
500        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
501        assert_eq!(primitive_array.ptype(), PType::I16);
502
503        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
504        assert_eq!(primitive_array_non_null.ptype(), PType::I16);
505    }
506
507    #[test]
508    fn test_int32_array_conversion() {
509        let arrow_array = Int32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
510        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
511
512        let arrow_array_non_null = Int32Array::from(vec![1000, 2000, 3000, 4000]);
513        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
514
515        assert_eq!(vortex_array.len(), 4);
516        assert_eq!(vortex_array_non_null.len(), 4);
517
518        // Verify metadata - should be PrimitiveArray with I32 ptype
519        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
520        assert_eq!(primitive_array.ptype(), PType::I32);
521
522        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
523        assert_eq!(primitive_array_non_null.ptype(), PType::I32);
524    }
525
526    #[test]
527    fn test_int64_array_conversion() {
528        let arrow_array = Int64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
529        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
530
531        let arrow_array_non_null = Int64Array::from(vec![10000_i64, 20000, 30000, 40000]);
532        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
533
534        assert_eq!(vortex_array.len(), 4);
535        assert_eq!(vortex_array_non_null.len(), 4);
536
537        // Verify metadata - should be PrimitiveArray with I64 ptype
538        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
539        assert_eq!(primitive_array.ptype(), PType::I64);
540
541        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
542        assert_eq!(primitive_array_non_null.ptype(), PType::I64);
543    }
544
545    #[test]
546    fn test_uint8_array_conversion() {
547        let arrow_array = UInt8Array::from(vec![Some(1), None, Some(3), Some(4)]);
548        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
549
550        let arrow_array_non_null = UInt8Array::from(vec![1_u8, 2, 3, 4]);
551        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
552
553        assert_eq!(vortex_array.len(), 4);
554        assert_eq!(vortex_array_non_null.len(), 4);
555
556        // Verify metadata - should be PrimitiveArray with U8 ptype
557        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
558        assert_eq!(primitive_array.ptype(), PType::U8);
559
560        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
561        assert_eq!(primitive_array_non_null.ptype(), PType::U8);
562    }
563
564    #[test]
565    fn test_uint16_array_conversion() {
566        let arrow_array = UInt16Array::from(vec![Some(100), None, Some(300), Some(400)]);
567        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
568
569        let arrow_array_non_null = UInt16Array::from(vec![100_u16, 200, 300, 400]);
570        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
571
572        assert_eq!(vortex_array.len(), 4);
573        assert_eq!(vortex_array_non_null.len(), 4);
574
575        // Verify metadata - should be PrimitiveArray with U16 ptype
576        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
577        assert_eq!(primitive_array.ptype(), PType::U16);
578
579        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
580        assert_eq!(primitive_array_non_null.ptype(), PType::U16);
581    }
582
583    #[test]
584    fn test_uint32_array_conversion() {
585        let arrow_array = UInt32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
586        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
587
588        let arrow_array_non_null = UInt32Array::from(vec![1000_u32, 2000, 3000, 4000]);
589        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
590
591        assert_eq!(vortex_array.len(), 4);
592        assert_eq!(vortex_array_non_null.len(), 4);
593
594        // Verify metadata - should be PrimitiveArray with U32 ptype
595        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
596        assert_eq!(primitive_array.ptype(), PType::U32);
597
598        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
599        assert_eq!(primitive_array_non_null.ptype(), PType::U32);
600    }
601
602    #[test]
603    fn test_uint64_array_conversion() {
604        let arrow_array = UInt64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
605        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
606
607        let arrow_array_non_null = UInt64Array::from(vec![10000_u64, 20000, 30000, 40000]);
608        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
609
610        assert_eq!(vortex_array.len(), 4);
611        assert_eq!(vortex_array_non_null.len(), 4);
612
613        // Verify metadata - should be PrimitiveArray with U64 ptype
614        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
615        assert_eq!(primitive_array.ptype(), PType::U64);
616
617        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
618        assert_eq!(primitive_array_non_null.ptype(), PType::U64);
619    }
620
621    #[test]
622    fn test_float16_array_conversion() {
623        let values = vec![
624            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5)),
625            None,
626            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(3.5)),
627        ];
628        let arrow_array = arrow_array::PrimitiveArray::<Float16Type>::from(values);
629        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
630
631        let non_null_values = vec![
632            <Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5),
633            <Float16Type as ArrowPrimitiveType>::Native::from_f32(2.5),
634        ];
635        let arrow_array_non_null =
636            arrow_array::PrimitiveArray::<Float16Type>::from(non_null_values);
637        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
638
639        assert_eq!(vortex_array.len(), 3);
640        assert_eq!(vortex_array_non_null.len(), 2);
641
642        // Verify metadata - should be PrimitiveArray with F16 ptype
643        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
644        assert_eq!(primitive_array.ptype(), PType::F16);
645
646        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
647        assert_eq!(primitive_array_non_null.ptype(), PType::F16);
648    }
649
650    #[test]
651    fn test_float32_array_conversion() {
652        let arrow_array = Float32Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
653        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
654
655        let arrow_array_non_null = Float32Array::from(vec![1.5_f32, 2.5, 3.5, 4.5]);
656        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
657
658        assert_eq!(vortex_array.len(), 4);
659        assert_eq!(vortex_array_non_null.len(), 4);
660
661        // Verify metadata - should be PrimitiveArray with F32 ptype
662        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
663        assert_eq!(primitive_array.ptype(), PType::F32);
664
665        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
666        assert_eq!(primitive_array_non_null.ptype(), PType::F32);
667    }
668
669    #[test]
670    fn test_float64_array_conversion() {
671        let arrow_array = Float64Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
672        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
673
674        let arrow_array_non_null = Float64Array::from(vec![1.5_f64, 2.5, 3.5, 4.5]);
675        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
676
677        assert_eq!(vortex_array.len(), 4);
678        assert_eq!(vortex_array_non_null.len(), 4);
679
680        // Verify metadata - should be PrimitiveArray with F64 ptype
681        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
682        assert_eq!(primitive_array.ptype(), PType::F64);
683
684        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
685        assert_eq!(primitive_array_non_null.ptype(), PType::F64);
686    }
687
688    // Test decimal array conversions
689    #[test]
690    fn test_decimal128_array_conversion() {
691        let mut builder = Decimal128Builder::with_capacity(4);
692        builder.append_value(12345);
693        builder.append_null();
694        builder.append_value(67890);
695        builder.append_value(11111);
696        let decimal_array = builder.finish().with_precision_and_scale(10, 2).unwrap();
697
698        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
699        assert_eq!(vortex_array.len(), 4);
700
701        let mut builder_non_null = Decimal128Builder::with_capacity(3);
702        builder_non_null.append_value(12345);
703        builder_non_null.append_value(67890);
704        builder_non_null.append_value(11111);
705        let decimal_array_non_null = builder_non_null
706            .finish()
707            .with_precision_and_scale(10, 2)
708            .unwrap();
709
710        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
711        assert_eq!(vortex_array_non_null.len(), 3);
712
713        // Verify metadata - should be DecimalArray with correct precision and scale
714        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
715        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 10);
716        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 2);
717
718        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
719        assert_eq!(
720            decimal_vortex_array_non_null.decimal_dtype().precision(),
721            10
722        );
723        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 2);
724    }
725
726    #[test]
727    fn test_decimal256_array_conversion() {
728        let mut builder = Decimal256Builder::with_capacity(4);
729        builder.append_value(arrow_buffer::i256::from_i128(12345));
730        builder.append_null();
731        builder.append_value(arrow_buffer::i256::from_i128(67890));
732        builder.append_value(arrow_buffer::i256::from_i128(11111));
733        let decimal_array = builder.finish().with_precision_and_scale(38, 10).unwrap();
734
735        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
736        assert_eq!(vortex_array.len(), 4);
737
738        let mut builder_non_null = Decimal256Builder::with_capacity(3);
739        builder_non_null.append_value(arrow_buffer::i256::from_i128(12345));
740        builder_non_null.append_value(arrow_buffer::i256::from_i128(67890));
741        builder_non_null.append_value(arrow_buffer::i256::from_i128(11111));
742        let decimal_array_non_null = builder_non_null
743            .finish()
744            .with_precision_and_scale(38, 10)
745            .unwrap();
746
747        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
748        assert_eq!(vortex_array_non_null.len(), 3);
749
750        // Verify metadata - should be DecimalArray with correct precision and scale
751        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
752        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 38);
753        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 10);
754
755        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
756        assert_eq!(
757            decimal_vortex_array_non_null.decimal_dtype().precision(),
758            38
759        );
760        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 10);
761    }
762
763    // Test temporal array conversions
764    #[test]
765    fn test_timestamp_second_array_conversion() {
766        let arrow_array =
767            TimestampSecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
768        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
769
770        let arrow_array_non_null = TimestampSecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
771        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
772
773        assert_eq!(vortex_array.len(), 4);
774        assert_eq!(vortex_array_non_null.len(), 4);
775
776        // Verify metadata - should be TemporalArray with Second time unit
777        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
778        assert_eq!(
779            temporal_array.temporal_metadata().time_unit(),
780            TimeUnit::Seconds
781        );
782
783        let temporal_array_non_null =
784            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
785        assert_eq!(
786            temporal_array_non_null.temporal_metadata().time_unit(),
787            TimeUnit::Seconds
788        );
789    }
790
791    #[test]
792    fn test_timestamp_millisecond_array_conversion() {
793        let arrow_array =
794            TimestampMillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
795        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
796
797        let arrow_array_non_null =
798            TimestampMillisecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
799        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
800
801        assert_eq!(vortex_array.len(), 4);
802        assert_eq!(vortex_array_non_null.len(), 4);
803    }
804
805    #[test]
806    fn test_timestamp_microsecond_array_conversion() {
807        let arrow_array =
808            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
809        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
810
811        let arrow_array_non_null =
812            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
813        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
814
815        assert_eq!(vortex_array.len(), 4);
816        assert_eq!(vortex_array_non_null.len(), 4);
817    }
818
819    #[test]
820    fn test_timestamp_timezone_microsecond_array_conversion() {
821        let arrow_array =
822            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)])
823                .with_timezone("UTC");
824        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
825
826        let arrow_array_non_null =
827            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]).with_timezone("UTC");
828        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
829
830        assert_eq!(vortex_array.len(), 4);
831        assert_eq!(
832            vortex_array.dtype(),
833            &DType::Extension(Arc::new(ExtDType::new(
834                TIMESTAMP_ID.clone(),
835                Arc::new(DType::Primitive(PType::I64, Nullability::Nullable)),
836                Some(
837                    TemporalMetadata::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string()))
838                        .into()
839                )
840            )))
841        );
842        assert_eq!(vortex_array_non_null.len(), 4);
843        assert_eq!(
844            vortex_array_non_null.dtype(),
845            &DType::Extension(Arc::new(ExtDType::new(
846                TIMESTAMP_ID.clone(),
847                Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)),
848                Some(
849                    TemporalMetadata::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string()))
850                        .into()
851                )
852            )))
853        );
854    }
855
856    #[test]
857    fn test_timestamp_nanosecond_array_conversion() {
858        let arrow_array =
859            TimestampNanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
860        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
861
862        let arrow_array_non_null = TimestampNanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
863        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
864
865        assert_eq!(vortex_array.len(), 4);
866        assert_eq!(vortex_array_non_null.len(), 4);
867    }
868
869    #[test]
870    fn test_time32_second_array_conversion() {
871        let arrow_array = Time32SecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
872        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
873
874        let arrow_array_non_null = Time32SecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
875        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
876
877        assert_eq!(vortex_array.len(), 4);
878        assert_eq!(vortex_array_non_null.len(), 4);
879
880        // Verify metadata - should be TemporalArray with Second time unit
881        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
882        assert_eq!(
883            temporal_array.temporal_metadata().time_unit(),
884            TimeUnit::Seconds
885        );
886
887        let temporal_array_non_null =
888            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
889        assert_eq!(
890            temporal_array_non_null.temporal_metadata().time_unit(),
891            TimeUnit::Seconds
892        );
893    }
894
895    #[test]
896    fn test_time32_millisecond_array_conversion() {
897        let arrow_array =
898            Time32MillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
899        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
900
901        let arrow_array_non_null = Time32MillisecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
902        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
903
904        assert_eq!(vortex_array.len(), 4);
905        assert_eq!(vortex_array_non_null.len(), 4);
906    }
907
908    #[test]
909    fn test_time64_microsecond_array_conversion() {
910        let arrow_array =
911            Time64MicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
912        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
913
914        let arrow_array_non_null = Time64MicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
915        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
916
917        assert_eq!(vortex_array.len(), 4);
918        assert_eq!(vortex_array_non_null.len(), 4);
919    }
920
921    #[test]
922    fn test_time64_nanosecond_array_conversion() {
923        let arrow_array =
924            Time64NanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
925        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
926
927        let arrow_array_non_null = Time64NanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
928        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
929
930        assert_eq!(vortex_array.len(), 4);
931        assert_eq!(vortex_array_non_null.len(), 4);
932    }
933
934    #[test]
935    fn test_date32_array_conversion() {
936        let arrow_array = Date32Array::from(vec![Some(18000), None, Some(18002), Some(18003)]);
937        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
938
939        let arrow_array_non_null = Date32Array::from(vec![18000_i32, 18001, 18002, 18003]);
940        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
941
942        assert_eq!(vortex_array.len(), 4);
943        assert_eq!(vortex_array_non_null.len(), 4);
944    }
945
946    #[test]
947    fn test_date64_array_conversion() {
948        let arrow_array = Date64Array::from(vec![
949            Some(1555200000000),
950            None,
951            Some(1555286400000),
952            Some(1555372800000),
953        ]);
954        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
955
956        let arrow_array_non_null = Date64Array::from(vec![
957            1555200000000_i64,
958            1555213600000,
959            1555286400000,
960            1555372800000,
961        ]);
962        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
963
964        assert_eq!(vortex_array.len(), 4);
965        assert_eq!(vortex_array_non_null.len(), 4);
966    }
967
968    // Test string/binary array conversions
969    #[test]
970    fn test_utf8_array_conversion() {
971        let arrow_array = StringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
972        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
973
974        let arrow_array_non_null = StringArray::from(vec!["hello", "world", "test", "vortex"]);
975        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
976
977        assert_eq!(vortex_array.len(), 4);
978        assert_eq!(vortex_array_non_null.len(), 4);
979
980        // Verify metadata - should be VarBinArray with Utf8 dtype
981        let varbin_array = vortex_array.as_::<VarBinVTable>();
982        assert_eq!(varbin_array.dtype(), &DType::Utf8(true.into()));
983
984        let varbin_array_non_null = vortex_array_non_null.as_::<VarBinVTable>();
985        assert_eq!(varbin_array_non_null.dtype(), &DType::Utf8(false.into()));
986    }
987
988    #[test]
989    fn test_large_utf8_array_conversion() {
990        let arrow_array =
991            LargeStringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
992        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
993
994        let arrow_array_non_null = LargeStringArray::from(vec!["hello", "world", "test", "vortex"]);
995        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
996
997        assert_eq!(vortex_array.len(), 4);
998        assert_eq!(vortex_array_non_null.len(), 4);
999    }
1000
1001    #[test]
1002    fn test_binary_array_conversion() {
1003        let arrow_array = BinaryArray::from(vec![
1004            Some("hello".as_bytes()),
1005            None,
1006            Some("world".as_bytes()),
1007            Some("test".as_bytes()),
1008        ]);
1009        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1010
1011        let arrow_array_non_null = BinaryArray::from(vec![
1012            "hello".as_bytes(),
1013            "world".as_bytes(),
1014            "test".as_bytes(),
1015            "vortex".as_bytes(),
1016        ]);
1017        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1018
1019        assert_eq!(vortex_array.len(), 4);
1020        assert_eq!(vortex_array_non_null.len(), 4);
1021    }
1022
1023    #[test]
1024    fn test_large_binary_array_conversion() {
1025        let arrow_array = LargeBinaryArray::from(vec![
1026            Some("hello".as_bytes()),
1027            None,
1028            Some("world".as_bytes()),
1029            Some("test".as_bytes()),
1030        ]);
1031        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1032
1033        let arrow_array_non_null = LargeBinaryArray::from(vec![
1034            "hello".as_bytes(),
1035            "world".as_bytes(),
1036            "test".as_bytes(),
1037            "vortex".as_bytes(),
1038        ]);
1039        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1040
1041        assert_eq!(vortex_array.len(), 4);
1042        assert_eq!(vortex_array_non_null.len(), 4);
1043    }
1044
1045    #[test]
1046    fn test_utf8_view_array_conversion() {
1047        let mut builder = StringViewBuilder::new();
1048        builder.append_value("hello");
1049        builder.append_null();
1050        builder.append_value("world");
1051        builder.append_value("test");
1052        let arrow_array = builder.finish();
1053        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1054
1055        let mut builder_non_null = StringViewBuilder::new();
1056        builder_non_null.append_value("hello");
1057        builder_non_null.append_value("world");
1058        builder_non_null.append_value("test");
1059        builder_non_null.append_value("vortex");
1060        let arrow_array_non_null = builder_non_null.finish();
1061        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1062
1063        assert_eq!(vortex_array.len(), 4);
1064        assert_eq!(vortex_array_non_null.len(), 4);
1065
1066        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1067        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1068        assert_eq!(
1069            varbin_view_array.buffers().len(),
1070            arrow_array.data_buffers().len()
1071        );
1072        assert_eq!(varbin_view_array.dtype(), &DType::Utf8(true.into()));
1073
1074        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1075        assert_eq!(
1076            varbin_view_array_non_null.buffers().len(),
1077            arrow_array_non_null.data_buffers().len()
1078        );
1079        assert_eq!(
1080            varbin_view_array_non_null.dtype(),
1081            &DType::Utf8(false.into())
1082        );
1083    }
1084
1085    #[test]
1086    fn test_binary_view_array_conversion() {
1087        let mut builder = BinaryViewBuilder::new();
1088        builder.append_value(b"hello");
1089        builder.append_null();
1090        builder.append_value(b"world");
1091        builder.append_value(b"test");
1092        let arrow_array = builder.finish();
1093        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1094
1095        let mut builder_non_null = BinaryViewBuilder::new();
1096        builder_non_null.append_value(b"hello");
1097        builder_non_null.append_value(b"world");
1098        builder_non_null.append_value(b"test");
1099        builder_non_null.append_value(b"vortex");
1100        let arrow_array_non_null = builder_non_null.finish();
1101        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1102
1103        assert_eq!(vortex_array.len(), 4);
1104        assert_eq!(vortex_array_non_null.len(), 4);
1105
1106        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1107        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1108        assert_eq!(
1109            varbin_view_array.buffers().len(),
1110            arrow_array.data_buffers().len()
1111        );
1112        assert_eq!(varbin_view_array.dtype(), &DType::Binary(true.into()));
1113
1114        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1115        assert_eq!(
1116            varbin_view_array_non_null.buffers().len(),
1117            arrow_array_non_null.data_buffers().len()
1118        );
1119        assert_eq!(
1120            varbin_view_array_non_null.dtype(),
1121            &DType::Binary(false.into())
1122        );
1123    }
1124
1125    // Test boolean array conversions
1126    #[test]
1127    fn test_boolean_array_conversion() {
1128        let arrow_array = BooleanArray::from(vec![Some(true), None, Some(false), Some(true)]);
1129        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1130
1131        let arrow_array_non_null = BooleanArray::from(vec![true, false, true, false]);
1132        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1133
1134        assert_eq!(vortex_array.len(), 4);
1135        assert_eq!(vortex_array_non_null.len(), 4);
1136    }
1137
1138    // Test struct array conversions
1139    #[test]
1140    fn test_struct_array_conversion() {
1141        let fields = vec![
1142            Field::new("field1", DataType::Int32, true),
1143            Field::new("field2", DataType::Utf8, false),
1144        ];
1145        let schema = Fields::from(fields);
1146
1147        let field1_data = Int32Array::from(vec![Some(1), None, Some(3)]);
1148        let field2_data = StringArray::from(vec!["a", "b", "c"]);
1149
1150        let arrow_array = StructArray::new(
1151            schema.clone(),
1152            vec![Arc::new(field1_data), Arc::new(field2_data)],
1153            None,
1154        );
1155
1156        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1157        assert_eq!(vortex_array.len(), 3);
1158
1159        // Verify metadata - should be StructArray with correct field names
1160        let struct_vortex_array = vortex_array.as_::<StructVTable>();
1161        assert_eq!(struct_vortex_array.names().len(), 2);
1162        assert_eq!(struct_vortex_array.names()[0], "field1".into());
1163        assert_eq!(struct_vortex_array.names()[1], "field2".into());
1164
1165        // Test nullable struct
1166        let nullable_array = StructArray::new(
1167            schema,
1168            vec![
1169                Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
1170                Arc::new(StringArray::from(vec!["a", "b", "c"])),
1171            ],
1172            Some(arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![
1173                true, false, true,
1174            ]))),
1175        );
1176
1177        let vortex_nullable_array = ArrayRef::from_arrow(&nullable_array, true);
1178        assert_eq!(vortex_nullable_array.len(), 3);
1179
1180        // Verify metadata for nullable struct
1181        let struct_vortex_nullable_array = vortex_nullable_array.as_::<StructVTable>();
1182        assert_eq!(struct_vortex_nullable_array.names().len(), 2);
1183        assert_eq!(struct_vortex_nullable_array.names()[0], "field1".into());
1184        assert_eq!(struct_vortex_nullable_array.names()[1], "field2".into());
1185    }
1186
1187    // Test list array conversions
1188    #[test]
1189    fn test_list_array_conversion() {
1190        let mut builder = ListBuilder::new(Int32Builder::new());
1191        builder.append_value([Some(1), None, Some(3)]);
1192        builder.append_null();
1193        builder.append_value([Some(4), Some(5)]);
1194        let arrow_array = builder.finish();
1195
1196        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1197        assert_eq!(vortex_array.len(), 3);
1198
1199        // Verify metadata - should be ListArray with correct offsets
1200        let list_vortex_array = vortex_array.as_::<ListVTable>();
1201        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1202        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1203        assert_eq!(offsets_array.ptype(), PType::I32);
1204
1205        // Test non-nullable list
1206        let mut builder_non_null = ListBuilder::new(Int32Builder::new());
1207        builder_non_null.append_value([Some(1), None, Some(3)]);
1208        builder_non_null.append_value([Some(4), Some(5)]);
1209        let arrow_array_non_null = builder_non_null.finish();
1210
1211        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1212        assert_eq!(vortex_array_non_null.len(), 2);
1213
1214        // Verify metadata for non-nullable list
1215        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1216        let offsets_array_non_null = list_vortex_array_non_null
1217            .offsets()
1218            .as_::<PrimitiveVTable>();
1219        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1220        assert_eq!(offsets_array_non_null.ptype(), PType::I32);
1221    }
1222
1223    #[test]
1224    fn test_large_list_array_conversion() {
1225        let mut builder = LargeListBuilder::new(Int32Builder::new());
1226        builder.append_value([Some(1), None, Some(3)]);
1227        builder.append_null();
1228        builder.append_value([Some(4), Some(5)]);
1229        let arrow_array = builder.finish();
1230
1231        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1232        assert_eq!(vortex_array.len(), 3);
1233
1234        // Verify metadata - should be ListArray with correct offsets (I64 for large lists)
1235        let list_vortex_array = vortex_array.as_::<ListVTable>();
1236        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1237        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1238        assert_eq!(offsets_array.ptype(), PType::I64); // Large lists use I64 offsets
1239
1240        // Test non-nullable large list
1241        let mut builder_non_null = LargeListBuilder::new(Int32Builder::new());
1242        builder_non_null.append_value([Some(1), None, Some(3)]);
1243        builder_non_null.append_value([Some(4), Some(5)]);
1244        let arrow_array_non_null = builder_non_null.finish();
1245
1246        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1247        assert_eq!(vortex_array_non_null.len(), 2);
1248
1249        // Verify metadata for non-nullable large list
1250        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1251        let offsets_array_non_null = list_vortex_array_non_null
1252            .offsets()
1253            .as_::<PrimitiveVTable>();
1254        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1255        assert_eq!(offsets_array_non_null.ptype(), PType::I64); // Large lists use I64 offsets
1256    }
1257
1258    // Test null array conversions
1259    #[test]
1260    fn test_null_array_conversion() {
1261        let arrow_array = NullArray::new(5);
1262        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1263        assert_eq!(vortex_array.len(), 5);
1264    }
1265
1266    // Test buffer conversions
1267    #[test]
1268    fn test_arrow_buffer_conversion() {
1269        let data = vec![1u8, 2, 3, 4, 5];
1270        let arrow_buffer = ArrowBuffer::from_vec(data);
1271        let vortex_array = arrow_buffer.into_array();
1272        assert_eq!(vortex_array.len(), 5);
1273    }
1274
1275    #[test]
1276    fn test_boolean_buffer_conversion() {
1277        let data = vec![true, false, true, false, true];
1278        let boolean_buffer = BooleanBuffer::from(data);
1279        let vortex_array = boolean_buffer.into_array();
1280        assert_eq!(vortex_array.len(), 5);
1281    }
1282
1283    #[test]
1284    fn test_scalar_buffer_conversion() {
1285        let data = vec![1i32, 2, 3, 4, 5];
1286        let scalar_buffer = ScalarBuffer::from(data);
1287        let vortex_array = scalar_buffer.into_array();
1288        assert_eq!(vortex_array.len(), 5);
1289    }
1290
1291    #[test]
1292    fn test_offset_buffer_conversion() {
1293        let data = vec![0i32, 2, 5, 8, 10];
1294        let offset_buffer = OffsetBuffer::new(ScalarBuffer::from(data));
1295        let vortex_array = offset_buffer.into_array();
1296        assert_eq!(vortex_array.len(), 5);
1297    }
1298
1299    // Test RecordBatch conversions
1300    #[test]
1301    fn test_record_batch_conversion() {
1302        let schema = Arc::new(Schema::new(vec![
1303            Field::new("field1", DataType::Int32, false),
1304            Field::new("field2", DataType::Utf8, false),
1305        ]));
1306
1307        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1308        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1309
1310        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1311
1312        let vortex_array = ArrayRef::from_arrow(record_batch, false);
1313        assert_eq!(vortex_array.len(), 4);
1314
1315        // Test with reference
1316        let schema = Arc::new(Schema::new(vec![
1317            Field::new("field1", DataType::Int32, false),
1318            Field::new("field2", DataType::Utf8, false),
1319        ]));
1320
1321        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1322        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1323
1324        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1325
1326        let vortex_array = ArrayRef::from_arrow(&record_batch, false);
1327        assert_eq!(vortex_array.len(), 4);
1328    }
1329
1330    // Test dynamic dispatch conversion
1331    #[test]
1332    fn test_dyn_array_conversion() {
1333        let int_array = Int32Array::from(vec![1, 2, 3, 4]);
1334        let dyn_array: &dyn ArrowArray = &int_array;
1335        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1336        assert_eq!(vortex_array.len(), 4);
1337
1338        let string_array = StringArray::from(vec!["a", "b", "c"]);
1339        let dyn_array: &dyn ArrowArray = &string_array;
1340        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1341        assert_eq!(vortex_array.len(), 3);
1342
1343        let bool_array = BooleanArray::from(vec![true, false, true]);
1344        let dyn_array: &dyn ArrowArray = &bool_array;
1345        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1346        assert_eq!(vortex_array.len(), 3);
1347    }
1348
1349    // Existing tests
1350    #[test]
1351    pub fn nullable_may_contain_non_nullable() {
1352        let null_struct_array_with_non_nullable_field = new_null_array(
1353            &DataType::Struct(Fields::from(vec![Field::new(
1354                "non_nullable_inner",
1355                DataType::Int32,
1356                false,
1357            )])),
1358            1,
1359        );
1360        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1361    }
1362
1363    #[test]
1364    pub fn nullable_may_contain_deeply_nested_non_nullable() {
1365        let null_struct_array_with_non_nullable_field = new_null_array(
1366            &DataType::Struct(Fields::from(vec![Field::new(
1367                "non_nullable_inner",
1368                DataType::Struct(Fields::from(vec![Field::new(
1369                    "non_nullable_deeper_inner",
1370                    DataType::Int32,
1371                    false,
1372                )])),
1373                false,
1374            )])),
1375            1,
1376        );
1377        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1378    }
1379
1380    #[test]
1381    #[should_panic]
1382    pub fn cannot_handle_nullable_struct_containing_non_nullable_dictionary() {
1383        let null_struct_array_with_non_nullable_field = new_null_array(
1384            &DataType::Struct(Fields::from(vec![Field::new(
1385                "non_nullable_deeper_inner",
1386                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1387                false,
1388            )])),
1389            1,
1390        );
1391
1392        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1393    }
1394}
vortex_array/arrow/convert.rs

vortex_array/arrow/
convert.rs
