vortex_array/arrow/
convert.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::sync::Arc;
5
6use arrow_array::cast::{AsArray, as_null_array};
7use arrow_array::types::{
8    ByteArrayType, ByteViewType, Date32Type, Date64Type, Decimal128Type, Decimal256Type,
9    Float16Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
10    Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
11    TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
12    TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
13};
14use arrow_array::{
15    Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray, GenericByteArray,
16    GenericByteViewArray, GenericListArray, NullArray as ArrowNullArray, OffsetSizeTrait,
17    PrimitiveArray as ArrowPrimitiveArray, RecordBatch, StructArray as ArrowStructArray,
18    make_array,
19};
20use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
21use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
22use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
23use itertools::Itertools;
24use vortex_buffer::{Alignment, Buffer, ByteBuffer};
25use vortex_dtype::datetime::TimeUnit;
26use vortex_dtype::{DType, DecimalDType, NativePType, PType};
27use vortex_error::{VortexExpect as _, vortex_panic};
28use vortex_scalar::i256;
29
30use crate::arrays::{
31    BoolArray, DecimalArray, ListArray, NullArray, PrimitiveArray, StructArray, TemporalArray,
32    VarBinArray, VarBinViewArray,
33};
34use crate::arrow::FromArrowArray;
35use crate::validity::Validity;
36use crate::{ArrayRef, IntoArray};
37
38impl IntoArray for ArrowBuffer {
39    fn into_array(self) -> ArrayRef {
40        PrimitiveArray::from_byte_buffer(
41            ByteBuffer::from_arrow_buffer(self, Alignment::of::<u8>()),
42            PType::U8,
43            Validity::NonNullable,
44        )
45        .into_array()
46    }
47}
48
49impl IntoArray for BooleanBuffer {
50    fn into_array(self) -> ArrayRef {
51        BoolArray::new(self, Validity::NonNullable).into_array()
52    }
53}
54
55impl<T> IntoArray for ScalarBuffer<T>
56where
57    T: ArrowNativeType + NativePType,
58{
59    fn into_array(self) -> ArrayRef {
60        PrimitiveArray::new(
61            Buffer::<T>::from_arrow_scalar_buffer(self),
62            Validity::NonNullable,
63        )
64        .into_array()
65    }
66}
67
68impl<O> IntoArray for OffsetBuffer<O>
69where
70    O: NativePType + OffsetSizeTrait,
71{
72    fn into_array(self) -> ArrayRef {
73        let primitive = PrimitiveArray::new(
74            Buffer::from_arrow_scalar_buffer(self.into_inner()),
75            Validity::NonNullable,
76        );
77
78        primitive.into_array()
79    }
80}
81
82macro_rules! impl_from_arrow_primitive {
83    ($T:path) => {
84        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
85            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
86                let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone());
87                let validity = nulls(value.nulls(), nullable);
88                PrimitiveArray::new(buffer, validity).into_array()
89            }
90        }
91    };
92}
93
94impl_from_arrow_primitive!(Int8Type);
95impl_from_arrow_primitive!(Int16Type);
96impl_from_arrow_primitive!(Int32Type);
97impl_from_arrow_primitive!(Int64Type);
98impl_from_arrow_primitive!(UInt8Type);
99impl_from_arrow_primitive!(UInt16Type);
100impl_from_arrow_primitive!(UInt32Type);
101impl_from_arrow_primitive!(UInt64Type);
102impl_from_arrow_primitive!(Float16Type);
103impl_from_arrow_primitive!(Float32Type);
104impl_from_arrow_primitive!(Float64Type);
105
106impl FromArrowArray<&ArrowPrimitiveArray<Decimal128Type>> for ArrayRef {
107    fn from_arrow(array: &ArrowPrimitiveArray<Decimal128Type>, nullable: bool) -> Self {
108        let decimal_type = DecimalDType::new(array.precision(), array.scale());
109        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
110        let validity = nulls(array.nulls(), nullable);
111        DecimalArray::new(buffer, decimal_type, validity).into_array()
112    }
113}
114
115impl FromArrowArray<&ArrowPrimitiveArray<Decimal256Type>> for ArrayRef {
116    fn from_arrow(array: &ArrowPrimitiveArray<Decimal256Type>, nullable: bool) -> Self {
117        let decimal_type = DecimalDType::new(array.precision(), array.scale());
118        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
119        // SAFETY: Our i256 implementation has the same bit-pattern representation of the
120        //  arrow_buffer::i256 type. It is safe to treat values held inside the buffer as values
121        //  of either type.
122        let buffer =
123            unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(buffer) };
124        let validity = nulls(array.nulls(), nullable);
125        DecimalArray::new(buffer, decimal_type, validity).into_array()
126    }
127}
128
129macro_rules! impl_from_arrow_temporal {
130    ($T:path) => {
131        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
132            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
133                temporal_array(value, nullable)
134            }
135        }
136    };
137}
138
139// timestamp
140impl_from_arrow_temporal!(TimestampSecondType);
141impl_from_arrow_temporal!(TimestampMillisecondType);
142impl_from_arrow_temporal!(TimestampMicrosecondType);
143impl_from_arrow_temporal!(TimestampNanosecondType);
144
145// time
146impl_from_arrow_temporal!(Time32SecondType);
147impl_from_arrow_temporal!(Time32MillisecondType);
148impl_from_arrow_temporal!(Time64MicrosecondType);
149impl_from_arrow_temporal!(Time64NanosecondType);
150
151// date
152impl_from_arrow_temporal!(Date32Type);
153impl_from_arrow_temporal!(Date64Type);
154
155fn temporal_array<T: ArrowPrimitiveType>(value: &ArrowPrimitiveArray<T>, nullable: bool) -> ArrayRef
156where
157    T::Native: NativePType,
158{
159    let arr = PrimitiveArray::new(
160        Buffer::from_arrow_scalar_buffer(value.values().clone()),
161        nulls(value.nulls(), nullable),
162    )
163    .into_array();
164
165    match value.data_type() {
166        DataType::Timestamp(time_unit, tz) => {
167            let tz = tz.as_ref().map(|s| s.to_string());
168            TemporalArray::new_timestamp(arr, time_unit.into(), tz).into()
169        }
170        DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
171        DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
172        DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::D).into(),
173        DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Ms).into(),
174        DataType::Duration(_) => unimplemented!(),
175        DataType::Interval(_) => unimplemented!(),
176        _ => vortex_panic!("Invalid temporal type: {}", value.data_type()),
177    }
178}
179
180impl<T: ByteArrayType> FromArrowArray<&GenericByteArray<T>> for ArrayRef
181where
182    <T as ByteArrayType>::Offset: NativePType,
183{
184    fn from_arrow(value: &GenericByteArray<T>, nullable: bool) -> Self {
185        let dtype = match T::DATA_TYPE {
186            DataType::Binary | DataType::LargeBinary => DType::Binary(nullable.into()),
187            DataType::Utf8 | DataType::LargeUtf8 => DType::Utf8(nullable.into()),
188            dt => vortex_panic!("Invalid data type for ByteArray: {dt}"),
189        };
190        VarBinArray::try_new(
191            value.offsets().clone().into_array(),
192            ByteBuffer::from_arrow_buffer(value.values().clone(), Alignment::of::<u8>()),
193            dtype,
194            nulls(value.nulls(), nullable),
195        )
196        .vortex_expect("Failed to convert Arrow GenericByteArray to Vortex VarBinArray")
197        .into_array()
198    }
199}
200
201impl<T: ByteViewType> FromArrowArray<&GenericByteViewArray<T>> for ArrayRef {
202    fn from_arrow(value: &GenericByteViewArray<T>, nullable: bool) -> Self {
203        let dtype = match T::DATA_TYPE {
204            DataType::BinaryView => DType::Binary(nullable.into()),
205            DataType::Utf8View => DType::Utf8(nullable.into()),
206            dt => vortex_panic!("Invalid data type for ByteViewArray: {dt}"),
207        };
208
209        let views_buffer = Buffer::from_byte_buffer(
210            Buffer::from_arrow_scalar_buffer(value.views().clone()).into_byte_buffer(),
211        );
212
213        // SAFETY: arrow-rs ByteViewArray already checks the same invariants, we inherit those
214        //  guarantees by zero-copy constructing from one.
215        unsafe {
216            VarBinViewArray::new_unchecked(
217                views_buffer,
218                Arc::from(
219                    value
220                        .data_buffers()
221                        .iter()
222                        .map(|b| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::<u8>()))
223                        .collect::<Vec<_>>(),
224                ),
225                dtype,
226                nulls(value.nulls(), nullable),
227            )
228            .into_array()
229        }
230    }
231}
232
233impl FromArrowArray<&ArrowBooleanArray> for ArrayRef {
234    fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self {
235        BoolArray::new(value.values().clone(), nulls(value.nulls(), nullable)).into_array()
236    }
237}
238
239/// Strip out the nulls from this array and return a new array without nulls.
240fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData {
241    if data.null_count() == 0 {
242        // No nulls to remove, return the array as is
243        return data;
244    }
245
246    let children = match data.data_type() {
247        DataType::Struct(fields) => Some(
248            fields
249                .iter()
250                .zip(data.child_data().iter())
251                .map(|(field, child_data)| {
252                    if field.is_nullable() {
253                        child_data.clone()
254                    } else {
255                        remove_nulls(child_data.clone())
256                    }
257                })
258                .collect_vec(),
259        ),
260        DataType::List(f)
261        | DataType::LargeList(f)
262        | DataType::ListView(f)
263        | DataType::LargeListView(f)
264        | DataType::FixedSizeList(f, _)
265            if !f.is_nullable() =>
266        {
267            // All list types only have one child
268            assert_eq!(
269                data.child_data().len(),
270                1,
271                "List types should have one child"
272            );
273            Some(vec![remove_nulls(data.child_data()[0].clone())])
274        }
275        _ => None,
276    };
277
278    let mut builder = data.into_builder().nulls(None);
279    if let Some(children) = children {
280        builder = builder.child_data(children);
281    }
282    builder
283        .build()
284        .vortex_expect("reconstructing array without nulls")
285}
286
287impl FromArrowArray<&ArrowStructArray> for ArrayRef {
288    fn from_arrow(value: &ArrowStructArray, nullable: bool) -> Self {
289        StructArray::try_new(
290            value.column_names().iter().copied().collect(),
291            value
292                .columns()
293                .iter()
294                .zip(value.fields())
295                .map(|(c, field)| {
296                    // Arrow pushes down nulls, even into non-nullable fields. So we strip them
297                    // out here because Vortex is a little more strict.
298                    if c.null_count() > 0 && !field.is_nullable() {
299                        let stripped = make_array(remove_nulls(c.into_data()));
300                        Self::from_arrow(stripped.as_ref(), false)
301                    } else {
302                        Self::from_arrow(c.as_ref(), field.is_nullable())
303                    }
304                })
305                .collect(),
306            value.len(),
307            nulls(value.nulls(), nullable),
308        )
309        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
310        .into_array()
311    }
312}
313
314impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListArray<O>> for ArrayRef {
315    fn from_arrow(value: &GenericListArray<O>, nullable: bool) -> Self {
316        // Extract the validity of the underlying element array
317        let elem_nullable = match value.data_type() {
318            DataType::List(field) => field.is_nullable(),
319            DataType::LargeList(field) => field.is_nullable(),
320            dt => vortex_panic!("Invalid data type for ListArray: {dt}"),
321        };
322        ListArray::try_new(
323            Self::from_arrow(value.values().as_ref(), elem_nullable),
324            // offsets are always non-nullable
325            value.offsets().clone().into_array(),
326            nulls(value.nulls(), nullable),
327        )
328        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
329        .into_array()
330    }
331}
332
333impl FromArrowArray<&ArrowNullArray> for ArrayRef {
334    fn from_arrow(value: &ArrowNullArray, nullable: bool) -> Self {
335        assert!(nullable);
336        NullArray::new(value.len()).into_array()
337    }
338}
339
340fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity {
341    if nullable {
342        nulls
343            .map(|nulls| {
344                if nulls.null_count() == nulls.len() {
345                    Validity::AllInvalid
346                } else {
347                    Validity::from(nulls.inner().clone())
348                }
349            })
350            .unwrap_or_else(|| Validity::AllValid)
351    } else {
352        assert!(nulls.map(|x| x.null_count() == 0).unwrap_or(true));
353        Validity::NonNullable
354    }
355}
356
357impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
358    fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> Self {
359        match array.data_type() {
360            DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable),
361            DataType::UInt8 => Self::from_arrow(array.as_primitive::<UInt8Type>(), nullable),
362            DataType::UInt16 => Self::from_arrow(array.as_primitive::<UInt16Type>(), nullable),
363            DataType::UInt32 => Self::from_arrow(array.as_primitive::<UInt32Type>(), nullable),
364            DataType::UInt64 => Self::from_arrow(array.as_primitive::<UInt64Type>(), nullable),
365            DataType::Int8 => Self::from_arrow(array.as_primitive::<Int8Type>(), nullable),
366            DataType::Int16 => Self::from_arrow(array.as_primitive::<Int16Type>(), nullable),
367            DataType::Int32 => Self::from_arrow(array.as_primitive::<Int32Type>(), nullable),
368            DataType::Int64 => Self::from_arrow(array.as_primitive::<Int64Type>(), nullable),
369            DataType::Float16 => Self::from_arrow(array.as_primitive::<Float16Type>(), nullable),
370            DataType::Float32 => Self::from_arrow(array.as_primitive::<Float32Type>(), nullable),
371            DataType::Float64 => Self::from_arrow(array.as_primitive::<Float64Type>(), nullable),
372            DataType::Utf8 => Self::from_arrow(array.as_string::<i32>(), nullable),
373            DataType::LargeUtf8 => Self::from_arrow(array.as_string::<i64>(), nullable),
374            DataType::Binary => Self::from_arrow(array.as_binary::<i32>(), nullable),
375            DataType::LargeBinary => Self::from_arrow(array.as_binary::<i64>(), nullable),
376            DataType::BinaryView => Self::from_arrow(array.as_binary_view(), nullable),
377            DataType::Utf8View => Self::from_arrow(array.as_string_view(), nullable),
378            DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),
379            DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),
380            DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),
381            DataType::Null => Self::from_arrow(as_null_array(array), nullable),
382            DataType::Timestamp(u, _) => match u {
383                ArrowTimeUnit::Second => {
384                    Self::from_arrow(array.as_primitive::<TimestampSecondType>(), nullable)
385                }
386                ArrowTimeUnit::Millisecond => {
387                    Self::from_arrow(array.as_primitive::<TimestampMillisecondType>(), nullable)
388                }
389                ArrowTimeUnit::Microsecond => {
390                    Self::from_arrow(array.as_primitive::<TimestampMicrosecondType>(), nullable)
391                }
392                ArrowTimeUnit::Nanosecond => {
393                    Self::from_arrow(array.as_primitive::<TimestampNanosecondType>(), nullable)
394                }
395            },
396            DataType::Date32 => Self::from_arrow(array.as_primitive::<Date32Type>(), nullable),
397            DataType::Date64 => Self::from_arrow(array.as_primitive::<Date64Type>(), nullable),
398            DataType::Time32(u) => match u {
399                ArrowTimeUnit::Second => {
400                    Self::from_arrow(array.as_primitive::<Time32SecondType>(), nullable)
401                }
402                ArrowTimeUnit::Millisecond => {
403                    Self::from_arrow(array.as_primitive::<Time32MillisecondType>(), nullable)
404                }
405                ArrowTimeUnit::Microsecond | ArrowTimeUnit::Nanosecond => unreachable!(),
406            },
407            DataType::Time64(u) => match u {
408                ArrowTimeUnit::Microsecond => {
409                    Self::from_arrow(array.as_primitive::<Time64MicrosecondType>(), nullable)
410                }
411                ArrowTimeUnit::Nanosecond => {
412                    Self::from_arrow(array.as_primitive::<Time64NanosecondType>(), nullable)
413                }
414                ArrowTimeUnit::Second | ArrowTimeUnit::Millisecond => unreachable!(),
415            },
416            DataType::Decimal128(..) => {
417                Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)
418            }
419            DataType::Decimal256(..) => {
420                Self::from_arrow(array.as_primitive::<Decimal256Type>(), nullable)
421            }
422            dt => vortex_panic!("Array encoding not implemented for Arrow data type {dt}"),
423        }
424    }
425}
426
427impl FromArrowArray<RecordBatch> for ArrayRef {
428    fn from_arrow(array: RecordBatch, nullable: bool) -> Self {
429        ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable)
430    }
431}
432
433impl FromArrowArray<&RecordBatch> for ArrayRef {
434    fn from_arrow(array: &RecordBatch, nullable: bool) -> Self {
435        Self::from_arrow(array.clone(), nullable)
436    }
437}
438
439#[cfg(test)]
440mod tests {
441    use std::sync::Arc;
442
443    use arrow_array::builder::{
444        BinaryViewBuilder, Decimal128Builder, Decimal256Builder, Int32Builder, LargeListBuilder,
445        ListBuilder, StringViewBuilder,
446    };
447    use arrow_array::types::{ArrowPrimitiveType, Float16Type};
448    use arrow_array::{
449        Array as ArrowArray, BinaryArray, BooleanArray, Date32Array, Date64Array, Float32Array,
450        Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, LargeBinaryArray,
451        LargeStringArray, NullArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray,
452        Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
453        TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
454        TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, new_null_array,
455    };
456    use arrow_buffer::{BooleanBuffer, Buffer as ArrowBuffer, OffsetBuffer, ScalarBuffer};
457    use arrow_schema::{DataType, Field, Fields, Schema};
458    use vortex_dtype::datetime::{TIMESTAMP_ID, TemporalMetadata, TimeUnit};
459    use vortex_dtype::{DType, ExtDType, Nullability, PType};
460
461    use crate::arrays::{
462        DecimalVTable, ListVTable, PrimitiveVTable, StructVTable, TemporalArray, VarBinVTable,
463        VarBinViewVTable,
464    };
465    use crate::arrow::FromArrowArray as _;
466    use crate::{ArrayRef, IntoArray};
467
468    // Test primitive array conversions
469    #[test]
470    fn test_int8_array_conversion() {
471        let arrow_array = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]);
472        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
473
474        let arrow_array_non_null = Int8Array::from(vec![1, 2, 3, 4]);
475        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
476
477        assert_eq!(vortex_array.len(), 4);
478        assert_eq!(vortex_array_non_null.len(), 4);
479
480        // Verify metadata - should be PrimitiveArray with I8 ptype
481        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
482        assert_eq!(primitive_array.ptype(), PType::I8);
483
484        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
485        assert_eq!(primitive_array_non_null.ptype(), PType::I8);
486    }
487
488    #[test]
489    fn test_int16_array_conversion() {
490        let arrow_array = Int16Array::from(vec![Some(100), None, Some(300), Some(400)]);
491        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
492
493        let arrow_array_non_null = Int16Array::from(vec![100, 200, 300, 400]);
494        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
495
496        assert_eq!(vortex_array.len(), 4);
497        assert_eq!(vortex_array_non_null.len(), 4);
498
499        // Verify metadata - should be PrimitiveArray with I16 ptype
500        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
501        assert_eq!(primitive_array.ptype(), PType::I16);
502
503        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
504        assert_eq!(primitive_array_non_null.ptype(), PType::I16);
505    }
506
507    #[test]
508    fn test_int32_array_conversion() {
509        let arrow_array = Int32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
510        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
511
512        let arrow_array_non_null = Int32Array::from(vec![1000, 2000, 3000, 4000]);
513        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
514
515        assert_eq!(vortex_array.len(), 4);
516        assert_eq!(vortex_array_non_null.len(), 4);
517
518        // Verify metadata - should be PrimitiveArray with I32 ptype
519        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
520        assert_eq!(primitive_array.ptype(), PType::I32);
521
522        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
523        assert_eq!(primitive_array_non_null.ptype(), PType::I32);
524    }
525
526    #[test]
527    fn test_int64_array_conversion() {
528        let arrow_array = Int64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
529        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
530
531        let arrow_array_non_null = Int64Array::from(vec![10000_i64, 20000, 30000, 40000]);
532        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
533
534        assert_eq!(vortex_array.len(), 4);
535        assert_eq!(vortex_array_non_null.len(), 4);
536
537        // Verify metadata - should be PrimitiveArray with I64 ptype
538        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
539        assert_eq!(primitive_array.ptype(), PType::I64);
540
541        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
542        assert_eq!(primitive_array_non_null.ptype(), PType::I64);
543    }
544
545    #[test]
546    fn test_uint8_array_conversion() {
547        let arrow_array = UInt8Array::from(vec![Some(1), None, Some(3), Some(4)]);
548        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
549
550        let arrow_array_non_null = UInt8Array::from(vec![1_u8, 2, 3, 4]);
551        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
552
553        assert_eq!(vortex_array.len(), 4);
554        assert_eq!(vortex_array_non_null.len(), 4);
555
556        // Verify metadata - should be PrimitiveArray with U8 ptype
557        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
558        assert_eq!(primitive_array.ptype(), PType::U8);
559
560        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
561        assert_eq!(primitive_array_non_null.ptype(), PType::U8);
562    }
563
564    #[test]
565    fn test_uint16_array_conversion() {
566        let arrow_array = UInt16Array::from(vec![Some(100), None, Some(300), Some(400)]);
567        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
568
569        let arrow_array_non_null = UInt16Array::from(vec![100_u16, 200, 300, 400]);
570        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
571
572        assert_eq!(vortex_array.len(), 4);
573        assert_eq!(vortex_array_non_null.len(), 4);
574
575        // Verify metadata - should be PrimitiveArray with U16 ptype
576        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
577        assert_eq!(primitive_array.ptype(), PType::U16);
578
579        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
580        assert_eq!(primitive_array_non_null.ptype(), PType::U16);
581    }
582
583    #[test]
584    fn test_uint32_array_conversion() {
585        let arrow_array = UInt32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
586        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
587
588        let arrow_array_non_null = UInt32Array::from(vec![1000_u32, 2000, 3000, 4000]);
589        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
590
591        assert_eq!(vortex_array.len(), 4);
592        assert_eq!(vortex_array_non_null.len(), 4);
593
594        // Verify metadata - should be PrimitiveArray with U32 ptype
595        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
596        assert_eq!(primitive_array.ptype(), PType::U32);
597
598        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
599        assert_eq!(primitive_array_non_null.ptype(), PType::U32);
600    }
601
602    #[test]
603    fn test_uint64_array_conversion() {
604        let arrow_array = UInt64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
605        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
606
607        let arrow_array_non_null = UInt64Array::from(vec![10000_u64, 20000, 30000, 40000]);
608        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
609
610        assert_eq!(vortex_array.len(), 4);
611        assert_eq!(vortex_array_non_null.len(), 4);
612
613        // Verify metadata - should be PrimitiveArray with U64 ptype
614        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
615        assert_eq!(primitive_array.ptype(), PType::U64);
616
617        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
618        assert_eq!(primitive_array_non_null.ptype(), PType::U64);
619    }
620
621    #[test]
622    fn test_float16_array_conversion() {
623        let values = vec![
624            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5)),
625            None,
626            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(3.5)),
627        ];
628        let arrow_array = arrow_array::PrimitiveArray::<Float16Type>::from(values);
629        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
630
631        let non_null_values = vec![
632            <Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5),
633            <Float16Type as ArrowPrimitiveType>::Native::from_f32(2.5),
634        ];
635        let arrow_array_non_null =
636            arrow_array::PrimitiveArray::<Float16Type>::from(non_null_values);
637        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
638
639        assert_eq!(vortex_array.len(), 3);
640        assert_eq!(vortex_array_non_null.len(), 2);
641
642        // Verify metadata - should be PrimitiveArray with F16 ptype
643        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
644        assert_eq!(primitive_array.ptype(), PType::F16);
645
646        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
647        assert_eq!(primitive_array_non_null.ptype(), PType::F16);
648    }
649
650    #[test]
651    fn test_float32_array_conversion() {
652        let arrow_array = Float32Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
653        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
654
655        let arrow_array_non_null = Float32Array::from(vec![1.5_f32, 2.5, 3.5, 4.5]);
656        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
657
658        assert_eq!(vortex_array.len(), 4);
659        assert_eq!(vortex_array_non_null.len(), 4);
660
661        // Verify metadata - should be PrimitiveArray with F32 ptype
662        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
663        assert_eq!(primitive_array.ptype(), PType::F32);
664
665        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
666        assert_eq!(primitive_array_non_null.ptype(), PType::F32);
667    }
668
669    #[test]
670    fn test_float64_array_conversion() {
671        let arrow_array = Float64Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
672        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
673
674        let arrow_array_non_null = Float64Array::from(vec![1.5_f64, 2.5, 3.5, 4.5]);
675        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
676
677        assert_eq!(vortex_array.len(), 4);
678        assert_eq!(vortex_array_non_null.len(), 4);
679
680        // Verify metadata - should be PrimitiveArray with F64 ptype
681        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
682        assert_eq!(primitive_array.ptype(), PType::F64);
683
684        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
685        assert_eq!(primitive_array_non_null.ptype(), PType::F64);
686    }
687
688    // Test decimal array conversions
689    #[test]
690    fn test_decimal128_array_conversion() {
691        let mut builder = Decimal128Builder::with_capacity(4);
692        builder.append_value(12345);
693        builder.append_null();
694        builder.append_value(67890);
695        builder.append_value(11111);
696        let decimal_array = builder.finish().with_precision_and_scale(10, 2).unwrap();
697
698        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
699        assert_eq!(vortex_array.len(), 4);
700
701        let mut builder_non_null = Decimal128Builder::with_capacity(3);
702        builder_non_null.append_value(12345);
703        builder_non_null.append_value(67890);
704        builder_non_null.append_value(11111);
705        let decimal_array_non_null = builder_non_null
706            .finish()
707            .with_precision_and_scale(10, 2)
708            .unwrap();
709
710        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
711        assert_eq!(vortex_array_non_null.len(), 3);
712
713        // Verify metadata - should be DecimalArray with correct precision and scale
714        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
715        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 10);
716        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 2);
717
718        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
719        assert_eq!(
720            decimal_vortex_array_non_null.decimal_dtype().precision(),
721            10
722        );
723        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 2);
724    }
725
726    #[test]
727    fn test_decimal256_array_conversion() {
728        let mut builder = Decimal256Builder::with_capacity(4);
729        builder.append_value(arrow_buffer::i256::from_i128(12345));
730        builder.append_null();
731        builder.append_value(arrow_buffer::i256::from_i128(67890));
732        builder.append_value(arrow_buffer::i256::from_i128(11111));
733        let decimal_array = builder.finish().with_precision_and_scale(38, 10).unwrap();
734
735        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
736        assert_eq!(vortex_array.len(), 4);
737
738        let mut builder_non_null = Decimal256Builder::with_capacity(3);
739        builder_non_null.append_value(arrow_buffer::i256::from_i128(12345));
740        builder_non_null.append_value(arrow_buffer::i256::from_i128(67890));
741        builder_non_null.append_value(arrow_buffer::i256::from_i128(11111));
742        let decimal_array_non_null = builder_non_null
743            .finish()
744            .with_precision_and_scale(38, 10)
745            .unwrap();
746
747        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
748        assert_eq!(vortex_array_non_null.len(), 3);
749
750        // Verify metadata - should be DecimalArray with correct precision and scale
751        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
752        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 38);
753        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 10);
754
755        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
756        assert_eq!(
757            decimal_vortex_array_non_null.decimal_dtype().precision(),
758            38
759        );
760        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 10);
761    }
762
763    // Test temporal array conversions
764    #[test]
765    fn test_timestamp_second_array_conversion() {
766        let arrow_array =
767            TimestampSecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
768        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
769
770        let arrow_array_non_null = TimestampSecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
771        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
772
773        assert_eq!(vortex_array.len(), 4);
774        assert_eq!(vortex_array_non_null.len(), 4);
775
776        // Verify metadata - should be TemporalArray with Second time unit
777        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
778        assert_eq!(temporal_array.temporal_metadata().time_unit(), TimeUnit::S);
779
780        let temporal_array_non_null =
781            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
782        assert_eq!(
783            temporal_array_non_null.temporal_metadata().time_unit(),
784            TimeUnit::S
785        );
786    }
787
788    #[test]
789    fn test_timestamp_millisecond_array_conversion() {
790        let arrow_array =
791            TimestampMillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
792        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
793
794        let arrow_array_non_null =
795            TimestampMillisecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
796        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
797
798        assert_eq!(vortex_array.len(), 4);
799        assert_eq!(vortex_array_non_null.len(), 4);
800    }
801
802    #[test]
803    fn test_timestamp_microsecond_array_conversion() {
804        let arrow_array =
805            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
806        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
807
808        let arrow_array_non_null =
809            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
810        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
811
812        assert_eq!(vortex_array.len(), 4);
813        assert_eq!(vortex_array_non_null.len(), 4);
814    }
815
816    #[test]
817    fn test_timestamp_timezone_microsecond_array_conversion() {
818        let arrow_array =
819            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)])
820                .with_timezone("UTC");
821        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
822
823        let arrow_array_non_null =
824            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]).with_timezone("UTC");
825        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
826
827        assert_eq!(vortex_array.len(), 4);
828        assert_eq!(
829            vortex_array.dtype(),
830            &DType::Extension(Arc::new(ExtDType::new(
831                TIMESTAMP_ID.clone(),
832                Arc::new(DType::Primitive(PType::I64, Nullability::Nullable)),
833                Some(TemporalMetadata::Timestamp(TimeUnit::Us, Some("UTC".to_string())).into())
834            )))
835        );
836        assert_eq!(vortex_array_non_null.len(), 4);
837        assert_eq!(
838            vortex_array_non_null.dtype(),
839            &DType::Extension(Arc::new(ExtDType::new(
840                TIMESTAMP_ID.clone(),
841                Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)),
842                Some(TemporalMetadata::Timestamp(TimeUnit::Us, Some("UTC".to_string())).into())
843            )))
844        );
845    }
846
847    #[test]
848    fn test_timestamp_nanosecond_array_conversion() {
849        let arrow_array =
850            TimestampNanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
851        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
852
853        let arrow_array_non_null = TimestampNanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
854        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
855
856        assert_eq!(vortex_array.len(), 4);
857        assert_eq!(vortex_array_non_null.len(), 4);
858    }
859
860    #[test]
861    fn test_time32_second_array_conversion() {
862        let arrow_array = Time32SecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
863        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
864
865        let arrow_array_non_null = Time32SecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
866        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
867
868        assert_eq!(vortex_array.len(), 4);
869        assert_eq!(vortex_array_non_null.len(), 4);
870
871        // Verify metadata - should be TemporalArray with Second time unit
872        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
873        assert_eq!(temporal_array.temporal_metadata().time_unit(), TimeUnit::S);
874
875        let temporal_array_non_null =
876            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
877        assert_eq!(
878            temporal_array_non_null.temporal_metadata().time_unit(),
879            TimeUnit::S
880        );
881    }
882
883    #[test]
884    fn test_time32_millisecond_array_conversion() {
885        let arrow_array =
886            Time32MillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
887        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
888
889        let arrow_array_non_null = Time32MillisecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
890        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
891
892        assert_eq!(vortex_array.len(), 4);
893        assert_eq!(vortex_array_non_null.len(), 4);
894    }
895
896    #[test]
897    fn test_time64_microsecond_array_conversion() {
898        let arrow_array =
899            Time64MicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
900        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
901
902        let arrow_array_non_null = Time64MicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
903        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
904
905        assert_eq!(vortex_array.len(), 4);
906        assert_eq!(vortex_array_non_null.len(), 4);
907    }
908
909    #[test]
910    fn test_time64_nanosecond_array_conversion() {
911        let arrow_array =
912            Time64NanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
913        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
914
915        let arrow_array_non_null = Time64NanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
916        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
917
918        assert_eq!(vortex_array.len(), 4);
919        assert_eq!(vortex_array_non_null.len(), 4);
920    }
921
922    #[test]
923    fn test_date32_array_conversion() {
924        let arrow_array = Date32Array::from(vec![Some(18000), None, Some(18002), Some(18003)]);
925        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
926
927        let arrow_array_non_null = Date32Array::from(vec![18000_i32, 18001, 18002, 18003]);
928        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
929
930        assert_eq!(vortex_array.len(), 4);
931        assert_eq!(vortex_array_non_null.len(), 4);
932    }
933
934    #[test]
935    fn test_date64_array_conversion() {
936        let arrow_array = Date64Array::from(vec![
937            Some(1555200000000),
938            None,
939            Some(1555286400000),
940            Some(1555372800000),
941        ]);
942        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
943
944        let arrow_array_non_null = Date64Array::from(vec![
945            1555200000000_i64,
946            1555213600000,
947            1555286400000,
948            1555372800000,
949        ]);
950        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
951
952        assert_eq!(vortex_array.len(), 4);
953        assert_eq!(vortex_array_non_null.len(), 4);
954    }
955
956    // Test string/binary array conversions
957    #[test]
958    fn test_utf8_array_conversion() {
959        let arrow_array = StringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
960        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
961
962        let arrow_array_non_null = StringArray::from(vec!["hello", "world", "test", "vortex"]);
963        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
964
965        assert_eq!(vortex_array.len(), 4);
966        assert_eq!(vortex_array_non_null.len(), 4);
967
968        // Verify metadata - should be VarBinArray with Utf8 dtype
969        let varbin_array = vortex_array.as_::<VarBinVTable>();
970        assert_eq!(varbin_array.dtype(), &DType::Utf8(true.into()));
971
972        let varbin_array_non_null = vortex_array_non_null.as_::<VarBinVTable>();
973        assert_eq!(varbin_array_non_null.dtype(), &DType::Utf8(false.into()));
974    }
975
976    #[test]
977    fn test_large_utf8_array_conversion() {
978        let arrow_array =
979            LargeStringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
980        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
981
982        let arrow_array_non_null = LargeStringArray::from(vec!["hello", "world", "test", "vortex"]);
983        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
984
985        assert_eq!(vortex_array.len(), 4);
986        assert_eq!(vortex_array_non_null.len(), 4);
987    }
988
989    #[test]
990    fn test_binary_array_conversion() {
991        let arrow_array = BinaryArray::from(vec![
992            Some("hello".as_bytes()),
993            None,
994            Some("world".as_bytes()),
995            Some("test".as_bytes()),
996        ]);
997        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
998
999        let arrow_array_non_null = BinaryArray::from(vec![
1000            "hello".as_bytes(),
1001            "world".as_bytes(),
1002            "test".as_bytes(),
1003            "vortex".as_bytes(),
1004        ]);
1005        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1006
1007        assert_eq!(vortex_array.len(), 4);
1008        assert_eq!(vortex_array_non_null.len(), 4);
1009    }
1010
1011    #[test]
1012    fn test_large_binary_array_conversion() {
1013        let arrow_array = LargeBinaryArray::from(vec![
1014            Some("hello".as_bytes()),
1015            None,
1016            Some("world".as_bytes()),
1017            Some("test".as_bytes()),
1018        ]);
1019        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1020
1021        let arrow_array_non_null = LargeBinaryArray::from(vec![
1022            "hello".as_bytes(),
1023            "world".as_bytes(),
1024            "test".as_bytes(),
1025            "vortex".as_bytes(),
1026        ]);
1027        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1028
1029        assert_eq!(vortex_array.len(), 4);
1030        assert_eq!(vortex_array_non_null.len(), 4);
1031    }
1032
1033    #[test]
1034    fn test_utf8_view_array_conversion() {
1035        let mut builder = StringViewBuilder::new();
1036        builder.append_value("hello");
1037        builder.append_null();
1038        builder.append_value("world");
1039        builder.append_value("test");
1040        let arrow_array = builder.finish();
1041        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1042
1043        let mut builder_non_null = StringViewBuilder::new();
1044        builder_non_null.append_value("hello");
1045        builder_non_null.append_value("world");
1046        builder_non_null.append_value("test");
1047        builder_non_null.append_value("vortex");
1048        let arrow_array_non_null = builder_non_null.finish();
1049        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1050
1051        assert_eq!(vortex_array.len(), 4);
1052        assert_eq!(vortex_array_non_null.len(), 4);
1053
1054        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1055        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1056        assert_eq!(
1057            varbin_view_array.buffers().len(),
1058            arrow_array.data_buffers().len()
1059        );
1060        assert_eq!(varbin_view_array.dtype(), &DType::Utf8(true.into()));
1061
1062        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1063        assert_eq!(
1064            varbin_view_array_non_null.buffers().len(),
1065            arrow_array_non_null.data_buffers().len()
1066        );
1067        assert_eq!(
1068            varbin_view_array_non_null.dtype(),
1069            &DType::Utf8(false.into())
1070        );
1071    }
1072
1073    #[test]
1074    fn test_binary_view_array_conversion() {
1075        let mut builder = BinaryViewBuilder::new();
1076        builder.append_value(b"hello");
1077        builder.append_null();
1078        builder.append_value(b"world");
1079        builder.append_value(b"test");
1080        let arrow_array = builder.finish();
1081        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1082
1083        let mut builder_non_null = BinaryViewBuilder::new();
1084        builder_non_null.append_value(b"hello");
1085        builder_non_null.append_value(b"world");
1086        builder_non_null.append_value(b"test");
1087        builder_non_null.append_value(b"vortex");
1088        let arrow_array_non_null = builder_non_null.finish();
1089        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1090
1091        assert_eq!(vortex_array.len(), 4);
1092        assert_eq!(vortex_array_non_null.len(), 4);
1093
1094        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1095        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1096        assert_eq!(
1097            varbin_view_array.buffers().len(),
1098            arrow_array.data_buffers().len()
1099        );
1100        assert_eq!(varbin_view_array.dtype(), &DType::Binary(true.into()));
1101
1102        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1103        assert_eq!(
1104            varbin_view_array_non_null.buffers().len(),
1105            arrow_array_non_null.data_buffers().len()
1106        );
1107        assert_eq!(
1108            varbin_view_array_non_null.dtype(),
1109            &DType::Binary(false.into())
1110        );
1111    }
1112
1113    // Test boolean array conversions
1114    #[test]
1115    fn test_boolean_array_conversion() {
1116        let arrow_array = BooleanArray::from(vec![Some(true), None, Some(false), Some(true)]);
1117        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1118
1119        let arrow_array_non_null = BooleanArray::from(vec![true, false, true, false]);
1120        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1121
1122        assert_eq!(vortex_array.len(), 4);
1123        assert_eq!(vortex_array_non_null.len(), 4);
1124    }
1125
1126    // Test struct array conversions
1127    #[test]
1128    fn test_struct_array_conversion() {
1129        let fields = vec![
1130            Field::new("field1", DataType::Int32, true),
1131            Field::new("field2", DataType::Utf8, false),
1132        ];
1133        let schema = Fields::from(fields);
1134
1135        let field1_data = Int32Array::from(vec![Some(1), None, Some(3)]);
1136        let field2_data = StringArray::from(vec!["a", "b", "c"]);
1137
1138        let arrow_array = StructArray::new(
1139            schema.clone(),
1140            vec![Arc::new(field1_data), Arc::new(field2_data)],
1141            None,
1142        );
1143
1144        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1145        assert_eq!(vortex_array.len(), 3);
1146
1147        // Verify metadata - should be StructArray with correct field names
1148        let struct_vortex_array = vortex_array.as_::<StructVTable>();
1149        assert_eq!(struct_vortex_array.names().len(), 2);
1150        assert_eq!(struct_vortex_array.names()[0], "field1".into());
1151        assert_eq!(struct_vortex_array.names()[1], "field2".into());
1152
1153        // Test nullable struct
1154        let nullable_array = StructArray::new(
1155            schema,
1156            vec![
1157                Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
1158                Arc::new(StringArray::from(vec!["a", "b", "c"])),
1159            ],
1160            Some(arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![
1161                true, false, true,
1162            ]))),
1163        );
1164
1165        let vortex_nullable_array = ArrayRef::from_arrow(&nullable_array, true);
1166        assert_eq!(vortex_nullable_array.len(), 3);
1167
1168        // Verify metadata for nullable struct
1169        let struct_vortex_nullable_array = vortex_nullable_array.as_::<StructVTable>();
1170        assert_eq!(struct_vortex_nullable_array.names().len(), 2);
1171        assert_eq!(struct_vortex_nullable_array.names()[0], "field1".into());
1172        assert_eq!(struct_vortex_nullable_array.names()[1], "field2".into());
1173    }
1174
1175    // Test list array conversions
1176    #[test]
1177    fn test_list_array_conversion() {
1178        let mut builder = ListBuilder::new(Int32Builder::new());
1179        builder.append_value([Some(1), None, Some(3)]);
1180        builder.append_null();
1181        builder.append_value([Some(4), Some(5)]);
1182        let arrow_array = builder.finish();
1183
1184        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1185        assert_eq!(vortex_array.len(), 3);
1186
1187        // Verify metadata - should be ListArray with correct offsets
1188        let list_vortex_array = vortex_array.as_::<ListVTable>();
1189        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1190        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1191        assert_eq!(offsets_array.ptype(), PType::I32);
1192
1193        // Test non-nullable list
1194        let mut builder_non_null = ListBuilder::new(Int32Builder::new());
1195        builder_non_null.append_value([Some(1), None, Some(3)]);
1196        builder_non_null.append_value([Some(4), Some(5)]);
1197        let arrow_array_non_null = builder_non_null.finish();
1198
1199        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1200        assert_eq!(vortex_array_non_null.len(), 2);
1201
1202        // Verify metadata for non-nullable list
1203        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1204        let offsets_array_non_null = list_vortex_array_non_null
1205            .offsets()
1206            .as_::<PrimitiveVTable>();
1207        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1208        assert_eq!(offsets_array_non_null.ptype(), PType::I32);
1209    }
1210
1211    #[test]
1212    fn test_large_list_array_conversion() {
1213        let mut builder = LargeListBuilder::new(Int32Builder::new());
1214        builder.append_value([Some(1), None, Some(3)]);
1215        builder.append_null();
1216        builder.append_value([Some(4), Some(5)]);
1217        let arrow_array = builder.finish();
1218
1219        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1220        assert_eq!(vortex_array.len(), 3);
1221
1222        // Verify metadata - should be ListArray with correct offsets (I64 for large lists)
1223        let list_vortex_array = vortex_array.as_::<ListVTable>();
1224        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1225        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1226        assert_eq!(offsets_array.ptype(), PType::I64); // Large lists use I64 offsets
1227
1228        // Test non-nullable large list
1229        let mut builder_non_null = LargeListBuilder::new(Int32Builder::new());
1230        builder_non_null.append_value([Some(1), None, Some(3)]);
1231        builder_non_null.append_value([Some(4), Some(5)]);
1232        let arrow_array_non_null = builder_non_null.finish();
1233
1234        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1235        assert_eq!(vortex_array_non_null.len(), 2);
1236
1237        // Verify metadata for non-nullable large list
1238        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1239        let offsets_array_non_null = list_vortex_array_non_null
1240            .offsets()
1241            .as_::<PrimitiveVTable>();
1242        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1243        assert_eq!(offsets_array_non_null.ptype(), PType::I64); // Large lists use I64 offsets
1244    }
1245
1246    // Test null array conversions
1247    #[test]
1248    fn test_null_array_conversion() {
1249        let arrow_array = NullArray::new(5);
1250        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1251        assert_eq!(vortex_array.len(), 5);
1252    }
1253
1254    // Test buffer conversions
1255    #[test]
1256    fn test_arrow_buffer_conversion() {
1257        let data = vec![1u8, 2, 3, 4, 5];
1258        let arrow_buffer = ArrowBuffer::from_vec(data);
1259        let vortex_array = arrow_buffer.into_array();
1260        assert_eq!(vortex_array.len(), 5);
1261    }
1262
1263    #[test]
1264    fn test_boolean_buffer_conversion() {
1265        let data = vec![true, false, true, false, true];
1266        let boolean_buffer = BooleanBuffer::from(data);
1267        let vortex_array = boolean_buffer.into_array();
1268        assert_eq!(vortex_array.len(), 5);
1269    }
1270
1271    #[test]
1272    fn test_scalar_buffer_conversion() {
1273        let data = vec![1i32, 2, 3, 4, 5];
1274        let scalar_buffer = ScalarBuffer::from(data);
1275        let vortex_array = scalar_buffer.into_array();
1276        assert_eq!(vortex_array.len(), 5);
1277    }
1278
1279    #[test]
1280    fn test_offset_buffer_conversion() {
1281        let data = vec![0i32, 2, 5, 8, 10];
1282        let offset_buffer = OffsetBuffer::new(ScalarBuffer::from(data));
1283        let vortex_array = offset_buffer.into_array();
1284        assert_eq!(vortex_array.len(), 5);
1285    }
1286
1287    // Test RecordBatch conversions
1288    #[test]
1289    fn test_record_batch_conversion() {
1290        let schema = Arc::new(Schema::new(vec![
1291            Field::new("field1", DataType::Int32, false),
1292            Field::new("field2", DataType::Utf8, false),
1293        ]));
1294
1295        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1296        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1297
1298        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1299
1300        let vortex_array = ArrayRef::from_arrow(record_batch, false);
1301        assert_eq!(vortex_array.len(), 4);
1302
1303        // Test with reference
1304        let schema = Arc::new(Schema::new(vec![
1305            Field::new("field1", DataType::Int32, false),
1306            Field::new("field2", DataType::Utf8, false),
1307        ]));
1308
1309        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1310        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1311
1312        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1313
1314        let vortex_array = ArrayRef::from_arrow(&record_batch, false);
1315        assert_eq!(vortex_array.len(), 4);
1316    }
1317
1318    // Test dynamic dispatch conversion
1319    #[test]
1320    fn test_dyn_array_conversion() {
1321        let int_array = Int32Array::from(vec![1, 2, 3, 4]);
1322        let dyn_array: &dyn ArrowArray = &int_array;
1323        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1324        assert_eq!(vortex_array.len(), 4);
1325
1326        let string_array = StringArray::from(vec!["a", "b", "c"]);
1327        let dyn_array: &dyn ArrowArray = &string_array;
1328        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1329        assert_eq!(vortex_array.len(), 3);
1330
1331        let bool_array = BooleanArray::from(vec![true, false, true]);
1332        let dyn_array: &dyn ArrowArray = &bool_array;
1333        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1334        assert_eq!(vortex_array.len(), 3);
1335    }
1336
1337    // Existing tests
1338    #[test]
1339    pub fn nullable_may_contain_non_nullable() {
1340        let null_struct_array_with_non_nullable_field = new_null_array(
1341            &DataType::Struct(Fields::from(vec![Field::new(
1342                "non_nullable_inner",
1343                DataType::Int32,
1344                false,
1345            )])),
1346            1,
1347        );
1348        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1349    }
1350
1351    #[test]
1352    pub fn nullable_may_contain_deeply_nested_non_nullable() {
1353        let null_struct_array_with_non_nullable_field = new_null_array(
1354            &DataType::Struct(Fields::from(vec![Field::new(
1355                "non_nullable_inner",
1356                DataType::Struct(Fields::from(vec![Field::new(
1357                    "non_nullable_deeper_inner",
1358                    DataType::Int32,
1359                    false,
1360                )])),
1361                false,
1362            )])),
1363            1,
1364        );
1365        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1366    }
1367
1368    #[test]
1369    #[should_panic]
1370    pub fn cannot_handle_nullable_struct_containing_non_nullable_dictionary() {
1371        let null_struct_array_with_non_nullable_field = new_null_array(
1372            &DataType::Struct(Fields::from(vec![Field::new(
1373                "non_nullable_deeper_inner",
1374                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1375                false,
1376            )])),
1377            1,
1378        );
1379
1380        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1381    }
1382}
vortex_array/arrow/convert.rs

vortex_array/arrow/
convert.rs
