vortex_array/arrow/
convert.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::sync::Arc;
5
6use arrow_array::cast::{AsArray, as_null_array};
7use arrow_array::types::{
8    ByteArrayType, ByteViewType, Date32Type, Date64Type, Decimal128Type, Decimal256Type,
9    Float16Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
10    Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
11    TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
12    TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
13};
14use arrow_array::{
15    Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray, GenericByteArray,
16    GenericByteViewArray, GenericListArray, NullArray as ArrowNullArray, OffsetSizeTrait,
17    PrimitiveArray as ArrowPrimitiveArray, RecordBatch, StructArray as ArrowStructArray,
18    make_array,
19};
20use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
21use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
22use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
23use itertools::Itertools;
24use vortex_buffer::{Alignment, Buffer, ByteBuffer};
25use vortex_dtype::datetime::TimeUnit;
26use vortex_dtype::{DType, DecimalDType, NativePType, PType};
27use vortex_error::{VortexExpect as _, vortex_panic};
28use vortex_scalar::i256;
29
30use crate::arrays::{
31    BoolArray, DecimalArray, ListArray, NullArray, PrimitiveArray, StructArray, TemporalArray,
32    VarBinArray, VarBinViewArray,
33};
34use crate::arrow::FromArrowArray;
35use crate::validity::Validity;
36use crate::{ArrayRef, IntoArray};
37
38impl IntoArray for ArrowBuffer {
39    fn into_array(self) -> ArrayRef {
40        PrimitiveArray::from_byte_buffer(
41            ByteBuffer::from_arrow_buffer(self, Alignment::of::<u8>()),
42            PType::U8,
43            Validity::NonNullable,
44        )
45        .into_array()
46    }
47}
48
49impl IntoArray for BooleanBuffer {
50    fn into_array(self) -> ArrayRef {
51        BoolArray::new(self, Validity::NonNullable).into_array()
52    }
53}
54
55impl<T> IntoArray for ScalarBuffer<T>
56where
57    T: ArrowNativeType + NativePType,
58{
59    fn into_array(self) -> ArrayRef {
60        PrimitiveArray::new(
61            Buffer::<T>::from_arrow_scalar_buffer(self),
62            Validity::NonNullable,
63        )
64        .into_array()
65    }
66}
67
68impl<O> IntoArray for OffsetBuffer<O>
69where
70    O: NativePType + OffsetSizeTrait,
71{
72    fn into_array(self) -> ArrayRef {
73        let primitive = PrimitiveArray::new(
74            Buffer::from_arrow_scalar_buffer(self.into_inner()),
75            Validity::NonNullable,
76        );
77
78        primitive.into_array()
79    }
80}
81
82macro_rules! impl_from_arrow_primitive {
83    ($ty:path) => {
84        impl FromArrowArray<&ArrowPrimitiveArray<$ty>> for ArrayRef {
85            fn from_arrow(value: &ArrowPrimitiveArray<$ty>, nullable: bool) -> Self {
86                let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone());
87                let validity = nulls(value.nulls(), nullable);
88                PrimitiveArray::new(buffer, validity).into_array()
89            }
90        }
91    };
92}
93
94impl_from_arrow_primitive!(Int8Type);
95impl_from_arrow_primitive!(Int16Type);
96impl_from_arrow_primitive!(Int32Type);
97impl_from_arrow_primitive!(Int64Type);
98impl_from_arrow_primitive!(UInt8Type);
99impl_from_arrow_primitive!(UInt16Type);
100impl_from_arrow_primitive!(UInt32Type);
101impl_from_arrow_primitive!(UInt64Type);
102impl_from_arrow_primitive!(Float16Type);
103impl_from_arrow_primitive!(Float32Type);
104impl_from_arrow_primitive!(Float64Type);
105
106impl FromArrowArray<&ArrowPrimitiveArray<Decimal128Type>> for ArrayRef {
107    fn from_arrow(array: &ArrowPrimitiveArray<Decimal128Type>, nullable: bool) -> Self {
108        let decimal_type = DecimalDType::new(array.precision(), array.scale());
109        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
110        let validity = nulls(array.nulls(), nullable);
111        DecimalArray::new(buffer, decimal_type, validity).into_array()
112    }
113}
114
115impl FromArrowArray<&ArrowPrimitiveArray<Decimal256Type>> for ArrayRef {
116    fn from_arrow(array: &ArrowPrimitiveArray<Decimal256Type>, nullable: bool) -> Self {
117        let decimal_type = DecimalDType::new(array.precision(), array.scale());
118        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
119        // SAFETY: Our i256 implementation has the same bit-pattern representation of the
120        //  arrow_buffer::i256 type. It is safe to treat values held inside the buffer as values
121        //  of either type.
122        let buffer =
123            unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(buffer) };
124        let validity = nulls(array.nulls(), nullable);
125        DecimalArray::new(buffer, decimal_type, validity).into_array()
126    }
127}
128
129macro_rules! impl_from_arrow_temporal {
130    ($ty:path) => {
131        impl FromArrowArray<&ArrowPrimitiveArray<$ty>> for ArrayRef {
132            fn from_arrow(value: &ArrowPrimitiveArray<$ty>, nullable: bool) -> Self {
133                temporal_array(value, nullable)
134            }
135        }
136    };
137}
138
139// timestamp
140impl_from_arrow_temporal!(TimestampSecondType);
141impl_from_arrow_temporal!(TimestampMillisecondType);
142impl_from_arrow_temporal!(TimestampMicrosecondType);
143impl_from_arrow_temporal!(TimestampNanosecondType);
144
145// time
146impl_from_arrow_temporal!(Time32SecondType);
147impl_from_arrow_temporal!(Time32MillisecondType);
148impl_from_arrow_temporal!(Time64MicrosecondType);
149impl_from_arrow_temporal!(Time64NanosecondType);
150
151// date
152impl_from_arrow_temporal!(Date32Type);
153impl_from_arrow_temporal!(Date64Type);
154
155fn temporal_array<T: ArrowPrimitiveType>(value: &ArrowPrimitiveArray<T>, nullable: bool) -> ArrayRef
156where
157    T::Native: NativePType,
158{
159    let arr = PrimitiveArray::new(
160        Buffer::from_arrow_scalar_buffer(value.values().clone()),
161        nulls(value.nulls(), nullable),
162    )
163    .into_array();
164
165    match value.data_type() {
166        DataType::Timestamp(time_unit, tz) => {
167            let tz = tz.as_ref().map(|s| s.to_string());
168            TemporalArray::new_timestamp(arr, time_unit.into(), tz).into()
169        }
170        DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
171        DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
172        DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::D).into(),
173        DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Ms).into(),
174        DataType::Duration(_) => unimplemented!(),
175        DataType::Interval(_) => unimplemented!(),
176        _ => vortex_panic!("Invalid temporal type: {}", value.data_type()),
177    }
178}
179
180impl<T: ByteArrayType> FromArrowArray<&GenericByteArray<T>> for ArrayRef
181where
182    <T as ByteArrayType>::Offset: NativePType,
183{
184    fn from_arrow(value: &GenericByteArray<T>, nullable: bool) -> Self {
185        let dtype = match T::DATA_TYPE {
186            DataType::Binary | DataType::LargeBinary => DType::Binary(nullable.into()),
187            DataType::Utf8 | DataType::LargeUtf8 => DType::Utf8(nullable.into()),
188            _ => vortex_panic!("Invalid data type for ByteArray: {}", T::DATA_TYPE),
189        };
190        VarBinArray::try_new(
191            value.offsets().clone().into_array(),
192            ByteBuffer::from_arrow_buffer(value.values().clone(), Alignment::of::<u8>()),
193            dtype,
194            nulls(value.nulls(), nullable),
195        )
196        .vortex_expect("Failed to convert Arrow GenericByteArray to Vortex VarBinArray")
197        .into_array()
198    }
199}
200
201impl<T: ByteViewType> FromArrowArray<&GenericByteViewArray<T>> for ArrayRef {
202    fn from_arrow(value: &GenericByteViewArray<T>, nullable: bool) -> Self {
203        let dtype = match T::DATA_TYPE {
204            DataType::BinaryView => DType::Binary(nullable.into()),
205            DataType::Utf8View => DType::Utf8(nullable.into()),
206            _ => vortex_panic!("Invalid data type for ByteViewArray: {}", T::DATA_TYPE),
207        };
208
209        let views_buffer = Buffer::from_byte_buffer(
210            Buffer::from_arrow_scalar_buffer(value.views().clone()).into_byte_buffer(),
211        );
212
213        // SAFETY: arrow-rs ByteViewArray already checks the same invariants, we inherit those
214        //  guarantees by zero-copy constructing from one.
215        unsafe {
216            VarBinViewArray::new_unchecked(
217                views_buffer,
218                Arc::from(
219                    value
220                        .data_buffers()
221                        .iter()
222                        .map(|b| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::<u8>()))
223                        .collect::<Vec<_>>(),
224                ),
225                dtype,
226                nulls(value.nulls(), nullable),
227            )
228            .into_array()
229        }
230    }
231}
232
233impl FromArrowArray<&ArrowBooleanArray> for ArrayRef {
234    fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self {
235        BoolArray::new(value.values().clone(), nulls(value.nulls(), nullable)).into_array()
236    }
237}
238
239/// Strip out the nulls from this array and return a new array without nulls.
240fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData {
241    if data.null_count() == 0 {
242        // No nulls to remove, return the array as is
243        return data;
244    }
245
246    let children = match data.data_type() {
247        DataType::Struct(fields) => Some(
248            fields
249                .iter()
250                .zip(data.child_data().iter())
251                .map(|(field, child_data)| {
252                    if field.is_nullable() {
253                        child_data.clone()
254                    } else {
255                        remove_nulls(child_data.clone())
256                    }
257                })
258                .collect_vec(),
259        ),
260        DataType::List(f)
261        | DataType::LargeList(f)
262        | DataType::ListView(f)
263        | DataType::LargeListView(f)
264        | DataType::FixedSizeList(f, _)
265            if !f.is_nullable() =>
266        {
267            // All list types only have one child
268            assert_eq!(
269                data.child_data().len(),
270                1,
271                "List types should have one child"
272            );
273            Some(vec![remove_nulls(data.child_data()[0].clone())])
274        }
275        _ => None,
276    };
277
278    let mut builder = data.into_builder().nulls(None);
279    if let Some(children) = children {
280        builder = builder.child_data(children);
281    }
282    builder
283        .build()
284        .vortex_expect("reconstructing array without nulls")
285}
286
287impl FromArrowArray<&ArrowStructArray> for ArrayRef {
288    fn from_arrow(value: &ArrowStructArray, nullable: bool) -> Self {
289        StructArray::try_new(
290            value.column_names().iter().copied().collect(),
291            value
292                .columns()
293                .iter()
294                .zip(value.fields())
295                .map(|(c, field)| {
296                    // Arrow pushes down nulls, even into non-nullable fields. So we strip them
297                    // out here because Vortex is a little more strict.
298                    if c.null_count() > 0 && !field.is_nullable() {
299                        let stripped = make_array(remove_nulls(c.into_data()));
300                        Self::from_arrow(stripped.as_ref(), false)
301                    } else {
302                        Self::from_arrow(c.as_ref(), field.is_nullable())
303                    }
304                })
305                .collect(),
306            value.len(),
307            nulls(value.nulls(), nullable),
308        )
309        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
310        .into_array()
311    }
312}
313
314impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListArray<O>> for ArrayRef {
315    fn from_arrow(value: &GenericListArray<O>, nullable: bool) -> Self {
316        // Extract the validity of the underlying element array
317        let elem_nullable = match value.data_type() {
318            DataType::List(field) => field.is_nullable(),
319            DataType::LargeList(field) => field.is_nullable(),
320            dt => vortex_panic!("Invalid data type for ListArray: {dt}"),
321        };
322        ListArray::try_new(
323            Self::from_arrow(value.values().as_ref(), elem_nullable),
324            // offsets are always non-nullable
325            value.offsets().clone().into_array(),
326            nulls(value.nulls(), nullable),
327        )
328        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
329        .into_array()
330    }
331}
332
333impl FromArrowArray<&ArrowNullArray> for ArrayRef {
334    fn from_arrow(value: &ArrowNullArray, nullable: bool) -> Self {
335        assert!(nullable);
336        NullArray::new(value.len()).into_array()
337    }
338}
339
340fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity {
341    if nullable {
342        nulls
343            .map(|nulls| {
344                if nulls.null_count() == nulls.len() {
345                    Validity::AllInvalid
346                } else {
347                    Validity::from(nulls.inner().clone())
348                }
349            })
350            .unwrap_or_else(|| Validity::AllValid)
351    } else {
352        assert!(nulls.map(|x| x.null_count() == 0).unwrap_or(true));
353        Validity::NonNullable
354    }
355}
356
357impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
358    fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> Self {
359        match array.data_type() {
360            DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable),
361            DataType::UInt8 => Self::from_arrow(array.as_primitive::<UInt8Type>(), nullable),
362            DataType::UInt16 => Self::from_arrow(array.as_primitive::<UInt16Type>(), nullable),
363            DataType::UInt32 => Self::from_arrow(array.as_primitive::<UInt32Type>(), nullable),
364            DataType::UInt64 => Self::from_arrow(array.as_primitive::<UInt64Type>(), nullable),
365            DataType::Int8 => Self::from_arrow(array.as_primitive::<Int8Type>(), nullable),
366            DataType::Int16 => Self::from_arrow(array.as_primitive::<Int16Type>(), nullable),
367            DataType::Int32 => Self::from_arrow(array.as_primitive::<Int32Type>(), nullable),
368            DataType::Int64 => Self::from_arrow(array.as_primitive::<Int64Type>(), nullable),
369            DataType::Float16 => Self::from_arrow(array.as_primitive::<Float16Type>(), nullable),
370            DataType::Float32 => Self::from_arrow(array.as_primitive::<Float32Type>(), nullable),
371            DataType::Float64 => Self::from_arrow(array.as_primitive::<Float64Type>(), nullable),
372            DataType::Utf8 => Self::from_arrow(array.as_string::<i32>(), nullable),
373            DataType::LargeUtf8 => Self::from_arrow(array.as_string::<i64>(), nullable),
374            DataType::Binary => Self::from_arrow(array.as_binary::<i32>(), nullable),
375            DataType::LargeBinary => Self::from_arrow(array.as_binary::<i64>(), nullable),
376            DataType::BinaryView => Self::from_arrow(array.as_binary_view(), nullable),
377            DataType::Utf8View => Self::from_arrow(array.as_string_view(), nullable),
378            DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),
379            DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),
380            DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),
381            DataType::Null => Self::from_arrow(as_null_array(array), nullable),
382            DataType::Timestamp(u, _) => match u {
383                ArrowTimeUnit::Second => {
384                    Self::from_arrow(array.as_primitive::<TimestampSecondType>(), nullable)
385                }
386                ArrowTimeUnit::Millisecond => {
387                    Self::from_arrow(array.as_primitive::<TimestampMillisecondType>(), nullable)
388                }
389                ArrowTimeUnit::Microsecond => {
390                    Self::from_arrow(array.as_primitive::<TimestampMicrosecondType>(), nullable)
391                }
392                ArrowTimeUnit::Nanosecond => {
393                    Self::from_arrow(array.as_primitive::<TimestampNanosecondType>(), nullable)
394                }
395            },
396            DataType::Date32 => Self::from_arrow(array.as_primitive::<Date32Type>(), nullable),
397            DataType::Date64 => Self::from_arrow(array.as_primitive::<Date64Type>(), nullable),
398            DataType::Time32(u) => match u {
399                ArrowTimeUnit::Second => {
400                    Self::from_arrow(array.as_primitive::<Time32SecondType>(), nullable)
401                }
402                ArrowTimeUnit::Millisecond => {
403                    Self::from_arrow(array.as_primitive::<Time32MillisecondType>(), nullable)
404                }
405                _ => unreachable!(),
406            },
407            DataType::Time64(u) => match u {
408                ArrowTimeUnit::Microsecond => {
409                    Self::from_arrow(array.as_primitive::<Time64MicrosecondType>(), nullable)
410                }
411                ArrowTimeUnit::Nanosecond => {
412                    Self::from_arrow(array.as_primitive::<Time64NanosecondType>(), nullable)
413                }
414                _ => unreachable!(),
415            },
416            DataType::Decimal128(..) => {
417                Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)
418            }
419            DataType::Decimal256(..) => {
420                Self::from_arrow(array.as_primitive::<Decimal256Type>(), nullable)
421            }
422            _ => vortex_panic!(
423                "Array encoding not implemented for Arrow data type {}",
424                array.data_type().clone()
425            ),
426        }
427    }
428}
429
430impl FromArrowArray<RecordBatch> for ArrayRef {
431    fn from_arrow(array: RecordBatch, nullable: bool) -> Self {
432        ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable)
433    }
434}
435
436impl FromArrowArray<&RecordBatch> for ArrayRef {
437    fn from_arrow(array: &RecordBatch, nullable: bool) -> Self {
438        Self::from_arrow(array.clone(), nullable)
439    }
440}
441
442#[cfg(test)]
443mod tests {
444    use std::sync::Arc;
445
446    use arrow_array::builder::{
447        BinaryViewBuilder, Decimal128Builder, Decimal256Builder, Int32Builder, LargeListBuilder,
448        ListBuilder, StringViewBuilder,
449    };
450    use arrow_array::types::{ArrowPrimitiveType, Float16Type};
451    use arrow_array::{
452        Array as ArrowArray, BinaryArray, BooleanArray, Date32Array, Date64Array, Float32Array,
453        Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, LargeBinaryArray,
454        LargeStringArray, NullArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray,
455        Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
456        TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
457        TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, new_null_array,
458    };
459    use arrow_buffer::{BooleanBuffer, Buffer as ArrowBuffer, OffsetBuffer, ScalarBuffer};
460    use arrow_schema::{DataType, Field, Fields, Schema};
461    use vortex_dtype::datetime::{TIMESTAMP_ID, TemporalMetadata, TimeUnit};
462    use vortex_dtype::{DType, ExtDType, Nullability, PType};
463
464    use crate::arrays::{
465        DecimalVTable, ListVTable, PrimitiveVTable, StructVTable, TemporalArray, VarBinVTable,
466        VarBinViewVTable,
467    };
468    use crate::arrow::FromArrowArray as _;
469    use crate::{ArrayRef, IntoArray};
470
471    // Test primitive array conversions
472    #[test]
473    fn test_int8_array_conversion() {
474        let arrow_array = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]);
475        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
476
477        let arrow_array_non_null = Int8Array::from(vec![1, 2, 3, 4]);
478        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
479
480        assert_eq!(vortex_array.len(), 4);
481        assert_eq!(vortex_array_non_null.len(), 4);
482
483        // Verify metadata - should be PrimitiveArray with I8 ptype
484        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
485        assert_eq!(primitive_array.ptype(), PType::I8);
486
487        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
488        assert_eq!(primitive_array_non_null.ptype(), PType::I8);
489    }
490
491    #[test]
492    fn test_int16_array_conversion() {
493        let arrow_array = Int16Array::from(vec![Some(100), None, Some(300), Some(400)]);
494        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
495
496        let arrow_array_non_null = Int16Array::from(vec![100, 200, 300, 400]);
497        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
498
499        assert_eq!(vortex_array.len(), 4);
500        assert_eq!(vortex_array_non_null.len(), 4);
501
502        // Verify metadata - should be PrimitiveArray with I16 ptype
503        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
504        assert_eq!(primitive_array.ptype(), PType::I16);
505
506        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
507        assert_eq!(primitive_array_non_null.ptype(), PType::I16);
508    }
509
510    #[test]
511    fn test_int32_array_conversion() {
512        let arrow_array = Int32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
513        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
514
515        let arrow_array_non_null = Int32Array::from(vec![1000, 2000, 3000, 4000]);
516        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
517
518        assert_eq!(vortex_array.len(), 4);
519        assert_eq!(vortex_array_non_null.len(), 4);
520
521        // Verify metadata - should be PrimitiveArray with I32 ptype
522        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
523        assert_eq!(primitive_array.ptype(), PType::I32);
524
525        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
526        assert_eq!(primitive_array_non_null.ptype(), PType::I32);
527    }
528
529    #[test]
530    fn test_int64_array_conversion() {
531        let arrow_array = Int64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
532        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
533
534        let arrow_array_non_null = Int64Array::from(vec![10000_i64, 20000, 30000, 40000]);
535        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
536
537        assert_eq!(vortex_array.len(), 4);
538        assert_eq!(vortex_array_non_null.len(), 4);
539
540        // Verify metadata - should be PrimitiveArray with I64 ptype
541        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
542        assert_eq!(primitive_array.ptype(), PType::I64);
543
544        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
545        assert_eq!(primitive_array_non_null.ptype(), PType::I64);
546    }
547
548    #[test]
549    fn test_uint8_array_conversion() {
550        let arrow_array = UInt8Array::from(vec![Some(1), None, Some(3), Some(4)]);
551        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
552
553        let arrow_array_non_null = UInt8Array::from(vec![1_u8, 2, 3, 4]);
554        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
555
556        assert_eq!(vortex_array.len(), 4);
557        assert_eq!(vortex_array_non_null.len(), 4);
558
559        // Verify metadata - should be PrimitiveArray with U8 ptype
560        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
561        assert_eq!(primitive_array.ptype(), PType::U8);
562
563        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
564        assert_eq!(primitive_array_non_null.ptype(), PType::U8);
565    }
566
567    #[test]
568    fn test_uint16_array_conversion() {
569        let arrow_array = UInt16Array::from(vec![Some(100), None, Some(300), Some(400)]);
570        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
571
572        let arrow_array_non_null = UInt16Array::from(vec![100_u16, 200, 300, 400]);
573        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
574
575        assert_eq!(vortex_array.len(), 4);
576        assert_eq!(vortex_array_non_null.len(), 4);
577
578        // Verify metadata - should be PrimitiveArray with U16 ptype
579        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
580        assert_eq!(primitive_array.ptype(), PType::U16);
581
582        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
583        assert_eq!(primitive_array_non_null.ptype(), PType::U16);
584    }
585
586    #[test]
587    fn test_uint32_array_conversion() {
588        let arrow_array = UInt32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
589        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
590
591        let arrow_array_non_null = UInt32Array::from(vec![1000_u32, 2000, 3000, 4000]);
592        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
593
594        assert_eq!(vortex_array.len(), 4);
595        assert_eq!(vortex_array_non_null.len(), 4);
596
597        // Verify metadata - should be PrimitiveArray with U32 ptype
598        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
599        assert_eq!(primitive_array.ptype(), PType::U32);
600
601        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
602        assert_eq!(primitive_array_non_null.ptype(), PType::U32);
603    }
604
605    #[test]
606    fn test_uint64_array_conversion() {
607        let arrow_array = UInt64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
608        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
609
610        let arrow_array_non_null = UInt64Array::from(vec![10000_u64, 20000, 30000, 40000]);
611        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
612
613        assert_eq!(vortex_array.len(), 4);
614        assert_eq!(vortex_array_non_null.len(), 4);
615
616        // Verify metadata - should be PrimitiveArray with U64 ptype
617        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
618        assert_eq!(primitive_array.ptype(), PType::U64);
619
620        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
621        assert_eq!(primitive_array_non_null.ptype(), PType::U64);
622    }
623
624    #[test]
625    fn test_float16_array_conversion() {
626        let values = vec![
627            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5)),
628            None,
629            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(3.5)),
630        ];
631        let arrow_array = arrow_array::PrimitiveArray::<Float16Type>::from(values);
632        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
633
634        let non_null_values = vec![
635            <Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5),
636            <Float16Type as ArrowPrimitiveType>::Native::from_f32(2.5),
637        ];
638        let arrow_array_non_null =
639            arrow_array::PrimitiveArray::<Float16Type>::from(non_null_values);
640        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
641
642        assert_eq!(vortex_array.len(), 3);
643        assert_eq!(vortex_array_non_null.len(), 2);
644
645        // Verify metadata - should be PrimitiveArray with F16 ptype
646        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
647        assert_eq!(primitive_array.ptype(), PType::F16);
648
649        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
650        assert_eq!(primitive_array_non_null.ptype(), PType::F16);
651    }
652
653    #[test]
654    fn test_float32_array_conversion() {
655        let arrow_array = Float32Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
656        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
657
658        let arrow_array_non_null = Float32Array::from(vec![1.5_f32, 2.5, 3.5, 4.5]);
659        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
660
661        assert_eq!(vortex_array.len(), 4);
662        assert_eq!(vortex_array_non_null.len(), 4);
663
664        // Verify metadata - should be PrimitiveArray with F32 ptype
665        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
666        assert_eq!(primitive_array.ptype(), PType::F32);
667
668        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
669        assert_eq!(primitive_array_non_null.ptype(), PType::F32);
670    }
671
672    #[test]
673    fn test_float64_array_conversion() {
674        let arrow_array = Float64Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
675        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
676
677        let arrow_array_non_null = Float64Array::from(vec![1.5_f64, 2.5, 3.5, 4.5]);
678        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
679
680        assert_eq!(vortex_array.len(), 4);
681        assert_eq!(vortex_array_non_null.len(), 4);
682
683        // Verify metadata - should be PrimitiveArray with F64 ptype
684        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
685        assert_eq!(primitive_array.ptype(), PType::F64);
686
687        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
688        assert_eq!(primitive_array_non_null.ptype(), PType::F64);
689    }
690
691    // Test decimal array conversions
692    #[test]
693    fn test_decimal128_array_conversion() {
694        let mut builder = Decimal128Builder::with_capacity(4);
695        builder.append_value(12345);
696        builder.append_null();
697        builder.append_value(67890);
698        builder.append_value(11111);
699        let decimal_array = builder.finish().with_precision_and_scale(10, 2).unwrap();
700
701        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
702        assert_eq!(vortex_array.len(), 4);
703
704        let mut builder_non_null = Decimal128Builder::with_capacity(3);
705        builder_non_null.append_value(12345);
706        builder_non_null.append_value(67890);
707        builder_non_null.append_value(11111);
708        let decimal_array_non_null = builder_non_null
709            .finish()
710            .with_precision_and_scale(10, 2)
711            .unwrap();
712
713        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
714        assert_eq!(vortex_array_non_null.len(), 3);
715
716        // Verify metadata - should be DecimalArray with correct precision and scale
717        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
718        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 10);
719        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 2);
720
721        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
722        assert_eq!(
723            decimal_vortex_array_non_null.decimal_dtype().precision(),
724            10
725        );
726        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 2);
727    }
728
729    #[test]
730    fn test_decimal256_array_conversion() {
731        let mut builder = Decimal256Builder::with_capacity(4);
732        builder.append_value(arrow_buffer::i256::from_i128(12345));
733        builder.append_null();
734        builder.append_value(arrow_buffer::i256::from_i128(67890));
735        builder.append_value(arrow_buffer::i256::from_i128(11111));
736        let decimal_array = builder.finish().with_precision_and_scale(38, 10).unwrap();
737
738        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
739        assert_eq!(vortex_array.len(), 4);
740
741        let mut builder_non_null = Decimal256Builder::with_capacity(3);
742        builder_non_null.append_value(arrow_buffer::i256::from_i128(12345));
743        builder_non_null.append_value(arrow_buffer::i256::from_i128(67890));
744        builder_non_null.append_value(arrow_buffer::i256::from_i128(11111));
745        let decimal_array_non_null = builder_non_null
746            .finish()
747            .with_precision_and_scale(38, 10)
748            .unwrap();
749
750        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
751        assert_eq!(vortex_array_non_null.len(), 3);
752
753        // Verify metadata - should be DecimalArray with correct precision and scale
754        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
755        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 38);
756        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 10);
757
758        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
759        assert_eq!(
760            decimal_vortex_array_non_null.decimal_dtype().precision(),
761            38
762        );
763        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 10);
764    }
765
766    // Test temporal array conversions
767    #[test]
768    fn test_timestamp_second_array_conversion() {
769        let arrow_array =
770            TimestampSecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
771        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
772
773        let arrow_array_non_null = TimestampSecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
774        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
775
776        assert_eq!(vortex_array.len(), 4);
777        assert_eq!(vortex_array_non_null.len(), 4);
778
779        // Verify metadata - should be TemporalArray with Second time unit
780        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
781        assert_eq!(temporal_array.temporal_metadata().time_unit(), TimeUnit::S);
782
783        let temporal_array_non_null =
784            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
785        assert_eq!(
786            temporal_array_non_null.temporal_metadata().time_unit(),
787            TimeUnit::S
788        );
789    }
790
791    #[test]
792    fn test_timestamp_millisecond_array_conversion() {
793        let arrow_array =
794            TimestampMillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
795        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
796
797        let arrow_array_non_null =
798            TimestampMillisecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
799        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
800
801        assert_eq!(vortex_array.len(), 4);
802        assert_eq!(vortex_array_non_null.len(), 4);
803    }
804
805    #[test]
806    fn test_timestamp_microsecond_array_conversion() {
807        let arrow_array =
808            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
809        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
810
811        let arrow_array_non_null =
812            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
813        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
814
815        assert_eq!(vortex_array.len(), 4);
816        assert_eq!(vortex_array_non_null.len(), 4);
817    }
818
819    #[test]
820    fn test_timestamp_timezone_microsecond_array_conversion() {
821        let arrow_array =
822            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)])
823                .with_timezone("UTC");
824        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
825
826        let arrow_array_non_null =
827            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]).with_timezone("UTC");
828        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
829
830        assert_eq!(vortex_array.len(), 4);
831        assert_eq!(
832            vortex_array.dtype(),
833            &DType::Extension(Arc::new(ExtDType::new(
834                TIMESTAMP_ID.clone(),
835                Arc::new(DType::Primitive(PType::I64, Nullability::Nullable)),
836                Some(TemporalMetadata::Timestamp(TimeUnit::Us, Some("UTC".to_string())).into())
837            )))
838        );
839        assert_eq!(vortex_array_non_null.len(), 4);
840        assert_eq!(
841            vortex_array_non_null.dtype(),
842            &DType::Extension(Arc::new(ExtDType::new(
843                TIMESTAMP_ID.clone(),
844                Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)),
845                Some(TemporalMetadata::Timestamp(TimeUnit::Us, Some("UTC".to_string())).into())
846            )))
847        );
848    }
849
850    #[test]
851    fn test_timestamp_nanosecond_array_conversion() {
852        let arrow_array =
853            TimestampNanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
854        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
855
856        let arrow_array_non_null = TimestampNanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
857        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
858
859        assert_eq!(vortex_array.len(), 4);
860        assert_eq!(vortex_array_non_null.len(), 4);
861    }
862
863    #[test]
864    fn test_time32_second_array_conversion() {
865        let arrow_array = Time32SecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
866        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
867
868        let arrow_array_non_null = Time32SecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
869        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
870
871        assert_eq!(vortex_array.len(), 4);
872        assert_eq!(vortex_array_non_null.len(), 4);
873
874        // Verify metadata - should be TemporalArray with Second time unit
875        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
876        assert_eq!(temporal_array.temporal_metadata().time_unit(), TimeUnit::S);
877
878        let temporal_array_non_null =
879            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
880        assert_eq!(
881            temporal_array_non_null.temporal_metadata().time_unit(),
882            TimeUnit::S
883        );
884    }
885
886    #[test]
887    fn test_time32_millisecond_array_conversion() {
888        let arrow_array =
889            Time32MillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
890        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
891
892        let arrow_array_non_null = Time32MillisecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
893        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
894
895        assert_eq!(vortex_array.len(), 4);
896        assert_eq!(vortex_array_non_null.len(), 4);
897    }
898
899    #[test]
900    fn test_time64_microsecond_array_conversion() {
901        let arrow_array =
902            Time64MicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
903        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
904
905        let arrow_array_non_null = Time64MicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
906        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
907
908        assert_eq!(vortex_array.len(), 4);
909        assert_eq!(vortex_array_non_null.len(), 4);
910    }
911
912    #[test]
913    fn test_time64_nanosecond_array_conversion() {
914        let arrow_array =
915            Time64NanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
916        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
917
918        let arrow_array_non_null = Time64NanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
919        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
920
921        assert_eq!(vortex_array.len(), 4);
922        assert_eq!(vortex_array_non_null.len(), 4);
923    }
924
925    #[test]
926    fn test_date32_array_conversion() {
927        let arrow_array = Date32Array::from(vec![Some(18000), None, Some(18002), Some(18003)]);
928        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
929
930        let arrow_array_non_null = Date32Array::from(vec![18000_i32, 18001, 18002, 18003]);
931        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
932
933        assert_eq!(vortex_array.len(), 4);
934        assert_eq!(vortex_array_non_null.len(), 4);
935    }
936
937    #[test]
938    fn test_date64_array_conversion() {
939        let arrow_array = Date64Array::from(vec![
940            Some(1555200000000),
941            None,
942            Some(1555286400000),
943            Some(1555372800000),
944        ]);
945        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
946
947        let arrow_array_non_null = Date64Array::from(vec![
948            1555200000000_i64,
949            1555213600000,
950            1555286400000,
951            1555372800000,
952        ]);
953        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
954
955        assert_eq!(vortex_array.len(), 4);
956        assert_eq!(vortex_array_non_null.len(), 4);
957    }
958
959    // Test string/binary array conversions
960    #[test]
961    fn test_utf8_array_conversion() {
962        let arrow_array = StringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
963        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
964
965        let arrow_array_non_null = StringArray::from(vec!["hello", "world", "test", "vortex"]);
966        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
967
968        assert_eq!(vortex_array.len(), 4);
969        assert_eq!(vortex_array_non_null.len(), 4);
970
971        // Verify metadata - should be VarBinArray with Utf8 dtype
972        let varbin_array = vortex_array.as_::<VarBinVTable>();
973        assert_eq!(varbin_array.dtype(), &DType::Utf8(true.into()));
974
975        let varbin_array_non_null = vortex_array_non_null.as_::<VarBinVTable>();
976        assert_eq!(varbin_array_non_null.dtype(), &DType::Utf8(false.into()));
977    }
978
979    #[test]
980    fn test_large_utf8_array_conversion() {
981        let arrow_array =
982            LargeStringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
983        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
984
985        let arrow_array_non_null = LargeStringArray::from(vec!["hello", "world", "test", "vortex"]);
986        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
987
988        assert_eq!(vortex_array.len(), 4);
989        assert_eq!(vortex_array_non_null.len(), 4);
990    }
991
992    #[test]
993    fn test_binary_array_conversion() {
994        let arrow_array = BinaryArray::from(vec![
995            Some("hello".as_bytes()),
996            None,
997            Some("world".as_bytes()),
998            Some("test".as_bytes()),
999        ]);
1000        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1001
1002        let arrow_array_non_null = BinaryArray::from(vec![
1003            "hello".as_bytes(),
1004            "world".as_bytes(),
1005            "test".as_bytes(),
1006            "vortex".as_bytes(),
1007        ]);
1008        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1009
1010        assert_eq!(vortex_array.len(), 4);
1011        assert_eq!(vortex_array_non_null.len(), 4);
1012    }
1013
1014    #[test]
1015    fn test_large_binary_array_conversion() {
1016        let arrow_array = LargeBinaryArray::from(vec![
1017            Some("hello".as_bytes()),
1018            None,
1019            Some("world".as_bytes()),
1020            Some("test".as_bytes()),
1021        ]);
1022        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1023
1024        let arrow_array_non_null = LargeBinaryArray::from(vec![
1025            "hello".as_bytes(),
1026            "world".as_bytes(),
1027            "test".as_bytes(),
1028            "vortex".as_bytes(),
1029        ]);
1030        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1031
1032        assert_eq!(vortex_array.len(), 4);
1033        assert_eq!(vortex_array_non_null.len(), 4);
1034    }
1035
1036    #[test]
1037    fn test_utf8_view_array_conversion() {
1038        let mut builder = StringViewBuilder::new();
1039        builder.append_value("hello");
1040        builder.append_null();
1041        builder.append_value("world");
1042        builder.append_value("test");
1043        let arrow_array = builder.finish();
1044        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1045
1046        let mut builder_non_null = StringViewBuilder::new();
1047        builder_non_null.append_value("hello");
1048        builder_non_null.append_value("world");
1049        builder_non_null.append_value("test");
1050        builder_non_null.append_value("vortex");
1051        let arrow_array_non_null = builder_non_null.finish();
1052        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1053
1054        assert_eq!(vortex_array.len(), 4);
1055        assert_eq!(vortex_array_non_null.len(), 4);
1056
1057        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1058        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1059        assert_eq!(
1060            varbin_view_array.buffers().len(),
1061            arrow_array.data_buffers().len()
1062        );
1063        assert_eq!(varbin_view_array.dtype(), &DType::Utf8(true.into()));
1064
1065        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1066        assert_eq!(
1067            varbin_view_array_non_null.buffers().len(),
1068            arrow_array_non_null.data_buffers().len()
1069        );
1070        assert_eq!(
1071            varbin_view_array_non_null.dtype(),
1072            &DType::Utf8(false.into())
1073        );
1074    }
1075
1076    #[test]
1077    fn test_binary_view_array_conversion() {
1078        let mut builder = BinaryViewBuilder::new();
1079        builder.append_value(b"hello");
1080        builder.append_null();
1081        builder.append_value(b"world");
1082        builder.append_value(b"test");
1083        let arrow_array = builder.finish();
1084        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1085
1086        let mut builder_non_null = BinaryViewBuilder::new();
1087        builder_non_null.append_value(b"hello");
1088        builder_non_null.append_value(b"world");
1089        builder_non_null.append_value(b"test");
1090        builder_non_null.append_value(b"vortex");
1091        let arrow_array_non_null = builder_non_null.finish();
1092        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1093
1094        assert_eq!(vortex_array.len(), 4);
1095        assert_eq!(vortex_array_non_null.len(), 4);
1096
1097        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1098        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1099        assert_eq!(
1100            varbin_view_array.buffers().len(),
1101            arrow_array.data_buffers().len()
1102        );
1103        assert_eq!(varbin_view_array.dtype(), &DType::Binary(true.into()));
1104
1105        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1106        assert_eq!(
1107            varbin_view_array_non_null.buffers().len(),
1108            arrow_array_non_null.data_buffers().len()
1109        );
1110        assert_eq!(
1111            varbin_view_array_non_null.dtype(),
1112            &DType::Binary(false.into())
1113        );
1114    }
1115
1116    // Test boolean array conversions
1117    #[test]
1118    fn test_boolean_array_conversion() {
1119        let arrow_array = BooleanArray::from(vec![Some(true), None, Some(false), Some(true)]);
1120        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1121
1122        let arrow_array_non_null = BooleanArray::from(vec![true, false, true, false]);
1123        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1124
1125        assert_eq!(vortex_array.len(), 4);
1126        assert_eq!(vortex_array_non_null.len(), 4);
1127    }
1128
1129    // Test struct array conversions
1130    #[test]
1131    fn test_struct_array_conversion() {
1132        let fields = vec![
1133            Field::new("field1", DataType::Int32, true),
1134            Field::new("field2", DataType::Utf8, false),
1135        ];
1136        let schema = Fields::from(fields);
1137
1138        let field1_data = Int32Array::from(vec![Some(1), None, Some(3)]);
1139        let field2_data = StringArray::from(vec!["a", "b", "c"]);
1140
1141        let arrow_array = StructArray::new(
1142            schema.clone(),
1143            vec![Arc::new(field1_data), Arc::new(field2_data)],
1144            None,
1145        );
1146
1147        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1148        assert_eq!(vortex_array.len(), 3);
1149
1150        // Verify metadata - should be StructArray with correct field names
1151        let struct_vortex_array = vortex_array.as_::<StructVTable>();
1152        assert_eq!(struct_vortex_array.names().len(), 2);
1153        assert_eq!(struct_vortex_array.names()[0], "field1".into());
1154        assert_eq!(struct_vortex_array.names()[1], "field2".into());
1155
1156        // Test nullable struct
1157        let nullable_array = StructArray::new(
1158            schema,
1159            vec![
1160                Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
1161                Arc::new(StringArray::from(vec!["a", "b", "c"])),
1162            ],
1163            Some(arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![
1164                true, false, true,
1165            ]))),
1166        );
1167
1168        let vortex_nullable_array = ArrayRef::from_arrow(&nullable_array, true);
1169        assert_eq!(vortex_nullable_array.len(), 3);
1170
1171        // Verify metadata for nullable struct
1172        let struct_vortex_nullable_array = vortex_nullable_array.as_::<StructVTable>();
1173        assert_eq!(struct_vortex_nullable_array.names().len(), 2);
1174        assert_eq!(struct_vortex_nullable_array.names()[0], "field1".into());
1175        assert_eq!(struct_vortex_nullable_array.names()[1], "field2".into());
1176    }
1177
1178    // Test list array conversions
1179    #[test]
1180    fn test_list_array_conversion() {
1181        let mut builder = ListBuilder::new(Int32Builder::new());
1182        builder.append_value([Some(1), None, Some(3)]);
1183        builder.append_null();
1184        builder.append_value([Some(4), Some(5)]);
1185        let arrow_array = builder.finish();
1186
1187        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1188        assert_eq!(vortex_array.len(), 3);
1189
1190        // Verify metadata - should be ListArray with correct offsets
1191        let list_vortex_array = vortex_array.as_::<ListVTable>();
1192        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1193        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1194        assert_eq!(offsets_array.ptype(), PType::I32);
1195
1196        // Test non-nullable list
1197        let mut builder_non_null = ListBuilder::new(Int32Builder::new());
1198        builder_non_null.append_value([Some(1), None, Some(3)]);
1199        builder_non_null.append_value([Some(4), Some(5)]);
1200        let arrow_array_non_null = builder_non_null.finish();
1201
1202        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1203        assert_eq!(vortex_array_non_null.len(), 2);
1204
1205        // Verify metadata for non-nullable list
1206        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1207        let offsets_array_non_null = list_vortex_array_non_null
1208            .offsets()
1209            .as_::<PrimitiveVTable>();
1210        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1211        assert_eq!(offsets_array_non_null.ptype(), PType::I32);
1212    }
1213
1214    #[test]
1215    fn test_large_list_array_conversion() {
1216        let mut builder = LargeListBuilder::new(Int32Builder::new());
1217        builder.append_value([Some(1), None, Some(3)]);
1218        builder.append_null();
1219        builder.append_value([Some(4), Some(5)]);
1220        let arrow_array = builder.finish();
1221
1222        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1223        assert_eq!(vortex_array.len(), 3);
1224
1225        // Verify metadata - should be ListArray with correct offsets (I64 for large lists)
1226        let list_vortex_array = vortex_array.as_::<ListVTable>();
1227        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1228        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1229        assert_eq!(offsets_array.ptype(), PType::I64); // Large lists use I64 offsets
1230
1231        // Test non-nullable large list
1232        let mut builder_non_null = LargeListBuilder::new(Int32Builder::new());
1233        builder_non_null.append_value([Some(1), None, Some(3)]);
1234        builder_non_null.append_value([Some(4), Some(5)]);
1235        let arrow_array_non_null = builder_non_null.finish();
1236
1237        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1238        assert_eq!(vortex_array_non_null.len(), 2);
1239
1240        // Verify metadata for non-nullable large list
1241        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1242        let offsets_array_non_null = list_vortex_array_non_null
1243            .offsets()
1244            .as_::<PrimitiveVTable>();
1245        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1246        assert_eq!(offsets_array_non_null.ptype(), PType::I64); // Large lists use I64 offsets
1247    }
1248
1249    // Test null array conversions
1250    #[test]
1251    fn test_null_array_conversion() {
1252        let arrow_array = NullArray::new(5);
1253        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1254        assert_eq!(vortex_array.len(), 5);
1255    }
1256
1257    // Test buffer conversions
1258    #[test]
1259    fn test_arrow_buffer_conversion() {
1260        let data = vec![1u8, 2, 3, 4, 5];
1261        let arrow_buffer = ArrowBuffer::from_vec(data);
1262        let vortex_array = arrow_buffer.into_array();
1263        assert_eq!(vortex_array.len(), 5);
1264    }
1265
1266    #[test]
1267    fn test_boolean_buffer_conversion() {
1268        let data = vec![true, false, true, false, true];
1269        let boolean_buffer = BooleanBuffer::from(data);
1270        let vortex_array = boolean_buffer.into_array();
1271        assert_eq!(vortex_array.len(), 5);
1272    }
1273
1274    #[test]
1275    fn test_scalar_buffer_conversion() {
1276        let data = vec![1i32, 2, 3, 4, 5];
1277        let scalar_buffer = ScalarBuffer::from(data);
1278        let vortex_array = scalar_buffer.into_array();
1279        assert_eq!(vortex_array.len(), 5);
1280    }
1281
1282    #[test]
1283    fn test_offset_buffer_conversion() {
1284        let data = vec![0i32, 2, 5, 8, 10];
1285        let offset_buffer = OffsetBuffer::new(ScalarBuffer::from(data));
1286        let vortex_array = offset_buffer.into_array();
1287        assert_eq!(vortex_array.len(), 5);
1288    }
1289
1290    // Test RecordBatch conversions
1291    #[test]
1292    fn test_record_batch_conversion() {
1293        let schema = Arc::new(Schema::new(vec![
1294            Field::new("field1", DataType::Int32, false),
1295            Field::new("field2", DataType::Utf8, false),
1296        ]));
1297
1298        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1299        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1300
1301        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1302
1303        let vortex_array = ArrayRef::from_arrow(record_batch, false);
1304        assert_eq!(vortex_array.len(), 4);
1305
1306        // Test with reference
1307        let schema = Arc::new(Schema::new(vec![
1308            Field::new("field1", DataType::Int32, false),
1309            Field::new("field2", DataType::Utf8, false),
1310        ]));
1311
1312        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1313        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1314
1315        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1316
1317        let vortex_array = ArrayRef::from_arrow(&record_batch, false);
1318        assert_eq!(vortex_array.len(), 4);
1319    }
1320
1321    // Test dynamic dispatch conversion
1322    #[test]
1323    fn test_dyn_array_conversion() {
1324        let int_array = Int32Array::from(vec![1, 2, 3, 4]);
1325        let dyn_array: &dyn ArrowArray = &int_array;
1326        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1327        assert_eq!(vortex_array.len(), 4);
1328
1329        let string_array = StringArray::from(vec!["a", "b", "c"]);
1330        let dyn_array: &dyn ArrowArray = &string_array;
1331        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1332        assert_eq!(vortex_array.len(), 3);
1333
1334        let bool_array = BooleanArray::from(vec![true, false, true]);
1335        let dyn_array: &dyn ArrowArray = &bool_array;
1336        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1337        assert_eq!(vortex_array.len(), 3);
1338    }
1339
1340    // Existing tests
1341    #[test]
1342    pub fn nullable_may_contain_non_nullable() {
1343        let null_struct_array_with_non_nullable_field = new_null_array(
1344            &DataType::Struct(Fields::from(vec![Field::new(
1345                "non_nullable_inner",
1346                DataType::Int32,
1347                false,
1348            )])),
1349            1,
1350        );
1351        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1352    }
1353
1354    #[test]
1355    pub fn nullable_may_contain_deeply_nested_non_nullable() {
1356        let null_struct_array_with_non_nullable_field = new_null_array(
1357            &DataType::Struct(Fields::from(vec![Field::new(
1358                "non_nullable_inner",
1359                DataType::Struct(Fields::from(vec![Field::new(
1360                    "non_nullable_deeper_inner",
1361                    DataType::Int32,
1362                    false,
1363                )])),
1364                false,
1365            )])),
1366            1,
1367        );
1368        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1369    }
1370
1371    #[test]
1372    #[should_panic]
1373    pub fn cannot_handle_nullable_struct_containing_non_nullable_dictionary() {
1374        let null_struct_array_with_non_nullable_field = new_null_array(
1375            &DataType::Struct(Fields::from(vec![Field::new(
1376                "non_nullable_deeper_inner",
1377                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1378                false,
1379            )])),
1380            1,
1381        );
1382
1383        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1384    }
1385}
vortex_array/arrow/convert.rs

vortex_array/arrow/
convert.rs
