vortex_array/arrow/
convert.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::sync::Arc;
5
6use arrow_array::array::{
7    Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray, GenericByteArray,
8    NullArray as ArrowNullArray, OffsetSizeTrait, PrimitiveArray as ArrowPrimitiveArray,
9    StructArray as ArrowStructArray,
10};
11use arrow_array::cast::{AsArray, as_null_array};
12use arrow_array::types::{
13    ByteArrayType, ByteViewType, Date32Type, Date64Type, Decimal128Type, Decimal256Type,
14    Float16Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type,
15    Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType,
16    TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
17    TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
18};
19use arrow_array::{GenericByteViewArray, GenericListArray, RecordBatch, make_array};
20use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
21use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
22use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
23use itertools::Itertools;
24use vortex_buffer::{Alignment, Buffer, ByteBuffer};
25use vortex_dtype::datetime::TimeUnit;
26use vortex_dtype::{DType, DecimalDType, NativePType, PType};
27use vortex_error::{VortexExpect as _, vortex_panic};
28use vortex_scalar::i256;
29
30use crate::arrays::{
31    BoolArray, DecimalArray, ListArray, NullArray, PrimitiveArray, StructArray, TemporalArray,
32    VarBinArray, VarBinViewArray,
33};
34use crate::arrow::FromArrowArray;
35use crate::validity::Validity;
36use crate::{ArrayRef, IntoArray};
37
38impl IntoArray for ArrowBuffer {
39    fn into_array(self) -> ArrayRef {
40        PrimitiveArray::from_byte_buffer(
41            ByteBuffer::from_arrow_buffer(self, Alignment::of::<u8>()),
42            PType::U8,
43            Validity::NonNullable,
44        )
45        .into_array()
46    }
47}
48
49impl IntoArray for BooleanBuffer {
50    fn into_array(self) -> ArrayRef {
51        BoolArray::new(self, Validity::NonNullable).into_array()
52    }
53}
54
55impl<T> IntoArray for ScalarBuffer<T>
56where
57    T: ArrowNativeType + NativePType,
58{
59    fn into_array(self) -> ArrayRef {
60        PrimitiveArray::new(
61            Buffer::<T>::from_arrow_scalar_buffer(self),
62            Validity::NonNullable,
63        )
64        .into_array()
65    }
66}
67
68impl<O> IntoArray for OffsetBuffer<O>
69where
70    O: NativePType + OffsetSizeTrait,
71{
72    fn into_array(self) -> ArrayRef {
73        let primitive = PrimitiveArray::new(
74            Buffer::from_arrow_scalar_buffer(self.into_inner()),
75            Validity::NonNullable,
76        );
77
78        primitive.into_array()
79    }
80}
81
82macro_rules! impl_from_arrow_primitive {
83    ($ty:path) => {
84        impl FromArrowArray<&ArrowPrimitiveArray<$ty>> for ArrayRef {
85            fn from_arrow(value: &ArrowPrimitiveArray<$ty>, nullable: bool) -> Self {
86                let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone());
87                let validity = nulls(value.nulls(), nullable);
88                PrimitiveArray::new(buffer, validity).into_array()
89            }
90        }
91    };
92}
93
94impl_from_arrow_primitive!(Int8Type);
95impl_from_arrow_primitive!(Int16Type);
96impl_from_arrow_primitive!(Int32Type);
97impl_from_arrow_primitive!(Int64Type);
98impl_from_arrow_primitive!(UInt8Type);
99impl_from_arrow_primitive!(UInt16Type);
100impl_from_arrow_primitive!(UInt32Type);
101impl_from_arrow_primitive!(UInt64Type);
102impl_from_arrow_primitive!(Float16Type);
103impl_from_arrow_primitive!(Float32Type);
104impl_from_arrow_primitive!(Float64Type);
105
106impl FromArrowArray<&ArrowPrimitiveArray<Decimal128Type>> for ArrayRef {
107    fn from_arrow(array: &ArrowPrimitiveArray<Decimal128Type>, nullable: bool) -> Self {
108        let decimal_type = DecimalDType::new(array.precision(), array.scale());
109        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
110        let validity = nulls(array.nulls(), nullable);
111        DecimalArray::new(buffer, decimal_type, validity).into_array()
112    }
113}
114
115impl FromArrowArray<&ArrowPrimitiveArray<Decimal256Type>> for ArrayRef {
116    fn from_arrow(array: &ArrowPrimitiveArray<Decimal256Type>, nullable: bool) -> Self {
117        let decimal_type = DecimalDType::new(array.precision(), array.scale());
118        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
119        // SAFETY: Our i256 implementation has the same bit-pattern representation of the
120        //  arrow_buffer::i256 type. It is safe to treat values held inside the buffer as values
121        //  of either type.
122        let buffer =
123            unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(buffer) };
124        let validity = nulls(array.nulls(), nullable);
125        DecimalArray::new(buffer, decimal_type, validity).into_array()
126    }
127}
128
129macro_rules! impl_from_arrow_temporal {
130    ($ty:path) => {
131        impl FromArrowArray<&ArrowPrimitiveArray<$ty>> for ArrayRef {
132            fn from_arrow(value: &ArrowPrimitiveArray<$ty>, nullable: bool) -> Self {
133                temporal_array(value, nullable)
134            }
135        }
136    };
137}
138
139// timestamp
140impl_from_arrow_temporal!(TimestampSecondType);
141impl_from_arrow_temporal!(TimestampMillisecondType);
142impl_from_arrow_temporal!(TimestampMicrosecondType);
143impl_from_arrow_temporal!(TimestampNanosecondType);
144
145// time
146impl_from_arrow_temporal!(Time32SecondType);
147impl_from_arrow_temporal!(Time32MillisecondType);
148impl_from_arrow_temporal!(Time64MicrosecondType);
149impl_from_arrow_temporal!(Time64NanosecondType);
150
151// date
152impl_from_arrow_temporal!(Date32Type);
153impl_from_arrow_temporal!(Date64Type);
154
155fn temporal_array<T: ArrowPrimitiveType>(value: &ArrowPrimitiveArray<T>, nullable: bool) -> ArrayRef
156where
157    T::Native: NativePType,
158{
159    let arr = PrimitiveArray::new(
160        Buffer::from_arrow_scalar_buffer(value.values().clone()),
161        nulls(value.nulls(), nullable),
162    )
163    .into_array();
164
165    match T::DATA_TYPE {
166        DataType::Timestamp(time_unit, tz) => {
167            let tz = tz.map(|s| s.to_string());
168            TemporalArray::new_timestamp(arr, time_unit.into(), tz).into()
169        }
170        DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
171        DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
172        DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::D).into(),
173        DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Ms).into(),
174        DataType::Duration(_) => unimplemented!(),
175        DataType::Interval(_) => unimplemented!(),
176        _ => vortex_panic!("Invalid temporal type: {}", T::DATA_TYPE),
177    }
178}
179
180impl<T: ByteArrayType> FromArrowArray<&GenericByteArray<T>> for ArrayRef
181where
182    <T as ByteArrayType>::Offset: NativePType,
183{
184    fn from_arrow(value: &GenericByteArray<T>, nullable: bool) -> Self {
185        let dtype = match T::DATA_TYPE {
186            DataType::Binary | DataType::LargeBinary => DType::Binary(nullable.into()),
187            DataType::Utf8 | DataType::LargeUtf8 => DType::Utf8(nullable.into()),
188            _ => vortex_panic!("Invalid data type for ByteArray: {}", T::DATA_TYPE),
189        };
190        VarBinArray::try_new(
191            value.offsets().clone().into_array(),
192            ByteBuffer::from_arrow_buffer(value.values().clone(), Alignment::of::<u8>()),
193            dtype,
194            nulls(value.nulls(), nullable),
195        )
196        .vortex_expect("Failed to convert Arrow GenericByteArray to Vortex VarBinArray")
197        .into_array()
198    }
199}
200
201impl<T: ByteViewType> FromArrowArray<&GenericByteViewArray<T>> for ArrayRef {
202    fn from_arrow(value: &GenericByteViewArray<T>, nullable: bool) -> Self {
203        let dtype = match T::DATA_TYPE {
204            DataType::BinaryView => DType::Binary(nullable.into()),
205            DataType::Utf8View => DType::Utf8(nullable.into()),
206            _ => vortex_panic!("Invalid data type for ByteViewArray: {}", T::DATA_TYPE),
207        };
208
209        let views_buffer = Buffer::from_byte_buffer(
210            Buffer::from_arrow_scalar_buffer(value.views().clone()).into_byte_buffer(),
211        );
212
213        VarBinViewArray::try_new(
214            views_buffer,
215            Arc::from(
216                value
217                    .data_buffers()
218                    .iter()
219                    .map(|b| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::<u8>()))
220                    .collect::<Vec<_>>(),
221            ),
222            dtype,
223            nulls(value.nulls(), nullable),
224        )
225        .vortex_expect("Failed to convert Arrow GenericByteViewArray to Vortex VarBinViewArray")
226        .into_array()
227    }
228}
229
230impl FromArrowArray<&ArrowBooleanArray> for ArrayRef {
231    fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self {
232        BoolArray::new(value.values().clone(), nulls(value.nulls(), nullable)).into_array()
233    }
234}
235
236/// Strip out the nulls from this array and return a new array without nulls.
237fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData {
238    if data.null_count() == 0 {
239        // No nulls to remove, return the array as is
240        return data;
241    }
242
243    let children = match data.data_type() {
244        DataType::Struct(fields) => Some(
245            fields
246                .iter()
247                .zip(data.child_data().iter())
248                .map(|(field, child_data)| {
249                    if field.is_nullable() {
250                        child_data.clone()
251                    } else {
252                        remove_nulls(child_data.clone())
253                    }
254                })
255                .collect_vec(),
256        ),
257        DataType::List(f)
258        | DataType::LargeList(f)
259        | DataType::ListView(f)
260        | DataType::LargeListView(f)
261        | DataType::FixedSizeList(f, _)
262            if !f.is_nullable() =>
263        {
264            // All list types only have one child
265            assert_eq!(
266                data.child_data().len(),
267                1,
268                "List types should have one child"
269            );
270            Some(vec![remove_nulls(data.child_data()[0].clone())])
271        }
272        _ => None,
273    };
274
275    let mut builder = data.into_builder().nulls(None);
276    if let Some(children) = children {
277        builder = builder.child_data(children);
278    }
279    builder
280        .build()
281        .vortex_expect("reconstructing array without nulls")
282}
283
284impl FromArrowArray<&ArrowStructArray> for ArrayRef {
285    fn from_arrow(value: &ArrowStructArray, nullable: bool) -> Self {
286        StructArray::try_new(
287            value.column_names().iter().copied().collect(),
288            value
289                .columns()
290                .iter()
291                .zip(value.fields())
292                .map(|(c, field)| {
293                    // Arrow pushes down nulls, even into non-nullable fields. So we strip them
294                    // out here because Vortex is a little more strict.
295                    if c.null_count() > 0 && !field.is_nullable() {
296                        let stripped = make_array(remove_nulls(c.into_data()));
297                        Self::from_arrow(stripped.as_ref(), false)
298                    } else {
299                        Self::from_arrow(c.as_ref(), field.is_nullable())
300                    }
301                })
302                .collect(),
303            value.len(),
304            nulls(value.nulls(), nullable),
305        )
306        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
307        .into_array()
308    }
309}
310
311impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListArray<O>> for ArrayRef {
312    fn from_arrow(value: &GenericListArray<O>, nullable: bool) -> Self {
313        // Extract the validity of the underlying element array
314        let elem_nullable = match value.data_type() {
315            DataType::List(field) => field.is_nullable(),
316            DataType::LargeList(field) => field.is_nullable(),
317            dt => vortex_panic!("Invalid data type for ListArray: {dt}"),
318        };
319        ListArray::try_new(
320            Self::from_arrow(value.values().as_ref(), elem_nullable),
321            // offsets are always non-nullable
322            value.offsets().clone().into_array(),
323            nulls(value.nulls(), nullable),
324        )
325        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
326        .into_array()
327    }
328}
329
330impl FromArrowArray<&ArrowNullArray> for ArrayRef {
331    fn from_arrow(value: &ArrowNullArray, nullable: bool) -> Self {
332        assert!(nullable);
333        NullArray::new(value.len()).into_array()
334    }
335}
336
337fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity {
338    if nullable {
339        nulls
340            .map(|nulls| {
341                if nulls.null_count() == nulls.len() {
342                    Validity::AllInvalid
343                } else {
344                    Validity::from(nulls.inner().clone())
345                }
346            })
347            .unwrap_or_else(|| Validity::AllValid)
348    } else {
349        assert!(nulls.map(|x| x.null_count() == 0).unwrap_or(true));
350        Validity::NonNullable
351    }
352}
353
354impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
355    fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> Self {
356        match array.data_type() {
357            DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable),
358            DataType::UInt8 => Self::from_arrow(array.as_primitive::<UInt8Type>(), nullable),
359            DataType::UInt16 => Self::from_arrow(array.as_primitive::<UInt16Type>(), nullable),
360            DataType::UInt32 => Self::from_arrow(array.as_primitive::<UInt32Type>(), nullable),
361            DataType::UInt64 => Self::from_arrow(array.as_primitive::<UInt64Type>(), nullable),
362            DataType::Int8 => Self::from_arrow(array.as_primitive::<Int8Type>(), nullable),
363            DataType::Int16 => Self::from_arrow(array.as_primitive::<Int16Type>(), nullable),
364            DataType::Int32 => Self::from_arrow(array.as_primitive::<Int32Type>(), nullable),
365            DataType::Int64 => Self::from_arrow(array.as_primitive::<Int64Type>(), nullable),
366            DataType::Float16 => Self::from_arrow(array.as_primitive::<Float16Type>(), nullable),
367            DataType::Float32 => Self::from_arrow(array.as_primitive::<Float32Type>(), nullable),
368            DataType::Float64 => Self::from_arrow(array.as_primitive::<Float64Type>(), nullable),
369            DataType::Utf8 => Self::from_arrow(array.as_string::<i32>(), nullable),
370            DataType::LargeUtf8 => Self::from_arrow(array.as_string::<i64>(), nullable),
371            DataType::Binary => Self::from_arrow(array.as_binary::<i32>(), nullable),
372            DataType::LargeBinary => Self::from_arrow(array.as_binary::<i64>(), nullable),
373            DataType::BinaryView => Self::from_arrow(array.as_binary_view(), nullable),
374            DataType::Utf8View => Self::from_arrow(array.as_string_view(), nullable),
375            DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),
376            DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),
377            DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),
378            DataType::Null => Self::from_arrow(as_null_array(array), nullable),
379            DataType::Timestamp(u, _) => match u {
380                ArrowTimeUnit::Second => {
381                    Self::from_arrow(array.as_primitive::<TimestampSecondType>(), nullable)
382                }
383                ArrowTimeUnit::Millisecond => {
384                    Self::from_arrow(array.as_primitive::<TimestampMillisecondType>(), nullable)
385                }
386                ArrowTimeUnit::Microsecond => {
387                    Self::from_arrow(array.as_primitive::<TimestampMicrosecondType>(), nullable)
388                }
389                ArrowTimeUnit::Nanosecond => {
390                    Self::from_arrow(array.as_primitive::<TimestampNanosecondType>(), nullable)
391                }
392            },
393            DataType::Date32 => Self::from_arrow(array.as_primitive::<Date32Type>(), nullable),
394            DataType::Date64 => Self::from_arrow(array.as_primitive::<Date64Type>(), nullable),
395            DataType::Time32(u) => match u {
396                ArrowTimeUnit::Second => {
397                    Self::from_arrow(array.as_primitive::<Time32SecondType>(), nullable)
398                }
399                ArrowTimeUnit::Millisecond => {
400                    Self::from_arrow(array.as_primitive::<Time32MillisecondType>(), nullable)
401                }
402                _ => unreachable!(),
403            },
404            DataType::Time64(u) => match u {
405                ArrowTimeUnit::Microsecond => {
406                    Self::from_arrow(array.as_primitive::<Time64MicrosecondType>(), nullable)
407                }
408                ArrowTimeUnit::Nanosecond => {
409                    Self::from_arrow(array.as_primitive::<Time64NanosecondType>(), nullable)
410                }
411                _ => unreachable!(),
412            },
413            DataType::Decimal128(..) => {
414                Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)
415            }
416            DataType::Decimal256(..) => {
417                Self::from_arrow(array.as_primitive::<Decimal256Type>(), nullable)
418            }
419            _ => vortex_panic!(
420                "Array encoding not implemented for Arrow data type {}",
421                array.data_type().clone()
422            ),
423        }
424    }
425}
426
427impl FromArrowArray<RecordBatch> for ArrayRef {
428    fn from_arrow(array: RecordBatch, nullable: bool) -> Self {
429        ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable)
430    }
431}
432
433impl FromArrowArray<&RecordBatch> for ArrayRef {
434    fn from_arrow(array: &RecordBatch, nullable: bool) -> Self {
435        Self::from_arrow(array.clone(), nullable)
436    }
437}
438
439#[cfg(test)]
440mod tests {
441    use std::sync::Arc;
442
443    use arrow_array::builder::{
444        BinaryViewBuilder, Decimal128Builder, Decimal256Builder, Int32Builder, LargeListBuilder,
445        ListBuilder, StringViewBuilder,
446    };
447    use arrow_array::types::{ArrowPrimitiveType, Float16Type};
448    use arrow_array::{
449        Array as ArrowArray, BinaryArray, BooleanArray, Date32Array, Date64Array, Float32Array,
450        Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, LargeBinaryArray,
451        LargeStringArray, NullArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray,
452        Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
453        TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
454        TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, new_null_array,
455    };
456    use arrow_buffer::{BooleanBuffer, Buffer as ArrowBuffer, OffsetBuffer, ScalarBuffer};
457    use arrow_schema::{DataType, Field, Fields, Schema};
458    use vortex_dtype::datetime::TimeUnit;
459    use vortex_dtype::{DType, PType};
460
461    use crate::arrays::{
462        DecimalVTable, ListVTable, PrimitiveVTable, StructVTable, TemporalArray, VarBinVTable,
463        VarBinViewVTable,
464    };
465    use crate::arrow::FromArrowArray as _;
466    use crate::{ArrayRef, IntoArray};
467
468    // Test primitive array conversions
469    #[test]
470    fn test_int8_array_conversion() {
471        let arrow_array = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]);
472        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
473
474        let arrow_array_non_null = Int8Array::from(vec![1, 2, 3, 4]);
475        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
476
477        assert_eq!(vortex_array.len(), 4);
478        assert_eq!(vortex_array_non_null.len(), 4);
479
480        // Verify metadata - should be PrimitiveArray with I8 ptype
481        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
482        assert_eq!(primitive_array.ptype(), PType::I8);
483
484        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
485        assert_eq!(primitive_array_non_null.ptype(), PType::I8);
486    }
487
488    #[test]
489    fn test_int16_array_conversion() {
490        let arrow_array = Int16Array::from(vec![Some(100), None, Some(300), Some(400)]);
491        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
492
493        let arrow_array_non_null = Int16Array::from(vec![100, 200, 300, 400]);
494        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
495
496        assert_eq!(vortex_array.len(), 4);
497        assert_eq!(vortex_array_non_null.len(), 4);
498
499        // Verify metadata - should be PrimitiveArray with I16 ptype
500        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
501        assert_eq!(primitive_array.ptype(), PType::I16);
502
503        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
504        assert_eq!(primitive_array_non_null.ptype(), PType::I16);
505    }
506
507    #[test]
508    fn test_int32_array_conversion() {
509        let arrow_array = Int32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
510        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
511
512        let arrow_array_non_null = Int32Array::from(vec![1000, 2000, 3000, 4000]);
513        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
514
515        assert_eq!(vortex_array.len(), 4);
516        assert_eq!(vortex_array_non_null.len(), 4);
517
518        // Verify metadata - should be PrimitiveArray with I32 ptype
519        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
520        assert_eq!(primitive_array.ptype(), PType::I32);
521
522        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
523        assert_eq!(primitive_array_non_null.ptype(), PType::I32);
524    }
525
526    #[test]
527    fn test_int64_array_conversion() {
528        let arrow_array = Int64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
529        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
530
531        let arrow_array_non_null = Int64Array::from(vec![10000_i64, 20000, 30000, 40000]);
532        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
533
534        assert_eq!(vortex_array.len(), 4);
535        assert_eq!(vortex_array_non_null.len(), 4);
536
537        // Verify metadata - should be PrimitiveArray with I64 ptype
538        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
539        assert_eq!(primitive_array.ptype(), PType::I64);
540
541        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
542        assert_eq!(primitive_array_non_null.ptype(), PType::I64);
543    }
544
545    #[test]
546    fn test_uint8_array_conversion() {
547        let arrow_array = UInt8Array::from(vec![Some(1), None, Some(3), Some(4)]);
548        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
549
550        let arrow_array_non_null = UInt8Array::from(vec![1_u8, 2, 3, 4]);
551        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
552
553        assert_eq!(vortex_array.len(), 4);
554        assert_eq!(vortex_array_non_null.len(), 4);
555
556        // Verify metadata - should be PrimitiveArray with U8 ptype
557        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
558        assert_eq!(primitive_array.ptype(), PType::U8);
559
560        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
561        assert_eq!(primitive_array_non_null.ptype(), PType::U8);
562    }
563
564    #[test]
565    fn test_uint16_array_conversion() {
566        let arrow_array = UInt16Array::from(vec![Some(100), None, Some(300), Some(400)]);
567        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
568
569        let arrow_array_non_null = UInt16Array::from(vec![100_u16, 200, 300, 400]);
570        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
571
572        assert_eq!(vortex_array.len(), 4);
573        assert_eq!(vortex_array_non_null.len(), 4);
574
575        // Verify metadata - should be PrimitiveArray with U16 ptype
576        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
577        assert_eq!(primitive_array.ptype(), PType::U16);
578
579        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
580        assert_eq!(primitive_array_non_null.ptype(), PType::U16);
581    }
582
583    #[test]
584    fn test_uint32_array_conversion() {
585        let arrow_array = UInt32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
586        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
587
588        let arrow_array_non_null = UInt32Array::from(vec![1000_u32, 2000, 3000, 4000]);
589        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
590
591        assert_eq!(vortex_array.len(), 4);
592        assert_eq!(vortex_array_non_null.len(), 4);
593
594        // Verify metadata - should be PrimitiveArray with U32 ptype
595        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
596        assert_eq!(primitive_array.ptype(), PType::U32);
597
598        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
599        assert_eq!(primitive_array_non_null.ptype(), PType::U32);
600    }
601
602    #[test]
603    fn test_uint64_array_conversion() {
604        let arrow_array = UInt64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
605        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
606
607        let arrow_array_non_null = UInt64Array::from(vec![10000_u64, 20000, 30000, 40000]);
608        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
609
610        assert_eq!(vortex_array.len(), 4);
611        assert_eq!(vortex_array_non_null.len(), 4);
612
613        // Verify metadata - should be PrimitiveArray with U64 ptype
614        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
615        assert_eq!(primitive_array.ptype(), PType::U64);
616
617        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
618        assert_eq!(primitive_array_non_null.ptype(), PType::U64);
619    }
620
621    #[test]
622    fn test_float16_array_conversion() {
623        let values = vec![
624            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5)),
625            None,
626            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(3.5)),
627        ];
628        let arrow_array = arrow_array::PrimitiveArray::<Float16Type>::from(values);
629        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
630
631        let non_null_values = vec![
632            <Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5),
633            <Float16Type as ArrowPrimitiveType>::Native::from_f32(2.5),
634        ];
635        let arrow_array_non_null =
636            arrow_array::PrimitiveArray::<Float16Type>::from(non_null_values);
637        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
638
639        assert_eq!(vortex_array.len(), 3);
640        assert_eq!(vortex_array_non_null.len(), 2);
641
642        // Verify metadata - should be PrimitiveArray with F16 ptype
643        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
644        assert_eq!(primitive_array.ptype(), PType::F16);
645
646        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
647        assert_eq!(primitive_array_non_null.ptype(), PType::F16);
648    }
649
650    #[test]
651    fn test_float32_array_conversion() {
652        let arrow_array = Float32Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
653        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
654
655        let arrow_array_non_null = Float32Array::from(vec![1.5_f32, 2.5, 3.5, 4.5]);
656        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
657
658        assert_eq!(vortex_array.len(), 4);
659        assert_eq!(vortex_array_non_null.len(), 4);
660
661        // Verify metadata - should be PrimitiveArray with F32 ptype
662        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
663        assert_eq!(primitive_array.ptype(), PType::F32);
664
665        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
666        assert_eq!(primitive_array_non_null.ptype(), PType::F32);
667    }
668
669    #[test]
670    fn test_float64_array_conversion() {
671        let arrow_array = Float64Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
672        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
673
674        let arrow_array_non_null = Float64Array::from(vec![1.5_f64, 2.5, 3.5, 4.5]);
675        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
676
677        assert_eq!(vortex_array.len(), 4);
678        assert_eq!(vortex_array_non_null.len(), 4);
679
680        // Verify metadata - should be PrimitiveArray with F64 ptype
681        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
682        assert_eq!(primitive_array.ptype(), PType::F64);
683
684        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
685        assert_eq!(primitive_array_non_null.ptype(), PType::F64);
686    }
687
688    // Test decimal array conversions
689    #[test]
690    fn test_decimal128_array_conversion() {
691        let mut builder = Decimal128Builder::with_capacity(4);
692        builder.append_value(12345);
693        builder.append_null();
694        builder.append_value(67890);
695        builder.append_value(11111);
696        let decimal_array = builder.finish().with_precision_and_scale(10, 2).unwrap();
697
698        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
699        assert_eq!(vortex_array.len(), 4);
700
701        let mut builder_non_null = Decimal128Builder::with_capacity(3);
702        builder_non_null.append_value(12345);
703        builder_non_null.append_value(67890);
704        builder_non_null.append_value(11111);
705        let decimal_array_non_null = builder_non_null
706            .finish()
707            .with_precision_and_scale(10, 2)
708            .unwrap();
709
710        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
711        assert_eq!(vortex_array_non_null.len(), 3);
712
713        // Verify metadata - should be DecimalArray with correct precision and scale
714        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
715        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 10);
716        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 2);
717
718        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
719        assert_eq!(
720            decimal_vortex_array_non_null.decimal_dtype().precision(),
721            10
722        );
723        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 2);
724    }
725
726    #[test]
727    fn test_decimal256_array_conversion() {
728        let mut builder = Decimal256Builder::with_capacity(4);
729        builder.append_value(arrow_buffer::i256::from_i128(12345));
730        builder.append_null();
731        builder.append_value(arrow_buffer::i256::from_i128(67890));
732        builder.append_value(arrow_buffer::i256::from_i128(11111));
733        let decimal_array = builder.finish().with_precision_and_scale(38, 10).unwrap();
734
735        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
736        assert_eq!(vortex_array.len(), 4);
737
738        let mut builder_non_null = Decimal256Builder::with_capacity(3);
739        builder_non_null.append_value(arrow_buffer::i256::from_i128(12345));
740        builder_non_null.append_value(arrow_buffer::i256::from_i128(67890));
741        builder_non_null.append_value(arrow_buffer::i256::from_i128(11111));
742        let decimal_array_non_null = builder_non_null
743            .finish()
744            .with_precision_and_scale(38, 10)
745            .unwrap();
746
747        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
748        assert_eq!(vortex_array_non_null.len(), 3);
749
750        // Verify metadata - should be DecimalArray with correct precision and scale
751        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
752        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 38);
753        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 10);
754
755        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
756        assert_eq!(
757            decimal_vortex_array_non_null.decimal_dtype().precision(),
758            38
759        );
760        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 10);
761    }
762
763    // Test temporal array conversions
764    #[test]
765    fn test_timestamp_second_array_conversion() {
766        let arrow_array =
767            TimestampSecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
768        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
769
770        let arrow_array_non_null = TimestampSecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
771        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
772
773        assert_eq!(vortex_array.len(), 4);
774        assert_eq!(vortex_array_non_null.len(), 4);
775
776        // Verify metadata - should be TemporalArray with Second time unit
777        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
778        assert_eq!(temporal_array.temporal_metadata().time_unit(), TimeUnit::S);
779
780        let temporal_array_non_null =
781            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
782        assert_eq!(
783            temporal_array_non_null.temporal_metadata().time_unit(),
784            TimeUnit::S
785        );
786    }
787
788    #[test]
789    fn test_timestamp_millisecond_array_conversion() {
790        let arrow_array =
791            TimestampMillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
792        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
793
794        let arrow_array_non_null =
795            TimestampMillisecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
796        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
797
798        assert_eq!(vortex_array.len(), 4);
799        assert_eq!(vortex_array_non_null.len(), 4);
800    }
801
802    #[test]
803    fn test_timestamp_microsecond_array_conversion() {
804        let arrow_array =
805            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
806        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
807
808        let arrow_array_non_null =
809            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
810        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
811
812        assert_eq!(vortex_array.len(), 4);
813        assert_eq!(vortex_array_non_null.len(), 4);
814    }
815
816    #[test]
817    fn test_timestamp_nanosecond_array_conversion() {
818        let arrow_array =
819            TimestampNanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
820        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
821
822        let arrow_array_non_null = TimestampNanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
823        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
824
825        assert_eq!(vortex_array.len(), 4);
826        assert_eq!(vortex_array_non_null.len(), 4);
827    }
828
829    #[test]
830    fn test_time32_second_array_conversion() {
831        let arrow_array = Time32SecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
832        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
833
834        let arrow_array_non_null = Time32SecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
835        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
836
837        assert_eq!(vortex_array.len(), 4);
838        assert_eq!(vortex_array_non_null.len(), 4);
839
840        // Verify metadata - should be TemporalArray with Second time unit
841        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
842        assert_eq!(temporal_array.temporal_metadata().time_unit(), TimeUnit::S);
843
844        let temporal_array_non_null =
845            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
846        assert_eq!(
847            temporal_array_non_null.temporal_metadata().time_unit(),
848            TimeUnit::S
849        );
850    }
851
852    #[test]
853    fn test_time32_millisecond_array_conversion() {
854        let arrow_array =
855            Time32MillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
856        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
857
858        let arrow_array_non_null = Time32MillisecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
859        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
860
861        assert_eq!(vortex_array.len(), 4);
862        assert_eq!(vortex_array_non_null.len(), 4);
863    }
864
865    #[test]
866    fn test_time64_microsecond_array_conversion() {
867        let arrow_array =
868            Time64MicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
869        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
870
871        let arrow_array_non_null = Time64MicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
872        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
873
874        assert_eq!(vortex_array.len(), 4);
875        assert_eq!(vortex_array_non_null.len(), 4);
876    }
877
878    #[test]
879    fn test_time64_nanosecond_array_conversion() {
880        let arrow_array =
881            Time64NanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
882        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
883
884        let arrow_array_non_null = Time64NanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
885        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
886
887        assert_eq!(vortex_array.len(), 4);
888        assert_eq!(vortex_array_non_null.len(), 4);
889    }
890
891    #[test]
892    fn test_date32_array_conversion() {
893        let arrow_array = Date32Array::from(vec![Some(18000), None, Some(18002), Some(18003)]);
894        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
895
896        let arrow_array_non_null = Date32Array::from(vec![18000_i32, 18001, 18002, 18003]);
897        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
898
899        assert_eq!(vortex_array.len(), 4);
900        assert_eq!(vortex_array_non_null.len(), 4);
901    }
902
903    #[test]
904    fn test_date64_array_conversion() {
905        let arrow_array = Date64Array::from(vec![
906            Some(1555200000000),
907            None,
908            Some(1555286400000),
909            Some(1555372800000),
910        ]);
911        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
912
913        let arrow_array_non_null = Date64Array::from(vec![
914            1555200000000_i64,
915            1555213600000,
916            1555286400000,
917            1555372800000,
918        ]);
919        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
920
921        assert_eq!(vortex_array.len(), 4);
922        assert_eq!(vortex_array_non_null.len(), 4);
923    }
924
925    // Test string/binary array conversions
926    #[test]
927    fn test_utf8_array_conversion() {
928        let arrow_array = StringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
929        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
930
931        let arrow_array_non_null = StringArray::from(vec!["hello", "world", "test", "vortex"]);
932        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
933
934        assert_eq!(vortex_array.len(), 4);
935        assert_eq!(vortex_array_non_null.len(), 4);
936
937        // Verify metadata - should be VarBinArray with Utf8 dtype
938        let varbin_array = vortex_array.as_::<VarBinVTable>();
939        assert_eq!(varbin_array.dtype(), &DType::Utf8(true.into()));
940
941        let varbin_array_non_null = vortex_array_non_null.as_::<VarBinVTable>();
942        assert_eq!(varbin_array_non_null.dtype(), &DType::Utf8(false.into()));
943    }
944
945    #[test]
946    fn test_large_utf8_array_conversion() {
947        let arrow_array =
948            LargeStringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
949        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
950
951        let arrow_array_non_null = LargeStringArray::from(vec!["hello", "world", "test", "vortex"]);
952        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
953
954        assert_eq!(vortex_array.len(), 4);
955        assert_eq!(vortex_array_non_null.len(), 4);
956    }
957
958    #[test]
959    fn test_binary_array_conversion() {
960        let arrow_array = BinaryArray::from(vec![
961            Some("hello".as_bytes()),
962            None,
963            Some("world".as_bytes()),
964            Some("test".as_bytes()),
965        ]);
966        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
967
968        let arrow_array_non_null = BinaryArray::from(vec![
969            "hello".as_bytes(),
970            "world".as_bytes(),
971            "test".as_bytes(),
972            "vortex".as_bytes(),
973        ]);
974        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
975
976        assert_eq!(vortex_array.len(), 4);
977        assert_eq!(vortex_array_non_null.len(), 4);
978    }
979
980    #[test]
981    fn test_large_binary_array_conversion() {
982        let arrow_array = LargeBinaryArray::from(vec![
983            Some("hello".as_bytes()),
984            None,
985            Some("world".as_bytes()),
986            Some("test".as_bytes()),
987        ]);
988        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
989
990        let arrow_array_non_null = LargeBinaryArray::from(vec![
991            "hello".as_bytes(),
992            "world".as_bytes(),
993            "test".as_bytes(),
994            "vortex".as_bytes(),
995        ]);
996        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
997
998        assert_eq!(vortex_array.len(), 4);
999        assert_eq!(vortex_array_non_null.len(), 4);
1000    }
1001
1002    #[test]
1003    fn test_utf8_view_array_conversion() {
1004        let mut builder = StringViewBuilder::new();
1005        builder.append_value("hello");
1006        builder.append_null();
1007        builder.append_value("world");
1008        builder.append_value("test");
1009        let arrow_array = builder.finish();
1010        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1011
1012        let mut builder_non_null = StringViewBuilder::new();
1013        builder_non_null.append_value("hello");
1014        builder_non_null.append_value("world");
1015        builder_non_null.append_value("test");
1016        builder_non_null.append_value("vortex");
1017        let arrow_array_non_null = builder_non_null.finish();
1018        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1019
1020        assert_eq!(vortex_array.len(), 4);
1021        assert_eq!(vortex_array_non_null.len(), 4);
1022
1023        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1024        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1025        assert_eq!(
1026            varbin_view_array.buffers().len(),
1027            arrow_array.data_buffers().len()
1028        );
1029        assert_eq!(varbin_view_array.dtype(), &DType::Utf8(true.into()));
1030
1031        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1032        assert_eq!(
1033            varbin_view_array_non_null.buffers().len(),
1034            arrow_array_non_null.data_buffers().len()
1035        );
1036        assert_eq!(
1037            varbin_view_array_non_null.dtype(),
1038            &DType::Utf8(false.into())
1039        );
1040    }
1041
1042    #[test]
1043    fn test_binary_view_array_conversion() {
1044        let mut builder = BinaryViewBuilder::new();
1045        builder.append_value(b"hello");
1046        builder.append_null();
1047        builder.append_value(b"world");
1048        builder.append_value(b"test");
1049        let arrow_array = builder.finish();
1050        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1051
1052        let mut builder_non_null = BinaryViewBuilder::new();
1053        builder_non_null.append_value(b"hello");
1054        builder_non_null.append_value(b"world");
1055        builder_non_null.append_value(b"test");
1056        builder_non_null.append_value(b"vortex");
1057        let arrow_array_non_null = builder_non_null.finish();
1058        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1059
1060        assert_eq!(vortex_array.len(), 4);
1061        assert_eq!(vortex_array_non_null.len(), 4);
1062
1063        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1064        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1065        assert_eq!(
1066            varbin_view_array.buffers().len(),
1067            arrow_array.data_buffers().len()
1068        );
1069        assert_eq!(varbin_view_array.dtype(), &DType::Binary(true.into()));
1070
1071        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1072        assert_eq!(
1073            varbin_view_array_non_null.buffers().len(),
1074            arrow_array_non_null.data_buffers().len()
1075        );
1076        assert_eq!(
1077            varbin_view_array_non_null.dtype(),
1078            &DType::Binary(false.into())
1079        );
1080    }
1081
1082    // Test boolean array conversions
1083    #[test]
1084    fn test_boolean_array_conversion() {
1085        let arrow_array = BooleanArray::from(vec![Some(true), None, Some(false), Some(true)]);
1086        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1087
1088        let arrow_array_non_null = BooleanArray::from(vec![true, false, true, false]);
1089        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1090
1091        assert_eq!(vortex_array.len(), 4);
1092        assert_eq!(vortex_array_non_null.len(), 4);
1093    }
1094
1095    // Test struct array conversions
1096    #[test]
1097    fn test_struct_array_conversion() {
1098        let fields = vec![
1099            Field::new("field1", DataType::Int32, true),
1100            Field::new("field2", DataType::Utf8, false),
1101        ];
1102        let schema = Fields::from(fields);
1103
1104        let field1_data = Int32Array::from(vec![Some(1), None, Some(3)]);
1105        let field2_data = StringArray::from(vec!["a", "b", "c"]);
1106
1107        let arrow_array = StructArray::new(
1108            schema.clone(),
1109            vec![Arc::new(field1_data), Arc::new(field2_data)],
1110            None,
1111        );
1112
1113        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1114        assert_eq!(vortex_array.len(), 3);
1115
1116        // Verify metadata - should be StructArray with correct field names
1117        let struct_vortex_array = vortex_array.as_::<StructVTable>();
1118        assert_eq!(struct_vortex_array.names().len(), 2);
1119        assert_eq!(struct_vortex_array.names()[0], "field1".into());
1120        assert_eq!(struct_vortex_array.names()[1], "field2".into());
1121
1122        // Test nullable struct
1123        let nullable_array = StructArray::new(
1124            schema,
1125            vec![
1126                Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
1127                Arc::new(StringArray::from(vec!["a", "b", "c"])),
1128            ],
1129            Some(arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![
1130                true, false, true,
1131            ]))),
1132        );
1133
1134        let vortex_nullable_array = ArrayRef::from_arrow(&nullable_array, true);
1135        assert_eq!(vortex_nullable_array.len(), 3);
1136
1137        // Verify metadata for nullable struct
1138        let struct_vortex_nullable_array = vortex_nullable_array.as_::<StructVTable>();
1139        assert_eq!(struct_vortex_nullable_array.names().len(), 2);
1140        assert_eq!(struct_vortex_nullable_array.names()[0], "field1".into());
1141        assert_eq!(struct_vortex_nullable_array.names()[1], "field2".into());
1142    }
1143
1144    // Test list array conversions
1145    #[test]
1146    fn test_list_array_conversion() {
1147        let mut builder = ListBuilder::new(Int32Builder::new());
1148        builder.append_value([Some(1), None, Some(3)]);
1149        builder.append_null();
1150        builder.append_value([Some(4), Some(5)]);
1151        let arrow_array = builder.finish();
1152
1153        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1154        assert_eq!(vortex_array.len(), 3);
1155
1156        // Verify metadata - should be ListArray with correct offsets
1157        let list_vortex_array = vortex_array.as_::<ListVTable>();
1158        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1159        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1160        assert_eq!(offsets_array.ptype(), PType::I32);
1161
1162        // Test non-nullable list
1163        let mut builder_non_null = ListBuilder::new(Int32Builder::new());
1164        builder_non_null.append_value([Some(1), None, Some(3)]);
1165        builder_non_null.append_value([Some(4), Some(5)]);
1166        let arrow_array_non_null = builder_non_null.finish();
1167
1168        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1169        assert_eq!(vortex_array_non_null.len(), 2);
1170
1171        // Verify metadata for non-nullable list
1172        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1173        let offsets_array_non_null = list_vortex_array_non_null
1174            .offsets()
1175            .as_::<PrimitiveVTable>();
1176        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1177        assert_eq!(offsets_array_non_null.ptype(), PType::I32);
1178    }
1179
1180    #[test]
1181    fn test_large_list_array_conversion() {
1182        let mut builder = LargeListBuilder::new(Int32Builder::new());
1183        builder.append_value([Some(1), None, Some(3)]);
1184        builder.append_null();
1185        builder.append_value([Some(4), Some(5)]);
1186        let arrow_array = builder.finish();
1187
1188        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1189        assert_eq!(vortex_array.len(), 3);
1190
1191        // Verify metadata - should be ListArray with correct offsets (I64 for large lists)
1192        let list_vortex_array = vortex_array.as_::<ListVTable>();
1193        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1194        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1195        assert_eq!(offsets_array.ptype(), PType::I64); // Large lists use I64 offsets
1196
1197        // Test non-nullable large list
1198        let mut builder_non_null = LargeListBuilder::new(Int32Builder::new());
1199        builder_non_null.append_value([Some(1), None, Some(3)]);
1200        builder_non_null.append_value([Some(4), Some(5)]);
1201        let arrow_array_non_null = builder_non_null.finish();
1202
1203        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1204        assert_eq!(vortex_array_non_null.len(), 2);
1205
1206        // Verify metadata for non-nullable large list
1207        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1208        let offsets_array_non_null = list_vortex_array_non_null
1209            .offsets()
1210            .as_::<PrimitiveVTable>();
1211        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1212        assert_eq!(offsets_array_non_null.ptype(), PType::I64); // Large lists use I64 offsets
1213    }
1214
1215    // Test null array conversions
1216    #[test]
1217    fn test_null_array_conversion() {
1218        let arrow_array = NullArray::new(5);
1219        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1220        assert_eq!(vortex_array.len(), 5);
1221    }
1222
1223    // Test buffer conversions
1224    #[test]
1225    fn test_arrow_buffer_conversion() {
1226        let data = vec![1u8, 2, 3, 4, 5];
1227        let arrow_buffer = ArrowBuffer::from_vec(data);
1228        let vortex_array = arrow_buffer.into_array();
1229        assert_eq!(vortex_array.len(), 5);
1230    }
1231
1232    #[test]
1233    fn test_boolean_buffer_conversion() {
1234        let data = vec![true, false, true, false, true];
1235        let boolean_buffer = BooleanBuffer::from(data);
1236        let vortex_array = boolean_buffer.into_array();
1237        assert_eq!(vortex_array.len(), 5);
1238    }
1239
1240    #[test]
1241    fn test_scalar_buffer_conversion() {
1242        let data = vec![1i32, 2, 3, 4, 5];
1243        let scalar_buffer = ScalarBuffer::from(data);
1244        let vortex_array = scalar_buffer.into_array();
1245        assert_eq!(vortex_array.len(), 5);
1246    }
1247
1248    #[test]
1249    fn test_offset_buffer_conversion() {
1250        let data = vec![0i32, 2, 5, 8, 10];
1251        let offset_buffer = OffsetBuffer::new(ScalarBuffer::from(data));
1252        let vortex_array = offset_buffer.into_array();
1253        assert_eq!(vortex_array.len(), 5);
1254    }
1255
1256    // Test RecordBatch conversions
1257    #[test]
1258    fn test_record_batch_conversion() {
1259        let schema = Arc::new(Schema::new(vec![
1260            Field::new("field1", DataType::Int32, false),
1261            Field::new("field2", DataType::Utf8, false),
1262        ]));
1263
1264        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1265        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1266
1267        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1268
1269        let vortex_array = ArrayRef::from_arrow(record_batch, false);
1270        assert_eq!(vortex_array.len(), 4);
1271
1272        // Test with reference
1273        let schema = Arc::new(Schema::new(vec![
1274            Field::new("field1", DataType::Int32, false),
1275            Field::new("field2", DataType::Utf8, false),
1276        ]));
1277
1278        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1279        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1280
1281        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1282
1283        let vortex_array = ArrayRef::from_arrow(&record_batch, false);
1284        assert_eq!(vortex_array.len(), 4);
1285    }
1286
1287    // Test dynamic dispatch conversion
1288    #[test]
1289    fn test_dyn_array_conversion() {
1290        let int_array = Int32Array::from(vec![1, 2, 3, 4]);
1291        let dyn_array: &dyn ArrowArray = &int_array;
1292        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1293        assert_eq!(vortex_array.len(), 4);
1294
1295        let string_array = StringArray::from(vec!["a", "b", "c"]);
1296        let dyn_array: &dyn ArrowArray = &string_array;
1297        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1298        assert_eq!(vortex_array.len(), 3);
1299
1300        let bool_array = BooleanArray::from(vec![true, false, true]);
1301        let dyn_array: &dyn ArrowArray = &bool_array;
1302        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1303        assert_eq!(vortex_array.len(), 3);
1304    }
1305
1306    // Existing tests
1307    #[test]
1308    pub fn nullable_may_contain_non_nullable() {
1309        let null_struct_array_with_non_nullable_field = new_null_array(
1310            &DataType::Struct(Fields::from(vec![Field::new(
1311                "non_nullable_inner",
1312                DataType::Int32,
1313                false,
1314            )])),
1315            1,
1316        );
1317        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1318    }
1319
1320    #[test]
1321    pub fn nullable_may_contain_deeply_nested_non_nullable() {
1322        let null_struct_array_with_non_nullable_field = new_null_array(
1323            &DataType::Struct(Fields::from(vec![Field::new(
1324                "non_nullable_inner",
1325                DataType::Struct(Fields::from(vec![Field::new(
1326                    "non_nullable_deeper_inner",
1327                    DataType::Int32,
1328                    false,
1329                )])),
1330                false,
1331            )])),
1332            1,
1333        );
1334        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1335    }
1336
1337    #[test]
1338    #[should_panic]
1339    pub fn cannot_handle_nullable_struct_containing_non_nullable_dictionary() {
1340        let null_struct_array_with_non_nullable_field = new_null_array(
1341            &DataType::Struct(Fields::from(vec![Field::new(
1342                "non_nullable_deeper_inner",
1343                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1344                false,
1345            )])),
1346            1,
1347        );
1348
1349        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1350    }
1351}
vortex_array/arrow/convert.rs

vortex_array/arrow/
convert.rs
