vortex_array/arrow/
convert.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::sync::Arc;
5
6use arrow_array::cast::{AsArray, as_null_array};
7use arrow_array::types::{
8    ByteArrayType, ByteViewType, Date32Type, Date64Type, Decimal32Type, Decimal64Type,
9    Decimal128Type, Decimal256Type, Float16Type, Float32Type, Float64Type, Int8Type, Int16Type,
10    Int32Type, Int64Type, Time32MillisecondType, Time32SecondType, Time64MicrosecondType,
11    Time64NanosecondType, TimestampMicrosecondType, TimestampMillisecondType,
12    TimestampNanosecondType, TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
13};
14use arrow_array::{
15    Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray,
16    FixedSizeListArray as ArrowFixedSizeListArray, GenericByteArray, GenericByteViewArray,
17    GenericListArray, NullArray as ArrowNullArray, OffsetSizeTrait,
18    PrimitiveArray as ArrowPrimitiveArray, RecordBatch, StructArray as ArrowStructArray,
19    make_array,
20};
21use arrow_buffer::buffer::{NullBuffer, OffsetBuffer};
22use arrow_buffer::{ArrowNativeType, BooleanBuffer, Buffer as ArrowBuffer, ScalarBuffer};
23use arrow_schema::{DataType, TimeUnit as ArrowTimeUnit};
24use itertools::Itertools;
25use vortex_buffer::{Alignment, Buffer, ByteBuffer};
26use vortex_dtype::datetime::TimeUnit;
27use vortex_dtype::{DType, DecimalDType, NativePType, PType};
28use vortex_error::{VortexExpect as _, vortex_panic};
29use vortex_scalar::i256;
30
31use crate::arrays::{
32    BoolArray, DecimalArray, FixedSizeListArray, ListArray, NullArray, PrimitiveArray, StructArray,
33    TemporalArray, VarBinArray, VarBinViewArray,
34};
35use crate::arrow::FromArrowArray;
36use crate::validity::Validity;
37use crate::{ArrayRef, IntoArray};
38
39impl IntoArray for ArrowBuffer {
40    fn into_array(self) -> ArrayRef {
41        PrimitiveArray::from_byte_buffer(
42            ByteBuffer::from_arrow_buffer(self, Alignment::of::<u8>()),
43            PType::U8,
44            Validity::NonNullable,
45        )
46        .into_array()
47    }
48}
49
50impl IntoArray for BooleanBuffer {
51    fn into_array(self) -> ArrayRef {
52        BoolArray::from_bool_buffer(self, Validity::NonNullable).into_array()
53    }
54}
55
56impl<T> IntoArray for ScalarBuffer<T>
57where
58    T: ArrowNativeType + NativePType,
59{
60    fn into_array(self) -> ArrayRef {
61        PrimitiveArray::new(
62            Buffer::<T>::from_arrow_scalar_buffer(self),
63            Validity::NonNullable,
64        )
65        .into_array()
66    }
67}
68
69impl<O> IntoArray for OffsetBuffer<O>
70where
71    O: NativePType + OffsetSizeTrait,
72{
73    fn into_array(self) -> ArrayRef {
74        let primitive = PrimitiveArray::new(
75            Buffer::from_arrow_scalar_buffer(self.into_inner()),
76            Validity::NonNullable,
77        );
78
79        primitive.into_array()
80    }
81}
82
83macro_rules! impl_from_arrow_primitive {
84    ($T:path) => {
85        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
86            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
87                let buffer = Buffer::from_arrow_scalar_buffer(value.values().clone());
88                let validity = nulls(value.nulls(), nullable);
89                PrimitiveArray::new(buffer, validity).into_array()
90            }
91        }
92    };
93}
94
95impl_from_arrow_primitive!(Int8Type);
96impl_from_arrow_primitive!(Int16Type);
97impl_from_arrow_primitive!(Int32Type);
98impl_from_arrow_primitive!(Int64Type);
99impl_from_arrow_primitive!(UInt8Type);
100impl_from_arrow_primitive!(UInt16Type);
101impl_from_arrow_primitive!(UInt32Type);
102impl_from_arrow_primitive!(UInt64Type);
103impl_from_arrow_primitive!(Float16Type);
104impl_from_arrow_primitive!(Float32Type);
105impl_from_arrow_primitive!(Float64Type);
106
107impl FromArrowArray<&ArrowPrimitiveArray<Decimal32Type>> for ArrayRef {
108    fn from_arrow(array: &ArrowPrimitiveArray<Decimal32Type>, nullable: bool) -> Self {
109        let decimal_type = DecimalDType::new(array.precision(), array.scale());
110        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
111        let validity = nulls(array.nulls(), nullable);
112        DecimalArray::new(buffer, decimal_type, validity).into_array()
113    }
114}
115
116impl FromArrowArray<&ArrowPrimitiveArray<Decimal64Type>> for ArrayRef {
117    fn from_arrow(array: &ArrowPrimitiveArray<Decimal64Type>, nullable: bool) -> Self {
118        let decimal_type = DecimalDType::new(array.precision(), array.scale());
119        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
120        let validity = nulls(array.nulls(), nullable);
121        DecimalArray::new(buffer, decimal_type, validity).into_array()
122    }
123}
124
125impl FromArrowArray<&ArrowPrimitiveArray<Decimal128Type>> for ArrayRef {
126    fn from_arrow(array: &ArrowPrimitiveArray<Decimal128Type>, nullable: bool) -> Self {
127        let decimal_type = DecimalDType::new(array.precision(), array.scale());
128        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
129        let validity = nulls(array.nulls(), nullable);
130        DecimalArray::new(buffer, decimal_type, validity).into_array()
131    }
132}
133
134impl FromArrowArray<&ArrowPrimitiveArray<Decimal256Type>> for ArrayRef {
135    fn from_arrow(array: &ArrowPrimitiveArray<Decimal256Type>, nullable: bool) -> Self {
136        let decimal_type = DecimalDType::new(array.precision(), array.scale());
137        let buffer = Buffer::from_arrow_scalar_buffer(array.values().clone());
138        // SAFETY: Our i256 implementation has the same bit-pattern representation of the
139        //  arrow_buffer::i256 type. It is safe to treat values held inside the buffer as values
140        //  of either type.
141        let buffer =
142            unsafe { std::mem::transmute::<Buffer<arrow_buffer::i256>, Buffer<i256>>(buffer) };
143        let validity = nulls(array.nulls(), nullable);
144        DecimalArray::new(buffer, decimal_type, validity).into_array()
145    }
146}
147
148macro_rules! impl_from_arrow_temporal {
149    ($T:path) => {
150        impl FromArrowArray<&ArrowPrimitiveArray<$T>> for ArrayRef {
151            fn from_arrow(value: &ArrowPrimitiveArray<$T>, nullable: bool) -> Self {
152                temporal_array(value, nullable)
153            }
154        }
155    };
156}
157
158// timestamp
159impl_from_arrow_temporal!(TimestampSecondType);
160impl_from_arrow_temporal!(TimestampMillisecondType);
161impl_from_arrow_temporal!(TimestampMicrosecondType);
162impl_from_arrow_temporal!(TimestampNanosecondType);
163
164// time
165impl_from_arrow_temporal!(Time32SecondType);
166impl_from_arrow_temporal!(Time32MillisecondType);
167impl_from_arrow_temporal!(Time64MicrosecondType);
168impl_from_arrow_temporal!(Time64NanosecondType);
169
170// date
171impl_from_arrow_temporal!(Date32Type);
172impl_from_arrow_temporal!(Date64Type);
173
174fn temporal_array<T: ArrowPrimitiveType>(value: &ArrowPrimitiveArray<T>, nullable: bool) -> ArrayRef
175where
176    T::Native: NativePType,
177{
178    let arr = PrimitiveArray::new(
179        Buffer::from_arrow_scalar_buffer(value.values().clone()),
180        nulls(value.nulls(), nullable),
181    )
182    .into_array();
183
184    match value.data_type() {
185        DataType::Timestamp(time_unit, tz) => {
186            let tz = tz.as_ref().map(|s| s.to_string());
187            TemporalArray::new_timestamp(arr, time_unit.into(), tz).into()
188        }
189        DataType::Time32(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
190        DataType::Time64(time_unit) => TemporalArray::new_time(arr, time_unit.into()).into(),
191        DataType::Date32 => TemporalArray::new_date(arr, TimeUnit::Days).into(),
192        DataType::Date64 => TemporalArray::new_date(arr, TimeUnit::Milliseconds).into(),
193        DataType::Duration(_) => unimplemented!(),
194        DataType::Interval(_) => unimplemented!(),
195        _ => vortex_panic!("Invalid temporal type: {}", value.data_type()),
196    }
197}
198
199impl<T: ByteArrayType> FromArrowArray<&GenericByteArray<T>> for ArrayRef
200where
201    <T as ByteArrayType>::Offset: NativePType,
202{
203    fn from_arrow(value: &GenericByteArray<T>, nullable: bool) -> Self {
204        let dtype = match T::DATA_TYPE {
205            DataType::Binary | DataType::LargeBinary => DType::Binary(nullable.into()),
206            DataType::Utf8 | DataType::LargeUtf8 => DType::Utf8(nullable.into()),
207            dt => vortex_panic!("Invalid data type for ByteArray: {dt}"),
208        };
209        VarBinArray::try_new(
210            value.offsets().clone().into_array(),
211            ByteBuffer::from_arrow_buffer(value.values().clone(), Alignment::of::<u8>()),
212            dtype,
213            nulls(value.nulls(), nullable),
214        )
215        .vortex_expect("Failed to convert Arrow GenericByteArray to Vortex VarBinArray")
216        .into_array()
217    }
218}
219
220impl<T: ByteViewType> FromArrowArray<&GenericByteViewArray<T>> for ArrayRef {
221    fn from_arrow(value: &GenericByteViewArray<T>, nullable: bool) -> Self {
222        let dtype = match T::DATA_TYPE {
223            DataType::BinaryView => DType::Binary(nullable.into()),
224            DataType::Utf8View => DType::Utf8(nullable.into()),
225            dt => vortex_panic!("Invalid data type for ByteViewArray: {dt}"),
226        };
227
228        let views_buffer = Buffer::from_byte_buffer(
229            Buffer::from_arrow_scalar_buffer(value.views().clone()).into_byte_buffer(),
230        );
231
232        // SAFETY: arrow-rs ByteViewArray already checks the same invariants, we inherit those
233        //  guarantees by zero-copy constructing from one.
234        unsafe {
235            VarBinViewArray::new_unchecked(
236                views_buffer,
237                Arc::from(
238                    value
239                        .data_buffers()
240                        .iter()
241                        .map(|b| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::<u8>()))
242                        .collect::<Vec<_>>(),
243                ),
244                dtype,
245                nulls(value.nulls(), nullable),
246            )
247            .into_array()
248        }
249    }
250}
251
252impl FromArrowArray<&ArrowBooleanArray> for ArrayRef {
253    fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self {
254        BoolArray::from_bool_buffer(value.values().clone(), nulls(value.nulls(), nullable))
255            .into_array()
256    }
257}
258
259/// Strip out the nulls from this array and return a new array without nulls.
260fn remove_nulls(data: arrow_data::ArrayData) -> arrow_data::ArrayData {
261    if data.null_count() == 0 {
262        // No nulls to remove, return the array as is
263        return data;
264    }
265
266    let children = match data.data_type() {
267        DataType::Struct(fields) => Some(
268            fields
269                .iter()
270                .zip(data.child_data().iter())
271                .map(|(field, child_data)| {
272                    if field.is_nullable() {
273                        child_data.clone()
274                    } else {
275                        remove_nulls(child_data.clone())
276                    }
277                })
278                .collect_vec(),
279        ),
280        DataType::List(f)
281        | DataType::LargeList(f)
282        | DataType::ListView(f)
283        | DataType::LargeListView(f)
284        | DataType::FixedSizeList(f, _)
285            if !f.is_nullable() =>
286        {
287            // All list types only have one child
288            assert_eq!(
289                data.child_data().len(),
290                1,
291                "List types should have one child"
292            );
293            Some(vec![remove_nulls(data.child_data()[0].clone())])
294        }
295        _ => None,
296    };
297
298    let mut builder = data.into_builder().nulls(None);
299    if let Some(children) = children {
300        builder = builder.child_data(children);
301    }
302    builder
303        .build()
304        .vortex_expect("reconstructing array without nulls")
305}
306
307impl FromArrowArray<&ArrowStructArray> for ArrayRef {
308    fn from_arrow(value: &ArrowStructArray, nullable: bool) -> Self {
309        StructArray::try_new(
310            value.column_names().iter().copied().collect(),
311            value
312                .columns()
313                .iter()
314                .zip(value.fields())
315                .map(|(c, field)| {
316                    // Arrow pushes down nulls, even into non-nullable fields. So we strip them
317                    // out here because Vortex is a little more strict.
318                    if c.null_count() > 0 && !field.is_nullable() {
319                        let stripped = make_array(remove_nulls(c.into_data()));
320                        Self::from_arrow(stripped.as_ref(), false)
321                    } else {
322                        Self::from_arrow(c.as_ref(), field.is_nullable())
323                    }
324                })
325                .collect(),
326            value.len(),
327            nulls(value.nulls(), nullable),
328        )
329        .vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
330        .into_array()
331    }
332}
333
334impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListArray<O>> for ArrayRef {
335    fn from_arrow(value: &GenericListArray<O>, nullable: bool) -> Self {
336        // Extract the validity of the underlying element array
337        let elem_nullable = match value.data_type() {
338            DataType::List(field) => field.is_nullable(),
339            DataType::LargeList(field) => field.is_nullable(),
340            dt => vortex_panic!("Invalid data type for ListArray: {dt}"),
341        };
342        ListArray::try_new(
343            Self::from_arrow(value.values().as_ref(), elem_nullable),
344            // offsets are always non-nullable
345            value.offsets().clone().into_array(),
346            nulls(value.nulls(), nullable),
347        )
348        .vortex_expect("Failed to convert Arrow ListArray to Vortex ListArray")
349        .into_array()
350    }
351}
352
353impl FromArrowArray<&ArrowFixedSizeListArray> for ArrayRef {
354    fn from_arrow(array: &ArrowFixedSizeListArray, nullable: bool) -> Self {
355        let DataType::FixedSizeList(field, list_size) = array.data_type() else {
356            vortex_panic!("Invalid data type for ListArray: {}", array.data_type());
357        };
358
359        FixedSizeListArray::try_new(
360            Self::from_arrow(array.values().as_ref(), field.is_nullable()),
361            *list_size as u32,
362            nulls(array.nulls(), nullable),
363            array.len(),
364        )
365        .vortex_expect("Failed to convert Arrow FixedSizeListArray to Vortex FixedSizeListArray")
366        .into_array()
367    }
368}
369
370impl FromArrowArray<&ArrowNullArray> for ArrayRef {
371    fn from_arrow(value: &ArrowNullArray, nullable: bool) -> Self {
372        assert!(nullable);
373        NullArray::new(value.len()).into_array()
374    }
375}
376
377fn nulls(nulls: Option<&NullBuffer>, nullable: bool) -> Validity {
378    if nullable {
379        nulls
380            .map(|nulls| {
381                if nulls.null_count() == nulls.len() {
382                    Validity::AllInvalid
383                } else {
384                    Validity::from(nulls.inner().clone())
385                }
386            })
387            .unwrap_or_else(|| Validity::AllValid)
388    } else {
389        assert!(nulls.map(|x| x.null_count() == 0).unwrap_or(true));
390        Validity::NonNullable
391    }
392}
393
394impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
395    fn from_arrow(array: &dyn ArrowArray, nullable: bool) -> Self {
396        match array.data_type() {
397            DataType::Boolean => Self::from_arrow(array.as_boolean(), nullable),
398            DataType::UInt8 => Self::from_arrow(array.as_primitive::<UInt8Type>(), nullable),
399            DataType::UInt16 => Self::from_arrow(array.as_primitive::<UInt16Type>(), nullable),
400            DataType::UInt32 => Self::from_arrow(array.as_primitive::<UInt32Type>(), nullable),
401            DataType::UInt64 => Self::from_arrow(array.as_primitive::<UInt64Type>(), nullable),
402            DataType::Int8 => Self::from_arrow(array.as_primitive::<Int8Type>(), nullable),
403            DataType::Int16 => Self::from_arrow(array.as_primitive::<Int16Type>(), nullable),
404            DataType::Int32 => Self::from_arrow(array.as_primitive::<Int32Type>(), nullable),
405            DataType::Int64 => Self::from_arrow(array.as_primitive::<Int64Type>(), nullable),
406            DataType::Float16 => Self::from_arrow(array.as_primitive::<Float16Type>(), nullable),
407            DataType::Float32 => Self::from_arrow(array.as_primitive::<Float32Type>(), nullable),
408            DataType::Float64 => Self::from_arrow(array.as_primitive::<Float64Type>(), nullable),
409            DataType::Utf8 => Self::from_arrow(array.as_string::<i32>(), nullable),
410            DataType::LargeUtf8 => Self::from_arrow(array.as_string::<i64>(), nullable),
411            DataType::Binary => Self::from_arrow(array.as_binary::<i32>(), nullable),
412            DataType::LargeBinary => Self::from_arrow(array.as_binary::<i64>(), nullable),
413            DataType::BinaryView => Self::from_arrow(array.as_binary_view(), nullable),
414            DataType::Utf8View => Self::from_arrow(array.as_string_view(), nullable),
415            DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),
416            DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),
417            DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),
418            DataType::FixedSizeList(..) => Self::from_arrow(array.as_fixed_size_list(), nullable),
419            DataType::Null => Self::from_arrow(as_null_array(array), nullable),
420            DataType::Timestamp(u, _) => match u {
421                ArrowTimeUnit::Second => {
422                    Self::from_arrow(array.as_primitive::<TimestampSecondType>(), nullable)
423                }
424                ArrowTimeUnit::Millisecond => {
425                    Self::from_arrow(array.as_primitive::<TimestampMillisecondType>(), nullable)
426                }
427                ArrowTimeUnit::Microsecond => {
428                    Self::from_arrow(array.as_primitive::<TimestampMicrosecondType>(), nullable)
429                }
430                ArrowTimeUnit::Nanosecond => {
431                    Self::from_arrow(array.as_primitive::<TimestampNanosecondType>(), nullable)
432                }
433            },
434            DataType::Date32 => Self::from_arrow(array.as_primitive::<Date32Type>(), nullable),
435            DataType::Date64 => Self::from_arrow(array.as_primitive::<Date64Type>(), nullable),
436            DataType::Time32(u) => match u {
437                ArrowTimeUnit::Second => {
438                    Self::from_arrow(array.as_primitive::<Time32SecondType>(), nullable)
439                }
440                ArrowTimeUnit::Millisecond => {
441                    Self::from_arrow(array.as_primitive::<Time32MillisecondType>(), nullable)
442                }
443                ArrowTimeUnit::Microsecond | ArrowTimeUnit::Nanosecond => unreachable!(),
444            },
445            DataType::Time64(u) => match u {
446                ArrowTimeUnit::Microsecond => {
447                    Self::from_arrow(array.as_primitive::<Time64MicrosecondType>(), nullable)
448                }
449                ArrowTimeUnit::Nanosecond => {
450                    Self::from_arrow(array.as_primitive::<Time64NanosecondType>(), nullable)
451                }
452                ArrowTimeUnit::Second | ArrowTimeUnit::Millisecond => unreachable!(),
453            },
454            DataType::Decimal32(..) => {
455                Self::from_arrow(array.as_primitive::<Decimal32Type>(), nullable)
456            }
457            DataType::Decimal64(..) => {
458                Self::from_arrow(array.as_primitive::<Decimal64Type>(), nullable)
459            }
460            DataType::Decimal128(..) => {
461                Self::from_arrow(array.as_primitive::<Decimal128Type>(), nullable)
462            }
463            DataType::Decimal256(..) => {
464                Self::from_arrow(array.as_primitive::<Decimal256Type>(), nullable)
465            }
466            dt => vortex_panic!("Array encoding not implemented for Arrow data type {dt}"),
467        }
468    }
469}
470
471impl FromArrowArray<RecordBatch> for ArrayRef {
472    fn from_arrow(array: RecordBatch, nullable: bool) -> Self {
473        ArrayRef::from_arrow(&arrow_array::StructArray::from(array), nullable)
474    }
475}
476
477impl FromArrowArray<&RecordBatch> for ArrayRef {
478    fn from_arrow(array: &RecordBatch, nullable: bool) -> Self {
479        Self::from_arrow(array.clone(), nullable)
480    }
481}
482
483#[cfg(test)]
484mod tests {
485    use std::sync::Arc;
486
487    use arrow_array::builder::{
488        BinaryViewBuilder, Decimal128Builder, Decimal256Builder, Int32Builder, LargeListBuilder,
489        ListBuilder, StringViewBuilder,
490    };
491    use arrow_array::types::{ArrowPrimitiveType, Float16Type};
492    use arrow_array::{
493        Array as ArrowArray, BinaryArray, BooleanArray, Date32Array, Date64Array, Float32Array,
494        Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, LargeBinaryArray,
495        LargeStringArray, NullArray, RecordBatch, StringArray, StructArray, Time32MillisecondArray,
496        Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
497        TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
498        TimestampSecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array, new_null_array,
499    };
500    use arrow_buffer::{BooleanBuffer, Buffer as ArrowBuffer, OffsetBuffer, ScalarBuffer};
501    use arrow_schema::{DataType, Field, Fields, Schema};
502    use vortex_dtype::datetime::{TIMESTAMP_ID, TemporalMetadata, TimeUnit};
503    use vortex_dtype::{DType, ExtDType, Nullability, PType};
504
505    use crate::arrays::{
506        DecimalVTable, ListVTable, PrimitiveVTable, StructVTable, TemporalArray, VarBinVTable,
507        VarBinViewVTable,
508    };
509    use crate::arrow::FromArrowArray as _;
510    use crate::{ArrayRef, IntoArray};
511
512    // Test primitive array conversions
513    #[test]
514    fn test_int8_array_conversion() {
515        let arrow_array = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]);
516        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
517
518        let arrow_array_non_null = Int8Array::from(vec![1, 2, 3, 4]);
519        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
520
521        assert_eq!(vortex_array.len(), 4);
522        assert_eq!(vortex_array_non_null.len(), 4);
523
524        // Verify metadata - should be PrimitiveArray with I8 ptype
525        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
526        assert_eq!(primitive_array.ptype(), PType::I8);
527
528        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
529        assert_eq!(primitive_array_non_null.ptype(), PType::I8);
530    }
531
532    #[test]
533    fn test_int16_array_conversion() {
534        let arrow_array = Int16Array::from(vec![Some(100), None, Some(300), Some(400)]);
535        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
536
537        let arrow_array_non_null = Int16Array::from(vec![100, 200, 300, 400]);
538        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
539
540        assert_eq!(vortex_array.len(), 4);
541        assert_eq!(vortex_array_non_null.len(), 4);
542
543        // Verify metadata - should be PrimitiveArray with I16 ptype
544        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
545        assert_eq!(primitive_array.ptype(), PType::I16);
546
547        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
548        assert_eq!(primitive_array_non_null.ptype(), PType::I16);
549    }
550
551    #[test]
552    fn test_int32_array_conversion() {
553        let arrow_array = Int32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
554        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
555
556        let arrow_array_non_null = Int32Array::from(vec![1000, 2000, 3000, 4000]);
557        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
558
559        assert_eq!(vortex_array.len(), 4);
560        assert_eq!(vortex_array_non_null.len(), 4);
561
562        // Verify metadata - should be PrimitiveArray with I32 ptype
563        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
564        assert_eq!(primitive_array.ptype(), PType::I32);
565
566        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
567        assert_eq!(primitive_array_non_null.ptype(), PType::I32);
568    }
569
570    #[test]
571    fn test_int64_array_conversion() {
572        let arrow_array = Int64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
573        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
574
575        let arrow_array_non_null = Int64Array::from(vec![10000_i64, 20000, 30000, 40000]);
576        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
577
578        assert_eq!(vortex_array.len(), 4);
579        assert_eq!(vortex_array_non_null.len(), 4);
580
581        // Verify metadata - should be PrimitiveArray with I64 ptype
582        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
583        assert_eq!(primitive_array.ptype(), PType::I64);
584
585        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
586        assert_eq!(primitive_array_non_null.ptype(), PType::I64);
587    }
588
589    #[test]
590    fn test_uint8_array_conversion() {
591        let arrow_array = UInt8Array::from(vec![Some(1), None, Some(3), Some(4)]);
592        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
593
594        let arrow_array_non_null = UInt8Array::from(vec![1_u8, 2, 3, 4]);
595        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
596
597        assert_eq!(vortex_array.len(), 4);
598        assert_eq!(vortex_array_non_null.len(), 4);
599
600        // Verify metadata - should be PrimitiveArray with U8 ptype
601        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
602        assert_eq!(primitive_array.ptype(), PType::U8);
603
604        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
605        assert_eq!(primitive_array_non_null.ptype(), PType::U8);
606    }
607
608    #[test]
609    fn test_uint16_array_conversion() {
610        let arrow_array = UInt16Array::from(vec![Some(100), None, Some(300), Some(400)]);
611        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
612
613        let arrow_array_non_null = UInt16Array::from(vec![100_u16, 200, 300, 400]);
614        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
615
616        assert_eq!(vortex_array.len(), 4);
617        assert_eq!(vortex_array_non_null.len(), 4);
618
619        // Verify metadata - should be PrimitiveArray with U16 ptype
620        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
621        assert_eq!(primitive_array.ptype(), PType::U16);
622
623        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
624        assert_eq!(primitive_array_non_null.ptype(), PType::U16);
625    }
626
627    #[test]
628    fn test_uint32_array_conversion() {
629        let arrow_array = UInt32Array::from(vec![Some(1000), None, Some(3000), Some(4000)]);
630        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
631
632        let arrow_array_non_null = UInt32Array::from(vec![1000_u32, 2000, 3000, 4000]);
633        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
634
635        assert_eq!(vortex_array.len(), 4);
636        assert_eq!(vortex_array_non_null.len(), 4);
637
638        // Verify metadata - should be PrimitiveArray with U32 ptype
639        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
640        assert_eq!(primitive_array.ptype(), PType::U32);
641
642        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
643        assert_eq!(primitive_array_non_null.ptype(), PType::U32);
644    }
645
646    #[test]
647    fn test_uint64_array_conversion() {
648        let arrow_array = UInt64Array::from(vec![Some(10000), None, Some(30000), Some(40000)]);
649        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
650
651        let arrow_array_non_null = UInt64Array::from(vec![10000_u64, 20000, 30000, 40000]);
652        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
653
654        assert_eq!(vortex_array.len(), 4);
655        assert_eq!(vortex_array_non_null.len(), 4);
656
657        // Verify metadata - should be PrimitiveArray with U64 ptype
658        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
659        assert_eq!(primitive_array.ptype(), PType::U64);
660
661        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
662        assert_eq!(primitive_array_non_null.ptype(), PType::U64);
663    }
664
665    #[test]
666    fn test_float16_array_conversion() {
667        let values = vec![
668            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5)),
669            None,
670            Some(<Float16Type as ArrowPrimitiveType>::Native::from_f32(3.5)),
671        ];
672        let arrow_array = arrow_array::PrimitiveArray::<Float16Type>::from(values);
673        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
674
675        let non_null_values = vec![
676            <Float16Type as ArrowPrimitiveType>::Native::from_f32(1.5),
677            <Float16Type as ArrowPrimitiveType>::Native::from_f32(2.5),
678        ];
679        let arrow_array_non_null =
680            arrow_array::PrimitiveArray::<Float16Type>::from(non_null_values);
681        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
682
683        assert_eq!(vortex_array.len(), 3);
684        assert_eq!(vortex_array_non_null.len(), 2);
685
686        // Verify metadata - should be PrimitiveArray with F16 ptype
687        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
688        assert_eq!(primitive_array.ptype(), PType::F16);
689
690        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
691        assert_eq!(primitive_array_non_null.ptype(), PType::F16);
692    }
693
694    #[test]
695    fn test_float32_array_conversion() {
696        let arrow_array = Float32Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
697        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
698
699        let arrow_array_non_null = Float32Array::from(vec![1.5_f32, 2.5, 3.5, 4.5]);
700        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
701
702        assert_eq!(vortex_array.len(), 4);
703        assert_eq!(vortex_array_non_null.len(), 4);
704
705        // Verify metadata - should be PrimitiveArray with F32 ptype
706        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
707        assert_eq!(primitive_array.ptype(), PType::F32);
708
709        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
710        assert_eq!(primitive_array_non_null.ptype(), PType::F32);
711    }
712
713    #[test]
714    fn test_float64_array_conversion() {
715        let arrow_array = Float64Array::from(vec![Some(1.5), None, Some(3.5), Some(4.5)]);
716        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
717
718        let arrow_array_non_null = Float64Array::from(vec![1.5_f64, 2.5, 3.5, 4.5]);
719        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
720
721        assert_eq!(vortex_array.len(), 4);
722        assert_eq!(vortex_array_non_null.len(), 4);
723
724        // Verify metadata - should be PrimitiveArray with F64 ptype
725        let primitive_array = vortex_array.as_::<PrimitiveVTable>();
726        assert_eq!(primitive_array.ptype(), PType::F64);
727
728        let primitive_array_non_null = vortex_array_non_null.as_::<PrimitiveVTable>();
729        assert_eq!(primitive_array_non_null.ptype(), PType::F64);
730    }
731
732    // Test decimal array conversions
733    #[test]
734    fn test_decimal128_array_conversion() {
735        let mut builder = Decimal128Builder::with_capacity(4);
736        builder.append_value(12345);
737        builder.append_null();
738        builder.append_value(67890);
739        builder.append_value(11111);
740        let decimal_array = builder.finish().with_precision_and_scale(10, 2).unwrap();
741
742        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
743        assert_eq!(vortex_array.len(), 4);
744
745        let mut builder_non_null = Decimal128Builder::with_capacity(3);
746        builder_non_null.append_value(12345);
747        builder_non_null.append_value(67890);
748        builder_non_null.append_value(11111);
749        let decimal_array_non_null = builder_non_null
750            .finish()
751            .with_precision_and_scale(10, 2)
752            .unwrap();
753
754        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
755        assert_eq!(vortex_array_non_null.len(), 3);
756
757        // Verify metadata - should be DecimalArray with correct precision and scale
758        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
759        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 10);
760        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 2);
761
762        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
763        assert_eq!(
764            decimal_vortex_array_non_null.decimal_dtype().precision(),
765            10
766        );
767        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 2);
768    }
769
770    #[test]
771    fn test_decimal256_array_conversion() {
772        let mut builder = Decimal256Builder::with_capacity(4);
773        builder.append_value(arrow_buffer::i256::from_i128(12345));
774        builder.append_null();
775        builder.append_value(arrow_buffer::i256::from_i128(67890));
776        builder.append_value(arrow_buffer::i256::from_i128(11111));
777        let decimal_array = builder.finish().with_precision_and_scale(38, 10).unwrap();
778
779        let vortex_array = ArrayRef::from_arrow(&decimal_array, true);
780        assert_eq!(vortex_array.len(), 4);
781
782        let mut builder_non_null = Decimal256Builder::with_capacity(3);
783        builder_non_null.append_value(arrow_buffer::i256::from_i128(12345));
784        builder_non_null.append_value(arrow_buffer::i256::from_i128(67890));
785        builder_non_null.append_value(arrow_buffer::i256::from_i128(11111));
786        let decimal_array_non_null = builder_non_null
787            .finish()
788            .with_precision_and_scale(38, 10)
789            .unwrap();
790
791        let vortex_array_non_null = ArrayRef::from_arrow(&decimal_array_non_null, false);
792        assert_eq!(vortex_array_non_null.len(), 3);
793
794        // Verify metadata - should be DecimalArray with correct precision and scale
795        let decimal_vortex_array = vortex_array.as_::<DecimalVTable>();
796        assert_eq!(decimal_vortex_array.decimal_dtype().precision(), 38);
797        assert_eq!(decimal_vortex_array.decimal_dtype().scale(), 10);
798
799        let decimal_vortex_array_non_null = vortex_array_non_null.as_::<DecimalVTable>();
800        assert_eq!(
801            decimal_vortex_array_non_null.decimal_dtype().precision(),
802            38
803        );
804        assert_eq!(decimal_vortex_array_non_null.decimal_dtype().scale(), 10);
805    }
806
807    // Test temporal array conversions
808    #[test]
809    fn test_timestamp_second_array_conversion() {
810        let arrow_array =
811            TimestampSecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
812        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
813
814        let arrow_array_non_null = TimestampSecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
815        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
816
817        assert_eq!(vortex_array.len(), 4);
818        assert_eq!(vortex_array_non_null.len(), 4);
819
820        // Verify metadata - should be TemporalArray with Second time unit
821        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
822        assert_eq!(
823            temporal_array.temporal_metadata().time_unit(),
824            TimeUnit::Seconds
825        );
826
827        let temporal_array_non_null =
828            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
829        assert_eq!(
830            temporal_array_non_null.temporal_metadata().time_unit(),
831            TimeUnit::Seconds
832        );
833    }
834
835    #[test]
836    fn test_timestamp_millisecond_array_conversion() {
837        let arrow_array =
838            TimestampMillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
839        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
840
841        let arrow_array_non_null =
842            TimestampMillisecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
843        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
844
845        assert_eq!(vortex_array.len(), 4);
846        assert_eq!(vortex_array_non_null.len(), 4);
847    }
848
849    #[test]
850    fn test_timestamp_microsecond_array_conversion() {
851        let arrow_array =
852            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
853        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
854
855        let arrow_array_non_null =
856            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
857        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
858
859        assert_eq!(vortex_array.len(), 4);
860        assert_eq!(vortex_array_non_null.len(), 4);
861    }
862
863    #[test]
864    fn test_timestamp_timezone_microsecond_array_conversion() {
865        let arrow_array =
866            TimestampMicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)])
867                .with_timezone("UTC");
868        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
869
870        let arrow_array_non_null =
871            TimestampMicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]).with_timezone("UTC");
872        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
873
874        assert_eq!(vortex_array.len(), 4);
875        assert_eq!(
876            vortex_array.dtype(),
877            &DType::Extension(Arc::new(ExtDType::new(
878                TIMESTAMP_ID.clone(),
879                Arc::new(DType::Primitive(PType::I64, Nullability::Nullable)),
880                Some(
881                    TemporalMetadata::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string()))
882                        .into()
883                )
884            )))
885        );
886        assert_eq!(vortex_array_non_null.len(), 4);
887        assert_eq!(
888            vortex_array_non_null.dtype(),
889            &DType::Extension(Arc::new(ExtDType::new(
890                TIMESTAMP_ID.clone(),
891                Arc::new(DType::Primitive(PType::I64, Nullability::NonNullable)),
892                Some(
893                    TemporalMetadata::Timestamp(TimeUnit::Microseconds, Some("UTC".to_string()))
894                        .into()
895                )
896            )))
897        );
898    }
899
900    #[test]
901    fn test_timestamp_nanosecond_array_conversion() {
902        let arrow_array =
903            TimestampNanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
904        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
905
906        let arrow_array_non_null = TimestampNanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
907        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
908
909        assert_eq!(vortex_array.len(), 4);
910        assert_eq!(vortex_array_non_null.len(), 4);
911    }
912
913    #[test]
914    fn test_time32_second_array_conversion() {
915        let arrow_array = Time32SecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
916        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
917
918        let arrow_array_non_null = Time32SecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
919        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
920
921        assert_eq!(vortex_array.len(), 4);
922        assert_eq!(vortex_array_non_null.len(), 4);
923
924        // Verify metadata - should be TemporalArray with Second time unit
925        let temporal_array = TemporalArray::try_from(vortex_array.clone()).unwrap();
926        assert_eq!(
927            temporal_array.temporal_metadata().time_unit(),
928            TimeUnit::Seconds
929        );
930
931        let temporal_array_non_null =
932            TemporalArray::try_from(vortex_array_non_null.clone()).unwrap();
933        assert_eq!(
934            temporal_array_non_null.temporal_metadata().time_unit(),
935            TimeUnit::Seconds
936        );
937    }
938
939    #[test]
940    fn test_time32_millisecond_array_conversion() {
941        let arrow_array =
942            Time32MillisecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
943        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
944
945        let arrow_array_non_null = Time32MillisecondArray::from(vec![1000_i32, 2000, 3000, 4000]);
946        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
947
948        assert_eq!(vortex_array.len(), 4);
949        assert_eq!(vortex_array_non_null.len(), 4);
950    }
951
952    #[test]
953    fn test_time64_microsecond_array_conversion() {
954        let arrow_array =
955            Time64MicrosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
956        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
957
958        let arrow_array_non_null = Time64MicrosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
959        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
960
961        assert_eq!(vortex_array.len(), 4);
962        assert_eq!(vortex_array_non_null.len(), 4);
963    }
964
965    #[test]
966    fn test_time64_nanosecond_array_conversion() {
967        let arrow_array =
968            Time64NanosecondArray::from(vec![Some(1000), None, Some(3000), Some(4000)]);
969        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
970
971        let arrow_array_non_null = Time64NanosecondArray::from(vec![1000_i64, 2000, 3000, 4000]);
972        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
973
974        assert_eq!(vortex_array.len(), 4);
975        assert_eq!(vortex_array_non_null.len(), 4);
976    }
977
978    #[test]
979    fn test_date32_array_conversion() {
980        let arrow_array = Date32Array::from(vec![Some(18000), None, Some(18002), Some(18003)]);
981        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
982
983        let arrow_array_non_null = Date32Array::from(vec![18000_i32, 18001, 18002, 18003]);
984        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
985
986        assert_eq!(vortex_array.len(), 4);
987        assert_eq!(vortex_array_non_null.len(), 4);
988    }
989
990    #[test]
991    fn test_date64_array_conversion() {
992        let arrow_array = Date64Array::from(vec![
993            Some(1555200000000),
994            None,
995            Some(1555286400000),
996            Some(1555372800000),
997        ]);
998        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
999
1000        let arrow_array_non_null = Date64Array::from(vec![
1001            1555200000000_i64,
1002            1555213600000,
1003            1555286400000,
1004            1555372800000,
1005        ]);
1006        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1007
1008        assert_eq!(vortex_array.len(), 4);
1009        assert_eq!(vortex_array_non_null.len(), 4);
1010    }
1011
1012    // Test string/binary array conversions
1013    #[test]
1014    fn test_utf8_array_conversion() {
1015        let arrow_array = StringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
1016        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1017
1018        let arrow_array_non_null = StringArray::from(vec!["hello", "world", "test", "vortex"]);
1019        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1020
1021        assert_eq!(vortex_array.len(), 4);
1022        assert_eq!(vortex_array_non_null.len(), 4);
1023
1024        // Verify metadata - should be VarBinArray with Utf8 dtype
1025        let varbin_array = vortex_array.as_::<VarBinVTable>();
1026        assert_eq!(varbin_array.dtype(), &DType::Utf8(true.into()));
1027
1028        let varbin_array_non_null = vortex_array_non_null.as_::<VarBinVTable>();
1029        assert_eq!(varbin_array_non_null.dtype(), &DType::Utf8(false.into()));
1030    }
1031
1032    #[test]
1033    fn test_large_utf8_array_conversion() {
1034        let arrow_array =
1035            LargeStringArray::from(vec![Some("hello"), None, Some("world"), Some("test")]);
1036        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1037
1038        let arrow_array_non_null = LargeStringArray::from(vec!["hello", "world", "test", "vortex"]);
1039        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1040
1041        assert_eq!(vortex_array.len(), 4);
1042        assert_eq!(vortex_array_non_null.len(), 4);
1043    }
1044
1045    #[test]
1046    fn test_binary_array_conversion() {
1047        let arrow_array = BinaryArray::from(vec![
1048            Some("hello".as_bytes()),
1049            None,
1050            Some("world".as_bytes()),
1051            Some("test".as_bytes()),
1052        ]);
1053        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1054
1055        let arrow_array_non_null = BinaryArray::from(vec![
1056            "hello".as_bytes(),
1057            "world".as_bytes(),
1058            "test".as_bytes(),
1059            "vortex".as_bytes(),
1060        ]);
1061        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1062
1063        assert_eq!(vortex_array.len(), 4);
1064        assert_eq!(vortex_array_non_null.len(), 4);
1065    }
1066
1067    #[test]
1068    fn test_large_binary_array_conversion() {
1069        let arrow_array = LargeBinaryArray::from(vec![
1070            Some("hello".as_bytes()),
1071            None,
1072            Some("world".as_bytes()),
1073            Some("test".as_bytes()),
1074        ]);
1075        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1076
1077        let arrow_array_non_null = LargeBinaryArray::from(vec![
1078            "hello".as_bytes(),
1079            "world".as_bytes(),
1080            "test".as_bytes(),
1081            "vortex".as_bytes(),
1082        ]);
1083        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1084
1085        assert_eq!(vortex_array.len(), 4);
1086        assert_eq!(vortex_array_non_null.len(), 4);
1087    }
1088
1089    #[test]
1090    fn test_utf8_view_array_conversion() {
1091        let mut builder = StringViewBuilder::new();
1092        builder.append_value("hello");
1093        builder.append_null();
1094        builder.append_value("world");
1095        builder.append_value("test");
1096        let arrow_array = builder.finish();
1097        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1098
1099        let mut builder_non_null = StringViewBuilder::new();
1100        builder_non_null.append_value("hello");
1101        builder_non_null.append_value("world");
1102        builder_non_null.append_value("test");
1103        builder_non_null.append_value("vortex");
1104        let arrow_array_non_null = builder_non_null.finish();
1105        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1106
1107        assert_eq!(vortex_array.len(), 4);
1108        assert_eq!(vortex_array_non_null.len(), 4);
1109
1110        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1111        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1112        assert_eq!(
1113            varbin_view_array.buffers().len(),
1114            arrow_array.data_buffers().len()
1115        );
1116        assert_eq!(varbin_view_array.dtype(), &DType::Utf8(true.into()));
1117
1118        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1119        assert_eq!(
1120            varbin_view_array_non_null.buffers().len(),
1121            arrow_array_non_null.data_buffers().len()
1122        );
1123        assert_eq!(
1124            varbin_view_array_non_null.dtype(),
1125            &DType::Utf8(false.into())
1126        );
1127    }
1128
1129    #[test]
1130    fn test_binary_view_array_conversion() {
1131        let mut builder = BinaryViewBuilder::new();
1132        builder.append_value(b"hello");
1133        builder.append_null();
1134        builder.append_value(b"world");
1135        builder.append_value(b"test");
1136        let arrow_array = builder.finish();
1137        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1138
1139        let mut builder_non_null = BinaryViewBuilder::new();
1140        builder_non_null.append_value(b"hello");
1141        builder_non_null.append_value(b"world");
1142        builder_non_null.append_value(b"test");
1143        builder_non_null.append_value(b"vortex");
1144        let arrow_array_non_null = builder_non_null.finish();
1145        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1146
1147        assert_eq!(vortex_array.len(), 4);
1148        assert_eq!(vortex_array_non_null.len(), 4);
1149
1150        // Verify metadata - should be VarBinViewArray with correct buffer count and dtype
1151        let varbin_view_array = vortex_array.as_::<VarBinViewVTable>();
1152        assert_eq!(
1153            varbin_view_array.buffers().len(),
1154            arrow_array.data_buffers().len()
1155        );
1156        assert_eq!(varbin_view_array.dtype(), &DType::Binary(true.into()));
1157
1158        let varbin_view_array_non_null = vortex_array_non_null.as_::<VarBinViewVTable>();
1159        assert_eq!(
1160            varbin_view_array_non_null.buffers().len(),
1161            arrow_array_non_null.data_buffers().len()
1162        );
1163        assert_eq!(
1164            varbin_view_array_non_null.dtype(),
1165            &DType::Binary(false.into())
1166        );
1167    }
1168
1169    // Test boolean array conversions
1170    #[test]
1171    fn test_boolean_array_conversion() {
1172        let arrow_array = BooleanArray::from(vec![Some(true), None, Some(false), Some(true)]);
1173        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1174
1175        let arrow_array_non_null = BooleanArray::from(vec![true, false, true, false]);
1176        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1177
1178        assert_eq!(vortex_array.len(), 4);
1179        assert_eq!(vortex_array_non_null.len(), 4);
1180    }
1181
1182    // Test struct array conversions
1183    #[test]
1184    fn test_struct_array_conversion() {
1185        let fields = vec![
1186            Field::new("field1", DataType::Int32, true),
1187            Field::new("field2", DataType::Utf8, false),
1188        ];
1189        let schema = Fields::from(fields);
1190
1191        let field1_data = Int32Array::from(vec![Some(1), None, Some(3)]);
1192        let field2_data = StringArray::from(vec!["a", "b", "c"]);
1193
1194        let arrow_array = StructArray::new(
1195            schema.clone(),
1196            vec![Arc::new(field1_data), Arc::new(field2_data)],
1197            None,
1198        );
1199
1200        let vortex_array = ArrayRef::from_arrow(&arrow_array, false);
1201        assert_eq!(vortex_array.len(), 3);
1202
1203        // Verify metadata - should be StructArray with correct field names
1204        let struct_vortex_array = vortex_array.as_::<StructVTable>();
1205        assert_eq!(struct_vortex_array.names().len(), 2);
1206        assert_eq!(struct_vortex_array.names()[0], "field1");
1207        assert_eq!(struct_vortex_array.names()[1], "field2");
1208
1209        // Test nullable struct
1210        let nullable_array = StructArray::new(
1211            schema,
1212            vec![
1213                Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
1214                Arc::new(StringArray::from(vec!["a", "b", "c"])),
1215            ],
1216            Some(arrow_buffer::NullBuffer::new(BooleanBuffer::from(vec![
1217                true, false, true,
1218            ]))),
1219        );
1220
1221        let vortex_nullable_array = ArrayRef::from_arrow(&nullable_array, true);
1222        assert_eq!(vortex_nullable_array.len(), 3);
1223
1224        // Verify metadata for nullable struct
1225        let struct_vortex_nullable_array = vortex_nullable_array.as_::<StructVTable>();
1226        assert_eq!(struct_vortex_nullable_array.names().len(), 2);
1227        assert_eq!(struct_vortex_nullable_array.names()[0], "field1");
1228        assert_eq!(struct_vortex_nullable_array.names()[1], "field2");
1229    }
1230
1231    // Test list array conversions
1232    #[test]
1233    fn test_list_array_conversion() {
1234        let mut builder = ListBuilder::new(Int32Builder::new());
1235        builder.append_value([Some(1), None, Some(3)]);
1236        builder.append_null();
1237        builder.append_value([Some(4), Some(5)]);
1238        let arrow_array = builder.finish();
1239
1240        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1241        assert_eq!(vortex_array.len(), 3);
1242
1243        // Verify metadata - should be ListArray with correct offsets
1244        let list_vortex_array = vortex_array.as_::<ListVTable>();
1245        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1246        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1247        assert_eq!(offsets_array.ptype(), PType::I32);
1248
1249        // Test non-nullable list
1250        let mut builder_non_null = ListBuilder::new(Int32Builder::new());
1251        builder_non_null.append_value([Some(1), None, Some(3)]);
1252        builder_non_null.append_value([Some(4), Some(5)]);
1253        let arrow_array_non_null = builder_non_null.finish();
1254
1255        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1256        assert_eq!(vortex_array_non_null.len(), 2);
1257
1258        // Verify metadata for non-nullable list
1259        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1260        let offsets_array_non_null = list_vortex_array_non_null
1261            .offsets()
1262            .as_::<PrimitiveVTable>();
1263        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1264        assert_eq!(offsets_array_non_null.ptype(), PType::I32);
1265    }
1266
1267    #[test]
1268    fn test_large_list_array_conversion() {
1269        let mut builder = LargeListBuilder::new(Int32Builder::new());
1270        builder.append_value([Some(1), None, Some(3)]);
1271        builder.append_null();
1272        builder.append_value([Some(4), Some(5)]);
1273        let arrow_array = builder.finish();
1274
1275        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1276        assert_eq!(vortex_array.len(), 3);
1277
1278        // Verify metadata - should be ListArray with correct offsets (I64 for large lists)
1279        let list_vortex_array = vortex_array.as_::<ListVTable>();
1280        let offsets_array = list_vortex_array.offsets().as_::<PrimitiveVTable>();
1281        assert_eq!(offsets_array.len(), 4); // n+1 offsets for n lists
1282        assert_eq!(offsets_array.ptype(), PType::I64); // Large lists use I64 offsets
1283
1284        // Test non-nullable large list
1285        let mut builder_non_null = LargeListBuilder::new(Int32Builder::new());
1286        builder_non_null.append_value([Some(1), None, Some(3)]);
1287        builder_non_null.append_value([Some(4), Some(5)]);
1288        let arrow_array_non_null = builder_non_null.finish();
1289
1290        let vortex_array_non_null = ArrayRef::from_arrow(&arrow_array_non_null, false);
1291        assert_eq!(vortex_array_non_null.len(), 2);
1292
1293        // Verify metadata for non-nullable large list
1294        let list_vortex_array_non_null = vortex_array_non_null.as_::<ListVTable>();
1295        let offsets_array_non_null = list_vortex_array_non_null
1296            .offsets()
1297            .as_::<PrimitiveVTable>();
1298        assert_eq!(offsets_array_non_null.len(), 3); // n+1 offsets for n lists
1299        assert_eq!(offsets_array_non_null.ptype(), PType::I64); // Large lists use I64 offsets
1300    }
1301
1302    // Test null array conversions
1303    #[test]
1304    fn test_null_array_conversion() {
1305        let arrow_array = NullArray::new(5);
1306        let vortex_array = ArrayRef::from_arrow(&arrow_array, true);
1307        assert_eq!(vortex_array.len(), 5);
1308    }
1309
1310    // Test buffer conversions
1311    #[test]
1312    fn test_arrow_buffer_conversion() {
1313        let data = vec![1u8, 2, 3, 4, 5];
1314        let arrow_buffer = ArrowBuffer::from_vec(data);
1315        let vortex_array = arrow_buffer.into_array();
1316        assert_eq!(vortex_array.len(), 5);
1317    }
1318
1319    #[test]
1320    fn test_boolean_buffer_conversion() {
1321        let data = vec![true, false, true, false, true];
1322        let boolean_buffer = BooleanBuffer::from(data);
1323        let vortex_array = boolean_buffer.into_array();
1324        assert_eq!(vortex_array.len(), 5);
1325    }
1326
1327    #[test]
1328    fn test_scalar_buffer_conversion() {
1329        let data = vec![1i32, 2, 3, 4, 5];
1330        let scalar_buffer = ScalarBuffer::from(data);
1331        let vortex_array = scalar_buffer.into_array();
1332        assert_eq!(vortex_array.len(), 5);
1333    }
1334
1335    #[test]
1336    fn test_offset_buffer_conversion() {
1337        let data = vec![0i32, 2, 5, 8, 10];
1338        let offset_buffer = OffsetBuffer::new(ScalarBuffer::from(data));
1339        let vortex_array = offset_buffer.into_array();
1340        assert_eq!(vortex_array.len(), 5);
1341    }
1342
1343    // Test RecordBatch conversions
1344    #[test]
1345    fn test_record_batch_conversion() {
1346        let schema = Arc::new(Schema::new(vec![
1347            Field::new("field1", DataType::Int32, false),
1348            Field::new("field2", DataType::Utf8, false),
1349        ]));
1350
1351        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1352        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1353
1354        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1355
1356        let vortex_array = ArrayRef::from_arrow(record_batch, false);
1357        assert_eq!(vortex_array.len(), 4);
1358
1359        // Test with reference
1360        let schema = Arc::new(Schema::new(vec![
1361            Field::new("field1", DataType::Int32, false),
1362            Field::new("field2", DataType::Utf8, false),
1363        ]));
1364
1365        let field1_data = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
1366        let field2_data = Arc::new(StringArray::from(vec!["a", "b", "c", "d"]));
1367
1368        let record_batch = RecordBatch::try_new(schema, vec![field1_data, field2_data]).unwrap();
1369
1370        let vortex_array = ArrayRef::from_arrow(&record_batch, false);
1371        assert_eq!(vortex_array.len(), 4);
1372    }
1373
1374    // Test dynamic dispatch conversion
1375    #[test]
1376    fn test_dyn_array_conversion() {
1377        let int_array = Int32Array::from(vec![1, 2, 3, 4]);
1378        let dyn_array: &dyn ArrowArray = &int_array;
1379        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1380        assert_eq!(vortex_array.len(), 4);
1381
1382        let string_array = StringArray::from(vec!["a", "b", "c"]);
1383        let dyn_array: &dyn ArrowArray = &string_array;
1384        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1385        assert_eq!(vortex_array.len(), 3);
1386
1387        let bool_array = BooleanArray::from(vec![true, false, true]);
1388        let dyn_array: &dyn ArrowArray = &bool_array;
1389        let vortex_array = ArrayRef::from_arrow(dyn_array, false);
1390        assert_eq!(vortex_array.len(), 3);
1391    }
1392
1393    // Existing tests
1394    #[test]
1395    pub fn nullable_may_contain_non_nullable() {
1396        let null_struct_array_with_non_nullable_field = new_null_array(
1397            &DataType::Struct(Fields::from(vec![Field::new(
1398                "non_nullable_inner",
1399                DataType::Int32,
1400                false,
1401            )])),
1402            1,
1403        );
1404        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1405    }
1406
1407    #[test]
1408    pub fn nullable_may_contain_deeply_nested_non_nullable() {
1409        let null_struct_array_with_non_nullable_field = new_null_array(
1410            &DataType::Struct(Fields::from(vec![Field::new(
1411                "non_nullable_inner",
1412                DataType::Struct(Fields::from(vec![Field::new(
1413                    "non_nullable_deeper_inner",
1414                    DataType::Int32,
1415                    false,
1416                )])),
1417                false,
1418            )])),
1419            1,
1420        );
1421        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1422    }
1423
1424    #[test]
1425    #[should_panic]
1426    pub fn cannot_handle_nullable_struct_containing_non_nullable_dictionary() {
1427        let null_struct_array_with_non_nullable_field = new_null_array(
1428            &DataType::Struct(Fields::from(vec![Field::new(
1429                "non_nullable_deeper_inner",
1430                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
1431                false,
1432            )])),
1433            1,
1434        );
1435
1436        ArrayRef::from_arrow(null_struct_array_with_non_nullable_field.as_ref(), true);
1437    }
1438}
vortex_array/arrow/convert.rs

vortex_array/arrow/
convert.rs
