1mod binary_array;
21
22use crate::types::*;
23use arrow_buffer::{ArrowNativeType, NullBuffer, OffsetBuffer, ScalarBuffer};
24use arrow_data::ArrayData;
25use arrow_schema::{DataType, IntervalUnit, TimeUnit};
26use std::any::Any;
27use std::sync::Arc;
28
29pub use binary_array::*;
30
31mod boolean_array;
32pub use boolean_array::*;
33
34mod byte_array;
35pub use byte_array::*;
36
37mod dictionary_array;
38pub use dictionary_array::*;
39
40mod fixed_size_binary_array;
41pub use fixed_size_binary_array::*;
42
43mod fixed_size_list_array;
44pub use fixed_size_list_array::*;
45
46mod list_array;
47pub use list_array::*;
48
49mod map_array;
50pub use map_array::*;
51
52mod null_array;
53pub use null_array::*;
54
55mod primitive_array;
56pub use primitive_array::*;
57
58mod string_array;
59pub use string_array::*;
60
61mod struct_array;
62pub use struct_array::*;
63
64mod union_array;
65pub use union_array::*;
66
67mod run_array;
68
69pub use run_array::*;
70
71mod byte_view_array;
72
73pub use byte_view_array::*;
74
75mod list_view_array;
76
77pub use list_view_array::*;
78
79use crate::iterator::ArrayIter;
80
81pub unsafe trait Array: std::fmt::Debug + Send + Sync {
92 fn as_any(&self) -> &dyn Any;
115
116 fn to_data(&self) -> ArrayData;
118
119 fn into_data(self) -> ArrayData;
123
124 fn data_type(&self) -> &DataType;
137
138 fn slice(&self, offset: usize, length: usize) -> ArrayRef;
152
153 fn len(&self) -> usize;
165
166 fn is_empty(&self) -> bool;
178
179 fn shrink_to_fit(&mut self) {}
184
185 fn offset(&self) -> usize;
201
202 fn nulls(&self) -> Option<&NullBuffer>;
215
216 fn logical_nulls(&self) -> Option<NullBuffer> {
235 self.nulls().cloned()
236 }
237
238 fn is_null(&self, index: usize) -> bool {
260 self.nulls().map(|n| n.is_null(index)).unwrap_or_default()
261 }
262
263 fn is_valid(&self, index: usize) -> bool {
277 !self.is_null(index)
278 }
279
280 fn null_count(&self) -> usize {
296 self.nulls().map(|n| n.null_count()).unwrap_or_default()
297 }
298
299 fn logical_null_count(&self) -> usize {
316 self.logical_nulls()
317 .map(|n| n.null_count())
318 .unwrap_or_default()
319 }
320
321 fn is_nullable(&self) -> bool {
335 self.logical_null_count() != 0
336 }
337
338 fn get_buffer_memory_size(&self) -> usize;
343
344 fn get_array_memory_size(&self) -> usize;
348}
349
350pub type ArrayRef = Arc<dyn Array>;
352
353unsafe impl Array for ArrayRef {
355 fn as_any(&self) -> &dyn Any {
356 self.as_ref().as_any()
357 }
358
359 fn to_data(&self) -> ArrayData {
360 self.as_ref().to_data()
361 }
362
363 fn into_data(self) -> ArrayData {
364 self.to_data()
365 }
366
367 fn data_type(&self) -> &DataType {
368 self.as_ref().data_type()
369 }
370
371 fn slice(&self, offset: usize, length: usize) -> ArrayRef {
372 self.as_ref().slice(offset, length)
373 }
374
375 fn len(&self) -> usize {
376 self.as_ref().len()
377 }
378
379 fn is_empty(&self) -> bool {
380 self.as_ref().is_empty()
381 }
382
383 fn shrink_to_fit(&mut self) {
385 if let Some(slf) = Arc::get_mut(self) {
386 slf.shrink_to_fit();
387 } else {
388 }
390 }
391
392 fn offset(&self) -> usize {
393 self.as_ref().offset()
394 }
395
396 fn nulls(&self) -> Option<&NullBuffer> {
397 self.as_ref().nulls()
398 }
399
400 fn logical_nulls(&self) -> Option<NullBuffer> {
401 self.as_ref().logical_nulls()
402 }
403
404 fn is_null(&self, index: usize) -> bool {
405 self.as_ref().is_null(index)
406 }
407
408 fn is_valid(&self, index: usize) -> bool {
409 self.as_ref().is_valid(index)
410 }
411
412 fn null_count(&self) -> usize {
413 self.as_ref().null_count()
414 }
415
416 fn logical_null_count(&self) -> usize {
417 self.as_ref().logical_null_count()
418 }
419
420 fn is_nullable(&self) -> bool {
421 self.as_ref().is_nullable()
422 }
423
424 fn get_buffer_memory_size(&self) -> usize {
425 self.as_ref().get_buffer_memory_size()
426 }
427
428 fn get_array_memory_size(&self) -> usize {
429 self.as_ref().get_array_memory_size()
430 }
431}
432
433unsafe impl<T: Array> Array for &T {
434 fn as_any(&self) -> &dyn Any {
435 T::as_any(self)
436 }
437
438 fn to_data(&self) -> ArrayData {
439 T::to_data(self)
440 }
441
442 fn into_data(self) -> ArrayData {
443 self.to_data()
444 }
445
446 fn data_type(&self) -> &DataType {
447 T::data_type(self)
448 }
449
450 fn slice(&self, offset: usize, length: usize) -> ArrayRef {
451 T::slice(self, offset, length)
452 }
453
454 fn len(&self) -> usize {
455 T::len(self)
456 }
457
458 fn is_empty(&self) -> bool {
459 T::is_empty(self)
460 }
461
462 fn offset(&self) -> usize {
463 T::offset(self)
464 }
465
466 fn nulls(&self) -> Option<&NullBuffer> {
467 T::nulls(self)
468 }
469
470 fn logical_nulls(&self) -> Option<NullBuffer> {
471 T::logical_nulls(self)
472 }
473
474 fn is_null(&self, index: usize) -> bool {
475 T::is_null(self, index)
476 }
477
478 fn is_valid(&self, index: usize) -> bool {
479 T::is_valid(self, index)
480 }
481
482 fn null_count(&self) -> usize {
483 T::null_count(self)
484 }
485
486 fn logical_null_count(&self) -> usize {
487 T::logical_null_count(self)
488 }
489
490 fn is_nullable(&self) -> bool {
491 T::is_nullable(self)
492 }
493
494 fn get_buffer_memory_size(&self) -> usize {
495 T::get_buffer_memory_size(self)
496 }
497
498 fn get_array_memory_size(&self) -> usize {
499 T::get_array_memory_size(self)
500 }
501}
502
503pub trait ArrayAccessor: Array {
584 type Item: Send + Sync;
586
587 fn value(&self, index: usize) -> Self::Item;
591
592 unsafe fn value_unchecked(&self, index: usize) -> Self::Item;
596}
597
598pub trait StringArrayType<'a>: ArrayAccessor<Item = &'a str> + Sized {
606 fn is_ascii(&self) -> bool;
608
609 fn iter(&self) -> ArrayIter<Self>;
611}
612
613impl<'a, O: OffsetSizeTrait> StringArrayType<'a> for &'a GenericStringArray<O> {
614 fn is_ascii(&self) -> bool {
615 GenericStringArray::<O>::is_ascii(self)
616 }
617
618 fn iter(&self) -> ArrayIter<Self> {
619 GenericStringArray::<O>::iter(self)
620 }
621}
622impl<'a> StringArrayType<'a> for &'a StringViewArray {
623 fn is_ascii(&self) -> bool {
624 StringViewArray::is_ascii(self)
625 }
626
627 fn iter(&self) -> ArrayIter<Self> {
628 StringViewArray::iter(self)
629 }
630}
631
632pub trait BinaryArrayType<'a>: ArrayAccessor<Item = &'a [u8]> + Sized {
641 fn iter(&self) -> ArrayIter<Self>;
643}
644
645impl<'a, O: OffsetSizeTrait> BinaryArrayType<'a> for &'a GenericBinaryArray<O> {
646 fn iter(&self) -> ArrayIter<Self> {
647 GenericBinaryArray::<O>::iter(self)
648 }
649}
650impl<'a> BinaryArrayType<'a> for &'a BinaryViewArray {
651 fn iter(&self) -> ArrayIter<Self> {
652 BinaryViewArray::iter(self)
653 }
654}
655impl<'a> BinaryArrayType<'a> for &'a FixedSizeBinaryArray {
656 fn iter(&self) -> ArrayIter<Self> {
657 FixedSizeBinaryArray::iter(self)
658 }
659}
660
661impl PartialEq for dyn Array + '_ {
662 fn eq(&self, other: &Self) -> bool {
663 self.to_data().eq(&other.to_data())
664 }
665}
666
667impl<T: Array> PartialEq<T> for dyn Array + '_ {
668 fn eq(&self, other: &T) -> bool {
669 self.to_data().eq(&other.to_data())
670 }
671}
672
673impl PartialEq for NullArray {
674 fn eq(&self, other: &NullArray) -> bool {
675 self.to_data().eq(&other.to_data())
676 }
677}
678
679impl<T: ArrowPrimitiveType> PartialEq for PrimitiveArray<T> {
680 fn eq(&self, other: &PrimitiveArray<T>) -> bool {
681 self.to_data().eq(&other.to_data())
682 }
683}
684
685impl<K: ArrowDictionaryKeyType> PartialEq for DictionaryArray<K> {
686 fn eq(&self, other: &Self) -> bool {
687 self.to_data().eq(&other.to_data())
688 }
689}
690
691impl PartialEq for BooleanArray {
692 fn eq(&self, other: &BooleanArray) -> bool {
693 self.to_data().eq(&other.to_data())
694 }
695}
696
697impl<OffsetSize: OffsetSizeTrait> PartialEq for GenericStringArray<OffsetSize> {
698 fn eq(&self, other: &Self) -> bool {
699 self.to_data().eq(&other.to_data())
700 }
701}
702
703impl<OffsetSize: OffsetSizeTrait> PartialEq for GenericBinaryArray<OffsetSize> {
704 fn eq(&self, other: &Self) -> bool {
705 self.to_data().eq(&other.to_data())
706 }
707}
708
709impl PartialEq for FixedSizeBinaryArray {
710 fn eq(&self, other: &Self) -> bool {
711 self.to_data().eq(&other.to_data())
712 }
713}
714
715impl<OffsetSize: OffsetSizeTrait> PartialEq for GenericListArray<OffsetSize> {
716 fn eq(&self, other: &Self) -> bool {
717 self.to_data().eq(&other.to_data())
718 }
719}
720
721impl<OffsetSize: OffsetSizeTrait> PartialEq for GenericListViewArray<OffsetSize> {
722 fn eq(&self, other: &Self) -> bool {
723 self.to_data().eq(&other.to_data())
724 }
725}
726
727impl PartialEq for MapArray {
728 fn eq(&self, other: &Self) -> bool {
729 self.to_data().eq(&other.to_data())
730 }
731}
732
733impl PartialEq for FixedSizeListArray {
734 fn eq(&self, other: &Self) -> bool {
735 self.to_data().eq(&other.to_data())
736 }
737}
738
739impl PartialEq for StructArray {
740 fn eq(&self, other: &Self) -> bool {
741 self.to_data().eq(&other.to_data())
742 }
743}
744
745impl<T: ByteViewType + ?Sized> PartialEq for GenericByteViewArray<T> {
746 fn eq(&self, other: &Self) -> bool {
747 self.to_data().eq(&other.to_data())
748 }
749}
750
751impl<R: RunEndIndexType> PartialEq for RunArray<R> {
752 fn eq(&self, other: &Self) -> bool {
753 self.to_data().eq(&other.to_data())
754 }
755}
756
757pub fn make_array(data: ArrayData) -> ArrayRef {
760 match data.data_type() {
761 DataType::Boolean => Arc::new(BooleanArray::from(data)) as ArrayRef,
762 DataType::Int8 => Arc::new(Int8Array::from(data)) as ArrayRef,
763 DataType::Int16 => Arc::new(Int16Array::from(data)) as ArrayRef,
764 DataType::Int32 => Arc::new(Int32Array::from(data)) as ArrayRef,
765 DataType::Int64 => Arc::new(Int64Array::from(data)) as ArrayRef,
766 DataType::UInt8 => Arc::new(UInt8Array::from(data)) as ArrayRef,
767 DataType::UInt16 => Arc::new(UInt16Array::from(data)) as ArrayRef,
768 DataType::UInt32 => Arc::new(UInt32Array::from(data)) as ArrayRef,
769 DataType::UInt64 => Arc::new(UInt64Array::from(data)) as ArrayRef,
770 DataType::Float16 => Arc::new(Float16Array::from(data)) as ArrayRef,
771 DataType::Float32 => Arc::new(Float32Array::from(data)) as ArrayRef,
772 DataType::Float64 => Arc::new(Float64Array::from(data)) as ArrayRef,
773 DataType::Date32 => Arc::new(Date32Array::from(data)) as ArrayRef,
774 DataType::Date64 => Arc::new(Date64Array::from(data)) as ArrayRef,
775 DataType::Time32(TimeUnit::Second) => Arc::new(Time32SecondArray::from(data)) as ArrayRef,
776 DataType::Time32(TimeUnit::Millisecond) => {
777 Arc::new(Time32MillisecondArray::from(data)) as ArrayRef
778 }
779 DataType::Time64(TimeUnit::Microsecond) => {
780 Arc::new(Time64MicrosecondArray::from(data)) as ArrayRef
781 }
782 DataType::Time64(TimeUnit::Nanosecond) => {
783 Arc::new(Time64NanosecondArray::from(data)) as ArrayRef
784 }
785 DataType::Timestamp(TimeUnit::Second, _) => {
786 Arc::new(TimestampSecondArray::from(data)) as ArrayRef
787 }
788 DataType::Timestamp(TimeUnit::Millisecond, _) => {
789 Arc::new(TimestampMillisecondArray::from(data)) as ArrayRef
790 }
791 DataType::Timestamp(TimeUnit::Microsecond, _) => {
792 Arc::new(TimestampMicrosecondArray::from(data)) as ArrayRef
793 }
794 DataType::Timestamp(TimeUnit::Nanosecond, _) => {
795 Arc::new(TimestampNanosecondArray::from(data)) as ArrayRef
796 }
797 DataType::Interval(IntervalUnit::YearMonth) => {
798 Arc::new(IntervalYearMonthArray::from(data)) as ArrayRef
799 }
800 DataType::Interval(IntervalUnit::DayTime) => {
801 Arc::new(IntervalDayTimeArray::from(data)) as ArrayRef
802 }
803 DataType::Interval(IntervalUnit::MonthDayNano) => {
804 Arc::new(IntervalMonthDayNanoArray::from(data)) as ArrayRef
805 }
806 DataType::Duration(TimeUnit::Second) => {
807 Arc::new(DurationSecondArray::from(data)) as ArrayRef
808 }
809 DataType::Duration(TimeUnit::Millisecond) => {
810 Arc::new(DurationMillisecondArray::from(data)) as ArrayRef
811 }
812 DataType::Duration(TimeUnit::Microsecond) => {
813 Arc::new(DurationMicrosecondArray::from(data)) as ArrayRef
814 }
815 DataType::Duration(TimeUnit::Nanosecond) => {
816 Arc::new(DurationNanosecondArray::from(data)) as ArrayRef
817 }
818 DataType::Binary => Arc::new(BinaryArray::from(data)) as ArrayRef,
819 DataType::LargeBinary => Arc::new(LargeBinaryArray::from(data)) as ArrayRef,
820 DataType::FixedSizeBinary(_) => Arc::new(FixedSizeBinaryArray::from(data)) as ArrayRef,
821 DataType::BinaryView => Arc::new(BinaryViewArray::from(data)) as ArrayRef,
822 DataType::Utf8 => Arc::new(StringArray::from(data)) as ArrayRef,
823 DataType::LargeUtf8 => Arc::new(LargeStringArray::from(data)) as ArrayRef,
824 DataType::Utf8View => Arc::new(StringViewArray::from(data)) as ArrayRef,
825 DataType::List(_) => Arc::new(ListArray::from(data)) as ArrayRef,
826 DataType::LargeList(_) => Arc::new(LargeListArray::from(data)) as ArrayRef,
827 DataType::ListView(_) => Arc::new(ListViewArray::from(data)) as ArrayRef,
828 DataType::LargeListView(_) => Arc::new(LargeListViewArray::from(data)) as ArrayRef,
829 DataType::Struct(_) => Arc::new(StructArray::from(data)) as ArrayRef,
830 DataType::Map(_, _) => Arc::new(MapArray::from(data)) as ArrayRef,
831 DataType::Union(_, _) => Arc::new(UnionArray::from(data)) as ArrayRef,
832 DataType::FixedSizeList(_, _) => Arc::new(FixedSizeListArray::from(data)) as ArrayRef,
833 DataType::Dictionary(key_type, _) => match key_type.as_ref() {
834 DataType::Int8 => Arc::new(DictionaryArray::<Int8Type>::from(data)) as ArrayRef,
835 DataType::Int16 => Arc::new(DictionaryArray::<Int16Type>::from(data)) as ArrayRef,
836 DataType::Int32 => Arc::new(DictionaryArray::<Int32Type>::from(data)) as ArrayRef,
837 DataType::Int64 => Arc::new(DictionaryArray::<Int64Type>::from(data)) as ArrayRef,
838 DataType::UInt8 => Arc::new(DictionaryArray::<UInt8Type>::from(data)) as ArrayRef,
839 DataType::UInt16 => Arc::new(DictionaryArray::<UInt16Type>::from(data)) as ArrayRef,
840 DataType::UInt32 => Arc::new(DictionaryArray::<UInt32Type>::from(data)) as ArrayRef,
841 DataType::UInt64 => Arc::new(DictionaryArray::<UInt64Type>::from(data)) as ArrayRef,
842 dt => unimplemented!("Unexpected dictionary key type {dt}"),
843 },
844 DataType::RunEndEncoded(run_ends_type, _) => match run_ends_type.data_type() {
845 DataType::Int16 => Arc::new(RunArray::<Int16Type>::from(data)) as ArrayRef,
846 DataType::Int32 => Arc::new(RunArray::<Int32Type>::from(data)) as ArrayRef,
847 DataType::Int64 => Arc::new(RunArray::<Int64Type>::from(data)) as ArrayRef,
848 dt => unimplemented!("Unexpected data type for run_ends array {dt}"),
849 },
850 DataType::Null => Arc::new(NullArray::from(data)) as ArrayRef,
851 DataType::Decimal32(_, _) => Arc::new(Decimal32Array::from(data)) as ArrayRef,
852 DataType::Decimal64(_, _) => Arc::new(Decimal64Array::from(data)) as ArrayRef,
853 DataType::Decimal128(_, _) => Arc::new(Decimal128Array::from(data)) as ArrayRef,
854 DataType::Decimal256(_, _) => Arc::new(Decimal256Array::from(data)) as ArrayRef,
855 dt => unimplemented!("Unexpected data type {dt}"),
856 }
857}
858
859pub fn new_empty_array(data_type: &DataType) -> ArrayRef {
872 let data = ArrayData::new_empty(data_type);
873 make_array(data)
874}
875
876pub fn new_null_array(data_type: &DataType, length: usize) -> ArrayRef {
890 make_array(ArrayData::new_null(data_type, length))
891}
892
893unsafe fn get_offsets<O: ArrowNativeType>(data: &ArrayData) -> OffsetBuffer<O> {
899 match data.is_empty() && data.buffers()[0].is_empty() {
900 true => OffsetBuffer::new_empty(),
901 false => {
902 let buffer =
903 ScalarBuffer::new(data.buffers()[0].clone(), data.offset(), data.len() + 1);
904 unsafe { OffsetBuffer::new_unchecked(buffer) }
907 }
908 }
909}
910
911fn print_long_array<A, F>(array: &A, f: &mut std::fmt::Formatter, print_item: F) -> std::fmt::Result
913where
914 A: Array,
915 F: Fn(&A, usize, &mut std::fmt::Formatter) -> std::fmt::Result,
916{
917 let head = std::cmp::min(10, array.len());
918
919 for i in 0..head {
920 if array.is_null(i) {
921 writeln!(f, " null,")?;
922 } else {
923 write!(f, " ")?;
924 print_item(array, i, f)?;
925 writeln!(f, ",")?;
926 }
927 }
928 if array.len() > 10 {
929 if array.len() > 20 {
930 writeln!(f, " ...{} elements...,", array.len() - 20)?;
931 }
932
933 let tail = std::cmp::max(head, array.len() - 10);
934
935 for i in tail..array.len() {
936 if array.is_null(i) {
937 writeln!(f, " null,")?;
938 } else {
939 write!(f, " ")?;
940 print_item(array, i, f)?;
941 writeln!(f, ",")?;
942 }
943 }
944 }
945 Ok(())
946}
947
948#[cfg(test)]
949mod tests {
950 use super::*;
951 use crate::cast::{as_union_array, downcast_array};
952 use crate::downcast_run_array;
953 use arrow_buffer::MutableBuffer;
954 use arrow_schema::{Field, Fields, UnionFields, UnionMode};
955
956 #[test]
957 fn test_empty_primitive() {
958 let array = new_empty_array(&DataType::Int32);
959 let a = array.as_any().downcast_ref::<Int32Array>().unwrap();
960 assert_eq!(a.len(), 0);
961 let expected: &[i32] = &[];
962 assert_eq!(a.values(), expected);
963 }
964
965 #[test]
966 fn test_empty_variable_sized() {
967 let array = new_empty_array(&DataType::Utf8);
968 let a = array.as_any().downcast_ref::<StringArray>().unwrap();
969 assert_eq!(a.len(), 0);
970 assert_eq!(a.value_offsets()[0], 0i32);
971 }
972
973 #[test]
974 fn test_empty_list_primitive() {
975 let data_type = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, false)));
976 let array = new_empty_array(&data_type);
977 let a = array.as_any().downcast_ref::<ListArray>().unwrap();
978 assert_eq!(a.len(), 0);
979 assert_eq!(a.value_offsets()[0], 0i32);
980 }
981
982 #[test]
983 fn test_null_boolean() {
984 let array = new_null_array(&DataType::Boolean, 9);
985 let a = array.as_any().downcast_ref::<BooleanArray>().unwrap();
986 assert_eq!(a.len(), 9);
987 for i in 0..9 {
988 assert!(a.is_null(i));
989 }
990 }
991
992 #[test]
993 fn test_null_primitive() {
994 let array = new_null_array(&DataType::Int32, 9);
995 let a = array.as_any().downcast_ref::<Int32Array>().unwrap();
996 assert_eq!(a.len(), 9);
997 for i in 0..9 {
998 assert!(a.is_null(i));
999 }
1000 }
1001
1002 #[test]
1003 fn test_null_struct() {
1004 let struct_type = DataType::Struct(vec![Field::new("data", DataType::Int64, false)].into());
1007 let array = new_null_array(&struct_type, 9);
1008
1009 let a = array.as_any().downcast_ref::<StructArray>().unwrap();
1010 assert_eq!(a.len(), 9);
1011 assert_eq!(a.column(0).len(), 9);
1012 for i in 0..9 {
1013 assert!(a.is_null(i));
1014 }
1015
1016 a.slice(0, 5);
1018 }
1019
1020 #[test]
1021 fn test_null_variable_sized() {
1022 let array = new_null_array(&DataType::Utf8, 9);
1023 let a = array.as_any().downcast_ref::<StringArray>().unwrap();
1024 assert_eq!(a.len(), 9);
1025 assert_eq!(a.value_offsets()[9], 0i32);
1026 for i in 0..9 {
1027 assert!(a.is_null(i));
1028 }
1029 }
1030
1031 #[test]
1032 fn test_null_list_primitive() {
1033 let data_type = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
1034 let array = new_null_array(&data_type, 9);
1035 let a = array.as_any().downcast_ref::<ListArray>().unwrap();
1036 assert_eq!(a.len(), 9);
1037 assert_eq!(a.value_offsets()[9], 0i32);
1038 for i in 0..9 {
1039 assert!(a.is_null(i));
1040 }
1041 }
1042
1043 #[test]
1044 fn test_null_map() {
1045 let data_type = DataType::Map(
1046 Arc::new(Field::new(
1047 "entry",
1048 DataType::Struct(Fields::from(vec![
1049 Field::new("key", DataType::Utf8, false),
1050 Field::new("value", DataType::Int32, true),
1051 ])),
1052 false,
1053 )),
1054 false,
1055 );
1056 let array = new_null_array(&data_type, 9);
1057 let a = array.as_any().downcast_ref::<MapArray>().unwrap();
1058 assert_eq!(a.len(), 9);
1059 assert_eq!(a.value_offsets()[9], 0i32);
1060 for i in 0..9 {
1061 assert!(a.is_null(i));
1062 }
1063 }
1064
1065 #[test]
1066 fn test_null_dictionary() {
1067 let values =
1068 vec![None, None, None, None, None, None, None, None, None] as Vec<Option<&str>>;
1069
1070 let array: DictionaryArray<Int8Type> = values.into_iter().collect();
1071 let array = Arc::new(array) as ArrayRef;
1072
1073 let null_array = new_null_array(array.data_type(), 9);
1074 assert_eq!(&array, &null_array);
1075 assert_eq!(
1076 array.to_data().buffers()[0].len(),
1077 null_array.to_data().buffers()[0].len()
1078 );
1079 }
1080
1081 #[test]
1082 fn test_null_union() {
1083 for mode in [UnionMode::Sparse, UnionMode::Dense] {
1084 let data_type = DataType::Union(
1085 UnionFields::try_new(
1086 vec![2, 1],
1087 vec![
1088 Field::new("foo", DataType::Int32, true),
1089 Field::new("bar", DataType::Int64, true),
1090 ],
1091 )
1092 .unwrap(),
1093 mode,
1094 );
1095 let array = new_null_array(&data_type, 4);
1096
1097 let array = as_union_array(array.as_ref());
1098 assert_eq!(array.len(), 4);
1099 assert_eq!(array.null_count(), 0);
1100 assert_eq!(array.logical_null_count(), 4);
1101
1102 for i in 0..4 {
1103 let a = array.value(i);
1104 assert_eq!(a.len(), 1);
1105 assert_eq!(a.null_count(), 1);
1106 assert_eq!(a.logical_null_count(), 1);
1107 assert!(a.is_null(0))
1108 }
1109
1110 array.to_data().validate_full().unwrap();
1111 }
1112 }
1113
1114 #[test]
1115 #[allow(unused_parens)]
1116 fn test_null_runs() {
1117 for r in [DataType::Int16, DataType::Int32, DataType::Int64] {
1118 let data_type = DataType::RunEndEncoded(
1119 Arc::new(Field::new("run_ends", r, false)),
1120 Arc::new(Field::new("values", DataType::Utf8, true)),
1121 );
1122
1123 let array = new_null_array(&data_type, 4);
1124 let array = array.as_ref();
1125
1126 downcast_run_array! {
1127 array => {
1128 assert_eq!(array.len(), 4);
1129 assert_eq!(array.null_count(), 0);
1130 assert_eq!(array.logical_null_count(), 4);
1131 assert_eq!(array.values().len(), 1);
1132 assert_eq!(array.values().null_count(), 1);
1133 assert_eq!(array.run_ends().len(), 4);
1134 assert_eq!(array.run_ends().values(), &[4]);
1135
1136 let idx = array.get_physical_indices(&[0, 1, 2, 3]).unwrap();
1137 assert_eq!(idx, &[0,0,0,0]);
1138 }
1139 d => unreachable!("{d}")
1140 }
1141 }
1142 }
1143
1144 #[test]
1145 fn test_null_fixed_size_binary() {
1146 for size in [1, 2, 7] {
1147 let array = new_null_array(&DataType::FixedSizeBinary(size), 6);
1148 let array = array
1149 .as_ref()
1150 .as_any()
1151 .downcast_ref::<FixedSizeBinaryArray>()
1152 .unwrap();
1153
1154 assert_eq!(array.len(), 6);
1155 assert_eq!(array.null_count(), 6);
1156 assert_eq!(array.logical_null_count(), 6);
1157 array.iter().for_each(|x| assert!(x.is_none()));
1158 }
1159 }
1160
1161 #[test]
1162 fn test_memory_size_null() {
1163 let null_arr = NullArray::new(32);
1164
1165 assert_eq!(0, null_arr.get_buffer_memory_size());
1166 assert_eq!(
1167 std::mem::size_of::<usize>(),
1168 null_arr.get_array_memory_size()
1169 );
1170 }
1171
1172 #[test]
1173 fn test_memory_size_primitive() {
1174 let arr = PrimitiveArray::<Int64Type>::from_iter_values(0..128);
1175 let empty = PrimitiveArray::<Int64Type>::from(ArrayData::new_empty(arr.data_type()));
1176
1177 assert_eq!(
1179 arr.get_array_memory_size() - empty.get_array_memory_size(),
1180 128 * std::mem::size_of::<i64>()
1181 );
1182 }
1183
1184 #[test]
1185 fn test_memory_size_primitive_sliced() {
1186 let arr = PrimitiveArray::<Int64Type>::from_iter_values(0..128);
1187 let slice1 = arr.slice(0, 64);
1188 let slice2 = arr.slice(64, 64);
1189
1190 assert_eq!(slice1.get_array_memory_size(), arr.get_array_memory_size());
1192 assert_eq!(slice2.get_array_memory_size(), arr.get_array_memory_size());
1193 }
1194
1195 #[test]
1196 fn test_memory_size_primitive_nullable() {
1197 let arr: PrimitiveArray<Int64Type> = (0..128)
1198 .map(|i| if i % 20 == 0 { Some(i) } else { None })
1199 .collect();
1200 let empty_with_bitmap = PrimitiveArray::<Int64Type>::from(
1201 ArrayData::builder(arr.data_type().clone())
1202 .add_buffer(MutableBuffer::new(0).into())
1203 .null_bit_buffer(Some(MutableBuffer::new_null(0).into()))
1204 .build()
1205 .unwrap(),
1206 );
1207
1208 assert_eq!(
1212 std::mem::size_of::<PrimitiveArray<Int64Type>>(),
1213 empty_with_bitmap.get_array_memory_size()
1214 );
1215
1216 assert_eq!(
1219 arr.get_array_memory_size() - empty_with_bitmap.get_array_memory_size(),
1220 128 * std::mem::size_of::<i64>() + 64
1221 );
1222 }
1223
1224 #[test]
1225 fn test_memory_size_dictionary() {
1226 let values = PrimitiveArray::<Int64Type>::from_iter_values(0..16);
1227 let keys = PrimitiveArray::<Int16Type>::from_iter_values(
1228 (0..256).map(|i| (i % values.len()) as i16),
1229 );
1230
1231 let dict_data_type = DataType::Dictionary(
1232 Box::new(keys.data_type().clone()),
1233 Box::new(values.data_type().clone()),
1234 );
1235 let dict_data = keys
1236 .into_data()
1237 .into_builder()
1238 .data_type(dict_data_type)
1239 .child_data(vec![values.into_data()])
1240 .build()
1241 .unwrap();
1242
1243 let empty_data = ArrayData::new_empty(&DataType::Dictionary(
1244 Box::new(DataType::Int16),
1245 Box::new(DataType::Int64),
1246 ));
1247
1248 let arr = DictionaryArray::<Int16Type>::from(dict_data);
1249 let empty = DictionaryArray::<Int16Type>::from(empty_data);
1250
1251 let expected_keys_size = 256 * std::mem::size_of::<i16>();
1252 assert_eq!(
1253 arr.keys().get_array_memory_size() - empty.keys().get_array_memory_size(),
1254 expected_keys_size
1255 );
1256
1257 let expected_values_size = 16 * std::mem::size_of::<i64>();
1258 assert_eq!(
1259 arr.values().get_array_memory_size() - empty.values().get_array_memory_size(),
1260 expected_values_size
1261 );
1262
1263 let expected_size = expected_keys_size + expected_values_size;
1264 assert_eq!(
1265 arr.get_array_memory_size() - empty.get_array_memory_size(),
1266 expected_size
1267 );
1268 }
1269
1270 fn compute_my_thing(arr: &dyn Array) -> bool {
1272 !arr.is_empty()
1273 }
1274
1275 #[test]
1276 fn test_array_ref_as_array() {
1277 let arr: Int32Array = vec![1, 2, 3].into_iter().map(Some).collect();
1278
1279 assert!(compute_my_thing(&arr));
1281
1282 let arr: ArrayRef = Arc::new(arr);
1284 assert!(compute_my_thing(&arr));
1285 assert!(compute_my_thing(arr.as_ref()));
1286 }
1287
1288 #[test]
1289 fn test_downcast_array() {
1290 let array: Int32Array = vec![1, 2, 3].into_iter().map(Some).collect();
1291
1292 let boxed: ArrayRef = Arc::new(array);
1293 let array: Int32Array = downcast_array(&boxed);
1294
1295 let expected: Int32Array = vec![1, 2, 3].into_iter().map(Some).collect();
1296 assert_eq!(array, expected);
1297 }
1298}