1pub mod binary;
55pub mod boolean;
56pub mod fixed_size_binary;
57pub mod fixed_size_list;
58pub mod iter;
59pub mod list;
60pub mod null;
61pub mod offset;
62pub mod primitive;
63pub mod string;
64pub mod structs;
65pub mod temporal;
66
67use std::any::Any;
68use std::sync::Arc;
69
70pub use crate::array::binary::{BinaryArray, GenericBinaryArray, LargeBinaryArray};
71pub use crate::array::boolean::BooleanArray;
72pub use crate::array::fixed_size_binary::FixedSizeBinaryArray;
73pub use crate::array::fixed_size_list::FixedSizeListArray;
74pub use crate::array::iter::{ArrayAccessor, ArrayIter};
75pub use crate::array::list::ListArray;
76pub use crate::array::null::NullArray;
77pub use crate::array::offset::{OffsetSizeTrait, validate_offsets};
78pub use crate::array::primitive::{
79 Float16Array, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array,
80 PrimitiveArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array,
81};
82pub use crate::array::string::{GenericStringArray, LargeStringArray, StringArray};
83pub use crate::array::structs::StructArray;
84pub use crate::array::temporal::{DurationArray, TimestampArray};
85
86use crate::buffer::Bitmap;
87use crate::datatype::{DataType, Field};
88use crate::error::{DataError, Result};
89use crate::sealed::Sealed;
90
91pub type ArrayRef = Arc<dyn Array>;
96
97pub trait Array: std::fmt::Debug + Send + Sync + Sealed + 'static {
101 fn as_any(&self) -> &dyn Any;
103
104 fn data_type(&self) -> &DataType;
106
107 fn len(&self) -> usize;
109
110 #[inline]
112 fn is_empty(&self) -> bool {
113 self.len() == 0
114 }
115
116 fn validity(&self) -> Option<&Bitmap>;
121
122 #[inline]
124 fn null_count(&self) -> usize {
125 self.validity().map_or(0, Bitmap::count_unset)
126 }
127
128 #[inline]
131 fn is_null(&self, index: usize) -> bool {
132 !self.is_valid(index)
133 }
134
135 #[inline]
137 fn is_valid(&self, index: usize) -> bool {
138 index < self.len() && self.validity().is_none_or(|bits| bits.value(index))
139 }
140
141 fn slice(&self, offset: usize, len: usize) -> ArrayRef;
157
158 fn buffer_memory_size(&self) -> usize;
164
165 fn children(&self) -> Vec<&ArrayRef> {
167 Vec::new()
168 }
169
170 fn equals(&self, other: &dyn Array) -> bool;
176}
177
178impl PartialEq for dyn Array {
179 #[inline]
180 fn eq(&self, other: &Self) -> bool {
181 self.equals(other)
182 }
183}
184
185impl PartialEq<dyn Array> for ArrayRef {
186 #[inline]
187 fn eq(&self, other: &dyn Array) -> bool {
188 self.as_ref().equals(other)
189 }
190}
191
192pub trait ArrayExt {
203 fn downcast<A: Array>(&self) -> Option<&A>;
205
206 fn try_downcast<A: Array>(&self) -> Result<&A>;
213}
214
215impl ArrayExt for dyn Array {
216 #[inline]
217 fn downcast<A: Array>(&self) -> Option<&A> {
218 self.as_any().downcast_ref::<A>()
219 }
220
221 fn try_downcast<A: Array>(&self) -> Result<&A> {
222 self.downcast::<A>()
223 .ok_or_else(|| DataError::downcast_failed::<A>(self.data_type().clone()))
224 }
225}
226
227impl ArrayExt for ArrayRef {
228 #[inline]
229 fn downcast<A: Array>(&self) -> Option<&A> {
230 self.as_ref().downcast::<A>()
231 }
232
233 #[inline]
234 fn try_downcast<A: Array>(&self) -> Result<&A> {
235 self.as_ref().try_downcast::<A>()
236 }
237}
238
239pub trait IntoArrayRef: Array + Sized {
248 fn into_array_ref(self) -> ArrayRef {
250 Arc::new(self)
251 }
252}
253
254impl<A: Array + Sized> IntoArrayRef for A {}
255
256pub fn try_slice(array: &ArrayRef, offset: usize, len: usize) -> Result<ArrayRef> {
271 if offset.saturating_add(len) > array.len() {
272 return Err(DataError::SliceOutOfBounds {
273 offset,
274 len,
275 available: array.len(),
276 });
277 }
278 Ok(slice_array(array, offset, len))
279}
280
281#[must_use]
293pub fn slice_array(array: &ArrayRef, offset: usize, len: usize) -> ArrayRef {
294 if offset == 0 && len >= array.len() {
295 return Arc::clone(array);
296 }
297 array.slice(offset, len)
298}
299
300pub fn new_empty_array(data_type: &DataType) -> Result<ArrayRef> {
315 new_null_array(data_type, 0)
316}
317
318pub fn new_null_array(data_type: &DataType, len: usize) -> Result<ArrayRef> {
339 let validity = Bitmap::new_unset(len);
340 Ok(match data_type {
341 DataType::Null => Arc::new(NullArray::new(len)),
342 DataType::Bool => Arc::new(BooleanArray::new(Bitmap::new_unset(len), Some(validity))?),
343 DataType::Int8 => Arc::new(Int8Array::new_null(len)),
344 DataType::Int16 => Arc::new(Int16Array::new_null(len)),
345 DataType::Int32 => Arc::new(Int32Array::new_null(len)),
346 DataType::Int64 => Arc::new(Int64Array::new_null(len)),
347 DataType::UInt8 => Arc::new(UInt8Array::new_null(len)),
348 DataType::UInt16 => Arc::new(UInt16Array::new_null(len)),
349 DataType::UInt32 => Arc::new(UInt32Array::new_null(len)),
350 DataType::UInt64 => Arc::new(UInt64Array::new_null(len)),
351 DataType::Float16 => Arc::new(Float16Array::new_null(len)),
352 DataType::Float32 => Arc::new(Float32Array::new_null(len)),
353 DataType::Float64 => Arc::new(Float64Array::new_null(len)),
354 DataType::Timestamp => Arc::new(TimestampArray::new_null(len)),
355 DataType::Duration => Arc::new(DurationArray::new_null(len)),
356 DataType::Binary => Arc::new(BinaryArray::new_null(len)),
357 DataType::LargeBinary => Arc::new(LargeBinaryArray::new_null(len)),
358 DataType::Utf8 => Arc::new(StringArray::new_null(len)),
359 DataType::LargeUtf8 => Arc::new(LargeStringArray::new_null(len)),
360 DataType::FixedSizeBinary(size) => Arc::new(FixedSizeBinaryArray::new_null(*size, len)?),
361 DataType::FixedSizeList(field, size) => Arc::new(FixedSizeListArray::new_null(
362 field.as_ref().clone(),
363 *size,
364 len,
365 )?),
366 DataType::List(field) => Arc::new(ListArray::new_null(field.as_ref().clone(), len)),
367 DataType::Struct(fields) => Arc::new(StructArray::new_null(fields.clone(), len)?),
368 })
369}
370
371#[must_use]
383pub fn array_kinds() -> &'static [&'static str] {
384 &[
385 "NullArray",
386 "BooleanArray",
387 "PrimitiveArray",
388 "BinaryArray",
389 "LargeBinaryArray",
390 "StringArray",
391 "LargeStringArray",
392 "FixedSizeBinaryArray",
393 "FixedSizeListArray",
394 "ListArray",
395 "StructArray",
396 "TimestampArray",
397 "DurationArray",
398 ]
399}
400
401pub(crate) static NULL_TYPE: DataType = DataType::Null;
410pub(crate) static BOOL_TYPE: DataType = DataType::Bool;
412pub(crate) static BINARY_TYPE: DataType = DataType::Binary;
414pub(crate) static LARGE_BINARY_TYPE: DataType = DataType::LargeBinary;
416pub(crate) static UTF8_TYPE: DataType = DataType::Utf8;
418pub(crate) static LARGE_UTF8_TYPE: DataType = DataType::LargeUtf8;
420pub(crate) static TIMESTAMP_TYPE: DataType = DataType::Timestamp;
422pub(crate) static DURATION_TYPE: DataType = DataType::Duration;
424
425pub(crate) fn check_validity(validity: Option<&Bitmap>, len: usize) -> Result<()> {
434 match validity {
435 Some(bits) if bits.len() != len => Err(DataError::ValidityLengthMismatch {
436 array_len: len,
437 validity_len: bits.len(),
438 }),
439 _ => Ok(()),
440 }
441}
442
443#[inline]
445pub(crate) fn clamp_window(array_len: usize, offset: usize, len: usize) -> (usize, usize) {
446 let offset = offset.min(array_len);
447 (offset, len.min(array_len - offset))
448}
449
450#[inline]
452pub(crate) fn validity_eq(left: Option<&Bitmap>, right: Option<&Bitmap>, len: usize) -> bool {
453 match (left, right) {
454 (None, None) => true,
455 (Some(bits), None) | (None, Some(bits)) => bits.count_set() == len,
456 (Some(a), Some(b)) => a == b,
457 }
458}
459
460pub(crate) fn check_fixed_size(size: i32) -> Result<usize> {
462 if size <= 0 {
463 return Err(DataError::InvalidFixedSize { size });
464 }
465 usize::try_from(size).map_err(|_| DataError::InvalidFixedSize { size })
466}
467
468pub(crate) fn debug_array<T: std::fmt::Debug>(
471 f: &mut std::fmt::Formatter<'_>,
472 name: &str,
473 data_type: &DataType,
474 len: usize,
475 null_count: usize,
476 values: impl Iterator<Item = Option<T>>,
477) -> std::fmt::Result {
478 const PREVIEW: usize = 10;
479 let mut rendered = Vec::with_capacity(PREVIEW);
480 for value in values.take(PREVIEW) {
481 rendered.push(match value {
482 Some(value) => format!("{value:?}"),
483 None => "null".to_owned(),
484 });
485 }
486 if len > PREVIEW {
487 rendered.push("…".to_owned());
488 }
489 write!(
490 f,
491 "{name}[{data_type}; len={len}, nulls={null_count}] {{{}}}",
492 rendered.join(", ")
493 )
494}
495
496pub(crate) fn child_field(data_type: &DataType) -> Option<&Field> {
498 match data_type {
499 DataType::List(field) | DataType::FixedSizeList(field, _) => Some(field.as_ref()),
500 _ => None,
501 }
502}
503
504#[cfg(test)]
505mod tests {
506 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
507
508 use super::*;
509
510 fn every_data_type() -> Vec<DataType> {
511 vec![
512 DataType::Null,
513 DataType::Bool,
514 DataType::Int8,
515 DataType::Int16,
516 DataType::Int32,
517 DataType::Int64,
518 DataType::UInt8,
519 DataType::UInt16,
520 DataType::UInt32,
521 DataType::UInt64,
522 DataType::Float16,
523 DataType::Float32,
524 DataType::Float64,
525 DataType::Binary,
526 DataType::LargeBinary,
527 DataType::Utf8,
528 DataType::LargeUtf8,
529 DataType::FixedSizeBinary(4),
530 DataType::fixed_size_list(Field::new("xyz", DataType::Float32, false), 3),
531 DataType::list(Field::new("item", DataType::Utf8, true)),
532 DataType::strukt([Field::new("a", DataType::Int32, true)]),
533 DataType::Timestamp,
534 DataType::Duration,
535 ]
536 }
537
538 #[test]
539 fn arrays_are_send_and_sync() {
540 const fn assert_send_sync<T: Send + Sync + ?Sized>() {}
541 assert_send_sync::<dyn Array>();
542 assert_send_sync::<ArrayRef>();
543 }
544
545 #[test]
546 fn null_arrays_exist_for_every_data_type() {
547 for data_type in every_data_type() {
548 let array =
549 new_null_array(&data_type, 5).unwrap_or_else(|e| panic!("{data_type}: {e}"));
550 assert_eq!(array.len(), 5, "{data_type}");
551 assert_eq!(array.null_count(), 5, "{data_type}");
552 assert_eq!(array.data_type(), &data_type, "{data_type}");
553 for i in 0..5 {
554 assert!(array.is_null(i), "{data_type} index {i}");
555 assert!(!array.is_valid(i), "{data_type} index {i}");
556 }
557 assert!(array.is_null(99), "out of range reads as null");
558 }
559 }
560
561 #[test]
562 fn empty_arrays_exist_for_every_data_type() {
563 for data_type in every_data_type() {
564 let array = new_empty_array(&data_type).unwrap();
565 assert!(array.is_empty(), "{data_type}");
566 assert_eq!(array.null_count(), 0, "{data_type}");
567 assert_eq!(Array::slice(array.as_ref(), 0, 0).len(), 0);
568 }
569 }
570
571 #[test]
572 fn null_arrays_reject_invalid_fixed_widths() {
573 assert!(new_null_array(&DataType::FixedSizeBinary(0), 1).is_err());
574 assert!(new_null_array(&DataType::FixedSizeBinary(-3), 1).is_err());
575 assert!(
576 new_null_array(
577 &DataType::fixed_size_list(Field::new("x", DataType::Int8, true), 0),
578 1
579 )
580 .is_err()
581 );
582 }
583
584 #[test]
585 fn kinds_cover_every_family() {
586 assert_eq!(array_kinds().len(), 13);
587 let mut sorted = array_kinds().to_vec();
588 sorted.sort_unstable();
589 sorted.dedup();
590 assert_eq!(sorted.len(), array_kinds().len(), "no duplicates");
591 }
592
593 #[test]
594 fn downcasting() {
595 let column: ArrayRef = Int32Array::from_values([1, 2]).into_array_ref();
596 assert!(column.downcast::<Int32Array>().is_some());
597 assert!(column.downcast::<StringArray>().is_none());
598 assert_eq!(column.try_downcast::<Int32Array>().unwrap().len(), 2);
599 let err = column.try_downcast::<BooleanArray>().unwrap_err();
600 assert!(matches!(err, DataError::DowncastFailed { .. }));
601 assert!(column.as_ref().downcast::<Int32Array>().is_some());
602 }
603
604 #[test]
605 fn slice_array_short_circuits_full_ranges() {
606 let column: ArrayRef = Int32Array::from_values([1, 2, 3]).into_array_ref();
607 let whole = slice_array(&column, 0, 3);
608 assert!(Arc::ptr_eq(&column, &whole));
609 let over = slice_array(&column, 0, 99);
610 assert!(Arc::ptr_eq(&column, &over));
611 let part = slice_array(&column, 1, 2);
612 assert!(!Arc::ptr_eq(&column, &part));
613 assert_eq!(part.len(), 2);
614 }
615
616 #[test]
617 fn try_slice_reports_out_of_range() {
618 let column: ArrayRef = Int32Array::from_values([1, 2, 3]).into_array_ref();
619 assert_eq!(try_slice(&column, 1, 2).unwrap().len(), 2);
620 assert_eq!(
621 try_slice(&column, 2, 2).unwrap_err(),
622 DataError::SliceOutOfBounds {
623 offset: 2,
624 len: 2,
625 available: 3
626 }
627 );
628 assert!(try_slice(&column, usize::MAX, 1).is_err());
629 }
630
631 #[test]
632 fn equality_across_trait_objects() {
633 let a: ArrayRef = Int32Array::from_values([1, 2, 3]).into_array_ref();
634 let b: ArrayRef = Int32Array::from_values([1, 2, 3]).into_array_ref();
635 let c: ArrayRef = Int32Array::from_values([1, 2]).into_array_ref();
636 let d: ArrayRef = Int64Array::from_values([1, 2, 3]).into_array_ref();
637 assert_eq!(a, b);
638 assert_ne!(a, c);
639 assert_ne!(a, d, "different data types are never equal");
640 assert_eq!(vec![Arc::clone(&a)], vec![b]);
641 }
642
643 #[test]
644 fn sliced_arrays_equal_freshly_built_ones() {
645 let full: ArrayRef = Int32Array::from_values([9, 1, 2, 3, 9]).into_array_ref();
646 let window = slice_array(&full, 1, 3);
647 let fresh: ArrayRef = Int32Array::from_values([1, 2, 3]).into_array_ref();
648 assert_eq!(window, fresh);
649 }
650
651 #[test]
652 fn helper_predicates() {
653 assert!(check_validity(None, 3).is_ok());
654 let bits = Bitmap::new_set(3);
655 assert!(check_validity(Some(&bits), 3).is_ok());
656 assert_eq!(
657 check_validity(Some(&bits), 4).unwrap_err(),
658 DataError::ValidityLengthMismatch {
659 array_len: 4,
660 validity_len: 3
661 }
662 );
663 assert_eq!(clamp_window(5, 3, 99), (3, 2));
664 assert_eq!(clamp_window(5, 99, 1), (5, 0));
665 assert_eq!(check_fixed_size(4).unwrap(), 4);
666 assert!(check_fixed_size(0).is_err());
667 assert!(check_fixed_size(-1).is_err());
668 assert!(validity_eq(None, Some(&Bitmap::new_set(3)), 3));
669 assert!(!validity_eq(None, Some(&Bitmap::new_unset(3)), 3));
670 assert!(child_field(&DataType::Int8).is_none());
671 assert!(child_field(&DataType::list(Field::new("i", DataType::Int8, true))).is_some());
672 }
673}