1use std::any::Any;
26use std::fmt;
27use std::sync::Arc;
28
29use crate::array::iter::{ArrayAccessor, ArrayIter};
30use crate::array::{Array, ArrayRef, check_validity, clamp_window, debug_array, validity_eq};
31use crate::buffer::{Bitmap, ScalarBuffer};
32use crate::datatype::{ArrowNativeType, DataType, F16};
33use crate::error::{DataError, Result};
34use crate::sealed::Sealed;
35
36#[derive(Clone)]
40pub struct PrimitiveArray<T: ArrowNativeType> {
41 data_type: DataType,
44 values: ScalarBuffer<T>,
46 validity: Option<Bitmap>,
48}
49
50pub type Int8Array = PrimitiveArray<i8>;
52pub type Int16Array = PrimitiveArray<i16>;
54pub type Int32Array = PrimitiveArray<i32>;
56pub type Int64Array = PrimitiveArray<i64>;
58pub type UInt8Array = PrimitiveArray<u8>;
60pub type UInt16Array = PrimitiveArray<u16>;
62pub type UInt32Array = PrimitiveArray<u32>;
64pub type UInt64Array = PrimitiveArray<u64>;
66pub type Float16Array = PrimitiveArray<F16>;
68pub type Float32Array = PrimitiveArray<f32>;
70pub type Float64Array = PrimitiveArray<f64>;
72
73impl<T: ArrowNativeType> PrimitiveArray<T> {
74 pub fn try_new(values: ScalarBuffer<T>, validity: Option<Bitmap>) -> Result<Self> {
92 Self::try_new_with_type(T::DATA_TYPE, values, validity)
93 }
94
95 pub fn try_new_with_type(
107 data_type: DataType,
108 values: ScalarBuffer<T>,
109 validity: Option<Bitmap>,
110 ) -> Result<Self> {
111 if data_type.primitive_width() != Some(T::WIDTH) || !data_type.is_primitive() {
112 return Err(DataError::type_mismatch(T::DATA_TYPE, data_type));
113 }
114 check_validity(validity.as_ref(), values.len())?;
115 Ok(Self {
116 data_type,
117 values,
118 validity,
119 })
120 }
121
122 #[must_use]
132 pub fn from_values(values: impl IntoIterator<Item = T>) -> Self {
133 let values: Vec<T> = values.into_iter().collect();
134 Self {
135 data_type: T::DATA_TYPE,
136 values: ScalarBuffer::from_slice(&values),
137 validity: None,
138 }
139 }
140
141 #[must_use]
143 pub fn from_opt_iter(values: impl IntoIterator<Item = Option<T>>) -> Self {
144 let iter = values.into_iter();
145 let (lower, _) = iter.size_hint();
146 let mut slots = Vec::with_capacity(lower);
147 let mut validity = crate::buffer::BitmapBuilder::with_capacity(lower);
148 let mut any_null = false;
149 for value in iter {
150 match value {
151 Some(value) => {
152 slots.push(value);
153 validity.append(true);
154 }
155 None => {
156 slots.push(T::default());
157 validity.append(false);
158 any_null = true;
159 }
160 }
161 }
162 Self {
163 data_type: T::DATA_TYPE,
164 values: ScalarBuffer::from_slice(&slots),
165 validity: any_null.then(|| validity.finish()),
166 }
167 }
168
169 #[must_use]
171 pub fn new_null(len: usize) -> Self {
172 Self {
173 data_type: T::DATA_TYPE,
174 values: ScalarBuffer::zeroed(len),
175 validity: Some(Bitmap::new_unset(len)),
176 }
177 }
178
179 pub(crate) fn from_parts(
185 data_type: DataType,
186 values: ScalarBuffer<T>,
187 validity: Option<Bitmap>,
188 ) -> Self {
189 debug_assert_eq!(data_type.primitive_width(), Some(T::WIDTH));
190 debug_assert!(
191 validity
192 .as_ref()
193 .is_none_or(|bits| bits.len() == values.len())
194 );
195 Self {
196 data_type,
197 values,
198 validity,
199 }
200 }
201
202 #[inline]
208 #[must_use]
209 pub fn values(&self) -> &[T] {
210 self.values.as_slice()
211 }
212
213 #[inline]
215 #[must_use]
216 pub const fn values_buffer(&self) -> &ScalarBuffer<T> {
217 &self.values
218 }
219
220 #[inline]
224 #[must_use]
225 pub fn value(&self, index: usize) -> Option<T> {
226 self.values.get(index)
227 }
228
229 #[inline]
232 #[must_use]
233 pub fn get(&self, index: usize) -> Option<T> {
234 if self.is_valid_index(index) {
235 self.values.get(index)
236 } else {
237 None
238 }
239 }
240
241 #[inline]
243 pub fn iter(&self) -> ArrayIter<&Self> {
244 ArrayIter::new(self)
245 }
246
247 pub fn values_iter(&self) -> impl Iterator<Item = T> + '_ {
249 self.iter().flatten()
250 }
251
252 #[must_use]
255 pub fn slice(&self, offset: usize, len: usize) -> Self {
256 let (offset, len) = clamp_window(self.len(), offset, len);
257 Self {
258 data_type: self.data_type.clone(),
259 values: self.values.slice(offset, len),
260 validity: self.validity.as_ref().map(|bits| bits.slice(offset, len)),
261 }
262 }
263
264 pub fn try_slice(&self, offset: usize, len: usize) -> Result<Self> {
270 if offset.saturating_add(len) > self.len() {
271 return Err(DataError::SliceOutOfBounds {
272 offset,
273 len,
274 available: self.len(),
275 });
276 }
277 Ok(self.slice(offset, len))
278 }
279
280 pub fn with_validity(&self, validity: Option<Bitmap>) -> Result<Self> {
286 check_validity(validity.as_ref(), self.len())?;
287 Ok(Self {
288 data_type: self.data_type.clone(),
289 values: self.values.clone(),
290 validity,
291 })
292 }
293
294 #[inline]
296 fn is_valid_index(&self, index: usize) -> bool {
297 index < self.values.len() && self.validity.as_ref().is_none_or(|bits| bits.value(index))
298 }
299}
300
301impl<T: ArrowNativeType> Sealed for PrimitiveArray<T> {}
302
303impl<T: ArrowNativeType> Array for PrimitiveArray<T> {
304 fn as_any(&self) -> &dyn Any {
305 self
306 }
307
308 fn data_type(&self) -> &DataType {
309 &self.data_type
310 }
311
312 fn len(&self) -> usize {
313 self.values.len()
314 }
315
316 fn validity(&self) -> Option<&Bitmap> {
317 self.validity.as_ref()
318 }
319
320 fn slice(&self, offset: usize, len: usize) -> ArrayRef {
321 Arc::new(Self::slice(self, offset, len))
322 }
323
324 fn buffer_memory_size(&self) -> usize {
325 self.values.inner().backing_len()
326 + self
327 .validity
328 .as_ref()
329 .map_or(0, |bits| bits.buffer().backing_len())
330 }
331
332 fn equals(&self, other: &dyn Array) -> bool {
333 let Some(other) = other.as_any().downcast_ref::<Self>() else {
334 return false;
335 };
336 if self.data_type != other.data_type || self.len() != other.len() {
337 return false;
338 }
339 if !validity_eq(self.validity.as_ref(), other.validity.as_ref(), self.len()) {
340 return false;
341 }
342 (0..self.len()).all(|index| match (self.get(index), other.get(index)) {
344 (Some(a), Some(b)) => a.value_eq(&b),
345 (None, None) => true,
346 _ => false,
347 })
348 }
349}
350
351impl<T: ArrowNativeType> ArrayAccessor for &PrimitiveArray<T> {
352 type Item = T;
353
354 #[inline]
355 fn accessor_len(&self) -> usize {
356 self.values.len()
357 }
358
359 #[inline]
360 fn accessor_get(&self, index: usize) -> Option<T> {
361 PrimitiveArray::get(self, index)
362 }
363}
364
365impl<T: ArrowNativeType> PartialEq for PrimitiveArray<T> {
366 #[inline]
367 fn eq(&self, other: &Self) -> bool {
368 self.equals(other)
369 }
370}
371
372impl<T: ArrowNativeType> FromIterator<T> for PrimitiveArray<T> {
373 fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
374 Self::from_values(iter)
375 }
376}
377
378impl<T: ArrowNativeType> FromIterator<Option<T>> for PrimitiveArray<T> {
379 fn from_iter<I: IntoIterator<Item = Option<T>>>(iter: I) -> Self {
380 Self::from_opt_iter(iter)
381 }
382}
383
384impl<T: ArrowNativeType, const N: usize> From<[T; N]> for PrimitiveArray<T> {
385 fn from(values: [T; N]) -> Self {
386 Self::from_values(values)
387 }
388}
389
390impl<T: ArrowNativeType> From<Vec<T>> for PrimitiveArray<T> {
391 fn from(values: Vec<T>) -> Self {
392 Self::from_values(values)
393 }
394}
395
396impl<'a, T: ArrowNativeType> IntoIterator for &'a PrimitiveArray<T> {
397 type Item = Option<T>;
398 type IntoIter = ArrayIter<&'a PrimitiveArray<T>>;
399
400 fn into_iter(self) -> Self::IntoIter {
401 self.iter()
402 }
403}
404
405impl<T: ArrowNativeType> fmt::Debug for PrimitiveArray<T> {
406 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
407 debug_array(
408 f,
409 "PrimitiveArray",
410 &self.data_type,
411 self.len(),
412 self.null_count(),
413 self.iter(),
414 )
415 }
416}
417
418#[cfg(test)]
419mod tests {
420 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
421
422 use super::*;
423 use crate::ALIGNMENT;
424
425 #[test]
426 fn empty_array() {
427 let array = Int32Array::from_values([] as [i32; 0]);
428 assert!(array.is_empty());
429 assert_eq!(array.len(), 0);
430 assert_eq!(array.null_count(), 0);
431 assert_eq!(array.values(), &[] as &[i32]);
432 assert_eq!(array.get(0), None);
433 assert_eq!(array.value(0), None);
434 assert!(array.is_null(0), "out of range reads as null");
435 assert_eq!(array.iter().count(), 0);
436 assert_eq!(array.data_type(), &DataType::Int32);
437 }
438
439 #[test]
440 fn no_null_array() {
441 let array = Int64Array::from_values([10, 20, 30]);
442 assert_eq!(array.len(), 3);
443 assert_eq!(array.null_count(), 0);
444 assert!(array.validity().is_none());
445 assert_eq!(array.values(), &[10, 20, 30]);
446 for index in 0..3 {
447 assert!(array.is_valid(index));
448 assert_eq!(array.get(index), Some((index as i64 + 1) * 10));
449 }
450 assert_eq!(array.get(3), None);
451 assert_eq!(
452 array.iter().collect::<Vec<_>>(),
453 vec![Some(10), Some(20), Some(30)]
454 );
455 }
456
457 #[test]
458 fn all_null_array() {
459 let array = UInt16Array::new_null(4);
460 assert_eq!(array.len(), 4);
461 assert_eq!(array.null_count(), 4);
462 assert_eq!(array.values(), &[0, 0, 0, 0]);
463 for index in 0..4 {
464 assert!(array.is_null(index));
465 assert_eq!(array.get(index), None);
466 assert_eq!(array.value(index), Some(0), "raw slot is still readable");
467 }
468 assert_eq!(array.iter().flatten().count(), 0);
469 }
470
471 #[test]
472 fn mixed_nulls() {
473 let array = Float64Array::from_opt_iter([Some(1.5), None, Some(-2.0), None]);
474 assert_eq!(array.len(), 4);
475 assert_eq!(array.null_count(), 2);
476 assert_eq!(array.get(0), Some(1.5));
477 assert_eq!(array.get(1), None);
478 assert_eq!(array.value(1), Some(0.0), "null slot holds the default");
479 assert_eq!(array.values_iter().collect::<Vec<_>>(), vec![1.5, -2.0]);
480 }
481
482 #[test]
483 fn from_opt_iter_without_nulls_skips_the_bitmap() {
484 let array = Int8Array::from_opt_iter([Some(1), Some(2)]);
485 assert!(
486 array.validity().is_none(),
487 "no nulls means no validity buffer"
488 );
489 assert_eq!(array.null_count(), 0);
490 }
491
492 #[test]
493 fn slicing_is_zero_copy_and_clamping() {
494 let array = Int32Array::from_opt_iter((0..20).map(|i| (i % 3 != 0).then_some(i)));
495 let window = array.slice(5, 10);
496 assert_eq!(window.len(), 10);
497 for index in 0..10 {
498 assert_eq!(window.get(index), array.get(index + 5));
499 }
500 assert_eq!(array.slice(18, 99).len(), 2);
501 assert_eq!(array.slice(99, 5).len(), 0);
502 assert_eq!(array.slice(0, 0).len(), 0);
503
504 assert_eq!(
506 window.values_buffer().inner().backing_len(),
507 array.values_buffer().inner().backing_len()
508 );
509 }
510
511 #[test]
512 fn nested_slices_compose() {
513 let array = Int32Array::from_values(0..32);
514 let a = array.slice(4, 20);
515 let b = a.slice(6, 5);
516 assert_eq!(b.values(), &[10, 11, 12, 13, 14]);
517 }
518
519 #[test]
520 fn try_slice_reports_out_of_range() {
521 let array = Int32Array::from_values([1, 2, 3]);
522 assert_eq!(array.try_slice(1, 2).unwrap().len(), 2);
523 assert_eq!(
524 array.try_slice(2, 2).unwrap_err(),
525 DataError::SliceOutOfBounds {
526 offset: 2,
527 len: 2,
528 available: 3
529 }
530 );
531 }
532
533 #[test]
534 fn values_are_64_byte_aligned() {
535 for len in [1usize, 3, 16, 100, 1000] {
536 let array = Int64Array::from_values((0..len).map(|i| i as i64));
537 assert_eq!(
538 array.values().as_ptr() as usize % ALIGNMENT,
539 0,
540 "len {len} lost the alignment"
541 );
542 }
543 }
544
545 #[test]
546 fn validity_length_is_checked() {
547 let values = ScalarBuffer::from_slice(&[1i32, 2, 3]);
548 assert!(Int32Array::try_new(values.clone(), None).is_ok());
549 assert!(Int32Array::try_new(values.clone(), Some(Bitmap::new_set(3))).is_ok());
550 assert_eq!(
551 Int32Array::try_new(values, Some(Bitmap::new_set(2))).unwrap_err(),
552 DataError::ValidityLengthMismatch {
553 array_len: 3,
554 validity_len: 2
555 }
556 );
557 }
558
559 #[test]
560 fn with_validity_replaces_the_bitmap() {
561 let array = Int32Array::from_values([1, 2, 3]);
562 let masked = array
563 .with_validity(Some([true, false, true].into_iter().collect()))
564 .unwrap();
565 assert_eq!(masked.null_count(), 1);
566 assert_eq!(array.null_count(), 0, "the original is untouched");
567 assert!(array.with_validity(Some(Bitmap::new_set(9))).is_err());
568 assert_eq!(masked.with_validity(None).unwrap().null_count(), 0);
569 }
570
571 #[test]
572 fn typed_constructor_accepts_matching_widths_only() {
573 let values = ScalarBuffer::from_slice(&[1i64, 2]);
574 assert!(Int64Array::try_new_with_type(DataType::Timestamp, values.clone(), None).is_ok());
575 assert!(Int64Array::try_new_with_type(DataType::Duration, values.clone(), None).is_ok());
576 assert!(Int64Array::try_new_with_type(DataType::Float64, values.clone(), None).is_ok());
577 assert!(Int64Array::try_new_with_type(DataType::Int32, values.clone(), None).is_err());
578 assert!(Int64Array::try_new_with_type(DataType::Utf8, values, None).is_err());
579 }
580
581 #[test]
582 fn equality_ignores_null_slot_contents() {
583 let a = Int32Array::from_opt_iter([Some(1), None, Some(3)]);
584 let b = Int32Array::try_new(
585 ScalarBuffer::from_slice(&[1, 999, 3]),
586 Some([true, false, true].into_iter().collect()),
587 )
588 .unwrap();
589 assert_eq!(a, b, "the garbage in the null slot must not matter");
590 assert_ne!(a, Int32Array::from_opt_iter([Some(1), Some(2), Some(3)]));
591 assert_ne!(a, Int32Array::from_opt_iter([Some(1), None]));
592 }
593
594 #[test]
595 fn equality_treats_absent_validity_as_all_valid() {
596 let bare = Int32Array::from_values([1, 2]);
597 let masked = Int32Array::try_new(
598 ScalarBuffer::from_slice(&[1, 2]),
599 Some([true, true].into_iter().collect()),
600 )
601 .unwrap();
602 assert_eq!(bare, masked);
603 }
604
605 #[test]
606 fn equality_across_data_types_is_false() {
607 let stamps = Int64Array::try_new_with_type(
608 DataType::Timestamp,
609 ScalarBuffer::from_slice(&[1, 2]),
610 None,
611 )
612 .unwrap();
613 assert_ne!(stamps, Int64Array::from_values([1, 2]));
614 }
615
616 #[test]
617 fn nan_arrays_are_equal_to_themselves() {
618 let array = Float32Array::from_values([f32::NAN, 1.0]);
619 assert_eq!(array, array.clone());
620 assert_eq!(
621 Float64Array::from_values([f64::NAN]),
622 Float64Array::from_values([f64::NAN])
623 );
624 assert_eq!(
625 Float16Array::from_values([F16::NAN]),
626 Float16Array::from_values([F16::NAN])
627 );
628 assert_eq!(
629 Float32Array::from_values([0.0]),
630 Float32Array::from_values([-0.0]),
631 "IEEE signed zeros still compare equal"
632 );
633 }
634
635 #[test]
636 fn sliced_arrays_compare_by_logical_content() {
637 let full = Int32Array::from_opt_iter([Some(9), Some(1), None, Some(3), Some(9)]);
638 let window = full.slice(1, 3);
639 assert_eq!(window, Int32Array::from_opt_iter([Some(1), None, Some(3)]));
640 }
641
642 #[test]
643 fn float16_columns_round_trip() {
644 let array = Float16Array::from_opt_iter([
645 Some(F16::from_f32(1.5)),
646 None,
647 Some(F16::from_f32(-0.25)),
648 ]);
649 assert_eq!(array.data_type(), &DataType::Float16);
650 assert_eq!(array.get(0).map(F16::to_f32), Some(1.5));
651 assert_eq!(array.get(1), None);
652 assert_eq!(array.get(2).map(F16::to_f32), Some(-0.25));
653 assert_eq!(array.values().len(), 3);
654 }
655
656 #[test]
657 fn every_native_width_builds_an_array() {
658 assert_eq!(Int8Array::from_values([1i8]).data_type(), &DataType::Int8);
659 assert_eq!(
660 Int16Array::from_values([1i16]).data_type(),
661 &DataType::Int16
662 );
663 assert_eq!(
664 Int32Array::from_values([1i32]).data_type(),
665 &DataType::Int32
666 );
667 assert_eq!(
668 Int64Array::from_values([1i64]).data_type(),
669 &DataType::Int64
670 );
671 assert_eq!(UInt8Array::from_values([1u8]).data_type(), &DataType::UInt8);
672 assert_eq!(
673 UInt16Array::from_values([1u16]).data_type(),
674 &DataType::UInt16
675 );
676 assert_eq!(
677 UInt32Array::from_values([1u32]).data_type(),
678 &DataType::UInt32
679 );
680 assert_eq!(
681 UInt64Array::from_values([1u64]).data_type(),
682 &DataType::UInt64
683 );
684 assert_eq!(
685 Float32Array::from_values([1.0f32]).data_type(),
686 &DataType::Float32
687 );
688 assert_eq!(
689 Float64Array::from_values([1.0f64]).data_type(),
690 &DataType::Float64
691 );
692 }
693
694 #[test]
695 fn conversions_and_collect() {
696 let from_array: Int32Array = [1, 2, 3].into();
697 assert_eq!(from_array.len(), 3);
698 let from_vec: Int32Array = vec![1, 2].into();
699 assert_eq!(from_vec.len(), 2);
700 let collected: Int32Array = (0..4).collect();
701 assert_eq!(collected.values(), &[0, 1, 2, 3]);
702 let opt_collected: Int32Array = [Some(1), None].into_iter().collect();
703 assert_eq!(opt_collected.null_count(), 1);
704 assert_eq!((&collected).into_iter().count(), 4);
705 }
706
707 #[test]
708 fn buffer_memory_size_counts_both_buffers() {
709 let bare = Int64Array::from_values([1, 2, 3]);
710 assert!(bare.buffer_memory_size() >= 24);
711 let masked = bare
712 .with_validity(Some([true, false, true].into_iter().collect()))
713 .unwrap();
714 assert!(masked.buffer_memory_size() > bare.buffer_memory_size());
715 }
716
717 #[test]
718 fn debug_output_is_readable() {
719 let array = Int32Array::from_opt_iter([Some(1), None, Some(3)]);
720 let rendered = format!("{array:?}");
721 assert!(rendered.contains("Int32"), "{rendered}");
722 assert!(rendered.contains("len=3"), "{rendered}");
723 assert!(rendered.contains("nulls=1"), "{rendered}");
724 assert!(rendered.contains("null"), "{rendered}");
725
726 let long = Int32Array::from_values(0..50);
727 assert!(format!("{long:?}").contains('…'));
728 }
729
730 #[test]
731 fn trait_object_slice_matches_the_inherent_one() {
732 let array = Int32Array::from_values([1, 2, 3, 4]);
733 let sliced = Array::slice(&array, 1, 2);
734 assert_eq!(sliced.len(), 2);
735 assert_eq!(
736 sliced
737 .as_any()
738 .downcast_ref::<Int32Array>()
739 .map(Int32Array::values),
740 Some(&[2, 3][..])
741 );
742 }
743}