1use crate::array::{get_offsets_from_buffer, print_long_array};
6use crate::builder::GenericByteBuilder;
7use crate::iterator::ArrayIter;
8use crate::types::ByteArrayType;
9use crate::types::bytes::ByteArrayNativeType;
10use crate::{Array, ArrayAccessor, ArrayRef, OffsetSizeTrait, Scalar};
11use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
12use arrow_buffer::{NullBuffer, OffsetBuffer};
13use arrow_data::{ArrayData, ArrayDataBuilder};
14use arrow_schema::{ArrowError, DataType};
15use std::any::Any;
16use std::sync::Arc;
17
18pub struct GenericByteArray<T: ByteArrayType> {
75 data_type: DataType,
76 value_offsets: OffsetBuffer<T::Offset>,
77 value_data: Buffer,
78 nulls: Option<NullBuffer>,
79}
80
81impl<T: ByteArrayType> Clone for GenericByteArray<T> {
82 fn clone(&self) -> Self {
83 Self {
84 data_type: T::DATA_TYPE,
85 value_offsets: self.value_offsets.clone(),
86 value_data: self.value_data.clone(),
87 nulls: self.nulls.clone(),
88 }
89 }
90}
91
92impl<T: ByteArrayType> GenericByteArray<T> {
93 pub const DATA_TYPE: DataType = T::DATA_TYPE;
95
96 pub fn new(
102 offsets: OffsetBuffer<T::Offset>,
103 values: Buffer,
104 nulls: Option<NullBuffer>,
105 ) -> Self {
106 Self::try_new(offsets, values, nulls).unwrap()
107 }
108
109 pub fn try_new(
116 offsets: OffsetBuffer<T::Offset>,
117 values: Buffer,
118 nulls: Option<NullBuffer>,
119 ) -> Result<Self, ArrowError> {
120 let len = offsets.len() - 1;
121
122 T::validate(&offsets, &values)?;
124
125 if let Some(n) = nulls.as_ref()
126 && n.len() != len
127 {
128 return Err(ArrowError::InvalidArgumentError(format!(
129 "Incorrect length of null buffer for {}{}Array, expected {len} got {}",
130 T::Offset::PREFIX,
131 T::PREFIX,
132 n.len(),
133 )));
134 }
135
136 Ok(Self {
137 data_type: T::DATA_TYPE,
138 value_offsets: offsets,
139 value_data: values,
140 nulls,
141 })
142 }
143
144 pub unsafe fn new_unchecked(
150 offsets: OffsetBuffer<T::Offset>,
151 values: Buffer,
152 nulls: Option<NullBuffer>,
153 ) -> Self {
154 if cfg!(feature = "force_validate") {
155 return Self::new(offsets, values, nulls);
156 }
157 Self {
158 data_type: T::DATA_TYPE,
159 value_offsets: offsets,
160 value_data: values,
161 nulls,
162 }
163 }
164
165 pub fn new_null(len: usize) -> Self {
167 Self {
168 data_type: T::DATA_TYPE,
169 value_offsets: OffsetBuffer::new_zeroed(len),
170 value_data: MutableBuffer::new(0).into(),
171 nulls: Some(NullBuffer::new_null(len)),
172 }
173 }
174
175 pub fn new_scalar(value: impl AsRef<T::Native>) -> Scalar<Self> {
177 Scalar::new(Self::from_iter_values(std::iter::once(value)))
178 }
179
180 pub fn new_repeated(value: impl AsRef<T::Native>, repeat_count: usize) -> Self {
189 let s: &[u8] = value.as_ref().as_ref();
190 let value_offsets = OffsetBuffer::from_repeated_length(s.len(), repeat_count);
191 let bytes: Buffer = {
192 let mut mutable_buffer = MutableBuffer::with_capacity(0);
193 mutable_buffer.repeat_slice_n_times(s, repeat_count);
194
195 mutable_buffer.into()
196 };
197
198 Self {
199 data_type: T::DATA_TYPE,
200 value_data: bytes,
201 value_offsets,
202 nulls: None,
203 }
204 }
205
206 pub fn try_new_repeated(
223 value: impl AsRef<T::Native>,
224 repeat_count: usize,
225 ) -> Result<Self, ArrowError> {
226 let s: &[u8] = value.as_ref().as_ref();
227 let value_offsets = OffsetBuffer::try_from_repeated_length(s.len(), repeat_count)
228 .map_err(|err| ArrowError::OffsetOverflowError(err.value().unwrap_or(usize::MAX)))?;
229 let bytes: Buffer = {
230 let mut mutable_buffer = MutableBuffer::with_capacity(0);
231 mutable_buffer
232 .try_repeat_slice_n_times(s, repeat_count)
233 .map_err(|e| ArrowError::MemoryError(e.to_string()))?;
234
235 mutable_buffer.into()
236 };
237
238 Ok(Self {
239 data_type: T::DATA_TYPE,
240 value_data: bytes,
241 value_offsets,
242 nulls: None,
243 })
244 }
245
246 pub fn from_iter_values<Ptr, I>(iter: I) -> Self
251 where
252 Ptr: AsRef<T::Native>,
253 I: IntoIterator<Item = Ptr>,
254 {
255 let iter = iter.into_iter();
256 let (lower, upper) = iter.size_hint();
259 let capacity = upper.unwrap_or(lower);
260
261 let mut offsets = MutableBuffer::new(
262 capacity
263 .saturating_add(1)
264 .saturating_mul(std::mem::size_of::<T::Offset>()),
265 );
266 offsets.push(T::Offset::usize_as(0));
267
268 let mut values = MutableBuffer::new(0);
269 for s in iter {
270 let s: &[u8] = s.as_ref().as_ref();
271 values.extend_from_slice(s);
272 offsets.push(T::Offset::usize_as(values.len()));
273 }
274
275 T::Offset::from_usize(values.len()).expect("offset overflow");
276 let offsets = Buffer::from(offsets);
277
278 let value_offsets = unsafe { OffsetBuffer::new_unchecked(offsets.into()) };
280
281 Self {
282 data_type: T::DATA_TYPE,
283 value_data: values.into(),
284 value_offsets,
285 nulls: None,
286 }
287 }
288
289 pub fn into_parts(self) -> (OffsetBuffer<T::Offset>, Buffer, Option<NullBuffer>) {
291 (self.value_offsets, self.value_data, self.nulls)
292 }
293
294 #[inline]
298 pub fn value_length(&self, i: usize) -> T::Offset {
299 let offsets = self.value_offsets();
300 offsets[i + 1] - offsets[i]
301 }
302
303 #[inline]
308 pub fn offsets(&self) -> &OffsetBuffer<T::Offset> {
309 &self.value_offsets
310 }
311
312 #[inline]
318 pub fn values(&self) -> &Buffer {
319 &self.value_data
320 }
321
322 pub fn value_data(&self) -> &[u8] {
328 self.value_data.as_slice()
329 }
330
331 pub fn is_ascii(&self) -> bool {
333 let offsets = &self.value_offsets;
334 self.value_data()[offsets.first().as_usize()..offsets.last().as_usize()].is_ascii()
335 }
336
337 #[inline]
339 pub fn value_offsets(&self) -> &[T::Offset] {
340 &self.value_offsets
341 }
342
343 pub unsafe fn value_unchecked(&self, i: usize) -> &T::Native {
351 let end = *unsafe { self.value_offsets().get_unchecked(i + 1) };
352 let start = *unsafe { self.value_offsets().get_unchecked(i) };
353
354 let b = unsafe {
364 std::slice::from_raw_parts(
365 self.value_data
366 .as_ptr()
367 .offset(start.to_isize().unwrap_unchecked()),
368 (end - start).to_usize().unwrap_unchecked(),
369 )
370 };
371
372 unsafe { T::Native::from_bytes_unchecked(b) }
375 }
376
377 pub fn value(&self, i: usize) -> &T::Native {
385 assert!(
386 i < self.len(),
387 "Trying to access an element at index {} from a {}{}Array of length {}",
388 i,
389 T::Offset::PREFIX,
390 T::PREFIX,
391 self.len()
392 );
393 unsafe { self.value_unchecked(i) }
396 }
397
398 pub fn iter(&self) -> ArrayIter<&Self> {
400 ArrayIter::new(self)
401 }
402
403 pub fn slice(&self, offset: usize, length: usize) -> Self {
408 Self {
409 data_type: T::DATA_TYPE,
410 value_offsets: self.value_offsets.slice(offset, length),
411 value_data: self.value_data.clone(),
412 nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
413 }
414 }
415
416 pub fn into_builder(self) -> Result<GenericByteBuilder<T>, Self> {
425 let (offsets, values, nulls) = self.into_parts();
426
427 let rebuild = |offsets: Buffer, values: Buffer, nulls: Option<NullBuffer>| {
429 unsafe {
431 Self::new_unchecked(OffsetBuffer::new_unchecked(offsets.into()), values, nulls)
432 }
433 };
434
435 let offsets = match offsets.into_inner().into_inner().into_mutable() {
439 Ok(offsets) => offsets,
440 Err(offsets) => return Err(rebuild(offsets, values, nulls)),
441 };
442 let values = match values.into_mutable() {
443 Ok(values) => values,
444 Err(values) => return Err(rebuild(offsets.into(), values, nulls)),
445 };
446
447 let nulls = nulls.map(|nulls| {
452 let bits = nulls.into_inner();
453 if bits.offset() == 0 {
454 let len = bits.len();
455 bits.into_inner()
456 .into_mutable()
457 .unwrap_or_else(|buffer| buffer[..len.div_ceil(8)].to_vec().into())
458 } else {
459 bits.sliced().to_vec().into()
461 }
462 });
463
464 Ok(unsafe { GenericByteBuilder::new_from_buffer(offsets, values, nulls) })
466 }
467}
468
469impl<T: ByteArrayType> std::fmt::Debug for GenericByteArray<T> {
470 fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
471 write!(f, "{}{}Array\n[\n", T::Offset::PREFIX, T::PREFIX)?;
472 print_long_array(self, f, &mut |index, f| {
473 std::fmt::Debug::fmt(&self.value(index), f)
474 })?;
475 write!(f, "]")
476 }
477}
478
479unsafe impl<T: ByteArrayType> Array for GenericByteArray<T> {
481 fn as_any(&self) -> &dyn Any {
482 self
483 }
484
485 fn to_data(&self) -> ArrayData {
486 self.clone().into()
487 }
488
489 fn into_data(self) -> ArrayData {
490 self.into()
491 }
492
493 fn data_type(&self) -> &DataType {
494 &self.data_type
495 }
496
497 fn slice(&self, offset: usize, length: usize) -> ArrayRef {
498 Arc::new(self.slice(offset, length))
499 }
500
501 fn len(&self) -> usize {
502 self.value_offsets.len() - 1
503 }
504
505 fn is_empty(&self) -> bool {
506 self.value_offsets.len() <= 1
507 }
508
509 fn shrink_to_fit(&mut self) {
510 self.value_offsets.shrink_to_fit();
511 self.value_data.shrink_to_fit();
512 if let Some(nulls) = &mut self.nulls {
513 nulls.shrink_to_fit();
514 }
515 }
516
517 fn offset(&self) -> usize {
518 0
519 }
520
521 fn nulls(&self) -> Option<&NullBuffer> {
522 self.nulls.as_ref()
523 }
524
525 fn logical_null_count(&self) -> usize {
526 self.null_count()
528 }
529
530 fn get_buffer_memory_size(&self) -> usize {
531 let mut sum = self.value_offsets.inner().inner().capacity();
532 sum += self.value_data.capacity();
533 if let Some(x) = &self.nulls {
534 sum += x.buffer().capacity()
535 }
536 sum
537 }
538
539 fn get_array_memory_size(&self) -> usize {
540 std::mem::size_of::<Self>() + self.get_buffer_memory_size()
541 }
542
543 #[cfg(feature = "pool")]
544 fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
545 self.value_offsets.claim(pool);
546 self.value_data.claim(pool);
547 if let Some(nulls) = &self.nulls {
548 nulls.claim(pool);
549 }
550 }
551}
552
553impl<'a, T: ByteArrayType> ArrayAccessor for &'a GenericByteArray<T> {
554 type Item = &'a T::Native;
555
556 fn value(&self, index: usize) -> Self::Item {
557 GenericByteArray::value(self, index)
558 }
559
560 unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
561 unsafe { GenericByteArray::value_unchecked(self, index) }
562 }
563}
564
565impl<T: ByteArrayType> From<ArrayData> for GenericByteArray<T> {
566 fn from(data: ArrayData) -> Self {
567 let (data_type, len, nulls, offset, mut buffers, _child_data) = data.into_parts();
568 assert_eq!(
569 data_type,
570 Self::DATA_TYPE,
571 "{}{}Array expects DataType::{}",
572 T::Offset::PREFIX,
573 T::PREFIX,
574 Self::DATA_TYPE
575 );
576 assert_eq!(
577 buffers.len(),
578 2,
579 "{}{}Array data should contain 2 buffers only (offsets and values)",
580 T::Offset::PREFIX,
581 T::PREFIX,
582 );
583 let value_data = buffers.pop().expect("checked above");
585 let offset_buffer = buffers.pop().expect("checked above");
586
587 let value_offsets = unsafe { get_offsets_from_buffer(offset_buffer, offset, len) };
590 Self {
591 data_type,
592 value_offsets,
593 value_data,
594 nulls,
595 }
596 }
597}
598
599impl<T: ByteArrayType> From<GenericByteArray<T>> for ArrayData {
600 fn from(array: GenericByteArray<T>) -> Self {
601 let len = array.len();
602
603 let offsets = array.value_offsets.into_inner().into_inner();
604 let builder = ArrayDataBuilder::new(array.data_type)
605 .len(len)
606 .buffers(vec![offsets, array.value_data])
607 .nulls(array.nulls);
608
609 unsafe { builder.build_unchecked() }
610 }
611}
612
613impl<'a, T: ByteArrayType> IntoIterator for &'a GenericByteArray<T> {
614 type Item = Option<&'a T::Native>;
615 type IntoIter = ArrayIter<Self>;
616
617 fn into_iter(self) -> Self::IntoIter {
618 ArrayIter::new(self)
619 }
620}
621
622impl<'a, Ptr, T: ByteArrayType> FromIterator<&'a Option<Ptr>> for GenericByteArray<T>
623where
624 Ptr: AsRef<T::Native> + 'a,
625{
626 fn from_iter<I: IntoIterator<Item = &'a Option<Ptr>>>(iter: I) -> Self {
627 iter.into_iter()
628 .map(|o| o.as_ref().map(|p| p.as_ref()))
629 .collect()
630 }
631}
632
633impl<Ptr, T: ByteArrayType> FromIterator<Option<Ptr>> for GenericByteArray<T>
634where
635 Ptr: AsRef<T::Native>,
636{
637 fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
638 let iter = iter.into_iter();
639 let mut builder = GenericByteBuilder::with_capacity(iter.size_hint().0, 1024);
640 builder.extend(iter);
641 builder.finish()
642 }
643}
644
645#[cfg(test)]
646mod tests {
647 use crate::{Array, BinaryArray, LargeBinaryArray, StringArray};
648 use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer};
649 use arrow_schema::ArrowError;
650
651 #[test]
654 fn from_iter_values_untrusted_size_hint() {
655 let no_upper_bound = (0..20).filter(|i| i % 2 == 0).map(|i| format!("v{i}"));
657 assert_eq!(no_upper_bound.size_hint(), (0, Some(20)));
658 let array = StringArray::from_iter_values(no_upper_bound);
659 assert_eq!(array.len(), 10);
660 assert_eq!(array.value(0), "v0");
661 assert_eq!(array.value(9), "v18");
662
663 let too_large_upper_bound = (0..).map(|i| format!("v{i}")).take_while(|v| v != "v3");
665 assert_eq!(too_large_upper_bound.size_hint(), (0, None));
666 let array = StringArray::from_iter_values(too_large_upper_bound);
667 assert_eq!(array.len(), 3);
668 assert_eq!(array.value(2), "v2");
669 }
670
671 #[test]
672 fn try_new() {
673 let data = Buffer::from_slice_ref("helloworld");
674 let offsets = OffsetBuffer::new(vec![0, 5, 10].into());
675 StringArray::new(offsets.clone(), data.clone(), None);
676
677 let nulls = NullBuffer::new_null(3);
678 let err =
679 StringArray::try_new(offsets.clone(), data.clone(), Some(nulls.clone())).unwrap_err();
680 assert_eq!(
681 err.to_string(),
682 "Invalid argument error: Incorrect length of null buffer for StringArray, expected 2 got 3"
683 );
684
685 let err = BinaryArray::try_new(offsets.clone(), data.clone(), Some(nulls)).unwrap_err();
686 assert_eq!(
687 err.to_string(),
688 "Invalid argument error: Incorrect length of null buffer for BinaryArray, expected 2 got 3"
689 );
690
691 let non_utf8_data = Buffer::from_slice_ref(b"he\xFFloworld");
692 let err = StringArray::try_new(offsets.clone(), non_utf8_data.clone(), None).unwrap_err();
693 assert_eq!(
694 err.to_string(),
695 "Invalid argument error: Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 2"
696 );
697
698 BinaryArray::new(offsets, non_utf8_data, None);
699
700 let offsets = OffsetBuffer::new(vec![0, 5, 11].into());
701 let err = StringArray::try_new(offsets.clone(), data.clone(), None).unwrap_err();
702 assert_eq!(
703 err.to_string(),
704 "Invalid argument error: Offset of 11 exceeds length of values 10"
705 );
706
707 let err = BinaryArray::try_new(offsets.clone(), data, None).unwrap_err();
708 assert_eq!(
709 err.to_string(),
710 "Invalid argument error: Maximum offset of 11 is larger than values of length 10"
711 );
712
713 let non_ascii_data = Buffer::from_slice_ref("heìloworld");
714 StringArray::new(offsets.clone(), non_ascii_data.clone(), None);
715 BinaryArray::new(offsets, non_ascii_data.clone(), None);
716
717 let offsets = OffsetBuffer::new(vec![0, 3, 10].into());
718 let err = StringArray::try_new(offsets.clone(), non_ascii_data.clone(), None).unwrap_err();
719 assert_eq!(
720 err.to_string(),
721 "Invalid argument error: Split UTF-8 codepoint at offset 3"
722 );
723
724 BinaryArray::new(offsets, non_ascii_data, None);
725 }
726
727 #[test]
728 fn create_repeated() {
729 let arr = BinaryArray::new_repeated(b"hello", 3);
730 assert_eq!(arr.len(), 3);
731 assert_eq!(arr.value(0), b"hello");
732 assert_eq!(arr.value(1), b"hello");
733 assert_eq!(arr.value(2), b"hello");
734
735 let arr = StringArray::new_repeated("world", 2);
736 assert_eq!(arr.len(), 2);
737 assert_eq!(arr.value(0), "world");
738 assert_eq!(arr.value(1), "world");
739 }
740
741 #[test]
742 #[should_panic(expected = "total length overflow: does not fit in usize")]
743 fn create_repeated_usize_overflow_1() {
744 let _arr = BinaryArray::new_repeated(b"hello", (usize::MAX / "hello".len()) + 1);
745 }
746
747 #[test]
748 #[should_panic(expected = "total length overflow: does not fit in usize")]
749 fn create_repeated_usize_overflow_2() {
750 let _arr = BinaryArray::new_repeated(b"hello", usize::MAX);
751 }
752
753 #[test]
754 #[should_panic(expected = "offset overflow")]
755 fn create_repeated_i32_offset_overflow_1() {
756 let _arr = BinaryArray::new_repeated(b"hello", usize::MAX / "hello".len());
757 }
758
759 #[test]
760 #[should_panic(expected = "offset overflow")]
761 fn create_repeated_i32_offset_overflow_2() {
762 let _arr = BinaryArray::new_repeated(b"hello", ((i32::MAX as usize) / "hello".len()) + 1);
763 }
764
765 #[test]
766 fn try_new_repeated() {
767 let arr = BinaryArray::try_new_repeated(b"\x00\x01\xff", 3).unwrap();
769 assert_eq!(arr.len(), 3);
770 assert_eq!(arr.null_count(), 0);
771 for i in 0..3 {
772 assert_eq!(arr.value(i), b"\x00\x01\xff");
773 }
774 assert_eq!(arr.value_offsets(), &[0, 3, 6, 9]);
775
776 let arr = BinaryArray::try_new_repeated(b"test", 0).unwrap();
778 assert_eq!(arr.len(), 0);
779 assert_eq!(arr.value_offsets(), &[0]);
780
781 let arr = BinaryArray::try_new_repeated(b"", 5).unwrap();
783 assert_eq!(arr.len(), 5);
784 assert_eq!(arr.value_offsets(), &[0, 0, 0, 0, 0, 0]);
785 }
786
787 #[test]
788 fn try_new_repeated_overflow() {
789 let err =
791 BinaryArray::try_new_repeated(b"hello", (usize::MAX / "hello".len()) + 1).unwrap_err();
792 assert!(matches!(err, ArrowError::OffsetOverflowError(usize::MAX)));
793
794 let err = BinaryArray::try_new_repeated(b"a", (i32::MAX as usize) + 1).unwrap_err();
796 assert!(matches!(
797 err,
798 ArrowError::OffsetOverflowError(val) if val == (i32::MAX as usize) + 1
799 ));
800
801 #[cfg(target_pointer_width = "64")]
803 {
804 let err = LargeBinaryArray::try_new_repeated(b"x", i64::MAX as usize).unwrap_err();
806 assert!(matches!(err, ArrowError::OffsetOverflowError(_)));
807
808 let err =
809 LargeBinaryArray::try_new_repeated(b"a", (i64::MAX as usize) + 1).unwrap_err();
810 assert!(matches!(
811 err,
812 ArrowError::OffsetOverflowError(9223372036854775808)
813 ));
814 }
815 }
816}