1use crate::array::{get_offsets, print_long_array};
19use crate::builder::GenericByteBuilder;
20use crate::iterator::ArrayIter;
21use crate::types::ByteArrayType;
22use crate::types::bytes::ByteArrayNativeType;
23use crate::{Array, ArrayAccessor, ArrayRef, OffsetSizeTrait, Scalar};
24use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
25use arrow_buffer::{NullBuffer, OffsetBuffer};
26use arrow_data::{ArrayData, ArrayDataBuilder};
27use arrow_schema::{ArrowError, DataType};
28use std::any::Any;
29use std::sync::Arc;
30
31pub struct GenericByteArray<T: ByteArrayType> {
88 data_type: DataType,
89 value_offsets: OffsetBuffer<T::Offset>,
90 value_data: Buffer,
91 nulls: Option<NullBuffer>,
92}
93
94impl<T: ByteArrayType> Clone for GenericByteArray<T> {
95 fn clone(&self) -> Self {
96 Self {
97 data_type: T::DATA_TYPE,
98 value_offsets: self.value_offsets.clone(),
99 value_data: self.value_data.clone(),
100 nulls: self.nulls.clone(),
101 }
102 }
103}
104
105impl<T: ByteArrayType> GenericByteArray<T> {
106 pub const DATA_TYPE: DataType = T::DATA_TYPE;
108
109 pub fn new(
115 offsets: OffsetBuffer<T::Offset>,
116 values: Buffer,
117 nulls: Option<NullBuffer>,
118 ) -> Self {
119 Self::try_new(offsets, values, nulls).unwrap()
120 }
121
122 pub fn try_new(
129 offsets: OffsetBuffer<T::Offset>,
130 values: Buffer,
131 nulls: Option<NullBuffer>,
132 ) -> Result<Self, ArrowError> {
133 let len = offsets.len() - 1;
134
135 T::validate(&offsets, &values)?;
137
138 if let Some(n) = nulls.as_ref() {
139 if n.len() != len {
140 return Err(ArrowError::InvalidArgumentError(format!(
141 "Incorrect length of null buffer for {}{}Array, expected {len} got {}",
142 T::Offset::PREFIX,
143 T::PREFIX,
144 n.len(),
145 )));
146 }
147 }
148
149 Ok(Self {
150 data_type: T::DATA_TYPE,
151 value_offsets: offsets,
152 value_data: values,
153 nulls,
154 })
155 }
156
157 pub unsafe fn new_unchecked(
163 offsets: OffsetBuffer<T::Offset>,
164 values: Buffer,
165 nulls: Option<NullBuffer>,
166 ) -> Self {
167 if cfg!(feature = "force_validate") {
168 return Self::new(offsets, values, nulls);
169 }
170 Self {
171 data_type: T::DATA_TYPE,
172 value_offsets: offsets,
173 value_data: values,
174 nulls,
175 }
176 }
177
178 pub fn new_null(len: usize) -> Self {
180 Self {
181 data_type: T::DATA_TYPE,
182 value_offsets: OffsetBuffer::new_zeroed(len),
183 value_data: MutableBuffer::new(0).into(),
184 nulls: Some(NullBuffer::new_null(len)),
185 }
186 }
187
188 pub fn new_scalar(value: impl AsRef<T::Native>) -> Scalar<Self> {
190 Scalar::new(Self::from_iter_values(std::iter::once(value)))
191 }
192
193 pub fn new_repeated(value: impl AsRef<T::Native>, repeat_count: usize) -> Self {
199 let s: &[u8] = value.as_ref().as_ref();
200 let value_offsets = OffsetBuffer::from_repeated_length(s.len(), repeat_count);
201 let bytes: Buffer = {
202 let mut mutable_buffer = MutableBuffer::with_capacity(0);
203 mutable_buffer.repeat_slice_n_times(s, repeat_count);
204
205 mutable_buffer.into()
206 };
207
208 Self {
209 data_type: T::DATA_TYPE,
210 value_data: bytes,
211 value_offsets,
212 nulls: None,
213 }
214 }
215
216 pub fn from_iter_values<Ptr, I>(iter: I) -> Self
218 where
219 Ptr: AsRef<T::Native>,
220 I: IntoIterator<Item = Ptr>,
221 {
222 let iter = iter.into_iter();
223 let (_, data_len) = iter.size_hint();
224 let data_len = data_len.expect("Iterator must be sized"); let mut offsets = MutableBuffer::new((data_len + 1) * std::mem::size_of::<T::Offset>());
227 offsets.push(T::Offset::usize_as(0));
228
229 let mut values = MutableBuffer::new(0);
230 for s in iter {
231 let s: &[u8] = s.as_ref().as_ref();
232 values.extend_from_slice(s);
233 offsets.push(T::Offset::usize_as(values.len()));
234 }
235
236 T::Offset::from_usize(values.len()).expect("offset overflow");
237 let offsets = Buffer::from(offsets);
238
239 let value_offsets = unsafe { OffsetBuffer::new_unchecked(offsets.into()) };
241
242 Self {
243 data_type: T::DATA_TYPE,
244 value_data: values.into(),
245 value_offsets,
246 nulls: None,
247 }
248 }
249
250 pub fn into_parts(self) -> (OffsetBuffer<T::Offset>, Buffer, Option<NullBuffer>) {
252 (self.value_offsets, self.value_data, self.nulls)
253 }
254
255 #[inline]
259 pub fn value_length(&self, i: usize) -> T::Offset {
260 let offsets = self.value_offsets();
261 offsets[i + 1] - offsets[i]
262 }
263
264 #[inline]
269 pub fn offsets(&self) -> &OffsetBuffer<T::Offset> {
270 &self.value_offsets
271 }
272
273 #[inline]
278 pub fn values(&self) -> &Buffer {
279 &self.value_data
280 }
281
282 pub fn value_data(&self) -> &[u8] {
284 self.value_data.as_slice()
285 }
286
287 pub fn is_ascii(&self) -> bool {
289 let offsets = self.value_offsets();
290 let start = offsets.first().unwrap();
291 let end = offsets.last().unwrap();
292 self.value_data()[start.as_usize()..end.as_usize()].is_ascii()
293 }
294
295 #[inline]
297 pub fn value_offsets(&self) -> &[T::Offset] {
298 &self.value_offsets
299 }
300
301 pub unsafe fn value_unchecked(&self, i: usize) -> &T::Native {
309 let end = *unsafe { self.value_offsets().get_unchecked(i + 1) };
310 let start = *unsafe { self.value_offsets().get_unchecked(i) };
311
312 let b = unsafe {
322 std::slice::from_raw_parts(
323 self.value_data
324 .as_ptr()
325 .offset(start.to_isize().unwrap_unchecked()),
326 (end - start).to_usize().unwrap_unchecked(),
327 )
328 };
329
330 unsafe { T::Native::from_bytes_unchecked(b) }
333 }
334
335 pub fn value(&self, i: usize) -> &T::Native {
343 assert!(
344 i < self.len(),
345 "Trying to access an element at index {} from a {}{}Array of length {}",
346 i,
347 T::Offset::PREFIX,
348 T::PREFIX,
349 self.len()
350 );
351 unsafe { self.value_unchecked(i) }
354 }
355
356 pub fn iter(&self) -> ArrayIter<&Self> {
358 ArrayIter::new(self)
359 }
360
361 pub fn slice(&self, offset: usize, length: usize) -> Self {
363 Self {
364 data_type: T::DATA_TYPE,
365 value_offsets: self.value_offsets.slice(offset, length),
366 value_data: self.value_data.clone(),
367 nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
368 }
369 }
370
371 pub fn into_builder(self) -> Result<GenericByteBuilder<T>, Self> {
380 let (offsets, values, nulls) = self.into_parts();
381
382 let rebuild = |offsets: Buffer, values: Buffer, nulls: Option<NullBuffer>| {
384 unsafe {
386 Self::new_unchecked(OffsetBuffer::new_unchecked(offsets.into()), values, nulls)
387 }
388 };
389
390 let offsets = match offsets.into_inner().into_inner().into_mutable() {
394 Ok(offsets) => offsets,
395 Err(offsets) => return Err(rebuild(offsets, values, nulls)),
396 };
397 let values = match values.into_mutable() {
398 Ok(values) => values,
399 Err(values) => return Err(rebuild(offsets.into(), values, nulls)),
400 };
401
402 let nulls = nulls.map(|nulls| {
407 let bits = nulls.into_inner();
408 if bits.offset() == 0 {
409 let len = bits.len();
410 bits.into_inner()
411 .into_mutable()
412 .unwrap_or_else(|buffer| buffer[..len.div_ceil(8)].to_vec().into())
413 } else {
414 bits.sliced().to_vec().into()
416 }
417 });
418
419 Ok(unsafe { GenericByteBuilder::new_from_buffer(offsets, values, nulls) })
421 }
422}
423
424impl<T: ByteArrayType> std::fmt::Debug for GenericByteArray<T> {
425 fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
426 write!(f, "{}{}Array\n[\n", T::Offset::PREFIX, T::PREFIX)?;
427 print_long_array(self, f, |array, index, f| {
428 std::fmt::Debug::fmt(&array.value(index), f)
429 })?;
430 write!(f, "]")
431 }
432}
433
434unsafe impl<T: ByteArrayType> Array for GenericByteArray<T> {
436 fn as_any(&self) -> &dyn Any {
437 self
438 }
439
440 fn to_data(&self) -> ArrayData {
441 self.clone().into()
442 }
443
444 fn into_data(self) -> ArrayData {
445 self.into()
446 }
447
448 fn data_type(&self) -> &DataType {
449 &self.data_type
450 }
451
452 fn slice(&self, offset: usize, length: usize) -> ArrayRef {
453 Arc::new(self.slice(offset, length))
454 }
455
456 fn len(&self) -> usize {
457 self.value_offsets.len() - 1
458 }
459
460 fn is_empty(&self) -> bool {
461 self.value_offsets.len() <= 1
462 }
463
464 fn shrink_to_fit(&mut self) {
465 self.value_offsets.shrink_to_fit();
466 self.value_data.shrink_to_fit();
467 if let Some(nulls) = &mut self.nulls {
468 nulls.shrink_to_fit();
469 }
470 }
471
472 fn offset(&self) -> usize {
473 0
474 }
475
476 fn nulls(&self) -> Option<&NullBuffer> {
477 self.nulls.as_ref()
478 }
479
480 fn logical_null_count(&self) -> usize {
481 self.null_count()
483 }
484
485 fn get_buffer_memory_size(&self) -> usize {
486 let mut sum = self.value_offsets.inner().inner().capacity();
487 sum += self.value_data.capacity();
488 if let Some(x) = &self.nulls {
489 sum += x.buffer().capacity()
490 }
491 sum
492 }
493
494 fn get_array_memory_size(&self) -> usize {
495 std::mem::size_of::<Self>() + self.get_buffer_memory_size()
496 }
497}
498
499impl<'a, T: ByteArrayType> ArrayAccessor for &'a GenericByteArray<T> {
500 type Item = &'a T::Native;
501
502 fn value(&self, index: usize) -> Self::Item {
503 GenericByteArray::value(self, index)
504 }
505
506 unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
507 unsafe { GenericByteArray::value_unchecked(self, index) }
508 }
509}
510
511impl<T: ByteArrayType> From<ArrayData> for GenericByteArray<T> {
512 fn from(data: ArrayData) -> Self {
513 assert_eq!(
514 data.data_type(),
515 &Self::DATA_TYPE,
516 "{}{}Array expects DataType::{}",
517 T::Offset::PREFIX,
518 T::PREFIX,
519 Self::DATA_TYPE
520 );
521 assert_eq!(
522 data.buffers().len(),
523 2,
524 "{}{}Array data should contain 2 buffers only (offsets and values)",
525 T::Offset::PREFIX,
526 T::PREFIX,
527 );
528 let value_offsets = unsafe { get_offsets(&data) };
531 let value_data = data.buffers()[1].clone();
532 Self {
533 value_offsets,
534 value_data,
535 data_type: T::DATA_TYPE,
536 nulls: data.nulls().cloned(),
537 }
538 }
539}
540
541impl<T: ByteArrayType> From<GenericByteArray<T>> for ArrayData {
542 fn from(array: GenericByteArray<T>) -> Self {
543 let len = array.len();
544
545 let offsets = array.value_offsets.into_inner().into_inner();
546 let builder = ArrayDataBuilder::new(array.data_type)
547 .len(len)
548 .buffers(vec![offsets, array.value_data])
549 .nulls(array.nulls);
550
551 unsafe { builder.build_unchecked() }
552 }
553}
554
555impl<'a, T: ByteArrayType> IntoIterator for &'a GenericByteArray<T> {
556 type Item = Option<&'a T::Native>;
557 type IntoIter = ArrayIter<Self>;
558
559 fn into_iter(self) -> Self::IntoIter {
560 ArrayIter::new(self)
561 }
562}
563
564impl<'a, Ptr, T: ByteArrayType> FromIterator<&'a Option<Ptr>> for GenericByteArray<T>
565where
566 Ptr: AsRef<T::Native> + 'a,
567{
568 fn from_iter<I: IntoIterator<Item = &'a Option<Ptr>>>(iter: I) -> Self {
569 iter.into_iter()
570 .map(|o| o.as_ref().map(|p| p.as_ref()))
571 .collect()
572 }
573}
574
575impl<Ptr, T: ByteArrayType> FromIterator<Option<Ptr>> for GenericByteArray<T>
576where
577 Ptr: AsRef<T::Native>,
578{
579 fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
580 let iter = iter.into_iter();
581 let mut builder = GenericByteBuilder::with_capacity(iter.size_hint().0, 1024);
582 builder.extend(iter);
583 builder.finish()
584 }
585}
586
587#[cfg(test)]
588mod tests {
589 use crate::{Array, BinaryArray, StringArray};
590 use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer};
591
592 #[test]
593 fn try_new() {
594 let data = Buffer::from_slice_ref("helloworld");
595 let offsets = OffsetBuffer::new(vec![0, 5, 10].into());
596 StringArray::new(offsets.clone(), data.clone(), None);
597
598 let nulls = NullBuffer::new_null(3);
599 let err =
600 StringArray::try_new(offsets.clone(), data.clone(), Some(nulls.clone())).unwrap_err();
601 assert_eq!(
602 err.to_string(),
603 "Invalid argument error: Incorrect length of null buffer for StringArray, expected 2 got 3"
604 );
605
606 let err = BinaryArray::try_new(offsets.clone(), data.clone(), Some(nulls)).unwrap_err();
607 assert_eq!(
608 err.to_string(),
609 "Invalid argument error: Incorrect length of null buffer for BinaryArray, expected 2 got 3"
610 );
611
612 let non_utf8_data = Buffer::from_slice_ref(b"he\xFFloworld");
613 let err = StringArray::try_new(offsets.clone(), non_utf8_data.clone(), None).unwrap_err();
614 assert_eq!(
615 err.to_string(),
616 "Invalid argument error: Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 2"
617 );
618
619 BinaryArray::new(offsets, non_utf8_data, None);
620
621 let offsets = OffsetBuffer::new(vec![0, 5, 11].into());
622 let err = StringArray::try_new(offsets.clone(), data.clone(), None).unwrap_err();
623 assert_eq!(
624 err.to_string(),
625 "Invalid argument error: Offset of 11 exceeds length of values 10"
626 );
627
628 let err = BinaryArray::try_new(offsets.clone(), data, None).unwrap_err();
629 assert_eq!(
630 err.to_string(),
631 "Invalid argument error: Maximum offset of 11 is larger than values of length 10"
632 );
633
634 let non_ascii_data = Buffer::from_slice_ref("heìloworld");
635 StringArray::new(offsets.clone(), non_ascii_data.clone(), None);
636 BinaryArray::new(offsets, non_ascii_data.clone(), None);
637
638 let offsets = OffsetBuffer::new(vec![0, 3, 10].into());
639 let err = StringArray::try_new(offsets.clone(), non_ascii_data.clone(), None).unwrap_err();
640 assert_eq!(
641 err.to_string(),
642 "Invalid argument error: Split UTF-8 codepoint at offset 3"
643 );
644
645 BinaryArray::new(offsets, non_ascii_data, None);
646 }
647
648 #[test]
649 fn create_repeated() {
650 let arr = BinaryArray::new_repeated(b"hello", 3);
651 assert_eq!(arr.len(), 3);
652 assert_eq!(arr.value(0), b"hello");
653 assert_eq!(arr.value(1), b"hello");
654 assert_eq!(arr.value(2), b"hello");
655
656 let arr = StringArray::new_repeated("world", 2);
657 assert_eq!(arr.len(), 2);
658 assert_eq!(arr.value(0), "world");
659 assert_eq!(arr.value(1), "world");
660 }
661
662 #[test]
663 #[should_panic(expected = "usize overflow")]
664 fn create_repeated_usize_overflow_1() {
665 let _arr = BinaryArray::new_repeated(b"hello", (usize::MAX / "hello".len()) + 1);
666 }
667
668 #[test]
669 #[should_panic(expected = "usize overflow")]
670 fn create_repeated_usize_overflow_2() {
671 let _arr = BinaryArray::new_repeated(b"hello", usize::MAX);
672 }
673
674 #[test]
675 #[should_panic(expected = "offset overflow")]
676 fn create_repeated_i32_offset_overflow_1() {
677 let _arr = BinaryArray::new_repeated(b"hello", usize::MAX / "hello".len());
678 }
679
680 #[test]
681 #[should_panic(expected = "offset overflow")]
682 fn create_repeated_i32_offset_overflow_2() {
683 let _arr = BinaryArray::new_repeated(b"hello", ((i32::MAX as usize) / "hello".len()) + 1);
684 }
685}