1use std::any::Any;
32use std::fmt;
33use std::sync::Arc;
34
35use crate::array::iter::{ArrayAccessor, ArrayIter};
36use crate::array::offset::{OffsetSizeTrait, validate_offsets};
37use crate::array::{
38 Array, ArrayRef, BINARY_TYPE, LARGE_BINARY_TYPE, check_validity, clamp_window, debug_array,
39 validity_eq,
40};
41use crate::buffer::{Bitmap, Buffer, ScalarBuffer};
42use crate::datatype::DataType;
43use crate::error::{DataError, Result};
44use crate::sealed::Sealed;
45
46#[derive(Clone)]
51pub struct GenericBinaryArray<O: OffsetSizeTrait> {
52 offsets: ScalarBuffer<O>,
54 values: Buffer,
56 validity: Option<Bitmap>,
58}
59
60pub type BinaryArray = GenericBinaryArray<i32>;
62pub type LargeBinaryArray = GenericBinaryArray<i64>;
64
65impl<O: OffsetSizeTrait> GenericBinaryArray<O> {
66 pub fn try_new(
86 offsets: ScalarBuffer<O>,
87 values: Buffer,
88 validity: Option<Bitmap>,
89 ) -> Result<Self> {
90 let len = validate_offsets(offsets.as_slice(), values.len())?;
91 check_validity(validity.as_ref(), len)?;
92 Ok(Self {
93 offsets,
94 values,
95 validity,
96 })
97 }
98
99 #[must_use]
114 pub unsafe fn new_unchecked(
115 offsets: ScalarBuffer<O>,
116 values: Buffer,
117 validity: Option<Bitmap>,
118 ) -> Self {
119 Self::from_parts(offsets, values, validity)
120 }
121
122 pub(crate) fn from_parts(
129 offsets: ScalarBuffer<O>,
130 values: Buffer,
131 validity: Option<Bitmap>,
132 ) -> Self {
133 debug_assert!(validate_offsets(offsets.as_slice(), values.len()).is_ok());
134 debug_assert!(
135 validity
136 .as_ref()
137 .is_none_or(|bits| bits.len() + 1 == offsets.len())
138 );
139 Self {
140 offsets,
141 values,
142 validity,
143 }
144 }
145
146 #[must_use]
156 pub fn from_values<V: AsRef<[u8]>>(values: impl IntoIterator<Item = V>) -> Self {
157 Self::build(values.into_iter().map(Some))
158 }
159
160 #[must_use]
162 pub fn from_opt_iter<V: AsRef<[u8]>>(values: impl IntoIterator<Item = Option<V>>) -> Self {
163 Self::build(values.into_iter())
164 }
165
166 #[must_use]
168 pub fn new_null(len: usize) -> Self {
169 Self {
170 offsets: ScalarBuffer::zeroed(len + 1),
171 values: Buffer::new(),
172 validity: Some(Bitmap::new_unset(len)),
173 }
174 }
175
176 fn build<V: AsRef<[u8]>>(values: impl Iterator<Item = Option<V>>) -> Self {
179 let (lower, _) = values.size_hint();
180 let mut offsets = Vec::with_capacity(lower + 1);
181 let mut data = crate::buffer::AlignedBuf::new();
182 let mut validity = crate::buffer::BitmapBuilder::with_capacity(lower);
183 let mut any_null = false;
184 offsets.push(O::ZERO);
185 for value in values {
186 match value {
187 Some(value) => {
188 data.extend_from_slice(value.as_ref());
189 validity.append(true);
190 }
191 None => {
192 validity.append(false);
193 any_null = true;
194 }
195 }
196 offsets.push(O::from_usize(data.len()).unwrap_or(O::ZERO));
199 }
200 Self {
201 offsets: ScalarBuffer::from_slice(&offsets),
202 values: Buffer::from(data),
203 validity: any_null.then(|| validity.finish()),
204 }
205 }
206
207 #[inline]
209 #[must_use]
210 pub fn value_offsets(&self) -> &[O] {
211 self.offsets.as_slice()
212 }
213
214 #[inline]
227 #[must_use]
228 pub const fn offsets_buffer(&self) -> &ScalarBuffer<O> {
229 &self.offsets
230 }
231
232 #[inline]
234 #[must_use]
235 pub const fn value_data(&self) -> &Buffer {
236 &self.values
237 }
238
239 #[inline]
243 #[must_use]
244 pub fn value(&self, index: usize) -> Option<&[u8]> {
245 let start = self.offsets.get(index)?.to_usize()?;
246 let end = self.offsets.get(index + 1)?.to_usize()?;
247 self.values.as_slice().get(start..end)
248 }
249
250 #[inline]
253 #[must_use]
254 pub fn get(&self, index: usize) -> Option<&[u8]> {
255 if self.is_valid_index(index) {
256 self.value(index)
257 } else {
258 None
259 }
260 }
261
262 #[inline]
264 #[must_use]
265 pub fn value_length(&self, index: usize) -> Option<usize> {
266 let start = self.offsets.get(index)?.to_usize()?;
267 let end = self.offsets.get(index + 1)?.to_usize()?;
268 end.checked_sub(start)
269 }
270
271 #[must_use]
273 pub fn total_value_bytes(&self) -> usize {
274 let offsets = self.offsets.as_slice();
275 match (offsets.first(), offsets.last()) {
276 (Some(first), Some(last)) => {
277 let start = first.to_usize().unwrap_or(0);
278 let end = last.to_usize().unwrap_or(0);
279 end.saturating_sub(start)
280 }
281 _ => 0,
282 }
283 }
284
285 #[inline]
287 pub fn iter(&self) -> ArrayIter<&Self> {
288 ArrayIter::new(self)
289 }
290
291 #[must_use]
296 pub fn slice(&self, offset: usize, len: usize) -> Self {
297 let (offset, len) = clamp_window(self.len(), offset, len);
298 Self {
299 offsets: self.offsets.slice(offset, len + 1),
300 values: self.values.clone(),
301 validity: self.validity.as_ref().map(|bits| bits.slice(offset, len)),
302 }
303 }
304
305 pub fn try_slice(&self, offset: usize, len: usize) -> Result<Self> {
311 if offset.saturating_add(len) > self.len() {
312 return Err(DataError::SliceOutOfBounds {
313 offset,
314 len,
315 available: self.len(),
316 });
317 }
318 Ok(self.slice(offset, len))
319 }
320
321 pub fn with_validity(&self, validity: Option<Bitmap>) -> Result<Self> {
327 check_validity(validity.as_ref(), self.len())?;
328 Ok(Self {
329 offsets: self.offsets.clone(),
330 values: self.values.clone(),
331 validity,
332 })
333 }
334
335 #[inline]
337 fn is_valid_index(&self, index: usize) -> bool {
338 index < self.len() && self.validity.as_ref().is_none_or(|bits| bits.value(index))
339 }
340}
341
342impl<O: OffsetSizeTrait> Sealed for GenericBinaryArray<O> {}
343
344impl<O: OffsetSizeTrait> Array for GenericBinaryArray<O> {
345 fn as_any(&self) -> &dyn Any {
346 self
347 }
348
349 fn data_type(&self) -> &DataType {
350 if O::IS_LARGE {
351 &LARGE_BINARY_TYPE
352 } else {
353 &BINARY_TYPE
354 }
355 }
356
357 fn len(&self) -> usize {
358 self.offsets.len().saturating_sub(1)
359 }
360
361 fn validity(&self) -> Option<&Bitmap> {
362 self.validity.as_ref()
363 }
364
365 fn slice(&self, offset: usize, len: usize) -> ArrayRef {
366 Arc::new(Self::slice(self, offset, len))
367 }
368
369 fn buffer_memory_size(&self) -> usize {
370 self.offsets.inner().backing_len()
371 + self.values.backing_len()
372 + self
373 .validity
374 .as_ref()
375 .map_or(0, |bits| bits.buffer().backing_len())
376 }
377
378 fn equals(&self, other: &dyn Array) -> bool {
379 let Some(other) = other.as_any().downcast_ref::<Self>() else {
380 return false;
381 };
382 if self.len() != other.len()
383 || !validity_eq(self.validity.as_ref(), other.validity.as_ref(), self.len())
384 {
385 return false;
386 }
387 (0..self.len()).all(|index| self.get(index) == other.get(index))
388 }
389}
390
391impl<'a, O: OffsetSizeTrait> ArrayAccessor for &'a GenericBinaryArray<O> {
392 type Item = &'a [u8];
393
394 #[inline]
395 fn accessor_len(&self) -> usize {
396 Array::len(*self)
397 }
398
399 #[inline]
400 fn accessor_get(&self, index: usize) -> Option<&'a [u8]> {
401 let array: &'a GenericBinaryArray<O> = self;
402 array.get(index)
403 }
404}
405
406impl<O: OffsetSizeTrait> PartialEq for GenericBinaryArray<O> {
407 #[inline]
408 fn eq(&self, other: &Self) -> bool {
409 self.equals(other)
410 }
411}
412
413impl<O: OffsetSizeTrait> Eq for GenericBinaryArray<O> {}
414
415impl<O: OffsetSizeTrait, V: AsRef<[u8]>> FromIterator<Option<V>> for GenericBinaryArray<O> {
416 fn from_iter<I: IntoIterator<Item = Option<V>>>(iter: I) -> Self {
417 Self::from_opt_iter(iter)
418 }
419}
420
421impl<'a, O: OffsetSizeTrait> IntoIterator for &'a GenericBinaryArray<O> {
422 type Item = Option<&'a [u8]>;
423 type IntoIter = ArrayIter<&'a GenericBinaryArray<O>>;
424
425 fn into_iter(self) -> Self::IntoIter {
426 self.iter()
427 }
428}
429
430impl<O: OffsetSizeTrait> fmt::Debug for GenericBinaryArray<O> {
431 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
432 debug_array(
433 f,
434 if O::IS_LARGE {
435 "LargeBinaryArray"
436 } else {
437 "BinaryArray"
438 },
439 self.data_type(),
440 self.len(),
441 self.null_count(),
442 self.iter(),
443 )
444 }
445}
446
447#[cfg(test)]
448mod tests {
449 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
450
451 use super::*;
452
453 #[test]
454 fn empty_array() {
455 let array = BinaryArray::from_values([] as [&[u8]; 0]);
456 assert!(array.is_empty());
457 assert_eq!(array.len(), 0);
458 assert_eq!(array.null_count(), 0);
459 assert_eq!(array.value_offsets(), &[0]);
460 assert_eq!(array.get(0), None);
461 assert_eq!(array.value_length(0), None);
462 assert_eq!(array.total_value_bytes(), 0);
463 assert_eq!(array.iter().count(), 0);
464 }
465
466 #[test]
467 fn no_null_array() {
468 let array = BinaryArray::from_values([&b"aa"[..], &b""[..], &b"cccc"[..]]);
469 assert_eq!(array.len(), 3);
470 assert_eq!(array.null_count(), 0);
471 assert!(array.validity().is_none());
472 assert_eq!(array.value_offsets(), &[0, 2, 2, 6]);
473 assert_eq!(array.get(0), Some(&b"aa"[..]));
474 assert_eq!(array.get(1), Some(&b""[..]));
475 assert_eq!(array.get(2), Some(&b"cccc"[..]));
476 assert_eq!(array.get(3), None);
477 assert_eq!(array.value_length(1), Some(0));
478 assert_eq!(array.total_value_bytes(), 6);
479 assert_eq!(array.value_data().as_slice(), b"aacccc");
480 }
481
482 #[test]
483 fn all_null_array() {
484 let array = LargeBinaryArray::new_null(3);
485 assert_eq!(array.len(), 3);
486 assert_eq!(array.null_count(), 3);
487 assert_eq!(array.data_type(), &DataType::LargeBinary);
488 for index in 0..3 {
489 assert!(array.is_null(index));
490 assert_eq!(array.get(index), None);
491 assert_eq!(array.value(index), Some(&b""[..]));
492 }
493 }
494
495 #[test]
496 fn mixed_nulls() {
497 let array =
498 BinaryArray::from_opt_iter([Some(&b"one"[..]), None, Some(&b"three"[..]), None]);
499 assert_eq!(array.len(), 4);
500 assert_eq!(array.null_count(), 2);
501 assert_eq!(array.get(0), Some(&b"one"[..]));
502 assert_eq!(array.get(1), None);
503 assert_eq!(array.value(1), Some(&b""[..]), "null slots span zero bytes");
504 assert_eq!(array.get(2), Some(&b"three"[..]));
505 assert_eq!(array.total_value_bytes(), 8);
506 assert_eq!(
507 array.iter().flatten().collect::<Vec<_>>(),
508 vec![&b"one"[..], &b"three"[..]]
509 );
510 }
511
512 #[test]
513 fn slicing_narrows_offsets_only() {
514 let array = BinaryArray::from_values([&b"a"[..], &b"bb"[..], &b"ccc"[..], &b"dddd"[..]]);
515 let window = array.slice(1, 2);
516 assert_eq!(window.len(), 2);
517 assert_eq!(window.get(0), Some(&b"bb"[..]));
518 assert_eq!(window.get(1), Some(&b"ccc"[..]));
519 assert_eq!(
520 window.value_offsets(),
521 &[1, 3, 6],
522 "a sliced array keeps the original offsets"
523 );
524 assert_eq!(
525 window.value_data().as_slice(),
526 array.value_data().as_slice(),
527 "the value region is shared, not rebuilt"
528 );
529 assert_eq!(window.total_value_bytes(), 5);
530 }
531
532 #[test]
533 fn slicing_clamps_and_composes() {
534 let array = BinaryArray::from_values((0..20).map(|i| vec![b'x'; i]));
535 assert_eq!(array.slice(18, 99).len(), 2);
536 assert_eq!(array.slice(99, 1).len(), 0);
537 assert_eq!(array.slice(0, 0).len(), 0);
538 let nested = array.slice(4, 10).slice(3, 4);
539 assert_eq!(nested.len(), 4);
540 assert_eq!(nested.value_length(0), Some(7));
541 assert_eq!(
542 array.try_slice(19, 2).unwrap_err(),
543 DataError::SliceOutOfBounds {
544 offset: 19,
545 len: 2,
546 available: 20
547 }
548 );
549 }
550
551 #[test]
552 fn sliced_arrays_revalidate() {
553 let array = BinaryArray::from_values([&b"a"[..], &b"bb"[..], &b"ccc"[..]]);
554 let window = array.slice(1, 2);
555 let rebuilt = BinaryArray::try_new(
558 ScalarBuffer::from_slice(window.value_offsets()),
559 window.value_data().clone(),
560 None,
561 )
562 .unwrap();
563 assert_eq!(rebuilt, window);
564 }
565
566 #[test]
567 fn try_new_validates_offsets() {
568 let values = Buffer::from_slice(b"abcdef");
569 assert!(
570 BinaryArray::try_new(ScalarBuffer::from_slice(&[0, 3, 6]), values.clone(), None)
571 .is_ok()
572 );
573 assert!(matches!(
574 BinaryArray::try_new(ScalarBuffer::from_slice(&[0, 9]), values.clone(), None),
575 Err(DataError::OffsetOutOfBounds { .. })
576 ));
577 assert!(matches!(
578 BinaryArray::try_new(ScalarBuffer::from_slice(&[0, 5, 2]), values.clone(), None),
579 Err(DataError::NonMonotonicOffsets { .. })
580 ));
581 assert!(matches!(
582 BinaryArray::try_new(ScalarBuffer::from_slice(&[-1, 2]), values.clone(), None),
583 Err(DataError::NegativeOffset { .. })
584 ));
585 assert!(matches!(
586 BinaryArray::try_new(
587 ScalarBuffer::from_slice(&[] as &[i32]),
588 values.clone(),
589 None
590 ),
591 Err(DataError::OffsetBufferLength { .. })
592 ));
593 assert!(matches!(
594 BinaryArray::try_new(
595 ScalarBuffer::from_slice(&[0, 3]),
596 values,
597 Some(Bitmap::new_set(5))
598 ),
599 Err(DataError::ValidityLengthMismatch { .. })
600 ));
601 }
602
603 #[test]
604 fn unchecked_constructor_matches_the_checked_one() {
605 let offsets = ScalarBuffer::from_slice(&[0i32, 2, 5]);
606 let values = Buffer::from_slice(b"aabbb");
607 let checked = BinaryArray::try_new(offsets.clone(), values.clone(), None).unwrap();
608 let unchecked = unsafe { BinaryArray::new_unchecked(offsets, values, None) };
610 assert_eq!(checked, unchecked);
611 }
612
613 #[test]
614 fn both_offset_widths_work() {
615 let small = BinaryArray::from_values([&b"x"[..]]);
616 let large = LargeBinaryArray::from_values([&b"x"[..]]);
617 assert_eq!(small.data_type(), &DataType::Binary);
618 assert_eq!(large.data_type(), &DataType::LargeBinary);
619 assert_eq!(small.get(0), large.get(0));
620 assert_eq!(small.value_offsets().len(), 2);
621 assert_eq!(large.value_offsets().len(), 2);
622 }
623
624 #[test]
625 fn equality_ignores_null_slot_contents_and_offset_bases() {
626 let a = BinaryArray::from_opt_iter([Some(&b"aa"[..]), None, Some(&b"cc"[..])]);
627 let padded = BinaryArray::from_opt_iter([
628 Some(&b"zz"[..]),
629 Some(&b"aa"[..]),
630 None,
631 Some(&b"cc"[..]),
632 ]);
633 assert_eq!(a, padded.slice(1, 3));
634 assert_ne!(a, BinaryArray::from_values([&b"aa"[..], &b"cc"[..]]));
635 assert_ne!(
636 a,
637 BinaryArray::from_opt_iter([Some(&b"aa"[..]), Some(&b"bb"[..]), Some(&b"cc"[..])])
638 );
639 }
640
641 #[test]
642 fn equality_across_offset_widths_is_false() {
643 let small: ArrayRef = Arc::new(BinaryArray::from_values([&b"x"[..]]));
644 let large: ArrayRef = Arc::new(LargeBinaryArray::from_values([&b"x"[..]]));
645 assert_ne!(small, large);
646 }
647
648 #[test]
649 fn with_validity_replaces_the_bitmap() {
650 let array = BinaryArray::from_values([&b"a"[..], &b"b"[..]]);
651 let masked = array
652 .with_validity(Some([true, false].into_iter().collect()))
653 .unwrap();
654 assert_eq!(masked.null_count(), 1);
655 assert_eq!(masked.get(1), None);
656 assert!(array.with_validity(Some(Bitmap::new_set(9))).is_err());
657 }
658
659 #[test]
660 fn collects_from_iterators() {
661 let collected: BinaryArray = [Some(b"a".to_vec()), None].into_iter().collect();
662 assert_eq!(collected.len(), 2);
663 assert_eq!(collected.null_count(), 1);
664 let owned = BinaryArray::from_values(vec![vec![1u8, 2], vec![3]]);
665 assert_eq!(owned.get(0), Some(&[1u8, 2][..]));
666 assert_eq!((&owned).into_iter().count(), 2);
667 }
668
669 #[test]
670 fn debug_output_names_the_offset_width() {
671 let small = BinaryArray::from_opt_iter([Some(&b"a"[..]), None]);
672 let rendered = format!("{small:?}");
673 assert!(rendered.starts_with("BinaryArray"), "{rendered}");
674 assert!(rendered.contains("nulls=1"), "{rendered}");
675
676 let large = LargeBinaryArray::from_values([&b"a"[..]]);
677 assert!(format!("{large:?}").starts_with("LargeBinaryArray"));
678 }
679
680 #[test]
681 fn buffer_memory_size_counts_three_buffers() {
682 let array = BinaryArray::from_values([&b"abcdef"[..]; 100]);
683 let bare = array.buffer_memory_size();
684 assert!(bare >= 600);
685 let masked = array.with_validity(Some(Bitmap::new_set(100))).unwrap();
686 assert!(masked.buffer_memory_size() > bare);
687 }
688
689 #[test]
690 fn large_values_round_trip() {
691 let big = vec![7u8; 100_000];
692 let array = LargeBinaryArray::from_values([big.as_slice(), b"tail"]);
693 assert_eq!(array.value_length(0), Some(100_000));
694 assert_eq!(array.get(1), Some(&b"tail"[..]));
695 assert_eq!(array.total_value_bytes(), 100_004);
696 }
697}