1use crate::buffer::ScalarBuffer;
6use crate::{ArrowNativeType, MutableBuffer, NullBuffer, OffsetBufferBuilder, OverflowError};
7use std::ops::Deref;
8
9#[derive(Debug, Clone, PartialEq, Eq)]
46pub struct OffsetBuffer<O: ArrowNativeType>(ScalarBuffer<O>);
47
48impl<O: ArrowNativeType> OffsetBuffer<O> {
49 pub fn new(buffer: ScalarBuffer<O>) -> Self {
56 assert!(!buffer.is_empty(), "offsets cannot be empty");
57 assert!(
58 buffer[0] >= O::usize_as(0),
59 "offsets must be greater than 0"
60 );
61 assert!(
62 buffer.windows(2).all(|w| w[0] <= w[1]),
63 "offsets must be monotonically increasing"
64 );
65 Self(buffer)
66 }
67
68 pub unsafe fn new_unchecked(buffer: ScalarBuffer<O>) -> Self {
75 Self(buffer)
76 }
77
78 pub fn new_empty() -> Self {
80 let buffer = MutableBuffer::from_len_zeroed(std::mem::size_of::<O>());
81 Self(buffer.into_buffer().into())
82 }
83
84 pub fn new_zeroed(len: usize) -> Self {
92 Self::try_new_zeroed(len).unwrap_or_else(|err| panic!("{err}"))
93 }
94
95 pub fn try_new_zeroed(len: usize) -> Result<Self, OverflowError> {
102 let len_bytes = len
103 .checked_add(1)
104 .and_then(|o| o.checked_mul(std::mem::size_of::<O>()))
105 .ok_or_else(|| OverflowError::new::<usize>("buffer length"))?;
106 let buffer = MutableBuffer::try_from_len_zeroed(len_bytes)
107 .map_err(|_| OverflowError::new::<usize>("buffer length"))?;
108 Ok(Self(buffer.into_buffer().into()))
109 }
110
111 pub fn from_lengths<I>(lengths: I) -> Self
126 where
127 I: IntoIterator<Item = usize>,
128 {
129 Self::try_from_lengths(lengths).unwrap_or_else(|err| panic!("{err}"))
130 }
131
132 pub fn try_from_lengths<I>(lengths: I) -> Result<Self, OverflowError>
145 where
146 I: IntoIterator<Item = usize>,
147 {
148 let iter = lengths.into_iter();
149 let mut out = Vec::with_capacity(iter.size_hint().0 + 1);
150 out.push(O::usize_as(0));
151
152 let mut acc = 0_usize;
153 for length in iter {
154 acc = acc
155 .checked_add(length)
156 .ok_or_else(|| OverflowError::new::<usize>("total length"))?;
157 out.push(O::usize_as(acc))
158 }
159 O::from_usize(acc).ok_or_else(|| OverflowError::new::<O>("offset").with_value(acc))?;
161 Ok(Self(out.into()))
162 }
163
164 pub fn from_repeated_length(length: usize, n: usize) -> Self {
179 Self::try_from_repeated_length(length, n).unwrap_or_else(|err| panic!("{err}"))
180 }
181
182 pub fn try_from_repeated_length(length: usize, n: usize) -> Result<Self, OverflowError> {
195 if n == 0 {
196 return Ok(Self::new_empty());
197 }
198
199 if length == 0 {
200 return Self::try_new_zeroed(n);
201 }
202
203 let total_length = length
205 .checked_mul(n)
206 .ok_or_else(|| OverflowError::new::<usize>("total length"))?;
207
208 O::from_usize(total_length)
209 .ok_or_else(|| OverflowError::new::<O>("offset").with_value(total_length))?;
210
211 let len_bytes = n
212 .checked_add(1)
213 .and_then(|o| o.checked_mul(std::mem::size_of::<O>()))
214 .ok_or_else(|| OverflowError::new::<usize>("buffer length"))?;
215
216 let mut buffer = MutableBuffer::try_with_capacity(len_bytes)
217 .map_err(|_| OverflowError::new::<usize>("buffer length"))?;
218
219 for index in 0..=n {
220 unsafe {
222 buffer.push_unchecked(O::usize_as(index * length));
223 }
224 }
225
226 Ok(Self(buffer.into_buffer().into()))
227 }
228
229 #[inline]
240 pub fn first(&self) -> O {
241 self.0
242 .first()
243 .copied()
244 .expect("An `OffsetBuffer` is never empty")
245 }
246
247 #[inline]
258 pub fn last(&self) -> O {
259 self.0
260 .last()
261 .copied()
262 .expect("An `OffsetBuffer` is never empty")
263 }
264
265 pub fn lengths(&self) -> impl ExactSizeIterator<Item = usize> + '_ {
294 self.0.windows(2).map(|x| x[1].as_usize() - x[0].as_usize())
295 }
296
297 pub fn shrink_to_fit(&mut self) {
299 self.0.shrink_to_fit();
300 }
301
302 pub fn inner(&self) -> &ScalarBuffer<O> {
304 &self.0
305 }
306
307 pub fn into_inner(self) -> ScalarBuffer<O> {
309 self.0
310 }
311
312 #[cfg(feature = "pool")]
314 pub fn claim(&self, pool: &dyn crate::MemoryPool) {
315 self.0.claim(pool);
316 }
317
318 pub fn slice(&self, offset: usize, len: usize) -> Self {
324 Self(self.0.slice(offset, len.saturating_add(1)))
325 }
326
327 #[inline]
331 pub fn ptr_eq(&self, other: &Self) -> bool {
332 self.0.ptr_eq(&other.0)
333 }
334
335 pub fn has_non_empty_nulls(&self, null_buffer: Option<&NullBuffer>) -> bool {
371 let Some(null_buffer) = null_buffer else {
372 return false;
373 };
374
375 assert_eq!(
376 self.len() - 1,
377 null_buffer.len(),
378 "The length of the offsets should be 1 more than the length of the null buffer"
379 );
380
381 if null_buffer.null_count() == 0 {
382 return false;
383 }
384
385 let initial_offset = self[0];
387 let last_offset = self[self.len() - 1];
388
389 if null_buffer.null_count() == self.len() - 1 {
391 return last_offset != initial_offset;
392 }
393
394 let mut valid_slices_iter = null_buffer.valid_slices();
395
396 let (start, end) = valid_slices_iter.next().unwrap();
398
399 if self[start] != initial_offset {
401 return true;
402 }
403
404 let mut end_offset_of_last_valid_value = self[end];
407
408 for (start, end) in valid_slices_iter {
409 if self[start] != end_offset_of_last_valid_value {
412 return true;
413 }
414
415 end_offset_of_last_valid_value = self[end];
418 }
419
420 end_offset_of_last_valid_value != last_offset
421 }
422
423 pub fn subtract(self, rhs: O) -> Self
445 where
446 O: std::ops::Sub<Output = O> + std::cmp::PartialOrd + num_traits::CheckedSub,
447 {
448 if rhs == O::usize_as(0) {
449 return self;
450 }
451
452 let len = self.len();
453
454 assert!(
456 self[0] >= rhs,
457 "shifted offsets will become negative which is not allowed"
458 );
459
460 if rhs < O::usize_as(0) {
462 self[len - 1].checked_sub(&rhs).expect("must not overflow");
463 }
464
465 let shifted_offsets: Vec<O> = match self.into_inner().into_inner().into_vec() {
467 Ok(mut v) => {
469 for offset in &mut v {
470 *offset = *offset - rhs;
471 }
472 v
473 }
474 Err(buffer) => {
476 let offsets = ScalarBuffer::<O>::from(buffer);
477 offsets.iter().map(|offset| *offset - rhs).collect()
478 }
479 };
480 let shifted_buffer = ScalarBuffer::from(shifted_offsets);
481 unsafe { Self::new_unchecked(shifted_buffer) }
486 }
487}
488
489impl<T: ArrowNativeType> Deref for OffsetBuffer<T> {
490 type Target = [T];
491
492 #[inline]
493 fn deref(&self) -> &Self::Target {
494 &self.0
495 }
496}
497
498impl<T: ArrowNativeType> AsRef<[T]> for OffsetBuffer<T> {
499 #[inline]
500 fn as_ref(&self) -> &[T] {
501 self
502 }
503}
504
505impl<O: ArrowNativeType> From<OffsetBufferBuilder<O>> for OffsetBuffer<O> {
506 fn from(value: OffsetBufferBuilder<O>) -> Self {
507 value.finish()
508 }
509}
510
511impl<O: ArrowNativeType> Default for OffsetBuffer<O> {
512 fn default() -> Self {
513 Self::new_empty()
514 }
515}
516
517#[cfg(test)]
518mod tests {
519 use super::*;
520
521 #[test]
522 #[should_panic(expected = "offsets cannot be empty")]
523 fn empty_offsets() {
524 OffsetBuffer::new(Vec::<i32>::new().into());
525 }
526
527 #[test]
528 #[should_panic(expected = "offsets must be greater than 0")]
529 fn negative_offsets() {
530 OffsetBuffer::new(vec![-1, 0, 1].into());
531 }
532
533 #[test]
534 fn try_from_lengths_overflow() {
535 let lengths = [u32::MAX as usize, 1];
537
538 let err = OffsetBuffer::<i32>::try_from_lengths(lengths).unwrap_err();
539 assert_eq!(
540 err.to_string(),
541 "offset overflow: 4294967296 does not fit in i32"
542 );
543 assert!(OffsetBuffer::<i64>::try_from_lengths(lengths).is_ok());
544
545 let err = OffsetBuffer::<i32>::try_from_lengths([usize::MAX, 1]).unwrap_err();
546 assert_eq!(
547 err.to_string(),
548 "total length overflow: does not fit in usize"
549 );
550
551 assert!(std::panic::catch_unwind(|| OffsetBuffer::<i32>::from_lengths(lengths)).is_err());
553 }
554
555 #[test]
556 fn try_from_repeated_length_overflow() {
557 assert_eq!(
558 OffsetBuffer::<i32>::try_from_repeated_length(2, usize::MAX)
559 .unwrap_err()
560 .to_string(),
561 "total length overflow: does not fit in usize"
562 );
563 assert_eq!(
564 OffsetBuffer::<i32>::try_from_repeated_length(u32::MAX as usize, 2)
565 .unwrap_err()
566 .to_string(),
567 "offset overflow: 8589934590 does not fit in i32"
568 );
569 assert_eq!(
570 OffsetBuffer::<i32>::try_from_repeated_length(4, 3)
571 .unwrap()
572 .as_ref(),
573 &[0, 4, 8, 12]
574 );
575 }
576
577 #[test]
578 fn try_new_zeroed_overflow() {
579 assert_eq!(
580 OffsetBuffer::<i64>::try_new_zeroed(usize::MAX)
581 .unwrap_err()
582 .to_string(),
583 "buffer length overflow: does not fit in usize"
584 );
585 let max_layout_len = isize::MAX as usize / std::mem::size_of::<i64>();
586 assert!(OffsetBuffer::<i64>::try_new_zeroed(max_layout_len).is_err());
587 assert_eq!(
588 OffsetBuffer::<i32>::try_new_zeroed(3).unwrap().as_ref(),
589 &[0; 4]
590 );
591 }
592
593 #[test]
594 fn offsets() {
595 OffsetBuffer::new(vec![0, 1, 2, 3].into());
596
597 let offsets = OffsetBuffer::<i32>::new_zeroed(3);
598 assert_eq!(offsets.as_ref(), &[0; 4]);
599
600 let offsets = OffsetBuffer::<i32>::new_zeroed(0);
601 assert_eq!(offsets.as_ref(), &[0; 1]);
602 }
603
604 #[test]
605 #[should_panic(expected = "overflow")]
606 fn offsets_new_zeroed_overflow() {
607 OffsetBuffer::<i32>::new_zeroed(usize::MAX);
608 }
609
610 #[test]
611 #[should_panic(expected = "offsets must be monotonically increasing")]
612 fn non_monotonic_offsets() {
613 OffsetBuffer::new(vec![1, 2, 0].into());
614 }
615
616 #[test]
617 fn from_lengths() {
618 let buffer = OffsetBuffer::<i32>::from_lengths([2, 6, 3, 7, 2]);
619 assert_eq!(buffer.as_ref(), &[0, 2, 8, 11, 18, 20]);
620
621 let half_max = i32::MAX / 2;
622 let buffer = OffsetBuffer::<i32>::from_lengths([half_max as usize, half_max as usize]);
623 assert_eq!(buffer.as_ref(), &[0, half_max, half_max * 2]);
624 }
625
626 #[test]
627 #[should_panic(expected = "offset overflow")]
628 fn from_lengths_offset_overflow() {
629 OffsetBuffer::<i32>::from_lengths([i32::MAX as usize, 1]);
630 }
631
632 #[test]
633 #[should_panic(expected = "total length overflow: does not fit in usize")]
634 fn from_lengths_usize_overflow() {
635 OffsetBuffer::<i32>::from_lengths([usize::MAX, 1]);
636 }
637
638 #[test]
639 #[should_panic(expected = "offset overflow")]
640 fn from_repeated_lengths_offset_length_overflow() {
641 OffsetBuffer::<i32>::from_repeated_length(i32::MAX as usize / 4, 5);
642 }
643
644 #[test]
645 #[should_panic(expected = "offset overflow")]
646 fn from_repeated_lengths_offset_repeat_overflow() {
647 OffsetBuffer::<i32>::from_repeated_length(1, i32::MAX as usize + 1);
648 }
649
650 #[test]
651 #[should_panic(expected = "offset overflow")]
652 fn from_repeated_lengths_usize_length_overflow() {
653 OffsetBuffer::<i32>::from_repeated_length(usize::MAX, 1);
654 }
655
656 #[test]
657 #[should_panic(expected = "total length overflow: does not fit in usize")]
658 fn from_repeated_lengths_usize_length_usize_overflow() {
659 OffsetBuffer::<i32>::from_repeated_length(usize::MAX, 2);
660 }
661
662 #[test]
663 #[should_panic(expected = "offset overflow")]
664 fn from_repeated_lengths_usize_repeat_overflow() {
665 OffsetBuffer::<i32>::from_repeated_length(1, usize::MAX);
666 }
667
668 #[test]
669 fn get_lengths() {
670 let offsets = OffsetBuffer::<i32>::new(ScalarBuffer::<i32>::from(vec![0, 1, 4, 9]));
671 assert_eq!(offsets.lengths().collect::<Vec<usize>>(), vec![1, 3, 5]);
672 }
673
674 #[test]
675 fn get_lengths_should_be_with_fixed_size() {
676 let offsets = OffsetBuffer::<i32>::new(ScalarBuffer::<i32>::from(vec![0, 1, 4, 9]));
677 let iter = offsets.lengths();
678 assert_eq!(iter.size_hint(), (3, Some(3)));
679 assert_eq!(iter.len(), 3);
680 }
681
682 #[test]
683 fn get_lengths_from_empty_offset_buffer_should_be_empty_iterator() {
684 let offsets = OffsetBuffer::<i32>::new_empty();
685 assert_eq!(offsets.lengths().collect::<Vec<usize>>(), vec![]);
686 }
687
688 #[test]
689 fn impl_eq() {
690 fn are_equal<T: Eq>(a: &T, b: &T) -> bool {
691 a.eq(b)
692 }
693
694 assert!(
695 are_equal(
696 &OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 1, 4, 9])),
697 &OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 1, 4, 9]))
698 ),
699 "OffsetBuffer should implement Eq."
700 );
701 }
702
703 #[test]
704 fn impl_default() {
705 let default = OffsetBuffer::<i32>::default();
706 assert_eq!(default.as_ref(), &[0]);
707 }
708
709 #[test]
710 fn from_repeated_length_basic() {
711 let buffer = OffsetBuffer::<i32>::from_repeated_length(4, 3);
713 assert_eq!(buffer.as_ref(), &[0, 4, 8, 12]);
714
715 let lengths: Vec<usize> = buffer.lengths().collect();
717 assert_eq!(lengths, vec![4, 4, 4]);
718 }
719
720 #[test]
721 fn from_repeated_length_single_repeat() {
722 let buffer = OffsetBuffer::<i32>::from_repeated_length(5, 1);
724 assert_eq!(buffer.as_ref(), &[0, 5]);
725
726 let lengths: Vec<usize> = buffer.lengths().collect();
727 assert_eq!(lengths, vec![5]);
728 }
729
730 #[test]
731 fn from_repeated_length_zero_repeats() {
732 let buffer = OffsetBuffer::<i32>::from_repeated_length(10, 0);
733 assert_eq!(buffer, OffsetBuffer::<i32>::new_empty());
734 }
735
736 #[test]
737 fn from_repeated_length_zero_length() {
738 let buffer = OffsetBuffer::<i32>::from_repeated_length(0, 5);
740 assert_eq!(buffer.as_ref(), &[0, 0, 0, 0, 0, 0]);
741
742 let lengths: Vec<usize> = buffer.lengths().collect();
744 assert_eq!(lengths, vec![0, 0, 0, 0, 0]);
745 }
746
747 #[test]
748 fn from_repeated_length_large_values() {
749 let buffer = OffsetBuffer::<i32>::from_repeated_length(1000, 100);
751 assert_eq!(buffer[0], 0);
752
753 let lengths: Vec<usize> = buffer.lengths().collect();
755 assert_eq!(lengths.len(), 100);
756 assert!(lengths.iter().all(|&len| len == 1000));
757 }
758
759 #[test]
760 fn from_repeated_length_unit_length() {
761 let buffer = OffsetBuffer::<i32>::from_repeated_length(1, 10);
763 assert_eq!(buffer.as_ref(), &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
764
765 let lengths: Vec<usize> = buffer.lengths().collect();
766 assert_eq!(lengths, vec![1; 10]);
767 }
768
769 #[test]
770 fn from_repeated_length_max_safe_values() {
771 let third_max = (i32::MAX / 3) as usize;
774 let buffer = OffsetBuffer::<i32>::from_repeated_length(third_max, 2);
775 assert_eq!(
776 buffer.as_ref(),
777 &[0, third_max as i32, (third_max * 2) as i32]
778 );
779 }
780
781 #[test]
786 fn has_non_empty_nulls_none_null_buffer() {
787 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 5, 8]));
789 assert!(!offsets.has_non_empty_nulls(None));
790 }
791
792 #[test]
793 fn has_non_empty_nulls_all_valid() {
794 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 5, 8]));
796 let nulls = NullBuffer::new_valid(3);
797 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
798 }
799
800 #[test]
801 fn has_non_empty_nulls_all_null_empty_offsets() {
802 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 0, 0, 0]));
804 let nulls = NullBuffer::new_null(3);
805 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
806 }
807
808 #[test]
809 fn has_non_empty_nulls_all_null_non_empty_offsets() {
810 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 2, 5, 7]));
812 let nulls = NullBuffer::new_null(3);
813 assert!(offsets.has_non_empty_nulls(Some(&nulls)));
814 }
815
816 #[test]
817 fn has_non_empty_nulls_all_null_nonzero_but_equal_offsets() {
818 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![5, 5, 5]));
820 let nulls = NullBuffer::new_null(2);
821 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
822 }
823
824 #[test]
825 fn has_non_empty_nulls_leading_nulls_with_data() {
826 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 5, 8]));
830 let nulls = NullBuffer::from(vec![false, true, true]);
831 assert!(offsets.has_non_empty_nulls(Some(&nulls)));
832 }
833
834 #[test]
835 fn has_non_empty_nulls_leading_nulls_without_data() {
836 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 0, 3, 6]));
840 let nulls = NullBuffer::from(vec![false, true, true]);
841 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
842 }
843
844 #[test]
845 fn has_non_empty_nulls_only_trailing_null_has_data() {
846 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 0, 3, 6, 8]));
850 let nulls = NullBuffer::from(vec![false, true, true, false]);
851 assert!(offsets.has_non_empty_nulls(Some(&nulls)));
852 }
853
854 #[test]
855 fn has_non_empty_nulls_trailing_nulls_without_data() {
856 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 6, 6]));
860 let nulls = NullBuffer::from(vec![true, true, false]);
861 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
862 }
863
864 #[test]
865 fn has_non_empty_nulls_middle_nulls_with_data() {
866 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 7, 10]));
870 let nulls = NullBuffer::from(vec![true, false, true]);
871 assert!(offsets.has_non_empty_nulls(Some(&nulls)));
872 }
873
874 #[test]
875 fn has_non_empty_nulls_middle_nulls_without_data() {
876 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 3, 6]));
880 let nulls = NullBuffer::from(vec![true, false, true]);
881 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
882 }
883
884 #[test]
885 fn has_non_empty_nulls_alternating_null_valid_all_empty() {
886 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 0, 3, 3, 6, 6]));
890 let nulls = NullBuffer::from(vec![false, true, false, true, false]);
891 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
892
893 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 0, 3, 3, 6, 6, 9]));
895 let nulls = NullBuffer::from(vec![false, true, false, true, false, true]);
896 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
897 }
898
899 #[test]
900 fn has_non_empty_nulls_multiple_null_regions_second_has_data() {
901 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 0, 3, 5, 6]));
905 let nulls = NullBuffer::from(vec![false, true, false, true]);
906 assert!(offsets.has_non_empty_nulls(Some(&nulls)));
907 }
908
909 #[test]
910 fn has_non_empty_nulls_multiple_null_regions_later_gap_has_data() {
911 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 0, 3, 3, 6, 8, 10]));
918 let nulls = NullBuffer::from(vec![false, true, false, true, false, true]);
919 assert!(offsets.has_non_empty_nulls(Some(&nulls)));
920 }
921
922 #[test]
923 fn has_non_empty_nulls_single_element_null_empty() {
924 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 0]));
926 let nulls = NullBuffer::new_null(1);
927 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
928 }
929
930 #[test]
931 fn has_non_empty_nulls_single_element_null_non_empty() {
932 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 5]));
934 let nulls = NullBuffer::new_null(1);
935 assert!(offsets.has_non_empty_nulls(Some(&nulls)));
936 }
937
938 #[test]
939 fn has_non_empty_nulls_single_element_valid() {
940 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 5]));
942 let nulls = NullBuffer::new_valid(1);
943 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
944 }
945
946 #[test]
947 fn has_non_empty_nulls_consecutive_nulls_between_valid_slices() {
948 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 2, 2, 2, 5, 8]));
953 let nulls = NullBuffer::from(vec![true, false, false, true, true]);
954 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
955 }
956
957 #[test]
958 fn has_non_empty_nulls_consecutive_nulls_between_valid_slices_with_data() {
959 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 2, 3, 4, 5, 8]));
966 let nulls = NullBuffer::from(vec![true, false, false, true, true]);
967 assert!(offsets.has_non_empty_nulls(Some(&nulls)));
968 }
969
970 #[test]
971 fn has_non_empty_nulls_nonzero_initial_offset_all_null_equal() {
972 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![10, 10, 10]));
974 let nulls = NullBuffer::new_null(2);
975 assert!(!offsets.has_non_empty_nulls(Some(&nulls)));
976 }
977
978 #[test]
979 fn has_non_empty_nulls_nonzero_initial_offset_with_data() {
980 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![10, 15, 20]));
984 let nulls = NullBuffer::from(vec![false, true]);
985 assert!(offsets.has_non_empty_nulls(Some(&nulls)));
986 }
987
988 #[test]
989 fn has_non_empty_nulls_sliced_no_nulls_in_null_region() {
990 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 3, 6, 6, 9]));
994 let sliced = offsets.slice(1, 3);
995 let nulls = NullBuffer::from(vec![false, true, false]);
996 assert!(!sliced.has_non_empty_nulls(Some(&nulls)));
997 }
998
999 #[test]
1000 fn has_non_empty_nulls_sliced_null_has_data() {
1001 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 7, 10, 15]));
1005 let sliced = offsets.slice(1, 2);
1006 let nulls = NullBuffer::from(vec![false, true]);
1007 assert!(sliced.has_non_empty_nulls(Some(&nulls)));
1008 }
1009
1010 #[test]
1011 #[should_panic(
1012 expected = "The length of the offsets should be 1 more than the length of the null buffer"
1013 )]
1014 fn has_non_empty_nulls_all_valid_mismatched_lengths_too_short() {
1015 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 5, 8]));
1017 let nulls = NullBuffer::new_valid(2); offsets.has_non_empty_nulls(Some(&nulls));
1019 }
1020
1021 #[test]
1022 #[should_panic(
1023 expected = "The length of the offsets should be 1 more than the length of the null buffer"
1024 )]
1025 fn has_non_empty_nulls_all_valid_mismatched_lengths_too_long() {
1026 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 5, 8]));
1028 let nulls = NullBuffer::new_valid(5); offsets.has_non_empty_nulls(Some(&nulls));
1030 }
1031
1032 #[test]
1033 #[should_panic(expected = "shifted offsets will become negative which is not allowed")]
1034 fn should_panic_for_subtract_by_value_that_will_cause_offsets_to_be_less_than_zero() {
1035 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 6]));
1037 offsets.subtract(1);
1038 }
1039
1040 #[test]
1041 fn subtract_by_value_that_will_cause_offsets_to_be_less_than_zero_for_outside_the_slice() {
1042 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 1, 4, 7]));
1043 let sliced = offsets.slice(1, 2); drop(offsets);
1045 assert_eq!(sliced.as_ref(), &[1, 4, 7]);
1046
1047 let result = sliced.subtract(1);
1048 assert_eq!(result.as_ref(), &[0, 3, 6]);
1049 assert_eq!(result.len(), 3);
1050 }
1051
1052 #[test]
1053 #[should_panic(expected = "must not overflow")]
1054 fn should_panic_subtract_by_value_that_will_cause_offsets_to_overflow() {
1055 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 5, i32::MAX]));
1058 offsets.subtract(-1);
1059 }
1060
1061 #[test]
1062 fn subtract_by_value_that_will_cause_offsets_to_overflow_outside_the_slice() {
1063 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 6, i32::MAX]));
1064 let sliced = offsets.slice(0, 2); assert_eq!(sliced.as_ref(), &[0, 3, 6]);
1066
1067 let result = sliced.subtract(-1);
1068 assert_eq!(result.as_ref(), &[1, 4, 7]);
1069 assert_eq!(result.len(), 3);
1070 }
1071
1072 #[test]
1073 fn when_shift_is_0_subtract_should_reuse_the_buffer_even_when_it_is_shared() {
1074 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 6]));
1077 let shared = offsets.clone(); let result = offsets.subtract(0);
1079 assert!(
1080 result.ptr_eq(&shared),
1081 "subtract(0) must return the same underlying buffer, even when shared"
1082 );
1083 }
1084
1085 #[test]
1086 fn should_reuse_the_underline_data_when_the_buffer_is_not_shared() {
1087 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![2, 5, 8]));
1090 let ptr_before = offsets.as_ptr();
1091 let result = offsets.subtract(2);
1092 assert_eq!(
1093 ptr_before,
1094 result.as_ptr(),
1095 "a non-shared buffer should be mutated in place, reusing the allocation"
1096 );
1097 assert_eq!(result.as_ref(), &[0, 3, 6]);
1098 }
1099
1100 #[test]
1101 fn should_create_a_new_buffer_when_the_buffer_is_shared() {
1102 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![2, 5, 8]));
1103 let shared = offsets.clone();
1104 let ptr_before = offsets.as_ptr();
1105 let result = offsets.subtract(2);
1106 assert_ne!(
1107 ptr_before,
1108 result.as_ptr(),
1109 "a shared buffer must not be mutated in place; a new allocation is created"
1110 );
1111 assert_eq!(result.as_ref(), &[0, 3, 6]);
1112 assert_eq!(shared.as_ref(), &[2, 5, 8]);
1114 }
1115
1116 #[test]
1117 fn when_shift_is_negative_it_should_shift_offsets_in_the_right_direction() {
1118 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 6]));
1120 let result = offsets.subtract(-2);
1121 assert_eq!(result.as_ref(), &[2, 5, 8]);
1122 }
1123
1124 #[test]
1126 fn for_sliced_unshared_buffer_shift_should_not_reuse_buffer() {
1127 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![1, 3, 6, 9, 12]));
1129 let sliced = offsets.slice(1, 2); drop(offsets); assert_eq!(sliced.as_ref(), &[3, 6, 9]);
1132
1133 let ptr_before = sliced.as_ptr();
1134 let result = sliced.subtract(1);
1135
1136 assert_ne!(
1137 ptr_before,
1138 result.as_ptr(),
1139 "should not be reused until #10118 is merged"
1140 );
1141
1142 assert_eq!(result.as_ref(), &[2, 5, 8]);
1143 }
1144
1145 #[test]
1146 fn for_sliced_but_start_at_0_unshared_buffer_shift_should_reuse_buffer() {
1147 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![1, 3, 6, 9, 12]));
1148 let sliced = offsets.slice(0, 2);
1149 drop(offsets); assert_eq!(sliced.as_ref(), &[1, 3, 6]);
1151
1152 let ptr_before = sliced.as_ptr();
1153 let result = sliced.subtract(1);
1154
1155 assert_eq!(ptr_before, result.as_ptr(), "should be reused");
1156
1157 assert_eq!(result.as_ref(), &[0, 2, 5]);
1158 }
1159
1160 #[test]
1161 fn for_sliced_shared_buffer_shifted_buffer_should_only_include_the_sliced_data() {
1162 let offsets = OffsetBuffer::new(ScalarBuffer::<i32>::from(vec![0, 3, 6, 9, 12]));
1167 let sliced = offsets.slice(1, 2);
1168 assert_eq!(sliced.as_ref(), &[3, 6, 9]);
1169
1170 let result = sliced.subtract(3);
1171
1172 assert_eq!(
1173 result.as_ref(),
1174 &[0, 3, 6],
1175 "shifted result should contain only the sliced data"
1176 );
1177 assert_eq!(result.len(), 3);
1178
1179 let underlying_buffer = result.inner().inner();
1181 assert_eq!(underlying_buffer.ptr_offset(), 0);
1182 assert_eq!(underlying_buffer.len(), 3 * std::mem::size_of::<i32>());
1183 }
1184}