Skip to main content

arrow_array/array/
byte_array.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::array::{get_offsets_from_buffer, print_long_array};
6use crate::builder::GenericByteBuilder;
7use crate::iterator::ArrayIter;
8use crate::types::ByteArrayType;
9use crate::types::bytes::ByteArrayNativeType;
10use crate::{Array, ArrayAccessor, ArrayRef, OffsetSizeTrait, Scalar};
11use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
12use arrow_buffer::{NullBuffer, OffsetBuffer};
13use arrow_data::{ArrayData, ArrayDataBuilder};
14use arrow_schema::{ArrowError, DataType};
15use std::any::Any;
16use std::sync::Arc;
17
18/// An array of [variable length byte arrays](https://arrow.apache.org/docs/format/Columnar.html#variable-size-binary-layout)
19///
20/// See [`StringArray`] and [`LargeStringArray`] for storing utf8 encoded string data
21///
22/// See [`BinaryArray`] and [`LargeBinaryArray`] for storing arbitrary bytes
23///
24/// # Example: From a Vec
25///
26/// ```
27/// # use arrow_array::{Array, GenericByteArray, types::Utf8Type};
28/// let arr: GenericByteArray<Utf8Type> = vec!["hello", "world", ""].into();
29/// assert_eq!(arr.value_data(), b"helloworld");
30/// assert_eq!(arr.value_offsets(), &[0, 5, 10, 10]);
31/// let values: Vec<_> = arr.iter().collect();
32/// assert_eq!(values, &[Some("hello"), Some("world"), Some("")]);
33/// ```
34///
35/// # Example: From an optional Vec
36///
37/// ```
38/// # use arrow_array::{Array, GenericByteArray, types::Utf8Type};
39/// let arr: GenericByteArray<Utf8Type> = vec![Some("hello"), Some("world"), Some(""), None].into();
40/// assert_eq!(arr.value_data(), b"helloworld");
41/// assert_eq!(arr.value_offsets(), &[0, 5, 10, 10, 10]);
42/// let values: Vec<_> = arr.iter().collect();
43/// assert_eq!(values, &[Some("hello"), Some("world"), Some(""), None]);
44/// ```
45///
46/// # Example: From an iterator of option
47///
48/// ```
49/// # use arrow_array::{Array, GenericByteArray, types::Utf8Type};
50/// let arr: GenericByteArray<Utf8Type> = (0..5).map(|x| (x % 2 == 0).then(|| x.to_string())).collect();
51/// let values: Vec<_> = arr.iter().collect();
52/// assert_eq!(values, &[Some("0"), None, Some("2"), None, Some("4")]);
53/// ```
54///
55/// # Example: Using Builder
56///
57/// ```
58/// # use arrow_array::Array;
59/// # use arrow_array::builder::GenericByteBuilder;
60/// # use arrow_array::types::Utf8Type;
61/// let mut builder = GenericByteBuilder::<Utf8Type>::new();
62/// builder.append_value("hello");
63/// builder.append_null();
64/// builder.append_value("world");
65/// let array = builder.finish();
66/// let values: Vec<_> = array.iter().collect();
67/// assert_eq!(values, &[Some("hello"), None, Some("world")]);
68/// ```
69///
70/// [`StringArray`]: crate::StringArray
71/// [`LargeStringArray`]: crate::LargeStringArray
72/// [`BinaryArray`]: crate::BinaryArray
73/// [`LargeBinaryArray`]: crate::LargeBinaryArray
74pub struct GenericByteArray<T: ByteArrayType> {
75    data_type: DataType,
76    value_offsets: OffsetBuffer<T::Offset>,
77    value_data: Buffer,
78    nulls: Option<NullBuffer>,
79}
80
81impl<T: ByteArrayType> Clone for GenericByteArray<T> {
82    fn clone(&self) -> Self {
83        Self {
84            data_type: T::DATA_TYPE,
85            value_offsets: self.value_offsets.clone(),
86            value_data: self.value_data.clone(),
87            nulls: self.nulls.clone(),
88        }
89    }
90}
91
92impl<T: ByteArrayType> GenericByteArray<T> {
93    /// Data type of the array.
94    pub const DATA_TYPE: DataType = T::DATA_TYPE;
95
96    /// Create a new [`GenericByteArray`] from the provided parts, panicking on failure
97    ///
98    /// # Panics
99    ///
100    /// Panics if [`GenericByteArray::try_new`] returns an error
101    pub fn new(
102        offsets: OffsetBuffer<T::Offset>,
103        values: Buffer,
104        nulls: Option<NullBuffer>,
105    ) -> Self {
106        Self::try_new(offsets, values, nulls).unwrap()
107    }
108
109    /// Create a new [`GenericByteArray`] from the provided parts, returning an error on failure
110    ///
111    /// # Errors
112    ///
113    /// * `offsets.len() - 1 != nulls.len()`
114    /// * Any consecutive pair of `offsets` does not denote a valid slice of `values`
115    pub fn try_new(
116        offsets: OffsetBuffer<T::Offset>,
117        values: Buffer,
118        nulls: Option<NullBuffer>,
119    ) -> Result<Self, ArrowError> {
120        let len = offsets.len() - 1;
121
122        // Verify that each pair of offsets is a valid slices of values
123        T::validate(&offsets, &values)?;
124
125        if let Some(n) = nulls.as_ref()
126            && n.len() != len
127        {
128            return Err(ArrowError::InvalidArgumentError(format!(
129                "Incorrect length of null buffer for {}{}Array, expected {len} got {}",
130                T::Offset::PREFIX,
131                T::PREFIX,
132                n.len(),
133            )));
134        }
135
136        Ok(Self {
137            data_type: T::DATA_TYPE,
138            value_offsets: offsets,
139            value_data: values,
140            nulls,
141        })
142    }
143
144    /// Create a new [`GenericByteArray`] from the provided parts, without validation
145    ///
146    /// # Safety
147    ///
148    /// Safe if [`Self::try_new`] would not error
149    pub unsafe fn new_unchecked(
150        offsets: OffsetBuffer<T::Offset>,
151        values: Buffer,
152        nulls: Option<NullBuffer>,
153    ) -> Self {
154        if cfg!(feature = "force_validate") {
155            return Self::new(offsets, values, nulls);
156        }
157        Self {
158            data_type: T::DATA_TYPE,
159            value_offsets: offsets,
160            value_data: values,
161            nulls,
162        }
163    }
164
165    /// Create a new [`GenericByteArray`] of length `len` where all values are null
166    pub fn new_null(len: usize) -> Self {
167        Self {
168            data_type: T::DATA_TYPE,
169            value_offsets: OffsetBuffer::new_zeroed(len),
170            value_data: MutableBuffer::new(0).into(),
171            nulls: Some(NullBuffer::new_null(len)),
172        }
173    }
174
175    /// Create a new [`Scalar`] from `v`
176    pub fn new_scalar(value: impl AsRef<T::Native>) -> Scalar<Self> {
177        Scalar::new(Self::from_iter_values(std::iter::once(value)))
178    }
179
180    /// Create a new [`GenericByteArray`] where `value` is repeated `repeat_count` times.
181    ///
182    /// # Panics
183    ///
184    /// Panics if value's length multiplied by `repeat_count` overflows `usize`
185    /// or exceeds `T::Offset::MAX`.
186    ///
187    /// Use [`Self::try_new_repeated`] for a fallible version.
188    pub fn new_repeated(value: impl AsRef<T::Native>, repeat_count: usize) -> Self {
189        let s: &[u8] = value.as_ref().as_ref();
190        let value_offsets = OffsetBuffer::from_repeated_length(s.len(), repeat_count);
191        let bytes: Buffer = {
192            let mut mutable_buffer = MutableBuffer::with_capacity(0);
193            mutable_buffer.repeat_slice_n_times(s, repeat_count);
194
195            mutable_buffer.into()
196        };
197
198        Self {
199            data_type: T::DATA_TYPE,
200            value_data: bytes,
201            value_offsets,
202            nulls: None,
203        }
204    }
205
206    /// Create a new [`GenericByteArray`] where `value` is repeated `repeat_count` times.
207    ///
208    /// # Errors
209    ///
210    /// Returns an error if value's length multiplied by `repeat_count` overflows `usize`,
211    /// exceeds `T::Offset::MAX`, or if the buffer cannot be allocated.
212    ///
213    /// # Example
214    /// ```
215    /// # use arrow_array::{Array, StringArray};
216    /// let array = StringArray::try_new_repeated("hello", 3).unwrap();
217    /// assert_eq!(array.len(), 3);
218    /// assert_eq!(array.value(0), "hello");
219    /// assert_eq!(array.value(1), "hello");
220    /// assert_eq!(array.value(2), "hello");
221    /// ```
222    pub fn try_new_repeated(
223        value: impl AsRef<T::Native>,
224        repeat_count: usize,
225    ) -> Result<Self, ArrowError> {
226        let s: &[u8] = value.as_ref().as_ref();
227        let value_offsets = OffsetBuffer::try_from_repeated_length(s.len(), repeat_count)
228            .map_err(|err| ArrowError::OffsetOverflowError(err.value().unwrap_or(usize::MAX)))?;
229        let bytes: Buffer = {
230            let mut mutable_buffer = MutableBuffer::with_capacity(0);
231            mutable_buffer
232                .try_repeat_slice_n_times(s, repeat_count)
233                .map_err(|e| ArrowError::MemoryError(e.to_string()))?;
234
235            mutable_buffer.into()
236        };
237
238        Ok(Self {
239            data_type: T::DATA_TYPE,
240            value_data: bytes,
241            value_offsets,
242            nulls: None,
243        })
244    }
245
246    /// Creates a [`GenericByteArray`] based on an iterator of values without nulls
247    ///
248    /// # Panics
249    /// Panics if the total length of the values exceeds `T::Offset::MAX`
250    pub fn from_iter_values<Ptr, I>(iter: I) -> Self
251    where
252        Ptr: AsRef<T::Native>,
253        I: IntoIterator<Item = Ptr>,
254    {
255        let iter = iter.into_iter();
256        // The size hint is only used to pre-allocate: an iterator is free to yield
257        // a different number of items than it reports.
258        let (lower, upper) = iter.size_hint();
259        let capacity = upper.unwrap_or(lower);
260
261        let mut offsets = MutableBuffer::new(
262            capacity
263                .saturating_add(1)
264                .saturating_mul(std::mem::size_of::<T::Offset>()),
265        );
266        offsets.push(T::Offset::usize_as(0));
267
268        let mut values = MutableBuffer::new(0);
269        for s in iter {
270            let s: &[u8] = s.as_ref().as_ref();
271            values.extend_from_slice(s);
272            offsets.push(T::Offset::usize_as(values.len()));
273        }
274
275        T::Offset::from_usize(values.len()).expect("offset overflow");
276        let offsets = Buffer::from(offsets);
277
278        // Safety: valid by construction
279        let value_offsets = unsafe { OffsetBuffer::new_unchecked(offsets.into()) };
280
281        Self {
282            data_type: T::DATA_TYPE,
283            value_data: values.into(),
284            value_offsets,
285            nulls: None,
286        }
287    }
288
289    /// Deconstruct this array into its constituent parts
290    pub fn into_parts(self) -> (OffsetBuffer<T::Offset>, Buffer, Option<NullBuffer>) {
291        (self.value_offsets, self.value_data, self.nulls)
292    }
293
294    /// Returns the length for value at index `i`.
295    /// # Panics
296    /// Panics if index `i` is out of bounds.
297    #[inline]
298    pub fn value_length(&self, i: usize) -> T::Offset {
299        let offsets = self.value_offsets();
300        offsets[i + 1] - offsets[i]
301    }
302
303    /// Returns a reference to the offsets of this array
304    ///
305    /// Unlike [`Self::value_offsets`] this returns the [`OffsetBuffer`]
306    /// allowing for zero-copy cloning
307    #[inline]
308    pub fn offsets(&self) -> &OffsetBuffer<T::Offset> {
309        &self.value_offsets
310    }
311
312    /// Returns the values of this array
313    ///
314    /// Unlike [`Self::value_data`] this returns the [`Buffer`]
315    /// allowing for zero-copy cloning. Like [`Self::value_data`], it can
316    /// include bytes that are not part of any value.
317    #[inline]
318    pub fn values(&self) -> &Buffer {
319        &self.value_data
320    }
321
322    /// Returns the raw value data
323    ///
324    /// This can include bytes before the first offset or after the last one,
325    /// for example when the array is a slice of a larger array. Those bytes are
326    /// not part of any value, and in a string array they need not be valid UTF-8.
327    pub fn value_data(&self) -> &[u8] {
328        self.value_data.as_slice()
329    }
330
331    /// Returns true if all data within this array is ASCII
332    pub fn is_ascii(&self) -> bool {
333        let offsets = &self.value_offsets;
334        self.value_data()[offsets.first().as_usize()..offsets.last().as_usize()].is_ascii()
335    }
336
337    /// Returns the offset values in the offsets buffer
338    #[inline]
339    pub fn value_offsets(&self) -> &[T::Offset] {
340        &self.value_offsets
341    }
342
343    /// Returns the element at index `i`
344    ///
345    /// Note: This method does not check for nulls and the value is arbitrary
346    /// if [`is_null`](Self::is_null) returns true for the index.
347    ///
348    /// # Safety
349    /// Caller is responsible for ensuring that the index is within the bounds of the array
350    pub unsafe fn value_unchecked(&self, i: usize) -> &T::Native {
351        let end = *unsafe { self.value_offsets().get_unchecked(i + 1) };
352        let start = *unsafe { self.value_offsets().get_unchecked(i) };
353
354        // Soundness
355        // pointer alignment & location is ensured by RawPtrBox
356        // buffer bounds/offset is ensured by the value_offset invariants
357
358        // Safety of `to_isize().unwrap()`
359        // `start` and `end` are &OffsetSize, which is a generic type that implements the
360        // OffsetSizeTrait. Currently, only i32 and i64 implement OffsetSizeTrait,
361        // both of which should cleanly cast to isize on an architecture that supports
362        // 32/64-bit offsets
363        let b = unsafe {
364            std::slice::from_raw_parts(
365                self.value_data
366                    .as_ptr()
367                    .offset(start.to_isize().unwrap_unchecked()),
368                (end - start).to_usize().unwrap_unchecked(),
369            )
370        };
371
372        // SAFETY:
373        // ArrayData is valid
374        unsafe { T::Native::from_bytes_unchecked(b) }
375    }
376
377    /// Returns the element at index `i`
378    ///
379    /// Note: This method does not check for nulls and the value is arbitrary
380    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
381    ///
382    /// # Panics
383    /// Panics if index `i` is out of bounds.
384    pub fn value(&self, i: usize) -> &T::Native {
385        assert!(
386            i < self.len(),
387            "Trying to access an element at index {} from a {}{}Array of length {}",
388            i,
389            T::Offset::PREFIX,
390            T::PREFIX,
391            self.len()
392        );
393        // SAFETY:
394        // Verified length above
395        unsafe { self.value_unchecked(i) }
396    }
397
398    /// constructs a new iterator
399    pub fn iter(&self) -> ArrayIter<&Self> {
400        ArrayIter::new(self)
401    }
402
403    /// Returns a zero-copy slice of this array with the indicated offset and length.
404    ///
405    /// # Panics
406    /// Panics if `offset + length > self.len()`
407    pub fn slice(&self, offset: usize, length: usize) -> Self {
408        Self {
409            data_type: T::DATA_TYPE,
410            value_offsets: self.value_offsets.slice(offset, length),
411            value_data: self.value_data.clone(),
412            nulls: self.nulls.as_ref().map(|n| n.slice(offset, length)),
413        }
414    }
415
416    /// Converts this array into a [`GenericByteBuilder`] that reuses its buffers,
417    /// if possible.
418    ///
419    /// Returns `Err(self)` if the offsets or values cannot be reused, for
420    /// example because they are shared with another array, or because this is a
421    /// sliced array with a non-zero first offset.
422    ///
423    /// The null buffer is copied if it cannot be reused.
424    pub fn into_builder(self) -> Result<GenericByteBuilder<T>, Self> {
425        let (offsets, values, nulls) = self.into_parts();
426
427        // Puts the parts back together into the original array
428        let rebuild = |offsets: Buffer, values: Buffer, nulls: Option<NullBuffer>| {
429            // SAFETY: the parts come unchanged from `self`
430            unsafe {
431                Self::new_unchecked(OffsetBuffer::new_unchecked(offsets.into()), values, nulls)
432            }
433        };
434
435        // A buffer can be reused only if this array is its only owner and it
436        // starts at the beginning of its allocation. Otherwise, the original
437        // array is returned.
438        let offsets = match offsets.into_inner().into_inner().into_mutable() {
439            Ok(offsets) => offsets,
440            Err(offsets) => return Err(rebuild(offsets, values, nulls)),
441        };
442        let values = match values.into_mutable() {
443            Ok(values) => values,
444            Err(values) => return Err(rebuild(offsets.into(), values, nulls)),
445        };
446
447        // The builder needs a null buffer whose first bit is for the first row.
448        // That is the case when the bit offset is 0, and then the buffer is reused
449        // if this array is its only owner. Otherwise, the bits are copied, which is
450        // cheap as there is one bit per row.
451        let nulls = nulls.map(|nulls| {
452            let bits = nulls.into_inner();
453            if bits.offset() == 0 {
454                let len = bits.len();
455                bits.into_inner()
456                    .into_mutable()
457                    .unwrap_or_else(|buffer| buffer[..len.div_ceil(8)].to_vec().into())
458            } else {
459                // `sliced` returns the bits starting at the first row
460                bits.sliced().to_vec().into()
461            }
462        });
463
464        // SAFETY: the buffers come from a valid array
465        Ok(unsafe { GenericByteBuilder::new_from_buffer(offsets, values, nulls) })
466    }
467}
468
469impl<T: ByteArrayType> std::fmt::Debug for GenericByteArray<T> {
470    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
471        write!(f, "{}{}Array\n[\n", T::Offset::PREFIX, T::PREFIX)?;
472        print_long_array(self, f, &mut |index, f| {
473            std::fmt::Debug::fmt(&self.value(index), f)
474        })?;
475        write!(f, "]")
476    }
477}
478
479/// SAFETY: Correctly implements the contract of Arrow Arrays
480unsafe impl<T: ByteArrayType> Array for GenericByteArray<T> {
481    fn as_any(&self) -> &dyn Any {
482        self
483    }
484
485    fn to_data(&self) -> ArrayData {
486        self.clone().into()
487    }
488
489    fn into_data(self) -> ArrayData {
490        self.into()
491    }
492
493    fn data_type(&self) -> &DataType {
494        &self.data_type
495    }
496
497    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
498        Arc::new(self.slice(offset, length))
499    }
500
501    fn len(&self) -> usize {
502        self.value_offsets.len() - 1
503    }
504
505    fn is_empty(&self) -> bool {
506        self.value_offsets.len() <= 1
507    }
508
509    fn shrink_to_fit(&mut self) {
510        self.value_offsets.shrink_to_fit();
511        self.value_data.shrink_to_fit();
512        if let Some(nulls) = &mut self.nulls {
513            nulls.shrink_to_fit();
514        }
515    }
516
517    fn offset(&self) -> usize {
518        0
519    }
520
521    fn nulls(&self) -> Option<&NullBuffer> {
522        self.nulls.as_ref()
523    }
524
525    fn logical_null_count(&self) -> usize {
526        // More efficient that the default implementation
527        self.null_count()
528    }
529
530    fn get_buffer_memory_size(&self) -> usize {
531        let mut sum = self.value_offsets.inner().inner().capacity();
532        sum += self.value_data.capacity();
533        if let Some(x) = &self.nulls {
534            sum += x.buffer().capacity()
535        }
536        sum
537    }
538
539    fn get_array_memory_size(&self) -> usize {
540        std::mem::size_of::<Self>() + self.get_buffer_memory_size()
541    }
542
543    #[cfg(feature = "pool")]
544    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
545        self.value_offsets.claim(pool);
546        self.value_data.claim(pool);
547        if let Some(nulls) = &self.nulls {
548            nulls.claim(pool);
549        }
550    }
551}
552
553impl<'a, T: ByteArrayType> ArrayAccessor for &'a GenericByteArray<T> {
554    type Item = &'a T::Native;
555
556    fn value(&self, index: usize) -> Self::Item {
557        GenericByteArray::value(self, index)
558    }
559
560    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
561        unsafe { GenericByteArray::value_unchecked(self, index) }
562    }
563}
564
565impl<T: ByteArrayType> From<ArrayData> for GenericByteArray<T> {
566    fn from(data: ArrayData) -> Self {
567        let (data_type, len, nulls, offset, mut buffers, _child_data) = data.into_parts();
568        assert_eq!(
569            data_type,
570            Self::DATA_TYPE,
571            "{}{}Array expects DataType::{}",
572            T::Offset::PREFIX,
573            T::PREFIX,
574            Self::DATA_TYPE
575        );
576        assert_eq!(
577            buffers.len(),
578            2,
579            "{}{}Array data should contain 2 buffers only (offsets and values)",
580            T::Offset::PREFIX,
581            T::PREFIX,
582        );
583        // buffers are offset then value, so pop in reverse
584        let value_data = buffers.pop().expect("checked above");
585        let offset_buffer = buffers.pop().expect("checked above");
586
587        // SAFETY:
588        // ArrayData is valid, and verified type above
589        let value_offsets = unsafe { get_offsets_from_buffer(offset_buffer, offset, len) };
590        Self {
591            data_type,
592            value_offsets,
593            value_data,
594            nulls,
595        }
596    }
597}
598
599impl<T: ByteArrayType> From<GenericByteArray<T>> for ArrayData {
600    fn from(array: GenericByteArray<T>) -> Self {
601        let len = array.len();
602
603        let offsets = array.value_offsets.into_inner().into_inner();
604        let builder = ArrayDataBuilder::new(array.data_type)
605            .len(len)
606            .buffers(vec![offsets, array.value_data])
607            .nulls(array.nulls);
608
609        unsafe { builder.build_unchecked() }
610    }
611}
612
613impl<'a, T: ByteArrayType> IntoIterator for &'a GenericByteArray<T> {
614    type Item = Option<&'a T::Native>;
615    type IntoIter = ArrayIter<Self>;
616
617    fn into_iter(self) -> Self::IntoIter {
618        ArrayIter::new(self)
619    }
620}
621
622impl<'a, Ptr, T: ByteArrayType> FromIterator<&'a Option<Ptr>> for GenericByteArray<T>
623where
624    Ptr: AsRef<T::Native> + 'a,
625{
626    fn from_iter<I: IntoIterator<Item = &'a Option<Ptr>>>(iter: I) -> Self {
627        iter.into_iter()
628            .map(|o| o.as_ref().map(|p| p.as_ref()))
629            .collect()
630    }
631}
632
633impl<Ptr, T: ByteArrayType> FromIterator<Option<Ptr>> for GenericByteArray<T>
634where
635    Ptr: AsRef<T::Native>,
636{
637    fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
638        let iter = iter.into_iter();
639        let mut builder = GenericByteBuilder::with_capacity(iter.size_hint().0, 1024);
640        builder.extend(iter);
641        builder.finish()
642    }
643}
644
645#[cfg(test)]
646mod tests {
647    use crate::{Array, BinaryArray, LargeBinaryArray, StringArray};
648    use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer};
649    use arrow_schema::ArrowError;
650
651    /// `from_iter_values` must work with iterators that report no upper size bound,
652    /// and must not trust the size hint it does get.
653    #[test]
654    fn from_iter_values_untrusted_size_hint() {
655        // No upper bound at all:
656        let no_upper_bound = (0..20).filter(|i| i % 2 == 0).map(|i| format!("v{i}"));
657        assert_eq!(no_upper_bound.size_hint(), (0, Some(20)));
658        let array = StringArray::from_iter_values(no_upper_bound);
659        assert_eq!(array.len(), 10);
660        assert_eq!(array.value(0), "v0");
661        assert_eq!(array.value(9), "v18");
662
663        // Upper bound larger than the number of yielded values:
664        let too_large_upper_bound = (0..).map(|i| format!("v{i}")).take_while(|v| v != "v3");
665        assert_eq!(too_large_upper_bound.size_hint(), (0, None));
666        let array = StringArray::from_iter_values(too_large_upper_bound);
667        assert_eq!(array.len(), 3);
668        assert_eq!(array.value(2), "v2");
669    }
670
671    #[test]
672    fn try_new() {
673        let data = Buffer::from_slice_ref("helloworld");
674        let offsets = OffsetBuffer::new(vec![0, 5, 10].into());
675        StringArray::new(offsets.clone(), data.clone(), None);
676
677        let nulls = NullBuffer::new_null(3);
678        let err =
679            StringArray::try_new(offsets.clone(), data.clone(), Some(nulls.clone())).unwrap_err();
680        assert_eq!(
681            err.to_string(),
682            "Invalid argument error: Incorrect length of null buffer for StringArray, expected 2 got 3"
683        );
684
685        let err = BinaryArray::try_new(offsets.clone(), data.clone(), Some(nulls)).unwrap_err();
686        assert_eq!(
687            err.to_string(),
688            "Invalid argument error: Incorrect length of null buffer for BinaryArray, expected 2 got 3"
689        );
690
691        let non_utf8_data = Buffer::from_slice_ref(b"he\xFFloworld");
692        let err = StringArray::try_new(offsets.clone(), non_utf8_data.clone(), None).unwrap_err();
693        assert_eq!(
694            err.to_string(),
695            "Invalid argument error: Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 2"
696        );
697
698        BinaryArray::new(offsets, non_utf8_data, None);
699
700        let offsets = OffsetBuffer::new(vec![0, 5, 11].into());
701        let err = StringArray::try_new(offsets.clone(), data.clone(), None).unwrap_err();
702        assert_eq!(
703            err.to_string(),
704            "Invalid argument error: Offset of 11 exceeds length of values 10"
705        );
706
707        let err = BinaryArray::try_new(offsets.clone(), data, None).unwrap_err();
708        assert_eq!(
709            err.to_string(),
710            "Invalid argument error: Maximum offset of 11 is larger than values of length 10"
711        );
712
713        let non_ascii_data = Buffer::from_slice_ref("heìloworld");
714        StringArray::new(offsets.clone(), non_ascii_data.clone(), None);
715        BinaryArray::new(offsets, non_ascii_data.clone(), None);
716
717        let offsets = OffsetBuffer::new(vec![0, 3, 10].into());
718        let err = StringArray::try_new(offsets.clone(), non_ascii_data.clone(), None).unwrap_err();
719        assert_eq!(
720            err.to_string(),
721            "Invalid argument error: Split UTF-8 codepoint at offset 3"
722        );
723
724        BinaryArray::new(offsets, non_ascii_data, None);
725    }
726
727    #[test]
728    fn create_repeated() {
729        let arr = BinaryArray::new_repeated(b"hello", 3);
730        assert_eq!(arr.len(), 3);
731        assert_eq!(arr.value(0), b"hello");
732        assert_eq!(arr.value(1), b"hello");
733        assert_eq!(arr.value(2), b"hello");
734
735        let arr = StringArray::new_repeated("world", 2);
736        assert_eq!(arr.len(), 2);
737        assert_eq!(arr.value(0), "world");
738        assert_eq!(arr.value(1), "world");
739    }
740
741    #[test]
742    #[should_panic(expected = "total length overflow: does not fit in usize")]
743    fn create_repeated_usize_overflow_1() {
744        let _arr = BinaryArray::new_repeated(b"hello", (usize::MAX / "hello".len()) + 1);
745    }
746
747    #[test]
748    #[should_panic(expected = "total length overflow: does not fit in usize")]
749    fn create_repeated_usize_overflow_2() {
750        let _arr = BinaryArray::new_repeated(b"hello", usize::MAX);
751    }
752
753    #[test]
754    #[should_panic(expected = "offset overflow")]
755    fn create_repeated_i32_offset_overflow_1() {
756        let _arr = BinaryArray::new_repeated(b"hello", usize::MAX / "hello".len());
757    }
758
759    #[test]
760    #[should_panic(expected = "offset overflow")]
761    fn create_repeated_i32_offset_overflow_2() {
762        let _arr = BinaryArray::new_repeated(b"hello", ((i32::MAX as usize) / "hello".len()) + 1);
763    }
764
765    #[test]
766    fn try_new_repeated() {
767        // BinaryArray with multiple elements
768        let arr = BinaryArray::try_new_repeated(b"\x00\x01\xff", 3).unwrap();
769        assert_eq!(arr.len(), 3);
770        assert_eq!(arr.null_count(), 0);
771        for i in 0..3 {
772            assert_eq!(arr.value(i), b"\x00\x01\xff");
773        }
774        assert_eq!(arr.value_offsets(), &[0, 3, 6, 9]);
775
776        // Zero repeats
777        let arr = BinaryArray::try_new_repeated(b"test", 0).unwrap();
778        assert_eq!(arr.len(), 0);
779        assert_eq!(arr.value_offsets(), &[0]);
780
781        // Empty values with non-zero repeats
782        let arr = BinaryArray::try_new_repeated(b"", 5).unwrap();
783        assert_eq!(arr.len(), 5);
784        assert_eq!(arr.value_offsets(), &[0, 0, 0, 0, 0, 0]);
785    }
786
787    #[test]
788    fn try_new_repeated_overflow() {
789        // usize multiplication overflow
790        let err =
791            BinaryArray::try_new_repeated(b"hello", (usize::MAX / "hello".len()) + 1).unwrap_err();
792        assert!(matches!(err, ArrowError::OffsetOverflowError(usize::MAX)));
793
794        // i32 offset overflow (fits in usize, exceeds i32::MAX)
795        let err = BinaryArray::try_new_repeated(b"a", (i32::MAX as usize) + 1).unwrap_err();
796        assert!(matches!(
797            err,
798            ArrowError::OffsetOverflowError(val) if val == (i32::MAX as usize) + 1
799        ));
800
801        // i64 offset overflow (fits in usize, exceeds i64::MAX on 64-bit platforms)
802        #[cfg(target_pointer_width = "64")]
803        {
804            // Valid within i64 offset bounds, but offset-buffer allocation overflows memory/usize
805            let err = LargeBinaryArray::try_new_repeated(b"x", i64::MAX as usize).unwrap_err();
806            assert!(matches!(err, ArrowError::OffsetOverflowError(_)));
807
808            let err =
809                LargeBinaryArray::try_new_repeated(b"a", (i64::MAX as usize) + 1).unwrap_err();
810            assert!(matches!(
811                err,
812                ArrowError::OffsetOverflowError(9223372036854775808)
813            ));
814        }
815    }
816}