Skip to main content

arrow_array/builder/
generic_bytes_builder.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5use crate::builder::ArrayBuilder;
6use crate::types::{ByteArrayType, GenericBinaryType, GenericStringType};
7use crate::{Array, ArrayRef, GenericByteArray, OffsetSizeTrait};
8use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, NullBufferBuilder, ScalarBuffer};
9use arrow_data::ArrayDataBuilder;
10use arrow_schema::ArrowError;
11use std::any::Any;
12use std::sync::Arc;
13
14/// Builder for [`GenericByteArray`]
15///
16/// For building strings, see docs on [`GenericStringBuilder`].
17/// For building binary, see docs on [`GenericBinaryBuilder`].
18pub struct GenericByteBuilder<T: ByteArrayType> {
19    value_builder: Vec<u8>,
20    offsets_builder: Vec<T::Offset>,
21    null_buffer_builder: NullBufferBuilder,
22}
23
24impl<T: ByteArrayType> GenericByteBuilder<T> {
25    /// Creates a new [`GenericByteBuilder`].
26    pub fn new() -> Self {
27        Self::with_capacity(1024, 1024)
28    }
29
30    /// Creates a new [`GenericByteBuilder`].
31    ///
32    /// - `item_capacity` is the number of items to pre-allocate.
33    ///   The size of the preallocated buffer of offsets is the number of items plus one.
34    /// - `data_capacity` is the total number of bytes of data to pre-allocate
35    ///   (for all items, not per item).
36    pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {
37        let mut offsets_builder = Vec::with_capacity(item_capacity + 1);
38        offsets_builder.push(T::Offset::from_usize(0).unwrap());
39        Self {
40            value_builder: Vec::with_capacity(data_capacity),
41            offsets_builder,
42            null_buffer_builder: NullBufferBuilder::new(item_capacity),
43        }
44    }
45
46    /// Creates a new  [`GenericByteBuilder`] from buffers.
47    ///
48    /// Any bytes in `value_buffer` after the last offset are dropped. The first
49    /// bit of `null_buffer` is for the first row.
50    ///
51    /// # Safety
52    ///
53    /// This doesn't verify buffer contents as it assumes the buffers are from
54    /// existing and valid [`GenericByteArray`].
55    pub unsafe fn new_from_buffer(
56        offsets_buffer: MutableBuffer,
57        mut value_buffer: MutableBuffer,
58        null_buffer: Option<MutableBuffer>,
59    ) -> Self {
60        let offsets_builder: Vec<T::Offset> =
61            ScalarBuffer::<T::Offset>::from(offsets_buffer).into();
62
63        // New values are appended at the end of the values buffer, so it must end
64        // at the last offset
65        value_buffer.truncate(offsets_builder.last().unwrap().as_usize());
66        let value_builder: Vec<u8> = ScalarBuffer::<u8>::from(value_buffer).into();
67
68        let null_buffer_builder = null_buffer
69            .map(|buffer| NullBufferBuilder::new_from_buffer(buffer, offsets_builder.len() - 1))
70            .unwrap_or_else(|| NullBufferBuilder::new_with_len(offsets_builder.len() - 1));
71
72        Self {
73            value_builder,
74            offsets_builder,
75            null_buffer_builder,
76        }
77    }
78
79    #[inline]
80    fn next_offset(&self) -> T::Offset {
81        T::Offset::from_usize(self.value_builder.len()).expect("byte array offset overflow")
82    }
83
84    /// Appends a value into the builder.
85    ///
86    /// See the [GenericStringBuilder] documentation for examples of
87    /// incrementally building string values with multiple `write!` calls.
88    ///
89    /// # Panics
90    ///
91    /// Panics if the resulting length of [`Self::values_slice`] would exceed
92    /// `T::Offset::MAX` bytes.
93    ///
94    /// For example, this can happen with [`StringArray`] or [`BinaryArray`]
95    /// where the total length of all values exceeds 2GB
96    ///
97    /// [`StringArray`]: crate::StringArray
98    /// [`BinaryArray`]: crate::BinaryArray
99    #[inline]
100    pub fn append_value(&mut self, value: impl AsRef<T::Native>) {
101        self.value_builder
102            .extend_from_slice(value.as_ref().as_ref());
103        self.null_buffer_builder.append(true);
104        self.offsets_builder.push(self.next_offset());
105    }
106
107    /// Appends a value of type `T` into the builder `n` times.
108    ///
109    /// See [`Self::append_value`] for more panic information.
110    ///
111    /// # Panics
112    ///
113    /// Panics for the same reasons as [`Self::append_value`]
114    #[inline]
115    pub fn append_value_n(&mut self, value: impl AsRef<T::Native>, n: usize) {
116        let bytes: &[u8] = value.as_ref().as_ref();
117        self.value_builder.reserve(bytes.len() * n);
118        self.offsets_builder.reserve(n);
119        for _ in 0..n {
120            self.value_builder.extend_from_slice(bytes);
121            self.offsets_builder.push(self.next_offset());
122        }
123        self.null_buffer_builder.append_n_non_nulls(n);
124    }
125
126    /// Append an `Option` value into the builder.
127    ///
128    /// - A `None` value will append a null value.
129    /// - A `Some` value will append the value.
130    ///
131    /// See [`Self::append_value`] for more panic information.
132    ///
133    /// # Panics
134    ///
135    /// Panics for the same reasons as [`Self::append_value`]
136    #[inline]
137    pub fn append_option(&mut self, value: Option<impl AsRef<T::Native>>) {
138        match value {
139            None => self.append_null(),
140            Some(v) => self.append_value(v),
141        }
142    }
143
144    /// Append a null value into the builder.
145    #[inline]
146    pub fn append_null(&mut self) {
147        self.null_buffer_builder.append(false);
148        self.offsets_builder.push(self.next_offset());
149    }
150
151    /// Appends `n` `null`s into the builder.
152    #[inline]
153    pub fn append_nulls(&mut self, n: usize) {
154        self.null_buffer_builder.append_n_nulls(n);
155        let next_offset = self.next_offset();
156        self.offsets_builder
157            .extend(std::iter::repeat_n(next_offset, n));
158    }
159
160    /// Appends array values and null to this builder as is
161    /// (this means that underlying null values are copied as is).
162    #[inline]
163    pub fn append_array(&mut self, array: &GenericByteArray<T>) -> Result<(), ArrowError> {
164        use num_traits::CheckedAdd;
165        if array.len() == 0 {
166            return Ok(());
167        }
168
169        let offsets = array.offsets();
170
171        // If the offsets are contiguous, we can append them directly avoiding the need to align
172        // for example, when the first appended array is not sliced (starts at offset 0)
173        if self.next_offset() == offsets[0] {
174            self.offsets_builder.extend_from_slice(&offsets[1..]);
175        } else {
176            // Shifting all the offsets
177            let shift: T::Offset = self.next_offset() - offsets[0];
178
179            if shift.checked_add(&offsets[offsets.len() - 1]).is_none() {
180                return Err(ArrowError::OffsetOverflowError(
181                    shift.as_usize() + offsets[offsets.len() - 1].as_usize(),
182                ));
183            }
184
185            self.offsets_builder
186                .extend(offsets[1..].iter().map(|&offset| offset + shift));
187        }
188
189        // Append underlying values, starting from the first offset and ending at the last offset
190        self.value_builder.extend_from_slice(
191            &array.values().as_slice()[offsets[0].as_usize()..offsets[array.len()].as_usize()],
192        );
193
194        if let Some(null_buffer) = array.nulls() {
195            self.null_buffer_builder.append_buffer(null_buffer);
196        } else {
197            self.null_buffer_builder.append_n_non_nulls(array.len());
198        }
199        Ok(())
200    }
201
202    /// Builds the [`GenericByteArray`] and reset this builder.
203    pub fn finish(&mut self) -> GenericByteArray<T> {
204        let array_type = T::DATA_TYPE;
205        let array_builder = ArrayDataBuilder::new(array_type)
206            .len(self.len())
207            .add_buffer(std::mem::take(&mut self.offsets_builder).into())
208            .add_buffer(std::mem::take(&mut self.value_builder).into())
209            .nulls(self.null_buffer_builder.finish());
210
211        self.offsets_builder.push(self.next_offset());
212        // SAFETY: builder is constructed from valid offset and value buffers maintained by the builder
213        let array_data = unsafe { array_builder.build_unchecked() };
214        GenericByteArray::from(array_data)
215    }
216
217    /// Builds the [`GenericByteArray`] without resetting the builder.
218    pub fn finish_cloned(&self) -> GenericByteArray<T> {
219        let array_type = T::DATA_TYPE;
220        let offset_buffer = Buffer::from_slice_ref(self.offsets_builder.as_slice());
221        let value_buffer = Buffer::from_slice_ref(self.value_builder.as_slice());
222        let array_builder = ArrayDataBuilder::new(array_type)
223            .len(self.len())
224            .add_buffer(offset_buffer)
225            .add_buffer(value_buffer)
226            .nulls(self.null_buffer_builder.finish_cloned());
227
228        // SAFETY: builder is constructed from valid offset and value buffers maintained by the builder
229        let array_data = unsafe { array_builder.build_unchecked() };
230        GenericByteArray::from(array_data)
231    }
232
233    /// Returns the current values buffer as a slice
234    pub fn values_slice(&self) -> &[u8] {
235        self.value_builder.as_slice()
236    }
237
238    /// Returns the current values buffer capacity, in bytes.
239    pub fn values_capacity(&self) -> usize {
240        self.value_builder.capacity()
241    }
242
243    /// Returns the current offsets buffer as a slice
244    pub fn offsets_slice(&self) -> &[T::Offset] {
245        self.offsets_builder.as_slice()
246    }
247
248    /// Returns the current offsets buffer capacity, in offsets.
249    pub fn offsets_capacity(&self) -> usize {
250        self.offsets_builder.capacity()
251    }
252
253    /// Returns the current null buffer as a slice
254    pub fn validity_slice(&self) -> Option<&[u8]> {
255        self.null_buffer_builder.as_slice()
256    }
257
258    /// Returns the current null buffer allocated capacity, in bytes.
259    pub fn validity_capacity(&self) -> usize {
260        self.null_buffer_builder.allocated_size()
261    }
262
263    /// Returns the current null buffer as a mutable slice
264    pub fn validity_slice_mut(&mut self) -> Option<&mut [u8]> {
265        self.null_buffer_builder.as_slice_mut()
266    }
267}
268
269impl<T: ByteArrayType> std::fmt::Debug for GenericByteBuilder<T> {
270    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
271        write!(f, "{}{}Builder", T::Offset::PREFIX, T::PREFIX)?;
272        f.debug_struct("")
273            .field("value_builder", &self.value_builder)
274            .field("offsets_builder", &self.offsets_builder)
275            .field("null_buffer_builder", &self.null_buffer_builder)
276            .finish()
277    }
278}
279
280impl<T: ByteArrayType> Default for GenericByteBuilder<T> {
281    fn default() -> Self {
282        Self::new()
283    }
284}
285
286impl<T: ByteArrayType> ArrayBuilder for GenericByteBuilder<T> {
287    /// Returns the number of binary slots in the builder
288    fn len(&self) -> usize {
289        self.null_buffer_builder.len()
290    }
291
292    /// Builds the array and reset this builder.
293    fn finish(&mut self) -> ArrayRef {
294        Arc::new(self.finish())
295    }
296
297    /// Builds the array without resetting the builder.
298    fn finish_cloned(&self) -> ArrayRef {
299        Arc::new(self.finish_cloned())
300    }
301
302    /// Returns the builder as a non-mutable `Any` reference.
303    fn as_any(&self) -> &dyn Any {
304        self
305    }
306
307    /// Returns the builder as a mutable `Any` reference.
308    fn as_any_mut(&mut self) -> &mut dyn Any {
309        self
310    }
311
312    /// Returns the boxed builder as a box of `Any`.
313    fn into_box_any(self: Box<Self>) -> Box<dyn Any> {
314        self
315    }
316}
317
318impl<T: ByteArrayType, V: AsRef<T::Native>> Extend<Option<V>> for GenericByteBuilder<T> {
319    #[inline]
320    fn extend<I: IntoIterator<Item = Option<V>>>(&mut self, iter: I) {
321        for v in iter {
322            self.append_option(v)
323        }
324    }
325}
326
327/// Array builder for [`GenericStringArray`][crate::GenericStringArray]
328///
329/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
330/// [`GenericByteBuilder::append_null`].
331///
332/// This builder also implements [`std::fmt::Write`] with any written data
333/// included in the next appended value. This allows using [`std::fmt::Display`]
334/// with standard Rust idioms like `write!` and `writeln!` to write data
335/// directly to the builder without intermediate allocations.
336///
337/// # Example writing strings with `append_value`
338/// ```
339/// # use arrow_array::builder::GenericStringBuilder;
340/// let mut builder = GenericStringBuilder::<i32>::new();
341///
342/// // Write one string value
343/// builder.append_value("foobarbaz");
344///
345/// // Write a second string
346/// builder.append_value("v2");
347///
348/// let array = builder.finish();
349/// assert_eq!(array.value(0), "foobarbaz");
350/// assert_eq!(array.value(1), "v2");
351/// ```
352///
353/// # Example incrementally writing strings with `std::fmt::Write`
354///
355/// ```
356/// # use std::fmt::Write;
357/// # use arrow_array::builder::GenericStringBuilder;
358/// let mut builder = GenericStringBuilder::<i32>::new();
359///
360/// // Write data in multiple `write!` calls
361/// write!(builder, "foo").unwrap();
362/// write!(builder, "bar").unwrap();
363/// // The next call to append_value finishes the current string
364/// // including all previously written strings.
365/// builder.append_value("baz");
366///
367/// // Write second value with a single write call
368/// write!(builder, "v2").unwrap();
369/// // finish the value by calling append_value with an empty string
370/// builder.append_value("");
371///
372/// let array = builder.finish();
373/// assert_eq!(array.value(0), "foobarbaz");
374/// assert_eq!(array.value(1), "v2");
375/// ```
376pub type GenericStringBuilder<O> = GenericByteBuilder<GenericStringType<O>>;
377
378impl<O: OffsetSizeTrait> std::fmt::Write for GenericStringBuilder<O> {
379    fn write_str(&mut self, s: &str) -> std::fmt::Result {
380        self.value_builder.extend_from_slice(s.as_bytes());
381        Ok(())
382    }
383}
384
385/// A byte size value representing the number of bytes to allocate per string in [`GenericStringBuilder`]
386///
387/// To create a [`GenericStringBuilder`] using `.with_capacity` we are required to provide: \
388/// - `item_capacity` - the row count \
389/// - `data_capacity` - total string byte count \
390///
391/// We will use the `AVERAGE_STRING_LENGTH` * row_count for `data_capacity`. \
392///
393/// These capacities are preallocation hints used to improve performance,
394/// but consequences of passing a hint too large or too small should be negligible.
395const AVERAGE_STRING_LENGTH: usize = 16;
396/// Trait for string-like array builders
397///
398/// This trait provides unified interface for builders that append string-like data
399/// such as [`GenericStringBuilder<O>`] and [`crate::builder::StringViewBuilder`]
400pub trait StringLikeArrayBuilder: ArrayBuilder {
401    /// Returns a human-readable type name for the builder.
402    fn type_name() -> &'static str;
403
404    /// Creates a new builder with the given row capacity.
405    fn with_capacity(capacity: usize) -> Self;
406
407    /// Appends a non-null string value to the builder.
408    fn append_value(&mut self, value: &str);
409
410    /// Appends a null value to the builder.
411    fn append_null(&mut self);
412}
413
414impl<O: OffsetSizeTrait> StringLikeArrayBuilder for GenericStringBuilder<O> {
415    fn type_name() -> &'static str {
416        std::any::type_name::<Self>()
417    }
418    fn with_capacity(capacity: usize) -> Self {
419        Self::with_capacity(capacity, capacity * AVERAGE_STRING_LENGTH)
420    }
421    fn append_value(&mut self, value: &str) {
422        Self::append_value(self, value);
423    }
424    fn append_null(&mut self) {
425        Self::append_null(self);
426    }
427}
428
429/// A byte size value representing the number of bytes to allocate per binary in [`GenericBinaryBuilder`]
430///
431/// To create a [`GenericBinaryBuilder`] using `.with_capacity` we are required to provide: \
432/// - `item_capacity` - the row count \
433/// - `data_capacity` - total binary byte count \
434///
435/// We will use the `AVERAGE_BINARY_LENGTH` * row_count for `data_capacity`. \
436///
437/// These capacities are preallocation hints used to improve performance,
438/// but consequences of passing a hint too large or too small should be negligible.
439const AVERAGE_BINARY_LENGTH: usize = 128;
440/// Trait for binary-like array builders
441///
442/// This trait provides unified interface for builders that append binary-like data
443/// such as [`GenericBinaryBuilder<O>`] and [`crate::builder::BinaryViewBuilder`]
444pub trait BinaryLikeArrayBuilder: ArrayBuilder {
445    /// Returns a human-readable type name for the builder.
446    fn type_name() -> &'static str;
447
448    /// Creates a new builder with the given row capacity.
449    fn with_capacity(capacity: usize) -> Self;
450
451    /// Appends a non-null string value to the builder.
452    fn append_value(&mut self, value: &[u8]);
453
454    /// Appends a null value to the builder.
455    fn append_null(&mut self);
456}
457
458impl<O: OffsetSizeTrait> BinaryLikeArrayBuilder for GenericBinaryBuilder<O> {
459    fn type_name() -> &'static str {
460        std::any::type_name::<Self>()
461    }
462    fn with_capacity(capacity: usize) -> Self {
463        Self::with_capacity(capacity, capacity * AVERAGE_BINARY_LENGTH)
464    }
465    fn append_value(&mut self, value: &[u8]) {
466        Self::append_value(self, value);
467    }
468    fn append_null(&mut self) {
469        Self::append_null(self);
470    }
471}
472
473///  Array builder for [`GenericBinaryArray`][crate::GenericBinaryArray]
474///
475/// Values can be appended using [`GenericByteBuilder::append_value`], and nulls with
476/// [`GenericByteBuilder::append_null`].
477///
478/// # Example
479/// ```
480/// # use arrow_array::builder::GenericBinaryBuilder;
481/// let mut builder = GenericBinaryBuilder::<i32>::new();
482///
483/// // Write data
484/// builder.append_value("foo");
485///
486/// // Write second value
487/// builder.append_value(&[0,1,2]);
488///
489/// let array = builder.finish();
490/// // binary values
491/// assert_eq!(array.value(0), b"foo");
492/// assert_eq!(array.value(1), b"\x00\x01\x02");
493/// ```
494///
495/// # Example incrementally writing bytes with `write_bytes`
496///
497/// ```
498/// # use std::io::Write;
499/// # use arrow_array::builder::GenericBinaryBuilder;
500/// let mut builder = GenericBinaryBuilder::<i32>::new();
501///
502/// // Write data in multiple `write_bytes` calls
503/// write!(builder, "foo").unwrap();
504/// write!(builder, "bar").unwrap();
505/// // The next call to append_value finishes the current string
506/// // including all previously written strings.
507/// builder.append_value("baz");
508///
509/// // Write second value with a single write call
510/// write!(builder, "v2").unwrap();
511/// // finish the value by calling append_value with an empty string
512/// builder.append_value("");
513///
514/// let array = builder.finish();
515/// assert_eq!(array.value(0), "foobarbaz".as_bytes());
516/// assert_eq!(array.value(1), "v2".as_bytes());
517/// ```
518pub type GenericBinaryBuilder<O> = GenericByteBuilder<GenericBinaryType<O>>;
519
520impl<O: OffsetSizeTrait> std::io::Write for GenericBinaryBuilder<O> {
521    fn write(&mut self, bs: &[u8]) -> std::io::Result<usize> {
522        self.value_builder.extend_from_slice(bs);
523        Ok(bs.len())
524    }
525
526    fn flush(&mut self) -> std::io::Result<()> {
527        Ok(())
528    }
529}
530
531#[cfg(test)]
532mod tests {
533    use super::*;
534    use crate::GenericStringArray;
535    use crate::array::Array;
536    use arrow_buffer::NullBuffer;
537    use std::fmt::Write as _;
538    use std::io::Write as _;
539
540    fn _test_generic_binary_builder<O: OffsetSizeTrait>() {
541        let mut builder = GenericBinaryBuilder::<O>::new();
542
543        builder.append_value(b"hello");
544        builder.append_value(b"");
545        builder.append_null();
546        builder.append_value(b"rust");
547
548        let array = builder.finish();
549
550        assert_eq!(4, array.len());
551        assert_eq!(1, array.null_count());
552        assert_eq!(b"hello", array.value(0));
553        assert_eq!([] as [u8; 0], array.value(1));
554        assert!(array.is_null(2));
555        assert_eq!(b"rust", array.value(3));
556        assert_eq!(O::from_usize(5).unwrap(), array.value_offsets()[2]);
557        assert_eq!(O::from_usize(4).unwrap(), array.value_length(3));
558    }
559
560    #[test]
561    fn test_binary_builder() {
562        _test_generic_binary_builder::<i32>()
563    }
564
565    #[test]
566    fn test_large_binary_builder() {
567        _test_generic_binary_builder::<i64>()
568    }
569
570    fn _test_generic_binary_builder_all_nulls<O: OffsetSizeTrait>() {
571        let mut builder = GenericBinaryBuilder::<O>::new();
572        builder.append_null();
573        builder.append_null();
574        builder.append_null();
575        builder.append_nulls(2);
576        assert_eq!(5, builder.len());
577        assert!(!builder.is_empty());
578
579        let array = builder.finish();
580        assert_eq!(5, array.null_count());
581        assert_eq!(5, array.len());
582        assert!(array.is_null(0));
583        assert!(array.is_null(1));
584        assert!(array.is_null(2));
585        assert!(array.is_null(3));
586        assert!(array.is_null(4));
587    }
588
589    #[test]
590    fn test_binary_builder_all_nulls() {
591        _test_generic_binary_builder_all_nulls::<i32>()
592    }
593
594    #[test]
595    fn test_large_binary_builder_all_nulls() {
596        _test_generic_binary_builder_all_nulls::<i64>()
597    }
598
599    fn _test_generic_binary_builder_reset<O: OffsetSizeTrait>() {
600        let mut builder = GenericBinaryBuilder::<O>::new();
601
602        builder.append_value(b"hello");
603        builder.append_value(b"");
604        builder.append_null();
605        builder.append_value(b"rust");
606        builder.finish();
607
608        assert!(builder.is_empty());
609
610        builder.append_value(b"parquet");
611        builder.append_null();
612        builder.append_value(b"arrow");
613        builder.append_value(b"");
614        builder.append_nulls(2);
615        builder.append_value(b"hi");
616        let array = builder.finish();
617
618        assert_eq!(7, array.len());
619        assert_eq!(3, array.null_count());
620        assert_eq!(b"parquet", array.value(0));
621        assert!(array.is_null(1));
622        assert!(array.is_null(4));
623        assert!(array.is_null(5));
624        assert_eq!(b"arrow", array.value(2));
625        assert_eq!(b"", array.value(1));
626        assert_eq!(b"hi", array.value(6));
627
628        assert_eq!(O::zero(), array.value_offsets()[0]);
629        assert_eq!(O::from_usize(7).unwrap(), array.value_offsets()[2]);
630        assert_eq!(O::from_usize(14).unwrap(), array.value_offsets()[7]);
631        assert_eq!(O::from_usize(5).unwrap(), array.value_length(2));
632    }
633
634    #[test]
635    fn test_binary_builder_reset() {
636        _test_generic_binary_builder_reset::<i32>()
637    }
638
639    #[test]
640    fn test_large_binary_builder_reset() {
641        _test_generic_binary_builder_reset::<i64>()
642    }
643
644    fn _test_generic_string_array_builder<O: OffsetSizeTrait>() {
645        let mut builder = GenericStringBuilder::<O>::new();
646        let owned = "arrow".to_owned();
647
648        builder.append_value("hello");
649        builder.append_value("");
650        builder.append_value(&owned);
651        builder.append_null();
652        builder.append_option(Some("rust"));
653        builder.append_option(None::<&str>);
654        builder.append_option(None::<String>);
655        builder.append_nulls(2);
656        builder.append_value("parquet");
657        assert_eq!(10, builder.len());
658
659        assert_eq!(
660            GenericStringArray::<O>::from(vec![
661                Some("hello"),
662                Some(""),
663                Some("arrow"),
664                None,
665                Some("rust"),
666                None,
667                None,
668                None,
669                None,
670                Some("parquet")
671            ]),
672            builder.finish()
673        );
674    }
675
676    #[test]
677    fn test_string_array_builder() {
678        _test_generic_string_array_builder::<i32>()
679    }
680
681    #[test]
682    fn test_large_string_array_builder() {
683        _test_generic_string_array_builder::<i64>()
684    }
685
686    fn _test_generic_string_array_builder_finish<O: OffsetSizeTrait>() {
687        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
688
689        builder.append_value("hello");
690        builder.append_value("rust");
691        builder.append_null();
692
693        builder.finish();
694        assert!(builder.is_empty());
695        assert_eq!(&[O::zero()], builder.offsets_slice());
696
697        builder.append_value("arrow");
698        builder.append_value("parquet");
699        let arr = builder.finish();
700        // array should not have null buffer because there is not `null` value.
701        assert!(arr.nulls().is_none());
702        assert_eq!(GenericStringArray::<O>::from(vec!["arrow", "parquet"]), arr,)
703    }
704
705    #[test]
706    fn test_string_array_builder_finish() {
707        _test_generic_string_array_builder_finish::<i32>()
708    }
709
710    #[test]
711    fn test_large_string_array_builder_finish() {
712        _test_generic_string_array_builder_finish::<i64>()
713    }
714
715    fn _test_generic_string_array_builder_finish_cloned<O: OffsetSizeTrait>() {
716        let mut builder = GenericStringBuilder::<O>::with_capacity(3, 11);
717
718        builder.append_value("hello");
719        builder.append_value("rust");
720        builder.append_null();
721
722        let mut arr = builder.finish_cloned();
723        assert!(!builder.is_empty());
724        assert_eq!(3, arr.len());
725
726        builder.append_value("arrow");
727        builder.append_value("parquet");
728        arr = builder.finish();
729
730        assert!(arr.nulls().is_some());
731        assert_eq!(&[O::zero()], builder.offsets_slice());
732        assert_eq!(5, arr.len());
733    }
734
735    #[test]
736    fn test_string_array_builder_finish_cloned() {
737        _test_generic_string_array_builder_finish_cloned::<i32>()
738    }
739
740    #[test]
741    fn test_large_string_array_builder_finish_cloned() {
742        _test_generic_string_array_builder_finish_cloned::<i64>()
743    }
744
745    #[test]
746    fn test_extend() {
747        let mut builder = GenericStringBuilder::<i32>::new();
748        builder.extend(["a", "b", "c", "", "a", "b", "c"].into_iter().map(Some));
749        builder.extend(["d", "cupcakes", "hello"].into_iter().map(Some));
750        let array = builder.finish();
751        assert_eq!(array.value_offsets(), &[0, 1, 2, 3, 3, 4, 5, 6, 7, 15, 20]);
752        assert_eq!(array.value_data(), b"abcabcdcupcakeshello");
753    }
754
755    #[test]
756    fn test_write_str() {
757        let mut builder = GenericStringBuilder::<i32>::new();
758        write!(builder, "foo").unwrap();
759        builder.append_value("");
760        writeln!(builder, "bar").unwrap();
761        builder.append_value("");
762        write!(builder, "fiz").unwrap();
763        write!(builder, "buz").unwrap();
764        builder.append_value("");
765        let a = builder.finish();
766        let r: Vec<_> = a.iter().flatten().collect();
767        assert_eq!(r, &["foo", "bar\n", "fizbuz"])
768    }
769
770    #[test]
771    fn test_write_bytes() {
772        let mut builder = GenericBinaryBuilder::<i32>::new();
773        write!(builder, "foo").unwrap();
774        builder.append_value("");
775        writeln!(builder, "bar").unwrap();
776        builder.append_value("");
777        write!(builder, "fiz").unwrap();
778        write!(builder, "buz").unwrap();
779        builder.append_value("");
780        let a = builder.finish();
781        let r: Vec<_> = a.iter().flatten().collect();
782        assert_eq!(
783            r,
784            &[b"foo".as_slice(), b"bar\n".as_slice(), b"fizbuz".as_slice()]
785        )
786    }
787
788    #[test]
789    fn test_append_array_without_nulls() {
790        let input = vec![
791            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
792            "thank", "you", "for", "asking",
793        ];
794        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
795        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
796        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
797
798        let mut builder = GenericStringBuilder::<i32>::new();
799        builder.append_array(&arr1).unwrap();
800        builder.append_array(&arr2).unwrap();
801        builder.append_array(&arr3).unwrap();
802
803        let actual = builder.finish();
804        let expected = GenericStringArray::<i32>::from(input);
805
806        assert_eq!(actual, expected);
807    }
808
809    #[test]
810    fn test_append_array_with_nulls() {
811        let input = vec![
812            Some("hello"),
813            None,
814            Some("how"),
815            None,
816            None,
817            None,
818            None,
819            Some("I"),
820            Some("am"),
821            Some("doing"),
822            Some("well"),
823        ];
824        let arr1 = GenericStringArray::<i32>::from(input[..3].to_vec());
825        let arr2 = GenericStringArray::<i32>::from(input[3..7].to_vec());
826        let arr3 = GenericStringArray::<i32>::from(input[7..].to_vec());
827
828        let mut builder = GenericStringBuilder::<i32>::new();
829        builder.append_array(&arr1).unwrap();
830        builder.append_array(&arr2).unwrap();
831        builder.append_array(&arr3).unwrap();
832
833        let actual = builder.finish();
834        let expected = GenericStringArray::<i32>::from(input);
835
836        assert_eq!(actual, expected);
837    }
838
839    #[test]
840    fn test_append_empty_array() {
841        let arr = GenericStringArray::<i32>::from(Vec::<&str>::new());
842        let mut builder = GenericStringBuilder::<i32>::new();
843        builder.append_array(&arr).unwrap();
844        let result = builder.finish();
845        assert_eq!(result.len(), 0);
846    }
847
848    #[test]
849    fn test_append_array_with_offset_not_starting_at_0() {
850        let input = vec![
851            Some("hello"),
852            None,
853            Some("how"),
854            None,
855            None,
856            None,
857            None,
858            Some("I"),
859            Some("am"),
860            Some("doing"),
861            Some("well"),
862        ];
863        let full_array = GenericStringArray::<i32>::from(input);
864        let sliced = full_array.slice(1, 4);
865
866        assert_ne!(sliced.offsets()[0].as_usize(), 0);
867        assert_ne!(sliced.offsets().last(), full_array.offsets().last());
868
869        let mut builder = GenericStringBuilder::<i32>::new();
870        builder.append_array(&sliced).unwrap();
871        let actual = builder.finish();
872
873        let expected = GenericStringArray::<i32>::from(vec![None, Some("how"), None, None]);
874
875        assert_eq!(actual, expected);
876    }
877
878    #[test]
879    fn test_append_underlying_null_values_added_as_is() {
880        let input_1_array_with_nulls = {
881            let input = vec![
882                "hello", "world", "how", "are", "you", "doing", "today", "I", "am",
883            ];
884            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
885
886            GenericStringArray::<i32>::new(
887                offsets,
888                buffer,
889                Some(NullBuffer::from(&[
890                    true, false, true, false, false, true, true, true, false,
891                ])),
892            )
893        };
894        let input_2_array_with_nulls = {
895            let input = vec!["doing", "well", "thank", "you", "for", "asking"];
896            let (offsets, buffer, _) = GenericStringArray::<i32>::from(input).into_parts();
897
898            GenericStringArray::<i32>::new(
899                offsets,
900                buffer,
901                Some(NullBuffer::from(&[false, false, true, false, true, true])),
902            )
903        };
904
905        let mut builder = GenericStringBuilder::<i32>::new();
906        builder.append_array(&input_1_array_with_nulls).unwrap();
907        builder.append_array(&input_2_array_with_nulls).unwrap();
908
909        let actual = builder.finish();
910        let expected = GenericStringArray::<i32>::from(vec![
911            Some("hello"),
912            None, // world
913            Some("how"),
914            None, // are
915            None, // you
916            Some("doing"),
917            Some("today"),
918            Some("I"),
919            None, // am
920            None, // doing
921            None, // well
922            Some("thank"),
923            None, // "you",
924            Some("for"),
925            Some("asking"),
926        ]);
927
928        assert_eq!(actual, expected);
929
930        let expected_underlying_buffer = Buffer::from(
931            [
932                "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing",
933                "well", "thank", "you", "for", "asking",
934            ]
935            .join("")
936            .as_bytes(),
937        );
938        assert_eq!(actual.values(), &expected_underlying_buffer);
939    }
940
941    #[test]
942    fn append_array_with_continues_indices() {
943        let input = vec![
944            "hello", "world", "how", "are", "you", "doing", "today", "I", "am", "doing", "well",
945            "thank", "you", "for", "asking",
946        ];
947        let full_array = GenericStringArray::<i32>::from(input);
948        let slice1 = full_array.slice(0, 3);
949        let slice2 = full_array.slice(3, 4);
950        let slice3 = full_array.slice(7, full_array.len() - 7);
951
952        let mut builder = GenericStringBuilder::<i32>::new();
953        builder.append_array(&slice1).unwrap();
954        builder.append_array(&slice2).unwrap();
955        builder.append_array(&slice3).unwrap();
956
957        let actual = builder.finish();
958
959        assert_eq!(actual, full_array);
960    }
961
962    #[test]
963    fn test_append_array_offset_overflow_precise() {
964        let mut builder = GenericStringBuilder::<i32>::new();
965
966        let initial_string = "x".repeat(i32::MAX as usize - 100);
967        builder.append_value(&initial_string);
968
969        let overflow_string = "y".repeat(200);
970        let overflow_array = GenericStringArray::<i32>::from(vec![overflow_string.as_str()]);
971
972        let result = builder.append_array(&overflow_array);
973
974        assert!(matches!(result, Err(ArrowError::OffsetOverflowError(_))));
975    }
976
977    #[test]
978    fn test_append_value_n() {
979        let mut builder = GenericStringBuilder::<i32>::new();
980        builder.append_value("hello");
981        builder.append_value_n("world", 3);
982        builder.append_null();
983        let array = builder.finish();
984
985        assert_eq!(5, array.len());
986        assert_eq!(1, array.null_count());
987        assert_eq!("hello", array.value(0));
988        assert_eq!("world", array.value(1));
989        assert_eq!("world", array.value(2));
990        assert_eq!("world", array.value(3));
991        assert!(array.is_null(4));
992    }
993}