arrow_array/array/
binary_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::types::{ByteArrayType, GenericBinaryType};
19use crate::{Array, GenericByteArray, GenericListArray, GenericStringArray, OffsetSizeTrait};
20use arrow_data::ArrayData;
21use arrow_schema::DataType;
22
23/// A [`GenericByteArray`] for storing `[u8]`
24pub type GenericBinaryArray<OffsetSize> = GenericByteArray<GenericBinaryType<OffsetSize>>;
25
26impl<OffsetSize: OffsetSizeTrait> GenericBinaryArray<OffsetSize> {
27    /// Creates a [GenericBinaryArray] from a vector of byte slices
28    ///
29    /// See also [`Self::from_iter_values`]
30    pub fn from_vec(v: Vec<&[u8]>) -> Self {
31        Self::from_iter_values(v)
32    }
33
34    /// Creates a [GenericBinaryArray] from a vector of Optional (null) byte slices
35    pub fn from_opt_vec(v: Vec<Option<&[u8]>>) -> Self {
36        v.into_iter().collect()
37    }
38
39    fn from_list(v: GenericListArray<OffsetSize>) -> Self {
40        let v = v.into_data();
41        assert_eq!(
42            v.child_data().len(),
43            1,
44            "BinaryArray can only be created from list array of u8 values \
45             (i.e. List<PrimitiveArray<u8>>)."
46        );
47        let child_data = &v.child_data()[0];
48
49        assert_eq!(
50            child_data.child_data().len(),
51            0,
52            "BinaryArray can only be created from list array of u8 values \
53             (i.e. List<PrimitiveArray<u8>>)."
54        );
55        assert_eq!(
56            child_data.data_type(),
57            &DataType::UInt8,
58            "BinaryArray can only be created from List<u8> arrays, mismatched data types."
59        );
60        assert_eq!(
61            child_data.null_count(),
62            0,
63            "The child array cannot contain null values."
64        );
65
66        let builder = ArrayData::builder(Self::DATA_TYPE)
67            .len(v.len())
68            .offset(v.offset())
69            .add_buffer(v.buffers()[0].clone())
70            .add_buffer(child_data.buffers()[0].slice(child_data.offset()))
71            .nulls(v.nulls().cloned());
72
73        let data = unsafe { builder.build_unchecked() };
74        Self::from(data)
75    }
76
77    /// Returns an iterator that returns the values of `array.value(i)` for an iterator with each element `i`
78    pub fn take_iter<'a>(
79        &'a self,
80        indexes: impl Iterator<Item = Option<usize>> + 'a,
81    ) -> impl Iterator<Item = Option<&'a [u8]>> {
82        indexes.map(|opt_index| opt_index.map(|index| self.value(index)))
83    }
84
85    /// Returns an iterator that returns the values of `array.value(i)` for an iterator with each element `i`
86    /// # Safety
87    ///
88    /// caller must ensure that the indexes in the iterator are less than the `array.len()`
89    pub unsafe fn take_iter_unchecked<'a>(
90        &'a self,
91        indexes: impl Iterator<Item = Option<usize>> + 'a,
92    ) -> impl Iterator<Item = Option<&'a [u8]>> {
93        unsafe { indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index))) }
94    }
95}
96
97impl<OffsetSize: OffsetSizeTrait> From<Vec<Option<&[u8]>>> for GenericBinaryArray<OffsetSize> {
98    fn from(v: Vec<Option<&[u8]>>) -> Self {
99        Self::from_opt_vec(v)
100    }
101}
102
103impl<OffsetSize: OffsetSizeTrait> From<Vec<&[u8]>> for GenericBinaryArray<OffsetSize> {
104    fn from(v: Vec<&[u8]>) -> Self {
105        Self::from_iter_values(v)
106    }
107}
108
109impl<T: OffsetSizeTrait> From<GenericListArray<T>> for GenericBinaryArray<T> {
110    fn from(v: GenericListArray<T>) -> Self {
111        Self::from_list(v)
112    }
113}
114
115impl<OffsetSize: OffsetSizeTrait> From<GenericStringArray<OffsetSize>>
116    for GenericBinaryArray<OffsetSize>
117{
118    fn from(value: GenericStringArray<OffsetSize>) -> Self {
119        let builder = value
120            .into_data()
121            .into_builder()
122            .data_type(GenericBinaryType::<OffsetSize>::DATA_TYPE);
123
124        // Safety:
125        // A StringArray is a valid BinaryArray
126        Self::from(unsafe { builder.build_unchecked() })
127    }
128}
129
130/// A [`GenericBinaryArray`] of `[u8]` using `i32` offsets
131///
132/// The byte length of each element is represented by an i32.
133///
134/// # Examples
135///
136/// Create a BinaryArray from a vector of byte slices.
137///
138/// ```
139/// use arrow_array::{Array, BinaryArray};
140/// let values: Vec<&[u8]> =
141///     vec![b"one", b"two", b"", b"three"];
142/// let array = BinaryArray::from_vec(values);
143/// assert_eq!(4, array.len());
144/// assert_eq!(b"one", array.value(0));
145/// assert_eq!(b"two", array.value(1));
146/// assert_eq!(b"", array.value(2));
147/// assert_eq!(b"three", array.value(3));
148/// ```
149///
150/// Create a BinaryArray from a vector of Optional (null) byte slices.
151///
152/// ```
153/// use arrow_array::{Array, BinaryArray};
154/// let values: Vec<Option<&[u8]>> =
155///     vec![Some(b"one"), Some(b"two"), None, Some(b""), Some(b"three")];
156/// let array = BinaryArray::from_opt_vec(values);
157/// assert_eq!(5, array.len());
158/// assert_eq!(b"one", array.value(0));
159/// assert_eq!(b"two", array.value(1));
160/// assert_eq!(b"", array.value(3));
161/// assert_eq!(b"three", array.value(4));
162/// assert!(!array.is_null(0));
163/// assert!(!array.is_null(1));
164/// assert!(array.is_null(2));
165/// assert!(!array.is_null(3));
166/// assert!(!array.is_null(4));
167/// ```
168///
169/// See [`GenericByteArray`] for more information and examples
170pub type BinaryArray = GenericBinaryArray<i32>;
171
172/// A [`GenericBinaryArray`] of `[u8]` using `i64` offsets
173///
174/// # Examples
175///
176/// Create a LargeBinaryArray from a vector of byte slices.
177///
178/// ```
179/// use arrow_array::{Array, LargeBinaryArray};
180/// let values: Vec<&[u8]> =
181///     vec![b"one", b"two", b"", b"three"];
182/// let array = LargeBinaryArray::from_vec(values);
183/// assert_eq!(4, array.len());
184/// assert_eq!(b"one", array.value(0));
185/// assert_eq!(b"two", array.value(1));
186/// assert_eq!(b"", array.value(2));
187/// assert_eq!(b"three", array.value(3));
188/// ```
189///
190/// Create a LargeBinaryArray from a vector of Optional (null) byte slices.
191///
192/// ```
193/// use arrow_array::{Array, LargeBinaryArray};
194/// let values: Vec<Option<&[u8]>> =
195///     vec![Some(b"one"), Some(b"two"), None, Some(b""), Some(b"three")];
196/// let array = LargeBinaryArray::from_opt_vec(values);
197/// assert_eq!(5, array.len());
198/// assert_eq!(b"one", array.value(0));
199/// assert_eq!(b"two", array.value(1));
200/// assert_eq!(b"", array.value(3));
201/// assert_eq!(b"three", array.value(4));
202/// assert!(!array.is_null(0));
203/// assert!(!array.is_null(1));
204/// assert!(array.is_null(2));
205/// assert!(!array.is_null(3));
206/// assert!(!array.is_null(4));
207/// ```
208///
209/// See [`GenericByteArray`] for more information and examples
210pub type LargeBinaryArray = GenericBinaryArray<i64>;
211
212#[cfg(test)]
213mod tests {
214    use super::*;
215    use crate::{ListArray, StringArray};
216    use arrow_buffer::Buffer;
217    use arrow_schema::Field;
218    use std::sync::Arc;
219
220    #[test]
221    fn test_binary_array() {
222        let values = b"helloparquet";
223        let offsets: [i32; 4] = [0, 5, 5, 12];
224
225        // Array data: ["hello", "", "parquet"]
226        let array_data = ArrayData::builder(DataType::Binary)
227            .len(3)
228            .add_buffer(Buffer::from_slice_ref(offsets))
229            .add_buffer(Buffer::from_slice_ref(values))
230            .build()
231            .unwrap();
232        let binary_array = BinaryArray::from(array_data);
233        assert_eq!(3, binary_array.len());
234        assert_eq!(0, binary_array.null_count());
235        assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
236        assert_eq!([b'h', b'e', b'l', b'l', b'o'], unsafe {
237            binary_array.value_unchecked(0)
238        });
239        assert_eq!([] as [u8; 0], binary_array.value(1));
240        assert_eq!([] as [u8; 0], unsafe { binary_array.value_unchecked(1) });
241        assert_eq!(
242            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
243            binary_array.value(2)
244        );
245        assert_eq!([b'p', b'a', b'r', b'q', b'u', b'e', b't'], unsafe {
246            binary_array.value_unchecked(2)
247        });
248        assert_eq!(5, binary_array.value_offsets()[2]);
249        assert_eq!(7, binary_array.value_length(2));
250        for i in 0..3 {
251            assert!(binary_array.is_valid(i));
252            assert!(!binary_array.is_null(i));
253        }
254    }
255
256    #[test]
257    fn test_binary_array_with_offsets() {
258        let values = b"helloparquet";
259        let offsets: [i32; 4] = [0, 5, 5, 12];
260
261        // Test binary array with offset
262        let array_data = ArrayData::builder(DataType::Binary)
263            .len(2)
264            .offset(1)
265            .add_buffer(Buffer::from_slice_ref(offsets))
266            .add_buffer(Buffer::from_slice_ref(values))
267            .build()
268            .unwrap();
269        let binary_array = BinaryArray::from(array_data);
270        assert_eq!(
271            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
272            binary_array.value(1)
273        );
274        assert_eq!(5, binary_array.value_offsets()[0]);
275        assert_eq!(0, binary_array.value_length(0));
276        assert_eq!(5, binary_array.value_offsets()[1]);
277        assert_eq!(7, binary_array.value_length(1));
278    }
279
280    #[test]
281    fn test_large_binary_array() {
282        let values = b"helloparquet";
283        let offsets: [i64; 4] = [0, 5, 5, 12];
284
285        // Array data: ["hello", "", "parquet"]
286        let array_data = ArrayData::builder(DataType::LargeBinary)
287            .len(3)
288            .add_buffer(Buffer::from_slice_ref(offsets))
289            .add_buffer(Buffer::from_slice_ref(values))
290            .build()
291            .unwrap();
292        let binary_array = LargeBinaryArray::from(array_data);
293        assert_eq!(3, binary_array.len());
294        assert_eq!(0, binary_array.null_count());
295        assert_eq!([b'h', b'e', b'l', b'l', b'o'], binary_array.value(0));
296        assert_eq!([b'h', b'e', b'l', b'l', b'o'], unsafe {
297            binary_array.value_unchecked(0)
298        });
299        assert_eq!([] as [u8; 0], binary_array.value(1));
300        assert_eq!([] as [u8; 0], unsafe { binary_array.value_unchecked(1) });
301        assert_eq!(
302            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
303            binary_array.value(2)
304        );
305        assert_eq!([b'p', b'a', b'r', b'q', b'u', b'e', b't'], unsafe {
306            binary_array.value_unchecked(2)
307        });
308        assert_eq!(5, binary_array.value_offsets()[2]);
309        assert_eq!(7, binary_array.value_length(2));
310        for i in 0..3 {
311            assert!(binary_array.is_valid(i));
312            assert!(!binary_array.is_null(i));
313        }
314    }
315
316    #[test]
317    fn test_large_binary_array_with_offsets() {
318        let values = b"helloparquet";
319        let offsets: [i64; 4] = [0, 5, 5, 12];
320
321        // Test binary array with offset
322        let array_data = ArrayData::builder(DataType::LargeBinary)
323            .len(2)
324            .offset(1)
325            .add_buffer(Buffer::from_slice_ref(offsets))
326            .add_buffer(Buffer::from_slice_ref(values))
327            .build()
328            .unwrap();
329        let binary_array = LargeBinaryArray::from(array_data);
330        assert_eq!(
331            [b'p', b'a', b'r', b'q', b'u', b'e', b't'],
332            binary_array.value(1)
333        );
334        assert_eq!([b'p', b'a', b'r', b'q', b'u', b'e', b't'], unsafe {
335            binary_array.value_unchecked(1)
336        });
337        assert_eq!(5, binary_array.value_offsets()[0]);
338        assert_eq!(0, binary_array.value_length(0));
339        assert_eq!(5, binary_array.value_offsets()[1]);
340        assert_eq!(7, binary_array.value_length(1));
341    }
342
343    fn _test_generic_binary_array_from_list_array<O: OffsetSizeTrait>() {
344        let values = b"helloparquet";
345        let child_data = ArrayData::builder(DataType::UInt8)
346            .len(12)
347            .add_buffer(Buffer::from(values))
348            .build()
349            .unwrap();
350        let offsets = [0, 5, 5, 12].map(|n| O::from_usize(n).unwrap());
351
352        // Array data: ["hello", "", "parquet"]
353        let array_data1 = ArrayData::builder(GenericBinaryArray::<O>::DATA_TYPE)
354            .len(3)
355            .add_buffer(Buffer::from_slice_ref(offsets))
356            .add_buffer(Buffer::from_slice_ref(values))
357            .build()
358            .unwrap();
359        let binary_array1 = GenericBinaryArray::<O>::from(array_data1);
360
361        let data_type = GenericListArray::<O>::DATA_TYPE_CONSTRUCTOR(Arc::new(
362            Field::new_list_field(DataType::UInt8, false),
363        ));
364
365        let array_data2 = ArrayData::builder(data_type)
366            .len(3)
367            .add_buffer(Buffer::from_slice_ref(offsets))
368            .add_child_data(child_data)
369            .build()
370            .unwrap();
371        let list_array = GenericListArray::<O>::from(array_data2);
372        let binary_array2 = GenericBinaryArray::<O>::from(list_array);
373
374        assert_eq!(binary_array1.len(), binary_array2.len());
375        assert_eq!(binary_array1.null_count(), binary_array2.null_count());
376        assert_eq!(binary_array1.value_offsets(), binary_array2.value_offsets());
377        for i in 0..binary_array1.len() {
378            assert_eq!(binary_array1.value(i), binary_array2.value(i));
379            assert_eq!(binary_array1.value(i), unsafe {
380                binary_array2.value_unchecked(i)
381            });
382            assert_eq!(binary_array1.value_length(i), binary_array2.value_length(i));
383        }
384    }
385
386    #[test]
387    fn test_binary_array_from_list_array() {
388        _test_generic_binary_array_from_list_array::<i32>();
389    }
390
391    #[test]
392    fn test_large_binary_array_from_list_array() {
393        _test_generic_binary_array_from_list_array::<i64>();
394    }
395
396    fn _test_generic_binary_array_from_list_array_with_offset<O: OffsetSizeTrait>() {
397        let values = b"HelloArrowAndParquet";
398        // b"ArrowAndParquet"
399        let child_data = ArrayData::builder(DataType::UInt8)
400            .len(15)
401            .offset(5)
402            .add_buffer(Buffer::from(values))
403            .build()
404            .unwrap();
405
406        let offsets = [0, 5, 8, 15].map(|n| O::from_usize(n).unwrap());
407        let null_buffer = Buffer::from_slice_ref([0b101]);
408        let data_type = GenericListArray::<O>::DATA_TYPE_CONSTRUCTOR(Arc::new(
409            Field::new_list_field(DataType::UInt8, false),
410        ));
411
412        // [None, Some(b"Parquet")]
413        let array_data = ArrayData::builder(data_type)
414            .len(2)
415            .offset(1)
416            .add_buffer(Buffer::from_slice_ref(offsets))
417            .null_bit_buffer(Some(null_buffer))
418            .add_child_data(child_data)
419            .build()
420            .unwrap();
421        let list_array = GenericListArray::<O>::from(array_data);
422        let binary_array = GenericBinaryArray::<O>::from(list_array);
423
424        assert_eq!(2, binary_array.len());
425        assert_eq!(1, binary_array.null_count());
426        assert!(binary_array.is_null(0));
427        assert!(binary_array.is_valid(1));
428        assert_eq!(b"Parquet", binary_array.value(1));
429    }
430
431    #[test]
432    fn test_binary_array_from_list_array_with_offset() {
433        _test_generic_binary_array_from_list_array_with_offset::<i32>();
434    }
435
436    #[test]
437    fn test_large_binary_array_from_list_array_with_offset() {
438        _test_generic_binary_array_from_list_array_with_offset::<i64>();
439    }
440
441    fn _test_generic_binary_array_from_list_array_with_child_nulls_failed<O: OffsetSizeTrait>() {
442        let values = b"HelloArrow";
443        let child_data = ArrayData::builder(DataType::UInt8)
444            .len(10)
445            .add_buffer(Buffer::from(values))
446            .null_bit_buffer(Some(Buffer::from_slice_ref([0b1010101010])))
447            .build()
448            .unwrap();
449
450        let offsets = [0, 5, 10].map(|n| O::from_usize(n).unwrap());
451        let data_type = GenericListArray::<O>::DATA_TYPE_CONSTRUCTOR(Arc::new(
452            Field::new_list_field(DataType::UInt8, true),
453        ));
454
455        // [None, Some(b"Parquet")]
456        let array_data = ArrayData::builder(data_type)
457            .len(2)
458            .add_buffer(Buffer::from_slice_ref(offsets))
459            .add_child_data(child_data)
460            .build()
461            .unwrap();
462        let list_array = GenericListArray::<O>::from(array_data);
463        drop(GenericBinaryArray::<O>::from(list_array));
464    }
465
466    #[test]
467    #[should_panic(expected = "The child array cannot contain null values.")]
468    fn test_binary_array_from_list_array_with_child_nulls_failed() {
469        _test_generic_binary_array_from_list_array_with_child_nulls_failed::<i32>();
470    }
471
472    #[test]
473    #[should_panic(expected = "The child array cannot contain null values.")]
474    fn test_large_binary_array_from_list_array_with_child_nulls_failed() {
475        _test_generic_binary_array_from_list_array_with_child_nulls_failed::<i64>();
476    }
477
478    fn test_generic_binary_array_from_opt_vec<T: OffsetSizeTrait>() {
479        let values: Vec<Option<&[u8]>> =
480            vec![Some(b"one"), Some(b"two"), None, Some(b""), Some(b"three")];
481        let array = GenericBinaryArray::<T>::from_opt_vec(values);
482        assert_eq!(array.len(), 5);
483        assert_eq!(array.value(0), b"one");
484        assert_eq!(array.value(1), b"two");
485        assert_eq!(array.value(3), b"");
486        assert_eq!(array.value(4), b"three");
487        assert!(!array.is_null(0));
488        assert!(!array.is_null(1));
489        assert!(array.is_null(2));
490        assert!(!array.is_null(3));
491        assert!(!array.is_null(4));
492    }
493
494    #[test]
495    fn test_large_binary_array_from_opt_vec() {
496        test_generic_binary_array_from_opt_vec::<i64>()
497    }
498
499    #[test]
500    fn test_binary_array_from_opt_vec() {
501        test_generic_binary_array_from_opt_vec::<i32>()
502    }
503
504    #[test]
505    fn test_binary_array_from_unbound_iter() {
506        // iterator that doesn't declare (upper) size bound
507        let value_iter = (0..)
508            .scan(0usize, |pos, i| {
509                if *pos < 10 {
510                    *pos += 1;
511                    Some(Some(format!("value {i}")))
512                } else {
513                    // actually returns up to 10 values
514                    None
515                }
516            })
517            // limited using take()
518            .take(100);
519
520        let (_, upper_size_bound) = value_iter.size_hint();
521        // the upper bound, defined by take above, is 100
522        assert_eq!(upper_size_bound, Some(100));
523        let binary_array: BinaryArray = value_iter.collect();
524        // but the actual number of items in the array should be 10
525        assert_eq!(binary_array.len(), 10);
526    }
527
528    #[test]
529    #[should_panic(
530        expected = "BinaryArray can only be created from List<u8> arrays, mismatched data types."
531    )]
532    fn test_binary_array_from_incorrect_list_array() {
533        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
534        let values_data = ArrayData::builder(DataType::UInt32)
535            .len(12)
536            .add_buffer(Buffer::from_slice_ref(values))
537            .build()
538            .unwrap();
539        let offsets: [i32; 4] = [0, 5, 5, 12];
540
541        let data_type = DataType::List(Arc::new(Field::new_list_field(DataType::UInt32, false)));
542        let array_data = ArrayData::builder(data_type)
543            .len(3)
544            .add_buffer(Buffer::from_slice_ref(offsets))
545            .add_child_data(values_data)
546            .build()
547            .unwrap();
548        let list_array = ListArray::from(array_data);
549        drop(BinaryArray::from(list_array));
550    }
551
552    #[test]
553    #[should_panic(
554        expected = "Trying to access an element at index 4 from a BinaryArray of length 3"
555    )]
556    fn test_binary_array_get_value_index_out_of_bound() {
557        let values: [u8; 12] = [104, 101, 108, 108, 111, 112, 97, 114, 113, 117, 101, 116];
558        let offsets: [i32; 4] = [0, 5, 5, 12];
559        let array_data = ArrayData::builder(DataType::Binary)
560            .len(3)
561            .add_buffer(Buffer::from_slice_ref(offsets))
562            .add_buffer(Buffer::from_slice_ref(values))
563            .build()
564            .unwrap();
565        let binary_array = BinaryArray::from(array_data);
566        binary_array.value(4);
567    }
568
569    #[test]
570    #[should_panic(expected = "LargeBinaryArray expects DataType::LargeBinary")]
571    fn test_binary_array_validation() {
572        let array = BinaryArray::from_iter_values([&[1, 2]]);
573        let _ = LargeBinaryArray::from(array.into_data());
574    }
575
576    #[test]
577    fn test_binary_array_all_null() {
578        let data = vec![None];
579        let array = BinaryArray::from(data);
580        array
581            .into_data()
582            .validate_full()
583            .expect("All null array has valid array data");
584    }
585
586    #[test]
587    fn test_large_binary_array_all_null() {
588        let data = vec![None];
589        let array = LargeBinaryArray::from(data);
590        array
591            .into_data()
592            .validate_full()
593            .expect("All null array has valid array data");
594    }
595
596    #[test]
597    fn test_empty_offsets() {
598        let string = BinaryArray::from(
599            ArrayData::builder(DataType::Binary)
600                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
601                .build()
602                .unwrap(),
603        );
604        assert_eq!(string.value_offsets(), &[0]);
605        let string = LargeBinaryArray::from(
606            ArrayData::builder(DataType::LargeBinary)
607                .buffers(vec![Buffer::from(&[]), Buffer::from(&[])])
608                .build()
609                .unwrap(),
610        );
611        assert_eq!(string.len(), 0);
612        assert_eq!(string.value_offsets(), &[0]);
613    }
614
615    #[test]
616    fn test_to_from_string() {
617        let s = StringArray::from_iter_values(["a", "b", "c", "d"]);
618        let b = BinaryArray::from(s.clone());
619        let sa = StringArray::from(b); // Performs UTF-8 validation again
620
621        assert_eq!(s, sa);
622    }
623}
arrow_array/array/binary_array.rs

arrow_array/array/
binary_array.rs
