arrow_array/array/
fixed_size_binary_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::array::print_long_array;
19use crate::iterator::FixedSizeBinaryIter;
20use crate::{Array, ArrayAccessor, ArrayRef, FixedSizeListArray, Scalar};
21use arrow_buffer::buffer::NullBuffer;
22use arrow_buffer::{bit_util, ArrowNativeType, BooleanBuffer, Buffer, MutableBuffer};
23use arrow_data::{ArrayData, ArrayDataBuilder};
24use arrow_schema::{ArrowError, DataType};
25use std::any::Any;
26use std::sync::Arc;
27
28/// An array of [fixed size binary arrays](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout)
29///
30/// # Examples
31///
32/// Create an array from an iterable argument of byte slices.
33///
34/// ```
35///    use arrow_array::{Array, FixedSizeBinaryArray};
36///    let input_arg = vec![ vec![1, 2], vec![3, 4], vec![5, 6] ];
37///    let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
38///
39///    assert_eq!(3, arr.len());
40///
41/// ```
42/// Create an array from an iterable argument of sparse byte slices.
43/// Sparsity means that the input argument can contain `None` items.
44/// ```
45///    use arrow_array::{Array, FixedSizeBinaryArray};
46///    let input_arg = vec![ None, Some(vec![7, 8]), Some(vec![9, 10]), None, Some(vec![13, 14]) ];
47///    let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
48///    assert_eq!(5, arr.len())
49///
50/// ```
51///
52#[derive(Clone)]
53pub struct FixedSizeBinaryArray {
54    data_type: DataType, // Must be DataType::FixedSizeBinary(value_length)
55    value_data: Buffer,
56    nulls: Option<NullBuffer>,
57    len: usize,
58    value_length: i32,
59}
60
61impl FixedSizeBinaryArray {
62    /// Create a new [`FixedSizeBinaryArray`] with `size` element size, panicking on failure
63    ///
64    /// # Panics
65    ///
66    /// Panics if [`Self::try_new`] returns an error
67    pub fn new(size: i32, values: Buffer, nulls: Option<NullBuffer>) -> Self {
68        Self::try_new(size, values, nulls).unwrap()
69    }
70
71    /// Create a new [`Scalar`] from `value`
72    pub fn new_scalar(value: impl AsRef<[u8]>) -> Scalar<Self> {
73        let v = value.as_ref();
74        Scalar::new(Self::new(v.len() as _, Buffer::from(v), None))
75    }
76
77    /// Create a new [`FixedSizeBinaryArray`] from the provided parts, returning an error on failure
78    ///
79    /// # Errors
80    ///
81    /// * `size < 0`
82    /// * `values.len() / size != nulls.len()`
83    pub fn try_new(
84        size: i32,
85        values: Buffer,
86        nulls: Option<NullBuffer>,
87    ) -> Result<Self, ArrowError> {
88        let data_type = DataType::FixedSizeBinary(size);
89        let s = size.to_usize().ok_or_else(|| {
90            ArrowError::InvalidArgumentError(format!("Size cannot be negative, got {}", size))
91        })?;
92
93        let len = values.len() / s;
94        if let Some(n) = nulls.as_ref() {
95            if n.len() != len {
96                return Err(ArrowError::InvalidArgumentError(format!(
97                    "Incorrect length of null buffer for FixedSizeBinaryArray, expected {} got {}",
98                    len,
99                    n.len(),
100                )));
101            }
102        }
103
104        Ok(Self {
105            data_type,
106            value_data: values,
107            value_length: size,
108            nulls,
109            len,
110        })
111    }
112
113    /// Create a new [`FixedSizeBinaryArray`] of length `len` where all values are null
114    ///
115    /// # Panics
116    ///
117    /// Panics if
118    ///
119    /// * `size < 0`
120    /// * `size * len` would overflow `usize`
121    pub fn new_null(size: i32, len: usize) -> Self {
122        let capacity = size.to_usize().unwrap().checked_mul(len).unwrap();
123        Self {
124            data_type: DataType::FixedSizeBinary(size),
125            value_data: MutableBuffer::new(capacity).into(),
126            nulls: Some(NullBuffer::new_null(len)),
127            value_length: size,
128            len,
129        }
130    }
131
132    /// Deconstruct this array into its constituent parts
133    pub fn into_parts(self) -> (i32, Buffer, Option<NullBuffer>) {
134        (self.value_length, self.value_data, self.nulls)
135    }
136
137    /// Returns the element at index `i` as a byte slice.
138    /// # Panics
139    /// Panics if index `i` is out of bounds.
140    pub fn value(&self, i: usize) -> &[u8] {
141        assert!(
142            i < self.len(),
143            "Trying to access an element at index {} from a FixedSizeBinaryArray of length {}",
144            i,
145            self.len()
146        );
147        let offset = i + self.offset();
148        unsafe {
149            let pos = self.value_offset_at(offset);
150            std::slice::from_raw_parts(
151                self.value_data.as_ptr().offset(pos as isize),
152                (self.value_offset_at(offset + 1) - pos) as usize,
153            )
154        }
155    }
156
157    /// Returns the element at index `i` as a byte slice.
158    /// # Safety
159    /// Caller is responsible for ensuring that the index is within the bounds of the array
160    pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] {
161        let offset = i + self.offset();
162        let pos = self.value_offset_at(offset);
163        std::slice::from_raw_parts(
164            self.value_data.as_ptr().offset(pos as isize),
165            (self.value_offset_at(offset + 1) - pos) as usize,
166        )
167    }
168
169    /// Returns the offset for the element at index `i`.
170    ///
171    /// Note this doesn't do any bound checking, for performance reason.
172    #[inline]
173    pub fn value_offset(&self, i: usize) -> i32 {
174        self.value_offset_at(self.offset() + i)
175    }
176
177    /// Returns the length for an element.
178    ///
179    /// All elements have the same length as the array is a fixed size.
180    #[inline]
181    pub fn value_length(&self) -> i32 {
182        self.value_length
183    }
184
185    /// Returns the values of this array.
186    ///
187    /// Unlike [`Self::value_data`] this returns the [`Buffer`]
188    /// allowing for zero-copy cloning.
189    #[inline]
190    pub fn values(&self) -> &Buffer {
191        &self.value_data
192    }
193
194    /// Returns the raw value data.
195    pub fn value_data(&self) -> &[u8] {
196        self.value_data.as_slice()
197    }
198
199    /// Returns a zero-copy slice of this array with the indicated offset and length.
200    pub fn slice(&self, offset: usize, len: usize) -> Self {
201        assert!(
202            offset.saturating_add(len) <= self.len,
203            "the length + offset of the sliced FixedSizeBinaryArray cannot exceed the existing length"
204        );
205
206        let size = self.value_length as usize;
207
208        Self {
209            data_type: self.data_type.clone(),
210            nulls: self.nulls.as_ref().map(|n| n.slice(offset, len)),
211            value_length: self.value_length,
212            value_data: self.value_data.slice_with_length(offset * size, len * size),
213            len,
214        }
215    }
216
217    /// Create an array from an iterable argument of sparse byte slices.
218    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
219    /// contain `None` items.
220    ///
221    /// # Examples
222    ///
223    /// ```
224    /// use arrow_array::FixedSizeBinaryArray;
225    /// let input_arg = vec![
226    ///     None,
227    ///     Some(vec![7, 8]),
228    ///     Some(vec![9, 10]),
229    ///     None,
230    ///     Some(vec![13, 14]),
231    ///     None,
232    /// ];
233    /// let array = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
234    /// ```
235    ///
236    /// # Errors
237    ///
238    /// Returns error if argument has length zero, or sizes of nested slices don't match.
239    #[deprecated(
240        note = "This function will fail if the iterator produces only None values; prefer `try_from_sparse_iter_with_size`"
241    )]
242    pub fn try_from_sparse_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
243    where
244        T: Iterator<Item = Option<U>>,
245        U: AsRef<[u8]>,
246    {
247        let mut len = 0;
248        let mut size = None;
249        let mut byte = 0;
250
251        let iter_size_hint = iter.size_hint().0;
252        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
253        let mut buffer = MutableBuffer::new(0);
254
255        let mut prepend = 0;
256        iter.try_for_each(|item| -> Result<(), ArrowError> {
257            // extend null bitmask by one byte per each 8 items
258            if byte == 0 {
259                null_buf.push(0u8);
260                byte = 8;
261            }
262            byte -= 1;
263
264            if let Some(slice) = item {
265                let slice = slice.as_ref();
266                if let Some(size) = size {
267                    if size != slice.len() {
268                        return Err(ArrowError::InvalidArgumentError(format!(
269                            "Nested array size mismatch: one is {}, and the other is {}",
270                            size,
271                            slice.len()
272                        )));
273                    }
274                } else {
275                    let len = slice.len();
276                    size = Some(len);
277                    // Now that we know how large each element is we can reserve
278                    // sufficient capacity in the underlying mutable buffer for
279                    // the data.
280                    buffer.reserve(iter_size_hint * len);
281                    buffer.extend_zeros(slice.len() * prepend);
282                }
283                bit_util::set_bit(null_buf.as_slice_mut(), len);
284                buffer.extend_from_slice(slice);
285            } else if let Some(size) = size {
286                buffer.extend_zeros(size);
287            } else {
288                prepend += 1;
289            }
290
291            len += 1;
292
293            Ok(())
294        })?;
295
296        if len == 0 {
297            return Err(ArrowError::InvalidArgumentError(
298                "Input iterable argument has no data".to_owned(),
299            ));
300        }
301
302        let null_buf = BooleanBuffer::new(null_buf.into(), 0, len);
303        let nulls = Some(NullBuffer::new(null_buf)).filter(|n| n.null_count() > 0);
304
305        let size = size.unwrap_or(0) as i32;
306        Ok(Self {
307            data_type: DataType::FixedSizeBinary(size),
308            value_data: buffer.into(),
309            nulls,
310            value_length: size,
311            len,
312        })
313    }
314
315    /// Create an array from an iterable argument of sparse byte slices.
316    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
317    /// contain `None` items. In cases where the iterator returns only `None` values, this
318    /// also takes a size parameter to ensure that the a valid FixedSizeBinaryArray is still
319    /// created.
320    ///
321    /// # Examples
322    ///
323    /// ```
324    /// use arrow_array::FixedSizeBinaryArray;
325    /// let input_arg = vec![
326    ///     None,
327    ///     Some(vec![7, 8]),
328    ///     Some(vec![9, 10]),
329    ///     None,
330    ///     Some(vec![13, 14]),
331    ///     None,
332    /// ];
333    /// let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
334    /// ```
335    ///
336    /// # Errors
337    ///
338    /// Returns error if argument has length zero, or sizes of nested slices don't match.
339    pub fn try_from_sparse_iter_with_size<T, U>(mut iter: T, size: i32) -> Result<Self, ArrowError>
340    where
341        T: Iterator<Item = Option<U>>,
342        U: AsRef<[u8]>,
343    {
344        let mut len = 0;
345        let mut byte = 0;
346
347        let iter_size_hint = iter.size_hint().0;
348        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
349        let mut buffer = MutableBuffer::new(iter_size_hint * (size as usize));
350
351        iter.try_for_each(|item| -> Result<(), ArrowError> {
352            // extend null bitmask by one byte per each 8 items
353            if byte == 0 {
354                null_buf.push(0u8);
355                byte = 8;
356            }
357            byte -= 1;
358
359            if let Some(slice) = item {
360                let slice = slice.as_ref();
361                if size as usize != slice.len() {
362                    return Err(ArrowError::InvalidArgumentError(format!(
363                        "Nested array size mismatch: one is {}, and the other is {}",
364                        size,
365                        slice.len()
366                    )));
367                }
368
369                bit_util::set_bit(null_buf.as_slice_mut(), len);
370                buffer.extend_from_slice(slice);
371            } else {
372                buffer.extend_zeros(size as usize);
373            }
374
375            len += 1;
376
377            Ok(())
378        })?;
379
380        let null_buf = BooleanBuffer::new(null_buf.into(), 0, len);
381        let nulls = Some(NullBuffer::new(null_buf)).filter(|n| n.null_count() > 0);
382
383        Ok(Self {
384            data_type: DataType::FixedSizeBinary(size),
385            value_data: buffer.into(),
386            nulls,
387            len,
388            value_length: size,
389        })
390    }
391
392    /// Create an array from an iterable argument of byte slices.
393    ///
394    /// # Examples
395    ///
396    /// ```
397    /// use arrow_array::FixedSizeBinaryArray;
398    /// let input_arg = vec![
399    ///     vec![1, 2],
400    ///     vec![3, 4],
401    ///     vec![5, 6],
402    /// ];
403    /// let array = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
404    /// ```
405    ///
406    /// # Errors
407    ///
408    /// Returns error if argument has length zero, or sizes of nested slices don't match.
409    pub fn try_from_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
410    where
411        T: Iterator<Item = U>,
412        U: AsRef<[u8]>,
413    {
414        let mut len = 0;
415        let mut size = None;
416        let iter_size_hint = iter.size_hint().0;
417        let mut buffer = MutableBuffer::new(0);
418
419        iter.try_for_each(|item| -> Result<(), ArrowError> {
420            let slice = item.as_ref();
421            if let Some(size) = size {
422                if size != slice.len() {
423                    return Err(ArrowError::InvalidArgumentError(format!(
424                        "Nested array size mismatch: one is {}, and the other is {}",
425                        size,
426                        slice.len()
427                    )));
428                }
429            } else {
430                let len = slice.len();
431                size = Some(len);
432                buffer.reserve(iter_size_hint * len);
433            }
434
435            buffer.extend_from_slice(slice);
436
437            len += 1;
438
439            Ok(())
440        })?;
441
442        if len == 0 {
443            return Err(ArrowError::InvalidArgumentError(
444                "Input iterable argument has no data".to_owned(),
445            ));
446        }
447
448        let size = size.unwrap_or(0).try_into().unwrap();
449        Ok(Self {
450            data_type: DataType::FixedSizeBinary(size),
451            value_data: buffer.into(),
452            nulls: None,
453            value_length: size,
454            len,
455        })
456    }
457
458    #[inline]
459    fn value_offset_at(&self, i: usize) -> i32 {
460        self.value_length * i as i32
461    }
462
463    /// constructs a new iterator
464    pub fn iter(&self) -> FixedSizeBinaryIter<'_> {
465        FixedSizeBinaryIter::new(self)
466    }
467}
468
469impl From<ArrayData> for FixedSizeBinaryArray {
470    fn from(data: ArrayData) -> Self {
471        assert_eq!(
472            data.buffers().len(),
473            1,
474            "FixedSizeBinaryArray data should contain 1 buffer only (values)"
475        );
476        let value_length = match data.data_type() {
477            DataType::FixedSizeBinary(len) => *len,
478            _ => panic!("Expected data type to be FixedSizeBinary"),
479        };
480
481        let size = value_length as usize;
482        let value_data =
483            data.buffers()[0].slice_with_length(data.offset() * size, data.len() * size);
484
485        Self {
486            data_type: data.data_type().clone(),
487            nulls: data.nulls().cloned(),
488            len: data.len(),
489            value_data,
490            value_length,
491        }
492    }
493}
494
495impl From<FixedSizeBinaryArray> for ArrayData {
496    fn from(array: FixedSizeBinaryArray) -> Self {
497        let builder = ArrayDataBuilder::new(array.data_type)
498            .len(array.len)
499            .buffers(vec![array.value_data])
500            .nulls(array.nulls);
501
502        unsafe { builder.build_unchecked() }
503    }
504}
505
506/// Creates a `FixedSizeBinaryArray` from `FixedSizeList<u8>` array
507impl From<FixedSizeListArray> for FixedSizeBinaryArray {
508    fn from(v: FixedSizeListArray) -> Self {
509        let value_len = v.value_length();
510        let v = v.into_data();
511        assert_eq!(
512            v.child_data().len(),
513            1,
514            "FixedSizeBinaryArray can only be created from list array of u8 values \
515             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
516        );
517        let child_data = &v.child_data()[0];
518
519        assert_eq!(
520            child_data.child_data().len(),
521            0,
522            "FixedSizeBinaryArray can only be created from list array of u8 values \
523             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
524        );
525        assert_eq!(
526            child_data.data_type(),
527            &DataType::UInt8,
528            "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
529        );
530        assert_eq!(
531            child_data.null_count(),
532            0,
533            "The child array cannot contain null values."
534        );
535
536        let builder = ArrayData::builder(DataType::FixedSizeBinary(value_len))
537            .len(v.len())
538            .offset(v.offset())
539            .add_buffer(child_data.buffers()[0].slice(child_data.offset()))
540            .nulls(v.nulls().cloned());
541
542        let data = unsafe { builder.build_unchecked() };
543        Self::from(data)
544    }
545}
546
547impl From<Vec<Option<&[u8]>>> for FixedSizeBinaryArray {
548    fn from(v: Vec<Option<&[u8]>>) -> Self {
549        #[allow(deprecated)]
550        Self::try_from_sparse_iter(v.into_iter()).unwrap()
551    }
552}
553
554impl From<Vec<&[u8]>> for FixedSizeBinaryArray {
555    fn from(v: Vec<&[u8]>) -> Self {
556        Self::try_from_iter(v.into_iter()).unwrap()
557    }
558}
559
560impl<const N: usize> From<Vec<&[u8; N]>> for FixedSizeBinaryArray {
561    fn from(v: Vec<&[u8; N]>) -> Self {
562        Self::try_from_iter(v.into_iter()).unwrap()
563    }
564}
565
566impl std::fmt::Debug for FixedSizeBinaryArray {
567    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
568        write!(f, "FixedSizeBinaryArray<{}>\n[\n", self.value_length())?;
569        print_long_array(self, f, |array, index, f| {
570            std::fmt::Debug::fmt(&array.value(index), f)
571        })?;
572        write!(f, "]")
573    }
574}
575
576impl Array for FixedSizeBinaryArray {
577    fn as_any(&self) -> &dyn Any {
578        self
579    }
580
581    fn to_data(&self) -> ArrayData {
582        self.clone().into()
583    }
584
585    fn into_data(self) -> ArrayData {
586        self.into()
587    }
588
589    fn data_type(&self) -> &DataType {
590        &self.data_type
591    }
592
593    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
594        Arc::new(self.slice(offset, length))
595    }
596
597    fn len(&self) -> usize {
598        self.len
599    }
600
601    fn is_empty(&self) -> bool {
602        self.len == 0
603    }
604
605    fn shrink_to_fit(&mut self) {
606        self.value_data.shrink_to_fit();
607        if let Some(nulls) = &mut self.nulls {
608            nulls.shrink_to_fit();
609        }
610    }
611
612    fn offset(&self) -> usize {
613        0
614    }
615
616    fn nulls(&self) -> Option<&NullBuffer> {
617        self.nulls.as_ref()
618    }
619
620    fn logical_null_count(&self) -> usize {
621        // More efficient that the default implementation
622        self.null_count()
623    }
624
625    fn get_buffer_memory_size(&self) -> usize {
626        let mut sum = self.value_data.capacity();
627        if let Some(n) = &self.nulls {
628            sum += n.buffer().capacity();
629        }
630        sum
631    }
632
633    fn get_array_memory_size(&self) -> usize {
634        std::mem::size_of::<Self>() + self.get_buffer_memory_size()
635    }
636}
637
638impl<'a> ArrayAccessor for &'a FixedSizeBinaryArray {
639    type Item = &'a [u8];
640
641    fn value(&self, index: usize) -> Self::Item {
642        FixedSizeBinaryArray::value(self, index)
643    }
644
645    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
646        FixedSizeBinaryArray::value_unchecked(self, index)
647    }
648}
649
650impl<'a> IntoIterator for &'a FixedSizeBinaryArray {
651    type Item = Option<&'a [u8]>;
652    type IntoIter = FixedSizeBinaryIter<'a>;
653
654    fn into_iter(self) -> Self::IntoIter {
655        FixedSizeBinaryIter::<'a>::new(self)
656    }
657}
658
659#[cfg(test)]
660mod tests {
661    use crate::RecordBatch;
662    use arrow_schema::{Field, Schema};
663
664    use super::*;
665
666    #[test]
667    fn test_fixed_size_binary_array() {
668        let values: [u8; 15] = *b"hellotherearrow";
669
670        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
671            .len(3)
672            .add_buffer(Buffer::from(&values[..]))
673            .build()
674            .unwrap();
675        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
676        assert_eq!(3, fixed_size_binary_array.len());
677        assert_eq!(0, fixed_size_binary_array.null_count());
678        assert_eq!(
679            [b'h', b'e', b'l', b'l', b'o'],
680            fixed_size_binary_array.value(0)
681        );
682        assert_eq!(
683            [b't', b'h', b'e', b'r', b'e'],
684            fixed_size_binary_array.value(1)
685        );
686        assert_eq!(
687            [b'a', b'r', b'r', b'o', b'w'],
688            fixed_size_binary_array.value(2)
689        );
690        assert_eq!(5, fixed_size_binary_array.value_length());
691        assert_eq!(10, fixed_size_binary_array.value_offset(2));
692        for i in 0..3 {
693            assert!(fixed_size_binary_array.is_valid(i));
694            assert!(!fixed_size_binary_array.is_null(i));
695        }
696
697        // Test binary array with offset
698        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
699            .len(2)
700            .offset(1)
701            .add_buffer(Buffer::from(&values[..]))
702            .build()
703            .unwrap();
704        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
705        assert_eq!(
706            [b't', b'h', b'e', b'r', b'e'],
707            fixed_size_binary_array.value(0)
708        );
709        assert_eq!(
710            [b'a', b'r', b'r', b'o', b'w'],
711            fixed_size_binary_array.value(1)
712        );
713        assert_eq!(2, fixed_size_binary_array.len());
714        assert_eq!(0, fixed_size_binary_array.value_offset(0));
715        assert_eq!(5, fixed_size_binary_array.value_length());
716        assert_eq!(5, fixed_size_binary_array.value_offset(1));
717    }
718
719    #[test]
720    fn test_fixed_size_binary_array_from_fixed_size_list_array() {
721        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
722        let values_data = ArrayData::builder(DataType::UInt8)
723            .len(12)
724            .offset(2)
725            .add_buffer(Buffer::from_slice_ref(values))
726            .build()
727            .unwrap();
728        // [null, [10, 11, 12, 13]]
729        let array_data = unsafe {
730            ArrayData::builder(DataType::FixedSizeList(
731                Arc::new(Field::new("item", DataType::UInt8, false)),
732                4,
733            ))
734            .len(2)
735            .offset(1)
736            .add_child_data(values_data)
737            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101])))
738            .build_unchecked()
739        };
740        let list_array = FixedSizeListArray::from(array_data);
741        let binary_array = FixedSizeBinaryArray::from(list_array);
742
743        assert_eq!(2, binary_array.len());
744        assert_eq!(1, binary_array.null_count());
745        assert!(binary_array.is_null(0));
746        assert!(binary_array.is_valid(1));
747        assert_eq!(&[10, 11, 12, 13], binary_array.value(1));
748    }
749
750    #[test]
751    #[should_panic(
752        expected = "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays"
753    )]
754    // Different error messages, so skip for now
755    // https://github.com/apache/arrow-rs/issues/1545
756    #[cfg(not(feature = "force_validate"))]
757    fn test_fixed_size_binary_array_from_incorrect_fixed_size_list_array() {
758        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
759        let values_data = ArrayData::builder(DataType::UInt32)
760            .len(12)
761            .add_buffer(Buffer::from_slice_ref(values))
762            .build()
763            .unwrap();
764
765        let array_data = unsafe {
766            ArrayData::builder(DataType::FixedSizeList(
767                Arc::new(Field::new("item", DataType::Binary, false)),
768                4,
769            ))
770            .len(3)
771            .add_child_data(values_data)
772            .build_unchecked()
773        };
774        let list_array = FixedSizeListArray::from(array_data);
775        drop(FixedSizeBinaryArray::from(list_array));
776    }
777
778    #[test]
779    #[should_panic(expected = "The child array cannot contain null values.")]
780    fn test_fixed_size_binary_array_from_fixed_size_list_array_with_child_nulls_failed() {
781        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
782        let values_data = ArrayData::builder(DataType::UInt8)
783            .len(12)
784            .add_buffer(Buffer::from_slice_ref(values))
785            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101010101010])))
786            .build()
787            .unwrap();
788
789        let array_data = unsafe {
790            ArrayData::builder(DataType::FixedSizeList(
791                Arc::new(Field::new("item", DataType::UInt8, false)),
792                4,
793            ))
794            .len(3)
795            .add_child_data(values_data)
796            .build_unchecked()
797        };
798        let list_array = FixedSizeListArray::from(array_data);
799        drop(FixedSizeBinaryArray::from(list_array));
800    }
801
802    #[test]
803    fn test_fixed_size_binary_array_fmt_debug() {
804        let values: [u8; 15] = *b"hellotherearrow";
805
806        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
807            .len(3)
808            .add_buffer(Buffer::from(&values[..]))
809            .build()
810            .unwrap();
811        let arr = FixedSizeBinaryArray::from(array_data);
812        assert_eq!(
813            "FixedSizeBinaryArray<5>\n[\n  [104, 101, 108, 108, 111],\n  [116, 104, 101, 114, 101],\n  [97, 114, 114, 111, 119],\n]",
814            format!("{arr:?}")
815        );
816    }
817
818    #[test]
819    fn test_fixed_size_binary_array_from_iter() {
820        let input_arg = vec![vec![1, 2], vec![3, 4], vec![5, 6]];
821        let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
822
823        assert_eq!(2, arr.value_length());
824        assert_eq!(3, arr.len())
825    }
826
827    #[test]
828    fn test_all_none_fixed_size_binary_array_from_sparse_iter() {
829        let none_option: Option<[u8; 32]> = None;
830        let input_arg = vec![none_option, none_option, none_option];
831        #[allow(deprecated)]
832        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
833        assert_eq!(0, arr.value_length());
834        assert_eq!(3, arr.len())
835    }
836
837    #[test]
838    fn test_fixed_size_binary_array_from_sparse_iter() {
839        let input_arg = vec![
840            None,
841            Some(vec![7, 8]),
842            Some(vec![9, 10]),
843            None,
844            Some(vec![13, 14]),
845        ];
846        #[allow(deprecated)]
847        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.iter().cloned()).unwrap();
848        assert_eq!(2, arr.value_length());
849        assert_eq!(5, arr.len());
850
851        let arr =
852            FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
853        assert_eq!(2, arr.value_length());
854        assert_eq!(5, arr.len());
855    }
856
857    #[test]
858    fn test_fixed_size_binary_array_from_sparse_iter_with_size_all_none() {
859        let input_arg = vec![None, None, None, None, None] as Vec<Option<Vec<u8>>>;
860
861        let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 16)
862            .unwrap();
863        assert_eq!(16, arr.value_length());
864        assert_eq!(5, arr.len())
865    }
866
867    #[test]
868    fn test_fixed_size_binary_array_from_vec() {
869        let values = vec!["one".as_bytes(), b"two", b"six", b"ten"];
870        let array = FixedSizeBinaryArray::from(values);
871        assert_eq!(array.len(), 4);
872        assert_eq!(array.null_count(), 0);
873        assert_eq!(array.logical_null_count(), 0);
874        assert_eq!(array.value(0), b"one");
875        assert_eq!(array.value(1), b"two");
876        assert_eq!(array.value(2), b"six");
877        assert_eq!(array.value(3), b"ten");
878        assert!(!array.is_null(0));
879        assert!(!array.is_null(1));
880        assert!(!array.is_null(2));
881        assert!(!array.is_null(3));
882    }
883
884    #[test]
885    #[should_panic(expected = "Nested array size mismatch: one is 3, and the other is 5")]
886    fn test_fixed_size_binary_array_from_vec_incorrect_length() {
887        let values = vec!["one".as_bytes(), b"two", b"three", b"four"];
888        let _ = FixedSizeBinaryArray::from(values);
889    }
890
891    #[test]
892    fn test_fixed_size_binary_array_from_opt_vec() {
893        let values = vec![
894            Some("one".as_bytes()),
895            Some(b"two"),
896            None,
897            Some(b"six"),
898            Some(b"ten"),
899        ];
900        let array = FixedSizeBinaryArray::from(values);
901        assert_eq!(array.len(), 5);
902        assert_eq!(array.value(0), b"one");
903        assert_eq!(array.value(1), b"two");
904        assert_eq!(array.value(3), b"six");
905        assert_eq!(array.value(4), b"ten");
906        assert!(!array.is_null(0));
907        assert!(!array.is_null(1));
908        assert!(array.is_null(2));
909        assert!(!array.is_null(3));
910        assert!(!array.is_null(4));
911    }
912
913    #[test]
914    #[should_panic(expected = "Nested array size mismatch: one is 3, and the other is 5")]
915    fn test_fixed_size_binary_array_from_opt_vec_incorrect_length() {
916        let values = vec![
917            Some("one".as_bytes()),
918            Some(b"two"),
919            None,
920            Some(b"three"),
921            Some(b"four"),
922        ];
923        let _ = FixedSizeBinaryArray::from(values);
924    }
925
926    #[test]
927    fn fixed_size_binary_array_all_null() {
928        let data = vec![None] as Vec<Option<String>>;
929        let array =
930            FixedSizeBinaryArray::try_from_sparse_iter_with_size(data.into_iter(), 0).unwrap();
931        array
932            .into_data()
933            .validate_full()
934            .expect("All null array has valid array data");
935    }
936
937    #[test]
938    // Test for https://github.com/apache/arrow-rs/issues/1390
939    fn fixed_size_binary_array_all_null_in_batch_with_schema() {
940        let schema = Schema::new(vec![Field::new("a", DataType::FixedSizeBinary(2), true)]);
941
942        let none_option: Option<[u8; 2]> = None;
943        let item = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
944            vec![none_option, none_option, none_option].into_iter(),
945            2,
946        )
947        .unwrap();
948
949        // Should not panic
950        RecordBatch::try_new(Arc::new(schema), vec![Arc::new(item)]).unwrap();
951    }
952
953    #[test]
954    #[should_panic(
955        expected = "Trying to access an element at index 4 from a FixedSizeBinaryArray of length 3"
956    )]
957    fn test_fixed_size_binary_array_get_value_index_out_of_bound() {
958        let values = vec![Some("one".as_bytes()), Some(b"two"), None];
959        let array = FixedSizeBinaryArray::from(values);
960
961        array.value(4);
962    }
963
964    #[test]
965    fn test_constructors() {
966        let buffer = Buffer::from_vec(vec![0_u8; 10]);
967        let a = FixedSizeBinaryArray::new(2, buffer.clone(), None);
968        assert_eq!(a.len(), 5);
969
970        let nulls = NullBuffer::new_null(5);
971        FixedSizeBinaryArray::new(2, buffer.clone(), Some(nulls));
972
973        let a = FixedSizeBinaryArray::new(3, buffer.clone(), None);
974        assert_eq!(a.len(), 3);
975
976        let nulls = NullBuffer::new_null(3);
977        FixedSizeBinaryArray::new(3, buffer.clone(), Some(nulls));
978
979        let err = FixedSizeBinaryArray::try_new(-1, buffer.clone(), None).unwrap_err();
980
981        assert_eq!(
982            err.to_string(),
983            "Invalid argument error: Size cannot be negative, got -1"
984        );
985
986        let nulls = NullBuffer::new_null(3);
987        let err = FixedSizeBinaryArray::try_new(2, buffer, Some(nulls)).unwrap_err();
988        assert_eq!(err.to_string(), "Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3");
989    }
990}
arrow_array/array/fixed_size_binary_array.rs

arrow_array/array/
fixed_size_binary_array.rs
