arrow_array/array/
fixed_size_binary_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::array::print_long_array;
19use crate::iterator::FixedSizeBinaryIter;
20use crate::{Array, ArrayAccessor, ArrayRef, FixedSizeListArray, Scalar};
21use arrow_buffer::buffer::NullBuffer;
22use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, bit_util};
23use arrow_data::{ArrayData, ArrayDataBuilder};
24use arrow_schema::{ArrowError, DataType};
25use std::any::Any;
26use std::sync::Arc;
27
28/// An array of [fixed-size binary values](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout)
29///
30/// Each element in a [`FixedSizeBinaryArray`] has `value_length` bytes, where
31/// `value_length` is defined by the schema.
32///
33/// This array type is useful for storing fixed-length values such as 16-byte
34/// UUIDs (`value_length = 16`).
35///
36/// # Layout
37///
38/// Values in a [`FixedSizeBinaryArray`] are stored contiguously in a single
39/// buffer. The byte offset for the `i`-th element can be calculated as
40/// `i * value_length`.
41///
42/// Nulls are stored in a standard optional Arrow [`NullBuffer`].
43///
44/// For example, a 100-value [`FixedSizeBinaryArray`] with `value_length = 12`
45/// is shown below.
46///
47/// ```text
48/// ┌──────────────────────────────────────────┐
49/// │ Computed byte offsets                    │
50/// │          ┌──────────────────────┐ ┌────┐ │
51/// │          │┌────────────────────┐│ │    │ │
52/// │       0  ││value 0  (12 bytes) ││ │ 1  │ │
53/// │          │├────────────────────┤│ │    │ │
54/// │       12 ││value 1  (12 bytes) ││ │ 0  │ │
55/// │          │├────────────────────┤│ │    │ │
56/// │       24 ││value 2  (12 bytes) ││ │ 1  │ │
57/// │          │└────────────────────┘│ │    │ │
58/// │          │         ...          │ │... │ │
59/// │          │┌───────────────────┐ │ │    │ │
60/// │     1188 ││value 99 (12 bytes)│ │ │ 1  │ │
61/// │          │└───────────────────┘ │ │    │ │
62/// │          └──────────────────────┘ └────┘ │
63/// │           value_data              nulls  │
64/// └──────────────────────────────────────────┘
65/// ```
66///
67/// # Examples
68///
69/// Create an array from an iterable argument of byte slices.
70///
71/// ```
72///    use arrow_array::{Array, FixedSizeBinaryArray};
73///    let input_arg = vec![ vec![1, 2], vec![3, 4], vec![5, 6] ];
74///    let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
75///
76///    assert_eq!(3, arr.len());
77///
78/// ```
79/// Create an array from an iterable argument of sparse byte slices.
80/// Sparsity means that the input argument can contain `None` items.
81/// ```
82///    use arrow_array::{Array, FixedSizeBinaryArray};
83///    let input_arg = vec![ None, Some(vec![7, 8]), Some(vec![9, 10]), None, Some(vec![13, 14]) ];
84///    let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
85///    assert_eq!(5, arr.len())
86///
87/// ```
88///
89#[derive(Clone)]
90pub struct FixedSizeBinaryArray {
91    /// Must be DataType::FixedSizeBinary(value_size)
92    data_type: DataType,
93    /// `len` values, each `value_size` bytes
94    value_data: Buffer,
95    /// Optional Null Buffer
96    nulls: Option<NullBuffer>,
97    /// Number of elements in the array
98    len: usize,
99    /// size of each element, validated to fit in a positive i32
100    ///
101    /// Corresponds to the [`byteWidth` field] in the Arrow Spec
102    ///
103    /// note: Arrow stores `value_len` using i32. This implementation stores it
104    /// as a usize to ensure correct offset calculations.
105    ///
106    /// [`byteWidth` field]: https://github.com/apache/arrow/blob/2a89d03bbefd620b42126b8e00f8ae57e99cd638/format/Schema.fbs#L211
107    value_size: usize,
108}
109
110impl FixedSizeBinaryArray {
111    /// Create a new [`FixedSizeBinaryArray`] with `value_length` bytes per element, panicking on
112    /// failure
113    ///
114    /// # Panics
115    ///
116    /// Panics if [`Self::try_new`] returns an error
117    pub fn new(value_length: i32, values: Buffer, nulls: Option<NullBuffer>) -> Self {
118        Self::try_new(value_length, values, nulls).unwrap()
119    }
120
121    /// Create a new [`Scalar`] from `value`
122    pub fn new_scalar(value: impl AsRef<[u8]>) -> Scalar<Self> {
123        let v = value.as_ref();
124        let value_length =
125            i32::try_from(v.len()).expect("FixedSizeBinaryArray value length exceeds i32");
126        Scalar::new(Self::new(value_length, Buffer::from(v), None))
127    }
128
129    /// Create a new [`FixedSizeBinaryArray`] from the provided parts, returning an error on failure
130    ///
131    /// Creating an array with `value_length == 0` will try to get the length from the null
132    /// buffer. If no null buffer is provided, the resulting array will have length zero.
133    /// You can use [`Self::try_new_with_len`] to provide the length
134    ///
135    /// # Errors
136    ///
137    /// * `value_length < 0`
138    /// * `values.len() / value_length != nulls.len()`
139    /// * `value_length == 0 && values.len() != 0`
140    pub fn try_new(
141        value_length: i32,
142        values: Buffer,
143        nulls: Option<NullBuffer>,
144    ) -> Result<Self, ArrowError> {
145        let value_size = value_length.to_usize().ok_or_else(|| {
146            ArrowError::InvalidArgumentError(format!(
147                "Value length cannot be negative, got {value_length}"
148            ))
149        })?;
150
151        let len = match values.len().checked_div(value_size) {
152            Some(len) => len,
153            None => nulls.as_ref().map(|n| n.len()).unwrap_or(0),
154        };
155
156        Self::try_new_with_len(value_length, values, nulls, len)
157    }
158
159    /// Create a new [`FixedSizeBinaryArray`] from the provided parts and number of elements, returning an error on failure
160    ///
161    /// This is useful when the length cannot be determinated from the provided values (in case of `value_length == 0`) or nulls (`nulls.is_none()`).
162    ///
163    /// # Errors
164    ///
165    /// * `value_length < 0`
166    /// * `values.len() / value_length != len`
167    /// * `value_length == 0 && values.len() != 0`
168    /// * `nulls.len() != len`
169    /// * `value_length != 0 && values.len() / value_length != len`
170    pub fn try_new_with_len(
171        value_length: i32,
172        values: Buffer,
173        nulls: Option<NullBuffer>,
174        len: usize,
175    ) -> Result<Self, ArrowError> {
176        let data_type = DataType::FixedSizeBinary(value_length);
177        let value_size = value_length.to_usize().ok_or_else(|| {
178            ArrowError::InvalidArgumentError(format!(
179                "Value length cannot be negative, got {value_length}"
180            ))
181        })?;
182
183        if let Some(nulls) = &nulls {
184            if nulls.len() != len {
185                return Err(ArrowError::InvalidArgumentError(format!(
186                    "Incorrect length of null buffer for FixedSizeBinaryArray, expected {} got {}",
187                    len,
188                    nulls.len(),
189                )));
190            }
191        }
192
193        if value_size != 0 && values.len() / value_size != len {
194            return Err(ArrowError::InvalidArgumentError(format!(
195                "Incorrect length of values buffer for FixedSizeBinaryArray, expected {} got {}",
196                len,
197                values.len() / value_size,
198            )));
199        }
200
201        if value_size == 0 && !values.is_empty() {
202            return Err(ArrowError::InvalidArgumentError(
203                "Buffer cannot have non-zero length if the value length is zero".to_owned(),
204            ));
205        }
206
207        Ok(Self {
208            data_type,
209            value_data: values,
210            value_size,
211            nulls,
212            len,
213        })
214    }
215
216    /// Create a new [`FixedSizeBinaryArray`] of length `len` where all values are null
217    ///
218    /// # Panics
219    ///
220    /// Panics if
221    ///
222    /// * `value_length < 0`
223    /// * `value_length * len` would overflow `usize`
224    /// * `value_length * len * 8` would overflow `usize`
225    pub fn new_null(value_length: i32, len: usize) -> Self {
226        const BITS_IN_A_BYTE: usize = 8;
227        let value_size = value_length.to_usize().unwrap();
228        let capacity_in_bytes = value_size.checked_mul(len).unwrap();
229        let capacity_in_bits = capacity_in_bytes.checked_mul(BITS_IN_A_BYTE).unwrap();
230        Self {
231            data_type: DataType::FixedSizeBinary(value_length),
232            value_data: MutableBuffer::new_null(capacity_in_bits).into(),
233            nulls: Some(NullBuffer::new_null(len)),
234            value_size,
235            len,
236        }
237    }
238
239    /// Deconstruct this array into its constituent parts
240    pub fn into_parts(self) -> (i32, Buffer, Option<NullBuffer>) {
241        let value_length = self.value_length();
242        (value_length, self.value_data, self.nulls)
243    }
244
245    /// Returns the element at index `i` as a byte slice.
246    ///
247    /// Note: This method does not check for nulls and the value is arbitrary
248    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
249    ///
250    /// # Panics
251    /// Panics if index `i` is out of bounds.
252    pub fn value(&self, i: usize) -> &[u8] {
253        let len = self.len();
254        assert!(
255            i < len,
256            "Trying to access an element at index {i} from a FixedSizeBinaryArray of length {len}",
257        );
258        let position = i * self.value_size;
259        unsafe {
260            std::slice::from_raw_parts(self.value_data.as_ptr().add(position), self.value_size)
261        }
262    }
263
264    /// Returns the element at index `i` as a byte slice.
265    ///
266    /// Note: This method does not check for nulls and the value is arbitrary
267    /// if [`is_null`](Self::is_null) returns true for the index.
268    ///
269    /// # Safety
270    ///
271    /// Caller is responsible for ensuring that the index is within the bounds
272    /// of the array
273    pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] {
274        let position = i * self.value_size;
275        unsafe {
276            std::slice::from_raw_parts(self.value_data.as_ptr().add(position), self.value_size)
277        }
278    }
279
280    /// Returns the offset for the element at index `i`.
281    ///
282    /// Note this doesn't do any bound checking, for performance reason.
283    ///
284    /// # Panics
285    ///
286    /// Panics if the computed byte offset exceeds `i32::MAX`.
287    #[deprecated(since = "59.0.0", note = "Use i * value_size() instead")]
288    #[inline]
289    pub fn value_offset(&self, i: usize) -> i32 {
290        self.value_length() * i as i32
291    }
292
293    /// Returns the length for an element.
294    ///
295    /// All elements have the same length as the array is a fixed size.
296    ///
297    /// Returns an `i32` to be compatible with the Arrow spec.
298    ///
299    /// Use [`Self::value_size`] to return a `usize`.
300    #[inline]
301    pub fn value_length(&self) -> i32 {
302        // This is safe: constructor validated that value_size was a valid i32
303        self.value_size as i32
304    }
305
306    /// Return the length for an element, as a usize.
307    ///
308    /// All elements have the same length as the array is a fixed size.
309    ///
310    /// Note: This value will always fit, without overflow, into an i32
311    #[inline]
312    pub fn value_size(&self) -> usize {
313        self.value_size
314    }
315
316    /// Returns the values of this array.
317    ///
318    /// Unlike [`Self::value_data`] this returns the [`Buffer`]
319    /// allowing for zero-copy cloning.
320    #[inline]
321    pub fn values(&self) -> &Buffer {
322        &self.value_data
323    }
324
325    /// Returns the raw value data.
326    pub fn value_data(&self) -> &[u8] {
327        self.value_data.as_slice()
328    }
329
330    /// Returns a zero-copy slice of this array with the indicated offset and length.
331    pub fn slice(&self, offset: usize, len: usize) -> Self {
332        assert!(
333            offset.saturating_add(len) <= self.len,
334            "the length + offset of the sliced FixedSizeBinaryArray cannot exceed the existing length"
335        );
336        let offset_bytes = offset
337            .checked_mul(self.value_size)
338            .expect("offset overflow");
339        let len_bytes = len.checked_mul(self.value_size).expect("offset overflow");
340
341        Self {
342            data_type: self.data_type.clone(),
343            nulls: self.nulls.as_ref().map(|n| n.slice(offset, len)),
344            value_size: self.value_size,
345            value_data: self.value_data.slice_with_length(offset_bytes, len_bytes),
346            len,
347        }
348    }
349
350    /// Create an array from an iterable argument of sparse byte slices.
351    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
352    /// contain `None` items.
353    ///
354    /// # Examples
355    ///
356    /// ```
357    /// use arrow_array::FixedSizeBinaryArray;
358    /// let input_arg = vec![
359    ///     None,
360    ///     Some(vec![7, 8]),
361    ///     Some(vec![9, 10]),
362    ///     None,
363    ///     Some(vec![13, 14]),
364    ///     None,
365    /// ];
366    /// let array = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
367    /// ```
368    ///
369    /// # Errors
370    ///
371    /// Returns error if argument has length zero, or sizes of nested slices don't match.
372    #[deprecated(
373        since = "28.0.0",
374        note = "This function will fail if the iterator produces only None values; prefer `try_from_sparse_iter_with_size`"
375    )]
376    pub fn try_from_sparse_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
377    where
378        T: Iterator<Item = Option<U>>,
379        U: AsRef<[u8]>,
380    {
381        let mut len = 0;
382        let mut value_size = None;
383        let mut byte = 0;
384
385        let iter_size_hint = iter.size_hint().0;
386        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
387        let mut buffer = MutableBuffer::new(0);
388
389        let mut prepend = 0;
390        iter.try_for_each(|item| -> Result<(), ArrowError> {
391            // extend null bitmask by one byte per each 8 items
392            if byte == 0 {
393                null_buf.push(0u8);
394                byte = 8;
395            }
396            byte -= 1;
397
398            if let Some(slice) = item {
399                let slice = slice.as_ref();
400                if let Some(size) = value_size {
401                    if size != slice.len() {
402                        return Err(ArrowError::InvalidArgumentError(format!(
403                            "Nested array size mismatch: one is {}, and the other is {}",
404                            size,
405                            slice.len()
406                        )));
407                    }
408                } else {
409                    let len = slice.len();
410                    value_size = Some(len);
411                    // Now that we know how large each element is we can reserve
412                    // sufficient capacity in the underlying mutable buffer for
413                    // the data.
414                    if let Some(capacity) = iter_size_hint.checked_mul(len) {
415                        buffer.reserve(capacity);
416                    }
417                    let prepend_zeros = slice.len().checked_mul(prepend).ok_or_else(|| {
418                        ArrowError::InvalidArgumentError(format!(
419                            "FixedSizeBinaryArray error: value size {} * prepend {prepend} exceeds usize",
420                            slice.len()
421                        ))
422                    })?;
423                    buffer.extend_zeros(prepend_zeros);
424                }
425                bit_util::set_bit(null_buf.as_slice_mut(), len);
426                buffer.extend_from_slice(slice);
427            } else if let Some(size) = value_size {
428                buffer.extend_zeros(size);
429            } else {
430                prepend += 1;
431            }
432
433            len += 1;
434
435            Ok(())
436        })?;
437
438        if len == 0 {
439            return Err(ArrowError::InvalidArgumentError(
440                "Input iterable argument has no data".to_owned(),
441            ));
442        }
443
444        let nulls = NullBuffer::from_unsliced_buffer(null_buf, len);
445
446        let value_size = value_size.unwrap_or(0);
447        let value_length = value_size.try_into().map_err(|_| {
448            ArrowError::InvalidArgumentError(format!(
449                "FixedSizeBinaryArray value length exceeds i32, got {value_size}"
450            ))
451        })?;
452        Ok(Self {
453            data_type: DataType::FixedSizeBinary(value_length),
454            value_data: buffer.into(),
455            nulls,
456            value_size,
457            len,
458        })
459    }
460
461    /// Create an array from an iterable argument of sparse byte slices.
462    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
463    /// contain `None` items. In cases where the iterator returns only `None` values, this
464    /// also takes a `value_length` parameter to ensure that a valid
465    /// [`FixedSizeBinaryArray`] is still created.
466    ///
467    /// # Examples
468    ///
469    /// ```
470    /// use arrow_array::FixedSizeBinaryArray;
471    /// let input_arg = vec![
472    ///     None,
473    ///     Some(vec![7, 8]),
474    ///     Some(vec![9, 10]),
475    ///     None,
476    ///     Some(vec![13, 14]),
477    ///     None,
478    /// ];
479    /// let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
480    /// ```
481    ///
482    /// # Errors
483    ///
484    /// Returns error if argument has length zero, or sizes of nested slices don't match.
485    pub fn try_from_sparse_iter_with_size<T, U>(
486        mut iter: T,
487        value_length: i32,
488    ) -> Result<Self, ArrowError>
489    where
490        T: Iterator<Item = Option<U>>,
491        U: AsRef<[u8]>,
492    {
493        let value_size = value_length.to_usize().ok_or_else(|| {
494            ArrowError::InvalidArgumentError(format!(
495                "Value length cannot be negative, got {value_length}"
496            ))
497        })?;
498        let mut len = 0;
499        let mut byte = 0;
500
501        let iter_size_hint = iter.size_hint().0;
502        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
503        let capacity = iter_size_hint.checked_mul(value_size).ok_or_else(|| {
504            ArrowError::InvalidArgumentError(format!(
505                "FixedSizeBinaryArray error: value size {value_size} * len hint {iter_size_hint} exceeds usize"
506            ))
507        })?;
508        let mut buffer = MutableBuffer::new(capacity);
509
510        iter.try_for_each(|item| -> Result<(), ArrowError> {
511            // extend null bitmask by one byte per each 8 items
512            if byte == 0 {
513                null_buf.push(0u8);
514                byte = 8;
515            }
516            byte -= 1;
517
518            if let Some(slice) = item {
519                let slice = slice.as_ref();
520                if value_size != slice.len() {
521                    return Err(ArrowError::InvalidArgumentError(format!(
522                        "Nested array size mismatch: one is {}, and the other is {}",
523                        value_length,
524                        slice.len()
525                    )));
526                }
527
528                bit_util::set_bit(null_buf.as_slice_mut(), len);
529                buffer.extend_from_slice(slice);
530            } else {
531                buffer.extend_zeros(value_size);
532            }
533
534            len += 1;
535
536            Ok(())
537        })?;
538
539        let nulls = NullBuffer::from_unsliced_buffer(null_buf, len);
540
541        Ok(Self {
542            data_type: DataType::FixedSizeBinary(value_length),
543            value_data: buffer.into(),
544            nulls,
545            len,
546            value_size,
547        })
548    }
549
550    /// Create an array from an iterable argument of byte slices.
551    ///
552    /// # Examples
553    ///
554    /// ```
555    /// use arrow_array::FixedSizeBinaryArray;
556    /// let input_arg = vec![
557    ///     vec![1, 2],
558    ///     vec![3, 4],
559    ///     vec![5, 6],
560    /// ];
561    /// let array = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
562    /// ```
563    ///
564    /// # Errors
565    ///
566    /// Returns error if argument has length zero, or sizes of nested slices don't match.
567    pub fn try_from_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
568    where
569        T: Iterator<Item = U>,
570        U: AsRef<[u8]>,
571    {
572        let mut len = 0;
573        let mut value_size = None;
574        let iter_size_hint = iter.size_hint().0;
575        let mut buffer = MutableBuffer::new(0);
576
577        iter.try_for_each(|item| -> Result<(), ArrowError> {
578            let slice = item.as_ref();
579            if let Some(value_size) = value_size {
580                if value_size != slice.len() {
581                    return Err(ArrowError::InvalidArgumentError(format!(
582                        "Nested array size mismatch: one is {value_size}, and the other is {}",
583                        slice.len()
584                    )));
585                }
586            } else {
587                let len = slice.len();
588                value_size = Some(len);
589                if let Some(capacity) = iter_size_hint.checked_mul(len) {
590                    buffer.reserve(capacity);
591                }
592            }
593
594            buffer.extend_from_slice(slice);
595
596            len += 1;
597
598            Ok(())
599        })?;
600
601        if len == 0 {
602            return Err(ArrowError::InvalidArgumentError(
603                "Input iterable argument has no data".to_owned(),
604            ));
605        }
606
607        let value_size = value_size.unwrap_or(0);
608        let value_length = value_size.try_into().map_err(|_| {
609            ArrowError::InvalidArgumentError(format!(
610                "FixedSizeBinaryArray value length exceeds i32, got {value_size}"
611            ))
612        })?;
613        Ok(Self {
614            data_type: DataType::FixedSizeBinary(value_length),
615            value_data: buffer.into(),
616            nulls: None,
617            value_size,
618            len,
619        })
620    }
621
622    /// constructs a new iterator
623    pub fn iter(&self) -> FixedSizeBinaryIter<'_> {
624        FixedSizeBinaryIter::new(self)
625    }
626}
627
628impl From<ArrayData> for FixedSizeBinaryArray {
629    fn from(data: ArrayData) -> Self {
630        let (data_type, len, nulls, offset, buffers, _child_data) = data.into_parts();
631
632        assert_eq!(
633            buffers.len(),
634            1,
635            "FixedSizeBinaryArray data should contain 1 buffer only (values)"
636        );
637        let value_length = match data_type {
638            DataType::FixedSizeBinary(len) => len,
639            _ => panic!("Expected data type to be FixedSizeBinary"),
640        };
641
642        let value_size = value_length
643            .to_usize()
644            .expect("FixedSizeBinaryArray value length must be non-negative");
645        let value_data = buffers[0].slice_with_length(
646            offset.checked_mul(value_size).expect("offset overflow"),
647            len.checked_mul(value_size).expect("length overflow"),
648        );
649
650        Self {
651            data_type,
652            nulls,
653            len,
654            value_data,
655            value_size,
656        }
657    }
658}
659
660impl From<FixedSizeBinaryArray> for ArrayData {
661    fn from(array: FixedSizeBinaryArray) -> Self {
662        let builder = ArrayDataBuilder::new(array.data_type)
663            .len(array.len)
664            .buffers(vec![array.value_data])
665            .nulls(array.nulls);
666
667        unsafe { builder.build_unchecked() }
668    }
669}
670
671/// Creates a `FixedSizeBinaryArray` from `FixedSizeList<u8>` array
672impl From<FixedSizeListArray> for FixedSizeBinaryArray {
673    fn from(v: FixedSizeListArray) -> Self {
674        let value_len = v.value_length();
675        let v = v.into_data();
676        assert_eq!(
677            v.child_data().len(),
678            1,
679            "FixedSizeBinaryArray can only be created from list array of u8 values \
680             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
681        );
682        let child_data = &v.child_data()[0];
683
684        assert_eq!(
685            child_data.child_data().len(),
686            0,
687            "FixedSizeBinaryArray can only be created from list array of u8 values \
688             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
689        );
690        assert_eq!(
691            child_data.data_type(),
692            &DataType::UInt8,
693            "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
694        );
695        assert_eq!(
696            child_data.null_count(),
697            0,
698            "The child array cannot contain null values."
699        );
700
701        let builder = ArrayData::builder(DataType::FixedSizeBinary(value_len))
702            .len(v.len())
703            .offset(v.offset())
704            .add_buffer(child_data.buffers()[0].slice(child_data.offset()))
705            .nulls(v.nulls().cloned());
706
707        let data = unsafe { builder.build_unchecked() };
708        Self::from(data)
709    }
710}
711
712impl TryFrom<Vec<Option<&[u8]>>> for FixedSizeBinaryArray {
713    type Error = ArrowError;
714
715    fn try_from(v: Vec<Option<&[u8]>>) -> Result<Self, Self::Error> {
716        #[allow(deprecated)]
717        Self::try_from_sparse_iter(v.into_iter())
718    }
719}
720
721impl TryFrom<Vec<&[u8]>> for FixedSizeBinaryArray {
722    type Error = ArrowError;
723
724    fn try_from(v: Vec<&[u8]>) -> Result<Self, Self::Error> {
725        Self::try_from_iter(v.into_iter())
726    }
727}
728
729impl<const N: usize> TryFrom<Vec<Option<&[u8; N]>>> for FixedSizeBinaryArray {
730    type Error = ArrowError;
731
732    fn try_from(v: Vec<Option<&[u8; N]>>) -> Result<Self, Self::Error> {
733        N.try_into()
734            .map_err(|_| {
735                ArrowError::InvalidArgumentError(format!(
736                    "FixedSizeBinaryArray value length exceeds i32, got {N}"
737                ))
738            })
739            .and_then(|x| Self::try_from_sparse_iter_with_size(v.into_iter(), x))
740    }
741}
742
743impl<const N: usize> TryFrom<Vec<&[u8; N]>> for FixedSizeBinaryArray {
744    type Error = ArrowError;
745
746    fn try_from(v: Vec<&[u8; N]>) -> Result<Self, Self::Error> {
747        Self::try_from_iter(v.into_iter())
748    }
749}
750
751impl std::fmt::Debug for FixedSizeBinaryArray {
752    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
753        write!(f, "FixedSizeBinaryArray<{}>\n[\n", self.value_length())?;
754        print_long_array(self, f, |array, index, f| {
755            std::fmt::Debug::fmt(&array.value(index), f)
756        })?;
757        write!(f, "]")
758    }
759}
760
761/// SAFETY: Correctly implements the contract of Arrow Arrays
762unsafe impl Array for FixedSizeBinaryArray {
763    fn as_any(&self) -> &dyn Any {
764        self
765    }
766
767    fn to_data(&self) -> ArrayData {
768        self.clone().into()
769    }
770
771    fn into_data(self) -> ArrayData {
772        self.into()
773    }
774
775    fn data_type(&self) -> &DataType {
776        &self.data_type
777    }
778
779    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
780        Arc::new(self.slice(offset, length))
781    }
782
783    fn len(&self) -> usize {
784        self.len
785    }
786
787    fn is_empty(&self) -> bool {
788        self.len == 0
789    }
790
791    fn shrink_to_fit(&mut self) {
792        self.value_data.shrink_to_fit();
793        if let Some(nulls) = &mut self.nulls {
794            nulls.shrink_to_fit();
795        }
796    }
797
798    fn offset(&self) -> usize {
799        // Slices are normalized by slicing `value_data`/`nulls` directly;
800        // FSB does not retain a separate logical element offset.
801        0
802    }
803
804    fn nulls(&self) -> Option<&NullBuffer> {
805        self.nulls.as_ref()
806    }
807
808    fn logical_null_count(&self) -> usize {
809        // More efficient that the default implementation
810        self.null_count()
811    }
812
813    fn get_buffer_memory_size(&self) -> usize {
814        let mut sum = self.value_data.capacity();
815        if let Some(n) = &self.nulls {
816            sum += n.buffer().capacity();
817        }
818        sum
819    }
820
821    fn get_array_memory_size(&self) -> usize {
822        std::mem::size_of::<Self>() + self.get_buffer_memory_size()
823    }
824
825    #[cfg(feature = "pool")]
826    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
827        self.value_data.claim(pool);
828        if let Some(nulls) = &self.nulls {
829            nulls.claim(pool);
830        }
831    }
832}
833
834impl<'a> ArrayAccessor for &'a FixedSizeBinaryArray {
835    type Item = &'a [u8];
836
837    fn value(&self, index: usize) -> Self::Item {
838        FixedSizeBinaryArray::value(self, index)
839    }
840
841    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
842        unsafe { FixedSizeBinaryArray::value_unchecked(self, index) }
843    }
844}
845
846impl<'a> IntoIterator for &'a FixedSizeBinaryArray {
847    type Item = Option<&'a [u8]>;
848    type IntoIter = FixedSizeBinaryIter<'a>;
849
850    fn into_iter(self) -> Self::IntoIter {
851        FixedSizeBinaryIter::<'a>::new(self)
852    }
853}
854
855#[cfg(test)]
856mod tests {
857    use super::*;
858    use crate::RecordBatch;
859    use arrow_schema::{Field, Schema};
860
861    #[test]
862    fn test_fixed_size_binary_array() {
863        let values = b"hellotherearrow";
864
865        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
866            .len(3)
867            .add_buffer(Buffer::from(values))
868            .build()
869            .unwrap();
870        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
871        assert_eq!(3, fixed_size_binary_array.len());
872        assert_eq!(0, fixed_size_binary_array.null_count());
873        assert_eq!(
874            [b'h', b'e', b'l', b'l', b'o'],
875            fixed_size_binary_array.value(0)
876        );
877        assert_eq!(
878            [b't', b'h', b'e', b'r', b'e'],
879            fixed_size_binary_array.value(1)
880        );
881        assert_eq!(
882            [b'a', b'r', b'r', b'o', b'w'],
883            fixed_size_binary_array.value(2)
884        );
885        assert_eq!(5, fixed_size_binary_array.value_length());
886        for i in 0..3 {
887            assert!(fixed_size_binary_array.is_valid(i));
888            assert!(!fixed_size_binary_array.is_null(i));
889        }
890
891        // Test binary array with offset
892        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
893            .len(2)
894            .offset(1)
895            .add_buffer(Buffer::from(values))
896            .build()
897            .unwrap();
898        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
899        assert_eq!(
900            [b't', b'h', b'e', b'r', b'e'],
901            fixed_size_binary_array.value(0)
902        );
903        assert_eq!(
904            [b'a', b'r', b'r', b'o', b'w'],
905            fixed_size_binary_array.value(1)
906        );
907        assert_eq!(2, fixed_size_binary_array.len());
908        assert_eq!(5, fixed_size_binary_array.value_length());
909    }
910
911    #[test]
912    fn test_fixed_size_binary_array_from_fixed_size_list_array() {
913        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
914        let values_data = ArrayData::builder(DataType::UInt8)
915            .len(12)
916            .offset(2)
917            .add_buffer(Buffer::from_slice_ref(values))
918            .build()
919            .unwrap();
920        // [null, [10, 11, 12, 13]]
921        let array_data = unsafe {
922            ArrayData::builder(DataType::FixedSizeList(
923                Arc::new(Field::new_list_field(DataType::UInt8, false)),
924                4,
925            ))
926            .len(2)
927            .offset(1)
928            .add_child_data(values_data)
929            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101])))
930            .build_unchecked()
931        };
932        let list_array = FixedSizeListArray::from(array_data);
933        let binary_array = FixedSizeBinaryArray::from(list_array);
934
935        assert_eq!(2, binary_array.len());
936        assert_eq!(1, binary_array.null_count());
937        assert!(binary_array.is_null(0));
938        assert!(binary_array.is_valid(1));
939        assert_eq!(&[10, 11, 12, 13], binary_array.value(1));
940    }
941
942    #[test]
943    #[should_panic(
944        expected = "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays"
945    )]
946    // Different error messages, so skip for now
947    // https://github.com/apache/arrow-rs/issues/1545
948    #[cfg(not(feature = "force_validate"))]
949    fn test_fixed_size_binary_array_from_incorrect_fixed_size_list_array() {
950        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
951        let values_data = ArrayData::builder(DataType::UInt32)
952            .len(12)
953            .add_buffer(Buffer::from_slice_ref(values))
954            .build()
955            .unwrap();
956
957        let array_data = unsafe {
958            ArrayData::builder(DataType::FixedSizeList(
959                Arc::new(Field::new_list_field(DataType::Binary, false)),
960                4,
961            ))
962            .len(3)
963            .add_child_data(values_data)
964            .build_unchecked()
965        };
966        let list_array = FixedSizeListArray::from(array_data);
967        drop(FixedSizeBinaryArray::from(list_array));
968    }
969
970    #[test]
971    #[should_panic(expected = "The child array cannot contain null values.")]
972    fn test_fixed_size_binary_array_from_fixed_size_list_array_with_child_nulls_failed() {
973        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
974        let values_data = ArrayData::builder(DataType::UInt8)
975            .len(12)
976            .add_buffer(Buffer::from_slice_ref(values))
977            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101010101010])))
978            .build()
979            .unwrap();
980
981        let array_data = unsafe {
982            ArrayData::builder(DataType::FixedSizeList(
983                Arc::new(Field::new_list_field(DataType::UInt8, false)),
984                4,
985            ))
986            .len(3)
987            .add_child_data(values_data)
988            .build_unchecked()
989        };
990        let list_array = FixedSizeListArray::from(array_data);
991        drop(FixedSizeBinaryArray::from(list_array));
992    }
993
994    #[test]
995    fn test_fixed_size_binary_array_fmt_debug() {
996        let values = b"hellotherearrow";
997
998        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
999            .len(3)
1000            .add_buffer(Buffer::from(values))
1001            .build()
1002            .unwrap();
1003        let arr = FixedSizeBinaryArray::from(array_data);
1004        assert_eq!(
1005            "FixedSizeBinaryArray<5>\n[\n  [104, 101, 108, 108, 111],\n  [116, 104, 101, 114, 101],\n  [97, 114, 114, 111, 119],\n]",
1006            format!("{arr:?}")
1007        );
1008    }
1009
1010    #[test]
1011    fn test_fixed_size_binary_array_from_iter() {
1012        let input_arg = vec![vec![1, 2], vec![3, 4], vec![5, 6]];
1013        let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
1014
1015        assert_eq!(2, arr.value_length());
1016        assert_eq!(3, arr.len())
1017    }
1018
1019    #[test]
1020    fn test_all_none_fixed_size_binary_array_from_sparse_iter() {
1021        let none_option: Option<[u8; 32]> = None;
1022        let input_arg = vec![none_option, none_option, none_option];
1023        #[allow(deprecated)]
1024        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
1025        assert_eq!(0, arr.value_length());
1026        assert_eq!(3, arr.len())
1027    }
1028
1029    #[test]
1030    fn test_fixed_size_binary_array_from_sparse_iter() {
1031        let input_arg = vec![
1032            None,
1033            Some(vec![7, 8]),
1034            Some(vec![9, 10]),
1035            None,
1036            Some(vec![13, 14]),
1037        ];
1038        #[allow(deprecated)]
1039        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.iter().cloned()).unwrap();
1040        assert_eq!(2, arr.value_length());
1041        assert_eq!(5, arr.len());
1042
1043        let arr =
1044            FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
1045        assert_eq!(2, arr.value_length());
1046        assert_eq!(5, arr.len());
1047    }
1048
1049    #[test]
1050    fn test_fixed_size_binary_array_from_sparse_iter_with_size_all_none() {
1051        let input_arg = vec![None, None, None, None, None] as Vec<Option<Vec<u8>>>;
1052
1053        let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 16)
1054            .unwrap();
1055        assert_eq!(16, arr.value_length());
1056        assert_eq!(5, arr.len())
1057    }
1058
1059    #[test]
1060    fn test_fixed_size_binary_array_from_vec() {
1061        let values = vec!["one".as_bytes(), b"two", b"six", b"ten"];
1062        let array = FixedSizeBinaryArray::try_from(values).unwrap();
1063        assert_eq!(array.len(), 4);
1064        assert_eq!(array.null_count(), 0);
1065        assert_eq!(array.logical_null_count(), 0);
1066        assert_eq!(array.value(0), b"one");
1067        assert_eq!(array.value(1), b"two");
1068        assert_eq!(array.value(2), b"six");
1069        assert_eq!(array.value(3), b"ten");
1070        assert!(!array.is_null(0));
1071        assert!(!array.is_null(1));
1072        assert!(!array.is_null(2));
1073        assert!(!array.is_null(3));
1074    }
1075
1076    #[test]
1077    fn test_fixed_size_binary_array_from_vec_incorrect_length() {
1078        let values = vec!["one".as_bytes(), b"two", b"three", b"four"];
1079        assert!(FixedSizeBinaryArray::try_from(values).is_err());
1080    }
1081
1082    #[test]
1083    fn test_fixed_size_binary_array_from_opt_vec() {
1084        let values = vec![
1085            Some("one".as_bytes()),
1086            Some(b"two"),
1087            None,
1088            Some(b"six"),
1089            Some(b"ten"),
1090        ];
1091        let array = FixedSizeBinaryArray::try_from(values).unwrap();
1092        assert_eq!(array.len(), 5);
1093        assert_eq!(array.value(0), b"one");
1094        assert_eq!(array.value(1), b"two");
1095        assert_eq!(array.value(3), b"six");
1096        assert_eq!(array.value(4), b"ten");
1097        assert!(!array.is_null(0));
1098        assert!(!array.is_null(1));
1099        assert!(array.is_null(2));
1100        assert!(!array.is_null(3));
1101        assert!(!array.is_null(4));
1102    }
1103
1104    #[test]
1105    fn test_fixed_size_binary_array_from_opt_vec_incorrect_length() {
1106        let values = vec![
1107            Some("one".as_bytes()),
1108            Some(b"two"),
1109            None,
1110            Some(b"three"),
1111            Some(b"four"),
1112        ];
1113        assert!(FixedSizeBinaryArray::try_from(values).is_err());
1114    }
1115
1116    #[test]
1117    fn fixed_size_binary_array_all_null() {
1118        let data = vec![None] as Vec<Option<String>>;
1119        let array =
1120            FixedSizeBinaryArray::try_from_sparse_iter_with_size(data.into_iter(), 0).unwrap();
1121        array
1122            .into_data()
1123            .validate_full()
1124            .expect("All null array has valid array data");
1125    }
1126
1127    #[test]
1128    // Test for https://github.com/apache/arrow-rs/issues/1390
1129    fn fixed_size_binary_array_all_null_in_batch_with_schema() {
1130        let schema = Schema::new(vec![Field::new("a", DataType::FixedSizeBinary(2), true)]);
1131
1132        let none_option: Option<[u8; 2]> = None;
1133        let item = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1134            vec![none_option, none_option, none_option].into_iter(),
1135            2,
1136        )
1137        .unwrap();
1138
1139        // Should not panic
1140        RecordBatch::try_new(Arc::new(schema), vec![Arc::new(item)]).unwrap();
1141    }
1142
1143    #[test]
1144    #[should_panic(
1145        expected = "Trying to access an element at index 4 from a FixedSizeBinaryArray of length 3"
1146    )]
1147    fn test_fixed_size_binary_array_get_value_index_out_of_bound() {
1148        let values = vec![Some("one".as_bytes()), Some(b"two"), None];
1149        let array = FixedSizeBinaryArray::try_from(values).unwrap();
1150
1151        array.value(4);
1152    }
1153
1154    #[test]
1155    fn test_constructors() {
1156        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1157        let a = FixedSizeBinaryArray::new(2, buffer.clone(), None);
1158        assert_eq!(a.len(), 5);
1159
1160        let nulls = NullBuffer::new_null(5);
1161        FixedSizeBinaryArray::new(2, buffer.clone(), Some(nulls));
1162
1163        let null_array = FixedSizeBinaryArray::new_null(4, 3);
1164        assert_eq!(null_array.len(), 3);
1165        assert_eq!(null_array.values().len(), 12);
1166
1167        let a = FixedSizeBinaryArray::new(3, buffer.clone(), None);
1168        assert_eq!(a.len(), 3);
1169
1170        let nulls = NullBuffer::new_null(3);
1171        FixedSizeBinaryArray::new(3, buffer.clone(), Some(nulls));
1172
1173        let err = FixedSizeBinaryArray::try_new(-1, buffer.clone(), None).unwrap_err();
1174
1175        assert_eq!(
1176            err.to_string(),
1177            "Invalid argument error: Value length cannot be negative, got -1"
1178        );
1179
1180        let nulls = NullBuffer::new_null(3);
1181        let err = FixedSizeBinaryArray::try_new(2, buffer.clone(), Some(nulls)).unwrap_err();
1182        assert_eq!(
1183            err.to_string(),
1184            "Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3"
1185        );
1186
1187        let zero_sized = FixedSizeBinaryArray::new(0, Buffer::default(), None);
1188        assert_eq!(zero_sized.len(), 0);
1189        assert_eq!(zero_sized.null_count(), 0);
1190        assert_eq!(zero_sized.values().len(), 0);
1191
1192        let nulls = NullBuffer::new_null(3);
1193        let zero_sized_with_nulls = FixedSizeBinaryArray::new(0, Buffer::default(), Some(nulls));
1194        assert_eq!(zero_sized_with_nulls.len(), 3);
1195        assert_eq!(zero_sized_with_nulls.null_count(), 3);
1196        assert_eq!(zero_sized_with_nulls.values().len(), 0);
1197
1198        let zero_sized_with_non_empty_buffer_err =
1199            FixedSizeBinaryArray::try_new(0, buffer, None).unwrap_err();
1200        assert_eq!(
1201            zero_sized_with_non_empty_buffer_err.to_string(),
1202            "Invalid argument error: Buffer cannot have non-zero length if the value length is zero"
1203        );
1204    }
1205
1206    #[test]
1207    fn test_try_new_with_len() {
1208        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1209
1210        let a = FixedSizeBinaryArray::try_new_with_len(2, buffer.clone(), None, 5).unwrap();
1211        assert_eq!(a.len(), 5);
1212
1213        let nulls = NullBuffer::new_null(5);
1214        let a = FixedSizeBinaryArray::try_new_with_len(2, buffer, Some(nulls), 5).unwrap();
1215        assert_eq!(a.len(), 5);
1216        assert_eq!(a.null_count(), 5);
1217
1218        let a = FixedSizeBinaryArray::try_new_with_len(2, Buffer::default(), None, 0).unwrap();
1219        assert_eq!(a.len(), 0);
1220    }
1221
1222    #[test]
1223    fn test_try_new_with_len_zero_width() {
1224        // Zero-width with no nulls: the case where the length cannot be inferred from the parts
1225        let a = FixedSizeBinaryArray::try_new_with_len(0, Buffer::default(), None, 5).unwrap();
1226        assert_eq!(a.len(), 5);
1227        assert_eq!(a.null_count(), 0);
1228        assert_eq!(a.values().len(), 0);
1229
1230        let nulls = NullBuffer::new_null(3);
1231        let a =
1232            FixedSizeBinaryArray::try_new_with_len(0, Buffer::default(), Some(nulls), 3).unwrap();
1233        assert_eq!(a.len(), 3);
1234        assert_eq!(a.null_count(), 3);
1235    }
1236
1237    #[test]
1238    fn test_try_new_with_len_negative_value_length() {
1239        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1240        let err = FixedSizeBinaryArray::try_new_with_len(-1, buffer, None, 5).unwrap_err();
1241        assert_eq!(
1242            err.to_string(),
1243            "Invalid argument error: Value length cannot be negative, got -1"
1244        );
1245    }
1246
1247    #[test]
1248    fn test_try_new_with_len_incorrect_null_buffer_length() {
1249        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1250        let nulls = NullBuffer::new_null(3);
1251        let err = FixedSizeBinaryArray::try_new_with_len(2, buffer, Some(nulls), 5).unwrap_err();
1252        assert_eq!(
1253            err.to_string(),
1254            "Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3"
1255        );
1256    }
1257
1258    #[test]
1259    fn test_try_new_with_len_incorrect_values_buffer_length() {
1260        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1261        let err = FixedSizeBinaryArray::try_new_with_len(2, buffer, None, 3).unwrap_err();
1262        assert_eq!(
1263            err.to_string(),
1264            "Invalid argument error: Incorrect length of values buffer for FixedSizeBinaryArray, expected 3 got 5"
1265        );
1266    }
1267
1268    #[test]
1269    fn test_try_new_with_len_zero_width_non_empty_buffer() {
1270        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1271        let err = FixedSizeBinaryArray::try_new_with_len(0, buffer, None, 5).unwrap_err();
1272        assert_eq!(
1273            err.to_string(),
1274            "Invalid argument error: Buffer cannot have non-zero length if the value length is zero"
1275        );
1276    }
1277}
arrow_array/array/fixed_size_binary_array.rs

arrow_array/array/
fixed_size_binary_array.rs
