arrow_array/array/
fixed_size_binary_array.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::array::print_long_array;
19use crate::iterator::FixedSizeBinaryIter;
20use crate::{Array, ArrayAccessor, ArrayRef, FixedSizeListArray, Scalar};
21use arrow_buffer::buffer::NullBuffer;
22use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, bit_util};
23use arrow_data::{ArrayData, ArrayDataBuilder};
24use arrow_schema::{ArrowError, DataType};
25use std::any::Any;
26use std::sync::Arc;
27
28/// An array of [fixed-size binary values](https://arrow.apache.org/docs/format/Columnar.html#fixed-size-primitive-layout)
29///
30/// Each element in a [`FixedSizeBinaryArray`] has `value_length` bytes, where
31/// `value_length` is defined by the schema.
32///
33/// This array type is useful for storing fixed-length values such as 16-byte
34/// UUIDs (`value_length = 16`).
35///
36/// # Layout
37///
38/// Values in a [`FixedSizeBinaryArray`] are stored contiguously in a single
39/// buffer. The byte offset for the `i`-th element can be calculated as
40/// `i * value_length`.
41///
42/// Nulls are stored in a standard optional Arrow [`NullBuffer`].
43///
44/// For example, a 100-value [`FixedSizeBinaryArray`] with `value_length = 12`
45/// is shown below.
46///
47/// ```text
48/// ┌──────────────────────────────────────────┐
49/// │ Computed byte offsets                    │
50/// │          ┌──────────────────────┐ ┌────┐ │
51/// │          │┌────────────────────┐│ │    │ │
52/// │       0  ││value 0  (12 bytes) ││ │ 1  │ │
53/// │          │├────────────────────┤│ │    │ │
54/// │       12 ││value 1  (12 bytes) ││ │ 0  │ │
55/// │          │├────────────────────┤│ │    │ │
56/// │       24 ││value 2  (12 bytes) ││ │ 1  │ │
57/// │          │└────────────────────┘│ │    │ │
58/// │          │         ...          │ │... │ │
59/// │          │┌───────────────────┐ │ │    │ │
60/// │     1188 ││value 99 (12 bytes)│ │ │ 1  │ │
61/// │          │└───────────────────┘ │ │    │ │
62/// │          └──────────────────────┘ └────┘ │
63/// │           value_data              nulls  │
64/// └──────────────────────────────────────────┘
65/// ```
66///
67/// # Examples
68///
69/// Create an array from an iterable argument of byte slices.
70///
71/// ```
72///    use arrow_array::{Array, FixedSizeBinaryArray};
73///    let input_arg = vec![ vec![1, 2], vec![3, 4], vec![5, 6] ];
74///    let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
75///
76///    assert_eq!(3, arr.len());
77///
78/// ```
79/// Create an array from an iterable argument of sparse byte slices.
80/// Sparsity means that the input argument can contain `None` items.
81/// ```
82///    use arrow_array::{Array, FixedSizeBinaryArray};
83///    let input_arg = vec![ None, Some(vec![7, 8]), Some(vec![9, 10]), None, Some(vec![13, 14]) ];
84///    let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
85///    assert_eq!(5, arr.len())
86///
87/// ```
88///
89#[derive(Clone)]
90pub struct FixedSizeBinaryArray {
91    /// Must be DataType::FixedSizeBinary(value_size)
92    data_type: DataType,
93    /// `len` values, each `value_size` bytes
94    value_data: Buffer,
95    /// Optional Null Buffer
96    nulls: Option<NullBuffer>,
97    /// Number of elements in the array
98    len: usize,
99    /// size of each element, validated to fit in a positive i32
100    ///
101    /// Corresponds to the [`byteWidth` field] in the Arrow Spec
102    ///
103    /// note: Arrow stores `value_len` using i32. This implementation stores it
104    /// as a usize to ensure correct offset calculations.
105    ///
106    /// [`byteWidth` field]: https://github.com/apache/arrow/blob/2a89d03bbefd620b42126b8e00f8ae57e99cd638/format/Schema.fbs#L211
107    value_size: usize,
108}
109
110impl FixedSizeBinaryArray {
111    /// Create a new [`FixedSizeBinaryArray`] with `value_length` bytes per element, panicking on
112    /// failure
113    ///
114    /// # Panics
115    ///
116    /// Panics if [`Self::try_new`] returns an error
117    pub fn new(value_length: i32, values: Buffer, nulls: Option<NullBuffer>) -> Self {
118        Self::try_new(value_length, values, nulls).unwrap()
119    }
120
121    /// Create a new [`FixedSizeBinaryArray`] from the provided parts without validation.
122    ///
123    /// # Safety
124    /// - `value_length >= 0`
125    /// - `values.len() == len * value_length as usize`
126    /// - `nulls.len() == len` if `nulls` is `Some`
127    pub unsafe fn new_unchecked(
128        value_length: i32,
129        values: Buffer,
130        nulls: Option<NullBuffer>,
131        len: usize,
132    ) -> Self {
133        if cfg!(feature = "force_validate") {
134            return Self::try_new_with_len(value_length, values, nulls, len).unwrap();
135        }
136        Self {
137            data_type: DataType::FixedSizeBinary(value_length),
138            value_data: values,
139            value_size: value_length as usize,
140            nulls,
141            len,
142        }
143    }
144
145    /// Create a new [`Scalar`] from `value`
146    pub fn new_scalar(value: impl AsRef<[u8]>) -> Scalar<Self> {
147        let v = value.as_ref();
148        let value_length =
149            i32::try_from(v.len()).expect("FixedSizeBinaryArray value length exceeds i32");
150        Scalar::new(Self::new(value_length, Buffer::from(v), None))
151    }
152
153    /// Create a new [`FixedSizeBinaryArray`] from the provided parts, returning an error on failure
154    ///
155    /// Creating an array with `value_length == 0` will try to get the length from the null
156    /// buffer. If no null buffer is provided, the resulting array will have length zero.
157    /// You can use [`Self::try_new_with_len`] to provide the length
158    ///
159    /// # Errors
160    ///
161    /// * `value_length < 0`
162    /// * `values.len() / value_length != nulls.len()`
163    /// * `value_length == 0 && values.len() != 0`
164    pub fn try_new(
165        value_length: i32,
166        values: Buffer,
167        nulls: Option<NullBuffer>,
168    ) -> Result<Self, ArrowError> {
169        let value_size = value_length.to_usize().ok_or_else(|| {
170            ArrowError::InvalidArgumentError(format!(
171                "Value length cannot be negative, got {value_length}"
172            ))
173        })?;
174
175        let len = match values.len().checked_div(value_size) {
176            Some(len) => len,
177            None => nulls.as_ref().map(|n| n.len()).unwrap_or(0),
178        };
179
180        Self::try_new_with_len(value_length, values, nulls, len)
181    }
182
183    /// Create a new [`FixedSizeBinaryArray`] from the provided parts and number of elements, returning an error on failure
184    ///
185    /// This is useful when the length cannot be determinated from the provided values (in case of `value_length == 0`) or nulls (`nulls.is_none()`).
186    ///
187    /// # Errors
188    ///
189    /// * `value_length < 0`
190    /// * `values.len() / value_length != len`
191    /// * `value_length == 0 && values.len() != 0`
192    /// * `nulls.len() != len`
193    /// * `value_length != 0 && values.len() / value_length != len`
194    pub fn try_new_with_len(
195        value_length: i32,
196        values: Buffer,
197        nulls: Option<NullBuffer>,
198        len: usize,
199    ) -> Result<Self, ArrowError> {
200        let data_type = DataType::FixedSizeBinary(value_length);
201        let value_size = value_length.to_usize().ok_or_else(|| {
202            ArrowError::InvalidArgumentError(format!(
203                "Value length cannot be negative, got {value_length}"
204            ))
205        })?;
206
207        if let Some(nulls) = &nulls {
208            if nulls.len() != len {
209                return Err(ArrowError::InvalidArgumentError(format!(
210                    "Incorrect length of null buffer for FixedSizeBinaryArray, expected {} got {}",
211                    len,
212                    nulls.len(),
213                )));
214            }
215        }
216
217        if value_size != 0 && values.len() / value_size != len {
218            return Err(ArrowError::InvalidArgumentError(format!(
219                "Incorrect length of values buffer for FixedSizeBinaryArray, expected {} got {}",
220                len,
221                values.len() / value_size,
222            )));
223        }
224
225        if value_size == 0 && !values.is_empty() {
226            return Err(ArrowError::InvalidArgumentError(
227                "Buffer cannot have non-zero length if the value length is zero".to_owned(),
228            ));
229        }
230
231        Ok(Self {
232            data_type,
233            value_data: values,
234            value_size,
235            nulls,
236            len,
237        })
238    }
239
240    /// Create a new [`FixedSizeBinaryArray`] of length `len` where all values are null
241    ///
242    /// # Panics
243    ///
244    /// Panics if
245    ///
246    /// * `value_length < 0`
247    /// * `value_length * len` would overflow `usize`
248    /// * `value_length * len * 8` would overflow `usize`
249    pub fn new_null(value_length: i32, len: usize) -> Self {
250        const BITS_IN_A_BYTE: usize = 8;
251        let value_size = value_length.to_usize().unwrap();
252        let capacity_in_bytes = value_size.checked_mul(len).unwrap();
253        let capacity_in_bits = capacity_in_bytes.checked_mul(BITS_IN_A_BYTE).unwrap();
254        Self {
255            data_type: DataType::FixedSizeBinary(value_length),
256            value_data: MutableBuffer::new_null(capacity_in_bits).into(),
257            nulls: Some(NullBuffer::new_null(len)),
258            value_size,
259            len,
260        }
261    }
262
263    /// Deconstruct this array into its constituent parts
264    pub fn into_parts(self) -> (i32, Buffer, Option<NullBuffer>) {
265        let value_length = self.value_length();
266        (value_length, self.value_data, self.nulls)
267    }
268
269    /// Returns the element at index `i` as a byte slice.
270    ///
271    /// Note: This method does not check for nulls and the value is arbitrary
272    /// (but still well-defined) if [`is_null`](Self::is_null) returns true for the index.
273    ///
274    /// # Panics
275    /// Panics if index `i` is out of bounds.
276    pub fn value(&self, i: usize) -> &[u8] {
277        let len = self.len();
278        assert!(
279            i < len,
280            "Trying to access an element at index {i} from a FixedSizeBinaryArray of length {len}",
281        );
282        let position = i * self.value_size;
283        unsafe {
284            std::slice::from_raw_parts(self.value_data.as_ptr().add(position), self.value_size)
285        }
286    }
287
288    /// Returns the element at index `i` as a byte slice.
289    ///
290    /// Note: This method does not check for nulls and the value is arbitrary
291    /// if [`is_null`](Self::is_null) returns true for the index.
292    ///
293    /// # Safety
294    ///
295    /// Caller is responsible for ensuring that the index is within the bounds
296    /// of the array
297    pub unsafe fn value_unchecked(&self, i: usize) -> &[u8] {
298        let position = i * self.value_size;
299        unsafe {
300            std::slice::from_raw_parts(self.value_data.as_ptr().add(position), self.value_size)
301        }
302    }
303
304    /// Returns the offset for the element at index `i`.
305    ///
306    /// Note this doesn't do any bound checking, for performance reason.
307    ///
308    /// # Panics
309    ///
310    /// Panics if the computed byte offset exceeds `i32::MAX`.
311    #[deprecated(since = "59.0.0", note = "Use i * value_size() instead")]
312    #[inline]
313    pub fn value_offset(&self, i: usize) -> i32 {
314        self.value_length() * i as i32
315    }
316
317    /// Returns the length for an element.
318    ///
319    /// All elements have the same length as the array is a fixed size.
320    ///
321    /// Returns an `i32` to be compatible with the Arrow spec.
322    ///
323    /// Use [`Self::value_size`] to return a `usize`.
324    #[inline]
325    pub fn value_length(&self) -> i32 {
326        // This is safe: constructor validated that value_size was a valid i32
327        self.value_size as i32
328    }
329
330    /// Return the length for an element, as a usize.
331    ///
332    /// All elements have the same length as the array is a fixed size.
333    ///
334    /// Note: This value will always fit, without overflow, into an i32
335    #[inline]
336    pub fn value_size(&self) -> usize {
337        self.value_size
338    }
339
340    /// Returns the values of this array.
341    ///
342    /// Unlike [`Self::value_data`] this returns the [`Buffer`]
343    /// allowing for zero-copy cloning.
344    #[inline]
345    pub fn values(&self) -> &Buffer {
346        &self.value_data
347    }
348
349    /// Returns the raw value data.
350    pub fn value_data(&self) -> &[u8] {
351        self.value_data.as_slice()
352    }
353
354    /// Returns a zero-copy slice of this array with the indicated offset and length.
355    pub fn slice(&self, offset: usize, len: usize) -> Self {
356        assert!(
357            offset.saturating_add(len) <= self.len,
358            "the length + offset of the sliced FixedSizeBinaryArray cannot exceed the existing length"
359        );
360        let offset_bytes = offset
361            .checked_mul(self.value_size)
362            .expect("offset overflow");
363        let len_bytes = len.checked_mul(self.value_size).expect("offset overflow");
364
365        Self {
366            data_type: self.data_type.clone(),
367            nulls: self.nulls.as_ref().map(|n| n.slice(offset, len)),
368            value_size: self.value_size,
369            value_data: self.value_data.slice_with_length(offset_bytes, len_bytes),
370            len,
371        }
372    }
373
374    /// Create an array from an iterable argument of sparse byte slices.
375    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
376    /// contain `None` items.
377    ///
378    /// # Examples
379    ///
380    /// ```
381    /// use arrow_array::FixedSizeBinaryArray;
382    /// let input_arg = vec![
383    ///     None,
384    ///     Some(vec![7, 8]),
385    ///     Some(vec![9, 10]),
386    ///     None,
387    ///     Some(vec![13, 14]),
388    ///     None,
389    /// ];
390    /// let array = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
391    /// ```
392    ///
393    /// # Errors
394    ///
395    /// Returns error if argument has length zero, or sizes of nested slices don't match.
396    #[deprecated(
397        since = "28.0.0",
398        note = "This function will fail if the iterator produces only None values; prefer `try_from_sparse_iter_with_size`"
399    )]
400    pub fn try_from_sparse_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
401    where
402        T: Iterator<Item = Option<U>>,
403        U: AsRef<[u8]>,
404    {
405        let mut len = 0;
406        let mut value_size = None;
407        let mut byte = 0;
408
409        let iter_size_hint = iter.size_hint().0;
410        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
411        let mut buffer = MutableBuffer::new(0);
412
413        let mut prepend = 0;
414        iter.try_for_each(|item| -> Result<(), ArrowError> {
415            // extend null bitmask by one byte per each 8 items
416            if byte == 0 {
417                null_buf.push(0u8);
418                byte = 8;
419            }
420            byte -= 1;
421
422            if let Some(slice) = item {
423                let slice = slice.as_ref();
424                if let Some(size) = value_size {
425                    if size != slice.len() {
426                        return Err(ArrowError::InvalidArgumentError(format!(
427                            "Nested array size mismatch: one is {}, and the other is {}",
428                            size,
429                            slice.len()
430                        )));
431                    }
432                } else {
433                    let len = slice.len();
434                    value_size = Some(len);
435                    // Now that we know how large each element is we can reserve
436                    // sufficient capacity in the underlying mutable buffer for
437                    // the data.
438                    if let Some(capacity) = iter_size_hint.checked_mul(len) {
439                        buffer.reserve(capacity);
440                    }
441                    let prepend_zeros = slice.len().checked_mul(prepend).ok_or_else(|| {
442                        ArrowError::InvalidArgumentError(format!(
443                            "FixedSizeBinaryArray error: value size {} * prepend {prepend} exceeds usize",
444                            slice.len()
445                        ))
446                    })?;
447                    buffer.extend_zeros(prepend_zeros);
448                }
449                bit_util::set_bit(null_buf.as_slice_mut(), len);
450                buffer.extend_from_slice(slice);
451            } else if let Some(size) = value_size {
452                buffer.extend_zeros(size);
453            } else {
454                prepend += 1;
455            }
456
457            len += 1;
458
459            Ok(())
460        })?;
461
462        if len == 0 {
463            return Err(ArrowError::InvalidArgumentError(
464                "Input iterable argument has no data".to_owned(),
465            ));
466        }
467
468        let nulls = NullBuffer::from_unsliced_buffer(null_buf, len);
469
470        let value_size = value_size.unwrap_or(0);
471        let value_length = value_size.try_into().map_err(|_| {
472            ArrowError::InvalidArgumentError(format!(
473                "FixedSizeBinaryArray value length exceeds i32, got {value_size}"
474            ))
475        })?;
476        Ok(Self {
477            data_type: DataType::FixedSizeBinary(value_length),
478            value_data: buffer.into(),
479            nulls,
480            value_size,
481            len,
482        })
483    }
484
485    /// Create an array from an iterable argument of sparse byte slices.
486    /// Sparsity means that items returned by the iterator are optional, i.e input argument can
487    /// contain `None` items. In cases where the iterator returns only `None` values, this
488    /// also takes a `value_length` parameter to ensure that a valid
489    /// [`FixedSizeBinaryArray`] is still created.
490    ///
491    /// # Examples
492    ///
493    /// ```
494    /// use arrow_array::FixedSizeBinaryArray;
495    /// let input_arg = vec![
496    ///     None,
497    ///     Some(vec![7, 8]),
498    ///     Some(vec![9, 10]),
499    ///     None,
500    ///     Some(vec![13, 14]),
501    ///     None,
502    /// ];
503    /// let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
504    /// ```
505    ///
506    /// # Errors
507    ///
508    /// Returns error if argument has length zero, or sizes of nested slices don't match.
509    pub fn try_from_sparse_iter_with_size<T, U>(
510        mut iter: T,
511        value_length: i32,
512    ) -> Result<Self, ArrowError>
513    where
514        T: Iterator<Item = Option<U>>,
515        U: AsRef<[u8]>,
516    {
517        let value_size = value_length.to_usize().ok_or_else(|| {
518            ArrowError::InvalidArgumentError(format!(
519                "Value length cannot be negative, got {value_length}"
520            ))
521        })?;
522        let mut len = 0;
523        let mut byte = 0;
524
525        let iter_size_hint = iter.size_hint().0;
526        let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
527        let capacity = iter_size_hint.checked_mul(value_size).ok_or_else(|| {
528            ArrowError::InvalidArgumentError(format!(
529                "FixedSizeBinaryArray error: value size {value_size} * len hint {iter_size_hint} exceeds usize"
530            ))
531        })?;
532        let mut buffer = MutableBuffer::new(capacity);
533
534        iter.try_for_each(|item| -> Result<(), ArrowError> {
535            // extend null bitmask by one byte per each 8 items
536            if byte == 0 {
537                null_buf.push(0u8);
538                byte = 8;
539            }
540            byte -= 1;
541
542            if let Some(slice) = item {
543                let slice = slice.as_ref();
544                if value_size != slice.len() {
545                    return Err(ArrowError::InvalidArgumentError(format!(
546                        "Nested array size mismatch: one is {}, and the other is {}",
547                        value_length,
548                        slice.len()
549                    )));
550                }
551
552                bit_util::set_bit(null_buf.as_slice_mut(), len);
553                buffer.extend_from_slice(slice);
554            } else {
555                buffer.extend_zeros(value_size);
556            }
557
558            len += 1;
559
560            Ok(())
561        })?;
562
563        let nulls = NullBuffer::from_unsliced_buffer(null_buf, len);
564
565        Ok(Self {
566            data_type: DataType::FixedSizeBinary(value_length),
567            value_data: buffer.into(),
568            nulls,
569            len,
570            value_size,
571        })
572    }
573
574    /// Create an array from an iterable argument of byte slices.
575    ///
576    /// # Examples
577    ///
578    /// ```
579    /// use arrow_array::FixedSizeBinaryArray;
580    /// let input_arg = vec![
581    ///     vec![1, 2],
582    ///     vec![3, 4],
583    ///     vec![5, 6],
584    /// ];
585    /// let array = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
586    /// ```
587    ///
588    /// # Errors
589    ///
590    /// Returns error if argument has length zero, or sizes of nested slices don't match.
591    pub fn try_from_iter<T, U>(mut iter: T) -> Result<Self, ArrowError>
592    where
593        T: Iterator<Item = U>,
594        U: AsRef<[u8]>,
595    {
596        let mut len = 0;
597        let mut value_size = None;
598        let iter_size_hint = iter.size_hint().0;
599        let mut buffer = MutableBuffer::new(0);
600
601        iter.try_for_each(|item| -> Result<(), ArrowError> {
602            let slice = item.as_ref();
603            if let Some(value_size) = value_size {
604                if value_size != slice.len() {
605                    return Err(ArrowError::InvalidArgumentError(format!(
606                        "Nested array size mismatch: one is {value_size}, and the other is {}",
607                        slice.len()
608                    )));
609                }
610            } else {
611                let len = slice.len();
612                value_size = Some(len);
613                if let Some(capacity) = iter_size_hint.checked_mul(len) {
614                    buffer.reserve(capacity);
615                }
616            }
617
618            buffer.extend_from_slice(slice);
619
620            len += 1;
621
622            Ok(())
623        })?;
624
625        if len == 0 {
626            return Err(ArrowError::InvalidArgumentError(
627                "Input iterable argument has no data".to_owned(),
628            ));
629        }
630
631        let value_size = value_size.unwrap_or(0);
632        let value_length = value_size.try_into().map_err(|_| {
633            ArrowError::InvalidArgumentError(format!(
634                "FixedSizeBinaryArray value length exceeds i32, got {value_size}"
635            ))
636        })?;
637        Ok(Self {
638            data_type: DataType::FixedSizeBinary(value_length),
639            value_data: buffer.into(),
640            nulls: None,
641            value_size,
642            len,
643        })
644    }
645
646    /// constructs a new iterator
647    pub fn iter(&self) -> FixedSizeBinaryIter<'_> {
648        FixedSizeBinaryIter::new(self)
649    }
650}
651
652impl From<ArrayData> for FixedSizeBinaryArray {
653    fn from(data: ArrayData) -> Self {
654        let (data_type, len, nulls, offset, buffers, _child_data) = data.into_parts();
655
656        assert_eq!(
657            buffers.len(),
658            1,
659            "FixedSizeBinaryArray data should contain 1 buffer only (values)"
660        );
661        let value_length = match data_type {
662            DataType::FixedSizeBinary(len) => len,
663            _ => panic!("Expected data type to be FixedSizeBinary"),
664        };
665
666        let value_size = value_length
667            .to_usize()
668            .expect("FixedSizeBinaryArray value length must be non-negative");
669        let value_data = buffers[0].slice_with_length(
670            offset.checked_mul(value_size).expect("offset overflow"),
671            len.checked_mul(value_size).expect("length overflow"),
672        );
673
674        Self {
675            data_type,
676            nulls,
677            len,
678            value_data,
679            value_size,
680        }
681    }
682}
683
684impl From<FixedSizeBinaryArray> for ArrayData {
685    fn from(array: FixedSizeBinaryArray) -> Self {
686        let builder = ArrayDataBuilder::new(array.data_type)
687            .len(array.len)
688            .buffers(vec![array.value_data])
689            .nulls(array.nulls);
690
691        unsafe { builder.build_unchecked() }
692    }
693}
694
695/// Creates a `FixedSizeBinaryArray` from `FixedSizeList<u8>` array
696impl From<FixedSizeListArray> for FixedSizeBinaryArray {
697    fn from(v: FixedSizeListArray) -> Self {
698        let value_len = v.value_length();
699        let v = v.into_data();
700        assert_eq!(
701            v.child_data().len(),
702            1,
703            "FixedSizeBinaryArray can only be created from list array of u8 values \
704             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
705        );
706        let child_data = &v.child_data()[0];
707
708        assert_eq!(
709            child_data.child_data().len(),
710            0,
711            "FixedSizeBinaryArray can only be created from list array of u8 values \
712             (i.e. FixedSizeList<PrimitiveArray<u8>>)."
713        );
714        assert_eq!(
715            child_data.data_type(),
716            &DataType::UInt8,
717            "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
718        );
719        assert_eq!(
720            child_data.null_count(),
721            0,
722            "The child array cannot contain null values."
723        );
724
725        let builder = ArrayData::builder(DataType::FixedSizeBinary(value_len))
726            .len(v.len())
727            .offset(v.offset())
728            .add_buffer(child_data.buffers()[0].slice(child_data.offset()))
729            .nulls(v.nulls().cloned());
730
731        let data = unsafe { builder.build_unchecked() };
732        Self::from(data)
733    }
734}
735
736impl TryFrom<Vec<Option<&[u8]>>> for FixedSizeBinaryArray {
737    type Error = ArrowError;
738
739    fn try_from(v: Vec<Option<&[u8]>>) -> Result<Self, Self::Error> {
740        #[allow(deprecated)]
741        Self::try_from_sparse_iter(v.into_iter())
742    }
743}
744
745impl TryFrom<Vec<&[u8]>> for FixedSizeBinaryArray {
746    type Error = ArrowError;
747
748    fn try_from(v: Vec<&[u8]>) -> Result<Self, Self::Error> {
749        Self::try_from_iter(v.into_iter())
750    }
751}
752
753impl<const N: usize> TryFrom<Vec<Option<&[u8; N]>>> for FixedSizeBinaryArray {
754    type Error = ArrowError;
755
756    fn try_from(v: Vec<Option<&[u8; N]>>) -> Result<Self, Self::Error> {
757        N.try_into()
758            .map_err(|_| {
759                ArrowError::InvalidArgumentError(format!(
760                    "FixedSizeBinaryArray value length exceeds i32, got {N}"
761                ))
762            })
763            .and_then(|x| Self::try_from_sparse_iter_with_size(v.into_iter(), x))
764    }
765}
766
767impl<const N: usize> TryFrom<Vec<&[u8; N]>> for FixedSizeBinaryArray {
768    type Error = ArrowError;
769
770    fn try_from(v: Vec<&[u8; N]>) -> Result<Self, Self::Error> {
771        Self::try_from_iter(v.into_iter())
772    }
773}
774
775impl std::fmt::Debug for FixedSizeBinaryArray {
776    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
777        write!(f, "FixedSizeBinaryArray<{}>\n[\n", self.value_length())?;
778        print_long_array(self, f, |array, index, f| {
779            std::fmt::Debug::fmt(&array.value(index), f)
780        })?;
781        write!(f, "]")
782    }
783}
784
785/// SAFETY: Correctly implements the contract of Arrow Arrays
786unsafe impl Array for FixedSizeBinaryArray {
787    fn as_any(&self) -> &dyn Any {
788        self
789    }
790
791    fn to_data(&self) -> ArrayData {
792        self.clone().into()
793    }
794
795    fn into_data(self) -> ArrayData {
796        self.into()
797    }
798
799    fn data_type(&self) -> &DataType {
800        &self.data_type
801    }
802
803    fn slice(&self, offset: usize, length: usize) -> ArrayRef {
804        Arc::new(self.slice(offset, length))
805    }
806
807    fn len(&self) -> usize {
808        self.len
809    }
810
811    fn is_empty(&self) -> bool {
812        self.len == 0
813    }
814
815    fn shrink_to_fit(&mut self) {
816        self.value_data.shrink_to_fit();
817        if let Some(nulls) = &mut self.nulls {
818            nulls.shrink_to_fit();
819        }
820    }
821
822    fn offset(&self) -> usize {
823        // Slices are normalized by slicing `value_data`/`nulls` directly;
824        // FSB does not retain a separate logical element offset.
825        0
826    }
827
828    fn nulls(&self) -> Option<&NullBuffer> {
829        self.nulls.as_ref()
830    }
831
832    fn logical_null_count(&self) -> usize {
833        // More efficient that the default implementation
834        self.null_count()
835    }
836
837    fn get_buffer_memory_size(&self) -> usize {
838        let mut sum = self.value_data.capacity();
839        if let Some(n) = &self.nulls {
840            sum += n.buffer().capacity();
841        }
842        sum
843    }
844
845    fn get_array_memory_size(&self) -> usize {
846        std::mem::size_of::<Self>() + self.get_buffer_memory_size()
847    }
848
849    #[cfg(feature = "pool")]
850    fn claim(&self, pool: &dyn arrow_buffer::MemoryPool) {
851        self.value_data.claim(pool);
852        if let Some(nulls) = &self.nulls {
853            nulls.claim(pool);
854        }
855    }
856}
857
858impl<'a> ArrayAccessor for &'a FixedSizeBinaryArray {
859    type Item = &'a [u8];
860
861    fn value(&self, index: usize) -> Self::Item {
862        FixedSizeBinaryArray::value(self, index)
863    }
864
865    unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
866        unsafe { FixedSizeBinaryArray::value_unchecked(self, index) }
867    }
868}
869
870impl<'a> IntoIterator for &'a FixedSizeBinaryArray {
871    type Item = Option<&'a [u8]>;
872    type IntoIter = FixedSizeBinaryIter<'a>;
873
874    fn into_iter(self) -> Self::IntoIter {
875        FixedSizeBinaryIter::<'a>::new(self)
876    }
877}
878
879#[cfg(test)]
880mod tests {
881    use super::*;
882    use crate::RecordBatch;
883    use arrow_schema::{Field, Schema};
884
885    #[test]
886    fn test_fixed_size_binary_array() {
887        let values = b"hellotherearrow";
888
889        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
890            .len(3)
891            .add_buffer(Buffer::from(values))
892            .build()
893            .unwrap();
894        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
895        assert_eq!(3, fixed_size_binary_array.len());
896        assert_eq!(0, fixed_size_binary_array.null_count());
897        assert_eq!(
898            [b'h', b'e', b'l', b'l', b'o'],
899            fixed_size_binary_array.value(0)
900        );
901        assert_eq!(
902            [b't', b'h', b'e', b'r', b'e'],
903            fixed_size_binary_array.value(1)
904        );
905        assert_eq!(
906            [b'a', b'r', b'r', b'o', b'w'],
907            fixed_size_binary_array.value(2)
908        );
909        assert_eq!(5, fixed_size_binary_array.value_length());
910        for i in 0..3 {
911            assert!(fixed_size_binary_array.is_valid(i));
912            assert!(!fixed_size_binary_array.is_null(i));
913        }
914
915        // Test binary array with offset
916        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
917            .len(2)
918            .offset(1)
919            .add_buffer(Buffer::from(values))
920            .build()
921            .unwrap();
922        let fixed_size_binary_array = FixedSizeBinaryArray::from(array_data);
923        assert_eq!(
924            [b't', b'h', b'e', b'r', b'e'],
925            fixed_size_binary_array.value(0)
926        );
927        assert_eq!(
928            [b'a', b'r', b'r', b'o', b'w'],
929            fixed_size_binary_array.value(1)
930        );
931        assert_eq!(2, fixed_size_binary_array.len());
932        assert_eq!(5, fixed_size_binary_array.value_length());
933    }
934
935    #[test]
936    fn test_fixed_size_binary_array_from_fixed_size_list_array() {
937        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13];
938        let values_data = ArrayData::builder(DataType::UInt8)
939            .len(12)
940            .offset(2)
941            .add_buffer(Buffer::from_slice_ref(values))
942            .build()
943            .unwrap();
944        // [null, [10, 11, 12, 13]]
945        let array_data = unsafe {
946            ArrayData::builder(DataType::FixedSizeList(
947                Arc::new(Field::new_list_field(DataType::UInt8, false)),
948                4,
949            ))
950            .len(2)
951            .offset(1)
952            .add_child_data(values_data)
953            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101])))
954            .build_unchecked()
955        };
956        let list_array = FixedSizeListArray::from(array_data);
957        let binary_array = FixedSizeBinaryArray::from(list_array);
958
959        assert_eq!(2, binary_array.len());
960        assert_eq!(1, binary_array.null_count());
961        assert!(binary_array.is_null(0));
962        assert!(binary_array.is_valid(1));
963        assert_eq!(&[10, 11, 12, 13], binary_array.value(1));
964    }
965
966    #[test]
967    #[should_panic(
968        expected = "FixedSizeBinaryArray can only be created from FixedSizeList<u8> arrays"
969    )]
970    // Different error messages, so skip for now
971    // https://github.com/apache/arrow-rs/issues/1545
972    #[cfg(not(feature = "force_validate"))]
973    fn test_fixed_size_binary_array_from_incorrect_fixed_size_list_array() {
974        let values: [u32; 12] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
975        let values_data = ArrayData::builder(DataType::UInt32)
976            .len(12)
977            .add_buffer(Buffer::from_slice_ref(values))
978            .build()
979            .unwrap();
980
981        let array_data = unsafe {
982            ArrayData::builder(DataType::FixedSizeList(
983                Arc::new(Field::new_list_field(DataType::Binary, false)),
984                4,
985            ))
986            .len(3)
987            .add_child_data(values_data)
988            .build_unchecked()
989        };
990        let list_array = FixedSizeListArray::from(array_data);
991        drop(FixedSizeBinaryArray::from(list_array));
992    }
993
994    #[test]
995    #[should_panic(expected = "The child array cannot contain null values.")]
996    fn test_fixed_size_binary_array_from_fixed_size_list_array_with_child_nulls_failed() {
997        let values = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
998        let values_data = ArrayData::builder(DataType::UInt8)
999            .len(12)
1000            .add_buffer(Buffer::from_slice_ref(values))
1001            .null_bit_buffer(Some(Buffer::from_slice_ref([0b101010101010])))
1002            .build()
1003            .unwrap();
1004
1005        let array_data = unsafe {
1006            ArrayData::builder(DataType::FixedSizeList(
1007                Arc::new(Field::new_list_field(DataType::UInt8, false)),
1008                4,
1009            ))
1010            .len(3)
1011            .add_child_data(values_data)
1012            .build_unchecked()
1013        };
1014        let list_array = FixedSizeListArray::from(array_data);
1015        drop(FixedSizeBinaryArray::from(list_array));
1016    }
1017
1018    #[test]
1019    fn test_fixed_size_binary_array_fmt_debug() {
1020        let values = b"hellotherearrow";
1021
1022        let array_data = ArrayData::builder(DataType::FixedSizeBinary(5))
1023            .len(3)
1024            .add_buffer(Buffer::from(values))
1025            .build()
1026            .unwrap();
1027        let arr = FixedSizeBinaryArray::from(array_data);
1028        assert_eq!(
1029            "FixedSizeBinaryArray<5>\n[\n  [104, 101, 108, 108, 111],\n  [116, 104, 101, 114, 101],\n  [97, 114, 114, 111, 119],\n]",
1030            format!("{arr:?}")
1031        );
1032    }
1033
1034    #[test]
1035    fn test_fixed_size_binary_array_from_iter() {
1036        let input_arg = vec![vec![1, 2], vec![3, 4], vec![5, 6]];
1037        let arr = FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap();
1038
1039        assert_eq!(2, arr.value_length());
1040        assert_eq!(3, arr.len())
1041    }
1042
1043    #[test]
1044    fn test_all_none_fixed_size_binary_array_from_sparse_iter() {
1045        let none_option: Option<[u8; 32]> = None;
1046        let input_arg = vec![none_option, none_option, none_option];
1047        #[allow(deprecated)]
1048        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.into_iter()).unwrap();
1049        assert_eq!(0, arr.value_length());
1050        assert_eq!(3, arr.len())
1051    }
1052
1053    #[test]
1054    fn test_fixed_size_binary_array_from_sparse_iter() {
1055        let input_arg = vec![
1056            None,
1057            Some(vec![7, 8]),
1058            Some(vec![9, 10]),
1059            None,
1060            Some(vec![13, 14]),
1061        ];
1062        #[allow(deprecated)]
1063        let arr = FixedSizeBinaryArray::try_from_sparse_iter(input_arg.iter().cloned()).unwrap();
1064        assert_eq!(2, arr.value_length());
1065        assert_eq!(5, arr.len());
1066
1067        let arr =
1068            FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 2).unwrap();
1069        assert_eq!(2, arr.value_length());
1070        assert_eq!(5, arr.len());
1071    }
1072
1073    #[test]
1074    fn test_fixed_size_binary_array_from_sparse_iter_with_size_all_none() {
1075        let input_arg = vec![None, None, None, None, None] as Vec<Option<Vec<u8>>>;
1076
1077        let arr = FixedSizeBinaryArray::try_from_sparse_iter_with_size(input_arg.into_iter(), 16)
1078            .unwrap();
1079        assert_eq!(16, arr.value_length());
1080        assert_eq!(5, arr.len())
1081    }
1082
1083    #[test]
1084    fn test_fixed_size_binary_array_from_vec() {
1085        let values = vec!["one".as_bytes(), b"two", b"six", b"ten"];
1086        let array = FixedSizeBinaryArray::try_from(values).unwrap();
1087        assert_eq!(array.len(), 4);
1088        assert_eq!(array.null_count(), 0);
1089        assert_eq!(array.logical_null_count(), 0);
1090        assert_eq!(array.value(0), b"one");
1091        assert_eq!(array.value(1), b"two");
1092        assert_eq!(array.value(2), b"six");
1093        assert_eq!(array.value(3), b"ten");
1094        assert!(!array.is_null(0));
1095        assert!(!array.is_null(1));
1096        assert!(!array.is_null(2));
1097        assert!(!array.is_null(3));
1098    }
1099
1100    #[test]
1101    fn test_fixed_size_binary_array_from_vec_incorrect_length() {
1102        let values = vec!["one".as_bytes(), b"two", b"three", b"four"];
1103        assert!(FixedSizeBinaryArray::try_from(values).is_err());
1104    }
1105
1106    #[test]
1107    fn test_fixed_size_binary_array_from_opt_vec() {
1108        let values = vec![
1109            Some("one".as_bytes()),
1110            Some(b"two"),
1111            None,
1112            Some(b"six"),
1113            Some(b"ten"),
1114        ];
1115        let array = FixedSizeBinaryArray::try_from(values).unwrap();
1116        assert_eq!(array.len(), 5);
1117        assert_eq!(array.value(0), b"one");
1118        assert_eq!(array.value(1), b"two");
1119        assert_eq!(array.value(3), b"six");
1120        assert_eq!(array.value(4), b"ten");
1121        assert!(!array.is_null(0));
1122        assert!(!array.is_null(1));
1123        assert!(array.is_null(2));
1124        assert!(!array.is_null(3));
1125        assert!(!array.is_null(4));
1126    }
1127
1128    #[test]
1129    fn test_fixed_size_binary_array_from_opt_vec_incorrect_length() {
1130        let values = vec![
1131            Some("one".as_bytes()),
1132            Some(b"two"),
1133            None,
1134            Some(b"three"),
1135            Some(b"four"),
1136        ];
1137        assert!(FixedSizeBinaryArray::try_from(values).is_err());
1138    }
1139
1140    #[test]
1141    fn fixed_size_binary_array_all_null() {
1142        let data = vec![None] as Vec<Option<String>>;
1143        let array =
1144            FixedSizeBinaryArray::try_from_sparse_iter_with_size(data.into_iter(), 0).unwrap();
1145        array
1146            .into_data()
1147            .validate_full()
1148            .expect("All null array has valid array data");
1149    }
1150
1151    #[test]
1152    // Test for https://github.com/apache/arrow-rs/issues/1390
1153    fn fixed_size_binary_array_all_null_in_batch_with_schema() {
1154        let schema = Schema::new(vec![Field::new("a", DataType::FixedSizeBinary(2), true)]);
1155
1156        let none_option: Option<[u8; 2]> = None;
1157        let item = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1158            vec![none_option, none_option, none_option].into_iter(),
1159            2,
1160        )
1161        .unwrap();
1162
1163        // Should not panic
1164        RecordBatch::try_new(Arc::new(schema), vec![Arc::new(item)]).unwrap();
1165    }
1166
1167    #[test]
1168    #[should_panic(
1169        expected = "Trying to access an element at index 4 from a FixedSizeBinaryArray of length 3"
1170    )]
1171    fn test_fixed_size_binary_array_get_value_index_out_of_bound() {
1172        let values = vec![Some("one".as_bytes()), Some(b"two"), None];
1173        let array = FixedSizeBinaryArray::try_from(values).unwrap();
1174
1175        array.value(4);
1176    }
1177
1178    #[test]
1179    fn test_constructors() {
1180        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1181        let a = FixedSizeBinaryArray::new(2, buffer.clone(), None);
1182        assert_eq!(a.len(), 5);
1183
1184        let nulls = NullBuffer::new_null(5);
1185        FixedSizeBinaryArray::new(2, buffer.clone(), Some(nulls));
1186
1187        let null_array = FixedSizeBinaryArray::new_null(4, 3);
1188        assert_eq!(null_array.len(), 3);
1189        assert_eq!(null_array.values().len(), 12);
1190
1191        let a = FixedSizeBinaryArray::new(3, buffer.clone(), None);
1192        assert_eq!(a.len(), 3);
1193
1194        let nulls = NullBuffer::new_null(3);
1195        FixedSizeBinaryArray::new(3, buffer.clone(), Some(nulls));
1196
1197        let err = FixedSizeBinaryArray::try_new(-1, buffer.clone(), None).unwrap_err();
1198
1199        assert_eq!(
1200            err.to_string(),
1201            "Invalid argument error: Value length cannot be negative, got -1"
1202        );
1203
1204        let nulls = NullBuffer::new_null(3);
1205        let err = FixedSizeBinaryArray::try_new(2, buffer.clone(), Some(nulls)).unwrap_err();
1206        assert_eq!(
1207            err.to_string(),
1208            "Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3"
1209        );
1210
1211        let zero_sized = FixedSizeBinaryArray::new(0, Buffer::default(), None);
1212        assert_eq!(zero_sized.len(), 0);
1213        assert_eq!(zero_sized.null_count(), 0);
1214        assert_eq!(zero_sized.values().len(), 0);
1215
1216        let nulls = NullBuffer::new_null(3);
1217        let zero_sized_with_nulls = FixedSizeBinaryArray::new(0, Buffer::default(), Some(nulls));
1218        assert_eq!(zero_sized_with_nulls.len(), 3);
1219        assert_eq!(zero_sized_with_nulls.null_count(), 3);
1220        assert_eq!(zero_sized_with_nulls.values().len(), 0);
1221
1222        let zero_sized_with_non_empty_buffer_err =
1223            FixedSizeBinaryArray::try_new(0, buffer, None).unwrap_err();
1224        assert_eq!(
1225            zero_sized_with_non_empty_buffer_err.to_string(),
1226            "Invalid argument error: Buffer cannot have non-zero length if the value length is zero"
1227        );
1228    }
1229
1230    #[test]
1231    fn test_try_new_with_len() {
1232        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1233
1234        let a = FixedSizeBinaryArray::try_new_with_len(2, buffer.clone(), None, 5).unwrap();
1235        assert_eq!(a.len(), 5);
1236
1237        let nulls = NullBuffer::new_null(5);
1238        let a = FixedSizeBinaryArray::try_new_with_len(2, buffer, Some(nulls), 5).unwrap();
1239        assert_eq!(a.len(), 5);
1240        assert_eq!(a.null_count(), 5);
1241
1242        let a = FixedSizeBinaryArray::try_new_with_len(2, Buffer::default(), None, 0).unwrap();
1243        assert_eq!(a.len(), 0);
1244    }
1245
1246    #[test]
1247    fn test_try_new_with_len_zero_width() {
1248        // Zero-width with no nulls: the case where the length cannot be inferred from the parts
1249        let a = FixedSizeBinaryArray::try_new_with_len(0, Buffer::default(), None, 5).unwrap();
1250        assert_eq!(a.len(), 5);
1251        assert_eq!(a.null_count(), 0);
1252        assert_eq!(a.values().len(), 0);
1253
1254        let nulls = NullBuffer::new_null(3);
1255        let a =
1256            FixedSizeBinaryArray::try_new_with_len(0, Buffer::default(), Some(nulls), 3).unwrap();
1257        assert_eq!(a.len(), 3);
1258        assert_eq!(a.null_count(), 3);
1259    }
1260
1261    #[test]
1262    fn test_try_new_with_len_negative_value_length() {
1263        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1264        let err = FixedSizeBinaryArray::try_new_with_len(-1, buffer, None, 5).unwrap_err();
1265        assert_eq!(
1266            err.to_string(),
1267            "Invalid argument error: Value length cannot be negative, got -1"
1268        );
1269    }
1270
1271    #[test]
1272    fn test_try_new_with_len_incorrect_null_buffer_length() {
1273        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1274        let nulls = NullBuffer::new_null(3);
1275        let err = FixedSizeBinaryArray::try_new_with_len(2, buffer, Some(nulls), 5).unwrap_err();
1276        assert_eq!(
1277            err.to_string(),
1278            "Invalid argument error: Incorrect length of null buffer for FixedSizeBinaryArray, expected 5 got 3"
1279        );
1280    }
1281
1282    #[test]
1283    fn test_try_new_with_len_incorrect_values_buffer_length() {
1284        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1285        let err = FixedSizeBinaryArray::try_new_with_len(2, buffer, None, 3).unwrap_err();
1286        assert_eq!(
1287            err.to_string(),
1288            "Invalid argument error: Incorrect length of values buffer for FixedSizeBinaryArray, expected 3 got 5"
1289        );
1290    }
1291
1292    #[test]
1293    fn test_try_new_with_len_zero_width_non_empty_buffer() {
1294        let buffer = Buffer::from_vec(vec![0_u8; 10]);
1295        let err = FixedSizeBinaryArray::try_new_with_len(0, buffer, None, 5).unwrap_err();
1296        assert_eq!(
1297            err.to_string(),
1298            "Invalid argument error: Buffer cannot have non-zero length if the value length is zero"
1299        );
1300    }
1301}
arrow_array/array/fixed_size_binary_array.rs

arrow_array/array/
fixed_size_binary_array.rs
