Skip to main content

astrs_data/array/
fixed_size_binary.rs

1//! [`FixedSizeBinaryArray`] — a column of equally sized byte blobs.
2//!
3//! No offset buffer: slot `i` is exactly `values[i * size .. (i + 1) * size]`.
4//! This is the layout AstRS uses for hashes, fixed-length IDs, small
5//! fixed-shape sensor records and the raw bytes of a fixed-format image tile,
6//! and it is the cheapest variable-free byte layout Arrow offers.
7//!
8//! ```
9//! use astrs_data::array::{Array, FixedSizeBinaryArray};
10//!
11//! let hashes = FixedSizeBinaryArray::try_from_values(4, [&b"aaaa"[..], &b"bbbb"[..]])?;
12//! assert_eq!(hashes.len(), 2);
13//! assert_eq!(hashes.value_size(), 4);
14//! assert_eq!(hashes.get(1), Some(&b"bbbb"[..]));
15//! # Ok::<(), astrs_data::DataError>(())
16//! ```
17
18use std::any::Any;
19use std::fmt;
20use std::sync::Arc;
21
22use crate::array::iter::{ArrayAccessor, ArrayIter};
23use crate::array::{
24    Array, ArrayRef, check_fixed_size, check_validity, clamp_window, debug_array, validity_eq,
25};
26use crate::buffer::{AlignedBuf, Bitmap, BitmapBuilder, Buffer};
27use crate::datatype::DataType;
28use crate::error::{DataError, Result};
29use crate::sealed::Sealed;
30
31/// A column of `size`-byte blobs with an optional validity bitmap.
32#[derive(Clone)]
33pub struct FixedSizeBinaryArray {
34    /// Always `DataType::FixedSizeBinary(size)`.
35    data_type: DataType,
36    /// Bytes per slot. Always positive.
37    size: usize,
38    /// `len * size` bytes.
39    values: Buffer,
40    /// Logical length, `values.len() / size`.
41    len: usize,
42    /// Validity, or `None` when every slot is valid.
43    validity: Option<Bitmap>,
44}
45
46impl FixedSizeBinaryArray {
47    /// Builds an array from a flat value buffer.
48    ///
49    /// # Errors
50    ///
51    /// * [`DataError::InvalidFixedSize`] when `size` is not positive.
52    /// * [`DataError::BufferLengthNotMultiple`] when the buffer is not a whole
53    ///   number of slots.
54    /// * [`DataError::ValidityLengthMismatch`] when the bitmap length is wrong.
55    ///
56    /// ```
57    /// use astrs_data::array::{Array, FixedSizeBinaryArray};
58    /// use astrs_data::Buffer;
59    ///
60    /// let array = FixedSizeBinaryArray::try_new(2, Buffer::from_slice(b"abcd"), None)?;
61    /// assert_eq!(array.len(), 2);
62    /// assert_eq!(array.get(0), Some(&b"ab"[..]));
63    /// # Ok::<(), astrs_data::DataError>(())
64    /// ```
65    pub fn try_new(size: i32, values: Buffer, validity: Option<Bitmap>) -> Result<Self> {
66        let width = check_fixed_size(size)?;
67        if !values.len().is_multiple_of(width) {
68            return Err(DataError::BufferLengthNotMultiple {
69                len: values.len(),
70                width,
71            });
72        }
73        let len = values.len() / width;
74        check_validity(validity.as_ref(), len)?;
75        Ok(Self {
76            data_type: DataType::FixedSizeBinary(size),
77            size: width,
78            values,
79            len,
80            validity,
81        })
82    }
83
84    /// Builds an array from equally sized slices.
85    ///
86    /// # Errors
87    ///
88    /// * [`DataError::InvalidFixedSize`] when `size` is not positive.
89    /// * [`DataError::ChildLengthMismatch`] when a value has the wrong length.
90    pub fn try_from_values<V: AsRef<[u8]>>(
91        size: i32,
92        values: impl IntoIterator<Item = V>,
93    ) -> Result<Self> {
94        Self::try_from_opt_iter(size, values.into_iter().map(Some))
95    }
96
97    /// Builds an array from optional equally sized slices, null slots holding
98    /// `size` zero bytes.
99    ///
100    /// # Errors
101    ///
102    /// * [`DataError::InvalidFixedSize`] when `size` is not positive.
103    /// * [`DataError::ChildLengthMismatch`] when a value has the wrong length.
104    ///
105    /// ```
106    /// use astrs_data::array::{Array, FixedSizeBinaryArray};
107    ///
108    /// let array = FixedSizeBinaryArray::try_from_opt_iter(2, [Some(&b"ab"[..]), None])?;
109    /// assert_eq!(array.null_count(), 1);
110    /// assert!(FixedSizeBinaryArray::try_from_values(2, [&b"abc"[..]]).is_err());
111    /// # Ok::<(), astrs_data::DataError>(())
112    /// ```
113    pub fn try_from_opt_iter<V: AsRef<[u8]>>(
114        size: i32,
115        values: impl IntoIterator<Item = Option<V>>,
116    ) -> Result<Self> {
117        let width = check_fixed_size(size)?;
118        let iter = values.into_iter();
119        let (lower, _) = iter.size_hint();
120        let mut data = AlignedBuf::with_capacity(lower.saturating_mul(width));
121        let mut validity = BitmapBuilder::with_capacity(lower);
122        let mut any_null = false;
123        for value in iter {
124            match value {
125                Some(value) => {
126                    let bytes = value.as_ref();
127                    if bytes.len() != width {
128                        return Err(DataError::ChildLengthMismatch {
129                            expected: width,
130                            actual: bytes.len(),
131                        });
132                    }
133                    data.extend_from_slice(bytes);
134                    validity.append(true);
135                }
136                None => {
137                    data.extend_zeroed(width);
138                    validity.append(false);
139                    any_null = true;
140                }
141            }
142        }
143        let len = validity.len();
144        Ok(Self {
145            data_type: DataType::FixedSizeBinary(size),
146            size: width,
147            values: Buffer::from(data),
148            len,
149            validity: any_null.then(|| validity.finish()),
150        })
151    }
152
153    /// An all-null array of `len` slots.
154    ///
155    /// # Errors
156    ///
157    /// [`DataError::InvalidFixedSize`] when `size` is not positive.
158    pub fn new_null(size: i32, len: usize) -> Result<Self> {
159        let width = check_fixed_size(size)?;
160        Ok(Self {
161            data_type: DataType::FixedSizeBinary(size),
162            size: width,
163            values: Buffer::zeroed(len.saturating_mul(width)),
164            len,
165            validity: Some(Bitmap::new_unset(len)),
166        })
167    }
168
169    /// Assembles the parts without revalidating.
170    ///
171    /// Safe, and crate-internal: [`crate::builder::FixedSizeBinaryBuilder`]
172    /// checks every value's width at `append`, so the finished buffer is a
173    /// whole number of slots by construction.
174    pub(crate) fn from_parts(size: i32, values: Buffer, validity: Option<Bitmap>) -> Self {
175        let width = usize::try_from(size).unwrap_or(1).max(1);
176        debug_assert_eq!(values.len() % width, 0);
177        Self {
178            data_type: DataType::FixedSizeBinary(size),
179            size: width,
180            len: values.len() / width,
181            values,
182            validity,
183        }
184    }
185
186    /// Bytes per slot.
187    #[inline]
188    #[must_use]
189    pub const fn value_size(&self) -> usize {
190        self.size
191    }
192
193    /// The flat value buffer, `len * size` bytes.
194    #[inline]
195    #[must_use]
196    pub const fn value_data(&self) -> &Buffer {
197        &self.values
198    }
199
200    /// The raw slot bytes at `index`, ignoring validity.
201    ///
202    /// Returns `None` only when `index` is out of range.
203    #[inline]
204    #[must_use]
205    pub fn value(&self, index: usize) -> Option<&[u8]> {
206        if index >= self.len {
207            return None;
208        }
209        let start = index * self.size;
210        self.values.as_slice().get(start..start + self.size)
211    }
212
213    /// The logical value at `index`: `None` when the slot is null or out of
214    /// range.
215    #[inline]
216    #[must_use]
217    pub fn get(&self, index: usize) -> Option<&[u8]> {
218        if self.is_valid_index(index) {
219            self.value(index)
220        } else {
221            None
222        }
223    }
224
225    /// Iterates over the logical values, `None` for nulls.
226    #[inline]
227    pub fn iter(&self) -> ArrayIter<&Self> {
228        ArrayIter::new(self)
229    }
230
231    /// A zero-copy sub-range, clamped to the array (the crate-wide slicing
232    /// convention).
233    #[must_use]
234    pub fn slice(&self, offset: usize, len: usize) -> Self {
235        let (offset, len) = clamp_window(self.len, offset, len);
236        Self {
237            data_type: self.data_type.clone(),
238            size: self.size,
239            values: self.values.slice(offset * self.size, len * self.size),
240            len,
241            validity: self.validity.as_ref().map(|bits| bits.slice(offset, len)),
242        }
243    }
244
245    /// Checked [`FixedSizeBinaryArray::slice`].
246    ///
247    /// # Errors
248    ///
249    /// [`DataError::SliceOutOfBounds`] when the window leaves the array.
250    pub fn try_slice(&self, offset: usize, len: usize) -> Result<Self> {
251        if offset.saturating_add(len) > self.len {
252            return Err(DataError::SliceOutOfBounds {
253                offset,
254                len,
255                available: self.len,
256            });
257        }
258        Ok(self.slice(offset, len))
259    }
260
261    /// Returns a copy with a different validity bitmap.
262    ///
263    /// # Errors
264    ///
265    /// [`DataError::ValidityLengthMismatch`] when the bitmap length is wrong.
266    pub fn with_validity(&self, validity: Option<Bitmap>) -> Result<Self> {
267        check_validity(validity.as_ref(), self.len)?;
268        Ok(Self {
269            data_type: self.data_type.clone(),
270            size: self.size,
271            values: self.values.clone(),
272            len: self.len,
273            validity,
274        })
275    }
276
277    /// Whether `index` is inside the array and not null.
278    #[inline]
279    fn is_valid_index(&self, index: usize) -> bool {
280        index < self.len && self.validity.as_ref().is_none_or(|bits| bits.value(index))
281    }
282}
283
284impl Sealed for FixedSizeBinaryArray {}
285
286impl Array for FixedSizeBinaryArray {
287    fn as_any(&self) -> &dyn Any {
288        self
289    }
290
291    fn data_type(&self) -> &DataType {
292        &self.data_type
293    }
294
295    fn len(&self) -> usize {
296        self.len
297    }
298
299    fn validity(&self) -> Option<&Bitmap> {
300        self.validity.as_ref()
301    }
302
303    fn slice(&self, offset: usize, len: usize) -> ArrayRef {
304        Arc::new(Self::slice(self, offset, len))
305    }
306
307    fn buffer_memory_size(&self) -> usize {
308        self.values.backing_len()
309            + self
310                .validity
311                .as_ref()
312                .map_or(0, |bits| bits.buffer().backing_len())
313    }
314
315    fn equals(&self, other: &dyn Array) -> bool {
316        let Some(other) = other.as_any().downcast_ref::<Self>() else {
317            return false;
318        };
319        if self.size != other.size
320            || self.len != other.len
321            || !validity_eq(self.validity.as_ref(), other.validity.as_ref(), self.len)
322        {
323            return false;
324        }
325        (0..self.len).all(|index| self.get(index) == other.get(index))
326    }
327}
328
329impl<'a> ArrayAccessor for &'a FixedSizeBinaryArray {
330    type Item = &'a [u8];
331
332    #[inline]
333    fn accessor_len(&self) -> usize {
334        self.len
335    }
336
337    #[inline]
338    fn accessor_get(&self, index: usize) -> Option<&'a [u8]> {
339        let array: &'a FixedSizeBinaryArray = self;
340        array.get(index)
341    }
342}
343
344impl PartialEq for FixedSizeBinaryArray {
345    #[inline]
346    fn eq(&self, other: &Self) -> bool {
347        self.equals(other)
348    }
349}
350
351impl Eq for FixedSizeBinaryArray {}
352
353impl<'a> IntoIterator for &'a FixedSizeBinaryArray {
354    type Item = Option<&'a [u8]>;
355    type IntoIter = ArrayIter<&'a FixedSizeBinaryArray>;
356
357    fn into_iter(self) -> Self::IntoIter {
358        self.iter()
359    }
360}
361
362impl fmt::Debug for FixedSizeBinaryArray {
363    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
364        debug_array(
365            f,
366            "FixedSizeBinaryArray",
367            &self.data_type,
368            self.len,
369            self.null_count(),
370            self.iter(),
371        )
372    }
373}
374
375#[cfg(test)]
376mod tests {
377    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
378
379    use super::*;
380
381    #[test]
382    fn empty_array() {
383        let array = FixedSizeBinaryArray::try_from_values(4, [] as [&[u8]; 0]).unwrap();
384        assert!(array.is_empty());
385        assert_eq!(array.value_size(), 4);
386        assert_eq!(array.get(0), None);
387        assert_eq!(array.iter().count(), 0);
388        assert_eq!(array.data_type(), &DataType::FixedSizeBinary(4));
389    }
390
391    #[test]
392    fn no_null_array() {
393        let array =
394            FixedSizeBinaryArray::try_from_values(3, [&b"abc"[..], &b"def"[..], &b"ghi"[..]])
395                .unwrap();
396        assert_eq!(array.len(), 3);
397        assert_eq!(array.null_count(), 0);
398        assert!(array.validity().is_none());
399        assert_eq!(array.get(0), Some(&b"abc"[..]));
400        assert_eq!(array.get(2), Some(&b"ghi"[..]));
401        assert_eq!(array.get(3), None);
402        assert_eq!(array.value_data().as_slice(), b"abcdefghi");
403    }
404
405    #[test]
406    fn all_null_array() {
407        let array = FixedSizeBinaryArray::new_null(2, 3).unwrap();
408        assert_eq!(array.len(), 3);
409        assert_eq!(array.null_count(), 3);
410        for index in 0..3 {
411            assert!(array.is_null(index));
412            assert_eq!(array.get(index), None);
413            assert_eq!(array.value(index), Some(&[0u8, 0][..]));
414        }
415    }
416
417    #[test]
418    fn mixed_nulls() {
419        let array =
420            FixedSizeBinaryArray::try_from_opt_iter(2, [Some(&b"ab"[..]), None, Some(&b"cd"[..])])
421                .unwrap();
422        assert_eq!(array.len(), 3);
423        assert_eq!(array.null_count(), 1);
424        assert_eq!(array.get(1), None);
425        assert_eq!(array.value(1), Some(&[0u8, 0][..]));
426        assert_eq!(
427            array.iter().flatten().collect::<Vec<_>>(),
428            vec![&b"ab"[..], &b"cd"[..]]
429        );
430    }
431
432    #[test]
433    fn wrong_width_values_are_rejected() {
434        assert_eq!(
435            FixedSizeBinaryArray::try_from_values(2, [&b"abc"[..]]).unwrap_err(),
436            DataError::ChildLengthMismatch {
437                expected: 2,
438                actual: 3
439            }
440        );
441        assert_eq!(
442            FixedSizeBinaryArray::try_from_values(2, [&b""[..]]).unwrap_err(),
443            DataError::ChildLengthMismatch {
444                expected: 2,
445                actual: 0
446            }
447        );
448    }
449
450    #[test]
451    fn invalid_sizes_are_rejected() {
452        assert_eq!(
453            FixedSizeBinaryArray::try_new(0, Buffer::new(), None).unwrap_err(),
454            DataError::InvalidFixedSize { size: 0 }
455        );
456        assert!(FixedSizeBinaryArray::try_new(-4, Buffer::new(), None).is_err());
457        assert!(FixedSizeBinaryArray::new_null(0, 3).is_err());
458        assert!(FixedSizeBinaryArray::try_from_values(-1, [] as [&[u8]; 0]).is_err());
459    }
460
461    #[test]
462    fn ragged_buffers_are_rejected() {
463        assert_eq!(
464            FixedSizeBinaryArray::try_new(4, Buffer::from_slice(b"abcdef"), None).unwrap_err(),
465            DataError::BufferLengthNotMultiple { len: 6, width: 4 }
466        );
467        assert!(
468            FixedSizeBinaryArray::try_new(2, Buffer::from_slice(b"abcd"), Some(Bitmap::new_set(3)))
469                .is_err()
470        );
471    }
472
473    #[test]
474    fn slicing_is_zero_copy_and_clamping() {
475        let array =
476            FixedSizeBinaryArray::try_from_values(2, (0..10).map(|i| vec![i as u8, (i * 2) as u8]))
477                .unwrap();
478        let window = array.slice(3, 4);
479        assert_eq!(window.len(), 4);
480        assert_eq!(window.get(0), Some(&[3u8, 6][..]));
481        assert_eq!(window.value_size(), 2);
482        assert_eq!(
483            window.value_data().backing_len(),
484            array.value_data().backing_len(),
485            "the values buffer is shared"
486        );
487        assert_eq!(array.slice(9, 99).len(), 1);
488        assert_eq!(array.slice(99, 1).len(), 0);
489        assert_eq!(array.slice(2, 3).slice(1, 1).get(0), Some(&[3u8, 6][..]));
490        assert!(array.try_slice(9, 2).is_err());
491    }
492
493    #[test]
494    fn equality_ignores_null_slot_contents() {
495        let a = FixedSizeBinaryArray::try_from_opt_iter(2, [Some(&b"ab"[..]), None]).unwrap();
496        let b = FixedSizeBinaryArray::try_new(
497            2,
498            Buffer::from_slice(b"abzz"),
499            Some([true, false].into_iter().collect()),
500        )
501        .unwrap();
502        assert_eq!(a, b);
503        assert_ne!(
504            a,
505            FixedSizeBinaryArray::try_from_values(2, [&b"ab"[..]]).unwrap()
506        );
507        assert_ne!(
508            a,
509            FixedSizeBinaryArray::try_from_opt_iter(2, [Some(&b"ab"[..]), Some(&b"cd"[..])])
510                .unwrap()
511        );
512    }
513
514    #[test]
515    fn arrays_of_different_widths_are_not_equal() {
516        let two = FixedSizeBinaryArray::try_from_values(2, [&b"ab"[..]]).unwrap();
517        let one = FixedSizeBinaryArray::try_from_values(1, [&b"a"[..]]).unwrap();
518        assert_ne!(two, one);
519    }
520
521    #[test]
522    fn with_validity_replaces_the_bitmap() {
523        let array = FixedSizeBinaryArray::try_from_values(1, [&b"a"[..], &b"b"[..]]).unwrap();
524        let masked = array
525            .with_validity(Some([true, false].into_iter().collect()))
526            .unwrap();
527        assert_eq!(masked.null_count(), 1);
528        assert!(array.with_validity(Some(Bitmap::new_set(9))).is_err());
529    }
530
531    #[test]
532    fn debug_output_is_readable() {
533        let array = FixedSizeBinaryArray::try_from_opt_iter(2, [Some(&b"ab"[..]), None]).unwrap();
534        let rendered = format!("{array:?}");
535        assert!(rendered.contains("FixedSizeBinary(2)"), "{rendered}");
536        assert!(rendered.contains("nulls=1"), "{rendered}");
537    }
538
539    #[test]
540    fn large_fixed_width_column() {
541        let array =
542            FixedSizeBinaryArray::try_from_values(32, (0..2_000).map(|i| vec![i as u8; 32]))
543                .unwrap();
544        assert_eq!(array.len(), 2_000);
545        assert_eq!(array.value_data().len(), 64_000);
546        assert_eq!(array.get(1_999), Some(&[(1_999u32 as u8); 32][..]));
547        assert!(array.buffer_memory_size() >= 64_000);
548    }
549}