Skip to main content

astrs_data/array/
offset.rs

1//! [`OffsetSizeTrait`] — the `i32`/`i64` switch behind the `Binary`/
2//! `LargeBinary` and `Utf8`/`LargeUtf8` pairs, plus shared offset validation.
3//!
4//! Arrow's variable-length layouts differ only in the width of their offset
5//! buffer, so `astrs-data` writes each family once, generic over the offset
6//! type, and exposes the two instantiations as aliases. The `Large` variants
7//! exist because a 4 KiB × 1080p image column overflows `i32` offsets after
8//! ~700 frames — a real limit for robotics payloads.
9//!
10//! ```
11//! use astrs_data::array::{Array, BinaryArray, LargeBinaryArray, OffsetSizeTrait};
12//!
13//! assert!(!<i32 as OffsetSizeTrait>::IS_LARGE);
14//! assert!(<i64 as OffsetSizeTrait>::IS_LARGE);
15//! assert_eq!(BinaryArray::from_opt_iter([Some(&b"x"[..])]).len(), 1);
16//! assert_eq!(LargeBinaryArray::from_opt_iter([Some(&b"x"[..])]).len(), 1);
17//! ```
18
19use crate::datatype::{ArrowNativeType, DataType};
20use crate::error::{DataError, Result};
21
22/// The offset width of a variable-length array: `i32` or `i64`.
23///
24/// Sealed through [`ArrowNativeType`], so the set stays exactly these two.
25pub trait OffsetSizeTrait: ArrowNativeType + Ord {
26    /// `true` for the 64-bit (`Large`) variants.
27    const IS_LARGE: bool;
28
29    /// The additive identity, the value every offset buffer starts at.
30    const ZERO: Self;
31
32    /// The `Binary`/`LargeBinary` type for this offset width.
33    const BINARY_TYPE: DataType;
34
35    /// The `Utf8`/`LargeUtf8` type for this offset width.
36    const UTF8_TYPE: DataType;
37
38    /// Converts to `usize`, or `None` when negative or too large for the host.
39    fn to_usize(self) -> Option<usize>;
40
41    /// Converts from `usize`, or `None` when the value does not fit.
42    fn from_usize(value: usize) -> Option<Self>;
43
44    /// Widens to `i64` for uniform error reporting.
45    fn as_i64(self) -> i64;
46}
47
48impl OffsetSizeTrait for i32 {
49    const IS_LARGE: bool = false;
50    const ZERO: Self = 0;
51    const BINARY_TYPE: DataType = DataType::Binary;
52    const UTF8_TYPE: DataType = DataType::Utf8;
53
54    #[inline]
55    fn to_usize(self) -> Option<usize> {
56        usize::try_from(self).ok()
57    }
58
59    #[inline]
60    fn from_usize(value: usize) -> Option<Self> {
61        Self::try_from(value).ok()
62    }
63
64    #[inline]
65    fn as_i64(self) -> i64 {
66        i64::from(self)
67    }
68}
69
70impl OffsetSizeTrait for i64 {
71    const IS_LARGE: bool = true;
72    const ZERO: Self = 0;
73    const BINARY_TYPE: DataType = DataType::LargeBinary;
74    const UTF8_TYPE: DataType = DataType::LargeUtf8;
75
76    #[inline]
77    fn to_usize(self) -> Option<usize> {
78        usize::try_from(self).ok()
79    }
80
81    #[inline]
82    fn from_usize(value: usize) -> Option<Self> {
83        Self::try_from(value).ok()
84    }
85
86    #[inline]
87    fn as_i64(self) -> i64 {
88        self
89    }
90}
91
92/// Validates an offset buffer against a value region.
93///
94/// This is the check that makes every `value(i)` accessor *total*: after it
95/// passes, `offsets[i] .. offsets[i + 1]` is guaranteed to be a valid range
96/// inside a `values_len`-byte region for every `i < len`. Clippy's `panic`
97/// lint cannot see slice-indexing panics, so this validation is the only thing
98/// standing between a malformed payload and a runtime abort.
99///
100/// # Errors
101///
102/// * [`DataError::OffsetBufferLength`] — not exactly `len + 1` entries.
103/// * [`DataError::NegativeOffset`] — an entry below zero.
104/// * [`DataError::NonMonotonicOffsets`] — an entry below its predecessor.
105/// * [`DataError::OffsetOutOfBounds`] — the last entry past `values_len`.
106///
107/// ```
108/// use astrs_data::array::validate_offsets;
109///
110/// assert!(validate_offsets(&[0i32, 2, 5], 5).is_ok());
111/// assert!(validate_offsets(&[0i32, 2, 5], 4).is_err());
112/// assert!(validate_offsets(&[0i32, 5, 2], 5).is_err());
113/// assert!(validate_offsets(&[] as &[i32], 0).is_err(), "needs at least one entry");
114/// ```
115pub fn validate_offsets<O: OffsetSizeTrait>(offsets: &[O], values_len: usize) -> Result<usize> {
116    let Some((&first, rest)) = offsets.split_first() else {
117        return Err(DataError::OffsetBufferLength {
118            expected: 1,
119            actual: 0,
120        });
121    };
122    if first < O::ZERO {
123        return Err(DataError::NegativeOffset {
124            index: 0,
125            offset: first.as_i64(),
126        });
127    }
128    let mut previous = first;
129    for (position, &offset) in rest.iter().enumerate() {
130        let index = position + 1;
131        if offset < O::ZERO {
132            return Err(DataError::NegativeOffset {
133                index,
134                offset: offset.as_i64(),
135            });
136        }
137        if offset < previous {
138            return Err(DataError::NonMonotonicOffsets {
139                index,
140                offset: offset.as_i64(),
141            });
142        }
143        previous = offset;
144    }
145    let last = previous.to_usize().ok_or(DataError::OffsetOutOfBounds {
146        index: offsets.len() - 1,
147        offset: usize::MAX,
148        values_len,
149    })?;
150    if last > values_len {
151        return Err(DataError::OffsetOutOfBounds {
152            index: offsets.len() - 1,
153            offset: last,
154            values_len,
155        });
156    }
157    Ok(offsets.len() - 1)
158}
159
160#[cfg(test)]
161mod tests {
162    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
163
164    use super::*;
165
166    #[test]
167    fn offset_width_constants() {
168        const { assert!(!<i32 as OffsetSizeTrait>::IS_LARGE) };
169        const { assert!(<i64 as OffsetSizeTrait>::IS_LARGE) };
170        assert_eq!(<i32 as OffsetSizeTrait>::BINARY_TYPE, DataType::Binary);
171        assert_eq!(<i64 as OffsetSizeTrait>::BINARY_TYPE, DataType::LargeBinary);
172        assert_eq!(<i32 as OffsetSizeTrait>::UTF8_TYPE, DataType::Utf8);
173        assert_eq!(<i64 as OffsetSizeTrait>::UTF8_TYPE, DataType::LargeUtf8);
174    }
175
176    #[test]
177    fn conversions_are_range_checked() {
178        assert_eq!(OffsetSizeTrait::to_usize(5i32), Some(5));
179        assert_eq!(OffsetSizeTrait::to_usize(-1i32), None);
180        assert_eq!(<i32 as OffsetSizeTrait>::from_usize(5), Some(5));
181        assert_eq!(<i32 as OffsetSizeTrait>::from_usize(usize::MAX), None);
182        assert_eq!(<i64 as OffsetSizeTrait>::from_usize(1 << 40), Some(1 << 40));
183        assert_eq!(OffsetSizeTrait::as_i64(-3i32), -3);
184        assert_eq!(OffsetSizeTrait::as_i64(-3i64), -3);
185    }
186
187    #[test]
188    fn valid_offsets_report_the_array_length() {
189        assert_eq!(validate_offsets(&[0i32], 0).unwrap(), 0);
190        assert_eq!(validate_offsets(&[0i32, 0, 0], 0).unwrap(), 2);
191        assert_eq!(validate_offsets(&[0i32, 3, 3, 7], 7).unwrap(), 3);
192        assert_eq!(validate_offsets(&[0i64, 4], 100).unwrap(), 1);
193        // Offsets need not start at zero: a sliced array keeps its window.
194        assert_eq!(validate_offsets(&[5i32, 9], 9).unwrap(), 1);
195    }
196
197    #[test]
198    fn empty_offset_buffer_is_rejected() {
199        assert_eq!(
200            validate_offsets(&[] as &[i32], 0).unwrap_err(),
201            DataError::OffsetBufferLength {
202                expected: 1,
203                actual: 0
204            }
205        );
206    }
207
208    #[test]
209    fn negative_offsets_are_rejected() {
210        assert_eq!(
211            validate_offsets(&[-1i32, 0], 0).unwrap_err(),
212            DataError::NegativeOffset {
213                index: 0,
214                offset: -1
215            }
216        );
217        assert_eq!(
218            validate_offsets(&[0i32, -1], 0).unwrap_err(),
219            DataError::NegativeOffset {
220                index: 1,
221                offset: -1
222            }
223        );
224    }
225
226    #[test]
227    fn non_monotonic_offsets_are_rejected() {
228        assert_eq!(
229            validate_offsets(&[0i32, 5, 2], 5).unwrap_err(),
230            DataError::NonMonotonicOffsets {
231                index: 2,
232                offset: 2
233            }
234        );
235        assert!(validate_offsets(&[0i32, 5, 5], 5).is_ok(), "equal is fine");
236    }
237
238    #[test]
239    fn offsets_past_the_value_region_are_rejected() {
240        assert_eq!(
241            validate_offsets(&[0i32, 9], 4).unwrap_err(),
242            DataError::OffsetOutOfBounds {
243                index: 1,
244                offset: 9,
245                values_len: 4
246            }
247        );
248        assert!(
249            validate_offsets(&[0i32, 4], 4).is_ok(),
250            "exactly at the end"
251        );
252    }
253}