vortex_array/arrays/varbinview/
view.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! The 16-byte view struct stored in variable-length binary vectors.
5
6use std::fmt;
7use std::hash::Hash;
8use std::hash::Hasher;
9use std::ops::Range;
10
11use static_assertions::assert_eq_align;
12use static_assertions::assert_eq_size;
13use vortex_error::VortexExpect;
14
15/// A view over a variable-length binary value.
16///
17/// Either an inlined representation (for values <= 12 bytes) or a reference
18/// to an external buffer (for values > 12 bytes).
19#[derive(Clone, Copy)]
20#[repr(C, align(16))]
21pub union BinaryView {
22    /// Numeric representation. This is logically `u128`, but we split it into the high and low
23    /// bits to preserve the alignment.
24    pub(crate) le_bytes: [u8; 16],
25
26    /// Inlined representation: strings <= 12 bytes
27    pub(crate) inlined: Inlined,
28
29    /// Reference type: strings > 12 bytes.
30    pub(crate) _ref: Ref,
31}
32
33assert_eq_align!(BinaryView, u128);
34assert_eq_size!(BinaryView, [u8; 16]);
35assert_eq_size!(Inlined, [u8; 16]);
36assert_eq_size!(Ref, [u8; 16]);
37
38/// Variant of a [`BinaryView`] that holds an inlined value.
39#[derive(Clone, Copy, Debug, PartialEq, Eq)]
40#[repr(C, align(8))]
41pub struct Inlined {
42    /// The size of the full value.
43    pub size: u32,
44    /// The full inlined value.
45    pub data: [u8; BinaryView::MAX_INLINED_SIZE],
46}
47
48impl Inlined {
49    /// Creates a new inlined representation from the provided value of constant size.
50    fn new<const N: usize>(value: &[u8]) -> Self {
51        debug_assert_eq!(value.len(), N);
52        let mut inlined = Self {
53            size: N.try_into().vortex_expect("inlined size must fit in u32"),
54            data: [0u8; BinaryView::MAX_INLINED_SIZE],
55        };
56        inlined.data[..N].copy_from_slice(&value[..N]);
57        inlined
58    }
59
60    /// Returns the full inlined value.
61    #[inline]
62    pub fn value(&self) -> &[u8] {
63        &self.data[0..(self.size as usize)]
64    }
65}
66
67/// Variant of a [`BinaryView`] that holds a reference to an external buffer.
68#[derive(Clone, Copy, Debug)]
69#[repr(C, align(8))]
70pub struct Ref {
71    /// The size of the full value.
72    pub size: u32,
73    /// The prefix bytes of the value (first 4 bytes).
74    pub prefix: [u8; 4],
75    /// The index of the buffer where the full value is stored.
76    pub buffer_index: u32,
77    /// The offset within the buffer where the full value starts.
78    pub offset: u32,
79}
80
81impl Ref {
82    /// Returns the range within the buffer where the full value is stored.
83    #[inline]
84    pub fn as_range(&self) -> Range<usize> {
85        self.offset as usize..(self.offset + self.size) as usize
86    }
87
88    /// Replaces the buffer index and offset of the reference, returning a new `Ref`.
89    #[inline]
90    pub fn with_buffer_and_offset(&self, buffer_index: u32, offset: u32) -> Ref {
91        Self {
92            size: self.size,
93            prefix: self.prefix,
94            buffer_index,
95            offset,
96        }
97    }
98}
99
100impl BinaryView {
101    /// Maximum size of an inlined binary value.
102    ///
103    /// cbindgen:ignore
104    pub const MAX_INLINED_SIZE: usize = 12;
105
106    /// Create a view from a value, block and offset
107    ///
108    /// Depending on the length of the provided value either a new inlined
109    /// or a reference view will be constructed.
110    ///
111    /// Adapted from arrow-rs <https://github.com/apache/arrow-rs/blob/f4fde769ab6e1a9b75f890b7f8b47bc22800830b/arrow-array/src/builder/generic_bytes_view_builder.rs#L524>
112    /// Explicitly enumerating inlined view produces code that avoids calling generic `ptr::copy_non_interleave` that's slower than explicit stores
113    #[inline(never)]
114    pub fn make_view(value: &[u8], block: u32, offset: u32) -> Self {
115        match value.len() {
116            0 => Self {
117                inlined: Inlined::new::<0>(value),
118            },
119            1 => Self {
120                inlined: Inlined::new::<1>(value),
121            },
122            2 => Self {
123                inlined: Inlined::new::<2>(value),
124            },
125            3 => Self {
126                inlined: Inlined::new::<3>(value),
127            },
128            4 => Self {
129                inlined: Inlined::new::<4>(value),
130            },
131            5 => Self {
132                inlined: Inlined::new::<5>(value),
133            },
134            6 => Self {
135                inlined: Inlined::new::<6>(value),
136            },
137            7 => Self {
138                inlined: Inlined::new::<7>(value),
139            },
140            8 => Self {
141                inlined: Inlined::new::<8>(value),
142            },
143            9 => Self {
144                inlined: Inlined::new::<9>(value),
145            },
146            10 => Self {
147                inlined: Inlined::new::<10>(value),
148            },
149            11 => Self {
150                inlined: Inlined::new::<11>(value),
151            },
152            12 => Self {
153                inlined: Inlined::new::<12>(value),
154            },
155            _ => Self::new_ref(
156                u32::try_from(value.len()).vortex_expect("value length must fit in u32"),
157                value[0..4]
158                    .try_into()
159                    .ok()
160                    .vortex_expect("prefix must be exactly 4 bytes"),
161                block,
162                offset,
163            ),
164        }
165    }
166
167    /// Create a new empty view
168    #[inline]
169    pub fn empty_view() -> Self {
170        Self { le_bytes: [0; 16] }
171    }
172
173    /// Create a reference view directly from its components, without inspecting the value.
174    ///
175    /// `size` must be greater than [`MAX_INLINED_SIZE`], and `prefix` must hold the first four
176    /// bytes of the value. This is the fast path for bulk view construction where the caller has
177    /// already established that the value is too long to inline; it assembles the 16-byte view as a
178    /// single `u128` so the compiler can emit one wide store per view.
179    ///
180    /// [`MAX_INLINED_SIZE`]: Self::MAX_INLINED_SIZE
181    #[inline]
182    pub fn new_ref(size: u32, prefix: [u8; 4], buffer_index: u32, offset: u32) -> Self {
183        debug_assert!(size as usize > Self::MAX_INLINED_SIZE);
184        // Matches the little-endian field order of `Ref` (size, prefix, buffer_index, offset),
185        // consistent with `le_bytes` and the `From<u128>`/`as_u128` representation.
186        Self::from(
187            u128::from(size)
188                | (u128::from(u32::from_le_bytes(prefix)) << 32)
189                | (u128::from(buffer_index) << 64)
190                | (u128::from(offset) << 96),
191        )
192    }
193
194    /// Create a new inlined binary view
195    ///
196    /// # Panics
197    ///
198    /// Panics if the provided string is too long to inline.
199    #[inline]
200    pub fn new_inlined(value: &[u8]) -> Self {
201        assert!(
202            value.len() <= Self::MAX_INLINED_SIZE,
203            "expected inlined value to be <= 12 bytes, was {}",
204            value.len()
205        );
206
207        Self::make_view(value, 0, 0)
208    }
209
210    /// Returns the length of the binary value.
211    #[inline]
212    pub fn len(&self) -> u32 {
213        unsafe { self.inlined.size }
214    }
215
216    /// Returns true if the binary value is empty.
217    #[inline]
218    pub fn is_empty(&self) -> bool {
219        self.len() == 0
220    }
221
222    /// Returns true if the binary value is inlined.
223    #[inline]
224    #[expect(
225        clippy::cast_possible_truncation,
226        reason = "MAX_INLINED_SIZE is a small constant"
227    )]
228    pub fn is_inlined(&self) -> bool {
229        self.len() <= (Self::MAX_INLINED_SIZE as u32)
230    }
231
232    /// Returns the inlined representation of the binary value.
233    pub fn as_inlined(&self) -> &Inlined {
234        debug_assert!(self.is_inlined());
235        unsafe { &self.inlined }
236    }
237
238    /// Returns the reference representation of the binary value.
239    pub fn as_view(&self) -> &Ref {
240        debug_assert!(!self.is_inlined());
241        unsafe { &self._ref }
242    }
243
244    /// Returns a mutable reference to the reference representation of the binary value.
245    pub fn as_view_mut(&mut self) -> &mut Ref {
246        unsafe { &mut self._ref }
247    }
248
249    /// Returns the bytes of this value, reading out of `buffers` when it is not inlined.
250    ///
251    /// Unlike [`VarBinViewData::bytes_at`](crate::arrays::varbinview::VarBinViewData::bytes_at)
252    /// this borrows
253    /// from slices the caller has already resolved instead of cloning a buffer handle, so it is
254    /// safe to call once per row in a loop.
255    ///
256    /// # Panics
257    ///
258    /// Panics if the view references a buffer or range not covered by `buffers`.
259    #[inline]
260    pub fn bytes<'a>(&'a self, buffers: &[&'a [u8]]) -> &'a [u8] {
261        if self.is_inlined() {
262            self.as_inlined().value()
263        } else {
264            let view = self.as_view();
265            &buffers[view.buffer_index as usize][view.as_range()]
266        }
267    }
268
269    /// Returns the binary view as u128 representation.
270    pub fn as_u128(&self) -> u128 {
271        // SAFETY: binary view always safe to read as u128 LE bytes
272        unsafe { u128::from_le_bytes(self.le_bytes) }
273    }
274}
275
276impl From<u128> for BinaryView {
277    fn from(value: u128) -> Self {
278        BinaryView {
279            le_bytes: value.to_le_bytes(),
280        }
281    }
282}
283
284impl From<Ref> for BinaryView {
285    fn from(value: Ref) -> Self {
286        BinaryView { _ref: value }
287    }
288}
289
290impl PartialEq for BinaryView {
291    fn eq(&self, other: &Self) -> bool {
292        let a = unsafe { std::mem::transmute::<&BinaryView, &u128>(self) };
293        let b = unsafe { std::mem::transmute::<&BinaryView, &u128>(other) };
294        a == b
295    }
296}
297impl Eq for BinaryView {}
298
299impl Hash for BinaryView {
300    fn hash<H: Hasher>(&self, state: &mut H) {
301        unsafe { std::mem::transmute::<&BinaryView, &u128>(self) }.hash(state);
302    }
303}
304
305impl Default for BinaryView {
306    fn default() -> Self {
307        Self::make_view(&[], 0, 0)
308    }
309}
310
311impl fmt::Debug for BinaryView {
312    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
313        let mut s = f.debug_struct("BinaryView");
314        if self.is_inlined() {
315            s.field("inline", &self.as_inlined());
316        } else {
317            s.field("ref", &self.as_view());
318        }
319        s.finish()
320    }
321}
322
323#[cfg(test)]
324mod tests {
325    use super::*;
326
327    #[rstest::rstest]
328    // Just past the inline boundary, typical, and large values.
329    #[case(13, 7, 42)]
330    #[case(20, 7, 42)]
331    #[case(255, 7, 42)]
332    #[case(4096, 7, 42)]
333    // Zero buffer index/offset and the `u32` extremes, to confirm the `u128` field assembly does
334    // not overflow into neighbouring fields.
335    #[case(13, 0, 0)]
336    #[case(13, u32::MAX, u32::MAX)]
337    fn new_ref_matches_make_view(#[case] len: u32, #[case] buffer_index: u32, #[case] offset: u32) {
338        // `new_ref` assembles the reference view as a `u128`; it must be byte-identical to the
339        // value-inspecting `make_view` for any value longer than the inline limit.
340        let value: Vec<u8> = (0..len)
341            .map(|i| u8::try_from(i % 251).vortex_expect("i % 251 fits in u8"))
342            .collect();
343        let prefix = [value[0], value[1], value[2], value[3]];
344        let made = BinaryView::make_view(&value, buffer_index, offset);
345        let built = BinaryView::new_ref(len, prefix, buffer_index, offset);
346        assert_eq!(made.as_u128(), built.as_u128(), "mismatch at len {len}");
347        assert!(!built.is_inlined());
348        let r = built.as_view();
349        assert_eq!(r.size, len);
350        assert_eq!(r.prefix, prefix);
351        assert_eq!(r.buffer_index, buffer_index);
352        assert_eq!(r.offset, offset);
353    }
354}
vortex_array/arrays/varbinview/view.rs

vortex_array/arrays/varbinview/
view.rs
