vortex_array/arrays/varbinview/
build_views.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use itertools::Itertools;
5use num_traits::AsPrimitive;
6use vortex_buffer::Buffer;
7use vortex_buffer::BufferMut;
8use vortex_buffer::ByteBuffer;
9
10pub use crate::arrays::varbinview::BinaryView;
11use crate::dtype::NativePType;
12
13/// Convert an offsets buffer to a buffer of element lengths.
14#[inline]
15pub fn offsets_to_lengths<P: NativePType>(offsets: &[P]) -> Buffer<P> {
16    offsets
17        .iter()
18        .tuple_windows::<(_, _)>()
19        .map(|(&start, &end)| end - start)
20        .collect()
21}
22
23/// Maximum number of buffer bytes that can be referenced by a single `BinaryView`
24pub const MAX_BUFFER_LEN: usize = i32::MAX as usize;
25
26/// Split a large buffer of input `bytes` holding string data into `VarBinView` buffers and views.
27///
28/// The values must be laid end-to-end in `bytes`, one per entry of `lens`, describing the whole
29/// buffer exactly. The returned buffers are zero-copy slices of `bytes`, numbered sequentially
30/// from `start_buf_index`.
31///
32/// `max_buffer_len` must not exceed [`MAX_BUFFER_LEN`], since every view offset is stored in a
33/// `u32` and offsets are bounded by `max_buffer_len`.
34///
35/// # Panics
36///
37/// Panics if the lengths do not describe `bytes` exactly, or if a single value exceeds
38/// `max_buffer_len`.
39pub fn build_views<P: NativePType + AsPrimitive<usize>>(
40    start_buf_index: u32,
41    max_buffer_len: usize,
42    bytes: ByteBuffer,
43    lens: &[P],
44) -> (Vec<ByteBuffer>, Buffer<BinaryView>) {
45    let mut views = BufferMut::with_capacity(lens.len());
46    let buffers = extend_views(
47        &mut views,
48        start_buf_index,
49        max_buffer_len,
50        &bytes,
51        lens.len(),
52        |i| lens[i].as_(),
53    );
54    (buffers, views.freeze())
55}
56
57/// [`build_views`] for values described by an offsets buffer instead of lengths.
58///
59/// `offsets` are absolute positions into `bytes` — the layout a `VarBinArray` stores — so there
60/// is one more offset than there are values, and the values need not start at the beginning of
61/// `bytes`: only `offsets[0]..offsets[last]` is referenced, and the returned buffers are zero-copy
62/// slices of that range.
63///
64/// # Panics
65///
66/// Panics if `offsets` is empty, not monotonically non-decreasing within `bytes`, or if a single
67/// value exceeds `max_buffer_len`.
68pub fn build_views_from_offsets<P: NativePType + AsPrimitive<usize>>(
69    start_buf_index: u32,
70    max_buffer_len: usize,
71    bytes: ByteBuffer,
72    offsets: &[P],
73) -> (Vec<ByteBuffer>, Buffer<BinaryView>) {
74    assert!(!offsets.is_empty(), "offsets must hold at least one entry");
75    let first: usize = offsets[0].as_();
76    let last: usize = offsets[offsets.len() - 1].as_();
77    let bytes = bytes.slice(first..last);
78
79    let count = offsets.len() - 1;
80    let mut views = BufferMut::with_capacity(count);
81    // Wrapping keeps corrupt non-monotonic offsets from panicking on the subtraction itself; the
82    // wrapped length then fails the in-bounds slicing (or `max_buffer_len`) checks in the loop.
83    let buffers = extend_views(
84        &mut views,
85        start_buf_index,
86        max_buffer_len,
87        &bytes,
88        count,
89        |i| {
90            AsPrimitive::<usize>::as_(offsets[i + 1])
91                .wrapping_sub(AsPrimitive::<usize>::as_(offsets[i]))
92        },
93    );
94    (buffers, views.freeze())
95}
96
97/// Appends one view per value straight into `views`, splitting `bytes` into buffers.
98///
99/// This is the core behind [`build_views`]: it writes into an existing views buffer so that a
100/// [`VarBinViewBuilder`](crate::builders::VarBinViewBuilder) can build views directly into its
101/// storage without an intermediate allocation. `len_at(i)` is the byte length of value `i`, and
102/// the `count` lengths must describe `bytes` exactly. The returned buffers are zero-copy slices
103/// of `bytes`, numbered sequentially from `start_buf_index`.
104pub(crate) fn extend_views(
105    views: &mut BufferMut<BinaryView>,
106    start_buf_index: u32,
107    max_buffer_len: usize,
108    bytes: &ByteBuffer,
109    count: usize,
110    len_at: impl Fn(usize) -> usize,
111) -> Vec<ByteBuffer> {
112    assert!(
113        max_buffer_len <= MAX_BUFFER_LEN,
114        "max_buffer_len cannot exceed MAX_BUFFER_LEN, offsets must fit in u32"
115    );
116
117    if bytes.len() <= max_buffer_len {
118        // Common case: the whole decoded heap fits within a single buffer, so no rollover can
119        // occur (`bytes.len()` is the total decoded size and therefore an upper bound on every
120        // offset).
121        extend_views_single_buffer(views, start_buf_index, bytes, count, len_at);
122        if bytes.is_empty() {
123            Vec::new()
124        } else {
125            vec![bytes.clone()]
126        }
127    } else {
128        extend_views_rolling(views, start_buf_index, max_buffer_len, bytes, count, len_at)
129    }
130}
131
132/// Build views when the whole heap fits in a single output buffer.
133///
134/// Because no rollover can occur, the hot loop drops the per-element rollover branch and constructs
135/// reference views inline, avoiding the out-of-line `BinaryView::make_view` call for the common
136/// long-string case. Every offset is bounded by `bytes.len()`, which the caller has guaranteed is
137/// at most [`MAX_BUFFER_LEN`], so the `usize -> u32` conversions cannot truncate.
138fn extend_views_single_buffer(
139    views: &mut BufferMut<BinaryView>,
140    buf_index: u32,
141    bytes: &ByteBuffer,
142    count: usize,
143    len_at: impl Fn(usize) -> usize,
144) {
145    views.reserve(count);
146    let base = views.len();
147
148    let data = bytes.as_slice();
149    let mut offset = 0usize;
150    // Write directly into the reserved spare capacity rather than `push_unchecked`. The latter
151    // advances the backing buffer's length on every call, which the optimizer cannot prove is
152    // loop-invariant, so it reloads and rewrites the output cursor through the stack each
153    // iteration. Writing into the spare slice keeps the cursor in a register and the length is
154    // set once after the loop.
155    let spare = &mut views.spare_capacity_mut()[..count];
156    for (i, slot) in spare.iter_mut().enumerate() {
157        let len = len_at(i);
158        let value = &data[offset..offset + len];
159        let view = if len > BinaryView::MAX_INLINED_SIZE {
160            let mut prefix = [0u8; 4];
161            prefix.copy_from_slice(&value[..4]);
162            BinaryView::new_ref(len.as_(), prefix, buf_index, offset.as_())
163        } else {
164            BinaryView::make_view(value, buf_index, offset.as_())
165        };
166        slot.write(view);
167        offset += len;
168    }
169    assert_eq!(
170        offset,
171        data.len(),
172        "value lengths must describe the byte heap exactly"
173    );
174    // SAFETY: the loop initialized exactly `count` contiguous views (`spare` has at least
175    //  `count` slots).
176    unsafe { views.set_len(base + count) };
177}
178
179/// Build views when the heap exceeds `max_buffer_len` and must be split across multiple buffers.
180///
181/// The buffer is rolled over every `max_buffer_len` bytes so that no view offset overflows the
182/// `u32` offset field. Each output buffer is a zero-copy slice of `bytes`.
183fn extend_views_rolling(
184    views: &mut BufferMut<BinaryView>,
185    start_buf_index: u32,
186    max_buffer_len: usize,
187    bytes: &ByteBuffer,
188    count: usize,
189    len_at: impl Fn(usize) -> usize,
190) -> Vec<ByteBuffer> {
191    views.reserve(count);
192    let mut buffers = Vec::new();
193    let mut buf_index = start_buf_index;
194
195    let data = bytes.as_slice();
196    // The absolute start of the current segment, and the offset of the next value within it.
197    let mut segment_start = 0usize;
198    let mut offset = 0usize;
199    for i in 0..count {
200        let len = len_at(i);
201        assert!(len <= max_buffer_len, "values cannot exceed max_buffer_len");
202
203        if offset + len > max_buffer_len {
204            // Roll the buffer every 2GiB, to avoid overflowing VarBinView offset field
205            buffers.push(bytes.slice(segment_start..segment_start + offset));
206            buf_index += 1;
207            segment_start += offset;
208            offset = 0;
209        }
210        let start = segment_start + offset;
211        let view = BinaryView::make_view(&data[start..start + len], buf_index, offset.as_());
212        views.push(view);
213        offset += len;
214    }
215    assert_eq!(
216        segment_start + offset,
217        data.len(),
218        "value lengths must describe the byte heap exactly"
219    );
220
221    if segment_start < data.len() {
222        buffers.push(bytes.slice(segment_start..data.len()));
223    }
224
225    buffers
226}
227
228#[cfg(test)]
229mod tests {
230    use rstest::rstest;
231    use vortex_buffer::ByteBuffer;
232    use vortex_buffer::ByteBufferMut;
233
234    use crate::arrays::varbinview::BinaryView;
235    use crate::arrays::varbinview::build_views::MAX_BUFFER_LEN;
236    use crate::arrays::varbinview::build_views::build_views;
237    use crate::arrays::varbinview::build_views::build_views_from_offsets;
238
239    /// Concatenate `values` into a single byte heap and return it alongside the per-element lengths,
240    /// matching the `(bytes, lens)` inputs that `build_views` consumes.
241    fn flatten(values: &[&[u8]]) -> (ByteBuffer, Vec<u32>) {
242        let mut bytes = ByteBufferMut::empty();
243        let mut lens = Vec::with_capacity(values.len());
244        for v in values {
245            bytes.extend_from_slice(v);
246            lens.push(u32::try_from(v.len()).unwrap());
247        }
248        (bytes.freeze(), lens)
249    }
250
251    /// Reconstruct the logical value behind each view by dereferencing it through the output
252    /// buffers. The first buffer corresponds to `start_buf_index`, so buffer indices are rebased by
253    /// that amount. This is the core correctness invariant: regardless of which code path built the
254    /// views, every view must point back at its original bytes.
255    fn reconstruct(
256        buffers: &[ByteBuffer],
257        views: &[BinaryView],
258        start_buf_index: u32,
259    ) -> Vec<Vec<u8>> {
260        views
261            .iter()
262            .map(|view| {
263                if view.is_inlined() {
264                    view.as_inlined().value().to_vec()
265                } else {
266                    let r = view.as_view();
267                    let buf = &buffers[(r.buffer_index - start_buf_index) as usize];
268                    buf[r.as_range()].to_vec()
269                }
270            })
271            .collect()
272    }
273
274    /// The single-buffer fast path (`bytes.len() <= max_buffer_len`) must reproduce every input
275    /// value exactly, emit a single output buffer holding the untouched heap, and reference only
276    /// `start_buf_index`. We cover a spread of value sets that mix inlined (<= 12 bytes) and
277    /// reference (> 12 bytes) lengths, including the 12/13 byte inline boundary, empty values, and a
278    /// fully-inlined set.
279    #[rstest]
280    #[case::mixed(&[b"a".as_slice(), b"this is a long reference value", b"short", b"another long value here!!"])]
281    #[case::inline_boundary(&[&[b'x'; 12] as &[u8], &[b'y'; 13], &[b'z'; 12], &[b'w'; 13]])]
282    #[case::all_inlined(&[b"".as_slice(), b"a", b"bb", b"ccc", b"dddddddddddd"])]
283    #[case::all_reference(&[&[b'a'; 100] as &[u8], &[b'b'; 50], &[b'c'; 4096]])]
284    #[case::empty_values_interleaved(&[b"".as_slice(), b"a long value that is referenced", b"", b"", b"trailing long reference value"])]
285    #[case::single_long(&[&[7u8; 1 << 16] as &[u8]])]
286    fn fast_path_roundtrip(#[case] values: &[&[u8]]) {
287        let (bytes, lens) = flatten(values);
288        let total = bytes.len();
289        let start_buf_index = 3;
290
291        // `max_buffer_len` strictly greater than the heap forces the single-buffer fast path.
292        let (buffers, views) = build_views(start_buf_index, total + 1, bytes, &lens);
293
294        assert_eq!(views.len(), values.len());
295        if total == 0 {
296            assert!(buffers.is_empty(), "empty heap must not allocate a buffer");
297        } else {
298            assert_eq!(buffers.len(), 1, "whole heap must stay in one buffer");
299            // The fast path adopts the input heap unchanged.
300            let concatenated: Vec<u8> = values.concat();
301            assert_eq!(buffers[0].as_slice(), concatenated.as_slice());
302        }
303        for view in views.iter() {
304            if !view.is_inlined() {
305                assert_eq!(view.as_view().buffer_index, start_buf_index);
306            }
307        }
308
309        let expected: Vec<Vec<u8>> = values.iter().map(|v| v.to_vec()).collect();
310        assert_eq!(reconstruct(&buffers, &views, start_buf_index), expected);
311    }
312
313    /// The output buffers must be zero-copy slices of the input heap, on both paths — a copy here
314    /// silently doubles the memory cost of every decode that feeds views.
315    #[test]
316    fn output_buffers_are_zero_copy() {
317        let values: &[&[u8]] = &[
318            b"first long reference value",
319            b"tiny",
320            b"second long reference value!!",
321            b"third looooong reference value",
322        ];
323        let (bytes, lens) = flatten(values);
324        let base = bytes.as_ptr();
325
326        // Fast path: the single output buffer is the input buffer.
327        let (buffers, _views) = build_views(0, bytes.len() + 1, bytes.clone(), &lens);
328        assert_eq!(buffers.len(), 1);
329        assert_eq!(buffers[0].as_ptr(), base, "fast path must not copy");
330
331        // Rolling path: every output buffer points into the input allocation.
332        let longest = values.iter().map(|v| v.len()).max().unwrap();
333        let (buffers, _views) = build_views(0, longest, bytes, &lens);
334        assert!(buffers.len() > 1);
335        let mut expected_ptr = base;
336        for buffer in &buffers {
337            assert_eq!(buffer.as_ptr(), expected_ptr, "rolling path must not copy");
338            // SAFETY: the buffers partition the input heap, so the next one starts where
339            // this one ends, still within (or one past) the original allocation.
340            expected_ptr = unsafe { expected_ptr.add(buffer.len()) };
341        }
342    }
343
344    /// Offsets and sizes are written into the `u32` `Ref` fields via `as_` truncation, so we must
345    /// confirm they stay correct once the running offset grows well past the 16-bit range (i.e. is
346    /// not narrowed to a smaller width). A ~9 MiB heap pushes offsets above 2^23 while remaining far
347    /// below `MAX_BUFFER_LEN`; each value encodes its index in its first bytes so a misplaced offset
348    /// would reconstruct the wrong bytes.
349    #[test]
350    fn fast_path_large_offsets() {
351        const N: usize = 9000;
352        const LEN: usize = 1000;
353        // The final offset is (N - 1) * LEN, which must exceed 2^23 to be a meaningful check.
354        const { assert!((N - 1) * LEN > (1 << 23)) };
355
356        let values: Vec<Vec<u8>> = (0..N)
357            .map(|i| {
358                let mut v = vec![0u8; LEN];
359                v[..4].copy_from_slice(&u32::try_from(i).unwrap().to_le_bytes());
360                v
361            })
362            .collect();
363        let refs: Vec<&[u8]> = values.iter().map(|v| v.as_slice()).collect();
364
365        let (bytes, lens) = flatten(&refs);
366        let total = bytes.len();
367
368        let (buffers, views) = build_views(0, total + 1, bytes, &lens);
369
370        assert_eq!(buffers.len(), 1);
371        // The recorded offset must equal the cumulative byte position, exactly, for every view.
372        for (i, view) in views.iter().enumerate() {
373            let r = view.as_view();
374            assert_eq!(r.offset as usize, i * LEN, "wrong offset for view {i}");
375            assert_eq!(r.size as usize, LEN);
376        }
377        assert_eq!(reconstruct(&buffers, &views, 0), values);
378    }
379
380    /// The fast path is taken when `bytes.len() <= max_buffer_len`, so equality at the boundary must
381    /// still produce a single buffer (not roll over to the slow path).
382    #[test]
383    fn fast_path_taken_at_exact_boundary() {
384        let (bytes, lens) =
385            flatten(&[b"this value is definitely long", b"and so is this one here"]);
386        let total = bytes.len();
387
388        let (buffers, views) = build_views(0, total, bytes, &lens);
389
390        assert_eq!(
391            buffers.len(),
392            1,
393            "len == max_buffer_len must stay on fast path"
394        );
395        assert_eq!(views.len(), 2);
396    }
397
398    /// For the same logical data, the fast path (single buffer) and the slow rollover path must
399    /// reconstruct identical values. Driving the slow path with a small `max_buffer_len` forces
400    /// buffer splitting while leaving the recovered values unchanged.
401    #[test]
402    fn fast_and_slow_paths_agree() {
403        let values: &[&[u8]] = &[
404            b"first long reference value",
405            b"tiny",
406            b"second long reference value!!",
407            b"third looooong reference value",
408        ];
409        let expected: Vec<Vec<u8>> = values.iter().map(|v| v.to_vec()).collect();
410
411        let (fast_bytes, lens) = flatten(values);
412        let total = fast_bytes.len();
413        let (fast_buffers, fast_views) = build_views(0, total + 1, fast_bytes, &lens);
414        assert_eq!(fast_buffers.len(), 1);
415        assert_eq!(reconstruct(&fast_buffers, &fast_views, 0), expected);
416
417        // Force the rollover path: a small cap (>= the longest value) that the total heap exceeds.
418        let longest = values.iter().map(|v| v.len()).max().unwrap();
419        let (slow_bytes, _) = flatten(values);
420        let (slow_buffers, slow_views) = build_views(0, longest, slow_bytes, &lens);
421        assert!(
422            slow_buffers.len() > 1,
423            "small cap should split into many buffers"
424        );
425        assert_eq!(reconstruct(&slow_buffers, &slow_views, 0), expected);
426
427        // Same logical contents regardless of how the heap was partitioned.
428        assert_eq!(
429            reconstruct(&fast_buffers, &fast_views, 0),
430            reconstruct(&slow_buffers, &slow_views, 0)
431        );
432    }
433
434    /// Empty input must yield no buffers and no views, exercising the `bytes.is_empty()` branch.
435    #[test]
436    fn fast_path_empty_input() {
437        let lens: Vec<u32> = Vec::new();
438        let (buffers, views) = build_views(0, 1024, ByteBuffer::empty(), &lens);
439        assert!(buffers.is_empty());
440        assert!(views.is_empty());
441    }
442
443    /// The fast path must produce views byte-identical to the value-inspecting `make_view`, which is
444    /// what the slow path uses. This pins the inline/reference decision and field layout.
445    #[test]
446    fn fast_path_matches_make_view() {
447        let values: &[&[u8]] = &[b"inline", b"this is a long reference value", b""];
448        let (bytes, lens) = flatten(values);
449        let total = bytes.len();
450        let (_buffers, views) = build_views(0, total + 1, bytes, &lens);
451
452        let expected = [
453            BinaryView::make_view(b"inline", 0, 0),
454            BinaryView::make_view(b"this is a long reference value", 0, 6),
455            BinaryView::make_view(b"", 0, 36),
456        ];
457        assert_eq!(views.as_slice(), &expected);
458    }
459
460    /// The offsets-driven variant must agree with the lengths-driven one, reference only the
461    /// `offsets[0]..offsets[last]` range, and stay zero-copy — it exists so a `VarBinArray` heap
462    /// can feed views without materializing a lengths buffer or copying its bytes.
463    #[test]
464    fn from_offsets_matches_lengths_and_is_zero_copy() {
465        // A heap with a prefix and suffix outside the offsets range, as a sliced VarBin has.
466        let heap = ByteBuffer::copy_from(b"..a long value that is referenced!tiny..".as_slice());
467        let offsets: Vec<u32> = vec![2, 34, 38];
468
469        let (buffers, views) = build_views_from_offsets(5, MAX_BUFFER_LEN, heap.clone(), &offsets);
470
471        assert_eq!(buffers.len(), 1);
472        // Zero-copy: the buffer points at offset 2 of the original allocation.
473        // SAFETY: offset 2 is in bounds of the 40-byte heap.
474        assert_eq!(buffers[0].as_ptr(), unsafe { heap.as_ptr().add(2) });
475        assert_eq!(buffers[0].len(), 36);
476
477        assert_eq!(
478            reconstruct(&buffers, &views, 5),
479            vec![
480                b"a long value that is referenced!".to_vec(),
481                b"tiny".to_vec()
482            ]
483        );
484    }
485
486    /// Lengths that do not cover the heap exactly are a caller bug and must be rejected rather
487    /// than silently emitting views over a partially-covered buffer.
488    #[test]
489    #[should_panic(expected = "value lengths must describe the byte heap exactly")]
490    fn short_lengths_panic() {
491        let (bytes, _) = flatten(&[b"a long value that is referenced", b"tiny"]);
492        build_views(0, MAX_BUFFER_LEN, bytes, &[31u32]);
493    }
494
495    // TODO(someone): ideally CI would run this in release mode as well, since debug builds make the
496    // ~2.25 GiB allocation and fill loop substantially slower.
497    /// Slow regression for the single-buffer fast-path guard. The fast path is only valid when the
498    /// whole heap fits in one buffer (`bytes.len() <= max_buffer_len`); once the heap exceeds
499    /// [`MAX_BUFFER_LEN`] (`i32::MAX`, ~2.0 GiB) `build_views` must roll the heap into multiple
500    /// buffers, resetting the per-buffer offset, so no view references an offset past the
501    /// `i32`-bounded buffer limit.
502    ///
503    /// We build a heap just past `i32::MAX` and assert it rolls over into more than one buffer, that
504    /// no buffer exceeds `MAX_BUFFER_LEN`, and that values straddling the rollover boundary (where
505    /// the second buffer's offsets restart from zero) reconstruct exactly. If the guard regressed and
506    /// the fast path swallowed the whole heap, it would emit a single >2 GiB buffer with offsets past
507    /// `i32::MAX`, which the buffer-count and buffer-size assertions catch.
508    ///
509    /// Allocates ~2.25 GiB, so it is gated to CI and skipped when `VORTEX_SKIP_SLOW_TESTS` is set:
510    ///
511    /// ```text
512    /// CI=1 cargo test --release -p vortex-array build_views_offsets_overflow
513    /// ```
514    ///
515    /// [`MAX_BUFFER_LEN`]: super::MAX_BUFFER_LEN
516    #[test_with::env(CI)]
517    #[test_with::no_env(VORTEX_SKIP_SLOW_TESTS)]
518    fn build_views_offsets_overflow_i32() {
519        const STRING_LEN: usize = 64 * 1024;
520        // Comfortably past MAX_BUFFER_LEN (`i32::MAX` ~= 2.0 GiB) so the heap must roll over.
521        const TOTAL_BYTES: usize = (1usize << 31) + (256 << 20); // ~2.25 GiB
522        const N: usize = TOTAL_BYTES / STRING_LEN;
523
524        // Each value's first 8 bytes encode its row index, so a misrouted offset is detectable.
525        let nth_string = |i: usize| {
526            let mut s = vec![b'x'; STRING_LEN];
527            s[..8].copy_from_slice(&(i as u64).to_le_bytes());
528            s
529        };
530
531        let mut bytes = ByteBufferMut::with_capacity(N * STRING_LEN);
532        let mut value = vec![b'x'; STRING_LEN];
533        for i in 0..N {
534            value[..8].copy_from_slice(&(i as u64).to_le_bytes());
535            bytes.extend_from_slice(&value);
536        }
537
538        let lens = vec![u32::try_from(STRING_LEN).unwrap(); N];
539        let (buffers, views) = build_views(0, MAX_BUFFER_LEN, bytes.freeze(), &lens);
540
541        assert_eq!(views.len(), N);
542        assert!(
543            buffers.len() >= 2,
544            "heap exceeding MAX_BUFFER_LEN must roll over into multiple buffers, got {}",
545            buffers.len()
546        );
547        for (i, b) in buffers.iter().enumerate() {
548            assert!(
549                b.len() <= MAX_BUFFER_LEN,
550                "buffer {i} of {} bytes exceeds MAX_BUFFER_LEN",
551                b.len()
552            );
553        }
554
555        // The boundary row is the first whose offset would cross MAX_BUFFER_LEN on the fast path.
556        let boundary = MAX_BUFFER_LEN / STRING_LEN;
557        for i in [0, boundary - 1, boundary, boundary + 1, N / 2, N - 1] {
558            let view = &views[i];
559            let r = view.as_view();
560            let got = &buffers[r.buffer_index as usize][r.as_range()];
561            assert_eq!(got, nth_string(i).as_slice(), "value mismatch at row {i}");
562            assert_eq!(r.size as usize, STRING_LEN);
563        }
564    }
565
566    #[test]
567    fn test_to_canonical_large() {
568        // We are testing generating views for raw data that should look like
569        //
570        //    aaaaaaaaaaaaa ("a"*13)
571        //    bbbbbbbbbbbbb ("b"*13)
572        //    ccccccccccccc ("c"*13)
573        //    ddddddddddddd ("d"*13)
574        //
575        // In real code, this would all fit in one buffer, but to unit test the splitting logic
576        // we split buffers at length 26, which should result in two buffers for the output array.
577        let raw_data =
578            ByteBuffer::copy_from("aaaaaaaaaaaaabbbbbbbbbbbbbcccccccccccccddddddddddddd");
579        let lens = vec![13u8; 4];
580
581        let (buffers, views) = build_views(0, 26, raw_data, &lens);
582
583        assert_eq!(
584            buffers,
585            vec![
586                ByteBuffer::copy_from("aaaaaaaaaaaaabbbbbbbbbbbbb"),
587                ByteBuffer::copy_from("cccccccccccccddddddddddddd"),
588            ]
589        );
590
591        assert_eq!(
592            views.as_slice(),
593            &[
594                BinaryView::make_view(b"aaaaaaaaaaaaa", 0, 0),
595                BinaryView::make_view(b"bbbbbbbbbbbbb", 0, 13),
596                BinaryView::make_view(b"ccccccccccccc", 1, 0),
597                BinaryView::make_view(b"ddddddddddddd", 1, 13),
598            ]
599        )
600    }
601
602    #[test]
603    #[should_panic(expected = "max_buffer_len cannot exceed MAX_BUFFER_LEN")]
604    fn test_max_buffer_len_too_large_panics() {
605        build_views(0, MAX_BUFFER_LEN + 1, ByteBuffer::copy_from("abc"), &[3u32]);
606    }
607}
vortex_array/arrays/varbinview/build_views.rs

vortex_array/arrays/varbinview/
build_views.rs
