Skip to main content

arrow_buffer/builder/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Buffer builders
6
7mod boolean;
8mod null;
9mod offset;
10
11pub use boolean::*;
12pub use null::*;
13pub use offset::*;
14
15use crate::{ArrowNativeType, Buffer, MutableBuffer};
16use std::marker::PhantomData;
17
18/// Builder for creating Arrow [`Buffer`] objects
19///
20/// A [`Buffer`] is the underlying data structure of Arrow's Arrays.
21///
22/// For all supported types, there are type definitions for the
23/// generic version of `BufferBuilder<T>`, e.g. `BufferBuilder`.
24///
25/// **Note it is typically faster to create buffers directly from `Vec`**.
26/// See example on [`Buffer`].
27///
28/// # See Also
29/// * [`BooleanBufferBuilder`]: for packing bits in [`BooleanBuffer`]s
30/// * [`NullBufferBuilder`]: for creating [`NullBuffer`]s of null values
31///
32/// [`BooleanBuffer`]: crate::BooleanBuffer
33/// [`NullBuffer`]: crate::NullBuffer
34///
35/// # Example:
36///
37/// ```
38/// # use arrow_buffer::builder::BufferBuilder;
39/// let mut builder = BufferBuilder::<u8>::new(100);
40/// builder.append_slice(&[42, 43, 44]);
41/// builder.append(45);
42/// let buffer = builder.finish();
43/// assert_eq!(unsafe { buffer.typed_data::<u8>() }, &[42, 43, 44, 45]);
44/// ```
45#[derive(Debug)]
46pub struct BufferBuilder<T: ArrowNativeType> {
47    buffer: MutableBuffer,
48    _marker: PhantomData<T>,
49}
50
51impl<T: ArrowNativeType> BufferBuilder<T> {
52    /// Creates a new builder with initial capacity for _at least_ `capacity`
53    /// elements of type `T`.
54    ///
55    /// The capacity can later be manually adjusted with the
56    /// [`reserve()`](BufferBuilder::reserve) method.
57    /// Also the
58    /// [`append()`](BufferBuilder::append),
59    /// [`append_slice()`](BufferBuilder::append_slice) and
60    /// [`advance()`](BufferBuilder::advance)
61    /// methods automatically increase the capacity if needed.
62    ///
63    /// # Example:
64    ///
65    /// ```
66    /// # use arrow_buffer::builder::BufferBuilder;
67    /// let mut builder = BufferBuilder::<u8>::new(10);
68    ///
69    /// assert!(builder.capacity() >= 10);
70    /// ```
71    #[inline]
72    pub fn new(capacity: usize) -> Self {
73        let buffer = MutableBuffer::new(capacity * std::mem::size_of::<T>());
74
75        Self {
76            buffer,
77            _marker: PhantomData,
78        }
79    }
80
81    /// Creates a new builder from a [`MutableBuffer`]
82    ///
83    /// # Safety
84    ///
85    /// - `buffer` bytes must be aligned to type `T`
86    pub unsafe fn new_from_buffer(buffer: MutableBuffer) -> Self {
87        Self {
88            buffer,
89            _marker: PhantomData,
90        }
91    }
92
93    /// Returns the current number of array elements in the internal buffer.
94    ///
95    /// # Example:
96    ///
97    /// ```
98    /// # use arrow_buffer::builder::BufferBuilder;
99    /// let mut builder = BufferBuilder::<u8>::new(10);
100    /// builder.append(42);
101    ///
102    /// assert_eq!(builder.len(), 1);
103    /// ```
104    pub fn len(&self) -> usize {
105        self.buffer.len() / std::mem::size_of::<T>()
106    }
107
108    /// Returns whether the internal buffer is empty.
109    ///
110    /// # Example:
111    ///
112    /// ```
113    /// # use arrow_buffer::builder::BufferBuilder;
114    /// let mut builder = BufferBuilder::<u8>::new(10);
115    /// builder.append(42);
116    ///
117    /// assert_eq!(builder.is_empty(), false);
118    /// ```
119    pub fn is_empty(&self) -> bool {
120        self.buffer.is_empty()
121    }
122
123    /// Returns the actual capacity (number of elements) of the internal buffer.
124    ///
125    /// Note: the internal capacity returned by this method might be larger than
126    /// what you'd expect after setting the capacity in the `new()` or `reserve()`
127    /// functions.
128    pub fn capacity(&self) -> usize {
129        let byte_capacity = self.buffer.capacity();
130        byte_capacity / std::mem::size_of::<T>()
131    }
132
133    /// Increases the number of elements in the internal buffer by `n`
134    /// and resizes the buffer as needed.
135    ///
136    /// The values of the newly added elements are 0.
137    /// This method is usually used when appending `NULL` values to the buffer
138    /// as they still require physical memory space.
139    ///
140    /// # Example:
141    ///
142    /// ```
143    /// # use arrow_buffer::builder::BufferBuilder;
144    /// let mut builder = BufferBuilder::<u8>::new(10);
145    /// builder.advance(2);
146    ///
147    /// assert_eq!(builder.len(), 2);
148    /// ```
149    #[inline]
150    pub fn advance(&mut self, i: usize) {
151        self.buffer.extend_zeros(i * std::mem::size_of::<T>());
152    }
153
154    /// Reserves memory for _at least_ `n` more elements of type `T`.
155    ///
156    /// # Example:
157    ///
158    /// ```
159    /// # use arrow_buffer::builder::BufferBuilder;
160    /// let mut builder = BufferBuilder::<u8>::new(10);
161    /// builder.reserve(10);
162    ///
163    /// assert!(builder.capacity() >= 20);
164    /// ```
165    #[inline]
166    pub fn reserve(&mut self, n: usize) {
167        self.buffer.reserve(n * std::mem::size_of::<T>());
168    }
169
170    /// Appends a value of type `T` into the builder,
171    /// growing the internal buffer as needed.
172    ///
173    /// # Example:
174    ///
175    /// ```
176    /// # use arrow_buffer::builder::BufferBuilder;
177    /// let mut builder = BufferBuilder::<u8>::new(10);
178    /// builder.append(42);
179    ///
180    /// assert_eq!(builder.len(), 1);
181    /// ```
182    #[inline]
183    pub fn append(&mut self, v: T) {
184        self.reserve(1);
185        self.buffer.push(v);
186    }
187
188    /// Appends a value of type `T` into the builder N times,
189    /// growing the internal buffer as needed.
190    ///
191    /// # Example:
192    ///
193    /// ```
194    /// # use arrow_buffer::builder::BufferBuilder;
195    /// let mut builder = BufferBuilder::<u8>::new(10);
196    /// builder.append_n(10, 42);
197    ///
198    /// assert_eq!(builder.len(), 10);
199    /// ```
200    #[inline]
201    pub fn append_n(&mut self, n: usize, v: T) {
202        self.reserve(n);
203        self.extend(std::iter::repeat_n(v, n))
204    }
205
206    /// Appends `n`, zero-initialized values
207    ///
208    /// # Example:
209    ///
210    /// ```
211    /// # use arrow_buffer::builder::BufferBuilder;
212    /// let mut builder = BufferBuilder::<u32>::new(10);
213    /// builder.append_n_zeroed(3);
214    ///
215    /// assert_eq!(builder.len(), 3);
216    /// assert_eq!(builder.as_slice(), &[0, 0, 0])
217    /// ```
218    #[inline]
219    pub fn append_n_zeroed(&mut self, n: usize) {
220        self.buffer.extend_zeros(n * std::mem::size_of::<T>());
221    }
222
223    /// Appends a slice of type `T`, growing the internal buffer as needed.
224    ///
225    /// # Example:
226    ///
227    /// ```
228    /// # use arrow_buffer::builder::BufferBuilder;
229    /// let mut builder = BufferBuilder::<u8>::new(10);
230    /// builder.append_slice(&[42, 44, 46]);
231    ///
232    /// assert_eq!(builder.len(), 3);
233    /// ```
234    #[inline]
235    pub fn append_slice(&mut self, slice: &[T]) {
236        self.buffer.extend_from_slice(slice);
237    }
238
239    /// View the contents of this buffer as a slice
240    ///
241    /// ```
242    /// # use arrow_buffer::builder::BufferBuilder;
243    /// let mut builder = BufferBuilder::<f64>::new(10);
244    /// builder.append(1.3);
245    /// builder.append_n(2, 2.3);
246    ///
247    /// assert_eq!(builder.as_slice(), &[1.3, 2.3, 2.3]);
248    /// ```
249    #[inline]
250    pub fn as_slice(&self) -> &[T] {
251        // SAFETY
252        //
253        // - MutableBuffer is aligned and initialized for len elements of T
254        // - MutableBuffer corresponds to a single allocation
255        // - MutableBuffer does not support modification whilst active immutable borrows
256        unsafe { std::slice::from_raw_parts(self.buffer.as_ptr().cast(), self.len()) }
257    }
258
259    /// View the contents of this buffer as a mutable slice
260    ///
261    /// # Example:
262    ///
263    /// ```
264    /// # use arrow_buffer::builder::BufferBuilder;
265    /// let mut builder = BufferBuilder::<f32>::new(10);
266    ///
267    /// builder.append_slice(&[1., 2., 3.4]);
268    /// assert_eq!(builder.as_slice(), &[1., 2., 3.4]);
269    ///
270    /// builder.as_slice_mut()[1] = 4.2;
271    /// assert_eq!(builder.as_slice(), &[1., 4.2, 3.4]);
272    /// ```
273    #[inline]
274    pub fn as_slice_mut(&mut self) -> &mut [T] {
275        // SAFETY
276        //
277        // - MutableBuffer is aligned and initialized for len elements of T
278        // - MutableBuffer corresponds to a single allocation
279        // - MutableBuffer does not support modification whilst active immutable borrows
280        unsafe { std::slice::from_raw_parts_mut(self.buffer.as_mut_ptr().cast(), self.len()) }
281    }
282
283    /// Shorten this BufferBuilder to `len` items
284    ///
285    /// If `len` is greater than the builder's current length, this has no effect
286    ///
287    /// # Example:
288    ///
289    /// ```
290    /// # use arrow_buffer::builder::BufferBuilder;
291    /// let mut builder = BufferBuilder::<u16>::new(10);
292    ///
293    /// builder.append_slice(&[42, 44, 46]);
294    /// assert_eq!(builder.as_slice(), &[42, 44, 46]);
295    ///
296    /// builder.truncate(2);
297    /// assert_eq!(builder.as_slice(), &[42, 44]);
298    ///
299    /// builder.append(12);
300    /// assert_eq!(builder.as_slice(), &[42, 44, 12]);
301    /// ```
302    #[inline]
303    pub fn truncate(&mut self, len: usize) {
304        self.buffer.truncate(len * std::mem::size_of::<T>());
305    }
306
307    /// # Safety
308    /// This requires the iterator be a trusted length. This could instead require
309    /// the iterator implement `TrustedLen` once that is stabilized.
310    #[inline]
311    pub unsafe fn append_trusted_len_iter(&mut self, iter: impl IntoIterator<Item = T>) {
312        let iter = iter.into_iter();
313        let len = iter
314            .size_hint()
315            .1
316            .expect("append_trusted_len_iter expects upper bound");
317        self.reserve(len);
318        self.extend(iter);
319    }
320
321    /// Resets this builder and returns an immutable [Buffer].
322    ///
323    /// Use [`Self::build`] when you don't need to reuse this builder.
324    ///
325    /// # Example:
326    ///
327    /// ```
328    /// # use arrow_buffer::builder::BufferBuilder;
329    /// let mut builder = BufferBuilder::<u8>::new(10);
330    /// builder.append_slice(&[42, 44, 46]);
331    /// let buffer = builder.finish();
332    /// assert_eq!(unsafe { buffer.typed_data::<u8>() }, &[42, 44, 46]);
333    /// ```
334    #[inline]
335    pub fn finish(&mut self) -> Buffer {
336        let buf = std::mem::take(&mut self.buffer);
337        buf.into()
338    }
339
340    /// Builds an immutable [Buffer] without resetting the builder.
341    ///
342    /// This consumes the builder. Use [`Self::finish`] to reuse it.
343    ///
344    /// # Example:
345    ///
346    /// ```
347    /// # use arrow_buffer::builder::BufferBuilder;
348    /// let mut builder = BufferBuilder::<u8>::new(10);
349    /// builder.append_slice(&[42, 44, 46]);
350    /// let buffer = builder.build();
351    /// assert_eq!(unsafe { buffer.typed_data::<u8>() }, &[42, 44, 46]);
352    /// ```
353    #[inline]
354    pub fn build(self) -> Buffer {
355        self.buffer.into()
356    }
357}
358
359impl<T: ArrowNativeType> Default for BufferBuilder<T> {
360    fn default() -> Self {
361        Self::new(0)
362    }
363}
364
365impl<T: ArrowNativeType> Extend<T> for BufferBuilder<T> {
366    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
367        self.buffer.extend(iter)
368    }
369}
370
371impl<T: ArrowNativeType> From<Vec<T>> for BufferBuilder<T> {
372    fn from(value: Vec<T>) -> Self {
373        let buffer = MutableBuffer::from(value);
374        // SAFETY
375        // - buffer is aligned to T
376        unsafe { Self::new_from_buffer(buffer) }
377    }
378}
379
380impl<T: ArrowNativeType> FromIterator<T> for BufferBuilder<T> {
381    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
382        let mut builder = Self::default();
383        builder.extend(iter);
384        builder
385    }
386}
387
388#[cfg(test)]
389mod tests {
390    use super::*;
391    use std::mem;
392
393    #[test]
394    fn default() {
395        let builder = BufferBuilder::<u32>::default();
396        assert!(builder.is_empty());
397        assert!(builder.buffer.is_empty());
398        assert_eq!(builder.buffer.capacity(), 0);
399    }
400
401    #[test]
402    fn from_iter() {
403        let input = [1u16, 2, 3, 4];
404        let builder = input.into_iter().collect::<BufferBuilder<_>>();
405        assert_eq!(builder.len(), 4);
406        assert_eq!(builder.buffer.len(), 4 * mem::size_of::<u16>());
407    }
408
409    #[test]
410    fn extend() {
411        let input = [1, 2];
412        let mut builder = input.into_iter().collect::<BufferBuilder<_>>();
413        assert_eq!(builder.len(), 2);
414        builder.extend([3, 4]);
415        assert_eq!(builder.len(), 4);
416    }
417
418    #[test]
419    fn truncate_safety() {
420        let mut builder = BufferBuilder::from(vec![40, -63, 90]);
421        assert_eq!(builder.len(), 3);
422        builder.truncate(151);
423        assert_eq!(builder.len(), 3);
424        builder.advance(219);
425        assert_eq!(builder.len(), 222);
426        let slice = builder.as_slice_mut();
427        assert_eq!(slice.len(), 222);
428    }
429
430    #[test]
431    #[should_panic(expected = "buffer length overflow")]
432    fn reserve_length_overflow() {
433        let mut builder = BufferBuilder::<u8>::new(1);
434        builder.append(0);
435        builder.reserve(usize::MAX);
436    }
437
438    #[test]
439    #[should_panic(expected = "buffer length overflow")]
440    fn append_n_zeroed_length_overflow() {
441        let mut builder = BufferBuilder::<u64>::new(1);
442        builder.append_n_zeroed(1);
443        builder.append_n_zeroed(usize::MAX / mem::size_of::<u64>());
444    }
445
446    #[test]
447    #[should_panic(expected = "buffer length overflow")]
448    fn advance_length_overflow() {
449        let mut builder = BufferBuilder::<u64>::new(1);
450        builder.advance(1);
451        builder.advance(usize::MAX / mem::size_of::<u64>());
452    }
453}