arrow_buffer/builder/mod.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4
5//! Buffer builders
6
7mod boolean;
8mod null;
9mod offset;
10
11pub use boolean::*;
12pub use null::*;
13pub use offset::*;
14
15use crate::{ArrowNativeType, Buffer, MutableBuffer};
16use std::marker::PhantomData;
17
18/// Builder for creating Arrow [`Buffer`] objects
19///
20/// A [`Buffer`] is the underlying data structure of Arrow's Arrays.
21///
22/// For all supported types, there are type definitions for the
23/// generic version of `BufferBuilder<T>`, e.g. `BufferBuilder`.
24///
25/// **Note it is typically faster to create buffers directly from `Vec`**.
26/// See example on [`Buffer`].
27///
28/// # See Also
29/// * [`BooleanBufferBuilder`]: for packing bits in [`BooleanBuffer`]s
30/// * [`NullBufferBuilder`]: for creating [`NullBuffer`]s of null values
31///
32/// [`BooleanBuffer`]: crate::BooleanBuffer
33/// [`NullBuffer`]: crate::NullBuffer
34///
35/// # Example:
36///
37/// ```
38/// # use arrow_buffer::builder::BufferBuilder;
39/// let mut builder = BufferBuilder::<u8>::new(100);
40/// builder.append_slice(&[42, 43, 44]);
41/// builder.append(45);
42/// let buffer = builder.finish();
43/// assert_eq!(unsafe { buffer.typed_data::<u8>() }, &[42, 43, 44, 45]);
44/// ```
45#[derive(Debug)]
46pub struct BufferBuilder<T: ArrowNativeType> {
47 buffer: MutableBuffer,
48 _marker: PhantomData<T>,
49}
50
51impl<T: ArrowNativeType> BufferBuilder<T> {
52 /// Creates a new builder with initial capacity for _at least_ `capacity`
53 /// elements of type `T`.
54 ///
55 /// The capacity can later be manually adjusted with the
56 /// [`reserve()`](BufferBuilder::reserve) method.
57 /// Also the
58 /// [`append()`](BufferBuilder::append),
59 /// [`append_slice()`](BufferBuilder::append_slice) and
60 /// [`advance()`](BufferBuilder::advance)
61 /// methods automatically increase the capacity if needed.
62 ///
63 /// # Example:
64 ///
65 /// ```
66 /// # use arrow_buffer::builder::BufferBuilder;
67 /// let mut builder = BufferBuilder::<u8>::new(10);
68 ///
69 /// assert!(builder.capacity() >= 10);
70 /// ```
71 #[inline]
72 pub fn new(capacity: usize) -> Self {
73 let buffer = MutableBuffer::new(capacity * std::mem::size_of::<T>());
74
75 Self {
76 buffer,
77 _marker: PhantomData,
78 }
79 }
80
81 /// Creates a new builder from a [`MutableBuffer`]
82 ///
83 /// # Safety
84 ///
85 /// - `buffer` bytes must be aligned to type `T`
86 pub unsafe fn new_from_buffer(buffer: MutableBuffer) -> Self {
87 Self {
88 buffer,
89 _marker: PhantomData,
90 }
91 }
92
93 /// Returns the current number of array elements in the internal buffer.
94 ///
95 /// # Example:
96 ///
97 /// ```
98 /// # use arrow_buffer::builder::BufferBuilder;
99 /// let mut builder = BufferBuilder::<u8>::new(10);
100 /// builder.append(42);
101 ///
102 /// assert_eq!(builder.len(), 1);
103 /// ```
104 pub fn len(&self) -> usize {
105 self.buffer.len() / std::mem::size_of::<T>()
106 }
107
108 /// Returns whether the internal buffer is empty.
109 ///
110 /// # Example:
111 ///
112 /// ```
113 /// # use arrow_buffer::builder::BufferBuilder;
114 /// let mut builder = BufferBuilder::<u8>::new(10);
115 /// builder.append(42);
116 ///
117 /// assert_eq!(builder.is_empty(), false);
118 /// ```
119 pub fn is_empty(&self) -> bool {
120 self.buffer.is_empty()
121 }
122
123 /// Returns the actual capacity (number of elements) of the internal buffer.
124 ///
125 /// Note: the internal capacity returned by this method might be larger than
126 /// what you'd expect after setting the capacity in the `new()` or `reserve()`
127 /// functions.
128 pub fn capacity(&self) -> usize {
129 let byte_capacity = self.buffer.capacity();
130 byte_capacity / std::mem::size_of::<T>()
131 }
132
133 /// Increases the number of elements in the internal buffer by `n`
134 /// and resizes the buffer as needed.
135 ///
136 /// The values of the newly added elements are 0.
137 /// This method is usually used when appending `NULL` values to the buffer
138 /// as they still require physical memory space.
139 ///
140 /// # Example:
141 ///
142 /// ```
143 /// # use arrow_buffer::builder::BufferBuilder;
144 /// let mut builder = BufferBuilder::<u8>::new(10);
145 /// builder.advance(2);
146 ///
147 /// assert_eq!(builder.len(), 2);
148 /// ```
149 #[inline]
150 pub fn advance(&mut self, i: usize) {
151 self.buffer.extend_zeros(i * std::mem::size_of::<T>());
152 }
153
154 /// Reserves memory for _at least_ `n` more elements of type `T`.
155 ///
156 /// # Example:
157 ///
158 /// ```
159 /// # use arrow_buffer::builder::BufferBuilder;
160 /// let mut builder = BufferBuilder::<u8>::new(10);
161 /// builder.reserve(10);
162 ///
163 /// assert!(builder.capacity() >= 20);
164 /// ```
165 #[inline]
166 pub fn reserve(&mut self, n: usize) {
167 self.buffer.reserve(n * std::mem::size_of::<T>());
168 }
169
170 /// Appends a value of type `T` into the builder,
171 /// growing the internal buffer as needed.
172 ///
173 /// # Example:
174 ///
175 /// ```
176 /// # use arrow_buffer::builder::BufferBuilder;
177 /// let mut builder = BufferBuilder::<u8>::new(10);
178 /// builder.append(42);
179 ///
180 /// assert_eq!(builder.len(), 1);
181 /// ```
182 #[inline]
183 pub fn append(&mut self, v: T) {
184 self.reserve(1);
185 self.buffer.push(v);
186 }
187
188 /// Appends a value of type `T` into the builder N times,
189 /// growing the internal buffer as needed.
190 ///
191 /// # Example:
192 ///
193 /// ```
194 /// # use arrow_buffer::builder::BufferBuilder;
195 /// let mut builder = BufferBuilder::<u8>::new(10);
196 /// builder.append_n(10, 42);
197 ///
198 /// assert_eq!(builder.len(), 10);
199 /// ```
200 #[inline]
201 pub fn append_n(&mut self, n: usize, v: T) {
202 self.reserve(n);
203 self.extend(std::iter::repeat_n(v, n))
204 }
205
206 /// Appends `n`, zero-initialized values
207 ///
208 /// # Example:
209 ///
210 /// ```
211 /// # use arrow_buffer::builder::BufferBuilder;
212 /// let mut builder = BufferBuilder::<u32>::new(10);
213 /// builder.append_n_zeroed(3);
214 ///
215 /// assert_eq!(builder.len(), 3);
216 /// assert_eq!(builder.as_slice(), &[0, 0, 0])
217 /// ```
218 #[inline]
219 pub fn append_n_zeroed(&mut self, n: usize) {
220 self.buffer.extend_zeros(n * std::mem::size_of::<T>());
221 }
222
223 /// Appends a slice of type `T`, growing the internal buffer as needed.
224 ///
225 /// # Example:
226 ///
227 /// ```
228 /// # use arrow_buffer::builder::BufferBuilder;
229 /// let mut builder = BufferBuilder::<u8>::new(10);
230 /// builder.append_slice(&[42, 44, 46]);
231 ///
232 /// assert_eq!(builder.len(), 3);
233 /// ```
234 #[inline]
235 pub fn append_slice(&mut self, slice: &[T]) {
236 self.buffer.extend_from_slice(slice);
237 }
238
239 /// View the contents of this buffer as a slice
240 ///
241 /// ```
242 /// # use arrow_buffer::builder::BufferBuilder;
243 /// let mut builder = BufferBuilder::<f64>::new(10);
244 /// builder.append(1.3);
245 /// builder.append_n(2, 2.3);
246 ///
247 /// assert_eq!(builder.as_slice(), &[1.3, 2.3, 2.3]);
248 /// ```
249 #[inline]
250 pub fn as_slice(&self) -> &[T] {
251 // SAFETY
252 //
253 // - MutableBuffer is aligned and initialized for len elements of T
254 // - MutableBuffer corresponds to a single allocation
255 // - MutableBuffer does not support modification whilst active immutable borrows
256 unsafe { std::slice::from_raw_parts(self.buffer.as_ptr().cast(), self.len()) }
257 }
258
259 /// View the contents of this buffer as a mutable slice
260 ///
261 /// # Example:
262 ///
263 /// ```
264 /// # use arrow_buffer::builder::BufferBuilder;
265 /// let mut builder = BufferBuilder::<f32>::new(10);
266 ///
267 /// builder.append_slice(&[1., 2., 3.4]);
268 /// assert_eq!(builder.as_slice(), &[1., 2., 3.4]);
269 ///
270 /// builder.as_slice_mut()[1] = 4.2;
271 /// assert_eq!(builder.as_slice(), &[1., 4.2, 3.4]);
272 /// ```
273 #[inline]
274 pub fn as_slice_mut(&mut self) -> &mut [T] {
275 // SAFETY
276 //
277 // - MutableBuffer is aligned and initialized for len elements of T
278 // - MutableBuffer corresponds to a single allocation
279 // - MutableBuffer does not support modification whilst active immutable borrows
280 unsafe { std::slice::from_raw_parts_mut(self.buffer.as_mut_ptr().cast(), self.len()) }
281 }
282
283 /// Shorten this BufferBuilder to `len` items
284 ///
285 /// If `len` is greater than the builder's current length, this has no effect
286 ///
287 /// # Example:
288 ///
289 /// ```
290 /// # use arrow_buffer::builder::BufferBuilder;
291 /// let mut builder = BufferBuilder::<u16>::new(10);
292 ///
293 /// builder.append_slice(&[42, 44, 46]);
294 /// assert_eq!(builder.as_slice(), &[42, 44, 46]);
295 ///
296 /// builder.truncate(2);
297 /// assert_eq!(builder.as_slice(), &[42, 44]);
298 ///
299 /// builder.append(12);
300 /// assert_eq!(builder.as_slice(), &[42, 44, 12]);
301 /// ```
302 #[inline]
303 pub fn truncate(&mut self, len: usize) {
304 self.buffer.truncate(len * std::mem::size_of::<T>());
305 }
306
307 /// # Safety
308 /// This requires the iterator be a trusted length. This could instead require
309 /// the iterator implement `TrustedLen` once that is stabilized.
310 #[inline]
311 pub unsafe fn append_trusted_len_iter(&mut self, iter: impl IntoIterator<Item = T>) {
312 let iter = iter.into_iter();
313 let len = iter
314 .size_hint()
315 .1
316 .expect("append_trusted_len_iter expects upper bound");
317 self.reserve(len);
318 self.extend(iter);
319 }
320
321 /// Resets this builder and returns an immutable [Buffer].
322 ///
323 /// Use [`Self::build`] when you don't need to reuse this builder.
324 ///
325 /// # Example:
326 ///
327 /// ```
328 /// # use arrow_buffer::builder::BufferBuilder;
329 /// let mut builder = BufferBuilder::<u8>::new(10);
330 /// builder.append_slice(&[42, 44, 46]);
331 /// let buffer = builder.finish();
332 /// assert_eq!(unsafe { buffer.typed_data::<u8>() }, &[42, 44, 46]);
333 /// ```
334 #[inline]
335 pub fn finish(&mut self) -> Buffer {
336 let buf = std::mem::take(&mut self.buffer);
337 buf.into()
338 }
339
340 /// Builds an immutable [Buffer] without resetting the builder.
341 ///
342 /// This consumes the builder. Use [`Self::finish`] to reuse it.
343 ///
344 /// # Example:
345 ///
346 /// ```
347 /// # use arrow_buffer::builder::BufferBuilder;
348 /// let mut builder = BufferBuilder::<u8>::new(10);
349 /// builder.append_slice(&[42, 44, 46]);
350 /// let buffer = builder.build();
351 /// assert_eq!(unsafe { buffer.typed_data::<u8>() }, &[42, 44, 46]);
352 /// ```
353 #[inline]
354 pub fn build(self) -> Buffer {
355 self.buffer.into()
356 }
357}
358
359impl<T: ArrowNativeType> Default for BufferBuilder<T> {
360 fn default() -> Self {
361 Self::new(0)
362 }
363}
364
365impl<T: ArrowNativeType> Extend<T> for BufferBuilder<T> {
366 fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
367 self.buffer.extend(iter)
368 }
369}
370
371impl<T: ArrowNativeType> From<Vec<T>> for BufferBuilder<T> {
372 fn from(value: Vec<T>) -> Self {
373 let buffer = MutableBuffer::from(value);
374 // SAFETY
375 // - buffer is aligned to T
376 unsafe { Self::new_from_buffer(buffer) }
377 }
378}
379
380impl<T: ArrowNativeType> FromIterator<T> for BufferBuilder<T> {
381 fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
382 let mut builder = Self::default();
383 builder.extend(iter);
384 builder
385 }
386}
387
388#[cfg(test)]
389mod tests {
390 use super::*;
391 use std::mem;
392
393 #[test]
394 fn default() {
395 let builder = BufferBuilder::<u32>::default();
396 assert!(builder.is_empty());
397 assert!(builder.buffer.is_empty());
398 assert_eq!(builder.buffer.capacity(), 0);
399 }
400
401 #[test]
402 fn from_iter() {
403 let input = [1u16, 2, 3, 4];
404 let builder = input.into_iter().collect::<BufferBuilder<_>>();
405 assert_eq!(builder.len(), 4);
406 assert_eq!(builder.buffer.len(), 4 * mem::size_of::<u16>());
407 }
408
409 #[test]
410 fn extend() {
411 let input = [1, 2];
412 let mut builder = input.into_iter().collect::<BufferBuilder<_>>();
413 assert_eq!(builder.len(), 2);
414 builder.extend([3, 4]);
415 assert_eq!(builder.len(), 4);
416 }
417
418 #[test]
419 fn truncate_safety() {
420 let mut builder = BufferBuilder::from(vec![40, -63, 90]);
421 assert_eq!(builder.len(), 3);
422 builder.truncate(151);
423 assert_eq!(builder.len(), 3);
424 builder.advance(219);
425 assert_eq!(builder.len(), 222);
426 let slice = builder.as_slice_mut();
427 assert_eq!(slice.len(), 222);
428 }
429
430 #[test]
431 #[should_panic(expected = "buffer length overflow")]
432 fn reserve_length_overflow() {
433 let mut builder = BufferBuilder::<u8>::new(1);
434 builder.append(0);
435 builder.reserve(usize::MAX);
436 }
437
438 #[test]
439 #[should_panic(expected = "buffer length overflow")]
440 fn append_n_zeroed_length_overflow() {
441 let mut builder = BufferBuilder::<u64>::new(1);
442 builder.append_n_zeroed(1);
443 builder.append_n_zeroed(usize::MAX / mem::size_of::<u64>());
444 }
445
446 #[test]
447 #[should_panic(expected = "buffer length overflow")]
448 fn advance_length_overflow() {
449 let mut builder = BufferBuilder::<u64>::new(1);
450 builder.advance(1);
451 builder.advance(usize::MAX / mem::size_of::<u64>());
452 }
453}