Skip to main content

vortex_array/builders/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Builders for Vortex arrays.
5//!
6//! Every logical type in Vortex has a canonical (uncompressed) in-memory encoding. This module
7//! provides pre-allocated builders to construct new canonical arrays.
8//!
9//! Canonical form is not recursive, and neither are these builders: appending an array to a nested
10//! builder keeps the child in the encoding it arrived in instead of decoding it. The fields of a
11//! [`StructArray`](crate::arrays::StructArray), the elements of a list, and the storage of an
12//! [`ExtensionArray`](crate::arrays::ExtensionArray) may therefore come back compressed, or as a
13//! [`ChunkedArray`](crate::arrays::ChunkedArray) when several arrays were appended in turn.
14//!
15//! ## Example:
16//!
17//! ```
18//! use vortex_array::builders::{ArrayBuilder, builder_with_capacity_in};
19//! use vortex_array::dtype::{DType, Nullability};
20//! use vortex_array::memory::BufferAllocatorRef;
21//! use vortex_array::{VortexSessionExecute};
22//! use vortex_session::VortexSession;
23//!
24//! // Create a new builder for string data.
25//! let mut builder = builder_with_capacity_in(&DType::Utf8(Nullability::NonNullable), 4, BufferAllocatorRef::static_ref());
26//!
27//! builder.append_scalar(&"a".into()).unwrap();
28//! builder.append_scalar(&"b".into()).unwrap();
29//! builder.append_scalar(&"c".into()).unwrap();
30//! builder.append_scalar(&"d".into()).unwrap();
31//!
32//! let strings = builder.finish();
33//! let mut ctx = VortexSession::empty().create_execution_ctx();
34//!
35//! assert_eq!(strings.execute_scalar(0, &mut ctx).unwrap(), "a".into());
36//! assert_eq!(strings.execute_scalar(1, &mut ctx).unwrap(), "b".into());
37//! assert_eq!(strings.execute_scalar(2, &mut ctx).unwrap(), "c".into());
38//! assert_eq!(strings.execute_scalar(3, &mut ctx).unwrap(), "d".into());
39//! ```
40
41use std::any::Any;
42use std::sync::Arc;
43
44use vortex_error::VortexResult;
45use vortex_error::vortex_panic;
46
47use crate::ArrayRef;
48use crate::ExecutionCtx;
49use crate::canonical::Canonical;
50use crate::dtype::DType;
51use crate::match_each_decimal_value_type;
52use crate::match_each_native_ptype;
53use crate::memory::BufferAllocatorRef;
54use crate::scalar::Scalar;
55
56mod lazy_null_builder;
57pub(crate) use lazy_null_builder::LazyBitBufferBuilder;
58
59mod bool;
60mod child;
61mod decimal;
62pub mod dict;
63mod extension;
64mod fixed_size_list;
65mod list;
66mod listview;
67mod map;
68mod null;
69mod primitive;
70mod struct_;
71mod validity;
72mod varbinview;
73
74pub use bool::*;
75pub(crate) use child::ChildBuilder;
76pub use decimal::*;
77pub use extension::*;
78pub use fixed_size_list::*;
79pub use list::*;
80pub use listview::*;
81pub use map::*;
82pub use null::*;
83pub use primitive::*;
84pub use struct_::*;
85pub(crate) use validity::ValidityBuilder;
86pub use varbinview::*;
87
88pub use crate::arrays::varbin::builder::VarBinBuilder;
89
90#[cfg(test)]
91mod tests;
92
93/// The default capacity for builders.
94///
95/// This is equal to the default capacity for Arrow Arrays.
96pub const DEFAULT_BUILDER_CAPACITY: usize = 1024;
97
98pub trait ArrayBuilder: Send {
99    fn as_any(&self) -> &dyn Any;
100
101    fn as_any_mut(&mut self) -> &mut dyn Any;
102
103    fn dtype(&self) -> &DType;
104
105    fn len(&self) -> usize;
106
107    fn is_empty(&self) -> bool {
108        self.len() == 0
109    }
110
111    /// Append a "zero" value to the array.
112    ///
113    /// Zero values are generally determined by [`Scalar::default_value`].
114    fn append_zero(&mut self) {
115        self.append_zeros(1)
116    }
117
118    /// Appends n "zero" values to the array.
119    ///
120    /// Zero values are generally determined by [`Scalar::default_value`].
121    fn append_zeros(&mut self, n: usize);
122
123    /// Append a "null" value to the array.
124    ///
125    /// Implementors should panic if this method is called on a non-nullable [`ArrayBuilder`].
126    fn append_null(&mut self) {
127        self.append_nulls(1)
128    }
129
130    /// The inner part of `append_nulls`.
131    ///
132    /// # Safety
133    ///
134    /// The array builder must be nullable.
135    unsafe fn append_nulls_unchecked(&mut self, n: usize);
136
137    /// Appends n "null" values to the array.
138    ///
139    /// Implementors should panic if this method is called on a non-nullable [`ArrayBuilder`].
140    fn append_nulls(&mut self, n: usize) {
141        assert!(
142            self.dtype().is_nullable(),
143            "tried to append {n} nulls to a non-nullable array builder"
144        );
145
146        // SAFETY: We check above that the array builder is nullable.
147        unsafe {
148            self.append_nulls_unchecked(n);
149        }
150    }
151
152    /// Appends a default value to the array.
153    fn append_default(&mut self) {
154        self.append_defaults(1)
155    }
156
157    /// Appends n default values to the array.
158    ///
159    /// If the array builder is nullable, then this has the behavior of `self.append_nulls(n)`.
160    /// If the array builder is non-nullable, then it has the behavior of `self.append_zeros(n)`.
161    fn append_defaults(&mut self, n: usize) {
162        if self.dtype().is_nullable() {
163            self.append_nulls(n);
164        } else {
165            self.append_zeros(n);
166        }
167    }
168
169    /// A generic function to append a scalar to the builder.
170    fn append_scalar(&mut self, scalar: &Scalar) -> VortexResult<()>;
171
172    /// Allocate space for extra `additional` items
173    fn reserve_exact(&mut self, additional: usize);
174
175    /// Constructs an Array from the builder components.
176    ///
177    /// The returned array is canonical at the top level only; its children keep whatever encoding
178    /// they were appended with.
179    ///
180    /// # Panics
181    ///
182    /// This function may panic if the builder's methods are called with invalid arguments. If only
183    /// the methods on this interface are used, the builder should not panic. However, specific
184    /// builders have interfaces that may be misused. For example, if the number of values in a
185    /// [PrimitiveBuilder]'s [vortex_buffer::BufferMut] does not match the number of validity bits,
186    /// the PrimitiveBuilder's [Self::finish] will panic.
187    fn finish(&mut self) -> ArrayRef;
188
189    /// Constructs a canonical array directly from the builder.
190    ///
191    /// This method provides a default implementation that creates an [`ArrayRef`] via `finish` and
192    /// then converts it to canonical form. Specific builders can override this with optimized
193    /// implementations that avoid the intermediate [`ArrayRef`] creation.
194    fn finish_into_canonical(&mut self, ctx: &mut ExecutionCtx) -> Canonical;
195}
196
197/// Matches a `&mut dyn ArrayBuilder` against every concrete list builder type, i.e. every
198/// [`ListBuilder`]`<O>` and [`ListViewBuilder`]`<O, S>` instantiation over the
199/// [`OffsetBuilderPType`](crate::dtype::OffsetBuilderPType) offset/size types (`u32`, `u64`, `i32`,
200/// `i64`).
201///
202/// Binds the downcast builder as `$builder` and evaluates `$body` with it, yielding
203/// `Some($body)`; yields `None` when the builder is not a list builder. List encodings dispatch
204/// through this matcher because the concrete list builders are generic over their offset/size
205/// integer types, which cannot be named through a `dyn ArrayBuilder`. The matcher is exhaustive
206/// because `OffsetBuilderPType` is sealed, so no other instantiations can be constructed.
207#[macro_export]
208macro_rules! match_each_list_builder {
209    ($dyn_builder:expr, | $builder:ident | $body:expr) => {{
210        let __dyn_builder: &mut dyn $crate::builders::ArrayBuilder = $dyn_builder;
211        match $crate::__match_each_list_builder!(
212            __dyn_builder,
213            $builder,
214            $body,
215            [u32, u64, i32, i64]
216        ) {
217            ::core::option::Option::Some(__result) => ::core::option::Option::Some(__result),
218            ::core::option::Option::None => $crate::__match_each_listview_builder!(
219                __dyn_builder,
220                $builder,
221                $body,
222                [u32, u64, i32, i64]
223            ),
224        }
225    }};
226}
227
228/// Matches a `&mut dyn ArrayBuilder` against every concrete [`ListViewBuilder`]`<O, S>`
229/// instantiation over the [`OffsetBuilderPType`](crate::dtype::OffsetBuilderPType) offset/size
230/// types (`u32`, `u64`, `i32`, `i64`), and only those.
231///
232/// Binds the downcast builder as `$builder` and evaluates `$body` with it, yielding
233/// `Some($body)`; yields `None` when the builder is not a list-view builder - including when it
234/// is a [`ListBuilder`]. Callers reach for this instead of
235/// [`match_each_list_builder!`](crate::match_each_list_builder) when the body needs methods only
236/// a list-view builder has, such as
237/// [`append_array_as_repeated_list`](ListViewBuilder::append_array_as_repeated_list).
238#[macro_export]
239macro_rules! match_each_listview_builder {
240    ($dyn_builder:expr, | $builder:ident | $body:expr) => {{
241        let __dyn_builder: &mut dyn $crate::builders::ArrayBuilder = $dyn_builder;
242        $crate::__match_each_listview_builder!(__dyn_builder, $builder, $body, [u32, u64, i32, i64])
243    }};
244}
245
246#[doc(hidden)]
247#[macro_export]
248macro_rules! __match_each_list_builder {
249    ($target:ident, $builder:ident, $body:expr, []) => {
250        ::core::option::Option::None
251    };
252    ($target:ident, $builder:ident, $body:expr, [$offset:ty $(, $rest:ty)*]) => {
253        if let ::core::option::Option::Some($builder) =
254            $crate::builders::ArrayBuilder::as_any_mut($target)
255                .downcast_mut::<$crate::builders::ListBuilder<$offset>>()
256        {
257            ::core::option::Option::Some($body)
258        } else {
259            $crate::__match_each_list_builder!($target, $builder, $body, [$($rest),*])
260        }
261    };
262}
263
264#[doc(hidden)]
265#[macro_export]
266macro_rules! __match_each_listview_builder {
267    ($target:ident, $builder:ident, $body:expr, []) => {
268        ::core::option::Option::None
269    };
270    ($target:ident, $builder:ident, $body:expr, [$offset:ty $(, $rest:ty)*]) => {
271        match $crate::__match_each_listview_builder_size!(
272            $target,
273            $builder,
274            $body,
275            $offset,
276            [u32, u64, i32, i64]
277        ) {
278            ::core::option::Option::Some(__result) => ::core::option::Option::Some(__result),
279            ::core::option::Option::None => $crate::__match_each_listview_builder!(
280                $target,
281                $builder,
282                $body,
283                [$($rest),*]
284            ),
285        }
286    };
287}
288
289#[doc(hidden)]
290#[macro_export]
291macro_rules! __match_each_listview_builder_size {
292    ($target:ident, $builder:ident, $body:expr, $offset:ty, []) => {
293        ::core::option::Option::None
294    };
295    ($target:ident, $builder:ident, $body:expr, $offset:ty, [$size:ty $(, $rest:ty)*]) => {
296        if let ::core::option::Option::Some($builder) =
297            $crate::builders::ArrayBuilder::as_any_mut($target)
298                .downcast_mut::<$crate::builders::ListViewBuilder<$offset, $size>>()
299        {
300            ::core::option::Option::Some($body)
301        } else {
302            $crate::__match_each_listview_builder_size!(
303                $target, $builder, $body, $offset, [$($rest),*]
304            )
305        }
306    };
307}
308
309/// Matches a `&mut dyn ArrayBuilder` against every concrete map builder type.
310///
311/// Binds the downcast builder as `$builder` and evaluates `$body` with it, yielding
312/// `Some($body)`; yields `None` when the builder is not a map builder.
313#[macro_export]
314macro_rules! match_each_map_builder {
315    ($dyn_builder:expr, | $builder:ident | $body:expr) => {{
316        let __dyn_builder: &mut dyn $crate::builders::ArrayBuilder = $dyn_builder;
317        $crate::__match_each_map_builder!(__dyn_builder, $builder, $body, [u32, u64, i32, i64])
318    }};
319}
320
321#[doc(hidden)]
322#[macro_export]
323macro_rules! __match_each_map_builder {
324    ($target:ident, $builder:ident, $body:expr, []) => {
325        ::core::option::Option::None
326    };
327    ($target:ident, $builder:ident, $body:expr, [$offset:ty $(, $rest:ty)*]) => {
328        match $crate::__match_each_map_builder_size!(
329            $target,
330            $builder,
331            $body,
332            $offset,
333            [u32, u64, i32, i64]
334        ) {
335            ::core::option::Option::Some(__result) => ::core::option::Option::Some(__result),
336            ::core::option::Option::None => $crate::__match_each_map_builder!(
337                $target,
338                $builder,
339                $body,
340                [$($rest),*]
341            ),
342        }
343    };
344}
345
346#[doc(hidden)]
347#[macro_export]
348macro_rules! __match_each_map_builder_size {
349    ($target:ident, $builder:ident, $body:expr, $offset:ty, []) => {
350        ::core::option::Option::None
351    };
352    ($target:ident, $builder:ident, $body:expr, $offset:ty, [$size:ty $(, $rest:ty)*]) => {
353        if let ::core::option::Option::Some($builder) =
354            $crate::builders::ArrayBuilder::as_any_mut($target)
355                .downcast_mut::<$crate::builders::MapBuilder<$offset, $size>>()
356        {
357            ::core::option::Option::Some($body)
358        } else {
359            $crate::__match_each_map_builder_size!(
360                $target, $builder, $body, $offset, [$($rest),*]
361            )
362        }
363    };
364}
365
366/// Construct a new canonical builder for the given [`DType`].
367#[deprecated(note = "use `builder_with_capacity_in` with an explicit allocator")]
368pub fn builder_with_capacity(dtype: &DType, capacity: usize) -> Box<dyn ArrayBuilder> {
369    builder_with_capacity_in(dtype, capacity, BufferAllocatorRef::static_ref())
370}
371
372/// Construct a new canonical builder using `allocator`.
373///
374/// # Example
375///
376/// ```
377/// use vortex_array::builders::{ArrayBuilder, builder_with_capacity_in};
378/// use vortex_array::dtype::{DType, Nullability};
379/// use vortex_array::memory::BufferAllocatorRef;
380/// use vortex_array::{VortexSessionExecute};
381/// use vortex_session::VortexSession;
382///
383/// // Create a new builder for string data.
384/// let mut builder = builder_with_capacity_in(&DType::Utf8(Nullability::NonNullable), 4, BufferAllocatorRef::static_ref());
385///
386/// builder.append_scalar(&"a".into()).unwrap();
387/// builder.append_scalar(&"b".into()).unwrap();
388/// builder.append_scalar(&"c".into()).unwrap();
389/// builder.append_scalar(&"d".into()).unwrap();
390///
391/// let strings = builder.finish();
392/// let mut ctx = VortexSession::empty().create_execution_ctx();
393///
394/// assert_eq!(strings.execute_scalar(0, &mut ctx).unwrap(), "a".into());
395/// assert_eq!(strings.execute_scalar(1, &mut ctx).unwrap(), "b".into());
396/// assert_eq!(strings.execute_scalar(2, &mut ctx).unwrap(), "c".into());
397/// assert_eq!(strings.execute_scalar(3, &mut ctx).unwrap(), "d".into());
398/// ```
399pub fn builder_with_capacity_in(
400    dtype: &DType,
401    capacity: usize,
402    allocator: &BufferAllocatorRef,
403) -> Box<dyn ArrayBuilder> {
404    match dtype {
405        DType::Null => Box::new(NullBuilder::new()),
406        DType::Bool(n) => Box::new(BoolBuilder::with_capacity_in(*n, capacity, allocator)),
407        DType::Primitive(ptype, n) => {
408            match_each_native_ptype!(ptype, |P| {
409                Box::new(PrimitiveBuilder::<P>::with_capacity_in(
410                    *n, capacity, allocator,
411                ))
412            })
413        }
414        DType::Decimal(decimal_type, n) => {
415            match_each_decimal_value_type!(decimal_type.canonical_values_type(), |D| {
416                Box::new(DecimalBuilder::with_capacity_in::<D>(
417                    capacity,
418                    *decimal_type,
419                    *n,
420                    allocator,
421                ))
422            })
423        }
424        DType::Utf8(n) => Box::new(VarBinViewBuilder::with_capacity_in(
425            DType::Utf8(*n),
426            capacity,
427            allocator.clone(),
428        )),
429        DType::Binary(n) => Box::new(VarBinViewBuilder::with_capacity_in(
430            DType::Binary(*n),
431            capacity,
432            allocator.clone(),
433        )),
434        DType::List(dtype, n) => Box::new(ListViewBuilder::<u64, u64>::with_capacity_in(
435            Arc::clone(dtype),
436            *n,
437            2 * capacity, // Arbitrarily choose 2 times the `offsets` capacity here.
438            capacity,
439            allocator,
440        )),
441        DType::Map(map_dtype, nullability) => Box::new(MapBuilder::<u64, u64>::with_capacity_in(
442            map_dtype.clone(),
443            *nullability,
444            capacity,
445            allocator,
446        )),
447        DType::FixedSizeList(elem_dtype, list_size, null) => {
448            Box::new(FixedSizeListBuilder::with_capacity_in(
449                Arc::clone(elem_dtype),
450                *list_size,
451                *null,
452                capacity,
453                allocator,
454            ))
455        }
456        DType::Struct(struct_dtype, n) => Box::new(StructBuilder::with_capacity_in(
457            struct_dtype.clone(),
458            *n,
459            capacity,
460            allocator,
461        )),
462        DType::Union(..) => vortex_panic!("TODO(connor)[Union]: unimplemented"),
463        DType::Variant(_) => {
464            vortex_panic!("Variant builders are not implemented")
465        }
466        DType::Extension(ext_dtype) => Box::new(ExtensionBuilder::with_capacity_in(
467            ext_dtype.clone(),
468            capacity,
469            allocator,
470        )),
471    }
472}