vortex_array/builders/mod.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Builders for Vortex arrays.
5//!
6//! Every logical type in Vortex has a canonical (uncompressed) in-memory encoding. This module
7//! provides pre-allocated builders to construct new canonical arrays.
8//!
9//! Canonical form is not recursive, and neither are these builders: appending an array to a nested
10//! builder keeps the child in the encoding it arrived in instead of decoding it. The fields of a
11//! [`StructArray`](crate::arrays::StructArray), the elements of a list, and the storage of an
12//! [`ExtensionArray`](crate::arrays::ExtensionArray) may therefore come back compressed, or as a
13//! [`ChunkedArray`](crate::arrays::ChunkedArray) when several arrays were appended in turn.
14//!
15//! ## Example:
16//!
17//! ```
18//! use vortex_array::builders::{ArrayBuilder, builder_with_capacity_in};
19//! use vortex_array::dtype::{DType, Nullability};
20//! use vortex_array::memory::BufferAllocatorRef;
21//! use vortex_array::{VortexSessionExecute};
22//! use vortex_session::VortexSession;
23//!
24//! // Create a new builder for string data.
25//! let mut builder = builder_with_capacity_in(&DType::Utf8(Nullability::NonNullable), 4, BufferAllocatorRef::static_ref());
26//!
27//! builder.append_scalar(&"a".into()).unwrap();
28//! builder.append_scalar(&"b".into()).unwrap();
29//! builder.append_scalar(&"c".into()).unwrap();
30//! builder.append_scalar(&"d".into()).unwrap();
31//!
32//! let strings = builder.finish();
33//! let mut ctx = VortexSession::empty().create_execution_ctx();
34//!
35//! assert_eq!(strings.execute_scalar(0, &mut ctx).unwrap(), "a".into());
36//! assert_eq!(strings.execute_scalar(1, &mut ctx).unwrap(), "b".into());
37//! assert_eq!(strings.execute_scalar(2, &mut ctx).unwrap(), "c".into());
38//! assert_eq!(strings.execute_scalar(3, &mut ctx).unwrap(), "d".into());
39//! ```
40
41use std::any::Any;
42use std::sync::Arc;
43
44use vortex_error::VortexResult;
45use vortex_error::vortex_panic;
46
47use crate::ArrayRef;
48use crate::ExecutionCtx;
49use crate::canonical::Canonical;
50use crate::dtype::DType;
51use crate::match_each_decimal_value_type;
52use crate::match_each_native_ptype;
53use crate::memory::BufferAllocatorRef;
54use crate::scalar::Scalar;
55
56mod lazy_null_builder;
57pub(crate) use lazy_null_builder::LazyBitBufferBuilder;
58
59mod bool;
60mod child;
61mod decimal;
62pub mod dict;
63mod extension;
64mod fixed_size_list;
65mod list;
66mod listview;
67mod map;
68mod null;
69mod primitive;
70mod struct_;
71mod validity;
72mod varbinview;
73
74pub use bool::*;
75pub(crate) use child::ChildBuilder;
76pub use decimal::*;
77pub use extension::*;
78pub use fixed_size_list::*;
79pub use list::*;
80pub use listview::*;
81pub use map::*;
82pub use null::*;
83pub use primitive::*;
84pub use struct_::*;
85pub(crate) use validity::ValidityBuilder;
86pub use varbinview::*;
87
88pub use crate::arrays::varbin::builder::VarBinBuilder;
89
90#[cfg(test)]
91mod tests;
92
93/// The default capacity for builders.
94///
95/// This is equal to the default capacity for Arrow Arrays.
96pub const DEFAULT_BUILDER_CAPACITY: usize = 1024;
97
98pub trait ArrayBuilder: Send {
99 fn as_any(&self) -> &dyn Any;
100
101 fn as_any_mut(&mut self) -> &mut dyn Any;
102
103 fn dtype(&self) -> &DType;
104
105 fn len(&self) -> usize;
106
107 fn is_empty(&self) -> bool {
108 self.len() == 0
109 }
110
111 /// Append a "zero" value to the array.
112 ///
113 /// Zero values are generally determined by [`Scalar::default_value`].
114 fn append_zero(&mut self) {
115 self.append_zeros(1)
116 }
117
118 /// Appends n "zero" values to the array.
119 ///
120 /// Zero values are generally determined by [`Scalar::default_value`].
121 fn append_zeros(&mut self, n: usize);
122
123 /// Append a "null" value to the array.
124 ///
125 /// Implementors should panic if this method is called on a non-nullable [`ArrayBuilder`].
126 fn append_null(&mut self) {
127 self.append_nulls(1)
128 }
129
130 /// The inner part of `append_nulls`.
131 ///
132 /// # Safety
133 ///
134 /// The array builder must be nullable.
135 unsafe fn append_nulls_unchecked(&mut self, n: usize);
136
137 /// Appends n "null" values to the array.
138 ///
139 /// Implementors should panic if this method is called on a non-nullable [`ArrayBuilder`].
140 fn append_nulls(&mut self, n: usize) {
141 assert!(
142 self.dtype().is_nullable(),
143 "tried to append {n} nulls to a non-nullable array builder"
144 );
145
146 // SAFETY: We check above that the array builder is nullable.
147 unsafe {
148 self.append_nulls_unchecked(n);
149 }
150 }
151
152 /// Appends a default value to the array.
153 fn append_default(&mut self) {
154 self.append_defaults(1)
155 }
156
157 /// Appends n default values to the array.
158 ///
159 /// If the array builder is nullable, then this has the behavior of `self.append_nulls(n)`.
160 /// If the array builder is non-nullable, then it has the behavior of `self.append_zeros(n)`.
161 fn append_defaults(&mut self, n: usize) {
162 if self.dtype().is_nullable() {
163 self.append_nulls(n);
164 } else {
165 self.append_zeros(n);
166 }
167 }
168
169 /// A generic function to append a scalar to the builder.
170 fn append_scalar(&mut self, scalar: &Scalar) -> VortexResult<()>;
171
172 /// Allocate space for extra `additional` items
173 fn reserve_exact(&mut self, additional: usize);
174
175 /// Constructs an Array from the builder components.
176 ///
177 /// The returned array is canonical at the top level only; its children keep whatever encoding
178 /// they were appended with.
179 ///
180 /// # Panics
181 ///
182 /// This function may panic if the builder's methods are called with invalid arguments. If only
183 /// the methods on this interface are used, the builder should not panic. However, specific
184 /// builders have interfaces that may be misused. For example, if the number of values in a
185 /// [PrimitiveBuilder]'s [vortex_buffer::BufferMut] does not match the number of validity bits,
186 /// the PrimitiveBuilder's [Self::finish] will panic.
187 fn finish(&mut self) -> ArrayRef;
188
189 /// Constructs a canonical array directly from the builder.
190 ///
191 /// This method provides a default implementation that creates an [`ArrayRef`] via `finish` and
192 /// then converts it to canonical form. Specific builders can override this with optimized
193 /// implementations that avoid the intermediate [`ArrayRef`] creation.
194 fn finish_into_canonical(&mut self, ctx: &mut ExecutionCtx) -> Canonical;
195}
196
197/// Matches a `&mut dyn ArrayBuilder` against every concrete list builder type, i.e. every
198/// [`ListBuilder`]`<O>` and [`ListViewBuilder`]`<O, S>` instantiation over the
199/// [`OffsetBuilderPType`](crate::dtype::OffsetBuilderPType) offset/size types (`u32`, `u64`, `i32`,
200/// `i64`).
201///
202/// Binds the downcast builder as `$builder` and evaluates `$body` with it, yielding
203/// `Some($body)`; yields `None` when the builder is not a list builder. List encodings dispatch
204/// through this matcher because the concrete list builders are generic over their offset/size
205/// integer types, which cannot be named through a `dyn ArrayBuilder`. The matcher is exhaustive
206/// because `OffsetBuilderPType` is sealed, so no other instantiations can be constructed.
207#[macro_export]
208macro_rules! match_each_list_builder {
209 ($dyn_builder:expr, | $builder:ident | $body:expr) => {{
210 let __dyn_builder: &mut dyn $crate::builders::ArrayBuilder = $dyn_builder;
211 match $crate::__match_each_list_builder!(
212 __dyn_builder,
213 $builder,
214 $body,
215 [u32, u64, i32, i64]
216 ) {
217 ::core::option::Option::Some(__result) => ::core::option::Option::Some(__result),
218 ::core::option::Option::None => $crate::__match_each_listview_builder!(
219 __dyn_builder,
220 $builder,
221 $body,
222 [u32, u64, i32, i64]
223 ),
224 }
225 }};
226}
227
228/// Matches a `&mut dyn ArrayBuilder` against every concrete [`ListViewBuilder`]`<O, S>`
229/// instantiation over the [`OffsetBuilderPType`](crate::dtype::OffsetBuilderPType) offset/size
230/// types (`u32`, `u64`, `i32`, `i64`), and only those.
231///
232/// Binds the downcast builder as `$builder` and evaluates `$body` with it, yielding
233/// `Some($body)`; yields `None` when the builder is not a list-view builder - including when it
234/// is a [`ListBuilder`]. Callers reach for this instead of
235/// [`match_each_list_builder!`](crate::match_each_list_builder) when the body needs methods only
236/// a list-view builder has, such as
237/// [`append_array_as_repeated_list`](ListViewBuilder::append_array_as_repeated_list).
238#[macro_export]
239macro_rules! match_each_listview_builder {
240 ($dyn_builder:expr, | $builder:ident | $body:expr) => {{
241 let __dyn_builder: &mut dyn $crate::builders::ArrayBuilder = $dyn_builder;
242 $crate::__match_each_listview_builder!(__dyn_builder, $builder, $body, [u32, u64, i32, i64])
243 }};
244}
245
246#[doc(hidden)]
247#[macro_export]
248macro_rules! __match_each_list_builder {
249 ($target:ident, $builder:ident, $body:expr, []) => {
250 ::core::option::Option::None
251 };
252 ($target:ident, $builder:ident, $body:expr, [$offset:ty $(, $rest:ty)*]) => {
253 if let ::core::option::Option::Some($builder) =
254 $crate::builders::ArrayBuilder::as_any_mut($target)
255 .downcast_mut::<$crate::builders::ListBuilder<$offset>>()
256 {
257 ::core::option::Option::Some($body)
258 } else {
259 $crate::__match_each_list_builder!($target, $builder, $body, [$($rest),*])
260 }
261 };
262}
263
264#[doc(hidden)]
265#[macro_export]
266macro_rules! __match_each_listview_builder {
267 ($target:ident, $builder:ident, $body:expr, []) => {
268 ::core::option::Option::None
269 };
270 ($target:ident, $builder:ident, $body:expr, [$offset:ty $(, $rest:ty)*]) => {
271 match $crate::__match_each_listview_builder_size!(
272 $target,
273 $builder,
274 $body,
275 $offset,
276 [u32, u64, i32, i64]
277 ) {
278 ::core::option::Option::Some(__result) => ::core::option::Option::Some(__result),
279 ::core::option::Option::None => $crate::__match_each_listview_builder!(
280 $target,
281 $builder,
282 $body,
283 [$($rest),*]
284 ),
285 }
286 };
287}
288
289#[doc(hidden)]
290#[macro_export]
291macro_rules! __match_each_listview_builder_size {
292 ($target:ident, $builder:ident, $body:expr, $offset:ty, []) => {
293 ::core::option::Option::None
294 };
295 ($target:ident, $builder:ident, $body:expr, $offset:ty, [$size:ty $(, $rest:ty)*]) => {
296 if let ::core::option::Option::Some($builder) =
297 $crate::builders::ArrayBuilder::as_any_mut($target)
298 .downcast_mut::<$crate::builders::ListViewBuilder<$offset, $size>>()
299 {
300 ::core::option::Option::Some($body)
301 } else {
302 $crate::__match_each_listview_builder_size!(
303 $target, $builder, $body, $offset, [$($rest),*]
304 )
305 }
306 };
307}
308
309/// Matches a `&mut dyn ArrayBuilder` against every concrete map builder type.
310///
311/// Binds the downcast builder as `$builder` and evaluates `$body` with it, yielding
312/// `Some($body)`; yields `None` when the builder is not a map builder.
313#[macro_export]
314macro_rules! match_each_map_builder {
315 ($dyn_builder:expr, | $builder:ident | $body:expr) => {{
316 let __dyn_builder: &mut dyn $crate::builders::ArrayBuilder = $dyn_builder;
317 $crate::__match_each_map_builder!(__dyn_builder, $builder, $body, [u32, u64, i32, i64])
318 }};
319}
320
321#[doc(hidden)]
322#[macro_export]
323macro_rules! __match_each_map_builder {
324 ($target:ident, $builder:ident, $body:expr, []) => {
325 ::core::option::Option::None
326 };
327 ($target:ident, $builder:ident, $body:expr, [$offset:ty $(, $rest:ty)*]) => {
328 match $crate::__match_each_map_builder_size!(
329 $target,
330 $builder,
331 $body,
332 $offset,
333 [u32, u64, i32, i64]
334 ) {
335 ::core::option::Option::Some(__result) => ::core::option::Option::Some(__result),
336 ::core::option::Option::None => $crate::__match_each_map_builder!(
337 $target,
338 $builder,
339 $body,
340 [$($rest),*]
341 ),
342 }
343 };
344}
345
346#[doc(hidden)]
347#[macro_export]
348macro_rules! __match_each_map_builder_size {
349 ($target:ident, $builder:ident, $body:expr, $offset:ty, []) => {
350 ::core::option::Option::None
351 };
352 ($target:ident, $builder:ident, $body:expr, $offset:ty, [$size:ty $(, $rest:ty)*]) => {
353 if let ::core::option::Option::Some($builder) =
354 $crate::builders::ArrayBuilder::as_any_mut($target)
355 .downcast_mut::<$crate::builders::MapBuilder<$offset, $size>>()
356 {
357 ::core::option::Option::Some($body)
358 } else {
359 $crate::__match_each_map_builder_size!(
360 $target, $builder, $body, $offset, [$($rest),*]
361 )
362 }
363 };
364}
365
366/// Construct a new canonical builder for the given [`DType`].
367#[deprecated(note = "use `builder_with_capacity_in` with an explicit allocator")]
368pub fn builder_with_capacity(dtype: &DType, capacity: usize) -> Box<dyn ArrayBuilder> {
369 builder_with_capacity_in(dtype, capacity, BufferAllocatorRef::static_ref())
370}
371
372/// Construct a new canonical builder using `allocator`.
373///
374/// # Example
375///
376/// ```
377/// use vortex_array::builders::{ArrayBuilder, builder_with_capacity_in};
378/// use vortex_array::dtype::{DType, Nullability};
379/// use vortex_array::memory::BufferAllocatorRef;
380/// use vortex_array::{VortexSessionExecute};
381/// use vortex_session::VortexSession;
382///
383/// // Create a new builder for string data.
384/// let mut builder = builder_with_capacity_in(&DType::Utf8(Nullability::NonNullable), 4, BufferAllocatorRef::static_ref());
385///
386/// builder.append_scalar(&"a".into()).unwrap();
387/// builder.append_scalar(&"b".into()).unwrap();
388/// builder.append_scalar(&"c".into()).unwrap();
389/// builder.append_scalar(&"d".into()).unwrap();
390///
391/// let strings = builder.finish();
392/// let mut ctx = VortexSession::empty().create_execution_ctx();
393///
394/// assert_eq!(strings.execute_scalar(0, &mut ctx).unwrap(), "a".into());
395/// assert_eq!(strings.execute_scalar(1, &mut ctx).unwrap(), "b".into());
396/// assert_eq!(strings.execute_scalar(2, &mut ctx).unwrap(), "c".into());
397/// assert_eq!(strings.execute_scalar(3, &mut ctx).unwrap(), "d".into());
398/// ```
399pub fn builder_with_capacity_in(
400 dtype: &DType,
401 capacity: usize,
402 allocator: &BufferAllocatorRef,
403) -> Box<dyn ArrayBuilder> {
404 match dtype {
405 DType::Null => Box::new(NullBuilder::new()),
406 DType::Bool(n) => Box::new(BoolBuilder::with_capacity_in(*n, capacity, allocator)),
407 DType::Primitive(ptype, n) => {
408 match_each_native_ptype!(ptype, |P| {
409 Box::new(PrimitiveBuilder::<P>::with_capacity_in(
410 *n, capacity, allocator,
411 ))
412 })
413 }
414 DType::Decimal(decimal_type, n) => {
415 match_each_decimal_value_type!(decimal_type.canonical_values_type(), |D| {
416 Box::new(DecimalBuilder::with_capacity_in::<D>(
417 capacity,
418 *decimal_type,
419 *n,
420 allocator,
421 ))
422 })
423 }
424 DType::Utf8(n) => Box::new(VarBinViewBuilder::with_capacity_in(
425 DType::Utf8(*n),
426 capacity,
427 allocator.clone(),
428 )),
429 DType::Binary(n) => Box::new(VarBinViewBuilder::with_capacity_in(
430 DType::Binary(*n),
431 capacity,
432 allocator.clone(),
433 )),
434 DType::List(dtype, n) => Box::new(ListViewBuilder::<u64, u64>::with_capacity_in(
435 Arc::clone(dtype),
436 *n,
437 2 * capacity, // Arbitrarily choose 2 times the `offsets` capacity here.
438 capacity,
439 allocator,
440 )),
441 DType::Map(map_dtype, nullability) => Box::new(MapBuilder::<u64, u64>::with_capacity_in(
442 map_dtype.clone(),
443 *nullability,
444 capacity,
445 allocator,
446 )),
447 DType::FixedSizeList(elem_dtype, list_size, null) => {
448 Box::new(FixedSizeListBuilder::with_capacity_in(
449 Arc::clone(elem_dtype),
450 *list_size,
451 *null,
452 capacity,
453 allocator,
454 ))
455 }
456 DType::Struct(struct_dtype, n) => Box::new(StructBuilder::with_capacity_in(
457 struct_dtype.clone(),
458 *n,
459 capacity,
460 allocator,
461 )),
462 DType::Union(..) => vortex_panic!("TODO(connor)[Union]: unimplemented"),
463 DType::Variant(_) => {
464 vortex_panic!("Variant builders are not implemented")
465 }
466 DType::Extension(ext_dtype) => Box::new(ExtensionBuilder::with_capacity_in(
467 ext_dtype.clone(),
468 capacity,
469 allocator,
470 )),
471 }
472}