vortex_array/array/vtable/mod.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! This module contains the VTable definitions for a Vortex encoding.
5//!
6//! A Vortex array encoding is implemented by a small static vtable type plus an associated
7//! `TypedArrayData` value stored in each array instance. The vtable owns behavior such as
8//! validation, serialization, execution, child traversal, scalar access, and validity access.
9//!
10//! The public [`ArrayRef`] API performs common precondition checks before calling
11//! into these traits. Implementations should focus on encoding-specific work and uphold the
12//! documented postconditions.
13
14mod operations;
15mod validity;
16
17use std::fmt::Debug;
18use std::fmt::Display;
19use std::fmt::Formatter;
20use std::hash::Hasher;
21
22pub use operations::*;
23pub use validity::*;
24use vortex_error::VortexExpect;
25use vortex_error::VortexResult;
26use vortex_error::vortex_bail;
27use vortex_error::vortex_ensure;
28use vortex_error::vortex_panic;
29use vortex_session::VortexSession;
30
31use crate::Array;
32use crate::ArrayRef;
33use crate::ArrayView;
34use crate::Canonical;
35use crate::EqMode;
36use crate::ExecutionResult;
37use crate::IntoArray;
38pub use crate::array::plugin::*;
39use crate::arrays::ConstantArray;
40use crate::arrays::constant::Constant;
41use crate::buffer::BufferHandle;
42use crate::builders::ArrayBuilder;
43use crate::dtype::DType;
44use crate::dtype::Nullability;
45use crate::executor::ExecutionCtx;
46use crate::hash::ArrayEq;
47use crate::hash::ArrayHash;
48use crate::patches::Patches;
49use crate::scalar::ScalarValue;
50use crate::serde::ArrayChildren;
51use crate::validity::Validity;
52
53/// The array [`VTable`] encapsulates logic for an Array type within Vortex.
54///
55/// The logic is split across several "VTable" traits to enable easier code organization than
56/// simply lumping everything into a single trait.
57///
58/// From this [`VTable`] trait, we derive implementations for the sealed `DynArrayData` trait and the
59/// public [`ArrayPlugin`] registry trait.
60///
61/// The functions defined in these vtable traits will typically document their pre- and
62/// post-conditions. The pre-conditions are validated inside the `DynArrayData` and [`ArrayRef`]
63/// implementations so do not need to be checked in the vtable implementations (for example, index
64/// out of bounds). Post-conditions are validated after invocation of the vtable function and will
65/// panic if violated.
66pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug {
67 /// Per-array data owned by this encoding, excluding child arrays.
68 ///
69 /// Child arrays belong in [`ArrayParts::slots`](crate::ArrayParts::slots) so traversal,
70 /// serialization, and layout writers can discover them generically.
71 type TypedArrayData: 'static + Send + Sync + Clone + Debug + Display + ArrayHash + ArrayEq;
72
73 /// Scalar and element-wise operation hooks for this encoding.
74 type OperationsVTable: OperationsVTable<Self>;
75 /// Validity hook for nullable instances of this encoding.
76 type ValidityVTable: ValidityVTable<Self>;
77
78 /// Returns the ID of the array.
79 fn id(&self) -> ArrayId;
80
81 /// Validates that externally supplied logical metadata matches the array data.
82 ///
83 /// This is called by [`Array::try_from_parts`](crate::Array::try_from_parts) before the array
84 /// is published. Implementations should check dtype, length, slot count, child dtypes/lengths,
85 /// metadata bounds, and any buffer shape invariants that unsafe accessors depend on.
86 fn validate(
87 &self,
88 data: &Self::TypedArrayData,
89 dtype: &DType,
90 len: usize,
91 slots: &[Option<ArrayRef>],
92 ) -> VortexResult<()>;
93
94 /// Returns the number of top-level buffers in the array.
95 fn nbuffers(array: ArrayView<'_, Self>) -> usize;
96
97 /// Returns the buffer at the given index.
98 ///
99 /// # Panics
100 /// Panics if `idx >= nbuffers(array)`.
101 fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle;
102
103 /// Returns the name of the buffer at the given index, or `None` if unnamed.
104 fn buffer_name(array: ArrayView<'_, Self>, idx: usize) -> Option<String>;
105
106 /// Rebuild this array with replacement top-level buffers.
107 ///
108 /// This is for physical rewrites that preserve `dtype`, `len`, child slots, buffer count, and
109 /// buffer lengths. The caller checks the generic invariants before dispatching here;
110 /// implementations should interpret the replacement buffers for their encoding-specific
111 /// in-memory representation.
112 fn with_buffers(
113 &self,
114 array: ArrayView<'_, Self>,
115 buffers: &[BufferHandle],
116 ) -> VortexResult<ArrayParts<Self>>;
117
118 /// Serialize encoding metadata into a byte buffer for IPC or file storage.
119 ///
120 /// Buffers and children are serialized separately through [`buffer`](Self::buffer),
121 /// [`nbuffers`](Self::nbuffers), and child traversal.
122 ///
123 /// Return `Ok(None)` if this encoding cannot be serialized. This includes in-memory-only
124 /// encodings and encodings whose wire format is owned by a separate [`ArrayPlugin`]. Do not
125 /// return an error for this case: the caller decides how to report an unserializable array.
126 /// Return an error only when an array that can be serialized fails to serialize.
127 fn serialize(
128 array: ArrayView<'_, Self>,
129 session: &VortexSession,
130 ) -> VortexResult<Option<Vec<u8>>>;
131
132 /// Deserialize an array from serialized metadata, buffers, and children.
133 ///
134 /// The returned [`ArrayParts`] are still validated by the generic adapter.
135 /// Deserializers should use the provided `session` to resolve plugin-owned metadata instead of
136 /// relying on global state.
137 fn deserialize(
138 &self,
139 dtype: &DType,
140 len: usize,
141 metadata: &[u8],
142 buffers: &[BufferHandle],
143 children: &dyn ArrayChildren,
144 session: &VortexSession,
145 ) -> VortexResult<ArrayParts<Self>>;
146
147 /// Writes the array's logical values into a canonical builder.
148 ///
149 /// The default implementation executes the full array to [`Canonical`] and appends that result.
150 /// Encodings may override this to avoid materializing an intermediate canonical array.
151 fn append_to_builder(
152 array: ArrayView<'_, Self>,
153 builder: &mut dyn ArrayBuilder,
154 ctx: &mut ExecutionCtx,
155 ) -> VortexResult<()> {
156 let canonical = array
157 .array()
158 .clone()
159 .execute::<Canonical>(ctx)?
160 .into_array();
161 canonical.append_to_builder(builder, ctx)
162 }
163
164 /// Returns the name of the slot at the given index.
165 ///
166 /// # Panics
167 /// Panics if `idx >= slots(array).len()`.
168 fn slot_name(array: ArrayView<'_, Self>, idx: usize) -> String;
169
170 /// Execute this array by returning an [`ExecutionResult`].
171 ///
172 /// Execution is **iterative**, not recursive. Instead of recursively executing children,
173 /// implementations should return [`ExecutionResult::execute_slot`] to request that the
174 /// scheduler execute a slot first, or [`ExecutionResult::done`] when the encoding can
175 /// produce a result directly.
176 ///
177 /// For good examples of this pattern, see:
178 /// - [`Dict::execute`](crate::arrays::dict::vtable::Dict::execute) — demonstrates
179 /// requiring children via `require_child!` and producing a result once they are canonical.
180 /// - `BitPacked::execute` (in `vortex-fastlanes`) — demonstrates requiring patches and
181 /// validity via `require_patches!`/`require_validity!`.
182 ///
183 /// Array execution is designed such that repeated execution of an array will eventually
184 /// converge to a canonical representation. Implementations of this function should therefore
185 /// ensure they make progress towards that goal.
186 ///
187 /// The returned array (in `Done`) must be logically equivalent to the input array. In other
188 /// words, the recursively canonicalized forms of both arrays must be equal.
189 ///
190 /// Debug builds will panic if the returned array is of the wrong type, wrong length, or
191 /// incorrectly contains null values.
192 fn execute(array: Array<Self>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult>;
193
194 /// Attempt to reduce the array to a simpler representation without changing logical values.
195 ///
196 /// Reductions are opportunistic and may return `Ok(None)` when no cheaper representation is
197 /// known.
198 fn reduce(array: ArrayView<'_, Self>) -> VortexResult<Option<ArrayRef>> {
199 _ = array;
200 Ok(None)
201 }
202
203 /// Attempt to reduce `parent` after this array appears as one of its children.
204 ///
205 /// This is used by lazy arrays to let child execution unlock parent simplifications.
206 fn reduce_parent(
207 array: ArrayView<'_, Self>,
208 parent: &ArrayRef,
209 child_idx: usize,
210 ) -> VortexResult<Option<ArrayRef>> {
211 _ = (array, parent, child_idx);
212 Ok(None)
213 }
214}
215
216/// Alias for migration — downstream code can start using `ArrayVTable`.
217pub use VTable as ArrayVTable;
218
219use crate::array::ArrayId;
220use crate::array::ArrayParts;
221
222/// Empty array metadata struct for encodings with no per-array metadata.
223#[derive(Clone, Debug, Default)]
224pub struct EmptyArrayData;
225
226impl ArrayEq for EmptyArrayData {
227 fn array_eq(&self, _other: &Self, _accuracy: EqMode) -> bool {
228 true
229 }
230}
231impl ArrayHash for EmptyArrayData {
232 fn array_hash<H: Hasher>(&self, _state: &mut H, _accuracy: EqMode) {}
233}
234
235impl Display for EmptyArrayData {
236 fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result {
237 Ok(())
238 }
239}
240
241/// Rebuild an array that has no top-level buffers.
242#[inline]
243pub fn with_empty_buffers<V: VTable>(
244 vtable: &V,
245 array: ArrayView<'_, V>,
246 buffers: &[BufferHandle],
247) -> VortexResult<ArrayParts<V>> {
248 vortex_ensure!(
249 buffers.is_empty(),
250 "Array {} expects 0 buffers, got {}",
251 array.encoding_id(),
252 buffers.len()
253 );
254 Ok(ArrayParts::new(
255 vtable.clone(),
256 array.dtype().clone(),
257 array.len(),
258 array.data().clone(),
259 array.slots().iter().cloned().collect(),
260 ))
261}
262
263/// Reject buffer replacement for encodings whose exposed buffers are not runtime backing buffers.
264#[inline]
265pub fn unsupported_buffer_replacement<V: VTable>(
266 array: ArrayView<'_, V>,
267 _buffers: &[BufferHandle],
268) -> VortexResult<ArrayParts<V>> {
269 vortex_bail!(
270 "Array {} does not support in-memory buffer replacement",
271 array.encoding_id()
272 )
273}
274
275/// Placeholder type used to indicate when a particular vtable is not supported by the encoding.
276pub struct NotSupported;
277
278/// Returns the validity as a child array if it produces one.
279#[inline]
280pub fn validity_to_child(validity: &Validity, len: usize) -> Option<ArrayRef> {
281 match validity {
282 Validity::NonNullable | Validity::AllValid => None,
283 Validity::AllInvalid => Some(ConstantArray::new(false, len).into_array()),
284 Validity::Array(array) => Some(array.clone()),
285 }
286}
287
288/// Reconstruct a [`Validity`] from an optional child array and nullability.
289///
290/// This is the inverse of [`validity_to_child`].
291#[inline]
292pub fn child_to_validity(child: Option<&ArrayRef>, nullability: Nullability) -> Validity {
293 match child {
294 Some(arr) => {
295 // Detect constant bool arrays created by validity_to_child.
296 // Use direct ScalarValue matching to avoid expensive scalar conversion.
297 if let Some(c) = arr.as_opt::<Constant>()
298 && let Some(ScalarValue::Bool(val)) = c.scalar().value()
299 {
300 return if *val {
301 Validity::AllValid
302 } else {
303 Validity::AllInvalid
304 };
305 }
306 Validity::Array(arr.clone())
307 }
308 None => Validity::from(nullability),
309 }
310}
311
312/// Returns 1 if validity produces a child, 0 otherwise.
313#[inline]
314pub fn validity_nchildren(validity: &Validity) -> usize {
315 match validity {
316 Validity::NonNullable | Validity::AllValid => 0,
317 Validity::AllInvalid | Validity::Array(_) => 1,
318 }
319}
320
321/// Returns the number of children produced by patches.
322#[inline]
323pub fn patches_nchildren(patches: &Patches) -> usize {
324 2 + patches.chunk_offsets().is_some() as usize
325}
326
327/// Returns the child at the given index within a patches component.
328#[inline]
329pub fn patches_child(patches: &Patches, idx: usize) -> ArrayRef {
330 match idx {
331 0 => patches.indices().clone(),
332 1 => patches.values().clone(),
333 2 => patches
334 .chunk_offsets()
335 .as_ref()
336 .vortex_expect("patch_chunk_offsets child out of bounds")
337 .clone(),
338 _ => vortex_panic!("patches child index {idx} out of bounds"),
339 }
340}
341
342/// Returns the name of the child at the given index within a patches component.
343#[inline]
344pub fn patches_child_name(idx: usize) -> &'static str {
345 match idx {
346 0 => "patch_indices",
347 1 => "patch_values",
348 2 => "patch_chunk_offsets",
349 _ => vortex_panic!("patches child name index {idx} out of bounds"),
350 }
351}