Skip to main content

vortex_array/array/vtable/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! This module contains the VTable definitions for a Vortex encoding.
5//!
6//! A Vortex array encoding is implemented by a small static vtable type plus an associated
7//! `TypedArrayData` value stored in each array instance. The vtable owns behavior such as
8//! validation, serialization, execution, child traversal, scalar access, and validity access.
9//!
10//! The public [`ArrayRef`] API performs common precondition checks before calling
11//! into these traits. Implementations should focus on encoding-specific work and uphold the
12//! documented postconditions.
13
14mod operations;
15mod validity;
16
17use std::fmt::Debug;
18use std::fmt::Display;
19use std::fmt::Formatter;
20use std::hash::Hasher;
21
22pub use operations::*;
23pub use validity::*;
24use vortex_error::VortexExpect;
25use vortex_error::VortexResult;
26use vortex_error::vortex_bail;
27use vortex_error::vortex_ensure;
28use vortex_error::vortex_panic;
29use vortex_session::VortexSession;
30
31use crate::Array;
32use crate::ArrayRef;
33use crate::ArrayView;
34use crate::Canonical;
35use crate::EqMode;
36use crate::ExecutionResult;
37use crate::IntoArray;
38pub use crate::array::plugin::*;
39use crate::arrays::ConstantArray;
40use crate::arrays::constant::Constant;
41use crate::buffer::BufferHandle;
42use crate::builders::ArrayBuilder;
43use crate::dtype::DType;
44use crate::dtype::Nullability;
45use crate::executor::ExecutionCtx;
46use crate::hash::ArrayEq;
47use crate::hash::ArrayHash;
48use crate::patches::Patches;
49use crate::scalar::ScalarValue;
50use crate::serde::ArrayChildren;
51use crate::validity::Validity;
52
53/// The array [`VTable`] encapsulates logic for an Array type within Vortex.
54///
55/// The logic is split across several "VTable" traits to enable easier code organization than
56/// simply lumping everything into a single trait.
57///
58/// From this [`VTable`] trait, we derive implementations for the sealed `DynArrayData` trait and the
59/// public [`ArrayPlugin`] registry trait.
60///
61/// The functions defined in these vtable traits will typically document their pre- and
62/// post-conditions. The pre-conditions are validated inside the `DynArrayData` and [`ArrayRef`]
63/// implementations so do not need to be checked in the vtable implementations (for example, index
64/// out of bounds). Post-conditions are validated after invocation of the vtable function and will
65/// panic if violated.
66pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug {
67    /// Per-array data owned by this encoding, excluding child arrays.
68    ///
69    /// Child arrays belong in [`ArrayParts::slots`](crate::ArrayParts::slots) so traversal,
70    /// serialization, and layout writers can discover them generically.
71    type TypedArrayData: 'static + Send + Sync + Clone + Debug + Display + ArrayHash + ArrayEq;
72
73    /// Scalar and element-wise operation hooks for this encoding.
74    type OperationsVTable: OperationsVTable<Self>;
75    /// Validity hook for nullable instances of this encoding.
76    type ValidityVTable: ValidityVTable<Self>;
77
78    /// Returns the ID of the array.
79    fn id(&self) -> ArrayId;
80
81    /// Validates that externally supplied logical metadata matches the array data.
82    ///
83    /// This is called by [`Array::try_from_parts`](crate::Array::try_from_parts) before the array
84    /// is published. Implementations should check dtype, length, slot count, child dtypes/lengths,
85    /// metadata bounds, and any buffer shape invariants that unsafe accessors depend on.
86    fn validate(
87        &self,
88        data: &Self::TypedArrayData,
89        dtype: &DType,
90        len: usize,
91        slots: &[Option<ArrayRef>],
92    ) -> VortexResult<()>;
93
94    /// Returns the number of top-level buffers in the array.
95    fn nbuffers(array: ArrayView<'_, Self>) -> usize;
96
97    /// Returns the buffer at the given index.
98    ///
99    /// # Panics
100    /// Panics if `idx >= nbuffers(array)`.
101    fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle;
102
103    /// Returns the name of the buffer at the given index, or `None` if unnamed.
104    fn buffer_name(array: ArrayView<'_, Self>, idx: usize) -> Option<String>;
105
106    /// Rebuild this array with replacement top-level buffers.
107    ///
108    /// This is for physical rewrites that preserve `dtype`, `len`, child slots, buffer count, and
109    /// buffer lengths. The caller checks the generic invariants before dispatching here;
110    /// implementations should interpret the replacement buffers for their encoding-specific
111    /// in-memory representation.
112    fn with_buffers(
113        &self,
114        array: ArrayView<'_, Self>,
115        buffers: &[BufferHandle],
116    ) -> VortexResult<ArrayParts<Self>>;
117
118    /// Serialize encoding metadata into a byte buffer for IPC or file storage.
119    ///
120    /// Buffers and children are serialized separately through [`buffer`](Self::buffer),
121    /// [`nbuffers`](Self::nbuffers), and child traversal.
122    ///
123    /// Return `Ok(None)` if this encoding cannot be serialized. This includes in-memory-only
124    /// encodings and encodings whose wire format is owned by a separate [`ArrayPlugin`]. Do not
125    /// return an error for this case: the caller decides how to report an unserializable array.
126    /// Return an error only when an array that can be serialized fails to serialize.
127    fn serialize(
128        array: ArrayView<'_, Self>,
129        session: &VortexSession,
130    ) -> VortexResult<Option<Vec<u8>>>;
131
132    /// Deserialize an array from serialized metadata, buffers, and children.
133    ///
134    /// The returned [`ArrayParts`] are still validated by the generic adapter.
135    /// Deserializers should use the provided `session` to resolve plugin-owned metadata instead of
136    /// relying on global state.
137    fn deserialize(
138        &self,
139        dtype: &DType,
140        len: usize,
141        metadata: &[u8],
142        buffers: &[BufferHandle],
143        children: &dyn ArrayChildren,
144        session: &VortexSession,
145    ) -> VortexResult<ArrayParts<Self>>;
146
147    /// Writes the array's logical values into a canonical builder.
148    ///
149    /// The default implementation executes the full array to [`Canonical`] and appends that result.
150    /// Encodings may override this to avoid materializing an intermediate canonical array.
151    fn append_to_builder(
152        array: ArrayView<'_, Self>,
153        builder: &mut dyn ArrayBuilder,
154        ctx: &mut ExecutionCtx,
155    ) -> VortexResult<()> {
156        let canonical = array
157            .array()
158            .clone()
159            .execute::<Canonical>(ctx)?
160            .into_array();
161        canonical.append_to_builder(builder, ctx)
162    }
163
164    /// Returns the name of the slot at the given index.
165    ///
166    /// # Panics
167    /// Panics if `idx >= slots(array).len()`.
168    fn slot_name(array: ArrayView<'_, Self>, idx: usize) -> String;
169
170    /// Execute this array by returning an [`ExecutionResult`].
171    ///
172    /// Execution is **iterative**, not recursive. Instead of recursively executing children,
173    /// implementations should return [`ExecutionResult::execute_slot`] to request that the
174    /// scheduler execute a slot first, or [`ExecutionResult::done`] when the encoding can
175    /// produce a result directly.
176    ///
177    /// For good examples of this pattern, see:
178    /// - [`Dict::execute`](crate::arrays::dict::vtable::Dict::execute) — demonstrates
179    ///   requiring children via `require_child!` and producing a result once they are canonical.
180    /// - `BitPacked::execute` (in `vortex-fastlanes`) — demonstrates requiring patches and
181    ///   validity via `require_patches!`/`require_validity!`.
182    ///
183    /// Array execution is designed such that repeated execution of an array will eventually
184    /// converge to a canonical representation. Implementations of this function should therefore
185    /// ensure they make progress towards that goal.
186    ///
187    /// The returned array (in `Done`) must be logically equivalent to the input array. In other
188    /// words, the recursively canonicalized forms of both arrays must be equal.
189    ///
190    /// Debug builds will panic if the returned array is of the wrong type, wrong length, or
191    /// incorrectly contains null values.
192    fn execute(array: Array<Self>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult>;
193
194    /// Attempt to reduce the array to a simpler representation without changing logical values.
195    ///
196    /// Reductions are opportunistic and may return `Ok(None)` when no cheaper representation is
197    /// known.
198    fn reduce(array: ArrayView<'_, Self>) -> VortexResult<Option<ArrayRef>> {
199        _ = array;
200        Ok(None)
201    }
202
203    /// Attempt to reduce `parent` after this array appears as one of its children.
204    ///
205    /// This is used by lazy arrays to let child execution unlock parent simplifications.
206    fn reduce_parent(
207        array: ArrayView<'_, Self>,
208        parent: &ArrayRef,
209        child_idx: usize,
210    ) -> VortexResult<Option<ArrayRef>> {
211        _ = (array, parent, child_idx);
212        Ok(None)
213    }
214}
215
216/// Alias for migration — downstream code can start using `ArrayVTable`.
217pub use VTable as ArrayVTable;
218
219use crate::array::ArrayId;
220use crate::array::ArrayParts;
221
222/// Empty array metadata struct for encodings with no per-array metadata.
223#[derive(Clone, Debug, Default)]
224pub struct EmptyArrayData;
225
226impl ArrayEq for EmptyArrayData {
227    fn array_eq(&self, _other: &Self, _accuracy: EqMode) -> bool {
228        true
229    }
230}
231impl ArrayHash for EmptyArrayData {
232    fn array_hash<H: Hasher>(&self, _state: &mut H, _accuracy: EqMode) {}
233}
234
235impl Display for EmptyArrayData {
236    fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result {
237        Ok(())
238    }
239}
240
241/// Rebuild an array that has no top-level buffers.
242#[inline]
243pub fn with_empty_buffers<V: VTable>(
244    vtable: &V,
245    array: ArrayView<'_, V>,
246    buffers: &[BufferHandle],
247) -> VortexResult<ArrayParts<V>> {
248    vortex_ensure!(
249        buffers.is_empty(),
250        "Array {} expects 0 buffers, got {}",
251        array.encoding_id(),
252        buffers.len()
253    );
254    Ok(ArrayParts::new(
255        vtable.clone(),
256        array.dtype().clone(),
257        array.len(),
258        array.data().clone(),
259        array.slots().iter().cloned().collect(),
260    ))
261}
262
263/// Reject buffer replacement for encodings whose exposed buffers are not runtime backing buffers.
264#[inline]
265pub fn unsupported_buffer_replacement<V: VTable>(
266    array: ArrayView<'_, V>,
267    _buffers: &[BufferHandle],
268) -> VortexResult<ArrayParts<V>> {
269    vortex_bail!(
270        "Array {} does not support in-memory buffer replacement",
271        array.encoding_id()
272    )
273}
274
275/// Placeholder type used to indicate when a particular vtable is not supported by the encoding.
276pub struct NotSupported;
277
278/// Returns the validity as a child array if it produces one.
279#[inline]
280pub fn validity_to_child(validity: &Validity, len: usize) -> Option<ArrayRef> {
281    match validity {
282        Validity::NonNullable | Validity::AllValid => None,
283        Validity::AllInvalid => Some(ConstantArray::new(false, len).into_array()),
284        Validity::Array(array) => Some(array.clone()),
285    }
286}
287
288/// Reconstruct a [`Validity`] from an optional child array and nullability.
289///
290/// This is the inverse of [`validity_to_child`].
291#[inline]
292pub fn child_to_validity(child: Option<&ArrayRef>, nullability: Nullability) -> Validity {
293    match child {
294        Some(arr) => {
295            // Detect constant bool arrays created by validity_to_child.
296            // Use direct ScalarValue matching to avoid expensive scalar conversion.
297            if let Some(c) = arr.as_opt::<Constant>()
298                && let Some(ScalarValue::Bool(val)) = c.scalar().value()
299            {
300                return if *val {
301                    Validity::AllValid
302                } else {
303                    Validity::AllInvalid
304                };
305            }
306            Validity::Array(arr.clone())
307        }
308        None => Validity::from(nullability),
309    }
310}
311
312/// Returns 1 if validity produces a child, 0 otherwise.
313#[inline]
314pub fn validity_nchildren(validity: &Validity) -> usize {
315    match validity {
316        Validity::NonNullable | Validity::AllValid => 0,
317        Validity::AllInvalid | Validity::Array(_) => 1,
318    }
319}
320
321/// Returns the number of children produced by patches.
322#[inline]
323pub fn patches_nchildren(patches: &Patches) -> usize {
324    2 + patches.chunk_offsets().is_some() as usize
325}
326
327/// Returns the child at the given index within a patches component.
328#[inline]
329pub fn patches_child(patches: &Patches, idx: usize) -> ArrayRef {
330    match idx {
331        0 => patches.indices().clone(),
332        1 => patches.values().clone(),
333        2 => patches
334            .chunk_offsets()
335            .as_ref()
336            .vortex_expect("patch_chunk_offsets child out of bounds")
337            .clone(),
338        _ => vortex_panic!("patches child index {idx} out of bounds"),
339    }
340}
341
342/// Returns the name of the child at the given index within a patches component.
343#[inline]
344pub fn patches_child_name(idx: usize) -> &'static str {
345    match idx {
346        0 => "patch_indices",
347        1 => "patch_values",
348        2 => "patch_chunk_offsets",
349        _ => vortex_panic!("patches child name index {idx} out of bounds"),
350    }
351}