Skip to main content

vortex_array/array/
plugin.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::fmt;
5use std::fmt::Debug;
6use std::fmt::Formatter;
7use std::sync::Arc;
8
9use vortex_buffer::ByteBuffer;
10use vortex_error::VortexResult;
11use vortex_error::vortex_ensure_eq;
12use vortex_session::VortexSession;
13
14use crate::ArrayRef;
15use crate::IntoArray;
16use crate::array::Array;
17use crate::array::ArrayId;
18use crate::array::VTable;
19use crate::buffer::BufferHandle;
20use crate::dtype::DType;
21use crate::serde::ArrayChildren;
22
23/// Reference-counted array plugin.
24pub type ArrayPluginRef = Arc<dyn ArrayPlugin>;
25
26/// The wire representation produced by an in-memory array's serializer.
27///
28/// A serializer may reuse the in-memory array's buffers and children with [`Self::from_array`],
29/// or return the wire representation of another encoding. The node must hold the logical values
30/// of the array under the dtype supplied by the containing format.
31#[derive(Clone, Debug)]
32pub struct ArraySerialization {
33    /// The concrete array ID to write on the wire.
34    pub serialized_id: ArrayId,
35    /// Encoding-specific metadata written into the array node.
36    pub metadata: Vec<u8>,
37    /// Top-level buffers written for this array node.
38    pub buffers: Vec<ByteBuffer>,
39    /// Child arrays to serialize recursively.
40    pub children: Vec<ArrayRef>,
41}
42
43impl ArraySerialization {
44    /// Create a wire representation from an ID, metadata, buffers, and children.
45    pub fn new(
46        serialized_id: ArrayId,
47        metadata: Vec<u8>,
48        buffers: Vec<ByteBuffer>,
49        children: Vec<ArrayRef>,
50    ) -> Self {
51        Self {
52            serialized_id,
53            metadata,
54            buffers,
55            children,
56        }
57    }
58
59    /// Reuse an in-memory array's buffers and children with the supplied serialized metadata.
60    pub fn from_array(serialized_id: ArrayId, array: &ArrayRef, metadata: Vec<u8>) -> Self {
61        Self::new(serialized_id, metadata, array.buffers(), array.children())
62    }
63}
64
65/// The borrowed wire components passed to an array deserializer.
66pub struct ArrayDeserialization<'a> {
67    /// The exact array ID found on the wire.
68    pub serialized_id: ArrayId,
69    /// The logical dtype supplied by the containing format.
70    pub dtype: &'a DType,
71    /// The logical array length supplied by the containing format.
72    pub len: usize,
73    /// Encoding-specific metadata from the array node.
74    pub metadata: &'a [u8],
75    /// Top-level buffers referenced by the array node.
76    pub buffers: &'a [BufferHandle],
77    /// Lazily decoded child arrays referenced by the array node.
78    pub children: &'a dyn ArrayChildren,
79}
80
81impl<'a> ArrayDeserialization<'a> {
82    /// Create borrowed deserialization input from a wire ID and its serialized components.
83    pub fn new(
84        serialized_id: ArrayId,
85        dtype: &'a DType,
86        len: usize,
87        metadata: &'a [u8],
88        buffers: &'a [BufferHandle],
89        children: &'a dyn ArrayChildren,
90    ) -> Self {
91        Self {
92            serialized_id,
93            dtype,
94            len,
95            metadata,
96            buffers,
97            children,
98        }
99    }
100}
101
102/// Registry trait for serializing and deserializing an in-memory array representation.
103///
104/// A plugin writes one in-memory encoding, identified by [`id`](Self::id). It declares the wire
105/// IDs that it reads and writes separately, so it can write another encoding's wire form without
106/// replacing that encoding's reader. The serialization context validates that the chosen wire ID
107/// is permitted before it is written.
108///
109/// A plugin can retain historical read IDs after it stops writing them. A reader that predates a
110/// newer ID has no registration for it and reports it as unknown.
111pub trait ArrayPlugin: 'static + Send + Sync {
112    /// Returns the ID of the in-memory encoding that this plugin writes.
113    fn id(&self) -> ArrayId;
114
115    /// Returns the wire IDs that [`deserialize`](Self::deserialize) accepts, oldest to newest.
116    ///
117    /// [`ArraySession::register`](crate::session::ArraySession::register) makes this plugin the
118    /// reader of each ID. An ID can stay in this list after the plugin stops writing it, so that
119    /// old files stay readable.
120    fn read_ids(&self) -> Vec<ArrayId>;
121
122    /// Returns the wire IDs that [`serialize`](Self::serialize) may return.
123    ///
124    /// An ID in this list does not have to be in [`read_ids`](Self::read_ids). A plugin can write
125    /// its arrays in the wire form of another encoding. The plugin that owns that form reads it.
126    /// The node must hold the logical values of the array under the dtype of the array.
127    fn write_ids(&self) -> Vec<ArrayId>;
128
129    /// Serialize `array` to its wire representation.
130    ///
131    /// This function is called only for arrays whose in-memory encoding matches [`id`](Self::id).
132    /// The returned ID must be declared by [`write_ids`](Self::write_ids). Return
133    /// `Ok(None)` when the array cannot be serialized.
134    fn serialize(
135        &self,
136        array: &ArrayRef,
137        session: &VortexSession,
138    ) -> VortexResult<Option<ArraySerialization>>;
139
140    /// Deserialize one recognized wire representation into the current in-memory array.
141    ///
142    /// `serialized_id` identifies the exact representation encountered on disk. The returned
143    /// array does not necessarily have to use this plugin's in-memory ID; this supports legacy
144    /// representations that are normalized into another current in-memory array. Implementations
145    /// must validate the contract of that exact ID rather than accepting every form understood by
146    /// the current in-memory representation under an older ID.
147    fn deserialize(
148        &self,
149        parts: ArrayDeserialization<'_>,
150        session: &VortexSession,
151    ) -> VortexResult<ArrayRef>;
152
153    /// Can this plugin emit an array with the given encoding.
154    ///
155    /// By default, this is just the [ID][Self::id] of the plugin, but
156    /// can be overridden if this plugin instance supports reading/writing multiple arrays.
157    fn is_supported_encoding(&self, id: &ArrayId) -> bool {
158        self.id() == *id
159    }
160}
161
162impl Debug for dyn ArrayPlugin {
163    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
164        f.debug_tuple("ArrayPlugin").field(&self.id()).finish()
165    }
166}
167
168impl<V: VTable> ArrayPlugin for V {
169    fn id(&self) -> ArrayId {
170        VTable::id(self)
171    }
172
173    fn read_ids(&self) -> Vec<ArrayId> {
174        vec![self.id()]
175    }
176
177    fn write_ids(&self) -> Vec<ArrayId> {
178        vec![self.id()]
179    }
180
181    fn serialize(
182        &self,
183        array: &ArrayRef,
184        session: &VortexSession,
185    ) -> VortexResult<Option<ArraySerialization>> {
186        vortex_ensure_eq!(
187            self.id(),
188            array.encoding_id(),
189            "array plugin cannot serialize an in-memory array of another encoding",
190        );
191        Ok(V::serialize(array.as_::<V>(), session)?
192            .map(|metadata| ArraySerialization::from_array(self.id(), array, metadata)))
193    }
194
195    fn deserialize(
196        &self,
197        parts: ArrayDeserialization<'_>,
198        session: &VortexSession,
199    ) -> VortexResult<ArrayRef> {
200        vortex_ensure_eq!(
201            self.id(),
202            parts.serialized_id,
203            "array plugin does not recognize the serialized ID",
204        );
205        Ok(Array::<V>::try_from_parts(V::deserialize(
206            self,
207            parts.dtype,
208            parts.len,
209            parts.metadata,
210            parts.buffers,
211            parts.children,
212            session,
213        )?)?
214        .into_array())
215    }
216}