vortex_array/array/plugin.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::fmt;
5use std::fmt::Debug;
6use std::fmt::Formatter;
7use std::sync::Arc;
8
9use vortex_buffer::ByteBuffer;
10use vortex_error::VortexResult;
11use vortex_error::vortex_ensure_eq;
12use vortex_session::VortexSession;
13
14use crate::ArrayRef;
15use crate::IntoArray;
16use crate::array::Array;
17use crate::array::ArrayId;
18use crate::array::VTable;
19use crate::buffer::BufferHandle;
20use crate::dtype::DType;
21use crate::serde::ArrayChildren;
22
23/// Reference-counted array plugin.
24pub type ArrayPluginRef = Arc<dyn ArrayPlugin>;
25
26/// The wire representation produced by an in-memory array's serializer.
27///
28/// A serializer may reuse the in-memory array's buffers and children with [`Self::from_array`],
29/// or return the wire representation of another encoding. The node must hold the logical values
30/// of the array under the dtype supplied by the containing format.
31#[derive(Clone, Debug)]
32pub struct ArraySerialization {
33 /// The concrete array ID to write on the wire.
34 pub serialized_id: ArrayId,
35 /// Encoding-specific metadata written into the array node.
36 pub metadata: Vec<u8>,
37 /// Top-level buffers written for this array node.
38 pub buffers: Vec<ByteBuffer>,
39 /// Child arrays to serialize recursively.
40 pub children: Vec<ArrayRef>,
41}
42
43impl ArraySerialization {
44 /// Create a wire representation from an ID, metadata, buffers, and children.
45 pub fn new(
46 serialized_id: ArrayId,
47 metadata: Vec<u8>,
48 buffers: Vec<ByteBuffer>,
49 children: Vec<ArrayRef>,
50 ) -> Self {
51 Self {
52 serialized_id,
53 metadata,
54 buffers,
55 children,
56 }
57 }
58
59 /// Reuse an in-memory array's buffers and children with the supplied serialized metadata.
60 pub fn from_array(serialized_id: ArrayId, array: &ArrayRef, metadata: Vec<u8>) -> Self {
61 Self::new(serialized_id, metadata, array.buffers(), array.children())
62 }
63}
64
65/// The borrowed wire components passed to an array deserializer.
66pub struct ArrayDeserialization<'a> {
67 /// The exact array ID found on the wire.
68 pub serialized_id: ArrayId,
69 /// The logical dtype supplied by the containing format.
70 pub dtype: &'a DType,
71 /// The logical array length supplied by the containing format.
72 pub len: usize,
73 /// Encoding-specific metadata from the array node.
74 pub metadata: &'a [u8],
75 /// Top-level buffers referenced by the array node.
76 pub buffers: &'a [BufferHandle],
77 /// Lazily decoded child arrays referenced by the array node.
78 pub children: &'a dyn ArrayChildren,
79}
80
81impl<'a> ArrayDeserialization<'a> {
82 /// Create borrowed deserialization input from a wire ID and its serialized components.
83 pub fn new(
84 serialized_id: ArrayId,
85 dtype: &'a DType,
86 len: usize,
87 metadata: &'a [u8],
88 buffers: &'a [BufferHandle],
89 children: &'a dyn ArrayChildren,
90 ) -> Self {
91 Self {
92 serialized_id,
93 dtype,
94 len,
95 metadata,
96 buffers,
97 children,
98 }
99 }
100}
101
102/// Registry trait for serializing and deserializing an in-memory array representation.
103///
104/// A plugin writes one in-memory encoding, identified by [`id`](Self::id). It declares the wire
105/// IDs that it reads and writes separately, so it can write another encoding's wire form without
106/// replacing that encoding's reader. The serialization context validates that the chosen wire ID
107/// is permitted before it is written.
108///
109/// A plugin can retain historical read IDs after it stops writing them. A reader that predates a
110/// newer ID has no registration for it and reports it as unknown.
111pub trait ArrayPlugin: 'static + Send + Sync {
112 /// Returns the ID of the in-memory encoding that this plugin writes.
113 fn id(&self) -> ArrayId;
114
115 /// Returns the wire IDs that [`deserialize`](Self::deserialize) accepts, oldest to newest.
116 ///
117 /// [`ArraySession::register`](crate::session::ArraySession::register) makes this plugin the
118 /// reader of each ID. An ID can stay in this list after the plugin stops writing it, so that
119 /// old files stay readable.
120 fn read_ids(&self) -> Vec<ArrayId>;
121
122 /// Returns the wire IDs that [`serialize`](Self::serialize) may return.
123 ///
124 /// An ID in this list does not have to be in [`read_ids`](Self::read_ids). A plugin can write
125 /// its arrays in the wire form of another encoding. The plugin that owns that form reads it.
126 /// The node must hold the logical values of the array under the dtype of the array.
127 fn write_ids(&self) -> Vec<ArrayId>;
128
129 /// Serialize `array` to its wire representation.
130 ///
131 /// This function is called only for arrays whose in-memory encoding matches [`id`](Self::id).
132 /// The returned ID must be declared by [`write_ids`](Self::write_ids). Return
133 /// `Ok(None)` when the array cannot be serialized.
134 fn serialize(
135 &self,
136 array: &ArrayRef,
137 session: &VortexSession,
138 ) -> VortexResult<Option<ArraySerialization>>;
139
140 /// Deserialize one recognized wire representation into the current in-memory array.
141 ///
142 /// `serialized_id` identifies the exact representation encountered on disk. The returned
143 /// array does not necessarily have to use this plugin's in-memory ID; this supports legacy
144 /// representations that are normalized into another current in-memory array. Implementations
145 /// must validate the contract of that exact ID rather than accepting every form understood by
146 /// the current in-memory representation under an older ID.
147 fn deserialize(
148 &self,
149 parts: ArrayDeserialization<'_>,
150 session: &VortexSession,
151 ) -> VortexResult<ArrayRef>;
152
153 /// Can this plugin emit an array with the given encoding.
154 ///
155 /// By default, this is just the [ID][Self::id] of the plugin, but
156 /// can be overridden if this plugin instance supports reading/writing multiple arrays.
157 fn is_supported_encoding(&self, id: &ArrayId) -> bool {
158 self.id() == *id
159 }
160}
161
162impl Debug for dyn ArrayPlugin {
163 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
164 f.debug_tuple("ArrayPlugin").field(&self.id()).finish()
165 }
166}
167
168impl<V: VTable> ArrayPlugin for V {
169 fn id(&self) -> ArrayId {
170 VTable::id(self)
171 }
172
173 fn read_ids(&self) -> Vec<ArrayId> {
174 vec![self.id()]
175 }
176
177 fn write_ids(&self) -> Vec<ArrayId> {
178 vec![self.id()]
179 }
180
181 fn serialize(
182 &self,
183 array: &ArrayRef,
184 session: &VortexSession,
185 ) -> VortexResult<Option<ArraySerialization>> {
186 vortex_ensure_eq!(
187 self.id(),
188 array.encoding_id(),
189 "array plugin cannot serialize an in-memory array of another encoding",
190 );
191 Ok(V::serialize(array.as_::<V>(), session)?
192 .map(|metadata| ArraySerialization::from_array(self.id(), array, metadata)))
193 }
194
195 fn deserialize(
196 &self,
197 parts: ArrayDeserialization<'_>,
198 session: &VortexSession,
199 ) -> VortexResult<ArrayRef> {
200 vortex_ensure_eq!(
201 self.id(),
202 parts.serialized_id,
203 "array plugin does not recognize the serialized ID",
204 );
205 Ok(Array::<V>::try_from_parts(V::deserialize(
206 self,
207 parts.dtype,
208 parts.len,
209 parts.metadata,
210 parts.buffers,
211 parts.children,
212 session,
213 )?)?
214 .into_array())
215 }
216}