Skip to main content

vortex_array/
serde.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::borrow::Cow;
5use std::fmt::Debug;
6use std::fmt::Formatter;
7use std::iter;
8use std::sync::Arc;
9
10use flatbuffers::FlatBufferBuilder;
11use flatbuffers::Follow;
12use flatbuffers::WIPOffset;
13use flatbuffers::root;
14use vortex_buffer::Alignment;
15use vortex_buffer::ByteBuffer;
16use vortex_error::VortexError;
17use vortex_error::VortexExpect;
18use vortex_error::VortexResult;
19use vortex_error::vortex_bail;
20use vortex_error::vortex_err;
21use vortex_error::vortex_panic;
22use vortex_session::VortexSession;
23use vortex_session::registry::ReadContext;
24use vortex_utils::aliases::hash_map::HashMap;
25
26use crate::ArrayContext;
27use crate::ArrayRef;
28use crate::ArraySlots;
29use crate::array::ArrayDeserialization;
30use crate::array::ArrayId;
31use crate::array::new_foreign_array;
32use crate::buffer::BufferHandle;
33use crate::dtype::DType;
34use crate::dtype::TryFromBytes;
35use crate::expr::stats::Precision;
36use crate::expr::stats::Stat;
37use crate::flatbuffers::FlatBuffer;
38use crate::flatbuffers::WriteFlatBuffer;
39use crate::flatbuffers::array as fba;
40use crate::flatbuffers::array::Compression;
41use crate::optimizer::mark_decoded_optimized;
42use crate::session::ArraySessionExt;
43use crate::stats::StatsSet;
44
45/// Options for serializing an array.
46#[derive(Default, Debug)]
47pub struct SerializeOptions {
48    /// The starting position within an external stream or file. This offset is used to compute
49    /// appropriate padding to enable zero-copy reads.
50    pub offset: usize,
51    /// Whether to include sufficient zero-copy padding.
52    pub include_padding: bool,
53}
54
55impl ArrayRef {
56    /// Serialize the array into a sequence of byte buffers that should be written contiguously.
57    /// This function returns a vec to avoid copying data buffers.
58    ///
59    /// Optionally, padding can be included to guarantee buffer alignment and ensure zero-copy
60    /// reads within the context of an external file or stream. In this case, the alignment of
61    /// the first byte buffer should be respected when writing the buffers to the stream or file.
62    ///
63    /// The format of this blob is a sequence of data buffers, possible with prefixed padding,
64    /// followed by a flatbuffer containing an [`fba::Array`] message, and ending with a
65    /// little-endian u32 describing the length of the flatbuffer message.
66    pub fn serialize(
67        &self,
68        ctx: &ArrayContext,
69        session: &VortexSession,
70        options: &SerializeOptions,
71    ) -> VortexResult<Vec<ByteBuffer>> {
72        // Resolve the wire representation once. Serializers may choose historical IDs and may
73        // provide downgraded buffers or children that differ from the in-memory array tree.
74        let root = ArrayNodeFlatBuffer::try_new(ctx, session, self)?;
75        let array_buffers = root.array.buffers();
76
77        // Allocate result buffers, including a possible padding buffer for each.
78        let mut buffers = vec![];
79        let mut fb_buffers = Vec::with_capacity(buffers.capacity());
80
81        // If we're including padding, we need to find the maximum required buffer alignment.
82        let max_alignment = array_buffers
83            .iter()
84            .map(|buf| buf.alignment())
85            .chain(iter::once(FlatBuffer::alignment()))
86            .max()
87            .unwrap_or_else(FlatBuffer::alignment);
88
89        // Create a shared buffer of zeros we can use for padding
90        let zeros = ByteBuffer::zeroed(max_alignment.as_usize());
91
92        // We push an empty buffer with the maximum alignment, so then subsequent buffers
93        // will be aligned. For subsequent buffers, we always push a 1-byte alignment.
94        buffers.push(ByteBuffer::zeroed_aligned(0, max_alignment));
95
96        // Keep track of where we are in the "file" to calculate padding.
97        let mut pos = options.offset;
98
99        // Push all the array buffers with padding as necessary.
100        for buffer in array_buffers {
101            let padding = if options.include_padding {
102                let padding = pos.next_multiple_of(buffer.alignment().as_usize()) - pos;
103                if padding > 0 {
104                    pos += padding;
105                    buffers.push(zeros.slice(0..padding));
106                }
107                padding
108            } else {
109                0
110            };
111
112            fb_buffers.push(fba::Buffer::new(
113                u16::try_from(padding).vortex_expect("padding fits into u16"),
114                buffer.alignment().exponent(),
115                Compression::None,
116                u32::try_from(buffer.len())
117                    .map_err(|_| vortex_err!("All buffers must fit into u32 for serialization"))?,
118            ));
119
120            pos += buffer.len();
121            buffers.push(buffer.aligned(Alignment::none()));
122        }
123
124        // Set up the flatbuffer builder
125        let mut fbb = FlatBufferBuilder::new();
126
127        let fb_root = root.try_write_flatbuffer(&mut fbb)?;
128
129        let fb_buffers = fbb.create_vector(&fb_buffers);
130        let fb_array = fba::Array::create(
131            &mut fbb,
132            &fba::ArrayArgs {
133                root: Some(fb_root),
134                buffers: Some(fb_buffers),
135            },
136        );
137        fbb.finish_minimal(fb_array);
138        let (fb_vec, fb_start) = fbb.collapse();
139        let fb_end = fb_vec.len();
140        let fb_buffer = ByteBuffer::from(fb_vec).slice(fb_start..fb_end);
141        let fb_length = fb_buffer.len();
142
143        if options.include_padding {
144            let padding = pos.next_multiple_of(FlatBuffer::alignment().as_usize()) - pos;
145            if padding > 0 {
146                buffers.push(zeros.slice(0..padding));
147            }
148        }
149        buffers.push(fb_buffer);
150
151        // Finally, we write down the u32 length for the flatbuffer.
152        buffers.push(ByteBuffer::from(
153            u32::try_from(fb_length)
154                .map_err(|_| vortex_err!("Array metadata flatbuffer must fit into u32 for serialization. Array encoding tree is too large."))?
155                .to_le_bytes()
156                .to_vec(),
157        ));
158
159        Ok(buffers)
160    }
161}
162
163#[derive(Clone, Debug)]
164struct ArraySerializationTree {
165    source: ArrayRef,
166    serialized_id: ArrayId,
167    metadata: Vec<u8>,
168    buffers: Vec<ByteBuffer>,
169    children: Vec<ArraySerializationTree>,
170}
171
172impl ArraySerializationTree {
173    fn try_new(session: &VortexSession, source: &ArrayRef) -> VortexResult<Self> {
174        let Some(serialization) = session.array_serialize(source)? else {
175            vortex_bail!(
176                "Array {} does not support serialization",
177                source.encoding_id()
178            );
179        };
180        let children = serialization
181            .children
182            .iter()
183            .map(|child| Self::try_new(session, child))
184            .collect::<VortexResult<Vec<_>>>()?;
185
186        Ok(Self {
187            source: source.clone(),
188            serialized_id: serialization.serialized_id,
189            metadata: serialization.metadata,
190            buffers: serialization.buffers,
191            children,
192        })
193    }
194
195    fn nbuffers_recursive(&self) -> usize {
196        self.buffers.len()
197            + self
198                .children
199                .iter()
200                .map(Self::nbuffers_recursive)
201                .sum::<usize>()
202    }
203
204    fn buffers(&self) -> Vec<ByteBuffer> {
205        let mut buffers = Vec::with_capacity(self.nbuffers_recursive());
206        self.append_buffers(&mut buffers);
207        buffers
208    }
209
210    fn append_buffers(&self, buffers: &mut Vec<ByteBuffer>) {
211        buffers.extend(self.buffers.iter().cloned());
212        for child in &self.children {
213            child.append_buffers(buffers);
214        }
215    }
216}
217
218/// A utility struct for creating an [`fba::ArrayNode`] flatbuffer.
219pub struct ArrayNodeFlatBuffer<'a> {
220    ctx: &'a ArrayContext,
221    array: ArraySerializationTree,
222}
223
224impl<'a> ArrayNodeFlatBuffer<'a> {
225    pub fn try_new(
226        ctx: &'a ArrayContext,
227        session: &'a VortexSession,
228        array: &ArrayRef,
229    ) -> VortexResult<Self> {
230        let array = ArraySerializationTree::try_new(session, array)?;
231        let n_buffers_recursive = array.nbuffers_recursive();
232        if n_buffers_recursive > u16::MAX as usize {
233            vortex_bail!(
234                "Array and all descendent arrays can have at most u16::MAX buffers: {}",
235                n_buffers_recursive
236            );
237        };
238        Ok(Self { ctx, array })
239    }
240
241    pub fn try_write_flatbuffer<'fb>(
242        &self,
243        fbb: &mut FlatBufferBuilder<'fb>,
244    ) -> VortexResult<WIPOffset<fba::ArrayNode<'fb>>> {
245        self.try_write_node(fbb, &self.array, 0)
246    }
247
248    fn try_write_node<'fb>(
249        &self,
250        fbb: &mut FlatBufferBuilder<'fb>,
251        array: &ArraySerializationTree,
252        buffer_idx: u16,
253    ) -> VortexResult<WIPOffset<fba::ArrayNode<'fb>>> {
254        let encoding_idx = self.ctx.intern(&array.serialized_id).ok_or_else(|| {
255            vortex_err!(
256                "Serialized array ID {} not permitted by ctx",
257                array.serialized_id
258            )
259        })?;
260
261        let metadata = Some(fbb.create_vector(array.metadata.as_slice()));
262
263        // Assign buffer indices for all child arrays.
264        let nbuffers = u16::try_from(array.buffers.len())
265            .map_err(|_| vortex_err!("Array can have at most u16::MAX buffers"))?;
266        let mut child_buffer_idx = buffer_idx + nbuffers;
267
268        let children = array
269            .children
270            .iter()
271            .map(|child| {
272                // Update the number of buffers required.
273                let msg = self.try_write_node(fbb, child, child_buffer_idx)?;
274
275                child_buffer_idx = u16::try_from(child.nbuffers_recursive())
276                    .ok()
277                    .and_then(|nbuffers| nbuffers.checked_add(child_buffer_idx))
278                    .ok_or_else(|| vortex_err!("Too many buffers (u16) for Array"))?;
279
280                Ok(msg)
281            })
282            .collect::<VortexResult<Vec<_>>>()?;
283        let children = Some(fbb.create_vector(&children));
284
285        let buffers = Some(fbb.create_vector_from_iter((0..nbuffers).map(|i| i + buffer_idx)));
286        let stats = Some(array.source.statistics().write_flatbuffer(fbb)?);
287
288        Ok(fba::ArrayNode::create(
289            fbb,
290            &fba::ArrayNodeArgs {
291                encoding: encoding_idx,
292                metadata,
293                children,
294                buffers,
295                stats,
296            },
297        ))
298    }
299}
300
301/// To minimize the serialized form, arrays do not persist their own dtype and length. Instead,
302/// parent arrays pass this information down during deserialization.
303pub trait ArrayChildren {
304    /// Returns the nth child of the array with the given dtype and length.
305    fn get(&self, index: usize, dtype: &DType, len: usize) -> VortexResult<ArrayRef>;
306
307    /// The number of children.
308    fn len(&self) -> usize;
309
310    /// Returns true if there are no children.
311    fn is_empty(&self) -> bool {
312        self.len() == 0
313    }
314}
315
316impl<T: AsRef<[ArrayRef]>> ArrayChildren for T {
317    fn get(&self, index: usize, dtype: &DType, len: usize) -> VortexResult<ArrayRef> {
318        let array = self.as_ref()[index].clone();
319        assert_eq!(array.len(), len);
320        assert_eq!(array.dtype(), dtype);
321        Ok(array)
322    }
323
324    fn len(&self) -> usize {
325        self.as_ref().len()
326    }
327}
328
329/// [`SerializedArray`] represents a parsed but not-yet-decoded deserialized array.
330/// It contains all the information from the serialized form, without anything extra. i.e.
331/// it is missing a [`DType`] and `len`, and the `encoding_id` is not yet resolved to a concrete
332/// vtable.
333///
334/// An [`SerializedArray`] can be fully decoded into an [`ArrayRef`] using the `decode` function.
335#[derive(Clone)]
336pub struct SerializedArray {
337    // Typed as fb::ArrayNode
338    flatbuffer: FlatBuffer,
339    // The location of the current fb::ArrayNode
340    flatbuffer_loc: usize,
341    buffers: Arc<[BufferHandle]>,
342}
343
344impl Debug for SerializedArray {
345    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
346        f.debug_struct("SerializedArray")
347            .field("encoding_id", &self.encoding_id())
348            .field("children", &(0..self.nchildren()).map(|i| self.child(i)))
349            .field(
350                "buffers",
351                &(0..self.nbuffers()).map(|i| self.buffer(i).ok()),
352            )
353            .field("metadata", &self.metadata())
354            .finish()
355    }
356}
357
358impl SerializedArray {
359    /// Decode an [`SerializedArray`] into an [`ArrayRef`].
360    pub fn decode(
361        &self,
362        dtype: &DType,
363        len: usize,
364        ctx: &ReadContext,
365        session: &VortexSession,
366    ) -> VortexResult<ArrayRef> {
367        let encoding_idx = self.flatbuffer().encoding();
368        let encoding_id = ctx
369            .resolve(encoding_idx)
370            .ok_or_else(|| vortex_err!("Unknown encoding index: {}", encoding_idx))?;
371        let Some(plugin) = session.arrays().registry().get(&encoding_id) else {
372            if session.allows_unknown() {
373                return self.decode_foreign(encoding_id, dtype, len, ctx);
374            }
375            vortex_bail!("Unknown encoding: {}", encoding_id);
376        };
377
378        let children = SerializedArrayChildren {
379            ser: self,
380            ctx,
381            session,
382        };
383
384        let buffers = self.collect_buffers()?;
385
386        let decoded = plugin.deserialize(
387            ArrayDeserialization::new(
388                encoding_id,
389                dtype,
390                len,
391                self.metadata(),
392                &buffers,
393                &children,
394            ),
395            session,
396        )?;
397
398        assert_eq!(
399            decoded.len(),
400            len,
401            "Array decoded from {} has incorrect length {}, expected {}",
402            encoding_id,
403            decoded.len(),
404            len
405        );
406        assert_eq!(
407            decoded.dtype(),
408            dtype,
409            "Array decoded from {} has incorrect dtype {}, expected {}",
410            encoding_id,
411            decoded.dtype(),
412            dtype,
413        );
414
415        assert!(
416            plugin.is_supported_encoding(&decoded.encoding_id()),
417            "Array decoded from {} has incorrect encoding {}",
418            encoding_id,
419            decoded.encoding_id(),
420        );
421
422        // Populate statistics from the serialized array.
423        if let Some(stats) = self.flatbuffer().stats() {
424            let mut stats = StatsSet::from_flatbuffer(&stats, dtype, session)?;
425            // Arrays with 0 or 1 elements are always (strict) sorted. Older writers could store
426            // a sortedness hint that says otherwise, so the intrinsic fact takes precedence.
427            if len <= 1 {
428                stats.set(Stat::IsSorted, Precision::Exact(true.into()));
429                stats.set(Stat::IsStrictSorted, Precision::Exact(true.into()));
430            }
431            decoded.statistics().set_all(stats);
432        }
433
434        mark_decoded_optimized(&decoded);
435        Ok(decoded)
436    }
437
438    fn decode_foreign(
439        &self,
440        encoding_id: ArrayId,
441        dtype: &DType,
442        len: usize,
443        ctx: &ReadContext,
444    ) -> VortexResult<ArrayRef> {
445        let children = (0..self.nchildren())
446            .map(|idx| {
447                let child = self.child(idx);
448                let child_encoding_idx = child.flatbuffer().encoding();
449                let child_encoding_id = ctx
450                    .resolve(child_encoding_idx)
451                    .ok_or_else(|| vortex_err!("Unknown encoding index: {}", child_encoding_idx))?;
452                child
453                    .decode_foreign(child_encoding_id, dtype, len, ctx)
454                    .map(Some)
455            })
456            .collect::<VortexResult<ArraySlots>>()?;
457
458        new_foreign_array(
459            encoding_id,
460            dtype.clone(),
461            len,
462            self.metadata().to_vec(),
463            self.collect_buffers()?.into_owned(),
464            children,
465        )
466    }
467
468    /// Returns the array encoding.
469    pub fn encoding_id(&self) -> u16 {
470        self.flatbuffer().encoding()
471    }
472
473    /// Returns the array metadata bytes.
474    pub fn metadata(&self) -> &[u8] {
475        self.flatbuffer()
476            .metadata()
477            .map(|metadata| metadata.bytes())
478            .unwrap_or(&[])
479    }
480
481    /// Returns the number of children.
482    pub fn nchildren(&self) -> usize {
483        self.flatbuffer()
484            .children()
485            .map_or(0, |children| children.len())
486    }
487
488    /// Returns the nth child of the array.
489    pub fn child(&self, idx: usize) -> SerializedArray {
490        let children = self
491            .flatbuffer()
492            .children()
493            .vortex_expect("Expected array to have children");
494        if idx >= children.len() {
495            vortex_panic!(
496                "Invalid child index {} for array with {} children",
497                idx,
498                children.len()
499            );
500        }
501        self.with_root(children.get(idx))
502    }
503
504    /// Returns the number of buffers.
505    pub fn nbuffers(&self) -> usize {
506        self.flatbuffer()
507            .buffers()
508            .map_or(0, |buffers| buffers.len())
509    }
510
511    /// Returns the nth buffer of the current array.
512    pub fn buffer(&self, idx: usize) -> VortexResult<BufferHandle> {
513        let buffer_idx = self
514            .flatbuffer()
515            .buffers()
516            .ok_or_else(|| vortex_err!("Array has no buffers"))?
517            .get(idx);
518        self.buffers
519            .get(buffer_idx as usize)
520            .cloned()
521            .ok_or_else(|| {
522                vortex_err!(
523                    "Invalid buffer index {} for array with {} buffers",
524                    buffer_idx,
525                    self.nbuffers()
526                )
527            })
528    }
529
530    /// Returns all buffers for the current array node.
531    ///
532    /// If buffer indices are contiguous, returns a zero-copy borrowed slice.
533    /// Otherwise falls back to collecting each buffer individually.
534    fn collect_buffers(&self) -> VortexResult<Cow<'_, [BufferHandle]>> {
535        let Some(fb_buffers) = self.flatbuffer().buffers() else {
536            return Ok(Cow::Borrowed(&[]));
537        };
538        let count = fb_buffers.len();
539        if count == 0 {
540            return Ok(Cow::Borrowed(&[]));
541        }
542        let start = fb_buffers.get(0) as usize;
543        let contiguous = fb_buffers
544            .iter()
545            .enumerate()
546            .all(|(i, idx)| idx as usize == start + i);
547        if contiguous {
548            self.buffers.get(start..start + count).map_or_else(
549                || {
550                    vortex_bail!(
551                        "buffer indices {}..{} out of range for {} buffers",
552                        start,
553                        start + count,
554                        self.buffers.len()
555                    )
556                },
557                |slice| Ok(Cow::Borrowed(slice)),
558            )
559        } else {
560            (0..count)
561                .map(|idx| self.buffer(idx))
562                .collect::<VortexResult<Vec<_>>>()
563                .map(Cow::Owned)
564        }
565    }
566
567    /// Returns the buffer lengths as stored in the flatbuffer metadata.
568    ///
569    /// This reads the buffer descriptors from the flatbuffer, which contain the
570    /// serialized length of each buffer. This is useful for displaying buffer sizes
571    /// without needing to access the actual buffer data.
572    pub fn buffer_lengths(&self) -> Vec<usize> {
573        let fb_array = root::<fba::Array>(self.flatbuffer.as_ref())
574            .vortex_expect("SerializedArray flatbuffer must be a valid Array");
575        fb_array
576            .buffers()
577            .map(|buffers| buffers.iter().map(|b| b.length() as usize).collect())
578            .unwrap_or_default()
579    }
580
581    /// Validate and align the array tree flatbuffer, returning the aligned buffer and root location.
582    fn validate_array_tree(array_tree: impl Into<ByteBuffer>) -> VortexResult<(FlatBuffer, usize)> {
583        let fb_buffer = FlatBuffer::align_from(array_tree.into());
584        let fb_array = root::<fba::Array>(fb_buffer.as_ref())?;
585        let fb_root = fb_array
586            .root()
587            .ok_or_else(|| vortex_err!("Array must have a root node"))?;
588        let flatbuffer_loc = fb_root._tab.loc();
589        Ok((fb_buffer, flatbuffer_loc))
590    }
591
592    /// Create an [`SerializedArray`] from a pre-existing array tree flatbuffer and pre-resolved buffer
593    /// handles.
594    ///
595    /// The caller is responsible for resolving buffers from whatever source (device segments, host
596    /// overrides, or a mix). The buffers must be in the same order as the `Array.buffers` descriptor
597    /// list in the flatbuffer.
598    pub fn from_flatbuffer_with_buffers(
599        array_tree: impl Into<ByteBuffer>,
600        buffers: Vec<BufferHandle>,
601    ) -> VortexResult<Self> {
602        let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?;
603        Ok(SerializedArray {
604            flatbuffer,
605            flatbuffer_loc,
606            buffers: buffers.into(),
607        })
608    }
609
610    /// Create an [`SerializedArray`] from a raw array tree flatbuffer (metadata only).
611    ///
612    /// This constructor creates a `SerializedArray` with no buffer data, useful for
613    /// inspecting the metadata when the actual buffer data is not needed
614    /// (e.g., displaying buffer sizes from inlined array tree metadata).
615    ///
616    /// Note: Calling `buffer()` on the returned `SerializedArray` will fail since
617    /// no actual buffer data is available.
618    pub fn from_array_tree(array_tree: impl Into<ByteBuffer>) -> VortexResult<Self> {
619        let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?;
620        Ok(SerializedArray {
621            flatbuffer,
622            flatbuffer_loc,
623            buffers: Arc::new([]),
624        })
625    }
626
627    /// Returns the root ArrayNode flatbuffer.
628    fn flatbuffer(&self) -> fba::ArrayNode<'_> {
629        unsafe { fba::ArrayNode::follow(self.flatbuffer.as_ref(), self.flatbuffer_loc) }
630    }
631
632    /// Returns a new [`SerializedArray`] with the given node as the root
633    // TODO(ngates): we may want a wrapper that avoids this clone.
634    fn with_root(&self, root: fba::ArrayNode) -> Self {
635        let mut this = self.clone();
636        this.flatbuffer_loc = root._tab.loc();
637        this
638    }
639
640    /// Create an [`SerializedArray`] from a pre-existing flatbuffer (ArrayNode) and a segment containing
641    /// only the data buffers (without the flatbuffer suffix).
642    ///
643    /// This is used when the flatbuffer is stored separately in layout metadata (e.g., when
644    /// `FLAT_LAYOUT_INLINE_ARRAY_NODE` is enabled).
645    pub fn from_flatbuffer_and_segment(
646        array_tree: ByteBuffer,
647        segment: BufferHandle,
648    ) -> VortexResult<Self> {
649        // HashMap::new doesn't allocate when empty, so this has no overhead
650        Self::from_flatbuffer_and_segment_with_overrides(array_tree, segment, &HashMap::new())
651    }
652
653    /// Create an [`SerializedArray`] from a pre-existing flatbuffer (ArrayNode) and a segment,
654    /// substituting host-resident buffer overrides for specific buffer indices.
655    ///
656    /// Buffers whose index appears in `buffer_overrides` are resolved from the provided
657    /// host data instead of the segment. All other buffers are sliced from the segment
658    /// using the padding and alignment described in the flatbuffer.
659    pub fn from_flatbuffer_and_segment_with_overrides(
660        array_tree: ByteBuffer,
661        segment: BufferHandle,
662        buffer_overrides: &HashMap<u32, ByteBuffer>,
663    ) -> VortexResult<Self> {
664        // We align each buffer individually, so we remove alignment requirements on the segment
665        // for host-resident buffers. Device buffers are sliced directly.
666        let segment = segment.ensure_aligned(Alignment::none())?;
667
668        // this can't return the validated array because there is no lifetime to give it, so we
669        // need to cast it below, which is safe.
670        let (fb_buffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?;
671        // SAFETY: fb_buffer was already validated by validate_array_tree above.
672        let fb_array = unsafe { fba::root_as_array_unchecked(fb_buffer.as_ref()) };
673
674        let mut offset = 0usize;
675        let buffers = fb_array
676            .buffers()
677            .unwrap_or_default()
678            .iter()
679            .enumerate()
680            .map(|(idx, fb_buf)| {
681                let idx = u32::try_from(idx).vortex_expect("buffer count must fit in u32");
682
683                // The padding, length, and resulting offsets all come from the flatbuffer, which
684                // may be corrupt. Use checked arithmetic so malformed metadata returns a
685                // `VortexError` rather than panicking (see issue #8819).
686                let buffer_len = fb_buf.length() as usize;
687                let start = offset
688                    .checked_add(fb_buf.padding() as usize)
689                    .ok_or_else(|| {
690                        vortex_err!("Buffer {idx} offset overflows when adding its padding")
691                    })?;
692                let end = start.checked_add(buffer_len).ok_or_else(|| {
693                    vortex_err!("Buffer {idx} offset overflows when adding its length")
694                })?;
695
696                // The alignment exponent comes from the flatbuffer and may be corrupt, so validate
697                // it rather than panicking on a too-large shift (see issue #8819).
698                let alignment =
699                    Alignment::try_from_untrusted_exponent(fb_buf.alignment_exponent())?;
700                let handle = if let Some(host_data) = buffer_overrides.get(&idx) {
701                    BufferHandle::new_host(host_data.clone()).ensure_aligned(alignment)?
702                } else {
703                    // Bounds-check against the segment so an out-of-range buffer returns a
704                    // `VortexError` rather than panicking when slicing (see issue #8819).
705                    if end > segment.len() {
706                        vortex_bail!(
707                            "Buffer {idx} at offset {start} with length {buffer_len} is out of \
708                             bounds of the {}-byte segment",
709                            segment.len(),
710                        );
711                    }
712                    segment.slice(start..end).ensure_aligned(alignment)?
713                };
714
715                offset = end;
716                Ok(handle)
717            })
718            .collect::<VortexResult<Arc<[_]>>>()?;
719
720        Ok(SerializedArray {
721            flatbuffer: fb_buffer,
722            flatbuffer_loc,
723            buffers,
724        })
725    }
726}
727
728struct SerializedArrayChildren<'a> {
729    ser: &'a SerializedArray,
730    ctx: &'a ReadContext,
731    session: &'a VortexSession,
732}
733
734impl ArrayChildren for SerializedArrayChildren<'_> {
735    fn get(&self, index: usize, dtype: &DType, len: usize) -> VortexResult<ArrayRef> {
736        self.ser
737            .child(index)
738            .decode(dtype, len, self.ctx, self.session)
739    }
740
741    fn len(&self) -> usize {
742        self.ser.nchildren()
743    }
744}
745
746impl TryFrom<ByteBuffer> for SerializedArray {
747    type Error = VortexError;
748
749    fn try_from(value: ByteBuffer) -> Result<Self, Self::Error> {
750        // The final 4 bytes contain the length of the flatbuffer.
751        if value.len() < 4 {
752            vortex_bail!("SerializedArray buffer is too short");
753        }
754
755        // We align each buffer individually, so we remove alignment requirements on the buffer.
756        let value = value.aligned(Alignment::none());
757
758        let fb_length = u32::try_from_le_bytes(&value.as_slice()[value.len() - 4..])? as usize;
759        if value.len() < 4 + fb_length {
760            vortex_bail!("SerializedArray buffer is too short for flatbuffer");
761        }
762
763        let fb_offset = value.len() - 4 - fb_length;
764        let array_tree = value.slice(fb_offset..fb_offset + fb_length);
765        let segment = BufferHandle::new_host(value.slice(0..fb_offset));
766
767        Self::from_flatbuffer_and_segment(array_tree, segment)
768    }
769}
770
771impl TryFrom<BufferHandle> for SerializedArray {
772    type Error = VortexError;
773
774    fn try_from(value: BufferHandle) -> Result<Self, Self::Error> {
775        Self::try_from(value.try_to_host_sync()?)
776    }
777}
778
779#[cfg(test)]
780mod tests {
781    use std::sync::LazyLock;
782    use std::sync::atomic::AtomicUsize;
783    use std::sync::atomic::Ordering;
784
785    use vortex_buffer::ByteBufferMut;
786    use vortex_error::vortex_ensure;
787    use vortex_error::vortex_ensure_eq;
788    use vortex_session::VortexSession;
789    use vortex_session::registry::CachedId;
790
791    use super::*;
792    use crate::Array;
793    use crate::ArrayPlugin;
794    use crate::ArraySerialization;
795    use crate::ArrayVTable;
796    use crate::IntoArray;
797    use crate::VortexSessionExecute;
798    use crate::arrays::Primitive;
799    use crate::arrays::PrimitiveArray;
800    use crate::assert_arrays_eq;
801
802    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
803
804    static SERIALIZER_CALLS: AtomicUsize = AtomicUsize::new(0);
805
806    fn old_primitive_id() -> ArrayId {
807        ArrayVTable::id(&Primitive)
808    }
809
810    fn new_primitive_id() -> ArrayId {
811        static ID: CachedId = CachedId::new("vortex.test.primitive_v2");
812        *ID
813    }
814
815    #[derive(Debug)]
816    struct VersionedPrimitivePlugin;
817
818    impl ArrayPlugin for VersionedPrimitivePlugin {
819        fn id(&self) -> ArrayId {
820            old_primitive_id()
821        }
822
823        fn read_ids(&self) -> Vec<ArrayId> {
824            vec![old_primitive_id(), new_primitive_id()]
825        }
826
827        fn write_ids(&self) -> Vec<ArrayId> {
828            vec![old_primitive_id(), new_primitive_id()]
829        }
830
831        fn serialize(
832            &self,
833            array: &ArrayRef,
834            _session: &VortexSession,
835        ) -> VortexResult<Option<ArraySerialization>> {
836            vortex_ensure_eq!(
837                array.encoding_id(),
838                self.id(),
839                "versioned primitive serializer received a different encoding",
840            );
841
842            let serialized_id = if array.len() <= 4 {
843                old_primitive_id()
844            } else {
845                new_primitive_id()
846            };
847
848            Ok(Some(ArraySerialization::from_array(
849                serialized_id,
850                array,
851                vec![],
852            )))
853        }
854
855        fn deserialize(
856            &self,
857            parts: ArrayDeserialization<'_>,
858            session: &VortexSession,
859        ) -> VortexResult<ArrayRef> {
860            vortex_ensure!(
861                parts.serialized_id == old_primitive_id()
862                    || parts.serialized_id == new_primitive_id(),
863                "versioned primitive deserializer does not recognize {}",
864                parts.serialized_id,
865            );
866            vortex_ensure!(
867                parts.serialized_id != old_primitive_id() || parts.len <= 4,
868                "old primitive wire ID cannot represent length {}",
869                parts.len,
870            );
871            Ok(Array::<Primitive>::try_from_parts(ArrayVTable::deserialize(
872                &Primitive,
873                parts.dtype,
874                parts.len,
875                parts.metadata,
876                parts.buffers,
877                parts.children,
878                session,
879            )?)?
880            .into_array())
881        }
882    }
883
884    #[derive(Debug)]
885    struct CountingVersionedPrimitivePlugin;
886
887    impl ArrayPlugin for CountingVersionedPrimitivePlugin {
888        fn id(&self) -> ArrayId {
889            VersionedPrimitivePlugin.id()
890        }
891
892        fn read_ids(&self) -> Vec<ArrayId> {
893            VersionedPrimitivePlugin.read_ids()
894        }
895
896        fn write_ids(&self) -> Vec<ArrayId> {
897            VersionedPrimitivePlugin.write_ids()
898        }
899
900        fn serialize(
901            &self,
902            array: &ArrayRef,
903            session: &VortexSession,
904        ) -> VortexResult<Option<ArraySerialization>> {
905            SERIALIZER_CALLS.fetch_add(1, Ordering::Relaxed);
906            VersionedPrimitivePlugin.serialize(array, session)
907        }
908
909        fn deserialize(
910            &self,
911            parts: ArrayDeserialization<'_>,
912            session: &VortexSession,
913        ) -> VortexResult<ArrayRef> {
914            VersionedPrimitivePlugin.deserialize(parts, session)
915        }
916    }
917
918    fn versioned_primitive_session() -> VortexSession {
919        let session = VortexSession::empty();
920        session.arrays().register(VersionedPrimitivePlugin);
921        session
922    }
923
924    fn restricted_context(ids: &[ArrayId]) -> ArrayContext {
925        ArrayContext::new(ids.to_vec()).with_allowed_ids(ids.iter().copied().collect())
926    }
927
928    fn serialize_blob(
929        array: &ArrayRef,
930        ctx: &ArrayContext,
931        session: &VortexSession,
932    ) -> VortexResult<ByteBuffer> {
933        let mut blob = ByteBufferMut::empty();
934        for buffer in array.serialize(ctx, session, &SerializeOptions::default())? {
935            blob.extend_from_slice(buffer.as_ref());
936        }
937        Ok(blob.freeze())
938    }
939
940    #[test]
941    fn serialization_preserves_plugin_selected_old_wire_id() -> VortexResult<()> {
942        let session = VortexSession::empty();
943        session.arrays().register(CountingVersionedPrimitivePlugin);
944        let ctx = restricted_context(&[old_primitive_id(), new_primitive_id()]);
945        let array = PrimitiveArray::from_iter([1i32, 2, 3, 4]).into_array();
946
947        SERIALIZER_CALLS.store(0, Ordering::Relaxed);
948        let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &session)?)?;
949        assert_eq!(SERIALIZER_CALLS.load(Ordering::Relaxed), 1);
950        assert_eq!(
951            ReadContext::new(ctx.to_ids()).resolve(serialized.encoding_id()),
952            Some(old_primitive_id())
953        );
954        let decoded = serialized.decode(
955            array.dtype(),
956            array.len(),
957            &ReadContext::new(ctx.to_ids()),
958            &session,
959        )?;
960        assert_arrays_eq!(
961            decoded,
962            PrimitiveArray::from_iter([1i32, 2, 3, 4]),
963            &mut session.create_execution_ctx()
964        );
965        Ok(())
966    }
967
968    #[test]
969    fn serialization_preserves_plugin_selected_new_wire_id() -> VortexResult<()> {
970        let session = versioned_primitive_session();
971        let ctx = restricted_context(&[old_primitive_id(), new_primitive_id()]);
972        let array = PrimitiveArray::from_iter(0..8i32).into_array();
973        let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &session)?)?;
974        let read_ctx = ReadContext::new(ctx.to_ids());
975
976        assert_eq!(
977            read_ctx.resolve(serialized.encoding_id()),
978            Some(new_primitive_id())
979        );
980        let decoded = serialized.decode(array.dtype(), array.len(), &read_ctx, &session)?;
981        assert_eq!(decoded.encoding_id(), old_primitive_id());
982        assert_arrays_eq!(
983            decoded,
984            PrimitiveArray::from_iter(0..8i32),
985            &mut session.create_execution_ctx()
986        );
987        Ok(())
988    }
989
990    #[test]
991    fn serialization_fails_when_serialized_id_is_not_permitted() -> VortexResult<()> {
992        let session = versioned_primitive_session();
993        let ctx = restricted_context(&[old_primitive_id()]);
994        let array = PrimitiveArray::from_iter(0..8i32).into_array();
995
996        let error = array
997            .serialize(&ctx, &session, &SerializeOptions::default())
998            .expect_err("the serialized ID is not permitted");
999        assert!(error.to_string().contains("not permitted by ctx"));
1000        Ok(())
1001    }
1002
1003    #[test]
1004    fn old_reader_rejects_a_new_serialized_id() -> VortexResult<()> {
1005        let writer_session = versioned_primitive_session();
1006        let ctx = restricted_context(&[new_primitive_id()]);
1007        let array = PrimitiveArray::from_iter(0..8i32).into_array();
1008        let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &writer_session)?)?;
1009        let read_ctx = ReadContext::new(ctx.to_ids());
1010
1011        let error = serialized
1012            .decode(array.dtype(), array.len(), &read_ctx, &SESSION)
1013            .expect_err("an old reader must not recognize the new wire ID");
1014        assert!(error.to_string().contains("Unknown encoding"));
1015        Ok(())
1016    }
1017
1018    #[test]
1019    fn deserializer_enforces_the_exact_wire_id_contract() -> VortexResult<()> {
1020        let session = versioned_primitive_session();
1021        let write_ctx = restricted_context(&[new_primitive_id()]);
1022        let array = PrimitiveArray::from_iter(0..8i32).into_array();
1023        let serialized = SerializedArray::try_from(serialize_blob(&array, &write_ctx, &session)?)?;
1024
1025        // Interpret the encoded index as the old ID to simulate a file that uses the old tag for
1026        // a representation outside that tag's frozen contract.
1027        let error = serialized
1028            .decode(
1029                array.dtype(),
1030                array.len(),
1031                &ReadContext::new([old_primitive_id()]),
1032                &session,
1033            )
1034            .expect_err("the old wire contract must be enforced by the current deserializer");
1035        assert!(error.to_string().contains("old primitive wire ID"));
1036        Ok(())
1037    }
1038
1039    /// A corrupt array tree can declare a buffer that extends past the backing segment. Slicing
1040    /// such a buffer must return a [`VortexError`] rather than panicking (see issue #8819).
1041    #[test]
1042    fn from_flatbuffer_and_segment_rejects_out_of_bounds_buffer() -> VortexResult<()> {
1043        let array_ctx = ArrayContext::empty();
1044
1045        // Serialize a simple array so we have a valid array tree flatbuffer whose declared buffer
1046        // lengths describe the trailing data segment.
1047        let serialized = PrimitiveArray::from_iter([1i32, 2, 3, 4])
1048            .into_array()
1049            .serialize(&array_ctx, &SESSION, &SerializeOptions::default())?;
1050
1051        let mut concat = ByteBufferMut::empty();
1052        for buf in serialized {
1053            concat.extend_from_slice(buf.as_ref());
1054        }
1055        let value = concat.freeze().aligned(Alignment::none());
1056
1057        // Split the blob into the trailing flatbuffer and the leading data segment, mirroring
1058        // `SerializedArray::try_from`.
1059        let fb_length = u32::try_from_le_bytes(&value.as_slice()[value.len() - 4..])? as usize;
1060        let fb_offset = value.len() - 4 - fb_length;
1061        assert!(
1062            fb_offset > 0,
1063            "the array must have at least one data buffer"
1064        );
1065        let array_tree = value.slice(fb_offset..fb_offset + fb_length);
1066
1067        // Truncate the data segment by one byte so the declared buffer no longer fits.
1068        let truncated = BufferHandle::new_host(value.slice(0..fb_offset - 1));
1069
1070        let Some(err) = SerializedArray::from_flatbuffer_and_segment(array_tree, truncated).err()
1071        else {
1072            vortex_bail!("out-of-bounds buffer must be rejected");
1073        };
1074        assert!(
1075            err.to_string().contains("out of bounds"),
1076            "unexpected error: {err}"
1077        );
1078
1079        Ok(())
1080    }
1081
1082    /// A corrupt array tree can declare a buffer alignment of up to 2^63, which the copy that
1083    /// satisfies it allocates as slack. It must be rejected (see issue #8819).
1084    #[test]
1085    fn from_flatbuffer_and_segment_rejects_excessive_buffer_alignment() -> VortexResult<()> {
1086        // Padding and length fit the segment exactly, so the exponent is the only defect.
1087        // `validate_array_tree` only requires a root node to be present, so an empty one will do.
1088        let mut fbb = FlatBufferBuilder::new();
1089        let fb_root = fba::ArrayNode::create(&mut fbb, &fba::ArrayNodeArgs::default());
1090        let fb_buffers = fbb.create_vector(&[fba::Buffer::new(0, 40, Compression::None, 4)]);
1091        let fb_array = fba::Array::create(
1092            &mut fbb,
1093            &fba::ArrayArgs {
1094                root: Some(fb_root),
1095                buffers: Some(fb_buffers),
1096            },
1097        );
1098        fbb.finish_minimal(fb_array);
1099        let (fb_vec, fb_start) = fbb.collapse();
1100        let fb_end = fb_vec.len();
1101        let array_tree = ByteBuffer::from(fb_vec).slice(fb_start..fb_end);
1102
1103        let segment = BufferHandle::new_host(ByteBuffer::from(vec![0u8; 4]));
1104        let Some(err) = SerializedArray::from_flatbuffer_and_segment(array_tree, segment).err()
1105        else {
1106            vortex_bail!("excessive buffer alignment must be rejected");
1107        };
1108        assert!(
1109            err.to_string().contains("exceeds"),
1110            "unexpected error: {err}"
1111        );
1112
1113        Ok(())
1114    }
1115}