1use std::borrow::Cow;
5use std::fmt::Debug;
6use std::fmt::Formatter;
7use std::iter;
8use std::sync::Arc;
9
10use flatbuffers::FlatBufferBuilder;
11use flatbuffers::Follow;
12use flatbuffers::WIPOffset;
13use flatbuffers::root;
14use vortex_buffer::Alignment;
15use vortex_buffer::ByteBuffer;
16use vortex_error::VortexError;
17use vortex_error::VortexExpect;
18use vortex_error::VortexResult;
19use vortex_error::vortex_bail;
20use vortex_error::vortex_err;
21use vortex_error::vortex_panic;
22use vortex_session::VortexSession;
23use vortex_session::registry::ReadContext;
24use vortex_utils::aliases::hash_map::HashMap;
25
26use crate::ArrayContext;
27use crate::ArrayRef;
28use crate::ArraySlots;
29use crate::array::ArrayDeserialization;
30use crate::array::ArrayId;
31use crate::array::new_foreign_array;
32use crate::buffer::BufferHandle;
33use crate::dtype::DType;
34use crate::dtype::TryFromBytes;
35use crate::expr::stats::Precision;
36use crate::expr::stats::Stat;
37use crate::flatbuffers::FlatBuffer;
38use crate::flatbuffers::WriteFlatBuffer;
39use crate::flatbuffers::array as fba;
40use crate::flatbuffers::array::Compression;
41use crate::optimizer::mark_decoded_optimized;
42use crate::session::ArraySessionExt;
43use crate::stats::StatsSet;
44
45#[derive(Default, Debug)]
47pub struct SerializeOptions {
48 pub offset: usize,
51 pub include_padding: bool,
53}
54
55impl ArrayRef {
56 pub fn serialize(
67 &self,
68 ctx: &ArrayContext,
69 session: &VortexSession,
70 options: &SerializeOptions,
71 ) -> VortexResult<Vec<ByteBuffer>> {
72 let root = ArrayNodeFlatBuffer::try_new(ctx, session, self)?;
75 let array_buffers = root.array.buffers();
76
77 let mut buffers = vec![];
79 let mut fb_buffers = Vec::with_capacity(buffers.capacity());
80
81 let max_alignment = array_buffers
83 .iter()
84 .map(|buf| buf.alignment())
85 .chain(iter::once(FlatBuffer::alignment()))
86 .max()
87 .unwrap_or_else(FlatBuffer::alignment);
88
89 let zeros = ByteBuffer::zeroed(max_alignment.as_usize());
91
92 buffers.push(ByteBuffer::zeroed_aligned(0, max_alignment));
95
96 let mut pos = options.offset;
98
99 for buffer in array_buffers {
101 let padding = if options.include_padding {
102 let padding = pos.next_multiple_of(buffer.alignment().as_usize()) - pos;
103 if padding > 0 {
104 pos += padding;
105 buffers.push(zeros.slice(0..padding));
106 }
107 padding
108 } else {
109 0
110 };
111
112 fb_buffers.push(fba::Buffer::new(
113 u16::try_from(padding).vortex_expect("padding fits into u16"),
114 buffer.alignment().exponent(),
115 Compression::None,
116 u32::try_from(buffer.len())
117 .map_err(|_| vortex_err!("All buffers must fit into u32 for serialization"))?,
118 ));
119
120 pos += buffer.len();
121 buffers.push(buffer.aligned(Alignment::none()));
122 }
123
124 let mut fbb = FlatBufferBuilder::new();
126
127 let fb_root = root.try_write_flatbuffer(&mut fbb)?;
128
129 let fb_buffers = fbb.create_vector(&fb_buffers);
130 let fb_array = fba::Array::create(
131 &mut fbb,
132 &fba::ArrayArgs {
133 root: Some(fb_root),
134 buffers: Some(fb_buffers),
135 },
136 );
137 fbb.finish_minimal(fb_array);
138 let (fb_vec, fb_start) = fbb.collapse();
139 let fb_end = fb_vec.len();
140 let fb_buffer = ByteBuffer::from(fb_vec).slice(fb_start..fb_end);
141 let fb_length = fb_buffer.len();
142
143 if options.include_padding {
144 let padding = pos.next_multiple_of(FlatBuffer::alignment().as_usize()) - pos;
145 if padding > 0 {
146 buffers.push(zeros.slice(0..padding));
147 }
148 }
149 buffers.push(fb_buffer);
150
151 buffers.push(ByteBuffer::from(
153 u32::try_from(fb_length)
154 .map_err(|_| vortex_err!("Array metadata flatbuffer must fit into u32 for serialization. Array encoding tree is too large."))?
155 .to_le_bytes()
156 .to_vec(),
157 ));
158
159 Ok(buffers)
160 }
161}
162
163#[derive(Clone, Debug)]
164struct ArraySerializationTree {
165 source: ArrayRef,
166 serialized_id: ArrayId,
167 metadata: Vec<u8>,
168 buffers: Vec<ByteBuffer>,
169 children: Vec<ArraySerializationTree>,
170}
171
172impl ArraySerializationTree {
173 fn try_new(session: &VortexSession, source: &ArrayRef) -> VortexResult<Self> {
174 let Some(serialization) = session.array_serialize(source)? else {
175 vortex_bail!(
176 "Array {} does not support serialization",
177 source.encoding_id()
178 );
179 };
180 let children = serialization
181 .children
182 .iter()
183 .map(|child| Self::try_new(session, child))
184 .collect::<VortexResult<Vec<_>>>()?;
185
186 Ok(Self {
187 source: source.clone(),
188 serialized_id: serialization.serialized_id,
189 metadata: serialization.metadata,
190 buffers: serialization.buffers,
191 children,
192 })
193 }
194
195 fn nbuffers_recursive(&self) -> usize {
196 self.buffers.len()
197 + self
198 .children
199 .iter()
200 .map(Self::nbuffers_recursive)
201 .sum::<usize>()
202 }
203
204 fn buffers(&self) -> Vec<ByteBuffer> {
205 let mut buffers = Vec::with_capacity(self.nbuffers_recursive());
206 self.append_buffers(&mut buffers);
207 buffers
208 }
209
210 fn append_buffers(&self, buffers: &mut Vec<ByteBuffer>) {
211 buffers.extend(self.buffers.iter().cloned());
212 for child in &self.children {
213 child.append_buffers(buffers);
214 }
215 }
216}
217
218pub struct ArrayNodeFlatBuffer<'a> {
220 ctx: &'a ArrayContext,
221 array: ArraySerializationTree,
222}
223
224impl<'a> ArrayNodeFlatBuffer<'a> {
225 pub fn try_new(
226 ctx: &'a ArrayContext,
227 session: &'a VortexSession,
228 array: &ArrayRef,
229 ) -> VortexResult<Self> {
230 let array = ArraySerializationTree::try_new(session, array)?;
231 let n_buffers_recursive = array.nbuffers_recursive();
232 if n_buffers_recursive > u16::MAX as usize {
233 vortex_bail!(
234 "Array and all descendent arrays can have at most u16::MAX buffers: {}",
235 n_buffers_recursive
236 );
237 };
238 Ok(Self { ctx, array })
239 }
240
241 pub fn try_write_flatbuffer<'fb>(
242 &self,
243 fbb: &mut FlatBufferBuilder<'fb>,
244 ) -> VortexResult<WIPOffset<fba::ArrayNode<'fb>>> {
245 self.try_write_node(fbb, &self.array, 0)
246 }
247
248 fn try_write_node<'fb>(
249 &self,
250 fbb: &mut FlatBufferBuilder<'fb>,
251 array: &ArraySerializationTree,
252 buffer_idx: u16,
253 ) -> VortexResult<WIPOffset<fba::ArrayNode<'fb>>> {
254 let encoding_idx = self.ctx.intern(&array.serialized_id).ok_or_else(|| {
255 vortex_err!(
256 "Serialized array ID {} not permitted by ctx",
257 array.serialized_id
258 )
259 })?;
260
261 let metadata = Some(fbb.create_vector(array.metadata.as_slice()));
262
263 let nbuffers = u16::try_from(array.buffers.len())
265 .map_err(|_| vortex_err!("Array can have at most u16::MAX buffers"))?;
266 let mut child_buffer_idx = buffer_idx + nbuffers;
267
268 let children = array
269 .children
270 .iter()
271 .map(|child| {
272 let msg = self.try_write_node(fbb, child, child_buffer_idx)?;
274
275 child_buffer_idx = u16::try_from(child.nbuffers_recursive())
276 .ok()
277 .and_then(|nbuffers| nbuffers.checked_add(child_buffer_idx))
278 .ok_or_else(|| vortex_err!("Too many buffers (u16) for Array"))?;
279
280 Ok(msg)
281 })
282 .collect::<VortexResult<Vec<_>>>()?;
283 let children = Some(fbb.create_vector(&children));
284
285 let buffers = Some(fbb.create_vector_from_iter((0..nbuffers).map(|i| i + buffer_idx)));
286 let stats = Some(array.source.statistics().write_flatbuffer(fbb)?);
287
288 Ok(fba::ArrayNode::create(
289 fbb,
290 &fba::ArrayNodeArgs {
291 encoding: encoding_idx,
292 metadata,
293 children,
294 buffers,
295 stats,
296 },
297 ))
298 }
299}
300
301pub trait ArrayChildren {
304 fn get(&self, index: usize, dtype: &DType, len: usize) -> VortexResult<ArrayRef>;
306
307 fn len(&self) -> usize;
309
310 fn is_empty(&self) -> bool {
312 self.len() == 0
313 }
314}
315
316impl<T: AsRef<[ArrayRef]>> ArrayChildren for T {
317 fn get(&self, index: usize, dtype: &DType, len: usize) -> VortexResult<ArrayRef> {
318 let array = self.as_ref()[index].clone();
319 assert_eq!(array.len(), len);
320 assert_eq!(array.dtype(), dtype);
321 Ok(array)
322 }
323
324 fn len(&self) -> usize {
325 self.as_ref().len()
326 }
327}
328
329#[derive(Clone)]
336pub struct SerializedArray {
337 flatbuffer: FlatBuffer,
339 flatbuffer_loc: usize,
341 buffers: Arc<[BufferHandle]>,
342}
343
344impl Debug for SerializedArray {
345 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
346 f.debug_struct("SerializedArray")
347 .field("encoding_id", &self.encoding_id())
348 .field("children", &(0..self.nchildren()).map(|i| self.child(i)))
349 .field(
350 "buffers",
351 &(0..self.nbuffers()).map(|i| self.buffer(i).ok()),
352 )
353 .field("metadata", &self.metadata())
354 .finish()
355 }
356}
357
358impl SerializedArray {
359 pub fn decode(
361 &self,
362 dtype: &DType,
363 len: usize,
364 ctx: &ReadContext,
365 session: &VortexSession,
366 ) -> VortexResult<ArrayRef> {
367 let encoding_idx = self.flatbuffer().encoding();
368 let encoding_id = ctx
369 .resolve(encoding_idx)
370 .ok_or_else(|| vortex_err!("Unknown encoding index: {}", encoding_idx))?;
371 let Some(plugin) = session.arrays().registry().get(&encoding_id) else {
372 if session.allows_unknown() {
373 return self.decode_foreign(encoding_id, dtype, len, ctx);
374 }
375 vortex_bail!("Unknown encoding: {}", encoding_id);
376 };
377
378 let children = SerializedArrayChildren {
379 ser: self,
380 ctx,
381 session,
382 };
383
384 let buffers = self.collect_buffers()?;
385
386 let decoded = plugin.deserialize(
387 ArrayDeserialization::new(
388 encoding_id,
389 dtype,
390 len,
391 self.metadata(),
392 &buffers,
393 &children,
394 ),
395 session,
396 )?;
397
398 assert_eq!(
399 decoded.len(),
400 len,
401 "Array decoded from {} has incorrect length {}, expected {}",
402 encoding_id,
403 decoded.len(),
404 len
405 );
406 assert_eq!(
407 decoded.dtype(),
408 dtype,
409 "Array decoded from {} has incorrect dtype {}, expected {}",
410 encoding_id,
411 decoded.dtype(),
412 dtype,
413 );
414
415 assert!(
416 plugin.is_supported_encoding(&decoded.encoding_id()),
417 "Array decoded from {} has incorrect encoding {}",
418 encoding_id,
419 decoded.encoding_id(),
420 );
421
422 if let Some(stats) = self.flatbuffer().stats() {
424 let mut stats = StatsSet::from_flatbuffer(&stats, dtype, session)?;
425 if len <= 1 {
428 stats.set(Stat::IsSorted, Precision::Exact(true.into()));
429 stats.set(Stat::IsStrictSorted, Precision::Exact(true.into()));
430 }
431 decoded.statistics().set_all(stats);
432 }
433
434 mark_decoded_optimized(&decoded);
435 Ok(decoded)
436 }
437
438 fn decode_foreign(
439 &self,
440 encoding_id: ArrayId,
441 dtype: &DType,
442 len: usize,
443 ctx: &ReadContext,
444 ) -> VortexResult<ArrayRef> {
445 let children = (0..self.nchildren())
446 .map(|idx| {
447 let child = self.child(idx);
448 let child_encoding_idx = child.flatbuffer().encoding();
449 let child_encoding_id = ctx
450 .resolve(child_encoding_idx)
451 .ok_or_else(|| vortex_err!("Unknown encoding index: {}", child_encoding_idx))?;
452 child
453 .decode_foreign(child_encoding_id, dtype, len, ctx)
454 .map(Some)
455 })
456 .collect::<VortexResult<ArraySlots>>()?;
457
458 new_foreign_array(
459 encoding_id,
460 dtype.clone(),
461 len,
462 self.metadata().to_vec(),
463 self.collect_buffers()?.into_owned(),
464 children,
465 )
466 }
467
468 pub fn encoding_id(&self) -> u16 {
470 self.flatbuffer().encoding()
471 }
472
473 pub fn metadata(&self) -> &[u8] {
475 self.flatbuffer()
476 .metadata()
477 .map(|metadata| metadata.bytes())
478 .unwrap_or(&[])
479 }
480
481 pub fn nchildren(&self) -> usize {
483 self.flatbuffer()
484 .children()
485 .map_or(0, |children| children.len())
486 }
487
488 pub fn child(&self, idx: usize) -> SerializedArray {
490 let children = self
491 .flatbuffer()
492 .children()
493 .vortex_expect("Expected array to have children");
494 if idx >= children.len() {
495 vortex_panic!(
496 "Invalid child index {} for array with {} children",
497 idx,
498 children.len()
499 );
500 }
501 self.with_root(children.get(idx))
502 }
503
504 pub fn nbuffers(&self) -> usize {
506 self.flatbuffer()
507 .buffers()
508 .map_or(0, |buffers| buffers.len())
509 }
510
511 pub fn buffer(&self, idx: usize) -> VortexResult<BufferHandle> {
513 let buffer_idx = self
514 .flatbuffer()
515 .buffers()
516 .ok_or_else(|| vortex_err!("Array has no buffers"))?
517 .get(idx);
518 self.buffers
519 .get(buffer_idx as usize)
520 .cloned()
521 .ok_or_else(|| {
522 vortex_err!(
523 "Invalid buffer index {} for array with {} buffers",
524 buffer_idx,
525 self.nbuffers()
526 )
527 })
528 }
529
530 fn collect_buffers(&self) -> VortexResult<Cow<'_, [BufferHandle]>> {
535 let Some(fb_buffers) = self.flatbuffer().buffers() else {
536 return Ok(Cow::Borrowed(&[]));
537 };
538 let count = fb_buffers.len();
539 if count == 0 {
540 return Ok(Cow::Borrowed(&[]));
541 }
542 let start = fb_buffers.get(0) as usize;
543 let contiguous = fb_buffers
544 .iter()
545 .enumerate()
546 .all(|(i, idx)| idx as usize == start + i);
547 if contiguous {
548 self.buffers.get(start..start + count).map_or_else(
549 || {
550 vortex_bail!(
551 "buffer indices {}..{} out of range for {} buffers",
552 start,
553 start + count,
554 self.buffers.len()
555 )
556 },
557 |slice| Ok(Cow::Borrowed(slice)),
558 )
559 } else {
560 (0..count)
561 .map(|idx| self.buffer(idx))
562 .collect::<VortexResult<Vec<_>>>()
563 .map(Cow::Owned)
564 }
565 }
566
567 pub fn buffer_lengths(&self) -> Vec<usize> {
573 let fb_array = root::<fba::Array>(self.flatbuffer.as_ref())
574 .vortex_expect("SerializedArray flatbuffer must be a valid Array");
575 fb_array
576 .buffers()
577 .map(|buffers| buffers.iter().map(|b| b.length() as usize).collect())
578 .unwrap_or_default()
579 }
580
581 fn validate_array_tree(array_tree: impl Into<ByteBuffer>) -> VortexResult<(FlatBuffer, usize)> {
583 let fb_buffer = FlatBuffer::align_from(array_tree.into());
584 let fb_array = root::<fba::Array>(fb_buffer.as_ref())?;
585 let fb_root = fb_array
586 .root()
587 .ok_or_else(|| vortex_err!("Array must have a root node"))?;
588 let flatbuffer_loc = fb_root._tab.loc();
589 Ok((fb_buffer, flatbuffer_loc))
590 }
591
592 pub fn from_flatbuffer_with_buffers(
599 array_tree: impl Into<ByteBuffer>,
600 buffers: Vec<BufferHandle>,
601 ) -> VortexResult<Self> {
602 let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?;
603 Ok(SerializedArray {
604 flatbuffer,
605 flatbuffer_loc,
606 buffers: buffers.into(),
607 })
608 }
609
610 pub fn from_array_tree(array_tree: impl Into<ByteBuffer>) -> VortexResult<Self> {
619 let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?;
620 Ok(SerializedArray {
621 flatbuffer,
622 flatbuffer_loc,
623 buffers: Arc::new([]),
624 })
625 }
626
627 fn flatbuffer(&self) -> fba::ArrayNode<'_> {
629 unsafe { fba::ArrayNode::follow(self.flatbuffer.as_ref(), self.flatbuffer_loc) }
630 }
631
632 fn with_root(&self, root: fba::ArrayNode) -> Self {
635 let mut this = self.clone();
636 this.flatbuffer_loc = root._tab.loc();
637 this
638 }
639
640 pub fn from_flatbuffer_and_segment(
646 array_tree: ByteBuffer,
647 segment: BufferHandle,
648 ) -> VortexResult<Self> {
649 Self::from_flatbuffer_and_segment_with_overrides(array_tree, segment, &HashMap::new())
651 }
652
653 pub fn from_flatbuffer_and_segment_with_overrides(
660 array_tree: ByteBuffer,
661 segment: BufferHandle,
662 buffer_overrides: &HashMap<u32, ByteBuffer>,
663 ) -> VortexResult<Self> {
664 let segment = segment.ensure_aligned(Alignment::none())?;
667
668 let (fb_buffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?;
671 let fb_array = unsafe { fba::root_as_array_unchecked(fb_buffer.as_ref()) };
673
674 let mut offset = 0usize;
675 let buffers = fb_array
676 .buffers()
677 .unwrap_or_default()
678 .iter()
679 .enumerate()
680 .map(|(idx, fb_buf)| {
681 let idx = u32::try_from(idx).vortex_expect("buffer count must fit in u32");
682
683 let buffer_len = fb_buf.length() as usize;
687 let start = offset
688 .checked_add(fb_buf.padding() as usize)
689 .ok_or_else(|| {
690 vortex_err!("Buffer {idx} offset overflows when adding its padding")
691 })?;
692 let end = start.checked_add(buffer_len).ok_or_else(|| {
693 vortex_err!("Buffer {idx} offset overflows when adding its length")
694 })?;
695
696 let alignment =
699 Alignment::try_from_untrusted_exponent(fb_buf.alignment_exponent())?;
700 let handle = if let Some(host_data) = buffer_overrides.get(&idx) {
701 BufferHandle::new_host(host_data.clone()).ensure_aligned(alignment)?
702 } else {
703 if end > segment.len() {
706 vortex_bail!(
707 "Buffer {idx} at offset {start} with length {buffer_len} is out of \
708 bounds of the {}-byte segment",
709 segment.len(),
710 );
711 }
712 segment.slice(start..end).ensure_aligned(alignment)?
713 };
714
715 offset = end;
716 Ok(handle)
717 })
718 .collect::<VortexResult<Arc<[_]>>>()?;
719
720 Ok(SerializedArray {
721 flatbuffer: fb_buffer,
722 flatbuffer_loc,
723 buffers,
724 })
725 }
726}
727
728struct SerializedArrayChildren<'a> {
729 ser: &'a SerializedArray,
730 ctx: &'a ReadContext,
731 session: &'a VortexSession,
732}
733
734impl ArrayChildren for SerializedArrayChildren<'_> {
735 fn get(&self, index: usize, dtype: &DType, len: usize) -> VortexResult<ArrayRef> {
736 self.ser
737 .child(index)
738 .decode(dtype, len, self.ctx, self.session)
739 }
740
741 fn len(&self) -> usize {
742 self.ser.nchildren()
743 }
744}
745
746impl TryFrom<ByteBuffer> for SerializedArray {
747 type Error = VortexError;
748
749 fn try_from(value: ByteBuffer) -> Result<Self, Self::Error> {
750 if value.len() < 4 {
752 vortex_bail!("SerializedArray buffer is too short");
753 }
754
755 let value = value.aligned(Alignment::none());
757
758 let fb_length = u32::try_from_le_bytes(&value.as_slice()[value.len() - 4..])? as usize;
759 if value.len() < 4 + fb_length {
760 vortex_bail!("SerializedArray buffer is too short for flatbuffer");
761 }
762
763 let fb_offset = value.len() - 4 - fb_length;
764 let array_tree = value.slice(fb_offset..fb_offset + fb_length);
765 let segment = BufferHandle::new_host(value.slice(0..fb_offset));
766
767 Self::from_flatbuffer_and_segment(array_tree, segment)
768 }
769}
770
771impl TryFrom<BufferHandle> for SerializedArray {
772 type Error = VortexError;
773
774 fn try_from(value: BufferHandle) -> Result<Self, Self::Error> {
775 Self::try_from(value.try_to_host_sync()?)
776 }
777}
778
779#[cfg(test)]
780mod tests {
781 use std::sync::LazyLock;
782 use std::sync::atomic::AtomicUsize;
783 use std::sync::atomic::Ordering;
784
785 use vortex_buffer::ByteBufferMut;
786 use vortex_error::vortex_ensure;
787 use vortex_error::vortex_ensure_eq;
788 use vortex_session::VortexSession;
789 use vortex_session::registry::CachedId;
790
791 use super::*;
792 use crate::Array;
793 use crate::ArrayPlugin;
794 use crate::ArraySerialization;
795 use crate::ArrayVTable;
796 use crate::IntoArray;
797 use crate::VortexSessionExecute;
798 use crate::arrays::Primitive;
799 use crate::arrays::PrimitiveArray;
800 use crate::assert_arrays_eq;
801
802 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
803
804 static SERIALIZER_CALLS: AtomicUsize = AtomicUsize::new(0);
805
806 fn old_primitive_id() -> ArrayId {
807 ArrayVTable::id(&Primitive)
808 }
809
810 fn new_primitive_id() -> ArrayId {
811 static ID: CachedId = CachedId::new("vortex.test.primitive_v2");
812 *ID
813 }
814
815 #[derive(Debug)]
816 struct VersionedPrimitivePlugin;
817
818 impl ArrayPlugin for VersionedPrimitivePlugin {
819 fn id(&self) -> ArrayId {
820 old_primitive_id()
821 }
822
823 fn read_ids(&self) -> Vec<ArrayId> {
824 vec![old_primitive_id(), new_primitive_id()]
825 }
826
827 fn write_ids(&self) -> Vec<ArrayId> {
828 vec![old_primitive_id(), new_primitive_id()]
829 }
830
831 fn serialize(
832 &self,
833 array: &ArrayRef,
834 _session: &VortexSession,
835 ) -> VortexResult<Option<ArraySerialization>> {
836 vortex_ensure_eq!(
837 array.encoding_id(),
838 self.id(),
839 "versioned primitive serializer received a different encoding",
840 );
841
842 let serialized_id = if array.len() <= 4 {
843 old_primitive_id()
844 } else {
845 new_primitive_id()
846 };
847
848 Ok(Some(ArraySerialization::from_array(
849 serialized_id,
850 array,
851 vec![],
852 )))
853 }
854
855 fn deserialize(
856 &self,
857 parts: ArrayDeserialization<'_>,
858 session: &VortexSession,
859 ) -> VortexResult<ArrayRef> {
860 vortex_ensure!(
861 parts.serialized_id == old_primitive_id()
862 || parts.serialized_id == new_primitive_id(),
863 "versioned primitive deserializer does not recognize {}",
864 parts.serialized_id,
865 );
866 vortex_ensure!(
867 parts.serialized_id != old_primitive_id() || parts.len <= 4,
868 "old primitive wire ID cannot represent length {}",
869 parts.len,
870 );
871 Ok(Array::<Primitive>::try_from_parts(ArrayVTable::deserialize(
872 &Primitive,
873 parts.dtype,
874 parts.len,
875 parts.metadata,
876 parts.buffers,
877 parts.children,
878 session,
879 )?)?
880 .into_array())
881 }
882 }
883
884 #[derive(Debug)]
885 struct CountingVersionedPrimitivePlugin;
886
887 impl ArrayPlugin for CountingVersionedPrimitivePlugin {
888 fn id(&self) -> ArrayId {
889 VersionedPrimitivePlugin.id()
890 }
891
892 fn read_ids(&self) -> Vec<ArrayId> {
893 VersionedPrimitivePlugin.read_ids()
894 }
895
896 fn write_ids(&self) -> Vec<ArrayId> {
897 VersionedPrimitivePlugin.write_ids()
898 }
899
900 fn serialize(
901 &self,
902 array: &ArrayRef,
903 session: &VortexSession,
904 ) -> VortexResult<Option<ArraySerialization>> {
905 SERIALIZER_CALLS.fetch_add(1, Ordering::Relaxed);
906 VersionedPrimitivePlugin.serialize(array, session)
907 }
908
909 fn deserialize(
910 &self,
911 parts: ArrayDeserialization<'_>,
912 session: &VortexSession,
913 ) -> VortexResult<ArrayRef> {
914 VersionedPrimitivePlugin.deserialize(parts, session)
915 }
916 }
917
918 fn versioned_primitive_session() -> VortexSession {
919 let session = VortexSession::empty();
920 session.arrays().register(VersionedPrimitivePlugin);
921 session
922 }
923
924 fn restricted_context(ids: &[ArrayId]) -> ArrayContext {
925 ArrayContext::new(ids.to_vec()).with_allowed_ids(ids.iter().copied().collect())
926 }
927
928 fn serialize_blob(
929 array: &ArrayRef,
930 ctx: &ArrayContext,
931 session: &VortexSession,
932 ) -> VortexResult<ByteBuffer> {
933 let mut blob = ByteBufferMut::empty();
934 for buffer in array.serialize(ctx, session, &SerializeOptions::default())? {
935 blob.extend_from_slice(buffer.as_ref());
936 }
937 Ok(blob.freeze())
938 }
939
940 #[test]
941 fn serialization_preserves_plugin_selected_old_wire_id() -> VortexResult<()> {
942 let session = VortexSession::empty();
943 session.arrays().register(CountingVersionedPrimitivePlugin);
944 let ctx = restricted_context(&[old_primitive_id(), new_primitive_id()]);
945 let array = PrimitiveArray::from_iter([1i32, 2, 3, 4]).into_array();
946
947 SERIALIZER_CALLS.store(0, Ordering::Relaxed);
948 let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &session)?)?;
949 assert_eq!(SERIALIZER_CALLS.load(Ordering::Relaxed), 1);
950 assert_eq!(
951 ReadContext::new(ctx.to_ids()).resolve(serialized.encoding_id()),
952 Some(old_primitive_id())
953 );
954 let decoded = serialized.decode(
955 array.dtype(),
956 array.len(),
957 &ReadContext::new(ctx.to_ids()),
958 &session,
959 )?;
960 assert_arrays_eq!(
961 decoded,
962 PrimitiveArray::from_iter([1i32, 2, 3, 4]),
963 &mut session.create_execution_ctx()
964 );
965 Ok(())
966 }
967
968 #[test]
969 fn serialization_preserves_plugin_selected_new_wire_id() -> VortexResult<()> {
970 let session = versioned_primitive_session();
971 let ctx = restricted_context(&[old_primitive_id(), new_primitive_id()]);
972 let array = PrimitiveArray::from_iter(0..8i32).into_array();
973 let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &session)?)?;
974 let read_ctx = ReadContext::new(ctx.to_ids());
975
976 assert_eq!(
977 read_ctx.resolve(serialized.encoding_id()),
978 Some(new_primitive_id())
979 );
980 let decoded = serialized.decode(array.dtype(), array.len(), &read_ctx, &session)?;
981 assert_eq!(decoded.encoding_id(), old_primitive_id());
982 assert_arrays_eq!(
983 decoded,
984 PrimitiveArray::from_iter(0..8i32),
985 &mut session.create_execution_ctx()
986 );
987 Ok(())
988 }
989
990 #[test]
991 fn serialization_fails_when_serialized_id_is_not_permitted() -> VortexResult<()> {
992 let session = versioned_primitive_session();
993 let ctx = restricted_context(&[old_primitive_id()]);
994 let array = PrimitiveArray::from_iter(0..8i32).into_array();
995
996 let error = array
997 .serialize(&ctx, &session, &SerializeOptions::default())
998 .expect_err("the serialized ID is not permitted");
999 assert!(error.to_string().contains("not permitted by ctx"));
1000 Ok(())
1001 }
1002
1003 #[test]
1004 fn old_reader_rejects_a_new_serialized_id() -> VortexResult<()> {
1005 let writer_session = versioned_primitive_session();
1006 let ctx = restricted_context(&[new_primitive_id()]);
1007 let array = PrimitiveArray::from_iter(0..8i32).into_array();
1008 let serialized = SerializedArray::try_from(serialize_blob(&array, &ctx, &writer_session)?)?;
1009 let read_ctx = ReadContext::new(ctx.to_ids());
1010
1011 let error = serialized
1012 .decode(array.dtype(), array.len(), &read_ctx, &SESSION)
1013 .expect_err("an old reader must not recognize the new wire ID");
1014 assert!(error.to_string().contains("Unknown encoding"));
1015 Ok(())
1016 }
1017
1018 #[test]
1019 fn deserializer_enforces_the_exact_wire_id_contract() -> VortexResult<()> {
1020 let session = versioned_primitive_session();
1021 let write_ctx = restricted_context(&[new_primitive_id()]);
1022 let array = PrimitiveArray::from_iter(0..8i32).into_array();
1023 let serialized = SerializedArray::try_from(serialize_blob(&array, &write_ctx, &session)?)?;
1024
1025 let error = serialized
1028 .decode(
1029 array.dtype(),
1030 array.len(),
1031 &ReadContext::new([old_primitive_id()]),
1032 &session,
1033 )
1034 .expect_err("the old wire contract must be enforced by the current deserializer");
1035 assert!(error.to_string().contains("old primitive wire ID"));
1036 Ok(())
1037 }
1038
1039 #[test]
1042 fn from_flatbuffer_and_segment_rejects_out_of_bounds_buffer() -> VortexResult<()> {
1043 let array_ctx = ArrayContext::empty();
1044
1045 let serialized = PrimitiveArray::from_iter([1i32, 2, 3, 4])
1048 .into_array()
1049 .serialize(&array_ctx, &SESSION, &SerializeOptions::default())?;
1050
1051 let mut concat = ByteBufferMut::empty();
1052 for buf in serialized {
1053 concat.extend_from_slice(buf.as_ref());
1054 }
1055 let value = concat.freeze().aligned(Alignment::none());
1056
1057 let fb_length = u32::try_from_le_bytes(&value.as_slice()[value.len() - 4..])? as usize;
1060 let fb_offset = value.len() - 4 - fb_length;
1061 assert!(
1062 fb_offset > 0,
1063 "the array must have at least one data buffer"
1064 );
1065 let array_tree = value.slice(fb_offset..fb_offset + fb_length);
1066
1067 let truncated = BufferHandle::new_host(value.slice(0..fb_offset - 1));
1069
1070 let Some(err) = SerializedArray::from_flatbuffer_and_segment(array_tree, truncated).err()
1071 else {
1072 vortex_bail!("out-of-bounds buffer must be rejected");
1073 };
1074 assert!(
1075 err.to_string().contains("out of bounds"),
1076 "unexpected error: {err}"
1077 );
1078
1079 Ok(())
1080 }
1081
1082 #[test]
1085 fn from_flatbuffer_and_segment_rejects_excessive_buffer_alignment() -> VortexResult<()> {
1086 let mut fbb = FlatBufferBuilder::new();
1089 let fb_root = fba::ArrayNode::create(&mut fbb, &fba::ArrayNodeArgs::default());
1090 let fb_buffers = fbb.create_vector(&[fba::Buffer::new(0, 40, Compression::None, 4)]);
1091 let fb_array = fba::Array::create(
1092 &mut fbb,
1093 &fba::ArrayArgs {
1094 root: Some(fb_root),
1095 buffers: Some(fb_buffers),
1096 },
1097 );
1098 fbb.finish_minimal(fb_array);
1099 let (fb_vec, fb_start) = fbb.collapse();
1100 let fb_end = fb_vec.len();
1101 let array_tree = ByteBuffer::from(fb_vec).slice(fb_start..fb_end);
1102
1103 let segment = BufferHandle::new_host(ByteBuffer::from(vec![0u8; 4]));
1104 let Some(err) = SerializedArray::from_flatbuffer_and_segment(array_tree, segment).err()
1105 else {
1106 vortex_bail!("excessive buffer alignment must be rejected");
1107 };
1108 assert!(
1109 err.to_string().contains("exceeds"),
1110 "unexpected error: {err}"
1111 );
1112
1113 Ok(())
1114 }
1115}