vortex_fastlanes/delta/vtable/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use fastlanes::FastLanes;
5use prost::Message;
6use vortex_array::ArrayRef;
7use vortex_array::ExecutionCtx;
8use vortex_array::IntoArray;
9use vortex_array::ProstMetadata;
10use vortex_array::buffer::BufferHandle;
11use vortex_array::dtype::DType;
12use vortex_array::dtype::PType;
13use vortex_array::match_each_unsigned_integer_ptype;
14use vortex_array::serde::ArrayChildren;
15use vortex_array::vtable;
16use vortex_array::vtable::ArrayId;
17use vortex_array::vtable::VTable;
18use vortex_array::vtable::ValidityVTableFromChildSliceHelper;
19use vortex_error::VortexResult;
20use vortex_error::vortex_ensure;
21use vortex_error::vortex_err;
22use vortex_session::VortexSession;
23
24use crate::DeltaArray;
25use crate::delta::array::delta_decompress::delta_decompress;
26
27mod array;
28mod operations;
29mod rules;
30mod slice;
31mod validity;
32mod visitor;
33
34vtable!(Delta);
35
36#[derive(Clone, prost::Message)]
37#[repr(C)]
38pub struct DeltaMetadata {
39    #[prost(uint64, tag = "1")]
40    deltas_len: u64,
41    #[prost(uint32, tag = "2")]
42    offset: u32, // must be <1024
43}
44
45impl VTable for DeltaVTable {
46    type Array = DeltaArray;
47
48    type Metadata = ProstMetadata<DeltaMetadata>;
49
50    type ArrayVTable = Self;
51    type OperationsVTable = Self;
52    type ValidityVTable = ValidityVTableFromChildSliceHelper;
53    type VisitorVTable = Self;
54
55    fn id(_array: &Self::Array) -> ArrayId {
56        Self::ID
57    }
58
59    fn reduce_parent(
60        array: &Self::Array,
61        parent: &ArrayRef,
62        child_idx: usize,
63    ) -> VortexResult<Option<ArrayRef>> {
64        rules::RULES.evaluate(array, parent, child_idx)
65    }
66
67    fn with_children(array: &mut Self::Array, children: Vec<ArrayRef>) -> VortexResult<()> {
68        // DeltaArray children order (from visit_children):
69        // 1. bases
70        // 2. deltas
71
72        vortex_ensure!(
73            children.len() == 2,
74            "Expected 2 children for Delta encoding, got {}",
75            children.len()
76        );
77
78        array.bases = children[0].clone();
79        array.deltas = children[1].clone();
80
81        Ok(())
82    }
83
84    fn metadata(array: &DeltaArray) -> VortexResult<Self::Metadata> {
85        Ok(ProstMetadata(DeltaMetadata {
86            deltas_len: array.deltas().len() as u64,
87            offset: array.offset() as u32,
88        }))
89    }
90
91    fn serialize(metadata: Self::Metadata) -> VortexResult<Option<Vec<u8>>> {
92        Ok(Some(metadata.0.encode_to_vec()))
93    }
94
95    fn deserialize(
96        bytes: &[u8],
97        _dtype: &DType,
98        _len: usize,
99        _buffers: &[BufferHandle],
100        _session: &VortexSession,
101    ) -> VortexResult<Self::Metadata> {
102        Ok(ProstMetadata(DeltaMetadata::decode(bytes)?))
103    }
104
105    fn build(
106        dtype: &DType,
107        len: usize,
108        metadata: &Self::Metadata,
109        _buffers: &[BufferHandle],
110        children: &dyn ArrayChildren,
111    ) -> VortexResult<DeltaArray> {
112        assert_eq!(children.len(), 2);
113        let ptype = PType::try_from(dtype)?;
114        let lanes = match_each_unsigned_integer_ptype!(ptype, |T| { <T as FastLanes>::LANES });
115
116        // Compute the length of the bases array
117        let deltas_len = usize::try_from(metadata.0.deltas_len)
118            .map_err(|_| vortex_err!("deltas_len {} overflowed usize", metadata.0.deltas_len))?;
119        let num_chunks = deltas_len / 1024;
120        let remainder_base_size = if deltas_len % 1024 > 0 { 1 } else { 0 };
121        let bases_len = num_chunks * lanes + remainder_base_size;
122
123        let bases = children.get(0, dtype, bases_len)?;
124        let deltas = children.get(1, dtype, deltas_len)?;
125
126        DeltaArray::try_new(bases, deltas, metadata.0.offset as usize, len)
127    }
128
129    fn execute(array: &Self::Array, ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
130        Ok(delta_decompress(array, ctx)?.into_array())
131    }
132}
133
134#[derive(Debug)]
135pub struct DeltaVTable;
136
137impl DeltaVTable {
138    pub const ID: ArrayId = ArrayId::new_ref("fastlanes.delta");
139}
140
141#[cfg(test)]
142mod tests {
143    use vortex_array::test_harness::check_metadata;
144
145    use super::DeltaMetadata;
146    use super::ProstMetadata;
147
148    #[cfg_attr(miri, ignore)]
149    #[test]
150    fn test_delta_metadata() {
151        check_metadata(
152            "delta.metadata",
153            ProstMetadata(DeltaMetadata {
154                offset: u32::MAX,
155                deltas_len: u64::MAX,
156            }),
157        );
158    }
159}
vortex_fastlanes/delta/vtable/mod.rs

vortex_fastlanes/delta/vtable/
mod.rs
