Skip to main content

vortex_array/arrays/varbin/vtable/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::hash::Hasher;
5
6use prost::Message;
7use vortex_error::VortexExpect;
8use vortex_error::VortexResult;
9use vortex_error::vortex_bail;
10use vortex_error::vortex_ensure;
11use vortex_error::vortex_ensure_eq;
12use vortex_error::vortex_err;
13use vortex_error::vortex_panic;
14use vortex_session::registry::CachedId;
15
16use crate::ArrayParts;
17use crate::ArrayRef;
18use crate::ExecutionCtx;
19use crate::ExecutionResult;
20use crate::IntoArray;
21use crate::VortexSessionExecute;
22use crate::array::Array;
23use crate::array::ArrayId;
24use crate::array::ArrayView;
25use crate::array::VTable;
26use crate::arrays::PrimitiveArray;
27use crate::arrays::varbin::VarBinArrayExt;
28use crate::arrays::varbin::VarBinArraySlotsExt;
29use crate::arrays::varbin::VarBinData;
30use crate::arrays::varbin::VarBinSlots;
31use crate::buffer::BufferHandle;
32use crate::builders::ArrayBuilder;
33use crate::builders::VarBinViewBuilder;
34use crate::dtype::DType;
35use crate::dtype::Nullability;
36use crate::dtype::PType;
37use crate::match_each_integer_ptype;
38use crate::match_each_varbin_builder;
39use crate::serde::ArrayChildren;
40use crate::validity::Validity;
41pub(crate) mod canonical;
42mod kernel;
43mod operations;
44mod validity;
45
46use canonical::varbin_to_canonical;
47use vortex_session::VortexSession;
48
49use crate::EqMode;
50use crate::arrays::varbin::compute::rules::PARENT_RULES;
51use crate::hash::ArrayEq;
52use crate::hash::ArrayHash;
53
54/// A [`VarBin`]-encoded Vortex array.
55pub type VarBinArray = Array<VarBin>;
56
57pub(crate) fn initialize(session: &VortexSession) {
58    kernel::initialize(session);
59}
60
61#[derive(Clone, prost::Message)]
62pub struct VarBinMetadata {
63    #[prost(int32, tag = "1")]
64    pub(crate) offsets_ptype: i32,
65}
66
67impl VarBinMetadata {
68    fn get_offsets_ptype(&self) -> VortexResult<PType> {
69        PType::try_from(self.offsets_ptype).map_err(|_| {
70            vortex_err!(
71                "VarBinMetadata offsets_ptype must be a known PType, got {}",
72                self.offsets_ptype
73            )
74        })
75    }
76}
77
78impl ArrayHash for VarBinData {
79    fn array_hash<H: Hasher>(&self, state: &mut H, accuracy: EqMode) {
80        self.bytes().array_hash(state, accuracy);
81    }
82}
83
84impl ArrayEq for VarBinData {
85    fn array_eq(&self, other: &Self, accuracy: EqMode) -> bool {
86        self.bytes().array_eq(other.bytes(), accuracy)
87    }
88}
89
90impl VTable for VarBin {
91    type TypedArrayData = VarBinData;
92
93    type OperationsVTable = Self;
94    type ValidityVTable = Self;
95    fn id(&self) -> ArrayId {
96        static ID: CachedId = CachedId::new("vortex.varbin");
97        *ID
98    }
99
100    fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
101        1
102    }
103
104    fn validate(
105        &self,
106        _data: &VarBinData,
107        dtype: &DType,
108        len: usize,
109        slots: &[Option<ArrayRef>],
110    ) -> VortexResult<()> {
111        vortex_ensure_eq!(slots.len(), VarBinSlots::COUNT);
112        let offsets = slots[VarBinSlots::OFFSETS]
113            .as_ref()
114            .vortex_expect("VarBinArray offsets slot");
115        vortex_ensure_eq!(
116            offsets.len().saturating_sub(1),
117            len,
118            "VarBinArray length does not match outer length",
119        );
120        vortex_ensure!(
121            matches!(dtype, DType::Binary(_) | DType::Utf8(_)),
122            "VarBinArray dtype must be binary or utf8, got {dtype}"
123        );
124        Ok(())
125    }
126
127    fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle {
128        match idx {
129            0 => array.bytes_handle().clone(),
130            _ => vortex_panic!("VarBinArray buffer index {idx} out of bounds"),
131        }
132    }
133
134    fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option<String> {
135        match idx {
136            0 => Some("bytes".to_string()),
137            _ => vortex_panic!("VarBinArray buffer_name index {idx} out of bounds"),
138        }
139    }
140
141    fn with_buffers(
142        &self,
143        array: ArrayView<'_, Self>,
144        buffers: &[BufferHandle],
145    ) -> VortexResult<ArrayParts<Self>> {
146        vortex_ensure_eq!(buffers.len(), 1);
147        let mut data = array.data().clone();
148        data.bytes = buffers[0].clone();
149        Ok(ArrayParts::new(
150            self.clone(),
151            array.dtype().clone(),
152            array.len(),
153            data,
154            array.slots().iter().cloned().collect(),
155        ))
156    }
157
158    fn serialize(
159        array: ArrayView<'_, Self>,
160        _session: &VortexSession,
161    ) -> VortexResult<Option<Vec<u8>>> {
162        Ok(Some(
163            VarBinMetadata {
164                offsets_ptype: PType::try_from(array.offsets().dtype())
165                    .vortex_expect("Must be a valid PType") as i32,
166            }
167            .encode_to_vec(),
168        ))
169    }
170
171    fn deserialize(
172        &self,
173        dtype: &DType,
174        len: usize,
175        metadata: &[u8],
176        buffers: &[BufferHandle],
177        children: &dyn ArrayChildren,
178        session: &VortexSession,
179    ) -> VortexResult<ArrayParts<Self>> {
180        let metadata = VarBinMetadata::decode(metadata)?;
181        let offsets_ptype = metadata.get_offsets_ptype()?;
182        let validity = if children.len() == 1 {
183            Validity::from(dtype.nullability())
184        } else if children.len() == 2 {
185            let validity = children.get(1, &Validity::DTYPE, len)?;
186            Validity::Array(validity)
187        } else {
188            vortex_bail!("Expected 1 or 2 children, got {}", children.len());
189        };
190
191        let offsets = children.get(
192            0,
193            &DType::Primitive(offsets_ptype, Nullability::NonNullable),
194            len + 1,
195        )?;
196
197        if buffers.len() != 1 {
198            vortex_bail!("Expected 1 buffer, got {}", buffers.len());
199        }
200        let bytes = buffers[0].clone().try_to_host_sync()?;
201
202        let data = VarBinData::try_build(
203            offsets.clone(),
204            bytes,
205            dtype.clone(),
206            validity.clone(),
207            &mut session.create_execution_ctx(),
208        )?;
209        let slots = VarBinData::make_slots(offsets, &validity, len);
210        Ok(ArrayParts::new(
211            self.clone(),
212            dtype.clone(),
213            len,
214            data,
215            slots,
216        ))
217    }
218
219    fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
220        VarBinSlots::NAMES[idx].to_string()
221    }
222
223    fn reduce_parent(
224        array: ArrayView<'_, Self>,
225        parent: &ArrayRef,
226        child_idx: usize,
227    ) -> VortexResult<Option<ArrayRef>> {
228        PARENT_RULES.evaluate(array, parent, child_idx)
229    }
230
231    fn append_to_builder(
232        array: ArrayView<'_, Self>,
233        builder: &mut dyn ArrayBuilder,
234        ctx: &mut ExecutionCtx,
235    ) -> VortexResult<()> {
236        if let Some(result) =
237            match_each_varbin_builder!(builder, |builder| builder.append_varbin(array, ctx))
238        {
239            return result;
240        }
241
242        // The two arms here are every builder a `Utf8`/`Binary` dtype has: all four
243        // `VarBinBuilder` widths above, and `VarBinViewBuilder` below.
244        let Some(builder) = builder.as_any_mut().downcast_mut::<VarBinViewBuilder>() else {
245            vortex_bail!("append_to_builder for VarBin requires a variable-binary builder")
246        };
247        append_to_varbinview(array, builder, ctx)
248    }
249
250    fn execute(array: Array<Self>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
251        Ok(ExecutionResult::done(
252            varbin_to_canonical(array.as_view(), ctx)?.into_array(),
253        ))
254    }
255}
256
257/// Hands the value bytes to `builder` as a data buffer with views built over them.
258///
259/// Canonicalizing first would build the same views, then pay for them twice more: once to wrap
260/// them in a `VarBinViewArray` the builder immediately unwraps, and once for
261/// `append_varbinview_array` to rewrite every view so its buffer index is rebased onto the
262/// builder's. Handing the heap and offsets to the builder instead makes the whole append one view
263/// per row with no byte copy — the builder adopts the referenced range of the heap as it is. That
264/// range is fully covered by the new views, so this stays valid for a compacting builder too.
265fn append_to_varbinview(
266    array: ArrayView<'_, VarBin>,
267    builder: &mut VarBinViewBuilder,
268    ctx: &mut ExecutionCtx,
269) -> VortexResult<()> {
270    let len = array.as_ref().len();
271    let validity = array.varbin_validity().execute_mask(len, ctx)?;
272
273    let parts = array.into_owned().into_data_parts();
274    let offsets = parts.offsets.execute::<PrimitiveArray>(ctx)?;
275    match_each_integer_ptype!(offsets.ptype(), |P| {
276        builder.append_buffer_with_offsets(
277            parts.bytes.unwrap_host(),
278            offsets.as_slice::<P>(),
279            &validity,
280        )
281    });
282    Ok(())
283}
284
285#[derive(Clone, Debug)]
286pub struct VarBin;
287
288#[cfg(test)]
289mod tests {
290    use std::sync::LazyLock;
291
292    use prost::Message;
293    use vortex_error::VortexResult;
294    use vortex_session::VortexSession;
295
296    use super::VarBin;
297    use super::VarBinMetadata;
298    use crate::ArrayRef;
299    use crate::array::VTable;
300    use crate::dtype::DType;
301    use crate::dtype::Nullability;
302    use crate::dtype::PType;
303
304    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
305
306    #[test]
307    fn test_deserialize_rejects_unknown_offsets_ptype() -> VortexResult<()> {
308        let metadata = VarBinMetadata { offsets_ptype: 11 }.encode_to_vec();
309        let children: Vec<ArrayRef> = Vec::new();
310        let result = VarBin.deserialize(
311            &DType::Utf8(Nullability::NonNullable),
312            0,
313            &metadata,
314            &[],
315            &children,
316            &SESSION,
317        );
318        assert!(result.is_err_and(|err| {
319            err.to_string()
320                .contains("VarBinMetadata offsets_ptype must be a known PType, got 11")
321        }));
322
323        let known = VarBinMetadata {
324            offsets_ptype: PType::U32 as i32,
325        };
326        assert_eq!(known.get_offsets_ptype()?, PType::U32);
327
328        Ok(())
329    }
330}