vortex_array/arrays/varbin/vtable/
mod.rs1use std::hash::Hasher;
5
6use prost::Message;
7use vortex_error::VortexExpect;
8use vortex_error::VortexResult;
9use vortex_error::vortex_bail;
10use vortex_error::vortex_ensure;
11use vortex_error::vortex_ensure_eq;
12use vortex_error::vortex_err;
13use vortex_error::vortex_panic;
14use vortex_session::registry::CachedId;
15
16use crate::ArrayParts;
17use crate::ArrayRef;
18use crate::ExecutionCtx;
19use crate::ExecutionResult;
20use crate::IntoArray;
21use crate::VortexSessionExecute;
22use crate::array::Array;
23use crate::array::ArrayId;
24use crate::array::ArrayView;
25use crate::array::VTable;
26use crate::arrays::PrimitiveArray;
27use crate::arrays::varbin::VarBinArrayExt;
28use crate::arrays::varbin::VarBinArraySlotsExt;
29use crate::arrays::varbin::VarBinData;
30use crate::arrays::varbin::VarBinSlots;
31use crate::buffer::BufferHandle;
32use crate::builders::ArrayBuilder;
33use crate::builders::VarBinViewBuilder;
34use crate::dtype::DType;
35use crate::dtype::Nullability;
36use crate::dtype::PType;
37use crate::match_each_integer_ptype;
38use crate::match_each_varbin_builder;
39use crate::serde::ArrayChildren;
40use crate::validity::Validity;
41pub(crate) mod canonical;
42mod kernel;
43mod operations;
44mod validity;
45
46use canonical::varbin_to_canonical;
47use vortex_session::VortexSession;
48
49use crate::EqMode;
50use crate::arrays::varbin::compute::rules::PARENT_RULES;
51use crate::hash::ArrayEq;
52use crate::hash::ArrayHash;
53
54pub type VarBinArray = Array<VarBin>;
56
57pub(crate) fn initialize(session: &VortexSession) {
58 kernel::initialize(session);
59}
60
61#[derive(Clone, prost::Message)]
62pub struct VarBinMetadata {
63 #[prost(int32, tag = "1")]
64 pub(crate) offsets_ptype: i32,
65}
66
67impl VarBinMetadata {
68 fn get_offsets_ptype(&self) -> VortexResult<PType> {
69 PType::try_from(self.offsets_ptype).map_err(|_| {
70 vortex_err!(
71 "VarBinMetadata offsets_ptype must be a known PType, got {}",
72 self.offsets_ptype
73 )
74 })
75 }
76}
77
78impl ArrayHash for VarBinData {
79 fn array_hash<H: Hasher>(&self, state: &mut H, accuracy: EqMode) {
80 self.bytes().array_hash(state, accuracy);
81 }
82}
83
84impl ArrayEq for VarBinData {
85 fn array_eq(&self, other: &Self, accuracy: EqMode) -> bool {
86 self.bytes().array_eq(other.bytes(), accuracy)
87 }
88}
89
90impl VTable for VarBin {
91 type TypedArrayData = VarBinData;
92
93 type OperationsVTable = Self;
94 type ValidityVTable = Self;
95 fn id(&self) -> ArrayId {
96 static ID: CachedId = CachedId::new("vortex.varbin");
97 *ID
98 }
99
100 fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
101 1
102 }
103
104 fn validate(
105 &self,
106 _data: &VarBinData,
107 dtype: &DType,
108 len: usize,
109 slots: &[Option<ArrayRef>],
110 ) -> VortexResult<()> {
111 vortex_ensure_eq!(slots.len(), VarBinSlots::COUNT);
112 let offsets = slots[VarBinSlots::OFFSETS]
113 .as_ref()
114 .vortex_expect("VarBinArray offsets slot");
115 vortex_ensure_eq!(
116 offsets.len().saturating_sub(1),
117 len,
118 "VarBinArray length does not match outer length",
119 );
120 vortex_ensure!(
121 matches!(dtype, DType::Binary(_) | DType::Utf8(_)),
122 "VarBinArray dtype must be binary or utf8, got {dtype}"
123 );
124 Ok(())
125 }
126
127 fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle {
128 match idx {
129 0 => array.bytes_handle().clone(),
130 _ => vortex_panic!("VarBinArray buffer index {idx} out of bounds"),
131 }
132 }
133
134 fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option<String> {
135 match idx {
136 0 => Some("bytes".to_string()),
137 _ => vortex_panic!("VarBinArray buffer_name index {idx} out of bounds"),
138 }
139 }
140
141 fn with_buffers(
142 &self,
143 array: ArrayView<'_, Self>,
144 buffers: &[BufferHandle],
145 ) -> VortexResult<ArrayParts<Self>> {
146 vortex_ensure_eq!(buffers.len(), 1);
147 let mut data = array.data().clone();
148 data.bytes = buffers[0].clone();
149 Ok(ArrayParts::new(
150 self.clone(),
151 array.dtype().clone(),
152 array.len(),
153 data,
154 array.slots().iter().cloned().collect(),
155 ))
156 }
157
158 fn serialize(
159 array: ArrayView<'_, Self>,
160 _session: &VortexSession,
161 ) -> VortexResult<Option<Vec<u8>>> {
162 Ok(Some(
163 VarBinMetadata {
164 offsets_ptype: PType::try_from(array.offsets().dtype())
165 .vortex_expect("Must be a valid PType") as i32,
166 }
167 .encode_to_vec(),
168 ))
169 }
170
171 fn deserialize(
172 &self,
173 dtype: &DType,
174 len: usize,
175 metadata: &[u8],
176 buffers: &[BufferHandle],
177 children: &dyn ArrayChildren,
178 session: &VortexSession,
179 ) -> VortexResult<ArrayParts<Self>> {
180 let metadata = VarBinMetadata::decode(metadata)?;
181 let offsets_ptype = metadata.get_offsets_ptype()?;
182 let validity = if children.len() == 1 {
183 Validity::from(dtype.nullability())
184 } else if children.len() == 2 {
185 let validity = children.get(1, &Validity::DTYPE, len)?;
186 Validity::Array(validity)
187 } else {
188 vortex_bail!("Expected 1 or 2 children, got {}", children.len());
189 };
190
191 let offsets = children.get(
192 0,
193 &DType::Primitive(offsets_ptype, Nullability::NonNullable),
194 len + 1,
195 )?;
196
197 if buffers.len() != 1 {
198 vortex_bail!("Expected 1 buffer, got {}", buffers.len());
199 }
200 let bytes = buffers[0].clone().try_to_host_sync()?;
201
202 let data = VarBinData::try_build(
203 offsets.clone(),
204 bytes,
205 dtype.clone(),
206 validity.clone(),
207 &mut session.create_execution_ctx(),
208 )?;
209 let slots = VarBinData::make_slots(offsets, &validity, len);
210 Ok(ArrayParts::new(
211 self.clone(),
212 dtype.clone(),
213 len,
214 data,
215 slots,
216 ))
217 }
218
219 fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
220 VarBinSlots::NAMES[idx].to_string()
221 }
222
223 fn reduce_parent(
224 array: ArrayView<'_, Self>,
225 parent: &ArrayRef,
226 child_idx: usize,
227 ) -> VortexResult<Option<ArrayRef>> {
228 PARENT_RULES.evaluate(array, parent, child_idx)
229 }
230
231 fn append_to_builder(
232 array: ArrayView<'_, Self>,
233 builder: &mut dyn ArrayBuilder,
234 ctx: &mut ExecutionCtx,
235 ) -> VortexResult<()> {
236 if let Some(result) =
237 match_each_varbin_builder!(builder, |builder| builder.append_varbin(array, ctx))
238 {
239 return result;
240 }
241
242 let Some(builder) = builder.as_any_mut().downcast_mut::<VarBinViewBuilder>() else {
245 vortex_bail!("append_to_builder for VarBin requires a variable-binary builder")
246 };
247 append_to_varbinview(array, builder, ctx)
248 }
249
250 fn execute(array: Array<Self>, ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
251 Ok(ExecutionResult::done(
252 varbin_to_canonical(array.as_view(), ctx)?.into_array(),
253 ))
254 }
255}
256
257fn append_to_varbinview(
266 array: ArrayView<'_, VarBin>,
267 builder: &mut VarBinViewBuilder,
268 ctx: &mut ExecutionCtx,
269) -> VortexResult<()> {
270 let len = array.as_ref().len();
271 let validity = array.varbin_validity().execute_mask(len, ctx)?;
272
273 let parts = array.into_owned().into_data_parts();
274 let offsets = parts.offsets.execute::<PrimitiveArray>(ctx)?;
275 match_each_integer_ptype!(offsets.ptype(), |P| {
276 builder.append_buffer_with_offsets(
277 parts.bytes.unwrap_host(),
278 offsets.as_slice::<P>(),
279 &validity,
280 )
281 });
282 Ok(())
283}
284
285#[derive(Clone, Debug)]
286pub struct VarBin;
287
288#[cfg(test)]
289mod tests {
290 use std::sync::LazyLock;
291
292 use prost::Message;
293 use vortex_error::VortexResult;
294 use vortex_session::VortexSession;
295
296 use super::VarBin;
297 use super::VarBinMetadata;
298 use crate::ArrayRef;
299 use crate::array::VTable;
300 use crate::dtype::DType;
301 use crate::dtype::Nullability;
302 use crate::dtype::PType;
303
304 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
305
306 #[test]
307 fn test_deserialize_rejects_unknown_offsets_ptype() -> VortexResult<()> {
308 let metadata = VarBinMetadata { offsets_ptype: 11 }.encode_to_vec();
309 let children: Vec<ArrayRef> = Vec::new();
310 let result = VarBin.deserialize(
311 &DType::Utf8(Nullability::NonNullable),
312 0,
313 &metadata,
314 &[],
315 &children,
316 &SESSION,
317 );
318 assert!(result.is_err_and(|err| {
319 err.to_string()
320 .contains("VarBinMetadata offsets_ptype must be a known PType, got 11")
321 }));
322
323 let known = VarBinMetadata {
324 offsets_ptype: PType::U32 as i32,
325 };
326 assert_eq!(known.get_offsets_ptype()?, PType::U32);
327
328 Ok(())
329 }
330}