vortex_array/arrays/listview/vtable/
mod.rs1use std::hash::Hash;
5use std::hash::Hasher;
6use std::sync::Arc;
7
8use prost::Message;
9use vortex_error::VortexExpect;
10use vortex_error::VortexResult;
11use vortex_error::vortex_bail;
12use vortex_error::vortex_ensure;
13use vortex_error::vortex_ensure_eq;
14use vortex_error::vortex_err;
15use vortex_error::vortex_panic;
16use vortex_session::VortexSession;
17use vortex_session::registry::CachedId;
18
19use crate::ArrayEq;
20use crate::ArrayHash;
21use crate::ArrayParts;
22use crate::ArrayRef;
23use crate::EqMode;
24use crate::ExecutionCtx;
25use crate::ExecutionResult;
26use crate::VortexSessionExecute;
27use crate::array::Array;
28use crate::array::ArrayId;
29use crate::array::ArrayView;
30use crate::array::VTable;
31use crate::array::with_empty_buffers;
32use crate::arrays::listview::ListViewArraySlotsExt;
33use crate::arrays::listview::ListViewData;
34use crate::arrays::listview::ListViewSlots;
35use crate::arrays::listview::compute::rules::PARENT_RULES;
36use crate::buffer::BufferHandle;
37use crate::builders::ArrayBuilder;
38use crate::dtype::DType;
39use crate::dtype::Nullability;
40use crate::dtype::PType;
41use crate::match_each_list_builder;
42use crate::serde::ArrayChildren;
43use crate::validity::Validity;
44mod kernel;
45mod operations;
46mod validity;
47pub type ListViewArray = Array<ListView>;
49
50pub(crate) fn initialize(session: &VortexSession) {
51 kernel::initialize(session);
52}
53
54#[derive(Clone, Debug)]
55pub struct ListView;
56
57#[derive(Clone, prost::Message)]
58pub struct ListViewMetadata {
59 #[prost(uint64, tag = "1")]
60 elements_len: u64,
61 #[prost(int32, tag = "2")]
62 offset_ptype: i32,
63 #[prost(int32, tag = "3")]
64 size_ptype: i32,
65}
66
67impl ListViewMetadata {
68 fn get_offset_ptype(&self) -> VortexResult<PType> {
69 PType::try_from(self.offset_ptype).map_err(|_| {
70 vortex_err!(
71 "ListViewMetadata offset_ptype must be a known PType, got {}",
72 self.offset_ptype
73 )
74 })
75 }
76
77 fn get_size_ptype(&self) -> VortexResult<PType> {
78 PType::try_from(self.size_ptype).map_err(|_| {
79 vortex_err!(
80 "ListViewMetadata size_ptype must be a known PType, got {}",
81 self.size_ptype
82 )
83 })
84 }
85}
86
87impl ArrayHash for ListViewData {
88 fn array_hash<H: Hasher>(&self, state: &mut H, _accuracy: EqMode) {
89 self.is_zero_copy_to_list().hash(state);
90 }
91}
92
93impl ArrayEq for ListViewData {
94 fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool {
95 self.is_zero_copy_to_list() == other.is_zero_copy_to_list()
96 }
97}
98
99impl VTable for ListView {
100 type TypedArrayData = ListViewData;
101
102 type OperationsVTable = Self;
103 type ValidityVTable = Self;
104 fn id(&self) -> ArrayId {
105 static ID: CachedId = CachedId::new("vortex.listview");
106 *ID
107 }
108
109 fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
110 0
111 }
112
113 fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle {
114 vortex_panic!("ListViewArray buffer index {idx} out of bounds")
115 }
116
117 fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option<String> {
118 vortex_panic!("ListViewArray buffer_name index {idx} out of bounds")
119 }
120
121 fn with_buffers(
122 &self,
123 array: ArrayView<'_, Self>,
124 buffers: &[BufferHandle],
125 ) -> VortexResult<ArrayParts<Self>> {
126 with_empty_buffers(self, array, buffers)
127 }
128
129 fn serialize(
130 array: ArrayView<'_, Self>,
131 _session: &VortexSession,
132 ) -> VortexResult<Option<Vec<u8>>> {
133 Ok(Some(
134 ListViewMetadata {
135 elements_len: array.elements().len() as u64,
136 offset_ptype: PType::try_from(array.offsets().dtype())? as i32,
137 size_ptype: PType::try_from(array.sizes().dtype())? as i32,
138 }
139 .encode_to_vec(),
140 ))
141 }
142
143 fn validate(
144 &self,
145 _data: &ListViewData,
146 dtype: &DType,
147 len: usize,
148 slots: &[Option<ArrayRef>],
149 ) -> VortexResult<()> {
150 vortex_ensure_eq!(slots.len(), ListViewSlots::COUNT);
151 let elements = slots[ListViewSlots::ELEMENTS]
152 .as_ref()
153 .vortex_expect("ListViewArray elements slot");
154 let offsets = slots[ListViewSlots::OFFSETS]
155 .as_ref()
156 .vortex_expect("ListViewArray offsets slot");
157 let sizes = slots[ListViewSlots::SIZES]
158 .as_ref()
159 .vortex_expect("ListViewArray sizes slot");
160 vortex_ensure!(
161 offsets.len() == len && sizes.len() == len,
162 "ListViewArray length {} does not match outer length {}",
163 offsets.len(),
164 len
165 );
166
167 let actual_dtype = DType::List(Arc::new(elements.dtype().clone()), dtype.nullability());
168 vortex_ensure_eq!(
169 &actual_dtype,
170 dtype,
171 "ListViewArray dtype does not match outer dtype",
172 );
173
174 Ok(())
175 }
176
177 fn deserialize(
178 &self,
179 dtype: &DType,
180 len: usize,
181 metadata: &[u8],
182
183 buffers: &[BufferHandle],
184 children: &dyn ArrayChildren,
185 session: &VortexSession,
186 ) -> VortexResult<ArrayParts<Self>> {
187 let metadata = ListViewMetadata::decode(metadata)?;
188 let offset_ptype = metadata.get_offset_ptype()?;
189 let size_ptype = metadata.get_size_ptype()?;
190 vortex_ensure!(
191 buffers.is_empty(),
192 "`ListViewArray::build` expects no buffers"
193 );
194
195 let DType::List(element_dtype, _) = dtype else {
196 vortex_bail!("Expected List dtype, got {:?}", dtype);
197 };
198
199 let validity = if children.len() == 3 {
200 Validity::from(dtype.nullability())
201 } else if children.len() == 4 {
202 let validity = children.get(3, &Validity::DTYPE, len)?;
203 Validity::Array(validity)
204 } else {
205 vortex_bail!(
206 "`ListViewArray::build` expects 3 or 4 children, got {}",
207 children.len()
208 );
209 };
210
211 let elements = children.get(
213 0,
214 element_dtype.as_ref(),
215 usize::try_from(metadata.elements_len)?,
216 )?;
217
218 let offsets = children.get(
220 1,
221 &DType::Primitive(offset_ptype, Nullability::NonNullable),
222 len,
223 )?;
224
225 let sizes = children.get(
227 2,
228 &DType::Primitive(size_ptype, Nullability::NonNullable),
229 len,
230 )?;
231
232 ListViewData::validate(
233 &elements,
234 &offsets,
235 &sizes,
236 &validity,
237 &mut session.create_execution_ctx(),
238 )?;
239 let data = ListViewData::try_new()?;
240 let slots = ListViewData::make_slots(&elements, &offsets, &sizes, &validity, len);
241 Ok(ArrayParts::new(
242 self.clone(),
243 dtype.clone(),
244 len,
245 data,
246 slots,
247 ))
248 }
249
250 fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
251 ListViewSlots::NAMES[idx].to_string()
252 }
253
254 fn execute(array: Array<Self>, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
255 Ok(ExecutionResult::done(array))
256 }
257
258 #[expect(clippy::cognitive_complexity)]
260 fn append_to_builder(
261 array: ArrayView<'_, Self>,
262 builder: &mut dyn ArrayBuilder,
263 ctx: &mut ExecutionCtx,
264 ) -> VortexResult<()> {
265 match match_each_list_builder!(&mut *builder, |b| b.append_listview_array(array, ctx)) {
266 Some(result) => result,
267 None => vortex_bail!(
268 "cannot append a ListView array of dtype {} to a {} builder",
269 array.dtype(),
270 builder.dtype()
271 ),
272 }
273 }
274
275 fn reduce_parent(
276 array: ArrayView<'_, Self>,
277 parent: &ArrayRef,
278 child_idx: usize,
279 ) -> VortexResult<Option<ArrayRef>> {
280 PARENT_RULES.evaluate(array, parent, child_idx)
281 }
282}
283
284#[cfg(test)]
285mod tests {
286 use std::sync::Arc;
287 use std::sync::LazyLock;
288
289 use prost::Message;
290 use rstest::rstest;
291 use vortex_error::VortexResult;
292 use vortex_session::VortexSession;
293
294 use super::ListView;
295 use super::ListViewMetadata;
296 use crate::ArrayRef;
297 use crate::array::VTable;
298 use crate::dtype::DType;
299 use crate::dtype::Nullability;
300 use crate::dtype::PType;
301
302 static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
303
304 #[rstest]
305 #[case::offset_ptype(
306 |raw| ListViewMetadata { offset_ptype: raw, ..Default::default() },
307 ListViewMetadata::get_offset_ptype,
308 "ListViewMetadata offset_ptype must be a known PType, got 11",
309 )]
310 #[case::size_ptype(
311 |raw| ListViewMetadata { size_ptype: raw, ..Default::default() },
312 ListViewMetadata::get_size_ptype,
313 "ListViewMetadata size_ptype must be a known PType, got 11",
314 )]
315 fn test_deserialize_rejects_unknown_ptype(
316 #[case] with_ptype: fn(i32) -> ListViewMetadata,
317 #[case] get_ptype: fn(&ListViewMetadata) -> VortexResult<PType>,
318 #[case] expected: &str,
319 ) -> VortexResult<()> {
320 let dtype = DType::List(
321 Arc::new(DType::Primitive(PType::I32, Nullability::NonNullable)),
322 Nullability::NonNullable,
323 );
324 let metadata = with_ptype(11).encode_to_vec();
325 let children: Vec<ArrayRef> = Vec::new();
326 let result = ListView.deserialize(&dtype, 0, &metadata, &[], &children, &SESSION);
327 assert!(result.is_err_and(|err| err.to_string().contains(expected)));
328
329 assert_eq!(get_ptype(&with_ptype(PType::U32 as i32))?, PType::U32);
330
331 Ok(())
332 }
333}