Skip to main content

vortex_array/arrays/listview/vtable/
mod.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use std::hash::Hash;
5use std::hash::Hasher;
6use std::sync::Arc;
7
8use prost::Message;
9use vortex_error::VortexExpect;
10use vortex_error::VortexResult;
11use vortex_error::vortex_bail;
12use vortex_error::vortex_ensure;
13use vortex_error::vortex_ensure_eq;
14use vortex_error::vortex_err;
15use vortex_error::vortex_panic;
16use vortex_session::VortexSession;
17use vortex_session::registry::CachedId;
18
19use crate::ArrayEq;
20use crate::ArrayHash;
21use crate::ArrayParts;
22use crate::ArrayRef;
23use crate::EqMode;
24use crate::ExecutionCtx;
25use crate::ExecutionResult;
26use crate::VortexSessionExecute;
27use crate::array::Array;
28use crate::array::ArrayId;
29use crate::array::ArrayView;
30use crate::array::VTable;
31use crate::array::with_empty_buffers;
32use crate::arrays::listview::ListViewArraySlotsExt;
33use crate::arrays::listview::ListViewData;
34use crate::arrays::listview::ListViewSlots;
35use crate::arrays::listview::compute::rules::PARENT_RULES;
36use crate::buffer::BufferHandle;
37use crate::builders::ArrayBuilder;
38use crate::dtype::DType;
39use crate::dtype::Nullability;
40use crate::dtype::PType;
41use crate::match_each_list_builder;
42use crate::serde::ArrayChildren;
43use crate::validity::Validity;
44mod kernel;
45mod operations;
46mod validity;
47/// A [`ListView`]-encoded Vortex array.
48pub type ListViewArray = Array<ListView>;
49
50pub(crate) fn initialize(session: &VortexSession) {
51    kernel::initialize(session);
52}
53
54#[derive(Clone, Debug)]
55pub struct ListView;
56
57#[derive(Clone, prost::Message)]
58pub struct ListViewMetadata {
59    #[prost(uint64, tag = "1")]
60    elements_len: u64,
61    #[prost(int32, tag = "2")]
62    offset_ptype: i32,
63    #[prost(int32, tag = "3")]
64    size_ptype: i32,
65}
66
67impl ListViewMetadata {
68    fn get_offset_ptype(&self) -> VortexResult<PType> {
69        PType::try_from(self.offset_ptype).map_err(|_| {
70            vortex_err!(
71                "ListViewMetadata offset_ptype must be a known PType, got {}",
72                self.offset_ptype
73            )
74        })
75    }
76
77    fn get_size_ptype(&self) -> VortexResult<PType> {
78        PType::try_from(self.size_ptype).map_err(|_| {
79            vortex_err!(
80                "ListViewMetadata size_ptype must be a known PType, got {}",
81                self.size_ptype
82            )
83        })
84    }
85}
86
87impl ArrayHash for ListViewData {
88    fn array_hash<H: Hasher>(&self, state: &mut H, _accuracy: EqMode) {
89        self.is_zero_copy_to_list().hash(state);
90    }
91}
92
93impl ArrayEq for ListViewData {
94    fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool {
95        self.is_zero_copy_to_list() == other.is_zero_copy_to_list()
96    }
97}
98
99impl VTable for ListView {
100    type TypedArrayData = ListViewData;
101
102    type OperationsVTable = Self;
103    type ValidityVTable = Self;
104    fn id(&self) -> ArrayId {
105        static ID: CachedId = CachedId::new("vortex.listview");
106        *ID
107    }
108
109    fn nbuffers(_array: ArrayView<'_, Self>) -> usize {
110        0
111    }
112
113    fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle {
114        vortex_panic!("ListViewArray buffer index {idx} out of bounds")
115    }
116
117    fn buffer_name(_array: ArrayView<'_, Self>, idx: usize) -> Option<String> {
118        vortex_panic!("ListViewArray buffer_name index {idx} out of bounds")
119    }
120
121    fn with_buffers(
122        &self,
123        array: ArrayView<'_, Self>,
124        buffers: &[BufferHandle],
125    ) -> VortexResult<ArrayParts<Self>> {
126        with_empty_buffers(self, array, buffers)
127    }
128
129    fn serialize(
130        array: ArrayView<'_, Self>,
131        _session: &VortexSession,
132    ) -> VortexResult<Option<Vec<u8>>> {
133        Ok(Some(
134            ListViewMetadata {
135                elements_len: array.elements().len() as u64,
136                offset_ptype: PType::try_from(array.offsets().dtype())? as i32,
137                size_ptype: PType::try_from(array.sizes().dtype())? as i32,
138            }
139            .encode_to_vec(),
140        ))
141    }
142
143    fn validate(
144        &self,
145        _data: &ListViewData,
146        dtype: &DType,
147        len: usize,
148        slots: &[Option<ArrayRef>],
149    ) -> VortexResult<()> {
150        vortex_ensure_eq!(slots.len(), ListViewSlots::COUNT);
151        let elements = slots[ListViewSlots::ELEMENTS]
152            .as_ref()
153            .vortex_expect("ListViewArray elements slot");
154        let offsets = slots[ListViewSlots::OFFSETS]
155            .as_ref()
156            .vortex_expect("ListViewArray offsets slot");
157        let sizes = slots[ListViewSlots::SIZES]
158            .as_ref()
159            .vortex_expect("ListViewArray sizes slot");
160        vortex_ensure!(
161            offsets.len() == len && sizes.len() == len,
162            "ListViewArray length {} does not match outer length {}",
163            offsets.len(),
164            len
165        );
166
167        let actual_dtype = DType::List(Arc::new(elements.dtype().clone()), dtype.nullability());
168        vortex_ensure_eq!(
169            &actual_dtype,
170            dtype,
171            "ListViewArray dtype does not match outer dtype",
172        );
173
174        Ok(())
175    }
176
177    fn deserialize(
178        &self,
179        dtype: &DType,
180        len: usize,
181        metadata: &[u8],
182
183        buffers: &[BufferHandle],
184        children: &dyn ArrayChildren,
185        session: &VortexSession,
186    ) -> VortexResult<ArrayParts<Self>> {
187        let metadata = ListViewMetadata::decode(metadata)?;
188        let offset_ptype = metadata.get_offset_ptype()?;
189        let size_ptype = metadata.get_size_ptype()?;
190        vortex_ensure!(
191            buffers.is_empty(),
192            "`ListViewArray::build` expects no buffers"
193        );
194
195        let DType::List(element_dtype, _) = dtype else {
196            vortex_bail!("Expected List dtype, got {:?}", dtype);
197        };
198
199        let validity = if children.len() == 3 {
200            Validity::from(dtype.nullability())
201        } else if children.len() == 4 {
202            let validity = children.get(3, &Validity::DTYPE, len)?;
203            Validity::Array(validity)
204        } else {
205            vortex_bail!(
206                "`ListViewArray::build` expects 3 or 4 children, got {}",
207                children.len()
208            );
209        };
210
211        // Get elements with the correct length from metadata.
212        let elements = children.get(
213            0,
214            element_dtype.as_ref(),
215            usize::try_from(metadata.elements_len)?,
216        )?;
217
218        // Get offsets with proper type from metadata.
219        let offsets = children.get(
220            1,
221            &DType::Primitive(offset_ptype, Nullability::NonNullable),
222            len,
223        )?;
224
225        // Get sizes with proper type from metadata.
226        let sizes = children.get(
227            2,
228            &DType::Primitive(size_ptype, Nullability::NonNullable),
229            len,
230        )?;
231
232        ListViewData::validate(
233            &elements,
234            &offsets,
235            &sizes,
236            &validity,
237            &mut session.create_execution_ctx(),
238        )?;
239        let data = ListViewData::try_new()?;
240        let slots = ListViewData::make_slots(&elements, &offsets, &sizes, &validity, len);
241        Ok(ArrayParts::new(
242            self.clone(),
243            dtype.clone(),
244            len,
245            data,
246            slots,
247        ))
248    }
249
250    fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String {
251        ListViewSlots::NAMES[idx].to_string()
252    }
253
254    fn execute(array: Array<Self>, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
255        Ok(ExecutionResult::done(array))
256    }
257
258    // The complexity comes from the expansion of `match_each_list_builder!`.
259    #[expect(clippy::cognitive_complexity)]
260    fn append_to_builder(
261        array: ArrayView<'_, Self>,
262        builder: &mut dyn ArrayBuilder,
263        ctx: &mut ExecutionCtx,
264    ) -> VortexResult<()> {
265        match match_each_list_builder!(&mut *builder, |b| b.append_listview_array(array, ctx)) {
266            Some(result) => result,
267            None => vortex_bail!(
268                "cannot append a ListView array of dtype {} to a {} builder",
269                array.dtype(),
270                builder.dtype()
271            ),
272        }
273    }
274
275    fn reduce_parent(
276        array: ArrayView<'_, Self>,
277        parent: &ArrayRef,
278        child_idx: usize,
279    ) -> VortexResult<Option<ArrayRef>> {
280        PARENT_RULES.evaluate(array, parent, child_idx)
281    }
282}
283
284#[cfg(test)]
285mod tests {
286    use std::sync::Arc;
287    use std::sync::LazyLock;
288
289    use prost::Message;
290    use rstest::rstest;
291    use vortex_error::VortexResult;
292    use vortex_session::VortexSession;
293
294    use super::ListView;
295    use super::ListViewMetadata;
296    use crate::ArrayRef;
297    use crate::array::VTable;
298    use crate::dtype::DType;
299    use crate::dtype::Nullability;
300    use crate::dtype::PType;
301
302    static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
303
304    #[rstest]
305    #[case::offset_ptype(
306        |raw| ListViewMetadata { offset_ptype: raw, ..Default::default() },
307        ListViewMetadata::get_offset_ptype,
308        "ListViewMetadata offset_ptype must be a known PType, got 11",
309    )]
310    #[case::size_ptype(
311        |raw| ListViewMetadata { size_ptype: raw, ..Default::default() },
312        ListViewMetadata::get_size_ptype,
313        "ListViewMetadata size_ptype must be a known PType, got 11",
314    )]
315    fn test_deserialize_rejects_unknown_ptype(
316        #[case] with_ptype: fn(i32) -> ListViewMetadata,
317        #[case] get_ptype: fn(&ListViewMetadata) -> VortexResult<PType>,
318        #[case] expected: &str,
319    ) -> VortexResult<()> {
320        let dtype = DType::List(
321            Arc::new(DType::Primitive(PType::I32, Nullability::NonNullable)),
322            Nullability::NonNullable,
323        );
324        let metadata = with_ptype(11).encode_to_vec();
325        let children: Vec<ArrayRef> = Vec::new();
326        let result = ListView.deserialize(&dtype, 0, &metadata, &[], &children, &SESSION);
327        assert!(result.is_err_and(|err| err.to_string().contains(expected)));
328
329        assert_eq!(get_ptype(&with_ptype(PType::U32 as i32))?, PType::U32);
330
331        Ok(())
332    }
333}