Skip to main content

vortex_array/arrays/listview/
conversion.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4use vortex_buffer::BufferAllocatorRef;
5use vortex_error::VortexExpect;
6use vortex_error::VortexResult;
7
8use crate::ArrayRef;
9use crate::Canonical;
10use crate::ExecutionCtx;
11use crate::IntoArray;
12use crate::arrays::ExtensionArray;
13use crate::arrays::FixedSizeListArray;
14use crate::arrays::ListArray;
15use crate::arrays::ListViewArray;
16use crate::arrays::PrimitiveArray;
17use crate::arrays::StructArray;
18use crate::arrays::extension::ExtensionArrayExt;
19use crate::arrays::fixed_size_list::FixedSizeListArrayExt;
20use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt;
21use crate::arrays::list::ListArrayExt;
22use crate::arrays::list::ListArraySlotsExt;
23use crate::arrays::listview::ListViewArrayExt;
24use crate::arrays::listview::ListViewArraySlotsExt;
25use crate::arrays::listview::ListViewRebuildMode;
26use crate::arrays::struct_::StructArrayExt;
27use crate::builders::PrimitiveBuilder;
28use crate::dtype::IntegerPType;
29use crate::dtype::Nullability;
30use crate::match_each_integer_ptype;
31
32/// Creates a `ListViewArray` from a `ListArray` by computing `sizes` from `offsets`.
33///
34/// The output `ListViewArray` will be zero-copyable back to a `ListArray`, and additionally it will
35/// not have any leading or trailing garbage data.
36pub fn list_view_from_list(list: ListArray, ctx: &mut ExecutionCtx) -> VortexResult<ListViewArray> {
37    // If the list is empty, create an empty `ListViewArray` with the same offset `DType` as the
38    // input.
39    if list.is_empty() {
40        return Ok(Canonical::empty(list.dtype()).into_listview());
41    }
42
43    // We reset the offsets here because mostly for convenience, and also because callers of this
44    // function might not expect the output `ListViewArray` to have a bunch of leading and trailing
45    // garbage data when they turn it back into a `ListArray`.
46    let list = list.reset_offsets(false, ctx)?;
47
48    // `reset_offsets` can leave a lazy subtraction in the offsets slot, which both the `sizes` below
49    // and the view's own offsets read. Execute it once here so that it does not run per reader.
50    let list_offsets = list.offsets().clone().execute::<PrimitiveArray>(ctx)?;
51
52    // Create `sizes` array by computing differences between consecutive offsets.
53    // We use the same `DType` for the sizes as the `offsets` array to ensure compatibility.
54    let sizes = match_each_integer_ptype!(list_offsets.ptype(), |O| {
55        build_sizes_from_offsets::<O>(&list_offsets, ctx.allocator())?
56    });
57
58    // We need to slice the `offsets` to remove the last element (`ListArray` has `n + 1` offsets).
59    debug_assert_eq!(list_offsets.len(), list.len() + 1);
60    let adjusted_offsets = list_offsets.into_array().slice(0..list.len())?;
61
62    // SAFETY: Since everything came from an existing valid `ListArray`, and the `sizes` were
63    // derived from valid and in-order `offsets`, we know these fields are valid.
64    // We also just came directly from a `ListArray`, so we know this is zero-copyable.
65    Ok(unsafe {
66        ListViewArray::new_unchecked(
67            list.elements().clone(),
68            adjusted_offsets,
69            sizes,
70            list.validity()?,
71        )
72        .with_zero_copy_to_list(true)
73    })
74}
75
76/// Builds a sizes array by computing differences between consecutive offsets.
77///
78/// `offsets` **must** be the `n + 1` sorted offsets of a non-empty `ListArray` of `n` rows.
79fn build_sizes_from_offsets<O: IntegerPType>(
80    offsets: &PrimitiveArray,
81    allocator: &BufferAllocatorRef,
82) -> VortexResult<ArrayRef> {
83    let offsets_slice = offsets.as_slice::<O>();
84    debug_assert!(offsets_slice.is_sorted());
85
86    let len = offsets_slice.len() - 1;
87    let mut sizes_builder =
88        PrimitiveBuilder::<O>::with_capacity_in(Nullability::NonNullable, len, allocator);
89
90    // Create `UninitRange` for direct memory access.
91    let mut sizes_range = sizes_builder.uninit_range(len);
92
93    // Compute sizes as the difference between consecutive offsets.
94    for i in 0..len {
95        let size = offsets_slice[i + 1] - offsets_slice[i];
96        sizes_range.set_value(i, size);
97    }
98
99    // SAFETY: We have initialized all values in the range.
100    unsafe {
101        sizes_range.finish();
102    }
103
104    Ok(sizes_builder.finish_into_primitive().into_array())
105}
106
107/// Creates a `ListArray` from a `ListViewArray`.
108///
109/// The resulting `ListArray` will not have any leading or trailing garbage data.
110///
111/// This operation is fast when `ListViewArray::is_zero_copy_to_list` is `true`. It is not exactly
112/// zero-copy, since it must append one final offset. Otherwise it falls back to the (very)
113/// expensive path and rebuilds the `ListArray` from scratch.
114pub fn list_from_list_view(
115    list_view: ListViewArray,
116    ctx: &mut ExecutionCtx,
117) -> VortexResult<ListArray> {
118    // Rebuild as zero-copyable to list array and also trim all leading and trailing elements.
119    let zctl_array = list_view.rebuild(ListViewRebuildMode::MakeExact, ctx)?;
120    debug_assert!(zctl_array.is_zero_copy_to_list());
121
122    let list_offsets = match_each_integer_ptype!(zctl_array.offsets().dtype().as_ptype(), |O| {
123        // SAFETY: We just made the array zero-copyable to `ListArray`, so the safety contract is
124        // upheld.
125        unsafe { build_list_offsets_from_list_view::<O>(&zctl_array, ctx) }
126    });
127
128    // SAFETY: Because the shape of the `ListViewArray` is zero-copyable to a `ListArray`, we
129    // can simply reuse all of the data (besides the offsets). We also trim all of the elements to
130    // make it easier for the caller to use the `ListArray`.
131    Ok(unsafe {
132        ListArray::new_unchecked(
133            zctl_array.elements().clone(),
134            list_offsets,
135            zctl_array.validity()?,
136        )
137    })
138}
139
140// TODO(connor)[ListView]: We can optimize this by always keeping extra memory in `ListViewArray`
141// offsets for an `n+1`th offset.
142/// Builds a `ListArray` offsets array from a `ListViewArray` by constructing `n + 1` offsets.
143///
144/// The last offset is computed as `last_offset + last_size`, and is zero for an empty array.
145///
146/// # Safety
147///
148/// The `ListViewArray` must have offsets that are sorted, and every size must be equal to the gap
149/// between `offset[i]` and `offset[i + 1]`.
150unsafe fn build_list_offsets_from_list_view<O: IntegerPType>(
151    list_view: &ListViewArray,
152    ctx: &mut ExecutionCtx,
153) -> ArrayRef {
154    let len = list_view.len();
155    let mut offsets_builder =
156        PrimitiveBuilder::<O>::with_capacity_in(Nullability::NonNullable, len + 1, ctx.allocator());
157
158    // Create uninit range for direct memory access.
159    let mut offsets_range = offsets_builder.uninit_range(len + 1);
160
161    let offsets = list_view
162        .offsets()
163        .clone()
164        .execute::<PrimitiveArray>(ctx)
165        .vortex_expect("list view offsets must be primitive after rebuild");
166    let offsets_slice = offsets.as_slice::<O>();
167    debug_assert!(offsets_slice.is_sorted());
168
169    // Copy the existing n offsets.
170    offsets_range.copy_from_slice(0, offsets_slice);
171
172    // Append the final offset (last offset + last size).
173    let final_offset = if len != 0 {
174        let last_offset = offsets_slice[len - 1];
175
176        let last_size = list_view.size_at(len - 1, ctx);
177        let last_size =
178            O::from_usize(last_size).vortex_expect("size somehow did not fit into offsets");
179
180        last_offset + last_size
181    } else {
182        O::zero()
183    };
184
185    offsets_range.set_value(len, final_offset);
186
187    // SAFETY: We have initialized all values in the range.
188    unsafe {
189        offsets_range.finish();
190    }
191
192    offsets_builder.finish_into_primitive().into_array()
193}
194
195/// Recursively converts all `ListViewArray`s to `ListArray`s in a nested array structure.
196///
197/// The conversion happens bottom-up, processing children before parents.
198pub fn recursive_list_from_list_view(
199    array: ArrayRef,
200    ctx: &mut ExecutionCtx,
201) -> VortexResult<ArrayRef> {
202    if !array.dtype().is_nested() {
203        return Ok(array);
204    }
205
206    let canonical = array.execute::<Canonical>(ctx)?;
207
208    Ok(match canonical {
209        Canonical::List(listview) => {
210            let converted_elements =
211                recursive_list_from_list_view(listview.elements().clone(), ctx)?;
212            debug_assert_eq!(converted_elements.len(), listview.elements().len());
213
214            // Avoid cloning if elements didn't change.
215            let listview_with_converted_elements =
216                if !ArrayRef::ptr_eq(&converted_elements, listview.elements()) {
217                    // SAFETY: We are effectively just replacing the child elements array, which
218                    // must have the same length, so all invariants are maintained.
219                    unsafe {
220                        ListViewArray::new_unchecked(
221                            converted_elements,
222                            listview.offsets().clone(),
223                            listview.sizes().clone(),
224                            listview.validity()?,
225                        )
226                        .with_zero_copy_to_list(listview.is_zero_copy_to_list())
227                    }
228                } else {
229                    listview
230                };
231
232            // Make the conversion to `ListArray`.
233            let list_array = list_from_list_view(listview_with_converted_elements, ctx)?;
234            list_array.into_array()
235        }
236        Canonical::FixedSizeList(fixed_size_list) => {
237            let converted_elements =
238                recursive_list_from_list_view(fixed_size_list.elements().clone(), ctx)?;
239
240            // Avoid cloning if elements didn't change.
241            if !ArrayRef::ptr_eq(&converted_elements, fixed_size_list.elements()) {
242                FixedSizeListArray::try_new(
243                    converted_elements,
244                    fixed_size_list.list_size(),
245                    fixed_size_list.validity()?,
246                    fixed_size_list.len(),
247                )
248                .vortex_expect(
249                    "FixedSizeListArray reconstruction should not fail with valid components",
250                )
251                .into_array()
252            } else {
253                fixed_size_list.into_array()
254            }
255        }
256        Canonical::Struct(struct_array) => {
257            let mut converted_fields =
258                Vec::with_capacity(struct_array.iter_unmasked_fields().len());
259            let mut any_changed = false;
260
261            for field in struct_array.iter_unmasked_fields() {
262                let converted_field = recursive_list_from_list_view(field.clone(), ctx)?;
263                // Avoid cloning if elements didn't change.
264                any_changed |= !ArrayRef::ptr_eq(&converted_field, field);
265                converted_fields.push(converted_field);
266            }
267
268            if any_changed {
269                StructArray::try_new(
270                    struct_array.names().clone(),
271                    converted_fields,
272                    struct_array.len(),
273                    struct_array.validity()?,
274                )
275                .vortex_expect("StructArray reconstruction should not fail with valid components")
276                .into_array()
277            } else {
278                struct_array.into_array()
279            }
280        }
281        Canonical::Extension(ext_array) => {
282            let converted_storage =
283                recursive_list_from_list_view(ext_array.storage_array().clone(), ctx)?;
284
285            // Avoid cloning if elements didn't change.
286            if !ArrayRef::ptr_eq(&converted_storage, ext_array.storage_array()) {
287                ExtensionArray::new(ext_array.ext_dtype().clone(), converted_storage).into_array()
288            } else {
289                ext_array.into_array()
290            }
291        }
292        _ => unreachable!(),
293    })
294}
295
296#[cfg(test)]
297mod tests {
298
299    use vortex_buffer::buffer;
300    use vortex_error::VortexExpect;
301    use vortex_error::VortexResult;
302
303    use super::super::tests::common::SESSION;
304    use super::super::tests::common::create_basic_listview;
305    use super::super::tests::common::create_empty_lists_listview;
306    use super::super::tests::common::create_nullable_listview;
307    use super::super::tests::common::create_overlapping_listview;
308    use super::recursive_list_from_list_view;
309    use crate::ArrayEq;
310    use crate::ArrayRef;
311    use crate::EqMode;
312    use crate::IntoArray;
313    use crate::VortexSessionExecute;
314    use crate::arrays::BoolArray;
315    use crate::arrays::FixedSizeList;
316    use crate::arrays::FixedSizeListArray;
317    use crate::arrays::List;
318    use crate::arrays::ListArray;
319    use crate::arrays::ListViewArray;
320    use crate::arrays::PrimitiveArray;
321    use crate::arrays::Struct;
322    use crate::arrays::StructArray;
323    use crate::arrays::VarBinViewArray;
324    use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt;
325    use crate::arrays::list::ListArrayExt;
326    use crate::arrays::list::ListArraySlotsExt;
327    use crate::arrays::listview::ListViewArraySlotsExt;
328    use crate::arrays::listview::list_from_list_view;
329    use crate::arrays::listview::list_view_from_list;
330    use crate::arrays::struct_::StructArrayExt;
331    use crate::assert_arrays_eq;
332    use crate::dtype::FieldNames;
333    use crate::validity::Validity;
334
335    #[test]
336    fn test_list_to_listview_basic() -> VortexResult<()> {
337        let mut ctx = SESSION.create_execution_ctx();
338        // Create a basic ListArray: [[0,1,2], [3,4], [5,6], [7,8,9]].
339        let elements = buffer![0i32, 1, 2, 3, 4, 5, 6, 7, 8, 9].into_array();
340        let offsets = buffer![0u32, 3, 5, 7, 10].into_array();
341        let list_array =
342            ListArray::try_new(elements.clone(), offsets, Validity::NonNullable, &mut ctx)?;
343
344        let list_view = list_view_from_list(list_array.clone(), &mut ctx)?;
345
346        // Verify structure.
347        assert_eq!(list_view.len(), 4);
348        assert_arrays_eq!(list_view.elements().clone(), elements, &mut ctx);
349
350        // Verify offsets (should be same but without last element).
351        let expected_offsets = buffer![0u32, 3, 5, 7].into_array();
352        assert_arrays_eq!(list_view.offsets().clone(), expected_offsets, &mut ctx);
353
354        // Verify sizes.
355        let expected_sizes = buffer![3u32, 2, 2, 3].into_array();
356        assert_arrays_eq!(list_view.sizes().clone(), expected_sizes, &mut ctx);
357
358        // Verify data integrity.
359        assert_arrays_eq!(list_view, list_array, &mut ctx);
360        Ok(())
361    }
362
363    #[test]
364    fn test_list_to_listview_resets_nonzero_offsets() -> VortexResult<()> {
365        let mut ctx = SESSION.create_execution_ctx();
366        let elements = buffer![0i32, 1, 2, 3, 4].into_array();
367        let offsets = buffer![2u16, 4, 5].into_array();
368        let list = ListArray::try_new(elements, offsets, Validity::NonNullable, &mut ctx)?;
369
370        let list_view = list_view_from_list(list.clone(), &mut ctx)?;
371
372        assert_arrays_eq!(
373            list_view.offsets().clone(),
374            buffer![0u16, 2].into_array(),
375            &mut ctx
376        );
377        assert_arrays_eq!(list_view, list, &mut ctx);
378        Ok(())
379    }
380
381    #[test]
382    fn test_listview_to_list_zero_copy() -> VortexResult<()> {
383        let mut ctx = SESSION.create_execution_ctx();
384        let list_view = create_basic_listview();
385        let list_array =
386            list_from_list_view(list_view.clone(), &mut SESSION.create_execution_ctx())?;
387
388        // Should have same elements.
389        assert_arrays_eq!(
390            list_array.elements().clone(),
391            list_view.elements().clone(),
392            &mut ctx
393        );
394
395        // ListArray offsets should have n+1 elements for n lists (add the final offset).
396        // Check that the first n offsets match.
397        let list_array_offsets_without_last = list_array.offsets().slice(0..list_view.len())?;
398        assert_arrays_eq!(
399            list_array_offsets_without_last,
400            list_view.offsets().clone(),
401            &mut ctx
402        );
403
404        // Verify data integrity.
405        assert_arrays_eq!(list_array, list_view, &mut ctx);
406        Ok(())
407    }
408
409    #[test]
410    fn test_empty_array_conversions() -> VortexResult<()> {
411        let mut ctx = SESSION.create_execution_ctx();
412        // Empty ListArray to ListViewArray.
413        let empty_elements = PrimitiveArray::from_iter::<[i32; 0]>([]).into_array();
414        let empty_offsets = buffer![0u32].into_array();
415        let empty_list = ListArray::try_new(
416            empty_elements,
417            empty_offsets,
418            Validity::NonNullable,
419            &mut ctx,
420        )?;
421
422        // This conversion will create an empty ListViewArray.
423        // Note: list_view_from_list handles the empty case specially.
424        let mut ctx = SESSION.create_execution_ctx();
425        let empty_list_view = list_view_from_list(empty_list.clone(), &mut ctx)?;
426        assert_eq!(empty_list_view.len(), 0);
427
428        // Convert back.
429        assert_arrays_eq!(empty_list_view, empty_list, &mut ctx);
430        let converted_back = list_from_list_view(empty_list_view, &mut ctx)?;
431        assert_eq!(converted_back.len(), 0);
432        assert_arrays_eq!(converted_back, empty_list, &mut ctx);
433        Ok(())
434    }
435
436    #[test]
437    fn test_nullable_conversions() -> VortexResult<()> {
438        let mut ctx = SESSION.create_execution_ctx();
439        // Create nullable ListArray: [[10,20], null, [50]].
440        let elements = buffer![10i32, 20, 30, 40, 50].into_array();
441        let offsets = buffer![0u32, 2, 4, 5].into_array();
442        let validity = Validity::Array(BoolArray::from_iter(vec![true, false, true]).into_array());
443        let nullable_list = ListArray::try_new(elements, offsets, validity.clone(), &mut ctx)?;
444
445        let nullable_list_view = list_view_from_list(nullable_list.clone(), &mut ctx)?;
446
447        // Verify validity is preserved.
448        assert!(
449            nullable_list_view
450                .validity()
451                .vortex_expect("listview validity should be derivable")
452                .array_eq(&validity, EqMode::Ptr)
453        );
454        assert_eq!(nullable_list_view.len(), 3);
455
456        // Round-trip conversion.
457        let converted_back = list_from_list_view(nullable_list_view, &mut ctx)?;
458        assert_arrays_eq!(converted_back, nullable_list, &mut ctx);
459        Ok(())
460    }
461
462    #[test]
463    fn test_non_zero_copy_listview_to_list() -> VortexResult<()> {
464        let mut ctx = SESSION.create_execution_ctx();
465        // Create ListViewArray with overlapping lists (not zero-copyable).
466        let list_view = create_overlapping_listview();
467        let list_array =
468            list_from_list_view(list_view.clone(), &mut SESSION.create_execution_ctx())?;
469
470        // The resulting ListArray should have monotonic offsets.
471        for i in 0..list_array.len() {
472            let start = list_array.offset_at(i, &mut ctx)?;
473            let end = list_array.offset_at(i + 1, &mut ctx)?;
474            assert!(end >= start, "Offsets should be monotonic after conversion");
475        }
476
477        // The data should still be correct even though it required a rebuild.
478        assert_arrays_eq!(list_array, list_view, &mut ctx);
479        Ok(())
480    }
481
482    #[test]
483    fn test_empty_sublists() -> VortexResult<()> {
484        let mut ctx = SESSION.create_execution_ctx();
485        let empty_lists_view = create_empty_lists_listview();
486
487        // Convert to ListArray.
488        let list_array = list_from_list_view(empty_lists_view.clone(), &mut ctx)?;
489        assert_eq!(list_array.len(), 4);
490
491        // All sublists should be empty.
492        for i in 0..list_array.len() {
493            assert_eq!(list_array.list_elements_at(i, &mut ctx)?.len(), 0);
494        }
495
496        // Round-trip.
497        let converted_back = list_view_from_list(list_array, &mut ctx)?;
498        assert_arrays_eq!(converted_back, empty_lists_view, &mut ctx);
499        Ok(())
500    }
501
502    #[test]
503    fn test_different_offset_types() -> VortexResult<()> {
504        let mut ctx = SESSION.create_execution_ctx();
505        // Test with i32 offsets.
506        let elements = buffer![1i32, 2, 3, 4, 5].into_array();
507        let i32_offsets = buffer![0i32, 2, 5].into_array();
508        let list_i32 = ListArray::try_new(
509            elements.clone(),
510            i32_offsets.clone(),
511            Validity::NonNullable,
512            &mut ctx,
513        )?;
514
515        let list_view_i32 = list_view_from_list(list_i32.clone(), &mut ctx)?;
516        assert_eq!(list_view_i32.offsets().dtype(), i32_offsets.dtype());
517        assert_eq!(list_view_i32.sizes().dtype(), i32_offsets.dtype());
518
519        // Test with i64 offsets.
520        let i64_offsets = buffer![0i64, 2, 5].into_array();
521        let list_i64 = ListArray::try_new(
522            elements,
523            i64_offsets.clone(),
524            Validity::NonNullable,
525            &mut ctx,
526        )?;
527
528        let list_view_i64 = list_view_from_list(list_i64.clone(), &mut ctx)?;
529        assert_eq!(list_view_i64.offsets().dtype(), i64_offsets.dtype());
530        assert_eq!(list_view_i64.sizes().dtype(), i64_offsets.dtype());
531
532        // Verify data integrity.
533        assert_arrays_eq!(list_view_i32, list_i32, &mut ctx);
534        assert_arrays_eq!(list_view_i64, list_i64, &mut ctx);
535        Ok(())
536    }
537
538    #[test]
539    fn test_round_trip_conversions() -> VortexResult<()> {
540        let mut ctx = SESSION.create_execution_ctx();
541
542        // Test 1: Basic round-trip.
543        let original = create_basic_listview();
544        let to_list = list_from_list_view(original.clone(), &mut ctx)?;
545        let back_to_view = list_view_from_list(to_list, &mut ctx)?;
546        assert_arrays_eq!(back_to_view, original, &mut ctx);
547
548        // Test 2: Nullable round-trip.
549        let nullable = create_nullable_listview();
550        let nullable_to_list = list_from_list_view(nullable.clone(), &mut ctx)?;
551        let nullable_back = list_view_from_list(nullable_to_list, &mut ctx)?;
552        assert_arrays_eq!(nullable_back, nullable, &mut ctx);
553
554        // Test 3: Non-zero-copyable round-trip.
555        let overlapping = create_overlapping_listview();
556
557        let overlapping_to_list = list_from_list_view(overlapping.clone(), &mut ctx)?;
558        let overlapping_back = list_view_from_list(overlapping_to_list, &mut ctx)?;
559        assert_arrays_eq!(overlapping_back, overlapping, &mut ctx);
560        Ok(())
561    }
562
563    #[test]
564    fn test_single_element_lists() -> VortexResult<()> {
565        let mut ctx = SESSION.create_execution_ctx();
566        // Create lists with single elements: [[100], [200], [300]].
567        let elements = buffer![100i32, 200, 300].into_array();
568        let offsets = buffer![0u32, 1, 2, 3].into_array();
569        let single_elem_list =
570            ListArray::try_new(elements, offsets, Validity::NonNullable, &mut ctx)?;
571
572        let list_view = list_view_from_list(single_elem_list.clone(), &mut ctx)?;
573        assert_eq!(list_view.len(), 3);
574
575        // Verify sizes are all 1.
576        let expected_sizes = buffer![1u32, 1, 1].into_array();
577        assert_arrays_eq!(list_view.sizes().clone(), expected_sizes, &mut ctx);
578
579        // Round-trip.
580        let converted_back = list_from_list_view(list_view, &mut ctx)?;
581        assert_arrays_eq!(converted_back, single_elem_list, &mut ctx);
582        Ok(())
583    }
584
585    #[test]
586    fn test_mixed_empty_and_non_empty_lists() -> VortexResult<()> {
587        let mut ctx = SESSION.create_execution_ctx();
588        // Create: [[1,2], [], [3], [], [4,5,6]].
589        let elements = buffer![1i32, 2, 3, 4, 5, 6].into_array();
590        let offsets = buffer![0u32, 2, 2, 3, 3, 6].into_array();
591        let mixed_list = ListArray::try_new(elements, offsets, Validity::NonNullable, &mut ctx)?;
592
593        let list_view = list_view_from_list(mixed_list.clone(), &mut ctx)?;
594        assert_eq!(list_view.len(), 5);
595
596        // Verify sizes.
597        let expected_sizes = buffer![2u32, 0, 1, 0, 3].into_array();
598        assert_arrays_eq!(list_view.sizes().clone(), expected_sizes, &mut ctx);
599
600        // Round-trip.
601        let converted_back = list_from_list_view(list_view, &mut ctx)?;
602        assert_arrays_eq!(converted_back, mixed_list, &mut ctx);
603        Ok(())
604    }
605
606    #[test]
607    fn test_recursive_simple_listview() -> VortexResult<()> {
608        let mut ctx = SESSION.create_execution_ctx();
609        let list_view = create_basic_listview();
610        let result = recursive_list_from_list_view(
611            list_view.clone().into_array(),
612            &mut SESSION.create_execution_ctx(),
613        )?;
614
615        assert!(result.is::<List>());
616        assert_eq!(result.len(), list_view.len());
617        assert_arrays_eq!(result, list_view.into_array(), &mut ctx);
618        Ok(())
619    }
620
621    #[test]
622    fn test_recursive_nested_listview() -> VortexResult<()> {
623        let mut ctx = SESSION.create_execution_ctx();
624        let inner_elements = buffer![1i32, 2, 3].into_array();
625        let inner_offsets = buffer![0u32, 2].into_array();
626        let inner_sizes = buffer![2u32, 1].into_array();
627        let inner_listview = unsafe {
628            ListViewArray::new_unchecked(
629                inner_elements,
630                inner_offsets,
631                inner_sizes,
632                Validity::NonNullable,
633            )
634            .with_zero_copy_to_list(true)
635        };
636
637        let outer_offsets = buffer![0u32, 1].into_array();
638        let outer_sizes = buffer![1u32, 1].into_array();
639        let outer_listview = unsafe {
640            ListViewArray::new_unchecked(
641                inner_listview.into_array(),
642                outer_offsets,
643                outer_sizes,
644                Validity::NonNullable,
645            )
646            .with_zero_copy_to_list(true)
647        };
648
649        let result = recursive_list_from_list_view(
650            outer_listview.clone().into_array(),
651            &mut SESSION.create_execution_ctx(),
652        )?;
653
654        assert!(result.as_::<List>().elements().is::<List>());
655        assert_eq!(result.len(), 2);
656        assert_arrays_eq!(result, outer_listview.into_array(), &mut ctx);
657        Ok(())
658    }
659
660    #[test]
661    fn test_recursive_struct_with_listview_fields() -> VortexResult<()> {
662        let mut ctx = SESSION.create_execution_ctx();
663        let listview_field = create_basic_listview().into_array();
664        let primitive_field = buffer![10i32, 20, 30, 40].into_array();
665
666        let struct_array = StructArray::try_new(
667            FieldNames::from(["lists", "values"]),
668            vec![listview_field, primitive_field],
669            4,
670            Validity::NonNullable,
671        )?;
672
673        let result = recursive_list_from_list_view(
674            struct_array.clone().into_array(),
675            &mut SESSION.create_execution_ctx(),
676        )?;
677
678        assert!(
679            result
680                .as_::<Struct>()
681                .unmasked_field_by_name("lists")?
682                .is::<List>()
683        );
684        assert_eq!(result.len(), 4);
685        assert_arrays_eq!(result, struct_array.into_array(), &mut ctx);
686        Ok(())
687    }
688
689    #[test]
690    fn test_recursive_fixed_size_list_with_listview_elements() -> VortexResult<()> {
691        let mut ctx = SESSION.create_execution_ctx();
692        let lv1_elements = buffer![1i32, 2].into_array();
693        let lv1_offsets = buffer![0u32].into_array();
694        let lv1_sizes = buffer![2u32].into_array();
695        let lv1 = unsafe {
696            ListViewArray::new_unchecked(
697                lv1_elements,
698                lv1_offsets,
699                lv1_sizes,
700                Validity::NonNullable,
701            )
702            .with_zero_copy_to_list(true)
703        };
704
705        let lv2_elements = buffer![3i32, 4].into_array();
706        let lv2_offsets = buffer![0u32].into_array();
707        let lv2_sizes = buffer![2u32].into_array();
708        let lv2 = unsafe {
709            ListViewArray::new_unchecked(
710                lv2_elements,
711                lv2_offsets,
712                lv2_sizes,
713                Validity::NonNullable,
714            )
715            .with_zero_copy_to_list(true)
716        };
717
718        let dtype = lv1.dtype().clone();
719        let chunked_listviews =
720            crate::arrays::ChunkedArray::try_new(vec![lv1.into_array(), lv2.into_array()], dtype)?;
721
722        let fixed_list =
723            FixedSizeListArray::new(chunked_listviews.into_array(), 1, Validity::NonNullable, 2);
724
725        let result = recursive_list_from_list_view(
726            fixed_list.clone().into_array(),
727            &mut SESSION.create_execution_ctx(),
728        )?;
729
730        assert!(result.as_::<FixedSizeList>().elements().is::<List>());
731        assert_eq!(result.len(), 2);
732        assert_arrays_eq!(result, fixed_list.into_array(), &mut ctx);
733        Ok(())
734    }
735
736    #[test]
737    fn test_recursive_deep_nesting() -> VortexResult<()> {
738        let mut ctx = SESSION.create_execution_ctx();
739        let innermost_elements = buffer![1i32, 2, 3].into_array();
740        let innermost_offsets = buffer![0u32, 2].into_array();
741        let innermost_sizes = buffer![2u32, 1].into_array();
742        let innermost_listview = unsafe {
743            ListViewArray::new_unchecked(
744                innermost_elements,
745                innermost_offsets,
746                innermost_sizes,
747                Validity::NonNullable,
748            )
749            .with_zero_copy_to_list(true)
750        };
751
752        let struct_array = StructArray::try_new(
753            FieldNames::from(["inner_lists"]),
754            vec![innermost_listview.into_array()],
755            2,
756            Validity::NonNullable,
757        )?;
758
759        let outer_offsets = buffer![0u32, 1].into_array();
760        let outer_sizes = buffer![1u32, 1].into_array();
761        let outer_listview = unsafe {
762            ListViewArray::new_unchecked(
763                struct_array.into_array(),
764                outer_offsets,
765                outer_sizes,
766                Validity::NonNullable,
767            )
768            .with_zero_copy_to_list(true)
769        };
770
771        let result = recursive_list_from_list_view(
772            outer_listview.clone().into_array(),
773            &mut SESSION.create_execution_ctx(),
774        )?;
775
776        assert!(
777            result
778                .as_::<List>()
779                .elements()
780                .as_::<Struct>()
781                .unmasked_field_by_name("inner_lists")?
782                .is::<List>()
783        );
784        assert_eq!(result.len(), 2);
785        assert_arrays_eq!(result, outer_listview.into_array(), &mut ctx);
786        Ok(())
787    }
788
789    #[test]
790    fn test_recursive_primitive_unchanged() -> VortexResult<()> {
791        let prim = buffer![1i32, 2, 3].into_array();
792        let prim_clone = prim.clone();
793        let result = recursive_list_from_list_view(prim, &mut SESSION.create_execution_ctx())?;
794
795        assert!(ArrayRef::ptr_eq(&result, &prim_clone));
796        Ok(())
797    }
798
799    #[test]
800    fn test_recursive_mixed_listview_and_list() -> VortexResult<()> {
801        let mut ctx = SESSION.create_execution_ctx();
802        let listview = create_basic_listview();
803        let list = list_from_list_view(listview.clone(), &mut ctx)?;
804
805        let struct_array = StructArray::try_new(
806            FieldNames::from(["listview_field", "list_field"]),
807            vec![listview.into_array(), list.into_array()],
808            4,
809            Validity::NonNullable,
810        )?;
811
812        let result = recursive_list_from_list_view(struct_array.clone().into_array(), &mut ctx)?;
813
814        assert!(
815            result
816                .as_::<Struct>()
817                .unmasked_field_by_name("listview_field")?
818                .is::<List>()
819        );
820        assert!(
821            result
822                .as_::<Struct>()
823                .unmasked_field_by_name("list_field")?
824                .is::<List>()
825        );
826        assert_eq!(result.len(), 4);
827        assert_arrays_eq!(result, struct_array.into_array(), &mut ctx);
828        Ok(())
829    }
830
831    /// Regression test for <https://github.com/vortex-data/vortex/issues/6882>.
832    ///
833    /// An empty `ListViewArray` constructed via `try_new` has `is_zero_copy_to_list: false`.
834    /// `list_from_list_view` should still succeed because empty arrays are trivially
835    /// zero-copyable.
836    #[test]
837    fn test_empty_listview_to_list_without_zctl_flag() -> VortexResult<()> {
838        let mut ctx = SESSION.create_execution_ctx();
839        let elements = VarBinViewArray::from_iter_str(Vec::<&str>::new()).into_array();
840        let offsets = PrimitiveArray::from_iter(Vec::<i16>::new()).into_array();
841        let sizes = PrimitiveArray::from_iter(Vec::<i16>::new()).into_array();
842        let list_view =
843            ListViewArray::try_new(elements, offsets, sizes, Validity::AllValid, &mut ctx)?;
844
845        // `try_new` sets `is_zero_copy_to_list: false`.
846        assert!(!list_view.is_zero_copy_to_list());
847
848        let list_array = list_from_list_view(list_view, &mut SESSION.create_execution_ctx())?;
849        assert_eq!(list_array.len(), 0);
850        Ok(())
851    }
852}