Skip to main content

vortex_array/
iter.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Iterator over slices of an array, and related utilities.
5
6use std::sync::Arc;
7
8use itertools::Itertools;
9use vortex_error::VortexResult;
10use vortex_error::vortex_ensure_eq;
11
12use crate::ArrayRef;
13use crate::IntoArray;
14use crate::arrays::Chunked;
15use crate::arrays::ChunkedArray;
16use crate::arrays::chunked::ChunkedArrayExt;
17use crate::dtype::DType;
18use crate::stream::ArrayStream;
19use crate::stream::ArrayStreamAdapter;
20
21/// Iterator of array with a known [`DType`].
22///
23/// It's up to implementations to guarantee all arrays have the same [`DType`].
24pub trait ArrayIterator: Iterator<Item = VortexResult<ArrayRef>> {
25    fn dtype(&self) -> &DType;
26}
27
28impl ArrayIterator for Box<dyn ArrayIterator + Send> {
29    #[inline]
30    fn dtype(&self) -> &DType {
31        self.as_ref().dtype()
32    }
33}
34
35pub struct ArrayIteratorAdapter<I> {
36    dtype: DType,
37    inner: I,
38}
39
40impl<I> ArrayIteratorAdapter<I> {
41    #[inline]
42    pub fn new(dtype: DType, inner: I) -> Self {
43        Self { dtype, inner }
44    }
45}
46
47impl<I> Iterator for ArrayIteratorAdapter<I>
48where
49    I: Iterator<Item = VortexResult<ArrayRef>>,
50{
51    type Item = VortexResult<ArrayRef>;
52
53    #[inline]
54    fn next(&mut self) -> Option<Self::Item> {
55        self.inner.next()
56    }
57
58    fn size_hint(&self) -> (usize, Option<usize>) {
59        self.inner.size_hint()
60    }
61}
62
63impl<I> ArrayIterator for ArrayIteratorAdapter<I>
64where
65    I: Iterator<Item = VortexResult<ArrayRef>>,
66{
67    #[inline]
68    fn dtype(&self) -> &DType {
69        &self.dtype
70    }
71}
72
73pub trait ArrayIteratorExt: ArrayIterator {
74    fn into_array_stream(self) -> impl ArrayStream
75    where
76        Self: Sized,
77    {
78        ArrayStreamAdapter::new(self.dtype().clone(), futures::stream::iter(self))
79    }
80
81    /// Collect the iterator into a single `Array`.
82    ///
83    /// If the iterator yields multiple chunks, they will be returned as a [`ChunkedArray`].
84    fn read_all(self) -> VortexResult<ArrayRef>
85    where
86        Self: Sized,
87    {
88        let dtype = self.dtype().clone();
89        let mut chunks = self.peekable();
90        let Some(first) = chunks.next().transpose()? else {
91            return Ok(ChunkedArray::try_new([], dtype)?.into_array());
92        };
93        if chunks.peek().is_none() {
94            return Ok(first);
95        }
96        let chunks = std::iter::once(Ok(first))
97            .chain(chunks)
98            .map(|chunk| -> VortexResult<_> {
99                let chunk = chunk?;
100                vortex_ensure_eq!(chunk.dtype(), &dtype);
101                Ok(chunk)
102            });
103        let expected_nchunks = chunks.size_hint().0;
104        chunks.process_results(|chunks| {
105            // SAFETY: the iterator validates the dtype of every successfully yielded chunk.
106            unsafe { ChunkedArray::new_unchecked_sized(chunks, dtype.clone(), expected_nchunks) }
107                .into_array()
108        })
109    }
110}
111
112impl<I: ArrayIterator> ArrayIteratorExt for I {}
113
114impl ArrayRef {
115    /// Create an [`ArrayIterator`] over the array.
116    pub fn to_array_iterator(&self) -> impl ArrayIterator + 'static {
117        let dtype = self.dtype().clone();
118        let iter = if let Some(chunked) = self.as_opt::<Chunked>() {
119            ArrayChunkIterator::Chunked(Arc::new(chunked.into_owned()), 0)
120        } else {
121            ArrayChunkIterator::Single(Some(self.clone()))
122        };
123        ArrayIteratorAdapter::new(dtype, iter)
124    }
125}
126
127/// We define a single iterator that can handle both chunked and non-chunked arrays.
128/// This avoids the need to create boxed static iterators for the two chunked and non-chunked cases.
129enum ArrayChunkIterator {
130    Single(Option<ArrayRef>),
131    Chunked(Arc<ChunkedArray>, usize),
132}
133
134impl Iterator for ArrayChunkIterator {
135    type Item = VortexResult<ArrayRef>;
136
137    #[inline]
138    fn next(&mut self) -> Option<Self::Item> {
139        match self {
140            ArrayChunkIterator::Single(array) => array.take().map(Ok),
141            ArrayChunkIterator::Chunked(chunked, idx) => (*idx < chunked.nchunks()).then(|| {
142                let chunk = chunked.chunk(*idx);
143                *idx += 1;
144                Ok(chunk.clone())
145            }),
146        }
147    }
148}