1use std::sync::Arc;
7
8use itertools::Itertools;
9use vortex_error::VortexResult;
10use vortex_error::vortex_ensure_eq;
11
12use crate::ArrayRef;
13use crate::IntoArray;
14use crate::arrays::Chunked;
15use crate::arrays::ChunkedArray;
16use crate::arrays::chunked::ChunkedArrayExt;
17use crate::dtype::DType;
18use crate::stream::ArrayStream;
19use crate::stream::ArrayStreamAdapter;
20
21pub trait ArrayIterator: Iterator<Item = VortexResult<ArrayRef>> {
25 fn dtype(&self) -> &DType;
26}
27
28impl ArrayIterator for Box<dyn ArrayIterator + Send> {
29 #[inline]
30 fn dtype(&self) -> &DType {
31 self.as_ref().dtype()
32 }
33}
34
35pub struct ArrayIteratorAdapter<I> {
36 dtype: DType,
37 inner: I,
38}
39
40impl<I> ArrayIteratorAdapter<I> {
41 #[inline]
42 pub fn new(dtype: DType, inner: I) -> Self {
43 Self { dtype, inner }
44 }
45}
46
47impl<I> Iterator for ArrayIteratorAdapter<I>
48where
49 I: Iterator<Item = VortexResult<ArrayRef>>,
50{
51 type Item = VortexResult<ArrayRef>;
52
53 #[inline]
54 fn next(&mut self) -> Option<Self::Item> {
55 self.inner.next()
56 }
57
58 fn size_hint(&self) -> (usize, Option<usize>) {
59 self.inner.size_hint()
60 }
61}
62
63impl<I> ArrayIterator for ArrayIteratorAdapter<I>
64where
65 I: Iterator<Item = VortexResult<ArrayRef>>,
66{
67 #[inline]
68 fn dtype(&self) -> &DType {
69 &self.dtype
70 }
71}
72
73pub trait ArrayIteratorExt: ArrayIterator {
74 fn into_array_stream(self) -> impl ArrayStream
75 where
76 Self: Sized,
77 {
78 ArrayStreamAdapter::new(self.dtype().clone(), futures::stream::iter(self))
79 }
80
81 fn read_all(self) -> VortexResult<ArrayRef>
85 where
86 Self: Sized,
87 {
88 let dtype = self.dtype().clone();
89 let mut chunks = self.peekable();
90 let Some(first) = chunks.next().transpose()? else {
91 return Ok(ChunkedArray::try_new([], dtype)?.into_array());
92 };
93 if chunks.peek().is_none() {
94 return Ok(first);
95 }
96 let chunks = std::iter::once(Ok(first))
97 .chain(chunks)
98 .map(|chunk| -> VortexResult<_> {
99 let chunk = chunk?;
100 vortex_ensure_eq!(chunk.dtype(), &dtype);
101 Ok(chunk)
102 });
103 let expected_nchunks = chunks.size_hint().0;
104 chunks.process_results(|chunks| {
105 unsafe { ChunkedArray::new_unchecked_sized(chunks, dtype.clone(), expected_nchunks) }
107 .into_array()
108 })
109 }
110}
111
112impl<I: ArrayIterator> ArrayIteratorExt for I {}
113
114impl ArrayRef {
115 pub fn to_array_iterator(&self) -> impl ArrayIterator + 'static {
117 let dtype = self.dtype().clone();
118 let iter = if let Some(chunked) = self.as_opt::<Chunked>() {
119 ArrayChunkIterator::Chunked(Arc::new(chunked.into_owned()), 0)
120 } else {
121 ArrayChunkIterator::Single(Some(self.clone()))
122 };
123 ArrayIteratorAdapter::new(dtype, iter)
124 }
125}
126
127enum ArrayChunkIterator {
130 Single(Option<ArrayRef>),
131 Chunked(Arc<ChunkedArray>, usize),
132}
133
134impl Iterator for ArrayChunkIterator {
135 type Item = VortexResult<ArrayRef>;
136
137 #[inline]
138 fn next(&mut self) -> Option<Self::Item> {
139 match self {
140 ArrayChunkIterator::Single(array) => array.take().map(Ok),
141 ArrayChunkIterator::Chunked(chunked, idx) => (*idx < chunked.nchunks()).then(|| {
142 let chunk = chunked.chunk(*idx);
143 *idx += 1;
144 Ok(chunk.clone())
145 }),
146 }
147 }
148}