vortex-array 0.68.0

Vortex in memory columnar data format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::future::Future;

use futures::TryStreamExt;
use vortex_error::VortexResult;

use crate::ArrayRef;
use crate::IntoArray;
use crate::arrays::ChunkedArray;
use crate::stream::ArrayStream;
use crate::stream::SendableArrayStream;

pub trait ArrayStreamExt: ArrayStream {
    /// Box the [`ArrayStream`] so that it can be sent between threads.
    fn boxed(self) -> SendableArrayStream
    where
        Self: Sized + Send + 'static,
    {
        Box::pin(self)
    }

    /// Collect the stream into a single `Array`.
    ///
    /// If the stream yields multiple chunks, they will be returned as a [`ChunkedArray`].
    fn read_all(self) -> impl Future<Output = VortexResult<ArrayRef>>
    where
        Self: Sized,
    {
        async move {
            let dtype = self.dtype().clone();
            let mut chunks: Vec<ArrayRef> = self.try_collect().await?;
            if chunks.len() == 1 {
                Ok(chunks.remove(0))
            } else {
                Ok(ChunkedArray::try_new(chunks, dtype)?.into_array())
            }
        }
    }
}

impl<S: ArrayStream> ArrayStreamExt for S {}