vortex_array/
lib.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
#![feature(once_cell_try)]
#![feature(trusted_len)]
//! Vortex crate containing core logic for encoding and memory representation of [arrays](ArrayData).
//!
//! At the heart of Vortex are [arrays](ArrayData) and [encodings](crate::encoding::EncodingVTable).
//! Arrays are typed views of memory buffers that hold [scalars](vortex_scalar::Scalar). These
//! buffers can be held in a number of physical encodings to perform lightweight compression that
//! exploits the particular data distribution of the array's values.
//!
//! Every data type recognized by Vortex also has a canonical physical encoding format, which
//! arrays can be [canonicalized](Canonical) into for ease of access in compute functions.
//!

pub use canonical::*;
pub use context::*;
pub use data::*;
pub use metadata::*;
pub use paste;
use vortex_dtype::DType;

use crate::encoding::ArrayEncodingRef;
use crate::nbytes::ArrayNBytes;
use crate::stats::ArrayStatistics;
use crate::validity::ArrayValidity;

pub mod accessor;
pub mod aliases;
pub mod array;
pub mod arrow;
pub mod builders;
mod canonical;
pub mod compress;
pub mod compute;
mod context;
mod data;
pub mod encoding;
pub mod iter;
mod macros;
mod metadata;
pub mod nbytes;
pub mod patches;
pub mod stats;
pub mod stream;
pub mod tree;
pub mod validity;
pub mod variants;
pub mod visitor;

pub mod flatbuffers {
    //! Re-exported autogenerated code from the core Vortex flatbuffer definitions.
    pub use vortex_flatbuffers::array::*;
}

/// A depth-first pre-order iterator over a ArrayData.
pub struct ArrayChildrenIterator {
    stack: Vec<ArrayData>,
}

impl ArrayChildrenIterator {
    pub fn new(array: ArrayData) -> Self {
        Self { stack: vec![array] }
    }
}

impl Iterator for ArrayChildrenIterator {
    type Item = ArrayData;

    fn next(&mut self) -> Option<Self::Item> {
        let next = self.stack.pop()?;
        for child in next.children().into_iter().rev() {
            self.stack.push(child);
        }
        Some(next)
    }
}

pub trait ToArrayData {
    fn to_array(&self) -> ArrayData;
}

/// Consume `self` and turn it into an [`ArrayData`] infallibly.
///
/// Implementation of this array should never fail.
pub trait IntoArrayData {
    fn into_array(self) -> ArrayData;
}

/// Collects together the behavior of an array.
pub trait ArrayTrait:
    AsRef<ArrayData>
    + ArrayEncodingRef
    + ArrayDType
    + ArrayLen
    + ArrayNBytes
    + IntoCanonical
    + ArrayValidity
    + ArrayStatistics
{
}

pub trait ArrayDType {
    // TODO(ngates): move into ArrayTrait?
    fn dtype(&self) -> &DType;
}

pub trait ArrayLen {
    fn len(&self) -> usize;

    fn is_empty(&self) -> bool;
}