vortex_array/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4#![cfg_attr(vortex_nightly, feature(portable_simd))]
5//! Vortex crate containing core logic for encoding and memory representation of [arrays](ArrayRef).
6//!
7//! At the heart of Vortex are [arrays](ArrayRef) and [encodings](EncodingRef).
8//! Arrays are typed views of memory buffers that hold [scalars](vortex_scalar::Scalar). These
9//! buffers can be held in a number of physical encodings to perform lightweight compression that
10//! exploits the particular data distribution of the array's values.
11//!
12//! Every data type recognized by Vortex also has a canonical physical encoding format, which
13//! arrays can be [canonicalized](Canonical) into for ease of access in compute functions.
14
15pub use array::*;
16pub use canonical::*;
17pub use context::*;
18pub use encoding::*;
19pub use hash::*;
20pub use mask_future::*;
21pub use metadata::*;
22use vortex_session::registry::Registry;
23use vortex_session::{Ref, SessionExt};
24
25use crate::arrays::{
26    BoolEncoding, ChunkedEncoding, ConstantEncoding, DecimalEncoding, ExtensionEncoding,
27    FixedSizeListEncoding, ListEncoding, ListViewEncoding, MaskedEncoding, NullEncoding,
28    PrimitiveEncoding, StructEncoding, VarBinEncoding, VarBinViewEncoding,
29};
30
31pub mod accessor;
32#[doc(hidden)]
33pub mod aliases;
34mod array;
35pub mod arrays;
36pub mod arrow;
37pub mod builders;
38mod canonical;
39pub mod compute;
40mod context;
41mod encoding;
42pub mod execution;
43pub mod expr;
44mod hash;
45pub mod iter;
46mod mask_future;
47mod metadata;
48pub mod optimizer;
49mod partial_ord;
50pub mod patches;
51pub mod pipeline;
52pub mod search_sorted;
53pub mod serde;
54pub mod stats;
55pub mod stream;
56#[cfg(any(test, feature = "test-harness"))]
57pub mod test_harness;
58pub mod validity;
59pub mod variants;
60pub mod vectors;
61pub mod vtable;
62
63pub mod flatbuffers {
64    //! Re-exported autogenerated code from the core Vortex flatbuffer definitions.
65    pub use vortex_flatbuffers::array::*;
66}
67
68pub type ArrayRegistry = Registry<EncodingRef>;
69
70#[derive(Debug)]
71pub struct ArraySession {
72    /// The set of registered array encodings.
73    registry: ArrayRegistry,
74}
75
76impl ArraySession {
77    pub fn registry(&self) -> &ArrayRegistry {
78        &self.registry
79    }
80
81    /// Register a new array encoding, replacing any existing encoding with the same ID.
82    pub fn register(&self, encoding: EncodingRef) {
83        self.registry.register(encoding)
84    }
85
86    /// Register many array encodings, replacing any existing encodings with the same ID.
87    pub fn register_many(&self, encodings: impl IntoIterator<Item = EncodingRef>) {
88        self.registry.register_many(encodings);
89    }
90}
91
92impl Default for ArraySession {
93    fn default() -> Self {
94        let encodings = ArrayRegistry::default();
95
96        // Register the canonical encodings.
97        encodings.register_many([
98            EncodingRef::new_ref(NullEncoding.as_ref()),
99            EncodingRef::new_ref(BoolEncoding.as_ref()),
100            EncodingRef::new_ref(PrimitiveEncoding.as_ref()),
101            EncodingRef::new_ref(DecimalEncoding.as_ref()),
102            EncodingRef::new_ref(VarBinViewEncoding.as_ref()),
103            EncodingRef::new_ref(ListViewEncoding.as_ref()),
104            EncodingRef::new_ref(FixedSizeListEncoding.as_ref()),
105            EncodingRef::new_ref(StructEncoding.as_ref()),
106            EncodingRef::new_ref(ExtensionEncoding.as_ref()),
107        ]);
108
109        // Register the utility encodings.
110        encodings.register_many([
111            EncodingRef::new_ref(ChunkedEncoding.as_ref()),
112            EncodingRef::new_ref(ConstantEncoding.as_ref()),
113            EncodingRef::new_ref(MaskedEncoding.as_ref()),
114            EncodingRef::new_ref(ListEncoding.as_ref()),
115            EncodingRef::new_ref(VarBinEncoding.as_ref()),
116        ]);
117
118        Self {
119            registry: encodings,
120        }
121    }
122}
123
124/// Session data for Vortex arrays.
125pub trait ArraySessionExt: SessionExt {
126    /// Returns the array encoding registry.
127    fn arrays(&self) -> Ref<'_, ArraySession> {
128        self.get::<ArraySession>()
129    }
130}
131impl<S: SessionExt> ArraySessionExt for S {}
vortex_array/lib.rs

vortex_array/
lib.rs
