vortex_tensor/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Types and functionality for working with tensors, vectors, and related mathematical constructs
5//! including unit vectors, spherical coordinates, and similarity measures such as cosine
6//! similarity.
7
8#![cfg_attr(
9    test,
10    allow(clippy::unwrap_used, clippy::expect_used, clippy::unwrap_in_result)
11)]
12
13use std::sync::Arc;
14
15use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin;
16use vortex_array::arrow::ArrowSessionExt;
17use vortex_array::dtype::session::DTypeSessionExt;
18use vortex_array::scalar_fn::session::ScalarFnSessionExt;
19use vortex_array::session::ArraySessionExt;
20use vortex_session::VortexSession;
21
22use crate::scalar_fns::cosine_similarity::CosineSimilarity;
23use crate::scalar_fns::inner_product::InnerProduct;
24use crate::scalar_fns::l2_denorm::L2Denorm;
25use crate::scalar_fns::l2_norm::L2Norm;
26use crate::types::fixed_shape_tensor::FixedShapeTensor;
27use crate::types::vector::Vector;
28
29pub mod matcher;
30pub mod scalar_fns;
31
32mod types;
33
34pub use types::fixed_shape_tensor;
35pub use types::vector;
36
37pub mod encodings;
38
39pub mod vector_search;
40
41mod utils;
42
43/// Environment variable that gates registration of the tensor scalar-fn array plugins (the array
44/// encodings that let [`CosineSimilarity`], [`InnerProduct`], [`L2Denorm`], and [`L2Norm`]
45/// persist in a Vortex file). When unset, only the scalar functions themselves
46/// are registered; readers of files containing serialized tensor scalar-fn arrays will fail to
47/// deserialize. Opt-in by setting the variable to any non-empty value.
48pub const SCALAR_FN_ARRAY_TENSOR_PLUGIN_ENV: &str = "VX_SCALAR_FN_ARRAY_TENSOR_PLUGIN";
49
50/// Initialize the Vortex tensor library with a Vortex session.
51pub fn initialize(session: &VortexSession) {
52    session.dtypes().register(Vector);
53    session.dtypes().register(FixedShapeTensor);
54
55    let arrow_session = session.arrow();
56    arrow_session.register_exporter(Arc::new(Vector));
57    arrow_session.register_importer(Arc::new(Vector));
58
59    let session_fns = session.scalar_fns();
60
61    session_fns.register(CosineSimilarity);
62    session_fns.register(InnerProduct);
63    session_fns.register(L2Denorm);
64    session_fns.register(L2Norm);
65
66    // Registering the scalar-fn array plugins lets the tensor scalar fns be serialized as array
67    // encodings inside Vortex files. Gate this on an env var so applications that do not intend
68    // to persist these encodings do not pay the registry cost or widen their stable-encoding
69    // surface unintentionally.
70    if std::env::var_os(SCALAR_FN_ARRAY_TENSOR_PLUGIN_ENV).is_some_and(|v| !v.is_empty()) {
71        let session_arrays = session.arrays();
72
73        session_arrays.register(ScalarFnArrayPlugin::new(CosineSimilarity));
74        session_arrays.register(ScalarFnArrayPlugin::new(InnerProduct));
75        session_arrays.register(ScalarFnArrayPlugin::new(L2Denorm));
76        session_arrays.register(ScalarFnArrayPlugin::new(L2Norm));
77    }
78}
79
80#[cfg(test)]
81mod tests {
82    use std::sync::LazyLock;
83
84    use vortex_array::session::ArraySession;
85    use vortex_session::VortexSession;
86
87    pub static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
88        let session = VortexSession::empty().with::<ArraySession>();
89        crate::initialize(&session);
90        session
91    });
92}
vortex_tensor/lib.rs

vortex_tensor/
lib.rs
