genegraph-storage 0.63.0

vector database: base Lance storage
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

//! #117-1: clustering artifact I/O, split out of [`StorageBackend`].
//!
//! Centroid maps, subcentroids, item norms and cluster assignments are one
//! consumer's ML-pipeline vocabulary. Keeping them on the base trait coupled
//! a general storage abstraction to that vocabulary and forced every
//! implementor to stub methods it does not support. As an extension
//! sub-trait, only clustering-capable backends implement it (#117-2).

use std::path::Path;

use crate::traits::backend::StorageBackend;
use smartcore::linalg::basic::matrix::DenseMatrix;

use crate::StorageResult;

/// Clustering-specific artifact save/load (centroid maps, subcentroid
/// lambdas, subcentroids, item norms, cluster assignments).
///
/// Requirements, not defaults: the on-disk layout is backend-specific.
/// A backend implements this trait only if it supports the clustering
/// pipeline artifacts.
pub trait ClusteringArtifacts: StorageBackend {
    /// Save centroid_map (vector of usize mapping items to centroids)
    async fn save_centroid_map(&self, map: &[usize], md_path: &Path) -> StorageResult<()>;

    /// Load centroid_map
    async fn load_centroid_map(&self) -> StorageResult<Vec<usize>>;

    /// Save subcentroid_lambdas (tau values for subcentroids)
    async fn save_subcentroid_lambdas(&self, lambdas: &[f64], md_path: &Path) -> StorageResult<()>;

    /// Load subcentroid_lambdas
    async fn load_subcentroid_lambdas(&self) -> StorageResult<Vec<f64>>;

    /// Save subcentroids (dense matrix)
    async fn save_subcentroids(
        &self,
        subcentroids: &DenseMatrix<f64>,
        md_path: &Path,
    ) -> StorageResult<()>;

    /// Load subcentroids
    async fn load_subcentroids(&self) -> StorageResult<Vec<Vec<f64>>>;

    /// Save item norms (precomputed L2 norms for fast distance computation)
    async fn save_item_norms(&self, item_norms: &[f64], md_path: &Path) -> StorageResult<()>;

    /// Load item norms
    async fn load_item_norms(&self) -> StorageResult<Vec<f64>>;

    /// Save cluster assignments (Vec<Option<usize>>)
    async fn save_cluster_assignments(
        &self,
        assignments: &[Option<usize>],
        md_path: &Path,
    ) -> StorageResult<()>;

    /// Load cluster assignments
    async fn load_cluster_assignments(&self) -> StorageResult<Vec<Option<usize>>>;
}