Struct LanceStorageGraph 

pub struct LanceStorageGraph { /* private fields */ }

Lance-based storage backend for ArrowSpace graph embeddings.

Stores dense and sparse matrices as Lance datasets using a columnar format (row, col, value for sparse; col_* for dense) schema for efficient random and columnar access.

Metadata must be seeded before any save_* call so the storage directory is initialized; the example below is executed as a doc-test on every cargo test run so it cannot go stale.

Examples

use genegraph_storage::lance_storage_graph::LanceStorageGraph;
use genegraph_storage::metadata::GeneMetadata;
use genegraph_storage::traits::backend::StorageBackend;
use genegraph_storage::traits::metadata::Metadata;
use smartcore::linalg::basic::arrays::{Array, Array2};
use smartcore::linalg::basic::matrix::DenseMatrix;

let base = std::env::temp_dir().join(format!("genegraph_doc_{}", std::process::id()));
let storage = LanceStorageGraph::new(
    base.to_string_lossy().to_string(),
    "doc_example".to_string(),
);

// some 2D data
let dense: Vec<Vec<f64>> = vec![vec![0.1, 0.4], vec![0.5, 0.2], vec![0.03, 0.8]];
let (nitems, nfeatures) = (dense.len(), dense[0].len());
let data = DenseMatrix::<f64>::from_iterator(
    dense.iter().flatten().copied(),
    nitems,
    nfeatures,
    0,
);

// seed metadata FIRST to initialize the storage directory
let md = GeneMetadata::seed_metadata("doc_example", nitems, nfeatures, &storage)
    .await
    .unwrap();
let md_path = storage.save_metadata(&md).await.unwrap();

// your data is saved in an efficient Lance format
storage.save_dense("my_dataset", &data, &md_path).await.unwrap();

// Loading back
let loaded = storage.load_dense("my_dataset").await.unwrap();
assert_eq!(loaded.shape(), (nitems, nfeatures));

Implementations

impl LanceStorageGraph

pub fn new(base: String, name: String) -> Self

Creates a new Lance storage backend.

This is used for on-the-fly creation. For proper setup use Genefold<...>::seed.

Arguments

• base - Base directory path for all storage files
• name - Name prefix for this storage instance

pub async fn spawn( base_path: String, ) -> Result<(Self, GeneMetadata), StorageError>

Spawn a LanceStorage from an existing seeded directory (with metadata.json)

pub fn scoped_generation(&self, generation: u64) -> Self

A handle over generation gen of this logical dataset: artifact paths resolve as {logical}__g{gen}_{key}.lance and the metadata path as {logical}__g{gen}_metadata.json — the per-generation commit pointer (#93, RFC #81-P5). The logical identity stays recoverable via crate::generations::logical_name.

Trait Implementations

impl Clone for LanceStorageGraph

fn clone(&self) -> LanceStorageGraph

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LanceStorageGraph

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl LanceStorage for LanceStorageGraph

async fn write_lance_batch_async( &self, uri: String, batch: RecordBatch, ) -> StorageResult<()>

async fn read_lance_all_batches_async( &self, uri: String, ) -> StorageResult<RecordBatch>

async fn save_primitive_column<T: ArrowPrimitiveType>( &self, key: &str, field_name: &str, values: Vec<T::Native>, md_path: &Path, ) -> StorageResult<()>

where Self: StorageBackend,

Writes a single-column primitive vector as a Lance dataset named <name>_<key>.lance and registers it in the metadata files map under key with filetype “vector” and shape (<len>, 1).

impl StorageBackend for LanceStorageGraph

fn basepath_to_uri(&self) -> StorageResult<String>

Converts the base path for the store to a file:// URI for Lance.

async fn save_dense( &self, key: &str, matrix: &DenseMatrix<f64>, md_path: &Path, ) -> StorageResult<()>

Save dense matrix using Lance-optimized vector format.

Each row of the matrix becomes a FixedSizeList entry for efficient vector operations. This format is optimized for vector search and enables Lance’s full-zip encoding.

Arguments

• filename - any name
• matrix - Dense matrix to save (N rows × F cols)
• md_path - Metadata file path for validation

async fn load_dense(&self, key: &str) -> StorageResult<DenseMatrix<f64>>

Load dense matrix from Lance-optimized vector format.

Reads FixedSizeList vectors and reconstructs a column-major DenseMatrix.

Arguments

• filename - any name previously assigned

Returns

Column-major DenseMatrix matching smartcore conventions

async fn load_dense_from_file( &self, path: &Path, ) -> StorageResult<DenseMatrix<f64>>

Load initial data using columnar format from a file path.

Async test helper that avoids any internal blocking runtimes.

async fn save_dense_to_file( data: &DenseMatrix<f64>, path: &Path, ) -> StorageResult<()>

Save dense matrix to file in columnar format (col_0, col_1, …, col_N)

Async test helper that avoids any internal blocking runtimes.

async fn save_centroid_map( &self, map: &[usize], md_path: &Path, ) -> StorageResult<()>

Save centroid_map (item-to-centroid assignments)

async fn load_centroid_map(&self) -> StorageResult<Vec<usize>>

Load centroid_map

async fn save_subcentroid_lambdas( &self, lambdas: &[f64], md_path: &Path, ) -> StorageResult<()>

Save subcentroid_lambdas (tau values for subcentroids)

async fn load_subcentroid_lambdas(&self) -> StorageResult<Vec<f64>>

Load subcentroid_lambdas

async fn save_subcentroids( &self, subcentroids: &DenseMatrix<f64>, md_path: &Path, ) -> StorageResult<()>

Save subcentroids (dense matrix)

async fn load_subcentroids(&self) -> StorageResult<Vec<Vec<f64>>>

Load subcentroids as Vec<Vec>

async fn save_item_norms( &self, item_norms: &[f64], md_path: &Path, ) -> StorageResult<()>

Save item norms vector

async fn load_item_norms(&self) -> StorageResult<Vec<f64>>

Load item norms vector

fn get_base(&self) -> String

Base directory of the instance

fn get_name(&self) -> String

Name of the instance

fn base_path(&self) -> PathBuf

Returns the base directory path.

fn metadata_path(&self) -> PathBuf

Returns the metadata path.

fn file_path(&self, key: &str) -> PathBuf

Compute the full Lance/parquet file path for a logical filetype.

async fn save_sparse( &self, key: &str, matrix: &CsMat<f64>, md_path: &Path, ) -> StorageResult<()>

Saves a sparse matrix. Requires metadata to exist.

async fn load_sparse(&self, key: &str) -> StorageResult<CsMat<f64>>

Loads a sparse matrix from storage.

async fn save_lambdas( &self, lambdas: &[f64], md_path: &Path, ) -> StorageResult<()>

Saves lambda eigenvalues. Requires metadata to exist.

async fn load_lambdas(&self) -> StorageResult<Vec<f64>>

Loads lambda eigenvalues from storage.

async fn save_vector( &self, key: &str, vector: &[f64], md_path: &Path, ) -> StorageResult<()>

save a generic f64 sequence

async fn save_index( &self, key: &str, vector: &[usize], md_path: &Path, ) -> StorageResult<()>

Save vectors that are not lambdas but indices.

async fn load_vector(&self, filename: &str) -> StorageResult<Vec<f64>>

async fn load_index(&self, filename: &str) -> StorageResult<Vec<usize>>

Load index or generic usize vector from storage.

async fn save_cluster_assignments( &self, assignments: &[Option<usize>], md_path: &Path, ) -> StorageResult<()>

Save cluster assignments (Vec<Option>)

async fn load_cluster_assignments(&self) -> StorageResult<Vec<Option<usize>>>

Load cluster assignments

fn exists(path: &str) -> (bool, Option<PathBuf>)

Returns true and the path to the metadata file if metadata file exists and is valid, false otherwise. This is used to avoid overwriting existing indexes.

fn path_to_uri(path: &Path) -> StorageResult<String>

Converts a full file path to a file:// URI for Lance.

fn validate_initialized(&self, md_path: &Path) -> StorageResult<()>

Validates that the storage directory is properly initialized with metadata.

fn to_dense_record_batch( &self, matrix: &DenseMatrix<f64>, ) -> Result<RecordBatch, StorageError>

Converts a dense matrix to a RecordBatch in vector format (Lance-optimized). Each row of the matrix becomes a single FixedSizeList entry.

fn from_dense_record_batch( &self, batch: &RecordBatch, ) -> Result<DenseMatrix<f64>, StorageError>

Reconstructs a dense matrix from a RecordBatch in vector format.

fn to_sparse_record_batch(&self, m: &CsMat<f64>) -> StorageResult<RecordBatch>

Converts a sparse CSR matrix to a RecordBatch in columnar format.

fn from_sparse_record_batch( &self, batch: RecordBatch, expected_rows: usize, expected_cols: usize, ) -> StorageResult<CsMat<f64>>

Reconstructs a sparse CSR matrix from a RecordBatch in columnar format.

async fn save_metadata(&self, metadata: &GeneMetadata) -> StorageResult<PathBuf>

Initializes storage by saving metadata. Must be called first.

async fn load_metadata(&self) -> StorageResult<GeneMetadata>

Loads metadata from storage.