Struct StorageManager 

pub struct StorageManager { /* private fields */ }

Implementations

impl StorageManager

pub async fn new(db_path: &str) -> Result<Self>

pub async fn new_lance_only(db_path: &str) -> Result<Self>

Create a storage manager for CLI tools. Use this for CLI tools that only need vector operations (index/search).

pub fn lance_path(&self) -> &str

pub async fn require_current_schema_for_writes(&self) -> Result<()>

pub async fn schema_status( &self, expected_schema: SchemaVersion, ) -> Result<SchemaStatusReport>

pub async fn missing_required_columns( table: &Table, target: SchemaVersion, ) -> Result<Vec<Field>>

pub async fn migrate_lance_schema( db_path: &str, target: SchemaVersion, check_only: bool, ) -> Result<SchemaMigrationReport>

pub fn cross_store_recovery_dir(&self) -> PathBuf

pub fn persist_cross_store_recovery_batch( &self, batch: &CrossStoreRecoveryBatch, ) -> Result<PathBuf>

pub fn update_cross_store_recovery_batch( &self, batch: &CrossStoreRecoveryBatch, ) -> Result<PathBuf>

pub fn clear_cross_store_recovery_batch(&self, batch_id: &str) -> Result<()>

pub fn list_cross_store_recovery_batches( &self, ) -> Result<Vec<CrossStoreRecoveryBatch>>

pub async fn refresh(&self) -> Result<()>

Refresh the table connection to see new data written by other processes. This clears the cached table reference, forcing it to be re-opened on next query.

pub async fn ensure_collection(&self) -> Result<()>

pub async fn add_to_store(&self, documents: Vec<ChromaDocument>) -> Result<()>

pub async fn search_store( &self, namespace: Option<&str>, embedding: Vec<f32>, k: usize, ) -> Result<Vec<ChromaDocument>>

pub async fn all_documents_page( &self, namespace: Option<&str>, offset: usize, limit: usize, ) -> Result<Vec<ChromaDocument>>

Return a single page of documents without running a vector search.

Used by admin/reporting paths that need deterministic limit/offset behavior without assuming any embedding dimension or creating a table on read.

pub async fn all_documents( &self, namespace: Option<&str>, limit: usize, ) -> Result<Vec<ChromaDocument>>

Return documents without running a vector search. Used by admin/reporting paths that need a bounded full-table scan starting from the first row.

pub async fn get_document( &self, namespace: &str, id: &str, ) -> Result<Option<ChromaDocument>>

pub async fn delete_document(&self, namespace: &str, id: &str) -> Result<usize>

pub async fn delete_documents( &self, namespace: &str, ids: &[&str], ) -> Result<usize>

Batch delete documents by IDs within a namespace.

Issues a single DELETE WHERE namespace = X AND id IN (...) per chunk, avoiding the per-document table scan that delete_document incurs when called in a loop. Predicate is split into 500-id chunks to keep SQL length bounded regardless of caller batch size.

pub async fn delete_namespace_documents(&self, namespace: &str) -> Result<usize>

pub async fn rename_namespace_atomic( &self, from: &str, to: &str, ) -> Result<usize>

pub fn get_collection_name(&self) -> &str

pub async fn get_filtered_in_namespace( &self, namespace: &str, filter: &str, ) -> Result<Vec<ChromaDocument>>

pub async fn search_store_with_layer( &self, namespace: Option<&str>, embedding: Vec<f32>, k: usize, layer_filter: Option<SliceLayer>, ) -> Result<Vec<ChromaDocument>>

Search with optional layer filtering for onion slice architecture

pub async fn get_children( &self, namespace: &str, parent_id: &str, ) -> Result<Vec<ChromaDocument>>

Get a document by ID and expand to get its children

pub async fn get_parent( &self, namespace: &str, child_id: &str, ) -> Result<Option<ChromaDocument>>

Get the parent of a document (drill up in onion hierarchy)

pub async fn has_content_hash( &self, namespace: &str, hash: &str, ) -> Result<bool>

Check if a content hash already exists in a namespace (for exact-match deduplication)

Returns Ok(false) if:

• Table doesn’t exist yet
• Table has old schema without content_hash column (graceful degradation)

pub async fn has_source_hash(&self, namespace: &str, hash: &str) -> Result<bool>

Check if any chunk in namespace already references the given source-document hash. Used by the indexing pipeline to skip re-embedding files that were already ingested (P4 — pre-index source-level dedup).

Returns Ok(false) if the table doesn’t exist yet, or if the table is on a pre-v4 schema without the source_hash column (graceful degradation — older namespaces should be backfilled via /admin/backfill-hashes).

pub async fn filter_existing_hashes<'a>( &self, namespace: &str, hashes: &'a [String], ) -> Result<Vec<&'a String>>

Filter a list of hashes to return only those that don’t exist in the namespace. This is more efficient than calling has_content_hash for each hash individually.

Returns all hashes as “new” if table has old schema without content_hash column.

pub async fn optimize(&self) -> Result<OptimizeStats>

Run all optimizations (compact + prune old versions)

pub async fn compact(&self) -> Result<OptimizeStats>

Compact small files into larger ones for better performance

pub async fn cleanup( &self, older_than_days: Option<u64>, ) -> Result<OptimizeStats>

Remove old versions older than specified duration (default: 7 days)

pub async fn stats(&self) -> Result<TableStats>

Get table statistics (row count, fragments, etc.)

pub async fn count_namespace(&self, namespace: &str) -> Result<usize>

Count rows in a specific namespace

pub async fn get_all_in_namespace( &self, namespace: &str, ) -> Result<Vec<ChromaDocument>>

Get all documents from a namespace (for migration/export)

Note: This uses a full table scan with namespace filter. For very large namespaces, consider batching.

pub async fn namespace_exists(&self, namespace: &str) -> Result<bool>

Check if a namespace exists (has any documents)

impl StorageManager

pub async fn garbage_collect(&self, config: &GcConfig) -> Result<GcStats>

Run garbage collection based on configuration

pub async fn run_gc(&self, config: &GcConfig) -> Result<GcStats>

👎Deprecated:

use garbage_collect

pub async fn list_namespaces(&self) -> Result<Vec<(String, usize)>>

List all unique namespaces in the database