brainwires_storage/
lib.rs

1#![deny(missing_docs)]
2//! Brainwires Storage — backend-agnostic persistent storage for the Brainwires
3//! Agent Framework.
4//!
5//! This crate provides conversation storage with semantic search, document
6//! ingestion with hybrid retrieval, three-tier memory hierarchy, image
7//! analysis storage, cross-process lock coordination, and reusable plan
8//! templates.
9//!
10//! # Unified Database Layer ([`databases`] module)
11//!
12//! One struct per database, one shared connection, implementing one or both
13//! of the core traits:
14//!
15//! - [`StorageBackend`] — generic CRUD + vector search for domain stores
16//! - [`VectorDatabase`](databases::VectorDatabase) — RAG embedding storage
17//!   with hybrid search
18//!
19//! ### Database backends
20//!
21//! | Backend | Struct | `StorageBackend` | `VectorDatabase` | Feature |
22//! |---------|--------|:---:|:---:|---------|
23//! | LanceDB | `LanceDatabase` | YES | YES | `lance-backend` (default) |
24//! | PostgreSQL | `PostgresDatabase` | YES | YES | `postgres-backend` |
25//! | MySQL | `MySqlDatabase` | YES | NO | `mysql-backend` |
26//! | SurrealDB | `SurrealDatabase` | YES | YES | `surrealdb-backend` |
27//! | Qdrant | `QdrantDatabase` | NO | YES | `qdrant-backend` |
28//! | Pinecone | `PineconeDatabase` | NO | YES | `pinecone-backend` |
29//! | Milvus | `MilvusDatabase` | NO | YES | `milvus-backend` |
30//! | Weaviate | `WeaviateDatabase` | NO | YES | `weaviate-backend` |
31//! | NornicDB | `NornicDatabase` | NO | YES | `nornicdb-backend` |
32//!
33//! Backends that implement both traits share a single connection — construct
34//! once, wrap in `Arc`, and pass to both domain stores and RAG subsystem.
35//!
36//! # Core Infrastructure
37//!
38//! - **`FastEmbedManager`** — text embeddings via FastEmbed ONNX model
39//!   (all-MiniLM-L6-v2, 384 dimensions)
40//! - **`CachedEmbeddingProvider`** — LRU-cached embedding provider (1000 entries)
41//!
42//! # Domain Stores ([`stores`] module)
43//!
44//! - **`MessageStore`** — conversation messages with vector search and TTL expiry
45//! - **`ConversationStore`** — conversation metadata with create-or-update semantics
46//! - **`TaskStore`** / **`AgentStateStore`** — task and agent state persistence
47//! - **`PlanStore`** — execution plan storage with markdown export
48//! - **`TemplateStore`** — reusable plan templates with `{{variable}}` substitution
49//! - **`LockStore`** — SQLite-backed cross-process lock coordination
50//!
51//! # Document Management
52//!
53//! - **`DocumentStore`** — hybrid search (vector + BM25 via RRF)
54//! - **`DocumentProcessor`** — PDF, DOCX, Markdown, plain text ingestion
55//! - **`DocumentChunker`** — paragraph/sentence-aware segmentation
56//! - **`DocumentMetadataStore`** — hash-based deduplication
57//!
58//! # Image Storage
59//!
60//! - **`ImageStore`** — analyzed images with semantic search over descriptions
61//!
62//! # Tiered Memory
63//!
64//! - **`TieredMemory`** — three-tier memory hierarchy (hot/warm/cold)
65//! - **`SummaryStore`** — compressed message summaries (warm tier)
66//! - **`FactStore`** — key facts extraction (cold tier)
67//! - **`TierMetadataStore`** — access tracking and importance scoring
68//!
69//! # Feature Flags
70//!
71//! | Feature | Default | Description |
72//! |---------|---------|-------------|
73//! | `native` | Yes | LanceDB backend + FastEmbed + SQLite locks + all native stores |
74//! | `lance-backend` | Yes (via `native`) | LanceDB embedded vector database |
75//! | `postgres-backend` | No | PostgreSQL + pgvector |
76//! | `mysql-backend` | No | MySQL / MariaDB |
77//! | `surrealdb-backend` | No | SurrealDB with native MTREE vector search |
78//! | `qdrant-backend` | No | Qdrant vector search |
79//! | `pinecone-backend` | No | Pinecone cloud vectors |
80//! | `milvus-backend` | No | Milvus vectors |
81//! | `weaviate-backend` | No | Weaviate search engine |
82//! | `nornicdb-backend` | No | NornicDB graph + vector |
83//! | `wasm` | No | WASM-compatible (pure types only) |
84
85// Re-export core types
86pub use brainwires_core;
87
88// ── Always available (pure types/logic) ──────────────────────────────────
89
90pub mod image_types;
91
92/// Unified database layer — one struct per database, shared connection,
93/// implementing [`StorageBackend`] and/or [`VectorDatabase`](databases::VectorDatabase).
94pub mod databases;
95
96/// BM25 keyword search using Tantivy.
97#[cfg(feature = "lance-backend")]
98pub mod bm25_search;
99/// Glob pattern matching utilities.
100#[cfg(feature = "lance-backend")]
101pub mod glob_utils;
102/// Platform-specific path utilities.
103#[cfg(feature = "lance-backend")]
104pub mod paths;
105
106#[cfg(feature = "native")]
107pub mod file_context;
108#[cfg(feature = "native")]
109pub mod tiered_memory;
110
111/// Embedding provider for vector operations.
112#[cfg(feature = "native")]
113pub mod embeddings;
114
115/// Domain stores for conversation, message, task, plan, and other data.
116pub mod stores;
117
118// Note: persistent_task_manager lives in this crate but requires brainwires-agents
119// which creates a cyclic dependency. It's compiled only when brought in by a
120// higher-level crate (e.g. brainwires facade) that can resolve the cycle.
121// TODO: Move to brainwires-agents or a bridge crate.
122
123// ── Re-exports (always available) ────────────────────────────────────────
124
125pub use databases::BackendCapabilities;
126pub use databases::traits::StorageBackend;
127pub use databases::types::record_get;
128pub use databases::types::{FieldDef, FieldType, FieldValue, Filter, Record, ScoredRecord};
129
130#[cfg(feature = "lance-backend")]
131pub use databases::LanceDatabase;
132
133pub use image_types::{
134    ImageFormat, ImageMetadata, ImageSearchRequest, ImageSearchResult, ImageStorage,
135};
136
137pub use stores::template_store::{PlanTemplate, TemplateStore};
138
139// ── Re-exports (native only) ─────────────────────────────────────────────
140
141#[cfg(feature = "native")]
142pub use embeddings::{
143    CachedEmbeddingProvider, EmbeddingProvider, EmbeddingProviderTrait, FastEmbedManager,
144};
145#[cfg(feature = "native")]
146pub use file_context::{FileChunk, FileContent, FileContextManager};
147#[cfg(feature = "native")]
148pub use stores::conversation_store::{ConversationMetadata, ConversationStore};
149#[cfg(feature = "native")]
150pub use stores::fact_store::FactStore;
151#[cfg(feature = "native")]
152pub use stores::image_store::ImageStore;
153#[cfg(feature = "native")]
154pub use stores::lock_store::{LockRecord, LockStats, LockStore};
155#[cfg(feature = "native")]
156pub use stores::message_store::{MessageMetadata, MessageStore};
157#[cfg(feature = "native")]
158pub use stores::plan_store::PlanStore;
159#[cfg(feature = "native")]
160pub use stores::summary_store::SummaryStore;
161#[cfg(feature = "native")]
162pub use stores::task_store::{AgentStateMetadata, AgentStateStore, TaskMetadata, TaskStore};
163#[cfg(feature = "native")]
164pub use stores::tier_metadata_store::TierMetadataStore;
165#[cfg(feature = "native")]
166pub use tiered_memory::{
167    CanonicalWriteToken, MemoryAuthority, MemoryTier, MultiFactorScore, TieredMemory,
168    TieredMemoryConfig, TieredSearchResult,
169};
170// persistent_task_manager re-exports moved to brainwires facade crate
171
172/// Prelude module for convenient imports
173pub mod prelude {
174    pub use super::stores::template_store::{PlanTemplate, TemplateStore};
175
176    #[cfg(feature = "native")]
177    pub use super::embeddings::{
178        CachedEmbeddingProvider, EmbeddingProvider, EmbeddingProviderTrait, FastEmbedManager,
179    };
180    #[cfg(feature = "native")]
181    pub use super::file_context::{FileContent, FileContextManager};
182    #[cfg(feature = "native")]
183    pub use super::stores::conversation_store::{ConversationMetadata, ConversationStore};
184    #[cfg(feature = "native")]
185    pub use super::stores::image_store::ImageStore;
186    #[cfg(feature = "native")]
187    pub use super::stores::lock_store::{LockRecord, LockStore};
188    #[cfg(feature = "native")]
189    pub use super::stores::message_store::{MessageMetadata, MessageStore};
190    #[cfg(feature = "native")]
191    pub use super::stores::plan_store::PlanStore;
192    #[cfg(feature = "native")]
193    pub use super::stores::task_store::{TaskMetadata, TaskStore};
194    #[cfg(feature = "native")]
195    pub use super::tiered_memory::{
196        CanonicalWriteToken, MemoryAuthority, MemoryTier, TieredMemory, TieredMemoryConfig,
197        TieredSearchResult,
198    };
199}
brainwires_storage/lib.rs

brainwires_storage/
lib.rs
