dataspool_rs/lib.rs
1//! # DataSpool: Efficient Data Bundling System
2//!
3//! DataSpool eliminates filesystem overhead by concatenating multiple items
4//! (cards, images, etc.) into a single indexed file with SQLite-based metadata.
5//!
6//! ## Features
7//!
8//! - **Spool Format**: Single file with byte-offset index
9//! - **Vector Database**: SQLite-based embeddings and metadata
10//! - **Random Access**: Direct byte offset seeks, no scanning
11//! - **Multiple Variants**: Cards, Images (future), Binary blobs
12//!
13//! ## Format
14//!
15//! ```text
16//! .spool file:
17//! [Magic: SP01][Version: 1]
18//! [Card Count: u32][Index Offset: u64]
19//! [Card 0 data...][Card 1 data...][Card N data...]
20//! [Index: offset0, len0, offset1, len1, ...]
21//!
22//! .db file (SQLite):
23//! - documents table (id, file_path, source, metadata, spool_offset, spool_length)
24//! - embeddings table (doc_id, vector BLOB)
25//! ```
26
27pub mod error;
28pub mod spool;
29pub mod persistent_store;
30
31pub use error::{DataSpoolError, Result};
32pub use spool::{SpoolBuilder, SpoolEntry, SpoolReader};
33pub use persistent_store::{DocumentRef, FileSearchResult, PersistentVectorStore};