second-brain-core 0.13.3

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
mod claim;
mod lexical;
mod outbox;
pub mod raw;
#[cfg(test)]
mod sealed_tests;
mod vector;

pub use claim::{CLAIM_FILE, IndexClaim};

pub use usearch::ScalarKind;
pub use vector::EMBEDDING_DIM;

use std::path::{Path, PathBuf};

use anyhow::{Result, anyhow};
use uuid::Uuid;

use crate::residency::IndexMemory;
use crate::sealed::SealKeys;
use crate::sqlite_store::SqliteStore;
use lexical::LexicalIndex;
use vector::VectorIndex;

const VECTOR_FILE: &str = "vectors.usearch";
const CHUNK_FILE: &str = "chunks.usearch";

/// Chunk vectors outnumber memory vectors several times over, and F16 halves
/// their RAM against F32 at no measurable cost to cosine ranking.
const CHUNK_SCALAR_KIND: ScalarKind = ScalarKind::F16;

/// `ReadWrite` indexes RAM-load usearch and may construct a tantivy writer.
/// `ReadOnly` indexes mmap usearch via `view()` and open tantivy purely for
/// search, so a multi-tenant service can hold thousands of them open without
/// paying either cost. See `Indexes::open_readonly`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IndexesMode {
    ReadWrite,
    ReadOnly,
}

/// The retrieval indexes beside brain.db: usearch for dense vectors (one
/// file for whole memories, one for their sentence chunks), tantivy for
/// BM25. Fed exclusively through the store's index_outbox (see
/// `Indexes::drain`), so SQLite remains the source of truth and both indexes
/// can always be rebuilt from it without re-embedding.
pub struct Indexes {
    vector: VectorIndex,
    chunks: VectorIndex,
    lexical: LexicalIndex,
    dir: PathBuf,
    mode: IndexesMode,
    unpersisted: bool,
    seal: Option<SealKeys>,
}

impl Indexes {
    /// Every file and directory the indexes keep in their directory.
    pub const FILE_NAMES: [&str; 3] = [VECTOR_FILE, CHUNK_FILE, "tantivy"];

    pub fn open(dir: &Path) -> Result<Self> {
        Self::open_with_kind(dir, ScalarKind::F32)
    }

    /// Same as `open`, with the scalar kind the vector index quantizes to
    /// made explicit. An existing vector file must have been written as that
    /// kind or this errors; `on_disk_scalar_kind` says which kind it was.
    pub fn open_with_kind(dir: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        std::fs::create_dir_all(dir)?;
        let vector = VectorIndex::open_with_kind(&dir.join(VECTOR_FILE), scalar_kind)?;
        let chunks = VectorIndex::open_with_kind(&dir.join(CHUNK_FILE), CHUNK_SCALAR_KIND)?;
        let lexical = LexicalIndex::open(&dir.join("tantivy"))?;
        Ok(Self {
            vector,
            chunks,
            lexical,
            dir: dir.to_path_buf(),
            mode: IndexesMode::ReadWrite,
            unpersisted: false,
            seal: None,
        })
    }

    /// Same as `open`, with every index file read and written sealed under
    /// `keys`. Files written before sealing open as they are; `reseal`
    /// seals them.
    pub fn open_keyed(dir: &Path, keys: &SealKeys) -> Result<Self> {
        Self::open_with_kind_keyed(dir, ScalarKind::F32, keys)
    }

    pub fn open_with_kind_keyed(
        dir: &Path,
        scalar_kind: ScalarKind,
        keys: &SealKeys,
    ) -> Result<Self> {
        std::fs::create_dir_all(dir)?;
        let vector = VectorIndex::open_with_kind_keyed(&dir.join(VECTOR_FILE), scalar_kind, keys)?;
        let chunks =
            VectorIndex::open_with_kind_keyed(&dir.join(CHUNK_FILE), CHUNK_SCALAR_KIND, keys)?;
        let lexical = LexicalIndex::open_keyed(&dir.join("tantivy"), keys)?;
        Ok(Self {
            vector,
            chunks,
            lexical,
            dir: dir.to_path_buf(),
            mode: IndexesMode::ReadWrite,
            unpersisted: false,
            seal: Some(keys.clone()),
        })
    }

    /// Opens both indexes read-only: usearch via mmap `view()`, tantivy with
    /// no `IndexWriter` ever constructed. Errors if either index is missing
    /// instead of creating one, and creates no files or directories, so an
    /// admission path can probe a tenant without side effects. The chunk
    /// index is the exception: an install that predates it has no chunk
    /// file until its writer first persists, so a missing one opens empty.
    pub fn open_readonly(dir: &Path) -> Result<Self> {
        Self::open_readonly_with_kind(dir, ScalarKind::F32)
    }

    /// Same as `open_readonly`, with the vector index's scalar kind made
    /// explicit; a vector file written as another kind is an error.
    pub fn open_readonly_with_kind(dir: &Path, scalar_kind: ScalarKind) -> Result<Self> {
        let vector = VectorIndex::open_view_with_kind(&dir.join(VECTOR_FILE), scalar_kind)?;
        let chunk_path = dir.join(CHUNK_FILE);
        let chunks = if chunk_path.exists() {
            VectorIndex::open_view_with_kind(&chunk_path, CHUNK_SCALAR_KIND)?
        } else {
            VectorIndex::empty_view_with_kind(&chunk_path, CHUNK_SCALAR_KIND)?
        };
        let lexical = LexicalIndex::open_reader(&dir.join("tantivy"))?;
        Ok(Self {
            vector,
            chunks,
            lexical,
            dir: dir.to_path_buf(),
            mode: IndexesMode::ReadOnly,
            unpersisted: false,
            seal: None,
        })
    }

    /// Same as `open_readonly_with_kind` over indexes sealed under `keys`.
    /// The vectors load into RAM, since a sealed file cannot be mmapped;
    /// a reader in the writer's own process uses `shared_reader` instead.
    pub fn open_readonly_with_kind_keyed(
        dir: &Path,
        scalar_kind: ScalarKind,
        keys: &SealKeys,
    ) -> Result<Self> {
        let vector = VectorIndex::open_loaded_with_kind(&dir.join(VECTOR_FILE), scalar_kind, keys)?;
        let chunk_path = dir.join(CHUNK_FILE);
        let chunks = if chunk_path.exists() {
            VectorIndex::open_loaded_with_kind(&chunk_path, CHUNK_SCALAR_KIND, keys)?
        } else {
            VectorIndex::empty_loaded_with_kind(&chunk_path, CHUNK_SCALAR_KIND, keys)?
        };
        let lexical = LexicalIndex::open_reader_keyed(&dir.join("tantivy"), keys)?;
        Ok(Self {
            vector,
            chunks,
            lexical,
            dir: dir.to_path_buf(),
            mode: IndexesMode::ReadOnly,
            unpersisted: false,
            seal: Some(keys.clone()),
        })
    }

    /// A `ReadOnly` handle that searches this writer's vectors in memory,
    /// seeing each change as it is made, and reads the same tantivy index,
    /// whose commits it sees after `reload_view`. Its `memory` counts no
    /// vector bytes, since they are this writer's.
    pub fn shared_reader(&self) -> Result<Indexes> {
        self.require_writable()?;
        Ok(Self {
            vector: self.vector.shared_reader(),
            chunks: self.chunks.shared_reader(),
            lexical: self.lexical.shared_reader()?,
            dir: self.dir.clone(),
            mode: IndexesMode::ReadOnly,
            unpersisted: false,
            seal: self.seal.clone(),
        })
    }

    /// None when the indexes were opened without keys.
    pub fn seal_keys(&self) -> Option<SealKeys> {
        self.seal.clone()
    }

    /// Files written from now on are sealed under `keys`' current key, and
    /// files under its retired keys still open. Errors on indexes opened
    /// without keys.
    pub fn set_seal_keys(&mut self, keys: SealKeys) -> Result<()> {
        self.lexical.set_seal_keys(keys.clone())?;
        self.vector.set_seal_keys(keys.clone());
        self.chunks.set_seal_keys(keys.clone());
        self.seal = Some(keys);
        Ok(())
    }

    /// Persists, then rewrites every index file not sealed under the
    /// current key (plaintext or under a retired key), so no file needs a
    /// retired key afterwards.
    pub fn reseal(&mut self) -> Result<()> {
        self.require_writable()?;
        if self.seal.is_none() {
            return Err(anyhow!(
                "indexes at {} were opened without keys; reopen them keyed to seal them",
                self.dir.display()
            ));
        }
        self.persist()?;
        self.lexical.reseal()?;
        Ok(())
    }

    /// The scalar kind the vector index in `dir` was written as, read from
    /// the usearch file header alone. `None` when no vector index has been
    /// persisted there yet, which is the normal state of a new tenant.
    pub fn on_disk_scalar_kind(dir: &Path) -> Result<Option<ScalarKind>> {
        VectorIndex::on_disk_kind(&dir.join(VECTOR_FILE))
    }

    pub fn on_disk_chunk_scalar_kind(dir: &Path) -> Result<Option<ScalarKind>> {
        VectorIndex::on_disk_kind(&dir.join(CHUNK_FILE))
    }

    pub fn mode(&self) -> IndexesMode {
        self.mode
    }

    pub fn scalar_kind(&self) -> ScalarKind {
        self.vector.scalar_kind()
    }

    pub(crate) fn vector_mut(&mut self) -> &mut VectorIndex {
        self.unpersisted = true;
        &mut self.vector
    }

    pub(crate) fn chunks_mut(&mut self) -> &mut VectorIndex {
        self.unpersisted = true;
        &mut self.chunks
    }

    pub(crate) fn lexical_mut(&mut self) -> &mut LexicalIndex {
        self.unpersisted = true;
        &mut self.lexical
    }

    /// True from the first change since the last successful `persist`, even
    /// when the operation that reached the indexes then failed: the files on
    /// disk may lack what this handle holds.
    pub fn has_unpersisted_changes(&self) -> bool {
        self.unpersisted
    }

    pub(crate) fn require_writable(&self) -> Result<()> {
        match self.mode {
            IndexesMode::ReadWrite => Ok(()),
            IndexesMode::ReadOnly => Err(anyhow!(
                "indexes at {} are open read-only; mutation is not permitted",
                self.dir.display()
            )),
        }
    }

    /// Returns (memory id, cosine similarity) pairs, best first. The store
    /// resolves usearch's u64 keys back to memory ids via vec_keys.
    pub fn vector_search(
        &self,
        store: &SqliteStore,
        embedding: &[f32],
        k: usize,
    ) -> Result<Vec<(Uuid, f32)>> {
        let hits = self.vector.search(embedding, k)?;
        let mut out = Vec::with_capacity(hits.len());
        for (key, similarity) in hits {
            if let Some(id) = store.memory_id_for_vec_key(key)? {
                out.push((id, similarity));
            }
        }
        Ok(out)
    }

    /// Returns (memory id, cosine similarity, chunk ord) for the best chunk
    /// of each memory among the top `k` chunk hits, best first, so fewer
    /// than `k` memories come back when several of one memory's chunks hit.
    pub fn chunk_search(
        &self,
        store: &SqliteStore,
        query: &[f32],
        k: usize,
    ) -> Result<Vec<(Uuid, f32, usize)>> {
        if self.chunks.len() == 0 {
            return Ok(Vec::new());
        }
        let mut seen = std::collections::HashSet::new();
        let mut out = Vec::new();
        for (key, similarity) in self.chunks.search(query, k)? {
            if let Some((id, ord)) = store.chunk_for_vec_key(key)?
                && seen.insert(id)
            {
                out.push((id, similarity, ord));
            }
        }
        Ok(out)
    }

    /// Returns (memory id, BM25 score) pairs, best first.
    pub fn bm25_search(&self, query: &str, k: usize) -> Result<Vec<(Uuid, f32)>> {
        self.lexical.search(query, k)
    }

    pub fn vector_len(&self) -> usize {
        self.vector.len()
    }

    pub fn chunk_vector_len(&self) -> usize {
        self.chunks.len()
    }

    pub fn lexical_len(&self) -> u64 {
        self.lexical.num_docs()
    }

    pub fn dir(&self) -> &Path {
        &self.dir
    }

    /// Index counts must match the store's indexable rows; drift means a
    /// missed outbox row or lost index file and calls for `rebuild`.
    pub fn consistent_with(&self, store: &SqliteStore) -> Result<bool> {
        let indexable = store.indexable_count()?;
        let embedded = store.vector_indexable_count(EMBEDDING_DIM)?;
        let embedded_chunks = store.embedded_chunk_count(EMBEDDING_DIM)?;
        Ok(self.vector.len() == embedded
            && self.chunks.len() == embedded_chunks
            && self.lexical.num_docs() as usize == indexable)
    }

    /// Consumes the handle once its lexical merges have finished. Dropping a
    /// tantivy writer kills its segment updater, which throws away a merge
    /// still running.
    pub fn wait_for_merges(mut self) -> Result<()> {
        self.lexical.close_writer()
    }

    /// Lets the lexical writer's merges finish and frees its indexing
    /// buffers, keeping the handle; the next change opens a new writer.
    pub fn close_lexical_writer(&mut self) -> Result<()> {
        self.lexical.close_writer()
    }

    pub fn persist(&mut self) -> Result<()> {
        self.require_writable()?;
        self.lexical.commit()?;
        self.vector.save()?;
        self.chunks.save()?;
        self.unpersisted = false;
        Ok(())
    }

    /// Rewrites the lexical index into fresh segments so text removed from
    /// it no longer sits in its files; the vector files hold no text.
    pub fn compact(&mut self) -> Result<()> {
        self.require_writable()?;
        self.lexical.compact()
    }

    /// Re-opens on-disk state so a `ReadOnly` handle observes what a writer
    /// elsewhere has since persisted. A no-op on `ReadWrite`: see
    /// `VectorIndex::reload_view` for why a mutable handle has nothing to
    /// pick up from disk. On a `shared_reader` only the tantivy reader
    /// reloads; the vectors are already the writer's.
    pub fn reload_view(&mut self) -> Result<()> {
        self.vector.reload_view()?;
        if self.dir.join(CHUNK_FILE).exists() {
            self.chunks.reload_view()?;
        }
        self.lexical.reload_reader()
    }

    /// Anonymous memory these indexes hold, chunk vectors included; a
    /// `shared_reader` holds no vectors of its own.
    pub fn memory(&self) -> IndexMemory {
        IndexMemory {
            vectors: self.vector.anonymous_bytes() + self.chunks.anonymous_bytes(),
            lexical: self.lexical.heap_bytes(),
        }
    }

    /// Bytes this tenant's indexes cost to keep open: usearch's own RAM
    /// accounting plus the on-disk footprint of the tantivy directory, which
    /// `memory_usage` never sees. Admission control budgets in bytes, not in
    /// a count of open tenants.
    pub fn resident_bytes(&self) -> usize {
        self.vector.memory_usage()
            + self.chunks.memory_usage()
            + dir_size(&self.dir.join("tantivy")) as usize
    }
}

/// A missing directory contributes zero bytes rather than erroring: for a
/// read-only handle beside an index that has never persisted anything,
/// "does not exist yet" and "zero bytes on disk" are the same fact. Entries
/// that vanish mid-walk (a concurrent writer mid-rename) are skipped for the
/// same reason `du` skips them: this is a best-effort accounting walk, not a
/// correctness-load-bearing read.
fn dir_size(dir: &Path) -> u64 {
    let Ok(entries) = std::fs::read_dir(dir) else {
        return 0;
    };
    let mut total = 0u64;
    for entry in entries.flatten() {
        let path = entry.path();
        if path.is_dir() {
            total += dir_size(&path);
        } else if let Ok(metadata) = entry.metadata() {
            total += metadata.len();
        }
    }
    total
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn unpersisted_changes_last_until_a_persist_succeeds() {
        let dir = tempfile::tempdir().unwrap();
        let mut idx = Indexes::open(dir.path()).unwrap();
        assert!(!idx.has_unpersisted_changes());
        idx.lexical_mut()
            .upsert(uuid::Uuid::new_v4(), "ferry timetable")
            .unwrap();
        assert!(idx.has_unpersisted_changes());
        idx.persist().unwrap();
        assert!(!idx.has_unpersisted_changes());
    }

    #[test]
    fn a_merge_started_by_the_last_commit_lands_before_the_handle_goes() {
        const MERGE_AT: usize = 8;
        let dir = tempfile::tempdir().unwrap();
        let mut idx = Indexes::open(dir.path()).unwrap();
        for i in 0..MERGE_AT {
            idx.lexical_mut()
                .upsert(uuid::Uuid::new_v4(), &format!("ferry timetable {i}"))
                .unwrap();
            idx.persist().unwrap();
        }
        idx.wait_for_merges().unwrap();

        let reopened = Indexes::open_readonly(dir.path()).unwrap();
        assert_eq!(reopened.lexical_len(), MERGE_AT as u64);
        assert!(
            reopened.lexical.segment_count() < MERGE_AT,
            "the merge of {MERGE_AT} segments was dropped: {} remain",
            reopened.lexical.segment_count()
        );
    }
}