second-brain-core 0.12.0

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::path::Path;

use anyhow::{Context, Result, anyhow};
use tantivy::collector::TopDocs;
use tantivy::directory::MmapDirectory;
use tantivy::query::QueryParser;
use tantivy::schema::{Field, STORED, STRING, Schema, TEXT, Value};
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term, doc};
use uuid::Uuid;

/// `ReadWrite` indexes may construct an `IndexWriter`, which takes a
/// cross-process exclusive lock file. `Reader` indexes never do: they exist
/// so a multi-tenant service can search beside a writing daemon, or beside
/// other readers, without contending for that lock and without risking the
/// zero-byte `.tantivy-writer.lock` a SIGKILLed writer can leave behind.
#[derive(Clone, Copy, PartialEq, Eq)]
enum LexicalMode {
    ReadWrite,
    Reader,
}

pub struct LexicalIndex {
    index: Index,
    // Our own handle on the same directory the index was opened from, kept
    // solely so `evict_dead_mmap_entries` can reach the mmap cache: `Index`
    // hands back a `ManagedDirectory`, which does not expose the concrete
    // `MmapDirectory` underneath. Cloning shares one `Arc`, so this is the
    // same cache the index uses, not a second one.
    directory: MmapDirectory,
    // Created on first mutation, not at open: the tantivy writer holds an
    // exclusive cross-process lock, and read-only consumers (recall, eval)
    // must be able to open the index beside a writing daemon.
    writer: Option<IndexWriter>,
    reader: IndexReader,
    id_field: Field,
    body_field: Field,
    mode: LexicalMode,
}

impl LexicalIndex {
    pub fn open(dir: &Path) -> Result<Self> {
        std::fs::create_dir_all(dir)
            .with_context(|| format!("creating tantivy dir {}", dir.display()))?;
        let mut builder = Schema::builder();
        let id_field = builder.add_text_field("memory_id", STRING | STORED);
        let body_field = builder.add_text_field("body", TEXT);
        let schema = builder.build();
        let directory = MmapDirectory::open(dir)?;
        let index =
            Index::open_or_create(directory.clone(), schema).context("opening tantivy index")?;
        let reader = index.reader().context("creating tantivy reader")?;
        Ok(Self {
            index,
            directory,
            writer: None,
            reader,
            id_field,
            body_field,
            mode: LexicalMode::ReadWrite,
        })
    }

    /// Opens an existing index purely for search: no `IndexWriter` is ever
    /// constructed here, so this never touches the writer lock file and can
    /// sit beside a writing daemon, or other readers, indefinitely. Errors
    /// instead of creating an index, because a reader that can create a
    /// tenant's index is a reader that can corrupt one.
    ///
    /// The reader uses `ReloadPolicy::Manual`: a service holding thousands
    /// of these open cannot afford a background filesystem-watch thread per
    /// tenant, so visibility of writer changes is pulled explicitly via
    /// `reload_reader`, not pushed on a delay.
    pub fn open_reader(dir: &Path) -> Result<Self> {
        let mmap_dir = MmapDirectory::open(dir)
            .with_context(|| format!("opening tantivy directory {}", dir.display()))?;
        let index = Index::open(mmap_dir.clone()).context("opening tantivy index")?;
        let schema = index.schema();
        let id_field = schema
            .get_field("memory_id")
            .context("tantivy schema missing memory_id field")?;
        let body_field = schema
            .get_field("body")
            .context("tantivy schema missing body field")?;
        let reader = index
            .reader_builder()
            .reload_policy(ReloadPolicy::Manual)
            .try_into()
            .context("creating tantivy reader")?;
        Ok(Self {
            index,
            directory: mmap_dir,
            writer: None,
            reader,
            id_field,
            body_field,
            mode: LexicalMode::Reader,
        })
    }

    /// Drops the cache entries tantivy keeps for files it has already unmapped.
    ///
    /// `MmapDirectory` memoizes one `Weak` per file it has ever mmapped, keyed
    /// by path. `MmapDirectory::delete` documents that it evicts a file's entry
    /// before deleting the file, but it does not: it only unlinks. Nothing else
    /// evicts either, because the eviction pass (`MmapCache::remove_weak_ref`)
    /// is reachable only through `get_cache_info`, which tantivy never calls
    /// itself. Every merged-away segment therefore leaves its path, its `Weak`,
    /// and the retained allocation of its dead `Arc` behind for the life of the
    /// process.
    ///
    /// That is unbounded here, because a daemon draining the index outbox
    /// commits every few seconds forever and each commit writes a fresh
    /// segment: measured at roughly 7 KB retained per commit against a
    /// fixed-size corpus, which is what OOM-killed the API server. Sweeping
    /// after every commit and every reader reload bounds the cache to the
    /// segments actually mapped.
    ///
    /// `get_cache_info` is called for that eviction, not for what it returns.
    fn evict_dead_mmap_entries(&self) {
        let _ = self.directory.get_cache_info();
    }

    fn writer_mut(&mut self) -> Result<&mut IndexWriter> {
        if self.mode == LexicalMode::Reader {
            return Err(anyhow!(
                "tantivy index is a read-only reader; mutation is not permitted"
            ));
        }
        if self.writer.is_none() {
            self.writer = Some(
                self.index
                    .writer(50_000_000)
                    .context("creating tantivy writer")?,
            );
        }
        Ok(self.writer.as_mut().unwrap())
    }

    /// Delete-then-add so a replayed outbox row cannot duplicate a document.
    pub fn upsert(&mut self, id: Uuid, body: &str) -> Result<()> {
        let id_field = self.id_field;
        let body_field = self.body_field;
        let id_str = id.to_string();
        let writer = self.writer_mut()?;
        writer.delete_term(Term::from_field_text(id_field, &id_str));
        writer.add_document(doc!(id_field => id_str, body_field => body))?;
        Ok(())
    }

    pub fn remove(&mut self, id: Uuid) -> Result<()> {
        let id_field = self.id_field;
        let id_str = id.to_string();
        self.writer_mut()?
            .delete_term(Term::from_field_text(id_field, &id_str));
        Ok(())
    }

    pub fn commit(&mut self) -> Result<()> {
        if self.mode == LexicalMode::Reader {
            return Err(anyhow!(
                "tantivy index is a read-only reader; commit is not permitted"
            ));
        }
        if let Some(writer) = self.writer.as_mut() {
            writer.commit().context("tantivy commit")?;
            self.reader.reload().context("tantivy reader reload")?;
            self.evict_dead_mmap_entries();
        }
        Ok(())
    }

    pub(crate) fn close_writer(&mut self) -> Result<()> {
        if let Some(writer) = self.writer.take() {
            writer
                .wait_merging_threads()
                .context("waiting for tantivy merges")?;
        }
        Ok(())
    }

    #[cfg(test)]
    pub(crate) fn segment_count(&self) -> usize {
        self.index
            .searchable_segment_ids()
            .map_or(0, |ids| ids.len())
    }

    /// Commits, then merges every segment into one and deletes the files the
    /// merge replaced. A deleted document's terms stay in its segment's files
    /// until that segment is merged away, and the merge policy may never pick
    /// a large old segment, so this is how a removed text leaves the disk.
    pub fn compact(&mut self) -> Result<()> {
        self.commit()?;
        if let Some(writer) = self.writer.take() {
            writer
                .wait_merging_threads()
                .context("waiting for tantivy merges")?;
        }
        let segments = self
            .index
            .searchable_segment_ids()
            .context("listing tantivy segments")?;
        let writer = self.writer_mut()?;
        if !segments.is_empty() {
            writer.merge(&segments).wait().context("tantivy merge")?;
        }
        writer
            .garbage_collect_files()
            .wait()
            .context("tantivy garbage collection")?;
        self.reader.reload().context("tantivy reader reload")?;
        self.evict_dead_mmap_entries();
        Ok(())
    }

    /// Returns (memory id, BM25 score) pairs, best first.
    pub fn search(&self, query_text: &str, k: usize) -> Result<Vec<(Uuid, f32)>> {
        let searcher = self.reader.searcher();
        let parser = QueryParser::for_index(&self.index, vec![self.body_field]);
        // Lenient parse: user queries are arbitrary text, not tantivy syntax,
        // and a stray `(` must degrade to a best-effort query, not an error.
        let (query, _errors) = parser.parse_query_lenient(query_text);
        let top = searcher.search(&query, &TopDocs::with_limit(k.max(1)).order_by_score())?;
        let mut out = Vec::with_capacity(top.len());
        for (score, address) in top {
            let document: TantivyDocument = searcher.doc(address)?;
            let id_str = document
                .get_first(self.id_field)
                .and_then(|v| v.as_str())
                .unwrap_or_default();
            if let Ok(id) = Uuid::parse_str(id_str) {
                out.push((id, score));
            }
        }
        Ok(out)
    }

    pub fn num_docs(&self) -> u64 {
        self.reader.searcher().num_docs()
    }

    /// Decompressed doc-store blocks the current searcher has cached, each
    /// counted at the configured block size, which a block passes by at most
    /// one document. Everything else a reader searches is the mapped segment
    /// files, and the writer's indexing arenas have no public size.
    pub fn heap_bytes(&self) -> u64 {
        let blocks = self.reader.searcher().doc_store_cache_stats().num_entries;
        (blocks * self.index.settings().docstore_blocksize) as u64
    }

    /// Reloads the reader so it observes segments committed since it was
    /// opened. Harmless on a `ReadWrite` instance: `commit` already reloads,
    /// so this just repeats an already-current reload.
    pub fn reload_reader(&mut self) -> Result<()> {
        self.reader.reload().context("tantivy reader reload")?;
        self.evict_dead_mmap_entries();
        Ok(())
    }

    /// Drop every document; used by full rebuild.
    pub fn reset(&mut self) -> Result<()> {
        self.writer_mut()?
            .delete_all_documents()
            .context("tantivy delete_all_documents")?;
        self.commit()
    }
}