second-brain-core 0.13.2

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::path::Path;

use anyhow::{Context, Result, anyhow};
use tantivy::collector::TopDocs;
use tantivy::directory::{Directory, MmapDirectory};
use tantivy::query::QueryParser;
use tantivy::schema::{Field, STORED, STRING, Schema, TEXT, Value};
use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term, doc};
use uuid::Uuid;

use crate::sealed::{SealKeys, SealedDirectory};

/// `ReadWrite` indexes may construct an `IndexWriter`, which takes a
/// cross-process exclusive lock file. `Reader` indexes never do: they exist
/// so a multi-tenant service can search beside a writing daemon, or beside
/// other readers, without contending for that lock and without risking the
/// zero-byte `.tantivy-writer.lock` a SIGKILLed writer can leave behind.
#[derive(Clone, Copy, PartialEq, Eq)]
enum LexicalMode {
    ReadWrite,
    Reader,
}

#[derive(Clone)]
enum LexicalDirectory {
    Mmap(MmapDirectory),
    Sealed(SealedDirectory),
}

impl LexicalDirectory {
    fn boxed(&self) -> Box<dyn Directory> {
        match self {
            Self::Mmap(directory) => Box::new(directory.clone()),
            Self::Sealed(directory) => Box::new(directory.clone()),
        }
    }

    fn sealed(&self) -> Result<&SealedDirectory> {
        match self {
            Self::Sealed(directory) => Ok(directory),
            Self::Mmap(_) => Err(anyhow!(
                "tantivy index was opened without keys; reopen it keyed to seal it"
            )),
        }
    }
}

fn schema() -> (Schema, Field, Field) {
    let mut builder = Schema::builder();
    let id_field = builder.add_text_field("memory_id", STRING | STORED);
    let body_field = builder.add_text_field("body", TEXT);
    (builder.build(), id_field, body_field)
}

pub struct LexicalIndex {
    index: Index,
    // Our own handle on the same directory the index was opened from, kept
    // so `evict_dead_mmap_entries` can reach the mmap cache and a sealed
    // directory its keys: `Index` hands back a `ManagedDirectory`, which
    // does not expose the concrete directory underneath. Cloning shares one
    // `Arc`, so this is the same cache and keys the index uses.
    directory: LexicalDirectory,
    // Created on first mutation, not at open: the tantivy writer holds an
    // exclusive cross-process lock, and read-only consumers (recall, eval)
    // must be able to open the index beside a writing daemon.
    writer: Option<IndexWriter>,
    reader: IndexReader,
    id_field: Field,
    body_field: Field,
    mode: LexicalMode,
}

impl LexicalIndex {
    pub fn open(dir: &Path) -> Result<Self> {
        std::fs::create_dir_all(dir)
            .with_context(|| format!("creating tantivy dir {}", dir.display()))?;
        Self::open_or_create(LexicalDirectory::Mmap(MmapDirectory::open(dir)?))
    }

    /// Same as `open`, with every file written sealed under `keys`. Files
    /// written before sealing open as they are until `reseal`.
    pub fn open_keyed(dir: &Path, keys: &SealKeys) -> Result<Self> {
        std::fs::create_dir_all(dir)
            .with_context(|| format!("creating tantivy dir {}", dir.display()))?;
        let directory = SealedDirectory::open(dir, keys.clone())?;
        directory.check_keys_open()?;
        directory.remove_leftover_tmp_files()?;
        Self::open_or_create(LexicalDirectory::Sealed(directory))
    }

    fn open_or_create(directory: LexicalDirectory) -> Result<Self> {
        let (schema, id_field, body_field) = schema();
        let index =
            Index::open_or_create(directory.boxed(), schema).context("opening tantivy index")?;
        let reader = index.reader().context("creating tantivy reader")?;
        Ok(Self {
            index,
            directory,
            writer: None,
            reader,
            id_field,
            body_field,
            mode: LexicalMode::ReadWrite,
        })
    }

    /// Opens an existing index purely for search: no `IndexWriter` is ever
    /// constructed here, so this never touches the writer lock file and can
    /// sit beside a writing daemon, or other readers, indefinitely. Errors
    /// instead of creating an index, because a reader that can create a
    /// tenant's index is a reader that can corrupt one.
    ///
    /// The reader uses `ReloadPolicy::Manual`: a service holding thousands
    /// of these open cannot afford a background filesystem-watch thread per
    /// tenant, so visibility of writer changes is pulled explicitly via
    /// `reload_reader`, not pushed on a delay.
    pub fn open_reader(dir: &Path) -> Result<Self> {
        let mmap_dir = MmapDirectory::open(dir)
            .with_context(|| format!("opening tantivy directory {}", dir.display()))?;
        Self::open_existing_reader(LexicalDirectory::Mmap(mmap_dir))
    }

    /// Same as `open_reader`, over a directory sealed under `keys`.
    pub fn open_reader_keyed(dir: &Path, keys: &SealKeys) -> Result<Self> {
        let directory = SealedDirectory::open(dir, keys.clone())?;
        directory.check_keys_open()?;
        Self::open_existing_reader(LexicalDirectory::Sealed(directory))
    }

    fn open_existing_reader(directory: LexicalDirectory) -> Result<Self> {
        let index = Index::open(directory.boxed()).context("opening tantivy index")?;
        let schema = index.schema();
        let id_field = schema
            .get_field("memory_id")
            .context("tantivy schema missing memory_id field")?;
        let body_field = schema
            .get_field("body")
            .context("tantivy schema missing body field")?;
        Self::reader_over(index, directory, id_field, body_field)
    }

    fn reader_over(
        index: Index,
        directory: LexicalDirectory,
        id_field: Field,
        body_field: Field,
    ) -> Result<Self> {
        let reader = index
            .reader_builder()
            .reload_policy(ReloadPolicy::Manual)
            .try_into()
            .context("creating tantivy reader")?;
        Ok(Self {
            index,
            directory,
            writer: None,
            reader,
            id_field,
            body_field,
            mode: LexicalMode::Reader,
        })
    }

    /// A reader over this same index, with a `ReloadPolicy::Manual` reader
    /// of its own: it sees what this handle has committed once it reloads.
    pub fn shared_reader(&self) -> Result<Self> {
        Self::reader_over(
            self.index.clone(),
            self.directory.clone(),
            self.id_field,
            self.body_field,
        )
    }

    /// Files written from now on are sealed under `keys`' current key.
    /// Errors when the index was opened without keys.
    pub fn set_seal_keys(&self, keys: SealKeys) -> Result<()> {
        self.directory.sealed()?.set_keys(keys);
        Ok(())
    }

    /// Closes the writer once its merges finish, then seals every file not
    /// sealed under the current key; returns how many were rewritten.
    pub fn reseal(&mut self) -> Result<usize> {
        if self.mode == LexicalMode::Reader {
            return Err(anyhow!(
                "tantivy index is a read-only reader; reseal is not permitted"
            ));
        }
        let directory = self.directory.sealed()?.clone();
        self.close_writer()?;
        directory.reseal_all()
    }

    /// Drops the cache entries tantivy keeps for files it has already unmapped.
    ///
    /// `MmapDirectory` memoizes one `Weak` per file it has ever mmapped, keyed
    /// by path. `MmapDirectory::delete` documents that it evicts a file's entry
    /// before deleting the file, but it does not: it only unlinks. Nothing else
    /// evicts either, because the eviction pass (`MmapCache::remove_weak_ref`)
    /// is reachable only through `get_cache_info`, which tantivy never calls
    /// itself. Every merged-away segment therefore leaves its path, its `Weak`,
    /// and the retained allocation of its dead `Arc` behind for the life of the
    /// process.
    ///
    /// That is unbounded here, because a daemon draining the index outbox
    /// commits every few seconds forever and each commit writes a fresh
    /// segment: measured at roughly 7 KB retained per commit against a
    /// fixed-size corpus, which is what OOM-killed the API server. Sweeping
    /// after every commit and every reader reload bounds the cache to the
    /// segments actually mapped.
    ///
    /// `get_cache_info` is called for that eviction, not for what it returns.
    fn evict_dead_mmap_entries(&self) {
        if let LexicalDirectory::Mmap(directory) = &self.directory {
            let _ = directory.get_cache_info();
        }
    }

    fn writer_mut(&mut self) -> Result<&mut IndexWriter> {
        if self.mode == LexicalMode::Reader {
            return Err(anyhow!(
                "tantivy index is a read-only reader; mutation is not permitted"
            ));
        }
        if self.writer.is_none() {
            self.writer = Some(
                self.index
                    .writer(50_000_000)
                    .context("creating tantivy writer")?,
            );
        }
        Ok(self.writer.as_mut().unwrap())
    }

    /// Delete-then-add so a replayed outbox row cannot duplicate a document.
    pub fn upsert(&mut self, id: Uuid, body: &str) -> Result<()> {
        let id_field = self.id_field;
        let body_field = self.body_field;
        let id_str = id.to_string();
        let writer = self.writer_mut()?;
        writer.delete_term(Term::from_field_text(id_field, &id_str));
        writer.add_document(doc!(id_field => id_str, body_field => body))?;
        Ok(())
    }

    pub fn remove(&mut self, id: Uuid) -> Result<()> {
        let id_field = self.id_field;
        let id_str = id.to_string();
        self.writer_mut()?
            .delete_term(Term::from_field_text(id_field, &id_str));
        Ok(())
    }

    pub fn commit(&mut self) -> Result<()> {
        if self.mode == LexicalMode::Reader {
            return Err(anyhow!(
                "tantivy index is a read-only reader; commit is not permitted"
            ));
        }
        if let Some(writer) = self.writer.as_mut() {
            writer.commit().context("tantivy commit")?;
            self.reader.reload().context("tantivy reader reload")?;
            self.evict_dead_mmap_entries();
        }
        Ok(())
    }

    /// Drops every change since the last commit.
    pub(crate) fn rollback(&mut self) -> Result<()> {
        if let Some(writer) = self.writer.as_mut() {
            writer.rollback().context("tantivy rollback")?;
        }
        Ok(())
    }

    pub(crate) fn close_writer(&mut self) -> Result<()> {
        if let Some(writer) = self.writer.take() {
            writer
                .wait_merging_threads()
                .context("waiting for tantivy merges")?;
        }
        Ok(())
    }

    #[cfg(test)]
    pub(crate) fn segment_count(&self) -> usize {
        self.index
            .searchable_segment_ids()
            .map_or(0, |ids| ids.len())
    }

    /// Commits, then merges every segment into one and deletes the files the
    /// merge replaced. A deleted document's terms stay in its segment's files
    /// until that segment is merged away, and the merge policy may never pick
    /// a large old segment, so this is how a removed text leaves the disk.
    pub fn compact(&mut self) -> Result<()> {
        self.commit()?;
        if let Some(writer) = self.writer.take() {
            writer
                .wait_merging_threads()
                .context("waiting for tantivy merges")?;
        }
        let segments = self
            .index
            .searchable_segment_ids()
            .context("listing tantivy segments")?;
        let writer = self.writer_mut()?;
        if !segments.is_empty() {
            writer.merge(&segments).wait().context("tantivy merge")?;
        }
        writer
            .garbage_collect_files()
            .wait()
            .context("tantivy garbage collection")?;
        self.reader.reload().context("tantivy reader reload")?;
        self.evict_dead_mmap_entries();
        Ok(())
    }

    /// Returns (memory id, BM25 score) pairs, best first.
    pub fn search(&self, query_text: &str, k: usize) -> Result<Vec<(Uuid, f32)>> {
        let searcher = self.reader.searcher();
        let parser = QueryParser::for_index(&self.index, vec![self.body_field]);
        // Lenient parse: user queries are arbitrary text, not tantivy syntax,
        // and a stray `(` must degrade to a best-effort query, not an error.
        let (query, _errors) = parser.parse_query_lenient(query_text);
        let top = searcher.search(&query, &TopDocs::with_limit(k.max(1)).order_by_score())?;
        let mut out = Vec::with_capacity(top.len());
        for (score, address) in top {
            let document: TantivyDocument = searcher.doc(address)?;
            let id_str = document
                .get_first(self.id_field)
                .and_then(|v| v.as_str())
                .unwrap_or_default();
            if let Ok(id) = Uuid::parse_str(id_str) {
                out.push((id, score));
            }
        }
        Ok(out)
    }

    pub fn num_docs(&self) -> u64 {
        self.reader.searcher().num_docs()
    }

    /// Decompressed doc-store blocks the current searcher has cached, each
    /// counted at the configured block size, which a block passes by at most
    /// one document. Everything else a reader searches is the mapped segment
    /// files, and the writer's indexing arenas have no public size.
    pub fn heap_bytes(&self) -> u64 {
        let blocks = self.reader.searcher().doc_store_cache_stats().num_entries;
        (blocks * self.index.settings().docstore_blocksize) as u64
    }

    /// Reloads the reader so it observes segments committed since it was
    /// opened. Harmless on a `ReadWrite` instance: `commit` already reloads,
    /// so this just repeats an already-current reload.
    pub fn reload_reader(&mut self) -> Result<()> {
        self.reader.reload().context("tantivy reader reload")?;
        self.evict_dead_mmap_entries();
        Ok(())
    }

    /// Drops every document at the next commit, so readers keep the old
    /// documents while a rebuild adds the new ones.
    pub fn reset(&mut self) -> Result<()> {
        self.writer_mut()?
            .delete_all_documents()
            .context("tantivy delete_all_documents")?;
        Ok(())
    }
}