second-brain-core 0.13.5

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::sync::Arc;
use std::time::Duration;

use anyhow::{Context, Result, bail};
use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
use ort::ep::CPU;

use crate::model_cache::{CachedModel, SnapshotLookup, model_cache_dir};
use crate::session_pool::SessionPool;

const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
const MODEL: EmbeddingModel = EmbeddingModel::BGESmallENV15;

/// Embedder backed by a fixed pool of independent sessions, for the same
/// reason as `Reranker`: `fastembed::TextEmbedding::embed` takes `&mut
/// self`, and ingestion is embed-bound, so a shared single-session mutex
/// serializes every writer in the process. Each slot keeps the original
/// lazy-load/idle-unload lifecycle independently.
pub struct Embedder {
    pool: Arc<SessionPool<TextEmbedding>>,
    checkout_timeout: Duration,
    activations: Activations,
}

/// What a session does with a run's activations once the run returns.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum Activations {
    /// onnxruntime's arena keeps each run's peak for reuse. Growth past the
    /// load and first call is not in `resident_bytes`.
    #[default]
    Retained,
    /// Each run frees its activations on return, so a slot keeps only what
    /// `resident_bytes` measured.
    Released,
}

/// Returned when every session in the pool is still busy past the checkout
/// timeout. Backpressure that surfaces as an error beats a request that
/// hangs forever.
#[derive(Debug)]
pub struct EmbedTimeoutError {
    pub waited: Duration,
    pub pool_size: usize,
}

impl std::fmt::Display for EmbedTimeoutError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "timed out after {:?} waiting for an idle embedding session (pool size {})",
            self.waited, self.pool_size
        )
    }
}

impl std::error::Error for EmbedTimeoutError {}

/// Snapshot of pool utilization for capacity planning.
#[derive(Debug, Clone, Copy, Default)]
pub struct EmbedMetricsSnapshot {
    pub loaded_sessions: usize,
    pub in_flight: usize,
    pub total_checkouts: u64,
    pub wait_p50: Duration,
    pub wait_max: Duration,
}

pub fn query_prompt(text: &str) -> String {
    format!("Represent this sentence for searching relevant passages: {text}")
}

fn load_model(show_progress: bool, activations: Activations) -> Result<TextEmbedding> {
    let mut options = InitOptions::new(MODEL)
        .with_cache_dir(model_cache_dir()?)
        .with_show_download_progress(show_progress);
    if activations == Activations::Released {
        // ort's CPU provider disables the arena unless asked to enable it.
        options = options.with_execution_providers(vec![CPU::default().build()]);
    }
    TextEmbedding::try_new(options).context("loading embedding model")
}

impl Embedder {
    pub fn new() -> Result<Self> {
        Self::with_sessions(1)
    }

    /// Builds a pool of `n` independent sessions, eager-loaded up front so
    /// construction fails fast exactly like the single-session constructor
    /// always has.
    pub fn with_sessions(n: usize) -> Result<Self> {
        Self::with_sessions_and_activations(n, Activations::default())
    }

    pub fn with_sessions_and_activations(n: usize, activations: Activations) -> Result<Self> {
        if n == 0 {
            bail!("embedding pool size must be at least 1");
        }
        let pool = SessionPool::new("embedding", n, |waited, pool_size| {
            EmbedTimeoutError { waited, pool_size }.into()
        });
        for i in 0..n {
            pool.install(i, || load_model(true, activations))
                .with_context(|| format!("initializing embedding session {i} of {n}"))?;
        }

        Ok(Self {
            pool: Arc::new(pool),
            checkout_timeout: DEFAULT_CHECKOUT_TIMEOUT,
            activations,
        })
    }

    /// Overrides the default checkout wait bound. Mainly useful for tests
    /// that need to observe the timeout path without a long-running test.
    pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
        self.checkout_timeout = timeout;
        self
    }

    pub fn pool_size(&self) -> usize {
        self.pool.len()
    }

    pub fn metrics(&self) -> EmbedMetricsSnapshot {
        let stats = self.pool.stats();
        EmbedMetricsSnapshot {
            loaded_sessions: stats.loaded_sessions,
            in_flight: stats.in_flight,
            total_checkouts: stats.total_checkouts,
            wait_p50: stats.wait_p50,
            wait_max: stats.wait_max,
        }
    }

    pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
        let activations = self.activations;
        self.pool.checkout(
            self.checkout_timeout,
            move || load_model(false, activations),
            |session| embed_one(session, text),
        )
    }

    // BGE retrieval is asymmetric: the instruction prefix goes on the query only,
    // documents stay bare, so embed() is left unchanged.
    pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
        self.embed(&query_prompt(text))
    }

    /// `embed_query` for a caller with a deadline: it takes only a loaded
    /// session, waiting at most `wait`, and fails with `EmbedTimeoutError`
    /// rather than load the model; a session dropped while idle is reloaded
    /// in the background for later calls.
    pub fn embed_query_within(&self, text: &str, wait: Duration) -> Result<Vec<f32>> {
        let prompt = query_prompt(text);
        let activations = self.activations;
        self.pool.checkout_warm(
            wait.min(self.checkout_timeout),
            move || load_model(false, activations),
            |session| embed_one(session, &prompt),
        )
    }

    pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
        let activations = self.activations;
        self.pool.checkout(
            self.checkout_timeout,
            move || load_model(false, activations),
            |session| {
                session
                    .embed(texts, None)
                    .context("generating batch embeddings")
            },
        )
    }

    pub fn drop_if_idle(&self, idle_secs: u64) {
        self.pool.drop_if_idle(idle_secs);
    }

    /// Allocator growth across every slot's load and first use; a dropped slot adds 0.
    pub fn resident_bytes(&self) -> u64 {
        self.pool.slot_resident_bytes().iter().sum()
    }

    /// Allocator growth across each slot's load and first use, 0 for a dropped slot.
    pub fn slot_resident_bytes(&self) -> Vec<u64> {
        self.pool.slot_resident_bytes()
    }

    pub fn dimension(&self) -> usize {
        384
    }
}

/// The embedding model's Hugging Face repo and the files it loads from a
/// snapshot.
pub fn cached_model() -> Result<CachedModel> {
    let info = TextEmbedding::get_model_info(&MODEL)?;
    Ok(CachedModel::new(
        &info.model_code,
        &info.model_file,
        &info.additional_files,
        SnapshotLookup::MainRef,
    ))
}

fn embed_one(session: &mut TextEmbedding, text: &str) -> Result<Vec<f32>> {
    session
        .embed(vec![text], None)
        .context("generating embedding")?
        .into_iter()
        .next()
        .ok_or_else(|| anyhow::anyhow!("no embedding returned"))
}

/// Embeds text for storage as a document: bare, never through
/// `embed_query`, because BGE retrieval is asymmetric and the instruction
/// prefix belongs on the query side only. A host that embeds somewhere other
/// than an in-process `Embedder` implements this to feed ingestion.
pub trait DocumentEmbedder: Send + Sync {
    fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
}

impl DocumentEmbedder for Embedder {
    fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
        self.embed(text)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::model_cache::TOKENIZER_FILES;

    #[test]
    fn cached_model_is_fastembeds_entry_for_the_loaded_model() {
        let info = TextEmbedding::get_model_info(&EmbeddingModel::BGESmallENV15).unwrap();

        let model = cached_model().unwrap();

        assert_eq!(model.code, info.model_code);
        assert!(model.files.contains(&info.model_file), "{model:?}");
        for file in TOKENIZER_FILES {
            assert!(model.files.iter().any(|f| f == file), "{model:?}");
        }
    }
}