second-brain-core 0.11.0

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::collections::VecDeque;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Condvar, Mutex, MutexGuard};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

use anyhow::{Context, Result, bail};
use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};

use crate::model_cache::model_cache_dir;

const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
const WAIT_SAMPLE_CAP: usize = 512;

/// Embedder backed by a fixed pool of independent sessions, for the same
/// reason as `Reranker`: `fastembed::TextEmbedding::embed` takes `&mut
/// self`, and ingestion is embed-bound, so a shared single-session mutex
/// serializes every writer in the process. Each slot keeps the original
/// lazy-load/idle-unload lifecycle independently.
pub struct Embedder {
    pool: SessionPool,
}

/// Returned when every session in the pool is still busy past the checkout
/// timeout. Backpressure that surfaces as an error beats a request that
/// hangs forever.
#[derive(Debug)]
pub struct EmbedTimeoutError {
    pub waited: Duration,
    pub pool_size: usize,
}

impl std::fmt::Display for EmbedTimeoutError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "timed out after {:?} waiting for an idle embedding session (pool size {})",
            self.waited, self.pool_size
        )
    }
}

impl std::error::Error for EmbedTimeoutError {}

/// Snapshot of pool utilization for capacity planning.
#[derive(Debug, Clone, Copy, Default)]
pub struct EmbedMetricsSnapshot {
    pub in_flight: usize,
    pub total_checkouts: u64,
    pub wait_p50: Duration,
    pub wait_max: Duration,
}

pub fn query_prompt(text: &str) -> String {
    format!("Represent this sentence for searching relevant passages: {text}")
}

fn now_secs() -> u64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap_or_default()
        .as_secs()
}

fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
    m.lock().unwrap_or_else(|e| e.into_inner())
}

#[derive(Default)]
struct PoolMetrics {
    in_flight: AtomicUsize,
    total_checkouts: AtomicU64,
    wait_max_ns: AtomicU64,
    // Bounded ring buffer rather than a full histogram crate: cheap enough
    // to sample on every checkout, and a sorted snapshot gives an exact
    // median over the recent window, which is all capacity planning needs.
    wait_samples: Mutex<VecDeque<u64>>,
}

impl PoolMetrics {
    fn record_wait(&self, ns: u64) {
        self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
        let mut samples = lock_or_recover(&self.wait_samples);
        if samples.len() >= WAIT_SAMPLE_CAP {
            samples.pop_front();
        }
        samples.push_back(ns);
    }

    fn p50_ns(&self) -> u64 {
        let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
            .iter()
            .copied()
            .collect();
        if samples.is_empty() {
            return 0;
        }
        samples.sort_unstable();
        samples[samples.len() / 2]
    }
}

/// Fixed-size pool of lazily-loaded, independently idle-unloadable sessions.
/// Checkout pulls an index off a free list guarded by a condvar, so waiters
/// are woken in order instead of spin-polling N mutexes with `try_lock`.
struct SessionPool {
    sessions: Vec<Mutex<Option<TextEmbedding>>>,
    last_used_secs: Vec<AtomicU64>,
    free: Mutex<VecDeque<usize>>,
    free_cv: Condvar,
    checkout_timeout: Duration,
    metrics: PoolMetrics,
}

/// Returns the checked-out slot to the free list on drop, including on a
/// panic unwind, so a failure inside one embed call cannot permanently
/// shrink the pool.
struct Checkout<'a> {
    pool: &'a SessionPool,
    idx: usize,
}

impl Drop for Checkout<'_> {
    fn drop(&mut self) {
        self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
        lock_or_recover(&self.pool.free).push_back(self.idx);
        self.pool.free_cv.notify_one();
    }
}

impl SessionPool {
    fn new(n: usize, checkout_timeout: Duration) -> Self {
        let mut sessions = Vec::with_capacity(n);
        let mut last_used_secs = Vec::with_capacity(n);
        let mut free = VecDeque::with_capacity(n);
        for i in 0..n {
            sessions.push(Mutex::new(None));
            last_used_secs.push(AtomicU64::new(0));
            free.push_back(i);
        }
        Self {
            sessions,
            last_used_secs,
            free: Mutex::new(free),
            free_cv: Condvar::new(),
            checkout_timeout,
            metrics: PoolMetrics::default(),
        }
    }

    fn len(&self) -> usize {
        self.sessions.len()
    }

    fn checkout<R>(
        &self,
        load: impl FnOnce() -> Result<TextEmbedding>,
        with_session: impl FnOnce(&mut TextEmbedding) -> Result<R>,
    ) -> Result<R> {
        let start = Instant::now();
        let guard = lock_or_recover(&self.free);
        let (mut guard, wait_result) = self
            .free_cv
            .wait_timeout_while(guard, self.checkout_timeout, |q| q.is_empty())
            .unwrap_or_else(|e| e.into_inner());
        if wait_result.timed_out() && guard.is_empty() {
            return Err(EmbedTimeoutError {
                waited: start.elapsed(),
                pool_size: self.len(),
            }
            .into());
        }
        let idx = guard.pop_front().expect("free slot present after wait");
        drop(guard);

        self.metrics.record_wait(start.elapsed().as_nanos() as u64);
        self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
        self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
        let _checkout = Checkout { pool: self, idx };

        let mut slot = lock_or_recover(&self.sessions[idx]);
        if slot.is_none() {
            tracing::info!("reloading embedding model after idle drop (slot {idx})");
            *slot = Some(load()?);
        }
        self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
        with_session(slot.as_mut().expect("session loaded above"))
    }

    fn drop_if_idle(&self, idle_secs: u64) {
        let now = now_secs();
        for (idx, last) in self.last_used_secs.iter().enumerate() {
            if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
                continue;
            }
            // try_lock, not lock: a busy slot is by definition not idle, and
            // this must never block behind an in-flight embed.
            if let Ok(mut slot) = self.sessions[idx].try_lock()
                && slot.is_some()
            {
                *slot = None;
                tracing::info!("embedding session {idx} dropped after {idle_secs}s idle");
            }
        }
    }
}

fn load_model(show_progress: bool) -> Result<TextEmbedding> {
    TextEmbedding::try_new(
        InitOptions::new(EmbeddingModel::BGESmallENV15)
            .with_cache_dir(model_cache_dir()?)
            .with_show_download_progress(show_progress),
    )
    .context("loading embedding model")
}

impl Embedder {
    pub fn new() -> Result<Self> {
        Self::with_sessions(1)
    }

    /// Builds a pool of `n` independent sessions, eager-loaded up front so
    /// construction fails fast exactly like the single-session constructor
    /// always has.
    pub fn with_sessions(n: usize) -> Result<Self> {
        if n == 0 {
            bail!("embedding pool size must be at least 1");
        }
        let pool = SessionPool::new(n, DEFAULT_CHECKOUT_TIMEOUT);
        for i in 0..n {
            let mut slot = lock_or_recover(&pool.sessions[i]);
            *slot = Some(
                load_model(true)
                    .with_context(|| format!("initializing embedding session {i} of {n}"))?,
            );
            pool.last_used_secs[i].store(now_secs(), Ordering::Relaxed);
        }

        Ok(Self { pool })
    }

    /// Overrides the default checkout wait bound. Mainly useful for tests
    /// that need to observe the timeout path without a long-running test.
    pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
        self.pool.checkout_timeout = timeout;
        self
    }

    pub fn pool_size(&self) -> usize {
        self.pool.len()
    }

    pub fn metrics(&self) -> EmbedMetricsSnapshot {
        EmbedMetricsSnapshot {
            in_flight: self.pool.metrics.in_flight.load(Ordering::Relaxed),
            total_checkouts: self.pool.metrics.total_checkouts.load(Ordering::Relaxed),
            wait_p50: Duration::from_nanos(self.pool.metrics.p50_ns()),
            wait_max: Duration::from_nanos(self.pool.metrics.wait_max_ns.load(Ordering::Relaxed)),
        }
    }

    pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
        let owned = text.to_string();
        let results = self.pool.checkout(
            || load_model(false),
            move |session| {
                session
                    .embed(vec![owned.as_str()], None)
                    .context("generating embedding")
            },
        )?;
        results
            .into_iter()
            .next()
            .ok_or_else(|| anyhow::anyhow!("no embedding returned"))
    }

    // BGE retrieval is asymmetric: the instruction prefix goes on the query only,
    // documents stay bare, so embed() is left unchanged.
    pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
        self.embed(&query_prompt(text))
    }

    pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
        let owned: Vec<String> = texts.iter().map(|t| t.to_string()).collect();
        self.pool.checkout(
            || load_model(false),
            move |session| {
                let refs: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
                session
                    .embed(refs, None)
                    .context("generating batch embeddings")
            },
        )
    }

    pub fn drop_if_idle(&self, idle_secs: u64) {
        self.pool.drop_if_idle(idle_secs);
    }

    pub fn dimension(&self) -> usize {
        384
    }
}

/// Embeds text for storage as a document: bare, never through
/// `embed_query`, because BGE retrieval is asymmetric and the instruction
/// prefix belongs on the query side only. A host that embeds somewhere other
/// than an in-process `Embedder` implements this to feed ingestion.
pub trait DocumentEmbedder: Send + Sync {
    fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
}

impl DocumentEmbedder for Embedder {
    fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
        self.embed(text)
    }
}