Skip to main content

second_brain_core/
embedding.rs

1use std::sync::Arc;
2use std::time::Duration;
3
4use anyhow::{Context, Result, bail};
5use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
6use ort::ep::CPU;
7
8use crate::model_cache::{CachedModel, SnapshotLookup, model_cache_dir};
9use crate::session_pool::SessionPool;
10
11const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
12const MODEL: EmbeddingModel = EmbeddingModel::BGESmallENV15;
13
14/// Embedder backed by a fixed pool of independent sessions, for the same
15/// reason as `Reranker`: `fastembed::TextEmbedding::embed` takes `&mut
16/// self`, and ingestion is embed-bound, so a shared single-session mutex
17/// serializes every writer in the process. Each slot keeps the original
18/// lazy-load/idle-unload lifecycle independently.
19pub struct Embedder {
20    pool: Arc<SessionPool<TextEmbedding>>,
21    checkout_timeout: Duration,
22    activations: Activations,
23}
24
25/// What a session does with a run's activations once the run returns.
26#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
27pub enum Activations {
28    /// onnxruntime's arena keeps each run's peak for reuse. Growth past the
29    /// load and first call is not in `resident_bytes`.
30    #[default]
31    Retained,
32    /// Each run frees its activations on return, so a slot keeps only what
33    /// `resident_bytes` measured.
34    Released,
35}
36
37/// Returned when every session in the pool is still busy past the checkout
38/// timeout. Backpressure that surfaces as an error beats a request that
39/// hangs forever.
40#[derive(Debug)]
41pub struct EmbedTimeoutError {
42    pub waited: Duration,
43    pub pool_size: usize,
44}
45
46impl std::fmt::Display for EmbedTimeoutError {
47    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48        write!(
49            f,
50            "timed out after {:?} waiting for an idle embedding session (pool size {})",
51            self.waited, self.pool_size
52        )
53    }
54}
55
56impl std::error::Error for EmbedTimeoutError {}
57
58/// Snapshot of pool utilization for capacity planning.
59#[derive(Debug, Clone, Copy, Default)]
60pub struct EmbedMetricsSnapshot {
61    pub loaded_sessions: usize,
62    pub in_flight: usize,
63    pub total_checkouts: u64,
64    pub wait_p50: Duration,
65    pub wait_max: Duration,
66}
67
68pub fn query_prompt(text: &str) -> String {
69    format!("Represent this sentence for searching relevant passages: {text}")
70}
71
72fn load_model(show_progress: bool, activations: Activations) -> Result<TextEmbedding> {
73    let mut options = InitOptions::new(MODEL)
74        .with_cache_dir(model_cache_dir()?)
75        .with_show_download_progress(show_progress);
76    if activations == Activations::Released {
77        // ort's CPU provider disables the arena unless asked to enable it.
78        options = options.with_execution_providers(vec![CPU::default().build()]);
79    }
80    TextEmbedding::try_new(options).context("loading embedding model")
81}
82
83impl Embedder {
84    pub fn new() -> Result<Self> {
85        Self::with_sessions(1)
86    }
87
88    /// Builds a pool of `n` independent sessions, eager-loaded up front so
89    /// construction fails fast exactly like the single-session constructor
90    /// always has.
91    pub fn with_sessions(n: usize) -> Result<Self> {
92        Self::with_sessions_and_activations(n, Activations::default())
93    }
94
95    pub fn with_sessions_and_activations(n: usize, activations: Activations) -> Result<Self> {
96        if n == 0 {
97            bail!("embedding pool size must be at least 1");
98        }
99        let pool = SessionPool::new("embedding", n, |waited, pool_size| {
100            EmbedTimeoutError { waited, pool_size }.into()
101        });
102        for i in 0..n {
103            pool.install(i, || load_model(true, activations))
104                .with_context(|| format!("initializing embedding session {i} of {n}"))?;
105        }
106
107        Ok(Self {
108            pool: Arc::new(pool),
109            checkout_timeout: DEFAULT_CHECKOUT_TIMEOUT,
110            activations,
111        })
112    }
113
114    /// Overrides the default checkout wait bound. Mainly useful for tests
115    /// that need to observe the timeout path without a long-running test.
116    pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
117        self.checkout_timeout = timeout;
118        self
119    }
120
121    pub fn pool_size(&self) -> usize {
122        self.pool.len()
123    }
124
125    pub fn metrics(&self) -> EmbedMetricsSnapshot {
126        let stats = self.pool.stats();
127        EmbedMetricsSnapshot {
128            loaded_sessions: stats.loaded_sessions,
129            in_flight: stats.in_flight,
130            total_checkouts: stats.total_checkouts,
131            wait_p50: stats.wait_p50,
132            wait_max: stats.wait_max,
133        }
134    }
135
136    pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
137        let activations = self.activations;
138        self.pool.checkout(
139            self.checkout_timeout,
140            move || load_model(false, activations),
141            |session| embed_one(session, text),
142        )
143    }
144
145    // BGE retrieval is asymmetric: the instruction prefix goes on the query only,
146    // documents stay bare, so embed() is left unchanged.
147    pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
148        self.embed(&query_prompt(text))
149    }
150
151    /// `embed_query` for a caller with a deadline: it takes only a loaded
152    /// session, waiting at most `wait`, and fails with `EmbedTimeoutError`
153    /// rather than load the model; a session dropped while idle is reloaded
154    /// in the background for later calls.
155    pub fn embed_query_within(&self, text: &str, wait: Duration) -> Result<Vec<f32>> {
156        let prompt = query_prompt(text);
157        let activations = self.activations;
158        self.pool.checkout_warm(
159            wait.min(self.checkout_timeout),
160            move || load_model(false, activations),
161            |session| embed_one(session, &prompt),
162        )
163    }
164
165    pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
166        let activations = self.activations;
167        self.pool.checkout(
168            self.checkout_timeout,
169            move || load_model(false, activations),
170            |session| {
171                session
172                    .embed(texts, None)
173                    .context("generating batch embeddings")
174            },
175        )
176    }
177
178    pub fn drop_if_idle(&self, idle_secs: u64) {
179        self.pool.drop_if_idle(idle_secs);
180    }
181
182    /// Allocator growth across every slot's load and first use; a dropped slot adds 0.
183    pub fn resident_bytes(&self) -> u64 {
184        self.pool.slot_resident_bytes().iter().sum()
185    }
186
187    /// Allocator growth across each slot's load and first use, 0 for a dropped slot.
188    pub fn slot_resident_bytes(&self) -> Vec<u64> {
189        self.pool.slot_resident_bytes()
190    }
191
192    pub fn dimension(&self) -> usize {
193        384
194    }
195}
196
197/// The embedding model's Hugging Face repo and the files it loads from a
198/// snapshot.
199pub fn cached_model() -> Result<CachedModel> {
200    let info = TextEmbedding::get_model_info(&MODEL)?;
201    Ok(CachedModel::new(
202        &info.model_code,
203        &info.model_file,
204        &info.additional_files,
205        SnapshotLookup::MainRef,
206    ))
207}
208
209fn embed_one(session: &mut TextEmbedding, text: &str) -> Result<Vec<f32>> {
210    session
211        .embed(vec![text], None)
212        .context("generating embedding")?
213        .into_iter()
214        .next()
215        .ok_or_else(|| anyhow::anyhow!("no embedding returned"))
216}
217
218/// Embeds text for storage as a document: bare, never through
219/// `embed_query`, because BGE retrieval is asymmetric and the instruction
220/// prefix belongs on the query side only. A host that embeds somewhere other
221/// than an in-process `Embedder` implements this to feed ingestion.
222pub trait DocumentEmbedder: Send + Sync {
223    fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
224}
225
226impl DocumentEmbedder for Embedder {
227    fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
228        self.embed(text)
229    }
230}
231
232#[cfg(test)]
233mod tests {
234    use super::*;
235    use crate::model_cache::TOKENIZER_FILES;
236
237    #[test]
238    fn cached_model_is_fastembeds_entry_for_the_loaded_model() {
239        let info = TextEmbedding::get_model_info(&EmbeddingModel::BGESmallENV15).unwrap();
240
241        let model = cached_model().unwrap();
242
243        assert_eq!(model.code, info.model_code);
244        assert!(model.files.contains(&info.model_file), "{model:?}");
245        for file in TOKENIZER_FILES {
246            assert!(model.files.iter().any(|f| f == file), "{model:?}");
247        }
248    }
249}