Skip to main content

second_brain_core/
embedding.rs

1use std::collections::VecDeque;
2use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
3use std::sync::{Condvar, Mutex, MutexGuard};
4use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
5
6use anyhow::{Context, Result, bail};
7use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
8
9use crate::model_cache::model_cache_dir;
10
11const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
12const WAIT_SAMPLE_CAP: usize = 512;
13
14/// Embedder backed by a fixed pool of independent sessions, for the same
15/// reason as `Reranker`: `fastembed::TextEmbedding::embed` takes `&mut
16/// self`, and ingestion is embed-bound, so a shared single-session mutex
17/// serializes every writer in the process. Each slot keeps the original
18/// lazy-load/idle-unload lifecycle independently.
19pub struct Embedder {
20    pool: SessionPool,
21}
22
23/// Returned when every session in the pool is still busy past the checkout
24/// timeout. Backpressure that surfaces as an error beats a request that
25/// hangs forever.
26#[derive(Debug)]
27pub struct EmbedTimeoutError {
28    pub waited: Duration,
29    pub pool_size: usize,
30}
31
32impl std::fmt::Display for EmbedTimeoutError {
33    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
34        write!(
35            f,
36            "timed out after {:?} waiting for an idle embedding session (pool size {})",
37            self.waited, self.pool_size
38        )
39    }
40}
41
42impl std::error::Error for EmbedTimeoutError {}
43
44/// Snapshot of pool utilization for capacity planning.
45#[derive(Debug, Clone, Copy, Default)]
46pub struct EmbedMetricsSnapshot {
47    pub in_flight: usize,
48    pub total_checkouts: u64,
49    pub wait_p50: Duration,
50    pub wait_max: Duration,
51}
52
53pub fn query_prompt(text: &str) -> String {
54    format!("Represent this sentence for searching relevant passages: {text}")
55}
56
57fn now_secs() -> u64 {
58    SystemTime::now()
59        .duration_since(UNIX_EPOCH)
60        .unwrap_or_default()
61        .as_secs()
62}
63
64fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
65    m.lock().unwrap_or_else(|e| e.into_inner())
66}
67
68#[derive(Default)]
69struct PoolMetrics {
70    in_flight: AtomicUsize,
71    total_checkouts: AtomicU64,
72    wait_max_ns: AtomicU64,
73    // Bounded ring buffer rather than a full histogram crate: cheap enough
74    // to sample on every checkout, and a sorted snapshot gives an exact
75    // median over the recent window, which is all capacity planning needs.
76    wait_samples: Mutex<VecDeque<u64>>,
77}
78
79impl PoolMetrics {
80    fn record_wait(&self, ns: u64) {
81        self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
82        let mut samples = lock_or_recover(&self.wait_samples);
83        if samples.len() >= WAIT_SAMPLE_CAP {
84            samples.pop_front();
85        }
86        samples.push_back(ns);
87    }
88
89    fn p50_ns(&self) -> u64 {
90        let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
91            .iter()
92            .copied()
93            .collect();
94        if samples.is_empty() {
95            return 0;
96        }
97        samples.sort_unstable();
98        samples[samples.len() / 2]
99    }
100}
101
102/// Fixed-size pool of lazily-loaded, independently idle-unloadable sessions.
103/// Checkout pulls an index off a free list guarded by a condvar, so waiters
104/// are woken in order instead of spin-polling N mutexes with `try_lock`.
105struct SessionPool {
106    sessions: Vec<Mutex<Option<TextEmbedding>>>,
107    last_used_secs: Vec<AtomicU64>,
108    free: Mutex<VecDeque<usize>>,
109    free_cv: Condvar,
110    checkout_timeout: Duration,
111    metrics: PoolMetrics,
112}
113
114/// Returns the checked-out slot to the free list on drop, including on a
115/// panic unwind, so a failure inside one embed call cannot permanently
116/// shrink the pool.
117struct Checkout<'a> {
118    pool: &'a SessionPool,
119    idx: usize,
120}
121
122impl Drop for Checkout<'_> {
123    fn drop(&mut self) {
124        self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
125        lock_or_recover(&self.pool.free).push_back(self.idx);
126        self.pool.free_cv.notify_one();
127    }
128}
129
130impl SessionPool {
131    fn new(n: usize, checkout_timeout: Duration) -> Self {
132        let mut sessions = Vec::with_capacity(n);
133        let mut last_used_secs = Vec::with_capacity(n);
134        let mut free = VecDeque::with_capacity(n);
135        for i in 0..n {
136            sessions.push(Mutex::new(None));
137            last_used_secs.push(AtomicU64::new(0));
138            free.push_back(i);
139        }
140        Self {
141            sessions,
142            last_used_secs,
143            free: Mutex::new(free),
144            free_cv: Condvar::new(),
145            checkout_timeout,
146            metrics: PoolMetrics::default(),
147        }
148    }
149
150    fn len(&self) -> usize {
151        self.sessions.len()
152    }
153
154    fn checkout<R>(
155        &self,
156        load: impl FnOnce() -> Result<TextEmbedding>,
157        with_session: impl FnOnce(&mut TextEmbedding) -> Result<R>,
158    ) -> Result<R> {
159        let start = Instant::now();
160        let guard = lock_or_recover(&self.free);
161        let (mut guard, wait_result) = self
162            .free_cv
163            .wait_timeout_while(guard, self.checkout_timeout, |q| q.is_empty())
164            .unwrap_or_else(|e| e.into_inner());
165        if wait_result.timed_out() && guard.is_empty() {
166            return Err(EmbedTimeoutError {
167                waited: start.elapsed(),
168                pool_size: self.len(),
169            }
170            .into());
171        }
172        let idx = guard.pop_front().expect("free slot present after wait");
173        drop(guard);
174
175        self.metrics.record_wait(start.elapsed().as_nanos() as u64);
176        self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
177        self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
178        let _checkout = Checkout { pool: self, idx };
179
180        let mut slot = lock_or_recover(&self.sessions[idx]);
181        if slot.is_none() {
182            tracing::info!("reloading embedding model after idle drop (slot {idx})");
183            *slot = Some(load()?);
184        }
185        self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
186        with_session(slot.as_mut().expect("session loaded above"))
187    }
188
189    fn drop_if_idle(&self, idle_secs: u64) {
190        let now = now_secs();
191        for (idx, last) in self.last_used_secs.iter().enumerate() {
192            if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
193                continue;
194            }
195            // try_lock, not lock: a busy slot is by definition not idle, and
196            // this must never block behind an in-flight embed.
197            if let Ok(mut slot) = self.sessions[idx].try_lock()
198                && slot.is_some()
199            {
200                *slot = None;
201                tracing::info!("embedding session {idx} dropped after {idle_secs}s idle");
202            }
203        }
204    }
205}
206
207fn load_model(show_progress: bool) -> Result<TextEmbedding> {
208    TextEmbedding::try_new(
209        InitOptions::new(EmbeddingModel::BGESmallENV15)
210            .with_cache_dir(model_cache_dir()?)
211            .with_show_download_progress(show_progress),
212    )
213    .context("loading embedding model")
214}
215
216impl Embedder {
217    pub fn new() -> Result<Self> {
218        Self::with_sessions(1)
219    }
220
221    /// Builds a pool of `n` independent sessions, eager-loaded up front so
222    /// construction fails fast exactly like the single-session constructor
223    /// always has.
224    pub fn with_sessions(n: usize) -> Result<Self> {
225        if n == 0 {
226            bail!("embedding pool size must be at least 1");
227        }
228        let pool = SessionPool::new(n, DEFAULT_CHECKOUT_TIMEOUT);
229        for i in 0..n {
230            let mut slot = lock_or_recover(&pool.sessions[i]);
231            *slot = Some(
232                load_model(true)
233                    .with_context(|| format!("initializing embedding session {i} of {n}"))?,
234            );
235            pool.last_used_secs[i].store(now_secs(), Ordering::Relaxed);
236        }
237
238        Ok(Self { pool })
239    }
240
241    /// Overrides the default checkout wait bound. Mainly useful for tests
242    /// that need to observe the timeout path without a long-running test.
243    pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
244        self.pool.checkout_timeout = timeout;
245        self
246    }
247
248    pub fn pool_size(&self) -> usize {
249        self.pool.len()
250    }
251
252    pub fn metrics(&self) -> EmbedMetricsSnapshot {
253        EmbedMetricsSnapshot {
254            in_flight: self.pool.metrics.in_flight.load(Ordering::Relaxed),
255            total_checkouts: self.pool.metrics.total_checkouts.load(Ordering::Relaxed),
256            wait_p50: Duration::from_nanos(self.pool.metrics.p50_ns()),
257            wait_max: Duration::from_nanos(self.pool.metrics.wait_max_ns.load(Ordering::Relaxed)),
258        }
259    }
260
261    pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
262        let owned = text.to_string();
263        let results = self.pool.checkout(
264            || load_model(false),
265            move |session| {
266                session
267                    .embed(vec![owned.as_str()], None)
268                    .context("generating embedding")
269            },
270        )?;
271        results
272            .into_iter()
273            .next()
274            .ok_or_else(|| anyhow::anyhow!("no embedding returned"))
275    }
276
277    // BGE retrieval is asymmetric: the instruction prefix goes on the query only,
278    // documents stay bare, so embed() is left unchanged.
279    pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
280        self.embed(&query_prompt(text))
281    }
282
283    pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
284        let owned: Vec<String> = texts.iter().map(|t| t.to_string()).collect();
285        self.pool.checkout(
286            || load_model(false),
287            move |session| {
288                let refs: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
289                session
290                    .embed(refs, None)
291                    .context("generating batch embeddings")
292            },
293        )
294    }
295
296    pub fn drop_if_idle(&self, idle_secs: u64) {
297        self.pool.drop_if_idle(idle_secs);
298    }
299
300    pub fn dimension(&self) -> usize {
301        384
302    }
303}
304
305/// Embeds text for storage as a document: bare, never through
306/// `embed_query`, because BGE retrieval is asymmetric and the instruction
307/// prefix belongs on the query side only. A host that embeds somewhere other
308/// than an in-process `Embedder` implements this to feed ingestion.
309pub trait DocumentEmbedder: Send + Sync {
310    fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
311}
312
313impl DocumentEmbedder for Embedder {
314    fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
315        self.embed(text)
316    }
317}