Skip to main content

second_brain_core/
embedding.rs

1use std::collections::VecDeque;
2use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
3use std::sync::{Condvar, Mutex, MutexGuard};
4use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
5
6use anyhow::{Context, Result, bail};
7use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
8
9const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
10const WAIT_SAMPLE_CAP: usize = 512;
11
12/// Embedder backed by a fixed pool of independent sessions, for the same
13/// reason as `Reranker`: `fastembed::TextEmbedding::embed` takes `&mut
14/// self`, and ingestion is embed-bound, so a shared single-session mutex
15/// serializes every writer in the process. Each slot keeps the original
16/// lazy-load/idle-unload lifecycle independently.
17pub struct Embedder {
18    pool: SessionPool,
19}
20
21/// Returned when every session in the pool is still busy past the checkout
22/// timeout. Backpressure that surfaces as an error beats a request that
23/// hangs forever.
24#[derive(Debug)]
25pub struct EmbedTimeoutError {
26    pub waited: Duration,
27    pub pool_size: usize,
28}
29
30impl std::fmt::Display for EmbedTimeoutError {
31    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
32        write!(
33            f,
34            "timed out after {:?} waiting for an idle embedding session (pool size {})",
35            self.waited, self.pool_size
36        )
37    }
38}
39
40impl std::error::Error for EmbedTimeoutError {}
41
42/// Snapshot of pool utilization for capacity planning.
43#[derive(Debug, Clone, Copy, Default)]
44pub struct EmbedMetricsSnapshot {
45    pub in_flight: usize,
46    pub total_checkouts: u64,
47    pub wait_p50: Duration,
48    pub wait_max: Duration,
49}
50
51pub fn query_prompt(text: &str) -> String {
52    format!("Represent this sentence for searching relevant passages: {text}")
53}
54
55fn now_secs() -> u64 {
56    SystemTime::now()
57        .duration_since(UNIX_EPOCH)
58        .unwrap_or_default()
59        .as_secs()
60}
61
62fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
63    m.lock().unwrap_or_else(|e| e.into_inner())
64}
65
66#[derive(Default)]
67struct PoolMetrics {
68    in_flight: AtomicUsize,
69    total_checkouts: AtomicU64,
70    wait_max_ns: AtomicU64,
71    // Bounded ring buffer rather than a full histogram crate: cheap enough
72    // to sample on every checkout, and a sorted snapshot gives an exact
73    // median over the recent window, which is all capacity planning needs.
74    wait_samples: Mutex<VecDeque<u64>>,
75}
76
77impl PoolMetrics {
78    fn record_wait(&self, ns: u64) {
79        self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
80        let mut samples = lock_or_recover(&self.wait_samples);
81        if samples.len() >= WAIT_SAMPLE_CAP {
82            samples.pop_front();
83        }
84        samples.push_back(ns);
85    }
86
87    fn p50_ns(&self) -> u64 {
88        let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
89            .iter()
90            .copied()
91            .collect();
92        if samples.is_empty() {
93            return 0;
94        }
95        samples.sort_unstable();
96        samples[samples.len() / 2]
97    }
98}
99
100/// Fixed-size pool of lazily-loaded, independently idle-unloadable sessions.
101/// Checkout pulls an index off a free list guarded by a condvar, so waiters
102/// are woken in order instead of spin-polling N mutexes with `try_lock`.
103struct SessionPool {
104    sessions: Vec<Mutex<Option<TextEmbedding>>>,
105    last_used_secs: Vec<AtomicU64>,
106    free: Mutex<VecDeque<usize>>,
107    free_cv: Condvar,
108    checkout_timeout: Duration,
109    metrics: PoolMetrics,
110}
111
112/// Returns the checked-out slot to the free list on drop, including on a
113/// panic unwind, so a failure inside one embed call cannot permanently
114/// shrink the pool.
115struct Checkout<'a> {
116    pool: &'a SessionPool,
117    idx: usize,
118}
119
120impl Drop for Checkout<'_> {
121    fn drop(&mut self) {
122        self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
123        lock_or_recover(&self.pool.free).push_back(self.idx);
124        self.pool.free_cv.notify_one();
125    }
126}
127
128impl SessionPool {
129    fn new(n: usize, checkout_timeout: Duration) -> Self {
130        let mut sessions = Vec::with_capacity(n);
131        let mut last_used_secs = Vec::with_capacity(n);
132        let mut free = VecDeque::with_capacity(n);
133        for i in 0..n {
134            sessions.push(Mutex::new(None));
135            last_used_secs.push(AtomicU64::new(0));
136            free.push_back(i);
137        }
138        Self {
139            sessions,
140            last_used_secs,
141            free: Mutex::new(free),
142            free_cv: Condvar::new(),
143            checkout_timeout,
144            metrics: PoolMetrics::default(),
145        }
146    }
147
148    fn len(&self) -> usize {
149        self.sessions.len()
150    }
151
152    fn checkout<R>(
153        &self,
154        load: impl FnOnce() -> Result<TextEmbedding>,
155        with_session: impl FnOnce(&mut TextEmbedding) -> Result<R>,
156    ) -> Result<R> {
157        let start = Instant::now();
158        let guard = lock_or_recover(&self.free);
159        let (mut guard, wait_result) = self
160            .free_cv
161            .wait_timeout_while(guard, self.checkout_timeout, |q| q.is_empty())
162            .unwrap_or_else(|e| e.into_inner());
163        if wait_result.timed_out() && guard.is_empty() {
164            return Err(EmbedTimeoutError {
165                waited: start.elapsed(),
166                pool_size: self.len(),
167            }
168            .into());
169        }
170        let idx = guard.pop_front().expect("free slot present after wait");
171        drop(guard);
172
173        self.metrics.record_wait(start.elapsed().as_nanos() as u64);
174        self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
175        self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
176        let _checkout = Checkout { pool: self, idx };
177
178        let mut slot = lock_or_recover(&self.sessions[idx]);
179        if slot.is_none() {
180            tracing::info!("reloading embedding model after idle drop (slot {idx})");
181            *slot = Some(load()?);
182        }
183        self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
184        with_session(slot.as_mut().expect("session loaded above"))
185    }
186
187    fn drop_if_idle(&self, idle_secs: u64) {
188        let now = now_secs();
189        for (idx, last) in self.last_used_secs.iter().enumerate() {
190            if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
191                continue;
192            }
193            // try_lock, not lock: a busy slot is by definition not idle, and
194            // this must never block behind an in-flight embed.
195            if let Ok(mut slot) = self.sessions[idx].try_lock()
196                && slot.is_some()
197            {
198                *slot = None;
199                tracing::info!("embedding session {idx} dropped after {idle_secs}s idle");
200            }
201        }
202    }
203}
204
205fn load_model(show_progress: bool) -> Result<TextEmbedding> {
206    TextEmbedding::try_new(
207        InitOptions::new(EmbeddingModel::BGESmallENV15).with_show_download_progress(show_progress),
208    )
209    .context("loading embedding model")
210}
211
212impl Embedder {
213    pub fn new() -> Result<Self> {
214        Self::with_sessions(1)
215    }
216
217    /// Builds a pool of `n` independent sessions, eager-loaded up front so
218    /// construction fails fast exactly like the single-session constructor
219    /// always has.
220    pub fn with_sessions(n: usize) -> Result<Self> {
221        if n == 0 {
222            bail!("embedding pool size must be at least 1");
223        }
224        let pool = SessionPool::new(n, DEFAULT_CHECKOUT_TIMEOUT);
225        for i in 0..n {
226            let mut slot = lock_or_recover(&pool.sessions[i]);
227            *slot = Some(
228                load_model(true)
229                    .with_context(|| format!("initializing embedding session {i} of {n}"))?,
230            );
231            pool.last_used_secs[i].store(now_secs(), Ordering::Relaxed);
232        }
233
234        Ok(Self { pool })
235    }
236
237    /// Overrides the default checkout wait bound. Mainly useful for tests
238    /// that need to observe the timeout path without a long-running test.
239    pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
240        self.pool.checkout_timeout = timeout;
241        self
242    }
243
244    pub fn pool_size(&self) -> usize {
245        self.pool.len()
246    }
247
248    pub fn metrics(&self) -> EmbedMetricsSnapshot {
249        EmbedMetricsSnapshot {
250            in_flight: self.pool.metrics.in_flight.load(Ordering::Relaxed),
251            total_checkouts: self.pool.metrics.total_checkouts.load(Ordering::Relaxed),
252            wait_p50: Duration::from_nanos(self.pool.metrics.p50_ns()),
253            wait_max: Duration::from_nanos(self.pool.metrics.wait_max_ns.load(Ordering::Relaxed)),
254        }
255    }
256
257    pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
258        let owned = text.to_string();
259        let results = self.pool.checkout(
260            || load_model(false),
261            move |session| {
262                session
263                    .embed(vec![owned.as_str()], None)
264                    .context("generating embedding")
265            },
266        )?;
267        results
268            .into_iter()
269            .next()
270            .ok_or_else(|| anyhow::anyhow!("no embedding returned"))
271    }
272
273    // BGE retrieval is asymmetric: the instruction prefix goes on the query only,
274    // documents stay bare, so embed() is left unchanged.
275    pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
276        self.embed(&query_prompt(text))
277    }
278
279    pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
280        let owned: Vec<String> = texts.iter().map(|t| t.to_string()).collect();
281        self.pool.checkout(
282            || load_model(false),
283            move |session| {
284                let refs: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
285                session
286                    .embed(refs, None)
287                    .context("generating batch embeddings")
288            },
289        )
290    }
291
292    pub fn drop_if_idle(&self, idle_secs: u64) {
293        self.pool.drop_if_idle(idle_secs);
294    }
295
296    pub fn dimension(&self) -> usize {
297        384
298    }
299}
300
301/// Embeds text for storage as a document: bare, never through
302/// `embed_query`, because BGE retrieval is asymmetric and the instruction
303/// prefix belongs on the query side only. A host that embeds somewhere other
304/// than an in-process `Embedder` implements this to feed ingestion.
305pub trait DocumentEmbedder: Send + Sync {
306    fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
307}
308
309impl DocumentEmbedder for Embedder {
310    fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
311        self.embed(text)
312    }
313}