second_brain_core/
embedding.rs1use std::collections::VecDeque;
2use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
3use std::sync::{Condvar, Mutex, MutexGuard};
4use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
5
6use anyhow::{Context, Result, bail};
7use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
8
9use crate::model_cache::model_cache_dir;
10
11const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
12const WAIT_SAMPLE_CAP: usize = 512;
13
14pub struct Embedder {
20 pool: SessionPool,
21}
22
23#[derive(Debug)]
27pub struct EmbedTimeoutError {
28 pub waited: Duration,
29 pub pool_size: usize,
30}
31
32impl std::fmt::Display for EmbedTimeoutError {
33 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
34 write!(
35 f,
36 "timed out after {:?} waiting for an idle embedding session (pool size {})",
37 self.waited, self.pool_size
38 )
39 }
40}
41
42impl std::error::Error for EmbedTimeoutError {}
43
44#[derive(Debug, Clone, Copy, Default)]
46pub struct EmbedMetricsSnapshot {
47 pub in_flight: usize,
48 pub total_checkouts: u64,
49 pub wait_p50: Duration,
50 pub wait_max: Duration,
51}
52
53pub fn query_prompt(text: &str) -> String {
54 format!("Represent this sentence for searching relevant passages: {text}")
55}
56
57fn now_secs() -> u64 {
58 SystemTime::now()
59 .duration_since(UNIX_EPOCH)
60 .unwrap_or_default()
61 .as_secs()
62}
63
64fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
65 m.lock().unwrap_or_else(|e| e.into_inner())
66}
67
68#[derive(Default)]
69struct PoolMetrics {
70 in_flight: AtomicUsize,
71 total_checkouts: AtomicU64,
72 wait_max_ns: AtomicU64,
73 wait_samples: Mutex<VecDeque<u64>>,
77}
78
79impl PoolMetrics {
80 fn record_wait(&self, ns: u64) {
81 self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
82 let mut samples = lock_or_recover(&self.wait_samples);
83 if samples.len() >= WAIT_SAMPLE_CAP {
84 samples.pop_front();
85 }
86 samples.push_back(ns);
87 }
88
89 fn p50_ns(&self) -> u64 {
90 let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
91 .iter()
92 .copied()
93 .collect();
94 if samples.is_empty() {
95 return 0;
96 }
97 samples.sort_unstable();
98 samples[samples.len() / 2]
99 }
100}
101
102struct SessionPool {
106 sessions: Vec<Mutex<Option<TextEmbedding>>>,
107 last_used_secs: Vec<AtomicU64>,
108 free: Mutex<VecDeque<usize>>,
109 free_cv: Condvar,
110 checkout_timeout: Duration,
111 metrics: PoolMetrics,
112}
113
114struct Checkout<'a> {
118 pool: &'a SessionPool,
119 idx: usize,
120}
121
122impl Drop for Checkout<'_> {
123 fn drop(&mut self) {
124 self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
125 lock_or_recover(&self.pool.free).push_back(self.idx);
126 self.pool.free_cv.notify_one();
127 }
128}
129
130impl SessionPool {
131 fn new(n: usize, checkout_timeout: Duration) -> Self {
132 let mut sessions = Vec::with_capacity(n);
133 let mut last_used_secs = Vec::with_capacity(n);
134 let mut free = VecDeque::with_capacity(n);
135 for i in 0..n {
136 sessions.push(Mutex::new(None));
137 last_used_secs.push(AtomicU64::new(0));
138 free.push_back(i);
139 }
140 Self {
141 sessions,
142 last_used_secs,
143 free: Mutex::new(free),
144 free_cv: Condvar::new(),
145 checkout_timeout,
146 metrics: PoolMetrics::default(),
147 }
148 }
149
150 fn len(&self) -> usize {
151 self.sessions.len()
152 }
153
154 fn checkout<R>(
155 &self,
156 load: impl FnOnce() -> Result<TextEmbedding>,
157 with_session: impl FnOnce(&mut TextEmbedding) -> Result<R>,
158 ) -> Result<R> {
159 let start = Instant::now();
160 let guard = lock_or_recover(&self.free);
161 let (mut guard, wait_result) = self
162 .free_cv
163 .wait_timeout_while(guard, self.checkout_timeout, |q| q.is_empty())
164 .unwrap_or_else(|e| e.into_inner());
165 if wait_result.timed_out() && guard.is_empty() {
166 return Err(EmbedTimeoutError {
167 waited: start.elapsed(),
168 pool_size: self.len(),
169 }
170 .into());
171 }
172 let idx = guard.pop_front().expect("free slot present after wait");
173 drop(guard);
174
175 self.metrics.record_wait(start.elapsed().as_nanos() as u64);
176 self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
177 self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
178 let _checkout = Checkout { pool: self, idx };
179
180 let mut slot = lock_or_recover(&self.sessions[idx]);
181 if slot.is_none() {
182 tracing::info!("reloading embedding model after idle drop (slot {idx})");
183 *slot = Some(load()?);
184 }
185 self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
186 with_session(slot.as_mut().expect("session loaded above"))
187 }
188
189 fn drop_if_idle(&self, idle_secs: u64) {
190 let now = now_secs();
191 for (idx, last) in self.last_used_secs.iter().enumerate() {
192 if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
193 continue;
194 }
195 if let Ok(mut slot) = self.sessions[idx].try_lock()
198 && slot.is_some()
199 {
200 *slot = None;
201 tracing::info!("embedding session {idx} dropped after {idle_secs}s idle");
202 }
203 }
204 }
205}
206
207fn load_model(show_progress: bool) -> Result<TextEmbedding> {
208 TextEmbedding::try_new(
209 InitOptions::new(EmbeddingModel::BGESmallENV15)
210 .with_cache_dir(model_cache_dir()?)
211 .with_show_download_progress(show_progress),
212 )
213 .context("loading embedding model")
214}
215
216impl Embedder {
217 pub fn new() -> Result<Self> {
218 Self::with_sessions(1)
219 }
220
221 pub fn with_sessions(n: usize) -> Result<Self> {
225 if n == 0 {
226 bail!("embedding pool size must be at least 1");
227 }
228 let pool = SessionPool::new(n, DEFAULT_CHECKOUT_TIMEOUT);
229 for i in 0..n {
230 let mut slot = lock_or_recover(&pool.sessions[i]);
231 *slot = Some(
232 load_model(true)
233 .with_context(|| format!("initializing embedding session {i} of {n}"))?,
234 );
235 pool.last_used_secs[i].store(now_secs(), Ordering::Relaxed);
236 }
237
238 Ok(Self { pool })
239 }
240
241 pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
244 self.pool.checkout_timeout = timeout;
245 self
246 }
247
248 pub fn pool_size(&self) -> usize {
249 self.pool.len()
250 }
251
252 pub fn metrics(&self) -> EmbedMetricsSnapshot {
253 EmbedMetricsSnapshot {
254 in_flight: self.pool.metrics.in_flight.load(Ordering::Relaxed),
255 total_checkouts: self.pool.metrics.total_checkouts.load(Ordering::Relaxed),
256 wait_p50: Duration::from_nanos(self.pool.metrics.p50_ns()),
257 wait_max: Duration::from_nanos(self.pool.metrics.wait_max_ns.load(Ordering::Relaxed)),
258 }
259 }
260
261 pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
262 let owned = text.to_string();
263 let results = self.pool.checkout(
264 || load_model(false),
265 move |session| {
266 session
267 .embed(vec![owned.as_str()], None)
268 .context("generating embedding")
269 },
270 )?;
271 results
272 .into_iter()
273 .next()
274 .ok_or_else(|| anyhow::anyhow!("no embedding returned"))
275 }
276
277 pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
280 self.embed(&query_prompt(text))
281 }
282
283 pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
284 let owned: Vec<String> = texts.iter().map(|t| t.to_string()).collect();
285 self.pool.checkout(
286 || load_model(false),
287 move |session| {
288 let refs: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
289 session
290 .embed(refs, None)
291 .context("generating batch embeddings")
292 },
293 )
294 }
295
296 pub fn drop_if_idle(&self, idle_secs: u64) {
297 self.pool.drop_if_idle(idle_secs);
298 }
299
300 pub fn dimension(&self) -> usize {
301 384
302 }
303}
304
305pub trait DocumentEmbedder: Send + Sync {
310 fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
311}
312
313impl DocumentEmbedder for Embedder {
314 fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
315 self.embed(text)
316 }
317}