second_brain_core/
embedding.rs1use std::collections::VecDeque;
2use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
3use std::sync::{Condvar, Mutex, MutexGuard};
4use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
5
6use anyhow::{Context, Result, bail};
7use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
8
9const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
10const WAIT_SAMPLE_CAP: usize = 512;
11
12pub struct Embedder {
18 pool: SessionPool,
19}
20
21#[derive(Debug)]
25pub struct EmbedTimeoutError {
26 pub waited: Duration,
27 pub pool_size: usize,
28}
29
30impl std::fmt::Display for EmbedTimeoutError {
31 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
32 write!(
33 f,
34 "timed out after {:?} waiting for an idle embedding session (pool size {})",
35 self.waited, self.pool_size
36 )
37 }
38}
39
40impl std::error::Error for EmbedTimeoutError {}
41
42#[derive(Debug, Clone, Copy, Default)]
44pub struct EmbedMetricsSnapshot {
45 pub in_flight: usize,
46 pub total_checkouts: u64,
47 pub wait_p50: Duration,
48 pub wait_max: Duration,
49}
50
51pub fn query_prompt(text: &str) -> String {
52 format!("Represent this sentence for searching relevant passages: {text}")
53}
54
55fn now_secs() -> u64 {
56 SystemTime::now()
57 .duration_since(UNIX_EPOCH)
58 .unwrap_or_default()
59 .as_secs()
60}
61
62fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
63 m.lock().unwrap_or_else(|e| e.into_inner())
64}
65
66#[derive(Default)]
67struct PoolMetrics {
68 in_flight: AtomicUsize,
69 total_checkouts: AtomicU64,
70 wait_max_ns: AtomicU64,
71 wait_samples: Mutex<VecDeque<u64>>,
75}
76
77impl PoolMetrics {
78 fn record_wait(&self, ns: u64) {
79 self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
80 let mut samples = lock_or_recover(&self.wait_samples);
81 if samples.len() >= WAIT_SAMPLE_CAP {
82 samples.pop_front();
83 }
84 samples.push_back(ns);
85 }
86
87 fn p50_ns(&self) -> u64 {
88 let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
89 .iter()
90 .copied()
91 .collect();
92 if samples.is_empty() {
93 return 0;
94 }
95 samples.sort_unstable();
96 samples[samples.len() / 2]
97 }
98}
99
100struct SessionPool {
104 sessions: Vec<Mutex<Option<TextEmbedding>>>,
105 last_used_secs: Vec<AtomicU64>,
106 free: Mutex<VecDeque<usize>>,
107 free_cv: Condvar,
108 checkout_timeout: Duration,
109 metrics: PoolMetrics,
110}
111
112struct Checkout<'a> {
116 pool: &'a SessionPool,
117 idx: usize,
118}
119
120impl Drop for Checkout<'_> {
121 fn drop(&mut self) {
122 self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
123 lock_or_recover(&self.pool.free).push_back(self.idx);
124 self.pool.free_cv.notify_one();
125 }
126}
127
128impl SessionPool {
129 fn new(n: usize, checkout_timeout: Duration) -> Self {
130 let mut sessions = Vec::with_capacity(n);
131 let mut last_used_secs = Vec::with_capacity(n);
132 let mut free = VecDeque::with_capacity(n);
133 for i in 0..n {
134 sessions.push(Mutex::new(None));
135 last_used_secs.push(AtomicU64::new(0));
136 free.push_back(i);
137 }
138 Self {
139 sessions,
140 last_used_secs,
141 free: Mutex::new(free),
142 free_cv: Condvar::new(),
143 checkout_timeout,
144 metrics: PoolMetrics::default(),
145 }
146 }
147
148 fn len(&self) -> usize {
149 self.sessions.len()
150 }
151
152 fn checkout<R>(
153 &self,
154 load: impl FnOnce() -> Result<TextEmbedding>,
155 with_session: impl FnOnce(&mut TextEmbedding) -> Result<R>,
156 ) -> Result<R> {
157 let start = Instant::now();
158 let guard = lock_or_recover(&self.free);
159 let (mut guard, wait_result) = self
160 .free_cv
161 .wait_timeout_while(guard, self.checkout_timeout, |q| q.is_empty())
162 .unwrap_or_else(|e| e.into_inner());
163 if wait_result.timed_out() && guard.is_empty() {
164 return Err(EmbedTimeoutError {
165 waited: start.elapsed(),
166 pool_size: self.len(),
167 }
168 .into());
169 }
170 let idx = guard.pop_front().expect("free slot present after wait");
171 drop(guard);
172
173 self.metrics.record_wait(start.elapsed().as_nanos() as u64);
174 self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
175 self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
176 let _checkout = Checkout { pool: self, idx };
177
178 let mut slot = lock_or_recover(&self.sessions[idx]);
179 if slot.is_none() {
180 tracing::info!("reloading embedding model after idle drop (slot {idx})");
181 *slot = Some(load()?);
182 }
183 self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
184 with_session(slot.as_mut().expect("session loaded above"))
185 }
186
187 fn drop_if_idle(&self, idle_secs: u64) {
188 let now = now_secs();
189 for (idx, last) in self.last_used_secs.iter().enumerate() {
190 if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
191 continue;
192 }
193 if let Ok(mut slot) = self.sessions[idx].try_lock()
196 && slot.is_some()
197 {
198 *slot = None;
199 tracing::info!("embedding session {idx} dropped after {idle_secs}s idle");
200 }
201 }
202 }
203}
204
205fn load_model(show_progress: bool) -> Result<TextEmbedding> {
206 TextEmbedding::try_new(
207 InitOptions::new(EmbeddingModel::BGESmallENV15).with_show_download_progress(show_progress),
208 )
209 .context("loading embedding model")
210}
211
212impl Embedder {
213 pub fn new() -> Result<Self> {
214 Self::with_sessions(1)
215 }
216
217 pub fn with_sessions(n: usize) -> Result<Self> {
221 if n == 0 {
222 bail!("embedding pool size must be at least 1");
223 }
224 let pool = SessionPool::new(n, DEFAULT_CHECKOUT_TIMEOUT);
225 for i in 0..n {
226 let mut slot = lock_or_recover(&pool.sessions[i]);
227 *slot = Some(
228 load_model(true)
229 .with_context(|| format!("initializing embedding session {i} of {n}"))?,
230 );
231 pool.last_used_secs[i].store(now_secs(), Ordering::Relaxed);
232 }
233
234 Ok(Self { pool })
235 }
236
237 pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
240 self.pool.checkout_timeout = timeout;
241 self
242 }
243
244 pub fn pool_size(&self) -> usize {
245 self.pool.len()
246 }
247
248 pub fn metrics(&self) -> EmbedMetricsSnapshot {
249 EmbedMetricsSnapshot {
250 in_flight: self.pool.metrics.in_flight.load(Ordering::Relaxed),
251 total_checkouts: self.pool.metrics.total_checkouts.load(Ordering::Relaxed),
252 wait_p50: Duration::from_nanos(self.pool.metrics.p50_ns()),
253 wait_max: Duration::from_nanos(self.pool.metrics.wait_max_ns.load(Ordering::Relaxed)),
254 }
255 }
256
257 pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
258 let owned = text.to_string();
259 let results = self.pool.checkout(
260 || load_model(false),
261 move |session| {
262 session
263 .embed(vec![owned.as_str()], None)
264 .context("generating embedding")
265 },
266 )?;
267 results
268 .into_iter()
269 .next()
270 .ok_or_else(|| anyhow::anyhow!("no embedding returned"))
271 }
272
273 pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
276 self.embed(&query_prompt(text))
277 }
278
279 pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
280 let owned: Vec<String> = texts.iter().map(|t| t.to_string()).collect();
281 self.pool.checkout(
282 || load_model(false),
283 move |session| {
284 let refs: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
285 session
286 .embed(refs, None)
287 .context("generating batch embeddings")
288 },
289 )
290 }
291
292 pub fn drop_if_idle(&self, idle_secs: u64) {
293 self.pool.drop_if_idle(idle_secs);
294 }
295
296 pub fn dimension(&self) -> usize {
297 384
298 }
299}
300
301pub trait DocumentEmbedder: Send + Sync {
306 fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
307}
308
309impl DocumentEmbedder for Embedder {
310 fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
311 self.embed(text)
312 }
313}