second_brain_core/
embedding.rs1use std::sync::Arc;
2use std::time::Duration;
3
4use anyhow::{Context, Result, bail};
5use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
6use ort::ep::CPU;
7
8use crate::model_cache::{CachedModel, SnapshotLookup, model_cache_dir};
9use crate::session_pool::SessionPool;
10
11const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
12const MODEL: EmbeddingModel = EmbeddingModel::BGESmallENV15;
13
14pub struct Embedder {
20 pool: Arc<SessionPool<TextEmbedding>>,
21 checkout_timeout: Duration,
22 activations: Activations,
23}
24
25#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
27pub enum Activations {
28 #[default]
31 Retained,
32 Released,
35}
36
37#[derive(Debug)]
41pub struct EmbedTimeoutError {
42 pub waited: Duration,
43 pub pool_size: usize,
44}
45
46impl std::fmt::Display for EmbedTimeoutError {
47 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48 write!(
49 f,
50 "timed out after {:?} waiting for an idle embedding session (pool size {})",
51 self.waited, self.pool_size
52 )
53 }
54}
55
56impl std::error::Error for EmbedTimeoutError {}
57
58#[derive(Debug, Clone, Copy, Default)]
60pub struct EmbedMetricsSnapshot {
61 pub loaded_sessions: usize,
62 pub in_flight: usize,
63 pub total_checkouts: u64,
64 pub wait_p50: Duration,
65 pub wait_max: Duration,
66}
67
68pub fn query_prompt(text: &str) -> String {
69 format!("Represent this sentence for searching relevant passages: {text}")
70}
71
72fn load_model(show_progress: bool, activations: Activations) -> Result<TextEmbedding> {
73 let mut options = InitOptions::new(MODEL)
74 .with_cache_dir(model_cache_dir()?)
75 .with_show_download_progress(show_progress);
76 if activations == Activations::Released {
77 options = options.with_execution_providers(vec![CPU::default().build()]);
79 }
80 TextEmbedding::try_new(options).context("loading embedding model")
81}
82
83impl Embedder {
84 pub fn new() -> Result<Self> {
85 Self::with_sessions(1)
86 }
87
88 pub fn with_sessions(n: usize) -> Result<Self> {
92 Self::with_sessions_and_activations(n, Activations::default())
93 }
94
95 pub fn with_sessions_and_activations(n: usize, activations: Activations) -> Result<Self> {
96 if n == 0 {
97 bail!("embedding pool size must be at least 1");
98 }
99 let pool = SessionPool::new("embedding", n, |waited, pool_size| {
100 EmbedTimeoutError { waited, pool_size }.into()
101 });
102 for i in 0..n {
103 pool.install(i, || load_model(true, activations))
104 .with_context(|| format!("initializing embedding session {i} of {n}"))?;
105 }
106
107 Ok(Self {
108 pool: Arc::new(pool),
109 checkout_timeout: DEFAULT_CHECKOUT_TIMEOUT,
110 activations,
111 })
112 }
113
114 pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
117 self.checkout_timeout = timeout;
118 self
119 }
120
121 pub fn pool_size(&self) -> usize {
122 self.pool.len()
123 }
124
125 pub fn metrics(&self) -> EmbedMetricsSnapshot {
126 let stats = self.pool.stats();
127 EmbedMetricsSnapshot {
128 loaded_sessions: stats.loaded_sessions,
129 in_flight: stats.in_flight,
130 total_checkouts: stats.total_checkouts,
131 wait_p50: stats.wait_p50,
132 wait_max: stats.wait_max,
133 }
134 }
135
136 pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
137 let activations = self.activations;
138 self.pool.checkout(
139 self.checkout_timeout,
140 move || load_model(false, activations),
141 |session| embed_one(session, text),
142 )
143 }
144
145 pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
148 self.embed(&query_prompt(text))
149 }
150
151 pub fn embed_query_within(&self, text: &str, wait: Duration) -> Result<Vec<f32>> {
156 let prompt = query_prompt(text);
157 let activations = self.activations;
158 self.pool.checkout_warm(
159 wait.min(self.checkout_timeout),
160 move || load_model(false, activations),
161 |session| embed_one(session, &prompt),
162 )
163 }
164
165 pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
166 let activations = self.activations;
167 self.pool.checkout(
168 self.checkout_timeout,
169 move || load_model(false, activations),
170 |session| {
171 session
172 .embed(texts, None)
173 .context("generating batch embeddings")
174 },
175 )
176 }
177
178 pub fn drop_if_idle(&self, idle_secs: u64) {
179 self.pool.drop_if_idle(idle_secs);
180 }
181
182 pub fn resident_bytes(&self) -> u64 {
184 self.pool.slot_resident_bytes().iter().sum()
185 }
186
187 pub fn slot_resident_bytes(&self) -> Vec<u64> {
189 self.pool.slot_resident_bytes()
190 }
191
192 pub fn dimension(&self) -> usize {
193 384
194 }
195}
196
197pub fn cached_model() -> Result<CachedModel> {
200 let info = TextEmbedding::get_model_info(&MODEL)?;
201 Ok(CachedModel::new(
202 &info.model_code,
203 &info.model_file,
204 &info.additional_files,
205 SnapshotLookup::MainRef,
206 ))
207}
208
209fn embed_one(session: &mut TextEmbedding, text: &str) -> Result<Vec<f32>> {
210 session
211 .embed(vec![text], None)
212 .context("generating embedding")?
213 .into_iter()
214 .next()
215 .ok_or_else(|| anyhow::anyhow!("no embedding returned"))
216}
217
218pub trait DocumentEmbedder: Send + Sync {
223 fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
224}
225
226impl DocumentEmbedder for Embedder {
227 fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
228 self.embed(text)
229 }
230}
231
232#[cfg(test)]
233mod tests {
234 use super::*;
235 use crate::model_cache::TOKENIZER_FILES;
236
237 #[test]
238 fn cached_model_is_fastembeds_entry_for_the_loaded_model() {
239 let info = TextEmbedding::get_model_info(&EmbeddingModel::BGESmallENV15).unwrap();
240
241 let model = cached_model().unwrap();
242
243 assert_eq!(model.code, info.model_code);
244 assert!(model.files.contains(&info.model_file), "{model:?}");
245 for file in TOKENIZER_FILES {
246 assert!(model.files.iter().any(|f| f == file), "{model:?}");
247 }
248 }
249}