use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result, bail};
use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
use ort::ep::CPU;
use crate::model_cache::{CachedModel, SnapshotLookup, model_cache_dir};
use crate::session_pool::SessionPool;
const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
const MODEL: EmbeddingModel = EmbeddingModel::BGESmallENV15;
pub struct Embedder {
pool: Arc<SessionPool<TextEmbedding>>,
checkout_timeout: Duration,
activations: Activations,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum Activations {
#[default]
Retained,
Released,
}
#[derive(Debug)]
pub struct EmbedTimeoutError {
pub waited: Duration,
pub pool_size: usize,
}
impl std::fmt::Display for EmbedTimeoutError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"timed out after {:?} waiting for an idle embedding session (pool size {})",
self.waited, self.pool_size
)
}
}
impl std::error::Error for EmbedTimeoutError {}
#[derive(Debug, Clone, Copy, Default)]
pub struct EmbedMetricsSnapshot {
pub loaded_sessions: usize,
pub in_flight: usize,
pub total_checkouts: u64,
pub wait_p50: Duration,
pub wait_max: Duration,
}
pub fn query_prompt(text: &str) -> String {
format!("Represent this sentence for searching relevant passages: {text}")
}
fn load_model(show_progress: bool, activations: Activations) -> Result<TextEmbedding> {
let mut options = InitOptions::new(MODEL)
.with_cache_dir(model_cache_dir()?)
.with_show_download_progress(show_progress);
if activations == Activations::Released {
options = options.with_execution_providers(vec![CPU::default().build()]);
}
TextEmbedding::try_new(options).context("loading embedding model")
}
impl Embedder {
pub fn new() -> Result<Self> {
Self::with_sessions(1)
}
pub fn with_sessions(n: usize) -> Result<Self> {
Self::with_sessions_and_activations(n, Activations::default())
}
pub fn with_sessions_and_activations(n: usize, activations: Activations) -> Result<Self> {
if n == 0 {
bail!("embedding pool size must be at least 1");
}
let pool = SessionPool::new("embedding", n, |waited, pool_size| {
EmbedTimeoutError { waited, pool_size }.into()
});
for i in 0..n {
pool.install(i, || load_model(true, activations))
.with_context(|| format!("initializing embedding session {i} of {n}"))?;
}
Ok(Self {
pool: Arc::new(pool),
checkout_timeout: DEFAULT_CHECKOUT_TIMEOUT,
activations,
})
}
pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
self.checkout_timeout = timeout;
self
}
pub fn pool_size(&self) -> usize {
self.pool.len()
}
pub fn metrics(&self) -> EmbedMetricsSnapshot {
let stats = self.pool.stats();
EmbedMetricsSnapshot {
loaded_sessions: stats.loaded_sessions,
in_flight: stats.in_flight,
total_checkouts: stats.total_checkouts,
wait_p50: stats.wait_p50,
wait_max: stats.wait_max,
}
}
pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
let activations = self.activations;
self.pool.checkout(
self.checkout_timeout,
move || load_model(false, activations),
|session| embed_one(session, text),
)
}
pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
self.embed(&query_prompt(text))
}
pub fn embed_query_within(&self, text: &str, wait: Duration) -> Result<Vec<f32>> {
let prompt = query_prompt(text);
let activations = self.activations;
self.pool.checkout_warm(
wait.min(self.checkout_timeout),
move || load_model(false, activations),
|session| embed_one(session, &prompt),
)
}
pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
let activations = self.activations;
self.pool.checkout(
self.checkout_timeout,
move || load_model(false, activations),
|session| {
session
.embed(texts, None)
.context("generating batch embeddings")
},
)
}
pub fn drop_if_idle(&self, idle_secs: u64) {
self.pool.drop_if_idle(idle_secs);
}
pub fn resident_bytes(&self) -> u64 {
self.pool.slot_resident_bytes().iter().sum()
}
pub fn slot_resident_bytes(&self) -> Vec<u64> {
self.pool.slot_resident_bytes()
}
pub fn dimension(&self) -> usize {
384
}
}
pub fn cached_model() -> Result<CachedModel> {
let info = TextEmbedding::get_model_info(&MODEL)?;
Ok(CachedModel::new(
&info.model_code,
&info.model_file,
&info.additional_files,
SnapshotLookup::MainRef,
))
}
fn embed_one(session: &mut TextEmbedding, text: &str) -> Result<Vec<f32>> {
session
.embed(vec![text], None)
.context("generating embedding")?
.into_iter()
.next()
.ok_or_else(|| anyhow::anyhow!("no embedding returned"))
}
pub trait DocumentEmbedder: Send + Sync {
fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
}
impl DocumentEmbedder for Embedder {
fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
self.embed(text)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::model_cache::TOKENIZER_FILES;
#[test]
fn cached_model_is_fastembeds_entry_for_the_loaded_model() {
let info = TextEmbedding::get_model_info(&EmbeddingModel::BGESmallENV15).unwrap();
let model = cached_model().unwrap();
assert_eq!(model.code, info.model_code);
assert!(model.files.contains(&info.model_file), "{model:?}");
for file in TOKENIZER_FILES {
assert!(model.files.iter().any(|f| f == file), "{model:?}");
}
}
}