use std::collections::VecDeque;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Condvar, Mutex, MutexGuard};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result, bail};
use fastembed::{EmbeddingModel, InitOptions, TextEmbedding};
const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
const WAIT_SAMPLE_CAP: usize = 512;
pub struct Embedder {
pool: SessionPool,
}
#[derive(Debug)]
pub struct EmbedTimeoutError {
pub waited: Duration,
pub pool_size: usize,
}
impl std::fmt::Display for EmbedTimeoutError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"timed out after {:?} waiting for an idle embedding session (pool size {})",
self.waited, self.pool_size
)
}
}
impl std::error::Error for EmbedTimeoutError {}
#[derive(Debug, Clone, Copy, Default)]
pub struct EmbedMetricsSnapshot {
pub in_flight: usize,
pub total_checkouts: u64,
pub wait_p50: Duration,
pub wait_max: Duration,
}
pub fn query_prompt(text: &str) -> String {
format!("Represent this sentence for searching relevant passages: {text}")
}
fn now_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
#[derive(Default)]
struct PoolMetrics {
in_flight: AtomicUsize,
total_checkouts: AtomicU64,
wait_max_ns: AtomicU64,
wait_samples: Mutex<VecDeque<u64>>,
}
impl PoolMetrics {
fn record_wait(&self, ns: u64) {
self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
let mut samples = lock_or_recover(&self.wait_samples);
if samples.len() >= WAIT_SAMPLE_CAP {
samples.pop_front();
}
samples.push_back(ns);
}
fn p50_ns(&self) -> u64 {
let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
.iter()
.copied()
.collect();
if samples.is_empty() {
return 0;
}
samples.sort_unstable();
samples[samples.len() / 2]
}
}
struct SessionPool {
sessions: Vec<Mutex<Option<TextEmbedding>>>,
last_used_secs: Vec<AtomicU64>,
free: Mutex<VecDeque<usize>>,
free_cv: Condvar,
checkout_timeout: Duration,
metrics: PoolMetrics,
}
struct Checkout<'a> {
pool: &'a SessionPool,
idx: usize,
}
impl Drop for Checkout<'_> {
fn drop(&mut self) {
self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
lock_or_recover(&self.pool.free).push_back(self.idx);
self.pool.free_cv.notify_one();
}
}
impl SessionPool {
fn new(n: usize, checkout_timeout: Duration) -> Self {
let mut sessions = Vec::with_capacity(n);
let mut last_used_secs = Vec::with_capacity(n);
let mut free = VecDeque::with_capacity(n);
for i in 0..n {
sessions.push(Mutex::new(None));
last_used_secs.push(AtomicU64::new(0));
free.push_back(i);
}
Self {
sessions,
last_used_secs,
free: Mutex::new(free),
free_cv: Condvar::new(),
checkout_timeout,
metrics: PoolMetrics::default(),
}
}
fn len(&self) -> usize {
self.sessions.len()
}
fn checkout<R>(
&self,
load: impl FnOnce() -> Result<TextEmbedding>,
with_session: impl FnOnce(&mut TextEmbedding) -> Result<R>,
) -> Result<R> {
let start = Instant::now();
let guard = lock_or_recover(&self.free);
let (mut guard, wait_result) = self
.free_cv
.wait_timeout_while(guard, self.checkout_timeout, |q| q.is_empty())
.unwrap_or_else(|e| e.into_inner());
if wait_result.timed_out() && guard.is_empty() {
return Err(EmbedTimeoutError {
waited: start.elapsed(),
pool_size: self.len(),
}
.into());
}
let idx = guard.pop_front().expect("free slot present after wait");
drop(guard);
self.metrics.record_wait(start.elapsed().as_nanos() as u64);
self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
let _checkout = Checkout { pool: self, idx };
let mut slot = lock_or_recover(&self.sessions[idx]);
if slot.is_none() {
tracing::info!("reloading embedding model after idle drop (slot {idx})");
*slot = Some(load()?);
}
self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
with_session(slot.as_mut().expect("session loaded above"))
}
fn drop_if_idle(&self, idle_secs: u64) {
let now = now_secs();
for (idx, last) in self.last_used_secs.iter().enumerate() {
if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
continue;
}
if let Ok(mut slot) = self.sessions[idx].try_lock()
&& slot.is_some()
{
*slot = None;
tracing::info!("embedding session {idx} dropped after {idle_secs}s idle");
}
}
}
}
fn load_model(show_progress: bool) -> Result<TextEmbedding> {
TextEmbedding::try_new(
InitOptions::new(EmbeddingModel::BGESmallENV15).with_show_download_progress(show_progress),
)
.context("loading embedding model")
}
impl Embedder {
pub fn new() -> Result<Self> {
Self::with_sessions(1)
}
pub fn with_sessions(n: usize) -> Result<Self> {
if n == 0 {
bail!("embedding pool size must be at least 1");
}
let pool = SessionPool::new(n, DEFAULT_CHECKOUT_TIMEOUT);
for i in 0..n {
let mut slot = lock_or_recover(&pool.sessions[i]);
*slot = Some(
load_model(true)
.with_context(|| format!("initializing embedding session {i} of {n}"))?,
);
pool.last_used_secs[i].store(now_secs(), Ordering::Relaxed);
}
Ok(Self { pool })
}
pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
self.pool.checkout_timeout = timeout;
self
}
pub fn pool_size(&self) -> usize {
self.pool.len()
}
pub fn metrics(&self) -> EmbedMetricsSnapshot {
EmbedMetricsSnapshot {
in_flight: self.pool.metrics.in_flight.load(Ordering::Relaxed),
total_checkouts: self.pool.metrics.total_checkouts.load(Ordering::Relaxed),
wait_p50: Duration::from_nanos(self.pool.metrics.p50_ns()),
wait_max: Duration::from_nanos(self.pool.metrics.wait_max_ns.load(Ordering::Relaxed)),
}
}
pub fn embed(&self, text: &str) -> Result<Vec<f32>> {
let owned = text.to_string();
let results = self.pool.checkout(
|| load_model(false),
move |session| {
session
.embed(vec![owned.as_str()], None)
.context("generating embedding")
},
)?;
results
.into_iter()
.next()
.ok_or_else(|| anyhow::anyhow!("no embedding returned"))
}
pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
self.embed(&query_prompt(text))
}
pub fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
let owned: Vec<String> = texts.iter().map(|t| t.to_string()).collect();
self.pool.checkout(
|| load_model(false),
move |session| {
let refs: Vec<&str> = owned.iter().map(|s| s.as_str()).collect();
session
.embed(refs, None)
.context("generating batch embeddings")
},
)
}
pub fn drop_if_idle(&self, idle_secs: u64) {
self.pool.drop_if_idle(idle_secs);
}
pub fn dimension(&self) -> usize {
384
}
}
pub trait DocumentEmbedder: Send + Sync {
fn embed_document(&self, text: &str) -> Result<Vec<f32>>;
}
impl DocumentEmbedder for Embedder {
fn embed_document(&self, text: &str) -> Result<Vec<f32>> {
self.embed(text)
}
}