use std::collections::VecDeque;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Condvar, Mutex, MutexGuard};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result, bail};
use fastembed::{RerankInitOptions, RerankerModel, TextRerank};
use super::onnx_rerank::{self, OnnxReranker, OnnxSessionConfig};
use crate::model_cache::model_cache_dir;
const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
const WAIT_SAMPLE_CAP: usize = 512;
const DEFAULT_DOC_CHARS: usize = 1024;
pub struct Reranker {
pool: SessionPool,
model_name: RerankerModel,
ep: RerankEp,
max_length: Option<usize>,
doc_chars: Option<usize>,
backend: RerankBackend,
intra_threads: usize,
inter_threads: usize,
}
enum RerankSession {
FastEmbed(TextRerank),
Onnx(OnnxReranker),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RerankBackend {
Onnx,
FastEmbed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RerankEp {
Cpu,
Cuda,
}
#[derive(Debug)]
pub struct RerankTimeoutError {
pub waited: Duration,
pub pool_size: usize,
}
impl std::fmt::Display for RerankTimeoutError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"timed out after {:?} waiting for an idle rerank session (pool size {})",
self.waited, self.pool_size
)
}
}
impl std::error::Error for RerankTimeoutError {}
pub trait RerankScorer: Send + Sync {
fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>>;
}
#[derive(Debug, Clone, Copy, Default)]
pub struct RerankMetricsSnapshot {
pub in_flight: usize,
pub total_checkouts: u64,
pub wait_p50: Duration,
pub wait_max: Duration,
}
fn now_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
#[derive(Default)]
struct PoolMetrics {
in_flight: AtomicUsize,
total_checkouts: AtomicU64,
wait_max_ns: AtomicU64,
wait_samples: Mutex<VecDeque<u64>>,
}
impl PoolMetrics {
fn record_wait(&self, ns: u64) {
self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
let mut samples = lock_or_recover(&self.wait_samples);
if samples.len() >= WAIT_SAMPLE_CAP {
samples.pop_front();
}
samples.push_back(ns);
}
fn p50_ns(&self) -> u64 {
let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
.iter()
.copied()
.collect();
if samples.is_empty() {
return 0;
}
samples.sort_unstable();
samples[samples.len() / 2]
}
}
struct SessionPool {
sessions: Vec<Mutex<Option<RerankSession>>>,
last_used_secs: Vec<AtomicU64>,
free: Mutex<VecDeque<usize>>,
free_cv: Condvar,
checkout_timeout: Duration,
metrics: PoolMetrics,
}
struct Checkout<'a> {
pool: &'a SessionPool,
idx: usize,
}
impl Drop for Checkout<'_> {
fn drop(&mut self) {
self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
lock_or_recover(&self.pool.free).push_back(self.idx);
self.pool.free_cv.notify_one();
}
}
impl SessionPool {
fn new(n: usize, checkout_timeout: Duration) -> Self {
let mut sessions = Vec::with_capacity(n);
let mut last_used_secs = Vec::with_capacity(n);
let mut free = VecDeque::with_capacity(n);
for i in 0..n {
sessions.push(Mutex::new(None));
last_used_secs.push(AtomicU64::new(0));
free.push_back(i);
}
Self {
sessions,
last_used_secs,
free: Mutex::new(free),
free_cv: Condvar::new(),
checkout_timeout,
metrics: PoolMetrics::default(),
}
}
fn len(&self) -> usize {
self.sessions.len()
}
fn checkout<R>(
&self,
load: impl FnOnce() -> Result<RerankSession>,
with_session: impl FnOnce(&mut RerankSession) -> Result<R>,
) -> Result<R> {
let start = Instant::now();
let guard = lock_or_recover(&self.free);
let (mut guard, wait_result) = self
.free_cv
.wait_timeout_while(guard, self.checkout_timeout, |q| q.is_empty())
.unwrap_or_else(|e| e.into_inner());
if wait_result.timed_out() && guard.is_empty() {
return Err(RerankTimeoutError {
waited: start.elapsed(),
pool_size: self.len(),
}
.into());
}
let idx = guard.pop_front().expect("free slot present after wait");
drop(guard);
self.metrics.record_wait(start.elapsed().as_nanos() as u64);
self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
let _checkout = Checkout { pool: self, idx };
let mut slot = lock_or_recover(&self.sessions[idx]);
if slot.is_none() {
tracing::info!("reloading rerank model after idle drop (slot {idx})");
*slot = Some(load()?);
}
self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
with_session(slot.as_mut().expect("session loaded above"))
}
fn drop_if_idle(&self, idle_secs: u64) {
let now = now_secs();
for (idx, last) in self.last_used_secs.iter().enumerate() {
if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
continue;
}
if let Ok(mut slot) = self.sessions[idx].try_lock()
&& slot.is_some()
{
*slot = None;
tracing::info!("rerank session {idx} dropped after {idle_secs}s idle");
}
}
}
}
pub fn parse_model(name: &str) -> Result<RerankerModel> {
match name {
"bge-reranker-v2-m3" => Ok(RerankerModel::BGERerankerV2M3),
"jina-reranker-v1-turbo" => Ok(RerankerModel::JINARerankerV1TurboEn),
other => {
bail!("unknown rerank model '{other}' (bge-reranker-v2-m3, jina-reranker-v1-turbo)")
}
}
}
fn parse_ep(raw: &str) -> Result<RerankEp> {
match raw {
"cpu" => Ok(RerankEp::Cpu),
"cuda" => Ok(RerankEp::Cuda),
other => bail!("unknown SECOND_BRAIN_RERANK_EP '{other}' (valid values: cpu, cuda)"),
}
}
pub(crate) fn ep_from_env() -> Result<RerankEp> {
match std::env::var("SECOND_BRAIN_RERANK_EP") {
Ok(v) => parse_ep(&v),
Err(_) => Ok(RerankEp::Cpu),
}
}
fn backend_from_env() -> RerankBackend {
match std::env::var("SECOND_BRAIN_RERANK_BACKEND") {
Ok(v) if v == "fastembed" => RerankBackend::FastEmbed,
_ => RerankBackend::Onnx,
}
}
fn intra_threads_from_env(pool_size: usize) -> Result<usize> {
match std::env::var("SECOND_BRAIN_RERANK_INTRA_THREADS") {
Ok(v) => {
let n: usize = v
.parse()
.with_context(|| format!("parsing SECOND_BRAIN_RERANK_INTRA_THREADS '{v}'"))?;
if n == 0 {
bail!("SECOND_BRAIN_RERANK_INTRA_THREADS must be at least 1");
}
Ok(n)
}
Err(_) => {
let cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
Ok((cores / pool_size.max(1)).max(1))
}
}
}
fn max_length_from_env() -> Result<Option<usize>> {
match std::env::var("SECOND_BRAIN_RERANK_MAX_LEN") {
Ok(v) => Ok(Some(v.parse::<usize>().with_context(|| {
format!("parsing SECOND_BRAIN_RERANK_MAX_LEN '{v}'")
})?)),
Err(_) => Ok(None),
}
}
fn doc_chars_from_env() -> Result<Option<usize>> {
match std::env::var("SECOND_BRAIN_RERANK_DOC_CHARS") {
Ok(v) => {
let n: usize = v
.parse()
.with_context(|| format!("parsing SECOND_BRAIN_RERANK_DOC_CHARS '{v}'"))?;
Ok(if n == 0 { None } else { Some(n) })
}
Err(_) => Ok(Some(DEFAULT_DOC_CHARS)),
}
}
fn checkout_timeout_from_env() -> Result<Duration> {
match std::env::var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") {
Ok(v) => {
let secs: u64 = v
.parse()
.with_context(|| format!("parsing SECOND_BRAIN_RERANK_CHECKOUT_SECS '{v}'"))?;
Ok(Duration::from_secs(secs))
}
Err(_) => Ok(DEFAULT_CHECKOUT_TIMEOUT),
}
}
#[allow(clippy::too_many_arguments)]
fn load_session(
backend: RerankBackend,
model_name: RerankerModel,
ep: RerankEp,
max_length: Option<usize>,
intra_threads: usize,
inter_threads: usize,
show_progress: bool,
) -> Result<RerankSession> {
match backend {
RerankBackend::FastEmbed => {
if ep == RerankEp::Cuda {
tracing::warn!(
"SECOND_BRAIN_RERANK_EP=cuda requested but unavailable on the fastembed \
rerank backend (ort is not a direct dependency along that path); falling \
back to cpu. Use the onnx backend (the default) for real GPU selection."
);
}
let mut opts = RerankInitOptions::new(model_name)
.with_cache_dir(model_cache_dir()?)
.with_show_download_progress(show_progress);
if let Some(len) = max_length {
opts = opts.with_max_length(len);
}
TextRerank::try_new(opts)
.map(RerankSession::FastEmbed)
.context("loading rerank model (fastembed backend)")
}
RerankBackend::Onnx => {
let config = OnnxSessionConfig {
intra_threads,
inter_threads,
opt_level: onnx_rerank::default_opt_level(),
ep,
max_length,
};
OnnxReranker::load(model_name, &config)
.map(RerankSession::Onnx)
.context("loading rerank model (onnx backend)")
}
}
}
fn truncate_chars(s: &str, max_chars: Option<usize>) -> String {
match max_chars {
Some(max) => s.chars().take(max).collect(),
None => s.to_string(),
}
}
pub fn sigmoid(score: f32) -> f32 {
1.0 / (1.0 + (-score).exp())
}
impl Reranker {
pub fn new(model_name: &str) -> Result<Self> {
Self::with_sessions(model_name, 1)
}
pub fn with_sessions(model_name: &str, n: usize) -> Result<Self> {
if n == 0 {
bail!("rerank pool size must be at least 1");
}
let parsed_model = parse_model(model_name)?;
let ep = ep_from_env()?;
let max_length = max_length_from_env()?;
let doc_chars = doc_chars_from_env()?;
let backend = backend_from_env();
let intra_threads = intra_threads_from_env(n)?;
let inter_threads = 1;
let pool = SessionPool::new(n, checkout_timeout_from_env()?);
for i in 0..n {
let mut slot = lock_or_recover(&pool.sessions[i]);
*slot = Some(
load_session(
backend,
parsed_model.clone(),
ep,
max_length,
intra_threads,
inter_threads,
true,
)
.with_context(|| format!("initializing rerank session {i} of {n}"))?,
);
pool.last_used_secs[i].store(now_secs(), Ordering::Relaxed);
}
Ok(Self {
pool,
model_name: parsed_model,
ep,
max_length,
doc_chars,
backend,
intra_threads,
inter_threads,
})
}
pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
self.pool.checkout_timeout = timeout;
self
}
pub fn pool_size(&self) -> usize {
self.pool.len()
}
pub fn metrics(&self) -> RerankMetricsSnapshot {
RerankMetricsSnapshot {
in_flight: self.pool.metrics.in_flight.load(Ordering::Relaxed),
total_checkouts: self.pool.metrics.total_checkouts.load(Ordering::Relaxed),
wait_p50: Duration::from_nanos(self.pool.metrics.p50_ns()),
wait_max: Duration::from_nanos(self.pool.metrics.wait_max_ns.load(Ordering::Relaxed)),
}
}
pub fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
if documents.is_empty() {
return Ok(Vec::new());
}
let truncated: Vec<String> = documents
.iter()
.map(|d| truncate_chars(d, self.doc_chars))
.collect();
let doc_refs: Vec<&str> = truncated.iter().map(String::as_str).collect();
let doc_count = doc_refs.len();
let model_name = self.model_name.clone();
let ep = self.ep;
let max_length = self.max_length;
let backend = self.backend;
let intra_threads = self.intra_threads;
let inter_threads = self.inter_threads;
let raw_scores: Vec<f32> = self.pool.checkout(
move || {
load_session(
backend,
model_name,
ep,
max_length,
intra_threads,
inter_threads,
false,
)
},
move |session| match session {
RerankSession::FastEmbed(session) => {
let results = session
.rerank(query, doc_refs, false, None)
.context("reranking (fastembed backend)")?;
let mut out = vec![0.0_f32; doc_count];
for r in results {
if let Some(slot) = out.get_mut(r.index) {
*slot = r.score;
}
}
Ok(out)
}
RerankSession::Onnx(session) => session
.rerank_batch(query, &doc_refs)
.context("reranking (onnx backend)"),
},
)?;
Ok(raw_scores.into_iter().map(sigmoid).collect())
}
pub fn drop_if_idle(&self, idle_secs: u64) {
self.pool.drop_if_idle(idle_secs);
}
}
impl RerankScorer for Reranker {
fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
Reranker::rerank(self, query, documents)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn model_names_parse_and_unknown_rejects() {
assert!(parse_model("bge-reranker-v2-m3").is_ok());
assert!(parse_model("jina-reranker-v1-turbo").is_ok());
assert!(parse_model("gpt-reranker").is_err());
}
#[test]
fn sigmoid_maps_logits_into_unit_interval() {
assert!((sigmoid(0.0) - 0.5).abs() < 1e-6);
assert!(sigmoid(8.0) > 0.99);
assert!(sigmoid(-8.0) < 0.01);
}
#[test]
fn ep_parsing_rejects_unknown_values_and_names_valid_ones() {
assert!(parse_ep("cpu").is_ok());
assert!(parse_ep("cuda").is_ok());
let err = parse_ep("tpu").unwrap_err();
assert!(err.to_string().contains("cpu"));
assert!(err.to_string().contains("cuda"));
}
#[test]
fn truncate_chars_never_splits_multibyte_utf8() {
let s = "a".repeat(9) + "\u{1F600}\u{1F600}\u{1F600}";
for max in 0..(s.chars().count() + 2) {
let t = truncate_chars(&s, Some(max));
assert!(std::str::from_utf8(t.as_bytes()).is_ok());
}
}
#[test]
fn intra_threads_defaults_to_cores_divided_by_pool_size() {
let cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
assert_eq!(intra_threads_from_env(1).unwrap(), cores.max(1));
assert_eq!(intra_threads_from_env(cores.max(2)).unwrap(), 1);
}
#[test]
fn doc_chars_from_env_defaults_to_1024_and_zero_opts_out_of_truncation() {
unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
assert_eq!(doc_chars_from_env().unwrap(), Some(DEFAULT_DOC_CHARS));
unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "0") };
assert_eq!(doc_chars_from_env().unwrap(), None);
unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "512") };
assert_eq!(doc_chars_from_env().unwrap(), Some(512));
unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
}
#[test]
fn checkout_timeout_from_env_defaults_to_30s_and_parses_override() {
unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
assert_eq!(
checkout_timeout_from_env().unwrap(),
DEFAULT_CHECKOUT_TIMEOUT
);
unsafe { std::env::set_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS", "5") };
assert_eq!(checkout_timeout_from_env().unwrap(), Duration::from_secs(5));
unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
}
}