1use std::collections::VecDeque;
2use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
3use std::sync::{Condvar, Mutex, MutexGuard};
4use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
5
6use anyhow::{Context, Result, bail};
7use fastembed::{RerankInitOptions, RerankerModel, TextRerank};
8
9use super::onnx_rerank::{self, OnnxReranker, OnnxSessionConfig};
10
11const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
12const WAIT_SAMPLE_CAP: usize = 512;
13
14const DEFAULT_DOC_CHARS: usize = 1024;
18
19pub struct Reranker {
26 pool: SessionPool,
27 model_name: RerankerModel,
28 ep: RerankEp,
29 max_length: Option<usize>,
30 doc_chars: Option<usize>,
31 backend: RerankBackend,
32 intra_threads: usize,
33 inter_threads: usize,
34}
35
36enum RerankSession {
40 FastEmbed(TextRerank),
41 Onnx(OnnxReranker),
42}
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53enum RerankBackend {
54 Onnx,
55 FastEmbed,
56}
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub(crate) enum RerankEp {
60 Cpu,
61 Cuda,
62}
63
64#[derive(Debug)]
71pub struct RerankTimeoutError {
72 pub waited: Duration,
73 pub pool_size: usize,
74}
75
76impl std::fmt::Display for RerankTimeoutError {
77 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
78 write!(
79 f,
80 "timed out after {:?} waiting for an idle rerank session (pool size {})",
81 self.waited, self.pool_size
82 )
83 }
84}
85
86impl std::error::Error for RerankTimeoutError {}
87
88pub trait RerankScorer: Send + Sync {
95 fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>>;
96}
97
98#[derive(Debug, Clone, Copy, Default)]
100pub struct RerankMetricsSnapshot {
101 pub in_flight: usize,
102 pub total_checkouts: u64,
103 pub wait_p50: Duration,
104 pub wait_max: Duration,
105}
106
107fn now_secs() -> u64 {
108 SystemTime::now()
109 .duration_since(UNIX_EPOCH)
110 .unwrap_or_default()
111 .as_secs()
112}
113
114fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
115 m.lock().unwrap_or_else(|e| e.into_inner())
116}
117
118#[derive(Default)]
119struct PoolMetrics {
120 in_flight: AtomicUsize,
121 total_checkouts: AtomicU64,
122 wait_max_ns: AtomicU64,
123 wait_samples: Mutex<VecDeque<u64>>,
127}
128
129impl PoolMetrics {
130 fn record_wait(&self, ns: u64) {
131 self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
132 let mut samples = lock_or_recover(&self.wait_samples);
133 if samples.len() >= WAIT_SAMPLE_CAP {
134 samples.pop_front();
135 }
136 samples.push_back(ns);
137 }
138
139 fn p50_ns(&self) -> u64 {
140 let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
141 .iter()
142 .copied()
143 .collect();
144 if samples.is_empty() {
145 return 0;
146 }
147 samples.sort_unstable();
148 samples[samples.len() / 2]
149 }
150}
151
152struct SessionPool {
156 sessions: Vec<Mutex<Option<RerankSession>>>,
157 last_used_secs: Vec<AtomicU64>,
158 free: Mutex<VecDeque<usize>>,
159 free_cv: Condvar,
160 checkout_timeout: Duration,
161 metrics: PoolMetrics,
162}
163
164struct Checkout<'a> {
168 pool: &'a SessionPool,
169 idx: usize,
170}
171
172impl Drop for Checkout<'_> {
173 fn drop(&mut self) {
174 self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
175 lock_or_recover(&self.pool.free).push_back(self.idx);
176 self.pool.free_cv.notify_one();
177 }
178}
179
180impl SessionPool {
181 fn new(n: usize, checkout_timeout: Duration) -> Self {
182 let mut sessions = Vec::with_capacity(n);
183 let mut last_used_secs = Vec::with_capacity(n);
184 let mut free = VecDeque::with_capacity(n);
185 for i in 0..n {
186 sessions.push(Mutex::new(None));
187 last_used_secs.push(AtomicU64::new(0));
188 free.push_back(i);
189 }
190 Self {
191 sessions,
192 last_used_secs,
193 free: Mutex::new(free),
194 free_cv: Condvar::new(),
195 checkout_timeout,
196 metrics: PoolMetrics::default(),
197 }
198 }
199
200 fn len(&self) -> usize {
201 self.sessions.len()
202 }
203
204 fn checkout<R>(
205 &self,
206 load: impl FnOnce() -> Result<RerankSession>,
207 with_session: impl FnOnce(&mut RerankSession) -> Result<R>,
208 ) -> Result<R> {
209 let start = Instant::now();
210 let guard = lock_or_recover(&self.free);
211 let (mut guard, wait_result) = self
212 .free_cv
213 .wait_timeout_while(guard, self.checkout_timeout, |q| q.is_empty())
214 .unwrap_or_else(|e| e.into_inner());
215 if wait_result.timed_out() && guard.is_empty() {
216 return Err(RerankTimeoutError {
217 waited: start.elapsed(),
218 pool_size: self.len(),
219 }
220 .into());
221 }
222 let idx = guard.pop_front().expect("free slot present after wait");
223 drop(guard);
224
225 self.metrics.record_wait(start.elapsed().as_nanos() as u64);
226 self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
227 self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
228 let _checkout = Checkout { pool: self, idx };
229
230 let mut slot = lock_or_recover(&self.sessions[idx]);
231 if slot.is_none() {
232 tracing::info!("reloading rerank model after idle drop (slot {idx})");
233 *slot = Some(load()?);
234 }
235 self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
236 with_session(slot.as_mut().expect("session loaded above"))
237 }
238
239 fn drop_if_idle(&self, idle_secs: u64) {
240 let now = now_secs();
241 for (idx, last) in self.last_used_secs.iter().enumerate() {
242 if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
243 continue;
244 }
245 if let Ok(mut slot) = self.sessions[idx].try_lock()
248 && slot.is_some()
249 {
250 *slot = None;
251 tracing::info!("rerank session {idx} dropped after {idle_secs}s idle");
252 }
253 }
254 }
255}
256
257pub fn parse_model(name: &str) -> Result<RerankerModel> {
258 match name {
259 "bge-reranker-v2-m3" => Ok(RerankerModel::BGERerankerV2M3),
260 "jina-reranker-v1-turbo" => Ok(RerankerModel::JINARerankerV1TurboEn),
261 other => {
262 bail!("unknown rerank model '{other}' (bge-reranker-v2-m3, jina-reranker-v1-turbo)")
263 }
264 }
265}
266
267fn parse_ep(raw: &str) -> Result<RerankEp> {
268 match raw {
269 "cpu" => Ok(RerankEp::Cpu),
270 "cuda" => Ok(RerankEp::Cuda),
271 other => bail!("unknown SECOND_BRAIN_RERANK_EP '{other}' (valid values: cpu, cuda)"),
272 }
273}
274
275pub(crate) fn ep_from_env() -> Result<RerankEp> {
276 match std::env::var("SECOND_BRAIN_RERANK_EP") {
277 Ok(v) => parse_ep(&v),
278 Err(_) => Ok(RerankEp::Cpu),
279 }
280}
281
282fn backend_from_env() -> RerankBackend {
283 match std::env::var("SECOND_BRAIN_RERANK_BACKEND") {
284 Ok(v) if v == "fastembed" => RerankBackend::FastEmbed,
285 _ => RerankBackend::Onnx,
286 }
287}
288
289fn intra_threads_from_env(pool_size: usize) -> Result<usize> {
294 match std::env::var("SECOND_BRAIN_RERANK_INTRA_THREADS") {
295 Ok(v) => {
296 let n: usize = v
297 .parse()
298 .with_context(|| format!("parsing SECOND_BRAIN_RERANK_INTRA_THREADS '{v}'"))?;
299 if n == 0 {
300 bail!("SECOND_BRAIN_RERANK_INTRA_THREADS must be at least 1");
301 }
302 Ok(n)
303 }
304 Err(_) => {
305 let cores = std::thread::available_parallelism()
306 .map(|n| n.get())
307 .unwrap_or(1);
308 Ok((cores / pool_size.max(1)).max(1))
309 }
310 }
311}
312
313fn max_length_from_env() -> Result<Option<usize>> {
314 match std::env::var("SECOND_BRAIN_RERANK_MAX_LEN") {
315 Ok(v) => Ok(Some(v.parse::<usize>().with_context(|| {
316 format!("parsing SECOND_BRAIN_RERANK_MAX_LEN '{v}'")
317 })?)),
318 Err(_) => Ok(None),
319 }
320}
321
322fn doc_chars_from_env() -> Result<Option<usize>> {
326 match std::env::var("SECOND_BRAIN_RERANK_DOC_CHARS") {
327 Ok(v) => {
328 let n: usize = v
329 .parse()
330 .with_context(|| format!("parsing SECOND_BRAIN_RERANK_DOC_CHARS '{v}'"))?;
331 Ok(if n == 0 { None } else { Some(n) })
332 }
333 Err(_) => Ok(Some(DEFAULT_DOC_CHARS)),
334 }
335}
336
337fn checkout_timeout_from_env() -> Result<Duration> {
338 match std::env::var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") {
339 Ok(v) => {
340 let secs: u64 = v
341 .parse()
342 .with_context(|| format!("parsing SECOND_BRAIN_RERANK_CHECKOUT_SECS '{v}'"))?;
343 Ok(Duration::from_secs(secs))
344 }
345 Err(_) => Ok(DEFAULT_CHECKOUT_TIMEOUT),
346 }
347}
348
349#[allow(clippy::too_many_arguments)]
361fn load_session(
362 backend: RerankBackend,
363 model_name: RerankerModel,
364 ep: RerankEp,
365 max_length: Option<usize>,
366 intra_threads: usize,
367 inter_threads: usize,
368 show_progress: bool,
369) -> Result<RerankSession> {
370 match backend {
371 RerankBackend::FastEmbed => {
372 if ep == RerankEp::Cuda {
373 tracing::warn!(
374 "SECOND_BRAIN_RERANK_EP=cuda requested but unavailable on the fastembed \
375 rerank backend (ort is not a direct dependency along that path); falling \
376 back to cpu. Use the onnx backend (the default) for real GPU selection."
377 );
378 }
379 let mut opts =
380 RerankInitOptions::new(model_name).with_show_download_progress(show_progress);
381 if let Some(len) = max_length {
382 opts = opts.with_max_length(len);
383 }
384 TextRerank::try_new(opts)
385 .map(RerankSession::FastEmbed)
386 .context("loading rerank model (fastembed backend)")
387 }
388 RerankBackend::Onnx => {
389 let config = OnnxSessionConfig {
390 intra_threads,
391 inter_threads,
392 opt_level: onnx_rerank::default_opt_level(),
393 ep,
394 max_length,
395 };
396 OnnxReranker::load(model_name, &config)
397 .map(RerankSession::Onnx)
398 .context("loading rerank model (onnx backend)")
399 }
400 }
401}
402
403fn truncate_chars(s: &str, max_chars: Option<usize>) -> String {
407 match max_chars {
408 Some(max) => s.chars().take(max).collect(),
409 None => s.to_string(),
410 }
411}
412
413pub fn sigmoid(score: f32) -> f32 {
416 1.0 / (1.0 + (-score).exp())
417}
418
419impl Reranker {
420 pub fn new(model_name: &str) -> Result<Self> {
421 Self::with_sessions(model_name, 1)
422 }
423
424 pub fn with_sessions(model_name: &str, n: usize) -> Result<Self> {
428 if n == 0 {
429 bail!("rerank pool size must be at least 1");
430 }
431 let parsed_model = parse_model(model_name)?;
432 let ep = ep_from_env()?;
433 let max_length = max_length_from_env()?;
434 let doc_chars = doc_chars_from_env()?;
435 let backend = backend_from_env();
436 let intra_threads = intra_threads_from_env(n)?;
437 let inter_threads = 1;
438
439 let pool = SessionPool::new(n, checkout_timeout_from_env()?);
440 for i in 0..n {
441 let mut slot = lock_or_recover(&pool.sessions[i]);
442 *slot = Some(
443 load_session(
444 backend,
445 parsed_model.clone(),
446 ep,
447 max_length,
448 intra_threads,
449 inter_threads,
450 true,
451 )
452 .with_context(|| format!("initializing rerank session {i} of {n}"))?,
453 );
454 pool.last_used_secs[i].store(now_secs(), Ordering::Relaxed);
455 }
456
457 Ok(Self {
458 pool,
459 model_name: parsed_model,
460 ep,
461 max_length,
462 doc_chars,
463 backend,
464 intra_threads,
465 inter_threads,
466 })
467 }
468
469 pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
472 self.pool.checkout_timeout = timeout;
473 self
474 }
475
476 pub fn pool_size(&self) -> usize {
477 self.pool.len()
478 }
479
480 pub fn metrics(&self) -> RerankMetricsSnapshot {
481 RerankMetricsSnapshot {
482 in_flight: self.pool.metrics.in_flight.load(Ordering::Relaxed),
483 total_checkouts: self.pool.metrics.total_checkouts.load(Ordering::Relaxed),
484 wait_p50: Duration::from_nanos(self.pool.metrics.p50_ns()),
485 wait_max: Duration::from_nanos(self.pool.metrics.wait_max_ns.load(Ordering::Relaxed)),
486 }
487 }
488
489 pub fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
491 if documents.is_empty() {
492 return Ok(Vec::new());
493 }
494 let truncated: Vec<String> = documents
495 .iter()
496 .map(|d| truncate_chars(d, self.doc_chars))
497 .collect();
498 let doc_refs: Vec<&str> = truncated.iter().map(String::as_str).collect();
499 let doc_count = doc_refs.len();
500
501 let model_name = self.model_name.clone();
502 let ep = self.ep;
503 let max_length = self.max_length;
504 let backend = self.backend;
505 let intra_threads = self.intra_threads;
506 let inter_threads = self.inter_threads;
507
508 let raw_scores: Vec<f32> = self.pool.checkout(
509 move || {
510 load_session(
511 backend,
512 model_name,
513 ep,
514 max_length,
515 intra_threads,
516 inter_threads,
517 false,
518 )
519 },
520 move |session| match session {
521 RerankSession::FastEmbed(session) => {
522 let results = session
523 .rerank(query, doc_refs, false, None)
524 .context("reranking (fastembed backend)")?;
525 let mut out = vec![0.0_f32; doc_count];
526 for r in results {
527 if let Some(slot) = out.get_mut(r.index) {
528 *slot = r.score;
529 }
530 }
531 Ok(out)
532 }
533 RerankSession::Onnx(session) => session
534 .rerank_batch(query, &doc_refs)
535 .context("reranking (onnx backend)"),
536 },
537 )?;
538
539 Ok(raw_scores.into_iter().map(sigmoid).collect())
540 }
541
542 pub fn drop_if_idle(&self, idle_secs: u64) {
543 self.pool.drop_if_idle(idle_secs);
544 }
545}
546
547impl RerankScorer for Reranker {
548 fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
549 Reranker::rerank(self, query, documents)
550 }
551}
552
553#[cfg(test)]
554mod tests {
555 use super::*;
556
557 #[test]
558 fn model_names_parse_and_unknown_rejects() {
559 assert!(parse_model("bge-reranker-v2-m3").is_ok());
560 assert!(parse_model("jina-reranker-v1-turbo").is_ok());
561 assert!(parse_model("gpt-reranker").is_err());
562 }
563
564 #[test]
565 fn sigmoid_maps_logits_into_unit_interval() {
566 assert!((sigmoid(0.0) - 0.5).abs() < 1e-6);
567 assert!(sigmoid(8.0) > 0.99);
568 assert!(sigmoid(-8.0) < 0.01);
569 }
570
571 #[test]
572 fn ep_parsing_rejects_unknown_values_and_names_valid_ones() {
573 assert!(parse_ep("cpu").is_ok());
574 assert!(parse_ep("cuda").is_ok());
575 let err = parse_ep("tpu").unwrap_err();
576 assert!(err.to_string().contains("cpu"));
577 assert!(err.to_string().contains("cuda"));
578 }
579
580 #[test]
581 fn truncate_chars_never_splits_multibyte_utf8() {
582 let s = "a".repeat(9) + "\u{1F600}\u{1F600}\u{1F600}";
583 for max in 0..(s.chars().count() + 2) {
584 let t = truncate_chars(&s, Some(max));
585 assert!(std::str::from_utf8(t.as_bytes()).is_ok());
586 }
587 }
588
589 #[test]
590 fn intra_threads_defaults_to_cores_divided_by_pool_size() {
591 let cores = std::thread::available_parallelism()
592 .map(|n| n.get())
593 .unwrap_or(1);
594 assert_eq!(intra_threads_from_env(1).unwrap(), cores.max(1));
595 assert_eq!(intra_threads_from_env(cores.max(2)).unwrap(), 1);
596 }
597
598 #[test]
599 fn doc_chars_from_env_defaults_to_1024_and_zero_opts_out_of_truncation() {
600 unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
603 assert_eq!(doc_chars_from_env().unwrap(), Some(DEFAULT_DOC_CHARS));
604
605 unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "0") };
606 assert_eq!(doc_chars_from_env().unwrap(), None);
607
608 unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "512") };
609 assert_eq!(doc_chars_from_env().unwrap(), Some(512));
610
611 unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
612 }
613
614 #[test]
615 fn checkout_timeout_from_env_defaults_to_30s_and_parses_override() {
616 unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
619 assert_eq!(
620 checkout_timeout_from_env().unwrap(),
621 DEFAULT_CHECKOUT_TIMEOUT
622 );
623
624 unsafe { std::env::set_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS", "5") };
625 assert_eq!(checkout_timeout_from_env().unwrap(), Duration::from_secs(5));
626
627 unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
628 }
629}