Skip to main content

second_brain_core/retrieval/
rerank.rs

1use std::collections::VecDeque;
2use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
3use std::sync::{Condvar, Mutex, MutexGuard};
4use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
5
6use anyhow::{Context, Result, bail};
7use fastembed::{RerankInitOptions, RerankerModel, TextRerank};
8
9use super::onnx_rerank::{self, OnnxReranker, OnnxSessionConfig};
10
11const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
12const WAIT_SAMPLE_CAP: usize = 512;
13
14/// evals/rerank-cost-curve.md: at every fuse_top, 1024-char documents track
15/// untruncated cross-encoder scores within 1-2 points of MRR while being
16/// 15-60% faster, so this is the default budget rather than "no truncation".
17const DEFAULT_DOC_CHARS: usize = 1024;
18
19/// Cross-encoder reranker backed by a fixed pool of independent sessions.
20/// Both backends' underlying session types take `&mut self` to run
21/// inference, so serving concurrent queries means N loaded sessions rather
22/// than one behind a shared lock. Each slot keeps the original lazy-load/
23/// idle-unload lifecycle independently, so one query going cold does not
24/// force every other slot to reload.
25pub struct Reranker {
26    pool: SessionPool,
27    model_name: RerankerModel,
28    ep: RerankEp,
29    max_length: Option<usize>,
30    doc_chars: Option<usize>,
31    backend: RerankBackend,
32    intra_threads: usize,
33    inter_threads: usize,
34}
35
36/// One loaded rerank session. `fastembed::TextRerank::rerank` and
37/// `ort::session::Session::run` both require exclusive (`&mut`) access, so a
38/// pool slot holds exactly one of these at a time regardless of backend.
39enum RerankSession {
40    FastEmbed(TextRerank),
41    Onnx(OnnxReranker),
42}
43
44/// `fastembed::TextRerank` hardcodes its session's intra-op thread count to
45/// `available_parallelism()` with no override in its public API (see
46/// `fastembed-5.13.4/src/reranking/impl.rs`), so a pool of N fastembed
47/// sessions asks the OS for N times the machine's core count and the
48/// sessions fight each other instead of scaling. The `onnx` backend owns its
49/// `ort::session::Session` directly and caps `intra_threads` per the pool
50/// size instead. `fastembed` stays available as an escape hatch via
51/// `SECOND_BRAIN_RERANK_BACKEND=fastembed`.
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53enum RerankBackend {
54    Onnx,
55    FastEmbed,
56}
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub(crate) enum RerankEp {
60    Cpu,
61    Cuda,
62}
63
64/// Returned when every session in the pool is still busy past the checkout
65/// timeout. Backpressure that surfaces as an error beats a request that
66/// hangs forever. A `RerankScorer` that is not a local pool (a remote
67/// service shedding load or timing out) returns it too, with `pool_size` 0,
68/// because it is the one error `Retriever::rank` degrades on rather than
69/// propagating.
70#[derive(Debug)]
71pub struct RerankTimeoutError {
72    pub waited: Duration,
73    pub pool_size: usize,
74}
75
76impl std::fmt::Display for RerankTimeoutError {
77    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
78        write!(
79            f,
80            "timed out after {:?} waiting for an idle rerank session (pool size {})",
81            self.waited, self.pool_size
82        )
83    }
84}
85
86impl std::error::Error for RerankTimeoutError {}
87
88/// What `Retriever::rank` needs from a cross-encoder: one score per document,
89/// in document order, on the 0..1 scale `RetrievalTunables::rerank_gate` is
90/// calibrated against (the sigmoid of the raw logit, which is what `Reranker`
91/// returns). A scorer that cannot answer right now must return
92/// `RerankTimeoutError`, directly or under anyhow context, so the query falls
93/// back to the cosine-gated path instead of failing.
94pub trait RerankScorer: Send + Sync {
95    fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>>;
96}
97
98/// Snapshot of pool utilization for capacity planning.
99#[derive(Debug, Clone, Copy, Default)]
100pub struct RerankMetricsSnapshot {
101    pub in_flight: usize,
102    pub total_checkouts: u64,
103    pub wait_p50: Duration,
104    pub wait_max: Duration,
105}
106
107fn now_secs() -> u64 {
108    SystemTime::now()
109        .duration_since(UNIX_EPOCH)
110        .unwrap_or_default()
111        .as_secs()
112}
113
114fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
115    m.lock().unwrap_or_else(|e| e.into_inner())
116}
117
118#[derive(Default)]
119struct PoolMetrics {
120    in_flight: AtomicUsize,
121    total_checkouts: AtomicU64,
122    wait_max_ns: AtomicU64,
123    // Bounded ring buffer rather than a full histogram crate: cheap enough
124    // to sample on every checkout, and a sorted snapshot gives an exact
125    // median over the recent window, which is all capacity planning needs.
126    wait_samples: Mutex<VecDeque<u64>>,
127}
128
129impl PoolMetrics {
130    fn record_wait(&self, ns: u64) {
131        self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
132        let mut samples = lock_or_recover(&self.wait_samples);
133        if samples.len() >= WAIT_SAMPLE_CAP {
134            samples.pop_front();
135        }
136        samples.push_back(ns);
137    }
138
139    fn p50_ns(&self) -> u64 {
140        let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
141            .iter()
142            .copied()
143            .collect();
144        if samples.is_empty() {
145            return 0;
146        }
147        samples.sort_unstable();
148        samples[samples.len() / 2]
149    }
150}
151
152/// Fixed-size pool of lazily-loaded, independently idle-unloadable sessions.
153/// Checkout pulls an index off a free list guarded by a condvar, so waiters
154/// are woken in order instead of spin-polling N mutexes with `try_lock`.
155struct SessionPool {
156    sessions: Vec<Mutex<Option<RerankSession>>>,
157    last_used_secs: Vec<AtomicU64>,
158    free: Mutex<VecDeque<usize>>,
159    free_cv: Condvar,
160    checkout_timeout: Duration,
161    metrics: PoolMetrics,
162}
163
164/// Returns the checked-out slot to the free list on drop, including on a
165/// panic unwind, so a failure inside one rerank call cannot permanently
166/// shrink the pool.
167struct Checkout<'a> {
168    pool: &'a SessionPool,
169    idx: usize,
170}
171
172impl Drop for Checkout<'_> {
173    fn drop(&mut self) {
174        self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
175        lock_or_recover(&self.pool.free).push_back(self.idx);
176        self.pool.free_cv.notify_one();
177    }
178}
179
180impl SessionPool {
181    fn new(n: usize, checkout_timeout: Duration) -> Self {
182        let mut sessions = Vec::with_capacity(n);
183        let mut last_used_secs = Vec::with_capacity(n);
184        let mut free = VecDeque::with_capacity(n);
185        for i in 0..n {
186            sessions.push(Mutex::new(None));
187            last_used_secs.push(AtomicU64::new(0));
188            free.push_back(i);
189        }
190        Self {
191            sessions,
192            last_used_secs,
193            free: Mutex::new(free),
194            free_cv: Condvar::new(),
195            checkout_timeout,
196            metrics: PoolMetrics::default(),
197        }
198    }
199
200    fn len(&self) -> usize {
201        self.sessions.len()
202    }
203
204    fn checkout<R>(
205        &self,
206        load: impl FnOnce() -> Result<RerankSession>,
207        with_session: impl FnOnce(&mut RerankSession) -> Result<R>,
208    ) -> Result<R> {
209        let start = Instant::now();
210        let guard = lock_or_recover(&self.free);
211        let (mut guard, wait_result) = self
212            .free_cv
213            .wait_timeout_while(guard, self.checkout_timeout, |q| q.is_empty())
214            .unwrap_or_else(|e| e.into_inner());
215        if wait_result.timed_out() && guard.is_empty() {
216            return Err(RerankTimeoutError {
217                waited: start.elapsed(),
218                pool_size: self.len(),
219            }
220            .into());
221        }
222        let idx = guard.pop_front().expect("free slot present after wait");
223        drop(guard);
224
225        self.metrics.record_wait(start.elapsed().as_nanos() as u64);
226        self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
227        self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
228        let _checkout = Checkout { pool: self, idx };
229
230        let mut slot = lock_or_recover(&self.sessions[idx]);
231        if slot.is_none() {
232            tracing::info!("reloading rerank model after idle drop (slot {idx})");
233            *slot = Some(load()?);
234        }
235        self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
236        with_session(slot.as_mut().expect("session loaded above"))
237    }
238
239    fn drop_if_idle(&self, idle_secs: u64) {
240        let now = now_secs();
241        for (idx, last) in self.last_used_secs.iter().enumerate() {
242            if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
243                continue;
244            }
245            // try_lock, not lock: a busy slot is by definition not idle, and
246            // this must never block behind an in-flight rerank.
247            if let Ok(mut slot) = self.sessions[idx].try_lock()
248                && slot.is_some()
249            {
250                *slot = None;
251                tracing::info!("rerank session {idx} dropped after {idle_secs}s idle");
252            }
253        }
254    }
255}
256
257pub fn parse_model(name: &str) -> Result<RerankerModel> {
258    match name {
259        "bge-reranker-v2-m3" => Ok(RerankerModel::BGERerankerV2M3),
260        "jina-reranker-v1-turbo" => Ok(RerankerModel::JINARerankerV1TurboEn),
261        other => {
262            bail!("unknown rerank model '{other}' (bge-reranker-v2-m3, jina-reranker-v1-turbo)")
263        }
264    }
265}
266
267fn parse_ep(raw: &str) -> Result<RerankEp> {
268    match raw {
269        "cpu" => Ok(RerankEp::Cpu),
270        "cuda" => Ok(RerankEp::Cuda),
271        other => bail!("unknown SECOND_BRAIN_RERANK_EP '{other}' (valid values: cpu, cuda)"),
272    }
273}
274
275pub(crate) fn ep_from_env() -> Result<RerankEp> {
276    match std::env::var("SECOND_BRAIN_RERANK_EP") {
277        Ok(v) => parse_ep(&v),
278        Err(_) => Ok(RerankEp::Cpu),
279    }
280}
281
282fn backend_from_env() -> RerankBackend {
283    match std::env::var("SECOND_BRAIN_RERANK_BACKEND") {
284        Ok(v) if v == "fastembed" => RerankBackend::FastEmbed,
285        _ => RerankBackend::Onnx,
286    }
287}
288
289/// Defaults to `cores / pool_size` (floor, minimum 1) so a full pool asks
290/// the OS for roughly one core's worth of intra-op threads per in-flight
291/// session instead of oversubscribing by `pool_size`x, which is the exact
292/// bottleneck this backend exists to avoid.
293fn intra_threads_from_env(pool_size: usize) -> Result<usize> {
294    match std::env::var("SECOND_BRAIN_RERANK_INTRA_THREADS") {
295        Ok(v) => {
296            let n: usize = v
297                .parse()
298                .with_context(|| format!("parsing SECOND_BRAIN_RERANK_INTRA_THREADS '{v}'"))?;
299            if n == 0 {
300                bail!("SECOND_BRAIN_RERANK_INTRA_THREADS must be at least 1");
301            }
302            Ok(n)
303        }
304        Err(_) => {
305            let cores = std::thread::available_parallelism()
306                .map(|n| n.get())
307                .unwrap_or(1);
308            Ok((cores / pool_size.max(1)).max(1))
309        }
310    }
311}
312
313fn max_length_from_env() -> Result<Option<usize>> {
314    match std::env::var("SECOND_BRAIN_RERANK_MAX_LEN") {
315        Ok(v) => Ok(Some(v.parse::<usize>().with_context(|| {
316            format!("parsing SECOND_BRAIN_RERANK_MAX_LEN '{v}'")
317        })?)),
318        Err(_) => Ok(None),
319    }
320}
321
322/// `0` is the escape hatch back to unlimited document length: a document
323/// budget of zero characters would score nothing, so that value is free to
324/// repurpose as "truncation off" rather than needing a second env var.
325fn doc_chars_from_env() -> Result<Option<usize>> {
326    match std::env::var("SECOND_BRAIN_RERANK_DOC_CHARS") {
327        Ok(v) => {
328            let n: usize = v
329                .parse()
330                .with_context(|| format!("parsing SECOND_BRAIN_RERANK_DOC_CHARS '{v}'"))?;
331            Ok(if n == 0 { None } else { Some(n) })
332        }
333        Err(_) => Ok(Some(DEFAULT_DOC_CHARS)),
334    }
335}
336
337fn checkout_timeout_from_env() -> Result<Duration> {
338    match std::env::var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") {
339        Ok(v) => {
340            let secs: u64 = v
341                .parse()
342                .with_context(|| format!("parsing SECOND_BRAIN_RERANK_CHECKOUT_SECS '{v}'"))?;
343            Ok(Duration::from_secs(secs))
344        }
345        Err(_) => Ok(DEFAULT_CHECKOUT_TIMEOUT),
346    }
347}
348
349/// Builds one session on the selected backend.
350///
351/// The `fastembed` backend cannot select a concrete execution provider: it
352/// is not a direct dependency of second-brain-core, and fastembed 5.13.4
353/// re-exports only the opaque `ExecutionProviderDispatch`, not a concrete
354/// provider type such as `CUDAExecutionProvider` — so there is no way from
355/// this crate to construct a value for
356/// `RerankInitOptions::with_execution_providers`. Requesting cuda on that
357/// path therefore always falls back to cpu, loudly. The `onnx` backend (the
358/// default) owns its `ort::Session` directly and can select CUDA for real;
359/// see `onnx_rerank::execution_providers`.
360#[allow(clippy::too_many_arguments)]
361fn load_session(
362    backend: RerankBackend,
363    model_name: RerankerModel,
364    ep: RerankEp,
365    max_length: Option<usize>,
366    intra_threads: usize,
367    inter_threads: usize,
368    show_progress: bool,
369) -> Result<RerankSession> {
370    match backend {
371        RerankBackend::FastEmbed => {
372            if ep == RerankEp::Cuda {
373                tracing::warn!(
374                    "SECOND_BRAIN_RERANK_EP=cuda requested but unavailable on the fastembed \
375                     rerank backend (ort is not a direct dependency along that path); falling \
376                     back to cpu. Use the onnx backend (the default) for real GPU selection."
377                );
378            }
379            let mut opts =
380                RerankInitOptions::new(model_name).with_show_download_progress(show_progress);
381            if let Some(len) = max_length {
382                opts = opts.with_max_length(len);
383            }
384            TextRerank::try_new(opts)
385                .map(RerankSession::FastEmbed)
386                .context("loading rerank model (fastembed backend)")
387        }
388        RerankBackend::Onnx => {
389            let config = OnnxSessionConfig {
390                intra_threads,
391                inter_threads,
392                opt_level: onnx_rerank::default_opt_level(),
393                ep,
394                max_length,
395            };
396            OnnxReranker::load(model_name, &config)
397                .map(RerankSession::Onnx)
398                .context("loading rerank model (onnx backend)")
399        }
400    }
401}
402
403/// Truncation happens exactly once, here, so a document scores identically
404/// no matter how long its untruncated form is. Character-based, not
405/// byte-based, so it can never split a multi-byte UTF-8 sequence.
406fn truncate_chars(s: &str, max_chars: Option<usize>) -> String {
407    match max_chars {
408        Some(max) => s.chars().take(max).collect(),
409        None => s.to_string(),
410    }
411}
412
413/// Cross-encoder raw scores are unbounded logits; the blend and the 0.30 gate
414/// both assume a 0..1 primary term, so scores pass through a sigmoid.
415pub fn sigmoid(score: f32) -> f32 {
416    1.0 / (1.0 + (-score).exp())
417}
418
419impl Reranker {
420    pub fn new(model_name: &str) -> Result<Self> {
421        Self::with_sessions(model_name, 1)
422    }
423
424    /// Builds a pool of `n` independent sessions, eager-loaded up front so
425    /// construction fails fast exactly like the single-session constructor
426    /// always has.
427    pub fn with_sessions(model_name: &str, n: usize) -> Result<Self> {
428        if n == 0 {
429            bail!("rerank pool size must be at least 1");
430        }
431        let parsed_model = parse_model(model_name)?;
432        let ep = ep_from_env()?;
433        let max_length = max_length_from_env()?;
434        let doc_chars = doc_chars_from_env()?;
435        let backend = backend_from_env();
436        let intra_threads = intra_threads_from_env(n)?;
437        let inter_threads = 1;
438
439        let pool = SessionPool::new(n, checkout_timeout_from_env()?);
440        for i in 0..n {
441            let mut slot = lock_or_recover(&pool.sessions[i]);
442            *slot = Some(
443                load_session(
444                    backend,
445                    parsed_model.clone(),
446                    ep,
447                    max_length,
448                    intra_threads,
449                    inter_threads,
450                    true,
451                )
452                .with_context(|| format!("initializing rerank session {i} of {n}"))?,
453            );
454            pool.last_used_secs[i].store(now_secs(), Ordering::Relaxed);
455        }
456
457        Ok(Self {
458            pool,
459            model_name: parsed_model,
460            ep,
461            max_length,
462            doc_chars,
463            backend,
464            intra_threads,
465            inter_threads,
466        })
467    }
468
469    /// Overrides the default checkout wait bound. Mainly useful for tests
470    /// that need to observe the timeout path without a long-running test.
471    pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
472        self.pool.checkout_timeout = timeout;
473        self
474    }
475
476    pub fn pool_size(&self) -> usize {
477        self.pool.len()
478    }
479
480    pub fn metrics(&self) -> RerankMetricsSnapshot {
481        RerankMetricsSnapshot {
482            in_flight: self.pool.metrics.in_flight.load(Ordering::Relaxed),
483            total_checkouts: self.pool.metrics.total_checkouts.load(Ordering::Relaxed),
484            wait_p50: Duration::from_nanos(self.pool.metrics.p50_ns()),
485            wait_max: Duration::from_nanos(self.pool.metrics.wait_max_ns.load(Ordering::Relaxed)),
486        }
487    }
488
489    /// Sigmoid-squashed relevance scores in the same order as `documents`.
490    pub fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
491        if documents.is_empty() {
492            return Ok(Vec::new());
493        }
494        let truncated: Vec<String> = documents
495            .iter()
496            .map(|d| truncate_chars(d, self.doc_chars))
497            .collect();
498        let doc_refs: Vec<&str> = truncated.iter().map(String::as_str).collect();
499        let doc_count = doc_refs.len();
500
501        let model_name = self.model_name.clone();
502        let ep = self.ep;
503        let max_length = self.max_length;
504        let backend = self.backend;
505        let intra_threads = self.intra_threads;
506        let inter_threads = self.inter_threads;
507
508        let raw_scores: Vec<f32> = self.pool.checkout(
509            move || {
510                load_session(
511                    backend,
512                    model_name,
513                    ep,
514                    max_length,
515                    intra_threads,
516                    inter_threads,
517                    false,
518                )
519            },
520            move |session| match session {
521                RerankSession::FastEmbed(session) => {
522                    let results = session
523                        .rerank(query, doc_refs, false, None)
524                        .context("reranking (fastembed backend)")?;
525                    let mut out = vec![0.0_f32; doc_count];
526                    for r in results {
527                        if let Some(slot) = out.get_mut(r.index) {
528                            *slot = r.score;
529                        }
530                    }
531                    Ok(out)
532                }
533                RerankSession::Onnx(session) => session
534                    .rerank_batch(query, &doc_refs)
535                    .context("reranking (onnx backend)"),
536            },
537        )?;
538
539        Ok(raw_scores.into_iter().map(sigmoid).collect())
540    }
541
542    pub fn drop_if_idle(&self, idle_secs: u64) {
543        self.pool.drop_if_idle(idle_secs);
544    }
545}
546
547impl RerankScorer for Reranker {
548    fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
549        Reranker::rerank(self, query, documents)
550    }
551}
552
553#[cfg(test)]
554mod tests {
555    use super::*;
556
557    #[test]
558    fn model_names_parse_and_unknown_rejects() {
559        assert!(parse_model("bge-reranker-v2-m3").is_ok());
560        assert!(parse_model("jina-reranker-v1-turbo").is_ok());
561        assert!(parse_model("gpt-reranker").is_err());
562    }
563
564    #[test]
565    fn sigmoid_maps_logits_into_unit_interval() {
566        assert!((sigmoid(0.0) - 0.5).abs() < 1e-6);
567        assert!(sigmoid(8.0) > 0.99);
568        assert!(sigmoid(-8.0) < 0.01);
569    }
570
571    #[test]
572    fn ep_parsing_rejects_unknown_values_and_names_valid_ones() {
573        assert!(parse_ep("cpu").is_ok());
574        assert!(parse_ep("cuda").is_ok());
575        let err = parse_ep("tpu").unwrap_err();
576        assert!(err.to_string().contains("cpu"));
577        assert!(err.to_string().contains("cuda"));
578    }
579
580    #[test]
581    fn truncate_chars_never_splits_multibyte_utf8() {
582        let s = "a".repeat(9) + "\u{1F600}\u{1F600}\u{1F600}";
583        for max in 0..(s.chars().count() + 2) {
584            let t = truncate_chars(&s, Some(max));
585            assert!(std::str::from_utf8(t.as_bytes()).is_ok());
586        }
587    }
588
589    #[test]
590    fn intra_threads_defaults_to_cores_divided_by_pool_size() {
591        let cores = std::thread::available_parallelism()
592            .map(|n| n.get())
593            .unwrap_or(1);
594        assert_eq!(intra_threads_from_env(1).unwrap(), cores.max(1));
595        assert_eq!(intra_threads_from_env(cores.max(2)).unwrap(), 1);
596    }
597
598    #[test]
599    fn doc_chars_from_env_defaults_to_1024_and_zero_opts_out_of_truncation() {
600        // SAFETY: this test owns SECOND_BRAIN_RERANK_DOC_CHARS exclusively;
601        // no other test in this binary reads or writes it.
602        unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
603        assert_eq!(doc_chars_from_env().unwrap(), Some(DEFAULT_DOC_CHARS));
604
605        unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "0") };
606        assert_eq!(doc_chars_from_env().unwrap(), None);
607
608        unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "512") };
609        assert_eq!(doc_chars_from_env().unwrap(), Some(512));
610
611        unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
612    }
613
614    #[test]
615    fn checkout_timeout_from_env_defaults_to_30s_and_parses_override() {
616        // SAFETY: this test owns SECOND_BRAIN_RERANK_CHECKOUT_SECS
617        // exclusively; no other test in this binary reads or writes it.
618        unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
619        assert_eq!(
620            checkout_timeout_from_env().unwrap(),
621            DEFAULT_CHECKOUT_TIMEOUT
622        );
623
624        unsafe { std::env::set_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS", "5") };
625        assert_eq!(checkout_timeout_from_env().unwrap(), Duration::from_secs(5));
626
627        unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
628    }
629}