Skip to main content

second_brain_core/retrieval/
rerank.rs

1use std::sync::Arc;
2use std::time::Duration;
3
4use anyhow::{Context, Result, bail};
5use fastembed::{RerankInitOptions, RerankerModel, TextRerank};
6
7use super::onnx_rerank::{self, OnnxReranker, OnnxSessionConfig};
8use crate::model_cache::model_cache_dir;
9use crate::session_pool::SessionPool;
10
11const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
12
13/// evals/rerank-cost-curve.md: at every fuse_top, 1024-char documents track
14/// untruncated cross-encoder scores within 1-2 points of MRR while being
15/// 15-60% faster, so this is the default budget rather than "no truncation".
16const DEFAULT_DOC_CHARS: usize = 1024;
17
18/// Cross-encoder reranker backed by a fixed pool of independent sessions.
19/// Both backends' underlying session types take `&mut self` to run
20/// inference, so serving concurrent queries means N loaded sessions rather
21/// than one behind a shared lock. Each slot keeps the original lazy-load/
22/// idle-unload lifecycle independently, so one query going cold does not
23/// force every other slot to reload.
24pub struct Reranker {
25    pool: Arc<SessionPool<RerankSession>>,
26    checkout_timeout: Duration,
27    model_name: RerankerModel,
28    ep: RerankEp,
29    max_length: Option<usize>,
30    doc_chars: Option<usize>,
31    backend: RerankBackend,
32    intra_threads: usize,
33    inter_threads: usize,
34}
35
36/// One loaded rerank session. `fastembed::TextRerank::rerank` and
37/// `ort::session::Session::run` both require exclusive (`&mut`) access, so a
38/// pool slot holds exactly one of these at a time regardless of backend.
39enum RerankSession {
40    FastEmbed(TextRerank),
41    Onnx(OnnxReranker),
42}
43
44/// `fastembed::TextRerank` hardcodes its session's intra-op thread count to
45/// `available_parallelism()` with no override in its public API (see
46/// `fastembed-5.13.4/src/reranking/impl.rs`), so a pool of N fastembed
47/// sessions asks the OS for N times the machine's core count and the
48/// sessions fight each other instead of scaling. The `onnx` backend owns its
49/// `ort::session::Session` directly and caps `intra_threads` per the pool
50/// size instead. `fastembed` stays available as an escape hatch via
51/// `SECOND_BRAIN_RERANK_BACKEND=fastembed`.
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53enum RerankBackend {
54    Onnx,
55    FastEmbed,
56}
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub(crate) enum RerankEp {
60    Cpu,
61    Cuda,
62}
63
64/// Returned when every session in the pool is still busy past the checkout
65/// timeout. Backpressure that surfaces as an error beats a request that
66/// hangs forever. A `RerankScorer` that is not a local pool (a remote
67/// service shedding load or timing out) returns it too, with `pool_size` 0,
68/// because it is the one error `Retriever::rank` degrades on rather than
69/// propagating.
70#[derive(Debug)]
71pub struct RerankTimeoutError {
72    pub waited: Duration,
73    pub pool_size: usize,
74}
75
76impl std::fmt::Display for RerankTimeoutError {
77    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
78        write!(
79            f,
80            "timed out after {:?} waiting for an idle rerank session (pool size {})",
81            self.waited, self.pool_size
82        )
83    }
84}
85
86impl std::error::Error for RerankTimeoutError {}
87
88/// What `Retriever::rank` needs from a cross-encoder: one score per document,
89/// in document order, on the 0..1 scale `RetrievalTunables::rerank_gate` is
90/// calibrated against (the sigmoid of the raw logit, which is what `Reranker`
91/// returns). A scorer that cannot answer right now must return
92/// `RerankTimeoutError`, directly or under anyhow context, so the query falls
93/// back to the cosine-gated path instead of failing.
94pub trait RerankScorer: Send + Sync {
95    fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>>;
96}
97
98/// Snapshot of pool utilization for capacity planning.
99#[derive(Debug, Clone, Copy, Default)]
100pub struct RerankMetricsSnapshot {
101    pub loaded_sessions: usize,
102    pub in_flight: usize,
103    pub total_checkouts: u64,
104    pub wait_p50: Duration,
105    pub wait_max: Duration,
106}
107
108pub fn parse_model(name: &str) -> Result<RerankerModel> {
109    match name {
110        "bge-reranker-v2-m3" => Ok(RerankerModel::BGERerankerV2M3),
111        "jina-reranker-v1-turbo" => Ok(RerankerModel::JINARerankerV1TurboEn),
112        other => {
113            bail!("unknown rerank model '{other}' (bge-reranker-v2-m3, jina-reranker-v1-turbo)")
114        }
115    }
116}
117
118fn parse_ep(raw: &str) -> Result<RerankEp> {
119    match raw {
120        "cpu" => Ok(RerankEp::Cpu),
121        "cuda" => Ok(RerankEp::Cuda),
122        other => bail!("unknown SECOND_BRAIN_RERANK_EP '{other}' (valid values: cpu, cuda)"),
123    }
124}
125
126pub(crate) fn ep_from_env() -> Result<RerankEp> {
127    match std::env::var("SECOND_BRAIN_RERANK_EP") {
128        Ok(v) => parse_ep(&v),
129        Err(_) => Ok(RerankEp::Cpu),
130    }
131}
132
133fn backend_from_env() -> RerankBackend {
134    match std::env::var("SECOND_BRAIN_RERANK_BACKEND") {
135        Ok(v) if v == "fastembed" => RerankBackend::FastEmbed,
136        _ => RerankBackend::Onnx,
137    }
138}
139
140/// Defaults to `cores / pool_size` (floor, minimum 1) so a full pool asks
141/// the OS for roughly one core's worth of intra-op threads per in-flight
142/// session instead of oversubscribing by `pool_size`x, which is the exact
143/// bottleneck this backend exists to avoid.
144fn intra_threads_from_env(pool_size: usize) -> Result<usize> {
145    match std::env::var("SECOND_BRAIN_RERANK_INTRA_THREADS") {
146        Ok(v) => {
147            let n: usize = v
148                .parse()
149                .with_context(|| format!("parsing SECOND_BRAIN_RERANK_INTRA_THREADS '{v}'"))?;
150            if n == 0 {
151                bail!("SECOND_BRAIN_RERANK_INTRA_THREADS must be at least 1");
152            }
153            Ok(n)
154        }
155        Err(_) => {
156            let cores = std::thread::available_parallelism()
157                .map(|n| n.get())
158                .unwrap_or(1);
159            Ok((cores / pool_size.max(1)).max(1))
160        }
161    }
162}
163
164fn max_length_from_env() -> Result<Option<usize>> {
165    match std::env::var("SECOND_BRAIN_RERANK_MAX_LEN") {
166        Ok(v) => Ok(Some(v.parse::<usize>().with_context(|| {
167            format!("parsing SECOND_BRAIN_RERANK_MAX_LEN '{v}'")
168        })?)),
169        Err(_) => Ok(None),
170    }
171}
172
173/// `0` is the escape hatch back to unlimited document length: a document
174/// budget of zero characters would score nothing, so that value is free to
175/// repurpose as "truncation off" rather than needing a second env var.
176fn doc_chars_from_env() -> Result<Option<usize>> {
177    match std::env::var("SECOND_BRAIN_RERANK_DOC_CHARS") {
178        Ok(v) => {
179            let n: usize = v
180                .parse()
181                .with_context(|| format!("parsing SECOND_BRAIN_RERANK_DOC_CHARS '{v}'"))?;
182            Ok(if n == 0 { None } else { Some(n) })
183        }
184        Err(_) => Ok(Some(DEFAULT_DOC_CHARS)),
185    }
186}
187
188fn checkout_timeout_from_env() -> Result<Duration> {
189    match std::env::var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") {
190        Ok(v) => {
191            let secs: u64 = v
192                .parse()
193                .with_context(|| format!("parsing SECOND_BRAIN_RERANK_CHECKOUT_SECS '{v}'"))?;
194            Ok(Duration::from_secs(secs))
195        }
196        Err(_) => Ok(DEFAULT_CHECKOUT_TIMEOUT),
197    }
198}
199
200/// Builds one session on the selected backend.
201///
202/// The `fastembed` backend cannot select a concrete execution provider: it
203/// is not a direct dependency of second-brain-core, and fastembed 5.13.4
204/// re-exports only the opaque `ExecutionProviderDispatch`, not a concrete
205/// provider type such as `CUDAExecutionProvider` — so there is no way from
206/// this crate to construct a value for
207/// `RerankInitOptions::with_execution_providers`. Requesting cuda on that
208/// path therefore always falls back to cpu, loudly. The `onnx` backend (the
209/// default) owns its `ort::Session` directly and can select CUDA for real;
210/// see `onnx_rerank::execution_providers`.
211#[allow(clippy::too_many_arguments)]
212fn load_session(
213    backend: RerankBackend,
214    model_name: RerankerModel,
215    ep: RerankEp,
216    max_length: Option<usize>,
217    intra_threads: usize,
218    inter_threads: usize,
219    show_progress: bool,
220) -> Result<RerankSession> {
221    match backend {
222        RerankBackend::FastEmbed => {
223            if ep == RerankEp::Cuda {
224                tracing::warn!(
225                    "SECOND_BRAIN_RERANK_EP=cuda requested but unavailable on the fastembed \
226                     rerank backend (ort is not a direct dependency along that path); falling \
227                     back to cpu. Use the onnx backend (the default) for real GPU selection."
228                );
229            }
230            let mut opts = RerankInitOptions::new(model_name)
231                .with_cache_dir(model_cache_dir()?)
232                .with_show_download_progress(show_progress);
233            if let Some(len) = max_length {
234                opts = opts.with_max_length(len);
235            }
236            TextRerank::try_new(opts)
237                .map(RerankSession::FastEmbed)
238                .context("loading rerank model (fastembed backend)")
239        }
240        RerankBackend::Onnx => {
241            let config = OnnxSessionConfig {
242                intra_threads,
243                inter_threads,
244                opt_level: onnx_rerank::default_opt_level(),
245                ep,
246                max_length,
247            };
248            OnnxReranker::load(model_name, &config)
249                .map(RerankSession::Onnx)
250                .context("loading rerank model (onnx backend)")
251        }
252    }
253}
254
255/// Truncation happens exactly once, here, so a document scores identically
256/// no matter how long its untruncated form is. Character-based, not
257/// byte-based, so it can never split a multi-byte UTF-8 sequence.
258fn truncate_chars(s: &str, max_chars: Option<usize>) -> String {
259    max_chars.map_or(s, |max| first_chars(s, max)).to_string()
260}
261
262/// At most the first `n` characters of `text`, cut on a character boundary.
263pub fn first_chars(text: &str, n: usize) -> &str {
264    text.char_indices()
265        .nth(n)
266        .map_or(text, |(end, _)| &text[..end])
267}
268
269/// Cross-encoder raw scores are unbounded logits; the blend and the 0.30 gate
270/// both assume a 0..1 primary term, so scores pass through a sigmoid.
271pub fn sigmoid(score: f32) -> f32 {
272    1.0 / (1.0 + (-score).exp())
273}
274
275impl Reranker {
276    pub fn new(model_name: &str) -> Result<Self> {
277        Self::with_sessions(model_name, 1)
278    }
279
280    /// Builds a pool of `n` independent sessions, eager-loaded up front so
281    /// construction fails fast exactly like the single-session constructor
282    /// always has.
283    pub fn with_sessions(model_name: &str, n: usize) -> Result<Self> {
284        if n == 0 {
285            bail!("rerank pool size must be at least 1");
286        }
287        let parsed_model = parse_model(model_name)?;
288        let ep = ep_from_env()?;
289        let max_length = max_length_from_env()?;
290        let doc_chars = doc_chars_from_env()?;
291        let backend = backend_from_env();
292        let intra_threads = intra_threads_from_env(n)?;
293        let inter_threads = 1;
294
295        let checkout_timeout = checkout_timeout_from_env()?;
296        let pool = SessionPool::new("rerank", n, |waited, pool_size| {
297            RerankTimeoutError { waited, pool_size }.into()
298        });
299        for i in 0..n {
300            pool.install(i, || {
301                load_session(
302                    backend,
303                    parsed_model.clone(),
304                    ep,
305                    max_length,
306                    intra_threads,
307                    inter_threads,
308                    true,
309                )
310            })
311            .with_context(|| format!("initializing rerank session {i} of {n}"))?;
312        }
313
314        Ok(Self {
315            pool: Arc::new(pool),
316            checkout_timeout,
317            model_name: parsed_model,
318            ep,
319            max_length,
320            doc_chars,
321            backend,
322            intra_threads,
323            inter_threads,
324        })
325    }
326
327    /// Overrides the default checkout wait bound. Mainly useful for tests
328    /// that need to observe the timeout path without a long-running test.
329    pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
330        self.checkout_timeout = timeout;
331        self
332    }
333
334    pub fn pool_size(&self) -> usize {
335        self.pool.len()
336    }
337
338    pub fn metrics(&self) -> RerankMetricsSnapshot {
339        let stats = self.pool.stats();
340        RerankMetricsSnapshot {
341            loaded_sessions: stats.loaded_sessions,
342            in_flight: stats.in_flight,
343            total_checkouts: stats.total_checkouts,
344            wait_p50: stats.wait_p50,
345            wait_max: stats.wait_max,
346        }
347    }
348
349    /// Sigmoid-squashed relevance scores in the same order as `documents`.
350    pub fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
351        self.score(query, documents, None)
352    }
353
354    /// `rerank` for a caller with a deadline: it takes only a session that
355    /// is loaded, waiting at most `wait` (and never past the pool's checkout
356    /// timeout), so it fails fast with `RerankTimeoutError` instead of
357    /// queueing behind other reranks or paying a model reload; a session
358    /// dropped while idle is reloaded in the background for later calls.
359    /// A rerank that gets a session runs to completion.
360    pub fn rerank_within(
361        &self,
362        query: &str,
363        documents: &[&str],
364        wait: Duration,
365    ) -> Result<Vec<f32>> {
366        self.score(query, documents, Some(wait.min(self.checkout_timeout)))
367    }
368
369    /// Starts, on a background thread, the reload a `rerank_within` would
370    /// start now (none while a loaded session is free), for a caller with a
371    /// deadline that will rerank soon but cannot wait for a load.
372    pub fn warm_in_background(&self) {
373        self.pool.warm_if_cold(self.loader());
374    }
375
376    fn score(
377        &self,
378        query: &str,
379        documents: &[&str],
380        warm_within: Option<Duration>,
381    ) -> Result<Vec<f32>> {
382        if documents.is_empty() {
383            return Ok(Vec::new());
384        }
385        let truncated: Vec<String> = documents
386            .iter()
387            .map(|d| truncate_chars(d, self.doc_chars))
388            .collect();
389        let doc_refs: Vec<&str> = truncated.iter().map(String::as_str).collect();
390        let doc_count = doc_refs.len();
391
392        let run = move |session: &mut RerankSession| match session {
393            RerankSession::FastEmbed(session) => {
394                let results = session
395                    .rerank(query, doc_refs, false, None)
396                    .context("reranking (fastembed backend)")?;
397                let mut out = vec![0.0_f32; doc_count];
398                for r in results {
399                    if let Some(slot) = out.get_mut(r.index) {
400                        *slot = r.score;
401                    }
402                }
403                Ok(out)
404            }
405            RerankSession::Onnx(session) => session
406                .rerank_batch(query, &doc_refs)
407                .context("reranking (onnx backend)"),
408        };
409        let raw_scores = match warm_within {
410            None => self
411                .pool
412                .checkout(self.checkout_timeout, self.loader(), run)?,
413            Some(wait) => self.pool.checkout_warm(wait, self.loader(), run)?,
414        };
415        Ok(raw_scores.into_iter().map(sigmoid).collect())
416    }
417
418    fn loader(&self) -> impl FnOnce() -> Result<RerankSession> + Send + 'static {
419        let model_name = self.model_name.clone();
420        let (ep, max_length, backend) = (self.ep, self.max_length, self.backend);
421        let (intra_threads, inter_threads) = (self.intra_threads, self.inter_threads);
422        move || {
423            load_session(
424                backend,
425                model_name,
426                ep,
427                max_length,
428                intra_threads,
429                inter_threads,
430                false,
431            )
432        }
433    }
434
435    pub fn drop_if_idle(&self, idle_secs: u64) {
436        self.pool.drop_if_idle(idle_secs);
437    }
438
439    /// Allocator growth across every slot's load and first use; a dropped slot adds 0.
440    pub fn resident_bytes(&self) -> u64 {
441        self.pool.slot_resident_bytes().iter().sum()
442    }
443
444    /// Allocator growth across each slot's load and first use, 0 for a dropped slot.
445    pub fn slot_resident_bytes(&self) -> Vec<u64> {
446        self.pool.slot_resident_bytes()
447    }
448}
449
450impl RerankScorer for Reranker {
451    fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
452        Reranker::rerank(self, query, documents)
453    }
454}
455
456#[cfg(test)]
457mod tests {
458    use super::*;
459
460    #[test]
461    fn model_names_parse_and_unknown_rejects() {
462        assert!(parse_model("bge-reranker-v2-m3").is_ok());
463        assert!(parse_model("jina-reranker-v1-turbo").is_ok());
464        assert!(parse_model("gpt-reranker").is_err());
465    }
466
467    #[test]
468    fn sigmoid_maps_logits_into_unit_interval() {
469        assert!((sigmoid(0.0) - 0.5).abs() < 1e-6);
470        assert!(sigmoid(8.0) > 0.99);
471        assert!(sigmoid(-8.0) < 0.01);
472    }
473
474    #[test]
475    fn ep_parsing_rejects_unknown_values_and_names_valid_ones() {
476        assert!(parse_ep("cpu").is_ok());
477        assert!(parse_ep("cuda").is_ok());
478        let err = parse_ep("tpu").unwrap_err();
479        assert!(err.to_string().contains("cpu"));
480        assert!(err.to_string().contains("cuda"));
481    }
482
483    #[test]
484    fn truncate_chars_never_splits_multibyte_utf8() {
485        let s = "a".repeat(9) + "\u{1F600}\u{1F600}\u{1F600}";
486        for max in 0..(s.chars().count() + 2) {
487            let t = truncate_chars(&s, Some(max));
488            assert!(std::str::from_utf8(t.as_bytes()).is_ok());
489        }
490    }
491
492    #[test]
493    fn intra_threads_defaults_to_cores_divided_by_pool_size() {
494        let cores = std::thread::available_parallelism()
495            .map(|n| n.get())
496            .unwrap_or(1);
497        assert_eq!(intra_threads_from_env(1).unwrap(), cores.max(1));
498        assert_eq!(intra_threads_from_env(cores.max(2)).unwrap(), 1);
499    }
500
501    #[test]
502    fn doc_chars_from_env_defaults_to_1024_and_zero_opts_out_of_truncation() {
503        // SAFETY: this test owns SECOND_BRAIN_RERANK_DOC_CHARS exclusively;
504        // no other test in this binary reads or writes it.
505        unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
506        assert_eq!(doc_chars_from_env().unwrap(), Some(DEFAULT_DOC_CHARS));
507
508        unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "0") };
509        assert_eq!(doc_chars_from_env().unwrap(), None);
510
511        unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "512") };
512        assert_eq!(doc_chars_from_env().unwrap(), Some(512));
513
514        unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
515    }
516
517    #[test]
518    fn checkout_timeout_from_env_defaults_to_30s_and_parses_override() {
519        // SAFETY: this test owns SECOND_BRAIN_RERANK_CHECKOUT_SECS
520        // exclusively; no other test in this binary reads or writes it.
521        unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
522        assert_eq!(
523            checkout_timeout_from_env().unwrap(),
524            DEFAULT_CHECKOUT_TIMEOUT
525        );
526
527        unsafe { std::env::set_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS", "5") };
528        assert_eq!(checkout_timeout_from_env().unwrap(), Duration::from_secs(5));
529
530        unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
531    }
532}