second_brain_core/retrieval/
rerank.rs1use std::sync::Arc;
2use std::time::Duration;
3
4use anyhow::{Context, Result, bail};
5use fastembed::{RerankInitOptions, RerankerModel, TextRerank};
6
7use super::onnx_rerank::{self, OnnxReranker, OnnxSessionConfig};
8use crate::model_cache::model_cache_dir;
9use crate::session_pool::SessionPool;
10
11const DEFAULT_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(30);
12
13const DEFAULT_DOC_CHARS: usize = 1024;
17
18pub struct Reranker {
25 pool: Arc<SessionPool<RerankSession>>,
26 checkout_timeout: Duration,
27 model_name: RerankerModel,
28 ep: RerankEp,
29 max_length: Option<usize>,
30 doc_chars: Option<usize>,
31 backend: RerankBackend,
32 intra_threads: usize,
33 inter_threads: usize,
34}
35
36enum RerankSession {
40 FastEmbed(TextRerank),
41 Onnx(OnnxReranker),
42}
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53enum RerankBackend {
54 Onnx,
55 FastEmbed,
56}
57
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub(crate) enum RerankEp {
60 Cpu,
61 Cuda,
62}
63
64#[derive(Debug)]
71pub struct RerankTimeoutError {
72 pub waited: Duration,
73 pub pool_size: usize,
74}
75
76impl std::fmt::Display for RerankTimeoutError {
77 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
78 write!(
79 f,
80 "timed out after {:?} waiting for an idle rerank session (pool size {})",
81 self.waited, self.pool_size
82 )
83 }
84}
85
86impl std::error::Error for RerankTimeoutError {}
87
88pub trait RerankScorer: Send + Sync {
95 fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>>;
96}
97
98#[derive(Debug, Clone, Copy, Default)]
100pub struct RerankMetricsSnapshot {
101 pub loaded_sessions: usize,
102 pub in_flight: usize,
103 pub total_checkouts: u64,
104 pub wait_p50: Duration,
105 pub wait_max: Duration,
106}
107
108pub fn parse_model(name: &str) -> Result<RerankerModel> {
109 match name {
110 "bge-reranker-v2-m3" => Ok(RerankerModel::BGERerankerV2M3),
111 "jina-reranker-v1-turbo" => Ok(RerankerModel::JINARerankerV1TurboEn),
112 other => {
113 bail!("unknown rerank model '{other}' (bge-reranker-v2-m3, jina-reranker-v1-turbo)")
114 }
115 }
116}
117
118fn parse_ep(raw: &str) -> Result<RerankEp> {
119 match raw {
120 "cpu" => Ok(RerankEp::Cpu),
121 "cuda" => Ok(RerankEp::Cuda),
122 other => bail!("unknown SECOND_BRAIN_RERANK_EP '{other}' (valid values: cpu, cuda)"),
123 }
124}
125
126pub(crate) fn ep_from_env() -> Result<RerankEp> {
127 match std::env::var("SECOND_BRAIN_RERANK_EP") {
128 Ok(v) => parse_ep(&v),
129 Err(_) => Ok(RerankEp::Cpu),
130 }
131}
132
133fn backend_from_env() -> RerankBackend {
134 match std::env::var("SECOND_BRAIN_RERANK_BACKEND") {
135 Ok(v) if v == "fastembed" => RerankBackend::FastEmbed,
136 _ => RerankBackend::Onnx,
137 }
138}
139
140fn intra_threads_from_env(pool_size: usize) -> Result<usize> {
145 match std::env::var("SECOND_BRAIN_RERANK_INTRA_THREADS") {
146 Ok(v) => {
147 let n: usize = v
148 .parse()
149 .with_context(|| format!("parsing SECOND_BRAIN_RERANK_INTRA_THREADS '{v}'"))?;
150 if n == 0 {
151 bail!("SECOND_BRAIN_RERANK_INTRA_THREADS must be at least 1");
152 }
153 Ok(n)
154 }
155 Err(_) => {
156 let cores = std::thread::available_parallelism()
157 .map(|n| n.get())
158 .unwrap_or(1);
159 Ok((cores / pool_size.max(1)).max(1))
160 }
161 }
162}
163
164fn max_length_from_env() -> Result<Option<usize>> {
165 match std::env::var("SECOND_BRAIN_RERANK_MAX_LEN") {
166 Ok(v) => Ok(Some(v.parse::<usize>().with_context(|| {
167 format!("parsing SECOND_BRAIN_RERANK_MAX_LEN '{v}'")
168 })?)),
169 Err(_) => Ok(None),
170 }
171}
172
173fn doc_chars_from_env() -> Result<Option<usize>> {
177 match std::env::var("SECOND_BRAIN_RERANK_DOC_CHARS") {
178 Ok(v) => {
179 let n: usize = v
180 .parse()
181 .with_context(|| format!("parsing SECOND_BRAIN_RERANK_DOC_CHARS '{v}'"))?;
182 Ok(if n == 0 { None } else { Some(n) })
183 }
184 Err(_) => Ok(Some(DEFAULT_DOC_CHARS)),
185 }
186}
187
188fn checkout_timeout_from_env() -> Result<Duration> {
189 match std::env::var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") {
190 Ok(v) => {
191 let secs: u64 = v
192 .parse()
193 .with_context(|| format!("parsing SECOND_BRAIN_RERANK_CHECKOUT_SECS '{v}'"))?;
194 Ok(Duration::from_secs(secs))
195 }
196 Err(_) => Ok(DEFAULT_CHECKOUT_TIMEOUT),
197 }
198}
199
200#[allow(clippy::too_many_arguments)]
212fn load_session(
213 backend: RerankBackend,
214 model_name: RerankerModel,
215 ep: RerankEp,
216 max_length: Option<usize>,
217 intra_threads: usize,
218 inter_threads: usize,
219 show_progress: bool,
220) -> Result<RerankSession> {
221 match backend {
222 RerankBackend::FastEmbed => {
223 if ep == RerankEp::Cuda {
224 tracing::warn!(
225 "SECOND_BRAIN_RERANK_EP=cuda requested but unavailable on the fastembed \
226 rerank backend (ort is not a direct dependency along that path); falling \
227 back to cpu. Use the onnx backend (the default) for real GPU selection."
228 );
229 }
230 let mut opts = RerankInitOptions::new(model_name)
231 .with_cache_dir(model_cache_dir()?)
232 .with_show_download_progress(show_progress);
233 if let Some(len) = max_length {
234 opts = opts.with_max_length(len);
235 }
236 TextRerank::try_new(opts)
237 .map(RerankSession::FastEmbed)
238 .context("loading rerank model (fastembed backend)")
239 }
240 RerankBackend::Onnx => {
241 let config = OnnxSessionConfig {
242 intra_threads,
243 inter_threads,
244 opt_level: onnx_rerank::default_opt_level(),
245 ep,
246 max_length,
247 };
248 OnnxReranker::load(model_name, &config)
249 .map(RerankSession::Onnx)
250 .context("loading rerank model (onnx backend)")
251 }
252 }
253}
254
255fn truncate_chars(s: &str, max_chars: Option<usize>) -> String {
259 max_chars.map_or(s, |max| first_chars(s, max)).to_string()
260}
261
262pub fn first_chars(text: &str, n: usize) -> &str {
264 text.char_indices()
265 .nth(n)
266 .map_or(text, |(end, _)| &text[..end])
267}
268
269pub fn sigmoid(score: f32) -> f32 {
272 1.0 / (1.0 + (-score).exp())
273}
274
275impl Reranker {
276 pub fn new(model_name: &str) -> Result<Self> {
277 Self::with_sessions(model_name, 1)
278 }
279
280 pub fn with_sessions(model_name: &str, n: usize) -> Result<Self> {
284 if n == 0 {
285 bail!("rerank pool size must be at least 1");
286 }
287 let parsed_model = parse_model(model_name)?;
288 let ep = ep_from_env()?;
289 let max_length = max_length_from_env()?;
290 let doc_chars = doc_chars_from_env()?;
291 let backend = backend_from_env();
292 let intra_threads = intra_threads_from_env(n)?;
293 let inter_threads = 1;
294
295 let checkout_timeout = checkout_timeout_from_env()?;
296 let pool = SessionPool::new("rerank", n, |waited, pool_size| {
297 RerankTimeoutError { waited, pool_size }.into()
298 });
299 for i in 0..n {
300 pool.install(i, || {
301 load_session(
302 backend,
303 parsed_model.clone(),
304 ep,
305 max_length,
306 intra_threads,
307 inter_threads,
308 true,
309 )
310 })
311 .with_context(|| format!("initializing rerank session {i} of {n}"))?;
312 }
313
314 Ok(Self {
315 pool: Arc::new(pool),
316 checkout_timeout,
317 model_name: parsed_model,
318 ep,
319 max_length,
320 doc_chars,
321 backend,
322 intra_threads,
323 inter_threads,
324 })
325 }
326
327 pub fn with_checkout_timeout(mut self, timeout: Duration) -> Self {
330 self.checkout_timeout = timeout;
331 self
332 }
333
334 pub fn pool_size(&self) -> usize {
335 self.pool.len()
336 }
337
338 pub fn metrics(&self) -> RerankMetricsSnapshot {
339 let stats = self.pool.stats();
340 RerankMetricsSnapshot {
341 loaded_sessions: stats.loaded_sessions,
342 in_flight: stats.in_flight,
343 total_checkouts: stats.total_checkouts,
344 wait_p50: stats.wait_p50,
345 wait_max: stats.wait_max,
346 }
347 }
348
349 pub fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
351 self.score(query, documents, None)
352 }
353
354 pub fn rerank_within(
361 &self,
362 query: &str,
363 documents: &[&str],
364 wait: Duration,
365 ) -> Result<Vec<f32>> {
366 self.score(query, documents, Some(wait.min(self.checkout_timeout)))
367 }
368
369 pub fn warm_in_background(&self) {
373 self.pool.warm_if_cold(self.loader());
374 }
375
376 fn score(
377 &self,
378 query: &str,
379 documents: &[&str],
380 warm_within: Option<Duration>,
381 ) -> Result<Vec<f32>> {
382 if documents.is_empty() {
383 return Ok(Vec::new());
384 }
385 let truncated: Vec<String> = documents
386 .iter()
387 .map(|d| truncate_chars(d, self.doc_chars))
388 .collect();
389 let doc_refs: Vec<&str> = truncated.iter().map(String::as_str).collect();
390 let doc_count = doc_refs.len();
391
392 let run = move |session: &mut RerankSession| match session {
393 RerankSession::FastEmbed(session) => {
394 let results = session
395 .rerank(query, doc_refs, false, None)
396 .context("reranking (fastembed backend)")?;
397 let mut out = vec![0.0_f32; doc_count];
398 for r in results {
399 if let Some(slot) = out.get_mut(r.index) {
400 *slot = r.score;
401 }
402 }
403 Ok(out)
404 }
405 RerankSession::Onnx(session) => session
406 .rerank_batch(query, &doc_refs)
407 .context("reranking (onnx backend)"),
408 };
409 let raw_scores = match warm_within {
410 None => self
411 .pool
412 .checkout(self.checkout_timeout, self.loader(), run)?,
413 Some(wait) => self.pool.checkout_warm(wait, self.loader(), run)?,
414 };
415 Ok(raw_scores.into_iter().map(sigmoid).collect())
416 }
417
418 fn loader(&self) -> impl FnOnce() -> Result<RerankSession> + Send + 'static {
419 let model_name = self.model_name.clone();
420 let (ep, max_length, backend) = (self.ep, self.max_length, self.backend);
421 let (intra_threads, inter_threads) = (self.intra_threads, self.inter_threads);
422 move || {
423 load_session(
424 backend,
425 model_name,
426 ep,
427 max_length,
428 intra_threads,
429 inter_threads,
430 false,
431 )
432 }
433 }
434
435 pub fn drop_if_idle(&self, idle_secs: u64) {
436 self.pool.drop_if_idle(idle_secs);
437 }
438
439 pub fn resident_bytes(&self) -> u64 {
441 self.pool.slot_resident_bytes().iter().sum()
442 }
443
444 pub fn slot_resident_bytes(&self) -> Vec<u64> {
446 self.pool.slot_resident_bytes()
447 }
448}
449
450impl RerankScorer for Reranker {
451 fn rerank(&self, query: &str, documents: &[&str]) -> Result<Vec<f32>> {
452 Reranker::rerank(self, query, documents)
453 }
454}
455
456#[cfg(test)]
457mod tests {
458 use super::*;
459
460 #[test]
461 fn model_names_parse_and_unknown_rejects() {
462 assert!(parse_model("bge-reranker-v2-m3").is_ok());
463 assert!(parse_model("jina-reranker-v1-turbo").is_ok());
464 assert!(parse_model("gpt-reranker").is_err());
465 }
466
467 #[test]
468 fn sigmoid_maps_logits_into_unit_interval() {
469 assert!((sigmoid(0.0) - 0.5).abs() < 1e-6);
470 assert!(sigmoid(8.0) > 0.99);
471 assert!(sigmoid(-8.0) < 0.01);
472 }
473
474 #[test]
475 fn ep_parsing_rejects_unknown_values_and_names_valid_ones() {
476 assert!(parse_ep("cpu").is_ok());
477 assert!(parse_ep("cuda").is_ok());
478 let err = parse_ep("tpu").unwrap_err();
479 assert!(err.to_string().contains("cpu"));
480 assert!(err.to_string().contains("cuda"));
481 }
482
483 #[test]
484 fn truncate_chars_never_splits_multibyte_utf8() {
485 let s = "a".repeat(9) + "\u{1F600}\u{1F600}\u{1F600}";
486 for max in 0..(s.chars().count() + 2) {
487 let t = truncate_chars(&s, Some(max));
488 assert!(std::str::from_utf8(t.as_bytes()).is_ok());
489 }
490 }
491
492 #[test]
493 fn intra_threads_defaults_to_cores_divided_by_pool_size() {
494 let cores = std::thread::available_parallelism()
495 .map(|n| n.get())
496 .unwrap_or(1);
497 assert_eq!(intra_threads_from_env(1).unwrap(), cores.max(1));
498 assert_eq!(intra_threads_from_env(cores.max(2)).unwrap(), 1);
499 }
500
501 #[test]
502 fn doc_chars_from_env_defaults_to_1024_and_zero_opts_out_of_truncation() {
503 unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
506 assert_eq!(doc_chars_from_env().unwrap(), Some(DEFAULT_DOC_CHARS));
507
508 unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "0") };
509 assert_eq!(doc_chars_from_env().unwrap(), None);
510
511 unsafe { std::env::set_var("SECOND_BRAIN_RERANK_DOC_CHARS", "512") };
512 assert_eq!(doc_chars_from_env().unwrap(), Some(512));
513
514 unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_DOC_CHARS") };
515 }
516
517 #[test]
518 fn checkout_timeout_from_env_defaults_to_30s_and_parses_override() {
519 unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
522 assert_eq!(
523 checkout_timeout_from_env().unwrap(),
524 DEFAULT_CHECKOUT_TIMEOUT
525 );
526
527 unsafe { std::env::set_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS", "5") };
528 assert_eq!(checkout_timeout_from_env().unwrap(), Duration::from_secs(5));
529
530 unsafe { std::env::remove_var("SECOND_BRAIN_RERANK_CHECKOUT_SECS") };
531 }
532}