use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use second_brain_core::embedding::Embedder;
use second_brain_core::retrieval::rerank::first_chars;
use second_brain_core::retrieval::{RerankScorer, RerankTimeoutError, Reranker, RetrievalTunables};
pub(super) trait TimedRerank: Send + Sync {
fn rerank(&self, query: &str, documents: &[&str]) -> anyhow::Result<Vec<f32>>;
fn rerank_within(
&self,
query: &str,
documents: &[&str],
wait: Duration,
) -> anyhow::Result<Vec<f32>>;
}
impl TimedRerank for Reranker {
fn rerank(&self, query: &str, documents: &[&str]) -> anyhow::Result<Vec<f32>> {
Reranker::rerank(self, query, documents)
}
fn rerank_within(
&self,
query: &str,
documents: &[&str],
wait: Duration,
) -> anyhow::Result<Vec<f32>> {
Reranker::rerank_within(self, query, documents, wait)
}
}
pub(super) struct Budgeted<'a> {
reranker: &'a dyn TimedRerank,
deadline: Option<Instant>,
doc_chars: Option<usize>,
failed: AtomicBool,
}
impl<'a> Budgeted<'a> {
pub(super) fn new(reranker: &'a dyn TimedRerank, deadline: Option<Instant>) -> Self {
Budgeted {
reranker,
deadline,
doc_chars: None,
failed: AtomicBool::new(false),
}
}
pub(super) fn with_doc_chars(mut self, doc_chars: Option<usize>) -> Self {
self.doc_chars = doc_chars;
self
}
pub(super) fn reranked(&self) -> bool {
!self.failed.load(Ordering::Relaxed)
}
}
impl RerankScorer for Budgeted<'_> {
fn rerank(&self, query: &str, documents: &[&str]) -> anyhow::Result<Vec<f32>> {
let capped: Vec<&str>;
let documents = match self.doc_chars {
Some(cap) => {
capped = documents.iter().map(|d| first_chars(d, cap)).collect();
&capped
}
None => documents,
};
let scores = match remaining(self.deadline) {
None => self.reranker.rerank(query, documents),
Some(Duration::ZERO) => Err(RerankTimeoutError {
waited: Duration::ZERO,
pool_size: 0,
}
.into()),
Some(wait) => self.reranker.rerank_within(query, documents, wait),
};
if scores.is_err() {
self.failed.store(true, Ordering::Relaxed);
}
scores
}
}
pub(super) fn doc_chars_for(doc_chars: Option<usize>) -> Result<Option<usize>, String> {
match doc_chars {
Some(0) => Err("rerank_doc_chars must be at least 1".to_string()),
cap => Ok(cap),
}
}
fn remaining(deadline: Option<Instant>) -> Option<Duration> {
deadline.map(|deadline| deadline.saturating_duration_since(Instant::now()))
}
pub(super) fn embed_query(
embedder: &Embedder,
text: &str,
deadline: Option<Instant>,
) -> anyhow::Result<Vec<f32>> {
match remaining(deadline) {
None => embedder.embed_query(text),
Some(wait) => embedder.embed_query_within(text, wait),
}
}
pub(super) fn tunables_for(
base: RetrievalTunables,
rerank_top: Option<usize>,
) -> Result<RetrievalTunables, String> {
let mut tunables = base;
match rerank_top {
None => {}
Some(0) => return Err("rerank_top must be at least 1".to_string()),
Some(top) => tunables.fuse_top = top.min(base.fuse_top),
}
Ok(tunables)
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use second_brain_core::query::{QueryFilters, QueryRequest};
use second_brain_core::retrieval::{Candidates, RerankTimeoutError, Retriever};
use second_brain_core::schema::{Memory, MemoryType};
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::Store;
use super::*;
struct Fake {
waits: Mutex<Vec<Duration>>,
documents: Mutex<Vec<String>>,
fails: bool,
}
impl TimedRerank for Fake {
fn rerank(&self, query: &str, documents: &[&str]) -> anyhow::Result<Vec<f32>> {
self.rerank_within(query, documents, Duration::MAX)
}
fn rerank_within(
&self,
_query: &str,
documents: &[&str],
wait: Duration,
) -> anyhow::Result<Vec<f32>> {
self.waits.lock().unwrap().push(wait);
self.documents
.lock()
.unwrap()
.extend(documents.iter().map(|d| d.to_string()));
if self.fails {
return Err(RerankTimeoutError {
waited: wait,
pool_size: 1,
}
.into());
}
Ok(vec![0.9; documents.len()])
}
}
fn fake(fails: bool) -> Fake {
Fake {
waits: Mutex::new(Vec::new()),
documents: Mutex::new(Vec::new()),
fails,
}
}
#[test]
fn a_rerank_waits_at_most_what_is_left_of_the_budget() {
let reranker = fake(false);
let budgeted = Budgeted::new(&reranker, Some(Instant::now() + Duration::from_millis(500)));
budgeted.rerank("q", &["a"]).unwrap();
let wait = reranker.waits.lock().unwrap()[0];
assert!(wait <= Duration::from_millis(500), "{wait:?}");
assert!(wait > Duration::from_millis(300), "{wait:?}");
assert!(budgeted.reranked());
}
#[test]
fn a_spent_budget_starts_no_rerank_and_no_budget_reranks_as_before() {
let reranker = fake(false);
let spent = Budgeted::new(&reranker, Some(Instant::now() - Duration::from_millis(1)));
let err = spent.rerank("q", &["a"]).unwrap_err();
assert!(
err.downcast_ref::<RerankTimeoutError>().is_some(),
"{err:#}"
);
assert!(!spent.reranked());
Budgeted::new(&reranker, None).rerank("q", &["a"]).unwrap();
assert_eq!(*reranker.waits.lock().unwrap(), vec![Duration::MAX]);
}
#[test]
fn a_failed_rerank_means_the_answer_was_not_reranked() {
let reranker = fake(true);
let budgeted = Budgeted::new(&reranker, None);
assert!(budgeted.rerank("q", &["a"]).is_err());
assert!(!budgeted.reranked());
}
#[test]
fn a_ranking_whose_rerank_finds_no_session_falls_back_to_cosine_unreranked() {
let store = SqliteStore::in_memory("m".to_string()).unwrap();
let mut dense = Vec::new();
for content in ["alpha", "beta"] {
let mut memory = Memory::new(
content.to_string(),
MemoryType::Semantic,
"test".to_string(),
String::new(),
);
memory.embedding = vec![1.0, 0.0];
store.store_memory(&memory).unwrap();
dense.push((memory.id, 0.9));
}
let request = QueryRequest {
text: "which letter".to_string(),
embedding: vec![1.0, 0.0],
limit: 10,
filters: QueryFilters::default(),
};
let pool = Candidates::new(dense, Vec::new());
let retriever = Retriever::default();
for (fails, reranked) in [(true, false), (false, true)] {
let reranker = fake(fails);
let budgeted = Budgeted::new(&reranker, Some(Instant::now() + Duration::from_secs(5)));
let results = retriever
.rank(&store, &request, &pool, Some(&budgeted))
.unwrap();
assert_eq!(results.len(), 2, "fails={fails}");
assert_eq!(budgeted.reranked(), reranked, "fails={fails}");
}
}
#[test]
fn a_document_cap_reranks_only_the_first_characters() {
let reranker = fake(false);
let budgeted = Budgeted::new(&reranker, None).with_doc_chars(Some(3));
budgeted.rerank("q", &["abcdef", "ab", "ünïcødé"]).unwrap();
Budgeted::new(&reranker, None)
.rerank("q", &["abcdef"])
.unwrap();
assert_eq!(
*reranker.documents.lock().unwrap(),
vec!["abc", "ab", "ünï", "abcdef"]
);
assert_eq!(doc_chars_for(None), Ok(None));
assert_eq!(doc_chars_for(Some(512)), Ok(Some(512)));
assert!(doc_chars_for(Some(0)).is_err());
}
#[test]
fn rerank_top_lowers_the_rerank_candidates_but_never_raises_them() {
let base = RetrievalTunables::default();
assert_eq!(tunables_for(base, None).unwrap().fuse_top, base.fuse_top);
assert_eq!(tunables_for(base, Some(10)).unwrap().fuse_top, 10);
assert_eq!(
tunables_for(base, Some(base.fuse_top + 50))
.unwrap()
.fuse_top,
base.fuse_top
);
assert!(tunables_for(base, Some(0)).is_err());
}
}