second-brain-api 0.13.5

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};

use second_brain_core::embedding::Embedder;
use second_brain_core::retrieval::rerank::first_chars;
use second_brain_core::retrieval::{RerankScorer, RerankTimeoutError, Reranker, RetrievalTunables};

/// A rerank with no deadline, and one that takes only a loaded session
/// within `wait`.
pub(super) trait TimedRerank: Send + Sync {
    fn rerank(&self, query: &str, documents: &[&str]) -> anyhow::Result<Vec<f32>>;

    fn rerank_within(
        &self,
        query: &str,
        documents: &[&str],
        wait: Duration,
    ) -> anyhow::Result<Vec<f32>>;
}

impl TimedRerank for Reranker {
    fn rerank(&self, query: &str, documents: &[&str]) -> anyhow::Result<Vec<f32>> {
        Reranker::rerank(self, query, documents)
    }

    fn rerank_within(
        &self,
        query: &str,
        documents: &[&str],
        wait: Duration,
    ) -> anyhow::Result<Vec<f32>> {
        Reranker::rerank_within(self, query, documents, wait)
    }
}

/// A reranker bound to one recall's deadline. It notes a failed rerank,
/// because ranking then falls back to cosine relevance and the answer must
/// not claim the cross-encoder bar.
pub(super) struct Budgeted<'a> {
    reranker: &'a dyn TimedRerank,
    deadline: Option<Instant>,
    doc_chars: Option<usize>,
    failed: AtomicBool,
}

impl<'a> Budgeted<'a> {
    pub(super) fn new(reranker: &'a dyn TimedRerank, deadline: Option<Instant>) -> Self {
        Budgeted {
            reranker,
            deadline,
            doc_chars: None,
            failed: AtomicBool::new(false),
        }
    }

    pub(super) fn with_doc_chars(mut self, doc_chars: Option<usize>) -> Self {
        self.doc_chars = doc_chars;
        self
    }

    pub(super) fn reranked(&self) -> bool {
        !self.failed.load(Ordering::Relaxed)
    }
}

impl RerankScorer for Budgeted<'_> {
    /// Past the deadline no rerank starts, because one that started would
    /// run to completion after the caller has gone.
    fn rerank(&self, query: &str, documents: &[&str]) -> anyhow::Result<Vec<f32>> {
        let capped: Vec<&str>;
        let documents = match self.doc_chars {
            Some(cap) => {
                capped = documents.iter().map(|d| first_chars(d, cap)).collect();
                &capped
            }
            None => documents,
        };
        let scores = match remaining(self.deadline) {
            None => self.reranker.rerank(query, documents),
            Some(Duration::ZERO) => Err(RerankTimeoutError {
                waited: Duration::ZERO,
                pool_size: 0,
            }
            .into()),
            Some(wait) => self.reranker.rerank_within(query, documents, wait),
        };
        if scores.is_err() {
            self.failed.store(true, Ordering::Relaxed);
        }
        scores
    }
}

/// A caller's document cap for the rerank; 0 would score nothing.
pub(super) fn doc_chars_for(doc_chars: Option<usize>) -> Result<Option<usize>, String> {
    match doc_chars {
        Some(0) => Err("rerank_doc_chars must be at least 1".to_string()),
        cap => Ok(cap),
    }
}

fn remaining(deadline: Option<Instant>) -> Option<Duration> {
    deadline.map(|deadline| deadline.saturating_duration_since(Instant::now()))
}

/// With a deadline the query embeds only on a loaded session, within what
/// is left of the budget.
pub(super) fn embed_query(
    embedder: &Embedder,
    text: &str,
    deadline: Option<Instant>,
) -> anyhow::Result<Vec<f32>> {
    match remaining(deadline) {
        None => embedder.embed_query(text),
        Some(wait) => embedder.embed_query_within(text, wait),
    }
}

/// The daemon's tunables with at most `rerank_top` candidates reranked; a
/// caller can shrink the rerank, never grow it past the daemon's setting.
pub(super) fn tunables_for(
    base: RetrievalTunables,
    rerank_top: Option<usize>,
) -> Result<RetrievalTunables, String> {
    let mut tunables = base;
    match rerank_top {
        None => {}
        Some(0) => return Err("rerank_top must be at least 1".to_string()),
        Some(top) => tunables.fuse_top = top.min(base.fuse_top),
    }
    Ok(tunables)
}

#[cfg(test)]
mod tests {
    use std::sync::Mutex;

    use second_brain_core::query::{QueryFilters, QueryRequest};
    use second_brain_core::retrieval::{Candidates, RerankTimeoutError, Retriever};
    use second_brain_core::schema::{Memory, MemoryType};
    use second_brain_core::sqlite_store::SqliteStore;
    use second_brain_core::store::Store;

    use super::*;

    struct Fake {
        waits: Mutex<Vec<Duration>>,
        documents: Mutex<Vec<String>>,
        fails: bool,
    }

    impl TimedRerank for Fake {
        fn rerank(&self, query: &str, documents: &[&str]) -> anyhow::Result<Vec<f32>> {
            self.rerank_within(query, documents, Duration::MAX)
        }

        fn rerank_within(
            &self,
            _query: &str,
            documents: &[&str],
            wait: Duration,
        ) -> anyhow::Result<Vec<f32>> {
            self.waits.lock().unwrap().push(wait);
            self.documents
                .lock()
                .unwrap()
                .extend(documents.iter().map(|d| d.to_string()));
            if self.fails {
                return Err(RerankTimeoutError {
                    waited: wait,
                    pool_size: 1,
                }
                .into());
            }
            Ok(vec![0.9; documents.len()])
        }
    }

    fn fake(fails: bool) -> Fake {
        Fake {
            waits: Mutex::new(Vec::new()),
            documents: Mutex::new(Vec::new()),
            fails,
        }
    }

    #[test]
    fn a_rerank_waits_at_most_what_is_left_of_the_budget() {
        let reranker = fake(false);
        let budgeted = Budgeted::new(&reranker, Some(Instant::now() + Duration::from_millis(500)));
        budgeted.rerank("q", &["a"]).unwrap();
        let wait = reranker.waits.lock().unwrap()[0];
        assert!(wait <= Duration::from_millis(500), "{wait:?}");
        assert!(wait > Duration::from_millis(300), "{wait:?}");
        assert!(budgeted.reranked());
    }

    #[test]
    fn a_spent_budget_starts_no_rerank_and_no_budget_reranks_as_before() {
        let reranker = fake(false);
        let spent = Budgeted::new(&reranker, Some(Instant::now() - Duration::from_millis(1)));
        let err = spent.rerank("q", &["a"]).unwrap_err();
        assert!(
            err.downcast_ref::<RerankTimeoutError>().is_some(),
            "{err:#}"
        );
        assert!(!spent.reranked());
        Budgeted::new(&reranker, None).rerank("q", &["a"]).unwrap();
        assert_eq!(*reranker.waits.lock().unwrap(), vec![Duration::MAX]);
    }

    #[test]
    fn a_failed_rerank_means_the_answer_was_not_reranked() {
        let reranker = fake(true);
        let budgeted = Budgeted::new(&reranker, None);
        assert!(budgeted.rerank("q", &["a"]).is_err());
        assert!(!budgeted.reranked());
    }

    #[test]
    fn a_ranking_whose_rerank_finds_no_session_falls_back_to_cosine_unreranked() {
        let store = SqliteStore::in_memory("m".to_string()).unwrap();
        let mut dense = Vec::new();
        for content in ["alpha", "beta"] {
            let mut memory = Memory::new(
                content.to_string(),
                MemoryType::Semantic,
                "test".to_string(),
                String::new(),
            );
            memory.embedding = vec![1.0, 0.0];
            store.store_memory(&memory).unwrap();
            dense.push((memory.id, 0.9));
        }
        let request = QueryRequest {
            text: "which letter".to_string(),
            embedding: vec![1.0, 0.0],
            limit: 10,
            filters: QueryFilters::default(),
        };
        let pool = Candidates::new(dense, Vec::new());
        let retriever = Retriever::default();

        for (fails, reranked) in [(true, false), (false, true)] {
            let reranker = fake(fails);
            let budgeted = Budgeted::new(&reranker, Some(Instant::now() + Duration::from_secs(5)));
            let results = retriever
                .rank(&store, &request, &pool, Some(&budgeted))
                .unwrap();
            assert_eq!(results.len(), 2, "fails={fails}");
            assert_eq!(budgeted.reranked(), reranked, "fails={fails}");
        }
    }

    #[test]
    fn a_document_cap_reranks_only_the_first_characters() {
        let reranker = fake(false);
        let budgeted = Budgeted::new(&reranker, None).with_doc_chars(Some(3));
        budgeted.rerank("q", &["abcdef", "ab", "ünïcødé"]).unwrap();
        Budgeted::new(&reranker, None)
            .rerank("q", &["abcdef"])
            .unwrap();
        assert_eq!(
            *reranker.documents.lock().unwrap(),
            vec!["abc", "ab", "ünï", "abcdef"]
        );
        assert_eq!(doc_chars_for(None), Ok(None));
        assert_eq!(doc_chars_for(Some(512)), Ok(Some(512)));
        assert!(doc_chars_for(Some(0)).is_err());
    }

    #[test]
    fn rerank_top_lowers_the_rerank_candidates_but_never_raises_them() {
        let base = RetrievalTunables::default();
        assert_eq!(tunables_for(base, None).unwrap().fuse_top, base.fuse_top);
        assert_eq!(tunables_for(base, Some(10)).unwrap().fuse_top, 10);
        assert_eq!(
            tunables_for(base, Some(base.fuse_top + 50))
                .unwrap()
                .fuse_top,
            base.fuse_top
        );
        assert!(tunables_for(base, Some(0)).is_err());
    }
}