second-brain-core 0.13.4

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::collections::HashSet;

use anyhow::Result;

use super::blend::ABSTAIN_RERANK_BAR;
use super::rerank::RerankScorer;
use super::rrf::{RRF_K, fuse};
use crate::index::raw::{RawHit, RawIndex};

const SEARCH_K: usize = 50;
const RERANK_TOP: usize = 20;

#[derive(Debug, Clone)]
pub struct RawAnswer {
    pub hit: RawHit,
    /// The cross-encoder score the passage was admitted on.
    pub relevance: f32,
}

/// The fused vector and BM25 candidates for `query`, at most twenty, for
/// `judge` to score. Split from judging so a caller can release the index
/// before the cross-encoder runs.
pub fn raw_candidates(
    index: &RawIndex,
    query: &str,
    query_embedding: &[f32],
    project: Option<&str>,
) -> Result<Vec<RawHit>> {
    let dense: Vec<u64> = index
        .vector_search(query_embedding, SEARCH_K, project)?
        .into_iter()
        .map(|(key, _)| key)
        .collect();
    let lexical: Vec<u64> = index
        .bm25_search(query, SEARCH_K, project)?
        .into_iter()
        .map(|(key, _)| key)
        .collect();
    let mut hits = Vec::new();
    for (key, _) in fuse(&[&dense, &lexical], RRF_K, RERANK_TOP) {
        if let Some(hit) = index.passage(key)? {
            hits.push(hit);
        }
    }
    Ok(hits)
}

/// The candidates the cross-encoder scores at `ABSTAIN_RERANK_BAR` or above,
/// best first and one per text, so an empty answer stays empty for a
/// question nothing stored can answer. Not truncated: the caller filters
/// first, then applies its limit.
pub fn judge(
    query: &str,
    hits: Vec<RawHit>,
    reranker: &dyn RerankScorer,
) -> Result<Vec<RawAnswer>> {
    if hits.is_empty() {
        return Ok(Vec::new());
    }
    let texts: Vec<&str> = hits.iter().map(|h| h.text.as_str()).collect();
    let scores = reranker.rerank(query, &texts)?;
    if scores.len() != hits.len() {
        anyhow::bail!(
            "reranker returned {} scores for {} raw passages",
            scores.len(),
            hits.len()
        );
    }
    let mut answers: Vec<RawAnswer> = hits
        .into_iter()
        .zip(scores)
        .filter(|(_, score)| *score >= ABSTAIN_RERANK_BAR)
        .map(|(hit, relevance)| RawAnswer { hit, relevance })
        .collect();
    answers.sort_by(|a, b| b.relevance.total_cmp(&a.relevance));
    let mut seen = HashSet::new();
    answers.retain(|a| seen.insert(a.hit.text.trim().to_string()));
    Ok(answers)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::index::EMBEDDING_DIM;
    use crate::index::raw::{RawOrigin, RawPassage};

    fn raw_fallback(
        index: &RawIndex,
        query: &str,
        query_embedding: &[f32],
        project: Option<&str>,
        reranker: &dyn RerankScorer,
        limit: usize,
    ) -> Result<Vec<RawAnswer>> {
        let hits = raw_candidates(index, query, query_embedding, project)?;
        let mut answers = judge(query, hits, reranker)?;
        answers.truncate(limit);
        Ok(answers)
    }

    struct TableScorer(Vec<(&'static str, f32)>);

    impl RerankScorer for TableScorer {
        fn rerank(&self, _: &str, docs: &[&str]) -> Result<Vec<f32>> {
            Ok(docs
                .iter()
                .map(|d| {
                    self.0
                        .iter()
                        .find(|(key, _)| d.contains(key))
                        .map_or(0.0, |(_, score)| *score)
                })
                .collect())
        }
    }

    fn unit(i: usize) -> Vec<f32> {
        let mut v = vec![0.0; EMBEDDING_DIM];
        v[i] = 1.0;
        v
    }

    fn index_with(passages: &[(&str, Option<&str>, usize)]) -> (tempfile::TempDir, RawIndex) {
        let dir = tempfile::tempdir().unwrap();
        let mut index = RawIndex::open(dir.path()).unwrap();
        let passages: Vec<_> = passages
            .iter()
            .map(|(text, project, axis)| {
                (
                    RawPassage {
                        text: text.to_string(),
                        at: None,
                        project: project.map(str::to_string),
                        memory: None,
                    },
                    unit(*axis),
                )
            })
            .collect();
        index
            .add_source(
                &RawOrigin::Transcript {
                    sha256: "aa".to_string(),
                },
                &passages,
            )
            .unwrap();
        index.commit().unwrap();
        (dir, index)
    }

    #[test]
    fn only_passages_the_cross_encoder_scores_at_the_bar_are_returned_best_first() {
        let (_dir, index) = index_with(&[
            ("the daemon port is 7200", None, 1),
            ("the daemon port was discussed", None, 1),
            ("the daemon runs on the pi", None, 1),
        ]);
        let scorer = TableScorer(vec![
            ("is 7200", 0.9),
            ("runs on the pi", ABSTAIN_RERANK_BAR + 0.01),
            ("discussed", ABSTAIN_RERANK_BAR - 0.01),
        ]);

        let answers = raw_fallback(&index, "daemon port", &unit(1), None, &scorer, 5).unwrap();

        let texts: Vec<&str> = answers.iter().map(|a| a.hit.text.as_str()).collect();
        assert_eq!(
            texts,
            vec!["the daemon port is 7200", "the daemon runs on the pi"]
        );
        assert!((answers[0].relevance - 0.9).abs() < 1e-6);
    }

    #[test]
    fn nothing_at_the_bar_is_an_empty_answer() {
        let (_dir, index) = index_with(&[("lunch was soup", None, 1)]);
        let scorer = TableScorer(vec![("soup", ABSTAIN_RERANK_BAR - 0.2)]);
        let answers = raw_fallback(&index, "daemon port", &unit(1), None, &scorer, 5).unwrap();
        assert!(answers.is_empty());
    }

    #[test]
    fn a_project_keeps_the_fallback_inside_it() {
        let (_dir, index) = index_with(&[
            ("deploy uses argo in garden", Some("-srv-garden"), 1),
            ("deploy uses argo in orchard", Some("-srv-orchard"), 1),
        ]);
        let scorer = TableScorer(vec![("deploy", 0.9)]);
        let answers = raw_fallback(
            &index,
            "deploy argo",
            &unit(1),
            Some("-srv-orchard"),
            &scorer,
            5,
        )
        .unwrap();
        assert_eq!(answers.len(), 1);
        assert_eq!(answers[0].hit.project.as_deref(), Some("-srv-orchard"));
    }

    #[test]
    fn an_empty_index_answers_nothing_without_calling_the_reranker() {
        struct Refuses;
        impl RerankScorer for Refuses {
            fn rerank(&self, _: &str, _: &[&str]) -> Result<Vec<f32>> {
                anyhow::bail!("must not be called")
            }
        }
        let dir = tempfile::tempdir().unwrap();
        let index = RawIndex::open(dir.path()).unwrap();
        assert!(
            raw_fallback(&index, "q", &unit(1), None, &Refuses, 5)
                .unwrap()
                .is_empty()
        );
    }

    #[test]
    fn a_reranker_that_drops_scores_is_an_error_not_a_shorter_answer() {
        struct Short;
        impl RerankScorer for Short {
            fn rerank(&self, _: &str, _: &[&str]) -> Result<Vec<f32>> {
                Ok(vec![0.99])
            }
        }
        let (_dir, index) = index_with(&[("alpha fact one", None, 1), ("alpha fact two", None, 1)]);
        let hits = raw_candidates(&index, "alpha fact", &unit(1), None).unwrap();
        assert_eq!(hits.len(), 2);
        assert!(judge("alpha fact", hits, &Short).is_err());
    }

    #[test]
    fn the_limit_caps_the_answer() {
        let (_dir, index) = index_with(&[
            ("alpha fact one", None, 1),
            ("alpha fact two", None, 2),
            ("alpha fact three", None, 3),
        ]);
        let scorer = TableScorer(vec![("alpha", 0.9)]);
        let answers = raw_fallback(&index, "alpha fact", &unit(1), None, &scorer, 2).unwrap();
        assert_eq!(answers.len(), 2);
    }
}