velesdb_memory/context/
relevance.rs

1//! Deterministic lexical relevance of a fragment to the request query — and
2//! [`DeterministicReranker`], the first [`Reranker`] implementation the crate
3//! ships (the trait was previously a bring-your-own plug-point only).
4//!
5//! No model, no randomness: the score is the fraction of the query's distinct
6//! lowercase alphanumeric terms that also appear in the fragment, in `[0, 1]`.
7//! Coarse on purpose — it only orders same-priority fragments during packing
8//! and documents each decision; it never invents or drops content.
9
10use std::collections::BTreeSet;
11
12use crate::model::Recollection;
13use crate::rerank::{RerankError, Reranker};
14
15/// The distinct lowercase alphanumeric terms of `text`. Build the query's
16/// set once per compile and score every fragment against it — re-tokenizing
17/// the query per fragment would be pure loop-invariant waste.
18pub(crate) fn terms(text: &str) -> BTreeSet<String> {
19    text.split(|c: char| !c.is_alphanumeric())
20        .filter(|term| !term.is_empty())
21        .map(str::to_lowercase)
22        .collect()
23}
24
25/// Fraction of the query's distinct terms found in `content`, in `[0, 1]`.
26/// An empty query scores every fragment `0.0`.
27#[allow(clippy::cast_precision_loss)] // term counts are far below 2^23
28pub(crate) fn lexical_relevance(query_terms: &BTreeSet<String>, content: &str) -> f32 {
29    if query_terms.is_empty() {
30        return 0.0;
31    }
32    let content_terms = terms(content);
33    let overlap = query_terms.intersection(&content_terms).count();
34    overlap as f32 / query_terms.len() as f32
35}
36
37/// The first shipped [`Reranker`]: re-orders a fused candidate pool by
38/// deterministic lexical overlap with the query, original (fused) order as
39/// the tie-break. Never invents or drops ids, never calls a model — safe to
40/// wire into
41/// [`recall_fused_reranked`](crate::service::MemoryService::recall_fused_reranked)
42/// where a cross-encoder would be overkill or non-reproducible.
43///
44/// Like every reranker in this crate it is **opt-in**: lexical overlap can
45/// demote a semantically relevant but differently-worded fact, so it suits
46/// keyword-anchored agent queries better than open conversational ones.
47#[derive(Debug, Clone, Copy, Default)]
48pub struct DeterministicReranker;
49
50impl Reranker for DeterministicReranker {
51    fn rerank(
52        &self,
53        query: &str,
54        candidates: Vec<Recollection>,
55    ) -> Result<Vec<Recollection>, RerankError> {
56        let query_terms = terms(query);
57        let mut indexed: Vec<(usize, f32, Recollection)> = candidates
58            .into_iter()
59            .enumerate()
60            .map(|(position, candidate)| {
61                let score = lexical_relevance(&query_terms, &candidate.content);
62                (position, score, candidate)
63            })
64            .collect();
65        indexed.sort_by(|a, b| b.1.total_cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
66        Ok(indexed
67            .into_iter()
68            .map(|(_, _, candidate)| candidate)
69            .collect())
70    }
71}
72
73#[cfg(test)]
74#[path = "relevance_tests.rs"]
75mod tests;
velesdb_memory/context/relevance.rs

velesdb_memory/context/
relevance.rs
