Skip to main content

paper/core/search/
pipeline.rs

1use super::filters::SearchFilters;
2use super::hit::{Confidence, Engine, Hit};
3use super::tier0::{tier0_search_filtered, HitSource, RawHit, MIN_QUERY_LEN};
4use super::tier1::tier1_search_filtered;
5use crate::core::config::SearchConfig;
6use crate::core::index::Connection;
7use crate::core::models::{Embedder, Reranker};
8use crate::core::text::{excerpt, is_searchable_query, query_terms, SNIPPET_CHARS};
9use anyhow::{bail, Result};
10use std::collections::HashMap;
11use std::path::PathBuf;
12
13/// Deterministic, non-generative embedder used in tests and in `--no-model`
14/// keyword-only operation.
15pub trait QueryEmbedder {
16    /// Embed a document chunk.
17    fn embed(&self, text: &str) -> Result<Vec<f32>>;
18
19    /// Embed a search query. Models with an asymmetric retrieval instruction
20    /// (BGE, E5) override this; symmetric models just use `embed`.
21    fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
22        self.embed(text)
23    }
24
25    /// The model identifier (e.g. "bge-small-en-v1.5"). Used to detect
26    /// index/search model mismatches.
27    fn model_id(&self) -> &str;
28}
29
30impl QueryEmbedder for Embedder {
31    fn embed(&self, text: &str) -> Result<Vec<f32>> {
32        Embedder::embed_document(self, text)
33    }
34
35    fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
36        Embedder::embed_query(self, text)
37    }
38
39    fn model_id(&self) -> &str {
40        Embedder::model_id(self)
41    }
42}
43
44/// Shared handle so the indexer and the search pipeline can reuse one model.
45pub type SharedEmbedder = std::sync::Arc<dyn QueryEmbedder + Send + Sync>;
46
47pub struct SearchPipeline {
48    conn: Connection,
49    embedder: Option<SharedEmbedder>,
50    reranker: Option<Reranker>,
51    config: SearchConfig,
52}
53
54impl SearchPipeline {
55    pub fn new(
56        conn: Connection,
57        embedder: Option<SharedEmbedder>,
58        reranker: Option<Reranker>,
59        config: SearchConfig,
60    ) -> Self {
61        Self {
62            conn,
63            embedder,
64            reranker,
65            config,
66        }
67    }
68
69    pub fn connection(&self) -> &Connection {
70        &self.conn
71    }
72
73    pub fn search(&self, query: &str, roots: &[PathBuf]) -> Result<Vec<Hit>> {
74        self.search_with(query, roots, &SearchFilters::default())
75    }
76
77    /// Search with structured filters (`--after`, `--before`, `--ext`, path).
78    pub fn search_with(
79        &self,
80        query: &str,
81        roots: &[PathBuf],
82        filters: &SearchFilters,
83    ) -> Result<Vec<Hit>> {
84        if !is_searchable_query(query) {
85            bail!(
86                "query must be at least {MIN_QUERY_LEN} characters \
87                 (one is enough for CJK and other scripts without spaces)"
88            );
89        }
90
91        // If we have an embedder and the index has a stored embedding model,
92        // they must match. This prevents silently searching with the wrong model.
93        if let Some(embedder) = &self.embedder {
94            if let Some(stored) = self.conn.stored_embedding_model()? {
95                if stored != embedder.model_id() {
96                    bail!(
97                        "embedding model mismatch: index was built with '{stored}' \
98                         but search is using '{current}'. Re-index or install the \
99                         correct model.",
100                        current = embedder.model_id()
101                    );
102                }
103            }
104        }
105
106        let kw_hits =
107            tier0_search_filtered(&self.conn, query, roots, filters, self.config.keyword_limit)?;
108
109        // Strong keyword evidence: skip the embedding model entirely. This keeps
110        // exact lookups (invoice numbers, names) fast and model-free.
111        //
112        // "Strong" means the top keyword hit carries every content word of the
113        // query, not merely "there are ten hits": ten partial `OR` matches of a
114        // vague question used to suppress semantic search exactly when it was
115        // most needed.
116        let mut sem_hits: Vec<RawHit> = Vec::new();
117        if !self.keyword_is_decisive(query, &kw_hits) {
118            if let Some(embedder) = &self.embedder {
119                let query_vec = embedder.embed_query(query)?;
120                sem_hits = tier1_search_filtered(
121                    &self.conn,
122                    &query_vec,
123                    roots,
124                    filters,
125                    self.config.semantic_limit,
126                )?
127                .into_iter()
128                .filter(|hit| hit.score >= self.config.semantic_min_score)
129                .collect();
130            }
131        }
132
133        let fused = rrf_fuse(
134            &kw_hits,
135            &sem_hits,
136            self.config.rrf_k,
137            self.config.fused_limit,
138        );
139        let relevance = relevance_map(&kw_hits, &sem_hits);
140        let terms = query_terms(query);
141        let mut hits = self.hydrate(&fused, &relevance, &terms)?;
142
143        if let Some(reranker) = &self.reranker {
144            // Only the top candidates go through the cross-encoder: it is one
145            // forward pass per pair, so reranking everything would be far slower.
146            let rest = hits.split_off(self.config.rerank_top.min(hits.len()));
147            let mut reranked = reranker.rerank(query, hits)?;
148            reranked.extend(rest);
149            hits = reranked;
150        }
151
152        for hit in &mut hits {
153            hit.confidence = self.confidence_for(hit, query, &kw_hits, &sem_hits);
154            hit.engine = Engine::Local;
155            hit.why_matched = why_matched(hit, &kw_hits, &sem_hits);
156        }
157        Ok(hits)
158    }
159
160    /// True when keyword search alone has already answered the query.
161    ///
162    /// The bar is lexical coverage of the *whole* query by the best hit, plus
163    /// enough corroborating results that this is not a fluke. A query with ten
164    /// weak partial matches still goes to the embedding model.
165    fn keyword_is_decisive(&self, query: &str, hits: &[RawHit]) -> bool {
166        if hits.len() < self.config.keyword_sufficiency_threshold {
167            return false;
168        }
169        let Some(top) = hits.first() else {
170            return false;
171        };
172        // Normalised relevance: 1.0 is the best match in this result set.
173        top.score >= 0.9 && full_term_coverage(query, &top.snippet)
174    }
175
176    /// Confidence describes the *evidence*, not the rank position.
177    ///
178    /// A lone keyword hit is not automatically strong evidence: normalising bm25
179    /// within a result set of size one would otherwise always yield 1.0.
180    fn confidence_for(
181        &self,
182        hit: &Hit,
183        query: &str,
184        keyword: &[RawHit],
185        semantic: &[RawHit],
186    ) -> Confidence {
187        let kw = keyword.iter().any(|h| h.chunk_id == hit.source_id);
188        let sem = semantic.iter().find(|h| h.chunk_id == hit.source_id);
189        match (kw, sem) {
190            // Corroborated by two independent retrievers.
191            (true, Some(_)) => Confidence::High,
192            // Every content word of the query literally appears in the chunk.
193            (true, None) if full_term_coverage(query, &hit.content) => Confidence::High,
194            (true, None) => Confidence::Medium,
195            (false, Some(sem)) if sem.score >= 0.75 => Confidence::High,
196            (false, Some(sem)) if sem.score >= 0.5 => Confidence::Medium,
197            _ => Confidence::Low,
198        }
199    }
200
201    /// Turn `(chunk_id, rrf_score)` pairs into full citations.
202    ///
203    /// RRF scores only define the *order* — they are tiny fractions like 0.016
204    /// and meaningless to a user. The displayed `score` is the underlying
205    /// normalised relevance, which is also what confidence is derived from.
206    fn hydrate(
207        &self,
208        fused: &[(i64, f32)],
209        relevance: &HashMap<i64, f32>,
210        terms: &[String],
211    ) -> Result<Vec<Hit>> {
212        let mut hits = Vec::with_capacity(fused.len());
213        let mut stmt = self.conn.inner().prepare(
214            "SELECT id, path, source_path, page, line_start, line_end, anchor, section, content
215             FROM chunks WHERE id = ?",
216        )?;
217        for &(id, score) in fused {
218            let row = stmt.query_row([id], |row| {
219                Ok(Hit {
220                    source_id: row.get(0)?,
221                    path: row.get(1)?,
222                    source_path: row.get(2)?,
223                    page: row.get(3)?,
224                    line_start: row.get(4)?,
225                    line_end: row.get(5)?,
226                    anchor: row.get(6)?,
227                    section: row.get(7)?,
228                    snippet: String::new(),
229                    content: row.get::<_, String>(8)?,
230                    score: relevance.get(&id).copied().unwrap_or(score),
231                    why_matched: String::new(),
232                    confidence: Confidence::Low,
233                    engine: Engine::Local,
234                })
235            });
236            match row {
237                Ok(mut hit) => {
238                    hit.snippet = excerpt(&hit.content, terms, SNIPPET_CHARS);
239                    hits.push(hit);
240                }
241                // A chunk can disappear between the scan and hydration.
242                Err(rusqlite::Error::QueryReturnedNoRows) => continue,
243                Err(e) => return Err(e.into()),
244            }
245        }
246        Ok(hits)
247    }
248}
249
250/// Reciprocal Rank Fusion. Results present in *both* lists accumulate two
251/// contributions and therefore outrank single-source results.
252pub fn rrf_fuse(keyword: &[RawHit], semantic: &[RawHit], k: i32, limit: usize) -> Vec<(i64, f32)> {
253    let k = k.max(1) as f32;
254    let mut scores: HashMap<i64, f32> = HashMap::new();
255    for list in [keyword, semantic] {
256        for (rank, hit) in list.iter().enumerate() {
257            *scores.entry(hit.chunk_id).or_insert(0.0) += 1.0 / (k + rank as f32 + 1.0);
258        }
259    }
260    let mut fused: Vec<(i64, f32)> = scores.into_iter().collect();
261    fused.sort_by(|a, b| {
262        b.1.partial_cmp(&a.1)
263            .unwrap_or(std::cmp::Ordering::Equal)
264            .then(a.0.cmp(&b.0))
265    });
266    fused.truncate(limit);
267    fused
268}
269
270/// Best available relevance per chunk: the semantic cosine similarity when we
271/// have one, otherwise the normalised keyword score.
272fn relevance_map(keyword: &[RawHit], semantic: &[RawHit]) -> HashMap<i64, f32> {
273    let mut map = HashMap::new();
274    for hit in keyword {
275        map.insert(hit.chunk_id, hit.score.clamp(0.0, 1.0));
276    }
277    for hit in semantic {
278        let entry = map.entry(hit.chunk_id).or_insert(0.0);
279        *entry = entry.max(hit.score.clamp(0.0, 1.0));
280    }
281    map
282}
283
284/// True when every content word of the query occurs in the text, i.e. an exact
285/// lexical match rather than a partial `OR` match.
286fn full_term_coverage(query: &str, text: &str) -> bool {
287    let text = text.to_lowercase();
288    let mut terms = query
289        .split_whitespace()
290        .map(|t| {
291            t.trim_matches(|c: char| !c.is_alphanumeric())
292                .to_lowercase()
293        })
294        .filter(|t| t.len() >= 3);
295    let Some(first) = terms.next() else {
296        return false;
297    };
298    text.contains(&first) && terms.all(|t| text.contains(&t))
299}
300
301/// Snippets are drawn from the stored `content`, not from the segmented index
302/// text: FTS5's `snippet()` would emit bigram-separated CJK and would need an
303/// unreliable desegmentation pass to look right.
304fn why_matched(hit: &Hit, keyword: &[RawHit], semantic: &[RawHit]) -> String {
305    let kw = keyword.iter().any(|h| h.chunk_id == hit.source_id);
306    let sem = semantic.iter().any(|h| h.chunk_id == hit.source_id);
307    match (kw, sem) {
308        (true, true) => "matched by keyword and semantic search".to_string(),
309        (true, false) => "matched by keyword search".to_string(),
310        (false, true) => "matched by semantic search".to_string(),
311        (false, false) => "matched".to_string(),
312    }
313}
314
315/// Used by `Hit::source` serialisation; kept here so `HitSource` stays imported.
316pub fn source_of(hit: &RawHit) -> HitSource {
317    hit.source
318}