1use super::filters::SearchFilters;
2use super::hit::{Confidence, Engine, Hit};
3use super::tier0::{tier0_search_filtered, HitSource, RawHit, MIN_QUERY_LEN};
4use super::tier1::tier1_search_filtered;
5use crate::core::config::SearchConfig;
6use crate::core::index::Connection;
7use crate::core::models::{Embedder, Reranker};
8use crate::core::text::{excerpt, is_searchable_query, query_terms, SNIPPET_CHARS};
9use anyhow::{bail, Result};
10use std::collections::HashMap;
11use std::path::PathBuf;
12
13pub trait QueryEmbedder {
16 fn embed(&self, text: &str) -> Result<Vec<f32>>;
18
19 fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
22 self.embed(text)
23 }
24
25 fn model_id(&self) -> &str;
28}
29
30impl QueryEmbedder for Embedder {
31 fn embed(&self, text: &str) -> Result<Vec<f32>> {
32 Embedder::embed_document(self, text)
33 }
34
35 fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
36 Embedder::embed_query(self, text)
37 }
38
39 fn model_id(&self) -> &str {
40 Embedder::model_id(self)
41 }
42}
43
44pub type SharedEmbedder = std::sync::Arc<dyn QueryEmbedder + Send + Sync>;
46
47pub struct SearchPipeline {
48 conn: Connection,
49 embedder: Option<SharedEmbedder>,
50 reranker: Option<Reranker>,
51 config: SearchConfig,
52}
53
54impl SearchPipeline {
55 pub fn new(
56 conn: Connection,
57 embedder: Option<SharedEmbedder>,
58 reranker: Option<Reranker>,
59 config: SearchConfig,
60 ) -> Self {
61 Self {
62 conn,
63 embedder,
64 reranker,
65 config,
66 }
67 }
68
69 pub fn connection(&self) -> &Connection {
70 &self.conn
71 }
72
73 pub fn search(&self, query: &str, roots: &[PathBuf]) -> Result<Vec<Hit>> {
74 self.search_with(query, roots, &SearchFilters::default())
75 }
76
77 pub fn search_with(
79 &self,
80 query: &str,
81 roots: &[PathBuf],
82 filters: &SearchFilters,
83 ) -> Result<Vec<Hit>> {
84 if !is_searchable_query(query) {
85 bail!(
86 "query must be at least {MIN_QUERY_LEN} characters \
87 (one is enough for CJK and other scripts without spaces)"
88 );
89 }
90
91 if let Some(embedder) = &self.embedder {
94 if let Some(stored) = self.conn.stored_embedding_model()? {
95 if stored != embedder.model_id() {
96 bail!(
97 "embedding model mismatch: index was built with '{stored}' \
98 but search is using '{current}'. Re-index or install the \
99 correct model.",
100 current = embedder.model_id()
101 );
102 }
103 }
104 }
105
106 let kw_hits =
107 tier0_search_filtered(&self.conn, query, roots, filters, self.config.keyword_limit)?;
108
109 let mut sem_hits: Vec<RawHit> = Vec::new();
117 if !self.keyword_is_decisive(query, &kw_hits) {
118 if let Some(embedder) = &self.embedder {
119 let query_vec = embedder.embed_query(query)?;
120 sem_hits = tier1_search_filtered(
121 &self.conn,
122 &query_vec,
123 roots,
124 filters,
125 self.config.semantic_limit,
126 )?
127 .into_iter()
128 .filter(|hit| hit.score >= self.config.semantic_min_score)
129 .collect();
130 }
131 }
132
133 let fused = rrf_fuse(
134 &kw_hits,
135 &sem_hits,
136 self.config.rrf_k,
137 self.config.fused_limit,
138 );
139 let relevance = relevance_map(&kw_hits, &sem_hits);
140 let terms = query_terms(query);
141 let mut hits = self.hydrate(&fused, &relevance, &terms)?;
142
143 if let Some(reranker) = &self.reranker {
144 let rest = hits.split_off(self.config.rerank_top.min(hits.len()));
147 let mut reranked = reranker.rerank(query, hits)?;
148 reranked.extend(rest);
149 hits = reranked;
150 }
151
152 for hit in &mut hits {
153 hit.confidence = self.confidence_for(hit, query, &kw_hits, &sem_hits);
154 hit.engine = Engine::Local;
155 hit.why_matched = why_matched(hit, &kw_hits, &sem_hits);
156 }
157 Ok(hits)
158 }
159
160 fn keyword_is_decisive(&self, query: &str, hits: &[RawHit]) -> bool {
166 if hits.len() < self.config.keyword_sufficiency_threshold {
167 return false;
168 }
169 let Some(top) = hits.first() else {
170 return false;
171 };
172 top.score >= 0.9 && full_term_coverage(query, &top.snippet)
174 }
175
176 fn confidence_for(
181 &self,
182 hit: &Hit,
183 query: &str,
184 keyword: &[RawHit],
185 semantic: &[RawHit],
186 ) -> Confidence {
187 let kw = keyword.iter().any(|h| h.chunk_id == hit.source_id);
188 let sem = semantic.iter().find(|h| h.chunk_id == hit.source_id);
189 match (kw, sem) {
190 (true, Some(_)) => Confidence::High,
192 (true, None) if full_term_coverage(query, &hit.content) => Confidence::High,
194 (true, None) => Confidence::Medium,
195 (false, Some(sem)) if sem.score >= 0.75 => Confidence::High,
196 (false, Some(sem)) if sem.score >= 0.5 => Confidence::Medium,
197 _ => Confidence::Low,
198 }
199 }
200
201 fn hydrate(
207 &self,
208 fused: &[(i64, f32)],
209 relevance: &HashMap<i64, f32>,
210 terms: &[String],
211 ) -> Result<Vec<Hit>> {
212 let mut hits = Vec::with_capacity(fused.len());
213 let mut stmt = self.conn.inner().prepare(
214 "SELECT id, path, source_path, page, line_start, line_end, anchor, section, content
215 FROM chunks WHERE id = ?",
216 )?;
217 for &(id, score) in fused {
218 let row = stmt.query_row([id], |row| {
219 Ok(Hit {
220 source_id: row.get(0)?,
221 path: row.get(1)?,
222 source_path: row.get(2)?,
223 page: row.get(3)?,
224 line_start: row.get(4)?,
225 line_end: row.get(5)?,
226 anchor: row.get(6)?,
227 section: row.get(7)?,
228 snippet: String::new(),
229 content: row.get::<_, String>(8)?,
230 score: relevance.get(&id).copied().unwrap_or(score),
231 why_matched: String::new(),
232 confidence: Confidence::Low,
233 engine: Engine::Local,
234 })
235 });
236 match row {
237 Ok(mut hit) => {
238 hit.snippet = excerpt(&hit.content, terms, SNIPPET_CHARS);
239 hits.push(hit);
240 }
241 Err(rusqlite::Error::QueryReturnedNoRows) => continue,
243 Err(e) => return Err(e.into()),
244 }
245 }
246 Ok(hits)
247 }
248}
249
250pub fn rrf_fuse(keyword: &[RawHit], semantic: &[RawHit], k: i32, limit: usize) -> Vec<(i64, f32)> {
253 let k = k.max(1) as f32;
254 let mut scores: HashMap<i64, f32> = HashMap::new();
255 for list in [keyword, semantic] {
256 for (rank, hit) in list.iter().enumerate() {
257 *scores.entry(hit.chunk_id).or_insert(0.0) += 1.0 / (k + rank as f32 + 1.0);
258 }
259 }
260 let mut fused: Vec<(i64, f32)> = scores.into_iter().collect();
261 fused.sort_by(|a, b| {
262 b.1.partial_cmp(&a.1)
263 .unwrap_or(std::cmp::Ordering::Equal)
264 .then(a.0.cmp(&b.0))
265 });
266 fused.truncate(limit);
267 fused
268}
269
270fn relevance_map(keyword: &[RawHit], semantic: &[RawHit]) -> HashMap<i64, f32> {
273 let mut map = HashMap::new();
274 for hit in keyword {
275 map.insert(hit.chunk_id, hit.score.clamp(0.0, 1.0));
276 }
277 for hit in semantic {
278 let entry = map.entry(hit.chunk_id).or_insert(0.0);
279 *entry = entry.max(hit.score.clamp(0.0, 1.0));
280 }
281 map
282}
283
284fn full_term_coverage(query: &str, text: &str) -> bool {
287 let text = text.to_lowercase();
288 let mut terms = query
289 .split_whitespace()
290 .map(|t| {
291 t.trim_matches(|c: char| !c.is_alphanumeric())
292 .to_lowercase()
293 })
294 .filter(|t| t.len() >= 3);
295 let Some(first) = terms.next() else {
296 return false;
297 };
298 text.contains(&first) && terms.all(|t| text.contains(&t))
299}
300
301fn why_matched(hit: &Hit, keyword: &[RawHit], semantic: &[RawHit]) -> String {
305 let kw = keyword.iter().any(|h| h.chunk_id == hit.source_id);
306 let sem = semantic.iter().any(|h| h.chunk_id == hit.source_id);
307 match (kw, sem) {
308 (true, true) => "matched by keyword and semantic search".to_string(),
309 (true, false) => "matched by keyword search".to_string(),
310 (false, true) => "matched by semantic search".to_string(),
311 (false, false) => "matched".to_string(),
312 }
313}
314
315pub fn source_of(hit: &RawHit) -> HitSource {
317 hit.source
318}