use super::filters::SearchFilters;
use super::hit::{Confidence, Engine, Hit};
use super::tier0::{tier0_search_filtered, HitSource, RawHit, MIN_QUERY_LEN};
use super::tier1::tier1_search_filtered;
use crate::core::config::SearchConfig;
use crate::core::index::Connection;
use crate::core::models::{Embedder, Reranker};
use crate::core::text::{excerpt, is_searchable_query, query_terms, SNIPPET_CHARS};
use anyhow::{bail, Result};
use std::collections::HashMap;
use std::path::PathBuf;
pub trait QueryEmbedder {
fn embed(&self, text: &str) -> Result<Vec<f32>>;
fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
self.embed(text)
}
fn model_id(&self) -> &str;
}
impl QueryEmbedder for Embedder {
fn embed(&self, text: &str) -> Result<Vec<f32>> {
Embedder::embed_document(self, text)
}
fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
Embedder::embed_query(self, text)
}
fn model_id(&self) -> &str {
Embedder::model_id(self)
}
}
pub type SharedEmbedder = std::sync::Arc<dyn QueryEmbedder + Send + Sync>;
pub struct SearchPipeline {
conn: Connection,
embedder: Option<SharedEmbedder>,
reranker: Option<Reranker>,
config: SearchConfig,
}
impl SearchPipeline {
pub fn new(
conn: Connection,
embedder: Option<SharedEmbedder>,
reranker: Option<Reranker>,
config: SearchConfig,
) -> Self {
Self {
conn,
embedder,
reranker,
config,
}
}
pub fn connection(&self) -> &Connection {
&self.conn
}
pub fn search(&self, query: &str, roots: &[PathBuf]) -> Result<Vec<Hit>> {
self.search_with(query, roots, &SearchFilters::default())
}
pub fn search_with(
&self,
query: &str,
roots: &[PathBuf],
filters: &SearchFilters,
) -> Result<Vec<Hit>> {
if !is_searchable_query(query) {
bail!(
"query must be at least {MIN_QUERY_LEN} characters \
(one is enough for CJK and other scripts without spaces)"
);
}
if let Some(embedder) = &self.embedder {
if let Some(stored) = self.conn.stored_embedding_model()? {
if stored != embedder.model_id() {
bail!(
"embedding model mismatch: index was built with '{stored}' \
but search is using '{current}'. Re-index or install the \
correct model.",
current = embedder.model_id()
);
}
}
}
let kw_hits =
tier0_search_filtered(&self.conn, query, roots, filters, self.config.keyword_limit)?;
let mut sem_hits: Vec<RawHit> = Vec::new();
if !self.keyword_is_decisive(query, &kw_hits) {
if let Some(embedder) = &self.embedder {
let query_vec = embedder.embed_query(query)?;
sem_hits = tier1_search_filtered(
&self.conn,
&query_vec,
roots,
filters,
self.config.semantic_limit,
)?
.into_iter()
.filter(|hit| hit.score >= self.config.semantic_min_score)
.collect();
}
}
let fused = rrf_fuse(
&kw_hits,
&sem_hits,
self.config.rrf_k,
self.config.fused_limit,
);
let relevance = relevance_map(&kw_hits, &sem_hits);
let terms = query_terms(query);
let mut hits = self.hydrate(&fused, &relevance, &terms)?;
if let Some(reranker) = &self.reranker {
let rest = hits.split_off(self.config.rerank_top.min(hits.len()));
let mut reranked = reranker.rerank(query, hits)?;
reranked.extend(rest);
hits = reranked;
}
for hit in &mut hits {
hit.confidence = self.confidence_for(hit, query, &kw_hits, &sem_hits);
hit.engine = Engine::Local;
hit.why_matched = why_matched(hit, &kw_hits, &sem_hits);
}
Ok(hits)
}
fn keyword_is_decisive(&self, query: &str, hits: &[RawHit]) -> bool {
if hits.len() < self.config.keyword_sufficiency_threshold {
return false;
}
let Some(top) = hits.first() else {
return false;
};
top.score >= 0.9 && full_term_coverage(query, &top.snippet)
}
fn confidence_for(
&self,
hit: &Hit,
query: &str,
keyword: &[RawHit],
semantic: &[RawHit],
) -> Confidence {
let kw = keyword.iter().any(|h| h.chunk_id == hit.source_id);
let sem = semantic.iter().find(|h| h.chunk_id == hit.source_id);
match (kw, sem) {
(true, Some(_)) => Confidence::High,
(true, None) if full_term_coverage(query, &hit.content) => Confidence::High,
(true, None) => Confidence::Medium,
(false, Some(sem)) if sem.score >= 0.75 => Confidence::High,
(false, Some(sem)) if sem.score >= 0.5 => Confidence::Medium,
_ => Confidence::Low,
}
}
fn hydrate(
&self,
fused: &[(i64, f32)],
relevance: &HashMap<i64, f32>,
terms: &[String],
) -> Result<Vec<Hit>> {
let mut hits = Vec::with_capacity(fused.len());
let mut stmt = self.conn.inner().prepare(
"SELECT id, path, source_path, page, line_start, line_end, anchor, section, content
FROM chunks WHERE id = ?",
)?;
for &(id, score) in fused {
let row = stmt.query_row([id], |row| {
Ok(Hit {
source_id: row.get(0)?,
path: row.get(1)?,
source_path: row.get(2)?,
page: row.get(3)?,
line_start: row.get(4)?,
line_end: row.get(5)?,
anchor: row.get(6)?,
section: row.get(7)?,
snippet: String::new(),
content: row.get::<_, String>(8)?,
score: relevance.get(&id).copied().unwrap_or(score),
why_matched: String::new(),
confidence: Confidence::Low,
engine: Engine::Local,
})
});
match row {
Ok(mut hit) => {
hit.snippet = excerpt(&hit.content, terms, SNIPPET_CHARS);
hits.push(hit);
}
Err(rusqlite::Error::QueryReturnedNoRows) => continue,
Err(e) => return Err(e.into()),
}
}
Ok(hits)
}
}
pub fn rrf_fuse(keyword: &[RawHit], semantic: &[RawHit], k: i32, limit: usize) -> Vec<(i64, f32)> {
let k = k.max(1) as f32;
let mut scores: HashMap<i64, f32> = HashMap::new();
for list in [keyword, semantic] {
for (rank, hit) in list.iter().enumerate() {
*scores.entry(hit.chunk_id).or_insert(0.0) += 1.0 / (k + rank as f32 + 1.0);
}
}
let mut fused: Vec<(i64, f32)> = scores.into_iter().collect();
fused.sort_by(|a, b| {
b.1.partial_cmp(&a.1)
.unwrap_or(std::cmp::Ordering::Equal)
.then(a.0.cmp(&b.0))
});
fused.truncate(limit);
fused
}
fn relevance_map(keyword: &[RawHit], semantic: &[RawHit]) -> HashMap<i64, f32> {
let mut map = HashMap::new();
for hit in keyword {
map.insert(hit.chunk_id, hit.score.clamp(0.0, 1.0));
}
for hit in semantic {
let entry = map.entry(hit.chunk_id).or_insert(0.0);
*entry = entry.max(hit.score.clamp(0.0, 1.0));
}
map
}
fn full_term_coverage(query: &str, text: &str) -> bool {
let text = text.to_lowercase();
let mut terms = query
.split_whitespace()
.map(|t| {
t.trim_matches(|c: char| !c.is_alphanumeric())
.to_lowercase()
})
.filter(|t| t.len() >= 3);
let Some(first) = terms.next() else {
return false;
};
text.contains(&first) && terms.all(|t| text.contains(&t))
}
fn why_matched(hit: &Hit, keyword: &[RawHit], semantic: &[RawHit]) -> String {
let kw = keyword.iter().any(|h| h.chunk_id == hit.source_id);
let sem = semantic.iter().any(|h| h.chunk_id == hit.source_id);
match (kw, sem) {
(true, true) => "matched by keyword and semantic search".to_string(),
(true, false) => "matched by keyword search".to_string(),
(false, true) => "matched by semantic search".to_string(),
(false, false) => "matched".to_string(),
}
}
pub fn source_of(hit: &RawHit) -> HitSource {
hit.source
}