1use std::collections::HashMap;
12use std::path::{Path, PathBuf};
13
14use anyhow::{Context, Result};
15use serde::{Deserialize, Serialize};
16use serde_json::{json, Value};
17use sha2::{Digest, Sha256};
18
19use crate::bm25;
20use crate::chunk::{chunk_file, Chunk};
21use crate::scan::{read_text, scan_project, ScannedFile};
22
23pub const DEFAULT_K: usize = 5;
24pub const MAX_K: usize = 20;
25
26pub const HOME_ENV: &str = "POWERSHELL_MCP_HOME";
28
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
31#[serde(rename_all = "lowercase")]
32pub enum QueryMode {
33 Hybrid,
34 Bm25,
35 Vector,
36}
37
38impl QueryMode {
39 pub fn parse(value: &str) -> Self {
42 match value.trim().to_ascii_lowercase().as_str() {
43 "vector" | "semantic" => QueryMode::Vector,
44 "bm25" | "keyword" | "lexical" => QueryMode::Bm25,
45 _ => QueryMode::Hybrid,
46 }
47 }
48}
49
50#[derive(Debug, Clone, Serialize, Deserialize)]
52pub struct ChunkMeta {
53 pub i: usize,
55 pub file: String,
57 pub line_start: usize,
59 pub line_end: usize,
61 pub text: String,
62}
63
64#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct RagManifest {
67 pub version: u32,
68 pub model: String,
70 pub dim: usize,
71 pub chunk_count: usize,
72 pub file_count: usize,
74 pub built_at: String,
75 pub project_root: String,
77 pub source_fingerprint: String,
80}
81
82#[derive(Debug, Clone)]
84pub struct BuildReport {
85 pub chunks: usize,
86 pub files: usize,
87 pub dim: usize,
88 pub model: String,
89}
90
91pub type EmbedFn<'a> = &'a dyn Fn(&[String]) -> anyhow::Result<Vec<Vec<f32>>>;
94
95pub fn default_rag_dir(project_root: &Path) -> PathBuf {
97 project_root.join(".rag-index")
98}
99
100pub fn rag_dir() -> PathBuf {
103 if let Ok(p) = std::env::var(HOME_ENV) {
104 if !p.trim().is_empty() {
105 return PathBuf::from(p);
106 }
107 }
108 std::env::current_dir()
109 .map(|c| c.join(".rag-index"))
110 .unwrap_or_else(|_| PathBuf::from(".rag-index"))
111}
112
113fn manifest_path(root: &Path) -> PathBuf {
114 root.join("manifest.json")
115}
116
117fn chunks_path(root: &Path) -> PathBuf {
118 root.join("chunks.jsonl")
119}
120
121fn embeddings_path(root: &Path) -> PathBuf {
122 root.join("embeddings.bin")
123}
124
125pub fn tantivy_dir(root: &Path) -> PathBuf {
126 root.join("tantivy")
127}
128
129pub fn build_index(project_root: &Path, root: &Path, embed: EmbedFn) -> Result<BuildReport> {
137 std::fs::create_dir_all(root).with_context(|| format!("create rag dir {}", root.display()))?;
138
139 let files = scan_project(project_root).context("scan project")?;
140 if files.is_empty() {
141 anyhow::bail!(
142 "no indexable files under {} (check DEFAULT_EXTS / EXCLUDED_DIRS in src/scan.rs)",
143 project_root.display()
144 );
145 }
146
147 let mut chunks: Vec<Chunk> = Vec::new();
148 let mut indexed_files = 0usize;
149 for f in &files {
150 let full = project_root.join(&f.rel_path);
151 let Some(text) = read_text(&full) else {
152 continue;
153 };
154 let file_chunks = chunk_file(chunks.len(), &f.rel_path, &text);
155 if !file_chunks.is_empty() {
156 indexed_files += 1;
157 }
158 chunks.extend(file_chunks);
159 }
160 if chunks.is_empty() {
161 anyhow::bail!("no chunks produced (all files empty, unreadable, or binary)");
162 }
163
164 let model = crate::embed::current_model();
165 let prefix = if model.uses_prefixes() {
166 crate::embed::PASSAGE_PREFIX
167 } else {
168 ""
169 };
170 let texts: Vec<String> = chunks
171 .iter()
172 .map(|c| format!("{prefix}{}", c.text))
173 .collect();
174 eprintln!("[rag] embedding {} chunks...", texts.len());
175 let vectors = embed(&texts).context("embed chunks")?;
176 let dim = vectors.first().map(|v| v.len()).unwrap_or(0);
177 if dim == 0 || vectors.len() != chunks.len() {
178 anyhow::bail!(
179 "embedding dimension/count mismatch: {} vectors, dim {}",
180 vectors.len(),
181 dim
182 );
183 }
184
185 let manifest = RagManifest {
186 version: 1,
187 model: model.id().to_string(),
188 dim,
189 chunk_count: chunks.len(),
190 file_count: indexed_files,
191 built_at: chrono::Utc::now().to_rfc3339(),
192 project_root: project_root
193 .canonicalize()
194 .unwrap_or_else(|_| project_root.to_path_buf())
195 .display()
196 .to_string(),
197 source_fingerprint: fingerprint_files(&files),
198 };
199
200 serde_json::to_writer_pretty(
201 std::io::BufWriter::new(std::fs::File::create(manifest_path(root))?),
202 &manifest,
203 )
204 .context("write manifest")?;
205
206 let mut chunks_out = String::new();
207 for c in &chunks {
208 let meta = ChunkMeta {
209 i: c.index,
210 file: c.file.clone(),
211 line_start: c.line_start,
212 line_end: c.line_end,
213 text: c.text.clone(),
214 };
215 chunks_out.push_str(&serde_json::to_string(&meta).context("serialize chunk")?);
216 chunks_out.push('\n');
217 }
218 std::fs::write(chunks_path(root), chunks_out).context("write chunks.jsonl")?;
219
220 let mut blob: Vec<u8> = Vec::with_capacity(vectors.len() * dim * 4);
221 for v in &vectors {
222 for f in v {
223 blob.extend_from_slice(&f.to_le_bytes());
224 }
225 }
226 std::fs::write(embeddings_path(root), blob).context("write embeddings.bin")?;
227
228 bm25::index_chunks(&tantivy_dir(root), &chunks).context("build tantivy index")?;
229
230 Ok(BuildReport {
231 chunks: chunks.len(),
232 files: indexed_files,
233 dim,
234 model: manifest.model,
235 })
236}
237
238pub fn fingerprint_files(files: &[ScannedFile]) -> String {
240 let mut digest = Sha256::new();
241 for f in files {
242 digest.update(f.rel_path.as_bytes());
243 digest.update(b"\t");
244 digest.update(f.bytes.to_le_bytes());
245 digest.update(b"\n");
246 }
247 let out = digest.finalize();
248 out[..6].iter().map(|b| format!("{b:02x}")).collect()
249}
250
251pub fn read_manifest(root: &Path) -> Result<RagManifest> {
258 let path = manifest_path(root);
259 let text = std::fs::read_to_string(&path).with_context(|| {
260 format!(
261 "no semantic index at {} (run `powershell-mcp build-index <corpus-root>` first)",
262 root.display()
263 )
264 })?;
265 serde_json::from_str(&text).context("parse rag manifest")
266}
267
268fn load_chunks(root: &Path) -> Result<Vec<ChunkMeta>> {
269 let path = chunks_path(root);
270 let text =
271 std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
272 let mut out = Vec::new();
273 for line in text.lines() {
274 if line.trim().is_empty() {
275 continue;
276 }
277 out.push(serde_json::from_str(line).context("parse chunk record")?);
278 }
279 Ok(out)
280}
281
282fn load_embeddings(root: &Path, count: usize, dim: usize) -> Result<Vec<Vec<f32>>> {
283 let path = embeddings_path(root);
284 let bytes = std::fs::read(&path).with_context(|| format!("read {}", path.display()))?;
285 let expected = count * dim * 4;
286 if bytes.len() != expected {
287 anyhow::bail!(
288 "embedding blob size mismatch: expected {} bytes, found {}",
289 expected,
290 bytes.len()
291 );
292 }
293 let mut out = Vec::with_capacity(count);
294 for i in 0..count {
295 let base = i * dim;
296 let mut v = Vec::with_capacity(dim);
297 for j in 0..dim {
298 let offset = (base + j) * 4;
299 let b = [
300 bytes[offset],
301 bytes[offset + 1],
302 bytes[offset + 2],
303 bytes[offset + 3],
304 ];
305 v.push(f32::from_le_bytes(b));
306 }
307 out.push(v);
308 }
309 Ok(out)
310}
311
312#[derive(Debug, Clone)]
315pub struct LoadedRag {
316 pub manifest: RagManifest,
317 pub chunks: Vec<ChunkMeta>,
318 pub embeddings: Vec<Vec<f32>>,
319}
320
321pub fn load(root: &Path) -> Result<LoadedRag> {
322 let manifest = read_manifest(root)?;
323 let chunks = load_chunks(root)?;
324 let embeddings = load_embeddings(root, manifest.chunk_count, manifest.dim)?;
325 Ok(LoadedRag {
326 manifest,
327 chunks,
328 embeddings,
329 })
330}
331
332pub fn ensure_model_matches(loaded: &LoadedRag) -> Result<()> {
335 let expected = crate::embed::current_model().id();
336 if loaded.manifest.model != expected {
337 anyhow::bail!(
338 "semantic index was built with model '{}' but this binary uses '{}'; run `powershell-mcp build-index` to rebuild",
339 loaded.manifest.model,
340 expected
341 );
342 }
343 Ok(())
344}
345
346#[derive(Debug, Clone)]
348pub struct ScoredChunk {
349 pub chunk: ChunkMeta,
350 pub score: f32,
351 pub vector_score: Option<f32>,
352 pub bm25_score: Option<f32>,
353}
354
355fn reciprocal_rank_fusion(lists: &[Vec<(usize, f32)>], top_k: usize) -> Vec<(usize, f32)> {
357 const K: f32 = 60.0;
358 let mut scores: HashMap<usize, f32> = HashMap::new();
359 for list in lists {
360 for (rank, (idx, _)) in list.iter().enumerate() {
361 *scores.entry(*idx).or_insert(0.0) += 1.0 / (K + rank as f32 + 1.0);
362 }
363 }
364 let mut ranked: Vec<(usize, f32)> = scores.into_iter().collect();
365 ranked.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
366 ranked.truncate(top_k);
367 ranked
368}
369
370pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
371 let an = norm(a);
372 let bn = norm(b);
373 if an > 0.0 && bn > 0.0 {
374 dot(a, b) / (an * bn)
375 } else {
376 0.0
377 }
378}
379
380fn norm(v: &[f32]) -> f32 {
381 v.iter().map(|x| x * x).sum::<f32>().sqrt()
382}
383
384fn dot(a: &[f32], b: &[f32]) -> f32 {
385 a.iter().zip(b.iter()).map(|(x, y)| x * y).sum()
386}
387
388pub fn retrieve(
391 loaded: &LoadedRag,
392 root: &Path,
393 question: &str,
394 mode: QueryMode,
395 k: usize,
396 query_embedding: Option<&[f32]>,
397) -> Result<Vec<ScoredChunk>> {
398 let k = k.clamp(1, MAX_K);
399 let tantivy_dir_path = tantivy_dir(root);
400 let chunks = &loaded.chunks;
401
402 match mode {
403 QueryMode::Bm25 => {
404 let hits = bm25::search_chunks(&tantivy_dir_path, question, k)?;
405 let mut out = Vec::with_capacity(hits.len());
406 for h in hits {
407 if let Some(chunk) = chunks.get(h.chunk_index) {
408 out.push(ScoredChunk {
409 chunk: chunk.clone(),
410 score: h.score,
411 vector_score: None,
412 bm25_score: Some(h.score),
413 });
414 }
415 }
416 Ok(out)
417 }
418 QueryMode::Vector => {
419 let q = query_embedding.ok_or_else(|| anyhow::anyhow!("query embedding required"))?;
420 let mut scored: Vec<(f32, usize)> = loaded
421 .embeddings
422 .iter()
423 .enumerate()
424 .map(|(idx, v)| (cosine_similarity(q, v), idx))
425 .collect();
426 scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
427 scored.truncate(k);
428 let mut out = Vec::with_capacity(scored.len());
429 for (sim, idx) in scored {
430 if let Some(chunk) = chunks.get(idx) {
431 out.push(ScoredChunk {
432 chunk: chunk.clone(),
433 score: sim,
434 vector_score: Some(sim),
435 bm25_score: None,
436 });
437 }
438 }
439 Ok(out)
440 }
441 QueryMode::Hybrid => {
442 let q = query_embedding.ok_or_else(|| anyhow::anyhow!("query embedding required"))?;
443 let pool = k * 4;
444 let bm = bm25::search_chunks(&tantivy_dir_path, question, pool)?;
445 let mut vec_scored: Vec<(f32, usize)> = loaded
446 .embeddings
447 .iter()
448 .enumerate()
449 .map(|(idx, v)| (cosine_similarity(q, v), idx))
450 .collect();
451 vec_scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
452 vec_scored.truncate(pool);
453
454 let bm_list: Vec<(usize, f32)> = bm.iter().map(|h| (h.chunk_index, h.score)).collect();
455 let vec_list: Vec<(usize, f32)> = vec_scored.iter().map(|(s, i)| (*i, *s)).collect();
456 let vmap: HashMap<usize, f32> = vec_scored.iter().map(|(s, i)| (*i, *s)).collect();
457 let bmap: HashMap<usize, f32> = bm.iter().map(|h| (h.chunk_index, h.score)).collect();
458
459 let ranked = reciprocal_rank_fusion(&[bm_list, vec_list], k);
460 let mut out = Vec::with_capacity(ranked.len());
461 for (idx, rrf) in ranked {
462 if let Some(chunk) = chunks.get(idx) {
463 out.push(ScoredChunk {
464 chunk: chunk.clone(),
465 score: rrf,
466 vector_score: vmap.get(&idx).copied(),
467 bm25_score: bmap.get(&idx).copied(),
468 });
469 }
470 }
471 Ok(out)
472 }
473 }
474}
475
476pub fn search(root: &Path, question: &str, mode: QueryMode, k: usize) -> Result<Vec<ScoredChunk>> {
479 let loaded = load(root)?;
480 let query_embedding: Option<Vec<f32>> = match mode {
481 QueryMode::Bm25 => None,
482 QueryMode::Vector | QueryMode::Hybrid => {
483 ensure_model_matches(&loaded)?;
484 let model = crate::embed::current_model();
485 let prefix = if model.uses_prefixes() {
486 crate::embed::QUERY_PREFIX
487 } else {
488 ""
489 };
490 let texts = vec![format!("{prefix}{question}")];
491 let embedded = crate::embed::embed_texts(&texts)?;
492 embedded.first().cloned()
493 }
494 };
495 retrieve(&loaded, root, question, mode, k, query_embedding.as_deref())
496}
497
498pub fn file_context<'a>(loaded: &'a LoadedRag, file: &str) -> Vec<&'a ChunkMeta> {
500 loaded.chunks.iter().filter(|c| c.file == file).collect()
501}
502
503pub fn scored_chunk_to_json(hit: &ScoredChunk) -> Value {
505 json!({
506 "i": hit.chunk.i,
507 "file": hit.chunk.file,
508 "line_start": hit.chunk.line_start,
509 "line_end": hit.chunk.line_end,
510 "text": hit.chunk.text,
511 "score": hit.score,
512 "bm25_score": hit.bm25_score,
513 "vector_score": hit.vector_score,
514 })
515}