Skip to main content

powershell_mcp/
rag.rs

1//! Hybrid vector + BM25 store for the PowerShell documentation corpus.
2//!
3//! The index is built offline (the build-index CLI) from a checkout of the
4//! MicrosoftDocs/PowerShell-Docs repository and lives under a RAG data
5//! directory (default <corpus>/.rag-index, overridable with
6//! POWERSHELL_MCP_HOME). Storage is three flat files plus the tantivy
7//! directory: manifest.json, chunks.jsonl, and a raw little-endian f32
8//! embeddings blob. Hybrid retrieval fuses BM25 and cosine-similarity
9//! rankings with reciprocal rank fusion.
10
11use std::collections::HashMap;
12use std::path::{Path, PathBuf};
13
14use anyhow::{Context, Result};
15use serde::{Deserialize, Serialize};
16use serde_json::{json, Value};
17use sha2::{Digest, Sha256};
18
19use crate::bm25;
20use crate::chunk::{chunk_file, Chunk};
21use crate::scan::{read_text, scan_project, ScannedFile};
22
23pub const DEFAULT_K: usize = 5;
24pub const MAX_K: usize = 20;
25
26/// RAG data directory env override, used by the server and CLI query.
27pub const HOME_ENV: &str = "POWERSHELL_MCP_HOME";
28
29/// Query mode.
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
31#[serde(rename_all = "lowercase")]
32pub enum QueryMode {
33    Hybrid,
34    Bm25,
35    Vector,
36}
37
38impl QueryMode {
39    /// "vector" | "semantic" -> Vector; "bm25" | "keyword" | "lexical" ->
40    /// Bm25; anything else (including unset) -> Hybrid.
41    pub fn parse(value: &str) -> Self {
42        match value.trim().to_ascii_lowercase().as_str() {
43            "vector" | "semantic" => QueryMode::Vector,
44            "bm25" | "keyword" | "lexical" => QueryMode::Bm25,
45            _ => QueryMode::Hybrid,
46        }
47    }
48}
49
50/// A chunk as persisted in chunks.jsonl.
51#[derive(Debug, Clone, Serialize, Deserialize)]
52pub struct ChunkMeta {
53    /// Global index within the corpus.
54    pub i: usize,
55    /// Path relative to the project root.
56    pub file: String,
57    /// 1-based inclusive first source line.
58    pub line_start: usize,
59    /// 1-based inclusive last source line.
60    pub line_end: usize,
61    pub text: String,
62}
63
64/// Index manifest.
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct RagManifest {
67    pub version: u32,
68    /// Embedding model identifier; guards against serving stale vectors.
69    pub model: String,
70    pub dim: usize,
71    pub chunk_count: usize,
72    /// Files that produced at least one chunk.
73    pub file_count: usize,
74    pub built_at: String,
75    /// Canonicalized project root the index was built from.
76    pub project_root: String,
77    /// SHA-256 over sorted "path<TAB>size" lines, so a changed project is
78    /// detectable.
79    pub source_fingerprint: String,
80}
81
82/// Summary of one build-index run.
83#[derive(Debug, Clone)]
84pub struct BuildReport {
85    pub chunks: usize,
86    pub files: usize,
87    pub dim: usize,
88    pub model: String,
89}
90
91/// Embedder callback injected for testability; the CLI passes the real
92/// fastembed embedder.
93pub type EmbedFn<'a> = &'a dyn Fn(&[String]) -> anyhow::Result<Vec<Vec<f32>>>;
94
95/// Default RAG data directory for a project: <project>/.rag-index.
96pub fn default_rag_dir(project_root: &Path) -> PathBuf {
97    project_root.join(".rag-index")
98}
99
100/// Resolve the active RAG data directory for query/serve: $POWERSHELL_MCP_HOME
101/// when set, else <cwd>/.rag-index.
102pub fn rag_dir() -> PathBuf {
103    if let Ok(p) = std::env::var(HOME_ENV) {
104        if !p.trim().is_empty() {
105            return PathBuf::from(p);
106        }
107    }
108    std::env::current_dir()
109        .map(|c| c.join(".rag-index"))
110        .unwrap_or_else(|_| PathBuf::from(".rag-index"))
111}
112
113fn manifest_path(root: &Path) -> PathBuf {
114    root.join("manifest.json")
115}
116
117fn chunks_path(root: &Path) -> PathBuf {
118    root.join("chunks.jsonl")
119}
120
121fn embeddings_path(root: &Path) -> PathBuf {
122    root.join("embeddings.bin")
123}
124
125pub fn tantivy_dir(root: &Path) -> PathBuf {
126    root.join("tantivy")
127}
128
129// ---------------------------------------------------------------------------
130// Index build
131// ---------------------------------------------------------------------------
132
133/// Build the hybrid index for a project directory into `root`. Scans the
134/// project, chunks every indexable text file, embeds the chunks, and writes
135/// the manifest, chunks, embedding blob, and tantivy index.
136pub fn build_index(project_root: &Path, root: &Path, embed: EmbedFn) -> Result<BuildReport> {
137    std::fs::create_dir_all(root).with_context(|| format!("create rag dir {}", root.display()))?;
138
139    let files = scan_project(project_root).context("scan project")?;
140    if files.is_empty() {
141        anyhow::bail!(
142            "no indexable files under {} (check DEFAULT_EXTS / EXCLUDED_DIRS in src/scan.rs)",
143            project_root.display()
144        );
145    }
146
147    let mut chunks: Vec<Chunk> = Vec::new();
148    let mut indexed_files = 0usize;
149    for f in &files {
150        let full = project_root.join(&f.rel_path);
151        let Some(text) = read_text(&full) else {
152            continue;
153        };
154        let file_chunks = chunk_file(chunks.len(), &f.rel_path, &text);
155        if !file_chunks.is_empty() {
156            indexed_files += 1;
157        }
158        chunks.extend(file_chunks);
159    }
160    if chunks.is_empty() {
161        anyhow::bail!("no chunks produced (all files empty, unreadable, or binary)");
162    }
163
164    let model = crate::embed::current_model();
165    let prefix = if model.uses_prefixes() {
166        crate::embed::PASSAGE_PREFIX
167    } else {
168        ""
169    };
170    let texts: Vec<String> = chunks
171        .iter()
172        .map(|c| format!("{prefix}{}", c.text))
173        .collect();
174    eprintln!("[rag] embedding {} chunks...", texts.len());
175    let vectors = embed(&texts).context("embed chunks")?;
176    let dim = vectors.first().map(|v| v.len()).unwrap_or(0);
177    if dim == 0 || vectors.len() != chunks.len() {
178        anyhow::bail!(
179            "embedding dimension/count mismatch: {} vectors, dim {}",
180            vectors.len(),
181            dim
182        );
183    }
184
185    let manifest = RagManifest {
186        version: 1,
187        model: model.id().to_string(),
188        dim,
189        chunk_count: chunks.len(),
190        file_count: indexed_files,
191        built_at: chrono::Utc::now().to_rfc3339(),
192        project_root: project_root
193            .canonicalize()
194            .unwrap_or_else(|_| project_root.to_path_buf())
195            .display()
196            .to_string(),
197        source_fingerprint: fingerprint_files(&files),
198    };
199
200    serde_json::to_writer_pretty(
201        std::io::BufWriter::new(std::fs::File::create(manifest_path(root))?),
202        &manifest,
203    )
204    .context("write manifest")?;
205
206    let mut chunks_out = String::new();
207    for c in &chunks {
208        let meta = ChunkMeta {
209            i: c.index,
210            file: c.file.clone(),
211            line_start: c.line_start,
212            line_end: c.line_end,
213            text: c.text.clone(),
214        };
215        chunks_out.push_str(&serde_json::to_string(&meta).context("serialize chunk")?);
216        chunks_out.push('\n');
217    }
218    std::fs::write(chunks_path(root), chunks_out).context("write chunks.jsonl")?;
219
220    let mut blob: Vec<u8> = Vec::with_capacity(vectors.len() * dim * 4);
221    for v in &vectors {
222        for f in v {
223            blob.extend_from_slice(&f.to_le_bytes());
224        }
225    }
226    std::fs::write(embeddings_path(root), blob).context("write embeddings.bin")?;
227
228    bm25::index_chunks(&tantivy_dir(root), &chunks).context("build tantivy index")?;
229
230    Ok(BuildReport {
231        chunks: chunks.len(),
232        files: indexed_files,
233        dim,
234        model: manifest.model,
235    })
236}
237
238/// First 12 hex chars of the SHA-256 over sorted "path<TAB>size" lines.
239pub fn fingerprint_files(files: &[ScannedFile]) -> String {
240    let mut digest = Sha256::new();
241    for f in files {
242        digest.update(f.rel_path.as_bytes());
243        digest.update(b"\t");
244        digest.update(f.bytes.to_le_bytes());
245        digest.update(b"\n");
246    }
247    let out = digest.finalize();
248    out[..6].iter().map(|b| format!("{b:02x}")).collect()
249}
250
251// ---------------------------------------------------------------------------
252// Load and query
253// ---------------------------------------------------------------------------
254
255/// Read the manifest, erroring with a helpful message when the index is
256/// missing (i.e. build-index has not been run).
257pub fn read_manifest(root: &Path) -> Result<RagManifest> {
258    let path = manifest_path(root);
259    let text = std::fs::read_to_string(&path).with_context(|| {
260        format!(
261            "no semantic index at {} (run `powershell-mcp build-index <corpus-root>` first)",
262            root.display()
263        )
264    })?;
265    serde_json::from_str(&text).context("parse rag manifest")
266}
267
268fn load_chunks(root: &Path) -> Result<Vec<ChunkMeta>> {
269    let path = chunks_path(root);
270    let text =
271        std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
272    let mut out = Vec::new();
273    for line in text.lines() {
274        if line.trim().is_empty() {
275            continue;
276        }
277        out.push(serde_json::from_str(line).context("parse chunk record")?);
278    }
279    Ok(out)
280}
281
282fn load_embeddings(root: &Path, count: usize, dim: usize) -> Result<Vec<Vec<f32>>> {
283    let path = embeddings_path(root);
284    let bytes = std::fs::read(&path).with_context(|| format!("read {}", path.display()))?;
285    let expected = count * dim * 4;
286    if bytes.len() != expected {
287        anyhow::bail!(
288            "embedding blob size mismatch: expected {} bytes, found {}",
289            expected,
290            bytes.len()
291        );
292    }
293    let mut out = Vec::with_capacity(count);
294    for i in 0..count {
295        let base = i * dim;
296        let mut v = Vec::with_capacity(dim);
297        for j in 0..dim {
298            let offset = (base + j) * 4;
299            let b = [
300                bytes[offset],
301                bytes[offset + 1],
302                bytes[offset + 2],
303                bytes[offset + 3],
304            ];
305            v.push(f32::from_le_bytes(b));
306        }
307        out.push(v);
308    }
309    Ok(out)
310}
311
312/// Loaded index: manifest + chunks + embeddings (heavy parts loaded lazily
313/// by the server and cached).
314#[derive(Debug, Clone)]
315pub struct LoadedRag {
316    pub manifest: RagManifest,
317    pub chunks: Vec<ChunkMeta>,
318    pub embeddings: Vec<Vec<f32>>,
319}
320
321pub fn load(root: &Path) -> Result<LoadedRag> {
322    let manifest = read_manifest(root)?;
323    let chunks = load_chunks(root)?;
324    let embeddings = load_embeddings(root, manifest.chunk_count, manifest.dim)?;
325    Ok(LoadedRag {
326        manifest,
327        chunks,
328        embeddings,
329    })
330}
331
332/// Error when the on-disk index was built with a different embedding model
333/// than the one this binary embeds with (stale vectors must not be served).
334pub fn ensure_model_matches(loaded: &LoadedRag) -> Result<()> {
335    let expected = crate::embed::current_model().id();
336    if loaded.manifest.model != expected {
337        anyhow::bail!(
338            "semantic index was built with model '{}' but this binary uses '{}'; run `powershell-mcp build-index` to rebuild",
339            loaded.manifest.model,
340            expected
341        );
342    }
343    Ok(())
344}
345
346/// A scored retrieval hit.
347#[derive(Debug, Clone)]
348pub struct ScoredChunk {
349    pub chunk: ChunkMeta,
350    pub score: f32,
351    pub vector_score: Option<f32>,
352    pub bm25_score: Option<f32>,
353}
354
355/// Reciprocal rank fusion: score(id) = sum over lists of 1 / (k + rank).
356fn reciprocal_rank_fusion(lists: &[Vec<(usize, f32)>], top_k: usize) -> Vec<(usize, f32)> {
357    const K: f32 = 60.0;
358    let mut scores: HashMap<usize, f32> = HashMap::new();
359    for list in lists {
360        for (rank, (idx, _)) in list.iter().enumerate() {
361            *scores.entry(*idx).or_insert(0.0) += 1.0 / (K + rank as f32 + 1.0);
362        }
363    }
364    let mut ranked: Vec<(usize, f32)> = scores.into_iter().collect();
365    ranked.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
366    ranked.truncate(top_k);
367    ranked
368}
369
370pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
371    let an = norm(a);
372    let bn = norm(b);
373    if an > 0.0 && bn > 0.0 {
374        dot(a, b) / (an * bn)
375    } else {
376        0.0
377    }
378}
379
380fn norm(v: &[f32]) -> f32 {
381    v.iter().map(|x| x * x).sum::<f32>().sqrt()
382}
383
384fn dot(a: &[f32], b: &[f32]) -> f32 {
385    a.iter().zip(b.iter()).map(|(x, y)| x * y).sum()
386}
387
388/// Run a hybrid/BM25/vector query over a loaded index. `query_embedding`
389/// may be None in BM25 mode; it is required for vector and hybrid modes.
390pub fn retrieve(
391    loaded: &LoadedRag,
392    root: &Path,
393    question: &str,
394    mode: QueryMode,
395    k: usize,
396    query_embedding: Option<&[f32]>,
397) -> Result<Vec<ScoredChunk>> {
398    let k = k.clamp(1, MAX_K);
399    let tantivy_dir_path = tantivy_dir(root);
400    let chunks = &loaded.chunks;
401
402    match mode {
403        QueryMode::Bm25 => {
404            let hits = bm25::search_chunks(&tantivy_dir_path, question, k)?;
405            let mut out = Vec::with_capacity(hits.len());
406            for h in hits {
407                if let Some(chunk) = chunks.get(h.chunk_index) {
408                    out.push(ScoredChunk {
409                        chunk: chunk.clone(),
410                        score: h.score,
411                        vector_score: None,
412                        bm25_score: Some(h.score),
413                    });
414                }
415            }
416            Ok(out)
417        }
418        QueryMode::Vector => {
419            let q = query_embedding.ok_or_else(|| anyhow::anyhow!("query embedding required"))?;
420            let mut scored: Vec<(f32, usize)> = loaded
421                .embeddings
422                .iter()
423                .enumerate()
424                .map(|(idx, v)| (cosine_similarity(q, v), idx))
425                .collect();
426            scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
427            scored.truncate(k);
428            let mut out = Vec::with_capacity(scored.len());
429            for (sim, idx) in scored {
430                if let Some(chunk) = chunks.get(idx) {
431                    out.push(ScoredChunk {
432                        chunk: chunk.clone(),
433                        score: sim,
434                        vector_score: Some(sim),
435                        bm25_score: None,
436                    });
437                }
438            }
439            Ok(out)
440        }
441        QueryMode::Hybrid => {
442            let q = query_embedding.ok_or_else(|| anyhow::anyhow!("query embedding required"))?;
443            let pool = k * 4;
444            let bm = bm25::search_chunks(&tantivy_dir_path, question, pool)?;
445            let mut vec_scored: Vec<(f32, usize)> = loaded
446                .embeddings
447                .iter()
448                .enumerate()
449                .map(|(idx, v)| (cosine_similarity(q, v), idx))
450                .collect();
451            vec_scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
452            vec_scored.truncate(pool);
453
454            let bm_list: Vec<(usize, f32)> = bm.iter().map(|h| (h.chunk_index, h.score)).collect();
455            let vec_list: Vec<(usize, f32)> = vec_scored.iter().map(|(s, i)| (*i, *s)).collect();
456            let vmap: HashMap<usize, f32> = vec_scored.iter().map(|(s, i)| (*i, *s)).collect();
457            let bmap: HashMap<usize, f32> = bm.iter().map(|h| (h.chunk_index, h.score)).collect();
458
459            let ranked = reciprocal_rank_fusion(&[bm_list, vec_list], k);
460            let mut out = Vec::with_capacity(ranked.len());
461            for (idx, rrf) in ranked {
462                if let Some(chunk) = chunks.get(idx) {
463                    out.push(ScoredChunk {
464                        chunk: chunk.clone(),
465                        score: rrf,
466                        vector_score: vmap.get(&idx).copied(),
467                        bm25_score: bmap.get(&idx).copied(),
468                    });
469                }
470            }
471            Ok(out)
472        }
473    }
474}
475
476/// Full query entry point used by the CLI: loads the index, embeds the
477/// question with the local model (vector/hybrid modes), and retrieves.
478pub fn search(root: &Path, question: &str, mode: QueryMode, k: usize) -> Result<Vec<ScoredChunk>> {
479    let loaded = load(root)?;
480    let query_embedding: Option<Vec<f32>> = match mode {
481        QueryMode::Bm25 => None,
482        QueryMode::Vector | QueryMode::Hybrid => {
483            ensure_model_matches(&loaded)?;
484            let model = crate::embed::current_model();
485            let prefix = if model.uses_prefixes() {
486                crate::embed::QUERY_PREFIX
487            } else {
488                ""
489            };
490            let texts = vec![format!("{prefix}{question}")];
491            let embedded = crate::embed::embed_texts(&texts)?;
492            embedded.first().cloned()
493        }
494    };
495    retrieve(&loaded, root, question, mode, k, query_embedding.as_deref())
496}
497
498/// Every chunk belonging to one file, in source order.
499pub fn file_context<'a>(loaded: &'a LoadedRag, file: &str) -> Vec<&'a ChunkMeta> {
500    loaded.chunks.iter().filter(|c| c.file == file).collect()
501}
502
503/// Render a ScoredChunk as the tool's result object.
504pub fn scored_chunk_to_json(hit: &ScoredChunk) -> Value {
505    json!({
506        "i": hit.chunk.i,
507        "file": hit.chunk.file,
508        "line_start": hit.chunk.line_start,
509        "line_end": hit.chunk.line_end,
510        "text": hit.chunk.text,
511        "score": hit.score,
512        "bm25_score": hit.bm25_score,
513        "vector_score": hit.vector_score,
514    })
515}