Skip to main content

powershell_mcp/
scan.rs

1//! Corpus scanning — walk the PowerShell-Docs checkout and collect the
2//! markdown files that should be indexed, with relative paths and sizes.
3//!
4//! The corpus is the official MicrosoftDocs/PowerShell-Docs repository
5//! (reference/ cmdlet pages, about topics, and docs-conceptual guides).
6//! Only markdown is indexed; supporting directories (media, mapping,
7//! breadcrumbs, CI, tests) and contributor-facing root documents are
8//! skipped.
9
10use anyhow::{Context, Result};
11use std::path::Path;
12
13/// Extensions indexed by default. The PowerShell corpus is markdown only.
14pub const DEFAULT_EXTS: &[&str] = &["md", "markdown"];
15
16/// Directories never scanned, at any depth. Generic noise plus the
17/// PowerShell-Docs directories that carry no retrievable prose.
18pub const EXCLUDED_DIRS: &[&str] = &[
19    ".git",
20    "target",
21    "node_modules",
22    ".venv",
23    "venv",
24    "__pycache__",
25    "dist",
26    "build",
27    "out",
28    ".idea",
29    ".vscode",
30    ".devcontainer",
31    ".opengrok",
32    ".rag-index",
33    "graphify-out",
34    ".cache",
35    "vendor",
36    "bin",
37    "artifacts",
38    "coverage",
39    ".next",
40    ".nuxt",
41    ".svelte-kit",
42    "Pods",
43    ".terraform",
44    // PowerShell-Docs specific: CI, tests, redirect stubs, media, and
45    // navigation support files hold no documentation prose.
46    ".github",
47    "tests",
48    "assets",
49    "redir",
50    "media",
51    "bread",
52    "mapping",
53];
54
55/// Lockfiles, generated manifests, and contributor-facing root documents
56/// that add noise without retrieval value.
57pub const EXCLUDED_FILES: &[&str] = &[
58    "Cargo.lock",
59    "package-lock.json",
60    "yarn.lock",
61    "pnpm-lock.yaml",
62    "poetry.lock",
63    "Pipfile.lock",
64    "uv.lock",
65    "composer.lock",
66    "Gemfile.lock",
67    "go.sum",
68    // PowerShell-Docs contributor-facing documents, not PowerShell
69    // reference material.
70    "CODE_OF_CONDUCT.md",
71    "CONTRIBUTING.md",
72    "SECURITY.md",
73    "ThirdPartyNotices.md",
74    "LICENSE.md",
75    "LICENSE-CODE.md",
76];
77
78/// Files larger than this are skipped (generated bundles, dumps).
79pub const MAX_FILE_BYTES: u64 = 2 * 1024 * 1024;
80
81/// One file the scanner wants to index.
82#[derive(Debug, Clone)]
83pub struct ScannedFile {
84    /// Path relative to the project root, forward slashes.
85    pub rel_path: String,
86    /// File size in bytes (used for the source fingerprint).
87    pub bytes: u64,
88}
89
90/// Walk a project root and return the indexable files, sorted by path so
91/// indices are reproducible.
92pub fn scan_project(root: &Path) -> Result<Vec<ScannedFile>> {
93    let mut files = Vec::new();
94    scan_dir(root, root, &mut files)?;
95    files.sort_by(|a, b| a.rel_path.cmp(&b.rel_path));
96    Ok(files)
97}
98
99fn scan_dir(root: &Path, dir: &Path, out: &mut Vec<ScannedFile>) -> Result<()> {
100    let entries = std::fs::read_dir(dir).with_context(|| format!("read dir {}", dir.display()))?;
101    for entry in entries {
102        let entry = entry.with_context(|| format!("read entry in {}", dir.display()))?;
103        let path = entry.path();
104        let name = entry.file_name();
105        let name_str = name.to_string_lossy();
106        if path.is_dir() {
107            if name_str.starts_with('.') || EXCLUDED_DIRS.contains(&name_str.as_ref()) {
108                continue;
109            }
110            scan_dir(root, &path, out)?;
111        } else if path.is_file() {
112            if name_str.starts_with('.') || EXCLUDED_FILES.contains(&name_str.as_ref()) {
113                continue;
114            }
115            let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
116            if !DEFAULT_EXTS.contains(&ext) {
117                continue;
118            }
119            let meta = entry
120                .metadata()
121                .with_context(|| format!("metadata {}", path.display()))?;
122            if meta.len() > MAX_FILE_BYTES {
123                continue;
124            }
125            let rel = path
126                .strip_prefix(root)
127                .with_context(|| format!("strip prefix for {}", path.display()))?;
128            out.push(ScannedFile {
129                rel_path: rel.to_string_lossy().to_string(),
130                bytes: meta.len(),
131            });
132        }
133    }
134    Ok(())
135}
136
137/// Read a file as lossy UTF-8, returning None when it is unreadable or
138/// contains NUL bytes in the first 8 KiB (a cheap binary check).
139pub fn read_text(path: &Path) -> Option<String> {
140    let bytes = std::fs::read(path).ok()?;
141    if bytes.iter().take(8192).any(|&b| b == 0) {
142        return None;
143    }
144    Some(String::from_utf8_lossy(&bytes).into_owned())
145}
146
147#[cfg(test)]
148mod tests {
149    use super::*;
150
151    /// A minimal PowerShell-Docs-shaped fixture: reference markdown,
152    /// about topics, conceptual guides, and noise that must be skipped.
153    fn docs_fixture(dir: &Path) {
154        std::fs::create_dir_all(dir.join("reference/7.4/Microsoft.PowerShell.Core/About")).unwrap();
155        std::fs::create_dir_all(dir.join("reference/docs-conceptual/learn")).unwrap();
156        std::fs::create_dir_all(dir.join("tests")).unwrap();
157        std::fs::create_dir_all(dir.join(".github")).unwrap();
158        std::fs::create_dir_all(dir.join("media")).unwrap();
159        std::fs::create_dir_all(dir.join("redir")).unwrap();
160        std::fs::create_dir_all(dir.join("reference/bread")).unwrap();
161        std::fs::create_dir_all(dir.join("reference/mapping")).unwrap();
162        std::fs::write(
163            dir.join("reference/7.4/Microsoft.PowerShell.Core/Get-Command.md"),
164            "---\ntitle: Get-Command\n---\n\n# Get-Command\n\nGets all commands.\n",
165        )
166        .unwrap();
167        std::fs::write(
168            dir.join("reference/7.4/Microsoft.PowerShell.Core/About/about_Arrays.md"),
169            "# about_Arrays\n\nDescribes arrays.\n",
170        )
171        .unwrap();
172        std::fs::write(
173            dir.join("reference/docs-conceptual/learn/learn-powershell.md"),
174            "# Learn PowerShell\n\nA tutorial.\n",
175        )
176        .unwrap();
177        std::fs::write(dir.join("README.md"), "# PowerShell Docs\n").unwrap();
178        // Noise the scanner must skip.
179        std::fs::write(dir.join("CODE_OF_CONDUCT.md"), "conduct\n").unwrap();
180        std::fs::write(dir.join("SECURITY.md"), "security\n").unwrap();
181        std::fs::write(dir.join("tests/unit.md"), "test docs\n").unwrap();
182        std::fs::write(dir.join(".github/pull_request_template.md"), "pr\n").unwrap();
183        std::fs::write(dir.join("media/logo.svg"), "<svg/>\n").unwrap();
184        std::fs::write(dir.join("reference/bread/toc.yml"), "items:\n").unwrap();
185        std::fs::write(dir.join("reference/mapping/monikerMapping.json"), "{}\n").unwrap();
186        std::fs::write(dir.join("redir/redirect.md"), "redirect\n").unwrap();
187        std::fs::write(dir.join("notes.txt"), "plain text\n").unwrap();
188    }
189
190    #[test]
191    fn scanner_finds_reference_markdown_and_skips_noise() {
192        let dir = tempfile::tempdir().expect("tempdir");
193        let root = dir.path();
194        docs_fixture(root);
195
196        let files = scan_project(root).unwrap();
197        let paths: Vec<&str> = files.iter().map(|f| f.rel_path.as_str()).collect();
198        assert_eq!(
199            paths,
200            vec![
201                "README.md",
202                "reference/7.4/Microsoft.PowerShell.Core/About/about_Arrays.md",
203                "reference/7.4/Microsoft.PowerShell.Core/Get-Command.md",
204                "reference/docs-conceptual/learn/learn-powershell.md",
205            ],
206            "only docs markdown is indexed; CI, tests, media, redirects, non-md files, and \
207             contributor documents must be skipped"
208        );
209    }
210
211    #[test]
212    fn read_text_rejects_binary() {
213        let dir = tempfile::tempdir().expect("tempdir");
214        let p = dir.path().join("blob.md");
215        let mut bytes = b"# heading\n".to_vec();
216        bytes.push(0);
217        std::fs::write(&p, bytes).unwrap();
218        assert!(read_text(&p).is_none());
219    }
220}