1use anyhow::{Context, Result};
11use std::path::Path;
12
13pub const DEFAULT_EXTS: &[&str] = &["md", "markdown"];
15
16pub const EXCLUDED_DIRS: &[&str] = &[
19 ".git",
20 "target",
21 "node_modules",
22 ".venv",
23 "venv",
24 "__pycache__",
25 "dist",
26 "build",
27 "out",
28 ".idea",
29 ".vscode",
30 ".devcontainer",
31 ".opengrok",
32 ".rag-index",
33 "graphify-out",
34 ".cache",
35 "vendor",
36 "bin",
37 "artifacts",
38 "coverage",
39 ".next",
40 ".nuxt",
41 ".svelte-kit",
42 "Pods",
43 ".terraform",
44 ".github",
47 "tests",
48 "assets",
49 "redir",
50 "media",
51 "bread",
52 "mapping",
53];
54
55pub const EXCLUDED_FILES: &[&str] = &[
58 "Cargo.lock",
59 "package-lock.json",
60 "yarn.lock",
61 "pnpm-lock.yaml",
62 "poetry.lock",
63 "Pipfile.lock",
64 "uv.lock",
65 "composer.lock",
66 "Gemfile.lock",
67 "go.sum",
68 "CODE_OF_CONDUCT.md",
71 "CONTRIBUTING.md",
72 "SECURITY.md",
73 "ThirdPartyNotices.md",
74 "LICENSE.md",
75 "LICENSE-CODE.md",
76];
77
78pub const MAX_FILE_BYTES: u64 = 2 * 1024 * 1024;
80
81#[derive(Debug, Clone)]
83pub struct ScannedFile {
84 pub rel_path: String,
86 pub bytes: u64,
88}
89
90pub fn scan_project(root: &Path) -> Result<Vec<ScannedFile>> {
93 let mut files = Vec::new();
94 scan_dir(root, root, &mut files)?;
95 files.sort_by(|a, b| a.rel_path.cmp(&b.rel_path));
96 Ok(files)
97}
98
99fn scan_dir(root: &Path, dir: &Path, out: &mut Vec<ScannedFile>) -> Result<()> {
100 let entries = std::fs::read_dir(dir).with_context(|| format!("read dir {}", dir.display()))?;
101 for entry in entries {
102 let entry = entry.with_context(|| format!("read entry in {}", dir.display()))?;
103 let path = entry.path();
104 let name = entry.file_name();
105 let name_str = name.to_string_lossy();
106 if path.is_dir() {
107 if name_str.starts_with('.') || EXCLUDED_DIRS.contains(&name_str.as_ref()) {
108 continue;
109 }
110 scan_dir(root, &path, out)?;
111 } else if path.is_file() {
112 if name_str.starts_with('.') || EXCLUDED_FILES.contains(&name_str.as_ref()) {
113 continue;
114 }
115 let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
116 if !DEFAULT_EXTS.contains(&ext) {
117 continue;
118 }
119 let meta = entry
120 .metadata()
121 .with_context(|| format!("metadata {}", path.display()))?;
122 if meta.len() > MAX_FILE_BYTES {
123 continue;
124 }
125 let rel = path
126 .strip_prefix(root)
127 .with_context(|| format!("strip prefix for {}", path.display()))?;
128 out.push(ScannedFile {
129 rel_path: rel.to_string_lossy().to_string(),
130 bytes: meta.len(),
131 });
132 }
133 }
134 Ok(())
135}
136
137pub fn read_text(path: &Path) -> Option<String> {
140 let bytes = std::fs::read(path).ok()?;
141 if bytes.iter().take(8192).any(|&b| b == 0) {
142 return None;
143 }
144 Some(String::from_utf8_lossy(&bytes).into_owned())
145}
146
147#[cfg(test)]
148mod tests {
149 use super::*;
150
151 fn docs_fixture(dir: &Path) {
154 std::fs::create_dir_all(dir.join("reference/7.4/Microsoft.PowerShell.Core/About")).unwrap();
155 std::fs::create_dir_all(dir.join("reference/docs-conceptual/learn")).unwrap();
156 std::fs::create_dir_all(dir.join("tests")).unwrap();
157 std::fs::create_dir_all(dir.join(".github")).unwrap();
158 std::fs::create_dir_all(dir.join("media")).unwrap();
159 std::fs::create_dir_all(dir.join("redir")).unwrap();
160 std::fs::create_dir_all(dir.join("reference/bread")).unwrap();
161 std::fs::create_dir_all(dir.join("reference/mapping")).unwrap();
162 std::fs::write(
163 dir.join("reference/7.4/Microsoft.PowerShell.Core/Get-Command.md"),
164 "---\ntitle: Get-Command\n---\n\n# Get-Command\n\nGets all commands.\n",
165 )
166 .unwrap();
167 std::fs::write(
168 dir.join("reference/7.4/Microsoft.PowerShell.Core/About/about_Arrays.md"),
169 "# about_Arrays\n\nDescribes arrays.\n",
170 )
171 .unwrap();
172 std::fs::write(
173 dir.join("reference/docs-conceptual/learn/learn-powershell.md"),
174 "# Learn PowerShell\n\nA tutorial.\n",
175 )
176 .unwrap();
177 std::fs::write(dir.join("README.md"), "# PowerShell Docs\n").unwrap();
178 std::fs::write(dir.join("CODE_OF_CONDUCT.md"), "conduct\n").unwrap();
180 std::fs::write(dir.join("SECURITY.md"), "security\n").unwrap();
181 std::fs::write(dir.join("tests/unit.md"), "test docs\n").unwrap();
182 std::fs::write(dir.join(".github/pull_request_template.md"), "pr\n").unwrap();
183 std::fs::write(dir.join("media/logo.svg"), "<svg/>\n").unwrap();
184 std::fs::write(dir.join("reference/bread/toc.yml"), "items:\n").unwrap();
185 std::fs::write(dir.join("reference/mapping/monikerMapping.json"), "{}\n").unwrap();
186 std::fs::write(dir.join("redir/redirect.md"), "redirect\n").unwrap();
187 std::fs::write(dir.join("notes.txt"), "plain text\n").unwrap();
188 }
189
190 #[test]
191 fn scanner_finds_reference_markdown_and_skips_noise() {
192 let dir = tempfile::tempdir().expect("tempdir");
193 let root = dir.path();
194 docs_fixture(root);
195
196 let files = scan_project(root).unwrap();
197 let paths: Vec<&str> = files.iter().map(|f| f.rel_path.as_str()).collect();
198 assert_eq!(
199 paths,
200 vec![
201 "README.md",
202 "reference/7.4/Microsoft.PowerShell.Core/About/about_Arrays.md",
203 "reference/7.4/Microsoft.PowerShell.Core/Get-Command.md",
204 "reference/docs-conceptual/learn/learn-powershell.md",
205 ],
206 "only docs markdown is indexed; CI, tests, media, redirects, non-md files, and \
207 contributor documents must be skipped"
208 );
209 }
210
211 #[test]
212 fn read_text_rejects_binary() {
213 let dir = tempfile::tempdir().expect("tempdir");
214 let p = dir.path().join("blob.md");
215 let mut bytes = b"# heading\n".to_vec();
216 bytes.push(0);
217 std::fs::write(&p, bytes).unwrap();
218 assert!(read_text(&p).is_none());
219 }
220}