1pub mod github;
9pub mod llms_txt;
10pub mod npm;
11
12use std::path::{Path, PathBuf};
13
14use crate::store::EntryFile;
15
16#[derive(Debug, Clone, PartialEq, Eq)]
19pub struct DocFile {
20 pub path: String,
21 pub content: String,
22}
23
24impl DocFile {
25 fn to_entry(&self) -> EntryFile {
26 EntryFile {
27 path: self.path.clone(),
28 content: self.content.clone(),
29 }
30 }
31}
32
33pub fn to_entry_files(files: &[DocFile]) -> Vec<EntryFile> {
35 files.iter().map(DocFile::to_entry).collect()
36}
37
38#[derive(Debug, Clone, Default, PartialEq, Eq)]
40pub struct FetchMeta {
41 pub commit: Option<String>,
43 pub ref_: Option<String>,
45 pub integrity: Option<String>,
47 pub tarball: Option<String>,
49 pub install_path: Option<PathBuf>,
51 pub urls: Option<Vec<String>>,
53}
54
55#[derive(Debug, Clone)]
57pub struct FetchResult {
58 pub files: Vec<DocFile>,
59 pub resolved_version: String,
60 pub store_path: Option<PathBuf>,
62 pub store_subpath: Option<String>,
65 pub from_store_cache: bool,
67 pub meta: FetchMeta,
68}
69
70const DOC_EXTS: &[&str] = &["md", "mdx", "txt", "rst"];
72
73pub const NPM_DOC_CANDIDATES: &[&str] = &[
75 "docs",
76 "doc",
77 "dist/docs",
78 "documentation",
79 "guide",
80 "guides",
81];
82pub const GITHUB_DOC_CANDIDATES: &[&str] = &["docs", "doc", "documentation", "guide", "guides"];
84
85pub fn is_doc_file(name: &str) -> bool {
87 match name.rsplit_once('.') {
88 Some((_, ext)) => DOC_EXTS.contains(&ext.to_ascii_lowercase().as_str()),
89 None => false,
90 }
91}
92
93pub fn collect_doc_files(base: &Path, current: &Path) -> std::io::Result<Vec<DocFile>> {
96 let mut files = Vec::new();
97 let canon_base = std::fs::canonicalize(base).ok();
100 collect_into(base, canon_base.as_deref(), current, &mut files)?;
101 files.sort_by(|a, b| a.path.cmp(&b.path));
102 Ok(files)
103}
104
105fn collect_into(
106 base: &Path,
107 canon_base: Option<&Path>,
108 current: &Path,
109 out: &mut Vec<DocFile>,
110) -> std::io::Result<()> {
111 for entry in std::fs::read_dir(current)? {
112 let entry = entry?;
113 let path = entry.path();
114 let file_type = entry.file_type()?;
115 if file_type.is_dir() {
116 collect_into(base, canon_base, &path, out)?;
117 } else if is_doc_file(&entry.file_name().to_string_lossy()) {
118 if file_type.is_symlink() {
125 match (canon_base, std::fs::canonicalize(&path)) {
126 (Some(cb), Ok(real)) if real.starts_with(cb) => {}
127 _ => continue, }
129 }
130 let rel = path.strip_prefix(base).unwrap_or(&path);
131 let rel_str = rel
132 .components()
133 .map(|c| c.as_os_str().to_string_lossy())
134 .collect::<Vec<_>>()
135 .join("/");
136 let content = std::fs::read_to_string(&path)?;
137 out.push(DocFile {
138 path: rel_str,
139 content,
140 });
141 }
142 }
143 Ok(())
144}
145
146pub fn detect_docs_path(dir: &Path, candidates: &[&str]) -> Option<String> {
148 candidates
149 .iter()
150 .find(|c| dir.join(c).is_dir())
151 .map(|c| c.to_string())
152}
153
154pub fn extract_docs_from_dir(
158 extracted: &Path,
159 repo: &str,
160 ref_: &str,
161 docs_path: Option<&str>,
162 candidates: &[&str],
163) -> anyhow::Result<Vec<DocFile>> {
164 let target = match docs_path.map(str::to_string).or_else(|| detect_docs_path(extracted, candidates)) {
165 Some(t) => t,
166 None => anyhow::bail!(
167 "No docs directory found in {repo}@{ref_}. Specify --path to point to the docs directory."
168 ),
169 };
170
171 let docs_dir = extracted.join(&target);
172 let resolved_base = std::fs::canonicalize(extracted)?;
175 let resolved_docs = if docs_dir.exists() {
176 std::fs::canonicalize(&docs_dir)?
177 } else {
178 docs_dir.clone()
179 };
180 if !resolved_docs.starts_with(&resolved_base) {
181 anyhow::bail!("Unsafe docsPath '{target}': must be a relative path within the repository");
182 }
183 if !docs_dir.exists() {
184 anyhow::bail!("Path \"{target}\" not found in {repo}@{ref_}");
185 }
186
187 if docs_dir.is_file() {
188 let content = std::fs::read_to_string(&docs_dir)?;
189 let name = docs_dir
190 .file_name()
191 .map(|n| n.to_string_lossy().into_owned())
192 .unwrap_or_default();
193 return Ok(vec![DocFile {
194 path: name,
195 content,
196 }]);
197 }
198 Ok(collect_doc_files(&docs_dir, &docs_dir)?)
199}
200
201#[cfg(test)]
202mod tests {
203 use super::*;
204
205 #[test]
206 fn is_doc_file_matches_known_extensions() {
207 for ok in ["a.md", "b.MDX", "c.txt", "d.rst"] {
208 assert!(is_doc_file(ok), "{ok}");
209 }
210 for no in ["a.png", "b.js", "noext", "c.mdxx"] {
211 assert!(!is_doc_file(no), "{no}");
212 }
213 }
214
215 #[test]
216 fn collect_and_extract_from_dir() {
217 let dir = tempfile::tempdir().unwrap();
218 let root = dir.path();
219 std::fs::create_dir_all(root.join("docs/sub")).unwrap();
220 std::fs::write(root.join("docs/a.md"), "A").unwrap();
221 std::fs::write(root.join("docs/sub/b.mdx"), "B").unwrap();
222 std::fs::write(root.join("docs/ignore.png"), "x").unwrap();
223
224 let files =
225 extract_docs_from_dir(root, "o/r", "v1", Some("docs"), GITHUB_DOC_CANDIDATES).unwrap();
226 let paths: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect();
227 assert_eq!(paths, vec!["a.md", "sub/b.mdx"]);
228 }
229
230 #[cfg(unix)]
233 #[test]
234 fn collect_skips_escaping_symlink_files() {
235 let dir = tempfile::tempdir().unwrap();
236 let root = dir.path();
237 let outside = root.join("secret.md");
238 std::fs::write(&outside, "TOP SECRET").unwrap();
239 let docs = root.join("docs");
240 std::fs::create_dir_all(&docs).unwrap();
241 std::fs::write(docs.join("real.md"), "REAL").unwrap();
242 std::os::unix::fs::symlink("../secret.md", docs.join("leak.md")).unwrap();
244 std::os::unix::fs::symlink("real.md", docs.join("alias.md")).unwrap();
246
247 let files = collect_doc_files(&docs, &docs).unwrap();
248 let paths: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect();
249 assert_eq!(paths, vec!["alias.md", "real.md"]);
251 assert!(files.iter().all(|f| f.content != "TOP SECRET"));
252 }
253
254 #[test]
255 fn extract_auto_detects_docs_dir() {
256 let dir = tempfile::tempdir().unwrap();
257 std::fs::create_dir_all(dir.path().join("doc")).unwrap();
258 std::fs::write(dir.path().join("doc/x.md"), "X").unwrap();
259 let files =
260 extract_docs_from_dir(dir.path(), "o/r", "v1", None, GITHUB_DOC_CANDIDATES).unwrap();
261 assert_eq!(files.len(), 1);
262 }
263
264 #[test]
265 fn extract_single_file_docs_path() {
266 let dir = tempfile::tempdir().unwrap();
267 std::fs::write(dir.path().join("README.md"), "hi").unwrap();
268 let files = extract_docs_from_dir(
269 dir.path(),
270 "o/r",
271 "v1",
272 Some("README.md"),
273 GITHUB_DOC_CANDIDATES,
274 )
275 .unwrap();
276 assert_eq!(
277 files,
278 vec![DocFile {
279 path: "README.md".into(),
280 content: "hi".into()
281 }]
282 );
283 }
284
285 #[test]
286 fn extract_missing_docs_errors() {
287 let dir = tempfile::tempdir().unwrap();
288 assert!(
289 extract_docs_from_dir(dir.path(), "o/r", "v1", None, GITHUB_DOC_CANDIDATES).is_err()
290 );
291 assert!(extract_docs_from_dir(
292 dir.path(),
293 "o/r",
294 "v1",
295 Some("nope"),
296 GITHUB_DOC_CANDIDATES
297 )
298 .is_err());
299 }
300}