Skip to main content

ask/sources/
mod.rs

1//! Source adapters — fetch docs from github / npm / llms-txt (web is deferred)
2//! and materialize them into the global store. Rust port of
3//! `packages/cli/src/sources/`.
4//!
5//! Shared here: the [`DocFile`]/[`FetchResult`] value types and the doc-file
6//! collection helpers reused by the npm and github adapters.
7
8pub mod github;
9pub mod llms_txt;
10pub mod npm;
11
12use std::path::{Path, PathBuf};
13
14use crate::store::EntryFile;
15
16/// A documentation file: a store-relative path (forward-slash separated) and its
17/// text content.
18#[derive(Debug, Clone, PartialEq, Eq)]
19pub struct DocFile {
20    pub path: String,
21    pub content: String,
22}
23
24impl DocFile {
25    fn to_entry(&self) -> EntryFile {
26        EntryFile {
27            path: self.path.clone(),
28            content: self.content.clone(),
29        }
30    }
31}
32
33/// Convert docs to store entry files.
34pub fn to_entry_files(files: &[DocFile]) -> Vec<EntryFile> {
35    files.iter().map(DocFile::to_entry).collect()
36}
37
38/// Source-specific metadata propagated to the resolved cache / lockfile.
39#[derive(Debug, Clone, Default, PartialEq, Eq)]
40pub struct FetchMeta {
41    /// GitHub commit sha (40 hex chars).
42    pub commit: Option<String>,
43    /// GitHub ref used (tag or branch name).
44    pub ref_: Option<String>,
45    /// npm Subresource Integrity hash from `dist.integrity`.
46    pub integrity: Option<String>,
47    /// npm tarball URL (omitted for local `node_modules` reads).
48    pub tarball: Option<String>,
49    /// Absolute path to the local `node_modules/<pkg>` dir when read locally.
50    pub install_path: Option<PathBuf>,
51    /// web / llms-txt source URL(s).
52    pub urls: Option<Vec<String>>,
53}
54
55/// The result of a source fetch (port of `FetchResult`).
56#[derive(Debug, Clone)]
57pub struct FetchResult {
58    pub files: Vec<DocFile>,
59    pub resolved_version: String,
60    /// Absolute path to the finalized store entry.
61    pub store_path: Option<PathBuf>,
62    /// Relative docs subpath inside `store_path` (github `docsPath`; empty
63    /// elsewhere).
64    pub store_subpath: Option<String>,
65    /// True when served from an existing verified store entry without a fetch.
66    pub from_store_cache: bool,
67    pub meta: FetchMeta,
68}
69
70/// Doc-file extensions collected from a docs directory.
71const DOC_EXTS: &[&str] = &["md", "mdx", "txt", "rst"];
72
73/// npm tarball docs-path auto-detect candidates (includes `dist/docs`).
74pub const NPM_DOC_CANDIDATES: &[&str] = &[
75    "docs",
76    "doc",
77    "dist/docs",
78    "documentation",
79    "guide",
80    "guides",
81];
82/// github checkout docs-path auto-detect candidates.
83pub const GITHUB_DOC_CANDIDATES: &[&str] = &["docs", "doc", "documentation", "guide", "guides"];
84
85/// Whether `name` has a documentation extension (case-insensitive).
86pub fn is_doc_file(name: &str) -> bool {
87    match name.rsplit_once('.') {
88        Some((_, ext)) => DOC_EXTS.contains(&ext.to_ascii_lowercase().as_str()),
89        None => false,
90    }
91}
92
93/// Recursively collect doc files under `current`, returning paths relative to
94/// `base` (forward-slash separated), sorted for determinism.
95pub fn collect_doc_files(base: &Path, current: &Path) -> std::io::Result<Vec<DocFile>> {
96    let mut files = Vec::new();
97    // Canonicalized base for the per-file symlink-escape guard in collect_into.
98    // `None` (base itself unresolvable) leaves non-symlink collection working.
99    let canon_base = std::fs::canonicalize(base).ok();
100    collect_into(base, canon_base.as_deref(), current, &mut files)?;
101    files.sort_by(|a, b| a.path.cmp(&b.path));
102    Ok(files)
103}
104
105fn collect_into(
106    base: &Path,
107    canon_base: Option<&Path>,
108    current: &Path,
109    out: &mut Vec<DocFile>,
110) -> std::io::Result<()> {
111    for entry in std::fs::read_dir(current)? {
112        let entry = entry?;
113        let path = entry.path();
114        let file_type = entry.file_type()?;
115        if file_type.is_dir() {
116            collect_into(base, canon_base, &path, out)?;
117        } else if is_doc_file(&entry.file_name().to_string_lossy()) {
118            // A symlinked doc file can point outside the docs root; the root-only
119            // containment guard in extract_docs_from_dir does not cover per-file
120            // links. Verify a symlink's real target stays within the canonical
121            // base before reading, so a checkout can't exfiltrate arbitrary files.
122            // Non-symlink files are physically inside the tree — no check needed,
123            // so normal collection stays byte-identical.
124            if file_type.is_symlink() {
125                match (canon_base, std::fs::canonicalize(&path)) {
126                    (Some(cb), Ok(real)) if real.starts_with(cb) => {}
127                    _ => continue, // escapes base, or unresolvable → skip
128                }
129            }
130            let rel = path.strip_prefix(base).unwrap_or(&path);
131            let rel_str = rel
132                .components()
133                .map(|c| c.as_os_str().to_string_lossy())
134                .collect::<Vec<_>>()
135                .join("/");
136            let content = std::fs::read_to_string(&path)?;
137            out.push(DocFile {
138                path: rel_str,
139                content,
140            });
141        }
142    }
143    Ok(())
144}
145
146/// First candidate that exists as a directory under `dir`, or `None`.
147pub fn detect_docs_path(dir: &Path, candidates: &[&str]) -> Option<String> {
148    candidates
149        .iter()
150        .find(|c| dir.join(c).is_dir())
151        .map(|c| c.to_string())
152}
153
154/// Resolve the docs directory (`docs_path` or auto-detected) inside `extracted`,
155/// guard against traversal via realpath containment, and collect its doc files
156/// (or a single file). Port of `extractDocsFromDir` / npm's equivalent.
157pub fn extract_docs_from_dir(
158    extracted: &Path,
159    repo: &str,
160    ref_: &str,
161    docs_path: Option<&str>,
162    candidates: &[&str],
163) -> anyhow::Result<Vec<DocFile>> {
164    let target = match docs_path.map(str::to_string).or_else(|| detect_docs_path(extracted, candidates)) {
165        Some(t) => t,
166        None => anyhow::bail!(
167            "No docs directory found in {repo}@{ref_}. Specify --path to point to the docs directory."
168        ),
169    };
170
171    let docs_dir = extracted.join(&target);
172    // Realpath containment: a symlink inside the checkout pointing outside must
173    // not let us read arbitrary files.
174    let resolved_base = std::fs::canonicalize(extracted)?;
175    let resolved_docs = if docs_dir.exists() {
176        std::fs::canonicalize(&docs_dir)?
177    } else {
178        docs_dir.clone()
179    };
180    if !resolved_docs.starts_with(&resolved_base) {
181        anyhow::bail!("Unsafe docsPath '{target}': must be a relative path within the repository");
182    }
183    if !docs_dir.exists() {
184        anyhow::bail!("Path \"{target}\" not found in {repo}@{ref_}");
185    }
186
187    if docs_dir.is_file() {
188        let content = std::fs::read_to_string(&docs_dir)?;
189        let name = docs_dir
190            .file_name()
191            .map(|n| n.to_string_lossy().into_owned())
192            .unwrap_or_default();
193        return Ok(vec![DocFile {
194            path: name,
195            content,
196        }]);
197    }
198    Ok(collect_doc_files(&docs_dir, &docs_dir)?)
199}
200
201#[cfg(test)]
202mod tests {
203    use super::*;
204
205    #[test]
206    fn is_doc_file_matches_known_extensions() {
207        for ok in ["a.md", "b.MDX", "c.txt", "d.rst"] {
208            assert!(is_doc_file(ok), "{ok}");
209        }
210        for no in ["a.png", "b.js", "noext", "c.mdxx"] {
211            assert!(!is_doc_file(no), "{no}");
212        }
213    }
214
215    #[test]
216    fn collect_and_extract_from_dir() {
217        let dir = tempfile::tempdir().unwrap();
218        let root = dir.path();
219        std::fs::create_dir_all(root.join("docs/sub")).unwrap();
220        std::fs::write(root.join("docs/a.md"), "A").unwrap();
221        std::fs::write(root.join("docs/sub/b.mdx"), "B").unwrap();
222        std::fs::write(root.join("docs/ignore.png"), "x").unwrap();
223
224        let files =
225            extract_docs_from_dir(root, "o/r", "v1", Some("docs"), GITHUB_DOC_CANDIDATES).unwrap();
226        let paths: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect();
227        assert_eq!(paths, vec!["a.md", "sub/b.mdx"]);
228    }
229
230    // A symlinked doc file that escapes the docs root must be skipped (not read),
231    // while an in-tree symlink resolving inside the root is still collected.
232    #[cfg(unix)]
233    #[test]
234    fn collect_skips_escaping_symlink_files() {
235        let dir = tempfile::tempdir().unwrap();
236        let root = dir.path();
237        let outside = root.join("secret.md");
238        std::fs::write(&outside, "TOP SECRET").unwrap();
239        let docs = root.join("docs");
240        std::fs::create_dir_all(&docs).unwrap();
241        std::fs::write(docs.join("real.md"), "REAL").unwrap();
242        // Escaping link: docs/leak.md -> ../secret.md (outside docs root).
243        std::os::unix::fs::symlink("../secret.md", docs.join("leak.md")).unwrap();
244        // In-tree link: docs/alias.md -> real.md (inside docs root).
245        std::os::unix::fs::symlink("real.md", docs.join("alias.md")).unwrap();
246
247        let files = collect_doc_files(&docs, &docs).unwrap();
248        let paths: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect();
249        // leak.md is dropped; real.md and the in-tree alias.md survive.
250        assert_eq!(paths, vec!["alias.md", "real.md"]);
251        assert!(files.iter().all(|f| f.content != "TOP SECRET"));
252    }
253
254    #[test]
255    fn extract_auto_detects_docs_dir() {
256        let dir = tempfile::tempdir().unwrap();
257        std::fs::create_dir_all(dir.path().join("doc")).unwrap();
258        std::fs::write(dir.path().join("doc/x.md"), "X").unwrap();
259        let files =
260            extract_docs_from_dir(dir.path(), "o/r", "v1", None, GITHUB_DOC_CANDIDATES).unwrap();
261        assert_eq!(files.len(), 1);
262    }
263
264    #[test]
265    fn extract_single_file_docs_path() {
266        let dir = tempfile::tempdir().unwrap();
267        std::fs::write(dir.path().join("README.md"), "hi").unwrap();
268        let files = extract_docs_from_dir(
269            dir.path(),
270            "o/r",
271            "v1",
272            Some("README.md"),
273            GITHUB_DOC_CANDIDATES,
274        )
275        .unwrap();
276        assert_eq!(
277            files,
278            vec![DocFile {
279                path: "README.md".into(),
280                content: "hi".into()
281            }]
282        );
283    }
284
285    #[test]
286    fn extract_missing_docs_errors() {
287        let dir = tempfile::tempdir().unwrap();
288        assert!(
289            extract_docs_from_dir(dir.path(), "o/r", "v1", None, GITHUB_DOC_CANDIDATES).is_err()
290        );
291        assert!(extract_docs_from_dir(
292            dir.path(),
293            "o/r",
294            "v1",
295            Some("nope"),
296            GITHUB_DOC_CANDIDATES
297        )
298        .is_err());
299    }
300}