pub mod github;
pub mod llms_txt;
pub mod npm;
use std::path::{Path, PathBuf};
use crate::store::EntryFile;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DocFile {
pub path: String,
pub content: String,
}
impl DocFile {
fn to_entry(&self) -> EntryFile {
EntryFile {
path: self.path.clone(),
content: self.content.clone(),
}
}
}
pub fn to_entry_files(files: &[DocFile]) -> Vec<EntryFile> {
files.iter().map(DocFile::to_entry).collect()
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FetchMeta {
pub commit: Option<String>,
pub ref_: Option<String>,
pub integrity: Option<String>,
pub tarball: Option<String>,
pub install_path: Option<PathBuf>,
pub urls: Option<Vec<String>>,
}
#[derive(Debug, Clone)]
pub struct FetchResult {
pub files: Vec<DocFile>,
pub resolved_version: String,
pub store_path: Option<PathBuf>,
pub store_subpath: Option<String>,
pub from_store_cache: bool,
pub meta: FetchMeta,
}
const DOC_EXTS: &[&str] = &["md", "mdx", "txt", "rst"];
pub const NPM_DOC_CANDIDATES: &[&str] = &[
"docs",
"doc",
"dist/docs",
"documentation",
"guide",
"guides",
];
pub const GITHUB_DOC_CANDIDATES: &[&str] = &["docs", "doc", "documentation", "guide", "guides"];
pub fn is_doc_file(name: &str) -> bool {
match name.rsplit_once('.') {
Some((_, ext)) => DOC_EXTS.contains(&ext.to_ascii_lowercase().as_str()),
None => false,
}
}
pub fn collect_doc_files(base: &Path, current: &Path) -> std::io::Result<Vec<DocFile>> {
let mut files = Vec::new();
let canon_base = std::fs::canonicalize(base).ok();
collect_into(base, canon_base.as_deref(), current, &mut files)?;
files.sort_by(|a, b| a.path.cmp(&b.path));
Ok(files)
}
fn collect_into(
base: &Path,
canon_base: Option<&Path>,
current: &Path,
out: &mut Vec<DocFile>,
) -> std::io::Result<()> {
for entry in std::fs::read_dir(current)? {
let entry = entry?;
let path = entry.path();
let file_type = entry.file_type()?;
if file_type.is_dir() {
collect_into(base, canon_base, &path, out)?;
} else if is_doc_file(&entry.file_name().to_string_lossy()) {
if file_type.is_symlink() {
match (canon_base, std::fs::canonicalize(&path)) {
(Some(cb), Ok(real)) if real.starts_with(cb) => {}
_ => continue, }
}
let rel = path.strip_prefix(base).unwrap_or(&path);
let rel_str = rel
.components()
.map(|c| c.as_os_str().to_string_lossy())
.collect::<Vec<_>>()
.join("/");
let content = std::fs::read_to_string(&path)?;
out.push(DocFile {
path: rel_str,
content,
});
}
}
Ok(())
}
pub fn detect_docs_path(dir: &Path, candidates: &[&str]) -> Option<String> {
candidates
.iter()
.find(|c| dir.join(c).is_dir())
.map(|c| c.to_string())
}
pub fn extract_docs_from_dir(
extracted: &Path,
repo: &str,
ref_: &str,
docs_path: Option<&str>,
candidates: &[&str],
) -> anyhow::Result<Vec<DocFile>> {
let target = match docs_path.map(str::to_string).or_else(|| detect_docs_path(extracted, candidates)) {
Some(t) => t,
None => anyhow::bail!(
"No docs directory found in {repo}@{ref_}. Specify --path to point to the docs directory."
),
};
let docs_dir = extracted.join(&target);
let resolved_base = std::fs::canonicalize(extracted)?;
let resolved_docs = if docs_dir.exists() {
std::fs::canonicalize(&docs_dir)?
} else {
docs_dir.clone()
};
if !resolved_docs.starts_with(&resolved_base) {
anyhow::bail!("Unsafe docsPath '{target}': must be a relative path within the repository");
}
if !docs_dir.exists() {
anyhow::bail!("Path \"{target}\" not found in {repo}@{ref_}");
}
if docs_dir.is_file() {
let content = std::fs::read_to_string(&docs_dir)?;
let name = docs_dir
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
return Ok(vec![DocFile {
path: name,
content,
}]);
}
Ok(collect_doc_files(&docs_dir, &docs_dir)?)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn is_doc_file_matches_known_extensions() {
for ok in ["a.md", "b.MDX", "c.txt", "d.rst"] {
assert!(is_doc_file(ok), "{ok}");
}
for no in ["a.png", "b.js", "noext", "c.mdxx"] {
assert!(!is_doc_file(no), "{no}");
}
}
#[test]
fn collect_and_extract_from_dir() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::create_dir_all(root.join("docs/sub")).unwrap();
std::fs::write(root.join("docs/a.md"), "A").unwrap();
std::fs::write(root.join("docs/sub/b.mdx"), "B").unwrap();
std::fs::write(root.join("docs/ignore.png"), "x").unwrap();
let files =
extract_docs_from_dir(root, "o/r", "v1", Some("docs"), GITHUB_DOC_CANDIDATES).unwrap();
let paths: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect();
assert_eq!(paths, vec!["a.md", "sub/b.mdx"]);
}
#[cfg(unix)]
#[test]
fn collect_skips_escaping_symlink_files() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
let outside = root.join("secret.md");
std::fs::write(&outside, "TOP SECRET").unwrap();
let docs = root.join("docs");
std::fs::create_dir_all(&docs).unwrap();
std::fs::write(docs.join("real.md"), "REAL").unwrap();
std::os::unix::fs::symlink("../secret.md", docs.join("leak.md")).unwrap();
std::os::unix::fs::symlink("real.md", docs.join("alias.md")).unwrap();
let files = collect_doc_files(&docs, &docs).unwrap();
let paths: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect();
assert_eq!(paths, vec!["alias.md", "real.md"]);
assert!(files.iter().all(|f| f.content != "TOP SECRET"));
}
#[test]
fn extract_auto_detects_docs_dir() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir_all(dir.path().join("doc")).unwrap();
std::fs::write(dir.path().join("doc/x.md"), "X").unwrap();
let files =
extract_docs_from_dir(dir.path(), "o/r", "v1", None, GITHUB_DOC_CANDIDATES).unwrap();
assert_eq!(files.len(), 1);
}
#[test]
fn extract_single_file_docs_path() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("README.md"), "hi").unwrap();
let files = extract_docs_from_dir(
dir.path(),
"o/r",
"v1",
Some("README.md"),
GITHUB_DOC_CANDIDATES,
)
.unwrap();
assert_eq!(
files,
vec![DocFile {
path: "README.md".into(),
content: "hi".into()
}]
);
}
#[test]
fn extract_missing_docs_errors() {
let dir = tempfile::tempdir().unwrap();
assert!(
extract_docs_from_dir(dir.path(), "o/r", "v1", None, GITHUB_DOC_CANDIDATES).is_err()
);
assert!(extract_docs_from_dir(
dir.path(),
"o/r",
"v1",
Some("nope"),
GITHUB_DOC_CANDIDATES
)
.is_err());
}
}