pleme-doc-gen 0.1.54

//! file_capture — typed byte-perfect file absorption for caixa.
//!
//! Per the operator's "absorb complete software into caixa lisp"
//! directive: the substrate extends from metadata-only round-trip to
//! whole-tree content round-trip. Every text file in the repo (under
//! filters + size cap) gets captured into a typed CapturedFile with
//! content + sha256; render reconstitutes byte-identical.
//!
//! What this captures:
//!   - All text files under repo root (UTF-8 decodable)
//!   - Up to per-file size cap (default 1MB)
//!   - Filtered to skip vendored / build / VCS artifacts
//!
//! What this does NOT capture:
//!   - Binary files (PNG, executables, .tar.gz, etc.)
//!   - Files matching common build-output / vendor patterns
//!   - Files exceeding the per-file size cap
//!
//! Round-trip semantics: original repo → reverse-with-capture →
//! .caixa.lisp containing :files [...] → render → byte-identical
//! working copy of the captured files (manifest + workflows + tests
//! still flow through the typed scaffolders).

use anyhow::{anyhow, Result};
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};

/// Typed representation of one captured file. Stored in Caixa as part
/// of the :files [...] slot.
#[derive(Debug, Clone)]
pub struct CapturedFile {
    /// Path relative to the repo root, forward-slash separated.
    pub path: String,
    /// SHA-256 of the raw bytes, hex-encoded.
    pub sha256: String,
    /// Byte count of the original content.
    pub size: usize,
    /// UTF-8 decoded content. Files that fail UTF-8 decode are
    /// skipped (captured as None at the walker layer).
    pub body: String,
}

/// Typed representation of one captured symlink. Restored via the
/// OS's symlink API. Per the stress-test finding on ripgrep's
/// `HomebrewFormula -> pkg/brew` + helix's `contrib/themes ->
/// ../runtime/themes`: symlinks are project structure that the
/// substrate must preserve for byte-perfect round-trip.
#[derive(Debug, Clone)]
pub struct CapturedSymlink {
    /// Path of the symlink itself, relative to repo root.
    pub path: String,
    /// Target the symlink points to (as-stored, may be relative).
    pub target: String,
}

/// Typed representation of a non-UTF-8 / binary file. Captured as
/// base64-encoded bytes when CaptureConfig.include_binaries is true.
/// Promotes the "encoding fixture / PNG / font silently skipped"
/// failure into a first-class typed pattern — any byte sequence can
/// flow through the substrate's typed-lisp surface.
#[derive(Debug, Clone)]
pub struct CapturedBinary {
    pub path: String,
    pub sha256: String,
    pub size: usize,
    /// Base64-encoded raw bytes (standard alphabet, no padding-elision).
    pub base64: String,
}

/// Configuration knobs for the file walker. Defaults are conservative
/// to keep .caixa.lisp sizes tractable + avoid leaking vendored content.
#[derive(Debug, Clone)]
pub struct CaptureConfig {
    /// Per-file byte cap. Files larger than this are skipped.
    pub max_file_bytes: usize,
    /// Whole-tree cap — stop capturing once we've collected this many
    /// total bytes. Prevents unbounded .caixa.lisp growth.
    pub max_total_bytes: usize,
    /// Directory names anywhere in the path that signal vendored /
    /// build content. Skipped wholesale. RESERVED for names that are
    /// NEVER legitimate as source modules (.git, node_modules,
    /// .pytest_cache, __pycache__, .venv, .terraform, .direnv, etc.).
    pub skip_dirs: Vec<String>,
    /// Directory names that mean "build output" ONLY when at the
    /// repo root — same name appearing as a source module deeper in
    /// the tree is preserved. E.g. `target/` at root = cargo build
    /// dir; `src/target/` = a real source module (cc-rs::target).
    pub skip_root_dirs: Vec<String>,
    /// File extensions that are definitely binary; skipped without
    /// reading the content (unless include_binaries is true).
    pub skip_extensions: Vec<String>,
    /// When true, files that fail UTF-8 decode OR have skip_extensions
    /// OR contain NUL bytes are captured into :binaries via base64
    /// rather than dropped. Extends absorption to PNGs, fonts,
    /// encoding fixtures, archives — anything bytewise representable.
    /// Default false to keep lisp sources tractable.
    pub include_binaries: bool,
}

impl Default for CaptureConfig {
    fn default() -> Self {
        Self {
            // 8 MB per file — large enough for generated unicode /
            // encoding tables (unicode-width's tables_test.rs is 1.36 MB;
            // ICU CLDR tables can hit 5 MB) while still rejecting
            // accidental .json corpus checkins.
            max_file_bytes: 8 * 1024 * 1024,
            max_total_bytes: 100 * 1024 * 1024, // 100 MB total
            // anywhere-denylist — directories that NEVER legitimately
            // appear as source modules in any project.
            skip_dirs: ["node_modules", ".git", "vendor",
                        ".venv", "venv", "__pycache__", ".pytest_cache",
                        ".idea", ".vscode", "Pods",
                        ".terraform", ".direnv", ".nuxt", ".next",
                        ".svelte-kit", ".turbo", ".parcel-cache"]
                .iter().map(|s| s.to_string()).collect(),
            // root-only denylist — names that are build-output at
            // repo root but VALID source modules deeper in the tree
            // (cc-rs::src/target is a real module; target/ at root
            // is cargo's build dir).
            skip_root_dirs: ["target", "dist", "build", "out", "_build"]
                .iter().map(|s| s.to_string()).collect(),
            skip_extensions: ["png", "jpg", "jpeg", "gif", "ico", "svg",
                              "pdf", "zip", "tar", "gz", "bz2", "xz",
                              "exe", "dll", "so", "dylib", "a", "o",
                              "wasm", "class", "jar", "war",
                              "mp3", "mp4", "mov", "wav", "ogg",
                              "ttf", "woff", "woff2", "eot"]
                .iter().map(|s| s.to_string()).collect(),
            include_binaries: false,
        }
    }
}

/// Walk `root`, capture every eligible file. Returns the typed list
/// plus aggregate stats.
pub fn capture(root: &Path, cfg: &CaptureConfig) -> Result<CaptureReport> {
    let mut report = CaptureReport::default();
    walk_dir(root, root, cfg, &mut report)?;
    Ok(report)
}

#[derive(Debug, Default, Clone)]
pub struct CaptureReport {
    pub files: Vec<CapturedFile>,
    pub symlinks: Vec<CapturedSymlink>,
    pub binaries: Vec<CapturedBinary>,
    pub total_bytes: usize,
    pub skipped_binary: usize,
    pub skipped_too_large: usize,
    pub skipped_total_cap: usize,
    pub skipped_dir: usize,
}

fn walk_dir(
    root: &Path,
    dir: &Path,
    cfg: &CaptureConfig,
    report: &mut CaptureReport,
) -> Result<()> {
    let entries = match std::fs::read_dir(dir) {
        Ok(e) => e,
        Err(_) => return Ok(()),
    };
    for entry in entries.flatten() {
        let path = entry.path();
        let name = entry.file_name();
        let name_str = name.to_string_lossy();
        // Detect symlinks via symlink_metadata (does NOT follow the
        // link). Capture as typed CapturedSymlink instead of
        // descending into / reading the target — preserves project
        // structure without double-capture or cycles.
        if let Ok(ftype) = entry.file_type() {
            if ftype.is_symlink() {
                if let Ok(target) = std::fs::read_link(&path) {
                    let rel = path.strip_prefix(root).unwrap_or(&path);
                    let rel_str = rel.to_string_lossy().replace('\\', "/");
                    let target_str = target.to_string_lossy().to_string();
                    report.symlinks.push(CapturedSymlink {
                        path: rel_str,
                        target: target_str,
                    });
                }
                continue;
            }
        }
        let meta = match entry.metadata() {
            Ok(m) => m,
            Err(_) => continue,
        };
        if meta.is_dir() {
            // skip_dirs is the canonical anywhere-denylist. Hidden
            // dirs (.git, .pytest_cache, .venv) belong here
            // explicitly; any hidden dir NOT in skip_dirs (.github,
            // .cargo, .vscode-extensions) is captured.
            if cfg.skip_dirs.iter().any(|s| s.as_str() == name_str.as_ref()) {
                report.skipped_dir += 1;
                continue;
            }
            // skip_root_dirs is the root-only denylist for build-
            // output names that are valid source modules deeper.
            // (cc-rs ships `src/target/` as a module; cargo creates
            // `target/` at root as build output — same name, very
            // different semantics.)
            if dir == root
                && cfg.skip_root_dirs.iter().any(|s| s.as_str() == name_str.as_ref())
            {
                report.skipped_dir += 1;
                continue;
            }
            walk_dir(root, &path, cfg, report)?;
            continue;
        }
        // Hidden files (.gitignore, .gitattributes, .env.example, .editorconfig,
        // .dockerignore, etc.) ARE captured — these are real project artifacts.
        if !meta.is_file() {
            continue;
        }
        let size = meta.len() as usize;
        if size > cfg.max_file_bytes {
            report.skipped_too_large += 1;
            continue;
        }
        if report.total_bytes + size > cfg.max_total_bytes {
            report.skipped_total_cap += 1;
            continue;
        }
        // Extension check — opt-in capture as binary when configured.
        let is_skip_ext = path.extension().and_then(|e| e.to_str())
            .map(|ext| cfg.skip_extensions.iter().any(|s| s.eq_ignore_ascii_case(ext)))
            .unwrap_or(false);
        if is_skip_ext && !cfg.include_binaries {
            report.skipped_binary += 1;
            continue;
        }
        // Read raw bytes — may be binary, may be UTF-8.
        let bytes = match std::fs::read(&path) {
            Ok(b) => b,
            Err(_) => continue,
        };
        // Branch on UTF-8 decodability + NUL-bytes.
        let has_nul = bytes.contains(&0u8);
        let body_utf8 = if !has_nul && !is_skip_ext {
            String::from_utf8(bytes.clone()).ok()
        } else {
            None
        };
        if body_utf8.is_none() {
            // Either non-UTF-8 OR contains NUL OR is in skip_extensions.
            // Capture as binary when include_binaries=true; else skip.
            if !cfg.include_binaries {
                report.skipped_binary += 1;
                continue;
            }
            let rel = path.strip_prefix(root).unwrap_or(&path);
            let rel_str = rel.to_string_lossy().replace('\\', "/");
            use base64::Engine;
            let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
            let mut hasher = Sha256::new();
            hasher.update(&bytes);
            let sha256 = format!("{:x}", hasher.finalize());
            report.total_bytes += size;
            report.binaries.push(CapturedBinary {
                path: rel_str,
                sha256,
                size,
                base64: b64,
            });
            continue;
        }
        let body = body_utf8.unwrap();
        let rel = path.strip_prefix(root).unwrap_or(&path);
        let rel_str = rel.to_string_lossy().replace('\\', "/");
        let mut hasher = Sha256::new();
        hasher.update(&bytes);
        let sha256 = format!("{:x}", hasher.finalize());
        report.total_bytes += size;
        report.files.push(CapturedFile {
            path: rel_str,
            sha256,
            size,
            body,
        });
    }
    Ok(())
}

/// Compute a sha256 hex digest of the given bytes. Public so render +
/// fidelity can call it for content-match verification.
pub fn hash_bytes(bytes: &[u8]) -> String {
    let mut hasher = Sha256::new();
    hasher.update(bytes);
    format!("{:x}", hasher.finalize())
}

/// Write all captured files into `out` directory, preserving relative
/// paths. Returns the list of paths actually written.
pub fn restore(out: &Path, files: &[CapturedFile]) -> Result<Vec<PathBuf>> {
    let mut written = Vec::new();
    for f in files {
        let dest = out.join(&f.path);
        if let Some(parent) = dest.parent() {
            std::fs::create_dir_all(parent)?;
        }
        std::fs::write(&dest, f.body.as_bytes())?;
        written.push(dest);
    }
    Ok(written)
}

/// Restore captured binaries — base64 decode + write raw bytes. Same
/// overlay semantics as restore() (later writes overwrite earlier).
pub fn restore_binaries(out: &Path, binaries: &[CapturedBinary]) -> Result<Vec<PathBuf>> {
    use base64::Engine;
    let mut written = Vec::new();
    for b in binaries {
        let dest = out.join(&b.path);
        if let Some(parent) = dest.parent() {
            std::fs::create_dir_all(parent)?;
        }
        let bytes = base64::engine::general_purpose::STANDARD.decode(&b.base64)
            .map_err(|e| anyhow!("base64 decode {}: {e}", b.path))?;
        std::fs::write(&dest, &bytes)?;
        written.push(dest);
    }
    Ok(written)
}

/// Restore captured symlinks at their original paths. Idempotent —
/// skip when a path already exists. Uses OS-appropriate symlink API.
pub fn restore_symlinks(out: &Path, symlinks: &[CapturedSymlink]) -> Result<Vec<PathBuf>> {
    let mut written = Vec::new();
    for s in symlinks {
        let dest = out.join(&s.path);
        if dest.exists() { continue; }
        if let Some(parent) = dest.parent() {
            std::fs::create_dir_all(parent)?;
        }
        #[cfg(unix)]
        std::os::unix::fs::symlink(&s.target, &dest)?;
        #[cfg(windows)]
        {
            // Best-effort on Windows: directory_symlink. Operators on
            // Windows generally have privilege issues with symlinks;
            // we silently fall through if creation fails.
            let _ = std::os::windows::fs::symlink_dir(&s.target, &dest)
                .or_else(|_| std::os::windows::fs::symlink_file(&s.target, &dest));
        }
        written.push(dest);
    }
    Ok(written)
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs;

    fn mk(files: &[(&str, &str)]) -> tempdir::TempDir {
        let tmp = tempdir::TempDir::new("fc").unwrap();
        for (rel, body) in files {
            let p = tmp.path().join(rel);
            if let Some(parent) = p.parent() { fs::create_dir_all(parent).unwrap(); }
            fs::write(&p, body).unwrap();
        }
        tmp
    }

    #[test]
    fn captures_text_files_with_correct_sha256() {
        let dir = mk(&[
            ("Cargo.toml", "[package]\nname=\"x\"\n"),
            ("src/lib.rs", "fn main() {}"),
        ]);
        let cfg = CaptureConfig::default();
        let r = capture(dir.path(), &cfg).unwrap();
        assert_eq!(r.files.len(), 2);
        let paths: Vec<&str> = r.files.iter().map(|f| f.path.as_str()).collect();
        assert!(paths.contains(&"Cargo.toml"));
        assert!(paths.contains(&"src/lib.rs"));
        // Verify sha256 is computed correctly.
        let cargo = r.files.iter().find(|f| f.path == "Cargo.toml").unwrap();
        assert_eq!(cargo.sha256, hash_bytes(b"[package]\nname=\"x\"\n"));
    }

    #[test]
    fn skips_target_node_modules_git_dirs() {
        let dir = mk(&[
            ("src/lib.rs", "// real"),
            ("target/debug/something.rs", "// build artifact"),
            ("node_modules/foo/index.js", "// dep"),
            (".git/HEAD", "ref: refs/heads/main"),
        ]);
        let r = capture(dir.path(), &CaptureConfig::default()).unwrap();
        let paths: Vec<&str> = r.files.iter().map(|f| f.path.as_str()).collect();
        assert_eq!(paths, vec!["src/lib.rs"]);
        assert!(r.skipped_dir >= 3);
    }

    #[test]
    fn captures_hidden_files_but_skips_hidden_dirs() {
        let dir = mk(&[
            ("src/lib.rs", "// real"),
            (".gitignore", "target/\n*.log\n"),
            (".editorconfig", "[*]\nindent_style = space\n"),
            (".git/HEAD", "ref: refs/heads/main"),
        ]);
        let r = capture(dir.path(), &CaptureConfig::default()).unwrap();
        let mut paths: Vec<&str> = r.files.iter().map(|f| f.path.as_str()).collect();
        paths.sort();
        assert_eq!(paths, vec![".editorconfig", ".gitignore", "src/lib.rs"]);
    }

    #[test]
    fn skips_binary_extensions_without_reading() {
        let dir = mk(&[
            ("logo.png", "fake png bytes"),
            ("README.md", "real text"),
        ]);
        let r = capture(dir.path(), &CaptureConfig::default()).unwrap();
        let paths: Vec<&str> = r.files.iter().map(|f| f.path.as_str()).collect();
        assert_eq!(paths, vec!["README.md"]);
        assert!(r.skipped_binary >= 1);
    }

    #[test]
    fn skips_files_with_nul_bytes() {
        let dir = mk(&[
            ("data.dat", "before\0after"),
            ("README.md", "real"),
        ]);
        let r = capture(dir.path(), &CaptureConfig::default()).unwrap();
        let paths: Vec<&str> = r.files.iter().map(|f| f.path.as_str()).collect();
        assert_eq!(paths, vec!["README.md"]);
    }

    #[test]
    fn restore_writes_byte_identical_files() {
        let dir = mk(&[
            ("a/b.txt", "hello world\n"),
            ("c.md", "# title\n\nbody"),
        ]);
        let r = capture(dir.path(), &CaptureConfig::default()).unwrap();
        let restore_dir = tempdir::TempDir::new("restore").unwrap();
        let written = restore(restore_dir.path(), &r.files).unwrap();
        assert_eq!(written.len(), 2);
        let a = fs::read_to_string(restore_dir.path().join("a/b.txt")).unwrap();
        let c = fs::read_to_string(restore_dir.path().join("c.md")).unwrap();
        assert_eq!(a, "hello world\n");
        assert_eq!(c, "# title\n\nbody");
    }

    #[test]
    fn per_file_size_cap_skips_large_files() {
        let dir = mk(&[
            ("small.txt", "tiny"),
            ("big.txt", &"x".repeat(2_000_000)),
        ]);
        let mut cfg = CaptureConfig::default();
        cfg.max_file_bytes = 1024;
        let r = capture(dir.path(), &cfg).unwrap();
        assert_eq!(r.files.len(), 1);
        assert_eq!(r.skipped_too_large, 1);
    }
}