kardo_core/scanner/

discovery.rs

//! File discovery: walk a project directory, respect .gitignore, hash content.

use std::fs;
use std::path::{Path, PathBuf};
use std::time::SystemTime;

use chrono::{DateTime, Utc};
use ignore::WalkBuilder;
use serde::Serialize;
use sha2::{Digest, Sha256};

/// Extensions considered relevant for documentation analysis.
const DOC_EXTENSIONS: &[&str] = &[
    "md", "txt", "yml", "yaml", "json", "toml", "rst", "adoc",
];

/// Special filename stems always included regardless of extension.
const SPECIAL_NAMES: &[&str] = &[
    "README",
    "CHANGELOG",
    "LICENSE",
    "CONTRIBUTING",
    "CLAUDE",
    "AGENTS",
    "SECURITY",
    "CODE_OF_CONDUCT",
    ".cursorrules",
    ".clinerules",
];

/// Directory prefixes that are always scanned for doc files.
const DOC_DIRECTORIES: &[&str] = &[
    "docs/",
    "doc/",
    "documentation/",
    ".claude/",
];

/// A file discovered during project scanning.
#[derive(Debug, Clone, Serialize)]
pub struct DiscoveredFile {
    /// Absolute path to the file.
    pub path: PathBuf,
    /// Path relative to the project root.
    pub relative_path: String,
    /// File size in bytes.
    pub size: u64,
    /// Last modification time.
    pub modified_at: Option<DateTime<Utc>>,
    /// File extension (lowercase, without dot).
    pub extension: Option<String>,
    /// Whether this is a Markdown file.
    pub is_markdown: bool,
    /// SHA-256 hash of file content.
    pub content_hash: String,
}

/// Project file scanner. Walks directory respecting .gitignore.
pub struct FileDiscovery {
    project_root: PathBuf,
    excludes: Vec<String>,
    includes: Vec<String>,
    use_gitignore: bool,
    custom_ignore_file: Option<PathBuf>,
}

impl FileDiscovery {
    /// Create a new scanner for the given project root.
    pub fn new(project_root: impl AsRef<Path>) -> Self {
        Self {
            project_root: project_root.as_ref().to_path_buf(),
            excludes: Vec::new(),
            includes: Vec::new(),
            use_gitignore: true,
            custom_ignore_file: None,
        }
    }

    /// Add glob patterns to exclude from scanning.
    pub fn with_excludes(mut self, excludes: Vec<String>) -> Self {
        self.excludes = excludes;
        self
    }

    /// Add glob patterns to include (whitelist mode, or override excludes).
    pub fn with_includes(mut self, includes: Vec<String>) -> Self {
        self.includes = includes;
        self
    }

    /// Enable or disable .gitignore processing.
    pub fn with_gitignore(mut self, use_gitignore: bool) -> Self {
        self.use_gitignore = use_gitignore;
        self
    }

    /// Add a custom ignore file path (e.g. a shared .kardo.ignore).
    pub fn with_custom_ignore_file(mut self, path: impl AsRef<Path>) -> Self {
        self.custom_ignore_file = Some(path.as_ref().to_path_buf());
        self
    }

    /// Scan project directory, respecting .gitignore and filtering rules.
    /// Returns all relevant documentation files sorted by relative_path.
    pub fn discover(&self) -> Result<Vec<DiscoveredFile>, ScanError> {
        let project_root = self
            .project_root
            .canonicalize()
            .map_err(|e| ScanError::Io(format!("Cannot resolve project root: {}", e)))?;

        let mut builder = WalkBuilder::new(&project_root);
        builder
            .hidden(false)
            .git_ignore(self.use_gitignore)
            .git_global(self.use_gitignore)
            .git_exclude(self.use_gitignore)
            .follow_links(false);

        // Layer 3: auto-detect .kardo.ignore in any directory
        builder.add_custom_ignore_filename(".kardo.ignore");

        // Custom ignore file (--ignore-path)
        if let Some(ref ignore_path) = self.custom_ignore_file {
            if let Some(err) = builder.add_ignore(ignore_path) {
                return Err(ScanError::Io(format!(
                    "Cannot read ignore file '{}': {}",
                    ignore_path.display(),
                    err
                )));
            }
        }

        // Layer 4: CLI --exclude / --include via overrides
        if !self.excludes.is_empty() || !self.includes.is_empty() {
            let mut ob = ignore::overrides::OverrideBuilder::new(&project_root);

            // When both excludes and includes are present, add catch-all whitelist
            // so excludes act as blacklist (not whitelist-only mode)
            if !self.excludes.is_empty() && !self.includes.is_empty() {
                ob.add("**").map_err(|e| {
                    ScanError::Io(format!("Failed to add catch-all override: {}", e))
                })?;
            }

            for pattern in &self.excludes {
                let negated = format!("!{}", pattern);
                ob.add(&negated).map_err(|e| {
                    ScanError::Io(format!("Invalid exclude pattern '{}': {}", pattern, e))
                })?;
            }
            for pattern in &self.includes {
                ob.add(pattern).map_err(|e| {
                    ScanError::Io(format!("Invalid include pattern '{}': {}", pattern, e))
                })?;
            }

            let overrides = ob
                .build()
                .map_err(|e| ScanError::Io(format!("Failed to build overrides: {}", e)))?;
            builder.overrides(overrides);
        }

        let walker = builder.build();

        let mut files = Vec::new();

        for entry in walker {
            let entry = match entry {
                Ok(e) => e,
                Err(err) => {
                    log::warn!("Skipping entry: {}", err);
                    continue;
                }
            };

            if entry.file_type().map_or(true, |ft| !ft.is_file()) {
                continue;
            }

            let abs_path = entry.path().to_path_buf();
            let relative = abs_path
                .strip_prefix(&project_root)
                .unwrap_or(&abs_path)
                .to_string_lossy()
                .to_string();

            if !Self::is_relevant_file(&abs_path, &relative) {
                continue;
            }

            match Self::build_discovered_file(&abs_path, &relative) {
                Ok(df) => files.push(df),
                Err(err) => {
                    log::warn!("Skipping {}: {}", relative, err);
                }
            }
        }

        files.sort_by(|a, b| a.relative_path.cmp(&b.relative_path));
        Ok(files)
    }

    /// Check if a file is relevant for documentation analysis.
    fn is_relevant_file(abs_path: &Path, relative: &str) -> bool {
        // Skip .kardo directory
        if relative.starts_with(".kardo/") || relative.starts_with(".kardo\\") {
            return false;
        }

        // Files inside doc directories are always relevant
        for dir in DOC_DIRECTORIES {
            if relative.starts_with(dir) {
                return true;
            }
        }

        // Check special filename stems (case-insensitive on stem)
        let file_name = abs_path
            .file_name()
            .and_then(|n| n.to_str())
            .unwrap_or("");
        let stem = abs_path
            .file_stem()
            .and_then(|s| s.to_str())
            .unwrap_or("");

        for special in SPECIAL_NAMES {
            if file_name.eq_ignore_ascii_case(special)
                || stem.eq_ignore_ascii_case(special)
            {
                return true;
            }
        }

        // Check extension
        if let Some(ext) = abs_path.extension().and_then(|e| e.to_str()) {
            let ext_lower = ext.to_lowercase();
            return DOC_EXTENSIONS.contains(&ext_lower.as_str());
        }

        false
    }

    /// Compute SHA-256 hex hash of a file's contents.
    fn hash_file(path: &Path) -> Result<String, ScanError> {
        let content = fs::read(path)
            .map_err(|e| ScanError::Io(format!("Cannot read {}: {}", path.display(), e)))?;
        let mut hasher = Sha256::new();
        hasher.update(&content);
        Ok(format!("{:x}", hasher.finalize()))
    }

    /// Build a DiscoveredFile from a path, computing hash and metadata.
    fn build_discovered_file(abs_path: &Path, relative: &str) -> Result<DiscoveredFile, ScanError> {
        let metadata = fs::metadata(abs_path)
            .map_err(|e| ScanError::Io(format!("Cannot read metadata for {}: {}", relative, e)))?;

        let size = metadata.len();

        let modified_at: Option<DateTime<Utc>> = metadata.modified().ok().and_then(|t| {
            t.duration_since(SystemTime::UNIX_EPOCH)
                .ok()
                .and_then(|d| {
                    DateTime::from_timestamp(d.as_secs() as i64, d.subsec_nanos())
                })
        });

        let extension = abs_path
            .extension()
            .and_then(|e| e.to_str())
            .map(|e| e.to_lowercase());

        let is_markdown = extension.as_deref() == Some("md");

        let content_hash = Self::hash_file(abs_path)?;

        Ok(DiscoveredFile {
            path: abs_path.to_path_buf(),
            relative_path: relative.to_string(),
            size,
            modified_at,
            extension,
            is_markdown,
            content_hash,
        })
    }
}

/// Errors during file scanning.
#[derive(Debug, thiserror::Error)]
pub enum ScanError {
    #[error("IO error: {0}")]
    Io(String),
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs;
    use tempfile::TempDir;

    fn init_git_repo(root: &Path) {
        fs::create_dir_all(root.join(".git")).unwrap();
        fs::write(root.join(".git/HEAD"), "ref: refs/heads/main\n").unwrap();
        fs::create_dir_all(root.join(".git/objects")).unwrap();
        fs::create_dir_all(root.join(".git/refs")).unwrap();
        fs::write(
            root.join(".git/config"),
            "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n",
        )
        .unwrap();
    }

    fn create_test_project(tmp: &TempDir) {
        let root = tmp.path();
        init_git_repo(root);

        fs::write(root.join("README.md"), "# Test Project\n").unwrap();
        fs::write(root.join("CLAUDE.md"), "# Claude instructions\n").unwrap();
        fs::create_dir_all(root.join("docs")).unwrap();
        fs::write(root.join("docs/guide.md"), "# Guide\nSome content").unwrap();
        fs::write(root.join("docs/config.yml"), "key: value\n").unwrap();
        fs::write(root.join("docs/data.json"), "{}").unwrap();
        fs::write(root.join("docs/settings.toml"), "[settings]\n").unwrap();

        // Non-doc files
        fs::create_dir_all(root.join("src")).unwrap();
        fs::write(root.join("src/main.rs"), "fn main() {}").unwrap();
        fs::write(root.join("src/lib.rs"), "pub mod foo;").unwrap();

        // .gitignore
        fs::write(root.join(".gitignore"), "target/\n*.log\n").unwrap();

        // Ignored files
        fs::create_dir_all(root.join("target")).unwrap();
        fs::write(root.join("target/debug.log"), "log").unwrap();
        fs::write(root.join("build.log"), "log").unwrap();
    }

    #[test]
    fn test_discover_finds_doc_files() {
        let tmp = TempDir::new().unwrap();
        create_test_project(&tmp);

        let scanner = FileDiscovery::new(tmp.path());
        let files = scanner.discover().unwrap();
        let rel_paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();

        assert!(rel_paths.contains(&"README.md"), "Missing README.md, found: {:?}", rel_paths);
        assert!(rel_paths.contains(&"CLAUDE.md"), "Missing CLAUDE.md, found: {:?}", rel_paths);
        assert!(rel_paths.contains(&"docs/guide.md"));
        assert!(rel_paths.contains(&"docs/config.yml"));
        assert!(rel_paths.contains(&"docs/data.json"));
        assert!(rel_paths.contains(&"docs/settings.toml"));

        // Non-doc files excluded
        assert!(!rel_paths.contains(&"src/main.rs"));
        assert!(!rel_paths.contains(&"src/lib.rs"));
    }

    #[test]
    fn test_gitignore_respected() {
        let tmp = TempDir::new().unwrap();
        create_test_project(&tmp);

        let scanner = FileDiscovery::new(tmp.path());
        let files = scanner.discover().unwrap();
        let rel_paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();

        assert!(!rel_paths.contains(&"target/debug.log"));
        assert!(!rel_paths.contains(&"build.log"));
    }

    #[test]
    fn test_is_relevant_file_filter() {
        // Doc extensions
        assert!(FileDiscovery::is_relevant_file(Path::new("guide.md"), "guide.md"));
        assert!(FileDiscovery::is_relevant_file(Path::new("notes.txt"), "notes.txt"));
        assert!(FileDiscovery::is_relevant_file(Path::new("doc.rst"), "doc.rst"));
        assert!(FileDiscovery::is_relevant_file(Path::new("doc.adoc"), "doc.adoc"));

        // Non-doc extensions
        assert!(!FileDiscovery::is_relevant_file(Path::new("main.rs"), "main.rs"));
        assert!(!FileDiscovery::is_relevant_file(Path::new("app.py"), "app.py"));
        assert!(!FileDiscovery::is_relevant_file(Path::new("style.css"), "style.css"));

        // Special names
        assert!(FileDiscovery::is_relevant_file(Path::new("README"), "README"));
        assert!(FileDiscovery::is_relevant_file(Path::new("README.md"), "README.md"));
        assert!(FileDiscovery::is_relevant_file(Path::new("LICENSE"), "LICENSE"));
        assert!(FileDiscovery::is_relevant_file(Path::new("CLAUDE.md"), "CLAUDE.md"));
        assert!(FileDiscovery::is_relevant_file(Path::new("AGENTS.md"), "AGENTS.md"));
        assert!(FileDiscovery::is_relevant_file(Path::new(".cursorrules"), ".cursorrules"));
        assert!(FileDiscovery::is_relevant_file(Path::new(".clinerules"), ".clinerules"));

        // .kardo excluded
        assert!(!FileDiscovery::is_relevant_file(
            Path::new(".kardo/config.json"),
            ".kardo/config.json"
        ));
    }

    #[test]
    fn test_content_hashing() {
        let tmp = TempDir::new().unwrap();
        init_git_repo(tmp.path());

        // Same content = same hash
        fs::write(tmp.path().join("a.md"), "identical content").unwrap();
        fs::write(tmp.path().join("b.md"), "identical content").unwrap();
        // Different content = different hash
        fs::write(tmp.path().join("c.md"), "different content").unwrap();

        let scanner = FileDiscovery::new(tmp.path());
        let files = scanner.discover().unwrap();

        let a = files.iter().find(|f| f.relative_path == "a.md").unwrap();
        let b = files.iter().find(|f| f.relative_path == "b.md").unwrap();
        let c = files.iter().find(|f| f.relative_path == "c.md").unwrap();

        assert_eq!(a.content_hash, b.content_hash);
        assert_ne!(a.content_hash, c.content_hash);

        // Verify hash format: 64 hex chars
        assert_eq!(a.content_hash.len(), 64);
        assert!(a.content_hash.chars().all(|ch| ch.is_ascii_hexdigit()));
    }

    #[test]
    fn test_relative_paths_correct() {
        let tmp = TempDir::new().unwrap();
        init_git_repo(tmp.path());

        fs::create_dir_all(tmp.path().join("docs/deep/nested")).unwrap();
        fs::write(tmp.path().join("docs/deep/nested/file.md"), "deep").unwrap();
        fs::write(tmp.path().join("README.md"), "root").unwrap();

        let scanner = FileDiscovery::new(tmp.path());
        let files = scanner.discover().unwrap();

        let readme = files.iter().find(|f| f.relative_path == "README.md").unwrap();
        assert!(readme.path.is_absolute());

        let nested = files.iter().find(|f| f.relative_path == "docs/deep/nested/file.md").unwrap();
        assert!(nested.path.is_absolute());
        assert!(!nested.relative_path.starts_with('/'));
    }

    #[test]
    fn test_hidden_claude_dir_discovered() {
        let tmp = TempDir::new().unwrap();
        init_git_repo(tmp.path());

        fs::create_dir_all(tmp.path().join(".claude")).unwrap();
        fs::write(tmp.path().join(".claude/instructions"), "AI instructions").unwrap();
        fs::create_dir_all(tmp.path().join(".claude/research")).unwrap();
        fs::write(tmp.path().join(".claude/research/notes.md"), "notes").unwrap();

        let scanner = FileDiscovery::new(tmp.path());
        let files = scanner.discover().unwrap();
        let rel_paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();

        assert!(
            rel_paths.contains(&".claude/instructions"),
            "Missing .claude/instructions, found: {:?}",
            rel_paths
        );
        assert!(rel_paths.contains(&".claude/research/notes.md"));
    }

    #[test]
    fn test_kardo_dir_excluded() {
        let tmp = TempDir::new().unwrap();
        init_git_repo(tmp.path());

        fs::create_dir_all(tmp.path().join(".kardo")).unwrap();
        fs::write(tmp.path().join(".kardo/kardo.db"), "fake db").unwrap();
        fs::write(tmp.path().join(".kardo/config.json"), "{}").unwrap();
        fs::write(tmp.path().join("README.md"), "hi").unwrap();

        let scanner = FileDiscovery::new(tmp.path());
        let files = scanner.discover().unwrap();
        let rel_paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();

        assert!(!rel_paths.iter().any(|p| p.starts_with(".kardo/")));
    }

    #[test]
    fn test_file_metadata() {
        let tmp = TempDir::new().unwrap();
        init_git_repo(tmp.path());

        fs::write(tmp.path().join("README.md"), "# Hello\n").unwrap();

        let scanner = FileDiscovery::new(tmp.path());
        let files = scanner.discover().unwrap();
        let readme = files.iter().find(|f| f.relative_path == "README.md").unwrap();

        assert!(readme.is_markdown);
        assert_eq!(readme.extension.as_deref(), Some("md"));
        assert_eq!(readme.size, 8); // "# Hello\n"
        assert!(readme.modified_at.is_some());
        assert!(readme.path.is_absolute());
    }

    #[test]
    fn test_sorted_output() {
        let tmp = TempDir::new().unwrap();
        init_git_repo(tmp.path());

        fs::write(tmp.path().join("z-file.md"), "z").unwrap();
        fs::write(tmp.path().join("a-file.md"), "a").unwrap();
        fs::create_dir_all(tmp.path().join("m")).unwrap();
        fs::write(tmp.path().join("m/middle.md"), "m").unwrap();

        let scanner = FileDiscovery::new(tmp.path());
        let files = scanner.discover().unwrap();
        let rel_paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();

        let mut sorted = rel_paths.clone();
        sorted.sort();
        assert_eq!(rel_paths, sorted);
    }

    #[test]
    fn test_custom_kardo_ignore_file() {
        let tmp = TempDir::new().unwrap();
        let root = tmp.path();
        init_git_repo(root);

        fs::write(root.join("README.md"), "# Project").unwrap();
        fs::create_dir_all(root.join(".claude/research/calibration")).unwrap();
        fs::write(root.join(".claude/research/calibration/data.md"), "cal").unwrap();
        fs::write(root.join(".claude/research/notes.md"), "notes").unwrap();

        // Create .kardo.ignore that excludes calibration dir
        fs::write(root.join(".kardo.ignore"), ".claude/research/calibration/\n").unwrap();

        let scanner = FileDiscovery::new(root);
        let files = scanner.discover().unwrap();
        let rel_paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();

        assert!(rel_paths.contains(&"README.md"));
        assert!(rel_paths.contains(&".claude/research/notes.md"));
        assert!(
            !rel_paths.contains(&".claude/research/calibration/data.md"),
            "calibration data should be excluded by .kardo.ignore, found: {:?}",
            rel_paths
        );
    }

    #[test]
    fn test_exclude_pattern_via_builder() {
        let tmp = TempDir::new().unwrap();
        let root = tmp.path();
        init_git_repo(root);

        fs::write(root.join("README.md"), "# Project").unwrap();
        fs::create_dir_all(root.join(".claude/research")).unwrap();
        fs::write(root.join(".claude/research/notes.md"), "notes").unwrap();
        fs::create_dir_all(root.join("docs")).unwrap();
        fs::write(root.join("docs/guide.md"), "guide").unwrap();

        let scanner = FileDiscovery::new(root)
            .with_excludes(vec![".claude/research/**".into()]);
        let files = scanner.discover().unwrap();
        let rel_paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();

        assert!(rel_paths.contains(&"README.md"));
        assert!(rel_paths.contains(&"docs/guide.md"));
        assert!(
            !rel_paths.contains(&".claude/research/notes.md"),
            "research notes should be excluded, found: {:?}",
            rel_paths
        );
    }

    #[test]
    fn test_include_overrides_exclude() {
        let tmp = TempDir::new().unwrap();
        let root = tmp.path();
        init_git_repo(root);

        fs::write(root.join("README.md"), "# Project").unwrap();
        fs::create_dir_all(root.join("docs")).unwrap();
        fs::write(root.join("docs/guide.md"), "guide").unwrap();
        fs::write(root.join("docs/important.md"), "important").unwrap();
        fs::write(root.join("docs/draft.md"), "draft").unwrap();

        let scanner = FileDiscovery::new(root)
            .with_excludes(vec!["docs/**".into()])
            .with_includes(vec!["docs/important.md".into()]);
        let files = scanner.discover().unwrap();
        let rel_paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();

        // README.md should still be included (catch-all ** protects it)
        assert!(
            rel_paths.contains(&"README.md"),
            "README.md should be included, found: {:?}",
            rel_paths
        );
        // docs/important.md should survive despite docs/** exclude
        assert!(
            rel_paths.contains(&"docs/important.md"),
            "important.md should override exclude, found: {:?}",
            rel_paths
        );
        // Other docs should be excluded
        assert!(!rel_paths.contains(&"docs/guide.md"));
        assert!(!rel_paths.contains(&"docs/draft.md"));
    }

    #[test]
    fn test_no_gitignore_flag() {
        let tmp = TempDir::new().unwrap();
        let root = tmp.path();
        init_git_repo(root);

        fs::write(root.join(".gitignore"), "ignored.md\n").unwrap();
        fs::write(root.join("README.md"), "# Project").unwrap();
        fs::write(root.join("ignored.md"), "this is ignored by git").unwrap();

        // With gitignore (default) — ignored.md should not appear
        let with_gi = FileDiscovery::new(root);
        let files = with_gi.discover().unwrap();
        let paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();
        assert!(!paths.contains(&"ignored.md"), "should be gitignored");

        // Without gitignore — ignored.md should appear
        let without_gi = FileDiscovery::new(root).with_gitignore(false);
        let files = without_gi.discover().unwrap();
        let paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();
        assert!(
            paths.contains(&"ignored.md"),
            "should appear with gitignore disabled, found: {:?}",
            paths
        );
    }

    #[test]
    fn test_builder_backward_compat() {
        let tmp = TempDir::new().unwrap();
        create_test_project(&tmp);

        // Old way: FileDiscovery::new(&root).discover()
        let scanner = FileDiscovery::new(tmp.path());
        let files = scanner.discover().unwrap();
        let rel_paths: Vec<&str> = files.iter().map(|f| f.relative_path.as_str()).collect();

        // Should behave identically to before: find doc files, respect gitignore
        assert!(rel_paths.contains(&"README.md"));
        assert!(rel_paths.contains(&"CLAUDE.md"));
        assert!(rel_paths.contains(&"docs/guide.md"));
        assert!(!rel_paths.contains(&"src/main.rs"));
        assert!(!rel_paths.contains(&"target/debug.log"));
        assert!(!rel_paths.contains(&"build.log"));
    }
}