harn-hostlib 0.10.166

//! Filtered directory walker + sensitive-file filter.
//!
//! Depth-first walk that prunes noisy directories before descending and
//! refuses to ingest credential-shaped files. Kept self-contained so the
//! `tools/search` crate can use `ignore` for honoring `.gitignore` while
//! the indexer stays deterministic across systems regardless of the user's
//! ignore configuration.

use std::collections::HashSet;
use std::fs::Metadata;
use std::path::{Path, PathBuf};

use harn_languages::Language;

use super::imports_go::GO_MANIFEST;
use super::state::relative_path_under_root;

/// Maximum file size accepted by the indexer. Larger files are skipped to
/// avoid blowing up the index on minified bundles, generated protobufs, etc.
pub(crate) const MAX_FILE_BYTES: u64 = 2 * 1024 * 1024;

/// Directory names whose entire subtree should be skipped. Matched by
/// basename only.
const SKIP_DIRS: &[&str] = &[
    // VCS
    ".git",
    ".hg",
    ".svn",
    // Package managers / lockfiles / caches
    "node_modules",
    "bower_components",
    "vendor",
    "__pycache__",
    ".pytest_cache",
    ".mypy_cache",
    ".ruff_cache",
    ".venv",
    "venv",
    "env",
    ".tox",
    ".gradle",
    ".idea",
    ".vs",
    ".vscode",
    "target",
    "Pods",
    ".dart_tool",
    ".pub-cache",
    // JS/TS build output
    "dist",
    "build",
    ".next",
    ".nuxt",
    ".svelte-kit",
    ".turbo",
    ".parcel-cache",
    ".cache",
    // Swift / Apple
    ".build",
    "DerivedData",
    ".swiftpm",
    // Test coverage
    "coverage",
    ".nyc_output",
    // Managed host/runtime artifacts
    ".burin",
    ".burin-evals",
    ".burin-live-evals",
    ".harn",
    ".harn-runs",
    // Claude Code
    ".claude",
    // Misc
    ".DS_Store",
    ".tmp",
    ".temp",
    "tmp",
    "temp",
];

const ALLOWED_DOTFILES: &[&str] = &[".env.example", ".gitignore", ".dockerignore"];

/// The registry language of a file the indexer ingests; `None` for any
/// file the registry does not recognize by name or extension.
pub(crate) fn indexable_language(path: &Path) -> Option<Language> {
    Language::from_path(path)
}

pub(crate) fn is_indexable_file(path: &Path) -> bool {
    indexable_language(path).is_some()
}

/// Public alias for [`is_sensitive`] — the `state` module needs to call
/// the same predicate from outside this file.
pub(crate) fn is_sensitive_path(path: &Path) -> bool {
    is_sensitive(path)
}

/// Returns `true` if the path looks like a credentials/secrets file that
/// must never enter the index.
pub(crate) fn is_sensitive(path: &Path) -> bool {
    // Normalize Windows backslashes to `/` before the component scan below so
    // the `split('/')` SENSITIVE_DIRS check sees directory parts on every
    // platform. This mirrors the `to_posix`-style normalization already used
    // elsewhere in this crate (walker.rs, state.rs, fs_watch.rs).
    let lower = path
        .to_string_lossy()
        .replace('\\', "/")
        .to_ascii_lowercase();
    let base = Path::new(&lower)
        .file_name()
        .and_then(|s| s.to_str())
        .unwrap_or_default()
        .to_string();

    if EXACT_SENSITIVE.contains(&base.as_str()) {
        return true;
    }
    if BASE_PREFIXES.iter().any(|p| base.starts_with(p)) {
        return true;
    }
    if BASE_SUFFIXES.iter().any(|s| base.ends_with(s)) {
        return true;
    }
    if BASE_CONTAINS.iter().any(|s| base.contains(s)) {
        return true;
    }
    lower.split('/').any(|part| SENSITIVE_DIRS.contains(&part))
}

const EXACT_SENSITIVE: &[&str] = &[
    ".env",
    ".envrc",
    ".netrc",
    ".pgpass",
    "credentials",
    "credentials.json",
    "credentials.yml",
    "credentials.yaml",
    "secrets",
    "secrets.json",
    "secrets.yml",
    "secrets.yaml",
    "service-account.json",
    "id_rsa",
    "id_dsa",
    "id_ecdsa",
    "id_ed25519",
    "id_rsa.pub",
    "id_dsa.pub",
    "id_ecdsa.pub",
    "id_ed25519.pub",
    "authorized_keys",
    "known_hosts",
];

const BASE_PREFIXES: &[&str] = &[".env.", ".env_", "credentials.", "secrets."];

const BASE_SUFFIXES: &[&str] = &[
    ".pem",
    ".key",
    ".p12",
    ".pfx",
    ".keystore",
    ".jks",
    ".asc",
    ".gpg",
    ".crt",
    ".cer",
];

const BASE_CONTAINS: &[&str] = &[
    "private_key",
    "privatekey",
    "api_key",
    "apikey",
    "secret_key",
    "secretkey",
];

const SENSITIVE_DIRS: &[&str] = &[".ssh", ".aws", ".gnupg"];

/// Walk `root` recursively, yielding every absolute path that passes the
/// indexer's filters together with the `symlink_metadata` the filters
/// already had to read. Handing the metadata back lets an incremental
/// refresh decide "unchanged" from mtime and size without a second
/// `stat` per file. Walks are depth-first; the order within a directory
/// is sorted lexicographically so two runs over identical inputs return
/// identical lists (helpful for snapshot tests).
///
/// Module manifests (`go.mod`) are not indexed, but module resolution
/// reads them, so the walk that already lists every directory reports
/// them to `on_manifest` as workspace-relative paths. That makes a
/// created, edited, or removed manifest as visible to a refresh as a
/// source file, without probing every source directory's ancestors.
pub(crate) fn walk_workspace<F, M>(root: &Path, mut on_file: F, mut on_manifest: M)
where
    F: FnMut(&Path, &Metadata),
    M: FnMut(String),
{
    let mut stack: Vec<PathBuf> = vec![root.to_path_buf()];
    let skip_dirs: HashSet<&str> = SKIP_DIRS.iter().copied().collect();
    while let Some(dir) = stack.pop() {
        let entries = match std::fs::read_dir(&dir) {
            Ok(rd) => rd,
            Err(_) => continue,
        };
        let mut sorted: Vec<PathBuf> = entries.filter_map(|e| e.ok().map(|e| e.path())).collect();
        sorted.sort();
        for entry in sorted {
            let basename = entry
                .file_name()
                .and_then(|s| s.to_str())
                .unwrap_or_default()
                .to_string();
            if should_skip_basename(&basename, &skip_dirs) {
                continue;
            }
            let metadata = match std::fs::symlink_metadata(&entry) {
                Ok(m) => m,
                Err(_) => continue,
            };
            if metadata.file_type().is_symlink() {
                continue;
            }
            if metadata.is_dir() {
                if skip_dirs.contains(basename.as_str()) {
                    continue;
                }
                stack.push(entry);
            } else if metadata.is_file() {
                if basename == GO_MANIFEST {
                    if metadata.len() > MAX_FILE_BYTES || is_sensitive(&entry) {
                        continue;
                    }
                    if let Some(rel) = relative_path_under_root(root, &entry) {
                        on_manifest(rel);
                    }
                    continue;
                }
                if !is_indexable_file(&entry) {
                    continue;
                }
                if metadata.len() > MAX_FILE_BYTES {
                    continue;
                }
                if is_sensitive(&entry) {
                    continue;
                }
                on_file(&entry, &metadata);
            }
        }
    }
}

fn should_skip_basename(name: &str, skip_dirs: &HashSet<&str>) -> bool {
    if name.is_empty() {
        return true;
    }
    if skip_dirs.contains(name) {
        // Directories are filtered later (we still want to test the
        // directory-vs-file check before pruning), so don't short-circuit
        // here — the caller handles directory skipping after the metadata
        // probe.
        return false;
    }
    if name.starts_with('.') && name.len() > 1 {
        if ALLOWED_DOTFILES.contains(&name) {
            return false;
        }
        // Dot-prefixed dirs that aren't in `skip_dirs` (e.g. `.cargo`,
        // `.config`) are still walked — only dot-prefixed *files* are
        // dropped. We can't check
        // file-vs-dir here without another stat, so approximate by
        // returning false; the caller's metadata probe will let the dir
        // through and skip the dotfile via `is_indexable_file` (no
        // matching extension).
        return false;
    }
    false
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs;
    use tempfile::tempdir;

    #[test]
    fn indexable_extensions_cover_common_languages() {
        assert!(is_indexable_file(Path::new("foo.rs")));
        assert!(is_indexable_file(Path::new("foo.swift")));
        assert!(is_indexable_file(Path::new("Foo.SWIFT")));
        assert!(is_indexable_file(Path::new("Dockerfile")));
        assert!(!is_indexable_file(Path::new("foo.bin")));
        assert!(!is_indexable_file(Path::new("README"))); // no extension, not allowlisted
    }

    #[test]
    fn sensitive_filter_rejects_known_shapes() {
        assert!(is_sensitive(Path::new("/repo/.env")));
        assert!(is_sensitive(Path::new("/repo/.env.local")));
        assert!(is_sensitive(Path::new("/repo/secrets.yaml")));
        assert!(is_sensitive(Path::new("/repo/server.pem")));
        assert!(is_sensitive(Path::new("/repo/api_key.txt")));
        assert!(is_sensitive(Path::new("/Users/me/.ssh/id_rsa")));
        assert!(!is_sensitive(Path::new("/repo/src/main.rs")));
    }

    #[test]
    fn walk_skips_pruned_dirs_and_oversize_files() {
        let dir = tempdir().unwrap();
        let root = dir.path();
        fs::create_dir_all(root.join("src")).unwrap();
        fs::create_dir_all(root.join("node_modules/foo")).unwrap();
        fs::create_dir_all(root.join(".git/objects")).unwrap();
        fs::create_dir_all(root.join(".burin-evals/run")).unwrap();
        fs::create_dir_all(root.join(".burin-live-evals/run")).unwrap();
        fs::create_dir_all(root.join(".harn/state")).unwrap();
        fs::create_dir_all(root.join(".harn-runs/session")).unwrap();
        fs::write(root.join("src/main.rs"), b"fn main() {}").unwrap();
        fs::write(root.join("go.mod"), b"module example.com/root\n").unwrap();
        fs::write(root.join("src/go.mod"), b"module example.com/nested\n").unwrap();
        fs::write(root.join("node_modules/foo/go.mod"), b"module skipped\n").unwrap();
        fs::write(root.join("src/.env"), b"SECRET=x").unwrap();
        fs::write(root.join("node_modules/foo/lib.js"), b"x").unwrap();
        fs::write(root.join(".git/objects/pack"), b"git").unwrap();
        fs::write(root.join(".burin-evals/run/transcript.jsonl"), b"{}").unwrap();
        fs::write(root.join(".burin-live-evals/run/live.jsonl"), b"{}").unwrap();
        fs::write(root.join(".harn/state/session.json"), b"{}").unwrap();
        fs::write(root.join(".harn-runs/session/trace.harn"), b"fn noisy() {}").unwrap();
        fs::write(
            root.join("oversize.json"),
            vec![b'a'; (MAX_FILE_BYTES + 1) as usize],
        )
        .unwrap();

        let mut found: Vec<String> = Vec::new();
        let mut manifests: Vec<String> = Vec::new();
        walk_workspace(
            root,
            |p, _meta| {
                found.push(
                    p.strip_prefix(root)
                        .unwrap()
                        .to_string_lossy()
                        .into_owned()
                        .replace('\\', "/"),
                );
            },
            |rel| manifests.push(rel),
        );
        found.sort();
        manifests.sort();
        assert_eq!(found, vec!["src/main.rs"]);
        // Manifests are reported, never indexed, and pruned directories
        // hide theirs exactly as they hide source files.
        assert_eq!(manifests, vec!["go.mod", "src/go.mod"]);
    }

    #[test]
    fn indexed_languages_use_registry_names() {
        let name = |p: &str| indexable_language(Path::new(p)).map(Language::name);
        assert_eq!(name("a.rs"), Some("rust"));
        assert_eq!(name("a.tsx"), Some("tsx"));
        assert_eq!(name("a.hxx"), Some("cpp"));
        assert_eq!(name("Dockerfile"), Some("dockerfile"));
        assert_eq!(name("a.unknown"), None);
    }
}