fluidattacks_core_git/

fs.rs

use anyhow::{bail, Context, Result};
use std::ffi::OsString;
use std::fs;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process::Command;
use tempfile::{NamedTempFile, TempDir};
use walkdir::WalkDir;

use crate::cli::{path_from_bytes, path_to_bytes};

const EMPTY_BLOB: &str = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391";

pub fn remove_symlinks(dir: &Path) -> Result<()> {
    for entry in WalkDir::new(dir)
        .min_depth(1)
        .into_iter()
        .filter_map(Result::ok)
    {
        if entry.file_type().is_symlink() {
            remove_if_present(entry.path(), fs::remove_file)?;
        }
    }

    Ok(())
}

pub fn delete_gitignored_files(repo_path: &Path, patterns: &[String]) -> Result<()> {
    // Git refuses `--ignored` without a pattern, and none excludes nothing.
    if patterns.is_empty() {
        return Ok(());
    }
    // Git reports a missing tree as a failed command, without the errno.
    fs::read_dir(repo_path).with_context(|| format!("reading {}", repo_path.display()))?;
    let repo_path = &std::path::absolute(repo_path)
        .with_context(|| format!("resolving {}", repo_path.display()))?;
    for file in excluded_files(repo_path, patterns)? {
        remove_if_present(&repo_path.join(file), fs::remove_file)?;
    }

    remove_empty_dirs(repo_path)?;
    Ok(())
}

fn excluded_files(repo_path: &Path, patterns: &[String]) -> Result<Vec<PathBuf>> {
    let exclusions = exclusions_file(patterns)?;
    let mut exclude_from = OsString::from("--exclude-from=");
    exclude_from.push(exclusions.path());
    let git_dir = empty_git_dir()?;
    index_every_file(git_dir.path(), repo_path)?;

    // Only the index is read, so neither the config nor the `.gitignore`
    // files of `repo_path` add rules.
    let output = Command::new("git")
        .arg("--git-dir")
        .arg(git_dir.path())
        .arg("--work-tree")
        .arg(repo_path)
        // A case-insensitive clone would otherwise match `docs` with `Docs`.
        .args(["-c", "core.ignorecase=false"])
        .args(["ls-files", "-z", "--cached", "--ignored"])
        .arg(exclude_from)
        // Git fails to start in a directory that no longer exists, so it runs inside the tree.
        .current_dir(repo_path)
        .output()
        .context("running git ls-files")?;
    if !output.status.success() {
        bail!(
            "git ls-files failed: {}",
            String::from_utf8_lossy(&output.stderr)
        );
    }

    Ok(output
        .stdout
        .split(|byte| *byte == 0)
        .filter(|file| !file.is_empty())
        .map(path_from_bytes)
        .collect())
}

// `ls-files --others` stops at a nested repository, so every file goes into
// the index instead, by name alone: nothing reads the blob it points at.
fn index_every_file(git_dir: &Path, repo_path: &Path) -> Result<()> {
    let mut entries = Vec::new();
    for entry in WalkDir::new(repo_path)
        .min_depth(1)
        .into_iter()
        .filter_entry(|entry| entry.file_name() != ".git")
        .filter_map(Result::ok)
        .filter(|entry| !entry.file_type().is_dir())
    {
        if let Ok(path) = entry.path().strip_prefix(repo_path) {
            entries.extend_from_slice(format!("100644 {EMPTY_BLOB}\t").as_bytes());
            entries.extend_from_slice(&path_to_bytes(path));
            entries.push(0);
        }
    }
    let mut file = NamedTempFile::new().context("creating the index entries")?;
    file.write_all(&entries)
        .context("writing the index entries")?;

    let output = Command::new("git")
        .arg("--git-dir")
        .arg(git_dir)
        .args(["update-index", "-z", "--index-info"])
        .stdin(file.reopen().context("reading the index entries")?)
        .current_dir(git_dir)
        .output()
        .context("running git update-index")?;
    if !output.status.success() {
        bail!(
            "git update-index failed: {}",
            String::from_utf8_lossy(&output.stderr)
        );
    }
    Ok(())
}

fn empty_git_dir() -> Result<TempDir> {
    let git_dir = TempDir::new().context("creating the git dir")?;
    let output = Command::new("git")
        .args(["init", "-q", "--bare"])
        .arg(git_dir.path())
        .current_dir(git_dir.path())
        .output()
        .context("running git init")?;
    if !output.status.success() {
        bail!(
            "git init failed: {}",
            String::from_utf8_lossy(&output.stderr)
        );
    }
    Ok(git_dir)
}

// From a file git reads each exclusion as a `.gitignore` line, and the
// command line stays short however many there are.
fn exclusions_file(patterns: &[String]) -> Result<NamedTempFile> {
    let mut file = NamedTempFile::new().context("creating the exclusions file")?;
    // A line break would split one exclusion into several rules.
    for pattern in patterns.iter().filter(|pattern| !pattern.contains('\n')) {
        writeln!(file, "{pattern}").context("writing the exclusions file")?;
    }
    Ok(file)
}

pub fn copy_dir_all(src: &Path, dst: &Path) -> Result<()> {
    fs::create_dir_all(dst)?;
    for entry in fs::read_dir(src)? {
        let entry = entry?;
        let ty = entry.file_type()?;
        let dst_path = dst.join(entry.file_name());
        if ty.is_dir() {
            copy_dir_all(&entry.path(), &dst_path)?;
        } else {
            fs::copy(entry.path(), &dst_path)?;
            copy_modified(&entry.path(), &dst_path);
        }
    }
    Ok(())
}

fn copy_modified(src: &Path, dst: &Path) {
    let Ok(modified) = fs::metadata(src).and_then(|meta| meta.modified()) else {
        return;
    };
    if let Ok(file) = fs::File::open(dst) {
        let _ = file.set_modified(modified);
    }
}

fn remove_empty_dirs(root: &Path) -> Result<()> {
    let mut dirs: Vec<_> = Vec::new();

    for entry in WalkDir::new(root) {
        let entry = entry?;
        let path = entry.path();

        let Ok(rel) = path.strip_prefix(root) else {
            continue;
        };

        let rel_str = rel.to_string_lossy();

        if rel_str == ".git" || rel_str.starts_with(".git/") || rel_str.starts_with(".git\\") {
            continue;
        }

        if entry.file_type().is_dir() && path != root {
            dirs.push(path.to_path_buf());
        }
    }

    dirs.sort();
    dirs.reverse();

    for dir in dirs {
        let is_empty = match fs::read_dir(&dir) {
            Ok(mut entries) => entries.next().is_none(),
            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
            Err(error) => {
                return Err(error).with_context(|| format!("reading {}", dir.display()));
            }
        };
        if is_empty {
            remove_if_present(&dir, fs::remove_dir)?;
        }
    }

    Ok(())
}

// A path already gone is what the removal was after.
fn remove_if_present<'a>(path: &'a Path, remove: fn(&'a Path) -> io::Result<()>) -> Result<()> {
    match remove(path) {
        Err(error) if error.kind() != io::ErrorKind::NotFound => {
            Err(error).with_context(|| format!("removing {}", path.display()))
        }
        _ => Ok(()),
    }
}

#[cfg(test)]
mod tests {
    use super::{copy_dir_all, delete_gitignored_files, remove_symlinks};
    use std::time::{Duration, SystemTime};

    // A clone as the download leaves it, with every file in the index.
    fn indexed_repo(files: &[&str]) -> tempfile::TempDir {
        let dir = tempfile::tempdir().unwrap();
        for file in files {
            let path = dir.path().join(file);
            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
            std::fs::write(path, b"").unwrap();
        }
        for args in [&["init", "-q"][..], &["add", "-A"]] {
            let status = std::process::Command::new("git")
                .arg("-C")
                .arg(dir.path())
                .args(args)
                .status()
                .unwrap();
            assert!(status.success());
        }
        dir
    }

    fn exclusions(patterns: &[&str]) -> Vec<String> {
        patterns
            .iter()
            .map(|pattern| (*pattern).to_owned())
            .collect()
    }

    #[test]
    fn a_directory_pattern_deletes_what_is_under_it() {
        let dir = indexed_repo(&["docs/a.md", "docs/sub/b.md", "keep.py"]);
        let repo = dir.path();

        delete_gitignored_files(repo, &exclusions(&["docs"])).unwrap();

        assert!(!repo.join("docs").exists());
        assert!(repo.join("keep.py").exists());
        assert!(repo.join(".git/config").exists());
    }

    #[test]
    fn a_directory_that_is_not_a_repository_is_pruned_too() {
        let dir = tempfile::tempdir().unwrap();
        let root = dir.path();
        std::fs::create_dir_all(root.join("docs/sub")).unwrap();
        std::fs::write(root.join("docs/sub/a.md"), b"").unwrap();
        std::fs::write(root.join("keep.py"), b"").unwrap();

        delete_gitignored_files(root, &exclusions(&["docs"])).unwrap();

        assert!(!root.join("docs").exists());
        assert!(root.join("keep.py").exists());
    }

    #[test]
    fn a_file_in_a_nested_repository_is_deleted_too() {
        let dir = indexed_repo(&["top.log", "inner/a.log", "inner/keep.py"]);
        let repo = dir.path();
        let status = std::process::Command::new("git")
            .args(["init", "-q"])
            .arg(repo.join("inner"))
            .status()
            .unwrap();
        assert!(status.success());

        delete_gitignored_files(repo, &exclusions(&["*.log"])).unwrap();

        assert!(!repo.join("top.log").exists());
        assert!(!repo.join("inner/a.log").exists());
        assert!(repo.join("inner/keep.py").exists());
        assert!(repo.join("inner/.git/HEAD").exists());
    }

    #[test]
    fn the_gitignore_of_the_tree_adds_no_rules() {
        let dir = indexed_repo(&["keep.py", "a.log"]);
        let repo = dir.path();
        std::fs::write(repo.join(".gitignore"), b"*.py\n").unwrap();

        delete_gitignored_files(repo, &exclusions(&["*.log"])).unwrap();

        assert!(repo.join("keep.py").exists());
        assert!(!repo.join("a.log").exists());
    }

    #[test]
    fn no_exclusions_keep_every_file() {
        let dir = indexed_repo(&["keep.py"]);

        delete_gitignored_files(dir.path(), &[]).unwrap();

        assert!(dir.path().join("keep.py").exists());
    }

    #[test]
    fn an_exclusion_that_matches_nothing_does_not_disable_the_others() {
        let dir = indexed_repo(&["docs/a.md", "keep.py"]);
        let repo = dir.path();

        delete_gitignored_files(repo, &exclusions(&["docs", "[z-a]"])).unwrap();

        assert!(!repo.join("docs/a.md").exists());
        assert!(repo.join("keep.py").exists());
    }

    #[test]
    fn an_unclosed_bracket_matches_nothing() {
        let dir = indexed_repo(&["[main", "]"]);
        let repo = dir.path();

        delete_gitignored_files(repo, &exclusions(&["[main", "[]]"])).unwrap();

        assert!(repo.join("[main").exists());
        assert!(!repo.join("]").exists());
    }

    #[test]
    fn braces_match_literally() {
        let dir = indexed_repo(&["a/x", "{a,b}/x", "a}b", "cq"]);
        let repo = dir.path();

        delete_gitignored_files(repo, &exclusions(&["{a,b}/x", "a}b", "c[a-{]"])).unwrap();

        assert!(repo.join("a/x").exists());
        assert!(!repo.join("{a,b}/x").exists());
        assert!(!repo.join("a}b").exists());
        assert!(!repo.join("cq").exists());
    }

    #[test]
    fn a_backslash_takes_the_next_character_literally() {
        let dir = indexed_repo(&["a[{b}", "{a}", "ab"]);
        let repo = dir.path();

        delete_gitignored_files(repo, &exclusions(&[r"a\[{b}", r"\{a}"])).unwrap();

        assert!(!repo.join("a[{b}").exists());
        assert!(!repo.join("{a}").exists());
        assert!(repo.join("ab").exists());
    }

    #[test]
    fn a_named_class_matches_its_characters() {
        let dir = indexed_repo(&["f5", "fx"]);
        let repo = dir.path();

        delete_gitignored_files(repo, &exclusions(&["f[[:digit:]]"])).unwrap();

        assert!(!repo.join("f5").exists());
        assert!(repo.join("fx").exists());
    }

    #[test]
    fn an_exclusion_keeps_its_case_in_a_case_insensitive_clone() {
        let dir = indexed_repo(&["README.md"]);
        let repo = dir.path();
        let status = std::process::Command::new("git")
            .arg("-C")
            .arg(repo)
            .args(["config", "core.ignorecase", "true"])
            .status()
            .unwrap();
        assert!(status.success());

        delete_gitignored_files(repo, &exclusions(&["readme.md"])).unwrap();

        assert!(repo.join("README.md").exists());
    }

    #[test]
    fn an_exclusion_reads_as_a_gitignore_line() {
        let dir = indexed_repo(&["#a", "b", "c"]);
        let repo = dir.path();

        delete_gitignored_files(repo, &exclusions(&["#a", "b ", "c"])).unwrap();

        assert!(repo.join("#a").exists());
        assert!(!repo.join("b").exists());
        assert!(!repo.join("c").exists());
    }

    #[test]
    fn an_untracked_file_is_deleted_too() {
        let dir = indexed_repo(&["docs/a.md"]);
        let repo = dir.path();
        std::fs::write(repo.join("docs/new.md"), b"").unwrap();

        delete_gitignored_files(repo, &exclusions(&["docs"])).unwrap();

        assert!(!repo.join("docs").exists());
    }

    #[test]
    fn an_exclusion_with_a_line_break_adds_no_rules() {
        let dir = indexed_repo(&["a", "b"]);
        let repo = dir.path();

        delete_gitignored_files(repo, &exclusions(&["a\nb"])).unwrap();

        assert!(repo.join("a").exists());
        assert!(repo.join("b").exists());
    }

    #[test]
    fn a_copied_tree_keeps_the_mode_and_the_modification_time() {
        let dir = tempfile::tempdir().unwrap();
        let src = dir.path().join("src/objects");
        std::fs::create_dir_all(&src).unwrap();
        let object = src.join("a1b2c3");
        std::fs::write(&object, b"blob").unwrap();

        let modified = SystemTime::UNIX_EPOCH
            .checked_add(Duration::from_secs(1_700_000_000))
            .unwrap();
        std::fs::File::open(&object)
            .unwrap()
            .set_modified(modified)
            .unwrap();
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt;
            std::fs::set_permissions(&object, std::fs::Permissions::from_mode(0o444)).unwrap();
        }

        let dst = dir.path().join("dst");
        copy_dir_all(&dir.path().join("src"), &dst).unwrap();

        let copied = dst.join("objects/a1b2c3");
        let meta = std::fs::metadata(&copied).unwrap();
        assert_eq!(meta.modified().unwrap(), modified);
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt;
            assert_eq!(meta.permissions().mode() & 0o777, 0o444);
        }
    }

    #[test]
    #[cfg(unix)]
    fn an_unreadable_directory_does_not_stop_the_removal() {
        use std::os::unix::fs::PermissionsExt;

        let dir = tempfile::tempdir().unwrap();
        let target = dir.path().join("target.txt");
        std::fs::write(&target, b"content").unwrap();

        let blocked = dir.path().join("b_blocked");
        std::fs::create_dir(&blocked).unwrap();
        let late = dir.path().join("c_late.lnk");
        std::os::unix::fs::symlink(&target, &late).unwrap();
        std::fs::set_permissions(&blocked, std::fs::Permissions::from_mode(0o000)).unwrap();

        remove_symlinks(dir.path()).unwrap();

        std::fs::set_permissions(&blocked, std::fs::Permissions::from_mode(0o755)).unwrap();
        assert!(!late.is_symlink());
        assert!(target.exists());
    }

    #[test]
    #[cfg(unix)]
    fn a_symlink_that_cannot_be_removed_is_reported() {
        use std::os::unix::fs::PermissionsExt;

        let dir = tempfile::tempdir().unwrap();
        let sealed = dir.path().join("sealed");
        std::fs::create_dir(&sealed).unwrap();
        let link = sealed.join("link.lnk");
        std::os::unix::fs::symlink(dir.path().join("target.txt"), &link).unwrap();
        std::fs::set_permissions(&sealed, std::fs::Permissions::from_mode(0o500)).unwrap();

        // A privileged user, as the ci runner is, writes into the directory
        // regardless of its mode, and then there is no failure to report.
        if std::fs::write(sealed.join("probe"), b"").is_ok() {
            std::fs::set_permissions(&sealed, std::fs::Permissions::from_mode(0o755)).unwrap();
            return;
        }

        let result = remove_symlinks(dir.path());

        std::fs::set_permissions(&sealed, std::fs::Permissions::from_mode(0o755)).unwrap();
        assert!(result.is_err());
        assert!(link.is_symlink());
    }

    #[test]
    #[cfg(unix)]
    fn the_directory_itself_is_never_removed() {
        let dir = tempfile::tempdir().unwrap();
        let real = dir.path().join("real");
        std::fs::create_dir(&real).unwrap();
        let alias = dir.path().join("alias");
        std::os::unix::fs::symlink(&real, &alias).unwrap();

        remove_symlinks(&alias).unwrap();

        assert!(alias.is_symlink());
    }
}