ripvec_core/encoder/ripvec/

manifest.rs

//! In-memory manifest tracking indexed files for online reconciliation.
//!
//! Each entry stores cheap stat data — `(mtime, size, inode)` on Unix
//! (`inode = 0` on Windows / unavailable filesystems) — plus a blake3
//! content hash. Reconciliation runs on every search via
//! [`RipvecIndex::diff_against`](super::index::RipvecIndex::diff_against):
//!
//!   1. Walk the corpus with the same [`WalkOptions`] used at index
//!      construction.
//!   2. For each walked file: compare the stat tuple to the manifest
//!      entry. Match → guaranteed-unchanged, skip.
//!   3. For mismatches: read the file, blake3-hash, compare against the
//!      stored hash. Match → metadata-only change (vim save-no-edit,
//!      build-tool touch), update the manifest's stat tuple in place to
//!      short-circuit future diffs. Mismatch → record as `dirty`.
//!   4. Manifest entries not seen during the walk → `deleted`.
//!   5. Walked paths not in the manifest → `new`.
//!
//! If the resulting [`Diff`] is empty, the existing index is up-to-date
//! and no work is needed. Otherwise the caller rebuilds.
//!
//! # Why blake3 + the stat tuple
//!
//! The stat tuple is the cheap pre-filter: warm `stat()` is ~1 µs per
//! file, so the whole tuple check on a 200-file repo is sub-millisecond.
//! Most files won't have a stat change between queries; the cheap path
//! skips them entirely.
//!
//! When the stat tuple *does* mismatch, the question is whether content
//! actually changed. Reading + blake3'ing a typical 1-30 KB source file
//! costs ~1-20 µs warm — two orders of magnitude cheaper than the
//! ~1-5 ms cost of re-chunking and re-embedding it. The break-even is
//! "blake3 is worth it when more than 0.7% of stat changes are touches
//! rather than real edits"; real-world workflows have 5-50% touch rates
//! (vim `:w` with no edits, autoformatters that hash-equal their input,
//! build tools that touch source for dependency tracking).
//!
//! # Inode as a third dimension
//!
//! `(mtime, size)` alone has a rare blind spot: same-byte-count
//! content swaps. Atomic-rename saves (the modern editor default) bump
//! the inode, so adding `inode` to the tuple catches those without a
//! blake3 round-trip. Inode is best-effort: 0 on Windows, where we
//! fall back to `(mtime, size)`. The blake3 verification path still
//! guarantees correctness even when the inode signal is unavailable.

use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::time::SystemTime;

/// One file's tracked state in the manifest.
///
/// Constructed via [`FileEntry::from_bytes`] when the caller already has
/// the file bytes in hand (avoids a redundant read), or via
/// [`FileEntry::from_path`] when only the path is known.
#[derive(Debug, Clone)]
pub struct FileEntry {
    /// Last modification time, or `UNIX_EPOCH` if the platform doesn't
    /// expose it. Used as the first part of the cheap stat-tuple check.
    pub mtime: SystemTime,
    /// File size in bytes, second part of the stat tuple.
    pub size: u64,
    /// File inode number on Unix (`0` on Windows / unavailable). Third
    /// part of the stat tuple; catches atomic-rename saves where mtime
    /// and size could coincide with the previous entry.
    pub ino: u64,
    /// Blake3 content hash. Authoritative — when the stat tuple changes,
    /// this confirms whether content actually changed vs. a touch.
    pub blake3: [u8; 32],
}

impl FileEntry {
    /// Build an entry from filesystem metadata and the file's bytes.
    ///
    /// Use this when the caller has already read the file (e.g., during
    /// chunking) to avoid the redundant read for blake3 hashing.
    #[must_use]
    pub fn from_bytes(metadata: &std::fs::Metadata, bytes: &[u8]) -> Self {
        Self {
            mtime: metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH),
            size: metadata.len(),
            ino: inode(metadata),
            blake3: *blake3::hash(bytes).as_bytes(),
        }
    }

    /// Build an entry by reading the file from disk.
    ///
    /// Reads the file once for blake3. Use [`Self::from_bytes`] if the
    /// caller already has the bytes.
    ///
    /// # Errors
    ///
    /// Returns the I/O error if stat or read fails.
    pub fn from_path(path: &Path) -> std::io::Result<Self> {
        let metadata = std::fs::metadata(path)?;
        let bytes = std::fs::read(path)?;
        Ok(Self::from_bytes(&metadata, &bytes))
    }
}

/// Per-root manifest of indexed files.
///
/// Keys are absolute, canonical paths (matching the paths returned by
/// [`crate::walk::collect_files_with_options`]).
#[derive(Debug, Clone, Default)]
pub struct Manifest {
    pub files: HashMap<PathBuf, FileEntry>,
}

impl Manifest {
    /// Construct an empty manifest.
    #[must_use]
    pub fn new() -> Self {
        Self {
            files: HashMap::new(),
        }
    }

    /// Number of tracked files.
    #[must_use]
    pub fn len(&self) -> usize {
        self.files.len()
    }

    /// Whether the manifest tracks zero files.
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.files.is_empty()
    }

    /// Insert or replace an entry.
    pub fn insert(&mut self, path: PathBuf, entry: FileEntry) {
        self.files.insert(path, entry);
    }

    /// Look up an entry by path.
    #[must_use]
    pub fn get(&self, path: &Path) -> Option<&FileEntry> {
        self.files.get(path)
    }
}

/// Categorized filesystem changes detected by [`diff_against_walk`].
///
/// All three vectors hold absolute paths (matching the walk's output).
/// A [`Diff`] is "empty" only when every list is empty; the
/// [`Self::is_empty`] helper exists to make this the canonical
/// "no-work-needed" check.
///
/// The `touched_clean` field carries files whose stat tuple changed but
/// whose blake3 hash still matches — "touched without content change".
/// These are NOT structural changes (the diff is otherwise empty for
/// them) but the new `(path, FileEntry)` pairs need to be propagated
/// back into the owning index's manifest so that future diffs hit the
/// cheap stat-tuple path rather than re-blake3'ing every time.
///
/// `is_empty()` deliberately ignores `touched_clean`: a diff that
/// contains only touched-clean entries requires no index rebuild.
/// [`crate::encoder::ripvec::index::RipvecIndex::apply_diff`] handles
/// the manifest refresh separately.
#[derive(Debug, Default)]
pub struct Diff {
    /// Files present in both manifest and walk whose content changed.
    pub dirty: Vec<PathBuf>,
    /// Files present in the walk but not in the manifest.
    pub new: Vec<PathBuf>,
    /// Files present in the manifest but not in the walk.
    pub deleted: Vec<PathBuf>,
    /// Files whose stat tuple changed but whose blake3 hash matches
    /// (touch-without-content-change). Each entry carries the refreshed
    /// [`FileEntry`] (new mtime/size/ino, same blake3) so the caller can
    /// update the owning manifest without re-reading the file.
    ///
    /// This field is NOT included in [`Self::is_empty`] or
    /// [`Self::total`] — these files are not structural changes.
    pub touched_clean: Vec<(PathBuf, FileEntry)>,
}

impl Diff {
    /// Whether all *structural* change lists are empty.
    ///
    /// Returns `true` even when `touched_clean` is non-empty — those
    /// files require manifest stat-tuple refreshes but no index rebuild.
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.dirty.is_empty() && self.new.is_empty() && self.deleted.is_empty()
    }

    /// Total number of structurally changed files (dirty + new + deleted).
    ///
    /// Does not count `touched_clean` entries; they are not structural
    /// changes.
    #[must_use]
    pub fn total(&self) -> usize {
        self.dirty.len() + self.new.len() + self.deleted.len()
    }
}

/// Compare the manifest to the current filesystem state and produce a
/// [`Diff`].
///
/// The walked file set is supplied by the caller (typically via
/// [`crate::walk::collect_files_with_options`]) so this function does no
/// I/O for path discovery — only per-file stat and (on stat mismatch)
/// content read for blake3 verification.
///
/// # Mutation of the manifest
///
/// When a file's stat tuple changes but its blake3 hash still matches
/// the manifest entry (the touch-without-content-change case), this
/// function updates the entry's `(mtime, size, ino)` in place. This is
/// not a correctness step — the diff is the same with or without the
/// update — but it short-circuits future diffs on the same touched
/// file: the next call sees the new stat tuple, hits the cheap-path
/// match, and skips the blake3 read.
///
/// # Robustness
///
/// Files that vanish between the walk and the per-file stat (rare race)
/// are silently skipped; they will appear in `deleted` on the next
/// diff. Permission errors are treated similarly. The function never
/// fails — every call returns a [`Diff`].
pub fn diff_against_walk(manifest: &mut Manifest, current_files: &[PathBuf]) -> Diff {
    let mut diff = Diff::default();
    let mut seen: HashSet<&Path> = HashSet::with_capacity(current_files.len());

    for path in current_files {
        seen.insert(path.as_path());
        let Ok(metadata) = std::fs::metadata(path) else {
            // Vanished between walk and stat; let the next diff catch
            // it via the deleted-files pass.
            continue;
        };
        let mtime = metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH);
        let size = metadata.len();
        let ino = inode(&metadata);

        match manifest.files.get(path) {
            None => {
                diff.new.push(path.clone());
            }
            Some(entry) => {
                if entry.mtime == mtime && entry.size == size && entry.ino == ino {
                    // Stat tuple unchanged → content guaranteed
                    // unchanged. The cheap path.
                    continue;
                }
                // Stat changed; blake3 to distinguish real edits from
                // metadata-only touches.
                let Ok(bytes) = std::fs::read(path) else {
                    // Treat permission/read errors conservatively as
                    // dirty so the rebuild path notices.
                    diff.dirty.push(path.clone());
                    continue;
                };
                let new_hash = *blake3::hash(&bytes).as_bytes();
                if new_hash == entry.blake3 {
                    // Touch without content change. Build the refreshed
                    // entry (same blake3, new stat tuple) and record it
                    // in `touched_clean` so callers that work on a
                    // cloned manifest (e.g. `diff_against_filesystem`)
                    // can propagate the refresh even though the in-place
                    // update below affects only the passed-in manifest.
                    let refreshed = FileEntry {
                        mtime,
                        size,
                        ino,
                        blake3: new_hash,
                    };
                    diff.touched_clean.push((path.clone(), refreshed));
                    // Also update the passed-in manifest in place so
                    // direct callers (non-cloned path) hit the cheap
                    // stat-tuple path on the next call. Preserves the
                    // existing behavior for callers that own the manifest.
                    if let Some(entry_mut) = manifest.files.get_mut(path) {
                        entry_mut.mtime = mtime;
                        entry_mut.size = size;
                        entry_mut.ino = ino;
                    }
                } else {
                    diff.dirty.push(path.clone());
                }
            }
        }
    }

    // Manifest entries we didn't visit during the walk → deleted (or
    // filtered out of the walk by changed `WalkOptions`, which the
    // caller treats identically: drop the chunks).
    for path in manifest.files.keys() {
        if !seen.contains(path.as_path()) {
            diff.deleted.push(path.clone());
        }
    }

    diff
}

#[cfg(unix)]
fn inode(metadata: &std::fs::Metadata) -> u64 {
    use std::os::unix::fs::MetadataExt;
    metadata.ino()
}

#[cfg(not(unix))]
fn inode(_metadata: &std::fs::Metadata) -> u64 {
    0
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Write;
    use tempfile::TempDir;

    fn write_file(dir: &Path, name: &str, content: &[u8]) -> PathBuf {
        let path = dir.join(name);
        let mut f = std::fs::File::create(&path).unwrap();
        f.write_all(content).unwrap();
        path
    }

    fn manifest_with(path: PathBuf, content: &[u8]) -> Manifest {
        let metadata = std::fs::metadata(&path).unwrap();
        let entry = FileEntry::from_bytes(&metadata, content);
        let mut m = Manifest::new();
        m.insert(path, entry);
        m
    }

    #[test]
    fn empty_diff_against_empty_walk() {
        let mut m = Manifest::new();
        let diff = diff_against_walk(&mut m, &[]);
        assert!(diff.is_empty());
        assert_eq!(diff.total(), 0);
    }

    #[test]
    fn detects_new_file() {
        let dir = TempDir::new().unwrap();
        let p1 = write_file(dir.path(), "a.txt", b"hello");
        let mut m = Manifest::new();
        let diff = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert_eq!(diff.new, vec![p1]);
        assert!(diff.dirty.is_empty());
        assert!(diff.deleted.is_empty());
    }

    #[test]
    fn detects_deleted_file_via_missing_from_walk() {
        let dir = TempDir::new().unwrap();
        let p1 = write_file(dir.path(), "gone.txt", b"hello");
        let mut m = manifest_with(p1.clone(), b"hello");
        std::fs::remove_file(&p1).unwrap();
        // Caller walked the dir — empty since gone.txt is gone
        let diff = diff_against_walk(&mut m, &[]);
        assert_eq!(diff.deleted, vec![p1]);
        assert!(diff.dirty.is_empty());
        assert!(diff.new.is_empty());
    }

    #[test]
    fn unchanged_file_skipped_via_stat_tuple() {
        let dir = TempDir::new().unwrap();
        let p1 = write_file(dir.path(), "stable.txt", b"hello");
        let mut m = manifest_with(p1.clone(), b"hello");
        let diff = diff_against_walk(&mut m, &[p1]);
        assert!(diff.is_empty(), "stat tuple match must skip blake3");
    }

    #[test]
    fn detects_content_change_when_size_changes() {
        let dir = TempDir::new().unwrap();
        let p1 = write_file(dir.path(), "edit.txt", b"hello");
        let mut m = manifest_with(p1.clone(), b"hello");
        std::thread::sleep(std::time::Duration::from_millis(20));
        write_file(dir.path(), "edit.txt", b"hello world"); // size change
        let diff = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert_eq!(diff.dirty, vec![p1]);
    }

    #[test]
    fn detects_content_change_when_size_unchanged() {
        let dir = TempDir::new().unwrap();
        // Same byte count, different content
        let p1 = write_file(dir.path(), "rename-vars.rs", b"let foo = 1;");
        let mut m = manifest_with(p1.clone(), b"let foo = 1;");
        std::thread::sleep(std::time::Duration::from_millis(20));
        write_file(dir.path(), "rename-vars.rs", b"let bar = 1;"); // same size
        let diff = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert_eq!(diff.dirty, vec![p1], "blake3 must catch same-size change");
    }

    #[test]
    fn touched_but_unchanged_does_not_appear_in_diff() {
        let dir = TempDir::new().unwrap();
        let p1 = write_file(dir.path(), "touched.txt", b"identical");
        let mut m = manifest_with(p1.clone(), b"identical");
        let original_mtime = m.get(&p1).unwrap().mtime;
        std::thread::sleep(std::time::Duration::from_millis(20));
        // Rewrite same content → mtime updates, blake3 same
        write_file(dir.path(), "touched.txt", b"identical");
        let new_mtime_on_disk = std::fs::metadata(&p1).unwrap().modified().unwrap();
        assert_ne!(
            original_mtime, new_mtime_on_disk,
            "setup: mtime must differ for this test to mean anything"
        );

        let diff = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert!(
            diff.is_empty(),
            "touch-without-content-change must not appear in diff"
        );

        // Manifest's mtime must be refreshed so the next diff hits the
        // cheap stat-tuple path instead of re-blake3'ing.
        let refreshed = m.get(&p1).unwrap();
        assert_eq!(
            refreshed.mtime, new_mtime_on_disk,
            "manifest mtime must be refreshed on touch-without-change"
        );
    }

    // ── R4.2 tests ────────────────────────────────────────────────────────────

    /// R4.2: `diff_against_walk` must populate `touched_clean` when a file
    /// is rewritten with identical content (stat tuple changes, blake3 same).
    #[test]
    fn diff_against_walk_records_touched_clean() {
        let dir = TempDir::new().unwrap();
        let p1 = write_file(dir.path(), "touched_clean.txt", b"same content");
        let mut m = manifest_with(p1.clone(), b"same content");

        std::thread::sleep(std::time::Duration::from_millis(20));
        // Rewrite with identical bytes → mtime changes, blake3 unchanged.
        write_file(dir.path(), "touched_clean.txt", b"same content");

        let diff = diff_against_walk(&mut m, std::slice::from_ref(&p1));

        assert!(
            diff.is_empty(),
            "touched_clean file must not appear in dirty/new/deleted"
        );
        assert_eq!(
            diff.touched_clean.len(),
            1,
            "touched_clean must have exactly one entry; got {:?}",
            diff.touched_clean
                .iter()
                .map(|(p, _)| p)
                .collect::<Vec<_>>()
        );
        let (tc_path, tc_entry) = &diff.touched_clean[0];
        assert_eq!(
            tc_path, &p1,
            "touched_clean path must match the touched file"
        );
        // Blake3 must be the hash of "same content".
        let expected_hash = *blake3::hash(b"same content").as_bytes();
        assert_eq!(
            tc_entry.blake3, expected_hash,
            "touched_clean entry must carry the correct (unchanged) blake3"
        );
    }

    /// R4.2: A second `diff_against_walk` call on the same manifest (after
    /// the in-place refresh from the first call) must produce an empty
    /// `touched_clean` — the stat tuple now matches, so no blake3 is needed.
    #[test]
    fn repeated_touch_without_edit_pays_one_blake3_then_zero() {
        let dir = TempDir::new().unwrap();
        let p1 = write_file(dir.path(), "repeated_touch.txt", b"constant");
        let mut m = manifest_with(p1.clone(), b"constant");

        std::thread::sleep(std::time::Duration::from_millis(20));
        write_file(dir.path(), "repeated_touch.txt", b"constant"); // touch only

        // First pass: stat tuple mismatches → blake3 read → touched_clean populated.
        let diff1 = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert!(diff1.is_empty(), "first pass: no structural changes");
        assert_eq!(
            diff1.touched_clean.len(),
            1,
            "first pass: touched_clean must have one entry"
        );

        // Manifest was updated in-place by diff_against_walk. Second pass
        // should hit the cheap stat-tuple path: no blake3 read, empty
        // touched_clean.
        let diff2 = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert!(diff2.is_empty(), "second pass: no structural changes");
        assert!(
            diff2.touched_clean.is_empty(),
            "second pass: touched_clean must be empty after in-place refresh; \
             got {} entries",
            diff2.touched_clean.len()
        );
    }

    /// R4.2: A touch-without-content-change followed by a real edit must
    /// still be detected as dirty.  The stat-tuple refresh on the first pass
    /// must not mask the subsequent genuine content change.
    #[test]
    fn touched_then_real_edit_still_detected() {
        let dir = TempDir::new().unwrap();
        let p1 = write_file(dir.path(), "touch_then_edit.txt", b"v1");
        let mut m = manifest_with(p1.clone(), b"v1");

        // Pass 1: touch only (same content).
        std::thread::sleep(std::time::Duration::from_millis(20));
        write_file(dir.path(), "touch_then_edit.txt", b"v1");
        let diff1 = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert!(diff1.is_empty(), "pass 1: touch only, no structural diff");
        assert_eq!(diff1.touched_clean.len(), 1, "pass 1: one touched_clean");

        // Pass 2: real edit.
        std::thread::sleep(std::time::Duration::from_millis(20));
        write_file(dir.path(), "touch_then_edit.txt", b"v2 changed");
        let diff2 = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert_eq!(
            diff2.dirty,
            vec![p1.clone()],
            "pass 2: real edit must appear in dirty"
        );
        assert!(
            diff2.touched_clean.is_empty(),
            "pass 2: touched_clean must be empty when content changed"
        );
    }

    #[test]
    fn touched_unchanged_then_real_change_still_detected() {
        // Regression guard: the manifest update on touch-without-change
        // must not mask a subsequent real edit.
        let dir = TempDir::new().unwrap();
        let p1 = write_file(dir.path(), "twice.txt", b"original");
        let mut m = manifest_with(p1.clone(), b"original");

        std::thread::sleep(std::time::Duration::from_millis(20));
        write_file(dir.path(), "twice.txt", b"original"); // touch only
        let diff1 = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert!(diff1.is_empty(), "first pass: touch only");

        std::thread::sleep(std::time::Duration::from_millis(20));
        write_file(dir.path(), "twice.txt", b"modified"); // real change
        let diff2 = diff_against_walk(&mut m, std::slice::from_ref(&p1));
        assert_eq!(diff2.dirty, vec![p1], "second pass: real edit detected");
    }

    #[test]
    fn new_plus_deleted_plus_dirty_simultaneously() {
        let dir = TempDir::new().unwrap();
        let keep = write_file(dir.path(), "keep.txt", b"keep");
        let edit = write_file(dir.path(), "edit.txt", b"orig");
        let gone = write_file(dir.path(), "gone.txt", b"gone");
        let added_path = dir.path().join("added.txt"); // new file we'll write below

        let mut m = Manifest::new();
        let keep_meta = std::fs::metadata(&keep).unwrap();
        let edit_meta = std::fs::metadata(&edit).unwrap();
        let gone_meta = std::fs::metadata(&gone).unwrap();
        m.insert(keep.clone(), FileEntry::from_bytes(&keep_meta, b"keep"));
        m.insert(edit.clone(), FileEntry::from_bytes(&edit_meta, b"orig"));
        m.insert(gone.clone(), FileEntry::from_bytes(&gone_meta, b"gone"));

        std::thread::sleep(std::time::Duration::from_millis(20));
        write_file(dir.path(), "edit.txt", b"changed");
        std::fs::remove_file(&gone).unwrap();
        write_file(dir.path(), "added.txt", b"added");

        let walk = vec![keep.clone(), edit.clone(), added_path.clone()];
        let diff = diff_against_walk(&mut m, &walk);
        assert_eq!(diff.dirty, vec![edit]);
        assert_eq!(diff.new, vec![added_path]);
        assert_eq!(diff.deleted, vec![gone]);
        assert!(!diff.is_empty());
        assert_eq!(diff.total(), 3);
    }

    #[test]
    fn file_entry_from_path_round_trips_from_bytes() {
        let dir = TempDir::new().unwrap();
        let p = write_file(dir.path(), "x.txt", b"some content");
        let from_path = FileEntry::from_path(&p).unwrap();
        let metadata = std::fs::metadata(&p).unwrap();
        let from_bytes = FileEntry::from_bytes(&metadata, b"some content");
        assert_eq!(from_path.blake3, from_bytes.blake3);
        assert_eq!(from_path.size, from_bytes.size);
        // mtime may differ by stat-resolution if the OS updated atime
        // between calls; size + hash are the load-bearing invariants.
    }

    #[test]
    fn manifest_default_is_empty() {
        let m = Manifest::default();
        assert!(m.is_empty());
        assert_eq!(m.len(), 0);
    }

    #[cfg(unix)]
    #[test]
    fn inode_is_non_zero_on_unix() {
        let dir = TempDir::new().unwrap();
        let p = write_file(dir.path(), "x", b"data");
        let entry = FileEntry::from_path(&p).unwrap();
        assert!(entry.ino > 0, "Unix metadata must produce a non-zero inode");
    }
}