mbx_cache_core/agent/
file_digest.rs

1use crate::CacheDigest;
2use serde::{Deserialize, Serialize};
3use std::io::{self, Read as _};
4use std::path::{Path, PathBuf};
5#[cfg(target_os = "linux")]
6use std::sync::{Mutex, OnceLock};
7use std::time::SystemTime;
8
9const DIGEST_BUFFER_BYTES: usize = 64 * 1024;
10const TIMESTAMP_MACROS: &[&[u8]] = &[b"__DATE__", b"__TIME__", b"__TIMESTAMP__"];
11
12#[cfg(target_os = "linux")]
13const STATX_IDENTITY_MASK: u32 = 0x100 | 0x200 | 0x40 | 0x1000;
14
15/// Stable Linux UAPI layout. `libc` omits statx on older musl headers even
16/// though the kernel syscall and wire structure are available there.
17#[cfg(target_os = "linux")]
18#[repr(C)]
19struct LinuxStatxTimestamp {
20    seconds: i64,
21    nanos: u32,
22    reserved: i32,
23}
24
25#[cfg(target_os = "linux")]
26#[repr(C)]
27struct LinuxStatx {
28    mask: u32,
29    block_size: u32,
30    attributes: u64,
31    links: u32,
32    uid: u32,
33    gid: u32,
34    mode: u16,
35    reserved0: u16,
36    inode: u64,
37    size: u64,
38    blocks: u64,
39    attributes_mask: u64,
40    accessed: LinuxStatxTimestamp,
41    created: LinuxStatxTimestamp,
42    changed: LinuxStatxTimestamp,
43    modified: LinuxStatxTimestamp,
44    rdev_major: u32,
45    rdev_minor: u32,
46    device_major: u32,
47    device_minor: u32,
48    mount_id: u64,
49    direct_io_memory_alignment: u32,
50    direct_io_offset_alignment: u32,
51    subvolume: u64,
52    atomic_write_unit_min: u32,
53    atomic_write_unit_max: u32,
54    atomic_write_segments_max: u32,
55    direct_io_read_offset_alignment: u32,
56    atomic_write_unit_max_opt: u32,
57    reserved1: u32,
58    reserved2: [u64; 8],
59}
60
61#[cfg(target_os = "linux")]
62const _: () = assert!(std::mem::size_of::<LinuxStatx>() == 256);
63
64/// The on-disk identity a recorded file digest describes.
65///
66/// The same trade [`VerifiedBlob`] makes for CAS reads, offered to shims for
67/// the files they hash: an overwrite moves the modification time and a
68/// truncation changes the length, so a digest recorded against both stands
69/// until either does. Where the platform reports a metadata-change time the
70/// identity carries that too, and it is the part a writer cannot restore: a
71/// rewrite that puts the modification time back still moves the change time.
72/// Unix object identity also distinguishes an atomic replacement when both
73/// files happen to share one filesystem timestamp tick.
74#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
75#[serde(deny_unknown_fields)]
76pub struct FileIdentity {
77    /// Absolute path of the file.
78    pub path: PathBuf,
79    /// Length of the file in bytes.
80    pub len: u64,
81    /// Modification time of the file.
82    pub modified: SystemTime,
83    /// Platform metadata-change token, where one exists.
84    pub changed: Option<(i64, i64)>,
85    /// Stable object identity used to detect replacement independently of
86    /// timestamp resolution.
87    #[serde(default, skip_serializing_if = "Option::is_none")]
88    pub object: Option<FileObjectIdentity>,
89}
90
91/// The kernel identity of one file object on one mounted filesystem.
92#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
93#[serde(deny_unknown_fields)]
94pub struct FileObjectIdentity {
95    /// Device identity component. Linux NFS uses the major number reported by
96    /// `statx`; other Unix filesystems store the high half of `st_dev`.
97    pub device_major: u32,
98    /// Device identity component. Linux NFS uses the minor number reported by
99    /// `statx`; other Unix filesystems store the low half of `st_dev`.
100    pub device_minor: u32,
101    /// Mount identifier in this mount namespace, or zero when unavailable.
102    pub mount_id: u64,
103    /// Inode number within the mounted filesystem.
104    pub inode: u64,
105}
106
107impl FileIdentity {
108    /// Describe a file from metadata already in hand, or nothing when the
109    /// filesystem reports no modification time to compare against later.
110    pub fn describe(path: &Path, metadata: &std::fs::Metadata) -> Option<Self> {
111        Some(Self {
112            path: path.to_path_buf(),
113            len: metadata.len(),
114            modified: metadata.modified().ok()?,
115            changed: change_token(metadata),
116            object: metadata_object_identity(metadata),
117        })
118    }
119
120    /// Describe a file only when metadata can safely stand in for its digest.
121    ///
122    /// Linux NFS may revise cached change times without a write, so it uses a
123    /// cached object/length/mtime identity that deliberately omits ctime. This
124    /// is the same timestamp-freshness contract as Cargo and [`VerifiedBlob`];
125    /// forcing a server round trip per input would serialize large NFS builds.
126    /// If the kernel cannot supply every required field, callers hash instead.
127    pub fn for_digest_cache(path: &Path, metadata: &std::fs::Metadata) -> io::Result<Option<Self>> {
128        digest_cache_identity(
129            path,
130            metadata,
131            metadata_identity_is_unreliable(path, metadata)?,
132        )
133    }
134
135    /// Whether the file at this identity's path still has exactly this
136    /// identity, so the digest recorded against it still describes the bytes on
137    /// disk without reading them again.
138    ///
139    /// Length alone would miss an overwrite that keeps the size, and the
140    /// modification time can be put back by whoever rewrote the file. The
141    /// change time cannot be set from user space, so where the platform reports
142    /// one a rewrite that restores the old modification time still shows. A
143    /// file that has vanished is an error rather than a change, so the caller
144    /// can tell the two apart.
145    pub fn still_describes(&self) -> std::io::Result<bool> {
146        let metadata = std::fs::metadata(&self.path)?;
147        Ok(Self::for_digest_cache(&self.path, &metadata)?.as_ref() == Some(self))
148    }
149
150    /// Whether this identity is strong enough to stand in for a second read.
151    pub fn can_skip_content_verification(&self) -> bool {
152        self.changed.is_some() || self.object.is_some()
153    }
154}
155
156/// A pre-operation snapshot that can prove whether a file's contents changed.
157///
158/// Most filesystems provide a stable metadata-change token, so the inexpensive
159/// identity is sufficient. Linux NFS can reconcile client and server timestamps
160/// after a writer has closed the file, making two metadata reads disagree
161/// without any intervening write. Those files carry a content digest instead,
162/// while retaining length and file identity to detect replacement. Callers use
163/// the same comparison either way and do not need to know which filesystem
164/// supplied the file.
165#[derive(Debug, Clone, PartialEq, Eq)]
166pub struct FileSnapshot {
167    identity: FileIdentity,
168    content: Option<CacheDigest>,
169}
170
171impl FileSnapshot {
172    /// Capture the strongest comparison the file's filesystem can support.
173    pub fn capture(path: &Path) -> io::Result<Option<Self>> {
174        let metadata = std::fs::metadata(path)?;
175        capture_file_snapshot(
176            path,
177            &NoFileDigestCache,
178            metadata_identity_is_unreliable(path, &metadata)?,
179            metadata,
180        )
181    }
182
183    /// Capture a snapshot while reusing or publishing a content digest through
184    /// the session ledger when the filesystem requires one.
185    pub fn capture_with_cache(
186        path: &Path,
187        digests: &dyn FileDigestCache,
188    ) -> io::Result<Option<Self>> {
189        let metadata = std::fs::metadata(path)?;
190        capture_file_snapshot(
191            path,
192            digests,
193            metadata_identity_is_unreliable(path, &metadata)?,
194            metadata,
195        )
196    }
197
198    /// Whether `identity` and `content` still describe this snapshot.
199    ///
200    /// A content-backed snapshot ignores both timestamps: NFS can reconcile
201    /// mtime as well as ctime without a content change. Matching endpoint
202    /// digests cannot detect an intervening edit that restores the original
203    /// bytes. Digest-cache lookup still uses timestamps to decide when a
204    /// digest needs refreshing; this comparison performs no additional reads.
205    pub fn matches(&self, identity: Option<&FileIdentity>, content: &CacheDigest) -> bool {
206        self.content.as_ref().map_or_else(
207            || identity == Some(&self.identity),
208            |before| {
209                before == content
210                    && identity.is_some_and(|after| {
211                        self.identity.path == after.path
212                            && self.identity.len == after.len
213                            && self.identity.object == after.object
214                    })
215            },
216        )
217    }
218
219    /// Whether a mismatch proves the file's contents changed.
220    pub fn proves_content_change(&self) -> bool {
221        self.content.is_some() || self.identity.changed.is_some()
222    }
223}
224
225impl From<FileIdentity> for FileSnapshot {
226    fn from(identity: FileIdentity) -> Self {
227        Self {
228            identity,
229            content: None,
230        }
231    }
232}
233
234#[cfg(target_os = "linux")]
235fn metadata_identity_is_unreliable(path: &Path, metadata: &std::fs::Metadata) -> io::Result<bool> {
236    use std::mem::MaybeUninit;
237    use std::os::unix::ffi::OsStrExt as _;
238    use std::os::unix::fs::MetadataExt as _;
239
240    static FILESYSTEMS: OnceLock<Mutex<std::collections::BTreeMap<u64, bool>>> = OnceLock::new();
241    let filesystems = FILESYSTEMS.get_or_init(|| Mutex::new(std::collections::BTreeMap::new()));
242    if let Some(unreliable) = filesystems.lock().unwrap().get(&metadata.dev()).copied() {
243        return Ok(unreliable);
244    }
245
246    let path = std::ffi::CString::new(path.as_os_str().as_bytes())
247        .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))?;
248    let mut status = MaybeUninit::<libc::statfs>::zeroed();
249    // SAFETY: `path` is NUL-terminated and `status` points to writable,
250    // correctly sized storage. statfs initializes it before returning success.
251    let result = unsafe { libc::statfs(path.as_ptr(), status.as_mut_ptr()) };
252    if result != 0 {
253        return Err(io::Error::last_os_error());
254    }
255    // SAFETY: statfs returned success and initialized `status`.
256    let status = unsafe { status.assume_init() };
257    // libc gives NFS_SUPER_MAGIC a different signedness from statfs::f_type on musl.
258    let unreliable = status.f_type == 0x6969;
259    filesystems
260        .lock()
261        .unwrap()
262        .insert(metadata.dev(), unreliable);
263    Ok(unreliable)
264}
265
266#[cfg(not(target_os = "linux"))]
267fn metadata_identity_is_unreliable(
268    _path: &Path,
269    _metadata: &std::fs::Metadata,
270) -> io::Result<bool> {
271    Ok(false)
272}
273
274fn capture_file_snapshot(
275    path: &Path,
276    digests: &dyn FileDigestCache,
277    content_identity: bool,
278    metadata: std::fs::Metadata,
279) -> io::Result<Option<FileSnapshot>> {
280    let cache_identity = digest_cache_identity(path, &metadata, content_identity)?;
281    let Some(identity) = cache_identity
282        .clone()
283        .or_else(|| FileIdentity::describe(path, &metadata))
284    else {
285        return Ok(None);
286    };
287    let (identity, content) = if content_identity {
288        let resolved = cache_identity
289            .as_ref()
290            .and_then(|identity| {
291                digests
292                    .resolve(FileDigestScope::Content, std::slice::from_ref(identity))
293                    .pop()
294            })
295            .unwrap_or(FileDigestResolution::Unresolved);
296        let cached = match resolved {
297            FileDigestResolution::Digest(digest)
298                if cache_identity.as_ref().is_some_and(|identity| {
299                    identity.len == digest.size && identity.still_describes().unwrap_or(false)
300                }) =>
301            {
302                Some(digest)
303            }
304            FileDigestResolution::Digest(_)
305            | FileDigestResolution::EmbeddedTimestampMacro
306            | FileDigestResolution::Unresolved => None,
307        };
308        if let Some(digest) = cached {
309            (identity, Some(digest))
310        } else {
311            let observed = digest_file_validated(FileDigestScope::Content, path)?
312                .ok_or_else(|| io::Error::other("could not establish a file digest identity"))?;
313            let digest = observed
314                .resolution
315                .into_digest()
316                .ok_or_else(|| io::Error::other("content digest resolution returned no digest"))?;
317            if let Some(file) = observed.cache_identity
318                && file.len == digest.size
319            {
320                digests.record(
321                    FileDigestScope::Content,
322                    vec![RecordedFileDigest {
323                        file,
324                        digest: digest.clone(),
325                    }],
326                );
327            }
328            (observed.identity, Some(digest))
329        }
330    } else {
331        (identity, None)
332    };
333    Ok(Some(FileSnapshot { identity, content }))
334}
335
336fn digest_cache_identity(
337    path: &Path,
338    metadata: &std::fs::Metadata,
339    unreliable: bool,
340) -> io::Result<Option<FileIdentity>> {
341    if unreliable {
342        nfs_file_identity(path)
343    } else {
344        Ok(FileIdentity::describe(path, metadata))
345    }
346}
347
348#[cfg(target_os = "linux")]
349fn nfs_file_identity(path: &Path) -> io::Result<Option<FileIdentity>> {
350    use std::mem::MaybeUninit;
351    use std::os::unix::ffi::OsStrExt as _;
352
353    let path = std::ffi::CString::new(path.as_os_str().as_bytes())
354        .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))?;
355    let mut status = MaybeUninit::<LinuxStatx>::zeroed();
356    // SAFETY: `path` is NUL-terminated and `status` names writable storage.
357    let result = unsafe {
358        libc::syscall(
359            libc::SYS_statx,
360            libc::AT_FDCWD,
361            path.as_ptr(),
362            // AT_STATX_DONT_SYNC: the surrounding build already trusts cached
363            // mtimes, and FORCE_SYNC costs one NFS RPC for every dependency
364            // edge rather than every distinct file.
365            0x4000,
366            STATX_IDENTITY_MASK,
367            status.as_mut_ptr(),
368        )
369    };
370    if result != 0 {
371        let error = io::Error::last_os_error();
372        return match error.raw_os_error() {
373            Some(libc::ENOSYS | libc::EINVAL | libc::EOPNOTSUPP) => Ok(None),
374            _ => Err(error),
375        };
376    }
377    // SAFETY: statx returned success and initialized `status`.
378    let status = unsafe { status.assume_init() };
379    nfs_identity_from_statx(
380        Path::new(std::ffi::OsStr::from_bytes(path.to_bytes())),
381        &status,
382    )
383}
384
385#[cfg(target_os = "linux")]
386fn nfs_identity_from_statx(path: &Path, status: &LinuxStatx) -> io::Result<Option<FileIdentity>> {
387    if status.mask & STATX_IDENTITY_MASK != STATX_IDENTITY_MASK
388        || status.modified.nanos >= 1_000_000_000
389    {
390        return Ok(None);
391    }
392    let modified = system_time(status.modified.seconds, status.modified.nanos)?;
393    Ok(Some(FileIdentity {
394        path: path.to_path_buf(),
395        len: status.size,
396        modified,
397        changed: None,
398        object: Some(FileObjectIdentity {
399            device_major: status.device_major,
400            device_minor: status.device_minor,
401            mount_id: status.mount_id,
402            inode: status.inode,
403        }),
404    }))
405}
406
407#[cfg(not(target_os = "linux"))]
408fn nfs_file_identity(_path: &Path) -> io::Result<Option<FileIdentity>> {
409    Ok(None)
410}
411
412#[cfg(target_os = "linux")]
413fn system_time(seconds: i64, nanos: u32) -> io::Result<SystemTime> {
414    if seconds >= 0 {
415        SystemTime::UNIX_EPOCH.checked_add(std::time::Duration::new(seconds as u64, nanos))
416    } else {
417        SystemTime::UNIX_EPOCH
418            .checked_sub(std::time::Duration::from_secs(seconds.unsigned_abs()))
419            .and_then(|time| time.checked_add(std::time::Duration::from_nanos(nanos.into())))
420    }
421    .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "file timestamp is out of range"))
422}
423
424/// The metadata-change time as an opaque token, where the platform has one.
425#[cfg(unix)]
426fn change_token(metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
427    use std::os::unix::fs::MetadataExt;
428    Some((metadata.ctime(), metadata.ctime_nsec()))
429}
430
431/// Windows reports creation rather than metadata-change time, which a rewrite
432/// does not move, so no token is better than a misleading one.
433#[cfg(not(unix))]
434fn change_token(_metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
435    None
436}
437
438#[cfg(unix)]
439fn metadata_object_identity(metadata: &std::fs::Metadata) -> Option<FileObjectIdentity> {
440    use std::os::unix::fs::MetadataExt as _;
441
442    let device = metadata.dev();
443    Some(FileObjectIdentity {
444        device_major: (device >> 32) as u32,
445        device_minor: device as u32,
446        mount_id: 0,
447        inode: metadata.ino(),
448    })
449}
450
451#[cfg(not(unix))]
452fn metadata_object_identity(_metadata: &std::fs::Metadata) -> Option<FileObjectIdentity> {
453    None
454}
455
456/// A file digest recorded against the identity it was read under.
457#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
458#[serde(deny_unknown_fields)]
459pub struct RecordedFileDigest {
460    /// Identity of the file when its contents were hashed.
461    pub file: FileIdentity,
462    /// Digest of those contents.
463    pub digest: CacheDigest,
464}
465
466/// What a recorded file digest may stand in for.
467///
468/// Adapters prove different things when they read a file: the cc adapter's
469/// input scan also establishes that a source embeds no timestamp macro, which
470/// a digest recorded by the rustc adapter never checked. Scoping the ledger
471/// keeps one adapter's shortcut from resting on a property another adapter
472/// never established.
473#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
474#[serde(rename_all = "snake_case")]
475pub enum FileDigestScope {
476    /// The digest describes the file's contents and nothing more.
477    Content,
478    /// The digest describes a cc compiler input that also passed the
479    /// timestamp-macro scan.
480    CcInput,
481}
482
483/// The outcome of resolving one file under a digest scope.
484#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
485#[serde(tag = "kind", rename_all = "snake_case")]
486pub enum FileDigestResolution {
487    /// The file's content digest, with any scope-specific checks complete.
488    Digest(CacheDigest),
489    /// A C/C++ input contains a time-dependent preprocessor macro.
490    EmbeddedTimestampMacro,
491    /// No shared resolver was available; the caller must read the file.
492    Unresolved,
493}
494
495/// A file digest tied to the file object and pathname observed while reading.
496#[derive(Debug, Clone, PartialEq, Eq)]
497pub struct ValidatedFileDigest {
498    /// Scope-specific outcome of reading the file.
499    pub resolution: FileDigestResolution,
500    /// Identity of the pathname after the validated read.
501    pub identity: FileIdentity,
502    /// Identity suitable for reuse in the digest ledger, when available.
503    pub cache_identity: Option<FileIdentity>,
504}
505
506impl FileDigestResolution {
507    /// Extract the digest when resolution succeeded.
508    pub fn into_digest(self) -> Option<CacheDigest> {
509        match self {
510            Self::Digest(digest) => Some(digest),
511            Self::EmbeddedTimestampMacro | Self::Unresolved => None,
512        }
513    }
514}
515
516/// Read and hash a file once, applying the checks required by `scope` in that
517/// same pass.
518pub fn digest_file(scope: FileDigestScope, path: &Path) -> io::Result<FileDigestResolution> {
519    let file = std::fs::File::open(path)?;
520    digest_reader(scope, &file).map(|(resolution, _)| resolution)
521}
522
523/// Read and hash a regular file only when the open handle remains stable and
524/// the pathname still names that same file after the read.
525pub fn digest_file_validated(
526    scope: FileDigestScope,
527    path: &Path,
528) -> io::Result<Option<ValidatedFileDigest>> {
529    let file = std::fs::File::open(path)?;
530    let before = file.metadata()?;
531    if !before.is_file() {
532        return Err(io::Error::new(
533            io::ErrorKind::InvalidInput,
534            format!("digest path is not a regular file: {}", path.display()),
535        ));
536    }
537    let (resolution, size) = digest_reader(scope, &file)?;
538    let after = file.metadata()?;
539    let current_metadata = std::fs::metadata(path)?;
540    let unreliable = metadata_identity_is_unreliable(path, &current_metadata)?;
541    let cache_identity = digest_cache_identity(path, &current_metadata, unreliable)?;
542    let identity = cache_identity
543        .clone()
544        .or_else(|| FileIdentity::describe(path, &current_metadata));
545    let Some(identity) = identity else {
546        return Ok(None);
547    };
548    if !metadata_names_same_file(&after, &current_metadata) {
549        return Err(io::Error::new(
550            io::ErrorKind::WouldBlock,
551            "file was replaced while its contents were being read",
552        ));
553    }
554    if !handle_and_path_are_stable(path, &before, &after, &identity, unreliable, size)? {
555        return Err(io::Error::new(
556            io::ErrorKind::WouldBlock,
557            "file changed while its contents were being read",
558        ));
559    }
560    Ok(Some(ValidatedFileDigest {
561        resolution,
562        identity,
563        cache_identity,
564    }))
565}
566
567fn digest_reader(
568    scope: FileDigestScope,
569    file: &std::fs::File,
570) -> io::Result<(FileDigestResolution, u64)> {
571    let mut reader = std::io::BufReader::new(file);
572    let mut hasher = blake3::Hasher::new();
573    let mut size = 0_u64;
574    let longest_macro = TIMESTAMP_MACROS
575        .iter()
576        .map(|macro_name| macro_name.len())
577        .max()
578        .unwrap_or_default();
579    let mut window = Vec::with_capacity(DIGEST_BUFFER_BYTES + longest_macro);
580    let mut chunk = vec![0_u8; DIGEST_BUFFER_BYTES];
581    let mut found_timestamp_macro = false;
582    loop {
583        let read = reader.read(&mut chunk)?;
584        if read == 0 {
585            break;
586        }
587        hasher.update(&chunk[..read]);
588        size = size
589            .checked_add(read as u64)
590            .ok_or_else(|| io::Error::other("file length overflowed u64"))?;
591        if scope == FileDigestScope::CcInput && !found_timestamp_macro {
592            window.extend_from_slice(&chunk[..read]);
593            found_timestamp_macro = TIMESTAMP_MACROS
594                .iter()
595                .any(|macro_name| contains_subslice(&window, macro_name));
596            let keep = window.len().saturating_sub(longest_macro.saturating_sub(1));
597            window.drain(..keep);
598        }
599    }
600    if found_timestamp_macro {
601        Ok((FileDigestResolution::EmbeddedTimestampMacro, size))
602    } else {
603        Ok((
604            FileDigestResolution::Digest(CacheDigest {
605                algorithm: "blake3".into(),
606                hash: hasher.finalize().to_hex().to_string(),
607                size,
608            }),
609            size,
610        ))
611    }
612}
613
614fn handle_and_path_are_stable(
615    path: &Path,
616    before: &std::fs::Metadata,
617    after: &std::fs::Metadata,
618    current: &FileIdentity,
619    unreliable: bool,
620    digest_size: u64,
621) -> io::Result<bool> {
622    if before.len() != after.len() || after.len() != digest_size || current.len != digest_size {
623        return Ok(false);
624    }
625    if !metadata_names_same_file(before, after) {
626        return Ok(false);
627    }
628    if unreliable {
629        #[cfg(target_os = "linux")]
630        {
631            let before_modified = before.modified()?;
632            let after_modified = after.modified()?;
633            let object_matches = current.object.as_ref().is_none_or(|object| {
634                metadata_matches_object(before, object) && metadata_matches_object(after, object)
635            });
636            return Ok(before_modified == after_modified
637                && after_modified == current.modified
638                && object_matches);
639        }
640        #[cfg(not(target_os = "linux"))]
641        {
642            let _ = (path, before, after, current);
643            return Ok(false);
644        }
645    }
646    let before = FileIdentity::describe(path, before);
647    let after = FileIdentity::describe(path, after);
648    Ok(before.is_some() && before == after && after.as_ref() == Some(current))
649}
650
651#[cfg(unix)]
652fn metadata_names_same_file(left: &std::fs::Metadata, right: &std::fs::Metadata) -> bool {
653    use std::os::unix::fs::MetadataExt as _;
654
655    left.dev() == right.dev() && left.ino() == right.ino()
656}
657
658#[cfg(not(unix))]
659fn metadata_names_same_file(_left: &std::fs::Metadata, _right: &std::fs::Metadata) -> bool {
660    true
661}
662
663#[cfg(target_os = "linux")]
664fn metadata_matches_object(metadata: &std::fs::Metadata, object: &FileObjectIdentity) -> bool {
665    use std::os::unix::fs::MetadataExt as _;
666
667    let device = metadata.dev();
668    let major = ((device >> 8) & 0x0fff) | ((device >> 32) & 0xfffff000);
669    let minor = (device & 0x00ff) | ((device >> 12) & 0xffffff00);
670    u32::try_from(major).ok() == Some(object.device_major)
671        && u32::try_from(minor).ok() == Some(object.device_minor)
672        && metadata.ino() == object.inode
673}
674
675fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
676    !needle.is_empty()
677        && haystack.len() >= needle.len()
678        && haystack
679            .windows(needle.len())
680            .any(|window| window == needle)
681}
682
683/// Recorded digests a session may consult instead of rehashing a file.
684///
685/// The agent's file-digest ledger answers through this everywhere a caller
686/// holds one; the no-op implementation stands in where reuse must not happen,
687/// such as under verification.
688pub trait FileDigestCache: Send + Sync {
689    /// Resolve these identities, coalescing concurrent misses where supported.
690    fn resolve(&self, scope: FileDigestScope, files: &[FileIdentity]) -> Vec<FileDigestResolution> {
691        self.find(scope, files)
692            .into_iter()
693            .map(|digest| {
694                digest.map_or(
695                    FileDigestResolution::Unresolved,
696                    FileDigestResolution::Digest,
697                )
698            })
699            .collect()
700    }
701    /// Recorded digests for these identities, in request order.
702    fn find(&self, scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>>;
703    /// Record digests for files read under these identities.
704    fn record(&self, scope: FileDigestScope, entries: Vec<RecordedFileDigest>);
705}
706
707/// A [`FileDigestCache`] that remembers nothing and finds nothing.
708pub struct NoFileDigestCache;
709
710impl FileDigestCache for NoFileDigestCache {
711    fn resolve(
712        &self,
713        _scope: FileDigestScope,
714        files: &[FileIdentity],
715    ) -> Vec<FileDigestResolution> {
716        vec![FileDigestResolution::Unresolved; files.len()]
717    }
718
719    fn find(&self, _scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>> {
720        vec![None; files.len()]
721    }
722
723    fn record(&self, _scope: FileDigestScope, _entries: Vec<RecordedFileDigest>) {}
724}
725
726#[cfg(test)]
727mod tests {
728    use super::*;
729
730    #[cfg(unix)]
731    struct RecordingDigestCache {
732        answer: Option<CacheDigest>,
733        recorded: std::sync::Mutex<Vec<RecordedFileDigest>>,
734    }
735
736    #[cfg(unix)]
737    impl FileDigestCache for RecordingDigestCache {
738        fn find(
739            &self,
740            _scope: FileDigestScope,
741            files: &[FileIdentity],
742        ) -> Vec<Option<CacheDigest>> {
743            assert_eq!(files.len(), 1);
744            vec![self.answer.clone()]
745        }
746
747        fn record(&self, _scope: FileDigestScope, entries: Vec<RecordedFileDigest>) {
748            self.recorded.lock().unwrap().extend(entries);
749        }
750    }
751
752    #[cfg(unix)]
753    #[test]
754    fn content_snapshot_records_a_replacement_under_its_own_identity() {
755        let directory = tempfile::tempdir().unwrap();
756        let path = directory.path().join("input.rlib");
757        let replacement = directory.path().join("replacement.rlib");
758        std::fs::write(&path, b"original....").unwrap();
759        std::fs::File::options()
760            .write(true)
761            .open(&path)
762            .unwrap()
763            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
764            .unwrap();
765        std::fs::write(&replacement, b"replacement!").unwrap();
766        std::fs::File::options()
767            .write(true)
768            .open(&replacement)
769            .unwrap()
770            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
771            .unwrap();
772        let before = std::fs::metadata(&path).unwrap();
773        let old_identity = FileIdentity::describe(&path, &before).unwrap();
774        std::fs::remove_file(&path).unwrap();
775        std::fs::rename(&replacement, &path).unwrap();
776        let cache = RecordingDigestCache {
777            answer: None,
778            recorded: std::sync::Mutex::new(Vec::new()),
779        };
780
781        let snapshot = capture_file_snapshot(&path, &cache, true, before)
782            .unwrap()
783            .unwrap();
784        let current = FileIdentity::for_digest_cache(&path, &std::fs::metadata(&path).unwrap())
785            .unwrap()
786            .unwrap();
787        let digest = CacheDigest::blake3_file(&path).unwrap();
788
789        assert_eq!(current.modified, old_identity.modified);
790        assert_ne!(current.object, old_identity.object);
791        assert_ne!(current, old_identity);
792        assert!(snapshot.matches(Some(&current), &digest));
793        let recorded = cache.recorded.lock().unwrap();
794        assert_eq!(recorded.len(), 1);
795        assert_eq!(recorded[0].file, current);
796        assert_eq!(recorded[0].digest, digest);
797    }
798
799    #[cfg(unix)]
800    #[test]
801    fn content_snapshot_rejects_a_stale_digest_cache_hit() {
802        let directory = tempfile::tempdir().unwrap();
803        let path = directory.path().join("input.rlib");
804        let replacement = directory.path().join("replacement.rlib");
805        std::fs::write(&path, b"original....").unwrap();
806        std::fs::File::options()
807            .write(true)
808            .open(&path)
809            .unwrap()
810            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
811            .unwrap();
812        let before = std::fs::metadata(&path).unwrap();
813        std::fs::write(&replacement, b"replacement!").unwrap();
814        std::fs::File::options()
815            .write(true)
816            .open(&replacement)
817            .unwrap()
818            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
819            .unwrap();
820        std::fs::remove_file(&path).unwrap();
821        std::fs::rename(&replacement, &path).unwrap();
822        let cache = RecordingDigestCache {
823            answer: Some(CacheDigest::blake3(b"original....")),
824            recorded: std::sync::Mutex::new(Vec::new()),
825        };
826
827        let snapshot = capture_file_snapshot(&path, &cache, true, before)
828            .unwrap()
829            .unwrap();
830        let current = FileIdentity::for_digest_cache(&path, &std::fs::metadata(&path).unwrap())
831            .unwrap()
832            .unwrap();
833        let digest = CacheDigest::blake3_file(&path).unwrap();
834
835        assert!(snapshot.matches(Some(&current), &digest));
836        assert_ne!(snapshot.content, Some(CacheDigest::blake3(b"original....")));
837        assert_eq!(cache.recorded.lock().unwrap()[0].digest, digest);
838    }
839
840    #[cfg(unix)]
841    #[test]
842    fn a_replaced_path_does_not_validate_the_open_handle() {
843        let directory = tempfile::tempdir().unwrap();
844        let path = directory.path().join("input.rlib");
845        let replacement = directory.path().join("replacement.rlib");
846        std::fs::write(&path, b"old bytes").unwrap();
847        std::fs::write(&replacement, b"new bytes").unwrap();
848
849        let file = std::fs::File::open(&path).unwrap();
850        let before = file.metadata().unwrap();
851        std::fs::rename(&replacement, &path).unwrap();
852        let (_, size) = digest_reader(FileDigestScope::Content, &file).unwrap();
853        let after = file.metadata().unwrap();
854        let current_metadata = std::fs::metadata(&path).unwrap();
855        let current = FileIdentity::describe(&path, &current_metadata).unwrap();
856
857        assert!(!metadata_names_same_file(&after, &current_metadata));
858        assert!(
859            !handle_and_path_are_stable(&path, &before, &after, &current, false, size).unwrap()
860        );
861    }
862
863    #[test]
864    fn an_identity_describes_the_file_until_it_is_written_or_removed() {
865        let directory = tempfile::tempdir().unwrap();
866        let path = directory.path().join("input.rs");
867        std::fs::write(&path, b"fn main() {}").unwrap();
868        let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
869        assert!(identity.still_describes().unwrap());
870
871        std::thread::sleep(std::time::Duration::from_millis(20));
872        std::fs::write(&path, b"fn main() { }").unwrap();
873        assert!(!identity.still_describes().unwrap());
874
875        std::fs::remove_file(&path).unwrap();
876        assert!(identity.still_describes().is_err());
877    }
878
879    #[test]
880    fn a_metadata_snapshot_detects_a_metadata_change() {
881        let directory = tempfile::tempdir().unwrap();
882        let path = directory.path().join("input.rs");
883        std::fs::write(&path, b"fn main() {}").unwrap();
884        let snapshot = capture_file_snapshot(
885            &path,
886            &NoFileDigestCache,
887            false,
888            std::fs::metadata(&path).unwrap(),
889        )
890        .unwrap()
891        .unwrap();
892
893        std::fs::File::options()
894            .write(true)
895            .open(&path)
896            .unwrap()
897            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
898            .unwrap();
899        let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
900        let digest = CacheDigest::blake3_file(&path).unwrap();
901
902        assert!(!snapshot.matches(identity.as_ref(), &digest));
903        assert_eq!(snapshot.proves_content_change(), cfg!(unix));
904    }
905
906    #[test]
907    fn a_content_snapshot_ignores_timestamp_churn_but_detects_changed_bytes() {
908        let directory = tempfile::tempdir().unwrap();
909        let path = directory.path().join("input.rs");
910        std::fs::write(&path, b"fn main() {}").unwrap();
911        let snapshot = capture_file_snapshot(
912            &path,
913            &NoFileDigestCache,
914            true,
915            std::fs::metadata(&path).unwrap(),
916        )
917        .unwrap()
918        .unwrap();
919
920        let mut identity = snapshot.identity.clone();
921        identity.changed = identity
922            .changed
923            .map(|(seconds, nanos)| (seconds + 1, nanos));
924        let digest = CacheDigest::blake3_file(&path).unwrap();
925        assert!(snapshot.matches(Some(&identity), &digest));
926        assert!(snapshot.proves_content_change());
927
928        identity.modified = SystemTime::UNIX_EPOCH;
929        assert!(snapshot.matches(Some(&identity), &digest));
930
931        std::fs::write(&path, b"fn main(){ }").unwrap();
932        let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
933        let digest = CacheDigest::blake3_file(&path).unwrap();
934        assert!(!snapshot.matches(identity.as_ref(), &digest));
935    }
936
937    #[test]
938    fn reliable_metadata_keeps_the_native_identity() {
939        let directory = tempfile::tempdir().unwrap();
940        let path = directory.path().join("input.rs");
941        std::fs::write(&path, b"fn main() {}").unwrap();
942        let metadata = std::fs::metadata(&path).unwrap();
943
944        let identity = digest_cache_identity(&path, &metadata, false)
945            .unwrap()
946            .unwrap();
947
948        assert_eq!(identity.path, path);
949        assert_eq!(identity.len, 12);
950        assert_eq!(identity.object.is_some(), cfg!(unix));
951    }
952
953    #[test]
954    fn content_snapshot_ignores_nfs_timestamp_churn_but_not_content_or_identity_changes() {
955        let digest = CacheDigest::blake3(b"nfs bytes");
956        let identity = FileIdentity {
957            path: PathBuf::from("/nfs/input.rlib"),
958            len: digest.size,
959            modified: SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(10),
960            changed: Some((10, 1)),
961            object: Some(FileObjectIdentity {
962                device_major: 0,
963                device_minor: 42,
964                mount_id: 7,
965                inode: 99,
966            }),
967        };
968        let snapshot = FileSnapshot {
969            identity: identity.clone(),
970            content: Some(digest.clone()),
971        };
972        // Client/server reconciliation can move either timestamp in either
973        // direction. None of these observations proves another write.
974        for seconds in [9, 11] {
975            let mut after = identity.clone();
976            after.changed = Some((seconds, 500));
977            after.modified = SystemTime::UNIX_EPOCH + std::time::Duration::new(seconds as u64, 500);
978            assert!(snapshot.matches(Some(&after), &digest));
979        }
980
981        // Timestamp tolerance does not admit changed bytes, even when their
982        // length and all metadata remain the same.
983        let changed = CacheDigest::blake3(b"NFS bytes");
984        assert_eq!(changed.size, digest.size);
985        assert!(!snapshot.matches(Some(&identity), &changed));
986        assert!(!snapshot.matches(None, &digest));
987
988        let mut after = identity.clone();
989        after.object.as_mut().unwrap().inode += 1;
990        assert!(!snapshot.matches(Some(&after), &digest));
991
992        let mut after = identity.clone();
993        after.len += 1;
994        assert!(!snapshot.matches(Some(&after), &digest));
995
996        let mut after = identity;
997        after.path.set_file_name("replacement.rlib");
998        assert!(!snapshot.matches(Some(&after), &digest));
999    }
1000
1001    #[cfg(target_os = "linux")]
1002    #[test]
1003    fn incomplete_statx_metadata_falls_back_to_content_hashing() {
1004        // SAFETY: all-zero is a valid unpopulated statx result for this parser.
1005        let status = unsafe { std::mem::zeroed::<LinuxStatx>() };
1006        assert!(
1007            nfs_identity_from_statx(Path::new("/nfs/input.rlib"), &status)
1008                .unwrap()
1009                .is_none()
1010        );
1011    }
1012}
mbx_cache_core/agent/file_digest.rs

mbx_cache_core/agent/
file_digest.rs
