mbx_cache_core/agent/
file_digest.rs

1use crate::CacheDigest;
2use serde::{Deserialize, Serialize};
3use std::io::{self, Read as _};
4use std::path::{Path, PathBuf};
5#[cfg(target_os = "linux")]
6use std::sync::{Mutex, OnceLock};
7use std::time::SystemTime;
8
9const DIGEST_BUFFER_BYTES: usize = 64 * 1024;
10const TIMESTAMP_MACROS: &[&[u8]] = &[b"__DATE__", b"__TIME__", b"__TIMESTAMP__"];
11
12#[cfg(target_os = "linux")]
13const STATX_IDENTITY_MASK: u32 = 0x100 | 0x200 | 0x40 | 0x1000;
14
15/// Stable Linux UAPI layout. `libc` omits statx on older musl headers even
16/// though the kernel syscall and wire structure are available there.
17#[cfg(target_os = "linux")]
18#[repr(C)]
19struct LinuxStatxTimestamp {
20    seconds: i64,
21    nanos: u32,
22    reserved: i32,
23}
24
25#[cfg(target_os = "linux")]
26#[repr(C)]
27struct LinuxStatx {
28    mask: u32,
29    block_size: u32,
30    attributes: u64,
31    links: u32,
32    uid: u32,
33    gid: u32,
34    mode: u16,
35    reserved0: u16,
36    inode: u64,
37    size: u64,
38    blocks: u64,
39    attributes_mask: u64,
40    accessed: LinuxStatxTimestamp,
41    created: LinuxStatxTimestamp,
42    changed: LinuxStatxTimestamp,
43    modified: LinuxStatxTimestamp,
44    rdev_major: u32,
45    rdev_minor: u32,
46    device_major: u32,
47    device_minor: u32,
48    mount_id: u64,
49    direct_io_memory_alignment: u32,
50    direct_io_offset_alignment: u32,
51    subvolume: u64,
52    atomic_write_unit_min: u32,
53    atomic_write_unit_max: u32,
54    atomic_write_segments_max: u32,
55    direct_io_read_offset_alignment: u32,
56    atomic_write_unit_max_opt: u32,
57    reserved1: u32,
58    reserved2: [u64; 8],
59}
60
61#[cfg(target_os = "linux")]
62const _: () = assert!(std::mem::size_of::<LinuxStatx>() == 256);
63
64/// The on-disk identity a recorded file digest describes.
65///
66/// The same trade [`VerifiedBlob`] makes for CAS reads, offered to shims for
67/// the files they hash: an overwrite moves the modification time and a
68/// truncation changes the length, so a digest recorded against both stands
69/// until either does. Where the platform reports a metadata-change time the
70/// identity carries that too, and it is the part a writer cannot restore: a
71/// rewrite that puts the modification time back still moves the change time.
72/// Unix object identity also distinguishes an atomic replacement when both
73/// files happen to share one filesystem timestamp tick.
74#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
75#[serde(deny_unknown_fields)]
76pub struct FileIdentity {
77    /// Absolute path of the file.
78    pub path: PathBuf,
79    /// Length of the file in bytes.
80    pub len: u64,
81    /// Modification time of the file.
82    pub modified: SystemTime,
83    /// Platform metadata-change token, where one exists.
84    pub changed: Option<(i64, i64)>,
85    /// Stable object identity used to detect replacement independently of
86    /// timestamp resolution.
87    #[serde(default, skip_serializing_if = "Option::is_none")]
88    pub object: Option<FileObjectIdentity>,
89}
90
91/// The kernel identity of one file object on one mounted filesystem.
92#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
93#[serde(deny_unknown_fields)]
94pub struct FileObjectIdentity {
95    /// Device identity component. Linux NFS uses the major number reported by
96    /// `statx`; other Unix filesystems store the high half of `st_dev`.
97    pub device_major: u32,
98    /// Device identity component. Linux NFS uses the minor number reported by
99    /// `statx`; other Unix filesystems store the low half of `st_dev`.
100    pub device_minor: u32,
101    /// Mount identifier in this mount namespace, or zero when unavailable.
102    pub mount_id: u64,
103    /// Inode number within the mounted filesystem.
104    pub inode: u64,
105}
106
107impl FileIdentity {
108    /// Describe a file from metadata already in hand, or nothing when the
109    /// filesystem reports no modification time to compare against later.
110    pub fn describe(path: &Path, metadata: &std::fs::Metadata) -> Option<Self> {
111        Some(Self {
112            path: path.to_path_buf(),
113            len: metadata.len(),
114            modified: metadata.modified().ok()?,
115            changed: change_token(metadata),
116            object: metadata_object_identity(metadata),
117        })
118    }
119
120    /// Describe a file only when metadata can safely stand in for its digest.
121    ///
122    /// Linux NFS may revise cached change times without a write, so it uses a
123    /// cached object/length/mtime identity that deliberately omits ctime. This
124    /// is the same timestamp-freshness contract as Cargo and [`VerifiedBlob`];
125    /// forcing a server round trip per input would serialize large NFS builds.
126    /// If the kernel cannot supply every required field, callers hash instead.
127    pub fn for_digest_cache(path: &Path, metadata: &std::fs::Metadata) -> io::Result<Option<Self>> {
128        digest_cache_identity(
129            path,
130            metadata,
131            metadata_identity_is_unreliable(path, metadata)?,
132        )
133    }
134
135    /// Whether the file at this identity's path still has exactly this
136    /// identity, so the digest recorded against it still describes the bytes on
137    /// disk without reading them again.
138    ///
139    /// Length alone would miss an overwrite that keeps the size, and the
140    /// modification time can be put back by whoever rewrote the file. The
141    /// change time cannot be set from user space, so where the platform reports
142    /// one a rewrite that restores the old modification time still shows. A
143    /// file that has vanished is an error rather than a change, so the caller
144    /// can tell the two apart.
145    pub fn still_describes(&self) -> std::io::Result<bool> {
146        let metadata = std::fs::metadata(&self.path)?;
147        Ok(Self::for_digest_cache(&self.path, &metadata)?.as_ref() == Some(self))
148    }
149
150    /// Whether this identity is strong enough to stand in for a second read.
151    pub fn can_skip_content_verification(&self) -> bool {
152        self.changed.is_some() || self.object.is_some()
153    }
154}
155
156/// A pre-operation snapshot that can prove whether a file's contents changed.
157///
158/// Most filesystems provide a stable metadata-change token, so the inexpensive
159/// identity is sufficient. Linux NFS can reconcile client and server timestamps
160/// after a writer has closed the file, making two metadata reads disagree
161/// without any intervening write. Those files carry a content digest instead,
162/// while retaining length and file identity to detect replacement. Callers use
163/// the same comparison either way and do not need to know which filesystem
164/// supplied the file.
165#[derive(Debug, Clone, PartialEq, Eq)]
166pub struct FileSnapshot {
167    identity: FileIdentity,
168    content: Option<CacheDigest>,
169}
170
171impl FileSnapshot {
172    /// Capture the strongest comparison the file's filesystem can support.
173    pub fn capture(path: &Path) -> io::Result<Option<Self>> {
174        let metadata = std::fs::metadata(path)?;
175        let unreliable = metadata_identity_is_unreliable(path, &metadata)?;
176        capture_file_snapshot(path, &NoFileDigestCache, unreliable, unreliable, metadata)
177    }
178
179    /// Capture a snapshot while reusing or publishing a content digest through
180    /// the session ledger when the filesystem requires one.
181    pub fn capture_with_cache(
182        path: &Path,
183        digests: &dyn FileDigestCache,
184    ) -> io::Result<Option<Self>> {
185        let metadata = std::fs::metadata(path)?;
186        let unreliable = metadata_identity_is_unreliable(path, &metadata)?;
187        capture_file_snapshot(path, digests, unreliable, unreliable, metadata)
188    }
189
190    /// Capture a content-backed snapshot whatever the filesystem, for a file
191    /// whose metadata is expected to change without its bytes.
192    ///
193    /// A compiler artifact is one: rustc hardlinks the `.rmeta` it just
194    /// emitted into its incremental session directory when it finishes, which
195    /// bumps the change token of a file a pipelined dependent is already
196    /// reading. Comparing the bytes, length, and file object instead accepts
197    /// that and still rejects a replacement or a rewrite.
198    pub fn capture_content_with_cache(
199        path: &Path,
200        digests: &dyn FileDigestCache,
201    ) -> io::Result<Option<Self>> {
202        let metadata = std::fs::metadata(path)?;
203        // The ledger identity still follows the filesystem: recording under
204        // the ordinary identity and looking up under the NFS one would miss
205        // every time, and every dependent would re-hash the artifact.
206        let unreliable = metadata_identity_is_unreliable(path, &metadata)?;
207        capture_file_snapshot(path, digests, unreliable, true, metadata)
208    }
209
210    /// Whether `identity` and `content` still describe this snapshot.
211    ///
212    /// A content-backed snapshot ignores both timestamps: NFS can reconcile
213    /// mtime as well as ctime without a content change. Matching endpoint
214    /// digests cannot detect an intervening edit that restores the original
215    /// bytes. Digest-cache lookup still uses timestamps to decide when a
216    /// digest needs refreshing; this comparison performs no additional reads.
217    pub fn matches(&self, identity: Option<&FileIdentity>, content: &CacheDigest) -> bool {
218        self.content.as_ref().map_or_else(
219            || identity == Some(&self.identity),
220            |before| {
221                before == content
222                    && identity.is_some_and(|after| {
223                        self.identity.path == after.path
224                            && self.identity.len == after.len
225                            && self.identity.object == after.object
226                    })
227            },
228        )
229    }
230
231    /// Whether a mismatch proves the file's contents changed.
232    pub fn proves_content_change(&self) -> bool {
233        self.content.is_some() || self.identity.changed.is_some()
234    }
235}
236
237impl From<FileIdentity> for FileSnapshot {
238    fn from(identity: FileIdentity) -> Self {
239        Self {
240            identity,
241            content: None,
242        }
243    }
244}
245
246#[cfg(target_os = "linux")]
247fn metadata_identity_is_unreliable(path: &Path, metadata: &std::fs::Metadata) -> io::Result<bool> {
248    use std::mem::MaybeUninit;
249    use std::os::unix::ffi::OsStrExt as _;
250    use std::os::unix::fs::MetadataExt as _;
251
252    static FILESYSTEMS: OnceLock<Mutex<std::collections::BTreeMap<u64, bool>>> = OnceLock::new();
253    let filesystems = FILESYSTEMS.get_or_init(|| Mutex::new(std::collections::BTreeMap::new()));
254    if let Some(unreliable) = filesystems.lock().unwrap().get(&metadata.dev()).copied() {
255        return Ok(unreliable);
256    }
257
258    let path = std::ffi::CString::new(path.as_os_str().as_bytes())
259        .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))?;
260    let mut status = MaybeUninit::<libc::statfs>::zeroed();
261    // SAFETY: `path` is NUL-terminated and `status` points to writable,
262    // correctly sized storage. statfs initializes it before returning success.
263    let result = unsafe { libc::statfs(path.as_ptr(), status.as_mut_ptr()) };
264    if result != 0 {
265        return Err(io::Error::last_os_error());
266    }
267    // SAFETY: statfs returned success and initialized `status`.
268    let status = unsafe { status.assume_init() };
269    // libc gives NFS_SUPER_MAGIC a different signedness from statfs::f_type on musl.
270    let unreliable = status.f_type == 0x6969;
271    filesystems
272        .lock()
273        .unwrap()
274        .insert(metadata.dev(), unreliable);
275    Ok(unreliable)
276}
277
278#[cfg(not(target_os = "linux"))]
279fn metadata_identity_is_unreliable(
280    _path: &Path,
281    _metadata: &std::fs::Metadata,
282) -> io::Result<bool> {
283    Ok(false)
284}
285
286/// `unreliable` selects the ledger identity the filesystem calls for, as
287/// [`digest_file_validated`] does when it records one; `content_identity`
288/// asks for the bytes to be compared as well.
289fn capture_file_snapshot(
290    path: &Path,
291    digests: &dyn FileDigestCache,
292    unreliable: bool,
293    content_identity: bool,
294    metadata: std::fs::Metadata,
295) -> io::Result<Option<FileSnapshot>> {
296    let cache_identity = digest_cache_identity(path, &metadata, unreliable)?;
297    let Some(identity) = cache_identity
298        .clone()
299        .or_else(|| FileIdentity::describe(path, &metadata))
300    else {
301        return Ok(None);
302    };
303    let (identity, content) = if content_identity {
304        let resolved = cache_identity
305            .as_ref()
306            .and_then(|identity| {
307                digests
308                    .resolve(FileDigestScope::Content, std::slice::from_ref(identity))
309                    .pop()
310            })
311            .unwrap_or(FileDigestResolution::Unresolved);
312        let cached = match resolved {
313            FileDigestResolution::Digest(digest)
314                if cache_identity.as_ref().is_some_and(|identity| {
315                    identity.len == digest.size && identity.still_describes().unwrap_or(false)
316                }) =>
317            {
318                Some(digest)
319            }
320            FileDigestResolution::Digest(_)
321            | FileDigestResolution::EmbeddedTimestampMacro
322            | FileDigestResolution::Unresolved => None,
323        };
324        if let Some(digest) = cached {
325            (identity, Some(digest))
326        } else {
327            let observed = digest_file_validated(FileDigestScope::Content, path)?
328                .ok_or_else(|| io::Error::other("could not establish a file digest identity"))?;
329            let digest = observed
330                .resolution
331                .into_digest()
332                .ok_or_else(|| io::Error::other("content digest resolution returned no digest"))?;
333            if let Some(file) = observed.cache_identity
334                && file.len == digest.size
335            {
336                digests.record(
337                    FileDigestScope::Content,
338                    vec![RecordedFileDigest {
339                        file,
340                        digest: digest.clone(),
341                    }],
342                );
343            }
344            (observed.identity, Some(digest))
345        }
346    } else {
347        (identity, None)
348    };
349    Ok(Some(FileSnapshot { identity, content }))
350}
351
352fn digest_cache_identity(
353    path: &Path,
354    metadata: &std::fs::Metadata,
355    unreliable: bool,
356) -> io::Result<Option<FileIdentity>> {
357    if unreliable {
358        nfs_file_identity(path)
359    } else {
360        Ok(FileIdentity::describe(path, metadata))
361    }
362}
363
364#[cfg(target_os = "linux")]
365fn nfs_file_identity(path: &Path) -> io::Result<Option<FileIdentity>> {
366    use std::mem::MaybeUninit;
367    use std::os::unix::ffi::OsStrExt as _;
368
369    let path = std::ffi::CString::new(path.as_os_str().as_bytes())
370        .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))?;
371    let mut status = MaybeUninit::<LinuxStatx>::zeroed();
372    // SAFETY: `path` is NUL-terminated and `status` names writable storage.
373    let result = unsafe {
374        libc::syscall(
375            libc::SYS_statx,
376            libc::AT_FDCWD,
377            path.as_ptr(),
378            // AT_STATX_DONT_SYNC: the surrounding build already trusts cached
379            // mtimes, and FORCE_SYNC costs one NFS RPC for every dependency
380            // edge rather than every distinct file.
381            0x4000,
382            STATX_IDENTITY_MASK,
383            status.as_mut_ptr(),
384        )
385    };
386    if result != 0 {
387        let error = io::Error::last_os_error();
388        return match error.raw_os_error() {
389            Some(libc::ENOSYS | libc::EINVAL | libc::EOPNOTSUPP) => Ok(None),
390            _ => Err(error),
391        };
392    }
393    // SAFETY: statx returned success and initialized `status`.
394    let status = unsafe { status.assume_init() };
395    nfs_identity_from_statx(
396        Path::new(std::ffi::OsStr::from_bytes(path.to_bytes())),
397        &status,
398    )
399}
400
401#[cfg(target_os = "linux")]
402fn nfs_identity_from_statx(path: &Path, status: &LinuxStatx) -> io::Result<Option<FileIdentity>> {
403    if status.mask & STATX_IDENTITY_MASK != STATX_IDENTITY_MASK
404        || status.modified.nanos >= 1_000_000_000
405    {
406        return Ok(None);
407    }
408    let modified = system_time(status.modified.seconds, status.modified.nanos)?;
409    Ok(Some(FileIdentity {
410        path: path.to_path_buf(),
411        len: status.size,
412        modified,
413        changed: None,
414        object: Some(FileObjectIdentity {
415            device_major: status.device_major,
416            device_minor: status.device_minor,
417            mount_id: status.mount_id,
418            inode: status.inode,
419        }),
420    }))
421}
422
423#[cfg(not(target_os = "linux"))]
424fn nfs_file_identity(_path: &Path) -> io::Result<Option<FileIdentity>> {
425    Ok(None)
426}
427
428#[cfg(target_os = "linux")]
429fn system_time(seconds: i64, nanos: u32) -> io::Result<SystemTime> {
430    if seconds >= 0 {
431        SystemTime::UNIX_EPOCH.checked_add(std::time::Duration::new(seconds as u64, nanos))
432    } else {
433        SystemTime::UNIX_EPOCH
434            .checked_sub(std::time::Duration::from_secs(seconds.unsigned_abs()))
435            .and_then(|time| time.checked_add(std::time::Duration::from_nanos(nanos.into())))
436    }
437    .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "file timestamp is out of range"))
438}
439
440/// The metadata-change time as an opaque token, where the platform has one.
441#[cfg(unix)]
442fn change_token(metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
443    use std::os::unix::fs::MetadataExt;
444    Some((metadata.ctime(), metadata.ctime_nsec()))
445}
446
447/// Windows reports creation rather than metadata-change time, which a rewrite
448/// does not move, so no token is better than a misleading one.
449#[cfg(not(unix))]
450fn change_token(_metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
451    None
452}
453
454#[cfg(unix)]
455fn metadata_object_identity(metadata: &std::fs::Metadata) -> Option<FileObjectIdentity> {
456    use std::os::unix::fs::MetadataExt as _;
457
458    let device = metadata.dev();
459    Some(FileObjectIdentity {
460        device_major: (device >> 32) as u32,
461        device_minor: device as u32,
462        mount_id: 0,
463        inode: metadata.ino(),
464    })
465}
466
467#[cfg(not(unix))]
468fn metadata_object_identity(_metadata: &std::fs::Metadata) -> Option<FileObjectIdentity> {
469    None
470}
471
472/// A file digest recorded against the identity it was read under.
473#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
474#[serde(deny_unknown_fields)]
475pub struct RecordedFileDigest {
476    /// Identity of the file when its contents were hashed.
477    pub file: FileIdentity,
478    /// Digest of those contents.
479    pub digest: CacheDigest,
480}
481
482/// What a recorded file digest may stand in for.
483///
484/// Adapters prove different things when they read a file: the cc adapter's
485/// input scan also establishes that a source embeds no timestamp macro, which
486/// a digest recorded by the rustc adapter never checked. Scoping the ledger
487/// keeps one adapter's shortcut from resting on a property another adapter
488/// never established.
489#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
490#[serde(rename_all = "snake_case")]
491pub enum FileDigestScope {
492    /// The digest describes the file's contents and nothing more.
493    Content,
494    /// The digest describes a cc compiler input that also passed the
495    /// timestamp-macro scan.
496    CcInput,
497}
498
499/// The outcome of resolving one file under a digest scope.
500#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
501#[serde(tag = "kind", rename_all = "snake_case")]
502pub enum FileDigestResolution {
503    /// The file's content digest, with any scope-specific checks complete.
504    Digest(CacheDigest),
505    /// A C/C++ input contains a time-dependent preprocessor macro.
506    EmbeddedTimestampMacro,
507    /// No shared resolver was available; the caller must read the file.
508    Unresolved,
509}
510
511/// A file digest tied to the file object and pathname observed while reading.
512#[derive(Debug, Clone, PartialEq, Eq)]
513pub struct ValidatedFileDigest {
514    /// Scope-specific outcome of reading the file.
515    pub resolution: FileDigestResolution,
516    /// Identity of the pathname after the validated read.
517    pub identity: FileIdentity,
518    /// Identity suitable for reuse in the digest ledger, when available.
519    pub cache_identity: Option<FileIdentity>,
520}
521
522impl FileDigestResolution {
523    /// Extract the digest when resolution succeeded.
524    pub fn into_digest(self) -> Option<CacheDigest> {
525        match self {
526            Self::Digest(digest) => Some(digest),
527            Self::EmbeddedTimestampMacro | Self::Unresolved => None,
528        }
529    }
530}
531
532/// Read and hash a file once, applying the checks required by `scope` in that
533/// same pass.
534pub fn digest_file(scope: FileDigestScope, path: &Path) -> io::Result<FileDigestResolution> {
535    let file = std::fs::File::open(path)?;
536    digest_reader(scope, &file).map(|(resolution, _)| resolution)
537}
538
539/// Read and hash a regular file only when the open handle remains stable and
540/// the pathname still names that same file after the read.
541pub fn digest_file_validated(
542    scope: FileDigestScope,
543    path: &Path,
544) -> io::Result<Option<ValidatedFileDigest>> {
545    let file = std::fs::File::open(path)?;
546    let before = file.metadata()?;
547    if !before.is_file() {
548        return Err(io::Error::new(
549            io::ErrorKind::InvalidInput,
550            format!("digest path is not a regular file: {}", path.display()),
551        ));
552    }
553    let (resolution, size) = digest_reader(scope, &file)?;
554    let after = file.metadata()?;
555    let current_metadata = std::fs::metadata(path)?;
556    let unreliable = metadata_identity_is_unreliable(path, &current_metadata)?;
557    let cache_identity = digest_cache_identity(path, &current_metadata, unreliable)?;
558    let identity = cache_identity
559        .clone()
560        .or_else(|| FileIdentity::describe(path, &current_metadata));
561    let Some(identity) = identity else {
562        return Ok(None);
563    };
564    if !metadata_names_same_file(&after, &current_metadata) {
565        return Err(io::Error::new(
566            io::ErrorKind::WouldBlock,
567            "file was replaced while its contents were being read",
568        ));
569    }
570    if !handle_and_path_are_stable(path, &before, &after, &identity, unreliable, size)? {
571        return Err(io::Error::new(
572            io::ErrorKind::WouldBlock,
573            "file changed while its contents were being read",
574        ));
575    }
576    Ok(Some(ValidatedFileDigest {
577        resolution,
578        identity,
579        cache_identity,
580    }))
581}
582
583fn digest_reader(
584    scope: FileDigestScope,
585    file: &std::fs::File,
586) -> io::Result<(FileDigestResolution, u64)> {
587    let mut reader = std::io::BufReader::new(file);
588    let mut hasher = blake3::Hasher::new();
589    let mut size = 0_u64;
590    let longest_macro = TIMESTAMP_MACROS
591        .iter()
592        .map(|macro_name| macro_name.len())
593        .max()
594        .unwrap_or_default();
595    let mut window = Vec::with_capacity(DIGEST_BUFFER_BYTES + longest_macro);
596    let mut chunk = vec![0_u8; DIGEST_BUFFER_BYTES];
597    let mut found_timestamp_macro = false;
598    loop {
599        let read = reader.read(&mut chunk)?;
600        if read == 0 {
601            break;
602        }
603        hasher.update(&chunk[..read]);
604        size = size
605            .checked_add(read as u64)
606            .ok_or_else(|| io::Error::other("file length overflowed u64"))?;
607        if scope == FileDigestScope::CcInput && !found_timestamp_macro {
608            window.extend_from_slice(&chunk[..read]);
609            found_timestamp_macro = TIMESTAMP_MACROS
610                .iter()
611                .any(|macro_name| contains_subslice(&window, macro_name));
612            let keep = window.len().saturating_sub(longest_macro.saturating_sub(1));
613            window.drain(..keep);
614        }
615    }
616    if found_timestamp_macro {
617        Ok((FileDigestResolution::EmbeddedTimestampMacro, size))
618    } else {
619        Ok((
620            FileDigestResolution::Digest(CacheDigest {
621                algorithm: "blake3".into(),
622                hash: hasher.finalize().to_hex().to_string(),
623                size,
624            }),
625            size,
626        ))
627    }
628}
629
630fn handle_and_path_are_stable(
631    path: &Path,
632    before: &std::fs::Metadata,
633    after: &std::fs::Metadata,
634    current: &FileIdentity,
635    unreliable: bool,
636    digest_size: u64,
637) -> io::Result<bool> {
638    if before.len() != after.len() || after.len() != digest_size || current.len != digest_size {
639        return Ok(false);
640    }
641    if !metadata_names_same_file(before, after) {
642        return Ok(false);
643    }
644    if unreliable {
645        #[cfg(target_os = "linux")]
646        {
647            let before_modified = before.modified()?;
648            let after_modified = after.modified()?;
649            let object_matches = current.object.as_ref().is_none_or(|object| {
650                metadata_matches_object(before, object) && metadata_matches_object(after, object)
651            });
652            return Ok(before_modified == after_modified
653                && after_modified == current.modified
654                && object_matches);
655        }
656        #[cfg(not(target_os = "linux"))]
657        {
658            let _ = (path, before, after, current);
659            return Ok(false);
660        }
661    }
662    let before = FileIdentity::describe(path, before);
663    let after = FileIdentity::describe(path, after);
664    Ok(before.is_some() && before == after && after.as_ref() == Some(current))
665}
666
667#[cfg(unix)]
668fn metadata_names_same_file(left: &std::fs::Metadata, right: &std::fs::Metadata) -> bool {
669    use std::os::unix::fs::MetadataExt as _;
670
671    left.dev() == right.dev() && left.ino() == right.ino()
672}
673
674#[cfg(not(unix))]
675fn metadata_names_same_file(_left: &std::fs::Metadata, _right: &std::fs::Metadata) -> bool {
676    true
677}
678
679#[cfg(target_os = "linux")]
680fn metadata_matches_object(metadata: &std::fs::Metadata, object: &FileObjectIdentity) -> bool {
681    use std::os::unix::fs::MetadataExt as _;
682
683    let device = metadata.dev();
684    let major = ((device >> 8) & 0x0fff) | ((device >> 32) & 0xfffff000);
685    let minor = (device & 0x00ff) | ((device >> 12) & 0xffffff00);
686    u32::try_from(major).ok() == Some(object.device_major)
687        && u32::try_from(minor).ok() == Some(object.device_minor)
688        && metadata.ino() == object.inode
689}
690
691fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
692    !needle.is_empty()
693        && haystack.len() >= needle.len()
694        && haystack
695            .windows(needle.len())
696            .any(|window| window == needle)
697}
698
699/// Recorded digests a session may consult instead of rehashing a file.
700///
701/// The agent's file-digest ledger answers through this everywhere a caller
702/// holds one; the no-op implementation stands in where reuse must not happen,
703/// such as under verification.
704pub trait FileDigestCache: Send + Sync {
705    /// Resolve these identities, coalescing concurrent misses where supported.
706    fn resolve(&self, scope: FileDigestScope, files: &[FileIdentity]) -> Vec<FileDigestResolution> {
707        self.find(scope, files)
708            .into_iter()
709            .map(|digest| {
710                digest.map_or(
711                    FileDigestResolution::Unresolved,
712                    FileDigestResolution::Digest,
713                )
714            })
715            .collect()
716    }
717    /// Recorded digests for these identities, in request order.
718    fn find(&self, scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>>;
719    /// Record digests for files read under these identities.
720    fn record(&self, scope: FileDigestScope, entries: Vec<RecordedFileDigest>);
721}
722
723/// A [`FileDigestCache`] that remembers nothing and finds nothing.
724pub struct NoFileDigestCache;
725
726impl FileDigestCache for NoFileDigestCache {
727    fn resolve(
728        &self,
729        _scope: FileDigestScope,
730        files: &[FileIdentity],
731    ) -> Vec<FileDigestResolution> {
732        vec![FileDigestResolution::Unresolved; files.len()]
733    }
734
735    fn find(&self, _scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>> {
736        vec![None; files.len()]
737    }
738
739    fn record(&self, _scope: FileDigestScope, _entries: Vec<RecordedFileDigest>) {}
740}
741
742#[cfg(test)]
743mod tests {
744    use super::*;
745
746    /// rustc links a finished `.rmeta` into its incremental directory, which
747    /// changes the file's metadata but not its bytes. A metadata snapshot
748    /// cannot tell that from a rewrite; a content snapshot can, and still
749    /// notices when the bytes do change.
750    #[cfg(unix)]
751    #[test]
752    fn a_content_snapshot_survives_a_hardlink_and_a_metadata_one_does_not() {
753        let directory = tempfile::tempdir().unwrap();
754        let path = directory.path().join("libdep.rmeta");
755        std::fs::write(&path, b"metadata").unwrap();
756        let by_metadata = FileSnapshot::capture(&path).unwrap().unwrap();
757        let by_content = FileSnapshot::capture_content_with_cache(&path, &NoFileDigestCache)
758            .unwrap()
759            .unwrap();
760        let digest = CacheDigest::blake3_file(&path).unwrap();
761        // A new ctime, so the change token cannot be mistaken for the old one.
762        std::thread::sleep(std::time::Duration::from_millis(20));
763        std::fs::hard_link(&path, directory.path().join("metadata.rmeta")).unwrap();
764        let after = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
765
766        assert!(by_content.proves_content_change());
767        assert!(by_content.matches(Some(&after), &digest));
768        if by_metadata.proves_content_change() {
769            assert!(!by_metadata.matches(Some(&after), &digest));
770        }
771
772        std::fs::write(&path, b"rewritten").unwrap();
773        let rewritten = CacheDigest::blake3_file(&path).unwrap();
774        let after = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
775        assert!(!by_content.matches(Some(&after), &rewritten));
776    }
777
778    #[cfg(unix)]
779    struct RecordingDigestCache {
780        answer: Option<CacheDigest>,
781        recorded: std::sync::Mutex<Vec<RecordedFileDigest>>,
782    }
783
784    #[cfg(unix)]
785    impl FileDigestCache for RecordingDigestCache {
786        fn find(
787            &self,
788            _scope: FileDigestScope,
789            files: &[FileIdentity],
790        ) -> Vec<Option<CacheDigest>> {
791            assert_eq!(files.len(), 1);
792            vec![self.answer.clone()]
793        }
794
795        fn record(&self, _scope: FileDigestScope, entries: Vec<RecordedFileDigest>) {
796            self.recorded.lock().unwrap().extend(entries);
797        }
798    }
799
800    #[cfg(unix)]
801    #[test]
802    fn content_snapshot_records_a_replacement_under_its_own_identity() {
803        let directory = tempfile::tempdir().unwrap();
804        let path = directory.path().join("input.rlib");
805        let replacement = directory.path().join("replacement.rlib");
806        std::fs::write(&path, b"original....").unwrap();
807        std::fs::File::options()
808            .write(true)
809            .open(&path)
810            .unwrap()
811            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
812            .unwrap();
813        std::fs::write(&replacement, b"replacement!").unwrap();
814        std::fs::File::options()
815            .write(true)
816            .open(&replacement)
817            .unwrap()
818            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
819            .unwrap();
820        let before = std::fs::metadata(&path).unwrap();
821        let old_identity = FileIdentity::describe(&path, &before).unwrap();
822        std::fs::remove_file(&path).unwrap();
823        std::fs::rename(&replacement, &path).unwrap();
824        let cache = RecordingDigestCache {
825            answer: None,
826            recorded: std::sync::Mutex::new(Vec::new()),
827        };
828
829        let snapshot = capture_file_snapshot(&path, &cache, true, true, before)
830            .unwrap()
831            .unwrap();
832        let current = FileIdentity::for_digest_cache(&path, &std::fs::metadata(&path).unwrap())
833            .unwrap()
834            .unwrap();
835        let digest = CacheDigest::blake3_file(&path).unwrap();
836
837        assert_eq!(current.modified, old_identity.modified);
838        assert_ne!(current.object, old_identity.object);
839        assert_ne!(current, old_identity);
840        assert!(snapshot.matches(Some(&current), &digest));
841        let recorded = cache.recorded.lock().unwrap();
842        assert_eq!(recorded.len(), 1);
843        assert_eq!(recorded[0].file, current);
844        assert_eq!(recorded[0].digest, digest);
845    }
846
847    #[cfg(unix)]
848    #[test]
849    fn content_snapshot_rejects_a_stale_digest_cache_hit() {
850        let directory = tempfile::tempdir().unwrap();
851        let path = directory.path().join("input.rlib");
852        let replacement = directory.path().join("replacement.rlib");
853        std::fs::write(&path, b"original....").unwrap();
854        std::fs::File::options()
855            .write(true)
856            .open(&path)
857            .unwrap()
858            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
859            .unwrap();
860        let before = std::fs::metadata(&path).unwrap();
861        std::fs::write(&replacement, b"replacement!").unwrap();
862        std::fs::File::options()
863            .write(true)
864            .open(&replacement)
865            .unwrap()
866            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
867            .unwrap();
868        std::fs::remove_file(&path).unwrap();
869        std::fs::rename(&replacement, &path).unwrap();
870        let cache = RecordingDigestCache {
871            answer: Some(CacheDigest::blake3(b"original....")),
872            recorded: std::sync::Mutex::new(Vec::new()),
873        };
874
875        let snapshot = capture_file_snapshot(&path, &cache, true, true, before)
876            .unwrap()
877            .unwrap();
878        let current = FileIdentity::for_digest_cache(&path, &std::fs::metadata(&path).unwrap())
879            .unwrap()
880            .unwrap();
881        let digest = CacheDigest::blake3_file(&path).unwrap();
882
883        assert!(snapshot.matches(Some(&current), &digest));
884        assert_ne!(snapshot.content, Some(CacheDigest::blake3(b"original....")));
885        assert_eq!(cache.recorded.lock().unwrap()[0].digest, digest);
886    }
887
888    #[cfg(unix)]
889    #[test]
890    fn a_replaced_path_does_not_validate_the_open_handle() {
891        let directory = tempfile::tempdir().unwrap();
892        let path = directory.path().join("input.rlib");
893        let replacement = directory.path().join("replacement.rlib");
894        std::fs::write(&path, b"old bytes").unwrap();
895        std::fs::write(&replacement, b"new bytes").unwrap();
896
897        let file = std::fs::File::open(&path).unwrap();
898        let before = file.metadata().unwrap();
899        std::fs::rename(&replacement, &path).unwrap();
900        let (_, size) = digest_reader(FileDigestScope::Content, &file).unwrap();
901        let after = file.metadata().unwrap();
902        let current_metadata = std::fs::metadata(&path).unwrap();
903        let current = FileIdentity::describe(&path, &current_metadata).unwrap();
904
905        assert!(!metadata_names_same_file(&after, &current_metadata));
906        assert!(
907            !handle_and_path_are_stable(&path, &before, &after, &current, false, size).unwrap()
908        );
909    }
910
911    #[test]
912    fn an_identity_describes_the_file_until_it_is_written_or_removed() {
913        let directory = tempfile::tempdir().unwrap();
914        let path = directory.path().join("input.rs");
915        std::fs::write(&path, b"fn main() {}").unwrap();
916        let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
917        assert!(identity.still_describes().unwrap());
918
919        std::thread::sleep(std::time::Duration::from_millis(20));
920        std::fs::write(&path, b"fn main() { }").unwrap();
921        assert!(!identity.still_describes().unwrap());
922
923        std::fs::remove_file(&path).unwrap();
924        assert!(identity.still_describes().is_err());
925    }
926
927    #[test]
928    fn a_metadata_snapshot_detects_a_metadata_change() {
929        let directory = tempfile::tempdir().unwrap();
930        let path = directory.path().join("input.rs");
931        std::fs::write(&path, b"fn main() {}").unwrap();
932        let snapshot = capture_file_snapshot(
933            &path,
934            &NoFileDigestCache,
935            false,
936            false,
937            std::fs::metadata(&path).unwrap(),
938        )
939        .unwrap()
940        .unwrap();
941
942        std::fs::File::options()
943            .write(true)
944            .open(&path)
945            .unwrap()
946            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
947            .unwrap();
948        let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
949        let digest = CacheDigest::blake3_file(&path).unwrap();
950
951        assert!(!snapshot.matches(identity.as_ref(), &digest));
952        assert_eq!(snapshot.proves_content_change(), cfg!(unix));
953    }
954
955    #[test]
956    fn a_content_snapshot_ignores_timestamp_churn_but_detects_changed_bytes() {
957        let directory = tempfile::tempdir().unwrap();
958        let path = directory.path().join("input.rs");
959        std::fs::write(&path, b"fn main() {}").unwrap();
960        let snapshot = capture_file_snapshot(
961            &path,
962            &NoFileDigestCache,
963            true,
964            true,
965            std::fs::metadata(&path).unwrap(),
966        )
967        .unwrap()
968        .unwrap();
969
970        let mut identity = snapshot.identity.clone();
971        identity.changed = identity
972            .changed
973            .map(|(seconds, nanos)| (seconds + 1, nanos));
974        let digest = CacheDigest::blake3_file(&path).unwrap();
975        assert!(snapshot.matches(Some(&identity), &digest));
976        assert!(snapshot.proves_content_change());
977
978        identity.modified = SystemTime::UNIX_EPOCH;
979        assert!(snapshot.matches(Some(&identity), &digest));
980
981        std::fs::write(&path, b"fn main(){ }").unwrap();
982        let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
983        let digest = CacheDigest::blake3_file(&path).unwrap();
984        assert!(!snapshot.matches(identity.as_ref(), &digest));
985    }
986
987    #[test]
988    fn reliable_metadata_keeps_the_native_identity() {
989        let directory = tempfile::tempdir().unwrap();
990        let path = directory.path().join("input.rs");
991        std::fs::write(&path, b"fn main() {}").unwrap();
992        let metadata = std::fs::metadata(&path).unwrap();
993
994        let identity = digest_cache_identity(&path, &metadata, false)
995            .unwrap()
996            .unwrap();
997
998        assert_eq!(identity.path, path);
999        assert_eq!(identity.len, 12);
1000        assert_eq!(identity.object.is_some(), cfg!(unix));
1001    }
1002
1003    #[test]
1004    fn content_snapshot_ignores_nfs_timestamp_churn_but_not_content_or_identity_changes() {
1005        let digest = CacheDigest::blake3(b"nfs bytes");
1006        let identity = FileIdentity {
1007            path: PathBuf::from("/nfs/input.rlib"),
1008            len: digest.size,
1009            modified: SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(10),
1010            changed: Some((10, 1)),
1011            object: Some(FileObjectIdentity {
1012                device_major: 0,
1013                device_minor: 42,
1014                mount_id: 7,
1015                inode: 99,
1016            }),
1017        };
1018        let snapshot = FileSnapshot {
1019            identity: identity.clone(),
1020            content: Some(digest.clone()),
1021        };
1022        // Client/server reconciliation can move either timestamp in either
1023        // direction. None of these observations proves another write.
1024        for seconds in [9, 11] {
1025            let mut after = identity.clone();
1026            after.changed = Some((seconds, 500));
1027            after.modified = SystemTime::UNIX_EPOCH + std::time::Duration::new(seconds as u64, 500);
1028            assert!(snapshot.matches(Some(&after), &digest));
1029        }
1030
1031        // Timestamp tolerance does not admit changed bytes, even when their
1032        // length and all metadata remain the same.
1033        let changed = CacheDigest::blake3(b"NFS bytes");
1034        assert_eq!(changed.size, digest.size);
1035        assert!(!snapshot.matches(Some(&identity), &changed));
1036        assert!(!snapshot.matches(None, &digest));
1037
1038        let mut after = identity.clone();
1039        after.object.as_mut().unwrap().inode += 1;
1040        assert!(!snapshot.matches(Some(&after), &digest));
1041
1042        let mut after = identity.clone();
1043        after.len += 1;
1044        assert!(!snapshot.matches(Some(&after), &digest));
1045
1046        let mut after = identity;
1047        after.path.set_file_name("replacement.rlib");
1048        assert!(!snapshot.matches(Some(&after), &digest));
1049    }
1050
1051    #[cfg(target_os = "linux")]
1052    #[test]
1053    fn incomplete_statx_metadata_falls_back_to_content_hashing() {
1054        // SAFETY: all-zero is a valid unpopulated statx result for this parser.
1055        let status = unsafe { std::mem::zeroed::<LinuxStatx>() };
1056        assert!(
1057            nfs_identity_from_statx(Path::new("/nfs/input.rlib"), &status)
1058                .unwrap()
1059                .is_none()
1060        );
1061    }
1062}
mbx_cache_core/agent/file_digest.rs

mbx_cache_core/agent/
file_digest.rs
