mbx_cache_core/agent/
file_digest.rs

1use crate::CacheDigest;
2use serde::{Deserialize, Serialize};
3use std::io;
4use std::path::{Path, PathBuf};
5use std::time::SystemTime;
6
7/// The on-disk identity a recorded file digest describes.
8///
9/// The same trade [`VerifiedBlob`] makes for CAS reads, offered to shims for
10/// the files they hash: an overwrite moves the modification time and a
11/// truncation changes the length, so a digest recorded against both stands
12/// until either does. Where the platform reports a metadata-change time the
13/// identity carries that too, and it is the part a writer cannot restore: a
14/// rewrite that puts the modification time back still moves the change time,
15/// so only filesystems without one fall back to the freshness model the
16/// surrounding build tool already lives on.
17#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
18#[serde(deny_unknown_fields)]
19pub struct FileIdentity {
20    /// Absolute path of the file.
21    pub path: PathBuf,
22    /// Length of the file in bytes.
23    pub len: u64,
24    /// Modification time of the file.
25    pub modified: SystemTime,
26    /// Platform metadata-change token, where one exists.
27    pub changed: Option<(i64, i64)>,
28}
29
30impl FileIdentity {
31    /// Describe a file from metadata already in hand, or nothing when the
32    /// filesystem reports no modification time to compare against later.
33    pub fn describe(path: &Path, metadata: &std::fs::Metadata) -> Option<Self> {
34        Some(Self {
35            path: path.to_path_buf(),
36            len: metadata.len(),
37            modified: metadata.modified().ok()?,
38            changed: change_token(metadata),
39        })
40    }
41
42    /// Describe a file only when metadata can safely stand in for its digest.
43    ///
44    /// Linux NFS may revise cached timestamps without a write and may delay a
45    /// real writer's final timestamps. Returning no identity makes digest
46    /// users hash the file instead of trusting that ambiguous metadata.
47    pub fn for_digest_cache(path: &Path, metadata: &std::fs::Metadata) -> io::Result<Option<Self>> {
48        Ok(digest_cache_identity(
49            path,
50            metadata,
51            metadata_identity_is_unreliable(path)?,
52        ))
53    }
54
55    /// Whether the file at this identity's path still has exactly this
56    /// identity, so the digest recorded against it still describes the bytes on
57    /// disk without reading them again.
58    ///
59    /// Length alone would miss an overwrite that keeps the size, and the
60    /// modification time can be put back by whoever rewrote the file. The
61    /// change time cannot be set from user space, so where the platform reports
62    /// one a rewrite that restores the old modification time still shows. A
63    /// file that has vanished is an error rather than a change, so the caller
64    /// can tell the two apart.
65    pub fn still_describes(&self) -> std::io::Result<bool> {
66        let metadata = std::fs::metadata(&self.path)?;
67        Ok(Self::describe(&self.path, &metadata).as_ref() == Some(self))
68    }
69}
70
71/// A pre-operation snapshot that can prove whether a file's contents changed.
72///
73/// Most filesystems provide a stable metadata-change token, so the inexpensive
74/// identity is sufficient. Linux NFS can reconcile the client and server
75/// change times after a writer has closed the file, making two metadata reads
76/// disagree without any intervening write. Those files carry a content digest
77/// instead, while retaining length and modification time as an independent
78/// signal for a write that restored the original bytes. Callers use the same
79/// comparison either way and do not need to know which filesystem supplied the
80/// file.
81#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct FileSnapshot {
83    identity: FileIdentity,
84    content: Option<CacheDigest>,
85}
86
87impl FileSnapshot {
88    /// Capture the strongest comparison the file's filesystem can support.
89    pub fn capture(path: &Path) -> io::Result<Option<Self>> {
90        capture_file_snapshot(path, metadata_identity_is_unreliable(path)?)
91    }
92
93    /// Whether `identity` and `content` still describe this snapshot.
94    ///
95    /// A content-backed snapshot deliberately ignores the unreliable change
96    /// token, but still requires the modification time to remain stable. That
97    /// prevents a write followed by restoration of the original bytes from
98    /// passing only because the endpoint digests agree.
99    pub fn matches(&self, identity: Option<&FileIdentity>, content: &CacheDigest) -> bool {
100        self.content.as_ref().map_or_else(
101            || identity == Some(&self.identity),
102            |before| {
103                before == content
104                    && identity.is_some_and(|after| {
105                        self.identity.path == after.path
106                            && self.identity.len == after.len
107                            && self.identity.modified == after.modified
108                    })
109            },
110        )
111    }
112
113    /// Whether a mismatch proves the file's contents changed.
114    pub fn proves_content_change(&self) -> bool {
115        self.content.is_some() || self.identity.changed.is_some()
116    }
117}
118
119impl From<FileIdentity> for FileSnapshot {
120    fn from(identity: FileIdentity) -> Self {
121        Self {
122            identity,
123            content: None,
124        }
125    }
126}
127
128#[cfg(target_os = "linux")]
129fn metadata_identity_is_unreliable(path: &Path) -> io::Result<bool> {
130    use std::mem::MaybeUninit;
131    use std::os::unix::ffi::OsStrExt as _;
132
133    let path = std::ffi::CString::new(path.as_os_str().as_bytes())
134        .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))?;
135    let mut status = MaybeUninit::<libc::statfs>::zeroed();
136    // SAFETY: `path` is NUL-terminated and `status` points to writable,
137    // correctly sized storage. statfs initializes it before returning success.
138    let result = unsafe { libc::statfs(path.as_ptr(), status.as_mut_ptr()) };
139    if result != 0 {
140        return Err(io::Error::last_os_error());
141    }
142    // SAFETY: statfs returned success and initialized `status`.
143    let status = unsafe { status.assume_init() };
144    // libc gives NFS_SUPER_MAGIC a different signedness from statfs::f_type on musl.
145    Ok(status.f_type == 0x6969)
146}
147
148#[cfg(not(target_os = "linux"))]
149fn metadata_identity_is_unreliable(_path: &Path) -> io::Result<bool> {
150    Ok(false)
151}
152
153fn capture_file_snapshot(path: &Path, content_identity: bool) -> io::Result<Option<FileSnapshot>> {
154    let metadata = std::fs::metadata(path)?;
155    let Some(identity) = FileIdentity::describe(path, &metadata) else {
156        return Ok(None);
157    };
158    let content = content_identity
159        .then(|| {
160            CacheDigest::blake3_file(path).map_err(|error| io::Error::other(error.to_string()))
161        })
162        .transpose()?;
163    Ok(Some(FileSnapshot { identity, content }))
164}
165
166fn digest_cache_identity(
167    path: &Path,
168    metadata: &std::fs::Metadata,
169    unreliable: bool,
170) -> Option<FileIdentity> {
171    if unreliable {
172        None
173    } else {
174        FileIdentity::describe(path, metadata)
175    }
176}
177
178/// The metadata-change time as an opaque token, where the platform has one.
179#[cfg(unix)]
180fn change_token(metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
181    use std::os::unix::fs::MetadataExt;
182    Some((metadata.ctime(), metadata.ctime_nsec()))
183}
184
185/// Windows reports creation rather than metadata-change time, which a rewrite
186/// does not move, so no token is better than a misleading one.
187#[cfg(not(unix))]
188fn change_token(_metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
189    None
190}
191
192/// A file digest recorded against the identity it was read under.
193#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
194#[serde(deny_unknown_fields)]
195pub struct RecordedFileDigest {
196    /// Identity of the file when its contents were hashed.
197    pub file: FileIdentity,
198    /// Digest of those contents.
199    pub digest: CacheDigest,
200}
201
202/// What a recorded file digest may stand in for.
203///
204/// Adapters prove different things when they read a file: the cc adapter's
205/// input scan also establishes that a source embeds no timestamp macro, which
206/// a digest recorded by the rustc adapter never checked. Scoping the ledger
207/// keeps one adapter's shortcut from resting on a property another adapter
208/// never established.
209#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
210#[serde(rename_all = "snake_case")]
211pub enum FileDigestScope {
212    /// The digest describes the file's contents and nothing more.
213    Content,
214    /// The digest describes a cc compiler input that also passed the
215    /// timestamp-macro scan.
216    CcInput,
217}
218
219/// Recorded digests a session may consult instead of rehashing a file.
220///
221/// The agent's file-digest ledger answers through this everywhere a caller
222/// holds one; the no-op implementation stands in where reuse must not happen,
223/// such as under verification.
224pub trait FileDigestCache: Send + Sync {
225    /// Recorded digests for these identities, in request order.
226    fn find(&self, scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>>;
227    /// Record digests for files read under these identities.
228    fn record(&self, scope: FileDigestScope, entries: Vec<RecordedFileDigest>);
229}
230
231/// A [`FileDigestCache`] that remembers nothing and finds nothing.
232pub struct NoFileDigestCache;
233
234impl FileDigestCache for NoFileDigestCache {
235    fn find(&self, _scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>> {
236        vec![None; files.len()]
237    }
238
239    fn record(&self, _scope: FileDigestScope, _entries: Vec<RecordedFileDigest>) {}
240}
241
242#[cfg(test)]
243mod tests {
244    use super::*;
245
246    #[test]
247    fn an_identity_describes_the_file_until_it_is_written_or_removed() {
248        let directory = tempfile::tempdir().unwrap();
249        let path = directory.path().join("input.rs");
250        std::fs::write(&path, b"fn main() {}").unwrap();
251        let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
252        assert!(identity.still_describes().unwrap());
253
254        std::thread::sleep(std::time::Duration::from_millis(20));
255        std::fs::write(&path, b"fn main() { }").unwrap();
256        assert!(!identity.still_describes().unwrap());
257
258        std::fs::remove_file(&path).unwrap();
259        assert!(identity.still_describes().is_err());
260    }
261
262    #[test]
263    fn a_metadata_snapshot_detects_a_metadata_change() {
264        let directory = tempfile::tempdir().unwrap();
265        let path = directory.path().join("input.rs");
266        std::fs::write(&path, b"fn main() {}").unwrap();
267        let snapshot = capture_file_snapshot(&path, false).unwrap().unwrap();
268
269        std::fs::File::options()
270            .write(true)
271            .open(&path)
272            .unwrap()
273            .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
274            .unwrap();
275        let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
276        let digest = CacheDigest::blake3_file(&path).unwrap();
277
278        assert!(!snapshot.matches(identity.as_ref(), &digest));
279        assert_eq!(snapshot.proves_content_change(), cfg!(unix));
280    }
281
282    #[test]
283    fn a_content_snapshot_ignores_change_token_churn_but_detects_other_changes() {
284        let directory = tempfile::tempdir().unwrap();
285        let path = directory.path().join("input.rs");
286        std::fs::write(&path, b"fn main() {}").unwrap();
287        let snapshot = capture_file_snapshot(&path, true).unwrap().unwrap();
288
289        let mut identity =
290            FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
291        identity.changed = identity
292            .changed
293            .map(|(seconds, nanos)| (seconds + 1, nanos));
294        let digest = CacheDigest::blake3_file(&path).unwrap();
295        assert!(snapshot.matches(Some(&identity), &digest));
296        assert!(snapshot.proves_content_change());
297
298        identity.modified = SystemTime::UNIX_EPOCH;
299        assert!(!snapshot.matches(Some(&identity), &digest));
300
301        std::fs::write(&path, b"fn main(){ }").unwrap();
302        let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
303        let digest = CacheDigest::blake3_file(&path).unwrap();
304        assert!(!snapshot.matches(identity.as_ref(), &digest));
305    }
306
307    #[test]
308    fn unreliable_metadata_is_not_used_as_a_digest_cache_identity() {
309        let directory = tempfile::tempdir().unwrap();
310        let path = directory.path().join("input.rs");
311        std::fs::write(&path, b"fn main() {}").unwrap();
312        let metadata = std::fs::metadata(&path).unwrap();
313
314        let identity = digest_cache_identity(&path, &metadata, false).unwrap();
315
316        assert_eq!(identity.path, path);
317        assert_eq!(identity.len, 12);
318        assert!(digest_cache_identity(&identity.path, &metadata, true).is_none());
319    }
320}
mbx_cache_core/agent/file_digest.rs

mbx_cache_core/agent/
file_digest.rs
