1use crate::CacheDigest;
2use serde::{Deserialize, Serialize};
3use std::io::{self, Read as _};
4use std::path::{Path, PathBuf};
5#[cfg(target_os = "linux")]
6use std::sync::{Mutex, OnceLock};
7use std::time::SystemTime;
8
9const DIGEST_BUFFER_BYTES: usize = 64 * 1024;
10const TIMESTAMP_MACROS: &[&[u8]] = &[b"__DATE__", b"__TIME__", b"__TIMESTAMP__"];
11
12#[cfg(target_os = "linux")]
13const STATX_IDENTITY_MASK: u32 = 0x100 | 0x200 | 0x40 | 0x1000;
14
15#[cfg(target_os = "linux")]
18#[repr(C)]
19struct LinuxStatxTimestamp {
20 seconds: i64,
21 nanos: u32,
22 reserved: i32,
23}
24
25#[cfg(target_os = "linux")]
26#[repr(C)]
27struct LinuxStatx {
28 mask: u32,
29 block_size: u32,
30 attributes: u64,
31 links: u32,
32 uid: u32,
33 gid: u32,
34 mode: u16,
35 reserved0: u16,
36 inode: u64,
37 size: u64,
38 blocks: u64,
39 attributes_mask: u64,
40 accessed: LinuxStatxTimestamp,
41 created: LinuxStatxTimestamp,
42 changed: LinuxStatxTimestamp,
43 modified: LinuxStatxTimestamp,
44 rdev_major: u32,
45 rdev_minor: u32,
46 device_major: u32,
47 device_minor: u32,
48 mount_id: u64,
49 direct_io_memory_alignment: u32,
50 direct_io_offset_alignment: u32,
51 subvolume: u64,
52 atomic_write_unit_min: u32,
53 atomic_write_unit_max: u32,
54 atomic_write_segments_max: u32,
55 direct_io_read_offset_alignment: u32,
56 atomic_write_unit_max_opt: u32,
57 reserved1: u32,
58 reserved2: [u64; 8],
59}
60
61#[cfg(target_os = "linux")]
62const _: () = assert!(std::mem::size_of::<LinuxStatx>() == 256);
63
64#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
75#[serde(deny_unknown_fields)]
76pub struct FileIdentity {
77 pub path: PathBuf,
79 pub len: u64,
81 pub modified: SystemTime,
83 pub changed: Option<(i64, i64)>,
85 #[serde(default, skip_serializing_if = "Option::is_none")]
88 pub object: Option<FileObjectIdentity>,
89}
90
91#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
93#[serde(deny_unknown_fields)]
94pub struct FileObjectIdentity {
95 pub device_major: u32,
98 pub device_minor: u32,
101 pub mount_id: u64,
103 pub inode: u64,
105}
106
107impl FileIdentity {
108 pub fn describe(path: &Path, metadata: &std::fs::Metadata) -> Option<Self> {
111 Some(Self {
112 path: path.to_path_buf(),
113 len: metadata.len(),
114 modified: metadata.modified().ok()?,
115 changed: change_token(metadata),
116 object: metadata_object_identity(metadata),
117 })
118 }
119
120 pub fn for_digest_cache(path: &Path, metadata: &std::fs::Metadata) -> io::Result<Option<Self>> {
128 digest_cache_identity(
129 path,
130 metadata,
131 metadata_identity_is_unreliable(path, metadata)?,
132 )
133 }
134
135 pub fn still_describes(&self) -> std::io::Result<bool> {
146 let metadata = std::fs::metadata(&self.path)?;
147 Ok(Self::for_digest_cache(&self.path, &metadata)?.as_ref() == Some(self))
148 }
149
150 pub fn can_skip_content_verification(&self) -> bool {
152 self.changed.is_some() || self.object.is_some()
153 }
154}
155
156#[derive(Debug, Clone, PartialEq, Eq)]
169pub struct FileSnapshot {
170 identity: FileIdentity,
171 content: Option<CacheDigest>,
172 require_full_identity: bool,
173}
174
175impl FileSnapshot {
176 pub fn capture(path: &Path) -> io::Result<Option<Self>> {
178 Self::capture_with_cache(path, &NoFileDigestCache)
179 }
180
181 pub fn capture_with_cache(
184 path: &Path,
185 digests: &dyn FileDigestCache,
186 ) -> io::Result<Option<Self>> {
187 let observed_at = SystemTime::now();
188 let metadata = std::fs::metadata(path)?;
189 let unreliable = metadata_identity_is_unreliable(path, &metadata)?;
190 capture_file_snapshot(path, digests, unreliable, unreliable, metadata, observed_at)
191 }
192
193 pub fn capture_content_with_cache(
202 path: &Path,
203 digests: &dyn FileDigestCache,
204 ) -> io::Result<Option<Self>> {
205 let observed_at = SystemTime::now();
206 let metadata = std::fs::metadata(path)?;
207 let unreliable = metadata_identity_is_unreliable(path, &metadata)?;
211 capture_file_snapshot(path, digests, unreliable, true, metadata, observed_at)
212 }
213
214 pub fn matches(&self, identity: Option<&FileIdentity>, content: &CacheDigest) -> bool {
224 self.content.as_ref().map_or_else(
225 || identity == Some(&self.identity),
226 |before| {
227 before == content
228 && identity.is_some_and(|after| {
229 if self.require_full_identity {
230 &self.identity == after
231 } else {
232 self.identity.path == after.path
233 && self.identity.len == after.len
234 && self.identity.object == after.object
235 }
236 })
237 },
238 )
239 }
240
241 pub fn proves_content_change(&self) -> bool {
243 self.content.is_some() || self.identity.changed.is_some()
244 }
245}
246
247impl From<FileIdentity> for FileSnapshot {
248 fn from(identity: FileIdentity) -> Self {
249 Self {
250 identity,
251 content: None,
252 require_full_identity: true,
253 }
254 }
255}
256
257#[cfg(target_os = "linux")]
258fn metadata_identity_is_unreliable(path: &Path, metadata: &std::fs::Metadata) -> io::Result<bool> {
259 use std::mem::MaybeUninit;
260 use std::os::unix::ffi::OsStrExt as _;
261 use std::os::unix::fs::MetadataExt as _;
262
263 static FILESYSTEMS: OnceLock<Mutex<std::collections::BTreeMap<u64, bool>>> = OnceLock::new();
264 let filesystems = FILESYSTEMS.get_or_init(|| Mutex::new(std::collections::BTreeMap::new()));
265 if let Some(unreliable) = filesystems.lock().unwrap().get(&metadata.dev()).copied() {
266 return Ok(unreliable);
267 }
268
269 let path = std::ffi::CString::new(path.as_os_str().as_bytes())
270 .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))?;
271 let mut status = MaybeUninit::<libc::statfs>::zeroed();
272 let result = unsafe { libc::statfs(path.as_ptr(), status.as_mut_ptr()) };
275 if result != 0 {
276 return Err(io::Error::last_os_error());
277 }
278 let status = unsafe { status.assume_init() };
280 let unreliable = status.f_type == 0x6969;
282 filesystems
283 .lock()
284 .unwrap()
285 .insert(metadata.dev(), unreliable);
286 Ok(unreliable)
287}
288
289#[cfg(not(target_os = "linux"))]
290fn metadata_identity_is_unreliable(
291 _path: &Path,
292 _metadata: &std::fs::Metadata,
293) -> io::Result<bool> {
294 Ok(false)
295}
296
297fn capture_file_snapshot(
301 path: &Path,
302 digests: &dyn FileDigestCache,
303 unreliable: bool,
304 content_identity: bool,
305 metadata: std::fs::Metadata,
306 observed_at: SystemTime,
307) -> io::Result<Option<FileSnapshot>> {
308 let cache_identity = digest_cache_identity(path, &metadata, unreliable)?;
309 let Some(identity) = cache_identity
310 .clone()
311 .or_else(|| FileIdentity::describe(path, &metadata))
312 else {
313 return Ok(None);
314 };
315 let racy = identity_is_racy(&identity, observed_at);
319 let cache_identity = cache_identity.filter(|_| !racy);
320 let (identity, content) = if content_identity || racy {
321 let resolved = cache_identity
322 .as_ref()
323 .and_then(|identity| {
324 digests
325 .resolve(FileDigestScope::Content, std::slice::from_ref(identity))
326 .pop()
327 })
328 .unwrap_or(FileDigestResolution::Unresolved);
329 let cached = match resolved {
330 FileDigestResolution::Digest(digest)
331 if cache_identity.as_ref().is_some_and(|identity| {
332 identity.len == digest.size && identity.still_describes().unwrap_or(false)
333 }) =>
334 {
335 Some(digest)
336 }
337 FileDigestResolution::Digest(_)
338 | FileDigestResolution::EmbeddedTimestampMacro
339 | FileDigestResolution::Unresolved => None,
340 };
341 if let Some(digest) = cached {
342 (identity, Some(digest))
343 } else {
344 let observed = digest_file_validated(FileDigestScope::Content, path)?
345 .ok_or_else(|| io::Error::other("could not establish a file digest identity"))?;
346 let digest = observed
347 .resolution
348 .into_digest()
349 .ok_or_else(|| io::Error::other("content digest resolution returned no digest"))?;
350 if let Some(file) = observed.cache_identity
351 && file.len == digest.size
352 {
353 digests.record(
354 FileDigestScope::Content,
355 vec![RecordedFileDigest {
356 file,
357 digest: digest.clone(),
358 }],
359 );
360 }
361 (observed.identity, Some(digest))
362 }
363 } else {
364 (identity, None)
365 };
366 Ok(Some(FileSnapshot {
367 identity,
368 content,
369 require_full_identity: !unreliable && !content_identity,
370 }))
371}
372
373fn digest_cache_identity(
374 path: &Path,
375 metadata: &std::fs::Metadata,
376 unreliable: bool,
377) -> io::Result<Option<FileIdentity>> {
378 if unreliable {
379 nfs_file_identity(path)
380 } else {
381 Ok(FileIdentity::describe(path, metadata))
382 }
383}
384
385#[cfg(target_os = "linux")]
386fn nfs_file_identity(path: &Path) -> io::Result<Option<FileIdentity>> {
387 use std::mem::MaybeUninit;
388 use std::os::unix::ffi::OsStrExt as _;
389
390 let path = std::ffi::CString::new(path.as_os_str().as_bytes())
391 .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))?;
392 let mut status = MaybeUninit::<LinuxStatx>::zeroed();
393 let result = unsafe {
395 libc::syscall(
396 libc::SYS_statx,
397 libc::AT_FDCWD,
398 path.as_ptr(),
399 0x4000,
403 STATX_IDENTITY_MASK,
404 status.as_mut_ptr(),
405 )
406 };
407 if result != 0 {
408 let error = io::Error::last_os_error();
409 return match error.raw_os_error() {
410 Some(libc::ENOSYS | libc::EINVAL | libc::EOPNOTSUPP) => Ok(None),
411 _ => Err(error),
412 };
413 }
414 let status = unsafe { status.assume_init() };
416 nfs_identity_from_statx(
417 Path::new(std::ffi::OsStr::from_bytes(path.to_bytes())),
418 &status,
419 )
420}
421
422#[cfg(target_os = "linux")]
423fn nfs_identity_from_statx(path: &Path, status: &LinuxStatx) -> io::Result<Option<FileIdentity>> {
424 if status.mask & STATX_IDENTITY_MASK != STATX_IDENTITY_MASK
425 || status.modified.nanos >= 1_000_000_000
426 {
427 return Ok(None);
428 }
429 let modified = system_time(status.modified.seconds, status.modified.nanos)?;
430 Ok(Some(FileIdentity {
431 path: path.to_path_buf(),
432 len: status.size,
433 modified,
434 changed: None,
435 object: Some(FileObjectIdentity {
436 device_major: status.device_major,
437 device_minor: status.device_minor,
438 mount_id: status.mount_id,
439 inode: status.inode,
440 }),
441 }))
442}
443
444#[cfg(not(target_os = "linux"))]
445fn nfs_file_identity(_path: &Path) -> io::Result<Option<FileIdentity>> {
446 Ok(None)
447}
448
449#[cfg(target_os = "linux")]
450fn system_time(seconds: i64, nanos: u32) -> io::Result<SystemTime> {
451 if seconds >= 0 {
452 SystemTime::UNIX_EPOCH.checked_add(std::time::Duration::new(seconds as u64, nanos))
453 } else {
454 SystemTime::UNIX_EPOCH
455 .checked_sub(std::time::Duration::from_secs(seconds.unsigned_abs()))
456 .and_then(|time| time.checked_add(std::time::Duration::from_nanos(nanos.into())))
457 }
458 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "file timestamp is out of range"))
459}
460
461#[cfg(unix)]
463fn change_token(metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
464 use std::os::unix::fs::MetadataExt;
465 Some((metadata.ctime(), metadata.ctime_nsec()))
466}
467
468#[cfg(not(unix))]
471fn change_token(_metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
472 None
473}
474
475#[cfg(unix)]
476fn metadata_object_identity(metadata: &std::fs::Metadata) -> Option<FileObjectIdentity> {
477 use std::os::unix::fs::MetadataExt as _;
478
479 let device = metadata.dev();
480 Some(FileObjectIdentity {
481 device_major: (device >> 32) as u32,
482 device_minor: device as u32,
483 mount_id: 0,
484 inode: metadata.ino(),
485 })
486}
487
488#[cfg(not(unix))]
489fn metadata_object_identity(_metadata: &std::fs::Metadata) -> Option<FileObjectIdentity> {
490 None
491}
492
493#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
495#[serde(deny_unknown_fields)]
496pub struct RecordedFileDigest {
497 pub file: FileIdentity,
499 pub digest: CacheDigest,
501}
502
503#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
511#[serde(rename_all = "snake_case")]
512pub enum FileDigestScope {
513 Content,
515 CcInput,
518}
519
520#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
522#[serde(tag = "kind", rename_all = "snake_case")]
523pub enum FileDigestResolution {
524 Digest(CacheDigest),
526 EmbeddedTimestampMacro,
528 Unresolved,
530}
531
532#[derive(Debug, Clone, PartialEq, Eq)]
534pub struct ValidatedFileDigest {
535 pub resolution: FileDigestResolution,
537 pub identity: FileIdentity,
539 pub cache_identity: Option<FileIdentity>,
541 pub racy: bool,
544}
545
546impl FileDigestResolution {
547 pub fn into_digest(self) -> Option<CacheDigest> {
549 match self {
550 Self::Digest(digest) => Some(digest),
551 Self::EmbeddedTimestampMacro | Self::Unresolved => None,
552 }
553 }
554}
555
556pub fn digest_file(scope: FileDigestScope, path: &Path) -> io::Result<FileDigestResolution> {
559 let file = std::fs::File::open(path)?;
560 digest_reader(scope, &file).map(|(resolution, _)| resolution)
561}
562
563pub fn digest_file_validated(
566 scope: FileDigestScope,
567 path: &Path,
568) -> io::Result<Option<ValidatedFileDigest>> {
569 let started = SystemTime::now();
572 let file = std::fs::File::open(path)?;
573 let before = file.metadata()?;
574 if !before.is_file() {
575 return Err(io::Error::new(
576 io::ErrorKind::InvalidInput,
577 format!("digest path is not a regular file: {}", path.display()),
578 ));
579 }
580 let (resolution, size) = digest_reader(scope, &file)?;
581 let after = file.metadata()?;
582 let current_metadata = std::fs::metadata(path)?;
583 let unreliable = metadata_identity_is_unreliable(path, ¤t_metadata)?;
584 let cache_identity = digest_cache_identity(path, ¤t_metadata, unreliable)?;
585 let identity = cache_identity
586 .clone()
587 .or_else(|| FileIdentity::describe(path, ¤t_metadata));
588 let Some(identity) = identity else {
589 return Ok(None);
590 };
591 if !metadata_names_same_file(&after, ¤t_metadata) {
592 return Err(io::Error::new(
593 io::ErrorKind::WouldBlock,
594 "file was replaced while its contents were being read",
595 ));
596 }
597 if !handle_and_path_are_stable(path, &before, &after, &identity, unreliable, size)? {
598 return Err(io::Error::new(
599 io::ErrorKind::WouldBlock,
600 "file changed while its contents were being read",
601 ));
602 }
603 let racy = cache_identity
608 .as_ref()
609 .is_some_and(|identity| identity_is_racy(identity, started));
610 let cache_identity = cache_identity.filter(|_| !racy);
611 Ok(Some(ValidatedFileDigest {
612 resolution,
613 identity,
614 cache_identity,
615 racy,
616 }))
617}
618
619const FINE_TIMESTAMP_RACE_WINDOW: std::time::Duration = std::time::Duration::from_millis(20);
623
624const COARSE_TIMESTAMP_RACE_WINDOW: std::time::Duration = std::time::Duration::from_secs(2);
627
628fn identity_is_racy(identity: &FileIdentity, observed_at: SystemTime) -> bool {
637 let nanos = |time: SystemTime| {
638 time.duration_since(SystemTime::UNIX_EPOCH)
639 .map_or(0, |since| since.subsec_nanos())
640 };
641 let coarse = nanos(identity.modified) == 0
642 && identity
643 .changed
644 .is_none_or(|(_, changed_nanos)| changed_nanos == 0);
645 let window = if coarse {
646 COARSE_TIMESTAMP_RACE_WINDOW
647 } else {
648 FINE_TIMESTAMP_RACE_WINDOW
649 };
650 let recent = |time: SystemTime| {
651 observed_at
652 .duration_since(time)
653 .map_or(true, |age| age < window)
654 };
655 recent(identity.modified)
656 || identity.changed.is_some_and(|(seconds, changed_nanos)| {
657 u64::try_from(seconds).map_or(true, |seconds| {
658 recent(
659 SystemTime::UNIX_EPOCH
660 + std::time::Duration::new(seconds, changed_nanos.try_into().unwrap_or(0)),
661 )
662 })
663 })
664}
665
666fn digest_reader(
667 scope: FileDigestScope,
668 file: &std::fs::File,
669) -> io::Result<(FileDigestResolution, u64)> {
670 let mut reader = std::io::BufReader::new(file);
671 let mut hasher = blake3::Hasher::new();
672 let mut size = 0_u64;
673 let longest_macro = TIMESTAMP_MACROS
674 .iter()
675 .map(|macro_name| macro_name.len())
676 .max()
677 .unwrap_or_default();
678 let mut window = Vec::with_capacity(DIGEST_BUFFER_BYTES + longest_macro);
679 let mut chunk = vec![0_u8; DIGEST_BUFFER_BYTES];
680 let mut found_timestamp_macro = false;
681 loop {
682 let read = reader.read(&mut chunk)?;
683 if read == 0 {
684 break;
685 }
686 hasher.update(&chunk[..read]);
687 size = size
688 .checked_add(read as u64)
689 .ok_or_else(|| io::Error::other("file length overflowed u64"))?;
690 if scope == FileDigestScope::CcInput && !found_timestamp_macro {
691 window.extend_from_slice(&chunk[..read]);
692 found_timestamp_macro = TIMESTAMP_MACROS
693 .iter()
694 .any(|macro_name| memchr::memmem::find(&window, macro_name).is_some());
695 let keep = window.len().saturating_sub(longest_macro.saturating_sub(1));
696 window.drain(..keep);
697 }
698 }
699 if found_timestamp_macro {
700 Ok((FileDigestResolution::EmbeddedTimestampMacro, size))
701 } else {
702 Ok((
703 FileDigestResolution::Digest(CacheDigest {
704 algorithm: "blake3".into(),
705 hash: hasher.finalize().to_hex().to_string(),
706 size,
707 }),
708 size,
709 ))
710 }
711}
712
713fn handle_and_path_are_stable(
714 path: &Path,
715 before: &std::fs::Metadata,
716 after: &std::fs::Metadata,
717 current: &FileIdentity,
718 unreliable: bool,
719 digest_size: u64,
720) -> io::Result<bool> {
721 if before.len() != after.len() || after.len() != digest_size || current.len != digest_size {
722 return Ok(false);
723 }
724 if !metadata_names_same_file(before, after) {
725 return Ok(false);
726 }
727 if unreliable {
728 #[cfg(target_os = "linux")]
729 {
730 let before_modified = before.modified()?;
731 let after_modified = after.modified()?;
732 let object_matches = current.object.as_ref().is_none_or(|object| {
733 metadata_matches_object(before, object) && metadata_matches_object(after, object)
734 });
735 return Ok(before_modified == after_modified
736 && after_modified == current.modified
737 && object_matches);
738 }
739 #[cfg(not(target_os = "linux"))]
740 {
741 let _ = (path, before, after, current);
742 return Ok(false);
743 }
744 }
745 let before = FileIdentity::describe(path, before);
746 let after = FileIdentity::describe(path, after);
747 Ok(before.is_some() && before == after && after.as_ref() == Some(current))
748}
749
750#[cfg(unix)]
751fn metadata_names_same_file(left: &std::fs::Metadata, right: &std::fs::Metadata) -> bool {
752 use std::os::unix::fs::MetadataExt as _;
753
754 left.dev() == right.dev() && left.ino() == right.ino()
755}
756
757#[cfg(not(unix))]
758fn metadata_names_same_file(_left: &std::fs::Metadata, _right: &std::fs::Metadata) -> bool {
759 true
760}
761
762#[cfg(target_os = "linux")]
763fn metadata_matches_object(metadata: &std::fs::Metadata, object: &FileObjectIdentity) -> bool {
764 use std::os::unix::fs::MetadataExt as _;
765
766 let device = metadata.dev();
767 let major = ((device >> 8) & 0x0fff) | ((device >> 32) & 0xfffff000);
768 let minor = (device & 0x00ff) | ((device >> 12) & 0xffffff00);
769 u32::try_from(major).ok() == Some(object.device_major)
770 && u32::try_from(minor).ok() == Some(object.device_minor)
771 && metadata.ino() == object.inode
772}
773
774pub trait FileDigestCache: Send + Sync {
780 fn resolve(&self, scope: FileDigestScope, files: &[FileIdentity]) -> Vec<FileDigestResolution> {
782 self.find(scope, files)
783 .into_iter()
784 .map(|digest| {
785 digest.map_or(
786 FileDigestResolution::Unresolved,
787 FileDigestResolution::Digest,
788 )
789 })
790 .collect()
791 }
792 fn find(&self, scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>>;
794 fn record(&self, scope: FileDigestScope, entries: Vec<RecordedFileDigest>);
796}
797
798pub struct NoFileDigestCache;
800
801impl FileDigestCache for NoFileDigestCache {
802 fn resolve(
803 &self,
804 _scope: FileDigestScope,
805 files: &[FileIdentity],
806 ) -> Vec<FileDigestResolution> {
807 vec![FileDigestResolution::Unresolved; files.len()]
808 }
809
810 fn find(&self, _scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>> {
811 vec![None; files.len()]
812 }
813
814 fn record(&self, _scope: FileDigestScope, _entries: Vec<RecordedFileDigest>) {}
815}
816
817#[cfg(test)]
818mod tests {
819 use super::*;
820
821 #[cfg(target_os = "linux")]
822 #[test]
823 fn an_nfs_snapshot_consults_the_session_digest_ledger() {
824 struct CountingCache(std::sync::atomic::AtomicUsize);
825 impl FileDigestCache for CountingCache {
826 fn find(
827 &self,
828 scope: FileDigestScope,
829 files: &[FileIdentity],
830 ) -> Vec<Option<CacheDigest>> {
831 assert_eq!(scope, FileDigestScope::Content);
832 assert_eq!(files.len(), 1);
833 assert_eq!(files[0].changed, None);
834 assert!(files[0].object.is_some());
835 self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
836 vec![None]
837 }
838 fn record(&self, _scope: FileDigestScope, _entries: Vec<RecordedFileDigest>) {}
839 }
840 let directory = tempfile::tempdir().unwrap();
841 let path = directory.path().join("input.rlib");
842 std::fs::write(&path, b"nfs bytes").unwrap();
843 if nfs_file_identity(&path).unwrap().is_none() {
844 eprintln!("skipping: statx cannot supply the NFS ledger identity");
845 return;
846 }
847 let cache = CountingCache(std::sync::atomic::AtomicUsize::new(0));
848 let metadata = std::fs::metadata(&path).unwrap();
849 let observed_at = metadata.modified().unwrap() + std::time::Duration::from_secs(3);
851 let snapshot = capture_file_snapshot(&path, &cache, true, true, metadata, observed_at)
852 .unwrap()
853 .unwrap();
854 assert_eq!(snapshot.content, Some(CacheDigest::blake3(b"nfs bytes")));
855 assert_eq!(cache.0.load(std::sync::atomic::Ordering::Relaxed), 1);
856 }
857
858 #[test]
859 fn timestamp_macros_are_detected_at_every_chunk_boundary_split() {
860 let directory = tempfile::tempdir().unwrap();
861 let path = directory.path().join("input.h");
862 for macro_name in TIMESTAMP_MACROS {
863 for split in 0..=macro_name.len() {
865 let mut bytes = vec![b'x'; DIGEST_BUFFER_BYTES - split];
866 bytes.extend_from_slice(macro_name);
867 std::fs::write(&path, &bytes).unwrap();
868 assert_eq!(
869 digest_file(FileDigestScope::CcInput, &path).unwrap(),
870 FileDigestResolution::EmbeddedTimestampMacro,
871 "macro {macro_name:?}, split {split}"
872 );
873 assert_eq!(
875 digest_file(FileDigestScope::Content, &path).unwrap(),
876 FileDigestResolution::Digest(CacheDigest::blake3(&bytes))
877 );
878 }
879 }
880 }
881
882 #[test]
883 fn timestamp_scan_accepts_empty_short_and_near_match_inputs() {
884 let directory = tempfile::tempdir().unwrap();
885 let path = directory.path().join("input.h");
886 for bytes in [
887 b"".as_slice(),
888 b"_",
889 b"__DATE_ __TIME_ __TIMESTAMP_",
890 b"__date__ __time__ __timestamp__",
891 &[0xff, 0, b'_', b'_'],
892 ] {
893 std::fs::write(&path, bytes).unwrap();
894 assert_eq!(
895 digest_file(FileDigestScope::CcInput, &path).unwrap(),
896 FileDigestResolution::Digest(CacheDigest::blake3(bytes))
897 );
898 }
899 }
900
901 #[cfg(unix)]
906 #[test]
907 fn a_content_snapshot_survives_a_hardlink_and_a_metadata_one_does_not() {
908 let directory = tempfile::tempdir().unwrap();
909 let path = directory.path().join("libdep.rmeta");
910 std::fs::write(&path, b"metadata").unwrap();
911 let metadata = std::fs::metadata(&path).unwrap();
912 let observed_at = metadata.modified().unwrap() + std::time::Duration::from_secs(3);
913 let by_metadata = capture_file_snapshot(
914 &path,
915 &NoFileDigestCache,
916 false,
917 false,
918 metadata,
919 observed_at,
920 )
921 .unwrap()
922 .unwrap();
923 let by_content = FileSnapshot::capture_content_with_cache(&path, &NoFileDigestCache)
924 .unwrap()
925 .unwrap();
926 let digest = CacheDigest::blake3_file(&path).unwrap();
927 std::thread::sleep(std::time::Duration::from_millis(20));
929 std::fs::hard_link(&path, directory.path().join("metadata.rmeta")).unwrap();
930 let after = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
931
932 assert!(by_content.proves_content_change());
933 assert!(by_content.matches(Some(&after), &digest));
934 if by_metadata.proves_content_change() {
935 assert!(!by_metadata.matches(Some(&after), &digest));
936 }
937
938 std::fs::write(&path, b"rewritten").unwrap();
939 let rewritten = CacheDigest::blake3_file(&path).unwrap();
940 let after = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
941 assert!(!by_content.matches(Some(&after), &rewritten));
942 }
943
944 #[cfg(unix)]
945 struct RecordingDigestCache {
946 answer: Option<CacheDigest>,
947 recorded: std::sync::Mutex<Vec<RecordedFileDigest>>,
948 }
949
950 #[cfg(unix)]
951 impl FileDigestCache for RecordingDigestCache {
952 fn find(
953 &self,
954 _scope: FileDigestScope,
955 files: &[FileIdentity],
956 ) -> Vec<Option<CacheDigest>> {
957 assert_eq!(files.len(), 1);
958 vec![self.answer.clone()]
959 }
960
961 fn record(&self, _scope: FileDigestScope, entries: Vec<RecordedFileDigest>) {
962 self.recorded.lock().unwrap().extend(entries);
963 }
964 }
965
966 #[cfg(unix)]
967 #[test]
968 fn content_snapshot_records_a_replacement_under_its_own_identity() {
969 let directory = tempfile::tempdir().unwrap();
970 let path = directory.path().join("input.rlib");
971 let replacement = directory.path().join("replacement.rlib");
972 std::fs::write(&path, b"original....").unwrap();
973 std::fs::File::options()
974 .write(true)
975 .open(&path)
976 .unwrap()
977 .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
978 .unwrap();
979 std::fs::write(&replacement, b"replacement!").unwrap();
980 std::fs::File::options()
981 .write(true)
982 .open(&replacement)
983 .unwrap()
984 .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
985 .unwrap();
986 let before = std::fs::metadata(&path).unwrap();
987 let old_identity = FileIdentity::describe(&path, &before).unwrap();
988 std::fs::remove_file(&path).unwrap();
989 std::fs::rename(&replacement, &path).unwrap();
990 let cache = RecordingDigestCache {
991 answer: None,
992 recorded: std::sync::Mutex::new(Vec::new()),
993 };
994
995 settle_past_timestamp_race();
996 let snapshot = capture_file_snapshot(&path, &cache, true, true, before, SystemTime::now())
997 .unwrap()
998 .unwrap();
999 let current = FileIdentity::for_digest_cache(&path, &std::fs::metadata(&path).unwrap())
1000 .unwrap()
1001 .unwrap();
1002 let digest = CacheDigest::blake3_file(&path).unwrap();
1003
1004 assert_eq!(current.modified, old_identity.modified);
1005 assert_ne!(current.object, old_identity.object);
1006 assert_ne!(current, old_identity);
1007 assert!(snapshot.matches(Some(¤t), &digest));
1008 let recorded = cache.recorded.lock().unwrap();
1009 assert_eq!(recorded.len(), 1);
1010 assert_eq!(recorded[0].file, current);
1011 assert_eq!(recorded[0].digest, digest);
1012 }
1013
1014 #[cfg(unix)]
1015 #[test]
1016 fn content_snapshot_rejects_a_stale_digest_cache_hit() {
1017 let directory = tempfile::tempdir().unwrap();
1018 let path = directory.path().join("input.rlib");
1019 let replacement = directory.path().join("replacement.rlib");
1020 std::fs::write(&path, b"original....").unwrap();
1021 std::fs::File::options()
1022 .write(true)
1023 .open(&path)
1024 .unwrap()
1025 .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
1026 .unwrap();
1027 let before = std::fs::metadata(&path).unwrap();
1028 std::fs::write(&replacement, b"replacement!").unwrap();
1029 std::fs::File::options()
1030 .write(true)
1031 .open(&replacement)
1032 .unwrap()
1033 .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
1034 .unwrap();
1035 std::fs::remove_file(&path).unwrap();
1036 std::fs::rename(&replacement, &path).unwrap();
1037 let cache = RecordingDigestCache {
1038 answer: Some(CacheDigest::blake3(b"original....")),
1039 recorded: std::sync::Mutex::new(Vec::new()),
1040 };
1041
1042 settle_past_timestamp_race();
1043 let snapshot = capture_file_snapshot(&path, &cache, true, true, before, SystemTime::now())
1044 .unwrap()
1045 .unwrap();
1046 let current = FileIdentity::for_digest_cache(&path, &std::fs::metadata(&path).unwrap())
1047 .unwrap()
1048 .unwrap();
1049 let digest = CacheDigest::blake3_file(&path).unwrap();
1050
1051 assert!(snapshot.matches(Some(¤t), &digest));
1052 assert_ne!(snapshot.content, Some(CacheDigest::blake3(b"original....")));
1053 assert_eq!(cache.recorded.lock().unwrap()[0].digest, digest);
1054 }
1055
1056 #[cfg(unix)]
1057 #[test]
1058 fn a_replaced_path_does_not_validate_the_open_handle() {
1059 let directory = tempfile::tempdir().unwrap();
1060 let path = directory.path().join("input.rlib");
1061 let replacement = directory.path().join("replacement.rlib");
1062 std::fs::write(&path, b"old bytes").unwrap();
1063 std::fs::write(&replacement, b"new bytes").unwrap();
1064
1065 let file = std::fs::File::open(&path).unwrap();
1066 let before = file.metadata().unwrap();
1067 std::fs::rename(&replacement, &path).unwrap();
1068 let (_, size) = digest_reader(FileDigestScope::Content, &file).unwrap();
1069 let after = file.metadata().unwrap();
1070 let current_metadata = std::fs::metadata(&path).unwrap();
1071 let current = FileIdentity::describe(&path, ¤t_metadata).unwrap();
1072
1073 assert!(!metadata_names_same_file(&after, ¤t_metadata));
1074 assert!(
1075 !handle_and_path_are_stable(&path, &before, &after, ¤t, false, size).unwrap()
1076 );
1077 }
1078
1079 #[test]
1080 fn an_identity_describes_the_file_until_it_is_written_or_removed() {
1081 let directory = tempfile::tempdir().unwrap();
1082 let path = directory.path().join("input.rs");
1083 std::fs::write(&path, b"fn main() {}").unwrap();
1084 let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
1085 assert!(identity.still_describes().unwrap());
1086
1087 std::thread::sleep(std::time::Duration::from_millis(20));
1088 std::fs::write(&path, b"fn main() { }").unwrap();
1089 assert!(!identity.still_describes().unwrap());
1090
1091 std::fs::remove_file(&path).unwrap();
1092 assert!(identity.still_describes().is_err());
1093 }
1094
1095 #[test]
1096 fn a_racy_snapshot_rejects_a_same_length_rewrite_with_identical_metadata() {
1097 let directory = tempfile::tempdir().unwrap();
1098 let path = directory.path().join("input.rs");
1099 std::fs::write(&path, b"fn main() {}").unwrap();
1100 let metadata = std::fs::metadata(&path).unwrap();
1101 let observed_at = metadata.modified().unwrap();
1102 let snapshot = capture_file_snapshot(
1103 &path,
1104 &NoFileDigestCache,
1105 false,
1106 false,
1107 metadata,
1108 observed_at,
1109 )
1110 .unwrap()
1111 .unwrap();
1112 let identity = snapshot.identity.clone();
1113 let original = CacheDigest::blake3_file(&path).unwrap();
1114 assert!(snapshot.matches(Some(&identity), &original));
1115
1116 std::fs::write(&path, b"fn main(){ }").unwrap();
1117 let rewritten = CacheDigest::blake3_file(&path).unwrap();
1118 assert_eq!(original.size, rewritten.size);
1119 assert_ne!(original, rewritten);
1120 assert!(!snapshot.matches(Some(&identity), &rewritten));
1123 assert!(snapshot.proves_content_change());
1124 }
1125
1126 #[cfg(unix)]
1127 #[test]
1128 fn a_racy_local_snapshot_rejects_an_edit_that_restores_the_original_bytes() {
1129 let directory = tempfile::tempdir().unwrap();
1130 let path = directory.path().join("input.rs");
1131 std::fs::write(&path, b"fn main() {}").unwrap();
1132 let metadata = std::fs::metadata(&path).unwrap();
1133 let observed_at = metadata.modified().unwrap();
1134 let snapshot = capture_file_snapshot(
1135 &path,
1136 &NoFileDigestCache,
1137 false,
1138 false,
1139 metadata,
1140 observed_at,
1141 )
1142 .unwrap()
1143 .unwrap();
1144 let original = CacheDigest::blake3_file(&path).unwrap();
1145 assert!(snapshot.matches(Some(&snapshot.identity), &original));
1146
1147 std::fs::write(&path, b"fn main(){ }").unwrap();
1148 std::fs::write(&path, b"fn main() {}").unwrap();
1149 let restored = CacheDigest::blake3_file(&path).unwrap();
1150 assert_eq!(original, restored);
1151 let mut after = snapshot.identity.clone();
1155 let (seconds, nanos) = after.changed.unwrap();
1156 after.changed = Some((seconds + 1, nanos));
1157 assert!(!snapshot.matches(Some(&after), &restored));
1158 }
1159
1160 #[test]
1161 fn a_settled_snapshot_keeps_the_metadata_only_path() {
1162 let directory = tempfile::tempdir().unwrap();
1163 let path = directory.path().join("input.rs");
1164 std::fs::write(&path, b"fn main() {}").unwrap();
1165 let metadata = std::fs::metadata(&path).unwrap();
1166 let observed_at = metadata.modified().unwrap() + std::time::Duration::from_secs(3);
1167 std::fs::remove_file(&path).unwrap();
1170 let snapshot = capture_file_snapshot(
1171 &path,
1172 &NoFileDigestCache,
1173 false,
1174 false,
1175 metadata,
1176 observed_at,
1177 )
1178 .unwrap()
1179 .unwrap();
1180 assert!(snapshot.content.is_none());
1181 }
1182
1183 #[cfg(unix)]
1184 #[test]
1185 fn a_racy_snapshot_does_not_reuse_a_digest_with_identical_metadata() {
1186 let directory = tempfile::tempdir().unwrap();
1187 let path = directory.path().join("input.rs");
1188 std::fs::write(&path, b"fn main() {}").unwrap();
1189 let metadata = std::fs::metadata(&path).unwrap();
1190 let observed_at = metadata.modified().unwrap();
1191 let cache = RecordingDigestCache {
1192 answer: Some(CacheDigest::blake3(b"fn main(){ }")),
1193 recorded: std::sync::Mutex::new(Vec::new()),
1194 };
1195 for content_identity in [false, true] {
1196 let snapshot = capture_file_snapshot(
1197 &path,
1198 &cache,
1199 false,
1200 content_identity,
1201 metadata.clone(),
1202 observed_at,
1203 )
1204 .unwrap()
1205 .unwrap();
1206 assert_eq!(snapshot.content, Some(CacheDigest::blake3(b"fn main() {}")));
1207 }
1208 }
1209
1210 #[test]
1211 fn a_metadata_snapshot_detects_a_metadata_change() {
1212 let directory = tempfile::tempdir().unwrap();
1213 let path = directory.path().join("input.rs");
1214 std::fs::write(&path, b"fn main() {}").unwrap();
1215 let metadata = std::fs::metadata(&path).unwrap();
1216 let observed_at = metadata.modified().unwrap() + std::time::Duration::from_secs(3);
1217 let snapshot = capture_file_snapshot(
1218 &path,
1219 &NoFileDigestCache,
1220 false,
1221 false,
1222 metadata,
1223 observed_at,
1224 )
1225 .unwrap()
1226 .unwrap();
1227
1228 std::fs::File::options()
1229 .write(true)
1230 .open(&path)
1231 .unwrap()
1232 .set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
1233 .unwrap();
1234 let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
1235 let digest = CacheDigest::blake3_file(&path).unwrap();
1236
1237 assert!(!snapshot.matches(identity.as_ref(), &digest));
1238 assert_eq!(snapshot.proves_content_change(), cfg!(unix));
1239 }
1240
1241 #[test]
1242 fn a_content_snapshot_ignores_timestamp_churn_but_detects_changed_bytes() {
1243 for (unreliable, content_identity) in [(true, false), (true, true), (false, true)] {
1244 let directory = tempfile::tempdir().unwrap();
1245 let path = directory.path().join("input.rs");
1246 std::fs::write(&path, b"fn main() {}").unwrap();
1247 let snapshot = capture_file_snapshot(
1248 &path,
1249 &NoFileDigestCache,
1250 unreliable,
1251 content_identity,
1252 std::fs::metadata(&path).unwrap(),
1253 SystemTime::now(),
1254 )
1255 .unwrap()
1256 .unwrap();
1257
1258 let mut identity = snapshot.identity.clone();
1259 identity.changed = identity
1260 .changed
1261 .map(|(seconds, nanos)| (seconds + 1, nanos));
1262 let digest = CacheDigest::blake3_file(&path).unwrap();
1263 assert!(snapshot.matches(Some(&identity), &digest));
1264 assert!(snapshot.proves_content_change());
1265
1266 identity.modified = SystemTime::UNIX_EPOCH;
1267 assert!(snapshot.matches(Some(&identity), &digest));
1268
1269 std::fs::write(&path, b"fn main(){ }").unwrap();
1270 let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
1271 let digest = CacheDigest::blake3_file(&path).unwrap();
1272 assert!(!snapshot.matches(identity.as_ref(), &digest));
1273 }
1274 }
1275
1276 #[test]
1277 fn an_nfs_object_identity_can_skip_a_second_content_read() {
1278 let identity = FileIdentity {
1279 path: PathBuf::from("/nfs/input.rlib"),
1280 len: 9,
1281 modified: SystemTime::UNIX_EPOCH,
1282 changed: None,
1283 object: Some(FileObjectIdentity {
1284 device_major: 0,
1285 device_minor: 42,
1286 mount_id: 7,
1287 inode: 99,
1288 }),
1289 };
1290 assert!(identity.can_skip_content_verification());
1291 let without_object = FileIdentity {
1292 object: None,
1293 ..identity
1294 };
1295 assert!(!without_object.can_skip_content_verification());
1296 }
1297
1298 #[test]
1299 fn reliable_metadata_keeps_the_native_identity() {
1300 let directory = tempfile::tempdir().unwrap();
1301 let path = directory.path().join("input.rs");
1302 std::fs::write(&path, b"fn main() {}").unwrap();
1303 let metadata = std::fs::metadata(&path).unwrap();
1304
1305 let identity = digest_cache_identity(&path, &metadata, false)
1306 .unwrap()
1307 .unwrap();
1308
1309 assert_eq!(identity.path, path);
1310 assert_eq!(identity.len, 12);
1311 assert_eq!(identity.object.is_some(), cfg!(unix));
1312 }
1313
1314 #[test]
1315 fn content_snapshot_ignores_nfs_timestamp_churn_but_not_content_or_identity_changes() {
1316 let digest = CacheDigest::blake3(b"nfs bytes");
1317 let identity = FileIdentity {
1318 path: PathBuf::from("/nfs/input.rlib"),
1319 len: digest.size,
1320 modified: SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(10),
1321 changed: Some((10, 1)),
1322 object: Some(FileObjectIdentity {
1323 device_major: 0,
1324 device_minor: 42,
1325 mount_id: 7,
1326 inode: 99,
1327 }),
1328 };
1329 let snapshot = FileSnapshot {
1330 identity: identity.clone(),
1331 content: Some(digest.clone()),
1332 require_full_identity: false,
1333 };
1334 for seconds in [9, 11] {
1337 let mut after = identity.clone();
1338 after.changed = Some((seconds, 500));
1339 after.modified = SystemTime::UNIX_EPOCH + std::time::Duration::new(seconds as u64, 500);
1340 assert!(snapshot.matches(Some(&after), &digest));
1341 }
1342
1343 let changed = CacheDigest::blake3(b"NFS bytes");
1346 assert_eq!(changed.size, digest.size);
1347 assert!(!snapshot.matches(Some(&identity), &changed));
1348 assert!(!snapshot.matches(None, &digest));
1349
1350 let mut after = identity.clone();
1351 after.object.as_mut().unwrap().inode += 1;
1352 assert!(!snapshot.matches(Some(&after), &digest));
1353
1354 let mut after = identity.clone();
1355 after.len += 1;
1356 assert!(!snapshot.matches(Some(&after), &digest));
1357
1358 let mut after = identity;
1359 after.path.set_file_name("replacement.rlib");
1360 assert!(!snapshot.matches(Some(&after), &digest));
1361 }
1362
1363 #[cfg(target_os = "linux")]
1364 #[test]
1365 fn incomplete_statx_metadata_falls_back_to_content_hashing() {
1366 let status = unsafe { std::mem::zeroed::<LinuxStatx>() };
1368 assert!(
1369 nfs_identity_from_statx(Path::new("/nfs/input.rlib"), &status)
1370 .unwrap()
1371 .is_none()
1372 );
1373 }
1374
1375 fn settle_past_timestamp_race() {
1378 std::thread::sleep(FINE_TIMESTAMP_RACE_WINDOW * 2);
1379 }
1380
1381 #[test]
1384 fn a_digest_of_a_recently_changed_file_is_not_offered_for_reuse() {
1385 let directory = tempfile::tempdir().unwrap();
1386 let path = directory.path().join("val.rs");
1387 std::fs::write(&path, b"pub const VAL: &str = \"v1\";").unwrap();
1388 std::fs::File::options()
1389 .write(true)
1390 .open(&path)
1391 .unwrap()
1392 .set_times(
1393 std::fs::FileTimes::new()
1394 .set_modified(SystemTime::now() + std::time::Duration::from_secs(3600)),
1395 )
1396 .unwrap();
1397
1398 let fresh = digest_file_validated(FileDigestScope::Content, &path)
1399 .unwrap()
1400 .unwrap();
1401 assert!(fresh.resolution.into_digest().is_some());
1402 assert!(fresh.racy);
1403 assert_eq!(fresh.cache_identity, None);
1404 }
1405
1406 #[test]
1407 fn a_digest_of_a_settled_file_is_offered_for_reuse() {
1408 let directory = tempfile::tempdir().unwrap();
1409 let path = directory.path().join("val.rs");
1410 std::fs::write(&path, b"pub const VAL: &str = \"v1\";").unwrap();
1411
1412 settle_past_timestamp_race();
1413 let settled = digest_file_validated(FileDigestScope::Content, &path)
1414 .unwrap()
1415 .unwrap();
1416 assert!(!settled.racy);
1417 assert!(settled.cache_identity.is_some());
1418 }
1419
1420 #[test]
1421 fn racy_window_follows_the_timestamp_resolution() {
1422 let at = |seconds: u64, nanos: u32| {
1423 SystemTime::UNIX_EPOCH + std::time::Duration::new(seconds, nanos)
1424 };
1425 let identity = |modified: SystemTime, changed: Option<(i64, i64)>| FileIdentity {
1426 path: PathBuf::from("/x"),
1427 len: 1,
1428 modified,
1429 changed,
1430 object: None,
1431 };
1432 let now = at(1_000, 500_000_000);
1433 let fine = |age_ms: u64| {
1434 let when = now - std::time::Duration::from_millis(age_ms);
1435 let since = when.duration_since(SystemTime::UNIX_EPOCH).unwrap();
1436 identity(
1437 when,
1438 Some((since.as_secs() as i64, i64::from(since.subsec_nanos()))),
1439 )
1440 };
1441 assert!(identity_is_racy(&fine(5), now));
1442 assert!(!identity_is_racy(&fine(25), now));
1443 let touched = identity(at(10, 7), Some((1_000, 499_000_000)));
1445 assert!(identity_is_racy(&touched, now));
1446 for with_ctime in [false, true] {
1448 let coarse = |age: u64| {
1449 identity(
1450 at(1_000 - age, 0),
1451 with_ctime.then_some((1_000 - age as i64, 0)),
1452 )
1453 };
1454 assert!(identity_is_racy(&coarse(1), at(1_000, 0)));
1455 assert!(!identity_is_racy(&coarse(3), at(1_000, 0)));
1456 }
1457 assert!(identity_is_racy(&identity(at(2_000, 1), None), now));
1459 }
1460}