use crate::CacheDigest;
use serde::{Deserialize, Serialize};
use std::io::{self, Read as _};
use std::path::{Path, PathBuf};
#[cfg(target_os = "linux")]
use std::sync::{Mutex, OnceLock};
use std::time::SystemTime;
const DIGEST_BUFFER_BYTES: usize = 64 * 1024;
const TIMESTAMP_MACROS: &[&[u8]] = &[b"__DATE__", b"__TIME__", b"__TIMESTAMP__"];
#[cfg(target_os = "linux")]
const STATX_IDENTITY_MASK: u32 = 0x100 | 0x200 | 0x40 | 0x1000;
#[cfg(target_os = "linux")]
#[repr(C)]
struct LinuxStatxTimestamp {
seconds: i64,
nanos: u32,
reserved: i32,
}
#[cfg(target_os = "linux")]
#[repr(C)]
struct LinuxStatx {
mask: u32,
block_size: u32,
attributes: u64,
links: u32,
uid: u32,
gid: u32,
mode: u16,
reserved0: u16,
inode: u64,
size: u64,
blocks: u64,
attributes_mask: u64,
accessed: LinuxStatxTimestamp,
created: LinuxStatxTimestamp,
changed: LinuxStatxTimestamp,
modified: LinuxStatxTimestamp,
rdev_major: u32,
rdev_minor: u32,
device_major: u32,
device_minor: u32,
mount_id: u64,
direct_io_memory_alignment: u32,
direct_io_offset_alignment: u32,
subvolume: u64,
atomic_write_unit_min: u32,
atomic_write_unit_max: u32,
atomic_write_segments_max: u32,
direct_io_read_offset_alignment: u32,
atomic_write_unit_max_opt: u32,
reserved1: u32,
reserved2: [u64; 8],
}
#[cfg(target_os = "linux")]
const _: () = assert!(std::mem::size_of::<LinuxStatx>() == 256);
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct FileIdentity {
pub path: PathBuf,
pub len: u64,
pub modified: SystemTime,
pub changed: Option<(i64, i64)>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub object: Option<FileObjectIdentity>,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct FileObjectIdentity {
pub device_major: u32,
pub device_minor: u32,
pub mount_id: u64,
pub inode: u64,
}
impl FileIdentity {
pub fn describe(path: &Path, metadata: &std::fs::Metadata) -> Option<Self> {
Some(Self {
path: path.to_path_buf(),
len: metadata.len(),
modified: metadata.modified().ok()?,
changed: change_token(metadata),
object: None,
})
}
pub fn for_digest_cache(path: &Path, metadata: &std::fs::Metadata) -> io::Result<Option<Self>> {
digest_cache_identity(
path,
metadata,
metadata_identity_is_unreliable(path, metadata)?,
)
}
pub fn still_describes(&self) -> std::io::Result<bool> {
let metadata = std::fs::metadata(&self.path)?;
Ok(Self::for_digest_cache(&self.path, &metadata)?.as_ref() == Some(self))
}
pub fn can_skip_content_verification(&self) -> bool {
self.changed.is_some() || self.object.is_some()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileSnapshot {
identity: FileIdentity,
content: Option<CacheDigest>,
}
impl FileSnapshot {
pub fn capture(path: &Path) -> io::Result<Option<Self>> {
let metadata = std::fs::metadata(path)?;
capture_file_snapshot(
path,
&NoFileDigestCache,
metadata_identity_is_unreliable(path, &metadata)?,
metadata,
)
}
pub fn capture_with_cache(
path: &Path,
digests: &dyn FileDigestCache,
) -> io::Result<Option<Self>> {
let metadata = std::fs::metadata(path)?;
capture_file_snapshot(
path,
digests,
metadata_identity_is_unreliable(path, &metadata)?,
metadata,
)
}
pub fn matches(&self, identity: Option<&FileIdentity>, content: &CacheDigest) -> bool {
self.content.as_ref().map_or_else(
|| identity == Some(&self.identity),
|before| {
before == content
&& identity.is_some_and(|after| {
self.identity.path == after.path
&& self.identity.len == after.len
&& self.identity.modified == after.modified
&& self.identity.object == after.object
})
},
)
}
pub fn proves_content_change(&self) -> bool {
self.content.is_some() || self.identity.changed.is_some()
}
}
impl From<FileIdentity> for FileSnapshot {
fn from(identity: FileIdentity) -> Self {
Self {
identity,
content: None,
}
}
}
#[cfg(target_os = "linux")]
fn metadata_identity_is_unreliable(path: &Path, metadata: &std::fs::Metadata) -> io::Result<bool> {
use std::mem::MaybeUninit;
use std::os::unix::ffi::OsStrExt as _;
use std::os::unix::fs::MetadataExt as _;
static FILESYSTEMS: OnceLock<Mutex<std::collections::BTreeMap<u64, bool>>> = OnceLock::new();
let filesystems = FILESYSTEMS.get_or_init(|| Mutex::new(std::collections::BTreeMap::new()));
if let Some(unreliable) = filesystems.lock().unwrap().get(&metadata.dev()).copied() {
return Ok(unreliable);
}
let path = std::ffi::CString::new(path.as_os_str().as_bytes())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))?;
let mut status = MaybeUninit::<libc::statfs>::zeroed();
let result = unsafe { libc::statfs(path.as_ptr(), status.as_mut_ptr()) };
if result != 0 {
return Err(io::Error::last_os_error());
}
let status = unsafe { status.assume_init() };
let unreliable = status.f_type == 0x6969;
filesystems
.lock()
.unwrap()
.insert(metadata.dev(), unreliable);
Ok(unreliable)
}
#[cfg(not(target_os = "linux"))]
fn metadata_identity_is_unreliable(
_path: &Path,
_metadata: &std::fs::Metadata,
) -> io::Result<bool> {
Ok(false)
}
fn capture_file_snapshot(
path: &Path,
digests: &dyn FileDigestCache,
content_identity: bool,
metadata: std::fs::Metadata,
) -> io::Result<Option<FileSnapshot>> {
let cache_identity = digest_cache_identity(path, &metadata, content_identity)?;
let Some(identity) = cache_identity
.clone()
.or_else(|| FileIdentity::describe(path, &metadata))
else {
return Ok(None);
};
let content = if content_identity {
let resolved = cache_identity
.as_ref()
.and_then(|identity| {
digests
.resolve(FileDigestScope::Content, std::slice::from_ref(identity))
.pop()
})
.unwrap_or(FileDigestResolution::Unresolved);
let (digest, fresh) = match resolved {
FileDigestResolution::Digest(digest) => (digest, false),
FileDigestResolution::EmbeddedTimestampMacro | FileDigestResolution::Unresolved => {
let digest = digest_file(FileDigestScope::Content, path)?
.into_digest()
.ok_or_else(|| {
io::Error::other("content digest resolution returned no digest")
})?;
(digest, true)
}
};
if fresh
&& let Some(file) = cache_identity
&& file.len == digest.size
{
digests.record(
FileDigestScope::Content,
vec![RecordedFileDigest {
file,
digest: digest.clone(),
}],
);
}
Some(digest)
} else {
None
};
Ok(Some(FileSnapshot { identity, content }))
}
fn digest_cache_identity(
path: &Path,
metadata: &std::fs::Metadata,
unreliable: bool,
) -> io::Result<Option<FileIdentity>> {
if unreliable {
nfs_file_identity(path)
} else {
Ok(FileIdentity::describe(path, metadata))
}
}
#[cfg(target_os = "linux")]
fn nfs_file_identity(path: &Path) -> io::Result<Option<FileIdentity>> {
use std::mem::MaybeUninit;
use std::os::unix::ffi::OsStrExt as _;
let path = std::ffi::CString::new(path.as_os_str().as_bytes())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))?;
let mut status = MaybeUninit::<LinuxStatx>::zeroed();
let result = unsafe {
libc::syscall(
libc::SYS_statx,
libc::AT_FDCWD,
path.as_ptr(),
0x4000,
STATX_IDENTITY_MASK,
status.as_mut_ptr(),
)
};
if result != 0 {
let error = io::Error::last_os_error();
return match error.raw_os_error() {
Some(libc::ENOSYS | libc::EINVAL | libc::EOPNOTSUPP) => Ok(None),
_ => Err(error),
};
}
let status = unsafe { status.assume_init() };
nfs_identity_from_statx(
Path::new(std::ffi::OsStr::from_bytes(path.to_bytes())),
&status,
)
}
#[cfg(target_os = "linux")]
fn nfs_identity_from_statx(path: &Path, status: &LinuxStatx) -> io::Result<Option<FileIdentity>> {
if status.mask & STATX_IDENTITY_MASK != STATX_IDENTITY_MASK
|| status.modified.nanos >= 1_000_000_000
{
return Ok(None);
}
let modified = system_time(status.modified.seconds, status.modified.nanos)?;
Ok(Some(FileIdentity {
path: path.to_path_buf(),
len: status.size,
modified,
changed: None,
object: Some(FileObjectIdentity {
device_major: status.device_major,
device_minor: status.device_minor,
mount_id: status.mount_id,
inode: status.inode,
}),
}))
}
#[cfg(not(target_os = "linux"))]
fn nfs_file_identity(_path: &Path) -> io::Result<Option<FileIdentity>> {
Ok(None)
}
#[cfg(target_os = "linux")]
fn system_time(seconds: i64, nanos: u32) -> io::Result<SystemTime> {
if seconds >= 0 {
SystemTime::UNIX_EPOCH.checked_add(std::time::Duration::new(seconds as u64, nanos))
} else {
SystemTime::UNIX_EPOCH
.checked_sub(std::time::Duration::from_secs(seconds.unsigned_abs()))
.and_then(|time| time.checked_add(std::time::Duration::from_nanos(nanos.into())))
}
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "file timestamp is out of range"))
}
#[cfg(unix)]
fn change_token(metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
use std::os::unix::fs::MetadataExt;
Some((metadata.ctime(), metadata.ctime_nsec()))
}
#[cfg(not(unix))]
fn change_token(_metadata: &std::fs::Metadata) -> Option<(i64, i64)> {
None
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RecordedFileDigest {
pub file: FileIdentity,
pub digest: CacheDigest,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FileDigestScope {
Content,
CcInput,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum FileDigestResolution {
Digest(CacheDigest),
EmbeddedTimestampMacro,
Unresolved,
}
impl FileDigestResolution {
pub fn into_digest(self) -> Option<CacheDigest> {
match self {
Self::Digest(digest) => Some(digest),
Self::EmbeddedTimestampMacro | Self::Unresolved => None,
}
}
}
pub fn digest_file(scope: FileDigestScope, path: &Path) -> io::Result<FileDigestResolution> {
let file = std::fs::File::open(path)?;
let mut reader = std::io::BufReader::new(file);
let mut hasher = blake3::Hasher::new();
let mut size = 0_u64;
let longest_macro = TIMESTAMP_MACROS
.iter()
.map(|macro_name| macro_name.len())
.max()
.unwrap_or_default();
let mut window = Vec::with_capacity(DIGEST_BUFFER_BYTES + longest_macro);
let mut chunk = vec![0_u8; DIGEST_BUFFER_BYTES];
let mut found_timestamp_macro = false;
loop {
let read = reader.read(&mut chunk)?;
if read == 0 {
break;
}
hasher.update(&chunk[..read]);
size = size
.checked_add(read as u64)
.ok_or_else(|| io::Error::other("file length overflowed u64"))?;
if scope == FileDigestScope::CcInput && !found_timestamp_macro {
window.extend_from_slice(&chunk[..read]);
found_timestamp_macro = TIMESTAMP_MACROS
.iter()
.any(|macro_name| contains_subslice(&window, macro_name));
let keep = window.len().saturating_sub(longest_macro.saturating_sub(1));
window.drain(..keep);
}
}
if found_timestamp_macro {
Ok(FileDigestResolution::EmbeddedTimestampMacro)
} else {
Ok(FileDigestResolution::Digest(CacheDigest {
algorithm: "blake3".into(),
hash: hasher.finalize().to_hex().to_string(),
size,
}))
}
}
fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
!needle.is_empty()
&& haystack.len() >= needle.len()
&& haystack
.windows(needle.len())
.any(|window| window == needle)
}
pub trait FileDigestCache: Send + Sync {
fn resolve(&self, scope: FileDigestScope, files: &[FileIdentity]) -> Vec<FileDigestResolution> {
self.find(scope, files)
.into_iter()
.map(|digest| {
digest.map_or(
FileDigestResolution::Unresolved,
FileDigestResolution::Digest,
)
})
.collect()
}
fn find(&self, scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>>;
fn record(&self, scope: FileDigestScope, entries: Vec<RecordedFileDigest>);
}
pub struct NoFileDigestCache;
impl FileDigestCache for NoFileDigestCache {
fn resolve(
&self,
_scope: FileDigestScope,
files: &[FileIdentity],
) -> Vec<FileDigestResolution> {
vec![FileDigestResolution::Unresolved; files.len()]
}
fn find(&self, _scope: FileDigestScope, files: &[FileIdentity]) -> Vec<Option<CacheDigest>> {
vec![None; files.len()]
}
fn record(&self, _scope: FileDigestScope, _entries: Vec<RecordedFileDigest>) {}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_identity_describes_the_file_until_it_is_written_or_removed() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("input.rs");
std::fs::write(&path, b"fn main() {}").unwrap();
let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap()).unwrap();
assert!(identity.still_describes().unwrap());
std::thread::sleep(std::time::Duration::from_millis(20));
std::fs::write(&path, b"fn main() { }").unwrap();
assert!(!identity.still_describes().unwrap());
std::fs::remove_file(&path).unwrap();
assert!(identity.still_describes().is_err());
}
#[test]
fn a_metadata_snapshot_detects_a_metadata_change() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("input.rs");
std::fs::write(&path, b"fn main() {}").unwrap();
let snapshot = capture_file_snapshot(
&path,
&NoFileDigestCache,
false,
std::fs::metadata(&path).unwrap(),
)
.unwrap()
.unwrap();
std::fs::File::options()
.write(true)
.open(&path)
.unwrap()
.set_times(std::fs::FileTimes::new().set_modified(SystemTime::UNIX_EPOCH))
.unwrap();
let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
let digest = CacheDigest::blake3_file(&path).unwrap();
assert!(!snapshot.matches(identity.as_ref(), &digest));
assert_eq!(snapshot.proves_content_change(), cfg!(unix));
}
#[test]
fn a_content_snapshot_ignores_change_token_churn_but_detects_other_changes() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("input.rs");
std::fs::write(&path, b"fn main() {}").unwrap();
let snapshot = capture_file_snapshot(
&path,
&NoFileDigestCache,
true,
std::fs::metadata(&path).unwrap(),
)
.unwrap()
.unwrap();
let mut identity = snapshot.identity.clone();
identity.changed = identity
.changed
.map(|(seconds, nanos)| (seconds + 1, nanos));
let digest = CacheDigest::blake3_file(&path).unwrap();
assert!(snapshot.matches(Some(&identity), &digest));
assert!(snapshot.proves_content_change());
identity.modified = SystemTime::UNIX_EPOCH;
assert!(!snapshot.matches(Some(&identity), &digest));
std::fs::write(&path, b"fn main(){ }").unwrap();
let identity = FileIdentity::describe(&path, &std::fs::metadata(&path).unwrap());
let digest = CacheDigest::blake3_file(&path).unwrap();
assert!(!snapshot.matches(identity.as_ref(), &digest));
}
#[test]
fn reliable_metadata_keeps_the_native_identity() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("input.rs");
std::fs::write(&path, b"fn main() {}").unwrap();
let metadata = std::fs::metadata(&path).unwrap();
let identity = digest_cache_identity(&path, &metadata, false)
.unwrap()
.unwrap();
assert_eq!(identity.path, path);
assert_eq!(identity.len, 12);
assert!(identity.object.is_none());
}
#[test]
fn content_snapshot_ignores_nfs_ctime_churn_but_not_object_replacement() {
let digest = CacheDigest::blake3(b"nfs bytes");
let identity = FileIdentity {
path: PathBuf::from("/nfs/input.rlib"),
len: digest.size,
modified: SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(10),
changed: Some((10, 1)),
object: Some(FileObjectIdentity {
device_major: 0,
device_minor: 42,
mount_id: 7,
inode: 99,
}),
};
let snapshot = FileSnapshot {
identity: identity.clone(),
content: Some(digest.clone()),
};
let mut after = identity;
after.changed = Some((9, 500));
assert!(snapshot.matches(Some(&after), &digest));
after.object.as_mut().unwrap().inode += 1;
assert!(!snapshot.matches(Some(&after), &digest));
after.object.as_mut().unwrap().inode -= 1;
after.modified += std::time::Duration::from_secs(1);
assert!(!snapshot.matches(Some(&after), &digest));
}
#[cfg(target_os = "linux")]
#[test]
fn incomplete_statx_metadata_falls_back_to_content_hashing() {
let status = unsafe { std::mem::zeroed::<LinuxStatx>() };
assert!(
nfs_identity_from_statx(Path::new("/nfs/input.rlib"), &status)
.unwrap()
.is_none()
);
}
}