use anyhow::{Context, Result};
use redb::ReadableDatabase;
use redb::ReadableTable;
use redb::TableDefinition;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::time::SystemTime;
const HASH_BUF_SIZE: usize = 64 * 1024;
use crate::build_context::BuildContext;
const MTIME_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("mtime_cache");
#[derive(Clone, Serialize, Deserialize)]
struct MtimeEntry {
stamp: crate::platform::FileStamp,
checksum: String,
}
#[derive(Default)]
pub struct PendingMtime {
entries: HashMap<String, MtimeEntry>,
}
const RECENT_CHANGE_SECS: i64 = 2;
fn get_mtime_db(ctx: &BuildContext) -> Result<std::sync::MutexGuard<'_, Option<redb::Database>>> {
let mut guard = ctx.mtime_db.lock().unwrap();
if guard.is_none() {
let dir = PathBuf::from(".rsconstruct");
crate::errors::ctx(
fs::create_dir_all(&dir),
"Failed to create .rsconstruct directory",
)?;
let db = crate::db::open_or_recreate(&dir.join("mtime.redb"), "Mtime cache")?;
*guard = Some(db);
}
Ok(guard)
}
pub fn file_checksum(ctx: &BuildContext, path: &Path) -> Result<String> {
let mut guard = ctx.checksum_cache.lock().unwrap();
let cache = guard.get_or_insert_with(HashMap::new);
if let Some(cached) = cache.get(path) {
return Ok(cached.clone());
}
let checksum = stream_file_checksum(path)?;
cache.insert(path.to_path_buf(), checksum.clone());
Ok(checksum)
}
pub fn forget_in_session(ctx: &BuildContext, paths: &[std::path::PathBuf]) {
let mut guard = ctx.checksum_cache.lock().unwrap();
if let Some(cache) = guard.as_mut() {
for path in paths {
cache.remove(path);
}
}
drop(guard);
ctx.combined_memo.lock().unwrap().forget(paths);
}
const MEMO_MIN_INPUTS: usize = 32;
#[derive(Default)]
pub struct CombinedMemo {
by_inputs: HashMap<Vec<PathBuf>, String>,
covered: std::collections::HashSet<PathBuf>,
}
impl CombinedMemo {
fn get(&self, inputs: &[PathBuf]) -> Option<String> {
self.by_inputs.get(inputs).cloned()
}
fn insert(&mut self, inputs: &[PathBuf], combined: String) {
self.covered.extend(inputs.iter().cloned());
self.by_inputs.insert(inputs.to_vec(), combined);
}
fn forget(&mut self, paths: &[PathBuf]) {
if paths.iter().any(|p| self.covered.contains(p)) {
self.by_inputs.clear();
self.covered.clear();
}
}
}
fn stream_file_checksum(path: &Path) -> Result<String> {
let mut file = fs::File::open(path)
.with_context(|| format!("Failed to open file for checksum: {}", path.display()))?;
let mut hasher = Sha256::new();
let mut buf = vec![0u8; HASH_BUF_SIZE];
loop {
let n = file
.read(&mut buf)
.with_context(|| format!("Failed to read file for checksum: {}", path.display()))?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
Ok(hex::encode(hasher.finalize()))
}
type DirtyMtimeEntry = (String, MtimeEntry);
fn fast_checksum(
ctx: &BuildContext,
path: &Path,
) -> Result<(String, ChecksumPath, Option<DirtyMtimeEntry>)> {
let metadata =
fs::metadata(path).with_context(|| format!("Failed to stat file: {}", path.display()))?;
let stamp = crate::platform::file_stamp(&metadata);
let path_str = path.display().to_string();
let staged = ctx
.mtime_pending
.lock()
.unwrap()
.entries
.get(&path_str)
.cloned();
let db_guard = get_mtime_db(ctx)?;
let cached = if staged.is_some() {
staged
} else if let Some(ref db) = *db_guard {
let read_txn = crate::errors::ctx(
db.begin_read(),
"Failed to begin read transaction for mtime cache",
)?;
match read_txn.open_table(MTIME_TABLE) {
Ok(table) => table
.get(path_str.as_str())
.with_context(|| format!("Failed to read the mtime cache entry of {path_str}"))?
.and_then(|data| serde_json::from_slice::<MtimeEntry>(data.value()).ok()),
Err(redb::TableError::TableDoesNotExist(_)) => None,
Err(e) => {
return Err(anyhow::Error::from(e).context("Failed to open the mtime cache table"));
}
}
} else {
None
};
drop(db_guard);
if let Some(ref entry) = cached
&& entry.stamp == stamp
{
let mut guard = ctx.checksum_cache.lock().unwrap();
let cache = guard.get_or_insert_with(HashMap::new);
cache.insert(path.to_path_buf(), entry.checksum.clone());
return Ok((entry.checksum.clone(), ChecksumPath::MtimeShortcut, None));
}
let checksum = stream_file_checksum(path)?;
let mut guard = ctx.checksum_cache.lock().unwrap();
let cache = guard.get_or_insert_with(HashMap::new);
cache.insert(path.to_path_buf(), checksum.clone());
drop(guard);
let now_secs = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map_or(i64::MAX, |d| i64::try_from(d.as_secs()).unwrap_or(i64::MAX));
if stamp.changed_within(now_secs, RECENT_CHANGE_SECS) {
return Ok((checksum, ChecksumPath::FullRead, None));
}
let new_entry = MtimeEntry {
stamp,
checksum: checksum.clone(),
};
Ok((
checksum,
ChecksumPath::FullRead,
Some((path_str, new_entry)),
))
}
fn stage_mtime_entries(ctx: &BuildContext, dirty: Vec<DirtyMtimeEntry>) {
if dirty.is_empty() {
return;
}
ctx.mtime_pending.lock().unwrap().entries.extend(dirty);
}
pub fn flush_mtime_cache(ctx: &BuildContext) -> Result<()> {
let dirty: Vec<DirtyMtimeEntry> =
std::mem::take(&mut ctx.mtime_pending.lock().unwrap().entries)
.into_iter()
.collect();
if dirty.is_empty() {
return Ok(());
}
let db_guard = get_mtime_db(ctx)?;
let db = crate::errors::ctx_opt(db_guard.as_ref(), "Mtime database not available")?;
let write_txn = crate::errors::ctx(
db.begin_write(),
"Failed to begin write transaction for mtime cache",
)?;
{
let mut table = write_txn
.open_table(MTIME_TABLE)
.context("Failed to open mtime cache table")?;
for (path_str, entry) in &dirty {
let value = serde_json::to_vec(entry).context("Failed to serialize mtime entry")?;
table
.insert(path_str.as_str(), value.as_slice())
.context("Failed to insert mtime entry")?;
}
}
crate::errors::ctx(write_txn.commit(), "Failed to commit mtime cache entries")?;
Ok(())
}
fn hash_checksums(checksums: &[String]) -> String {
let mut hasher = Sha256::new();
for c in checksums {
hasher.update((c.len() as u64).to_le_bytes());
hasher.update(c.as_bytes());
}
hex::encode(hasher.finalize())
}
pub fn hash_parts(parts: &[&str]) -> String {
let mut hasher = Sha256::new();
for p in parts {
hasher.update((p.len() as u64).to_le_bytes());
hasher.update(p.as_bytes());
}
hex::encode(hasher.finalize())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChecksumPath {
MtimeShortcut,
FullRead,
}
pub fn prune_mtime_cache(ctx: &BuildContext) -> Result<usize> {
if !PathBuf::from(".rsconstruct").join("mtime.redb").exists() {
return Ok(0);
}
let stale: Vec<String> = {
let db_guard = get_mtime_db(ctx)?;
let Some(db) = db_guard.as_ref() else {
return Ok(0);
};
let read_txn = crate::errors::ctx(
db.begin_read(),
"Failed to begin read transaction for mtime prune",
)?;
let Ok(table) = read_txn.open_table(MTIME_TABLE) else {
return Ok(0);
};
let mut stale = Vec::new();
for entry in table.iter().context("Failed to iterate mtime cache")? {
let (key, _) = entry.context("Failed to read mtime cache entry")?;
let path_str = key.value();
if !Path::new(path_str).exists() {
stale.push(path_str.to_string());
}
}
stale
};
if stale.is_empty() {
return Ok(0);
}
let db_guard = get_mtime_db(ctx)?;
let db = crate::errors::ctx_opt(db_guard.as_ref(), "Mtime database not available")?;
let write_txn = crate::errors::ctx(
db.begin_write(),
"Failed to begin write transaction for mtime prune",
)?;
{
let mut table = write_txn
.open_table(MTIME_TABLE)
.context("Failed to open mtime cache table for prune")?;
for path_str in &stale {
table
.remove(path_str.as_str())
.with_context(|| format!("Failed to remove mtime entry for {path_str}"))?;
}
}
crate::errors::ctx(write_txn.commit(), "Failed to commit mtime cache prune")?;
Ok(stale.len())
}
pub fn checksum_fast(ctx: &BuildContext, path: &Path) -> Result<(String, ChecksumPath)> {
if !ctx.mtime_enabled.load(std::sync::atomic::Ordering::Relaxed) {
return Ok((file_checksum(ctx, path)?, ChecksumPath::FullRead));
}
let (checksum, path_taken, dirty) = fast_checksum(ctx, path)?;
stage_mtime_entries(ctx, dirty.into_iter().collect());
Ok((checksum, path_taken))
}
pub fn checksum_output(ctx: &BuildContext, path: &Path) -> Result<(String, ChecksumPath)> {
if !ctx.mtime_enabled.load(std::sync::atomic::Ordering::Relaxed) {
return Ok((stream_file_checksum(path)?, ChecksumPath::FullRead));
}
let (checksum, path_taken, dirty) = fast_checksum(ctx, path)?;
stage_mtime_entries(ctx, dirty.into_iter().collect());
Ok((checksum, path_taken))
}
pub fn combined_input_checksum(ctx: &BuildContext, inputs: &[PathBuf]) -> Result<String> {
let memoize = inputs.len() >= MEMO_MIN_INPUTS;
if memoize && let Some(combined) = ctx.combined_memo.lock().unwrap().get(inputs) {
return Ok(combined);
}
let combined = compute_combined_input_checksum(ctx, inputs)?;
if memoize {
ctx.combined_memo
.lock()
.unwrap()
.insert(inputs, combined.clone());
}
Ok(combined)
}
fn compute_combined_input_checksum(ctx: &BuildContext, inputs: &[PathBuf]) -> Result<String> {
let mtime_enabled = ctx.mtime_enabled.load(std::sync::atomic::Ordering::Relaxed);
let mut checksums = Vec::with_capacity(inputs.len());
let mut dirty_entries = Vec::new();
for input in inputs {
if input.exists() {
if mtime_enabled {
let (checksum, _, dirty) = fast_checksum(ctx, input)?;
checksums.push(checksum);
if let Some(entry) = dirty {
dirty_entries.push(entry);
}
} else {
checksums.push(file_checksum(ctx, input)?);
}
} else {
checksums.push(format!("MISSING:{}", input.display()));
}
}
stage_mtime_entries(ctx, dirty_entries);
Ok(hash_checksums(&checksums))
}
pub fn bytes_checksum(data: &[u8]) -> String {
hex::encode(Sha256::digest(data))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn stamp_changes_when_content_is_rewritten_with_mtime_restored() {
let tmp = tempfile::TempDir::new().unwrap();
let path = tmp.path().join("f.txt");
fs::write(&path, "aaaa").unwrap();
let old_mtime = SystemTime::now() - std::time::Duration::from_secs(3600);
fs::File::options()
.write(true)
.open(&path)
.unwrap()
.set_modified(old_mtime)
.unwrap();
let before = crate::platform::file_stamp(&fs::metadata(&path).unwrap());
std::thread::sleep(std::time::Duration::from_millis(20));
fs::write(&path, "bbbb").unwrap();
fs::File::options()
.write(true)
.open(&path)
.unwrap()
.set_modified(old_mtime)
.unwrap();
let after = crate::platform::file_stamp(&fs::metadata(&path).unwrap());
assert_eq!(before.mtime_secs, after.mtime_secs, "mtime was restored");
assert_eq!(before.size, after.size, "same length");
assert_ne!(before, after, "ctime still records the rewrite");
}
#[test]
fn staged_entries_are_visible_at_once_and_written_by_flush() {
let tmp = tempfile::TempDir::new().unwrap();
let ctx = BuildContext::new();
*ctx.mtime_db.lock().unwrap() =
Some(redb::Database::create(tmp.path().join("mtime.redb")).unwrap());
let path = tmp.path().join("f.txt");
fs::write(&path, "content").unwrap();
let key = path.display().to_string();
let entry = MtimeEntry {
stamp: crate::platform::file_stamp(&fs::metadata(&path).unwrap()),
checksum: "staged".to_string(),
};
stage_mtime_entries(&ctx, vec![(key.clone(), entry)]);
assert_eq!(
checksum_fast(&ctx, &path).unwrap(),
("staged".to_string(), ChecksumPath::MtimeShortcut),
"the staged entry is used before the flush"
);
let stored = |ctx: &BuildContext| -> Option<String> {
let guard = ctx.mtime_db.lock().unwrap();
let txn = guard.as_ref().unwrap().begin_read().unwrap();
let table = txn.open_table(MTIME_TABLE).ok()?;
let data = table.get(key.as_str()).unwrap()?;
Some(
serde_json::from_slice::<MtimeEntry>(data.value())
.unwrap()
.checksum,
)
};
assert_eq!(stored(&ctx), None, "nothing is written before the flush");
flush_mtime_cache(&ctx).unwrap();
assert_eq!(stored(&ctx), Some("staged".to_string()));
assert!(ctx.mtime_pending.lock().unwrap().entries.is_empty());
}
#[test]
fn changed_within_uses_the_later_of_mtime_and_ctime() {
let stamp = crate::platform::FileStamp {
mtime_secs: 1_000,
mtime_nanos: 0,
ctime_secs: 5_000,
ctime_nanos: 0,
size: 4,
inode: 1,
};
assert!(stamp.changed_within(5_001, RECENT_CHANGE_SECS));
assert!(!stamp.changed_within(5_010, RECENT_CHANGE_SECS));
}
#[test]
fn hash_checksums_is_injection_proof() {
let joined_left = hash_checksums(&["a:b".to_string(), "c".to_string()]);
let joined_right = hash_checksums(&["a".to_string(), "b:c".to_string()]);
assert_ne!(joined_left, joined_right);
assert_ne!(
hash_checksums(&[]),
hash_checksums(&[String::new()]),
"no elements and one empty element must differ"
);
assert_eq!(
hash_checksums(&["x".to_string()]),
hash_checksums(&["x".to_string()]),
"must be deterministic"
);
}
#[test]
fn stream_checksum_matches_bytes_checksum_at_buffer_boundaries() {
let tmp = tempfile::TempDir::new().unwrap();
for size in [0, HASH_BUF_SIZE - 1, HASH_BUF_SIZE, HASH_BUF_SIZE + 1] {
let data = vec![0xABu8; size];
let path = tmp.path().join(format!("f{size}"));
fs::write(&path, &data).unwrap();
assert_eq!(
stream_file_checksum(&path).unwrap(),
bytes_checksum(&data),
"stream and one-shot checksums diverge at {size} bytes"
);
}
}
#[test]
fn combined_input_checksum_missing_and_order_semantics() {
let ctx = BuildContext::new();
ctx.set_mtime_check(false);
let tmp = tempfile::TempDir::new().unwrap();
let empty = tmp.path().join("empty.txt");
fs::write(&empty, b"").unwrap();
let missing = tmp.path().join("missing.txt");
assert_ne!(
combined_input_checksum(&ctx, std::slice::from_ref(&empty)).unwrap(),
combined_input_checksum(&ctx, std::slice::from_ref(&missing)).unwrap(),
"a missing input must not hash like an empty one"
);
let a = tmp.path().join("a.txt");
let b = tmp.path().join("b.txt");
fs::write(&a, b"aaa").unwrap();
fs::write(&b, b"bbb").unwrap();
assert_ne!(
combined_input_checksum(&ctx, &[a.clone(), b.clone()]).unwrap(),
combined_input_checksum(&ctx, &[b, a]).unwrap(),
"input order is part of the key"
);
}
#[test]
fn file_checksum_cache_evicts_on_forget() {
let ctx = BuildContext::new();
let tmp = tempfile::TempDir::new().unwrap();
let path = tmp.path().join("f.txt");
fs::write(&path, b"one").unwrap();
let first = file_checksum(&ctx, &path).unwrap();
fs::write(&path, b"two").unwrap();
assert_eq!(
file_checksum(&ctx, &path).unwrap(),
first,
"un-evicted reads serve the in-session cached checksum"
);
forget_in_session(&ctx, std::slice::from_ref(&path));
let second = file_checksum(&ctx, &path).unwrap();
assert_ne!(second, first, "eviction must expose the new content");
let fresh_ctx = BuildContext::new();
assert_eq!(
file_checksum(&fresh_ctx, &path).unwrap(),
second,
"post-eviction value must match a fresh context's view"
);
}
}