use anyhow::{Context, Result};
use std::fs;
use std::path::{Path, PathBuf};
use super::{CHECKSUM_PREFIX_LEN, CacheDescriptor, ObjectStore, TreeEntry, walk_files};
impl ObjectStore {
pub(super) fn descriptor_path(&self, descriptor_key: &str) -> PathBuf {
let (prefix, rest) = descriptor_key.split_at(CHECKSUM_PREFIX_LEN.min(descriptor_key.len()));
self.descriptors_dir.join(prefix).join(rest)
}
pub(super) fn store_descriptor(
&self,
cache_key: &str,
descriptor: &CacheDescriptor,
) -> Result<()> {
let path = self.descriptor_path(cache_key);
let parent = path
.parent()
.with_context(|| format!("Descriptor path has no parent: {}", path.display()))?;
fs::create_dir_all(parent).context("Failed to create descriptor directory")?;
let data =
serde_json::to_vec(descriptor).context("Failed to serialize cache descriptor")?;
let tmp_path = parent.join(format!(
".tmp-{}-{}",
std::process::id(),
super::blobs::NEXT_TMP_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
));
fs::write(&tmp_path, &data).with_context(|| {
format!(
"Failed to write descriptor temp file: {}",
tmp_path.display()
)
})?;
let mut perms = fs::metadata(&tmp_path)
.with_context(|| {
format!(
"Failed to read descriptor temp file metadata: {}",
tmp_path.display()
)
})?
.permissions();
perms.set_readonly(true);
fs::set_permissions(&tmp_path, perms).with_context(|| {
format!("Failed to set descriptor read-only: {}", tmp_path.display())
})?;
if let Err(e) = fs::rename(&tmp_path, &path) {
let mut writable = fs::metadata(&tmp_path).map(|m| m.permissions());
if let Ok(ref mut perms) = writable {
perms.set_readonly(false);
let _ = fs::set_permissions(&tmp_path, perms.clone());
}
let _ = fs::remove_file(&tmp_path);
return Err(e).with_context(|| {
format!("Failed to move descriptor into place: {}", path.display())
});
}
Ok(())
}
pub(super) fn get_descriptor(&self, cache_key: &str) -> Option<CacheDescriptor> {
let path = self.descriptor_path(cache_key);
let data = fs::read(&path).ok()?;
serde_json::from_slice(&data).ok()
}
pub(super) fn get_descriptor_pulling(
&self,
ctx: &crate::build_context::BuildContext,
cache_key: &str,
) -> Option<CacheDescriptor> {
if let Some(descriptor) = self.get_descriptor(cache_key) {
return Some(descriptor);
}
if !self.remote_pull {
return None;
}
match self.try_fetch_descriptor_from_remote(ctx, cache_key) {
Ok(descriptor) => descriptor,
Err(e) => {
crate::output::warn(&format!(
"failed to fetch descriptor from remote cache: {e}"
));
None
}
}
}
pub fn previous_tree_paths(&self, cache_key: &str) -> Vec<PathBuf> {
match self.get_descriptor(cache_key) {
Some(CacheDescriptor::Tree { entries }) => {
entries
.into_iter()
.filter_map(|e| match super::safe_entry_path(&e.path) {
Ok(_) => Some(PathBuf::from(e.path)),
Err(err) => {
crate::output::warn(&format!("Ignoring unsafe tree entry: {err}"));
None
}
})
.collect()
}
_ => Vec::new(),
}
}
fn store_and_push_descriptor(
&self,
ctx: &crate::build_context::BuildContext,
cache_key: &str,
descriptor: &CacheDescriptor,
) -> Result<()> {
self.store_descriptor(cache_key, descriptor)?;
if self.remote_push {
let data = serde_json::to_vec(descriptor)
.context("Failed to serialize cache descriptor for remote push")?;
self.try_push_descriptor_to_remote(ctx, cache_key, &data)?;
}
Ok(())
}
pub fn store_marker(
&self,
ctx: &crate::build_context::BuildContext,
cache_key: &str,
) -> Result<()> {
self.store_and_push_descriptor(ctx, cache_key, &CacheDescriptor::Marker)
}
pub fn store_blob_descriptor(
&self,
ctx: &crate::build_context::BuildContext,
cache_key: &str,
output_path: &Path,
) -> Result<bool> {
let content = fs::read(output_path)
.with_context(|| format!("Failed to read output: {}", output_path.display()))?;
let checksum = self.store_object(&content)?;
let mode = fs::metadata(output_path)
.ok()
.map(|m| crate::platform::get_mode(&m));
let changed = match self.get_descriptor(cache_key) {
Some(CacheDescriptor::Blob { checksum: prev, .. }) => prev != checksum,
_ => true,
};
if self.remote_push {
self.try_push_object_to_remote(ctx, &checksum)?;
}
self.store_and_push_descriptor(ctx, cache_key, &CacheDescriptor::Blob { checksum, mode })?;
Ok(changed)
}
pub fn store_tree_descriptor(
&self,
ctx: &crate::build_context::BuildContext,
cache_key: &str,
output_dirs: &[std::sync::Arc<PathBuf>],
output_files: &[PathBuf],
is_foreign: &dyn Fn(&Path) -> bool,
) -> Result<bool> {
let prev = self.get_descriptor(cache_key);
let mut entries = Vec::new();
for dir in output_dirs {
let dir: &Path = dir;
anyhow::ensure!(
dir.exists() && dir.is_dir(),
"Expected output directory not produced: {}",
dir.display()
);
for file_path in walk_files(dir) {
if is_foreign(&file_path) {
continue;
}
let content = fs::read(&file_path)
.with_context(|| format!("Failed to read: {}", file_path.display()))?;
let checksum = self.store_object(&content)?;
let mode = fs::metadata(&file_path)
.ok()
.map(|m| crate::platform::get_mode(&m));
if self.remote_push {
self.try_push_object_to_remote(ctx, &checksum)?;
}
entries.push(TreeEntry {
path: file_path.display().to_string(),
checksum,
mode,
});
}
}
for file_path in output_files {
anyhow::ensure!(
file_path.exists(),
"Expected output file not produced: {}",
file_path.display()
);
let content = fs::read(file_path)
.with_context(|| format!("Failed to read: {}", file_path.display()))?;
let checksum = self.store_object(&content)?;
let mode = fs::metadata(file_path)
.ok()
.map(|m| crate::platform::get_mode(&m));
if self.remote_push {
self.try_push_object_to_remote(ctx, &checksum)?;
}
entries.push(TreeEntry {
path: Self::path_string(file_path),
checksum,
mode,
});
}
entries.sort_by(|a, b| a.path.cmp(&b.path));
let changed = match prev {
Some(CacheDescriptor::Tree {
entries: ref prev_entries,
}) => {
entries.len() != prev_entries.len()
|| entries
.iter()
.zip(prev_entries.iter())
.any(|(a, b)| a.checksum != b.checksum || a.path != b.path)
}
_ => true,
};
self.store_and_push_descriptor(ctx, cache_key, &CacheDescriptor::Tree { entries })?;
Ok(changed)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::build_context::BuildContext;
#[test]
fn descriptor_path_shards_and_survives_short_keys() {
let tmp = tempfile::TempDir::new().unwrap();
let store = ObjectStore::new_in(tmp.path());
let path = store.descriptor_path("abcdef");
assert!(
path.ends_with(Path::new("ab").join("cdef")),
"expected ab/cdef sharding, got {}",
path.display()
);
let short = store.descriptor_path("a");
assert!(short.starts_with(tmp.path()));
}
#[test]
fn tree_change_detection_ignores_order_but_sees_content() {
let tmp = tempfile::TempDir::new().unwrap();
let store = ObjectStore::new_in(tmp.path());
let ctx = BuildContext::new();
let key = "abba7777";
let outdir = tmp.path().join("out");
fs::create_dir_all(&outdir).unwrap();
fs::write(outdir.join("a.txt"), b"one").unwrap();
fs::write(outdir.join("b.txt"), b"two").unwrap();
let dirs = [std::sync::Arc::new(outdir.clone())];
assert!(
store
.store_tree_descriptor(&ctx, key, &dirs, &[], &|_| false)
.unwrap(),
"first store is always a change"
);
assert!(
!store
.store_tree_descriptor(&ctx, key, &dirs, &[], &|_| false)
.unwrap(),
"identical re-store must not read as a change"
);
fs::write(outdir.join("a.txt"), b"changed").unwrap();
assert!(
store
.store_tree_descriptor(&ctx, key, &dirs, &[], &|_| false)
.unwrap(),
"content change must be detected"
);
}
#[test]
fn descriptor_overwrite_survives_read_only_previous() {
let tmp = tempfile::TempDir::new().unwrap();
let store = ObjectStore::new_in(tmp.path());
let ctx = BuildContext::new();
store.store_marker(&ctx, "cdcd1212").unwrap();
store.store_marker(&ctx, "cdcd1212").unwrap();
assert!(matches!(
store.get_descriptor("cdcd1212"),
Some(CacheDescriptor::Marker)
));
}
}