use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result};
use crate::util::{self, FileLock};
use serde::{Deserialize, Serialize};
use walkdir::WalkDir;
use crate::fingerprint::{self, ArtifactInfo, Unit};
const PARITY_SEED: &str = "epicenter-parity-v2";
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Toolchain {
pub release: String,
pub commit: String,
pub host: String,
}
impl Toolchain {
pub fn version_line(&self) -> String {
format!("rustc {} ({})", self.release, self.commit)
}
}
pub fn toolchain() -> Result<Toolchain> {
let mut command = util::command_for("rustc")?;
let output = command.arg("-vV").output().context("running `rustc -vV`")?;
if !output.status.success() {
anyhow::bail!("`rustc -vV` failed — is a toolchain installed?");
}
let text = String::from_utf8_lossy(&output.stdout);
let mut release = "unknown".to_string();
let mut commit = "unknown".to_string();
let mut host = "unknown".to_string();
for line in text.lines() {
if let Some(rest) = line.strip_prefix("rustc ") {
let mut parts = rest.split_whitespace();
if let Some(version) = parts.next() {
release = version.to_string();
}
if let Some(commit_part) = parts.next() {
let value = commit_part.trim_start_matches('(');
if !value.is_empty() {
commit = value.to_string();
}
}
} else if let Some(value) = line.strip_prefix("host: ") {
host = value.trim().to_string();
}
}
util::validate_component(&release, "rustc release")?;
util::validate_component(&commit, "rustc commit")?;
util::validate_component(&host, "rustc host")?;
Ok(Toolchain {
release,
commit,
host,
})
}
pub fn toolchain_install_time() -> Option<SystemTime> {
let mut command = util::command_for("rustc").ok()?;
let output = command.args(["--print", "sysroot"]).output().ok()?;
if !output.status.success() {
return None;
}
let sysroot = PathBuf::from(String::from_utf8_lossy(&output.stdout).trim());
let rustc = sysroot
.join("bin")
.join(if cfg!(windows) { "rustc.exe" } else { "rustc" });
fs::metadata(&rustc)
.and_then(|meta| meta.modified())
.ok()
.or_else(|| fs::metadata(&sysroot).and_then(|meta| meta.modified()).ok())
}
pub fn parity_key() -> Result<String> {
let tc = toolchain()?;
let key = format!(
"{}-{}-{}-{}",
tc.release,
tc.commit,
tc.host,
flags_digest()
);
util::validate_component(&key, "parity key")?;
Ok(key)
}
fn flags_digest() -> String {
let mut input = String::from(PARITY_SEED);
for name in [
"RUSTFLAGS",
"RUSTDOCFLAGS",
"CARGO_BUILD_RUSTFLAGS",
"CARGO_BUILD_RUSTDOCFLAGS",
"CARGO_BUILD_TARGET",
"RUSTC_WRAPPER",
"CARGO_BUILD_RUSTC_WRAPPER",
"RUSTC",
] {
input.push('\0');
input.push_str(name);
input.push('=');
input.push_str(&std::env::var(name).unwrap_or_default());
}
if let Ok(path) = crate::config::cargo_config_path() {
if let Ok(text) = fs::read_to_string(path) {
for key in ["rustflags", "rustdocflags", "target", "rustc-wrapper"] {
if let Some(value) = crate::config::read_table_key(&text, "build", key) {
input.push('\0');
input.push_str(key);
input.push('=');
input.push_str(&value);
}
}
}
}
if let Ok(mut cargo) = util::command_for("cargo") {
if let Ok(output) = cargo.arg("-V").output() {
if output.status.success() {
input.push('\0');
input.push_str("cargo=");
input.push_str(String::from_utf8_lossy(&output.stdout).trim());
}
}
}
format!("{:08x}", (fnv1a64(input.as_bytes()) >> 32) as u32)
}
fn fnv1a64(bytes: &[u8]) -> u64 {
let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
for byte in bytes {
hash ^= *byte as u64;
hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
}
hash
}
pub fn store_root(build_root: &Path) -> PathBuf {
build_root.parent().unwrap_or(build_root).join("store")
}
pub fn units_dir(store_root: &Path, parity: &str) -> PathBuf {
store_root.join(parity).join("units")
}
fn manifest_path(store_root: &Path, parity: &str) -> PathBuf {
store_root.join(parity).join("manifest.json")
}
fn store_lock_path(store_root: &Path, parity: &str) -> PathBuf {
store_root.join(format!(".lock-{parity}"))
}
fn ensure_store_root(store_root: &Path) -> Result<()> {
match fs::symlink_metadata(store_root) {
Ok(meta) => {
if !meta.is_dir() || meta.file_type().is_symlink() {
anyhow::bail!(
"store root {} is not a real directory",
store_root.display()
);
}
util::ensure_private_dir(store_root)?;
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
util::create_private_dir_all(store_root)?;
}
Err(err) => {
return Err(err).with_context(|| format!("inspecting {}", store_root.display()));
}
}
Ok(())
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct Manifest {
pub parity_key: String,
pub toolchain: String,
pub created: u64,
pub updated: u64,
pub units: Vec<UnitEntry>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UnitEntry {
pub name: String,
pub profile: String,
pub memo: String,
pub hash: String,
pub stamp: u64,
pub files: Vec<FileEntry>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileEntry {
pub path: String,
pub dest: String,
pub size: u64,
#[serde(default)]
pub sha256: String,
pub role: String,
}
impl Manifest {
pub fn validate(&self) -> Result<()> {
for unit in &self.units {
util::validate_component(&unit.name, "unit name")?;
util::validate_component(&unit.profile, "profile")?;
util::validate_component(&unit.memo, "memo file")?;
if !unit.hash.is_empty() && !fingerprint::looks_like_hash(&unit.hash) {
anyhow::bail!("invalid memo hash {:?} in unit {}", unit.hash, unit.name);
}
for file in &unit.files {
util::validate_rel(&file.path, "stored file path")?;
util::validate_rel(&file.dest, "destination path")?;
if !file.sha256.is_empty()
&& (file.sha256.len() != 64
|| !file.sha256.bytes().all(|byte| byte.is_ascii_hexdigit()))
{
anyhow::bail!("invalid SHA-256 for {} in unit {}", file.path, unit.name);
}
}
}
Ok(())
}
}
fn load_manifest(store_root: &Path, parity: &str) -> Result<Option<Manifest>> {
let path = manifest_path(store_root, parity);
if !path.is_file() {
return Ok(None);
}
let text = fs::read_to_string(&path).with_context(|| format!("reading {}", path.display()))?;
let manifest: Manifest =
serde_json::from_str(&text).with_context(|| format!("parsing {}", path.display()))?;
manifest
.validate()
.with_context(|| format!("validating {}", path.display()))?;
Ok(Some(manifest))
}
fn save_manifest(store_root: &Path, parity: &str, manifest: &Manifest) -> Result<()> {
util::validate_component(parity, "generation")?;
let dir = store_root.join(parity);
util::create_private_dir_all(&dir)?;
let what = serde_json::to_string_pretty(manifest)?;
let mut tmp = None;
for attempt in 0..100u32 {
let candidate = dir.join(format!(
"manifest.json.tmp-{}-{attempt}",
std::process::id()
));
match fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&candidate)
{
Ok(file) => {
tmp = Some((candidate, file));
break;
}
Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(err) => {
return Err(err).with_context(|| format!("creating {}", candidate.display()));
}
}
}
let Some((tmp, mut file)) = tmp else {
anyhow::bail!(
"could not create a unique manifest temp file in {}",
dir.display()
);
};
file.write_all(what.as_bytes())?;
file.write_all(b"\n")?;
file.sync_all()?;
drop(file);
fs::rename(&tmp, manifest_path(store_root, parity))?;
#[cfg(unix)]
if let Ok(parent) = fs::File::open(&dir) {
let _ = parent.sync_all();
}
Ok(())
}
pub fn generation_names(store_root: &Path) -> Result<Vec<String>> {
if !store_root.is_dir() {
return Ok(Vec::new());
}
let mut names = Vec::new();
for entry in
fs::read_dir(store_root).with_context(|| format!("reading {}", store_root.display()))?
{
let entry = entry?;
if !entry.file_type()?.is_dir() {
continue;
}
let name = entry.file_name().to_string_lossy().into_owned();
if !name.starts_with('.') {
names.push(name);
}
}
names.sort();
Ok(names)
}
#[derive(Debug, Clone)]
pub struct StoredUnit {
pub generation: String,
pub entry: UnitEntry,
pub dir: PathBuf,
}
pub fn stored_units(store_root: &Path) -> Result<Vec<StoredUnit>> {
let mut out = Vec::new();
for generation in generation_names(store_root)? {
let Some(manifest) = load_manifest(store_root, &generation)? else {
continue;
};
let units = units_dir(store_root, &generation);
for entry in manifest.units {
let dir = units.join(&entry.name);
out.push(StoredUnit {
generation: generation.clone(),
entry,
dir,
});
}
}
Ok(out)
}
pub fn stored_unit_rustc(unit: &StoredUnit) -> Option<String> {
let dir_meta = fs::symlink_metadata(&unit.dir).ok()?;
if !dir_meta.is_dir() || dir_meta.file_type().is_symlink() {
return None;
}
let rel = PathBuf::from(format!("{}.json", unit.entry.memo));
util::check_regular_chain(&unit.dir, &rel).ok()?;
let text = fs::read_to_string(unit.dir.join(&rel)).ok()?;
let value: serde_json::Value = serde_json::from_str(&text).ok()?;
match value.get("rustc") {
Some(serde_json::Value::Number(number)) => Some(number.to_string()),
Some(serde_json::Value::String(text)) => Some(text.clone()),
_ => None,
}
}
pub fn drop_units(store_root: &Path, generation: &str, names: &[String]) -> Result<usize> {
util::validate_component(generation, "generation")?;
let _lock = FileLock::acquire(&store_lock_path(store_root, generation))?;
let Some(mut manifest) = load_manifest(store_root, generation)? else {
return Ok(0);
};
let units = units_dir(store_root, generation);
let mut kept = Vec::with_capacity(manifest.units.len());
let mut dropped = 0;
for entry in manifest.units.drain(..) {
if !names.contains(&entry.name) {
kept.push(entry);
continue;
}
util::validate_component(&entry.name, "unit name")?;
let dir = units.join(&entry.name);
match fs::symlink_metadata(&dir) {
Ok(meta) if meta.file_type().is_symlink() => {
eprintln!("could not remove symlinked unit {}: refused", dir.display());
kept.push(entry);
}
Ok(meta) if meta.is_dir() => match fs::remove_dir_all(&dir) {
Ok(()) => {
dropped += 1;
}
Err(err) => {
eprintln!("could not remove {}: {err}", dir.display());
kept.push(entry);
}
},
Ok(_) => {
eprintln!("could not remove {}: not a directory", dir.display());
kept.push(entry);
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
dropped += 1;
}
Err(err) => {
eprintln!("could not remove {}: {err}", dir.display());
kept.push(entry);
}
}
}
manifest.units = kept;
if manifest.units.is_empty() {
let _ = fs::remove_dir_all(store_root.join(generation));
} else if dropped > 0 {
manifest.updated = now_secs();
save_manifest(store_root, generation, &manifest)?;
}
Ok(dropped)
}
#[derive(Debug, Default)]
pub struct HarvestReport {
pub parity: String,
pub toolchain: String,
pub stored: usize,
pub refreshed: usize,
pub unchanged: usize,
pub skipped_stale: usize,
pub skipped_incomplete: usize,
pub files: usize,
pub bytes: u64,
pub note: Option<String>,
}
pub fn harvest(
build_root: &Path,
store_root: &Path,
unit_filter: Option<&str>,
dry_run: bool,
) -> Result<HarvestReport> {
let tc = toolchain()?;
let parity = parity_key()?;
let mut report = HarvestReport {
parity: parity.clone(),
toolchain: tc.version_line(),
..Default::default()
};
let _lock = if dry_run {
None
} else {
ensure_store_root(store_root)?;
util::create_private_dir_all(&store_root.join(&parity))?;
util::create_private_dir_all(&units_dir(store_root, &parity))?;
let lock = FileLock::acquire(&store_lock_path(store_root, &parity))?;
sweep_staging(&units_dir(store_root, &parity));
Some(lock)
};
let rows: Vec<(Unit, ArtifactInfo)> = fingerprint::read_units(build_root)?
.into_iter()
.map(|unit| {
let arts = fingerprint::artifacts_for(build_root, &unit);
(unit, arts)
})
.collect();
let newest_activity = rows.iter().filter_map(|(_, arts)| arts.newest).max();
if let (Some(newest), Some(installed)) = (newest_activity, toolchain_install_time()) {
if newest < installed {
report.note = Some(format!(
"the shared cache predates the installed toolchain ({}) — build once, then harvest",
tc.version_line()
));
return Ok(report);
}
}
let active_gen = rows
.iter()
.filter(|(unit, _)| crate::who::rustc_of(unit) != "unknown")
.max_by_key(|(_, arts)| arts.newest)
.map(|(unit, _)| crate::who::rustc_of(unit))
.unwrap_or_else(|| "unknown".into());
let mut manifest = load_manifest(store_root, &parity)?.unwrap_or_else(|| Manifest {
parity_key: parity.clone(),
toolchain: tc.version_line(),
created: now_secs(),
updated: 0,
units: Vec::new(),
});
struct GroupPlan {
unit: Unit,
files: Vec<(PathBuf, String, String, String)>,
memo: String,
hash: String,
base: u64,
}
let mut plans: Vec<GroupPlan> = Vec::new();
for (unit, arts) in &rows {
if let Some(query) = unit_filter {
if !unit_name_matches(&unit.dir_name, query) {
continue;
}
}
let cohort = crate::who::rustc_of(unit);
if cohort != "unknown" && active_gen != "unknown" && cohort != active_gen {
report.skipped_stale += 1;
continue;
}
let unit_dir = build_root
.join(&unit.profile)
.join(".fingerprint")
.join(&unit.dir_name);
let profile_dir = build_root.join(&unit.profile);
let memo = unit.memo_file.clone();
let mut files: Vec<(PathBuf, String, String, String)> = Vec::new();
for entry in
fs::read_dir(&unit_dir).with_context(|| format!("reading {}", unit_dir.display()))?
{
let entry = entry?;
if !entry.file_type()?.is_file() {
continue;
}
let name = entry.file_name().to_string_lossy().into_owned();
if is_volatile(&name) {
continue;
}
let role = if name.starts_with("dep-") {
"dep-info"
} else if name.ends_with(".json") {
"fingerprint"
} else if fs::read_to_string(entry.path())
.map(|text| crate::fingerprint::looks_like_hash(text.trim()))
.unwrap_or(false)
{
"memo"
} else {
"extra"
};
files.push((
entry.path(),
name.clone(),
format!(".fingerprint/{}/{name}", unit.dir_name),
role.into(),
));
}
let mut artifact_count = 0;
for (path, _, _) in &arts.paths {
let name = path
.file_name()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_default();
if is_volatile(&name) {
continue;
}
let Ok(rel) = path.strip_prefix(&profile_dir) else {
continue;
};
let rel = rel.to_string_lossy().replace('\\', "/");
if rel.starts_with("deps/") && rel.ends_with(".d") {
continue;
}
let role = if rel.starts_with("build/") {
"build-tree"
} else {
"artifact"
};
files.push((path.clone(), rel.clone(), rel, role.into()));
artifact_count += 1;
}
if artifact_count == 0 {
report.skipped_incomplete += 1;
continue;
}
plans.push(GroupPlan {
base: group_stamp(&files)?,
memo,
hash: unit.memo_hash.clone().unwrap_or_default(),
files,
unit: unit.clone(),
});
}
let stored_stamps: std::collections::HashMap<String, u64> = manifest
.units
.iter()
.map(|entry| (entry.hash.clone(), entry.stamp))
.collect();
let index: std::collections::HashMap<&str, usize> = plans
.iter()
.enumerate()
.map(|(position, plan)| (plan.hash.as_str(), position))
.collect();
let mut plan_deps: Vec<Vec<usize>> = Vec::with_capacity(plans.len());
let mut stamps: Vec<u64> = Vec::with_capacity(plans.len());
for plan in &plans {
let mut deps = Vec::new();
let mut stamp = plan.base;
for dep in plan
.unit
.json
.as_ref()
.map(|json| json.deps.as_slice())
.unwrap_or(&[])
{
let Some(hash) = dep.memo_hash() else {
continue;
};
if let Some(&at) = index.get(hash.as_str()) {
deps.push(at);
} else if let Some(known) = stored_stamps.get(hash.as_str()) {
stamp = stamp.max(*known);
}
}
plan_deps.push(deps);
stamps.push(stamp);
}
let mut order: Vec<usize> = Vec::with_capacity(plans.len());
let mut settled = vec![false; plans.len()];
loop {
let mut progressed = false;
for at in 0..plans.len() {
if settled[at] || !plan_deps[at].iter().all(|&dep| settled[dep]) {
continue;
}
settled[at] = true;
order.push(at);
progressed = true;
}
if !progressed {
break;
}
}
for at in 0..plans.len() {
if !settled[at] {
order.push(at);
}
}
for &at in &order {
for dep in &plan_deps[at] {
stamps[at] = stamps[at].max(stamps[*dep]);
}
}
for (at, plan) in plans.iter().enumerate() {
let unit = &plan.unit;
let files = &plan.files;
let stamp = stamps[at];
let group_dir = units_dir(store_root, &parity).join(&unit.dir_name);
let mut new_entries: Vec<(&str, &str)> = files
.iter()
.map(|(_, path, _, role)| (path.as_str(), role.as_str()))
.collect();
new_entries.sort_unstable();
let already = manifest
.units
.iter()
.find(|entry| entry.name == unit.dir_name)
.map(|entry| {
let mut stored: Vec<(&str, &str)> = entry
.files
.iter()
.map(|file| (file.path.as_str(), file.role.as_str()))
.collect();
stored.sort_unstable();
entry.hash == plan.hash
&& entry.profile == unit.profile
&& entry.stamp == stamp
&& stored == new_entries
&& entry
.files
.iter()
.all(|file| !file.sha256.is_empty() && group_dir.join(&file.path).is_file())
})
.unwrap_or(false);
if already {
report.unchanged += 1;
continue;
}
let bytes: u64 = files
.iter()
.filter_map(|(src, _, _, _)| fs::metadata(src).ok())
.map(|meta| meta.len())
.sum();
if dry_run {
report.stored += 1;
report.files += files.len();
report.bytes += bytes;
continue;
}
let mut entries = Vec::with_capacity(files.len());
store_group(
store_root,
&parity,
&unit.dir_name,
stamp,
files,
&mut entries,
)
.with_context(|| format!("storing unit {}", unit.dir_name))?;
let entry = UnitEntry {
name: unit.dir_name.clone(),
profile: unit.profile.clone(),
memo: plan.memo.clone(),
hash: plan.hash.clone(),
stamp,
files: entries,
};
match manifest
.units
.iter_mut()
.find(|existing| existing.name == entry.name)
{
Some(slot) => {
*slot = entry;
report.refreshed += 1;
}
None => {
manifest.units.push(entry);
report.stored += 1;
}
}
report.files += files.len();
report.bytes += bytes;
}
if !dry_run {
manifest.units.sort_by(|a, b| {
(a.profile.as_str(), a.name.as_str()).cmp(&(b.profile.as_str(), b.name.as_str()))
});
manifest.updated = now_secs();
save_manifest(store_root, &parity, &manifest)?;
}
Ok(report)
}
fn store_group(
store_root: &Path,
parity: &str,
name: &str,
stamp: u64,
files: &[(PathBuf, String, String, String)],
entries: &mut Vec<FileEntry>,
) -> Result<()> {
util::validate_component(name, "unit name")?;
let units = units_dir(store_root, parity);
let final_dir = units.join(name);
let mut tmp = None;
for attempt in 0..100u32 {
let candidate = units.join(format!(".{name}.tmp-{}-{attempt}", std::process::id()));
match fs::create_dir(&candidate) {
Ok(()) => {
tmp = Some(candidate);
break;
}
Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(err) => {
return Err(err)
.with_context(|| format!("creating staging dir {}", candidate.display()));
}
}
}
let Some(tmp) = tmp else {
anyhow::bail!(
"could not create a unique staging dir in {}",
units.display()
);
};
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = fs::set_permissions(&tmp, fs::Permissions::from_mode(0o700));
}
if let Err(err) = stage_group(&tmp, files, stamp, entries) {
let _ = crate::util::force_remove_dir(&tmp);
return Err(err);
}
let old = units.join(format!(".{name}.old-{}-{}", std::process::id(), now_secs()));
match fs::symlink_metadata(&final_dir) {
Ok(meta) if meta.file_type().is_symlink() => {
let _ = crate::util::force_remove_dir(&tmp);
anyhow::bail!("refusing symlinked store unit {}", final_dir.display());
}
Ok(meta) if meta.is_dir() => fs::rename(&final_dir, &old)?,
Ok(_) => {
let _ = crate::util::force_remove_dir(&tmp);
anyhow::bail!("store unit path {} is not a directory", final_dir.display());
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => return Err(err).context("inspecting store unit"),
}
fs::rename(&tmp, &final_dir)?;
if fs::symlink_metadata(&old)
.map(|meta| meta.is_dir() && !meta.file_type().is_symlink())
.unwrap_or(false)
{
let _ = fs::remove_dir_all(&old);
}
Ok(())
}
fn stage_group(
tmp: &Path,
files: &[(PathBuf, String, String, String)],
stamp: u64,
entries: &mut Vec<FileEntry>,
) -> Result<()> {
for (src, path, dest, role) in files {
let rel = util::validate_rel(path, "stored file path")?;
let dst = tmp.join(&rel);
if let Some(parent) = dst.parent() {
util::create_private_dir_all(parent)?;
}
let (size, sha256) = util::copy_file_hashed(src, &dst)
.with_context(|| format!("copying {} -> {}", src.display(), dst.display()))?;
make_writable(&dst).with_context(|| format!("making {} writable", dst.display()))?;
set_stamp(&dst, stamp)?;
make_read_only(&dst)?;
entries.push(FileEntry {
path: path.clone(),
dest: dest.clone(),
size,
sha256,
role: role.clone(),
});
}
Ok(())
}
fn sweep_staging(units: &Path) {
let Ok(entries) = fs::read_dir(units) else {
return;
};
let now = SystemTime::now();
for entry in entries.flatten() {
let name = entry.file_name();
let name = name.to_string_lossy();
if !(name.contains(".tmp-") || name.contains(".old-")) {
continue;
}
let meta = match fs::symlink_metadata(entry.path()) {
Ok(meta) => meta,
Err(_) => continue,
};
if meta.file_type().is_symlink() {
let _ = fs::remove_file(entry.path());
continue;
}
if !meta.is_dir() {
continue;
}
let stale = meta
.modified()
.ok()
.and_then(|modified| now.duration_since(modified).ok())
.is_some_and(|age| age > Duration::from_secs(600));
if stale {
let _ = crate::util::force_remove_dir(&entry.path());
}
}
}
fn group_stamp(files: &[(PathBuf, String, String, String)]) -> Result<u64> {
let mut newest: Option<SystemTime> = None;
let mut metadata: Option<SystemTime> = None;
for (src, path, _, _) in files {
let modified = fs::metadata(src)
.and_then(|meta| meta.modified())
.with_context(|| format!("reading mtime of {}", src.display()))?;
if newest.map_or(true, |current| modified > current) {
newest = Some(modified);
}
if path.ends_with(".rmeta") && metadata.map_or(true, |current| modified > current) {
metadata = Some(modified);
}
}
Ok(match metadata.or(newest) {
Some(time) => ceil_secs(time),
None => now_secs(),
})
}
fn ceil_secs(time: SystemTime) -> u64 {
match time.duration_since(UNIX_EPOCH) {
Ok(duration) => {
let secs = duration.as_secs();
if duration.subsec_nanos() > 0 {
secs + 1
} else {
secs
}
}
Err(_) => 0,
}
}
fn now_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or(0)
}
fn set_stamp(path: &Path, stamp: u64) -> Result<()> {
let time = UNIX_EPOCH + Duration::from_secs(stamp);
let file =
fs::File::open(path).with_context(|| format!("opening {} to stamp it", path.display()))?;
file.set_modified(time)
.with_context(|| format!("stamping {}", path.display()))?;
Ok(())
}
#[cfg(unix)]
fn make_writable(path: &Path) -> std::io::Result<()> {
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(path)?.permissions();
perms.set_mode(0o644);
fs::set_permissions(path, perms)
}
#[cfg(not(unix))]
fn make_writable(path: &Path) -> std::io::Result<()> {
let mut perms = fs::metadata(path)?.permissions();
perms.set_readonly(false);
fs::set_permissions(path, perms)
}
fn make_read_only(path: &Path) -> Result<()> {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(path, fs::Permissions::from_mode(0o444))?;
}
#[cfg(not(unix))]
{
let _ = path;
}
Ok(())
}
fn is_volatile(name: &str) -> bool {
name == "invoked.timestamp" || name.contains(".kache")
}
#[derive(Debug, Default)]
pub struct RestoreReport {
pub parity: String,
pub target: PathBuf,
pub units_linked: usize,
pub units_already: usize,
pub files_linked: usize,
pub files_copied: usize,
pub files_existing: usize,
pub files_missing: usize,
pub bytes: u64,
pub errors: usize,
pub fatal_errors: usize,
pub note: Option<String>,
}
pub fn restore(
view_root: &Path,
store_root: &Path,
unit_filter: Option<&str>,
dry_run: bool,
) -> Result<RestoreReport> {
let parity = parity_key()?;
let mut report = RestoreReport {
parity: parity.clone(),
target: view_root.to_path_buf(),
..Default::default()
};
let Some(manifest) = load_manifest(store_root, &parity)? else {
report.note = Some(
"the vault holds no units for this generation yet — run `epicenter harvest` after a build"
.into(),
);
return Ok(report);
};
check_store_trust(store_root)?;
if !dry_run {
prepare_view_root(view_root)?;
}
let units = units_dir(store_root, &parity);
for entry in &manifest.units {
if let Some(query) = unit_filter {
if !unit_name_matches(&entry.name, query) {
continue;
}
}
let fp_dir = view_root
.join(&entry.profile)
.join(".fingerprint")
.join(&entry.name);
if read_small_regular(&fp_dir.join(&entry.memo))
.map(|text| text.trim() == entry.hash)
.unwrap_or(false)
{
report.units_already += 1;
continue;
}
let mut unit_touched = false;
for file in &entry.files {
let dest_rel = Path::new(&entry.profile).join(&file.dest);
let dst = view_root.join(&dest_rel);
match fs::symlink_metadata(&dst) {
Ok(meta) => {
if meta.file_type().is_symlink() {
report.errors += 1;
report.fatal_errors += 1;
eprintln!("restore: refusing symlink destination {}", dst.display());
} else {
report.files_existing += 1;
}
continue;
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => {
report.errors += 1;
eprintln!("restore: inspecting {}: {err}", dst.display());
continue;
}
}
let shared = file.path.starts_with("deps/") && !file.path.ends_with(".rmeta");
if dry_run {
unit_touched = true;
if shared {
report.files_linked += 1;
} else {
report.files_copied += 1;
}
report.bytes += file.size;
continue;
}
if let Some(parent) = dest_rel.parent() {
if let Err(err) = util::create_dir_chain_nofollow(view_root, parent) {
report.errors += 1;
eprintln!("restore: creating {}: {err}", parent.display());
continue;
}
}
let src = units.join(&entry.name).join(&file.path);
let rel = Path::new(&entry.name).join(&file.path);
match fs::symlink_metadata(&src) {
Ok(_) => {}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
report.files_missing += 1;
continue;
}
Err(err) => {
report.errors += 1;
report.fatal_errors += 1;
eprintln!("restore: inspecting {}: {err}", src.display());
continue;
}
}
let result = if shared {
util::check_regular_chain(&units, &rel)
.and_then(|_| verify_store_file(&src, file))
.and_then(|_| fs::hard_link(&src, &dst).map_err(Into::into))
} else {
util::check_regular_chain(&units, &rel)
.and_then(|_| install_private_copy(&src, &dst, entry.stamp, file))
};
match result {
Ok(()) => {
if shared {
report.files_linked += 1;
} else {
report.files_copied += 1;
}
report.bytes += file.size;
unit_touched = true;
}
Err(err) => {
report.errors += 1;
report.fatal_errors += 1;
eprintln!("restore: writing {}: {err:#}", dst.display());
continue;
}
}
}
if unit_touched {
report.units_linked += 1;
}
}
Ok(report)
}
fn check_store_trust(store_root: &Path) -> Result<()> {
let meta = fs::symlink_metadata(store_root)
.with_context(|| format!("inspecting store {}", store_root.display()))?;
if !meta.is_dir() || meta.file_type().is_symlink() {
anyhow::bail!("store {} is not a real directory", store_root.display());
}
if !util::owned_by_us(&meta) {
anyhow::bail!(
"store {} is owned by another user — refusing to execute its artifacts",
store_root.display()
);
}
util::ensure_private_dir(store_root)?;
Ok(())
}
fn prepare_view_root(view_root: &Path) -> Result<()> {
match fs::symlink_metadata(view_root) {
Ok(meta) => {
if !meta.is_dir() || meta.file_type().is_symlink() {
anyhow::bail!("view {} is not a real directory", view_root.display());
}
if !util::owned_by_us(&meta) {
anyhow::bail!(
"view {} is owned by another user — refusing to write into it",
view_root.display()
);
}
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
util::create_private_dir_all(view_root)?;
}
Err(err) => {
return Err(err).with_context(|| format!("inspecting view {}", view_root.display()));
}
}
Ok(())
}
fn verify_store_file(src: &Path, file: &FileEntry) -> Result<()> {
let meta = fs::symlink_metadata(src)
.with_context(|| format!("inspecting stored file {}", src.display()))?;
if !meta.file_type().is_file() || meta.file_type().is_symlink() {
anyhow::bail!("{} is not a regular file", src.display());
}
if !util::owned_by_us(&meta) {
anyhow::bail!("{} is owned by another user", src.display());
}
if meta.len() != file.size {
anyhow::bail!(
"{} size drift (recorded {}, found {})",
src.display(),
file.size,
meta.len()
);
}
if file.sha256.is_empty() {
anyhow::bail!("{} has no recorded SHA-256", src.display());
}
let found = util::sha256_file(src)?;
if found != file.sha256 {
anyhow::bail!("{} failed its SHA-256 check", src.display());
}
Ok(())
}
fn install_private_copy(src: &Path, dst: &Path, stamp: u64, file: &FileEntry) -> Result<()> {
let parent = dst
.parent()
.with_context(|| format!("destination {} has no parent", dst.display()))?;
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0);
let tmp = parent.join(format!(".epicenter-copy-{}-{nonce}", std::process::id()));
util::copy_file_verified(src, &tmp, file.size, &file.sha256)?;
make_writable(&tmp)?;
set_stamp(&tmp, stamp)?;
let installed = fs::hard_link(&tmp, dst);
let _ = fs::remove_file(&tmp);
installed.with_context(|| format!("installing {}", dst.display()))?;
Ok(())
}
fn read_small_regular(path: &Path) -> Option<String> {
let meta = fs::symlink_metadata(path).ok()?;
if !meta.file_type().is_file() || meta.file_type().is_symlink() || meta.len() > 4 * 1024 * 1024
{
return None;
}
fs::read_to_string(path).ok()
}
#[derive(Debug, Default)]
pub struct VerifyReport {
pub parity: String,
pub units: usize,
pub files: usize,
pub bytes: u64,
pub problems: Vec<String>,
pub warnings: Vec<String>,
pub ordering: Vec<String>,
}
impl VerifyReport {
pub fn is_clean(&self) -> bool {
self.problems.is_empty()
}
}
pub fn verify(store_root: &Path) -> Result<Vec<VerifyReport>> {
let mut reports = Vec::new();
for parity in generation_names(store_root)? {
let mut report = VerifyReport {
parity: parity.clone(),
..Default::default()
};
if let Err(err) = check_store_trust(store_root) {
report
.problems
.push(format!("unsafe store: {err:#} — no artifact was read"));
reports.push(report);
continue;
}
let Some(manifest) = load_manifest(store_root, &parity)? else {
report.problems.push(format!(
"generation {parity} has no readable manifest.json — it cannot be audited"
));
reports.push(report);
continue;
};
let units = units_dir(store_root, &parity);
for entry in &manifest.units {
for file in &entry.files {
let path = units.join(&entry.name).join(&file.path);
let rel = Path::new(&entry.name).join(&file.path);
if let Err(err) = util::check_regular_chain(&units, &rel) {
report.problems.push(format!(
"unsafe path: {} ({err:#})",
display_join(&path, &units)
));
continue;
}
let meta = match fs::symlink_metadata(&path) {
Ok(meta) => meta,
Err(_) => {
report
.problems
.push(format!("missing: {}", display_join(&path, units.as_path())));
continue;
}
};
if meta.file_type().is_symlink() {
report.problems.push(format!(
"symlink in store (refused): {}",
display_join(&path, &units)
));
continue;
}
if !meta.is_file() {
report.problems.push(format!(
"not a regular file: {}",
display_join(&path, &units)
));
continue;
}
if !util::owned_by_us(&meta) {
report.problems.push(format!(
"foreign-owned file: {}",
display_join(&path, &units)
));
}
report.files += 1;
report.bytes += meta.len();
if meta.len() != file.size {
report.problems.push(format!(
"size drift: {} (recorded {}, found {})",
display_join(&path, &units),
file.size,
meta.len()
));
}
let found_stamp = meta.modified().map(ceil_secs).unwrap_or(0);
if found_stamp != entry.stamp {
report.problems.push(format!(
"stamp drift: {} (recorded {}, found {})",
display_join(&path, &units),
entry.stamp,
found_stamp
));
}
if file.sha256.is_empty() {
report.problems.push(format!(
"no recorded SHA-256: {} (re-harvest this generation)",
display_join(&path, &units)
));
} else {
match util::sha256_file(&path) {
Ok(found) if found != file.sha256 => report.problems.push(format!(
"content drift: {} (recorded {}, found {})",
display_join(&path, &units),
&file.sha256[..file.sha256.len().min(12)],
&found[..found.len().min(12)]
)),
Ok(_) => {}
Err(err) => report.problems.push(format!(
"unreadable: {}: {err}",
display_join(&path, &units)
)),
}
}
if file.role == "memo" {
match read_small_regular(&path) {
Some(text) if file.path == entry.memo && text.trim() != entry.hash => {
let found = text.trim();
let detail = if crate::fingerprint::looks_like_hash(found) {
format!("found {found}")
} else {
"found non-hash content (not printed)".to_string()
};
report.problems.push(format!(
"hash drift: {} (recorded {}; {detail})",
display_join(&path, &units),
entry.hash
));
}
Some(text)
if file.path != entry.memo
&& !crate::fingerprint::looks_like_hash(text.trim()) =>
{
report.problems.push(format!(
"malformed memo file: {} (content is not a 16-hex hash)",
display_join(&path, &units)
));
}
Some(_) => {}
None => report.problems.push(format!(
"unreadable/symlinked memo file: {}",
display_join(&path, &units)
)),
}
}
}
}
let by_hash: std::collections::HashMap<&str, usize> = manifest
.units
.iter()
.enumerate()
.map(|(index, entry)| (entry.hash.as_str(), index))
.collect();
for entry in &manifest.units {
let json = entry
.files
.iter()
.find(|file| file.path.ends_with(".json"))
.and_then(|file| read_small_regular(&units.join(&entry.name).join(&file.path)))
.and_then(|text| crate::fingerprint::parse_fingerprint(&text));
let Some(json) = json else { continue };
for dep in &json.deps {
let Some(dep_index) = dep
.memo_hash()
.and_then(|hash| by_hash.get(hash.as_str()))
.copied()
else {
continue;
};
let dep_entry = &manifest.units[dep_index];
if dep_entry.stamp > entry.stamp {
report.ordering.push(format!(
"{} is older than its dependency {} ({} older) — cargo recompiles it after a restore",
entry.name,
dep_entry.name,
crate::util::human_duration(Duration::from_secs(dep_entry.stamp - entry.stamp))
));
}
}
}
report.units = manifest.units.len();
if units.is_dir() {
let known: std::collections::BTreeSet<&str> = manifest
.units
.iter()
.map(|unit| unit.name.as_str())
.collect();
for entry in fs::read_dir(&units)? {
let entry = entry?;
if !entry.file_type()?.is_dir() {
continue;
}
let name = entry.file_name().to_string_lossy().into_owned();
if name.starts_with('.') {
continue; }
if !known.contains(name.as_str()) {
report
.warnings
.push(format!("unrecorded unit group: units/{name}"));
}
}
}
reports.push(report);
}
Ok(reports)
}
fn display_join(path: &Path, base: &Path) -> String {
path.strip_prefix(base)
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/")
}
#[derive(Debug, Default)]
pub struct GenerationInfo {
pub parity: String,
pub active: bool,
pub units: usize,
pub files: usize,
pub bytes: u64,
pub linked: usize,
pub created: Option<u64>,
pub updated: Option<u64>,
}
pub fn generations(store_root: &Path) -> Result<Vec<GenerationInfo>> {
let active = parity_key().ok();
let mut out = Vec::new();
for parity in generation_names(store_root)? {
let mut info = GenerationInfo {
parity: parity.clone(),
active: active.as_deref() == Some(parity.as_str()),
..Default::default()
};
if let Some(manifest) = load_manifest(store_root, &parity)? {
info.units = manifest.units.len();
info.created = Some(manifest.created);
info.updated = Some(manifest.updated);
}
let units = units_dir(store_root, &parity);
if units.is_dir() {
for entry in WalkDir::new(&units).min_depth(1) {
let entry = entry?;
if !entry.file_type().is_file() {
continue;
}
let meta = entry.metadata()?;
info.files += 1;
info.bytes += meta.len();
if link_count(&meta) > 1 {
info.linked += 1;
}
}
}
out.push(info);
}
Ok(out)
}
pub(crate) fn link_count(meta: &fs::Metadata) -> u64 {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
meta.nlink()
}
#[cfg(not(unix))]
{
let _ = meta;
1
}
}
fn unit_name_matches(name: &str, query: &str) -> bool {
name.contains(query)
}
#[cfg(test)]
mod tests {
use super::*;
fn synthetic_build_dir(root: &Path) -> Result<()> {
let jsons =
r#"{"rustc":12019306335353385202,"features":"[]","declared_features":"[]","deps":[]}"#;
let lib = root.join("debug/.fingerprint/serde-102a33f055a8d229");
fs::create_dir_all(&lib)?;
fs::write(lib.join("lib-serde"), "5d82713a7f2d05e2")?;
fs::write(lib.join("lib-serde.json"), jsons)?;
fs::write(lib.join("dep-lib-serde"), [0u8; 24])?;
fs::write(lib.join("invoked.timestamp"), "")?;
fs::create_dir_all(root.join("debug/deps"))?;
fs::write(
root.join("debug/deps/libserde-102a33f055a8d229.rlib"),
[7u8; 64],
)?;
fs::write(
root.join("debug/deps/libserde-102a33f055a8d229.rmeta"),
[6u8; 48],
)?;
fs::write(
root.join("debug/deps/serde-102a33f055a8d229.d"),
b"side file",
)?;
fs::write(
root.join("debug/deps/libserde-102a33f055a8d229.kache-real"),
b"wrapper",
)?;
let compile = root.join("debug/.fingerprint/x-1122334455667788");
fs::create_dir_all(&compile)?;
fs::write(
compile.join("build-script-build-script-build"),
"aaaabbbbccccdddd",
)?;
fs::write(compile.join("build-script-build-script-build.json"), jsons)?;
fs::write(
compile.join("dep-build-script-build-script-build"),
[0u8; 8],
)?;
let compile_anchor = root.join("debug/build/x-1122334455667788");
fs::create_dir_all(&compile_anchor)?;
fs::write(
compile_anchor.join("build_script_build-1122334455667788"),
[3u8; 32],
)?;
fs::write(
compile_anchor.join("build_script_build-1122334455667788.d"),
b"dep",
)?;
fs::write(
compile_anchor.join("build_script_build-1122334455667788.kache-real"),
b"wrapper",
)?;
fs::write(compile_anchor.join("invoked.timestamp"), "")?;
let run = root.join("debug/.fingerprint/x-aabbccddeeff0011");
fs::create_dir_all(&run)?;
fs::write(
run.join("run-build-script-build-script-build"),
"eeeeffff00001111",
)?;
fs::write(run.join("run-build-script-build-script-build.json"), jsons)?;
let run_anchor = root.join("debug/build/x-aabbccddeeff0011");
fs::create_dir_all(&run_anchor)?;
fs::write(
run_anchor.join("output"),
b"cargo:rerun-if-changed=build.rs",
)?;
fs::write(run_anchor.join("stderr"), b"")?;
fs::write(run_anchor.join("invoked.timestamp"), "")?;
Ok(())
}
fn synthetic_husk(root: &Path) -> Result<()> {
let unit = root.join("debug/.fingerprint/husk-aaaabbbbccccdddd");
fs::create_dir_all(&unit)?;
fs::write(unit.join("lib-husk"), "0000111122223333")?;
Ok(())
}
#[test]
fn parity_key_is_stable_and_shaped() -> Result<()> {
let first = parity_key()?;
let second = parity_key()?;
assert_eq!(first, second);
let tc = toolchain()?;
assert!(first.starts_with(&format!("{}-{}", tc.release, tc.commit)));
assert!(first.ends_with(&flags_digest()));
Ok(())
}
#[test]
fn harvest_stores_groups_with_one_canonical_stamp() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
let report = harvest(build.path(), store.path(), None, false)?;
assert_eq!(report.stored, 3, "lib + build-script compile + run");
assert_eq!(report.skipped_incomplete, 0);
assert!(report.files >= 10, "got {}", report.files);
assert!(report.bytes > 0);
let parity = parity_key()?;
let manifest = load_manifest(store.path(), &parity)?.expect("manifest written");
assert_eq!(manifest.units.len(), 3);
let lib_entry = manifest
.units
.iter()
.find(|unit| unit.name == "serde-102a33f055a8d229")
.expect("lib unit recorded");
assert_eq!(lib_entry.hash, "5d82713a7f2d05e2");
assert_eq!(lib_entry.profile, "debug");
let group = units_dir(store.path(), &parity).join(&lib_entry.name);
for file in &lib_entry.files {
let meta = fs::metadata(group.join(&file.path))?;
assert_eq!(
ceil_secs(meta.modified()?),
lib_entry.stamp,
"{}",
file.path
);
assert_eq!(meta.len(), file.size);
}
assert!(!group.join("invoked.timestamp").exists());
for file in &lib_entry.files {
assert!(!file.path.contains("kache"), "{}", file.path);
assert!(!file.path.ends_with(".d"), "{}", file.path);
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let meta = fs::metadata(group.join(&lib_entry.files[0].path))?;
assert_eq!(
meta.permissions().mode() & 0o777,
0o444,
"store is read-only"
);
}
Ok(())
}
#[test]
fn harvest_is_idempotent_and_skips_incomplete_units() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
synthetic_husk(build.path())?;
let first = harvest(build.path(), store.path(), None, false)?;
assert_eq!(first.stored, 3);
assert_eq!(first.skipped_incomplete, 1, "husk without artifacts");
let second = harvest(build.path(), store.path(), None, false)?;
assert_eq!(second.stored, 0);
assert_eq!(second.unchanged, 3);
assert_eq!(second.skipped_incomplete, 1);
Ok(())
}
#[test]
fn harvest_refreshes_when_the_file_set_grows() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), Some("102a33f0"), false)?;
fs::write(
build.path().join("debug/deps/libserde-102a33f055a8d229.so"),
[1u8; 16],
)?;
let report = harvest(build.path(), store.path(), Some("102a33f0"), false)?;
assert_eq!(report.refreshed, 1);
assert_eq!(report.stored, 0);
let manifest = load_manifest(store.path(), &parity_key()?)?.expect("manifest");
let paths: Vec<&str> = manifest.units[0]
.files
.iter()
.map(|file| file.path.as_str())
.collect();
assert!(
paths.contains(&"deps/libserde-102a33f055a8d229.so"),
"{paths:?}"
);
Ok(())
}
#[test]
fn harvest_stores_all_memos_of_shared_dirs() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
let lib = build
.path()
.join("debug/.fingerprint/serde-102a33f055a8d229");
fs::write(lib.join("bin-serde"), "1111222233334444")?;
fs::write(lib.join("dep-bin-serde"), [0u8; 8])?;
let report = harvest(build.path(), store.path(), Some("102a33f0"), false)?;
assert_eq!(report.stored, 1);
let manifest = load_manifest(store.path(), &parity_key()?)?.expect("manifest");
let entry = &manifest.units[0];
let paths: Vec<&str> = entry.files.iter().map(|file| file.path.as_str()).collect();
assert!(paths.contains(&"bin-serde"), "{paths:?}");
assert!(paths.contains(&"dep-bin-serde"), "{paths:?}");
let reports = verify(store.path())?;
assert!(reports[0].is_clean(), "{:?}", reports[0].problems);
Ok(())
}
#[test]
fn harvest_dry_run_writes_nothing() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
let report = harvest(build.path(), store.path(), None, true)?;
assert_eq!(report.stored, 3);
assert_eq!(generation_names(store.path())?.len(), 0, "no generation");
Ok(())
}
#[test]
fn harvest_unit_filter_selects_one_group() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
let report = harvest(build.path(), store.path(), Some("102a33f0"), false)?;
assert_eq!(report.stored, 1);
let manifest = load_manifest(store.path(), &parity_key()?)?.expect("manifest");
assert_eq!(manifest.units.len(), 1);
assert_eq!(manifest.units[0].name, "serde-102a33f055a8d229");
Ok(())
}
#[test]
fn restore_links_hardlinks_and_never_overwrites() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
let view = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), None, false)?;
let occupied = view
.path()
.join("debug/deps/libserde-102a33f055a8d229.rlib");
fs::create_dir_all(occupied.parent().unwrap())?;
fs::write(&occupied, b"live artifact")?;
let report = restore(view.path(), store.path(), None, false)?;
assert_eq!(report.units_linked, 3);
assert_eq!(report.files_existing, 1, "the live artifact was skipped");
assert_eq!(fs::read(&occupied)?, b"live artifact");
assert_eq!(report.files_linked, 0, "no other deps/ artifact");
assert!(report.files_copied >= 12, "got {}", report.files_copied);
let memo = view
.path()
.join("debug/.fingerprint/serde-102a33f055a8d229/lib-serde");
assert_eq!(fs::read_to_string(&memo)?.trim(), "5d82713a7f2d05e2");
assert!(
view.path()
.join("debug/build/x-aabbccddeeff0011/output")
.is_file()
);
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
use std::os::unix::fs::PermissionsExt;
let parity = parity_key()?;
let stored = units_dir(store.path(), &parity).join("serde-102a33f055a8d229/lib-serde");
let view_meta = fs::metadata(&memo)?;
assert_ne!(
view_meta.ino(),
fs::metadata(&stored)?.ino(),
"state not shared"
);
assert_eq!(view_meta.nlink(), 1, "state owns its inode");
assert_ne!(view_meta.permissions().mode() & 0o200, 0, "state writable");
}
let again = restore(view.path(), store.path(), None, false)?;
assert_eq!(again.units_already, 3);
assert_eq!(again.files_linked, 0);
assert_eq!(again.files_copied, 0);
Ok(())
}
#[test]
fn restore_into_missing_profile_dirs_creates_them() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
let view = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), None, false)?;
let report = restore(view.path(), store.path(), None, false)?;
assert_eq!(report.errors, 0);
assert!(view.path().join("debug/.fingerprint").is_dir());
Ok(())
}
#[test]
fn restore_without_a_generation_reports_fail_open() -> Result<()> {
let view = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
let report = restore(view.path(), store.path(), None, false)?;
assert!(report.note.is_some());
assert_eq!(report.files_linked, 0);
Ok(())
}
#[test]
fn verify_passes_clean_and_flags_drift() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), None, false)?;
let reports = verify(store.path())?;
assert_eq!(reports.len(), 1);
assert!(reports[0].is_clean(), "{:?}", reports[0].problems);
assert_eq!(reports[0].units, 3);
let parity = parity_key()?;
let victim = units_dir(store.path(), &parity).join("serde-102a33f055a8d229/dep-lib-serde");
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&victim, fs::Permissions::from_mode(0o644))?;
}
fs::write(&victim, [9u8; 32])?;
let reports = verify(store.path())?;
assert!(!reports[0].is_clean());
assert!(
reports[0]
.problems
.iter()
.any(|line| line.contains("size drift")),
"{:?}",
reports[0].problems
);
Ok(())
}
#[test]
fn verify_reports_unrecorded_unit_groups() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), None, false)?;
let stray = units_dir(store.path(), &parity_key()?).join("stray-0000000000000000");
fs::create_dir_all(stray.join("deps"))?;
fs::write(stray.join("deps/x.rlib"), [0u8; 4])?;
let reports = verify(store.path())?;
assert!(
reports[0]
.warnings
.iter()
.any(|line| line.contains("stray-0000000000000000")),
"{:?}",
reports[0].warnings
);
Ok(())
}
#[test]
fn generations_summary_counts_store_files() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), None, false)?;
let generations = generations(store.path())?;
assert_eq!(generations.len(), 1);
assert!(generations[0].active, "current toolchain generation");
assert_eq!(generations[0].units, 3);
assert!(generations[0].files >= 10);
assert!(generations[0].linked <= generations[0].files);
Ok(())
}
#[test]
fn cached_diagnostics_are_stored_as_extra_and_verify_clean() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
fs::write(
build
.path()
.join("debug/.fingerprint/serde-102a33f055a8d229/output-lib-serde"),
r#"{"$message_type":"diagnostic","message":"warning: unused"}"#,
)?;
harvest(build.path(), store.path(), Some("102a33f0"), false)?;
let manifest = load_manifest(store.path(), &parity_key()?)?.expect("manifest");
let output = manifest.units[0]
.files
.iter()
.find(|file| file.path == "output-lib-serde")
.expect("cached diagnostics are stored");
assert_eq!(output.role, "extra");
let reports = verify(store.path())?;
assert!(reports[0].is_clean(), "{:?}", reports[0].problems);
Ok(())
}
#[test]
fn harvest_refreshes_when_a_role_drifted() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), Some("102a33f0"), false)?;
let parity = parity_key()?;
let mut manifest = load_manifest(store.path(), &parity)?.expect("manifest");
let memo_index = manifest.units[0]
.files
.iter()
.position(|file| file.path == manifest.units[0].memo)
.expect("memo recorded");
manifest.units[0].files[memo_index].role = "extra".to_string();
save_manifest(store.path(), &parity, &manifest)?;
let report = harvest(build.path(), store.path(), Some("102a33f0"), false)?;
assert_eq!(report.refreshed, 1, "role drift forces a refresh");
let manifest = load_manifest(store.path(), &parity)?.expect("manifest");
let role = &manifest.units[0].files[memo_index].role;
assert_eq!(role, "memo", "roles are rewritten");
Ok(())
}
#[test]
fn restore_shares_artifacts_but_copies_state_writable() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), Some("102a33f0"), false)?;
let view = tempfile::tempdir()?;
restore(view.path(), store.path(), Some("102a33f0"), false)?;
let state = view
.path()
.join("debug/.fingerprint/serde-102a33f055a8d229/lib-serde");
let artifact = view
.path()
.join("debug/deps/libserde-102a33f055a8d229.rlib");
let metadata = view
.path()
.join("debug/deps/libserde-102a33f055a8d229.rmeta");
assert!(state.is_file() && artifact.is_file() && metadata.is_file());
let parity = parity_key()?;
let manifest = load_manifest(store.path(), &parity)?.expect("manifest");
let entry = manifest
.units
.iter()
.find(|entry| entry.name == "serde-102a33f055a8d229")
.expect("stored unit");
assert_eq!(ceil_secs(fs::metadata(&state)?.modified()?), entry.stamp);
fs::write(&state, "ffffffffffffffff")?;
let stored = units_dir(store.path(), &parity)
.join("serde-102a33f055a8d229")
.join("lib-serde");
assert_eq!(fs::read_to_string(&stored)?, "5d82713a7f2d05e2");
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
assert!(
fs::metadata(&artifact)?.nlink() >= 2,
"artifact shares the store inode"
);
assert_eq!(fs::metadata(&state)?.nlink(), 1, "state is a private copy");
assert_eq!(
fs::metadata(&metadata)?.nlink(),
1,
".rmeta is a private copy (rustc opens it for write)"
);
}
Ok(())
}
fn synthetic_edge_dir(root: &Path, dep_at: u64, unit_at: u64) -> Result<()> {
fs::create_dir_all(root.join("debug/deps"))?;
let core = root.join("debug/.fingerprint/serde_core-b468be6fa8c1f936");
fs::create_dir_all(&core)?;
fs::write(core.join("lib-serde_core"), "71a9f323efa5ae82")?;
fs::write(core.join("dep-lib-serde_core"), [0u8; 8])?;
let core_rlib = root.join("debug/deps/libserde_core-b468be6fa8c1f936.rlib");
fs::write(&core_rlib, [1u8; 32])?;
let unit = root.join("debug/.fingerprint/serde-746d173581f1a314");
fs::create_dir_all(&unit)?;
fs::write(unit.join("lib-serde"), "5d82713a7f2d05e2")?;
fs::write(
unit.join("lib-serde.json"),
r#"{"rustc":5244445300240561903,"features":"[]","declared_features":"[]","deps":[[11029742160753049355,"serde_core",false,9416646317397223793]]}"#,
)?;
fs::write(unit.join("dep-lib-serde"), [0u8; 8])?;
let unit_rlib = root.join("debug/deps/libserde-746d173581f1a314.rlib");
fs::write(&unit_rlib, [1u8; 32])?;
set_stamp(&core.join("lib-serde_core"), dep_at)?;
set_stamp(&core.join("dep-lib-serde_core"), dep_at)?;
set_stamp(&core_rlib, dep_at)?;
set_stamp(&unit.join("lib-serde"), unit_at)?;
set_stamp(&unit.join("lib-serde.json"), unit_at)?;
set_stamp(&unit.join("dep-lib-serde"), unit_at)?;
set_stamp(&unit_rlib, unit_at)?;
Ok(())
}
#[test]
fn harvest_clamps_a_dependent_to_its_dependency() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
let now = now_secs();
synthetic_edge_dir(build.path(), now, now - 3600)?;
harvest(build.path(), store.path(), None, false)?;
let manifest = load_manifest(store.path(), &parity_key()?)?.expect("manifest");
let dependent = manifest
.units
.iter()
.find(|entry| entry.name == "serde-746d173581f1a314")
.expect("dependent stored");
let dependency = manifest
.units
.iter()
.find(|entry| entry.name == "serde_core-b468be6fa8c1f936")
.expect("dependency stored");
assert!(
dependent.stamp >= dependency.stamp,
"clamped: {} >= {}",
dependent.stamp,
dependency.stamp
);
let reports = verify(store.path())?;
assert!(reports[0].is_clean(), "{:?}", reports[0].problems);
assert!(reports[0].ordering.is_empty(), "{:?}", reports[0].ordering);
Ok(())
}
#[test]
fn harvest_stamps_groups_by_the_metadata_output() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
let now = now_secs();
set_stamp(
&build
.path()
.join("debug/deps/libserde-102a33f055a8d229.rmeta"),
now - 30,
)?;
set_stamp(
&build
.path()
.join("debug/deps/libserde-102a33f055a8d229.rlib"),
now,
)?;
harvest(build.path(), store.path(), Some("102a33f0"), false)?;
let manifest = load_manifest(store.path(), &parity_key()?)?.expect("manifest");
assert_eq!(manifest.units[0].stamp, now - 30);
Ok(())
}
#[test]
fn harvest_stores_read_only_sources() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
let source = build
.path()
.join("debug/.fingerprint/serde-102a33f055a8d229/dep-lib-serde");
let mut perms = fs::metadata(&source)?.permissions();
perms.set_readonly(true);
fs::set_permissions(&source, perms)?;
let report = harvest(build.path(), store.path(), Some("102a33f0"), false)?;
assert_eq!(report.stored, 1);
let parity = parity_key()?;
let manifest = load_manifest(store.path(), &parity)?.expect("manifest");
let stored = units_dir(store.path(), &parity)
.join(&manifest.units[0].name)
.join("dep-lib-serde");
assert!(stored.is_file(), "the state file is stored");
assert_eq!(
ceil_secs(fs::metadata(&stored)?.modified()?),
manifest.units[0].stamp,
"the stored copy carries the canonical stamp"
);
Ok(())
}
#[cfg(unix)]
#[test]
fn a_failed_store_drops_its_staging_dir() -> Result<()> {
use std::os::unix::fs::MetadataExt;
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
if fs::metadata(build.path())?.uid() == 0 {
return Ok(()); }
let artifact = build
.path()
.join("debug/deps/libserde-102a33f055a8d229.rlib");
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&artifact, fs::Permissions::from_mode(0o000))?;
}
let result = harvest(build.path(), store.path(), Some("102a33f0"), false);
assert!(result.is_err(), "an unreadable artifact aborts the store");
let units = units_dir(store.path(), &parity_key()?);
if units.is_dir() {
for entry in fs::read_dir(&units)? {
let name = entry?.file_name().to_string_lossy().into_owned();
assert!(!name.contains(".tmp-"), "staging leftover: {name}");
}
}
Ok(())
}
#[cfg(unix)]
#[test]
fn harvest_sweeps_stale_staging_dirs() -> Result<()> {
use std::os::unix::fs::PermissionsExt;
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
let units = units_dir(store.path(), &parity_key()?);
let stale = units.join(".serde-102a33f055a8d229.tmp-99999");
fs::create_dir_all(&stale)?;
let leftover = stale.join("dep-lib-serde");
fs::write(&leftover, b"x")?;
fs::set_permissions(&leftover, fs::Permissions::from_mode(0o444))?;
set_stamp(&stale, now_secs() - 7200)?;
let fresh = units.join(".serde-102a33f055a8d229.tmp-11111");
fs::create_dir_all(&fresh)?;
harvest(build.path(), store.path(), Some("102a33f0"), false)?;
assert!(!stale.exists(), "stale staging dir swept");
assert!(fresh.exists(), "fresh staging dir left alone");
Ok(())
}
#[test]
fn verify_flags_an_inverted_store_stamp() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
let now = now_secs();
synthetic_edge_dir(build.path(), now - 3600, now)?;
harvest(build.path(), store.path(), None, false)?;
let parity = parity_key()?;
let mut manifest = load_manifest(store.path(), &parity)?.expect("manifest");
let units = units_dir(store.path(), &parity);
let entry = manifest
.units
.iter_mut()
.find(|entry| entry.name == "serde-746d173581f1a314")
.expect("dependent stored");
entry.stamp = now - 7200;
for file in &entry.files {
set_stamp(&units.join(&entry.name).join(&file.path), now - 7200)?;
}
save_manifest(store.path(), &parity, &manifest)?;
let reports = verify(store.path())?;
assert!(reports[0].is_clean(), "{:?}", reports[0].problems);
assert_eq!(reports[0].ordering.len(), 1, "{:?}", reports[0].ordering);
assert!(reports[0].ordering[0].contains("serde-746d173581f1a314"));
assert!(reports[0].ordering[0].contains("serde_core-b468be6fa8c1f936"));
Ok(())
}
#[test]
fn manifest_validation_rejects_escapes() {
let unit = |name: &str, profile: &str, path: &str, dest: &str| UnitEntry {
name: name.into(),
profile: profile.into(),
memo: "lib-x".into(),
hash: "0123456789abcdef".into(),
stamp: 1,
files: vec![FileEntry {
path: path.into(),
dest: dest.into(),
size: 1,
sha256: String::new(),
role: "artifact".into(),
}],
};
let base = Manifest {
parity_key: "p".into(),
toolchain: "t".into(),
created: 1,
updated: 1,
units: vec![],
};
for unit in [
unit("../escape", "debug", "deps/x", "deps/x"),
unit("ok", "/etc", "deps/x", "deps/x"),
unit("ok", "debug", "/etc/passwd", "deps/x"),
unit("ok", "debug", "deps/x", "/root/.ssh/authorized_keys"),
unit("ok", "debug", "deps/../../x", "deps/x"),
] {
let manifest = Manifest {
units: vec![unit],
..base.clone()
};
assert!(manifest.validate().is_err(), "must reject {manifest:?}");
}
}
#[test]
fn harvest_records_sha256_and_verify_flags_content_drift() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), None, false)?;
let parity = parity_key()?;
let manifest = load_manifest(store.path(), &parity)?.expect("manifest");
assert!(
manifest
.units
.iter()
.flat_map(|unit| &unit.files)
.all(|file| !file.sha256.is_empty()),
"every stored file must carry a hash"
);
let entry = manifest
.units
.iter()
.find(|entry| entry.name == "serde-102a33f055a8d229")
.expect("unit");
let file = entry
.files
.iter()
.find(|file| file.path == "deps/libserde-102a33f055a8d229.rlib")
.expect("rlib");
let path = units_dir(store.path(), &parity)
.join(&entry.name)
.join(&file.path);
let mut bytes = fs::read(&path)?;
let last = bytes.len() - 1;
bytes[last] ^= 0x01; #[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&path, fs::Permissions::from_mode(0o644))?;
}
fs::write(&path, &bytes)?;
let stamp = UNIX_EPOCH + Duration::from_secs(entry.stamp);
fs::File::open(&path)?.set_modified(stamp)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&path, fs::Permissions::from_mode(0o444))?;
}
let reports = verify(store.path())?;
assert!(
reports[0]
.problems
.iter()
.any(|problem| problem.contains("content drift")),
"{:?}",
reports[0].problems
);
let view = tempfile::tempdir()?;
let report = restore(view.path(), store.path(), None, false)?;
assert!(report.errors > 0, "tampered artifact must not install");
assert!(
!view.path().join(&file.dest).exists(),
"tampered artifact must be absent from the view"
);
Ok(())
}
#[test]
fn concurrent_harvest_keeps_every_unit() -> Result<()> {
let store = tempfile::tempdir()?;
let mut builds = Vec::new();
for index in 0..6u32 {
let build = tempfile::tempdir()?;
let hex = format!("{:016x}", index + 1);
let unit = build
.path()
.join("debug/.fingerprint")
.join(format!("u{index}-{hex}"));
fs::create_dir_all(&unit)?;
fs::write(unit.join(format!("lib-u{index}")), "0123456789abcdef")?;
fs::write(
unit.join(format!("lib-u{index}.json")),
r#"{"rustc":1,"features":"[]","declared_features":"[]","deps":[]}"#,
)?;
fs::create_dir_all(build.path().join("debug/deps"))?;
fs::write(
build
.path()
.join("debug/deps")
.join(format!("libu{index}-{hex}.rlib")),
[index as u8; 8],
)?;
builds.push(build);
}
std::thread::scope(|scope| {
for build in &builds {
scope.spawn(|| harvest(build.path(), store.path(), None, false));
}
});
let manifest = load_manifest(store.path(), &parity_key()?)?.expect("manifest");
assert_eq!(manifest.units.len(), builds.len(), "{:?}", manifest.units);
Ok(())
}
#[cfg(unix)]
#[test]
fn store_directories_are_private() -> Result<()> {
use std::os::unix::fs::MetadataExt;
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
synthetic_build_dir(build.path())?;
harvest(build.path(), store.path(), None, false)?;
assert_eq!(
fs::metadata(store.path())?.mode() & 0o077,
0,
"store root 0700"
);
let generation = store.path().join(parity_key()?);
assert_eq!(
fs::metadata(&generation)?.mode() & 0o077,
0,
"generation 0700"
);
Ok(())
}
#[test]
fn verify_ordering_is_quiet_when_units_are_current() -> Result<()> {
let build = tempfile::tempdir()?;
let store = tempfile::tempdir()?;
let now = now_secs();
synthetic_edge_dir(build.path(), now - 3600, now)?;
harvest(build.path(), store.path(), None, false)?;
let reports = verify(store.path())?;
assert!(reports[0].is_clean(), "{:?}", reports[0].problems);
assert!(reports[0].ordering.is_empty(), "{:?}", reports[0].ordering);
Ok(())
}
}