use std::collections::{HashMap, HashSet};
use std::fs;
use std::io;
use std::path::Path;
use camino::{Utf8Component, Utf8Path, Utf8PathBuf};
use petgraph::graph::NodeIndex;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use crate::core::Hash32;
use crate::output::Output;
pub(crate) enum SnapshotEntry {
Page {
task: String,
output: Output,
content_hash: Hash32,
},
HashAsset { task: String },
StaticFile { source: Utf8PathBuf },
}
fn validate_dist_path(path: &Utf8Path) -> Result<(), crate::error::BuildError> {
let normalized = crate::output::normalize_path(path);
let safe = !path.as_str().is_empty()
&& !path.as_str().split('/').any(|component| component == ".")
&& normalized == path
&& path
.components()
.all(|component| matches!(component, Utf8Component::Normal(_)));
if safe {
Ok(())
} else {
Err(crate::error::BuildError::Other(anyhow::anyhow!(
"Output path `{path}` is outside the configured dist directory"
)))
}
}
fn existing_producer(entry: &SnapshotEntry) -> String {
match entry {
SnapshotEntry::Page { task, .. } | SnapshotEntry::HashAsset { task, .. } => task.clone(),
SnapshotEntry::StaticFile { source } => format!("static file `{source}`"),
}
}
fn output_conflict(path: Utf8PathBuf, existing: String, new: String) -> crate::error::BuildError {
crate::error::BuildError::Other(anyhow::anyhow!(
"Output conflict at `{path}`: produced by `{existing}` and `{new}`"
))
}
pub(crate) struct Snapshot {
entries: HashMap<Utf8PathBuf, SnapshotEntry>,
}
impl Snapshot {
pub(crate) fn new() -> Self {
Self {
entries: HashMap::new(),
}
}
pub(crate) fn insert_page(
&mut self,
_node: NodeIndex,
task_name: &str,
output: Output,
) -> Result<(), crate::error::BuildError> {
let path = output.path.clone();
validate_dist_path(&path)?;
if let Some(existing) = self.entries.get(&path) {
return Err(output_conflict(
path,
existing_producer(existing),
task_name.to_string(),
));
}
tracing::debug!("snapshot: page `{}` <- task `{}`", path, task_name);
let content_hash = Hash32::hash(&output.data);
self.entries.insert(
path,
SnapshotEntry::Page {
task: task_name.to_string(),
content_hash,
output,
},
);
Ok(())
}
pub(crate) fn insert_hash_asset(
&mut self,
_node: NodeIndex,
task_name: &str,
path: Utf8PathBuf,
) -> Result<(), crate::error::BuildError> {
validate_dist_path(&path)?;
if let Some(existing) = self.entries.get(&path) {
if matches!(existing, SnapshotEntry::HashAsset { .. }) {
return Ok(());
}
return Err(output_conflict(
path,
existing_producer(existing),
task_name.to_string(),
));
}
self.entries.insert(
path,
SnapshotEntry::HashAsset {
task: task_name.to_string(),
},
);
Ok(())
}
pub(crate) fn insert_static_file(
&mut self,
dist_rel: Utf8PathBuf,
source: Utf8PathBuf,
) -> Result<(), crate::error::BuildError> {
validate_dist_path(&dist_rel)?;
if let Some(existing) = self.entries.get(&dist_rel) {
return Err(output_conflict(
dist_rel,
existing_producer(existing),
format!("static file `{source}`"),
));
}
self.entries
.insert(dist_rel, SnapshotEntry::StaticFile { source });
Ok(())
}
pub(crate) fn page_count(&self) -> usize {
self.entries
.values()
.filter(|e| matches!(e, SnapshotEntry::Page { .. }))
.count()
}
pub(crate) fn commit(&self, dist: &camino::Utf8Path) -> io::Result<()> {
let dist = dist.as_std_path();
fs::create_dir_all(dist)?;
tracing::debug!(
"commit: {} total entries ({} pages, {} hash assets, {} static files)",
self.entries.len(),
self.entries
.values()
.filter(|e| matches!(e, SnapshotEntry::Page { .. }))
.count(),
self.entries
.values()
.filter(|e| matches!(e, SnapshotEntry::HashAsset { .. }))
.count(),
self.entries
.values()
.filter(|e| matches!(e, SnapshotEntry::StaticFile { .. }))
.count(),
);
let desired: HashSet<Utf8PathBuf> = self.entries.keys().cloned().collect();
let removed = remove_stale(dist, Utf8Path::new(""), &desired)?;
if removed > 0 {
tracing::info!("removed {} stale file(s) from dist", removed);
}
write_pages(
dist,
self.entries.iter().filter_map(|(path, entry)| match entry {
SnapshotEntry::Page {
output,
content_hash,
..
} => Some((path, output, content_hash)),
_ => None,
}),
)
}
pub(crate) fn commit_diff(&self, prev: &Snapshot, dist: &camino::Utf8Path) -> io::Result<()> {
let dist = dist.as_std_path();
fs::create_dir_all(dist)?;
tracing::debug!(
"commit_diff: {} prev entries -> {} new entries",
prev.entries.len(),
self.entries.len(),
);
let mut removed = 0;
let mut dirs_to_prune: HashSet<std::path::PathBuf> = HashSet::new();
for path in prev.entries.keys() {
if !self.entries.contains_key(path) {
let abs = dist.join(path.as_std_path());
tracing::debug!("removing stale dist file: {}", path);
match fs::remove_file(&abs) {
Ok(()) => removed += 1,
Err(e) if e.kind() == io::ErrorKind::NotFound => {
tracing::debug!("stale file already gone: {}", path);
}
Err(e) => return Err(e),
}
if let Some(parent) = abs.parent() {
dirs_to_prune.insert(parent.to_path_buf());
}
}
}
if removed > 0 {
tracing::info!("removed {} stale file(s) from dist", removed);
prune_empty_dirs(dist, dirs_to_prune)?;
}
write_pages(
dist,
self.entries.iter().filter_map(|(path, entry)| {
let SnapshotEntry::Page {
output,
content_hash,
..
} = entry
else {
return None;
};
match prev.entries.get(path) {
None => {
tracing::debug!("new page: {}", path);
Some((path, output, content_hash))
}
Some(SnapshotEntry::Page {
content_hash: prev_hash,
..
}) => {
let abs_path = dist.join(path.as_std_path());
if prev_hash != content_hash || !abs_path.exists() {
tracing::debug!("changed or missing page: {}", path);
Some((path, output, content_hash))
} else {
tracing::debug!("unchanged page, skipping: {}", path);
None
}
}
_ => {
tracing::debug!("new page (replaced non-page entry): {}", path);
Some((path, output, content_hash))
}
}
}),
)
}
pub(crate) fn to_meta(&self) -> SnapshotMeta {
let entries = self
.entries
.iter()
.map(|(path, entry)| {
let meta_entry = match entry {
SnapshotEntry::Page { content_hash, .. } => MetaEntry::Page {
content_hash: content_hash.to_bytes(),
},
SnapshotEntry::HashAsset { .. } => MetaEntry::HashAsset,
SnapshotEntry::StaticFile { .. } => MetaEntry::StaticFile,
};
(path.to_string(), meta_entry)
})
.collect();
SnapshotMeta { entries }
}
pub(crate) fn commit_diff_meta(
&self,
prev: &SnapshotMeta,
dist: &camino::Utf8Path,
) -> io::Result<()> {
let dist = dist.as_std_path();
fs::create_dir_all(dist)?;
tracing::debug!(
"commit_diff_meta: {} prev entries -> {} new entries",
prev.entries.len(),
self.entries.len(),
);
let mut removed = 0;
let mut dirs_to_prune: HashSet<std::path::PathBuf> = HashSet::new();
for path in prev.entries.keys() {
if !self.entries.contains_key(Utf8Path::new(path)) {
let abs = dist.join(path.as_str());
tracing::debug!("removing stale dist file: {}", path);
match fs::remove_file(&abs) {
Ok(()) => removed += 1,
Err(e) if e.kind() == io::ErrorKind::NotFound => {
tracing::debug!("stale file already gone: {}", path);
}
Err(e) => return Err(e),
}
if let Some(parent) = abs.parent() {
dirs_to_prune.insert(parent.to_path_buf());
}
}
}
if removed > 0 {
tracing::info!("removed {} stale file(s) from dist", removed);
prune_empty_dirs(dist, dirs_to_prune)?;
}
write_pages(
dist,
self.entries.iter().filter_map(|(path, entry)| {
let SnapshotEntry::Page {
output,
content_hash,
..
} = entry
else {
return None;
};
match prev.entries.get(path.as_str()) {
None => {
tracing::debug!("new page: {}", path);
Some((path, output, content_hash))
}
Some(MetaEntry::Page {
content_hash: prev_hash,
}) => {
let abs_path = dist.join(path.as_std_path());
if prev_hash != &content_hash.to_bytes() || !abs_path.exists() {
tracing::debug!("changed or missing page: {}", path);
Some((path, output, content_hash))
} else {
tracing::debug!("unchanged page, skipping: {}", path);
None
}
}
_ => {
tracing::debug!("new page (replaced non-page entry): {}", path);
Some((path, output, content_hash))
}
}
}),
)
}
}
#[derive(Serialize, Deserialize)]
pub(crate) struct SnapshotMeta {
entries: HashMap<String, MetaEntry>,
}
#[derive(Serialize, Deserialize)]
enum MetaEntry {
Page { content_hash: [u8; 32] },
HashAsset,
StaticFile,
}
impl SnapshotMeta {
const RELATIVE_PATH: &'static str = "snapshot/metadata.cbor";
pub(crate) fn load(cache_dir: &camino::Utf8Path) -> io::Result<Option<Self>> {
let path = cache_dir.join(Self::RELATIVE_PATH);
let file = match fs::File::open(&path) {
Ok(f) => f,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e),
};
tracing::debug!("loading snapshot meta from {}", path);
ciborium::from_reader(file)
.map(Some)
.map_err(io::Error::other)
}
pub(crate) fn save(&self, cache_dir: &camino::Utf8Path) -> io::Result<()> {
let path = cache_dir.join(Self::RELATIVE_PATH);
fs::create_dir_all(cache_dir.join("snapshot"))?;
let file = fs::File::create(&path)?;
tracing::debug!("saving snapshot meta to {}", path);
ciborium::into_writer(self, file).map_err(io::Error::other)
}
}
fn write_pages<'a>(
dist: &Path,
pages: impl Iterator<Item = (&'a Utf8PathBuf, &'a Output, &'a Hash32)>,
) -> io::Result<()> {
let pages: Vec<_> = pages.collect();
let parent_dirs: HashSet<std::path::PathBuf> = pages
.iter()
.filter_map(|(path, _, _)| {
dist.join(path.as_std_path())
.parent()
.map(|p| p.to_path_buf())
})
.collect();
for dir in parent_dirs {
fs::create_dir_all(dir)?;
}
pages.par_iter().try_for_each(|(path, output, _hash)| {
fs::write(dist.join(path.as_std_path()), &output.data)
})
}
fn prune_empty_dirs(dist: &Path, dirs: HashSet<std::path::PathBuf>) -> io::Result<()> {
let mut dirs: Vec<_> = dirs.into_iter().collect();
dirs.sort_by_key(|d| std::cmp::Reverse(d.components().count()));
for dir in dirs {
if dir == dist {
continue;
}
if fs::remove_dir(&dir).is_ok() {
tracing::debug!("pruned empty dir: {}", dir.display());
}
}
Ok(())
}
fn remove_stale(dist: &Path, rel: &Utf8Path, desired: &HashSet<Utf8PathBuf>) -> io::Result<usize> {
let dir = if rel.as_str().is_empty() {
dist.to_path_buf()
} else {
dist.join(rel.as_std_path())
};
let read_dir = match fs::read_dir(&dir) {
Ok(rd) => rd,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(0),
Err(e) => return Err(e),
};
let mut removed = 0;
for entry in read_dir {
let entry = entry?;
let name = entry.file_name();
let name_str = name.to_str().ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "non-UTF-8 filename in dist")
})?;
let entry_rel = if rel.as_str().is_empty() {
Utf8PathBuf::from(name_str)
} else {
rel.join(name_str)
};
let entry_abs = dist.join(entry_rel.as_std_path());
if entry.file_type()?.is_dir() {
removed += remove_stale(dist, &entry_rel, desired)?;
if fs::read_dir(&entry_abs)?.next().is_none() {
fs::remove_dir(&entry_abs)?;
}
} else if !desired.contains(entry_rel.as_path()) {
tracing::debug!("removing stale dist file: {}", entry_rel);
fs::remove_file(&entry_abs)?;
removed += 1;
}
}
Ok(removed)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Output, output::OutputData};
#[test]
fn rejects_paths_outside_dist() {
let mut snapshot = Snapshot::new();
let result = snapshot.insert_page(
NodeIndex::new(0),
"escape",
Output {
path: Utf8PathBuf::from("../escape.txt"),
data: OutputData::Utf8("bad".into()),
},
);
assert!(result.is_err());
}
#[test]
fn rejects_absolute_paths() {
let mut snapshot = Snapshot::new();
let result = snapshot.insert_page(
NodeIndex::new(0),
"escape",
Output {
path: Utf8PathBuf::from("/tmp/escape.txt"),
data: OutputData::Utf8("bad".into()),
},
);
assert!(result.is_err());
}
#[test]
fn rejects_duplicate_outputs() {
let mut snapshot = Snapshot::new();
let first = Output::binary("same.txt", b"first".to_vec());
let second = Output::binary("same.txt", b"second".to_vec());
assert!(
snapshot
.insert_page(NodeIndex::new(0), "first", first)
.is_ok()
);
let result = snapshot.insert_page(NodeIndex::new(1), "second", second);
assert!(result.is_err());
}
#[test]
fn rejects_current_dir_components() {
let mut snapshot = Snapshot::new();
let result = snapshot.insert_page(
NodeIndex::new(0),
"curdir",
Output {
path: Utf8PathBuf::from("same/./file.txt"),
data: OutputData::Utf8("bad".into()),
},
);
assert!(result.is_err());
}
}