use std::fs::{self, File, OpenOptions};
use std::io::{self, ErrorKind, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use crate::error::{Error, Result};
use crate::hash::{ContentHash, ContentHasher};
const PARTIAL_SUFFIX: &str = ".partial";
static NEXT_PARTIAL: AtomicU64 = AtomicU64::new(0);
fn create_partial(dir: &Path, hash: &ContentHash) -> io::Result<(File, PathBuf)> {
loop {
let n = NEXT_PARTIAL.fetch_add(1, Ordering::Relaxed);
let path = dir.join(format!(
"{}.{}-{n}{PARTIAL_SUFFIX}",
hash.to_hex(),
std::process::id()
));
match OpenOptions::new().write(true).create_new(true).open(&path) {
Ok(file) => return Ok((file, path)),
Err(e) if e.kind() == ErrorKind::AlreadyExists => continue,
Err(e) => return Err(e),
}
}
}
#[derive(Clone, Debug)]
pub struct Stager {
dir: PathBuf,
max: u64,
}
impl Stager {
pub fn new(dir: PathBuf, max: u64) -> Self {
Self { dir, max }
}
pub fn writer(&self, hash: ContentHash, len: u64) -> Result<StagingWriter> {
if len > self.max {
return Err(Error::TooLarge { len, max: self.max });
}
fs::create_dir_all(&self.dir)?;
let (file, partial) = create_partial(&self.dir, &hash)?;
Ok(StagingWriter {
file,
hasher: ContentHasher::new(),
written: 0,
len,
hash,
partial,
dest: self.dir.join(hash.to_hex()),
finished: false,
})
}
pub fn staged(&self, hash: &ContentHash) -> Option<PathBuf> {
let path = self.dir.join(hash.to_hex());
path.is_file().then_some(path)
}
}
pub struct StagingWriter {
file: File,
hasher: ContentHasher,
written: u64,
len: u64,
hash: ContentHash,
partial: PathBuf,
dest: PathBuf,
finished: bool,
}
impl StagingWriter {
pub fn write(&mut self, bytes: &[u8]) -> Result<()> {
let total = self.written.saturating_add(bytes.len() as u64);
if total > self.len {
return Err(Error::TooLarge {
len: total,
max: self.len,
});
}
self.file.write_all(bytes)?;
self.hasher.update(bytes);
self.written = total;
Ok(())
}
pub fn finish(mut self) -> Result<PathBuf> {
if self.written != self.len || self.hasher.clone().finish() != self.hash {
return Err(Error::ContentMismatch {
expected: self.hash,
});
}
self.file.flush()?;
self.file.sync_all()?;
if let Err(e) = fs::rename(&self.partial, &self.dest) {
if !self.dest.is_file() {
return Err(e.into());
}
tracing::debug!(hash = %self.hash, error = %e, "kept the content already staged");
return Ok(self.dest.clone());
}
self.finished = true;
Ok(self.dest.clone())
}
}
impl Write for StagingWriter {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
StagingWriter::write(self, buf).map_err(|e| match e {
Error::Io(e) => e,
other => io::Error::other(other),
})?;
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
self.file.flush()
}
}
impl Drop for StagingWriter {
fn drop(&mut self) {
if !self.finished {
let _ = fs::remove_file(&self.partial);
}
}
}
pub(crate) fn clear_partials(dir: &Path) -> io::Result<()> {
let entries = match fs::read_dir(dir) {
Ok(entries) => entries,
Err(e) if e.kind() == ErrorKind::NotFound => return Ok(()),
Err(e) => return Err(e),
};
for entry in entries {
let path = entry?.path();
if path.extension().is_some_and(|e| e == "partial") {
fs::remove_file(path)?;
}
}
Ok(())
}
pub(crate) fn finished(dir: &Path) -> io::Result<Vec<(ContentHash, PathBuf, fs::Metadata)>> {
let entries = match fs::read_dir(dir) {
Ok(entries) => entries,
Err(e) if e.kind() == ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(e),
};
let mut found = Vec::new();
for entry in entries {
let entry = entry?;
let Some(hash) = entry
.file_name()
.to_str()
.and_then(|name| name.parse::<ContentHash>().ok())
else {
continue;
};
let meta = entry.metadata()?;
if meta.is_file() {
found.push((hash, entry.path(), meta));
}
}
Ok(found)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::Error;
use crate::hash::{ContentHash, ContentHasher};
fn dir() -> (tempfile::TempDir, std::path::PathBuf) {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("staging");
(tmp, dir)
}
fn entries(dir: &std::path::Path) -> usize {
std::fs::read_dir(dir).map(|d| d.count()).unwrap_or(0)
}
#[test]
fn hasher_matches_of_bytes_across_split_updates() {
let mut h = ContentHasher::new();
h.update(b"ab");
h.update(b"c");
assert_eq!(h.finish(), ContentHash::of_bytes(b"abc"));
}
#[test]
fn finished_content_lands_at_its_hash() {
let (_t, dir) = dir();
let hash = ContentHash::of_bytes(b"hello");
let mut w = Stager::new(dir.clone(), 100).writer(hash, 5).unwrap();
w.write(b"hel").unwrap();
w.write(b"lo").unwrap();
let path = w.finish().unwrap();
assert_eq!(path, dir.join(hash.to_hex()));
assert_eq!(std::fs::read(&path).unwrap(), b"hello");
assert_eq!(entries(&dir), 1, "no .partial left");
}
#[test]
fn a_wrong_hash_is_refused_and_leaves_nothing() {
let (_t, dir) = dir();
let hash = ContentHash::of_bytes(b"hello");
let mut w = Stager::new(dir.clone(), 100).writer(hash, 5).unwrap();
w.write(b"world").unwrap();
assert!(matches!(
w.finish(),
Err(Error::ContentMismatch { expected }) if expected == hash
));
assert_eq!(entries(&dir), 0);
}
#[test]
fn a_short_write_is_refused_and_leaves_nothing() {
let (_t, dir) = dir();
let hash = ContentHash::of_bytes(b"hello");
let mut w = Stager::new(dir.clone(), 100).writer(hash, 5).unwrap();
w.write(b"hel").unwrap();
assert!(matches!(w.finish(), Err(Error::ContentMismatch { .. })));
assert_eq!(entries(&dir), 0);
}
#[test]
fn writing_past_len_is_refused() {
let (_t, dir) = dir();
let hash = ContentHash::of_bytes(b"abc");
let mut w = Stager::new(dir, 100).writer(hash, 3).unwrap();
assert!(matches!(
w.write(b"abcd"),
Err(Error::TooLarge { len: 4, max: 3 })
));
}
#[test]
fn a_len_over_the_max_is_refused_before_any_file() {
let (_t, dir) = dir();
let stager = Stager::new(dir.clone(), 10);
let err = stager.writer(ContentHash::of_bytes(b"x"), 11).err();
assert!(matches!(err, Some(Error::TooLarge { len: 11, max: 10 })));
assert_eq!(entries(&dir), 0);
}
#[test]
fn dropping_unfinished_removes_the_partial() {
let (_t, dir) = dir();
let hash = ContentHash::of_bytes(b"hello");
let mut w = Stager::new(dir.clone(), 100).writer(hash, 5).unwrap();
w.write(b"he").unwrap();
assert_eq!(entries(&dir), 1);
drop(w);
assert_eq!(entries(&dir), 0);
}
#[test]
fn a_zero_length_blob_finishes() {
let (_t, dir) = dir();
let hash = ContentHash::of_bytes(b"");
let w = Stager::new(dir.clone(), 100).writer(hash, 0).unwrap();
let path = w.finish().unwrap();
assert_eq!(std::fs::read(path).unwrap(), b"");
}
#[test]
fn staged_reports_only_finished_content() {
let (_t, dir) = dir();
let stager = Stager::new(dir, 100);
let hash = ContentHash::of_bytes(b"hi");
let mut w = stager.writer(hash, 2).unwrap();
w.write(b"hi").unwrap();
assert_eq!(stager.staged(&hash), None);
let path = w.finish().unwrap();
assert_eq!(stager.staged(&hash), Some(path));
}
#[test]
fn a_vanished_staging_dir_fails_and_leaves_no_partial() {
let (_t, dir) = dir();
let hash = ContentHash::of_bytes(b"hello");
let mut w = Stager::new(dir.clone(), 100).writer(hash, 5).unwrap();
std::fs::remove_dir_all(&dir).unwrap();
let result = w.write(b"hello").and_then(|()| w.finish().map(|_| ()));
assert!(result.is_err());
assert!(!dir.exists());
}
fn two_writers_for_one_hash(a_first: bool) {
let (_t, dir) = dir();
let stager = Stager::new(dir.clone(), 100);
let hash = ContentHash::of_bytes(b"hello");
let mut a = stager.writer(hash, 5).unwrap();
a.write(b"hel").unwrap();
let mut b = stager.writer(hash, 5).unwrap();
b.write(b"h").unwrap();
a.write(b"lo").unwrap();
b.write(b"el").unwrap();
let path = if a_first {
let path = a.finish().unwrap();
assert_eq!(std::fs::read(&path).unwrap(), b"hello");
b.write(b"lo").unwrap();
b.finish().unwrap()
} else {
b.write(b"lo").unwrap();
let path = b.finish().unwrap();
assert_eq!(std::fs::read(&path).unwrap(), b"hello");
a.finish().unwrap()
};
assert_eq!(std::fs::read(&path).unwrap(), b"hello");
assert_eq!(stager.staged(&hash), Some(path));
assert_eq!(entries(&dir), 1, "no .partial left");
}
#[test]
fn two_writers_for_one_hash_finishing_first_to_last() {
two_writers_for_one_hash(true);
}
#[test]
fn two_writers_for_one_hash_finishing_last_to_first() {
two_writers_for_one_hash(false);
}
#[test]
fn dropping_one_writer_leaves_the_other_alone() {
let (_t, dir) = dir();
let stager = Stager::new(dir.clone(), 100);
let hash = ContentHash::of_bytes(b"hello");
let mut a = stager.writer(hash, 5).unwrap();
a.write(b"he").unwrap();
let mut b = stager.writer(hash, 5).unwrap();
b.write(b"h").unwrap();
drop(b);
a.write(b"llo").unwrap();
let path = a.finish().unwrap();
assert_eq!(std::fs::read(path).unwrap(), b"hello");
assert_eq!(entries(&dir), 1);
}
#[test]
fn clear_partials_removes_the_partials_of_abandoned_writers() {
let (_t, dir) = dir();
let stager = Stager::new(dir.clone(), 100);
let hash = ContentHash::of_bytes(b"hello");
for _ in 0..2 {
let mut w = stager.writer(hash, 5).unwrap();
w.write(b"he").unwrap();
std::mem::forget(w);
}
assert_eq!(entries(&dir), 2, "each writer has its own partial");
clear_partials(&dir).unwrap();
assert_eq!(entries(&dir), 0);
}
#[test]
fn clear_partials_removes_only_partials() {
let (_t, dir) = dir();
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("aa"), b"keep").unwrap();
std::fs::write(dir.join("aa.partial"), b"drop").unwrap();
clear_partials(&dir).unwrap();
assert!(dir.join("aa").exists());
assert!(!dir.join("aa.partial").exists());
clear_partials(&dir.join("missing")).unwrap();
}
#[test]
fn a_failed_rename_onto_content_already_staged_counts_as_staged() {
let (_t, dir) = dir();
let stager = Stager::new(dir.clone(), 100);
let hash = ContentHash::of_bytes(b"hello");
let mut first = stager.writer(hash, 5).unwrap();
let mut second = stager.writer(hash, 5).unwrap();
first.write(b"hello").unwrap();
second.write(b"hello").unwrap();
first.finish().unwrap();
std::fs::remove_file(&second.partial).unwrap();
assert_eq!(second.finish().unwrap(), dir.join(hash.to_hex()));
assert_eq!(std::fs::read(dir.join(hash.to_hex())).unwrap(), b"hello");
assert_eq!(entries(&dir), 1);
}
#[test]
fn a_failed_rename_with_nothing_staged_is_an_error() {
let (_t, dir) = dir();
let hash = ContentHash::of_bytes(b"hello");
let mut w = Stager::new(dir.clone(), 100).writer(hash, 5).unwrap();
w.write(b"hello").unwrap();
std::fs::remove_file(&w.partial).unwrap();
assert!(matches!(w.finish(), Err(Error::Io(_))));
assert_eq!(entries(&dir), 0);
}
#[cfg(windows)]
#[test]
fn content_staged_while_the_same_hash_is_open_without_delete_sharing() {
use std::os::windows::fs::OpenOptionsExt;
const FILE_SHARE_READ: u32 = 0x1;
let (_t, dir) = dir();
let stager = Stager::new(dir.clone(), 100);
let hash = ContentHash::of_bytes(b"hello");
let mut first = stager.writer(hash, 5).unwrap();
first.write(b"hello").unwrap();
let staged = first.finish().unwrap();
let _held = OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ)
.open(&staged)
.unwrap();
let mut second = stager.writer(hash, 5).unwrap();
second.write(b"hello").unwrap();
assert_eq!(second.finish().unwrap(), staged);
assert_eq!(entries(&dir), 1, "the second partial is removed");
}
}