use std::path::{Path, PathBuf};
use anyhow::{Context, Result, anyhow, bail};
use znippy_common::{
ArtifactMeta, CompactReport, ZnippyArchive, compact_archive, get_all_files_meta,
};
pub use git_storage_trait::GcReport;
pub trait Gc {
fn run(&self, archive: &Path) -> Result<GcReport>;
fn name(&self) -> &'static str;
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HolgerTarget {
pub endpoint: String,
pub repository: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RetirementReceipt {
pub artifact: String,
pub bytes_shipped: u64,
pub digest_hex: String,
pub verified_at_destination: bool,
pub local_copy_removed: bool,
}
pub fn retire_to_holger(_generation: &Path, _target: &HolgerTarget) -> Result<RetirementReceipt> {
todo!(
"retire a pensioned generation to holger: deliberately empty. The contract is settled \
(ship the sealed archive, have holger verify it, only then unlink the local copy); \
the transport, the digest and the destination naming are not. Do not add a holger \
client from this file without deciding them first."
)
}
#[derive(Debug, Clone, Copy)]
pub struct CompactInPlace {
pub verify: bool,
}
impl Default for CompactInPlace {
fn default() -> Self {
Self { verify: true }
}
}
impl CompactInPlace {
pub fn new() -> Self {
Self::default()
}
}
impl Gc for CompactInPlace {
fn run(&self, archive: &Path) -> Result<GcReport> {
let CompactReport {
bytes_before,
bytes_after,
rows,
delta_rows,
} = compact_archive(archive)
.with_context(|| format!("compacting {} in place", archive.display()))?;
let verified = if self.verify {
read_back_every_entry(archive).with_context(|| {
format!(
"verifying {} after an in-place compaction — the original is already gone, \
which is exactly the window NewGeneration closes",
archive.display()
)
})?;
true
} else {
false
};
Ok(GcReport {
strategy: self.name(),
archive: archive.to_path_buf(),
retired: None,
bytes_before,
bytes_after,
rows,
delta_rows,
verified,
retired_packs: 0,
})
}
fn name(&self) -> &'static str {
"CompactInPlace"
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum StopAfter {
#[default]
Never,
Compact,
Verify,
Rename,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct NewGeneration {
pub stop_after: StopAfter,
}
impl NewGeneration {
pub fn new() -> Self {
Self::default()
}
pub fn stopping_after(stop: StopAfter) -> Self {
Self { stop_after: stop }
}
}
pub fn default_gc() -> NewGeneration {
NewGeneration::new()
}
pub fn next_generation(archive: &Path) -> Result<PathBuf> {
let name = archive
.file_name()
.and_then(|n| n.to_str())
.ok_or_else(|| anyhow!("{} has no usable file name", archive.display()))?;
let (stem, ext) = match name.rsplit_once('.') {
Some((s, e)) if !s.is_empty() => (s, Some(e)),
_ => (name, None),
};
let next = match stem.rsplit_once('.') {
Some((head, marker)) if is_generation(marker) => {
format!("{head}.g{}", marker[1..].parse::<u64>().unwrap_or(0) + 1)
}
_ => format!("{stem}.g1"),
};
let file = match ext {
Some(e) => format!("{next}.{e}"),
None => next,
};
Ok(archive.with_file_name(file))
}
fn is_generation(s: &str) -> bool {
s.len() > 1 && s.starts_with('g') && s[1..].bytes().all(|b| b.is_ascii_digit())
}
pub(crate) fn read_back_every_entry(archive: &Path) -> Result<u64> {
let manifest = manifest(archive)?;
let ar = ZnippyArchive::open(archive)
.with_context(|| format!("opening {} to verify it", archive.display()))?;
let mut bytes = 0u64;
for (path, expected) in &manifest {
let got = ar.extract_file_verified(path).with_context(|| {
format!(
"{} does not read back {path} — the new generation is not usable",
archive.display()
)
})?;
if got.len() as u64 != *expected {
bail!(
"{}: {path} reads back {} bytes, the index says {expected}",
archive.display(),
got.len()
);
}
bytes += got.len() as u64;
}
Ok(bytes)
}
fn manifest(archive: &Path) -> Result<Vec<(String, u64)>> {
let mut m: Vec<(String, u64)> = get_all_files_meta(archive)
.with_context(|| format!("listing {}", archive.display()))?
.into_iter()
.map(
|ArtifactMeta {
relative_path,
uncompressed_size,
..
}| { (relative_path, uncompressed_size) },
)
.collect();
m.sort();
Ok(m)
}
fn unique_sibling(archive: &Path, tag: &str) -> PathBuf {
let unique = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let mut p = archive.as_os_str().to_owned();
p.push(format!(".{tag}-{}-{unique}", std::process::id()));
PathBuf::from(p)
}
fn sync_dir(path: &Path) {
if let Some(parent) = path.parent()
&& let Ok(f) = std::fs::File::open(parent)
{
let _ = f.sync_all();
}
}
impl Gc for NewGeneration {
fn run(&self, archive: &Path) -> Result<GcReport> {
let bytes_before = std::fs::metadata(archive)
.with_context(|| format!("stat {}", archive.display()))?
.len();
let before = manifest(archive)?;
let target = next_generation(archive)?;
if target.exists() {
bail!(
"{} already exists — a previous GC left a generation behind, or two are running \
at once",
target.display()
);
}
let work = unique_sibling(archive, "gc");
std::fs::hard_link(archive, &work).with_context(|| {
format!(
"hard-linking {} to {} — a new generation is produced by compacting a second \
name for the same inode, which needs both on one filesystem",
archive.display(),
work.display()
)
})?;
let compacted = compact_archive(&work);
let CompactReport {
bytes_after,
rows,
delta_rows,
..
} = match compacted {
Ok(r) => r,
Err(e) => {
let _ = std::fs::remove_file(&work);
return Err(e.context(format!(
"compacting a new generation of {}",
archive.display()
)));
}
};
if self.stop_after == StopAfter::Compact {
bail!("interrupted after compact (test)");
}
if let Err(e) = read_back_every_entry(&work) {
let _ = std::fs::remove_file(&work);
return Err(e.context(format!(
"the new generation of {} did not verify — nothing was replaced",
archive.display()
)));
}
let after = manifest(&work)?;
if after != before {
let _ = std::fs::remove_file(&work);
bail!(
"the new generation of {} carries {} entries, the original {} — nothing was \
replaced",
archive.display(),
after.len(),
before.len()
);
}
if self.stop_after == StopAfter::Verify {
bail!("interrupted after verify (test)");
}
std::fs::rename(&work, &target)
.with_context(|| format!("naming the new generation {}", target.display()))?;
sync_dir(&target);
if self.stop_after == StopAfter::Rename {
bail!("interrupted after rename (test)");
}
std::fs::remove_file(archive).with_context(|| format!("retiring {}", archive.display()))?;
sync_dir(archive);
Ok(GcReport {
strategy: self.name(),
archive: target,
retired: Some(archive.to_path_buf()),
bytes_before,
bytes_after,
rows,
delta_rows,
verified: true,
retired_packs: 0,
})
}
fn name(&self) -> &'static str {
"NewGeneration"
}
}
#[cfg(test)]
mod tests {
use super::*;
use znippy_common::{
SupersedeOutcome, ZnippyArchive, ZnippyReader, create_archive, read_delta_map,
supersede_as_delta,
};
fn fixture(dir: &Path, name: &str) -> (PathBuf, Vec<Vec<u8>>) {
let archive = dir.join(name);
let mut st = 0x5151_2323_abcd_ef01u64;
let base: Vec<u8> = (0..600_000u32)
.map(|_| {
st = st.wrapping_add(0x9e37_79b9_7f4a_7c15);
let mut z = st;
z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
(z ^ (z >> 27)) as u8
})
.collect();
let mut gens: Vec<Vec<u8>> = vec![base];
for g in 1..4 {
let mut next = gens[g - 1].clone();
next.extend_from_slice(format!("generation {g} tail ").repeat(300).as_bytes());
gens.push(next);
}
let files: Vec<(String, Vec<u8>)> = gens
.iter()
.enumerate()
.map(|(i, b)| (format!("pack-{i}.pack"), b.clone()))
.collect();
create_archive(&archive, &files, 3).unwrap();
for i in (0..3).rev() {
let out = supersede_as_delta(
&archive,
&format!("pack-{i}.pack"),
&format!("pack-{}.pack", i + 1),
3 - 1 - i,
3,
)
.unwrap();
assert!(
matches!(out, SupersedeOutcome::Delta { .. }),
"gen {i}: {out:?}"
);
}
(archive, gens)
}
fn reads_back(archive: &Path, gens: &[Vec<u8>]) {
let ar = ZnippyArchive::open(archive)
.unwrap_or_else(|e| panic!("{} does not open: {e}", archive.display()));
for (i, want) in gens.iter().enumerate() {
assert_eq!(
&ar.extract_file(&format!("pack-{i}.pack")).unwrap(),
want,
"{} lost generation {i}",
archive.display()
);
}
}
#[test]
fn both_implementations_reclaim_the_dead_payload_and_change_no_entry() {
let dir = tempfile::tempdir().unwrap();
let raw = |gens: &[Vec<u8>]| gens.iter().map(|g| g.len() as u64).sum::<u64>();
for gc in [
&CompactInPlace::new() as &dyn Gc,
&NewGeneration::new() as &dyn Gc,
] {
let (archive, gens) = fixture(dir.path(), &format!("{}.znippy", gc.name()));
let before = std::fs::metadata(&archive).unwrap().len();
let report = gc.run(&archive).unwrap();
assert_eq!(report.strategy, gc.name());
assert_eq!(
report.rows,
4,
"{}: a GC must not change the row count",
gc.name()
);
assert_eq!(
report.delta_rows,
3,
"{}: the delta map must travel",
gc.name()
);
assert_eq!(report.bytes_before, before);
let on_disk = std::fs::metadata(&report.archive).unwrap().len();
assert_eq!(
report.bytes_after,
on_disk,
"{}: reported size is not the size",
gc.name()
);
assert!(
on_disk * 2 < raw(&gens),
"{}: the new archive is {on_disk} bytes for {} bytes of generations — the dead \
payload was not reclaimed",
gc.name(),
raw(&gens)
);
reads_back(&report.archive, &gens);
let map = read_delta_map(&report.archive).unwrap();
assert!(
map.iter().all(|(p, _, _)| p != "pack-3.pack"),
"{}: the live generation went behind a link: {map:?}",
gc.name()
);
}
}
#[test]
fn a_renames_over_the_original_and_b_produces_a_new_generation() {
let dir = tempfile::tempdir().unwrap();
let (a_path, gens) = fixture(dir.path(), "a.znippy");
let a = CompactInPlace::new().run(&a_path).unwrap();
assert_eq!(a.archive, a_path, "A moved the archive");
assert_eq!(a.retired, None);
assert!(a_path.exists(), "A removed the archive it compacted");
reads_back(&a_path, &gens);
let (b_path, gens) = fixture(dir.path(), "b.znippy");
let b = NewGeneration::new().run(&b_path).unwrap();
assert_eq!(b.archive, dir.path().join("b.g1.znippy"));
assert_eq!(b.retired, Some(b_path.clone()));
assert!(
!b_path.exists(),
"B kept the old generation after proving the new one"
);
assert!(b.archive.exists());
reads_back(&b.archive, &gens);
assert!(b.verified, "B must not report an unverified success");
let b2 = NewGeneration::new().run(&b.archive).unwrap();
assert_eq!(b2.archive, dir.path().join("b.g2.znippy"));
assert!(!b.archive.exists());
reads_back(&b2.archive, &gens);
}
#[test]
fn an_interruption_at_every_step_leaves_a_readable_archive() {
let dir = tempfile::tempdir().unwrap();
for (step, label) in [
(StopAfter::Compact, "compact"),
(StopAfter::Verify, "verify"),
(StopAfter::Rename, "rename"),
] {
let (archive, gens) = fixture(dir.path(), &format!("{label}.znippy"));
let target = next_generation(&archive).unwrap();
let err = NewGeneration::stopping_after(step)
.run(&archive)
.expect_err("the interruption did not stop the run");
assert!(err.to_string().contains(label), "wrong stop: {err}");
assert!(archive.exists(), "{label}: the original was unlinked early");
reads_back(&archive, &gens);
match step {
StopAfter::Rename => {
assert!(target.exists(), "the renamed generation is missing");
reads_back(&target, &gens);
}
_ => assert!(
!target.exists(),
"{label}: a generation took its permanent name before it was proven"
),
}
}
}
#[test]
fn an_existing_generation_is_never_overwritten() {
let dir = tempfile::tempdir().unwrap();
let (archive, gens) = fixture(dir.path(), "x.znippy");
let target = next_generation(&archive).unwrap();
std::fs::write(&target, b"not an archive").unwrap();
let err = NewGeneration::new()
.run(&archive)
.expect_err("overwrote a generation");
assert!(
err.to_string().contains("already exists"),
"wrong error: {err}"
);
assert_eq!(std::fs::read(&target).unwrap(), b"not an archive");
reads_back(&archive, &gens);
}
#[test]
fn retiring_to_holger_is_deliberately_empty() {
let target = HolgerTarget {
endpoint: "unspecified".into(),
repository: "nordisk/znippy".into(),
};
let outcome = std::panic::catch_unwind(|| {
let _ = retire_to_holger(Path::new("/nonexistent.g1.znippy"), &target);
});
let payload = outcome.expect_err(
"the holger retirement no longer panics — if it was implemented, delete this test in \
the same commit",
);
let msg = payload
.downcast_ref::<String>()
.cloned()
.or_else(|| payload.downcast_ref::<&str>().map(|s| s.to_string()))
.unwrap_or_default();
assert!(
msg.contains("deliberately empty"),
"it panics, but not as the placeholder: {msg}"
);
}
#[test]
fn generation_names_advance_in_the_stem() {
let n = |s: &str| {
next_generation(Path::new(s))
.unwrap()
.to_string_lossy()
.into_owned()
};
assert_eq!(n("/srv/repo.znippy"), "/srv/repo.g1.znippy");
assert_eq!(n("/srv/repo.g1.znippy"), "/srv/repo.g2.znippy");
assert_eq!(n("/srv/repo.g9.znippy"), "/srv/repo.g10.znippy");
assert_eq!(n("/srv/repo.git.znippy"), "/srv/repo.git.g1.znippy");
assert_eq!(n("/srv/repo"), "/srv/repo.g1");
}
}