use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{Result, anyhow};
use colored::Colorize;
use indicatif::{ProgressBar, ProgressStyle};
use tempfile::Builder;
use zbsdiff::zdelta::{self as delta};
use zoi_core::types::ZrepoLink;
use zoi_core::{cache, pgp, types, utils as core_utils};
use zoi_install::util as install_util;
use zync::{
AssemblyBuilder, ControlFile, GenerateOptions, Transport, TransportError
};
pub const ZREPO_HASH_META_KEY: &str = "zrepo_sha256";
const ZSYNC_SUFFIX: &str = ".zsync";
pub fn candidate_links(repo_config: &types::RepoConfig) -> Vec<&ZrepoLink> {
let (main_links, mirrors): (Vec<_>, Vec<_>) = repo_config
.zrepo
.iter()
.partition(|link| link.link_type == "main");
main_links.into_iter().chain(mirrors).collect()
}
pub fn control_url(link: &ZrepoLink) -> String {
match &link.control {
Some(control) if !control.is_empty() => control.clone(),
_ => format!("{}{ZSYNC_SUFFIX}", link.url.trim_end_matches('/'))
}
}
pub fn base_artifact_path(db_root: &Path, handle: &str) -> PathBuf {
db_root.join(format!("{handle}.zrepo"))
}
#[derive(Debug, Clone, Default)]
pub struct ZrepoStats {
pub file_count: usize,
pub tar_size: u64,
pub compressed_size: u64,
pub sha256: String
}
fn is_excluded_from_snapshot(relative: &Path) -> bool {
relative.components().any(|component| {
matches!(
component,
std::path::Component::Normal(name)
if name == ".git"
)
}) || relative
.file_name()
.is_some_and(|name| name.to_string_lossy().contains(".zrepo"))
}
fn snapshot_entries(registry_root: &Path) -> Vec<(String, PathBuf)> {
let mut entries: Vec<(String, PathBuf)> = Vec::new();
for entry in walkdir::WalkDir::new(registry_root)
.into_iter()
.filter_map(std::result::Result::ok)
{
if !entry.file_type().is_file() {
continue;
}
let Ok(relative) = entry.path().strip_prefix(registry_root) else {
continue;
};
if is_excluded_from_snapshot(relative) {
continue;
}
let name = relative.to_string_lossy().replace('\\', "/");
entries.push((name, entry.path().to_path_buf()));
}
entries.sort_by(|a, b| a.0.cmp(&b.0));
entries
}
fn encode_tar_entry(name: &str, path: &Path, data: &[u8]) -> Result<Vec<u8>> {
let mut header = tar::Header::new_gnu();
header.set_size(data.len() as u64);
header.set_mode(if is_executable(path) { 0o755 } else { 0o644 });
header.set_mtime(0);
header.set_uid(0);
header.set_gid(0);
header.set_path(name).map_err(|e| {
anyhow!("Failed to encode '{name}' into a tar header: {e}")
})?;
header.set_cksum();
let padding = (512 - (data.len() % 512)) % 512;
let mut entry = Vec::with_capacity(512 + data.len() + padding);
entry.extend_from_slice(header.as_bytes());
entry.extend_from_slice(data);
entry.resize(512 + data.len() + padding, 0);
Ok(entry)
}
fn build_snapshot_frames(
registry_root: &Path
) -> Result<(Vec<u8>, u64, usize)> {
let entries = snapshot_entries(registry_root);
let file_count = entries.len();
let mut frames = Vec::new();
let mut tar_size = 0u64;
for (name, path) in &entries {
let data = fs::read(path)
.map_err(|e| anyhow!("Failed to read '{}': {e}", path.display()))?;
let entry = encode_tar_entry(name, path, &data)?;
tar_size += entry.len() as u64;
let frame = delta::zstd_compress(&entry)
.map_err(|e| anyhow!("Failed to compress '{name}': {e}"))?;
frames.extend_from_slice(&frame);
}
Ok((frames, tar_size, file_count))
}
fn is_executable(path: &Path) -> bool {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::metadata(path).is_ok_and(|m| m.permissions().mode() & 0o111 != 0)
}
#[cfg(not(unix))]
{
let _ = path;
false
}
}
pub fn create_zrepo(
registry_root: &Path,
output: &Path,
sign_key: Option<&str>
) -> Result<ZrepoStats> {
if !registry_root.join("repo.yaml").is_file() {
return Err(anyhow!(
"'{}' has no repo.yaml, so it is not a registry and cannot be \
published as a snapshot.",
registry_root.display()
));
}
let (zrepo_bytes, tar_size, file_count) =
build_snapshot_frames(registry_root)?;
if let Some(parent) = output.parent()
&& !parent.as_os_str().is_empty()
{
fs::create_dir_all(parent)?;
}
fs::write(output, &zrepo_bytes)?;
let stats = ZrepoStats {
file_count,
tar_size,
compressed_size: zrepo_bytes.len() as u64,
sha256: sha256_bytes(&zrepo_bytes)
};
write_sidecars(output, &zrepo_bytes, sign_key)?;
write_control_file(output)?;
Ok(stats)
}
fn write_control_file(output: &Path) -> Result<()> {
let name = output.file_name().map_or_else(
|| "registry.zrepo".to_string(),
|n| n.to_string_lossy().to_string()
);
let control = zync::make(
output,
&GenerateOptions {
filename: Some(&name),
mtime: None,
urls: vec![name.clone()],
blocksize: None
}
)?;
let control_path = PathBuf::from(format!("{}.zsync", output.display()));
let mut file = fs::File::create(&control_path)?;
control.write(&mut file)?;
Ok(())
}
fn write_sidecars(
artifact: &Path,
bytes: &[u8],
sign_key: Option<&str>
) -> Result<()> {
let hash = sha256_bytes(bytes);
let hash_path = PathBuf::from(format!("{}.hash", artifact.display()));
fs::write(&hash_path, format!("{hash}\n"))?;
let size_path = PathBuf::from(format!("{}.size", artifact.display()));
fs::write(&size_path, format!("{}\n", bytes.len()))?;
if let Some(key_id) = sign_key {
let sig_path = PathBuf::from(format!("{}.sig", artifact.display()));
zoi_core::pgp::sign_detached(artifact, &sig_path, key_id)?;
}
Ok(())
}
fn sha256_bytes(data: &[u8]) -> String {
zoi_core::hash::calculate_reader_hash(
&mut Cursor::new(data),
zoi_core::hash::HashAlgorithm::Sha256
)
.map(|(_, digest)| digest)
.unwrap_or_default()
}
fn download(url: &str, dest: &Path, pb: Option<&ProgressBar>) -> Result<()> {
install_util::download_file_with_progress(url, dest, pb, None)
}
fn verify_signature(
link: &ZrepoLink,
artifact: &Path,
work_dir: &Path,
authorities: &[String],
pb: Option<&ProgressBar>,
verbose: bool
) -> Result<()> {
let Some(pgp_url) = &link.pgp else {
return Ok(());
};
if authorities.is_empty() {
if verbose {
println!(
"{}",
"No trusted PGP keys configured for this registry; skipping \
signature verification."
.yellow()
);
}
return Ok(());
}
let sig_path = work_dir.join("zrepo.sig");
if download(pgp_url, &sig_path, pb).is_err() {
if verbose {
println!(
"{} Signature file for the registry snapshot is unavailable.",
"Warning:".yellow()
);
}
return Ok(());
}
let certs = pgp::get_certs_by_name_or_fingerprint(authorities)?;
pgp::verify_detached_signature_multi_key(artifact, &sig_path, certs)?;
if verbose {
println!("{}", "Registry snapshot signature verified.".green());
}
Ok(())
}
fn verify_hash(
link: &ZrepoLink,
artifact: &Path,
pb: Option<&ProgressBar>
) -> Result<()> {
let Some(hash_url) = &link.hash else {
return Ok(());
};
let expected = install_util::get_expected_hash(hash_url, None)?;
if expected.is_empty() {
return Ok(());
}
if !install_util::verify_file_hash(artifact, &expected, pb)? {
return Err(anyhow!(
"Checksum verification failed for registry snapshot downloaded \
from {}",
link.url
));
}
Ok(())
}
fn extract_tar(tar_bytes: &[u8], dest: &Path) -> Result<()> {
fs::create_dir_all(dest)?;
let mut archive = tar::Archive::new(Cursor::new(tar_bytes));
archive.set_overwrite(true);
for entry in archive.entries()? {
let mut entry = entry?;
let relative = entry.path()?.into_owned();
let target = core_utils::safe_join(dest, &relative)?;
if entry.header().entry_type().is_dir() {
fs::create_dir_all(&target)?;
continue;
}
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)?;
}
let mut out = fs::File::create(&target)?;
std::io::copy(&mut entry, &mut out)?;
}
Ok(())
}
fn swap_into_place(staging_dir: &Path, target_dir: &Path) -> Result<()> {
let retired = target_dir.with_extension("zoi-old");
if retired.exists() {
let _ = fs::remove_dir_all(&retired);
}
let had_previous = target_dir.exists() || target_dir.is_symlink();
if had_previous {
if target_dir.is_symlink() {
fs::remove_file(target_dir)?;
} else {
fs::rename(target_dir, &retired)?;
}
}
if let Some(parent) = target_dir.parent() {
fs::create_dir_all(parent)?;
}
if let Err(e) = fs::rename(staging_dir, target_dir) {
if had_previous && retired.exists() {
let _ = fs::rename(&retired, target_dir);
}
return Err(anyhow!("Failed to install registry snapshot: {e}"));
}
if retired.exists() {
let _ = fs::remove_dir_all(&retired);
}
Ok(())
}
fn install_snapshot(
tar_bytes: &[u8],
zrepo_bytes: &[u8],
base_artifact: &Path,
target_dir: &Path,
work_dir: &Path
) -> Result<()> {
let staging_dir = work_dir.join("extract");
if staging_dir.exists() {
fs::remove_dir_all(&staging_dir)?;
}
extract_tar(tar_bytes, &staging_dir)?;
if !staging_dir.join("repo.yaml").exists() {
return Err(anyhow!(
"The registry snapshot does not contain a repo.yaml; it cannot be \
used to build a registry."
));
}
let staged_base = work_dir.join("staged.zrepo");
fs::write(&staged_base, zrepo_bytes)?;
if let Some(parent) = base_artifact.parent() {
fs::create_dir_all(parent)?;
}
fs::rename(&staged_base, base_artifact)?;
swap_into_place(&staging_dir, target_dir)
}
fn full_download(
link: &ZrepoLink,
base_artifact: &Path,
target_dir: &Path,
work_dir: &Path,
authorities: &[String],
pb: Option<&ProgressBar>,
verbose: bool
) -> Result<String> {
let download_path = work_dir.join("download.zrepo");
if download_path.exists() {
fs::remove_file(&download_path)?;
}
if let Some(p) = pb {
p.set_message("Downloading registry snapshot");
} else if verbose {
println!("Downloading registry snapshot from {}...", link.url.cyan());
}
download(&link.url, &download_path, pb)?;
verify_hash(link, &download_path, pb)?;
verify_signature(link, &download_path, work_dir, authorities, pb, verbose)?;
let zrepo_bytes = fs::read(&download_path)?;
let tar_bytes = delta::zstd_decompress(&zrepo_bytes).map_err(|e| {
anyhow!("Downloaded registry snapshot is not a valid zstd frame: {e}")
})?;
let hash = sha256_bytes(&zrepo_bytes);
install_snapshot(
&tar_bytes,
&zrepo_bytes,
base_artifact,
target_dir,
work_dir
)?;
Ok(hash)
}
pub fn sync_zrepo(
links: &[&ZrepoLink],
db_root: &Path,
handle: &str,
target_dir: &Path,
authorities: &[String],
pb: Option<&ProgressBar>,
verbose: bool
) -> Result<String> {
let base_artifact = base_artifact_path(db_root, handle);
let work_dir = Builder::new().prefix("zoi-zrepo").tempdir()?;
let work_path = work_dir.path().to_path_buf();
let mut last_error: Option<anyhow::Error> = None;
for link in links {
match try_link(
link,
&base_artifact,
target_dir,
&work_path,
authorities,
pb,
verbose
) {
Ok(hash) => return Ok(hash),
Err(e) => {
let msg =
format!("Sync with {} failed: {e}", link.url.yellow());
if let Some(p) = pb {
p.println(&msg);
} else {
eprintln!("{msg}");
}
last_error = Some(e);
}
}
}
Err(last_error
.unwrap_or_else(|| anyhow!("No .zrepo links are configured.")))
}
enum TransferUnavailable {
NotFetched(anyhow::Error),
Unusable(anyhow::Error)
}
struct ZoiTransport {
client: &'static reqwest::blocking::Client
}
impl Transport for ZoiTransport {
fn get(
&self,
url: &str
) -> Result<Box<dyn std::io::Read + Send>, TransportError> {
let response = self.client.get(url).send().map_err(|e| {
TransportError::Request {
url: url.to_string(),
reason: e.to_string()
}
})?;
Ok(Box::new(response))
}
fn get_range(
&self,
url: &str,
range: std::ops::Range<u64>
) -> Result<Box<dyn std::io::Read + Send>, TransportError> {
let last = range.end.saturating_sub(1).max(range.start);
let response = self
.client
.get(url)
.header("Range", format!("bytes={}-{}", range.start, last))
.send()
.map_err(|e| TransportError::Request {
url: url.to_string(),
reason: e.to_string()
})?;
let status = response.status().as_u16();
if status != 206 {
return Err(TransportError::UnexpectedStatus {
url: url.to_string(),
status
});
}
let wanted = range.end.saturating_sub(range.start);
Ok(Box::new(std::io::Read::take(response, wanted)))
}
}
fn resolve_snapshot_url(control_url: &str, target: &str) -> String {
if target.contains("://") {
return target.to_string();
}
let Some(cut) = control_url.rfind('/') else {
return target.to_string();
};
format!("{}{}", &control_url[..=cut], target)
}
fn zync_transfer(
link: &ZrepoLink,
base_artifact: &Path,
target_dir: &Path,
work_dir: &Path,
authorities: &[String],
pb: Option<&ProgressBar>,
verbose: bool
) -> Result<String, TransferUnavailable> {
let control_url = control_url(link);
let control_path = work_dir.join("control.zsync");
let unusable = |e: anyhow::Error| TransferUnavailable::Unusable(e);
download(&control_url, &control_path, pb)
.map_err(TransferUnavailable::NotFetched)?;
let control_bytes = fs::read(&control_path).map_err(|e| {
unusable(anyhow!("Failed to read the control file: {e}"))
})?;
let mut control = ControlFile::parse(Cursor::new(control_bytes))
.map_err(|e| unusable(anyhow!("Malformed control file: {e}")))?;
control.urls = control
.urls
.iter()
.flat_map(|url| {
cache::mirror_candidate_urls(&resolve_snapshot_url(
&control_url,
url
))
})
.collect();
if control.urls.is_empty() {
return Err(unusable(anyhow!(
"The control file names no snapshot to fetch."
)));
}
if let Some(p) = pb {
p.set_message("Updating registry snapshot");
} else if verbose {
println!("Updating registry snapshot from {control_url}...");
}
let blocksize = control.blocksize;
let progress_bar = pb.cloned();
let output = work_dir.join("assembled.zrepo");
let transport = ZoiTransport {
client: core_utils::get_http_client().map_err(unusable)?
};
let _ = &transport;
let mut assembly = AssemblyBuilder::new(control, &output)
.transport(Arc::new(transport))
.build()
.map_err(|e| unusable(anyhow!("Failed to start the transfer: {e}")))?;
assembly.set_progress(move |done: usize, total: usize| {
if let Some(p) = progress_bar.as_ref() {
p.set_length((total * blocksize) as u64);
p.set_position((done * blocksize) as u64);
}
});
if base_artifact.exists()
&& let Err(e) = assembly.submit_file(base_artifact)
&& verbose
{
println!(
"{}",
format!(
"Could not reuse the cached snapshot ({e}); fetching the \
whole registry."
)
.dimmed()
);
}
assembly
.fetch()
.map_err(|e| unusable(anyhow!("Snapshot transfer failed: {e}")))?;
assembly
.finish()
.map_err(|e| unusable(anyhow!("Failed to write the snapshot: {e}")))?;
verify_hash(link, &output, pb).map_err(unusable)?;
verify_signature(link, &output, work_dir, authorities, pb, verbose)
.map_err(unusable)?;
let zrepo_bytes = fs::read(&output).map_err(|e| unusable(e.into()))?;
let tar_bytes = delta::zstd_decompress(&zrepo_bytes).map_err(|e| {
unusable(anyhow!(
"Transferred registry snapshot is not a valid zstd stream: {e}"
))
})?;
let hash = sha256_bytes(&zrepo_bytes);
install_snapshot(
&tar_bytes,
&zrepo_bytes,
base_artifact,
target_dir,
work_dir
)
.map_err(unusable)?;
if verbose {
println!("{}", "Registry snapshot updated incrementally.".green());
}
Ok(hash)
}
fn try_link(
link: &ZrepoLink,
base_artifact: &Path,
target_dir: &Path,
work_dir: &Path,
authorities: &[String],
pb: Option<&ProgressBar>,
verbose: bool
) -> Result<String> {
match zync_transfer(
link,
base_artifact,
target_dir,
work_dir,
authorities,
pb,
verbose
) {
Ok(hash) => return Ok(hash),
Err(reason) => {
let declared = link.control.is_some();
let control_url = control_url(link);
let msg = match reason {
TransferUnavailable::NotFetched(e) if declared => {
Some(format!(
"{} The registry declares a control file at \
{control_url} but it could not be fetched ({e}). \
Falling back to a full download.",
"Warning:".yellow(),
))
}
TransferUnavailable::NotFetched(_) => {
if verbose {
println!(
"{}",
format!(
"No registry control file at {control_url}; \
downloading the full snapshot."
)
.dimmed()
);
}
None
}
TransferUnavailable::Unusable(e) => Some(format!(
"{} The registry control file could not be used ({}). \
Falling back to a full download.",
"Warning:".yellow(),
e
))
};
if let Some(msg) = msg {
if let Some(p) = pb {
p.println(&msg);
} else {
eprintln!("{msg}");
}
}
}
}
full_download(
link,
base_artifact,
target_dir,
work_dir,
authorities,
pb,
verbose
)
}
pub fn snapshot_progress_style() -> Result<ProgressStyle> {
Ok(ProgressStyle::default_bar()
.template(
"{spinner:.green} {msg:30.cyan.bold} [{bar:40.cyan/blue}] \
{bytes}/{total_bytes} ({bytes_per_sec}, {elapsed_precise})"
)?
.progress_chars("=>-"))
}
#[cfg(test)]
mod tests {
use super::*;
fn build_tar(entries: &[(&str, &str)]) -> Vec<u8> {
let mut builder = tar::Builder::new(Vec::new());
for (name, body) in entries {
let mut header = tar::Header::new_gnu();
header.set_size(body.len() as u64);
header.set_mode(0o644);
header.set_cksum();
builder
.append_data(&mut header, name, Cursor::new(body.as_bytes()))
.expect("tar append should succeed");
}
builder.into_inner().expect("tar build should succeed")
}
fn build_raw_tar(name: &str, body: &str) -> Vec<u8> {
let mut header = [0u8; 512];
let name_bytes = name.as_bytes();
assert!(
name_bytes.len() < 100,
"test helper only handles short names"
);
if let Some(slot) = header.get_mut(..name_bytes.len()) {
slot.copy_from_slice(name_bytes);
}
let octal =
|header: &mut [u8; 512], at: usize, len: usize, value: u64| {
let text = format!("{value:0width$o}", width = len - 1);
if let Some(slot) = header.get_mut(at..at + len - 1) {
slot.copy_from_slice(text.as_bytes());
}
};
octal(&mut header, 100, 8, 0o644);
octal(&mut header, 108, 8, 0);
octal(&mut header, 116, 8, 0);
octal(&mut header, 124, 12, body.len() as u64);
octal(&mut header, 136, 12, 0);
header[156] = b'0'; if let Some(slot) = header.get_mut(257..263) {
slot.copy_from_slice(b"ustar\0");
}
if let Some(slot) = header.get_mut(263..265) {
slot.copy_from_slice(b"00");
}
if let Some(slot) = header.get_mut(148..156) {
slot.fill(b' ');
}
let sum: u32 = header.iter().map(|b| u32::from(*b)).sum();
let checksum = format!("{sum:06o}\0 ");
if let Some(slot) = header.get_mut(148..156) {
slot.copy_from_slice(checksum.as_bytes());
}
let mut archive = header.to_vec();
archive.extend_from_slice(body.as_bytes());
let padding = (512 - body.len() % 512) % 512;
archive.extend(std::iter::repeat_n(0u8, padding));
archive.extend(std::iter::repeat_n(0u8, 1024)); archive
}
#[test]
fn extract_tar_refuses_to_escape_the_destination() {
let dir = tempfile::tempdir().expect("tempdir");
let tar_bytes = build_raw_tar("../escaped.txt", "pwned");
let err = extract_tar(&tar_bytes, dir.path())
.expect_err("a traversal entry must be rejected");
assert!(
err.to_string().contains("escapes its base directory"),
"unexpected error: {err}"
);
assert!(
!dir.path()
.parent()
.expect("tempdir has a parent")
.join("escaped.txt")
.exists(),
"nothing may be written outside the destination"
);
}
#[test]
fn extract_tar_refuses_absolute_paths() {
let dir = tempfile::tempdir().expect("tempdir");
let tar_bytes = build_raw_tar("/zoi-should-not-exist", "pwned");
extract_tar(&tar_bytes, dir.path())
.expect_err("an absolute entry must be rejected");
}
#[test]
fn extract_tar_refuses_a_deeply_nested_traversal() {
let dir = tempfile::tempdir().expect("tempdir");
let tar_bytes = build_raw_tar("a/../../escaped.txt", "pwned");
extract_tar(&tar_bytes, dir.path())
.expect_err("a nested traversal must be rejected");
}
#[test]
fn extract_tar_writes_nested_files() {
let dir = tempfile::tempdir().expect("tempdir");
let tar_bytes = build_tar(&[
("repo.yaml", "name: testreg\n"),
("main/ripgrep/ripgrep.pkg.lua", "metadata({})\n")
]);
extract_tar(&tar_bytes, dir.path()).expect("extraction should succeed");
assert_eq!(
fs::read_to_string(dir.path().join("repo.yaml"))
.expect("repo.yaml read"),
"name: testreg\n"
);
assert_eq!(
fs::read_to_string(dir.path().join("main/ripgrep/ripgrep.pkg.lua"))
.expect("pkg.lua read"),
"metadata({})\n"
);
}
#[test]
fn swap_into_place_replaces_an_existing_tree() {
let dir = tempfile::tempdir().expect("tempdir");
let staging = dir.path().join("staging");
let target = dir.path().join("registry");
fs::create_dir_all(&staging).expect("create staging");
fs::write(staging.join("repo.yaml"), "name: new\n")
.expect("write staging");
fs::create_dir_all(&target).expect("create target");
fs::write(target.join("repo.yaml"), "name: old\n").expect("write old");
fs::write(target.join("stale.txt"), "stale").expect("write stale");
swap_into_place(&staging, &target).expect("swap should succeed");
assert_eq!(
fs::read_to_string(target.join("repo.yaml")).expect("read"),
"name: new\n"
);
assert!(
!target.join("stale.txt").exists(),
"files from the previous snapshot must not survive the swap"
);
assert!(
!target.with_extension("zoi-old").exists(),
"the retired tree must be cleaned up"
);
}
#[test]
fn swap_into_place_replaces_a_local_registry_symlink() {
let dir = tempfile::tempdir().expect("tempdir");
let staging = dir.path().join("staging");
let target = dir.path().join("registry");
let local = dir.path().join("local-registry");
fs::create_dir_all(&staging).expect("create staging");
fs::write(staging.join("repo.yaml"), "name: new\n")
.expect("write staging");
fs::create_dir_all(&local).expect("create local");
fs::write(local.join("keep.txt"), "keep").expect("write local");
core_utils::symlink_dir(&local, &target).expect("create symlink");
swap_into_place(&staging, &target).expect("swap should succeed");
assert!(target.join("repo.yaml").exists(), "new tree installed");
assert!(
!target.is_symlink(),
"the symlink must be replaced by a real directory"
);
assert!(
local.join("keep.txt").exists(),
"a local registry must never be deleted through its symlink"
);
}
#[test]
fn install_snapshot_rejects_a_tree_without_repo_yaml() {
let dir = tempfile::tempdir().expect("tempdir");
let tar_bytes = build_tar(&[("packages.json", "{}")]);
let zrepo_bytes = delta::zstd_compress(&tar_bytes).expect("compress");
let err = install_snapshot(
&tar_bytes,
&zrepo_bytes,
&dir.path().join("testreg.zrepo"),
&dir.path().join("registry"),
dir.path()
)
.expect_err("a snapshot without repo.yaml must be rejected");
assert!(
err.to_string().contains("does not contain a repo.yaml"),
"unexpected error: {err}"
);
}
fn make_registry_tree(root: &Path, version: &str) {
fs::create_dir_all(root.join("main/ripgrep")).expect("create tree");
fs::write(
root.join("repo.yaml"),
"version: \"2\"\nname: testreg\ndescription: test\ngit: \
[]\nrepos: []\n"
)
.expect("write repo.yaml");
fs::write(
root.join("packages.json"),
"{\"version\":\"2\",\"packages\":{}}"
)
.expect("write packages.json");
fs::write(
root.join("main/ripgrep/ripgrep.pkg.lua"),
format!("metadata({{version=\"{version}\"}})\n")
)
.expect("write pkg.lua");
}
#[test]
fn create_zrepo_is_reproducible_for_an_unchanged_tree() {
let dir = tempfile::tempdir().expect("tempdir");
let tree = dir.path().join("registry");
make_registry_tree(&tree, "14.1.0");
let first = dir.path().join("first.zrepo");
let second = dir.path().join("second.zrepo");
let a = create_zrepo(&tree, &first, None).expect("first snapshot");
let b = create_zrepo(&tree, &second, None).expect("second snapshot");
assert_eq!(
fs::read(&first).expect("read first"),
fs::read(&second).expect("read second"),
"snapshotting the same tree twice must produce identical bytes"
);
assert_eq!(a.sha256, b.sha256);
assert_eq!(a.file_count, 3, "repo.yaml, packages.json, one pkg.lua");
}
#[test]
fn create_zrepo_ignores_git_plumbing_and_prior_snapshots() {
let dir = tempfile::tempdir().expect("tempdir");
let tree = dir.path().join("registry");
make_registry_tree(&tree, "14.1.0");
fs::create_dir_all(tree.join(".git/objects")).expect("create .git");
fs::write(tree.join(".git/HEAD"), "ref: refs/heads/main\n")
.expect("write HEAD");
fs::write(tree.join(".git/config"), "[core]\n").expect("write config");
fs::write(tree.join("testreg.zrepo"), "stale snapshot\n")
.expect("write stale snapshot");
let out = dir.path().join("out.zrepo");
let stats = create_zrepo(&tree, &out, None).expect("snapshot");
assert_eq!(
stats.file_count, 3,
"only the three registry files should be included"
);
let tar_bytes = delta::zstd_decompress(&fs::read(&out).expect("read"))
.expect("zstd");
let raw = String::from_utf8_lossy(&tar_bytes);
assert!(!raw.contains(".git"), "git plumbing must not be published");
assert!(
!raw.contains("stale snapshot"),
"a prior snapshot must not be published"
);
}
#[test]
fn create_zrepo_writes_hash_and_size_sidecars() {
let dir = tempfile::tempdir().expect("tempdir");
let tree = dir.path().join("registry");
make_registry_tree(&tree, "14.1.0");
let out = dir.path().join("testreg.zrepo");
let stats = create_zrepo(&tree, &out, None).expect("snapshot");
let hash = fs::read_to_string(dir.path().join("testreg.zrepo.hash"))
.expect("hash sidecar");
assert_eq!(hash.trim(), stats.sha256);
let size = fs::read_to_string(dir.path().join("testreg.zrepo.size"))
.expect("size sidecar");
assert_eq!(
size.trim().parse::<u64>().expect("parse size"),
stats.compressed_size
);
}
#[test]
fn create_zrepo_rejects_a_directory_that_is_not_a_registry() {
let dir = tempfile::tempdir().expect("tempdir");
let tree = dir.path().join("not-a-registry");
fs::create_dir_all(&tree).expect("create dir");
fs::write(tree.join("readme.md"), "hello\n").expect("write");
let err = create_zrepo(&tree, &dir.path().join("out.zrepo"), None)
.expect_err("a directory without repo.yaml must be rejected");
assert!(
err.to_string().contains("no repo.yaml"),
"unexpected error: {err}"
);
}
#[test]
fn published_artifacts_are_not_republished_into_the_next_snapshot() {
let dir = tempfile::tempdir().expect("tempdir");
let tree = dir.path().join("registry");
make_registry_tree(&tree, "14.1.0");
create_zrepo(&tree, &tree.join("testreg.zrepo"), None)
.expect("first publish");
assert!(
tree.join("testreg.zrepo.zsync").exists(),
"the control file should have been written beside the snapshot"
);
let second = dir.path().join("second.zrepo");
create_zrepo(&tree, &second, None).expect("second publish");
let bytes = fs::read(&second).expect("read");
let tar = delta::zstd_decompress(&bytes).expect("decompress");
let mut archive = tar::Archive::new(Cursor::new(tar));
let names: Vec<String> = archive
.entries()
.expect("entries should read")
.map(|entry| {
entry
.expect("entry")
.path()
.expect("path")
.to_string_lossy()
.to_string()
})
.collect();
for leaked in [
"testreg.zrepo",
"testreg.zrepo.hash",
"testreg.zrepo.size",
"testreg.zrepo.zsync"
] {
assert!(
!names.iter().any(|name| name == leaked),
"'{leaked}' leaked into the next snapshot: {names:?}"
);
}
assert!(names.contains(&"repo.yaml".to_string()), "{names:?}");
}
#[test]
fn every_registry_file_becomes_its_own_compressed_frame() {
let dir = tempfile::tempdir().expect("tempdir");
let tree = dir.path().join("registry");
make_registry_tree(&tree, "14.1.0");
let snapshot = dir.path().join("main.zrepo");
create_zrepo(&tree, &snapshot, None).expect("snapshot should build");
let bytes = fs::read(&snapshot).expect("read snapshot");
for name in ["repo.yaml", "packages.json"] {
let path = tree.join(name);
let data = fs::read(&path).expect("read entry");
let entry = encode_tar_entry(name, &path, &data).expect("encode");
let frame = delta::zstd_compress(&entry).expect("compress");
assert!(
contains(&bytes, &frame),
"'{name}' is not framed independently in the snapshot"
);
}
let tar = delta::zstd_decompress(&bytes).expect("valid zstd stream");
let mut archive = tar::Archive::new(Cursor::new(tar));
let names: Vec<String> = archive
.entries()
.expect("entries should read")
.map(|entry| {
entry
.expect("entry")
.path()
.expect("path")
.to_string_lossy()
.to_string()
})
.collect();
assert!(
names.contains(&"repo.yaml".to_string()),
"repo.yaml missing from {names:?}"
);
assert!(
names.iter().any(|n| n.ends_with("ripgrep.pkg.lua")),
"no package definitions in {names:?}"
);
}
#[test]
fn an_unchanged_file_produces_an_identical_frame_after_an_edit() {
let dir = tempfile::tempdir().expect("tempdir");
let tree = dir.path().join("registry");
make_registry_tree(&tree, "14.1.0");
let first = dir.path().join("v1.zrepo");
create_zrepo(&tree, &first, None).expect("first snapshot");
let first_bytes = fs::read(&first).expect("read first");
make_registry_tree(&tree, "14.1.1");
let second = dir.path().join("v2.zrepo");
create_zrepo(&tree, &second, None).expect("second snapshot");
let second_bytes = fs::read(&second).expect("read second");
assert_ne!(
first_bytes, second_bytes,
"the edit should have changed something"
);
for name in ["repo.yaml", "packages.json"] {
let path = tree.join(name);
let data = fs::read(&path).expect("read entry");
let entry = encode_tar_entry(name, &path, &data).expect("encode");
let frame = delta::zstd_compress(&entry).expect("compress");
assert!(
contains(&first_bytes, &frame),
"'{name}' missing from the first snapshot"
);
assert!(
contains(&second_bytes, &frame),
"'{name}' was re-framed by an unrelated edit"
);
}
}
fn contains(haystack: &[u8], needle: &[u8]) -> bool {
!needle.is_empty()
&& haystack
.windows(needle.len())
.any(|window| window == needle)
}
#[test]
fn create_zrepo_writes_a_zsync_control_file() {
let dir = tempfile::tempdir().expect("tempdir");
let tree = dir.path().join("registry");
make_registry_tree(&tree, "14.1.0");
let snapshot = dir.path().join("main.zrepo");
create_zrepo(&tree, &snapshot, None).expect("snapshot should build");
let control_path = dir.path().join("main.zrepo.zsync");
assert!(
control_path.exists(),
"a control file must be published beside the snapshot"
);
let control = ControlFile::parse(
fs::File::open(&control_path).expect("open control file")
)
.expect("control file should parse");
assert_eq!(
control.length,
fs::metadata(&snapshot).expect("stat").len()
);
assert_eq!(control.urls, vec!["main.zrepo".to_string()]);
}
#[test]
fn a_relative_control_url_resolves_against_the_control_file() {
assert_eq!(
resolve_snapshot_url(
"https://example.com/reg/main.zrepo.zsync",
"main.zrepo"
),
"https://example.com/reg/main.zrepo"
);
assert_eq!(
resolve_snapshot_url(
"https://example.com/reg/main.zrepo.zsync",
"https://cdn.example.net/main.zrepo"
),
"https://cdn.example.net/main.zrepo"
);
}
#[test]
fn the_control_url_defaults_to_the_conventional_suffix() {
let main = ZrepoLink {
link_type: "main".to_string(),
url: "https://example.com/reg/main.zrepo".to_string(),
control: None,
pgp: None,
hash: None,
size: None
};
assert_eq!(
control_url(&main),
"https://example.com/reg/main.zrepo.zsync"
);
let explicit = ZrepoLink {
control: Some("https://example.com/reg/index.zsync".to_string()),
..main
};
assert_eq!(
control_url(&explicit),
"https://example.com/reg/index.zsync"
);
}
}