use super::*;
use std::assert_matches;
#[test]
fn entry_candidates_resample_the_registry_for_each_entry() {
let registry: Vec<NodeCandidate> = (1u8..=7).map(|b| stripe_candidate(b, b)).collect();
let mut seen = HashSet::new();
let mut draws = HashSet::new();
for _ in 0..200 {
let sample = entry_candidates(®istry, None);
assert_eq!(sample.len(), discovery::SELECT_K);
let ids: std::collections::BTreeSet<PublicKey> = sample.iter().map(|c| c.node_id).collect();
seen.extend(ids.iter().copied());
draws.insert(ids);
}
assert_eq!(seen.len(), registry.len(), "every node reaches some entry");
assert!(draws.len() > 1, "entries draw different samples");
}
fn store_with_fresh_identities(dir: &Path, n: u8, now: u64) -> anyhow::Result<()> {
let store = decdn_client::PeerStore::open(dir);
for b in 1..=n {
store.upsert_identity(&stripe_candidate(b, b), now)?;
}
Ok(())
}
#[test]
fn cached_registry_needs_enough_fresh_identities() -> anyhow::Result<()> {
let cfg = decdn_client::StoreConfig::default();
let min = u8::try_from(cfg.min_fresh_candidates)?;
let now = 1_000_000;
let enough = tempfile::tempdir()?;
store_with_fresh_identities(enough.path(), min, now)?;
let cached = cached_registry(enough.path(), now).map(|c| c.len());
assert_eq!(cached, Some(cfg.min_fresh_candidates));
let few = tempfile::tempdir()?;
store_with_fresh_identities(few.path(), min - 1, now)?;
assert!(cached_registry(few.path(), now).is_none());
let later = now + cfg.identity_refresh_secs + 1;
assert!(cached_registry(enough.path(), later).is_none());
Ok(())
}
fn chain_against(
rpc: &std::net::TcpListener,
data_dir: &Path,
extra: &[&str],
) -> anyhow::Result<(ClientFetchArgs, fetch::ResolvedChain)> {
use clap::Parser as _;
#[derive(clap::Parser)]
struct T {
#[command(flatten)]
args: BundlePullArgs,
}
let rpc_url = format!("http://{}", rpc.local_addr()?);
let data_dir = data_dir.display().to_string();
let base = [
"t",
"-o",
"out",
"--hash",
"b3:aa",
"--rpc-url",
&rpc_url,
"--payment-pool-address",
"0x3333333333333333333333333333333333333333",
"--slash-judge-address",
"0x4444444444444444444444444444444444444444",
"--capacity-bond-address",
"0x5555555555555555555555555555555555555555",
"--data-dir",
&data_dir,
"--timeout-ms",
"200",
];
let common = T::try_parse_from(base.iter().chain(extra))?.args.common;
let chain = fetch::resolve_chain(&common, &decdn_common::config::FileConfig::default())?;
Ok((common, chain))
}
#[tokio::test]
async fn discover_candidates_skips_the_registry_read_on_fresh_identities() -> anyhow::Result<()> {
let min = u8::try_from(decdn_client::StoreConfig::default().min_fresh_candidates)?;
let dir = tempfile::tempdir()?;
store_with_fresh_identities(dir.path(), min, fetch::now_secs_cli())?;
let rpc = std::net::TcpListener::bind("127.0.0.1:0")?;
rpc.set_nonblocking(true)?;
let (common, chain) = chain_against(&rpc, dir.path(), &[])?;
let found = discover_candidates(&chain, &common).await?;
assert_eq!(found.len(), usize::from(min));
assert_matches!(
rpc.accept(),
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock,
"a cached run must not reach the RPC endpoint"
);
let (common, chain) = chain_against(&rpc, dir.path(), &["--rediscover"])?;
drop(discover_candidates(&chain, &common).await);
assert!(rpc.accept().is_ok(), "--rediscover reads the registry");
Ok(())
}
#[test]
fn a_blob_no_holder_has_fails_only_its_entry() {
let err = anyhow::Error::new(decdn_client::NoSourceHasBlob);
assert_eq!(entry_scope(&err), decdn_client::FatalScope::Item);
assert!(!ends_the_pull(&err));
}
#[test]
fn only_funding_needed_ends_the_whole_pull() {
let rejected = anyhow::Error::new(decdn_client::UpstreamVoucherRejected {
reason: decdn_protocol::client::VoucherRejectReason::SpendingCapExhausted,
bundle: None,
proof_generation: None,
})
.context("entry");
assert!(!ends_the_pull(&rejected));
let funding = anyhow::Error::new(decdn_client::NoAffordableSource {
deposit: alloy::primitives::U256::ZERO,
})
.context("entry");
assert_eq!(entry_scope(&funding), decdn_client::FatalScope::Command);
assert!(ends_the_pull(&funding));
}
#[test]
fn every_node_declining_fails_only_its_entry() {
let err =
anyhow::Error::new(decdn_client::NoNodeWillServe { reasons: vec![] }).context("entry");
assert_eq!(entry_scope(&err), decdn_client::FatalScope::Item);
assert!(!ends_the_pull(&err));
}
#[test]
fn an_over_cap_blob_fails_only_its_entry() {
let err = anyhow::Error::new(decdn_client::BlobTooLarge {
reached: 2,
ceiling: 1,
});
assert_eq!(entry_scope(&err), decdn_client::FatalScope::Item);
assert!(!ends_the_pull(&err));
}
#[test]
fn a_give_up_ends_the_whole_pull() {
let err = anyhow::Error::new(decdn_client::GaveUp {
idle: std::time::Duration::from_secs(1),
});
assert_eq!(entry_scope(&err), decdn_client::FatalScope::Item);
assert!(ends_the_pull(&err));
assert!(!ends_the_pull(&anyhow!("stream reset")));
}
#[test]
fn entry_retries_flag_is_gone() {
use clap::Parser as _;
#[derive(clap::Parser)]
struct T {
#[command(flatten)]
args: BundlePullArgs,
}
let base = ["t", "-o", "out", "--hash", "b3:aa"];
assert!(T::try_parse_from(base).is_ok());
let parsed = T::try_parse_from(base.iter().copied().chain(["--entry-retries", "2"]));
assert!(parsed.is_err());
}
#[tokio::test]
async fn a_command_fault_stops_every_later_entry() {
let started = std::sync::Mutex::new(Vec::new());
let mut settled = Vec::new();
let stopped = run_groups(
vec![0u8, 1, 2, 3],
1,
|i| {
started.lock().unwrap().push(i);
async move {
if i == 1 {
Err(anyhow::Error::new(decdn_client::NoAffordableSource {
deposit: alloy::primitives::U256::ZERO,
}))
} else {
Ok(())
}
}
},
|i, r: anyhow::Result<()>| {
settled.push(i);
r.err().filter(ends_the_pull)
},
)
.await;
let err = stopped.expect("the pull stops");
assert!(
err.downcast_ref::<decdn_client::NoAffordableSource>()
.is_some()
);
assert_eq!(*started.lock().unwrap(), vec![0, 1]);
assert_eq!(settled, vec![0, 1]);
}
#[tokio::test]
async fn a_donor_waiter_frees_its_slot_for_the_next_group() {
let gate = JobGate::new(1);
let order = std::sync::Mutex::new(Vec::new());
let donor_ready = tokio::sync::Notify::new();
let run = |i: u8| {
let (gate, order, donor_ready) = (&gate, &order, &donor_ready);
async move {
let slot = FetchSlot::acquire(gate).await?;
order.lock().unwrap().push(format!("{i}"));
if i == 0 {
let ready = donor_ready.notified();
slot.release();
ready.await;
slot.ensure().await?;
order.lock().unwrap().push("0 again".to_owned());
}
if i == 1 {
donor_ready.notify_one();
}
anyhow::Ok(())
}
};
let stopped = tokio::time::timeout(
std::time::Duration::from_secs(10),
run_groups(vec![0u8, 1, 2, 3], 4, run, |_, r| r.err()),
)
.await
.expect("a waiter that kept its slot would starve group 1");
assert!(stopped.is_none());
let order = order.lock().unwrap().clone();
assert_eq!(order.len(), 5);
assert_eq!(order.first().map(String::as_str), Some("0"));
assert_eq!(
order.get(1).map(String::as_str),
Some("1"),
"group 1 starts while group 0 waits"
);
}
#[tokio::test]
async fn a_retaken_slot_waits_behind_at_most_one_new_group() {
use std::task::Poll;
use futures_util::poll;
let gate = JobGate::new(1);
let waiter = FetchSlot::acquire(&gate).await.expect("first slot");
let mut b = std::pin::pin!(FetchSlot::acquire(&gate));
let mut c = std::pin::pin!(FetchSlot::acquire(&gate));
assert!(poll!(b.as_mut()).is_pending(), "b queues for the permit");
assert!(poll!(c.as_mut()).is_pending(), "c queues for admission");
waiter.release();
let Poll::Ready(Ok(b_slot)) = poll!(b.as_mut()) else {
panic!("b takes the released permit");
};
assert!(
poll!(c.as_mut()).is_pending(),
"c is admitted, queued for the permit"
);
let mut d = std::pin::pin!(FetchSlot::acquire(&gate));
assert!(poll!(d.as_mut()).is_pending(), "d queues for admission");
let mut retake = std::pin::pin!(waiter.ensure());
assert!(poll!(retake.as_mut()).is_pending());
drop(b_slot);
let Poll::Ready(Ok(c_slot)) = poll!(c.as_mut()) else {
panic!("c, admitted first, takes the next permit");
};
assert!(poll!(d.as_mut()).is_pending());
drop(c_slot);
assert_matches!(
poll!(retake.as_mut()),
Poll::Ready(Ok(())),
"the retake comes before d"
);
assert!(poll!(d.as_mut()).is_pending());
assert!(waiter.held());
}
#[tokio::test]
async fn a_command_fault_settles_the_items_it_drops() {
let settled = std::sync::atomic::AtomicU32::new(0);
let stopped = run_groups(
vec![0u8, 1],
2,
|i| {
let settled = &settled;
async move {
if i == 1 {
return Err(anyhow::Error::new(decdn_client::NoAffordableSource {
deposit: alloy::primitives::U256::ZERO,
}));
}
let on_drop = fetch::SettleOnDrop::new(|| {
settled.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
});
std::future::pending::<()>().await;
on_drop.disarm();
Ok(())
}
},
|_, r: anyhow::Result<()>| r.err().filter(ends_the_pull),
)
.await;
assert!(stopped.is_some(), "the sibling's fault ends the run");
assert_eq!(
settled.load(std::sync::atomic::Ordering::SeqCst),
1,
"the dropped item settled on drop"
);
}
#[tokio::test]
async fn an_item_fault_fails_only_its_entry() {
let mut failed = Vec::new();
let stopped = run_groups(
vec![0u8, 1, 2],
2,
|i| async move {
if i == 1 {
Err(anyhow::Error::new(decdn_client::BlobTooLarge {
reached: 2,
ceiling: 1,
}))
} else {
Ok(())
}
},
|i, r: anyhow::Result<()>| match r {
Ok(()) => None,
Err(e) => {
failed.push(i);
Some(e).filter(ends_the_pull)
}
},
)
.await;
assert!(stopped.is_none());
assert_eq!(failed, vec![1]);
}
#[tokio::test(start_paused = true)]
async fn a_stuck_entry_waits_while_another_progresses() {
let limit = std::time::Duration::from_secs(10);
let stop = decdn_client::StopPolicy::new(
false,
Some(limit),
Arc::new(decdn_client::ProgressClock::new()),
);
let begun = tokio::time::Instant::now();
let stopped = run_groups(
vec![0u8, 1],
2,
|i| {
let stop = stop.clone();
async move {
if i == 0 {
return Err(anyhow::Error::new(stop.expired().await));
}
for _ in 0..6 {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
stop.clock.tick();
}
Ok(())
}
},
|_, r: anyhow::Result<()>| r.err().filter(ends_the_pull),
)
.await;
assert!(
stopped
.as_ref()
.is_some_and(|e| e.downcast_ref::<decdn_client::GaveUp>().is_some())
);
assert_eq!(begun.elapsed(), std::time::Duration::from_secs(40));
}
#[test]
fn build_completed_updates_records_present_omits_failed() {
let tmp = tempfile::tempdir().expect("tmp");
std::fs::write(tmp.path().join("ok.txt"), b"data").expect("write");
std::fs::write(tmp.path().join("bad.txt"), b"old-bytes").expect("write");
let entries = [
ManifestEntry {
path: "ok.txt".into(),
hash: "b3:aa".into(),
size: Some(4),
chunks: Some(vec![ManifestChunk {
hash: "b3:bb".into(),
size: 4,
}]),
},
ManifestEntry {
path: "bad.txt".into(),
hash: "b3:cc".into(),
size: Some(9),
chunks: None,
},
];
let outcomes = vec![
EntryOutcome::Fetched(4),
EntryOutcome::Failed {
path: "bad.txt".into(),
err: "nope".into(),
},
];
let refs: Vec<&ManifestEntry> = entries.iter().collect();
let upd = build_completed_updates(&refs, &outcomes, tmp.path());
assert!(upd.contains_key("ok.txt"));
let rec = upd.get("ok.txt").expect("rec");
assert_eq!(rec.hash, "b3:aa");
assert_eq!(rec.size, 4);
assert!(rec.chunks.is_some());
assert!(!upd.contains_key("bad.txt"));
}
#[test]
fn build_completed_updates_records_linked_and_skipped() {
let tmp = tempfile::tempdir().expect("tmp");
std::fs::write(tmp.path().join("linked.txt"), b"aa").expect("write");
std::fs::write(tmp.path().join("skipped.txt"), b"bbbb").expect("write");
let entries = [
ManifestEntry {
path: "linked.txt".into(),
hash: "b3:l".into(),
size: Some(2),
chunks: None,
},
ManifestEntry {
path: "skipped.txt".into(),
hash: "b3:s".into(),
size: Some(4),
chunks: None,
},
];
let outcomes = vec![EntryOutcome::Linked, EntryOutcome::Skipped];
let refs: Vec<&ManifestEntry> = entries.iter().collect();
let upd = build_completed_updates(&refs, &outcomes, tmp.path());
assert_eq!(upd.get("linked.txt").expect("linked").hash, "b3:l");
assert_eq!(upd.get("skipped.txt").expect("skipped").hash, "b3:s");
}
#[test]
fn build_completed_updates_omits_entry_absent_from_outcomes() {
let tmp = tempfile::tempdir().expect("tmp");
std::fs::write(tmp.path().join("done.txt"), b"new").expect("write");
std::fs::write(tmp.path().join("pending.txt"), b"stale-old-bytes").expect("write");
let entries = [
ManifestEntry {
path: "done.txt".into(),
hash: "b3:done".into(),
size: Some(3),
chunks: None,
},
ManifestEntry {
path: "pending.txt".into(),
hash: "b3:new".into(), size: Some(3),
chunks: None,
},
];
let outcomes = vec![EntryOutcome::Fetched(3)]; let refs: Vec<&ManifestEntry> = entries.iter().collect();
let upd = build_completed_updates(&refs, &outcomes, tmp.path());
assert!(upd.contains_key("done.txt"));
assert!(
!upd.contains_key("pending.txt"),
"an entry not yet in outcomes must never be recorded"
);
}
fn saved_file(hash: &str, size: u64) -> bundle_manifest::SavedFile {
bundle_manifest::SavedFile {
hash: hash.into(),
size,
mtime: SavedMtime { secs: 1, nanos: 0 },
chunks: None,
}
}
fn one_update(path: &str, hash: &str, size: u64) -> BTreeMap<String, bundle_manifest::SavedFile> {
let mut m = BTreeMap::new();
m.insert(path.to_string(), saved_file(hash, size));
m
}
#[tokio::test]
async fn an_interrupted_drive_still_flushes_the_settled_entries() {
let tmp = tempfile::tempdir().expect("tmp");
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let (fire, mut interrupt) = Interrupt::manual();
let drive = async move {
tx.send(FlushBatch {
updates: one_update("done.bin", "b3:done", 1),
fetched_bytes: 1,
})
.expect("send");
fire.send(()).expect("fire");
std::future::pending::<()>().await;
};
let flush = flush_task(tmp.path(), SavedManifest::default(), rx);
assert!(drive_until_interrupted(drive, flush, &mut interrupt).await);
let saved = bundle_manifest::load(tmp.path());
assert_eq!(saved.get("done.bin").expect("done").hash, "b3:done");
}
#[tokio::test]
async fn a_drive_that_finishes_is_not_interrupted() {
let tmp = tempfile::tempdir().expect("tmp");
let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<FlushBatch>();
let drive = async move { drop(tx) };
let (_fire, mut interrupt) = Interrupt::manual();
let flush = flush_task(tmp.path(), SavedManifest::default(), rx);
assert!(!drive_until_interrupted(drive, flush, &mut interrupt).await);
}
#[tokio::test]
async fn flush_task_final_write_persists_all_batches() {
let tmp = tempfile::tempdir().expect("tmp");
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
tx.send(FlushBatch {
updates: one_update("a.bin", "b3:a", 1),
fetched_bytes: 1,
})
.expect("send a");
tx.send(FlushBatch {
updates: one_update("b.bin", "b3:b", 1),
fetched_bytes: 1,
})
.expect("send b");
drop(tx);
flush_task(tmp.path(), SavedManifest::default(), rx).await;
let saved = bundle_manifest::load(tmp.path());
assert_eq!(saved.get("a.bin").expect("a").hash, "b3:a");
assert_eq!(saved.get("b.bin").expect("b").hash, "b3:b");
}
#[tokio::test]
async fn flush_task_keeps_prior_untouched_entries() {
let tmp = tempfile::tempdir().expect("tmp");
let mut prior = SavedManifest::default();
bundle_manifest::merge(&mut prior, one_update("old.bin", "b3:old", 9));
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
tx.send(FlushBatch {
updates: one_update("new.bin", "b3:new", 2),
fetched_bytes: 2,
})
.expect("send");
drop(tx);
flush_task(tmp.path(), prior, rx).await;
let saved = bundle_manifest::load(tmp.path());
assert_eq!(saved.get("old.bin").expect("old kept").hash, "b3:old");
assert_eq!(saved.get("new.bin").expect("new recorded").hash, "b3:new");
}
#[tokio::test]
async fn flush_task_byte_cadence_writes_mid_stream() {
let tmp = tempfile::tempdir().expect("tmp");
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let handle = tokio::spawn({
let root = tmp.path().to_path_buf();
async move { flush_task(&root, SavedManifest::default(), rx).await }
});
tx.send(FlushBatch {
updates: one_update("big.bin", "b3:big", FLUSH_BYTES),
fetched_bytes: FLUSH_BYTES,
})
.expect("send");
let seen = tokio::time::timeout(std::time::Duration::from_secs(10), async {
loop {
if bundle_manifest::load(tmp.path()).get("big.bin").is_some() {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
})
.await
.is_ok();
assert!(
seen,
"byte-cadence flush should write before the channel closes"
);
drop(tx);
handle.await.expect("flush task join");
}
fn sel_entry(path: &str, size: u64) -> ManifestEntry {
ManifestEntry {
path: path.into(),
hash: "b3:00".into(),
size: Some(size),
chunks: None,
}
}
#[test]
fn parse_selection_drops_commented_and_deleted_lines() {
let entries = vec![
sel_entry("a.bin", 1),
sel_entry("b.bin", 2),
sel_entry("c.bin", 3),
];
let edited = "# header\na.bin\t1 B\n#b.bin\t2 B\n";
let kept = parse_selection(edited, entries).expect("parse");
let paths: Vec<&str> = kept.iter().map(|e| e.path.as_str()).collect();
assert_eq!(paths, vec!["a.bin"]);
}
#[test]
fn parse_selection_preserves_manifest_order_regardless_of_edit_order() {
let entries = vec![sel_entry("a.bin", 1), sel_entry("b.bin", 2)];
let edited = "b.bin\nb.bin is not a path\n".replace("b.bin is not a path", "a.bin");
let kept = parse_selection(&edited, entries).expect("parse");
let paths: Vec<&str> = kept.iter().map(|e| e.path.as_str()).collect();
assert_eq!(paths, vec!["a.bin", "b.bin"]);
}
#[test]
fn parse_selection_empty_when_everything_removed() {
let entries = vec![sel_entry("a.bin", 1)];
let kept = parse_selection("# all gone\n", entries).expect("parse");
assert!(kept.is_empty());
}
#[test]
fn parse_selection_rejects_unknown_path() {
let entries = vec![sel_entry("a.bin", 1)];
let err = parse_selection("a.bin\ntypo.bin\n", entries).unwrap_err();
let msg = format!("{err:#}");
assert!(msg.contains("typo.bin"), "{msg}");
}
#[test]
fn parse_selection_ignores_the_size_annotation_after_the_tab() {
let entries = vec![sel_entry("weights.bin", 1500)];
let kept = parse_selection("weights.bin\t1.5 KB\n", entries).expect("parse");
assert_eq!(kept.len(), 1);
}
#[test]
fn render_then_parse_unedited_keeps_every_entry() {
let entries = vec![sel_entry("a.bin", 1), sel_entry("dir/b.bin", 2000)];
let buffer = render_selection(&entries);
let kept = parse_selection(&buffer, entries).expect("parse");
let paths: Vec<&str> = kept.iter().map(|e| e.path.as_str()).collect();
assert_eq!(paths, vec!["a.bin", "dir/b.bin"]);
}
#[test]
fn editor_command_prefers_visual_then_editor_then_vi() {
assert_eq!(editor_command(Some("nano"), Some("vim")), vec!["nano"]);
assert_eq!(editor_command(None, Some("vim")), vec!["vim"]);
assert_eq!(editor_command(None, None), vec!["vi"]);
}
#[test]
fn editor_command_treats_blank_as_unset() {
assert_eq!(editor_command(Some(" "), Some("vim")), vec!["vim"]);
assert_eq!(editor_command(Some(""), None), vec!["vi"]);
}
#[test]
fn editor_command_splits_arguments() {
assert_eq!(
editor_command(Some("code --wait"), None),
vec!["code", "--wait"]
);
}
#[test]
fn check_selectable_rejects_hash_leading_and_tabbed_paths() {
assert!(check_selectable(&[sel_entry("#weird.bin", 1)]).is_err());
assert!(check_selectable(&[sel_entry("has\ttab.bin", 1)]).is_err());
assert!(check_selectable(&[sel_entry(" #weird.bin", 1)]).is_err());
assert!(check_selectable(&[sel_entry(" leading.bin", 1)]).is_err());
assert!(check_selectable(&[sel_entry("trailing.bin ", 1)]).is_err());
assert!(check_selectable(&[sel_entry("fine/name.bin", 1)]).is_ok());
}
#[test]
fn safe_join_builds_nested_path_under_root() {
let root = Path::new("/out");
let p = safe_join(root, "assets/app.js").unwrap();
assert_eq!(p, Path::new("/out/assets/app.js"));
}
#[test]
fn safe_join_rejects_parent_dir() {
let err = safe_join(Path::new("/out"), "a/../../etc/passwd").unwrap_err();
assert!(format!("{err:#}").contains(".."), "{err:#}");
}
#[test]
fn safe_join_rejects_absolute_and_empty_components() {
assert!(safe_join(Path::new("/out"), "/etc/passwd").is_err());
assert!(safe_join(Path::new("/out"), "a//b").is_err());
assert!(safe_join(Path::new("/out"), "a/").is_err());
assert!(safe_join(Path::new("/out"), "./a").is_err());
assert!(safe_join(Path::new("/out"), "").is_err());
}
#[test]
fn complement_runs_gap_in_the_middle() {
let donor = [(16384u64, 16384u64)];
assert_eq!(
complement_runs(&donor, 49152),
vec![(0, 16384), (32768, 16384)]
);
}
#[test]
fn complement_runs_empty_donor_is_the_whole_blob() {
assert_eq!(complement_runs(&[], 49152), vec![(0, 49152)]);
}
#[test]
fn complement_runs_full_coverage_is_empty() {
assert_eq!(complement_runs(&[(0, 49152)], 49152), Vec::new());
}
#[test]
fn complement_runs_coalesces_unsorted_overlapping_donor() {
let donor = [(32768u64, 16384u64), (16384u64, 16384u64)];
assert_eq!(complement_runs(&donor, 49152), vec![(0, 16384)]);
}
#[test]
fn complement_runs_total_zero_is_empty() {
assert_eq!(complement_runs(&[(0, 10)], 0), Vec::new());
}
const GROUP: u64 = CHUNK_GROUP_BYTES;
fn chunk(size: u64, byte: u8) -> ManifestChunk {
ManifestChunk {
hash: format!("b3:{}", blake3::Hash::from_bytes([byte; 32]).to_hex()),
size,
}
}
#[test]
fn hints_of_accumulates_offsets_and_validates_the_size_sum() {
let e = ManifestEntry {
path: "m.bin".into(),
hash: format!("b3:{}", blake3::Hash::from_bytes([0xff; 32]).to_hex()),
size: Some(100),
chunks: Some(vec![chunk(60, 0xaa), chunk(40, 0xbb)]),
};
let hints = hints_of(&e).expect("valid hints");
assert_eq!(hints.len(), 2);
assert_eq!(
(hints[0].hash, hints[0].offset, hints[0].len),
([0xaa; 32], 0, 60)
);
assert_eq!(
(hints[1].hash, hints[1].offset, hints[1].len),
([0xbb; 32], 60, 40)
);
}
#[test]
fn hints_of_rejects_a_size_sum_that_disagrees_with_the_whole_file() {
let e = ManifestEntry {
path: "m.bin".into(),
hash: format!("b3:{}", blake3::Hash::from_bytes([0xff; 32]).to_hex()),
size: Some(100),
chunks: Some(vec![chunk(60, 0xaa), chunk(30, 0xbb)]),
};
assert!(hints_of(&e).is_none());
}
#[test]
fn hints_of_needs_a_size_and_valid_chunk_hashes() {
let no_size = ManifestEntry {
path: "m.bin".into(),
hash: "b3:ff".into(),
size: None,
chunks: Some(vec![chunk(60, 0xaa)]),
};
assert!(hints_of(&no_size).is_none());
let bad_hash = ManifestEntry {
path: "m.bin".into(),
hash: "b3:ff".into(),
size: Some(4),
chunks: Some(vec![ManifestChunk {
hash: "not-a-hash".into(),
size: 4,
}]),
};
assert!(hints_of(&bad_hash).is_none());
}
#[test]
fn plan_dedup_aligns_a_donor_inward_and_takes_the_complement() {
let total = 3 * GROUP;
let h = [0x11; 32];
let mut index = HashMap::new();
index.insert(
h,
MaterializedRange {
source: PathBuf::from("/tmp/donorA"),
offset: 0,
len: GROUP,
},
);
let hints = [Hint {
hash: h,
offset: GROUP,
len: GROUP,
}];
let plan = plan_reassembly(&hints, &index, &FetchPlan::default(), [0; 32], total);
assert_eq!(plan.donor.len(), 1);
let d = &plan.donor[0];
assert_eq!(d.dst, (GROUP, GROUP));
assert_eq!(d.source, PathBuf::from("/tmp/donorA"));
assert_eq!(d.src_offset, 0);
assert_eq!(d.chunk_hash, h);
assert_eq!((d.chunk_src_offset, d.chunk_len), (0, GROUP));
assert_eq!(plan.drive, vec![(0, GROUP), (2 * GROUP, GROUP)]);
}
#[test]
fn plan_dedup_drops_partial_edge_groups_into_the_complement() {
let total = 4 * GROUP;
let h = [0x22; 32];
let mut index = HashMap::new();
index.insert(
h,
MaterializedRange {
source: PathBuf::from("/tmp/donorB"),
offset: 5000,
len: 2 * GROUP + 500,
},
);
let hints = [Hint {
hash: h,
offset: 100,
len: 2 * GROUP + 500,
}];
let plan = plan_reassembly(&hints, &index, &FetchPlan::default(), [0; 32], total);
assert_eq!(plan.donor.len(), 1);
let d = &plan.donor[0];
assert_eq!(d.dst, (GROUP, GROUP));
assert_eq!(d.src_offset, 5000 + (GROUP - 100));
assert_eq!(plan.drive, vec![(0, GROUP), (2 * GROUP, 2 * GROUP)]);
}
#[test]
fn plan_dedup_with_no_materialized_chunk_is_all_complement() {
let total = 2 * GROUP;
let hints = [Hint {
hash: [0x33; 32],
offset: 0,
len: GROUP,
}];
let plan = plan_reassembly(
&hints,
&HashMap::new(),
&FetchPlan::default(),
[0; 32],
total,
);
assert!(plan.donor.is_empty());
assert_eq!(plan.drive, vec![(0, total)]);
}
#[test]
fn plan_dedup_sub_group_hint_contributes_no_donor() {
let total = GROUP;
let h = [0x44; 32];
let mut index = HashMap::new();
index.insert(
h,
MaterializedRange {
source: PathBuf::from("/tmp/donorC"),
offset: 0,
len: 1000,
},
);
let hints = [Hint {
hash: h,
offset: 0,
len: 1000,
}];
let plan = plan_reassembly(&hints, &index, &FetchPlan::default(), [0; 32], total);
assert!(plan.donor.is_empty());
assert_eq!(plan.drive, vec![(0, total)]);
}
#[test]
fn plan_dedup_skips_a_donor_whose_claimed_length_disagrees() {
let total = 2 * GROUP;
let h = [0x55; 32];
let mut index = HashMap::new();
index.insert(
h,
MaterializedRange {
source: PathBuf::from("/tmp/donorX"),
offset: 0,
len: GROUP,
},
);
let hints = [Hint {
hash: h,
offset: 0,
len: 2 * GROUP,
}];
let plan = plan_reassembly(&hints, &index, &FetchPlan::default(), [0; 32], total);
assert!(
plan.donor.is_empty(),
"a length-mismatched donor must not be spliced"
);
assert_eq!(plan.drive, vec![(0, total)]);
}
#[test]
fn splice_donors_refetches_when_a_copy_would_run_past_the_donor_end() {
let tmp = tempfile::tempdir().expect("tmp");
let donor_path = tmp.path().join("donor");
let partial = tmp.path().join("out.partial");
let chunk_bytes: Vec<u8> = (0..GROUP)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
std::fs::write(&donor_path, &chunk_bytes).expect("write donor");
let chunk_hash = *blake3::hash(&chunk_bytes).as_bytes();
let total = 3 * GROUP;
std::fs::File::create(&partial)
.and_then(|f| f.set_len(total))
.expect("presize partial");
let donor = DonorRange {
dst: (GROUP, 2 * GROUP),
refetch: (GROUP, 2 * GROUP),
source: donor_path,
src_offset: 0,
chunk_hash,
chunk_src_offset: 0,
chunk_len: GROUP,
};
let failed = splice_donors(&partial, std::slice::from_ref(&donor))
.expect("a copy that EOFs must not fail the splice");
let refetch: Vec<(u64, u64)> = failed.iter().map(|d| d.refetch).collect();
assert_eq!(refetch, vec![(GROUP, 2 * GROUP)]);
let got = std::fs::read(&partial).expect("read partial");
assert!(
got.iter().all(|&b| b == 0),
"an EOF-ing donor copy must leave partial untouched"
);
}
#[tokio::test]
async fn reassemble_dedup_succeeds_when_the_first_drive_finalizes_the_blob() {
struct FinalizingDriver {
hash: [u8; 32],
staging: PathBuf,
drives: std::cell::Cell<u32>,
}
impl RangeDriver for FinalizingDriver {
fn hash(&self) -> [u8; 32] {
self.hash
}
fn staging(&self) -> &Path {
&self.staging
}
fn drive<'a>(
&'a self,
_ranges: &'a [(u64, u64)],
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + 'a>> {
Box::pin(async move {
self.drives.set(self.drives.get() + 1);
std::fs::write(&self.staging, b"finalized").expect("finalize staging");
Ok(())
})
}
}
let tmp = tempfile::tempdir().expect("tmp");
let staging = tmp.path().join("blob");
let driver = FinalizingDriver {
hash: [0x11; 32],
staging: staging.clone(),
drives: std::cell::Cell::new(0),
};
let plan = ReassemblePlan {
donor: vec![DonorRange {
dst: (GROUP, GROUP),
refetch: (GROUP, GROUP),
source: tmp.path().join("donor-never-read"),
src_offset: 0,
chunk_hash: [0x11; 32],
chunk_src_offset: 0,
chunk_len: GROUP,
}],
deferred: Vec::new(),
drive: vec![(0, GROUP)],
};
let index = ChunkIndex::default();
let fetch_plan = FetchPlan::default();
let resumed = [(GROUP, GROUP)];
let res = reassemble_dedup(
&driver,
&plan,
2 * GROUP,
&resumed,
None,
&index,
&fetch_plan,
None,
&|| {},
)
.await;
assert!(
res.is_ok(),
"a first pay-now drive that finalizes the blob must succeed, not \
fail on a missing .partial: {res:?}"
);
assert_eq!(driver.drives.get(), 1, "only the pay-now drive ran");
assert!(staging.try_exists().expect("stat staging"));
let outcome = res.expect("checked above");
assert_eq!(outcome.spliced_bytes, 0);
assert_eq!(outcome.resumed_bytes, GROUP);
assert_eq!(
EntryBytes::dedup(2 * GROUP, outcome),
EntryBytes {
paid: GROUP,
spliced: 0,
resumed: GROUP,
}
);
}
fn whole_deferred(hint: Hint) -> DeferredChunk {
DeferredChunk {
hint,
dst: (hint.offset, hint.len),
refetch: outward_groups((hint.offset, hint.len), u64::MAX),
}
}
fn mentry(whole: u8, size: u64, chunks: &[(u64, u8)]) -> ManifestEntry {
ManifestEntry {
path: format!("e{whole:02x}"),
hash: format!("b3:{}", blake3::Hash::from_bytes([whole; 32]).to_hex()),
size: Some(size),
chunks: Some(chunks.iter().map(|&(s, b)| chunk(s, b)).collect()),
}
}
#[test]
fn fetch_plan_assigns_shared_chunk_to_smallest_entry() {
let a = mentry(0x0a, 100, &[(40, 0xc0), (60, 0xc1)]);
let b = mentry(0x0b, 300, &[(60, 0xc1), (240, 0xc2)]);
let plan = build_fetch_plan(&[a, b]);
assert_eq!(plan.assigned.get(&[0xc1; 32]), Some(&[0x0a; 32]));
assert!(!plan.assigned.contains_key(&[0xc0; 32]));
assert!(!plan.assigned.contains_key(&[0xc2; 32]));
}
#[test]
fn fetch_plan_tie_breaks_by_whole_hash_and_is_order_independent() {
let p1 = build_fetch_plan(&[
mentry(0x0a, 60, &[(60, 0xc1)]),
mentry(0x0b, 60, &[(60, 0xc1)]),
]);
let p2 = build_fetch_plan(&[
mentry(0x0b, 60, &[(60, 0xc1)]),
mentry(0x0a, 60, &[(60, 0xc1)]),
]);
assert_eq!(p1.assigned, p2.assigned);
let winner = std::cmp::min([0x0a; 32], [0x0b; 32]);
assert_eq!(p1.assigned.get(&[0xc1; 32]), Some(&winner));
}
#[test]
fn plan_reassembly_defers_a_chunk_assigned_to_a_sibling() {
let total = 3 * GROUP + 50_000;
let b = mentry(0x0b, total, &[(3 * GROUP, 0x01), (50_000, 0x02)]);
let fetch_plan = build_fetch_plan(&[
mentry(0x0a, 3 * GROUP, &[(3 * GROUP, 0x01)]),
mentry(0x0b, total, &[(3 * GROUP, 0x01), (50_000, 0x02)]),
]);
let hints = hints_of(&b).expect("valid hints");
let plan = plan_reassembly(&hints, &HashMap::new(), &fetch_plan, [0x0b; 32], total);
assert_eq!(plan.deferred.len(), 1);
assert_eq!(plan.deferred[0].hint.hash, [0x01; 32]);
assert!(plan.donor.is_empty());
assert_eq!(plan.drive, vec![(3 * GROUP, 50_000)]);
}
#[test]
fn plan_reassembly_splices_its_own_assigned_chunk_once_a_sibling_registered_it() {
let a = mentry(0x0a, 3 * GROUP, &[(3 * GROUP, 0x01)]);
let fetch_plan = build_fetch_plan(&[
mentry(0x0a, 3 * GROUP, &[(3 * GROUP, 0x01)]),
mentry(
0x0b,
3 * GROUP + 50_000,
&[(3 * GROUP, 0x01), (50_000, 0x02)],
),
]);
assert_eq!(fetch_plan.assigned.get(&[0x01; 32]), Some(&[0x0a; 32]));
let registered = HashMap::from([(
[0x01; 32],
MaterializedRange {
source: PathBuf::from("b"),
offset: 0,
len: 3 * GROUP,
},
)]);
let hints = hints_of(&a).expect("valid hints");
let plan = plan_reassembly(&hints, ®istered, &fetch_plan, [0x0a; 32], 3 * GROUP);
assert_eq!(plan.donor.len(), 1);
assert!(
plan.drive.is_empty(),
"nothing is paid twice: {:?}",
plan.drive
);
}
#[test]
fn plan_reassembly_pays_a_sub_group_shared_chunk_it_cannot_splice() {
let b = mentry(0x0b, 200, &[(100, 0x09), (100, 0x08)]);
let fetch_plan = build_fetch_plan(&[
mentry(0x0a, 100, &[(100, 0x09)]),
mentry(0x0b, 200, &[(100, 0x09), (100, 0x08)]),
]);
let hints = hints_of(&b).expect("valid hints");
let plan = plan_reassembly(&hints, &HashMap::new(), &fetch_plan, [0x0b; 32], 200);
assert!(plan.deferred.is_empty());
assert_eq!(plan.drive, vec![(0, 200)]);
}
#[test]
fn groups_ordered_smallest_first_stable() {
let big = mentry(0x01, 900, &[(900, 0x91)]);
let small = mentry(0x02, 10, &[(10, 0x92)]);
let mid = mentry(0x03, 100, &[(100, 0x93)]);
let refs = vec![&big, &small, &mid];
let ordered = order_groups_smallest_first(group_by_hash(&refs));
let sizes: Vec<_> = ordered
.iter()
.map(|g| g.entries.iter().find_map(|e| e.size))
.collect();
assert_eq!(sizes, vec![Some(10), Some(100), Some(900)]);
}
#[test]
fn groups_ordered_by_any_declared_size_not_just_the_first() {
let whole = format!("b3:{}", blake3::Hash::from_bytes([0x42; 32]).to_hex());
let unsized_first = ManifestEntry {
path: "a".into(),
hash: whole.clone(),
size: None,
chunks: None,
};
let sized_dup = ManifestEntry {
path: "b".into(),
hash: whole,
size: Some(10),
chunks: None,
};
let big = mentry(0x01, 900, &[(900, 0x91)]);
let refs = vec![&big, &unsized_first, &sized_dup];
let ordered = order_groups_smallest_first(group_by_hash(&refs));
let sizes: Vec<_> = ordered
.iter()
.map(|g| g.entries.iter().find_map(|e| e.size))
.collect();
assert_eq!(sizes, vec![Some(10), Some(900)]);
}
#[test]
fn download_bytes_counts_shared_chunks_once() {
let a = mentry(0x0a, 3 * GROUP, &[(3 * GROUP, 0x01)]);
let total_b = 3 * GROUP + 50_000;
let b = mentry(0x0b, total_b, &[(3 * GROUP, 0x01), (50_000, 0x02)]);
let fetch_plan = build_fetch_plan(&[
mentry(0x0a, 3 * GROUP, &[(3 * GROUP, 0x01)]),
mentry(0x0b, total_b, &[(3 * GROUP, 0x01), (50_000, 0x02)]),
]);
let empty = HashMap::new();
let refs_a = vec![&a];
let refs_b = vec![&b];
let group_a = group_by_hash(&refs_a).pop().expect("group a");
let group_b = group_by_hash(&refs_b).pop().expect("group b");
assert_eq!(
blob_download_reconstruct(&group_a, &fetch_plan, &empty),
(3 * GROUP, 0)
);
assert_eq!(
blob_download_reconstruct(&group_b, &fetch_plan, &empty),
(50_000, 3 * GROUP)
);
assert_eq!(
download_bytes(&[a, b], &fetch_plan),
Some(3 * GROUP + 50_000)
);
}
struct RecordingDriver {
hash: [u8; 32],
staging: PathBuf,
content: Vec<u8>,
driven: std::sync::Mutex<Vec<(u64, u64)>>,
}
impl RangeDriver for RecordingDriver {
fn hash(&self) -> [u8; 32] {
self.hash
}
fn staging(&self) -> &Path {
&self.staging
}
fn drive<'a>(
&'a self,
ranges: &'a [(u64, u64)],
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + 'a>> {
Box::pin(async move {
use std::io::{Seek, SeekFrom, Write};
let total = u64::try_from(self.content.len()).expect("content len fits u64");
let (dir, stem) = fetch::ranged_store_location(&self.staging).expect("location");
ClientRangedStore::open_or_create(&dir, &stem, self.hash, total)
.expect("open ranged store");
let partial = self.staging.with_extension("partial");
let mut f = std::fs::OpenOptions::new()
.write(true)
.open(&partial)
.expect("open partial");
f.set_len(total).expect("size partial");
for &(off, len) in ranges {
self.driven.lock().expect("driven lock").push((off, len));
let start = usize::try_from(off).expect("offset fits usize");
let end = usize::try_from(off + len).expect("end fits usize");
f.seek(SeekFrom::Start(off)).expect("seek");
f.write_all(&self.content[start..end]).expect("write range");
}
f.sync_all().expect("sync partial");
Ok(())
})
}
}
#[tokio::test]
async fn a_fallback_drive_keeps_the_bytes_spliced_before_it() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let half = usize::try_from(2 * GROUP).expect("fits usize");
let donor_path = tmp.path().join("donor");
std::fs::write(&donor_path, &content[..half]).expect("write donor");
let donor_hash = *blake3::hash(&content[..half]).as_bytes();
let deferred_hash = *blake3::hash(&content[half..]).as_bytes();
let staging = tmp.path().join("blob");
let driver = RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
};
let plan = ReassemblePlan {
donor: vec![DonorRange {
dst: (0, 2 * GROUP),
refetch: (0, 2 * GROUP),
source: donor_path,
src_offset: 0,
chunk_hash: donor_hash,
chunk_src_offset: 0,
chunk_len: 2 * GROUP,
}],
deferred: vec![whole_deferred(Hint {
hash: deferred_hash,
offset: 2 * GROUP,
len: 2 * GROUP,
})],
drive: Vec::new(),
};
let index = ChunkIndex::default();
let mut fetch_plan = FetchPlan::default();
fetch_plan.assigned.insert(deferred_hash, [0xaa; 32]);
index.mark_finished([0xaa; 32]);
let outcome = reassemble_dedup(
&driver,
&plan,
total,
&[],
None,
&index,
&fetch_plan,
None,
&|| {},
)
.await
.expect("reassembly must succeed");
assert_eq!(std::fs::read(&staging).expect("read staging"), content);
assert_eq!(
driver.driven.lock().expect("driven lock").clone(),
vec![(2 * GROUP, 2 * GROUP)],
"only the deferred half is paid for; no self-heal re-drive"
);
assert_eq!(outcome.spliced_bytes, 2 * GROUP);
assert_eq!(outcome.resumed_bytes, 0);
}
#[tokio::test]
async fn a_splice_over_a_resumed_prefix_counts_the_overlap_as_spliced() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let half = usize::try_from(2 * GROUP).expect("fits usize");
let staging = tmp.path().join("blob");
ClientRangedStore::seed_checkpointed_prefix(tmp.path(), "blob", &content, 3 * GROUP)
.expect("seed the resume record");
let resumed = resumed_spans(&staging, whole);
assert_eq!(resumed, vec![(0, 3 * GROUP)]);
let donor_path = tmp.path().join("donor");
std::fs::write(&donor_path, &content[..half]).expect("write donor");
let donor_hash = *blake3::hash(&content[..half]).as_bytes();
let deferred_hash = *blake3::hash(&content[half..]).as_bytes();
let driver = RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
};
let plan = ReassemblePlan {
donor: vec![DonorRange {
dst: (0, 2 * GROUP),
refetch: (0, 2 * GROUP),
source: donor_path,
src_offset: 0,
chunk_hash: donor_hash,
chunk_src_offset: 0,
chunk_len: 2 * GROUP,
}],
deferred: vec![whole_deferred(Hint {
hash: deferred_hash,
offset: 2 * GROUP,
len: 2 * GROUP,
})],
drive: Vec::new(),
};
let index = ChunkIndex::default();
let mut fetch_plan = FetchPlan::default();
fetch_plan.assigned.insert(deferred_hash, [0xaa; 32]);
index.mark_finished([0xaa; 32]);
let outcome = reassemble_dedup(
&driver,
&plan,
total,
&resumed,
None,
&index,
&fetch_plan,
None,
&|| {},
)
.await
.expect("reassembly must succeed");
assert_eq!(std::fs::read(&staging).expect("read staging"), content);
assert_eq!(
outcome.spliced_bytes,
2 * GROUP,
"the splice keeps priority"
);
assert_eq!(outcome.resumed_bytes, GROUP, "only the unspliced prefix");
assert_eq!(
EntryBytes::dedup(total, outcome),
EntryBytes {
paid: GROUP,
spliced: 2 * GROUP,
resumed: GROUP,
}
);
}
#[test]
fn a_self_heal_outcome_counts_the_whole_resumed_prefix() {
let outcome = DedupOutcome::of(&[], &[(0, 3 * GROUP), (GROUP, GROUP)], 4 * GROUP, 2);
assert_eq!(outcome.spliced_bytes, 0);
assert_eq!(outcome.resumed_bytes, 3 * GROUP);
assert_eq!(EntryBytes::dedup(4 * GROUP, outcome).paid, GROUP);
}
#[test]
fn resumed_spans_read_the_record_an_earlier_run_left() {
let tmp = tempfile::tempdir().expect("tmp");
let content: Vec<u8> = (0..3 * GROUP + 99)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let len = u64::try_from(content.len()).expect("fits u64");
let hash = *blake3::hash(&content).as_bytes();
let staging = tmp.path().join("blob");
assert!(
resumed_spans(&staging, hash).is_empty(),
"no record, no spans"
);
ClientRangedStore::seed_checkpointed_prefix(tmp.path(), "blob", &content, GROUP)
.expect("seed the resume record");
let resumed = resumed_spans(&staging, hash);
assert_eq!(resumed, vec![(0, GROUP)]);
assert_eq!(
EntryBytes::whole_blob(len, span_bytes(&resumed)),
EntryBytes {
paid: len - GROUP,
spliced: 0,
resumed: GROUP,
}
);
assert_eq!(EntryBytes::whole_blob(10, 20).paid, 0);
assert_eq!(EntryBytes::whole_blob(10, 20).resumed, 10);
std::fs::write(tmp.path().join("blob.partial.ranges"), b"not json").expect("corrupt");
assert!(
resumed_spans(&staging, hash).is_empty(),
"an unreadable record resumes nothing"
);
}
fn two_donor_fixture(
dir: &Path,
content: &[u8],
cut: u64,
) -> (Vec<Hint>, HashMap<[u8; 32], MaterializedRange>) {
let c = usize::try_from(cut).expect("fits usize");
let total = u64::try_from(content.len()).expect("fits u64");
let mut hints = Vec::new();
let mut index = HashMap::new();
for (name, bytes, offset) in [("a", &content[..c], 0), ("b", &content[c..], cut)] {
let path = dir.join(name);
std::fs::write(&path, bytes).expect("write donor");
let hash = *blake3::hash(bytes).as_bytes();
let len = u64::try_from(bytes.len()).expect("fits u64");
hints.push(Hint { hash, offset, len });
index.insert(
hash,
MaterializedRange {
source: path,
offset: 0,
len,
},
);
}
assert_eq!(hints.iter().map(|h| h.len).sum::<u64>(), total);
(hints, index)
}
#[test]
fn plan_dedup_adjacent_donors_at_an_unaligned_boundary_leave_no_drive_gap() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content = vec![7u8; usize::try_from(total).expect("fits usize")];
let cut = GROUP + 5000;
let (hints, index) = two_donor_fixture(tmp.path(), &content, cut);
let plan = plan_reassembly(&hints, &index, &FetchPlan::default(), [0; 32], total);
assert!(plan.drive.is_empty(), "drive: {:?}", plan.drive);
let dsts: Vec<(u64, u64)> = plan.donor.iter().map(|d| d.dst).collect();
assert_eq!(dsts, vec![(0, cut), (cut, total - cut)]);
assert!(plan.donor.iter().all(|d| d.src_offset == 0));
assert_eq!(plan.donor[1].refetch, (GROUP, total - GROUP));
}
#[test]
fn plan_dedup_donor_next_to_a_driven_chunk_drives_the_shared_group_once() {
let total = 3 * GROUP;
let cut = GROUP + 5000;
let donor_hash = [0x44; 32];
let mut index = HashMap::new();
index.insert(
donor_hash,
MaterializedRange {
source: PathBuf::from("/tmp/donor"),
offset: 0,
len: cut,
},
);
let hints = [
Hint {
hash: donor_hash,
offset: 0,
len: cut,
},
Hint {
hash: [0x45; 32],
offset: cut,
len: total - cut,
},
];
let plan = plan_reassembly(&hints, &index, &FetchPlan::default(), [0; 32], total);
let dsts: Vec<(u64, u64)> = plan.donor.iter().map(|d| d.dst).collect();
assert_eq!(dsts, vec![(0, GROUP)]);
assert_eq!(plan.drive, vec![(GROUP, 2 * GROUP)]);
}
#[test]
fn plan_dedup_donor_meeting_a_deferred_chunk_leaves_no_drive_gap() {
let total = 3 * GROUP;
let cut = GROUP + 5000;
let (donor_hash, deferred_hash) = ([0x46; 32], [0x47; 32]);
let mut index = HashMap::new();
index.insert(
donor_hash,
MaterializedRange {
source: PathBuf::from("/tmp/donor"),
offset: 0,
len: cut,
},
);
let mut fetch_plan = FetchPlan::default();
fetch_plan.assigned.insert(deferred_hash, [0xaa; 32]);
let hints = [
Hint {
hash: donor_hash,
offset: 0,
len: cut,
},
Hint {
hash: deferred_hash,
offset: cut,
len: total - cut,
},
];
let plan = plan_reassembly(&hints, &index, &fetch_plan, [0x0b; 32], total);
assert!(plan.drive.is_empty(), "drive: {:?}", plan.drive);
assert_eq!(plan.donor[0].dst, (0, cut));
assert_eq!(plan.deferred[0].dst, (cut, total - cut));
assert_eq!(plan.deferred[0].refetch, (GROUP, 2 * GROUP));
}
#[test]
fn spliced_runs_merge_touching_spans_and_round_only_the_run_ends_inward() {
let total = 10 * GROUP;
let spans = [
(100, GROUP),
(GROUP + 100, 2 * GROUP),
(5 * GROUP + 1, 2 * GROUP),
];
assert_eq!(
spliced_runs(&spans, total),
vec![(GROUP, 3 * GROUP), (6 * GROUP, 7 * GROUP)]
);
assert!(spliced_runs(&[(1, GROUP)], total).is_empty());
}
#[test]
fn spliced_runs_keep_the_blob_end() {
let total = 2 * GROUP + 700;
assert_eq!(
spliced_runs(&[(GROUP, GROUP + 700)], total),
vec![(GROUP, total)]
);
}
#[tokio::test]
async fn reassemble_dedup_splices_a_boundary_group_from_two_donors() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let (hints, map) = two_donor_fixture(tmp.path(), &content, GROUP + 5000);
let plan = plan_reassembly(&hints, &map, &FetchPlan::default(), whole, total);
let staging = tmp.path().join("blob");
let driver = RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
};
let outcome = reassemble_dedup(
&driver,
&plan,
total,
&[],
None,
&ChunkIndex::default(),
&FetchPlan::default(),
None,
&|| {},
)
.await
.expect("reassembly must succeed");
assert_eq!(std::fs::read(&staging).expect("read staging"), content);
assert!(driver.driven.lock().expect("driven lock").is_empty());
assert_eq!(outcome.spliced_bytes, total);
}
#[tokio::test]
async fn reassemble_dedup_refetches_the_outward_group_span_of_a_failed_donor() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let cut = GROUP + 5000;
let (hints, map) = two_donor_fixture(tmp.path(), &content, cut);
let plan = plan_reassembly(&hints, &map, &FetchPlan::default(), whole, total);
std::fs::write(tmp.path().join("b"), vec![0u8; 16]).expect("corrupt donor b");
let staging = tmp.path().join("blob");
let driver = RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
};
let outcome = reassemble_dedup(
&driver,
&plan,
total,
&[],
None,
&ChunkIndex::default(),
&FetchPlan::default(),
None,
&|| {},
)
.await
.expect("reassembly must succeed");
assert_eq!(std::fs::read(&staging).expect("read staging"), content);
assert_eq!(
driver.driven.lock().expect("driven lock").clone(),
vec![(GROUP, total - GROUP)],
"the failed donor's whole groups are driven, boundary group included"
);
assert_eq!(outcome.spliced_bytes, GROUP);
assert_eq!(outcome.hints_ignored, 1);
}
#[tokio::test]
async fn reconcile_deferred_falls_back_to_the_outward_group_span() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let cut = GROUP + 5000;
let (hints, mut map) = two_donor_fixture(tmp.path(), &content, cut);
let b_hash = hints[1].hash;
map.remove(&b_hash);
let mut fetch_plan = FetchPlan::default();
fetch_plan.assigned.insert(b_hash, [0xaa; 32]);
let index = ChunkIndex::default();
index.mark_finished([0xaa; 32]);
let plan = plan_reassembly(&hints, &map, &fetch_plan, whole, total);
assert!(plan.drive.is_empty(), "drive: {:?}", plan.drive);
let staging = tmp.path().join("blob");
let driver = RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
};
let outcome = reassemble_dedup(
&driver,
&plan,
total,
&[],
None,
&index,
&fetch_plan,
None,
&|| {},
)
.await
.expect("reassembly must succeed");
assert_eq!(std::fs::read(&staging).expect("read staging"), content);
assert_eq!(
driver.driven.lock().expect("driven lock").clone(),
vec![(GROUP, total - GROUP)]
);
assert_eq!(outcome.spliced_bytes, GROUP);
}
#[tokio::test]
async fn spliced_bytes_exclude_a_first_splice_refetch_under_a_tail_splice() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let cut = GROUP + 5000;
let (hints, mut map) = two_donor_fixture(tmp.path(), &content, cut);
let a_hash = hints[0].hash;
map.remove(&a_hash);
let mut fetch_plan = FetchPlan::default();
fetch_plan.assigned.insert(a_hash, [0xaa; 32]);
let plan = plan_reassembly(&hints, &map, &fetch_plan, whole, total);
assert_eq!(plan.deferred.len(), 1);
assert_eq!(plan.donor.len(), 1);
let index = ChunkIndex::default();
index.register(Some(&hints[..1]), &tmp.path().join("a"));
std::fs::write(tmp.path().join("b"), vec![0u8; 16]).expect("corrupt donor b");
let staging = tmp.path().join("blob");
let driver = RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
};
let outcome = reassemble_dedup(
&driver,
&plan,
total,
&[],
None,
&index,
&fetch_plan,
None,
&|| {},
)
.await
.expect("reassembly must succeed");
assert_eq!(std::fs::read(&staging).expect("read staging"), content);
assert_eq!(
driver.driven.lock().expect("driven lock").clone(),
vec![(GROUP, total - GROUP)]
);
assert_eq!(outcome.spliced_bytes, GROUP);
assert_eq!(outcome.hints_ignored, 1);
}
fn net_spliced(spliced: &[(u64, u64)], driven: &[(u64, u64)], total: u64) -> u64 {
span_bytes(&uncovered_runs(spliced, driven, total))
}
#[test]
fn uncovered_runs_keeps_the_uncovered_parts_as_disjoint_spans() {
assert_eq!(
uncovered_runs(&[(300, 100), (0, 100)], &[(50, 100), (350, 10)], 1000),
vec![(0, 50), (300, 50), (360, 40)]
);
assert_eq!(
uncovered_runs(&[(0, 60), (40, 100)], &[], 100),
vec![(0, 100)]
);
assert!(uncovered_runs(&[(0, 100)], &[(0, 100)], 1000).is_empty());
}
#[test]
fn net_spliced_subtracts_only_the_driven_overlap() {
assert_eq!(net_spliced(&[(0, 100)], &[(200, 50)], 1000), 100);
assert_eq!(net_spliced(&[(0, 100)], &[(20, 30)], 1000), 70);
assert_eq!(net_spliced(&[(10, 20)], &[(0, 100)], 1000), 0);
assert_eq!(net_spliced(&[(0, 100)], &[(100, 100)], 1000), 100);
assert_eq!(
net_spliced(
&[(300, 100), (0, 100)],
&[(50, 100), (60, 10), (350, 100)],
1000
),
100
);
assert_eq!(net_spliced(&[], &[(0, 10)], 1000), 0);
assert_eq!(net_spliced(&[(0, 10)], &[], 1000), 10);
}
#[tokio::test]
async fn reconcile_deferred_splices_a_late_registered_donor() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let deferred_len = 2 * GROUP;
let dl = usize::try_from(deferred_len).expect("fits usize");
let chunk_hash = *blake3::hash(&content[..dl]).as_bytes();
let staging = tmp.path().join("blob");
let driver = RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
};
let plan = ReassemblePlan {
donor: Vec::new(),
deferred: vec![whole_deferred(Hint {
hash: chunk_hash,
offset: 0,
len: deferred_len,
})],
drive: vec![(deferred_len, 2 * GROUP)],
};
let index = std::sync::Arc::new(ChunkIndex::default());
let mut fetch_plan = FetchPlan::default();
fetch_plan.assigned.insert(chunk_hash, [0xaa; 32]);
let donor_path = tmp.path().join("donor");
std::fs::write(&donor_path, &content[..dl]).expect("write donor");
let index_bg = index.clone();
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
index_bg.register(
Some(&[Hint {
hash: chunk_hash,
offset: 0,
len: deferred_len,
}]),
&donor_path,
);
});
let res = reassemble_dedup(
&driver,
&plan,
total,
&[],
None,
&index,
&fetch_plan,
None,
&|| {},
)
.await;
assert!(res.is_ok(), "reassembly must succeed: {res:?}");
assert!(staging.try_exists().expect("stat staging"));
let got = std::fs::read(&staging).expect("read staging");
assert_eq!(
got, content,
"reassembled blob must match the whole content"
);
let driven_ranges = driver.driven.lock().expect("driven lock").clone();
assert!(
driven_ranges.iter().all(|&(off, _)| off >= deferred_len),
"the deferred range must be spliced, not driven: {driven_ranges:?}"
);
}
#[tokio::test]
async fn reconcile_deferred_pays_when_the_assigned_fetcher_never_produces_it() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let deferred_len = 2 * GROUP;
let dl = usize::try_from(deferred_len).expect("fits usize");
let chunk_hash = *blake3::hash(&content[..dl]).as_bytes();
let staging = tmp.path().join("blob");
let driver = RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
};
let plan = ReassemblePlan {
donor: Vec::new(),
deferred: vec![whole_deferred(Hint {
hash: chunk_hash,
offset: 0,
len: deferred_len,
})],
drive: vec![(deferred_len, 2 * GROUP)],
};
let index = std::sync::Arc::new(ChunkIndex::default());
let mut fetch_plan = FetchPlan::default();
fetch_plan.assigned.insert(chunk_hash, [0xaa; 32]);
let index_bg = index.clone();
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
index_bg.mark_finished([0xaa; 32]);
});
let res = reassemble_dedup(
&driver,
&plan,
total,
&[],
None,
&index,
&fetch_plan,
None,
&|| {},
)
.await;
assert!(res.is_ok(), "reassembly must succeed via fallback: {res:?}");
let got = std::fs::read(&staging).expect("read staging");
assert_eq!(got, content);
let driven_ranges = driver.driven.lock().expect("driven lock").clone();
assert!(
driven_ranges
.iter()
.any(|&(off, len)| off == 0 && len == deferred_len),
"the deferred range must be driven as a fallback: {driven_ranges:?}"
);
}
struct SlotDriver<'s, 'g> {
inner: RecordingDriver,
slot: &'s FetchSlot<'g>,
held_on_drive: std::sync::Mutex<Vec<bool>>,
}
impl RangeDriver for SlotDriver<'_, '_> {
fn hash(&self) -> [u8; 32] {
self.inner.hash()
}
fn staging(&self) -> &Path {
self.inner.staging()
}
fn drive<'a>(
&'a self,
ranges: &'a [(u64, u64)],
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + 'a>> {
Box::pin(async move {
self.slot.ensure().await?;
self.held_on_drive
.lock()
.expect("held lock")
.push(self.slot.held());
self.inner.drive(ranges).await
})
}
fn release_slot(&self) {
self.slot.release();
}
fn retake_slot(
&self,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + '_>> {
Box::pin(self.slot.ensure())
}
}
#[tokio::test]
async fn a_donor_waiter_gives_its_slot_to_a_queued_entry_and_retakes_it_to_drive() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let deferred_len = 2 * GROUP;
let dl = usize::try_from(deferred_len).expect("fits usize");
let chunk_hash = *blake3::hash(&content[..dl]).as_bytes();
let gate = JobGate::new(1);
let slot = FetchSlot::acquire(&gate).await.expect("first slot");
let staging = tmp.path().join("blob");
let driver = SlotDriver {
inner: RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
},
slot: &slot,
held_on_drive: std::sync::Mutex::new(Vec::new()),
};
let plan = ReassemblePlan {
donor: Vec::new(),
deferred: vec![whole_deferred(Hint {
hash: chunk_hash,
offset: 0,
len: deferred_len,
})],
drive: vec![(deferred_len, 2 * GROUP)],
};
let index = ChunkIndex::default();
let mut fetch_plan = FetchPlan::default();
fetch_plan.assigned.insert(chunk_hash, [0xaa; 32]);
let queued = async {
let got = tokio::time::timeout(std::time::Duration::from_secs(10), gate.permit())
.await
.is_ok_and(|permit| permit.is_ok());
index.mark_finished([0xaa; 32]);
got
};
let (res, queued_ran) = tokio::join!(
reassemble_dedup(
&driver,
&plan,
total,
&[],
None,
&index,
&fetch_plan,
None,
&|| {},
),
queued
);
assert!(res.is_ok(), "reassembly must succeed via fallback: {res:?}");
assert!(
queued_ran,
"the waiter must give its slot to the queued entry"
);
assert_eq!(std::fs::read(&staging).expect("read staging"), content);
let held = driver.held_on_drive.lock().expect("held lock").clone();
assert_eq!(
held,
vec![true, true],
"the complement drive and the fallback drive each run under a slot"
);
assert!(slot.held(), "the entry ends holding its slot again");
}
#[tokio::test]
async fn a_waiter_whose_donor_arrives_holds_its_slot_again_for_the_splice() {
let tmp = tempfile::tempdir().expect("tmp");
let total = 4 * GROUP;
let content: Vec<u8> = (0..total)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let whole = *blake3::hash(&content).as_bytes();
let deferred_len = 2 * GROUP;
let dl = usize::try_from(deferred_len).expect("fits usize");
let chunk_hash = *blake3::hash(&content[..dl]).as_bytes();
let hint = Hint {
hash: chunk_hash,
offset: 0,
len: deferred_len,
};
let gate = JobGate::new(1);
let slot = FetchSlot::acquire(&gate).await.expect("first slot");
let staging = tmp.path().join("blob");
let driver = SlotDriver {
inner: RecordingDriver {
hash: whole,
staging: staging.clone(),
content: content.clone(),
driven: std::sync::Mutex::new(Vec::new()),
},
slot: &slot,
held_on_drive: std::sync::Mutex::new(Vec::new()),
};
let plan = ReassemblePlan {
donor: Vec::new(),
deferred: vec![whole_deferred(hint)],
drive: vec![(deferred_len, 2 * GROUP)],
};
let index = ChunkIndex::default();
let mut fetch_plan = FetchPlan::default();
fetch_plan.assigned.insert(chunk_hash, [0xaa; 32]);
let donor_path = tmp.path().join("donor");
std::fs::write(&donor_path, &content[..dl]).expect("write donor");
let sibling = async {
let permit = gate.permit().await.expect("the waiter freed its slot");
index.register(Some(&[hint]), &donor_path);
drop(permit);
};
let (res, ()) = tokio::join!(
reassemble_dedup(
&driver,
&plan,
total,
&[],
None,
&index,
&fetch_plan,
None,
&|| {},
),
sibling
);
assert!(res.is_ok(), "reassembly must succeed: {res:?}");
assert_eq!(std::fs::read(&staging).expect("read staging"), content);
assert_eq!(
driver.held_on_drive.lock().expect("held lock").len(),
1,
"only the complement was driven: the donor was spliced"
);
assert!(slot.held(), "the entry holds its slot again after the wait");
}
#[tokio::test]
async fn run_end_sweep_keeps_a_retained_donor_and_removes_a_normal_one() {
let tmp = tempfile::tempdir().expect("tmp");
let normal = tmp.path().join("normal");
let retained = tmp.path().join("retained");
std::fs::write(&normal, b"n").expect("write normal");
std::fs::write(&retained, b"r").expect("write retained");
let index = ChunkIndex::default();
index.register(
Some(&[Hint {
hash: [1; 32],
offset: 0,
len: 1,
}]),
&normal,
);
index.register(
Some(&[Hint {
hash: [2; 32],
offset: 0,
len: 1,
}]),
&retained,
);
index.mark_retained(&retained);
sweep_donor_sources(&index).await;
assert!(
!normal.exists(),
"a normal donor source is swept at run end"
);
assert!(
retained.exists(),
"a retained (materialize-failed) donor source survives the sweep"
);
}
#[test]
fn chunk_index_seeded_source_survives_sweep() {
let tmp = tempfile::tempdir().expect("tmp");
let out = tmp.path().join("kept.bin");
std::fs::write(&out, b"donorbytes").expect("write");
let idx = ChunkIndex::default();
idx.seed_disk([7u8; 32], &out, 0, 10);
assert!(idx.sources().is_empty());
let guard = idx.map.lock().unwrap_or_else(PoisonError::into_inner);
assert!(guard.contains_key(&[7u8; 32]));
}
#[test]
fn splice_donors_copies_a_verified_subset_into_partial() {
let tmp = tempfile::tempdir().expect("tmp");
let donor_path = tmp.path().join("donor");
let partial = tmp.path().join("out.partial");
let chunk_bytes: Vec<u8> = (0..GROUP)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let pad_before = 100usize;
let mut donor_data = vec![0xEEu8; pad_before];
donor_data.extend_from_slice(&chunk_bytes);
donor_data.extend_from_slice(&[0xEE; 50]);
std::fs::write(&donor_path, &donor_data).expect("write donor");
let chunk_hash = *blake3::hash(&chunk_bytes).as_bytes();
let total = 2 * GROUP;
std::fs::File::create(&partial)
.and_then(|f| f.set_len(total))
.expect("presize partial");
let donor = DonorRange {
dst: (GROUP, GROUP),
refetch: (GROUP, GROUP),
source: donor_path,
src_offset: u64::try_from(pad_before).expect("pad_before fits in u64"),
chunk_hash,
chunk_src_offset: u64::try_from(pad_before).expect("pad_before fits in u64"),
chunk_len: GROUP,
};
let refetch = splice_donors(&partial, std::slice::from_ref(&donor)).expect("splice");
assert!(refetch.is_empty(), "{refetch:?}");
let got = std::fs::read(&partial).expect("read partial");
let g = usize::try_from(GROUP).expect("GROUP fits in usize");
assert_eq!(&got[g..2 * g], &chunk_bytes[..], "spliced subset mismatch");
assert!(
got[..g].iter().all(|&b| b == 0),
"untouched region must stay zero"
);
}
#[test]
fn splice_donors_refetches_a_corrupted_donor_chunk() {
let tmp = tempfile::tempdir().expect("tmp");
let donor_path = tmp.path().join("donor");
let partial = tmp.path().join("out.partial");
let chunk_bytes: Vec<u8> = (0..GROUP)
.map(|i| u8::try_from(i % 251).expect("i % 251 fits in u8"))
.collect();
let chunk_hash = *blake3::hash(&chunk_bytes).as_bytes();
let mut corrupted = chunk_bytes.clone();
let mid = corrupted.len() / 2;
corrupted[mid] ^= 0xFF;
std::fs::write(&donor_path, &corrupted).expect("write corrupted donor");
let total = 2 * GROUP;
std::fs::File::create(&partial)
.and_then(|f| f.set_len(total))
.expect("presize partial");
let donor = DonorRange {
dst: (GROUP, GROUP),
refetch: (GROUP, GROUP),
source: donor_path,
src_offset: 0,
chunk_hash,
chunk_src_offset: 0,
chunk_len: GROUP,
};
let failed = splice_donors(&partial, std::slice::from_ref(&donor)).expect("splice");
let refetch: Vec<(u64, u64)> = failed.iter().map(|d| d.refetch).collect();
assert_eq!(refetch, vec![donor.refetch]);
let got = std::fs::read(&partial).expect("read partial");
let g = usize::try_from(GROUP).expect("GROUP fits in usize");
assert!(
got[g..2 * g].iter().all(|&b| b == 0),
"untrusted donor must not write into partial"
);
}
#[test]
fn chunk_verified_matches_true_and_false() {
let tmp = tempfile::tempdir().expect("tmp");
let p = tmp.path().join("f");
let data = vec![9u8; 2000];
std::fs::write(&p, &data).expect("write");
let hash = *blake3::hash(&data).as_bytes();
let mut f = std::fs::File::open(&p).expect("open");
assert!(chunk_verified(&mut f, 0, 2000, hash));
let mut f2 = std::fs::File::open(&p).expect("open");
assert!(!chunk_verified(&mut f2, 0, 2000, [0u8; 32]));
}
#[test]
fn copy_exact_copies_only_the_requested_length() {
let tmp = tempfile::tempdir().expect("tmp");
let src_path = tmp.path().join("src");
let out_path = tmp.path().join("out");
std::fs::write(&src_path, b"hello world").expect("write src");
std::fs::write(&out_path, []).expect("write out");
let mut src = std::fs::File::open(&src_path).expect("open src");
let mut out = std::fs::OpenOptions::new()
.write(true)
.open(&out_path)
.expect("open out");
copy_exact(&mut src, &mut out, 5).expect("copy_exact");
drop(out);
let got = std::fs::read(&out_path).expect("read out");
assert_eq!(&got[..], b"hello");
}
#[test]
fn hash_partial_matches_direct_blake3_of_file_contents() {
let tmp = tempfile::tempdir().expect("tmp");
let p = tmp.path().join("f");
let data: Vec<u8> = (0..5000u32)
.map(|i| u8::try_from(i % 256).expect("i % 256 fits in u8"))
.collect();
std::fs::write(&p, &data).expect("write");
let got = hash_partial(&p).expect("hash_partial");
assert_eq!(got, *blake3::hash(&data).as_bytes());
}
#[test]
fn staging_path_is_the_plain_hex_final_blob_not_a_partial() {
let tmp = tempfile::tempdir().expect("tmp");
let hash = [0xabu8; 32];
let p = staging_path(tmp.path(), hash).expect("staging_path");
let name = p.file_name().and_then(|n| n.to_str()).expect("name");
assert!(
!name.ends_with(".partial"),
"staging file must be the plain finalized blob, got {name}"
);
assert_eq!(name, blake3::Hash::from_bytes(hash).to_hex().to_string());
assert!(p.starts_with(tmp.path().join(STAGING_DIR)));
}
#[test]
fn parse_manifest_accepts_v1_with_optional_size() {
let json = br#"{"version":1,"entries":[{"path":"a.txt","hash":"b3:ab","size":4},{"path":"b","hash":"b3:cd"}]}"#;
let m = parse_manifest(json).unwrap();
assert_eq!(m.entries.len(), 2);
assert_eq!(m.entries[0].size, Some(4));
assert_eq!(m.entries[1].size, None);
}
#[test]
fn parse_manifest_accepts_chunked_entry_in_order() {
let json = br#"{"version":1,"entries":[{"path":"m.bin","hash":"b3:whole","size":100,"chunks":[{"hash":"b3:c0","size":60},{"hash":"b3:c1","size":40}]}]}"#;
let m = parse_manifest(json).unwrap();
let chunks = m.entries[0].chunks.as_ref().expect("chunked entry");
let hashes: Vec<&str> = chunks.iter().map(|c| c.hash.as_str()).collect();
assert_eq!(hashes, vec!["b3:c0", "b3:c1"]);
}
#[test]
fn parse_manifest_plain_entry_has_no_chunks() {
let json = br#"{"version":1,"entries":[{"path":"a.txt","hash":"b3:ab","size":4}]}"#;
let m = parse_manifest(json).unwrap();
assert!(m.entries[0].chunks.is_none());
}
#[test]
fn parse_manifest_rejects_unsupported_version() {
let json = br#"{"version":2,"entries":[]}"#;
let err = parse_manifest(json).unwrap_err();
assert!(
format!("{err:#}").contains("unsupported bundle version 2"),
"{err:#}"
);
}
#[test]
fn report_errors_when_any_failed() {
let outcomes = vec![
EntryOutcome::Fetched(10),
EntryOutcome::Skipped,
EntryOutcome::Failed {
path: "x".into(),
err: "boom".into(),
},
];
let err = report(
&outcomes,
&[],
Transfer::default(),
DedupSummary::default(),
0,
Path::new("/out"),
true,
)
.unwrap_err();
assert!(format!("{err:#}").contains("1 entr"), "{err:#}");
}
#[test]
fn report_ok_when_none_failed() {
let outcomes = vec![EntryOutcome::Fetched(10), EntryOutcome::Skipped];
assert!(
report(
&outcomes,
&[],
Transfer::default(),
DedupSummary::default(),
0,
Path::new("/out"),
false
)
.is_ok()
);
}
#[test]
fn a_manifest_size_mismatch_warns_and_succeeds() {
let tmp = tempfile::tempdir().expect("tmp");
std::fs::write(tmp.path().join("big.bin"), b"twelve bytes").expect("write");
std::fs::write(tmp.path().join("ok.bin"), b"four").expect("write");
let entries = [
ManifestEntry {
path: "big.bin".into(),
hash: "b3:aa".into(),
size: Some(4),
chunks: None,
},
ManifestEntry {
path: "ok.bin".into(),
hash: "b3:bb".into(),
size: Some(4),
chunks: None,
},
];
let refs: Vec<&ManifestEntry> = entries.iter().collect();
let outcomes = vec![EntryOutcome::Fetched(12), EntryOutcome::Fetched(4)];
let updates = build_completed_updates(&refs, &outcomes, tmp.path());
let warnings = size_warnings(&refs, &updates);
assert_eq!(
warnings,
vec!["big.bin: manifest says 4 bytes, the blob is 12 bytes".to_string()]
);
assert!(
report(
&outcomes,
&warnings,
Transfer::default(),
DedupSummary::default(),
0,
tmp.path(),
false
)
.is_ok(),
"a size warning is not a failure"
);
}
#[test]
fn the_summary_counts_reused_and_excluded_entries() {
let outcomes = vec![
EntryOutcome::Fetched(10),
EntryOutcome::Linked,
EntryOutcome::Skipped,
EntryOutcome::Deduped(7),
EntryOutcome::Failed {
path: "x".into(),
err: "boom".into(),
},
];
let transfer = Transfer {
downloaded: 4,
reconstructed: 20,
resumed: 6,
};
let rep = pull_report(
&outcomes,
transfer,
DedupSummary::default(),
3,
Path::new("/out"),
);
assert_eq!(
counts_line(&rep),
"pulled into /out (1 fetched, 1 linked, 1 skipped, 1 reused, 3 excluded, 1 failed)"
);
assert_eq!(rep.reused_bytes, 7);
let json = serde_json::to_value(&rep).expect("serialize");
assert_eq!(json["reused"], 1);
assert_eq!(json["reused_bytes"], 7);
assert_eq!(json["excluded"], 3);
assert_eq!(json["resumed_bytes"], 6);
assert_eq!(json["downloaded"], 4);
assert!(json.get("deduped").is_none(), "{json}");
}
#[test]
fn human_bytes_scales_units() {
assert_eq!(human_bytes(0), "0 B");
assert_eq!(human_bytes(512), "512 B");
assert_eq!(human_bytes(1_500), "1.5 KB");
assert_eq!(human_bytes(27_600_000_000), "27.6 GB");
}
#[test]
fn transfer_line_shows_arrow_only_when_they_differ() {
assert_eq!(
transfer_line(Transfer {
downloaded: 27_600_000_000,
reconstructed: 27_600_000_000,
resumed: 0,
}),
"downloaded 27.6 GB"
);
assert_eq!(
transfer_line(Transfer {
downloaded: 13_800_000_000,
reconstructed: 27_600_000_000,
resumed: 0,
}),
"downloaded 13.8 GB → reconstructed 27.6 GB"
);
}
#[test]
fn group_transfer_counts_the_blob_once_and_every_written_copy() {
let outcomes = vec![
EntryOutcome::Fetched(100),
EntryOutcome::Linked,
EntryOutcome::Linked,
];
let t = group_transfer(&outcomes, Some(EntryBytes::whole_blob(100, 0)));
assert_eq!(t.downloaded, 100);
assert_eq!(t.reconstructed, 300);
}
#[test]
fn group_transfer_downloads_only_the_paid_bytes_of_a_spliced_blob() {
let outcomes = vec![EntryOutcome::Fetched(100), EntryOutcome::Linked];
let spliced = EntryBytes {
paid: 30,
spliced: 70,
resumed: 0,
};
let t = group_transfer(&outcomes, Some(spliced));
assert_eq!(t.downloaded, 30);
assert_eq!(t.reconstructed, 200);
assert_eq!(t.resumed, 0);
let t = group_transfer(&[EntryOutcome::Fetched(100)], Some(EntryBytes::default()));
assert_eq!(t.downloaded, 0);
assert_eq!(t.reconstructed, 100);
}
#[test]
fn group_transfer_tallies_a_resumed_prefix_apart_from_the_download() {
let outcomes = vec![EntryOutcome::Fetched(100), EntryOutcome::Linked];
let t = group_transfer(&outcomes, Some(EntryBytes::whole_blob(100, 40)));
assert_eq!(
t,
Transfer {
downloaded: 60,
reconstructed: 200,
resumed: 40,
}
);
let failed = vec![EntryOutcome::Failed {
path: "x".into(),
err: "boom".into(),
}];
let t = group_transfer(&failed, Some(EntryBytes::whole_blob(100, 40)));
assert_eq!(t, Transfer::default());
}
#[test]
fn group_transfer_skips_and_fails_contribute_nothing() {
let outcomes = vec![EntryOutcome::Fetched(100), EntryOutcome::Skipped];
let t = group_transfer(&outcomes, None);
assert_eq!(t.downloaded, 100);
assert_eq!(t.reconstructed, 100);
let none = vec![EntryOutcome::Failed {
path: "x".into(),
err: "boom".into(),
}];
let t = group_transfer(&none, None);
assert_eq!(t.downloaded, 0);
assert_eq!(t.reconstructed, 0);
}
fn entry(path: &str, hash: &str) -> ManifestEntry {
ManifestEntry {
path: path.into(),
hash: hash.into(),
size: None,
chunks: None,
}
}
#[test]
fn group_by_hash_collapses_duplicates_preserving_order() {
let entries = [
entry("a.txt", "b3:h1"),
entry("b.txt", "b3:h2"),
entry("c.txt", "b3:h1"),
];
let refs: Vec<&ManifestEntry> = entries.iter().collect();
let paths: Vec<Vec<&str>> = group_by_hash(&refs)
.iter()
.map(|group| group.entries.iter().map(|e| e.path.as_str()).collect())
.collect();
assert_eq!(paths, vec![vec!["a.txt", "c.txt"], vec!["b.txt"]]);
}
#[test]
fn group_by_hash_keeps_distinct_hashes_separate() {
let entries = [entry("a", "b3:1"), entry("b", "b3:2"), entry("c", "b3:3")];
let refs: Vec<&ManifestEntry> = entries.iter().collect();
let groups = group_by_hash(&refs);
assert_eq!(groups.len(), 3);
assert!(groups.iter().all(|group| group.entries.len() == 1));
}
fn strs(items: &[&str]) -> Vec<String> {
items.iter().map(|s| (*s).to_string()).collect()
}
fn filtered_paths(include: &[&str], exclude: &[&str], entries: Vec<ManifestEntry>) -> Vec<String> {
let filter = EntryFilter::compile(&strs(include), &strs(exclude)).unwrap();
filter
.apply_reporting(entries)
.kept
.into_iter()
.map(|e| e.path)
.collect()
}
fn sample() -> Vec<ManifestEntry> {
vec![
entry("models/a.bin", "b3:1"),
entry("models/b.txt", "b3:2"),
entry("docs/readme.md", "b3:3"),
entry("docs/deep/notes.txt", "b3:4"),
]
}
#[test]
fn entry_filter_passthrough_when_no_flags() {
assert_eq!(
filtered_paths(&[], &[], sample()),
vec![
"models/a.bin",
"models/b.txt",
"docs/readme.md",
"docs/deep/notes.txt"
]
);
}
#[test]
fn entry_filter_include_is_a_whitelist_gate() {
assert_eq!(
filtered_paths(&["models/*"], &[], sample()),
vec!["models/a.bin", "models/b.txt"]
);
}
#[test]
fn entry_filter_includes_are_ored() {
assert_eq!(
filtered_paths(&["models/*.bin", "docs/readme.md"], &[], sample()),
vec!["models/a.bin", "docs/readme.md"]
);
}
#[test]
fn entry_filter_excludes_are_ored() {
assert_eq!(
filtered_paths(&[], &["**/*.txt", "**/*.md"], sample()),
vec!["models/a.bin"]
);
}
#[test]
fn entry_filter_star_does_not_cross_slash() {
assert_eq!(
filtered_paths(&[], &["*.txt"], sample()),
vec![
"models/a.bin",
"models/b.txt",
"docs/readme.md",
"docs/deep/notes.txt"
]
);
}
#[test]
fn entry_filter_exclude_beats_include() {
assert_eq!(
filtered_paths(&["models/*"], &["**/*.txt"], sample()),
vec!["models/a.bin"]
);
}
#[test]
fn entry_filter_double_star_recurses() {
assert_eq!(
filtered_paths(&["docs/**"], &[], sample()),
vec!["docs/readme.md", "docs/deep/notes.txt"]
);
assert_eq!(
filtered_paths(&["docs/*"], &[], sample()),
vec!["docs/readme.md"]
);
}
#[test]
fn entry_filter_can_empty_the_set() {
assert!(filtered_paths(&["no/such/*"], &[], sample()).is_empty());
}
#[test]
fn entry_filter_reports_each_pattern_that_matched_nothing() {
let mut entries = sample();
entries.push(entry("gpt/metal/model.bin", "b3:5"));
let filter = EntryFilter::compile(
&strs(&["models/*", "no/such/*"]),
&strs(&["metal/*", "**/*.md"]),
)
.unwrap();
let pass = filter.apply_reporting(entries);
assert_eq!(
pass.kept
.iter()
.map(|e| e.path.as_str())
.collect::<Vec<_>>(),
vec!["models/a.bin", "models/b.txt"]
);
assert_eq!(pass.excluded, 3);
assert_eq!(
pass.unmatched,
vec![
Unmatched {
flag: "--include",
pattern: "no/such/*".into(),
suggestion: None,
},
Unmatched {
flag: "--exclude",
pattern: "metal/*".into(),
suggestion: Some("**/metal/*".into()),
},
]
);
assert_eq!(
pass.unmatched[0].warning(),
"--include 'no/such/*' matched no entries"
);
assert_eq!(
pass.unmatched[1].warning(),
"--exclude 'metal/*' matched no entries (a pattern matches the whole path from the \
bundle root; did you mean '**/metal/*'?)"
);
}
#[test]
fn entry_filter_hints_an_anchored_pattern_without_a_slash() {
let filter = EntryFilter::compile(&[], &strs(&["*.txt", "**/*.png"])).unwrap();
let pass = filter.apply_reporting(sample());
assert_eq!(pass.excluded, 0);
assert_eq!(
pass.unmatched
.iter()
.map(|u| (u.pattern.as_str(), u.suggestion.as_deref()))
.collect::<Vec<_>>(),
vec![("*.txt", Some("**/*.txt")), ("**/*.png", None)]
);
}
#[test]
fn entry_filter_reports_nothing_when_every_pattern_matches() {
let filter = EntryFilter::compile(&strs(&["models/*"]), &strs(&["docs/**"])).unwrap();
let pass = filter.apply_reporting(sample());
assert!(pass.unmatched.is_empty(), "{:?}", pass.unmatched);
assert_eq!(pass.excluded, 2);
let pass = EntryFilter::compile(&[], &[])
.unwrap()
.apply_reporting(sample());
assert!(pass.unmatched.is_empty());
assert_eq!(pass.excluded, 0);
}
#[test]
fn a_selection_adds_its_dropped_entries_to_the_excluded_count() {
let kept = Kept {
manifest: Manifest {
version: 1,
entries: sample(),
},
excluded: 2,
};
let kept = apply_selection(kept, |entries| {
parse_selection("models/a.bin\ndocs/readme.md\n", entries)
})
.unwrap();
assert_eq!(kept.manifest.entries.len(), 2);
assert_eq!(kept.excluded, 4);
}
#[test]
fn leftover_partials_counts_only_unselected_blobs() {
let tmp = tempfile::tempdir().expect("tmp");
assert_eq!(
leftover_partials(tmp.path(), &HashSet::new()),
Leftovers::default(),
"no staging dir, nothing left over"
);
let dir = tmp.path().join(STAGING_DIR);
std::fs::create_dir_all(&dir).expect("staging dir");
let hex = |b: u8| blake3::Hash::from_bytes([b; 32]).to_hex().to_string();
let write = |name: String, len: usize| {
std::fs::write(dir.join(name), vec![0u8; len]).expect("write");
};
write(format!("{}.partial", hex(1)), 500);
write(format!("{}.partial.ranges", hex(1)), 5);
write(format!("{}.partial", hex(2)), 300);
write(format!("{}.partial", hex(3)), 1000);
write(format!("{}.partial.ranges", hex(3)), 10);
write(hex(4), 200);
write(".tmpAbC123".into(), 50);
write("notes.txt".into(), 50);
let keep: HashSet<[u8; 32]> = [[1; 32], [2; 32]].into_iter().collect();
let on_disk =
|name: String| allocated_bytes(&std::fs::metadata(dir.join(name)).expect("metadata"));
let expected = on_disk(format!("{}.partial", hex(3)))
+ on_disk(format!("{}.partial.ranges", hex(3)))
+ on_disk(hex(4));
let leftovers = leftover_partials(tmp.path(), &keep);
assert_eq!(
leftovers,
Leftovers {
blobs: 2,
bytes: expected,
}
);
assert_eq!(
leftover_warning(
Path::new("/out/.decdn-partial"),
&Leftovers {
blobs: 2,
bytes: 1210,
}
)
.as_deref(),
Some(
"2 partial downloads (1.2 KB on disk) that this run did not use remain in \
/out/.decdn-partial; delete them to reclaim the space"
)
);
assert_eq!(
leftover_warning(
Path::new("/out/.decdn-partial"),
&Leftovers {
blobs: 1,
bytes: 1_500_000_000,
}
)
.as_deref(),
Some(
"1 partial download (1.5 GB on disk) that this run did not use remains in \
/out/.decdn-partial; delete it to reclaim the space"
)
);
assert_eq!(
leftover_warning(Path::new("/out"), &Leftovers::default()),
None
);
}
#[cfg(unix)]
#[test]
fn leftover_partials_count_a_sparse_partial_by_its_allocated_space() {
let tmp = tempfile::tempdir().expect("tmp");
let dir = tmp.path().join(STAGING_DIR);
std::fs::create_dir_all(&dir).expect("staging dir");
let hex = blake3::Hash::from_bytes([9; 32]).to_hex().to_string();
let file = std::fs::File::create(dir.join(format!("{hex}.partial"))).expect("create");
file.set_len(64 * 1024 * 1024).expect("size the partial");
let leftovers = leftover_partials(tmp.path(), &HashSet::new());
assert_eq!(leftovers.blobs, 1);
assert!(
leftovers.bytes < 1024 * 1024,
"a hole-only partial takes almost no space: {}",
leftovers.bytes
);
}
#[test]
fn entry_filter_rejects_bad_glob() {
let err = EntryFilter::compile(&strs(&["["]), &[]).unwrap_err();
assert!(
format!("{err:#}").contains("--include"),
"error should name the offending flag: {err:#}"
);
}
#[test]
fn link_or_copy_atomic_materializes_identical_bytes() {
let dir = tempfile::tempdir().unwrap();
let src = dir.path().join("src.bin");
let dest = dir.path().join("nested/dest.bin");
std::fs::write(&src, b"the-canonical-bytes").unwrap();
link_or_copy_atomic(&src, &dest).unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), b"the-canonical-bytes");
assert!(src.exists());
}
#[test]
fn link_or_copy_atomic_replaces_existing_dest() {
let dir = tempfile::tempdir().unwrap();
let src = dir.path().join("src.bin");
let dest = dir.path().join("dest.bin");
std::fs::write(&src, b"new").unwrap();
std::fs::write(&dest, b"stale-and-longer").unwrap();
link_or_copy_atomic(&src, &dest).unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), b"new");
}
#[test]
fn plan_slots_classifies_each_destination_independently() {
let dir = tempfile::tempdir().unwrap();
let out = dir.path();
std::fs::write(out.join("present.txt"), b"x").unwrap();
let entries = [entry("present.txt", "b3:h"), entry("absent.txt", "b3:h")];
let refs: Vec<&ManifestEntry> = entries.iter().collect();
let skip = HashSet::from(["present.txt".to_string()]);
let slots = plan_slots(&refs, out, false, &skip);
assert!(matches!(slots[0], Slot::Skip));
assert!(matches!(slots[1], Slot::Write { .. }));
let slots = plan_slots(&refs, out, true, &skip);
assert!(matches!(slots[0], Slot::Write { .. }));
assert!(matches!(slots[1], Slot::Write { .. }));
}
#[test]
fn plan_slots_marks_unsafe_paths_failed() {
let dir = tempfile::tempdir().unwrap();
let entries = [entry("../escape", "b3:h")];
let refs: Vec<&ManifestEntry> = entries.iter().collect();
let slots = plan_slots(&refs, dir.path(), false, &HashSet::new());
assert!(matches!(slots[0], Slot::Failed(_)));
}
#[test]
fn plan_slots_rejects_the_reserved_staging_dir() {
let dir = tempfile::tempdir().unwrap();
let entries = [entry(&format!("{STAGING_DIR}/deadbeef.partial"), "b3:h")];
let refs: Vec<&ManifestEntry> = entries.iter().collect();
let slots = plan_slots(&refs, dir.path(), false, &HashSet::new());
assert!(matches!(slots[0], Slot::Failed(_)));
}
#[tokio::test]
async fn resolve_disk_state_skips_matching_file_by_rehash() {
let tmp = tempfile::tempdir().expect("tmp");
let body = b"hello world";
std::fs::write(tmp.path().join("a.txt"), body).expect("write");
let h = format!("b3:{}", blake3::hash(body).to_hex());
let entries = [ManifestEntry {
path: "a.txt".into(),
hash: h,
size: Some(u64::try_from(body.len()).expect("len")),
chunks: None,
}];
let st = resolve_disk_state(&entries, &SavedManifest::default(), tmp.path(), false).await;
assert!(st.skip.contains("a.txt"));
}
#[tokio::test]
async fn resolve_disk_state_fast_skips_on_matching_record_without_rehash() {
let tmp = tempfile::tempdir().expect("tmp");
let body = b"actual on-disk bytes";
std::fs::write(tmp.path().join("a.txt"), body).expect("write");
let meta = std::fs::metadata(tmp.path().join("a.txt")).expect("meta");
let size = u64::try_from(body.len()).expect("len");
let mtime = SavedMtime::of(&meta).expect("mtime");
let claimed = "b3:0000000000000000000000000000000000000000000000000000000000000000".to_string();
assert_ne!(
claimed,
format!("b3:{}", blake3::hash(body).to_hex()),
"planted bytes must not match the claimed hash"
);
let mut updates = BTreeMap::new();
updates.insert(
"a.txt".to_string(),
bundle_manifest::SavedFile {
hash: claimed.clone(),
size,
mtime,
chunks: None,
},
);
bundle_manifest::merge_and_write(tmp.path(), SavedManifest::default(), updates)
.expect("write saved manifest");
let saved = bundle_manifest::load(tmp.path());
let entries = [ManifestEntry {
path: "a.txt".into(),
hash: claimed, size: Some(size),
chunks: None,
}];
let st = resolve_disk_state(&entries, &saved, tmp.path(), false).await;
assert!(st.skip.contains("a.txt"));
}
#[tokio::test]
async fn a_mismatched_entry_is_not_refetched_on_rerun() {
let tmp = tempfile::tempdir().expect("tmp");
let body = b"the blob is longer than the manifest says";
std::fs::write(tmp.path().join("a.bin"), body).expect("write");
let meta = std::fs::metadata(tmp.path().join("a.bin")).expect("meta");
let size = u64::try_from(body.len()).expect("len");
let hash = "b3:1111111111111111111111111111111111111111111111111111111111111111".to_string();
let mut updates = BTreeMap::new();
updates.insert(
"a.bin".to_string(),
bundle_manifest::SavedFile {
hash: hash.clone(),
size,
mtime: SavedMtime::of(&meta).expect("mtime"),
chunks: None,
},
);
bundle_manifest::merge_and_write(tmp.path(), SavedManifest::default(), updates)
.expect("write saved manifest");
let saved = bundle_manifest::load(tmp.path());
let entries = [ManifestEntry {
path: "a.bin".into(),
hash,
size: Some(size - 7),
chunks: None,
}];
let st = resolve_disk_state(&entries, &saved, tmp.path(), false).await;
assert!(st.skip.contains("a.bin"), "accepted by its hash");
}
#[tokio::test]
async fn resolve_disk_state_rehash_confirms_touched_but_identical_file() {
let tmp = tempfile::tempdir().expect("tmp");
let body = b"unchanged content";
std::fs::write(tmp.path().join("a.txt"), body).expect("write");
let real = format!("b3:{}", blake3::hash(body).to_hex());
let size = u64::try_from(body.len()).expect("len");
let stale = SavedMtime { secs: 1, nanos: 0 };
let meta = std::fs::metadata(tmp.path().join("a.txt")).expect("meta");
assert_ne!(
stale,
SavedMtime::of(&meta).expect("mtime"),
"stale mtime must differ from the file's real mtime"
);
let mut updates = BTreeMap::new();
updates.insert(
"a.txt".to_string(),
bundle_manifest::SavedFile {
hash: real.clone(),
size,
mtime: stale,
chunks: None,
},
);
bundle_manifest::merge_and_write(tmp.path(), SavedManifest::default(), updates)
.expect("write saved manifest");
let saved = bundle_manifest::load(tmp.path());
let entries = [ManifestEntry {
path: "a.txt".into(),
hash: real,
size: Some(size),
chunks: None,
}];
let st = resolve_disk_state(&entries, &saved, tmp.path(), false).await;
assert!(st.skip.contains("a.txt"));
}
#[tokio::test]
async fn resolve_disk_state_fetches_changed_file() {
let tmp = tempfile::tempdir().expect("tmp");
std::fs::write(tmp.path().join("a.txt"), b"OLD CONTENT").expect("write");
let new = format!("b3:{}", blake3::hash(b"NEW CONTENT").to_hex());
let entries = [ManifestEntry {
path: "a.txt".into(),
hash: new,
size: Some(11),
chunks: None,
}];
let st = resolve_disk_state(&entries, &SavedManifest::default(), tmp.path(), false).await;
assert!(!st.skip.contains("a.txt")); }
#[tokio::test]
async fn resolve_disk_state_absent_file_fetches() {
let tmp = tempfile::tempdir().expect("tmp");
let entries = [ManifestEntry {
path: "missing.txt".into(),
hash: "b3:00".into(),
size: Some(1),
chunks: None,
}];
let st = resolve_disk_state(&entries, &SavedManifest::default(), tmp.path(), false).await;
assert!(!st.skip.contains("missing.txt"));
}
#[tokio::test]
async fn resolve_disk_state_overwrite_skips_nothing() {
let tmp = tempfile::tempdir().expect("tmp");
let body = b"hello world";
std::fs::write(tmp.path().join("a.txt"), body).expect("write");
let h = format!("b3:{}", blake3::hash(body).to_hex());
let entries = [ManifestEntry {
path: "a.txt".into(),
hash: h,
size: Some(u64::try_from(body.len()).expect("len")),
chunks: None,
}];
let st = resolve_disk_state(&entries, &SavedManifest::default(), tmp.path(), true).await;
assert!(st.skip.is_empty());
}
#[tokio::test]
async fn resolve_disk_state_indexes_whole_file_donor_from_other_path() {
let tmp = tempfile::tempdir().expect("tmp");
let body = vec![7u8; 4096];
std::fs::create_dir_all(tmp.path().join("game1/lib")).expect("mkdir");
std::fs::write(tmp.path().join("game1/lib/dup.dll"), &body).expect("write");
let h = format!("b3:{}", blake3::hash(&body).to_hex());
let old_entries = [ManifestEntry {
path: "game1/lib/dup.dll".into(),
hash: h.clone(),
size: Some(4096),
chunks: None,
}];
let updates = build_completed_updates(
&old_entries.iter().collect::<Vec<_>>(),
&[EntryOutcome::Fetched(4096)],
tmp.path(),
);
bundle_manifest::merge_and_write(tmp.path(), SavedManifest::default(), updates)
.expect("write saved");
let saved = bundle_manifest::load(tmp.path());
let new_entries = vec![ManifestEntry {
path: "game2/lib/dup.dll".into(),
hash: h.clone(),
size: Some(4096),
chunks: None,
}];
let st = resolve_disk_state(&new_entries, &saved, tmp.path(), false).await;
let want = fetch::parse_hash(&h).expect("hash");
assert_eq!(
st.whole_file.get(&want),
Some(&tmp.path().join("game1/lib/dup.dll"))
);
}
#[tokio::test]
async fn resolve_disk_state_excludes_will_write_path_from_whole_file_index() {
let tmp = tempfile::tempdir().expect("tmp");
let old_body = vec![9u8; 32];
std::fs::write(tmp.path().join("changed.bin"), &old_body).expect("write old");
let old_hash = format!("b3:{}", blake3::hash(&old_body).to_hex());
let old_entries = [ManifestEntry {
path: "changed.bin".into(),
hash: old_hash.clone(),
size: Some(32),
chunks: None,
}];
let updates = build_completed_updates(
&old_entries.iter().collect::<Vec<_>>(),
&[EntryOutcome::Fetched(32)],
tmp.path(),
);
bundle_manifest::merge_and_write(tmp.path(), SavedManifest::default(), updates)
.expect("write saved");
let saved = bundle_manifest::load(tmp.path());
let new_hash = format!("b3:{}", blake3::hash(&[1u8; 32]).to_hex());
let new_entries = vec![ManifestEntry {
path: "changed.bin".into(),
hash: new_hash,
size: Some(32),
chunks: None,
}];
let st = resolve_disk_state(&new_entries, &saved, tmp.path(), false).await;
assert!(
!st.skip.contains("changed.bin"),
"changed content must fetch"
);
let old_want = fetch::parse_hash(&old_hash).expect("hash");
assert_eq!(
st.whole_file.get(&old_want),
None,
"a will-write path must never be indexed as a whole-file donor"
);
}
#[tokio::test]
async fn resolve_disk_state_overwrite_builds_no_whole_file_index() {
let tmp = tempfile::tempdir().expect("tmp");
let body = vec![7u8; 16];
std::fs::write(tmp.path().join("a.bin"), &body).expect("write");
let h = format!("b3:{}", blake3::hash(&body).to_hex());
let old = [ManifestEntry {
path: "a.bin".into(),
hash: h.clone(),
size: Some(16),
chunks: None,
}];
let updates = build_completed_updates(
&old.iter().collect::<Vec<_>>(),
&[EntryOutcome::Fetched(16)],
tmp.path(),
);
bundle_manifest::merge_and_write(tmp.path(), SavedManifest::default(), updates).expect("w");
let saved = bundle_manifest::load(tmp.path());
let st = resolve_disk_state(&old, &saved, tmp.path(), true).await;
assert!(st.whole_file.is_empty());
}
#[cfg(unix)]
#[tokio::test]
async fn resolve_disk_state_does_not_skip_a_symlink() {
let tmp = tempfile::tempdir().expect("tmp");
let body = b"hello world";
std::fs::write(tmp.path().join("target.bin"), body).expect("write target");
std::os::unix::fs::symlink(tmp.path().join("target.bin"), tmp.path().join("link.txt"))
.expect("symlink");
let h = format!("b3:{}", blake3::hash(body).to_hex());
let entries = [ManifestEntry {
path: "link.txt".into(),
hash: h,
size: Some(u64::try_from(body.len()).expect("len")),
chunks: None,
}];
let st = resolve_disk_state(&entries, &SavedManifest::default(), tmp.path(), false).await;
assert!(!st.skip.contains("link.txt"));
assert!(st.seed.is_empty());
}
#[tokio::test]
async fn resolve_disk_state_seeds_unchanged_and_old_chunks() {
let tmp = tempfile::tempdir().expect("tmp");
let body = vec![9u8; 20];
std::fs::write(tmp.path().join("u.bin"), &body).expect("write");
let uh = format!("b3:{}", blake3::hash(&body).to_hex());
let ch = format!("b3:{}", blake3::hash(&body).to_hex()); let entries = [ManifestEntry {
path: "u.bin".into(),
hash: uh,
size: Some(20),
chunks: Some(vec![ManifestChunk { hash: ch, size: 20 }]),
}];
let st = resolve_disk_state(&entries, &SavedManifest::default(), tmp.path(), false).await;
assert!(st.skip.contains("u.bin"));
assert_eq!(st.seed.len(), 1);
assert_eq!(st.seed[0].offset, 0);
assert_eq!(st.seed[0].len, 20);
}
#[tokio::test]
async fn resolve_disk_state_seeds_old_chunks_of_a_changed_file() {
let tmp = tempfile::tempdir().expect("tmp");
let old_body = vec![1u8; 20];
let path = tmp.path().join("c.bin");
std::fs::write(&path, &old_body).expect("write old");
let old_hash = format!("b3:{}", blake3::hash(&old_body).to_hex());
let old_chunk_hash = old_hash.clone(); let old_entries = [ManifestEntry {
path: "c.bin".into(),
hash: old_hash,
size: Some(20),
chunks: Some(vec![ManifestChunk {
hash: old_chunk_hash.clone(),
size: 20,
}]),
}];
let old_outcomes = vec![EntryOutcome::Fetched(20)];
let old_refs: Vec<&ManifestEntry> = old_entries.iter().collect();
let updates = build_completed_updates(&old_refs, &old_outcomes, tmp.path());
bundle_manifest::merge_and_write(tmp.path(), SavedManifest::default(), updates)
.expect("write saved manifest");
let saved = bundle_manifest::load(tmp.path());
let new_hash = format!("b3:{}", blake3::hash(&[2u8; 20]).to_hex());
let new_entries = vec![ManifestEntry {
path: "c.bin".into(),
hash: new_hash,
size: Some(20),
chunks: None,
}];
let st = resolve_disk_state(&new_entries, &saved, tmp.path(), false).await;
assert!(!st.skip.contains("c.bin"), "changed content must fetch");
assert!(!st.seed.is_empty());
let want_hash = fetch::parse_hash(&old_chunk_hash).expect("parse old chunk hash");
for donor in &st.seed {
assert_eq!(donor.hash, want_hash);
assert_eq!(donor.source, path);
assert_eq!(donor.offset, 0);
assert_eq!(donor.len, 20);
}
}
#[test]
fn materialize_replaces_dest_and_keeps_staging() {
let dir = tempfile::tempdir().unwrap();
let staging = dir.path().join("blob.partial");
std::fs::write(&staging, b"new-content").unwrap();
let dest = dir.path().join("out.bin");
std::fs::write(&dest, b"stale-old").unwrap();
let n = materialize(&staging, &dest).unwrap();
assert_eq!(n, b"new-content".len() as u64);
assert_eq!(std::fs::read(&dest).unwrap(), b"new-content");
assert!(staging.exists(), "staging must survive for retries");
assert_eq!(std::fs::read(&staging).unwrap(), b"new-content");
}
#[test]
fn first_write_moves_a_non_donor_blob_into_place() {
let dir = tempfile::tempdir().unwrap();
let staging = dir.path().join("blob");
std::fs::write(&staging, b"new-content").unwrap();
let dest = dir.path().join("sub").join("out.bin");
std::fs::create_dir_all(dest.parent().unwrap()).unwrap();
std::fs::write(&dest, b"stale-old").unwrap();
let n = first_write(&staging, &dest, false).unwrap();
assert_eq!(n, b"new-content".len() as u64);
assert_eq!(std::fs::read(&dest).unwrap(), b"new-content");
assert!(!staging.exists(), "the blob moved, it was not copied");
}
#[cfg(unix)]
#[test]
fn first_write_links_a_donor_blob_and_keeps_staging() {
use std::os::unix::fs::MetadataExt;
let dir = tempfile::tempdir().unwrap();
let staging = dir.path().join("blob");
std::fs::write(&staging, b"donor-bytes").unwrap();
let dest = dir.path().join("out.bin");
let n = first_write(&staging, &dest, true).unwrap();
assert_eq!(n, b"donor-bytes".len() as u64);
assert_eq!(std::fs::read(&dest).unwrap(), b"donor-bytes");
assert!(staging.exists(), "a donor's staging name survives");
assert_eq!(
std::fs::metadata(&staging).unwrap().ino(),
std::fs::metadata(&dest).unwrap().ino(),
"the first destination is a hard link, not a copy"
);
}
#[tokio::test]
async fn materialize_group_fetches_once_and_links_the_rest() {
let dir = tempfile::tempdir().unwrap();
let slots = vec![
Slot::Write {
label: "a",
dest: dir.path().join("a"),
},
Slot::Write {
label: "b",
dest: dir.path().join("b"),
},
];
let mat_calls = std::cell::Cell::new(0u32);
let link_calls = std::cell::Cell::new(0u32);
let outcomes = materialize_group(
slots,
|_dest| {
mat_calls.set(mat_calls.get() + 1);
async { Ok(7u64) }
},
|_src, _dest| {
link_calls.set(link_calls.get() + 1);
async { Ok(()) }
},
)
.await;
assert_eq!(mat_calls.get(), 1, "the paid path runs exactly once");
assert_eq!(
link_calls.get(),
1,
"the duplicate is linked, not re-fetched"
);
assert!(matches!(outcomes[0], EntryOutcome::Fetched(7)));
assert!(matches!(outcomes[1], EntryOutcome::Linked));
}
#[tokio::test]
async fn materialize_group_retries_next_path_when_first_write_fails() {
let dir = tempfile::tempdir().unwrap();
let slots = vec![
Slot::Write {
label: "a",
dest: dir.path().join("a"),
},
Slot::Write {
label: "b",
dest: dir.path().join("b"),
},
];
let mat_calls = std::cell::Cell::new(0u32);
let link_calls = std::cell::Cell::new(0u32);
let outcomes = materialize_group(
slots,
|_dest| {
let n = mat_calls.get();
mat_calls.set(n + 1);
async move {
if n == 0 {
Err(anyhow!("first write failed"))
} else {
Ok(5u64)
}
}
},
|_src, _dest| {
link_calls.set(link_calls.get() + 1);
async { Ok(()) }
},
)
.await;
assert_eq!(
mat_calls.get(),
2,
"the second writable path retries materialize"
);
assert_eq!(
link_calls.get(),
0,
"no canonical to link from until one succeeds"
);
assert!(matches!(outcomes[0], EntryOutcome::Failed { .. }));
assert!(matches!(outcomes[1], EntryOutcome::Fetched(5)));
}
#[tokio::test]
async fn materialize_from_donor_links_every_destination() {
let tmp = tempfile::tempdir().expect("tmp");
let donor = tmp.path().join("game1/lib/dup.dll");
std::fs::create_dir_all(donor.parent().expect("parent")).expect("mkdir");
let body = vec![3u8; 2048];
std::fs::write(&donor, &body).expect("write donor");
let dest = tmp.path().join("game2/lib/dup.dll");
let slots = vec![Slot::Write {
label: "game2/lib/dup.dll",
dest: dest.clone(),
}];
let outcomes = materialize_from_donor(slots, &donor, 2048).await;
assert!(matches!(outcomes.as_slice(), [EntryOutcome::Deduped(2048)]));
assert_eq!(std::fs::read(&dest).expect("read"), body);
}
#[tokio::test]
async fn materialize_from_donor_links_second_destination_as_linked() {
let tmp = tempfile::tempdir().expect("tmp");
let donor = tmp.path().join("donor.bin");
std::fs::write(&donor, b"same-bytes").expect("write donor");
let dest_a = tmp.path().join("a/out.bin");
let dest_b = tmp.path().join("b/out.bin");
let slots = vec![
Slot::Write {
label: "a/out.bin",
dest: dest_a.clone(),
},
Slot::Write {
label: "b/out.bin",
dest: dest_b.clone(),
},
];
let outcomes = materialize_from_donor(slots, &donor, 10).await;
assert!(matches!(outcomes[0], EntryOutcome::Deduped(10)));
assert!(matches!(outcomes[1], EntryOutcome::Linked));
assert_eq!(std::fs::read(&dest_a).expect("read a"), b"same-bytes");
assert_eq!(std::fs::read(&dest_b).expect("read b"), b"same-bytes");
}
#[tokio::test]
async fn materialize_from_donor_preserves_failed_slot() {
let tmp = tempfile::tempdir().expect("tmp");
let donor = tmp.path().join("donor.bin");
std::fs::write(&donor, b"bytes").expect("write donor");
let dest = tmp.path().join("out.bin");
let slots = vec![
Slot::Failed(EntryOutcome::failed("bad", &anyhow!("resolve failed"))),
Slot::Write {
label: "out.bin",
dest: dest.clone(),
},
];
let outcomes = materialize_from_donor(slots, &donor, 5).await;
assert!(matches!(outcomes[0], EntryOutcome::Failed { .. }));
assert!(matches!(outcomes[1], EntryOutcome::Deduped(5)));
}
#[test]
fn report_counts_linked_without_failing() {
let outcomes = vec![
EntryOutcome::Fetched(10),
EntryOutcome::Linked,
EntryOutcome::Skipped,
];
assert!(
report(
&outcomes,
&[],
Transfer::default(),
DedupSummary::default(),
0,
Path::new("/out"),
false
)
.is_ok()
);
}
#[test]
fn bundle_pull_namespace_flag_parses_and_rejects_zero() {
use clap::Parser;
#[derive(Parser)]
struct T {
#[command(flatten)]
a: decdn_common::cli::BundlePullArgs,
}
let ok = T::try_parse_from(["t", "-o", "out", "--hash", "b3:aa", "--namespace", "7"])
.expect("valid namespace parses");
assert_eq!(ok.a.namespace, Some(7));
assert!(
T::try_parse_from(["t", "-o", "out", "--hash", "b3:aa", "--namespace", "0"]).is_err(),
"namespace 0 is the reserved sentinel and must be rejected"
);
let none =
T::try_parse_from(["t", "-o", "out", "--hash", "b3:aa"]).expect("namespace is optional");
assert_eq!(none.a.namespace, None, "absent flag stays None");
}
#[tokio::test]
async fn dry_run_still_rejects_a_dangling_provider_address() {
use clap::Parser;
#[derive(Parser)]
struct T {
#[command(flatten)]
a: decdn_common::cli::BundlePullArgs,
}
let addr = "0x0000000000000000000000000000000000000001";
let args = T::parse_from([
"t",
"-o",
"out",
"--hash",
"b3:aa",
"--dry-run",
"--provider-address",
addr,
])
.a;
let err = super::bundle_pull(&args, None)
.await
.expect_err("a dangling --provider-address must be rejected even for --dry-run");
assert!(
format!("{err:#}").contains("--provider-address requires --node-id"),
"expected the validate() guard error, got: {err:#}"
);
}
fn pull_args(extra: &[&str]) -> BundlePullArgs {
use clap::Parser;
#[derive(Parser)]
struct T {
#[command(flatten)]
a: decdn_common::cli::BundlePullArgs,
}
T::parse_from(std::iter::once("t").chain(extra.iter().copied())).a
}
fn cache_manifest(out: &Path, manifest: &serde_json::Value) -> String {
let bytes = serde_json::to_vec(manifest).unwrap();
let hash = *blake3::hash(&bytes).as_bytes();
bundle_cache::store(out, hash, &bytes);
format!("b3:{}", blake3::Hash::from_bytes(hash).to_hex())
}
#[test]
fn dry_run_reads_a_cached_hash_manifest() {
let tmp = tempfile::tempdir().expect("tmp");
let out = tmp.path().to_str().expect("utf-8 tmp path");
let hash = cache_manifest(
tmp.path(),
&serde_json::json!({
"version": 1,
"entries": [
{"path": "openai-gpt-oss-20b/config.json", "hash": "b3:aa", "size": 10},
{"path": "openai-gpt-oss-20b/metal/model.bin", "hash": "b3:bb", "size": 20},
],
}),
);
for overwrite in [false, true] {
let mut argv = vec!["-o", out, "--hash", &hash, "--dry-run"];
if overwrite {
argv.push("--overwrite");
}
let manifest = dry_run_manifest(&pull_args(&argv))
.unwrap()
.expect("the cached manifest is read");
assert_eq!(
manifest
.entries
.iter()
.map(|e| e.path.as_str())
.collect::<Vec<_>>(),
vec![
"openai-gpt-oss-20b/config.json",
"openai-gpt-oss-20b/metal/model.bin"
],
"overwrite={overwrite}"
);
}
}
#[tokio::test]
async fn dry_run_has_no_manifest_for_an_uncached_hash() {
let tmp = tempfile::tempdir().expect("tmp");
let out = tmp.path().to_str().expect("utf-8 tmp path");
let hash = format!("b3:{}", "ab".repeat(32));
let args = pull_args(&["-o", out, "--hash", &hash, "--dry-run"]);
assert!(dry_run_manifest(&args).unwrap().is_none());
super::bundle_pull(&args, None).await.unwrap();
}
#[tokio::test]
async fn dry_run_fails_on_a_cached_manifest_that_does_not_parse() {
let tmp = tempfile::tempdir().expect("tmp");
let out = tmp.path().to_str().expect("utf-8 tmp path");
let hash = cache_manifest(
tmp.path(),
&serde_json::json!({"version": 2, "entries": []}),
);
let args = pull_args(&["-o", out, "--hash", &hash, "--dry-run"]);
let err = super::bundle_pull(&args, None)
.await
.expect_err("a cached non-v1 manifest fails the dry run");
let msg = format!("{err:#}");
assert!(msg.contains("unsupported bundle version 2"), "{msg}");
assert!(msg.contains(bundle_cache::CACHE_DIR), "{msg}");
}
#[tokio::test]
async fn dry_run_fails_naming_an_unreadable_cached_manifest() {
let tmp = tempfile::tempdir().expect("tmp");
let out = tmp.path().to_str().expect("utf-8 tmp path");
let raw = [0xab_u8; 32];
let hex = blake3::Hash::from_bytes(raw).to_hex();
std::fs::create_dir_all(bundle_cache::cache_path(tmp.path(), raw)).expect("mkdir");
let hash = format!("b3:{hex}");
let args = pull_args(&["-o", out, "--hash", &hash, "--dry-run"]);
let err = super::bundle_pull(&args, None)
.await
.expect_err("an unreadable cached manifest fails the dry run");
let msg = format!("{err:#}");
assert!(msg.contains(bundle_cache::CACHE_DIR), "{msg}");
assert!(msg.contains(hex.as_str()), "{msg}");
assert!(msg.contains("os error"), "{msg}");
}
#[tokio::test]
async fn dry_run_fails_naming_a_cached_manifest_that_fails_its_hash_check() {
let tmp = tempfile::tempdir().expect("tmp");
let out = tmp.path().to_str().expect("utf-8 tmp path");
let hash = cache_manifest(
tmp.path(),
&serde_json::json!({"version": 1, "entries": []}),
);
let raw = fetch::parse_hash(&hash).expect("hash");
std::fs::write(bundle_cache::cache_path(tmp.path(), raw), b"tampered").expect("tamper");
let args = pull_args(&["-o", out, "--hash", &hash, "--dry-run"]);
let err = super::bundle_pull(&args, None)
.await
.expect_err("a tampered cached manifest fails the dry run");
let msg = format!("{err:#}");
assert!(msg.contains("does not hash to the bundle"), "{msg}");
assert!(msg.contains(bundle_cache::CACHE_DIR), "{msg}");
}
#[test]
fn cached_manifest_reads_the_cache_unless_overwrite() {
let tmp = tempfile::tempdir().expect("tmp");
let out = tmp.path().to_str().expect("utf-8 tmp path");
let manifest = serde_json::json!({"version": 1, "entries": []});
let hash = cache_manifest(tmp.path(), &manifest);
let raw = fetch::parse_hash(&hash).expect("hash");
let cached = cached_manifest(&pull_args(&["-o", out, "--hash", &hash]), raw);
assert_eq!(cached, Some(serde_json::to_vec(&manifest).unwrap()));
let args = pull_args(&["-o", out, "--hash", &hash, "--overwrite"]);
assert_eq!(cached_manifest(&args, raw), None);
}
#[test]
fn cached_manifest_falls_back_to_a_fetch_on_an_unreadable_cache_file() {
let tmp = tempfile::tempdir().expect("tmp");
let out = tmp.path().to_str().expect("utf-8 tmp path");
let raw = [0xcd_u8; 32];
std::fs::create_dir_all(bundle_cache::cache_path(tmp.path(), raw)).expect("mkdir");
let hash = format!("b3:{}", blake3::Hash::from_bytes(raw).to_hex());
assert_eq!(
cached_manifest(&pull_args(&["-o", out, "--hash", &hash]), raw),
None
);
}
#[tokio::test]
async fn dry_run_rejects_a_malformed_hash() {
let tmp = tempfile::tempdir().expect("tmp");
let out = tmp.path().to_str().expect("utf-8 tmp path");
let args = pull_args(&["-o", out, "--hash", "b3:aa", "--dry-run"]);
let err = super::bundle_pull(&args, None)
.await
.expect_err("a malformed --hash fails the dry run");
assert!(format!("{err:#}").contains("invalid hash"), "{err:#}");
}
fn stripe_candidate(node: u8, operator: u8) -> NodeCandidate {
NodeCandidate {
node_id: iroh::SecretKey::from_bytes(&[node; 32]).public(),
eth_address: Address::from([operator; 20]),
region_hint: None,
multiaddrs: alloy::primitives::Bytes::new(),
}
}
#[test]
fn settle_group_run_records_outcomes_and_passes_a_stop_through() {
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let mut groups = vec![SettledGroup::default(), SettledGroup::default()];
let slot = || {
vec![Slot::Write {
label: "a.bin",
dest: PathBuf::from("a.bin"),
}]
};
let failed = GroupRun::fetch_failed(slot(), anyhow!("stream reset"));
assert!(settle_group_run(&mut groups, &tx, 0, failed, BTreeMap::new(), Vec::new()).is_none());
assert!(matches!(
groups[0].outcomes.as_slice(),
[EntryOutcome::Failed { .. }]
));
assert_eq!(groups[0].bytes, None, "a failed fetch paid for nothing");
assert!(rx.try_recv().is_err(), "nothing recorded, nothing flushed");
let disk = anyhow::Error::new(std::io::Error::from(std::io::ErrorKind::StorageFull))
.context("write staging");
let stopped = settle_group_run(
&mut groups,
&tx,
0,
GroupRun::fetch_failed(slot(), disk),
BTreeMap::new(),
Vec::new(),
);
assert!(stopped.is_some(), "a local disk fault ends the pull");
let mut updates = BTreeMap::new();
updates.insert(
"a.bin".to_string(),
bundle_manifest::SavedFile {
hash: "b3:00".into(),
size: 3,
mtime: SavedMtime { secs: 1, nanos: 0 },
chunks: None,
},
);
let landed = GroupRun::landed(vec![EntryOutcome::Fetched(3)], EntryBytes::whole_blob(3, 1));
assert!(settle_group_run(&mut groups, &tx, 1, landed, updates, Vec::new()).is_none());
assert!(matches!(
groups[1].outcomes.as_slice(),
[EntryOutcome::Fetched(3)]
));
assert_eq!(
groups[1].bytes,
Some(EntryBytes::whole_blob(3, 1)),
"the landed paid and resumed tallies are kept"
);
assert_eq!(rx.try_recv().expect("a flush batch").fetched_bytes, 3);
}
#[tokio::test]
async fn a_lane_widens_only_into_free_permits_and_gives_them_back() {
use decdn_client::GrowFor::{Extra, Restart};
let p1 = Address::repeat_byte(1);
let cap = LaneStreamCap::new(3);
let lease = cap.try_permit(p1).await.expect("the lane's own permit");
let extras = cap.extra_permits(p1).await;
assert!(extras.grant(Extra), "one extra stream");
assert!(!extras.grant(Extra), "the last free permit is kept back");
assert!(extras.grant(Restart), "a restart takes the last permit");
assert!(!extras.grant(Restart), "the cap is full");
extras.release_one();
assert!(extras.grant(Restart), "a released permit is free again");
extras.release_one();
extras.release_one();
drop(lease);
let all: Vec<_> = futures_util::future::join_all((0..3).map(|_| cap.try_permit(p1)))
.await
.into_iter()
.flatten()
.collect();
assert_eq!(all.len(), 3, "every permit came back");
}
#[tokio::test]
async fn a_cap_of_two_still_grants_an_extra_stream() {
let p1 = Address::repeat_byte(1);
let cap = LaneStreamCap::new(2);
let _lease = cap.try_permit(p1).await.expect("the lane's own permit");
let extras = cap.extra_permits(p1).await;
assert!(
extras.grant(decdn_client::GrowFor::Extra),
"the extra stream"
);
assert!(cap.try_permit(p1).await.is_none(), "the cap is full");
extras.release_one();
}
#[tokio::test]
async fn a_lane_running_a_stream_cannot_restart_into_a_claimed_permit() {
use decdn_client::GrowFor::{Extra, Restart};
let p1 = Address::repeat_byte(1);
let cap = LaneStreamCap::new(4);
let _b_lease = cap.try_permit(p1).await.expect("lane B's own permit");
let a_lease = cap.try_permit(p1).await.expect("lane A's own permit");
let a = cap.extra_permits(p1).await;
let b = cap.extra_permits(p1).await;
assert!(a.grant(Extra), "lane A's extra stream");
let c_lease = cap.try_permit(p1).await.expect("lane C's own permit");
assert!(!b.grant(Extra), "the cap is full: lane B claims");
drop(a_lease);
assert!(
!a.grant(Restart),
"lane A, still on its extra stream, waits behind lane B's claim"
);
assert!(!a.grant(Extra), "an extra stream waits too");
drop(c_lease);
assert!(
!a.grant(Restart),
"lane B's claim and the kept permit hold both free permits"
);
assert!(b.grant(Extra), "lane B takes the permit it claimed");
assert!(
a.grant(Restart),
"with lane B served, lane A's own claim comes first"
);
a.release_one();
a.release_one();
b.release_one();
}
#[tokio::test]
async fn a_lane_that_runs_no_stream_goes_ahead_of_a_claim_for_one_more() {
use decdn_client::GrowFor::{Extra, Restart};
let p1 = Address::repeat_byte(1);
let cap = LaneStreamCap::new(4);
let _b_lease = cap.try_permit(p1).await.expect("lane B's own permit");
let b = cap.extra_permits(p1).await;
assert!(b.grant(Extra), "lane B's first extra stream");
assert!(b.grant(Extra), "lane B's second extra stream");
assert!(
!b.grant(Extra),
"the last free permit is kept: lane B claims"
);
let a = cap.extra_permits(p1).await;
assert!(
a.grant(Restart),
"lane A, which gave its lease back, starts again on the free permit"
);
a.release_one();
b.release_one();
b.release_one();
}
#[tokio::test]
async fn an_ended_lane_claims_nothing() {
use decdn_client::GrowFor::{Extra, Restart};
let p1 = Address::repeat_byte(1);
let cap = LaneStreamCap::new(5);
let a_lease = cap.try_permit(p1).await.expect("lane A's own permit");
let _b_lease = cap.try_permit(p1).await.expect("lane B's own permit");
let _c_lease = cap.try_permit(p1).await.expect("lane C's own permit");
let a = cap.extra_permits(p1).await;
let b = cap.extra_permits(p1).await;
let c = cap.extra_permits(p1).await;
assert!(a.grant(Extra), "lane A's extra stream");
assert!(!b.grant(Extra), "lane B claims");
assert!(!c.grant(Extra), "lane C claims");
drop(a_lease);
assert!(
!a.grant(Restart),
"two claims and the kept permit hold three"
);
drop(b);
assert!(
!a.grant(Restart),
"lane B ended, and lane C's claim still holds a permit"
);
drop(c);
assert!(a.grant(Restart), "no claim is left ahead of lane A");
a.release_one();
a.release_one();
}
fn fill(streams: &ProviderStreams, n: usize) -> Vec<tokio::sync::OwnedSemaphorePermit> {
(0..n)
.map(|_| {
Arc::clone(&streams.semaphore)
.try_acquire_owned()
.expect("a free permit")
})
.collect()
}
#[test]
fn each_claim_ahead_holds_back_one_permit() {
let (b, c, d) = (1, 2, 3);
let streams = ProviderStreams::new(4);
let mut held = fill(&streams, 4);
assert!(streams.take(b, 0, false).is_none(), "lane B claims first");
assert!(streams.take(c, 0, false).is_none(), "lane C claims second");
held.truncate(2);
assert!(
streams.take(d, 0, false).is_none(),
"two claims ahead hold both free permits"
);
let mut reheld = fill(&streams, 1);
assert!(
streams.take(c, 0, false).is_none(),
"lane C waits behind lane B"
);
assert!(
streams.take(b, 0, false).is_some(),
"lane B is served first"
);
reheld.clear();
assert!(streams.take(c, 0, false).is_some(), "lane C is served next");
}
#[test]
fn a_later_ask_takes_only_what_the_claims_ahead_leave() {
let (b, c, d) = (1, 2, 3);
let streams = ProviderStreams::new(5);
let mut held = fill(&streams, 5);
assert!(streams.take(b, 1, false).is_none(), "lane B claims");
assert!(streams.take(c, 1, false).is_none(), "lane C claims");
held.truncate(1);
assert!(
streams.take(d, 0, false).is_some(),
"two claims and one kept permit leave a fourth free permit"
);
assert!(
streams.take(b, 1, false).is_some(),
"lane B is still served"
);
assert!(streams.take(c, 1, false).is_some(), "and lane C after it");
}
#[test]
fn a_renewed_claim_keeps_its_place() {
let (b, c) = (1, 2);
let streams = ProviderStreams::new(2);
let mut held = fill(&streams, 2);
assert!(streams.take(b, 0, false).is_none(), "lane B claims first");
assert!(streams.take(c, 0, false).is_none(), "lane C claims second");
assert!(streams.take(b, 0, false).is_none(), "lane B asks again");
held.truncate(1);
assert!(
streams.take(c, 0, false).is_none(),
"lane C still waits behind lane B"
);
assert!(
streams.take(b, 0, false).is_some(),
"lane B is served first"
);
}
#[test]
fn a_renewed_claim_takes_its_last_asks_keep() {
let (b, d) = (1, 2);
for (first, last, granted) in [(1, 0, true), (0, 1, false)] {
let streams = ProviderStreams::new(3);
let mut held = fill(&streams, 3);
assert!(streams.take(b, first, false).is_none(), "lane B claims");
assert!(streams.take(b, last, false).is_none(), "lane B asks again");
held.truncate(1);
assert_eq!(
streams.take(d, 0, false).is_some(),
granted,
"lane B's claim keeps {last} after its own permit"
);
}
}
#[test]
fn a_bare_ask_waits_only_behind_bare_claims() {
let (b, x, y) = (1, 2, 3);
let streams = ProviderStreams::new(3);
let mut held = fill(&streams, 3);
assert!(
streams.take(b, 1, false).is_none(),
"lane B claims one more"
);
assert!(
streams.take(x, 0, true).is_none(),
"lane X, with none, claims"
);
held.truncate(2);
assert!(
streams.take(y, 0, true).is_none(),
"lane Y, with none, waits behind lane X"
);
assert!(
streams.take(x, 0, true).is_some(),
"lane X goes ahead of lane B's claim"
);
}
#[test]
fn a_lane_is_not_held_back_by_its_own_claim() {
let a = 1;
let streams = ProviderStreams::new(2);
let _held = fill(&streams, 1);
assert!(
streams.take(a, 1, false).is_none(),
"the kept permit is not an extra stream's"
);
assert!(
streams.take(a, 0, false).is_some(),
"lane A's restart takes it past its own claim"
);
}
#[tokio::test(start_paused = true)]
async fn a_claim_lapses_when_its_lane_stops_asking() {
let (a, b) = (1, 2);
let streams = ProviderStreams::new(2);
let mut held = fill(&streams, 2);
assert!(streams.take(b, 1, false).is_none(), "lane B claims");
held.truncate(1);
assert!(
streams.take(a, 0, false).is_none(),
"lane B's claim holds the free permit"
);
tokio::time::advance(CLAIM_TTL / 2).await;
assert!(
streams.take(b, 1, false).is_none(),
"lane B asks again and renews its claim"
);
tokio::time::advance(CLAIM_TTL / 2 + std::time::Duration::from_millis(1)).await;
assert!(
streams.take(a, 0, false).is_none(),
"a renewed claim holds past the first ask's lapse"
);
tokio::time::advance(CLAIM_TTL).await;
assert!(
streams.take(a, 0, false).is_some(),
"a claim not renewed lapses"
);
}
#[tokio::test]
async fn lane_stream_cap_serializes_one_provider_and_frees_the_rest() {
let p1 = Address::repeat_byte(1);
let p2 = Address::repeat_byte(2);
let cap = LaneStreamCap::new(1);
let held = cap.try_permit(p1).await.unwrap();
assert!(
cap.try_permit(p1).await.is_none(),
"a second same-provider permit is taken while the first is held"
);
assert!(
cap.try_permit(p2).await.is_some(),
"a distinct provider does not share P1's permit"
);
drop(held);
assert!(
cap.try_permit(p1).await.is_some(),
"the permit is free once the first drops"
);
let cap2 = LaneStreamCap::new(2);
let a = cap2.try_permit(p1).await.unwrap();
let b = cap2
.try_permit(p1)
.await
.expect("two same-provider permits coexist at n == 2");
drop((a, b));
}
fn test_blob(len: usize) -> Vec<u8> {
(0..len)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect()
}
fn scripted_holder(
blob: &[u8],
provider: u8,
blip: Option<Arc<std::sync::atomic::AtomicBool>>,
) -> anyhow::Result<decdn_client::StreamCandidate<decdn_client::source::ScriptedSource>> {
let ledger = Arc::new(decdn_client::PoolLedger::new(
decdn_client::Cumulative::default(),
));
let mut source =
decdn_client::source::ScriptedSource::new(blob.to_vec())?.paying(Arc::clone(&ledger));
if let Some(fired) = blip {
source = source.fault_once_after(256 * 1024, move || {
fired.store(true, std::sync::atomic::Ordering::SeqCst);
anyhow!("connection reset")
});
}
Ok(decdn_client::StreamCandidate::new(
source,
Arc::new(std::sync::Mutex::new(decdn_client::PoolContext {
pool_id: alloy::primitives::B256::ZERO,
provider: Address::repeat_byte(provider),
deposit: alloy::primitives::U256::from(u128::MAX),
client_signer: Arc::new(PrivateKeySigner::random()),
voucher_domain: decdn_incentive::bind_node_id_domain(1, Address::ZERO),
prior_bytes_delivered: alloy::primitives::U256::ZERO,
prior_amount: alloy::primitives::U256::ZERO,
client_binding: None,
capability: None,
})),
ledger,
))
}
fn two_scripted_holders(
blob: &[u8],
first_blip: Option<Arc<std::sync::atomic::AtomicBool>>,
) -> anyhow::Result<(
decdn_client::StaticSources<decdn_client::source::ScriptedSource>,
Vec<decdn_client::Holder>,
)> {
let sources = decdn_client::StaticSources::new(vec![
scripted_holder(blob, 0xA1, first_blip)?,
scripted_holder(blob, 0xB2, None)?,
])?;
let holders = sources.holders();
Ok((sources, holders))
}
fn no_topups() -> decdn_client::source::FakeFunder {
decdn_client::source::FakeFunder::new(decdn_client::Recovery::Unavailable)
}
fn root(blob: &[u8]) -> [u8; 32] {
*blake3::hash(blob).as_bytes()
}
fn read_range(dest: &Path, offset: u64, len: u64) -> anyhow::Result<Vec<u8>> {
use std::io::{Read as _, Seek as _};
let mut f = std::fs::File::open(partial_path(dest))?;
f.seek(std::io::SeekFrom::Start(offset))?;
let mut buf = vec![0u8; usize::try_from(len)?];
f.read_exact(&mut buf)?;
Ok(buf)
}
#[tokio::test(start_paused = true)]
async fn a_range_entry_survives_a_holder_blip() -> anyhow::Result<()> {
let blob = test_blob(6 * 1024 * 1024);
let ranges = vec![(1024 * 1024, 2 * 1024 * 1024)];
let blipped = Arc::new(std::sync::atomic::AtomicBool::new(false));
let (provider, holders) = two_scripted_holders(&blob, Some(Arc::clone(&blipped)))?;
let staging = tempfile::tempdir()?;
let dest = staging.path().join("entry");
let downloader = Downloader::new(&provider, no_topups())
.holders(holders)
.max_lanes(2);
let stop = StopPolicy::new(
false,
Some(std::time::Duration::from_mins(1)),
Arc::default(),
);
downloader
.fetch_to_paths_until(
&[DownloadTarget {
hash: root(&blob),
total_bytes: u64::try_from(blob.len())?,
dest: &dest,
ranges: Some(&ranges),
}],
None,
&stop,
)
.await?;
assert!(
blipped.load(std::sync::atomic::Ordering::SeqCst),
"holder 0xA1 reset a stream mid-range"
);
assert_eq!(
read_range(&dest, 1024 * 1024, 2 * 1024 * 1024)?,
blob[1024 * 1024..3 * 1024 * 1024]
);
assert!(
!dest.exists(),
"an entry with donor bytes left is not promoted"
);
Ok(())
}
#[test]
fn entry_done_line_rates_a_plain_entry_on_its_download() {
let line = entry_done_line(
[1; 32],
4 << 20,
EntryBytes {
paid: 4 << 20,
spliced: 0,
resumed: 0,
},
std::time::Duration::from_secs(2),
);
assert!(line.ends_with(": 4.00 MiB in 2.0s (2.00 MiB/s)"), "{line}");
}
#[test]
fn entry_done_line_splits_a_dedup_entrys_spliced_bytes() {
let line = entry_done_line(
[1; 32],
12 << 20,
EntryBytes {
paid: 2 << 20,
spliced: 10 << 20,
resumed: 0,
},
std::time::Duration::from_secs(2),
);
assert!(
line.ends_with(
": 12.00 MiB in 2.0s (2.00 MiB downloaded at 1.00 MiB/s, 10.00 MiB spliced \
from disk)"
),
"{line}"
);
}
#[test]
fn entry_done_line_renders_a_fully_spliced_entry() {
let line = entry_done_line(
[1; 32],
12 << 20,
EntryBytes {
paid: 0,
spliced: 12 << 20,
resumed: 0,
},
std::time::Duration::from_secs(2),
);
assert!(
line.ends_with(": 12.00 MiB in 2.0s (0 B downloaded at --, 12.00 MiB spliced from disk)"),
"{line}"
);
}
#[test]
fn entry_done_line_splits_a_resumed_entrys_bytes() {
let line = entry_done_line(
[1; 32],
12 << 20,
EntryBytes {
paid: 2 << 20,
spliced: 6 << 20,
resumed: 4 << 20,
},
std::time::Duration::from_secs(2),
);
assert!(
line.ends_with(
": 12.00 MiB in 2.0s (2.00 MiB downloaded at 1.00 MiB/s, 4.00 MiB resumed, \
6.00 MiB spliced from disk)"
),
"{line}"
);
let line = entry_done_line(
[1; 32],
6 << 20,
EntryBytes {
paid: 2 << 20,
spliced: 0,
resumed: 4 << 20,
},
std::time::Duration::from_secs(2),
);
assert!(
line.ends_with(": 6.00 MiB in 2.0s (2.00 MiB downloaded at 1.00 MiB/s, 4.00 MiB resumed)"),
"{line}"
);
}
#[test]
fn entry_done_line_guards_a_zero_elapsed() {
let line = entry_done_line(
[1; 32],
1 << 20,
EntryBytes {
paid: 1 << 20,
spliced: 0,
resumed: 0,
},
std::time::Duration::ZERO,
);
assert!(line.ends_with(": 1.00 MiB in 0.0s (--)"), "{line}");
}