use std::sync::Arc;
use bytes::Bytes;
use flint_store::memory::MemoryStore;
use flint_store::{crc64_nvme, GenerationStamps, ObjectStore, PutCondition};
use super::inbox::{self, InboxEntry};
use super::lease::{self, ClaimOutcome};
use super::manifest;
use super::state::SyncerState;
use super::{now_unix, LeanConfig, LeanError, Syncer};
const PREFIX: &str = "tenant/proj1";
fn cfg_for(root: &std::path::Path) -> LeanConfig {
let mut c = LeanConfig::new(PREFIX, root);
instant_fence(&mut c);
c
}
fn instant_fence(c: &mut LeanConfig) {
c.claim_poll_secs = 0;
c.claim_quiet_spacing_secs = 0;
c.claim_deadline_secs = 2;
}
fn cfg_single(root: &std::path::Path) -> LeanConfig {
let mut c = LeanConfig::new(PREFIX, root);
c.chunked = false;
instant_fence(&mut c);
c
}
async fn syncer(store: &Arc<MemoryStore>, root: &std::path::Path) -> Syncer {
let cfg = cfg_for(root);
let state = SyncerState::open(cfg.state_dir()).unwrap();
Syncer {
store: store.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
noted_not_regular: Default::default(),
}
}
async fn claim_until_held(sc: &mut Syncer, max_steps: u32) -> bool {
for _ in 0..max_steps {
match lease::claim_step(sc, true).await.unwrap() {
ClaimOutcome::Claimed(_) => return true,
ClaimOutcome::Waiting { .. } => {}
}
}
false
}
fn write(root: &std::path::Path, rel: &str, content: &str) {
let p = root.join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, content).unwrap();
}
fn read(root: &std::path::Path, rel: &str) -> Option<String> {
std::fs::read_to_string(root.join(rel)).ok()
}
fn backdate_baseline(sc: &Syncer, rel: &str) {
let mut b = sc.state.load_baseline().unwrap();
if let Some(e) = b.entries.get_mut(rel) {
e.mtime_unix -= 10;
}
sc.state.save_baseline(&b).unwrap();
}
async fn hitl_write(
store: &Arc<MemoryStore>,
cfg: &LeanConfig,
path: &str,
content: &str,
author: &str,
) -> Result<String, LeanError> {
let key = cfg.file_key(path);
let cond = match store.head(&key).await {
Ok(meta) => {
if !inbox::hitl_may_overwrite(store.as_ref(), cfg, path, &meta.etag, meta.last_modified_unix, now_unix()).await? {
return Err(LeanError::State(format!("concurrent write: {path} is another writer's uncited upload")));
}
PutCondition::IfMatch(meta.etag)
}
Err(_) => PutCondition::IfNoneMatchAny,
};
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: format!("gateway-{author}"),
boundary_source: None,
posix: None,
};
let meta = store.put_whole(&key, body, &cond, &stamps, crc).await?;
inbox::gateway_append(
store.as_ref(),
cfg,
InboxEntry {
path: path.to_string(),
etag: meta.etag.clone(),
author: author.to_string(),
added_unix: now_unix(),
crc64_b64: Some(flint_store::crc64_to_b64(crc)),
},
)
.await?;
Ok(meta.etag)
}
#[tokio::test]
async fn checkout_publish_roundtrip_and_two_scan_delete() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap(); write(dir_a.path(), "src/main.rs", "fn main() {}");
write(dir_a.path(), "README.md", "hello");
let r = a.run_barrier().await.unwrap();
assert_eq!(r.uploaded.len(), 2);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(read(dir_b.path(), "src/main.rs").unwrap(), "fn main() {}");
std::fs::remove_file(dir_a.path().join("README.md")).unwrap();
let r1 = a.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty(), "first absence must not delete");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("README.md"));
let r2 = a.run_barrier().await.unwrap();
assert_eq!(r2.deleted, vec!["README.md".to_string()]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("README.md"));
assert!(store.head(&a.cfg.file_key("README.md")).await.is_err());
}
#[tokio::test]
async fn hitl_upload_survives_two_barriers_without_sync() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, "docs/upload.pdf", "user bytes", "dilip").await.unwrap();
write(dir.path(), "agent.txt", "agent work v2 — longer");
sc.run_barrier().await.unwrap();
sc.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "docs/upload.pdf").unwrap(), "user bytes");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("docs/upload.pdf"), "amputated!");
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "docs/upload.pdf").unwrap(), "user bytes");
}
async fn foreign_overwrite(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str, content: &str) {
let key = cfg.file_key(path);
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "a-stranger".to_string(),
boundary_source: None,
posix: None,
};
store
.put_whole(&key, body, &PutCondition::Unconditional, &stamps, crc)
.await
.expect("the foreign write lands");
}
#[tokio::test]
async fn a_published_workspace_refuses_a_foreign_write_instead_of_adopting_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.sole_writer = true;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "README.md", "the real readme");
sc.run_barrier().await.unwrap();
foreign_overwrite(&store, &sc.cfg, "README.md", "FOREIGN BYTES").await;
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
let err = sc2.checkout().await.expect_err("a published workspace must refuse");
let msg = format!("{err}");
assert!(msg.contains("SOLE WRITER"), "the refusal must say why: {msg}");
assert!(
!msg.contains("recover-staged"),
"that remedy no longer exists; nothing was staged here: {msg}"
);
assert!(
read(dir2.path(), "README.md").is_none(),
"it materialized the foreign bytes anyway"
);
}
#[tokio::test]
async fn an_ordinary_workspace_still_adopts_bytes_that_moved_past_the_manifest() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(!sc.cfg.sole_writer, "the default is an ordinary workspace");
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "README.md", "the real readme");
sc.run_barrier().await.unwrap();
foreign_overwrite(&store, &sc.cfg, "README.md", "newer human bytes").await;
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.checkout().await.expect("an ordinary workspace adopts");
assert_eq!(read(dir2.path(), "README.md").unwrap(), "newer human bytes");
}
#[tokio::test]
async fn the_published_flag_survives_the_pointer_round_trip() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.sole_writer = true;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "alpha");
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.sole_writer, "the pointer lost the flag");
write(dir.path(), "b.txt", "beta");
sc.run_barrier().await.unwrap();
let m2 = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m2.manifest.sole_writer, "the second publish dropped the flag");
}
#[tokio::test]
async fn a_published_mirror_does_not_probe_for_neighbours() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
store
.epoch_acquire(&format!("{PREFIX}/git/epoch"), "forge-1", None)
.await
.unwrap();
let seen = flint_store::layout::neighbours(
store.as_ref(),
&sc.cfg.prefix,
flint_store::layout::Writer::LeanWorkspace,
)
.await
.unwrap();
assert_eq!(seen.len(), 1, "the condition is there to be found");
assert!(!sc.cfg.sole_writer, "an ordinary workspace probes");
sc.cfg.sole_writer = true;
assert!(sc.cfg.sole_writer, "a published mirror does not");
}
#[tokio::test]
async fn ui_edit_vs_agent_edit_never_a_silent_winner() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "notes.md", "v1");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, "notes.md", "user version", "dilip").await.unwrap();
write(dir.path(), "notes.md", "agent version");
backdate_baseline(&sc, "notes.md");
sc.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "notes.md").unwrap(), "agent version");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let cited = &m.manifest.entries["notes.md"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"agent version");
let conflicts = sc.state.load_conflicts().unwrap();
let c = conflicts.iter().find(|c| c.kind == "consume-dirty").expect("conflict surfaced");
assert_eq!(c.path, "notes.md");
let preserved = c.preserved_key.as_ref().expect("foreign bytes preserved");
let (_, body) = store.get_whole(preserved, None).await.unwrap();
assert_eq!(&body[..], b"user version");
}
#[tokio::test]
async fn container_restart_never_resurrects_unpublished_delete() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "keep.txt", "keep");
write(dir.path(), "gone.txt", "delete me");
sc.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
drop(sc);
let mut sc = syncer(&store, dir.path()).await; let cr = sc.checkout().await.unwrap();
assert!(cr.resumed_live_tree, "marker present ⇒ live-tree row");
assert_eq!(cr.materialized, 0, "must not re-materialize");
assert!(read(dir.path(), "gone.txt").is_none(), "delete resurrected!");
assert!(claim_until_held(&mut sc, 1).await, "self-recognition must not wait");
sc.run_barrier().await.unwrap();
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("gone.txt"));
}
#[tokio::test]
async fn takeover_rotation_fences_the_straggler() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "from A");
a.run_barrier().await.unwrap();
let seq_before = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert!(claim_until_held(&mut a, 3).await, "a released cell is claimable at once");
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(
!claim_until_held(&mut b, 3).await,
"a fresh replacement must NOT claim instantly over a live-looking lease"
);
assert!(claim_until_held(&mut b, 10).await, "quiet polls exhausted ⇒ takeover");
let rotated = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap();
assert_eq!(rotated.manifest.seq, seq_before + 1, "rotation: seq++, content-identical");
assert_eq!(rotated.manifest.entries.len(), 1);
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "f.txt").unwrap(), "from A");
let err = lease::renew(&mut a).await.unwrap_err();
assert!(matches!(err, LeanError::Fenced(_)), "straggler must fence, got: {err:?}");
assert!(a.lease.is_none(), "a fenced holder must drop its lease");
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.seq, rotated.manifest.seq, "straggler CAS landed!");
let cited = &m.manifest.entries["f.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"from A", "straggler bytes reached a cited object");
write(dir_a.path(), "f.txt", "A, after the fence");
backdate_baseline(&a, "f.txt");
a.run_barrier().await.expect("a fenced writer's NEXT barrier publishes");
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap();
let cited = &m.manifest.entries["f.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"A, after the fence");
}
#[tokio::test]
async fn adopt_own_412_converges_without_conflict() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "big.bin", "payload");
let crashed_uuid = "crashed-barrier-uuid".to_string();
let mut intent = sc.state.load_intent().unwrap();
intent.flush_uuid = crashed_uuid.clone();
intent.keys = vec![sc.cfg.file_key("big.bin")];
sc.state.save_intent(&intent).unwrap();
sc.state.clear_intent_keys().unwrap(); let body = Bytes::from("payload");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 1,
epoch: 1,
flush_uuid: crashed_uuid,
boundary_source: None,
posix: None,
};
store
.put_whole(&sc.cfg.file_key("big.bin"), body, &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["big.bin".to_string()]);
assert!(r.parked.is_empty());
assert!(sc.state.load_conflicts().unwrap().is_empty(), "AdoptOwn must not conflict");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("big.bin"));
}
#[tokio::test]
async fn foreign_412_is_preserved_and_superseded_never_silently_overwritten() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
let body = Bytes::from("foreign bytes");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 9,
epoch: 0,
flush_uuid: "someone-else".into(),
boundary_source: None,
posix: None,
};
let foreign =
store.put_whole(&sc.cfg.file_key("f.txt"), body, &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
write(dir.path(), "f.txt", "agent bytes");
let r = sc.run_barrier().await.unwrap();
assert!(r.parked.is_empty(), "parked with no way out: {:?}", r.parked);
assert_eq!(r.uploaded, vec!["f.txt".to_string()]);
let (meta, body) = store.get_whole(&sc.cfg.file_key("f.txt"), None).await.unwrap();
assert_ne!(meta.etag, foreign.etag);
assert_eq!(&body[..], b"agent bytes");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries["f.txt"].etag, meta.etag);
let rec = sc
.state
.load_conflicts()
.unwrap()
.into_iter()
.find(|c| c.kind == "upload-412-preserved" && c.path == "f.txt")
.expect("no record surfaces the foreign write");
assert_eq!(rec.foreign_etag, foreign.etag);
let (_, kept) = store.get_whole(&rec.preserved_key.expect("not preserved"), None).await.unwrap();
assert_eq!(&kept[..], b"foreign bytes", "the preserved copy is not the foreign version");
}
#[tokio::test]
async fn gc_refuses_unrecognized_etag() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "doc.txt", "v1");
sc.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("doc.txt")).unwrap();
sc.run_barrier().await.unwrap();
let body = Bytes::from("user re-created");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "gateway-late".into(),
boundary_source: None,
posix: None,
};
let cur = store.head(&sc.cfg.file_key("doc.txt")).await.unwrap();
store
.put_whole(
&sc.cfg.file_key("doc.txt"),
body,
&PutCondition::IfMatch(cur.etag),
&stamps,
crc,
)
.await
.unwrap();
let r = sc.run_barrier().await.unwrap();
assert!(r.deleted.is_empty(), "GC deleted a foreign re-create!");
let (_, body) = store.get_whole(&sc.cfg.file_key("doc.txt"), None).await.unwrap();
assert_eq!(&body[..], b"user re-created");
let conflicts = sc.state.load_conflicts().unwrap();
assert!(conflicts.iter().any(|c| c.kind == "gc-skip" && c.path == "doc.txt"));
}
#[tokio::test]
async fn local_delete_loses_to_foreign_modify() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
sc.run_barrier().await.unwrap();
let body = Bytes::from("their v2");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 2,
epoch: 0,
flush_uuid: "other-writer".into(),
boundary_source: None,
posix: None,
};
let cur = store.head(&sc.cfg.file_key("shared.txt")).await.unwrap();
let newmeta = store
.put_whole(&sc.cfg.file_key("shared.txt"), body, &PutCondition::IfMatch(cur.etag), &stamps, crc)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.entries.get_mut("shared.txt").unwrap().etag = newmeta.etag.clone();
manifest::cas_write(store.as_ref(), &sc.cfg, &theirs, Some(&loaded.handle()), 0, "other-writer")
.await
.unwrap();
std::fs::remove_file(dir.path().join("shared.txt")).unwrap();
let r1 = sc.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty());
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries["shared.txt"].etag, newmeta.etag, "foreign entry dropped!");
let (_, body) = store.get_whole(&sc.cfg.file_key("shared.txt"), None).await.unwrap();
assert_eq!(&body[..], b"their v2");
let queued = sc.state.load_foreign_queue().unwrap();
assert!(queued.iter().any(|c| c.path == "shared.txt" && c.etag.as_deref() == Some(newmeta.etag.as_str())), "{queued:?}");
let r2 = sc.run_barrier().await.unwrap();
assert_eq!(r2.deleted, vec!["shared.txt".to_string()]);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("shared.txt"));
let conflicts = sc.state.load_conflicts().unwrap();
let c = conflicts
.iter()
.find(|c| c.kind == "consume-dirty" && c.path == "shared.txt")
.expect("delete-vs-edit conflict must surface");
let preserved = c.preserved_key.as_ref().expect("foreign bytes preserved");
let (_, body) = store.get_whole(preserved, None).await.unwrap();
assert_eq!(&body[..], b"their v2", "the edit must stay recoverable after the delete wins");
}
#[tokio::test]
async fn window_refuses_hitl_and_expiry_unwedges() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let sc = syncer(&store, dir.path()).await;
let entry = |p: &str| InboxEntry {
path: p.into(),
etag: "e".into(),
author: "dilip".into(),
added_unix: now_unix(),
crc64_b64: None,
};
inbox::open_window(store.as_ref(), &sc.cfg, 1, now_unix() + 300).await.unwrap();
let err = inbox::gateway_append(store.as_ref(), &sc.cfg, entry("a.txt")).await.unwrap_err();
assert!(matches!(err, LeanError::State(_)), "live window must refuse");
inbox::clear_window(store.as_ref(), &sc.cfg, 1, &[]).await.unwrap();
inbox::gateway_append(store.as_ref(), &sc.cfg, entry("a.txt")).await.unwrap();
inbox::open_window(store.as_ref(), &sc.cfg, 1, now_unix() - 10).await.unwrap();
inbox::gateway_append(store.as_ref(), &sc.cfg, entry("b.txt")).await.unwrap();
}
#[tokio::test]
async fn large_file_publishes_via_compose_and_roundtrips() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.whole_put_max = 8; assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
let big_v1: String = (0..200).map(|i| format!("line {i}\n")).collect();
write(dir.path(), "model.bin", &big_v1);
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["model.bin".to_string()]);
assert!(r.deferred.is_empty());
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "model.bin").unwrap(), big_v1);
let big_v2 = format!("{big_v1}and more\n");
write(dir.path(), "model.bin", &big_v2);
backdate_baseline(&sc, "model.bin");
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["model.bin".to_string()]);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let cited = &m.manifest.entries["model.bin"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(String::from_utf8(body.to_vec()).unwrap(), big_v2);
}
#[tokio::test]
async fn second_syncer_on_one_tree_refuses() {
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let _held = SyncerState::open(cfg.state_dir()).unwrap();
let Err(err) = SyncerState::open(cfg.state_dir()) else {
panic!("second open over a held workspace must refuse");
};
assert!(matches!(err, LeanError::State(_)));
}
#[tokio::test]
async fn checkout_budget_refuses_before_first_byte() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "big.txt", "0123456789012345678901234567890123456789");
sc.run_barrier().await.unwrap();
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.cfg.max_bytes = 10;
let err = sc2.checkout().await.unwrap_err();
assert!(matches!(err, LeanError::Budget(_)));
assert!(!sc2.state.marker_present(), "budget refusal must not gate-open the agent");
assert!(read(dir2.path(), "big.txt").is_none());
}
#[tokio::test]
async fn an_all_rejected_sync_scope_is_refused_not_widened_to_the_whole_tree() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
let err = sc
.sync_scoped(Some(vec!["../escape".into(), "./here".into()]))
.await
.expect_err("an all-rejected scope must not silently become the whole tree");
let msg = format!("{err}");
assert!(
msg.contains("NONE survived validation"),
"the refusal must say WHY, so a typo is not read as a sync bug: {msg}"
);
sc.sync_scoped(Some(vec!["inputs".into()])).await.expect("a valid scope must still sync");
}
#[tokio::test]
async fn sync_scan_first_dirty_wins_clean_applies() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "shared.txt", "v1");
write(dir_a.path(), "mine.txt", "agent latest — NEVER destroy");
a.run_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "shared.txt", "user v2", "dilip").await.unwrap();
write(dir_a.path(), "mine.txt", "agent latest v2");
backdate_baseline(&a, "mine.txt");
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.entries.remove("mine.txt");
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "remote-delete")
.await
.unwrap();
let r = a.sync().await.unwrap();
assert_eq!(read(dir_a.path(), "shared.txt").unwrap(), "user v2");
assert!(r.applied.contains(&"shared.txt".to_string()));
assert_eq!(read(dir_a.path(), "mine.txt").unwrap(), "agent latest v2");
assert!(r.conflicts.contains(&"mine.txt".to_string()));
assert!(r.deleted.is_empty());
}
use super::control;
use super::sentinel::{Due, Verb};
fn touch_sentinel(root: &std::path::Path, name: &str, body: &str) {
let dir = root.join(super::CONTROL_DIR);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join(name), body).unwrap();
}
fn control_exists(root: &std::path::Path, name: &str) -> bool {
root.join(super::CONTROL_DIR).join(name).exists()
}
#[tokio::test]
async fn flint_dir_never_scanned() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "real.txt", "data");
touch_sentinel(dir.path(), control::PUBLISH, "");
touch_sentinel(dir.path(), control::REMOTE_SEQ, "{}");
let scanned = super::scan::scan(dir.path()).unwrap();
assert!(scanned.contains_key("real.txt"));
assert!(
!scanned.keys().any(|k| k.starts_with(".flint")),
"the scan yielded a control path: {:?}",
scanned.keys().collect::<Vec<_>>()
);
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("real.txt"));
assert!(
!m.entries.keys().any(|k| k.starts_with(".flint")),
"a control path was CITED: {:?}",
m.entries.keys().collect::<Vec<_>>()
);
assert!(store.head(&a.cfg.file_key(".flint/publish")).await.is_err());
}
#[tokio::test]
async fn legacy_flint_citation_survives_upgrade() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "v1");
a.run_barrier().await.unwrap();
let key = a.cfg.file_key(".flint/legacy.txt");
let body = Bytes::from_static(b"legacy payload");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: "legacy".into(),
boundary_source: None,
posix: None,
};
let meta = store
.put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut m = loaded.manifest.clone();
m.seq += 1;
m.entries.insert(
".flint/legacy.txt".into(),
manifest::LeanEntry {
key: key.clone(),
etag: meta.etag.clone(),
crc64_b64: meta.crc64_b64.clone().unwrap(),
size: meta.size,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
},
);
let installed =
manifest::cas_write(store.as_ref(), &a.cfg, &m, Some(&loaded.handle()), 0, "legacy")
.await
.unwrap();
let mut b = a.state.load_baseline().unwrap();
b.entries.insert(
".flint/legacy.txt".into(),
super::state::BaselineEntry {
etag: meta.etag.clone(),
generation: 1,
size: meta.size,
mtime_unix: 0,
mtime_nanos: None,
crc64_b64: meta.crc64_b64.clone(),
},
);
b.inst_base.insert(".flint/legacy.txt".into(), meta.etag.clone());
b.prev_scan.insert(".flint/legacy.txt".into());
b.seq = m.seq;
b.manifest_etag = Some(installed.etag.clone());
a.state.save_baseline(&b).unwrap();
assert!(store.head(&key).await.is_ok());
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
let after = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
after.entries.contains_key(".flint/legacy.txt"),
"an upgrade DELETED a legacy citation"
);
assert!(store.head(&key).await.is_ok(), "an upgrade GC'd the legacy object");
}
#[tokio::test]
async fn checkout_never_materializes_control_citation() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "real.txt", "v1");
a.run_barrier().await.unwrap();
let key = a.cfg.file_key(".flint/legacy.txt");
let body = Bytes::from_static(b"legacy");
let crc = crc64_nvme(&body);
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfNoneMatchAny,
&GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: "legacy".into(),
boundary_source: None,
posix: None,
},
crc,
)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut m = loaded.manifest.clone();
m.seq += 1;
m.entries.insert(
".flint/legacy.txt".into(),
manifest::LeanEntry {
key,
etag: meta.etag.clone(),
crc64_b64: meta.crc64_b64.clone().unwrap(),
size: meta.size,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
},
);
manifest::cas_write(store.as_ref(), &a.cfg, &m, Some(&loaded.handle()), 0, "legacy")
.await
.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let r = b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "real.txt").unwrap(), "v1");
assert!(
!dir_b.path().join(".flint/legacy.txt").exists(),
"checkout materialized into the reserved control namespace"
);
assert_eq!(r.refused, 1);
assert!(b
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.path == ".flint/legacy.txt" && c.kind.starts_with("checkout-refused")));
}
#[tokio::test]
async fn capabilities_written_on_live_tree_restart() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join(super::CONTROL_DIR).join(control::CAPABILITIES))
.unwrap();
assert!(a.state.marker_present());
assert!(!control_exists(dir.path(), control::CAPABILITIES));
let r = a.checkout().await.unwrap();
assert!(r.resumed_live_tree);
assert!(!control_exists(dir.path(), control::CAPABILITIES));
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
let caps = a.read_capabilities().unwrap();
assert_eq!(caps.protocol, super::SENTINEL_PROTOCOL);
assert_eq!(caps.state, "live");
assert!(caps.verbs.iter().any(|v| v == "publish"));
assert!(caps.verbs.iter().any(|v| v == "sync"));
}
#[tokio::test]
async fn agent_guide_names_every_advertised_verb() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
let caps = a.read_capabilities().unwrap();
assert!(!caps.verbs.is_empty(), "the fixture advertises verbs");
let guide = std::fs::read_to_string(dir.path().join(super::CONTROL_DIR).join(control::AGENT_GUIDE)).unwrap();
assert_eq!(guide, control::AGENT_GUIDE_TEXT);
for v in &caps.verbs {
let named = match v.as_str() {
"remote-seq" => ".flint/remote.seq",
other => &format!(".flint/{other}"),
};
assert!(guide.contains(named), "the guide never names the advertised verb {v} as {named}");
}
assert!(
guide.contains(&format!("sentinel protocol {}", super::SENTINEL_PROTOCOL)),
"the guide names a protocol other than the marker's {}",
super::SENTINEL_PROTOCOL
);
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("f.txt"));
assert!(m.manifest.entries.keys().all(|k| !k.starts_with(".flint/")), "the guide was published as data");
}
#[tokio::test]
async fn preexisting_flint_disables_sentinels() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
touch_sentinel(dir.path(), control::PUBLISH, "app-owned payload");
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
assert!(!posture.enabled);
assert_eq!(posture.reason.as_deref(), Some("preexisting-flint-paths"));
a.write_capabilities(&posture).unwrap();
let caps = a.read_capabilities().unwrap();
assert!(caps.verbs.is_empty());
assert_eq!(caps.reason.as_deref(), Some("preexisting-flint-paths"));
a.checkout().await.unwrap();
let acks = a.sentinel_tick().await.unwrap();
assert!(acks.is_empty());
assert_eq!(
std::fs::read_to_string(dir.path().join(".flint/publish")).unwrap(),
"app-owned payload"
);
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
let again = a.sentinel_preflight().unwrap();
assert!(!again.enabled);
}
fn clear_min_interval(sc: &Syncer) {
let mut b = sc.load_budget().unwrap();
b.last_honor_unix = 0;
let bytes = serde_json::to_vec(&b).unwrap();
std::fs::write(sc.cfg.state_dir().join("sentinel-budget.json"), bytes).unwrap();
}
#[tokio::test]
async fn publish_sentinel_honored_and_acked() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
a.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &a.cfg).await.unwrap().map(|l| l.manifest.seq);
write(dir.path(), "model.json", "{}");
write(dir.path(), "model.json.index", "idx");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1","note":"step 1"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the sentinel was not honored");
let ack = &acks[0];
assert_eq!(ack.status, "ok");
assert_eq!(ack.boundary, "sentinel");
assert_eq!(ack.nonces, vec!["n-1".to_string()]);
assert!(ack.sentinel_mtime_unix_ns > 0);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("model.json"));
assert!(m.entries.contains_key("model.json.index"));
assert_eq!(ack.seq, Some(m.seq));
assert_ne!(before, Some(m.seq), "the manifest did not advance");
assert!(!control_exists(dir.path(), control::PUBLISH));
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
assert!(control_exists(dir.path(), control::PUBLISH_ACK));
assert_eq!(a.read_ack(Verb::Publish).unwrap().nonces, vec!["n-1".to_string()]);
}
#[tokio::test]
async fn a_publish_whose_delete_lost_to_a_peers_edit_is_partial() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir_a.path(), "shared.txt", "v1");
a.floor_tick().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "shared.txt", "B's v2");
backdate_baseline(&b, "shared.txt");
b.floor_tick().await.unwrap();
let theirs = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let b_etag = theirs.entries["shared.txt"].etag.clone();
assert_eq!(read(dir_a.path(), "shared.txt").as_deref(), Some("v1"), "fixture: B's edit already reached A");
std::fs::remove_file(dir_a.path().join("shared.txt")).unwrap();
touch_sentinel(dir_a.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(ack.seq, Some(m.seq), "fixture: the ack names a different seq than the one installed");
assert_eq!(
m.entries.get("shared.txt").map(|e| e.etag.as_str()),
Some(b_etag.as_str()),
"fixture: the merge did not keep B's entry, so the delete was not outranked"
);
assert_eq!(ack.status, "partial", "the ack says the delete is in seq {}, which still cites the file: {ack:?}", m.seq);
assert_eq!(ack.report.dropped, vec!["shared.txt".to_string()]);
touch_sentinel(dir_a.path(), control::PUBLISH, r#"{"nonce":"n-2"}"#);
a.poll_sentinels().unwrap();
clear_min_interval(&a);
let again = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
assert_eq!(again.status, "ok", "{again:?}");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(again.seq, Some(m.seq));
assert!(!m.entries.contains_key("shared.txt"), "the delete never published");
let c = again
.report
.conflicts
.iter()
.find(|c| c.kind == "consume-dirty" && c.path == "shared.txt")
.unwrap_or_else(|| panic!("no consume-dirty record on the ack: {again:?}"));
let (_, body) = store.get_whole(c.preserved_key.as_ref().expect("B's edit preserved"), None).await.unwrap();
assert_eq!(&body[..], b"B's v2");
}
#[tokio::test]
async fn a_publish_whose_upload_was_swept_is_partial() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.whole_put_max = 8; let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
let big: String = (0..200).map(|i| format!("line {i}\n")).collect();
write(dir.path(), "model.bin", &big);
write(dir.path(), "notes.txt", "n");
store.inject_compose_swept();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(ack.seq, Some(m.seq), "fixture: the ack names a different seq than the one installed");
assert!(m.entries.contains_key("notes.txt"), "fixture: the boundary installed nothing");
assert!(!m.entries.contains_key("model.bin"), "fixture: the swept upload was cited anyway");
assert_eq!(ack.status, "partial", "the ack says model.bin is in seq {}, which does not cite it: {ack:?}", m.seq);
assert_eq!(ack.report.dropped, vec!["model.bin".to_string()]);
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-2"}"#);
a.poll_sentinels().unwrap();
clear_min_interval(&a);
let again = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
assert_eq!(again.status, "ok", "{again:?}");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(again.seq, Some(m.seq));
assert!(m.entries.contains_key("model.bin"), "the re-touch did not publish the swept file");
}
#[tokio::test]
async fn a_drain_whose_upload_was_swept_does_not_attest() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.whole_put_max = 8; a.checkout().await.unwrap();
let big: String = (0..200).map(|i| format!("line {i}\n")).collect();
write(dir.path(), "model.bin", &big);
store.inject_compose_swept();
let drained = a.drain().await;
let cited = |m: Option<manifest::LeanManifest>| m.is_some_and(|m| m.entries.contains_key("model.bin"));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().map(|l| l.manifest);
assert!(!cited(m), "fixture: the swept upload was cited anyway");
assert!(drained.is_err(), "the drain attested a tree whose only copy of model.bin is on it: {drained:?}");
a.drain().await.expect("the retry did not drain");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().map(|l| l.manifest);
assert!(cited(m), "the retry did not publish model.bin");
}
#[tokio::test]
async fn a_drain_whose_delete_lost_to_a_peers_edit_does_not_attest() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "shared.txt", "v1");
a.floor_tick().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "shared.txt", "B's v2");
backdate_baseline(&b, "shared.txt");
b.floor_tick().await.unwrap();
std::fs::remove_file(dir_a.path().join("shared.txt")).unwrap();
let drained = a.drain().await;
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("shared.txt"), "fixture: the delete was not outranked");
assert!(drained.is_err(), "the drain attested a tree whose delete is not in the bucket: {drained:?}");
a.drain().await.expect("the retry did not drain");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("shared.txt"), "the retry did not publish the delete");
}
#[tokio::test]
async fn sentinel_ack_echoes_covered_nonces() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
for i in 0..5 {
touch_sentinel(dir.path(), control::PUBLISH, &format!(r#"{{"nonce":"n-{i}"}}"#));
a.poll_sentinels().unwrap();
}
let pending = a.load_pending(Verb::Publish).unwrap().unwrap();
assert_eq!(pending.nonces.len(), 5, "touches did not coalesce into one record");
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
for i in 0..5 {
assert!(
ack.nonces.contains(&format!("n-{i}")),
"nonce n-{i} was orphaned by coalescing: {:?}",
ack.nonces
);
}
assert!(ack.nonces.contains(&"n-2".to_string()));
}
#[tokio::test]
async fn min_interval_coalesces_into_one_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 3600; assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"first"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the first honor must be prompt");
let seq_after_first = acks[0].seq.unwrap();
write(dir.path(), "g.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"second"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert!(acks.is_empty(), "the min-interval did not hold the second honor");
assert_eq!(a.sentinel_due().unwrap(), Due::MinInterval);
assert_eq!(
a.load_pending(Verb::Publish).unwrap().unwrap().nonces,
vec!["second".to_string()]
);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.seq, seq_after_first, "a held honor still advanced the manifest");
let out = a.floor_tick().await.unwrap();
assert_eq!(out.acks.len(), 1);
assert_eq!(out.acks[0].nonces, vec!["second".to_string()]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("g.txt"));
}
#[tokio::test]
async fn budget_meters_bytes_not_calls() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.whole_put_max = 1024; a.cfg.sentinel_hourly_budget = 8; a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
let mut honors_large = 0;
let mut deferred_acks = 0;
for i in 0..6 {
write(dir.path(), "big.bin", &"x".repeat(4096 + i));
backdate_baseline(&a, "big.bin");
touch_sentinel(dir.path(), control::PUBLISH, &format!(r#"{{"nonce":"L{i}"}}"#));
let acks = a.sentinel_tick().await.unwrap();
honors_large += acks.len();
if acks.is_empty() && a.load_pending(Verb::Publish).unwrap().is_some() {
let out = a.floor_tick().await.unwrap();
deferred_acks += out
.acks
.iter()
.filter(|x| x.boundary == "sentinel-deferred")
.count();
}
}
assert!(
honors_large <= 3,
"a 4 KiB-over-ceiling storm was NOT throttled: {honors_large} prompt honors"
);
assert!(deferred_acks >= 1, "no ack was stamped sentinel-deferred");
assert_eq!(a.sentinel_due().unwrap(), Due::BudgetDeferred);
let store2 = Arc::new(MemoryStore::new());
let dir2 = tempfile::tempdir().unwrap();
let mut b = syncer(&store2, dir2.path()).await;
b.cfg.whole_put_max = 1024;
b.cfg.sentinel_hourly_budget = 8;
b.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut b, 3).await);
let posture = b.sentinel_preflight().unwrap();
b.write_capabilities(&posture).unwrap();
b.checkout().await.unwrap();
let mut honors_small = 0;
for i in 0..6 {
write(dir2.path(), "small.txt", &format!("v{i}"));
backdate_baseline(&b, "small.txt");
touch_sentinel(dir2.path(), control::PUBLISH, &format!(r#"{{"nonce":"S{i}"}}"#));
honors_small += b.sentinel_tick().await.unwrap().len();
}
assert_eq!(
honors_small, 6,
"the small-file storm was throttled — the meter is counting CALLS, not work"
);
assert!(b
.read_ack(Verb::Publish)
.map(|a| a.boundary == "sentinel")
.unwrap_or(false));
assert!(honors_small > honors_large);
}
#[tokio::test]
async fn no_diff_sentinel_honor_costs_no_budget() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_hourly_budget = 2;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
for i in 0..10 {
touch_sentinel(dir.path(), control::PUBLISH, &format!(r#"{{"nonce":"q{i}"}}"#));
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "a no-diff honor was throttled at touch {i}");
assert!(acks[0].report.no_change, "the tree was not actually quiet");
}
assert_eq!(a.load_budget().unwrap().spent(super::now_unix()), 0);
}
#[tokio::test]
async fn crash_between_consume_and_ack_reruns_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
a.run_barrier().await.unwrap();
let seq_before = a.state.load_baseline().unwrap().seq;
write(dir.path(), "late.txt", "written before the boundary");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"crashed"}"#);
a.poll_sentinels().unwrap();
assert!(a.load_pending(Verb::Publish).unwrap().is_some());
assert!(a.read_ack(Verb::Publish).is_none());
a.settle_pending_at_startup().await.unwrap();
let ack = a.read_ack(Verb::Publish).unwrap();
assert_eq!(ack.status, "ok");
assert!(ack.nonces.contains(&"crashed".to_string()));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(ack.seq, Some(m.seq));
assert!(m.seq > seq_before);
assert!(m.entries.contains_key("late.txt"));
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
}
#[tokio::test]
async fn restart_settles_pending_before_new_consume() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "a.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"old"}"#);
a.poll_sentinels().unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"new"}"#);
a.poll_sentinels().unwrap();
let pending = a.load_pending(Verb::Publish).unwrap().unwrap();
assert!(pending.nonces.contains(&"old".to_string()));
assert!(pending.nonces.contains(&"new".to_string()));
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
assert!(ack.nonces.contains(&"old".to_string()));
assert!(ack.nonces.contains(&"new".to_string()));
}
#[tokio::test]
async fn torn_pending_body_honored_as_bare_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"half-writ"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "a torn body wedged the verb");
assert_eq!(acks[0].status, "ok");
assert!(acks[0].nonces.is_empty());
assert!(acks[0].sentinel_mtime_unix_ns > 0, "the bare-touch mtime must still be covered");
assert!(a
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind == "sentinel-torn-body"));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("f.txt"));
}
#[tokio::test]
#[cfg(unix)]
async fn fifo_sentinel_skipped() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
let ctl = dir.path().join(super::CONTROL_DIR);
std::fs::create_dir_all(&ctl).unwrap();
let fifo = ctl.join(control::PUBLISH);
let cpath = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap();
let rc = unsafe { libc::mkfifo(cpath.as_ptr(), 0o644) };
assert_eq!(rc, 0, "could not create the FIFO fixture");
assert!(!std::fs::symlink_metadata(&fifo).unwrap().is_file());
let acks = a.sentinel_tick().await.unwrap();
assert!(acks.is_empty());
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
assert!(fifo.exists(), "the FIFO was consumed");
assert!(a
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind == "sentinel-not-regular-file"));
}
#[tokio::test]
async fn scoped_sync_preserves_out_of_scope_foreign_flow() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "inputs/data.txt", "v1");
write(dir_a.path(), "outputs/result.txt", "v1");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
for (path, content) in
[("inputs/data.txt", "foreign inputs v2"), ("outputs/result.txt", "foreign outputs v2")]
{
let key = a.cfg.file_key(path);
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let cur = store.head(&key).await.unwrap();
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfMatch(cur.etag),
&GenerationStamps {
generation: 2,
epoch: 0,
flush_uuid: "sibling".into(),
boundary_source: None,
posix: None,
},
crc,
)
.await
.unwrap();
let e = theirs.entries.get_mut(path).unwrap();
e.etag = meta.etag.clone();
e.crc64_b64 = meta.crc64_b64.clone().unwrap();
e.size = meta.size;
e.generation = 2;
}
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "sibling")
.await
.unwrap();
let foreign_out_etag = theirs.entries["outputs/result.txt"].etag.clone();
assert_ne!(
foreign_out_etag,
a.state.load_baseline().unwrap().inst_base["outputs/result.txt"]
);
let r = a.sync_scoped(Some(vec!["inputs/".into()])).await.unwrap();
assert_eq!(read(dir_a.path(), "inputs/data.txt").unwrap(), "foreign inputs v2");
assert!(r.applied.contains(&"inputs/data.txt".to_string()));
assert_eq!(read(dir_a.path(), "outputs/result.txt").unwrap(), "v1");
assert!(r.out_of_scope_foreign >= 1);
assert_ne!(
a.state.load_baseline().unwrap().inst_base["outputs/result.txt"],
foreign_out_etag,
"a scoped sync advanced the MERGE BASE for an out-of-scope path"
);
assert_eq!(r.seq, a.state.load_baseline().unwrap().seq);
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(
read(dir_a.path(), "outputs/result.txt").unwrap(),
"foreign outputs v2",
"the out-of-scope foreign change was LOST from the inbox flow"
);
}
#[test]
fn scope_matches_on_component_boundary() {
let s = super::sync::Scope::new(&["in".to_string(), "inputs/".to_string()]);
assert!(s.covers("in"));
assert!(s.covers("in/x.txt"));
assert!(s.covers("inputs/a.txt"));
assert!(!s.covers("internal/secret.txt"));
assert!(!s.covers("inputsX/a.txt"));
let s = super::sync::Scope::new(&["../etc".to_string()]);
assert!(s.is_empty());
}
#[tokio::test]
#[cfg(unix)]
async fn write_file_atomic_refuses_symlink_escape() {
let outside = tempfile::tempdir().unwrap();
let secret = outside.path().join("secret.txt");
std::fs::write(&secret, "ORIGINAL CREDENTIALS").unwrap();
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
std::os::unix::fs::symlink(outside.path(), dir.path().join("inputs")).unwrap();
assert!(dir.path().join("inputs/secret.txt").exists());
let scanned = super::scan::scan(dir.path()).unwrap();
assert!(!scanned.keys().any(|k| k.starts_with("inputs")));
assert!(super::barrier::contained_path(dir.path(), "inputs/secret.txt").is_err());
assert!(super::barrier::contained_path(dir.path(), "../escape.txt").is_err());
assert!(super::barrier::contained_path(dir.path(), "/etc/passwd").is_err());
assert!(super::barrier::contained_path(dir.path(), ".flint/publish").is_err());
assert!(super::barrier::contained_path(dir.path(), "ok/nested/f.txt").is_ok());
hitl_write(&store, &a.cfg, "inputs/secret.txt", "ATTACKER PAYLOAD", "attacker")
.await
.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(
std::fs::read_to_string(&secret).unwrap(),
"ORIGINAL CREDENTIALS",
"the syncer wrote THROUGH a planted symlink, outside the workspace"
);
assert!(a
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind.starts_with("consume-refused-containment")));
}
#[tokio::test]
async fn sync_rehonor_no_phantom_conflicts() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "shared.txt", "foreign v2", "ci").await.unwrap();
let r = a.sync().await.unwrap();
assert!(r.applied.contains(&"shared.txt".to_string()));
let mut b = a.state.load_baseline().unwrap();
let stale = b.entries.get_mut("shared.txt").unwrap();
stale.etag = "\"stale-pre-sync-etag\"".into();
stale.mtime_unix -= 10;
stale.size = 1;
a.state.save_baseline(&b).unwrap();
let scanned = super::scan::scan(dir.path()).unwrap();
let c = super::scan::classify(&scanned, &a.state.load_baseline().unwrap());
assert!(c.uploads.contains("shared.txt"), "the fixture is not actually dirty");
let r = a.sync().await.unwrap();
assert!(
!r.conflicts.contains(&"shared.txt".to_string()),
"a phantom conflict was reported for byte-identical content"
);
assert!(r.applied.contains(&"shared.txt".to_string()));
}
#[tokio::test]
async fn remote_seq_ticks_without_added_requests() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "v1");
a.floor_tick().await.unwrap();
let t0 = a.load_remote_seq();
assert!(t0.updated_unix > 0);
assert_eq!(t0.observed_seq, t0.integrated_seq, "no news, yet the ticker claims some");
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "foreign")
.await
.unwrap();
assert_eq!(t0.observed_seq, loaded.manifest.seq);
a.floor_tick().await.unwrap();
let t1 = a.load_remote_seq();
assert!(
t1.observed_seq >= theirs.seq,
"the ticker missed a foreign install: {} vs {}",
t1.observed_seq,
theirs.seq
);
assert!(t1.updated_unix >= t0.updated_unix);
}
#[tokio::test]
async fn remote_seq_reports_news_while_a_peers_change_waits_in_the_queue() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.floor_tick().await.unwrap();
b.checkout().await.unwrap();
b.floor_tick().await.unwrap();
let t0 = b.load_remote_seq();
assert_eq!(t0.observed_seq, t0.integrated_seq, "fixture: B has news before anyone wrote");
write(dir_a.path(), "x.txt", "A's v2");
backdate_baseline(&a, "x.txt");
a.floor_tick().await.unwrap();
write(dir_b.path(), "z.txt", "z");
b.floor_tick().await.unwrap();
assert!(
b.state.load_foreign_queue().unwrap().iter().any(|c| c.path == "x.txt"),
"fixture: B's merge queued nothing for x.txt"
);
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"), "fixture: the change already reached B's tree");
let t1 = b.load_remote_seq();
assert!(
t1.observed_seq > t1.integrated_seq,
"B's tree lacks A's change, yet remote.seq says no news: {t1:?}"
);
b.floor_tick().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's v2"), "A's change never reached B on the floor");
let t2 = b.load_remote_seq();
assert_eq!(t2.observed_seq, t2.integrated_seq, "news that was integrated is still reported: {t2:?}");
}
#[tokio::test]
async fn sync_request_is_carried_never_executed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "agent bytes");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.entries.remove("keep.txt");
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "remote-delete")
.await
.unwrap();
let tree_before = std::fs::read_to_string(dir.path().join("keep.txt")).unwrap();
a.carry_sync_request(super::now_unix(), "ci@example").unwrap();
for _ in 0..3 {
let _ = a.sentinel_tick().await.unwrap();
}
let t = a.load_remote_seq();
assert_eq!(t.sync_requested_by.as_deref(), Some("ci@example"));
assert!(t.sync_requested_unix.is_some());
assert_eq!(
std::fs::read_to_string(dir.path().join("keep.txt")).unwrap(),
tree_before,
"the syncer acted on a remote's sync request"
);
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"mine"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1);
assert!(!dir.path().join("keep.txt").exists(), "the agent's own sync did not run");
}
#[tokio::test]
async fn a_stale_ack_never_retires_a_fresh_bare_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "w.txt", "one");
touch_sentinel(dir.path(), control::PUBLISH, "");
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the first bare touch was not honored");
let first = a.read_ack(Verb::Publish).expect("no ack from the first honor");
assert_eq!(first.status, "ok");
std::thread::sleep(std::time::Duration::from_millis(1100));
write(dir.path(), "w.txt", "two");
touch_sentinel(dir.path(), control::PUBLISH, "");
let path = dir.path().join(super::CONTROL_DIR).join(control::PUBLISH);
let backdated = std::time::SystemTime::UNIX_EPOCH
+ std::time::Duration::from_nanos((first.sentinel_mtime_unix_ns as u64).saturating_sub(5));
let f = std::fs::File::options().write(true).open(&path).unwrap();
f.set_times(std::fs::FileTimes::new().set_modified(backdated)).unwrap();
drop(f);
assert!(a.consume_sentinel(Verb::Publish).unwrap(), "the second touch was not consumed");
let pending = a.load_pending(Verb::Publish).unwrap().expect("no pending record");
assert!(pending.nonces.is_empty(), "the fixture needs a BARE touch");
assert!(
pending.consumed_mtime_unix_ns <= first.sentinel_mtime_unix_ns,
"the mtime did not go backwards — the fixture never armed"
);
assert!(
!a.ack_matches(Verb::Publish, &pending),
"a stale ack retired a fresh boundary — it will never run"
);
let ack = a
.honor_pending(Verb::Publish, true)
.await
.unwrap()
.expect("the second boundary never ran");
assert_eq!(ack.status, "ok");
let second = a.read_ack(Verb::Publish).expect("no ack from the second honor");
assert!(second.completed_unix > first.completed_unix, "the ack was not re-minted");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let e = &m.entries["w.txt"];
let (_, body) = store.get_whole(&e.key, Some(&e.etag)).await.unwrap();
assert_eq!(&body[..], b"two", "the second boundary was retired without publishing");
}
#[tokio::test]
async fn a_fifo_at_the_sentinel_path_never_wedges_the_poll_arm() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let ctl = dir.path().join(super::CONTROL_DIR);
std::fs::create_dir_all(&ctl).unwrap();
let path = ctl.join(control::PUBLISH);
let c = std::ffi::CString::new(path.to_str().unwrap()).unwrap();
assert_eq!(unsafe { libc::mkfifo(c.as_ptr(), 0o644) }, 0, "mkfifo failed");
assert!(
!std::fs::metadata(&path).unwrap().is_file(),
"the fixture never armed — that is not a FIFO"
);
let r = super::sentinel::read_bounded(&path);
assert!(r.is_err(), "a writer-less FIFO read as a body instead of erroring");
let target = ctl.join("elsewhere");
std::fs::write(&target, "not the agent's").unwrap();
let link = ctl.join("publish.link");
std::os::unix::fs::symlink(&target, &link).unwrap();
assert!(
super::sentinel::read_bounded(&link).is_err(),
"a symlink at the sentinel path was followed"
);
let ticked = tokio::time::timeout(std::time::Duration::from_secs(5), a.sentinel_tick()).await;
let acks = ticked.expect("THE POLL ARM WEDGED on a FIFO at the sentinel path").unwrap();
assert!(acks.is_empty(), "a FIFO was honored as a boundary");
let conflicts = a.state.load_conflicts().unwrap();
assert!(
conflicts.iter().any(|c| c.kind == "sentinel-not-regular-file"),
"the non-regular sentinel left no conflict record"
);
for _ in 0..5 {
a.sentinel_tick().await.unwrap();
}
let after = a.state.load_conflicts().unwrap();
assert_eq!(
after.iter().filter(|c| c.kind == "sentinel-not-regular-file").count(),
1,
"the standing condition was re-recorded on every poll tick"
);
std::fs::remove_file(&path).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, "");
a.sentinel_tick().await.unwrap();
let c = std::ffi::CString::new(path.to_str().unwrap()).unwrap();
assert_eq!(unsafe { libc::mkfifo(c.as_ptr(), 0o644) }, 0, "second mkfifo failed");
a.sentinel_tick().await.unwrap();
assert_eq!(
a.state.load_conflicts().unwrap().iter()
.filter(|c| c.kind == "sentinel-not-regular-file")
.count(),
2,
"a RECURRENCE of the condition went unrecorded — the latch never clears"
);
}
#[tokio::test]
async fn status_renders_without_claiming_the_lease() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "held.txt", "x");
a.floor_tick().await.unwrap();
let s = super::status_report(&cfg_for(dir.path())).unwrap();
assert!(s.gauges.is_some());
assert_eq!(s.capabilities.unwrap().state, "live");
assert!(s.baseline_seq >= 1, "the tick's boundary never reached the baseline status reads");
assert!(s.incarnation_epoch.is_some());
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released);
assert_eq!(
cell.epoch,
a.state.load_incarnation().unwrap().unwrap().epoch,
"status rotated the epoch it was supposed to observe"
);
}
#[tokio::test]
async fn a_sentinel_boundary_carries_a_delete_made_before_the_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "k");
write(dir.path(), "gone.txt", "v1");
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("gone.txt"), "fixture: never published");
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-del"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the sentinel was not honored");
assert_eq!(acks[0].status, "ok");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("gone.txt"),
"the ack said ok at seq {:?} while the manifest still cites a file the agent \
deleted before the touch (report.deleted = {})",
acks[0].seq,
acks[0].report.deleted
);
}
#[tokio::test]
async fn only_a_declared_barrier_confirms_a_first_absence() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "gone.txt", "v1");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
let r = a.run_barrier().await.unwrap();
assert_eq!(r.first_absence, vec!["gone.txt".to_string()]);
assert_eq!(r.absences_confirmed, 0);
assert!(r.deleted.is_empty(), "the cadence barrier published a first absence");
let mut b = a.state.load_baseline().unwrap();
b.prev_scan.insert("gone.txt".into());
a.state.save_baseline(&b).unwrap();
let r = a.declared_barrier().await.unwrap();
assert_eq!(r.absences_confirmed, 1, "the declared barrier did not confirm the absence");
assert!(r.first_absence.is_empty());
assert_eq!(r.deleted, vec!["gone.txt".to_string()]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("gone.txt"));
}
#[tokio::test]
async fn an_unreadable_path_is_not_confirmed_as_deleted() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
if unsafe { libc::geteuid() } == 0 {
eprintln!("SKIPPED: running as root, mode bits cannot induce EACCES");
return;
}
let locked = dir.path().join("locked");
std::fs::create_dir(&locked).unwrap();
std::fs::write(locked.join("f.txt"), b"still here").unwrap();
std::fs::set_permissions(&locked, std::os::unix::fs::PermissionsExt::from_mode(0o000)).unwrap();
let mut classified = super::scan::Classified::default();
classified.first_absence.insert("locked/f.txt".into());
let err = a
.confirm_absences(&mut classified)
.expect_err("an unreadable path must not confirm a deletion");
assert!(
format!("{err}").contains("refusing to publish a deletion"),
"the refusal must say what it refused: {err}"
);
assert!(
classified.deletes.is_empty(),
"NOTHING may be queued for deletion off an errno that is not NotFound: {:?}",
classified.deletes
);
std::fs::set_permissions(&locked, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
}
#[tokio::test]
async fn the_confirmation_never_deletes_a_path_the_walk_merely_missed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
write(dir.path(), "renamed.txt", "here all along");
let mut classified = super::scan::Classified::default();
classified.first_absence.insert("renamed.txt".into());
classified.first_absence.insert("truly-gone.txt".into());
let confirmed = a.confirm_absences(&mut classified).unwrap();
assert_eq!(confirmed, 1);
assert_eq!(
classified.deletes.iter().cloned().collect::<Vec<_>>(),
vec!["truly-gone.txt".to_string()]
);
assert!(
classified.first_absence.contains("renamed.txt"),
"the confirmation promoted a path that is on disk — the walk race would publish \
a delete of a live file"
);
}
#[tokio::test]
async fn the_drain_carries_a_delete_made_before_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "gone.txt", "v1");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
a.drain().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("gone.txt"),
"the drain left the delete withheld — the successor's checkout resurrects it"
);
}
#[tokio::test]
async fn a_crash_between_the_cas_and_step_7_never_makes_our_own_entry_foreign() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
write(dir.path(), "f.txt", "v2 is longer");
backdate_baseline(&a, "f.txt");
let stale = a.state.load_baseline().unwrap();
a.run_barrier().await.unwrap();
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
a.state.save_baseline(&stale).unwrap();
assert_ne!(
stale.inst_base.get("f.txt"),
Some(&installed.manifest.entries["f.txt"].etag),
"fixture: the merge base is not actually behind the install"
);
std::fs::remove_file(dir.path().join("f.txt")).unwrap();
let r = a.declared_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("f.txt"),
"our own install read as a foreign change and swallowed the agent's delete \
(deleted={:?}, foreign_queued={})",
r.deleted,
r.foreign_queued
);
let ib = inbox::load(store.as_ref(), &a.cfg).await.unwrap();
assert!(
!ib.doc.entries.iter().any(|e| e.path == "f.txt"),
"a phantom foreign entry was queued for a path only this workspace ever wrote"
);
}
#[tokio::test]
async fn a_pending_sync_at_sigterm_never_cancels_the_drains_own_boundary() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
a.run_barrier().await.unwrap();
write(dir.path(), "work.txt", "W1");
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"s-1"}"#);
a.poll_sentinels().unwrap();
assert!(a.load_pending(Verb::Sync).unwrap().is_some());
let acks = a.drain().await.unwrap();
assert_eq!(acks.len(), 1, "the drain did not settle the owed sync ack");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
m.entries.contains_key("work.txt"),
"the drain skipped its cite-everything pass because a SYNC ack carried a seq — \
every byte since the last boundary died with the emptyDir"
);
}
#[tokio::test]
async fn a_planted_temp_sibling_is_never_written_through() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let outside = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
a.run_barrier().await.unwrap();
let victims = ["consume-victim", "ticker-victim", "state-victim"];
for v in victims {
std::fs::write(outside.path().join(v), "ORIGINAL").unwrap();
}
std::fs::create_dir_all(dir.path().join("inputs")).unwrap();
std::os::unix::fs::symlink(
outside.path().join("consume-victim"),
dir.path().join("inputs/config.json.flint-sync-tmp"),
)
.unwrap();
std::fs::create_dir_all(dir.path().join(super::CONTROL_DIR)).unwrap();
std::os::unix::fs::symlink(
outside.path().join("ticker-victim"),
dir.path().join(super::CONTROL_DIR).join("remote.seq.tmp"),
)
.unwrap();
std::os::unix::fs::symlink(
outside.path().join("state-victim"),
a.cfg.state_dir().join("baseline.tmp"),
)
.unwrap();
hitl_write(&store, &a.cfg, "inputs/config.json", "REMOTE-BYTES", "ui").await.unwrap();
a.run_barrier().await.unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-tmp"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the fixture never exercised the control writer");
for v in victims {
assert_eq!(
std::fs::read_to_string(outside.path().join(v)).unwrap(),
"ORIGINAL",
"the syncer wrote through a planted temp sibling ({v}) — an arbitrary-file-write \
primitive outside the workspace, with the bucket credentials"
);
}
assert_eq!(read(dir.path(), "inputs/config.json").as_deref(), Some("REMOTE-BYTES"));
assert!(control_exists(dir.path(), "remote.seq"));
}
#[tokio::test]
async fn a_barrier_echoes_the_running_binary_into_the_lease_cell() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
let cited = a.run_barrier().await.unwrap().seq.unwrap();
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released, "a barrier must hand the cell on when it is done");
let echo: flint_store::LeaseEcho = serde_json::from_str(
cell.echo.as_deref().expect("the handoff carried no observed-state echo"),
)
.expect("the echo is not a LeaseEcho");
assert_eq!(echo.last_cited_seq, cited, "the echo names no citation");
assert_eq!(echo.protocol, super::SENTINEL_PROTOCOL);
assert!(!echo.syncer_version.is_empty(), "no version ⇒ no mixed-fleet tell");
}
#[tokio::test]
async fn a_uds_boundary_and_a_file_sentinel_coalesce_into_one_ack() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
a.cfg.sentinel_min_interval_secs = 0;
write(dir.path(), "work.txt", "W1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"file:1"}"#);
a.poll_sentinels().unwrap();
a.request_boundary("uds:1", Some("from the socket".into())).unwrap();
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the two doors produced {} barriers", acks.len());
let ack = &acks[0];
assert_eq!(ack.status, "ok");
for n in ["uds:1", "file:1"] {
assert!(
ack.nonces.iter().any(|x| x == n),
"the ack does not cover {n}: {:?}",
ack.nonces
);
}
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("work.txt"), "the coalesced boundary published nothing");
write(dir.path(), "work2.txt", "W2");
a.request_boundary("uds:2", None).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"file:2"}"#);
a.poll_sentinels().unwrap();
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1);
for n in ["uds:2", "file:2"] {
assert!(acks[0].nonces.iter().any(|x| x == n), "socket-first lost {n}");
}
}
#[tokio::test]
async fn a_gateway_boundary_request_becomes_a_pending_sentinel_with_no_conflict() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
inbox::gateway_request(
store.as_ref(),
&a.cfg,
inbox::RequestedVerb::Boundary,
"ci@example",
)
.await
.unwrap();
write(dir.path(), "work.txt", "W1");
a.consume_inbox().await.unwrap();
let pending = a.load_pending(super::sentinel::Verb::Publish).unwrap();
let pending = pending.expect("the gateway request minted no pending sentinel");
assert!(
pending.nonces.iter().any(|n| n.contains("ci@example")),
"the requestor is not in the covered nonces: {:?}",
pending.nonces
);
assert_eq!(
std::fs::read_to_string(a.cfg.state_dir().join("conflicts.jsonl")).unwrap_or_default(),
"",
"the boundary request minted a conflict record"
);
let before = pending.nonces.len();
a.consume_inbox().await.unwrap();
assert_eq!(
a.load_pending(super::sentinel::Verb::Publish).unwrap().unwrap().nonces.len(),
before,
"the same gateway request was consumed twice"
);
}
#[tokio::test]
async fn a_gateway_sync_request_is_carried_and_never_executed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "LOCAL");
a.run_barrier().await.unwrap();
let mut m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
m.manifest.entries.remove("keep.txt");
m.manifest.seq += 1;
manifest::cas_write(store.as_ref(), &a.cfg, &m.manifest, Some(&m.handle()), 9, "foreign")
.await
.unwrap();
let tree_before = read(dir.path(), "keep.txt");
inbox::gateway_request(store.as_ref(), &a.cfg, inbox::RequestedVerb::Sync, "ci@example")
.await
.unwrap();
a.consume_inbox().await.unwrap();
assert_eq!(
read(dir.path(), "keep.txt"),
tree_before,
"the syncer EXECUTED a sync on a remote's say-so and rewrote the agent's tree"
);
assert!(tree_before.is_some(), "the fixture had nothing to lose");
let t: control::RemoteSeq = serde_json::from_slice(
&std::fs::read(dir.path().join(".flint/remote.seq")).unwrap(),
)
.unwrap();
assert!(t.sync_requested_unix.is_some(), "the request was not carried to the agent");
assert_eq!(t.sync_requested_by.as_deref(), Some("ci@example"));
assert!(a.load_pending(super::sentinel::Verb::Publish).unwrap().is_none());
}
#[test]
fn every_gauges_field_reaches_exactly_one_metric() {
let g = super::Gauges {
rpo_secs: 11,
withheld_reason: Some("parked-412".into()),
sentinel_budget_remaining: 44,
last_boundary: Some(super::gauges::LastBoundary {
source: "sentinel".into(),
seq: 66,
unix: 1_756_000_000,
}),
updated_unix: 1_756_000_100,
last_durable_unix: 1_756_000_050,
auth_paused_since_unix: Some(1_756_000_010),
};
let json = serde_json::to_value(&g).unwrap();
let fields: Vec<String> = json.as_object().unwrap().keys().cloned().collect();
assert!(fields.len() >= 7, "the fixture did not populate the struct: {fields:?}");
for f in &fields {
assert!(
super::metrics::COVERED_FIELDS.contains(&f.as_str()),
"gauges field {f:?} reaches no metric — an operator reading /metrics cannot see \
what an operator reading gauges.json can"
);
}
for f in super::metrics::COVERED_FIELDS {
assert!(
fields.iter().any(|x| x == f),
"metric table names {f:?}, which is not a gauges field any more"
);
}
let text = super::metrics::render(
&g,
&super::metrics::Labels { workspace: "proj1".into(), namespace: "agents".into() },
);
for (name, want) in [
("flint_lean_rpo_seconds", 11u64),
("flint_lean_sentinel_budget_remaining", 44),
("flint_lean_last_boundary_seq", 66),
("flint_lean_withheld_reason", 1), ("flint_lean_last_boundary_source", 1), ] {
let line = text
.lines()
.find(|l| l.starts_with(name) && !l.starts_with('#'))
.unwrap_or_else(|| panic!("no series for {name}"));
let got: u64 = line.rsplit(' ').next().unwrap().parse().unwrap();
assert_eq!(got, want, "{name} disagrees with the gauges it renders");
}
}
#[test]
fn the_label_key_set_is_exactly_workspace_and_namespace() {
let g = super::Gauges { rpo_secs: 1, ..Default::default() };
let text = super::metrics::render(
&g,
&super::metrics::Labels { workspace: "proj1".into(), namespace: "agents".into() },
);
let mut series = 0;
for line in text.lines().filter(|l| !l.starts_with('#')) {
let labels = line
.split_once('{')
.and_then(|(_, r)| r.split_once('}'))
.map(|(l, _)| l)
.unwrap_or_else(|| panic!("unlabelled series: {line}"));
let keys: Vec<&str> =
labels.split(',').map(|kv| kv.split_once('=').unwrap().0).collect();
assert_eq!(
keys,
vec!["workspace", "namespace"],
"series carries label keys beyond the fixed set: {line}"
);
series += 1;
}
assert!(series >= 8, "the renderer emitted almost nothing: {series}");
}
#[tokio::test]
async fn a_scrape_costs_no_bucket_request() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "work.txt", "W1");
a.run_barrier().await.unwrap();
let g = a.write_gauges(None).unwrap();
let before = store.list("").await.unwrap().len();
for _ in 0..25 {
let text = super::metrics::render(
&g,
&super::metrics::Labels { workspace: "w".into(), namespace: "n".into() },
);
assert!(text.contains("flint_lean_rpo_seconds"));
}
assert_eq!(
store.list("").await.unwrap().len(),
before,
"25 scrapes changed the bucket"
);
}
#[tokio::test]
async fn status_reports_a_standing_pending_sentinel() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let before = super::status_report(&a.cfg).unwrap();
assert!(before.pending_sentinels.is_empty(), "the fixture started with one pending");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"s-1"}"#);
a.poll_sentinels().unwrap();
assert!(
a.load_pending(super::sentinel::Verb::Publish).unwrap().is_some(),
"the fixture consumed nothing — the status field would be empty either way"
);
let r = super::status_report(&a.cfg).unwrap();
assert_eq!(
r.pending_sentinels,
vec!["publish".to_string()],
"status cannot see the pending record it exists to report"
);
}
#[tokio::test]
async fn a_resumed_checkout_never_adopts_a_stale_generation() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let dir2 = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "big.bin", "GENERATION-ONE");
a.run_barrier().await.unwrap();
write(dir2.path(), "big.bin", "GENERATION-ONE");
write(dir.path(), "big.bin", "GENERATION-TWO-IS-LONGER");
a.run_barrier().await.unwrap();
let cited = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert_eq!(
cited.manifest.entries["big.bin"].size,
"GENERATION-TWO-IS-LONGER".len() as u64,
"the fixture never advanced the manifest"
);
lease::release(&mut a).await.unwrap();
drop(a);
let mut b = syncer(&store, dir2.path()).await;
assert!(claim_until_held(&mut b, 8).await);
assert!(
!b.state.marker_present(),
"the fixture left a completion marker — this is the RESUME row"
);
b.checkout().await.unwrap();
let local = read(dir2.path(), "big.bin").unwrap();
assert_eq!(
local, "GENERATION-TWO-IS-LONGER",
"the resumed checkout adopted the STALE generation: the workspace holds {local:?} while \
the manifest cites the newer bytes, and the baseline claims they are the same — so the \
scan will never upload it and a sync will never re-fetch it"
);
}
#[tokio::test]
async fn every_boundary_says_which_clock_installed_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "one");
sc.floor_tick().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("a.txt"), "the fixture published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("cadence"),
"a cadence boundary does not say it was cadence"
);
write(dir.path(), "b.txt", "two");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n1"}"#);
sc.sentinel_tick().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("b.txt"), "the sentinel published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("sentinel"),
"a sentinel boundary does not say it was a sentinel"
);
write(dir.path(), "c.txt", "three");
sc.drain().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("c.txt"), "the drain published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("drain"),
"a drain boundary does not say it was a drain"
);
}
#[tokio::test]
async fn a_drained_sentinel_names_the_same_clock_in_the_ack_and_in_the_bucket() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "late.txt", "written before the drain");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"owed"}"#);
sc.poll_sentinels().unwrap();
assert!(
sc.load_pending(Verb::Publish).unwrap().is_some(),
"no pending record — the drain would not take the owed-ack arm"
);
let acks = sc.drain().await.unwrap();
let ack = acks.iter().find(|a| a.nonces.iter().any(|n| n == "owed")).expect("owed ack unsettled");
assert_eq!(ack.boundary, "drain");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("late.txt"), "the drain published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("drain"),
"the ack says drain and the bucket says something else"
);
}
#[tokio::test]
async fn sync_refuses_to_clobber_a_locally_dirty_path_and_says_so() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "dirty.txt", "published v1");
write(dir.path(), "clean.txt", "published v1");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
for (path, content) in
[("dirty.txt", "foreign bytes for dirty"), ("clean.txt", "foreign bytes for clean")]
{
let key = a.cfg.file_key(path);
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let cur = store.head(&key).await.unwrap();
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfMatch(cur.etag),
&GenerationStamps {
generation: 2,
epoch: 0,
flush_uuid: "sibling".into(),
boundary_source: None,
posix: None,
},
crc,
)
.await
.unwrap();
let e = theirs.entries.get_mut(path).unwrap();
e.etag = meta.etag.clone();
e.crc64_b64 = meta.crc64_b64.clone().unwrap();
e.size = meta.size;
e.generation = 2;
}
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "sibling")
.await
.unwrap();
write(dir.path(), "dirty.txt", "the agent's own unpublished work, longer");
let r = a.sync().await.unwrap();
assert_eq!(
read(dir.path(), "clean.txt").unwrap(),
"foreign bytes for clean",
"the control path was not applied — this sync did nothing at all"
);
assert!(r.applied.contains(&"clean.txt".to_string()));
assert_ne!(
read(dir.path(), "dirty.txt").unwrap(),
"foreign bytes for dirty",
"local and remote bytes are identical — nothing was in conflict"
);
assert_eq!(
read(dir.path(), "dirty.txt").unwrap(),
"the agent's own unpublished work, longer",
"sync CLOBBERED work the agent had not published"
);
assert!(
r.conflicts.contains(&"dirty.txt".to_string()),
"the conflict was not surfaced in the report: {:?}",
r.conflicts
);
let conflicts = a.state.load_conflicts().unwrap();
assert!(
conflicts.iter().any(|c| c.kind == "sync-dirty" && c.path == "dirty.txt"),
"no sync-dirty record was written: {conflicts:?}"
);
}
#[tokio::test]
async fn merge_applies_a_local_delete_only_where_theirs_is_unchanged() {
fn entry(etag: &str) -> manifest::LeanEntry {
manifest::LeanEntry {
key: "tenant/proj1/files/p.txt".into(),
etag: etag.into(),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 3,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
}
}
let cases: Vec<(&str, Option<&str>, Option<&str>, bool, bool)> = vec![
("theirs unchanged since our base", Some("e1"), Some("e1"), false, false),
("FOREIGN MODIFY under our delete", Some("e2"), Some("e1"), false, true),
("FOREIGN ADD under our delete", Some("e2"), None, false, true),
("absent from both", None, None, false, false),
("already deleted by someone else", None, Some("e1"), false, false),
("parked: never resolved this pass", Some("e1"), Some("e1"), true, true),
];
for (name, theirs_etag, base_etag, is_parked, must_survive) in cases {
let mut theirs = manifest::LeanManifest::default();
if let Some(e) = theirs_etag {
theirs.entries.insert("p.txt".to_string(), entry(e));
}
let mut base = std::collections::BTreeMap::new();
if let Some(b) = base_etag {
base.insert("p.txt".to_string(), b.to_string());
}
let mut deletes = std::collections::BTreeSet::new();
deletes.insert("p.txt".to_string());
let mut parked = std::collections::BTreeSet::new();
if is_parked {
parked.insert("p.txt".to_string());
}
assert_eq!(theirs.entries.contains_key("p.txt"), theirs_etag.is_some(), "{name}");
assert_eq!(base.contains_key("p.txt"), base_etag.is_some(), "{name}");
let (merged, _foreign) =
manifest::merge(&base, &theirs, &Default::default(), &deletes, &parked);
assert_eq!(
merged.entries.contains_key("p.txt"),
must_survive,
"{name}: our local delete {} the entry",
if must_survive { "destroyed" } else { "failed to remove" }
);
if must_survive {
if let Some(e) = theirs_etag {
assert_eq!(merged.entries["p.txt"].etag, e, "{name}: wrong bytes survived");
}
}
}
}
#[tokio::test]
async fn a_resumed_checkout_adopts_identical_bytes_and_refetches_same_size_impostors() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "same.txt", "these bytes never move");
write(dir.path(), "impostor.txt", "AAAA");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
let key = a.cfg.file_key("impostor.txt");
let body = Bytes::from_static(b"BBBB");
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfMatch(store.head(&key).await.unwrap().etag),
&GenerationStamps {
generation: 2,
epoch: 0,
flush_uuid: "sibling".into(),
boundary_source: None,
posix: None,
},
crc64_nvme(&Bytes::from_static(b"BBBB")),
)
.await
.unwrap();
let e = theirs.entries.get_mut("impostor.txt").unwrap();
e.etag = meta.etag.clone();
e.crc64_b64 = meta.crc64_b64.clone().unwrap();
e.size = meta.size;
e.generation = 2;
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "sibling")
.await
.unwrap();
assert_eq!(
theirs.entries["impostor.txt"].size,
std::fs::metadata(dir.path().join("impostor.txt")).unwrap().len(),
"the impostor differs in SIZE — this leg would pass without any crc at all"
);
assert_eq!(read(dir.path(), "impostor.txt").unwrap(), "AAAA");
std::fs::remove_file(dir.path().join(".flint-sync/checkout-complete")).unwrap();
let r = a.checkout().await.unwrap();
assert!(
r.skipped_present >= 1,
"nothing was adopted — the leg degenerates into 'checkout re-fetches everything'"
);
assert_eq!(
read(dir.path(), "same.txt").unwrap(),
"these bytes never move",
"an identical file was not left alone"
);
assert_eq!(
read(dir.path(), "impostor.txt").unwrap(),
"BBBB",
"checkout ADOPTED a same-size file whose bytes are not the cited bytes — \
the baseline now attests content the workspace does not hold"
);
}
#[tokio::test]
async fn a_takeover_rotation_carries_the_boundary_stamp_with_the_document() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
sc.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
manifest::cas_write_stamped(
store.as_ref(),
&sc.cfg,
&loaded.manifest,
Some(&loaded.handle()),
1,
"u-cited",
Some("sentinel"),
)
.await
.unwrap();
let lp = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(
lp.pointer.boundary_source.as_deref(),
Some("sentinel"),
"precondition: the cited manifest names its clock on the pointer"
);
manifest::rotate_for_takeover(store.as_ref(), &sc.cfg, 2).await.unwrap().unwrap();
let lp = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let doc = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(
lp.pointer.boundary_source.as_deref(),
Some("sentinel"),
"rotation must not drop the boundary stamp: a pointer reader would report an \
unknown clock for a workspace whose document still says `sentinel`"
);
assert_eq!(
lp.pointer.boundary_source, doc.manifest.boundary_source,
"the pointer and the document a reader assembles from it must never disagree"
);
assert_eq!(lp.pointer.seq, doc.manifest.seq, "the pointer's seq IS the document's seq");
}
fn entry(name: &str) -> manifest::LeanEntry {
manifest::LeanEntry {
key: format!("{PREFIX}/files/{name}"),
etag: "e".into(),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 1,
mode: 0o100644,
mtime_unix: 0,
generation: 1,
epoch: 1,
}
}
#[tokio::test]
async fn a_publish_writes_an_immutable_generation_and_a_pointer_that_names_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 1;
m.entries.insert("a.txt".into(), entry("x"));
manifest::cas_write(store.as_ref(), &cfg, &m, None, 3, "first").await.unwrap();
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(lp.pointer.seq, 1);
assert_eq!(lp.pointer.entries_seq, Some(1));
assert_eq!(lp.pointer.entries_key, lp.pointer.entries_key.clone());
assert!(store.head(lp.pointer.entries_key.as_deref().unwrap()).await.is_ok());
assert!(store.head(&cfg.manifest_key()).await.is_err());
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(loaded.manifest.seq, 1);
assert!(loaded.manifest.entries.contains_key("a.txt"));
assert!(!loaded.handle().legacy);
}
#[tokio::test]
async fn a_takeover_rotation_does_not_touch_the_entries_object() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 4;
for i in 0..50 {
m.entries.insert(format!("f{i:03}.txt"), entry("x"));
}
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "seed").await.unwrap();
let gen4 = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
let before = store.head(&gen4).await.unwrap();
let (rotated, _) = manifest::rotate_for_takeover(store.as_ref(), &cfg, 2).await.unwrap().unwrap();
assert_eq!(rotated.seq, 5, "rotation must bump the generation");
let after = store.head(&gen4).await.unwrap();
assert_eq!(before.etag, after.etag, "the entries object was rewritten by a rotation");
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(lp.pointer.seq, 5);
assert_eq!(lp.pointer.entries_seq, Some(4), "entries_seq must NOT move — a follower reads it to skip the GET");
assert_eq!(lp.pointer.entries_key.as_deref(), Some(gen4.as_str()));
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(loaded.manifest.entries.len(), 50);
assert_eq!(loaded.manifest.seq, 5, "the POINTER is the authority for seq, not the entries object");
}
#[tokio::test]
async fn migration_installs_the_pointer_and_poisons_the_legacy_key() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let mut old = manifest::LeanManifest::default();
old.seq = 7;
old.entries.insert("kept.txt".into(), entry("x"));
let bytes = old.to_bytes();
let crc = flint_store::crc64_nvme(&bytes);
let stamps = flint_store::GenerationStamps {
generation: 7,
epoch: 1,
flush_uuid: "legacy".into(),
boundary_source: None,
posix: None,
};
store
.put_whole(&cfg.manifest_key(), bytes.into(), &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert!(loaded.handle().legacy, "an un-migrated workspace must be recognised as legacy");
assert_eq!(loaded.manifest.seq, 7);
let mut next = loaded.manifest.clone();
next.seq += 1;
next.entries.insert("added.txt".into(), entry("x"));
manifest::cas_write(store.as_ref(), &cfg, &next, Some(&loaded.handle()), 2, "migrate")
.await
.unwrap();
let after = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert!(!after.handle().legacy);
assert_eq!(after.manifest.entries.len(), 2, "migration must not lose entries");
let (_, poisoned) = store.get_whole(&cfg.manifest_key(), None).await.unwrap();
assert!(
manifest::LeanManifest::parse(&poisoned).is_err(),
"the legacy key still parses as a manifest — an old syncer would serve a stale project from it"
);
assert!(String::from_utf8_lossy(&poisoned).contains("upgrade flint-sync"));
}
#[tokio::test]
async fn a_pointer_naming_a_missing_generation_refuses_rather_than_reading_empty() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 2;
m.entries.insert("a.txt".into(), entry("x"));
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "seed").await.unwrap();
let gen2 = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
store.delete(&gen2).await.unwrap();
match manifest::load(store.as_ref(), &cfg).await {
Err(LeanError::State(msg)) => {
assert!(msg.contains("does not exist"), "unexpected message: {msg}");
}
Ok(None) => panic!("a broken pointer read as an EMPTY workspace — the next barrier would re-seed over it"),
Ok(Some(_)) => panic!("a broken pointer read as a LOADED manifest"),
Err(e) => panic!("expected a State refusal, got {e:?}"),
}
}
#[tokio::test]
async fn a_second_writer_at_the_same_generation_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 1;
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "a").await.unwrap();
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
let mut mine = loaded.manifest.clone();
mine.seq += 1;
manifest::cas_write(store.as_ref(), &cfg, &mine, Some(&loaded.handle()), 1, "b").await.unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
let err = manifest::cas_write(store.as_ref(), &cfg, &theirs, Some(&loaded.handle()), 1, "c")
.await
.unwrap_err();
assert!(
matches!(err, LeanError::Store(flint_store::StoreError::PreconditionFailed(_))),
"expected a precondition failure, got {err:?}"
);
}
#[tokio::test]
async fn superseded_generations_are_reaped_but_the_live_one_and_a_window_survive() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let prefix = format!("{PREFIX}/.flint/lean/manifests/");
let mut handle = None;
for seq in 1..=12u64 {
let mut m = manifest::LeanManifest::default();
m.seq = seq;
m.entries.insert("a.txt".into(), entry("a.txt"));
let meta = manifest::cas_write(
store.as_ref(),
&cfg,
&m,
handle.as_ref(),
1,
&format!("flush-{seq}"),
)
.await
.unwrap();
handle = Some(manifest::ManifestHandle { etag: meta.etag, legacy: false, prev_chunks: Vec::new() });
}
assert_eq!(store.list(&prefix).await.unwrap().len(), 12);
let removed = manifest::sweep_generations(store.as_ref(), &cfg).await.unwrap();
let left = store.list(&prefix).await.unwrap();
assert_eq!(left.len(), manifest::KEEP_GENERATIONS + 1);
assert_eq!(removed, 12 - (manifest::KEEP_GENERATIONS + 1));
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
assert!(
left.iter().any(|o| Some(&o.key) == lp.pointer.entries_key.as_ref()),
"the reaper deleted the generation the pointer names"
);
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(loaded.manifest.seq, 12);
assert!(loaded.manifest.entries.contains_key("a.txt"));
}
#[tokio::test]
async fn the_reaper_collects_an_orphan_from_a_crash_before_the_pointer_cas() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 1;
m.entries.insert("a.txt".into(), entry("a.txt"));
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "live").await.unwrap();
let live = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
let bytes = m.to_bytes();
let crc = flint_store::crc64_nvme(&bytes);
let stamps = flint_store::GenerationStamps {
generation: 99,
epoch: 1,
flush_uuid: "crashed".into(),
boundary_source: None,
posix: None,
};
let orphan = cfg.generation_key(99, "crashed");
store.put_whole(&orphan, bytes.into(), &PutCondition::IfNoneMatchAny, &stamps, crc).await.unwrap();
assert_eq!(
manifest::sweep_generations(store.as_ref(), &cfg).await.unwrap(),
0,
"the sweep reaped a generation above the pointer that could still be an in-flight publish"
);
assert!(store.head(&orphan).await.is_ok());
store.backdate_epoch(&orphan, manifest::ORPHAN_GRACE_SECS + 60);
assert_eq!(manifest::sweep_generations(store.as_ref(), &cfg).await.unwrap(), 1);
let left: Vec<String> = store
.list(&format!("{PREFIX}/.flint/lean/manifests/"))
.await
.unwrap()
.into_iter()
.map(|o| o.key)
.collect();
assert!(!left.contains(&orphan), "the orphan survived the sweep");
assert!(left.contains(&live), "the LIVE generation was reaped");
assert!(manifest::load(store.as_ref(), &cfg).await.unwrap().is_some());
}
#[tokio::test]
async fn a_rotation_reads_and_writes_no_generation_object() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 3;
for i in 0..200 {
m.entries.insert(format!("f{i:04}.txt"), entry("f"));
}
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "seed").await.unwrap();
let gen = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
let before = store.head(&gen).await.unwrap();
store.reset_op_counts();
manifest::rotate_for_takeover(store.as_ref(), &cfg, 2).await.unwrap().unwrap();
let ops = store.op_counts();
assert!(
store.total_ops() <= 3,
"a rotation should be a couple of small requests, not {ops:?}"
);
let after = store.head(&gen).await.unwrap();
assert_eq!(before.etag, after.etag);
assert_eq!(
store.list(&format!("{PREFIX}/.flint/lean/manifests/")).await.unwrap().len(),
1,
"a rotation wrote a new generation object — it must reuse the standing one"
);
}
struct AuthRefusing {
inner: Arc<MemoryStore>,
refuse: std::sync::atomic::AtomicBool,
refuse_reads: std::sync::atomic::AtomicBool,
}
impl AuthRefusing {
fn new(inner: Arc<MemoryStore>) -> Self {
Self {
inner,
refuse: std::sync::atomic::AtomicBool::new(false),
refuse_reads: std::sync::atomic::AtomicBool::new(false),
}
}
fn set_refuse(&self, v: bool) {
self.refuse.store(v, std::sync::atomic::Ordering::SeqCst);
}
fn set_refuse_reads(&self, v: bool) {
self.refuse_reads.store(v, std::sync::atomic::Ordering::SeqCst);
}
}
#[async_trait::async_trait]
impl ObjectStore for AuthRefusing {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
if self.refuse_reads.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
if self.refuse_reads.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
if self.refuse_reads.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
if self.refuse_reads.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
if self.refuse.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_refused_credential_pauses_the_holder_without_fencing_it() {
let inner = Arc::new(MemoryStore::new());
let proxy = Arc::new(AuthRefusing::new(inner.clone()));
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store: proxy.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_none(),
"a healthy holder started out reading as credential-paused"
);
proxy.set_refuse(true);
let e = lease::renew(&mut a).await.unwrap_err();
assert!(e.is_auth(), "a 403 renewal did not classify as a credential fault: {e}");
assert!(
!matches!(e, LeanError::Fenced(_)),
"a credential refusal self-fenced a live writer: {e}"
);
assert!(a.lease.is_some(), "the refusal dropped the lease");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_some(),
"the refusal left no local evidence at all"
);
let gp = a.cfg.state_dir().join("gauges.json");
let mut g: serde_json::Value = serde_json::from_slice(&std::fs::read(&gp).unwrap()).unwrap();
g["auth_paused_since_unix"] = serde_json::json!(1_000_u64);
std::fs::write(&gp, serde_json::to_vec(&g).unwrap()).unwrap();
lease::renew(&mut a).await.unwrap_err();
assert_eq!(
a.load_gauges().unwrap().auth_paused_since_unix,
Some(1_000),
"a second refusal slid the pause clock forward, so the gauge no \
longer answers the question it exists to answer"
);
a.write_gauges(None).unwrap();
assert_eq!(
a.load_gauges().unwrap().auth_paused_since_unix,
Some(1_000),
"an ordinary gauge tick erased the credential pause"
);
proxy.set_refuse(false);
lease::renew(&mut a).await.expect("renew after the credentials came back");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_none(),
"the pause outlived the credentials being restored"
);
}
#[tokio::test]
async fn an_idle_writer_records_and_clears_a_credential_pause_on_its_floor_tick() {
let inner = Arc::new(MemoryStore::new());
let proxy = Arc::new(AuthRefusing::new(inner.clone()));
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store: proxy.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
noted_not_regular: Default::default(),
};
a.checkout().await.unwrap();
a.floor_tick().await.expect("an idle tick with working credentials");
assert!(a.load_gauges().unwrap().auth_paused_since_unix.is_none(), "a healthy idle writer reads as paused");
proxy.set_refuse_reads(true);
let e = a.floor_tick().await.expect_err("fixture: the idle tick made no refused request");
assert!(e.is_auth(), "the refused tick did not classify as a credential fault: {e}");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_some(),
"an idle writer's refused tick left no local evidence of the pause"
);
proxy.set_refuse_reads(false);
a.floor_tick().await.expect("an idle tick after the credentials came back");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_none(),
"the pause outlived the credentials being restored on an idle writer"
);
}
#[test]
fn is_auth_reads_through_the_from_conversion() {
fn via_question_mark() -> super::LeanResult<()> {
Err(flint_store::StoreError::Auth("ExpiredToken".into()))?;
Ok(())
}
assert!(via_question_mark().unwrap_err().is_auth());
assert!(!LeanError::State("x".into()).is_auth());
assert!(!LeanError::Fenced("x".into()).is_auth());
let other: LeanError = flint_store::StoreError::Other("boom".into()).into();
assert!(!other.is_auth(), "an ordinary store error read as a credential fault");
let pf: LeanError = flint_store::StoreError::PreconditionFailed("412".into()).into();
assert!(!pf.is_auth(), "a deposal read as a credential fault");
}
fn chunks_of(keys: &[String]) -> Vec<Vec<String>> {
let refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect();
let cuts = super::chunk::chunk_ranges_with(&refs, 8, 2, 32);
let mut out = Vec::new();
let mut start = 0;
for c in cuts {
out.push(keys[start..c].to_vec());
start = c;
}
out
}
fn fixed_chunks_of(keys: &[String], n: usize) -> Vec<Vec<String>> {
keys.chunks(n).map(|c| c.to_vec()).collect()
}
fn changed_chunks(a: &[Vec<String>], b: &[Vec<String>]) -> usize {
let before: std::collections::HashSet<_> = a.iter().collect();
b.iter().filter(|c| !before.contains(c)).count()
}
#[test]
fn a_front_insert_rewrites_one_chunk_not_the_project() {
let base: Vec<String> = (0..300).map(|i| format!("src/f{i:04}.txt")).collect();
let mut inserted = base.clone();
inserted.insert(0, "src/a000-brand-new.txt".to_string());
assert!(inserted.windows(2).all(|w| w[0] < w[1]), "fixture is not sorted");
let c0 = chunks_of(&base);
let c1 = chunks_of(&inserted);
assert!(c0.len() > 4, "fixture produced {} chunks — too few to say anything", c0.len());
let cd = changed_chunks(&c0, &c1);
let f0 = fixed_chunks_of(&base, 8);
let f1 = fixed_chunks_of(&inserted, 8);
let fd = changed_chunks(&f0, &f1);
assert!(
fd >= f0.len(),
"the control did not exhibit the cascade it exists to demonstrate ({fd} of {} chunks \
changed), so the assertion below proves nothing about this fixture",
f0.len()
);
assert!(
cd <= 2,
"a front insert rewrote {cd} of {} content-defined chunks (fixed-count rewrote {fd}) — \
boundaries are moving with POSITION, which is the failure mode §3 disqualifies",
c0.len()
);
}
#[test]
fn an_edit_far_from_a_chunk_leaves_it_alone() {
let base: Vec<String> = (0..300).map(|i| format!("src/f{i:04}.txt")).collect();
let mut edited = base.clone();
edited.insert(150, "src/f0149-inserted.txt".to_string());
edited.sort();
let c0 = chunks_of(&base);
let c1 = chunks_of(&edited);
assert_eq!(c0.first(), c1.first(), "the FIRST chunk moved for a change in the middle");
assert_eq!(c0.last(), c1.last(), "the LAST chunk moved for a change in the middle");
assert!(
changed_chunks(&c0, &c1) <= 2,
"a middle insert rewrote {} chunks",
changed_chunks(&c0, &c1)
);
}
#[test]
fn chunk_sizing_respects_min_max_and_keeps_the_tail() {
let keys: Vec<String> = (0..500).map(|i| format!("k{i:05}")).collect();
let refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect();
let cuts = super::chunk::chunk_ranges_with(&refs, 8, 3, 20);
assert_eq!(*cuts.last().unwrap(), keys.len(), "the tail run was dropped");
let mut prev = 0;
for (i, c) in cuts.iter().enumerate() {
let len = c - prev;
assert!(len <= 20, "chunk {i} has {len} entries, over the max of 20");
if i + 1 < cuts.len() {
assert!(len >= 3, "chunk {i} has {len} entries, under the min of 3");
}
prev = *c;
}
let total: usize = cuts.iter().scan(0, |p, c| { let l = c - *p; *p = *c; Some(l) }).sum();
assert_eq!(total, keys.len(), "chunks do not partition the key stream");
}
#[test]
fn chunk_boundaries_are_stable_and_content_addresses_are_not_crc() {
let keys: Vec<String> = (0..64).map(|i| format!("src/f{i:03}.txt")).collect();
let refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect();
let a = super::chunk::chunk_ranges_with(&refs, 8, 2, 32);
let b = super::chunk::chunk_ranges_with(&refs, 8, 2, 32);
assert_eq!(a, b, "chunking is not deterministic");
assert!(!a.is_empty() && *a.last().unwrap() == 64);
let x = super::chunk::chunk_address(b"{\"a\":1}");
let y = super::chunk::chunk_address(b"{\"a\":2}");
assert_ne!(x, y, "two different chunk bodies share an address");
assert_eq!(x.len(), 32, "address is not 128 bits of hex");
assert_eq!(x, super::chunk::chunk_address(b"{\"a\":1}"), "address is not stable");
}
fn entry_at(k: &str, seq: u64) -> super::manifest::LeanEntry {
super::manifest::LeanEntry {
key: format!("files/{k}"),
etag: format!("e-{k}-{seq}"),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 10,
mode: 0o644,
mtime_unix: 1_700_000_000,
generation: seq,
epoch: 1,
}
}
fn manifest_of(n: usize, seq: u64) -> super::manifest::LeanManifest {
super::manifest::LeanManifest {
seq,
entries: (0..n).map(|i| (format!("src/f{i:05}.txt"), entry_at(&format!("f{i:05}"), seq))).collect(),
sole_writer: false,
boundary_source: None,
}
}
#[tokio::test]
async fn a_three_file_publish_writes_chunks_proportional_to_the_change() {
let inner = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
let m0 = manifest_of(4000, 1);
inner.reset_op_counts();
let meta = manifest::cas_write_chunked(store.as_ref(), &cfg, &m0, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await
.expect("first chunked publish");
let first_puts = *inner.op_counts().get("put_whole").unwrap_or(&0);
let p0 = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer;
let chunks0 = match p0.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("the pointer is not chunked"),
};
assert!(chunks0.len() >= 4, "fixture made {} chunks — too few to measure", chunks0.len());
assert!(
first_puts as usize >= chunks0.len(),
"the first publish wrote {first_puts} objects for {} chunks — it cannot have written \
them all, so the comparison below is against nothing",
chunks0.len()
);
let mut m1 = m0.clone();
m1.seq = 2;
for i in [7usize, 1500, 3900] {
m1.entries.insert(format!("src/f{i:05}.txt"), entry_at(&format!("f{i:05}"), 99));
}
let h = super::manifest::ManifestHandle { etag: meta.etag.clone(), legacy: false, prev_chunks: Vec::new() };
inner.reset_op_counts();
manifest::cas_write_chunked(store.as_ref(), &cfg, &m1, Some(&h), &chunks0,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await
.expect("incremental chunked publish");
let puts = *inner.op_counts().get("put_whole").unwrap_or(&0);
eprintln!(
"chunked publish: {} chunks; full publish {first_puts} objects, 3-file publish {puts}",
chunks0.len()
);
assert!(
puts * 5 < first_puts,
"a three-file publish wrote {puts} objects against the full publish's {first_puts} \
over {} chunks — untouched chunks are being rewritten, which is O(entries) again",
chunks0.len()
);
assert!(puts <= 6, "a three-file publish wrote {puts} objects");
let back = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(back.manifest.entries.len(), 4000, "entries were lost across the chunked publish");
assert_eq!(back.manifest.entries, m1.entries, "the assembled manifest is not what was published");
}
#[tokio::test]
async fn a_missing_chunk_refuses_rather_than_shortening_the_manifest() {
let inner = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
let m = manifest_of(2000, 1);
manifest::cas_write_chunked(store.as_ref(), &cfg, &m, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None }).await.unwrap();
let p = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer;
let chunks = match p.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
assert!(chunks.len() >= 2);
store.delete(&cfg.chunk_key(&chunks[chunks.len() / 2].addr)).await.unwrap();
let msg = match manifest::load(store.as_ref(), &cfg).await {
Err(e) => e.to_string(),
Ok(m) => panic!(
"a manifest with a missing chunk LOADED, with {} entries",
m.map(|l| l.manifest.entries.len()).unwrap_or(0)
),
};
assert!(
msg.contains("hole"),
"a manifest with a missing chunk did not refuse; it said: {msg}"
);
}
#[test]
fn a_pointer_naming_both_layouts_or_neither_is_refused() {
let mut p = super::manifest::Pointer {
seq: 3,
entries_key: Some("k".into()),
entries_seq: Some(3),
chunks: Some(vec![]),
sole_writer: false,
boundary_source: None,
epoch: 1,
};
assert!(p.entries().unwrap_err().to_string().contains("BOTH"));
p.entries_key = None;
p.chunks = None;
assert!(p.entries().unwrap_err().to_string().contains("no entries at all"));
p.chunks = Some(vec![]);
assert!(matches!(p.entries().unwrap(), super::manifest::Entries::Chunked(c) if c.is_empty()));
}
#[test]
fn assemble_refuses_every_way_a_chunk_list_can_lie() {
let entries: std::collections::BTreeMap<String, super::manifest::LeanEntry> =
(0..40).map(|i| (format!("k{i:03}"), entry_at(&format!("k{i:03}"), 1))).collect();
let split = super::chunk::split_with(&entries, 4, 2, 8).unwrap();
assert!(split.len() >= 3, "fixture made {} chunks", split.len());
let refs: Vec<_> = split.iter().map(|(r, _)| r.clone()).collect();
let bodies: Vec<Vec<u8>> = split.iter().map(|(_, b)| b.clone()).collect();
let ok = super::chunk::assemble(&refs, &bodies).unwrap();
assert_eq!(ok, entries, "assemble does not round-trip its own split");
let must_fail = |r: Vec<super::chunk::ChunkRef>, b: Vec<Vec<u8>>, why: &str| {
assert!(
super::chunk::assemble(&r, &b).is_err(),
"assemble accepted a chunk list that {why}"
);
};
let mut r = refs.clone();
r[1].n += 1;
must_fail(r, bodies.clone(), "claims more entries than its body holds");
let mut b = bodies.clone();
b[1] = bodies[2].clone();
must_fail(refs.clone(), b, "returns a body that is not the addressed one");
let mut r = refs.clone();
let mut b = bodies.clone();
r.swap(0, 1);
b.swap(0, 1);
must_fail(r, b, "is not in increasing key order");
must_fail(refs.clone(), bodies[..bodies.len() - 1].to_vec(), "names more chunks than were fetched");
let mut r = refs.clone();
r[1].first = "zzz".into();
must_fail(r, bodies.clone(), "disagrees with its body about the first key");
}
struct SweepMidRead {
inner: Arc<MemoryStore>,
current_key: String,
doomed_key: String,
next_pointer: Vec<u8>,
move_pointer: bool,
fired: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl ObjectStore for SweepMidRead {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
if key == self.doomed_key
&& !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst)
{
if self.move_pointer {
let stamps = GenerationStamps {
generation: 9,
epoch: 1,
flush_uuid: "sweep".into(),
boundary_source: None,
posix: None,
};
let crc = crc64_nvme(&self.next_pointer);
self.inner
.put_whole(
&self.current_key,
Bytes::from(self.next_pointer.clone()),
&PutCondition::Unconditional,
&stamps,
crc,
)
.await?;
}
self.inner.delete(&self.doomed_key).await?;
return Err(flint_store::StoreError::NotFound(format!("swept: {key}")));
}
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_reader_swept_mid_read_restarts_onto_the_current_generation() {
for move_pointer in [true, false] {
let inner = Arc::new(MemoryStore::new());
let plain: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
let m1 = manifest_of(600, 1);
let meta1 =
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await
.unwrap();
let p1 = manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap().pointer;
let c1 = match p1.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new() };
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await
.unwrap();
let lp2 = manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap();
let ptr2_bytes = serde_json::to_vec(&lp2.pointer).unwrap();
let stamps = GenerationStamps {
generation: 1,
epoch: 1,
flush_uuid: "rewind".into(),
boundary_source: None,
posix: None,
};
let p1_bytes = serde_json::to_vec(&p1).unwrap();
let crc = crc64_nvme(&p1_bytes);
plain
.put_whole(&cfg.current_key(), Bytes::from(p1_bytes), &PutCondition::Unconditional, &stamps, crc)
.await
.unwrap();
let c2: Vec<String> = match lp2.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("not chunked"),
};
let doomed = c1
.iter()
.find(|r| !c2.contains(&r.addr))
.expect("generation 2 shares every chunk with generation 1 — nothing to sweep");
let store: Arc<dyn ObjectStore> = Arc::new(SweepMidRead {
inner: inner.clone(),
current_key: cfg.current_key(),
doomed_key: cfg.chunk_key(&doomed.addr),
next_pointer: ptr2_bytes.clone(),
move_pointer,
fired: std::sync::atomic::AtomicBool::new(false),
});
let got = manifest::load(store.as_ref(), &cfg).await;
if move_pointer {
let loaded = got.expect("a reader swept mid-read did not restart").unwrap();
assert_eq!(
loaded.manifest.entries, m2.entries,
"the restart did not land on the CURRENT generation — a reader that mixes \
generations is worse than one that fails"
);
assert_eq!(loaded.manifest.seq, 2);
} else {
let msg = match got {
Err(e) => e.to_string(),
Ok(_) => panic!(
"a chunk vanished under an UNCHANGED pointer and the load succeeded — \
that is a hole in the live manifest, not a race"
),
};
assert!(msg.contains("hole"), "wrong refusal: {msg}");
}
}
}
#[tokio::test]
async fn the_chunk_reaper_takes_orphans_and_nothing_else() {
let inner = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 0;
let m1 = manifest_of(600, 1);
let meta1 = manifest::cas_write_chunked(store.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await.unwrap();
let c1 = match manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap()
.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new() };
manifest::cas_write_chunked(store.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await.unwrap();
let live: Vec<String> = match manifest::load_pointer(store.as_ref(), &cfg).await.unwrap()
.unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("not chunked"),
};
let orphans: Vec<&super::chunk::ChunkRef> =
c1.iter().filter(|r| !live.contains(&r.addr)).collect();
assert!(!orphans.is_empty(), "generation 2 superseded no chunk — nothing to reap");
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
assert_eq!(n, orphans.len(), "the reaper took {n} chunks, expected {}", orphans.len());
for r in &orphans {
assert!(
store.head(&cfg.chunk_key(&r.addr)).await.is_err(),
"superseded chunk {} survived the sweep",
r.addr
);
}
for a in &live {
assert!(store.head(&cfg.chunk_key(a)).await.is_ok(), "the reaper took a LIVE chunk {a}");
}
let back = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(back.manifest.entries, m2.entries, "the sweep left the manifest short");
let meta2 = {
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
super::manifest::ManifestHandle { etag: lp.etag, legacy: false, prev_chunks: Vec::new() }
};
let mut m3 = m2.clone();
m3.seq = 3;
m3.entries.insert("src/f00300.txt".into(), entry_at("changed3", 43));
let c2: Vec<super::chunk::ChunkRef> = match manifest::load_pointer(store.as_ref(), &cfg)
.await.unwrap().unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
manifest::cas_write_chunked(store.as_ref(), &cfg, &m3, Some(&meta2), &c2,
manifest::PublishStamps { epoch: 1, flush_uuid: "u3", boundary_source: None })
.await.unwrap();
let before = store.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
cfg.orphan_grace_secs = 3600;
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
let after = store.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
assert_eq!(n, 0, "the reaper took {n} chunks that were inside the grace");
assert_eq!(before, after, "the chunk count moved despite the grace");
}
#[tokio::test]
async fn the_chunk_reaper_aborts_when_a_publish_lands_under_it() {
let inner = Arc::new(MemoryStore::new());
let plain: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 0;
let m1 = manifest_of(600, 1);
let meta1 = manifest::cas_write_chunked(plain.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await.unwrap();
let c1 = match manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap()
.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new() };
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await.unwrap();
let lp2 = manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap();
let mut moved = lp2.pointer.clone();
moved.seq = 99;
let moved_bytes = serde_json::to_vec(&moved).unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(PublishOnList {
inner: inner.clone(),
current_key: cfg.current_key(),
next_pointer: moved_bytes,
chunks_prefix: format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR),
fired: std::sync::atomic::AtomicBool::new(false),
});
let before = plain.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
let after = plain.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
assert_eq!(n, 0, "the reaper deleted {n} chunks after a publish landed under it");
assert_eq!(before, after, "chunks disappeared across an aborted pass");
}
struct PublishOnList {
inner: Arc<MemoryStore>,
current_key: String,
next_pointer: Vec<u8>,
chunks_prefix: String,
fired: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl ObjectStore for PublishOnList {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
let out = self.inner.list(prefix).await;
if prefix == self.chunks_prefix
&& !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst)
{
let stamps = GenerationStamps {
generation: 99,
epoch: 1,
flush_uuid: "raced".into(),
boundary_source: None,
posix: None,
};
let crc = crc64_nvme(&self.next_pointer);
self.inner
.put_whole(
&self.current_key,
Bytes::from(self.next_pointer.clone()),
&PutCondition::Unconditional,
&stamps,
crc,
)
.await?;
}
out
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_chunked_barrier_keeps_a_foreign_write_it_never_read() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunked = true;
sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..60 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
let p = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap().pointer;
let chunks = match p.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
super::manifest::Entries::Single { .. } => {
panic!("the barrier published a single generation object with cfg.chunked set")
}
};
assert!(
chunks.len() >= 3,
"PRECONDITION: {} chunk(s) — too few for a foreign write and a local one to land in \
different ones, which is the case that matters",
chunks.len()
);
hitl_write(&store, &sc.cfg, "src/f000.txt", "user version", "dilip").await.unwrap();
write(dir.path(), "src/f059.txt", "v2");
backdate_baseline(&sc, "src/f059.txt");
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(
m.manifest.entries.len(),
60,
"the chunked publish changed the entry count — a seam lost or duplicated a key"
);
let cited = &m.manifest.entries["src/f000.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(
&body[..],
b"user version",
"the chunked publish lost the foreign write — this is the splice failure the merge \
model refutes, reached through the barrier"
);
let mine = &m.manifest.entries["src/f059.txt"];
let (_, body) = store.get_whole(&mine.key, Some(&mine.etag)).await.unwrap();
assert_eq!(&body[..], b"v2", "the local edit did not land");
}
#[tokio::test]
async fn migrating_to_chunks_leaves_no_generation_objects_behind() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunked = false; sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
sc.cfg.orphan_grace_secs = 0;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..40 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
write(dir.path(), "src/f000.txt", "v2");
backdate_baseline(&sc, "src/f000.txt");
sc.run_barrier().await.unwrap();
let gens = format!("{}/{}/manifests/", sc.cfg.prefix, super::LEAN_DIR);
assert!(
!store.list(&gens).await.unwrap().is_empty(),
"PRECONDITION: no generation objects to migrate away from"
);
sc.cfg.chunked = true;
write(dir.path(), "src/f001.txt", "v2");
backdate_baseline(&sc, "src/f001.txt");
sc.run_barrier().await.unwrap();
let p = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap().pointer;
assert!(
matches!(p.entries().unwrap(), super::manifest::Entries::Chunked(_)),
"the workspace did not migrate to a chunk list"
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries.len(), 40, "entries were lost migrating to chunks");
manifest::sweep_generations(store.as_ref(), &sc.cfg).await.unwrap();
assert!(
store.list(&gens).await.unwrap().is_empty(),
"generation objects survived the migration: {:?}",
store.list(&gens).await.unwrap().iter().map(|o| &o.key).collect::<Vec<_>>()
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries.len(), 40, "the migration sweep broke the live manifest");
}
struct StaleListing {
inner: Arc<MemoryStore>,
}
#[async_trait::async_trait]
impl ObjectStore for StaleListing {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
let mut out = self.inner.list(prefix).await?;
for o in out.iter_mut() {
o.last_modified_unix = Some(0);
}
Ok(out)
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn the_chunk_reaper_judges_the_grace_now_not_when_it_listed() {
let inner = Arc::new(MemoryStore::new());
let plain: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 3600;
let m1 = manifest_of(600, 1);
let meta1 = manifest::cas_write_chunked(plain.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await.unwrap();
let c1 = match manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap()
.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new() };
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await.unwrap();
let prefix = format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR);
let before = plain.list(&prefix).await.unwrap().len();
let live: Vec<String> = match manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap()
.unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("not chunked"),
};
assert!(
c1.iter().any(|r| !live.contains(&r.addr)),
"PRECONDITION: nothing is unreferenced, so the reaper has no candidate to mis-judge"
);
let store: Arc<dyn ObjectStore> = Arc::new(StaleListing { inner: inner.clone() });
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
let after = plain.list(&prefix).await.unwrap().len();
assert_eq!(
n, 0,
"the reaper collected {n} chunk(s) on the LISTING's word that they were ancient — \
an adopt-rewrite landing after the listing is invisible to that, and rule 4 works \
by producing exactly such a rewrite"
);
assert_eq!(before, after, "chunks disappeared: {before} → {after}");
}
#[tokio::test]
async fn the_chunk_reaper_does_not_head_what_the_listing_shows_inside_the_grace() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 3600;
let mut m = manifest_of(600, 1);
let mut prev: Option<(super::manifest::ManifestHandle, Vec<super::chunk::ChunkRef>)> = None;
for seq in 1..=6u64 {
m.seq = seq;
m.entries.insert(format!("src/f{:05}.txt", seq * 97), entry_at(&format!("v{seq}"), seq));
let (h, c) = match &prev { Some((h, c)) => (Some(h), c.as_slice()), None => (None, &[][..]) };
manifest::cas_write_chunked(store.as_ref(), &cfg, &m, h, c,
manifest::PublishStamps { epoch: 1, flush_uuid: "u", boundary_source: None })
.await.unwrap();
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
let chunks = match lp.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
prev = Some((super::manifest::ManifestHandle { etag: lp.etag, legacy: false, prev_chunks: Vec::new() }, chunks));
}
let prefix = format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR);
let live: Vec<String> = prev.as_ref().unwrap().1.iter().map(|r| r.addr.clone()).collect();
let orphans = store.list(&prefix).await.unwrap().iter()
.filter(|o| !live.iter().any(|a| o.key.ends_with(a.as_str()))).count();
assert!(orphans >= 5, "PRECONDITION: five publishes left {orphans} unreferenced chunks");
store.reset_op_counts();
assert_eq!(manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap(), 0, "a chunk inside the grace was collected");
let heads = store.op_counts().get("head").copied().unwrap_or(0);
assert_eq!(heads, 0, "the reaper HEADed {heads} chunks the listing already showed inside the grace");
for o in store.list(&prefix).await.unwrap() {
store.backdate_epoch(&o.key, 3700);
}
store.reset_op_counts();
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
assert_eq!(n, orphans, "past the grace the reaper took {n} of {orphans}");
assert!(store.op_counts().get("head").copied().unwrap_or(0) >= orphans as u64, "a chunk was collected without a HEAD");
for a in &live {
assert!(store.head(&cfg.chunk_key(a)).await.is_ok(), "the reaper took a LIVE chunk {a}");
}
}
#[tokio::test]
async fn the_barrier_reaps_unreferenced_chunks_without_being_asked() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
sc.cfg.orphan_grace_secs = 0; assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..60 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
let prefix = format!("{}/{}/chunks/", sc.cfg.prefix, super::LEAN_DIR);
let after_first = store.list(&prefix).await.unwrap().len();
assert!(after_first >= 3, "PRECONDITION: {after_first} chunk(s) — too few to supersede any");
write(dir.path(), "src/f000.txt", "v2");
backdate_baseline(&sc, "src/f000.txt");
sc.run_barrier().await.unwrap();
let live: Vec<String> = match manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap()
.unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("the barrier did not publish a chunk list"),
};
let present: Vec<String> = store
.list(&prefix)
.await
.unwrap()
.into_iter()
.filter_map(|o| o.key.rsplit('/').next().map(|s| s.to_string()))
.collect();
let stray: Vec<&String> = present.iter().filter(|a| !live.contains(a)).collect();
assert!(
stray.is_empty(),
"the barrier left {} unreferenced chunk(s) in the bucket ({stray:?}) — the reaper is \
not reachable from a publish, which is the whole defect",
stray.len()
);
for a in &live {
assert!(present.contains(a), "the barrier's own sweep deleted live chunk {a}");
}
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries.len(), 60, "the sweep left the manifest short");
}
#[tokio::test]
async fn a_lost_renew_response_does_not_self_fence() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let key = a.cfg.epoch_key();
let stale = a.lease.clone().unwrap();
store.epoch_renew(&key, &stale, None).await.unwrap();
let landed = store.epoch_read(&key).await.unwrap().unwrap().token;
assert_ne!(landed, stale.token, "fixture: the renew did not move the token");
lease::renew(&mut a)
.await
.expect("a renew whose previous response was lost is not a deposal");
let cell = store.epoch_read(&key).await.unwrap().unwrap().token;
assert_eq!(
a.lease.as_ref().map(|l| l.token.as_str()),
Some(cell.as_str()),
"the holder did not adopt the cell's token after its lost renew"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 12).await, "quiet polls exhausted ⇒ takeover");
assert!(matches!(lease::renew(&mut a).await.unwrap_err(), LeanError::Fenced(_)));
assert!(a.lease.is_none(), "a fenced holder must drop its lease");
}
#[tokio::test]
async fn a_foreign_claim_refuses_the_syncer_before_it_claims() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
lease::verify_claim(&a).await.unwrap();
a.cfg.project_id = Some("team-a/p".into());
lease::verify_claim(&a).await.unwrap();
let key = a.cfg.claim_key();
let body = br#"{"project_id":"team-b/q","created_unix":1,"stamped_by":"op"}"#.to_vec();
let stamps = GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: "claim".into(),
boundary_source: None,
posix: None,
};
let crc = crc64_nvme(&body);
store
.put_whole(&key, Bytes::from(body), &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let err = lease::verify_claim(&a).await.unwrap_err();
assert!(
matches!(err, LeanError::Refused(_)),
"a foreign claim is a REFUSAL (exit EXIT_REFUSED, final for the delivery), not a \
state error a restart would retry: {err}"
);
let err = err.to_string();
assert!(
err.contains("team-b/q") && err.contains("team-a/p"),
"the refusal must name both projects: {err}"
);
assert!(a.lease.is_none());
a.cfg.project_id = Some("team-b/q".into());
lease::verify_claim(&a).await.unwrap();
}
#[test]
fn the_drain_attestation_is_written_by_the_drain_and_cleared_at_startup() {
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let st = SyncerState::open(cfg.state_dir()).unwrap();
assert!(!st.drained_path().exists(), "a fresh state dir must carry no attestation");
st.sync_tree().unwrap();
st.write_drained(Some(7), 1).unwrap();
let v: serde_json::Value = serde_json::from_slice(&std::fs::read(st.drained_path()).unwrap()).unwrap();
assert_eq!(v["seq"], 7);
assert_eq!(v["acks"], 1);
assert!(v["unix"].as_u64().unwrap() > 0);
st.clear_drained().unwrap();
assert!(!st.drained_path().exists());
st.clear_drained().unwrap(); }
#[tokio::test]
async fn ranged_checkout_is_byte_identical_and_parallel() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big: Vec<u8> = (0..5 * 1024 * 1024).map(|i| (i % 251) as u8).collect();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
write(dir_a.path(), "small.txt", "not ranged");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.range_get_min_bytes = 1024 * 1024;
b.cfg.range_get_chunk_bytes = 512 * 1024; b.cfg.range_get_parallelism = 4;
store.reset_peak_get_range_in_flight();
store.inject_get_range_delay_ms(5);
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(
std::fs::read(dir_b.path().join("weights.bin")).unwrap(),
big,
"ranged materialisation must be byte-identical"
);
assert_eq!(read(dir_b.path(), "small.txt").unwrap(), "not ranged");
let peak = store.peak_get_range_in_flight();
assert!(peak > 1, "ranges must overlap; peak in flight was {peak}");
}
fn ranged_default_tree() -> Vec<u8> {
(0..20 * 1024 * 1024).map(|i| (i % 251) as u8).collect()
}
#[tokio::test]
async fn ranged_checkout_on_by_default_uses_ranges() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big = ranged_default_tree();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
write(dir_a.path(), "small.txt", "under the threshold");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert_eq!(
b.cfg.range_get_min_bytes,
8 * 1024 * 1024,
"the shipped default moved to 8 MiB on the 2026-09-10 runcr drill"
);
store.reset_peak_get_range_in_flight();
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(
std::fs::read(dir_b.path().join("weights.bin")).unwrap(),
big,
"the DEFAULT path must be byte-identical, not merely fast"
);
assert_eq!(read(dir_b.path(), "small.txt").unwrap(), "under the threshold");
assert!(
store.peak_get_range_in_flight() > 0,
"an object over the default threshold must take the ranged path"
);
assert_eq!(cr.ranged, 1, "exactly the over-threshold object, not both");
}
#[tokio::test]
async fn ranged_checkout_explicitly_disabled_uses_no_ranges() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big = ranged_default_tree();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
write(dir_a.path(), "small.txt", "under the threshold");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.range_get_min_bytes = 0;
store.reset_peak_get_range_in_flight();
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(
std::fs::read(dir_b.path().join("weights.bin")).unwrap(),
big,
"the whole-object arm must materialise the same bytes"
);
assert_eq!(
store.peak_get_range_in_flight(),
0,
"the whole-object arm must issue no ranged reads"
);
assert_eq!(cr.ranged, 0);
}
async fn scoped_fixture(store: &Arc<MemoryStore>) -> tempfile::TempDir {
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "inputs/wanted.txt", "the file the agent edits");
write(dir_a.path(), "inputs/also.txt", "and this one");
for i in 0..6 {
write(dir_a.path(), &format!("outputs/big-{i}.bin"), &"x".repeat(4096));
}
a.run_barrier().await.unwrap();
dir_a
}
#[tokio::test]
async fn a_scoped_checkout_materialises_only_the_admitted_set() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(cr.materialized, 2, "only the admitted citations");
assert_eq!(cr.out_of_scope, 6, "and the declined count says the scope DID something");
assert_eq!(cr.scope.as_deref(), Some(&["inputs".to_string()][..]));
assert!(read(dir_b.path(), "inputs/wanted.txt").is_some());
assert!(
read(dir_b.path(), "outputs/big-0.bin").is_none(),
"an out-of-scope citation must not reach the tree"
);
let base = b.state.load_baseline().unwrap();
assert_eq!(
base.entries.keys().cloned().collect::<Vec<_>>(),
vec!["inputs/also.txt".to_string(), "inputs/wanted.txt".to_string()],
"an unadmitted path must never be cited"
);
assert_eq!(b.state.load_scope().unwrap().as_deref(), Some(&["inputs".to_string()][..]));
}
#[tokio::test]
async fn what_a_scoped_checkout_never_materialised_it_can_never_delete() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert!(claim_until_held(&mut b, 12).await, "quiet polls exhausted ⇒ takeover");
let r1 = b.run_barrier().await.unwrap();
let r2 = b.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty() && r2.deleted.is_empty(), "{:?} {:?}", r1.deleted, r2.deleted);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
m.entries.len(),
8,
"the six unadmitted citations survive a scoped workspace's barriers: {:?}",
m.entries.keys().collect::<Vec<_>>()
);
}
#[tokio::test]
async fn a_scoped_checkout_budget_is_computed_over_the_admitted_set() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_whole = tempfile::tempdir().unwrap();
let mut whole = syncer(&store, dir_whole.path()).await;
whole.cfg.max_bytes = 1024;
let err = whole.checkout().await.expect_err("the whole tree is over this budget");
assert!(matches!(err, LeanError::Budget(_)), "{err}");
let dir_scoped = tempfile::tempdir().unwrap();
let mut scoped = syncer(&store, dir_scoped.path()).await;
scoped.cfg.max_bytes = 1024;
let cr = scoped
.checkout_scoped(Some(vec!["inputs".into()]))
.await
.expect("a budget is a promise about what THIS checkout writes");
assert_eq!(cr.materialized, 2);
}
#[tokio::test]
async fn an_all_rejected_checkout_scope_is_refused_not_widened() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let err = b
.checkout_scoped(Some(vec!["../escape".into(), "./here".into()]))
.await
.expect_err("every entry malformed ⇒ an EMPTY scope ⇒ the whole manifest");
assert!(format!("{err}").contains("WHOLE MANIFEST"), "{err}");
assert!(
read(dir_b.path(), "outputs/big-0.bin").is_none(),
"the refusal must happen BEFORE the first byte"
);
assert!(!b.state.marker_present(), "and must not open the agent-start gate");
}
#[tokio::test]
async fn a_live_tree_refuses_a_checkout_whose_scope_disagrees() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
let err = b.checkout().await.expect_err("scoped tree, unscoped request");
let msg = format!("{err}");
assert!(msg.contains("UNSCOPED"), "the error must name BOTH sets: {msg}");
assert!(msg.contains("inputs"), "the error must name BOTH sets: {msg}");
let dir_c = tempfile::tempdir().unwrap();
let mut c = syncer(&store, dir_c.path()).await;
c.checkout().await.unwrap();
let err = c
.checkout_scoped(Some(vec!["inputs".into()]))
.await
.expect_err("unscoped tree, scoped request");
assert!(format!("{err}").contains("cannot change the admitted set"), "{err}");
let r = b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert!(r.resumed_live_tree);
}
#[tokio::test]
async fn a_scoped_checkout_leaves_the_merge_base_whole() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
let base = b.state.load_baseline().unwrap();
assert_eq!(base.entries.len(), 2, "the CITATIONS are scoped");
assert_eq!(
base.inst_base.len(),
8,
"the MERGE BASE is not: {:?}",
base.inst_base.keys().collect::<Vec<_>>()
);
}
#[tokio::test]
async fn a_narrowed_merge_base_makes_every_unadmitted_citation_foreign() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
a.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
a.sync().await.unwrap();
assert!(
read(dir_a.path(), "outputs/big-0.bin").is_none(),
"a whole-tree sync must leave the declined citations declined"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
{
let mut base = b.state.load_baseline().unwrap();
base.inst_base.retain(|p, _| p.starts_with("inputs/"));
b.state.save_baseline(&base).unwrap();
}
b.sync().await.unwrap();
assert!(
read(dir_b.path(), "outputs/big-0.bin").is_some(),
"THE CLIFF: absent from the merge base reads as CHANGED \
(manifest.rs `unwrap_or(true)`, sync.rs `unwrap_or(false)`), so every \
declined citation comes back one sync later"
);
let base = b.state.load_baseline().unwrap();
assert_eq!(base.entries.len(), 8, "and the scope is gone entirely");
}
#[tokio::test]
async fn an_unreadable_scope_is_not_read_as_unscoped() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
if unsafe { libc::geteuid() } == 0 {
eprintln!("SKIPPED: running as root, mode bits cannot induce EACCES");
return;
}
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
let sd = b.cfg.state_dir();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o000)).unwrap();
let got = b.state.load_scope();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
let err = got.expect_err("EACCES must not read as 'unscoped'");
assert!(format!("{err}").contains("refusing to"), "{err}");
}
#[tokio::test]
async fn a_whole_tree_checkout_clears_a_stale_scope() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.state.save_scope(Some(&["inputs".to_string()])).unwrap();
assert!(!b.state.marker_present());
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 8, "the whole manifest, not the stale scope's share");
assert_eq!(b.state.load_scope().unwrap(), None, "and the stale scope is GONE");
}
fn bulky_conflict(i: usize) -> super::state::ConflictRecord {
super::state::ConflictRecord {
path: format!("{}/{}", "p".repeat(900), i),
foreign_etag: format!("\"etag-{i}\""),
preserved_key: None,
kind: "test-bulk".into(),
at_unix: 1_700_000_000 + i as u64,
}
}
#[tokio::test]
async fn the_conflict_log_rotates_instead_of_growing_without_bound() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
for i in 0..2000 {
a.state.append_conflict(&bulky_conflict(i)).unwrap();
}
let sd = a.cfg.state_dir();
let live = std::fs::metadata(sd.join("conflicts.jsonl")).unwrap().len();
assert!(
std::fs::metadata(sd.join("conflicts.1.jsonl")).is_ok(),
"past the cap the live log must have rotated"
);
assert!(live <= 1 << 20, "the LIVE log is what gets parsed on every read: {live}");
let all = a.state.load_conflicts().unwrap();
assert_eq!(all.len(), 2000, "a single rotation loses nothing");
assert_eq!(a.state.conflicts_dropped().unwrap(), 0);
assert_eq!(all[0].at_unix, 1_700_000_000);
assert_eq!(all[1999].at_unix, 1_700_000_000 + 1999);
}
#[tokio::test]
async fn a_rotation_never_shortens_what_a_reader_has_already_counted() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
let mut prev = 0usize;
let mut saw_rotation = false;
for i in 0..2000 {
a.state.append_conflict(&bulky_conflict(i)).unwrap();
let n = a.state.load_conflicts().unwrap().len();
assert!(n >= prev, "the log SHRANK at append {i}: {prev} -> {n}");
if std::fs::metadata(a.cfg.state_dir().join("conflicts.1.jsonl")).is_ok() {
saw_rotation = true;
}
prev = n;
}
assert!(saw_rotation, "the run must actually cross a rotation or it proves nothing");
}
#[tokio::test]
async fn a_second_rotation_reports_what_it_dropped() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
for i in 0..4000 {
a.state.append_conflict(&bulky_conflict(i)).unwrap();
}
let dropped = a.state.conflicts_dropped().unwrap();
let held = a.state.load_conflicts().unwrap().len();
assert!(dropped > 0, "a second rotation DID drop records; the count must say so");
assert_eq!(
dropped as usize + held,
4000,
"dropped + held must account for every record: {dropped} + {held}"
);
}
#[tokio::test]
async fn an_unreadable_conflict_log_is_not_reported_as_empty() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
if unsafe { libc::geteuid() } == 0 {
eprintln!("SKIPPED: running as root, mode bits cannot induce EACCES");
return;
}
a.state.append_conflict(&bulky_conflict(1)).unwrap();
let sd = a.cfg.state_dir();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o000)).unwrap();
let got = a.state.load_conflicts();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
let err = got.expect_err("an unreadable log must not read as 'no conflicts'");
assert!(format!("{err}").contains("refusing to report"), "{err}");
}
#[test]
fn a_shortened_conflict_log_never_reports_an_empty_ack() {
use super::sentinel::conflicts_since;
let rec = |i: usize| bulky_conflict(i);
let grew = vec![rec(0), rec(1), rec(2)];
assert_eq!(conflicts_since(1, grew.clone()).len(), 2);
assert_eq!(conflicts_since(3, grew.clone()).len(), 0, "grew by nothing");
let shrank = vec![rec(7), rec(8)];
assert_eq!(
conflicts_since(5, shrank.clone()).len(),
2,
"a shortened log must not yield an empty conflict set"
);
}
struct ComposeWithoutChecksum(Arc<MemoryStore>);
#[async_trait::async_trait]
impl ObjectStore for ComposeWithoutChecksum {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
let mut m = self.0.compose_generation(spec).await?;
m.crc64_b64 = None;
Ok(m)
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
let mut out = self.0.list(prefix).await?;
for o in out.iter_mut() {
o.last_modified_unix = Some(0);
}
Ok(out)
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.0.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.0.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
async fn epoch_handoff(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<()> {
self.0.epoch_handoff(key, lease, echo).await
}
async fn epoch_enqueue(
&self,
key: &str,
observed: &flint_store::EpochState,
holder_id: &str,
) -> flint_store::StoreResult<flint_store::EpochState> {
self.0.epoch_enqueue(key, observed, holder_id).await
}
}
#[tokio::test]
async fn a_compose_that_reports_no_checksum_refuses_instead_of_citing() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(ComposeWithoutChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let cfg = {
let mut c = cfg_for(dir.path());
c.whole_put_max = 4096;
c
};
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store,
cfg,
state,
lease: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
std::fs::write(dir.path().join("big.bin"), vec![7u8; 20_000]).unwrap();
let err = a
.run_barrier()
.await
.expect_err("a publish with no checksum must refuse, not cite");
assert!(
format!("{err}").contains("refusing to cite bytes nothing vouched for"),
"{err}"
);
let m = manifest::load(mem.as_ref(), &a.cfg).await.unwrap();
assert!(
m.map(|l| l.manifest.entries.is_empty()).unwrap_or(true),
"a refused publish must leave no citation"
);
}
async fn rescope_fixture(store: &Arc<MemoryStore>) -> (tempfile::TempDir, Syncer) {
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(store, dir.path()).await;
b.checkout().await.unwrap();
assert!(claim_until_held(&mut b, 12).await, "quiet polls exhausted ⇒ takeover");
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 8);
(dir, b)
}
#[tokio::test]
async fn a_narrow_unwatches_without_publishing_a_single_deletion() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
let r = b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(r.uncited, 6, "six citations should have left the held set");
assert_eq!(r.unlinked, 6, "and six files the tree");
assert!(r.kept_dirty.is_empty());
assert!(read(dir.path(), "outputs/big-0.bin").is_none(), "the narrow did not unlink");
assert!(read(dir.path(), "inputs/wanted.txt").is_some(), "the narrow took an admitted path");
assert_eq!(b.state.load_scope().unwrap().as_deref(), Some(&["inputs".to_string()][..]));
let r1 = b.run_barrier().await.unwrap();
let r2 = b.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty() && r2.deleted.is_empty(), "{:?} {:?}", r1.deleted, r2.deleted);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 8, "a narrow published a deletion: {:?}", m.entries.keys());
}
#[tokio::test]
async fn the_same_unlink_without_the_uncite_publishes_deletions() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
for i in 0..6 {
std::fs::remove_file(dir.path().join(format!("outputs/big-{i}.bin"))).unwrap();
}
let r1 = b.run_barrier().await.unwrap();
let r2 = b.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty(), "first absence is not delete-eligible: {:?}", r1.deleted);
assert_eq!(r2.deleted.len(), 6, "the delete rule did not bite: {:?}", r2.deleted);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 2, "the bucket should have lost six objects here");
}
#[tokio::test]
async fn a_widen_materialises_and_cites_what_it_newly_admits() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert!(read(dir.path(), "outputs/big-0.bin").is_none());
let r = b.rescope(Some(vec!["inputs".into(), "outputs".into()])).await.unwrap();
assert_eq!(r.materialized, 6, "widen did not fetch the newly admitted set");
assert_eq!(r.already_held, 2, "and must not refetch what it already holds");
assert_eq!(r.uncited, 0);
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "widen did not materialise");
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 8);
let r1 = b.run_barrier().await.unwrap();
assert!(r1.uploaded.is_empty(), "widen re-uploaded its own fetch: {:?}", r1.uploaded);
}
#[tokio::test]
async fn a_narrow_leaves_the_merge_base_whole() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (_dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
let base = b.state.load_baseline().unwrap();
assert_eq!(base.entries.len(), 2, "the HELD set narrows");
assert_eq!(base.inst_base.len(), 8, "the MERGE BASE does not");
assert!(base.inst_base.contains_key("outputs/big-0.bin"));
assert!(!base.entries.contains_key("outputs/big-0.bin"));
assert!(!base.prev_scan.contains("outputs/big-0.bin"), "prev_scan must drop with entries");
}
#[tokio::test]
async fn a_crash_between_the_intent_and_the_unlink_converges() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
drop: (0..6).map(|i| format!("outputs/big-{i}.bin")).collect(),
})
.unwrap();
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "nothing applied yet");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed, "the barrier ran over a half-applied rescope");
assert!(r1.deleted.is_empty(), "{:?}", r1.deleted);
let r2 = b.run_barrier().await.unwrap();
assert!(!r2.rescope_replayed, "the replay must clear its own intent");
assert!(r2.deleted.is_empty(), "{:?}", r2.deleted);
assert!(read(dir.path(), "outputs/big-0.bin").is_none(), "the replay did not finish the job");
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 8, "converging cost the bucket objects");
}
#[tokio::test]
async fn a_crash_between_the_uncite_and_the_unlink_does_not_re_cite() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
drop: (0..6).map(|i| format!("outputs/big-{i}.bin")).collect(),
})
.unwrap();
let mut base = b.state.load_baseline().unwrap();
for i in 0..6 {
let p = format!("outputs/big-{i}.bin");
base.entries.remove(&p);
base.prev_scan.remove(&p);
}
b.state.save_baseline(&base).unwrap();
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "the files are still on disk");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed);
assert!(
r1.uploaded.is_empty(),
"the barrier re-cited what the narrow dropped: {:?}",
r1.uploaded
);
assert!(read(dir.path(), "outputs/big-0.bin").is_none());
assert_eq!(
b.state.load_baseline().unwrap().entries.len(),
2,
"the narrow was undone by its own recovery"
);
}
#[tokio::test]
async fn a_rescope_refuses_to_drop_a_path_with_unpublished_changes() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
write(dir.path(), "outputs/big-0.bin", "the user's unpublished edit");
let err = b.rescope(Some(vec!["inputs".into()])).await.unwrap_err();
match err {
LeanError::State(m) => assert!(m.contains("outputs/big-0.bin"), "{m}"),
e => panic!("wrong error: {e:?}"),
}
assert!(b.state.load_scope().unwrap().is_none(), "a refused rescope changed the scope");
assert!(b.state.load_scope_intent().unwrap().is_none(), "a refused rescope left an intent");
assert_eq!(
read(dir.path(), "outputs/big-0.bin").as_deref(),
Some("the user's unpublished edit"),
"a refused rescope touched the tree"
);
b.run_barrier().await.unwrap();
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
}
#[tokio::test]
async fn a_replay_keeps_a_dirty_path_instead_of_wedging_the_barrier() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
drop: (0..6).map(|i| format!("outputs/big-{i}.bin")).collect(),
})
.unwrap();
write(dir.path(), "outputs/big-0.bin", "edited after the intent landed");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed);
assert!(b.state.load_scope_intent().unwrap().is_none(), "the replay did not converge");
assert_eq!(
read(dir.path(), "outputs/big-0.bin").as_deref(),
Some("edited after the intent landed")
);
assert!(b.state.load_baseline().unwrap().entries.contains_key("outputs/big-0.bin"));
assert!(read(dir.path(), "outputs/big-1.bin").is_none(), "the clean ones still left");
assert!(
b.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.path == "outputs/big-0.bin" && c.kind == "rescope-kept-locally-dirty"),
"a kept path must be surfaced, not silently retained"
);
}
#[tokio::test]
async fn an_all_rejected_rescope_is_refused_not_widened() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (_dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
let err = b.rescope(Some(vec!["../escape".into(), "./here".into()])).await.unwrap_err();
assert!(matches!(err, LeanError::State(_)), "{err:?}");
assert_eq!(
b.state.load_scope().unwrap().as_deref(),
Some(&["inputs".to_string()][..]),
"a refused rescope widened the workspace"
);
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 2);
}
#[tokio::test]
async fn a_rescope_before_any_checkout_is_refused() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir.path()).await;
let err = b.rescope(Some(vec!["inputs".into()])).await.unwrap_err();
match err {
LeanError::State(m) => assert!(m.contains("checkout"), "{m}"),
e => panic!("wrong error: {e:?}"),
}
}
#[tokio::test]
async fn a_rescope_to_none_holds_everything_and_clears_the_scope() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 2);
let r = b.rescope(None).await.unwrap();
assert_eq!(r.materialized, 6);
assert!(b.state.load_scope().unwrap().is_none(), "the scope document must be REMOVED");
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 8);
assert!(read(dir.path(), "outputs/big-5.bin").is_some());
}
#[tokio::test]
async fn a_checkout_does_not_wait_out_a_standing_lease() {
let store = Arc::new(MemoryStore::new());
let pdir = tempfile::tempdir().unwrap();
let mut publisher = syncer(&store, pdir.path()).await;
assert!(claim_until_held(&mut publisher, 3).await);
publisher.checkout().await.unwrap();
write(pdir.path(), "README.md", "the real readme");
write(pdir.path(), "src/main.rs", "fn main() {}");
publisher.run_barrier().await.unwrap();
assert!(claim_until_held(&mut publisher, 3).await);
let held = publisher.lease.clone().expect("the publisher holds a lease");
let rdir = tempfile::tempdir().unwrap();
let mut reader = syncer(&store, rdir.path()).await;
let done = tokio::time::timeout(
std::time::Duration::from_secs(2),
super::verbs::run_verb(&mut reader, super::verbs::Step::Checkout),
)
.await;
match done {
Err(_) => panic!(
"the checkout verb waited on the publisher's lease — it has no business \
holding the publish fence"
),
Ok(r) => r.expect("the checkout itself must succeed"),
}
assert_eq!(read(rdir.path(), "README.md").unwrap(), "the real readme");
assert_eq!(read(rdir.path(), "src/main.rs").unwrap(), "fn main() {}");
let cell = store.epoch_read(&reader.cfg.epoch_key()).await.unwrap().expect("the cell stands");
assert_eq!(cell.holder_id, held.holder_id, "the reader deposed the publisher");
assert_eq!(cell.epoch, held.epoch, "the reader moved the epoch");
assert!(!cell.released, "the reader released someone else's lease");
assert!(reader.lease.is_none(), "the reader came away holding a lease");
}
#[tokio::test]
async fn a_checkout_issues_no_write_to_the_bucket() {
let store = Arc::new(MemoryStore::new());
let pdir = tempfile::tempdir().unwrap();
let mut publisher = syncer(&store, pdir.path()).await;
assert!(claim_until_held(&mut publisher, 3).await);
publisher.checkout().await.unwrap();
for i in 0..8 {
write(pdir.path(), &format!("f{i}.txt"), &format!("body {i}"));
}
publisher.run_barrier().await.unwrap();
let rdir = tempfile::tempdir().unwrap();
let mut reader = syncer(&store, rdir.path()).await;
store.reset_op_counts();
super::verbs::run_verb(&mut reader, super::verbs::Step::Checkout).await.unwrap();
let ops = store.op_counts();
for op in [
"put_whole",
"delete",
"delete_version",
"copy_object",
"compose_generation",
"epoch_acquire",
"epoch_renew",
"epoch_release",
"abort_upload",
] {
assert_eq!(
ops.get(op).copied().unwrap_or(0),
0,
"checkout called {op} — it is no longer a read, and `read_only_then` is no \
longer justified. Full shape: {ops:?}"
);
}
assert!(ops.values().sum::<u64>() > 0, "the checkout made no requests at all: {ops:?}");
assert_eq!(reader.state.load_baseline().unwrap().entries.len(), 8);
}
struct PublishMidCheckout {
inner: Arc<MemoryStore>,
cfg: LeanConfig,
trigger_key: String,
trigger_path: String,
fired: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl ObjectStore for PublishMidCheckout {
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
if key == self.trigger_key
&& !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst)
{
let loaded = manifest::load(self.inner.as_ref(), &self.cfg)
.await
.expect("the publisher reads the standing manifest")
.expect("there is one");
let body = Bytes::from_static(b"the publisher's next generation");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 1,
epoch: 1,
flush_uuid: "the-publisher".into(),
boundary_source: None,
posix: None,
};
let meta = self
.inner
.put_whole(&self.trigger_key, body, &PutCondition::Unconditional, &stamps, crc)
.await?;
let mut m = loaded.manifest.clone();
m.seq += 1;
m.entries.get_mut(&self.trigger_path).expect("the path is cited").etag = meta.etag;
manifest::cas_write(
self.inner.as_ref(),
&self.cfg,
&m,
Some(&loaded.handle()),
1,
"the-publisher",
)
.await
.expect("the publish lands");
}
self.inner.get_whole(key, if_match).await
}
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_publish_that_lands_mid_checkout_names_the_publisher_not_a_stranger() {
let inner = Arc::new(MemoryStore::new());
let pdir = tempfile::tempdir().unwrap();
let mut publisher = syncer(&inner, pdir.path()).await;
publisher.cfg.sole_writer = true;
assert!(claim_until_held(&mut publisher, 3).await);
publisher.checkout().await.unwrap();
write(pdir.path(), "README.md", "generation one");
publisher.run_barrier().await.unwrap();
let rdir = tempfile::tempdir().unwrap();
let mut reader = syncer(&inner, rdir.path()).await;
let racing = Arc::new(PublishMidCheckout {
inner: inner.clone(),
cfg: reader.cfg.clone(),
trigger_key: reader.cfg.file_key("README.md"),
trigger_path: "README.md".to_string(),
fired: std::sync::atomic::AtomicBool::new(false),
});
reader.store = racing as Arc<dyn ObjectStore>;
let err = reader.checkout().await.expect_err("the citation it loaded is one generation stale");
let msg = format!("{err}");
assert!(
msg.contains("its publisher published while the checkout was running"),
"the reader was not told who moved it: {msg}"
);
assert!(msg.contains("seq 1") && msg.contains("seq 2"), "name both generations: {msg}");
assert!(
msg.contains("Re-run checkout"),
"a reader one generation behind has a remedy; say it: {msg}"
);
assert!(msg.contains("SOLE WRITER"), "the underlying refusal must survive: {msg}");
assert!(
read(rdir.path(), "README.md").is_none(),
"it materialized a file it could not verify"
);
}
#[test]
fn siblings_racing_to_create_one_parent_do_not_refuse_each_other() {
let dir = tempfile::tempdir().unwrap();
for round in 0..50 {
let root = dir.path().join(format!("r{round}"));
std::fs::create_dir(&root).unwrap();
let go = Arc::new(std::sync::Barrier::new(8));
let hs: Vec<_> = (0..8)
.map(|i| {
let root = root.clone();
let go = go.clone();
std::thread::spawn(move || {
go.wait();
super::barrier::contained_path(&root, &format!("new/deeper/f-{i}"))
})
})
.collect();
for h in hs {
h.join().unwrap().expect("a sibling's mkdir is not a containment refusal");
}
assert!(root.join("new/deeper").is_dir());
}
}
#[test]
fn the_temp_retry_paths_replace_a_leftover_and_recreate_a_vanished_parent() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::create_dir_all(root.join("d")).unwrap();
std::fs::write(root.join("d/f.txt.flint-sync-tmp"), b"STALE").unwrap();
let st = super::barrier::write_file_atomic(&root.join("d/f.txt"), b"NEW", Some(0o644)).unwrap();
assert_eq!(std::fs::read(root.join("d/f.txt")).unwrap(), b"NEW");
assert_eq!(st.len(), 3, "the returned metadata is the written file's, not the leftover's");
assert!(!root.join("d/f.txt.flint-sync-tmp").exists(), "the temp name is gone after the rename");
let outside = tempfile::tempdir().unwrap();
std::fs::write(outside.path().join("victim"), b"ORIGINAL").unwrap();
std::os::unix::fs::symlink(outside.path().join("victim"), root.join("d/g.txt.flint-sync-tmp"))
.unwrap();
super::barrier::write_file_atomic(&root.join("d/g.txt"), b"NEW", None).unwrap();
assert_eq!(std::fs::read(outside.path().join("victim")).unwrap(), b"ORIGINAL");
assert_eq!(std::fs::read(root.join("d/g.txt")).unwrap(), b"NEW");
let (target, present) = super::barrier::contained_path_stat(root, "gone/h.txt").unwrap();
assert!(present.is_none(), "the walk reports an absent final component as None");
std::fs::remove_dir(root.join("gone")).unwrap();
super::barrier::write_file_atomic(&target, b"NEW", None).unwrap();
assert_eq!(std::fs::read(root.join("gone/h.txt")).unwrap(), b"NEW");
let (_, present) = super::barrier::contained_path_stat(root, "gone/h.txt").unwrap();
assert_eq!(present.map(|m| m.len()), Some(3));
let missing = root.join("nostate/x.json");
let err = super::safefs::write_via_tmp(&missing, &root.join("nostate/x.json.tmp"), b"{}", None)
.unwrap_err();
assert!(err.to_string().contains("not exclusively creatable"), "{err}");
assert!(!root.join("nostate").exists());
}
#[tokio::test]
async fn a_fresh_fetch_whose_bytes_do_not_match_the_manifest_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "model/config.json", "{\"layers\": 12, \"hidden\": 768}");
write(dir_a.path(), "README.md", "intact");
a.run_barrier().await.unwrap();
store.inject_corrupt_body(&a.cfg.file_key("model/config.json"), |b| b[10] ^= 0x01);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let err = b.checkout().await.expect_err("a corrupt fresh fetch must refuse");
let msg = err.to_string();
assert!(msg.contains("CRC-64") && msg.contains("model/config.json"), "{msg}");
assert!(
!dir_b.path().join("model/config.json").exists(),
"nothing may be written for a body that fails the check"
);
assert!(!dir_b.path().join("model/config.json.flint-sync-tmp").exists());
}
#[tokio::test]
async fn a_ranged_fetch_whose_bytes_do_not_match_the_manifest_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big: Vec<u8> = (0..5 * 1024 * 1024).map(|i| (i % 251) as u8).collect();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
a.run_barrier().await.unwrap();
store.inject_corrupt_body(&a.cfg.file_key("weights.bin"), |b| {
let i = b.len() / 2 + 777;
b[i] ^= 0x01;
});
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.range_get_min_bytes = 1024 * 1024;
b.cfg.range_get_chunk_bytes = 512 * 1024; b.cfg.range_get_parallelism = 4;
let err = b.checkout().await.expect_err("a corrupt ranged fetch must refuse");
let msg = err.to_string();
assert!(msg.contains("CRC-64") && msg.contains("10 ranges"), "{msg}");
assert!(!dir_b.path().join("weights.bin").exists(), "a wrong object must never be renamed into place");
assert!(!dir_b.path().join("weights.bin.flint-sync-tmp").exists(), "the temp file is cleaned up");
}
struct AttestsNoChecksum(Arc<MemoryStore>);
fn unattested(mut m: flint_store::ObjectMeta) -> flint_store::ObjectMeta {
m.crc64_b64 = None;
m
}
#[async_trait::async_trait]
impl ObjectStore for AttestsNoChecksum {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await.map(unattested)
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.put_whole(key, body, cond, stamps, crc).await.map(unattested)
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head(key).await.map(unattested)
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_whole(key, if_match).await.map(|(m, b)| (unattested(m), b))
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.0.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.0.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.0.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await.map(unattested)
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await.map(|(m, b)| (unattested(m), b))
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
async fn epoch_handoff(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<()> {
self.0.epoch_handoff(key, lease, echo).await
}
async fn epoch_enqueue(
&self,
key: &str,
observed: &flint_store::EpochState,
holder_id: &str,
) -> flint_store::StoreResult<flint_store::EpochState> {
self.0.epoch_enqueue(key, observed, holder_id).await
}
}
fn syncer_on(store: Arc<dyn ObjectStore>, root: &std::path::Path) -> Syncer {
let cfg = cfg_for(root);
let state = SyncerState::open(cfg.state_dir()).unwrap();
Syncer { store, cfg, state, lease: None, noted_not_regular: Default::default() }
}
fn crc_of(content: &str) -> String {
flint_store::crc64_to_b64(crc64_nvme(content.as_bytes()))
}
async fn assert_attests_nothing(mem: &Arc<MemoryStore>, store: &Arc<dyn ObjectStore>, key: &str) {
assert!(mem.head(key).await.unwrap().crc64_b64.is_some(), "the fixture object has a CRC");
assert!(store.head(key).await.unwrap().crc64_b64.is_none(), "HEAD attests a checksum");
let (m, _) = store.get_whole(key, None).await.unwrap();
assert!(m.crc64_b64.is_none(), "GET attests a checksum");
}
#[tokio::test]
async fn a_hitl_upload_on_a_backend_that_attests_no_checksum_is_cited_with_the_bytes_crc() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(AttestsNoChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer_on(store.clone(), dir.path());
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
let key = sc.cfg.file_key("docs/upload.pdf");
let body = Bytes::from_static(b"user bytes");
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "gateway-old".into(),
boundary_source: None,
posix: None,
};
let meta = mem
.put_whole(&key, body.clone(), &PutCondition::IfNoneMatchAny, &stamps, crc64_nvme(&body))
.await
.unwrap();
inbox::gateway_append(
store.as_ref(),
&sc.cfg,
InboxEntry {
path: "docs/upload.pdf".into(),
etag: meta.etag.clone(),
author: "dilip".into(),
added_unix: now_unix(),
crc64_b64: None,
},
)
.await
.unwrap();
assert_attests_nothing(&mem, &store, &key).await;
write(dir.path(), "agent.txt", "agent work v2 — longer");
sc.run_barrier().await.unwrap();
sc.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "docs/upload.pdf").unwrap(), "user bytes");
let b = sc.state.load_baseline().unwrap();
assert_eq!(
b.entries["docs/upload.pdf"].crc64_b64.as_deref(),
Some(crc_of("user bytes").as_str()),
"the baseline carries the CRC of the bytes the consume wrote"
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
let e = m.entries.get("docs/upload.pdf").expect("amputated!");
assert_eq!(e.crc64_b64, crc_of("user bytes"), "the repair cited the bytes' CRC");
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer_on(store.clone(), dir2.path());
sc2.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "docs/upload.pdf").unwrap(), "user bytes");
}
#[tokio::test]
async fn sync_on_a_backend_that_attests_no_checksum_still_sees_identical_bytes() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(AttestsNoChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let mut a = syncer_on(store.clone(), dir.path());
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
hitl_write(&mem, &a.cfg, "shared.txt", "foreign v2", "ci").await.unwrap();
assert_attests_nothing(&mem, &store, &a.cfg.file_key("shared.txt")).await;
let r = a.sync().await.unwrap();
assert!(r.applied.contains(&"shared.txt".to_string()));
let mut b = a.state.load_baseline().unwrap();
let stale = b.entries.get_mut("shared.txt").unwrap();
stale.etag = "\"stale-pre-sync-etag\"".into();
stale.mtime_unix -= 10;
stale.size = 1;
a.state.save_baseline(&b).unwrap();
let scanned = super::scan::scan(dir.path()).unwrap();
let c = super::scan::classify(&scanned, &a.state.load_baseline().unwrap());
assert!(c.uploads.contains("shared.txt"), "the fixture is not actually dirty");
let r = a.sync().await.unwrap();
assert!(
!r.conflicts.contains(&"shared.txt".to_string()),
"a phantom conflict was reported for byte-identical content"
);
assert!(r.applied.contains(&"shared.txt".to_string()));
}
#[tokio::test]
async fn a_consume_whose_bytes_do_not_match_the_writers_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, "docs/upload.pdf", "user bytes", "dilip").await.unwrap();
store.inject_corrupt_body(&sc.cfg.file_key("docs/upload.pdf"), |b| b[3] ^= 0x40);
sc.run_barrier().await.unwrap();
assert!(read(dir.path(), "docs/upload.pdf").is_none(), "corrupt bytes were written");
assert!(
sc.state.load_conflicts().unwrap().iter().any(|c| c.kind.starts_with("consume-refused-checksum")),
"the refusal was silent"
);
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(
ib.doc.entries.iter().any(|e| e.path == "docs/upload.pdf"),
"a refused entry must stay in the inbox, not be consumed away"
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("docs/upload.pdf"), "refused bytes were cited");
}
#[tokio::test]
async fn a_sync_whose_bytes_do_not_match_the_manifest_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "published bytes");
a.run_barrier().await.unwrap();
store.inject_corrupt_body(&a.cfg.file_key("f.txt"), |bytes| bytes[0] ^= 0x01);
let err = b.sync().await.expect_err("a sync of corrupt bytes must refuse");
assert!(err.to_string().contains("refusing to apply it"), "{err}");
assert!(read(dir_b.path(), "f.txt").is_none(), "corrupt bytes were written");
}
async fn hitl_remove(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str, author: &str) {
inbox::gateway_remove(
store.as_ref(),
cfg,
vec![inbox::Removal {
path: path.to_string(),
author: author.to_string(),
requested_unix: now_unix(),
moved_to: None,
refused: None,
}],
)
.await
.unwrap();
}
async fn hitl_rename(store: &Arc<MemoryStore>, cfg: &LeanConfig, from: &str, to: &str, author: &str) {
let m = manifest::load(store.as_ref(), cfg).await.unwrap().unwrap();
let src = m.manifest.entries.get(from).expect("source is cited");
let stamps = GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: format!("gateway-rename-{author}"),
boundary_source: None,
posix: None,
};
let dst = store
.copy_object(
&cfg.file_key(from),
Some(&src.etag),
&cfg.file_key(to),
&PutCondition::IfNoneMatchAny,
&stamps,
)
.await
.unwrap();
inbox::gateway_rename(
store.as_ref(),
cfg,
vec![InboxEntry {
path: to.to_string(),
etag: dst.etag,
author: author.to_string(),
added_unix: now_unix(),
crc64_b64: Some(src.crc64_b64.clone()),
}],
vec![inbox::Removal {
path: from.to_string(),
author: author.to_string(),
requested_unix: now_unix(),
moved_to: Some(to.to_string()),
refused: None,
}],
)
.await
.unwrap();
}
#[tokio::test]
async fn a_declared_removal_is_cited_out_in_one_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "README.md", "hello");
write(dir.path(), "src/main.rs", "fn main() {}");
sc.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest.seq;
hitl_remove(&store, &sc.cfg, "README.md", "dilip").await;
assert!(store.head(&sc.cfg.file_key("README.md")).await.is_ok());
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.pending_removal("README.md").is_some());
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.removed, vec!["README.md".to_string()], "declared this barrier");
assert_eq!(r.deleted, vec!["README.md".to_string()], "and GC'd this barrier, not next");
assert!(r.first_absence.is_empty(), "a declaration is not a first absence");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.seq, before + 1, "exactly one generation");
assert!(!m.manifest.entries.contains_key("README.md"));
assert!(m.manifest.entries.contains_key("src/main.rs"));
assert!(store.head(&sc.cfg.file_key("README.md")).await.is_err(), "object GC'd");
assert!(read(dir.path(), "README.md").is_none(), "unlinked from the agent's tree");
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.removals.is_empty(), "settled out of the cell");
assert!(!sc.state.load_baseline().unwrap().entries.contains_key("README.md"));
let r2 = sc.run_barrier().await.unwrap();
assert!(r2.no_change, "{r2:?}");
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 1);
assert!(read(dir_b.path(), "README.md").is_none());
}
#[tokio::test]
async fn a_declared_removal_of_a_dirty_path_applies_nothing_and_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
sc.run_barrier().await.unwrap();
write(dir.path(), "shared.txt", "the agent's unpublished v2");
backdate_baseline(&sc, "shared.txt");
hitl_remove(&store, &sc.cfg, "shared.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
assert!(r.removed.is_empty(), "nothing declared");
assert!(r.deleted.is_empty(), "nothing deleted");
assert_eq!(r.removals_refused, 1);
assert_eq!(read(dir.path(), "shared.txt").unwrap(), "the agent's unpublished v2");
assert_eq!(r.uploaded, vec!["shared.txt".to_string()], "the agent's edit publishes");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let (_, body) = store.get_whole(&sc.cfg.file_key("shared.txt"), None).await.unwrap();
assert_eq!(&body[..], b"the agent's unpublished v2");
assert!(m.manifest.entries.contains_key("shared.txt"));
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.pending_removal("shared.txt").is_none(), "not pending any more");
let rm = ib.doc.removals.iter().find(|r| r.path == "shared.txt").expect("kept, annotated");
let refusal = rm.refused.as_ref().expect("refused");
assert_eq!(refusal.kind, "removal-refused-dirty");
assert!(refusal.message.contains("dilip"), "names who asked: {}", refusal.message);
assert!(refusal.message.contains("kept"), "{}", refusal.message);
assert!(
sc.state.load_conflicts().unwrap().iter().any(|c| c.kind.starts_with("removal-refused-dirty")),
"the pod-side record too"
);
for _ in 0..2 {
let r = sc.run_barrier().await.unwrap();
assert!(r.deleted.is_empty() && r.removed.is_empty(), "{r:?}");
}
assert_eq!(read(dir.path(), "shared.txt").unwrap(), "the agent's unpublished v2");
assert!(store.head(&sc.cfg.file_key("shared.txt")).await.is_ok());
hitl_remove(&store, &sc.cfg, "shared.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["shared.txt".to_string()]);
assert!(inbox::load(store.as_ref(), &sc.cfg).await.unwrap().doc.removals.is_empty());
}
#[tokio::test]
async fn a_failed_unlink_publishes_no_deletion() {
if unsafe { libc::geteuid() } == 0 {
eprintln!("skipped: root ignores directory permissions");
return;
}
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "locked/f.txt", "keep");
sc.run_barrier().await.unwrap();
use std::os::unix::fs::PermissionsExt;
let locked = dir.path().join("locked");
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o555)).unwrap();
hitl_remove(&store, &sc.cfg, "locked/f.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap();
assert!(r.removed.is_empty() && r.deleted.is_empty(), "{r:?}");
assert_eq!(read(dir.path(), "locked/f.txt").unwrap(), "keep");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("locked/f.txt"), "still cited");
assert!(store.head(&sc.cfg.file_key("locked/f.txt")).await.is_ok(), "object untouched");
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.pending_removal("locked/f.txt").is_some(), "deferred, not refused");
assert!(sc
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind.starts_with("removal-unlink-failed")));
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["locked/f.txt".to_string()]);
assert!(read(dir.path(), "locked/f.txt").is_none());
}
#[tokio::test]
async fn a_rename_rides_one_manifest_generation() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "notes/a.txt", "the same bytes");
sc.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let src_crc = before.manifest.entries["notes/a.txt"].crc64_b64.clone();
hitl_rename(&store, &sc.cfg, "notes/a.txt", "docs/b.txt", "dilip").await;
assert!(store.head(&sc.cfg.file_key("docs/b.txt")).await.is_ok());
assert!(store.head(&sc.cfg.file_key("notes/a.txt")).await.is_ok());
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert_eq!(ib.doc.entries.len(), 1);
assert_eq!(ib.doc.removals.len(), 1);
assert_eq!(ib.doc.removals[0].moved_to.as_deref(), Some("docs/b.txt"));
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.consumed, 1);
assert_eq!(r.removed, vec!["notes/a.txt".to_string()]);
assert_eq!(r.deleted, vec!["notes/a.txt".to_string()]);
let after = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(after.manifest.seq, before.manifest.seq + 1, "ONE generation carries both halves");
assert!(!after.manifest.entries.contains_key("notes/a.txt"));
let dst = &after.manifest.entries["docs/b.txt"];
assert_eq!(dst.crc64_b64, src_crc, "the same bytes, attested");
assert!(store.head(&sc.cfg.file_key("notes/a.txt")).await.is_err(), "source GC'd");
assert_eq!(read(dir.path(), "docs/b.txt").unwrap(), "the same bytes");
assert!(read(dir.path(), "notes/a.txt").is_none());
assert!(inbox::load(store.as_ref(), &sc.cfg).await.unwrap().doc.removals.is_empty());
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 1);
assert_eq!(read(dir_b.path(), "docs/b.txt").unwrap(), "the same bytes");
}
#[tokio::test]
async fn a_rename_interrupted_before_the_source_unlink_re_applies() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "bytes");
sc.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest.seq;
hitl_rename(&store, &sc.cfg, "a.txt", "b.txt", "dilip").await;
sc.consume_inbox().await.unwrap();
assert_eq!(read(dir.path(), "b.txt").unwrap(), "bytes");
assert_eq!(read(dir.path(), "a.txt").unwrap(), "bytes", "the source is still there");
drop(sc);
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.removed, vec!["a.txt".to_string()]);
let after = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(after.manifest.seq, before + 1, "still one generation");
assert!(after.manifest.entries.contains_key("b.txt"));
assert!(!after.manifest.entries.contains_key("a.txt"));
assert!(read(dir.path(), "a.txt").is_none());
assert_eq!(read(dir.path(), "b.txt").unwrap(), "bytes");
}
#[tokio::test]
async fn a_journalled_declared_delete_survives_a_crash() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "gone.txt", "bytes");
write(dir.path(), "kept.txt", "bytes");
sc.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest.seq;
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
let mut intent = sc.state.load_intent().unwrap();
intent.declared_deletes = vec!["gone.txt".to_string()];
sc.state.save_intent(&intent).unwrap();
drop(sc);
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["gone.txt".to_string()], "one barrier, from the journal");
assert!(r.first_absence.is_empty());
let after = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(after.manifest.seq, before + 1);
assert!(!after.manifest.entries.contains_key("gone.txt"));
assert!(after.manifest.entries.contains_key("kept.txt"));
assert!(sc.state.load_intent().unwrap().declared_deletes.is_empty(), "cleared with the keys");
}
#[tokio::test]
async fn a_declared_removal_of_an_already_absent_path_declares_as_it_stands() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "both.txt", "bytes");
sc.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("both.txt")).unwrap();
hitl_remove(&store, &sc.cfg, "both.txt", "dilip").await;
hitl_remove(&store, &sc.cfg, "never/there.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["both.txt".to_string()]);
assert!(r.first_absence.is_empty());
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.removals.is_empty(), "both settled: {:?}", ib.doc.removals);
}
#[tokio::test]
async fn a_rename_whose_destination_the_agent_took_keeps_the_source() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "the user's bytes");
sc.run_barrier().await.unwrap();
write(dir.path(), "b.txt", "the agent's unpublished file");
hitl_rename(&store, &sc.cfg, "a.txt", "b.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.consumed, 1, "the destination entry is consumed (as a conflict)");
assert_eq!(r.removals_refused, 1);
assert!(r.removed.is_empty() && r.deleted.is_empty(), "{r:?}");
assert_eq!(read(dir.path(), "a.txt").unwrap(), "the user's bytes", "the source stays");
assert_eq!(read(dir.path(), "b.txt").unwrap(), "the agent's unpublished file");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("a.txt"), "still cited");
let (_, b) = store.get_whole(&sc.cfg.file_key("b.txt"), None).await.unwrap();
assert_eq!(&b[..], b"the agent's unpublished file", "the agent's version publishes");
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
let rm = ib.doc.removals.iter().find(|r| r.path == "a.txt").expect("refused, kept");
assert_eq!(rm.refused.as_ref().unwrap().kind, "removal-refused-destination-conflict");
let conflicts = sc.state.load_conflicts().unwrap();
let c = conflicts.iter().find(|c| c.kind == "consume-dirty" && c.path == "b.txt").unwrap();
let (_, kept) = store.get_whole(c.preserved_key.as_ref().unwrap(), None).await.unwrap();
assert_eq!(&kept[..], b"the user's bytes", "the moved bytes are preserved too");
}
struct FailPutTo(Arc<MemoryStore>, String, std::sync::atomic::AtomicBool);
impl FailPutTo {
fn arm(&self) {
self.2.store(true, std::sync::atomic::Ordering::SeqCst);
}
}
#[async_trait::async_trait]
impl ObjectStore for FailPutTo {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
if self.2.load(std::sync::atomic::Ordering::SeqCst) && key.ends_with(&self.1) {
return Err(flint_store::StoreError::Other(format!("injected failure on {key}")));
}
self.0.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_whole(key, if_match).await.map(|(m, b)| (unattested(m), b))
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.0.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.0.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.0.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await.map(|(m, b)| (unattested(m), b))
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
async fn epoch_handoff(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<()> {
self.0.epoch_handoff(key, lease, echo).await
}
async fn epoch_enqueue(
&self,
key: &str,
observed: &flint_store::EpochState,
holder_id: &str,
) -> flint_store::StoreResult<flint_store::EpochState> {
self.0.epoch_enqueue(key, observed, holder_id).await
}
}
#[tokio::test]
async fn a_consumed_hitl_write_survives_pod_replacement_before_the_cas() {
let inner = Arc::new(MemoryStore::new());
let failing = Arc::new(FailPutTo(
inner.clone(),
"/current".into(),
std::sync::atomic::AtomicBool::new(false),
));
let dir_a = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir_a.path());
let mut a = Syncer {
store: failing.clone() as Arc<dyn ObjectStore>,
state: SyncerState::open(cfg.state_dir()).unwrap(),
cfg,
lease: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "base.txt", "published");
a.run_barrier().await.unwrap();
let etag = hitl_write(&inner, &a.cfg, "ui/upload.txt", "acked to the user", "dilip").await.unwrap();
failing.arm();
let err = a.run_barrier().await.unwrap_err();
assert!(err.to_string().contains("injected"), "{err}");
assert_eq!(read(dir_a.path(), "ui/upload.txt").unwrap(), "acked to the user", "consumed into A's tree");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("ui/upload.txt"), "and not cited");
drop(a);
drop(dir_a);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&inner, dir_b.path()).await;
assert!(claim_until_held(&mut b, 3).await, "a released cell is claimable at once");
b.checkout().await.unwrap();
assert!(read(dir_b.path(), "ui/upload.txt").is_none(), "not cited, so not materialised");
let r = b.run_barrier().await.unwrap();
assert_eq!(r.consumed, 1, "the entry was still in the cell for the successor");
assert_eq!(read(dir_b.path(), "ui/upload.txt").unwrap(), "acked to the user");
let m = manifest::load(inner.as_ref(), &b.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries["ui/upload.txt"].etag, etag, "cited by the successor");
assert!(inbox::load(inner.as_ref(), &b.cfg).await.unwrap().doc.entries.is_empty(), "and then dropped");
}
#[tokio::test]
async fn an_invalid_sync_scope_is_acked_refused_and_never_wedges_publish() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"s1","scope":[]}"#);
a.sentinel_tick().await.expect("an invalid scope is the agent's error, not the syncer's");
let ack = a.read_ack(Verb::Sync).expect("no sync ack: the agent waits forever");
assert_eq!(ack.status, "refused-scope");
assert!(ack.nonces.contains(&"s1".to_string()), "the refusal does not name the agent's nonce");
assert!(
ack.reason.as_deref().unwrap_or("").contains("scope"),
"the refusal must say WHY: {:?}",
ack.reason
);
assert!(
a.load_pending(Verb::Sync).unwrap().is_none(),
"the pending record stood: every later tick repeats the same error"
);
write(dir.path(), "b.txt", "b");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"p1"}"#);
a.sentinel_tick().await.unwrap();
let ack = a.read_ack(Verb::Publish).expect("no publish ack after a refused sync");
assert_eq!(ack.status, "ok");
assert!(ack.nonces.contains(&"p1".to_string()));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("b.txt"), "the boundary the agent declared never reached the bucket");
}
#[tokio::test]
async fn a_refused_sync_pending_does_not_stop_the_cadence_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"s1","scope":["../escape"]}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
write(dir.path(), "c.txt", "c");
a.floor_tick().await.expect("a refused pending must not fail the floor tick");
assert_eq!(a.read_ack(Verb::Sync).map(|k| k.status), Some("refused-scope".into()));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("c.txt"), "the cadence barrier never ran behind the refused sync");
}
#[tokio::test]
async fn an_unscoped_sync_touch_widens_a_coalesced_scope_to_the_whole_tree() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"b","scope":["x/"]}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"a"}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
let p = a.load_pending(Verb::Sync).unwrap().unwrap();
assert!(p.scope.is_none(), "an unscoped touch was narrowed to {:?}", p.scope);
assert_eq!(p.nonces, vec!["b".to_string(), "a".to_string()]);
a.sentinel_tick().await.unwrap();
assert!(a.load_pending(Verb::Sync).unwrap().is_none());
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"c"}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"d","scope":["y/"]}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
let p = a.load_pending(Verb::Sync).unwrap().unwrap();
assert!(p.scope.is_none(), "a later scoped touch narrowed a whole-tree request to {:?}", p.scope);
}
#[tokio::test]
async fn a_publish_ack_carries_the_boundarys_conflict_records() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
write(dir.path(), "shared.txt", "the agent's longer edit");
hitl_write(&store, &a.cfg, "shared.txt", "UI", "ui").await.unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"p1"}"#);
a.sentinel_tick().await.unwrap();
let ack = a.read_ack(Verb::Publish).expect("no publish ack");
assert_eq!(ack.status, "ok");
assert!(
ack.report.conflicts.iter().any(|c| c.path == "shared.txt" && c.kind == "consume-dirty"),
"the publish ack carries no conflict record for the path the agent won: {:?}",
ack.report.conflicts
);
}
#[tokio::test]
async fn a_same_size_rewrite_within_the_scan_second_is_still_published() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "status.json", r#"{"ok":1}"#);
write(dir.path(), "other.txt", "untouched");
a.run_barrier().await.unwrap();
let sec = a.state.load_baseline().unwrap().entries["status.json"].mtime_unix;
write(dir.path(), "status.json", r#"{"ok":2}"#);
let f = std::fs::OpenOptions::new().write(true).open(dir.path().join("status.json")).unwrap();
f.set_modified(std::time::UNIX_EPOCH + std::time::Duration::new(sec as u64, 500_000_000)).unwrap();
drop(f);
let r = a.run_barrier().await.unwrap();
assert!(
r.uploaded.contains(&"status.json".to_string()),
"the rewrite was invisible: uploaded={:?} no_change={}",
r.uploaded,
r.no_change
);
assert!(!r.uploaded.contains(&"other.txt".to_string()), "nanosecond compare re-uploads everything");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, body) = store.get_whole(&a.cfg.file_key("status.json"), Some(&m.entries["status.json"].etag)).await.unwrap();
assert_eq!(&body[..], br#"{"ok":2}"#);
}
async fn vanished_base_recreates(answer_412: bool, compose: bool) {
let store = if answer_412 {
Arc::new(MemoryStore::new().with_if_match_missing_as_412())
} else {
Arc::new(MemoryStore::new())
};
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
if compose {
a.cfg.whole_put_max = 4; }
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "x.txt", "v1 body");
a.run_barrier().await.unwrap();
store.delete(&a.cfg.file_key("x.txt")).await.unwrap();
let probe = store
.put_whole(
&a.cfg.file_key("x.txt"),
Bytes::from_static(b"probe"),
&PutCondition::IfMatch("\"gone\"".into()),
&GenerationStamps { generation: 0, epoch: 0, flush_uuid: "probe".into(), boundary_source: None, posix: None },
crc64_nvme(b"probe"),
)
.await
.expect_err("If-Match on a missing key must fail");
assert_eq!(
matches!(probe, flint_store::StoreError::PreconditionFailed(_)),
answer_412,
"fixture: the double's answer is not the one this arm tests: {probe:?}"
);
write(dir.path(), "x.txt", "v2, longer than before");
let r = a.run_barrier().await.expect("a vanished base object failed the whole barrier");
assert!(r.uploaded.contains(&"x.txt".to_string()), "the edit was not published: {r:?}");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, body) = store.get_whole(&a.cfg.file_key("x.txt"), Some(&m.entries["x.txt"].etag)).await.unwrap();
assert_eq!(&body[..], b"v2, longer than before");
}
#[tokio::test]
async fn a_vanished_base_recreates_on_s3s_404() {
vanished_base_recreates(false, false).await;
}
#[tokio::test]
async fn a_vanished_base_recreates_on_ozones_412() {
vanished_base_recreates(true, false).await;
}
#[tokio::test]
async fn a_vanished_base_recreates_through_compose_on_s3s_404() {
vanished_base_recreates(false, true).await;
}
#[tokio::test]
async fn a_vanished_base_recreates_through_compose_on_ozones_412() {
vanished_base_recreates(true, true).await;
}
#[test]
fn containment_refuses_the_state_directory_too() {
let dir = tempfile::tempdir().unwrap();
super::barrier::contained_path(dir.path(), ".flint/x").expect_err("the control namespace must be refused");
super::barrier::contained_path(dir.path(), ".flint-sync/scope-intent.json")
.expect_err("the state directory must be refused");
super::barrier::contained_path(dir.path(), ".flint-sync").expect_err("the state directory itself");
super::barrier::contained_path(dir.path(), "ok/x").expect("an ordinary path");
super::barrier::contained_path(dir.path(), ".flint-syncer/notes").expect("a look-alike is not the state dir");
}
#[tokio::test]
async fn an_orphaned_consume_temp_is_never_published() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "a.txt", "real");
write(dir.path(), "a.txt.flint-sync-tmp", "torn consume leftover");
let r = a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("a.txt"));
assert!(
!m.entries.contains_key("a.txt.flint-sync-tmp") && !r.uploaded.iter().any(|p| p.ends_with(".flint-sync-tmp")),
"a crash-orphaned temp was published as data: {:?}",
r.uploaded
);
}
#[tokio::test]
async fn adopting_a_lost_renew_token_moves_the_cell() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let key = a.cfg.epoch_key();
let stale = a.lease.clone().unwrap();
store.epoch_renew(&key, &stale, None).await.unwrap();
let landed = store.epoch_read(&key).await.unwrap().unwrap().token;
lease::renew(&mut a).await.expect("a lost renew response is not a deposal");
let now = store.epoch_read(&key).await.unwrap().unwrap().token;
assert_ne!(now, landed, "the adoption wrote nothing: a challenger's quiet count keeps climbing");
assert_eq!(a.lease.as_ref().map(|l| l.token.as_str()), Some(now.as_str()), "the holder's token is not the cell's");
let cell = store.epoch_read(&key).await.unwrap().unwrap();
assert_eq!(cell.holder_id, a.lease.as_ref().unwrap().holder_id);
assert!(!cell.released);
}
type Hook = Box<dyn Fn() + Send + Sync>;
#[derive(Default)]
struct Hooks {
before_put: std::sync::Mutex<Option<(String, Hook)>>,
after_put: std::sync::Mutex<Option<(String, Hook)>>,
before_get_return: std::sync::Mutex<Option<(String, Hook)>>,
before_delete: std::sync::Mutex<Option<(String, Hook)>>,
after_head: std::sync::Mutex<Option<(String, Hook)>>,
compose_err_once: std::sync::atomic::AtomicBool,
acquire_err_once: std::sync::atomic::AtomicBool,
copy_fail: std::sync::atomic::AtomicBool,
put_fail_containing: std::sync::Mutex<Option<String>>,
handoff_races: std::sync::atomic::AtomicU32,
handoff_conflicts: std::sync::atomic::AtomicU32,
}
struct Hooked(Arc<MemoryStore>, Hooks);
impl Hooked {
fn before_put(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.before_put.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
fn after_put(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.after_put.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
fn before_get_return(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.before_get_return.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
fn before_delete(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.before_delete.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
fn after_head(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.after_head.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
}
fn take_hook(slot: &std::sync::Mutex<Option<(String, Hook)>>, key: &str) -> Option<Hook> {
let mut g = slot.lock().unwrap();
if g.as_ref().map(|(suf, _)| key.ends_with(suf.as_str())).unwrap_or(false) {
g.take().map(|(_, h)| h)
} else {
None
}
}
fn hooked_syncer(store: &Arc<Hooked>, root: &std::path::Path) -> Syncer {
let cfg = cfg_for(root);
Syncer {
store: store.clone() as Arc<dyn ObjectStore>,
state: SyncerState::open(cfg.state_dir()).unwrap(),
cfg,
lease: None,
noted_not_regular: Default::default(),
}
}
#[async_trait::async_trait]
impl ObjectStore for Hooked {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
if self.1.copy_fail.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Other("injected: preserve copy failed".into()));
}
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
if let Some(h) = take_hook(&self.1.before_put, key) {
h();
}
if let Some(needle) = self.1.put_fail_containing.lock().unwrap().as_deref() {
if key.contains(needle) {
return Err(flint_store::StoreError::Other(format!("injected: put refused for {key}")));
}
}
let r = self.0.put_whole(key, body, cond, stamps, crc).await;
if r.is_ok() {
if let Some(h) = take_hook(&self.1.after_put, key) {
h();
}
}
r
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
let r = self.0.compose_generation(spec).await;
if self.1.compose_err_once.swap(false, std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Other("injected: torn compose response".into()));
}
r
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
let r = self.0.head(key).await;
if let Some(h) = take_hook(&self.1.after_head, key) {
h();
}
r
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
let r = self.0.get_whole(key, if_match).await;
if let Some(h) = take_hook(&self.1.before_get_return, key) {
h();
}
r
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.0.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
if let Some(h) = take_hook(&self.1.before_delete, key) {
h();
}
self.0.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
if let Some(h) = take_hook(&self.1.before_delete, key) {
h();
}
self.0.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
let r = self.0.epoch_acquire(key, holder, observed).await;
if self.1.acquire_err_once.swap(false, std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Other("injected: lost acquire response".into()));
}
r
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
async fn epoch_handoff(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<()> {
use std::sync::atomic::Ordering::SeqCst;
if self.1.handoff_conflicts.fetch_update(SeqCst, SeqCst, |n| n.checked_sub(1)).is_ok() {
return Err(flint_store::StoreError::Conflict("injected: ConditionalRequestConflict".into()));
}
if let Ok(left) = self.1.handoff_races.fetch_update(SeqCst, SeqCst, |n| n.checked_sub(1)) {
let state = self.0.epoch_read(key).await?.expect("a cell to race");
self.0.epoch_enqueue(key, &state, &format!("synthetic-waiter-{left}")).await?;
}
self.0.epoch_handoff(key, lease, echo).await
}
async fn epoch_enqueue(
&self,
key: &str,
observed: &flint_store::EpochState,
holder_id: &str,
) -> flint_store::StoreResult<flint_store::EpochState> {
self.0.epoch_enqueue(key, observed, holder_id).await
}
}
#[tokio::test]
async fn the_manifest_cites_the_uploaded_length_not_the_scanned_one() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
a.cfg.whole_put_max = 1 << 20;
a.cfg.range_get_min_bytes = 1 << 20;
a.cfg.range_get_chunk_bytes = 1 << 20;
a.checkout().await.unwrap();
let path = dir.path().join("ckpt.bin");
std::fs::write(&path, vec![7u8; 4 << 20]).unwrap();
write(dir.path(), "aaa-first.txt", "uploads before ckpt.bin");
a.cfg.upload_fanout = 1;
let grow = path.clone();
hooked.before_put(&a.cfg.file_key("aaa-first.txt"), move || {
use std::io::Write;
std::fs::OpenOptions::new().append(true).open(&grow).unwrap().write_all(&vec![9u8; 2 << 20]).unwrap();
});
a.run_barrier().await.unwrap();
let head = inner.head(&a.cfg.file_key("ckpt.bin")).await.unwrap();
assert_eq!(head.size, 6 << 20, "fixture: the upload did not carry the grown file");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
m.entries["ckpt.bin"].size, head.size,
"the manifest cites a length the object does not have"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = hooked_syncer(&hooked, dir_b.path());
b.cfg.whole_put_max = 1 << 20;
b.cfg.range_get_min_bytes = 1 << 20;
b.cfg.range_get_chunk_bytes = 1 << 20;
assert!(claim_until_held(&mut b, 12).await, "takeover");
b.checkout().await.expect("the successor cannot check out the boundary: the workspace is unrecoverable");
assert_eq!(std::fs::metadata(dir_b.path().join("ckpt.bin")).unwrap().len(), 6 << 20);
}
#[tokio::test]
async fn a_crashed_compose_is_adopted_after_an_edit_not_parked_forever() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
a.cfg.whole_put_max = 1 << 20;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let path = dir.path().join("big.bin");
std::fs::write(&path, vec![1u8; 3 << 20]).unwrap();
hooked.1.compose_err_once.store(true, std::sync::atomic::Ordering::SeqCst);
a.run_barrier().await.expect_err("fixture: the torn compose did not fail the barrier");
inner.head(&a.cfg.file_key("big.bin")).await.expect("fixture: the compose did not land");
{
use std::io::Write;
std::fs::OpenOptions::new().append(true).open(&path).unwrap().write_all(&vec![2u8; 1 << 20]).unwrap();
}
let r = a.run_barrier().await.unwrap();
assert!(r.parked.is_empty(), "the crashed compose parked the path: {:?}", r.parked);
assert!(r.uploaded.contains(&"big.bin".to_string()), "not published: {r:?}");
let head = inner.head(&a.cfg.file_key("big.bin")).await.unwrap();
assert_eq!(head.size, 4 << 20, "the bucket holds the pre-edit bytes");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["big.bin"].size, 4 << 20);
}
#[tokio::test]
async fn a_consume_never_overwrites_a_write_that_landed_during_its_fetch() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
hitl_write(&inner, &a.cfg, "shared.txt", "UI", "ui").await.unwrap();
let late = dir.path().join("shared.txt");
hooked.before_get_return(&a.cfg.file_key("shared.txt"), move || {
std::fs::write(&late, "AGENT-LATE, a different length").unwrap();
});
a.run_barrier().await.unwrap();
assert_eq!(
read(dir.path(), "shared.txt").as_deref(),
Some("AGENT-LATE, a different length"),
"the agent's write was overwritten by a consume that checked the path before its fetch"
);
let recs = a.state.load_conflicts().unwrap();
assert!(
recs.iter().any(|c| c.path == "shared.txt" && c.kind == "consume-dirty" && c.preserved_key.is_some()),
"no record names the foreign version that lost: {recs:?}"
);
}
#[tokio::test]
async fn the_upload_refuses_a_symlink_swapped_in_after_the_scan() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
a.checkout().await.unwrap();
let outside = tempfile::tempdir().unwrap();
std::fs::write(outside.path().join("secret"), "SECRET").unwrap();
write(dir.path(), "cfg.yaml", "harmless");
write(dir.path(), "aaa-first.txt", "uploads before cfg.yaml");
a.cfg.upload_fanout = 1;
let p = dir.path().join("cfg.yaml");
let target = outside.path().join("secret");
hooked.before_put(&a.cfg.file_key("aaa-first.txt"), move || {
std::fs::remove_file(&p).unwrap();
std::os::unix::fs::symlink(&target, &p).unwrap();
});
let r = a.run_barrier().await.expect("a planted symlink must not fail the barrier");
assert!(!r.uploaded.contains(&"cfg.yaml".to_string()), "the link's target was published: {r:?}");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("cfg.yaml"), "the manifest cites the link's target");
assert!(inner.head(&a.cfg.file_key("cfg.yaml")).await.is_err(), "the outside bytes reached the bucket");
let recs = a.state.load_conflicts().unwrap();
assert!(recs.iter().any(|c| c.path == "cfg.yaml"), "nothing recorded the refusal: {recs:?}");
}
#[tokio::test]
async fn a_lost_acquire_response_still_rotates() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir_a = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir_a.path());
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "x");
a.run_barrier().await.unwrap();
let seq0 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert!(claim_until_held(&mut a, 3).await);
let dir_b = tempfile::tempdir().unwrap();
let mut b = hooked_syncer(&hooked, dir_b.path());
loop {
match lease::claim_step(&mut b, true).await.unwrap() {
lease::ClaimOutcome::Waiting { quiet_polls, .. } if quiet_polls >= 5 => break,
lease::ClaimOutcome::Waiting { .. } => {}
lease::ClaimOutcome::Claimed(_) => panic!("fixture: claimed before the threshold"),
}
}
hooked.1.acquire_err_once.store(true, std::sync::atomic::Ordering::SeqCst);
assert!(lease::claim_step(&mut b, true).await.is_err(), "fixture: the lost response did not surface");
let cell = inner.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
let b_id = b.state.load_incarnation().unwrap().unwrap().holder_id;
assert_eq!(cell.holder_id, b_id, "fixture: the acquire did not land");
let outcome = lease::claim_step(&mut b, true).await.unwrap();
assert!(matches!(outcome, lease::ClaimOutcome::Claimed(_)), "the retry did not claim");
let seq1 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert!(
seq1 > seq0,
"a lost acquire response skipped the takeover rotation (seq {seq0} → {seq1}): a straggler's CAS still matches"
);
}
#[tokio::test]
async fn a_parked_path_is_preserved_and_published_over_not_abandoned() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "dirty.txt", "published v1");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
let key = a.cfg.file_key("dirty.txt");
let body = Bytes::from("foreign bytes".to_string());
let crc = crc64_nvme(&body);
let cur = store.head(&key).await.unwrap();
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfMatch(cur.etag),
&GenerationStamps { generation: 2, epoch: 0, flush_uuid: "sibling".into(), boundary_source: None, posix: None },
crc,
)
.await
.unwrap();
let foreign_etag = meta.etag.clone();
let e = theirs.entries.get_mut("dirty.txt").unwrap();
e.etag = meta.etag.clone();
e.crc64_b64 = meta.crc64_b64.clone().unwrap();
e.size = meta.size;
e.generation = 2;
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "sibling").await.unwrap();
write(dir.path(), "dirty.txt", "the agent's own unpublished work, longer");
let r = a.run_barrier().await.unwrap();
assert!(r.parked.is_empty(), "the path was parked: {:?}", r.parked);
assert!(r.uploaded.contains(&"dirty.txt".to_string()), "the agent's version was not published: {r:?}");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, got) = store.get_whole(&key, Some(&m.entries["dirty.txt"].etag)).await.unwrap();
assert_eq!(&got[..], b"the agent's own unpublished work, longer", "the boundary does not carry the agent's bytes");
let rec = a
.state
.load_conflicts()
.unwrap()
.into_iter()
.find(|c| c.path == "dirty.txt" && c.kind == "upload-412-preserved")
.expect("no record names the foreign version that was superseded");
assert_eq!(rec.foreign_etag, foreign_etag);
let preserved = rec.preserved_key.expect("the foreign bytes were not preserved");
let (_, kept) = store.get_whole(&preserved, None).await.unwrap();
assert_eq!(&kept[..], b"foreign bytes", "the preserved copy is not the foreign version");
}
#[tokio::test]
async fn a_boundary_with_a_standing_park_is_partial_and_the_drain_does_not_attest_it() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
write(dir.path(), "dirty.txt", "published v1");
a.run_barrier().await.unwrap();
let key = a.cfg.file_key("dirty.txt");
let body = Bytes::from("foreign bytes".to_string());
let crc = crc64_nvme(&body);
let cur = inner.head(&key).await.unwrap();
inner
.put_whole(
&key,
body,
&PutCondition::IfMatch(cur.etag),
&GenerationStamps { generation: 2, epoch: 0, flush_uuid: "straggler".into(), boundary_source: None, posix: None },
crc,
)
.await
.unwrap();
write(dir.path(), "dirty.txt", "the agent's own unpublished work, longer");
*hooked.1.put_fail_containing.lock().unwrap() = Some("/conflicts/".into());
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"p1"}"#);
a.sentinel_tick().await.unwrap();
let ack = a.read_ack(Verb::Publish).expect("no ack");
assert_eq!(ack.status, "partial", "a boundary that does not carry the agent's file acked ok: {ack:?}");
assert_eq!(ack.report.dropped, vec!["dirty.txt".to_string()]);
assert_eq!(ack.report.parked, 1);
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"p2"}"#);
assert!(a.consume_sentinel(Verb::Publish).unwrap());
let drained = a.drain().await;
assert!(drained.is_err(), "the drain attested a boundary with a standing park: {drained:?}");
assert!(
!a.cfg.state_dir().join(super::state::DRAINED).exists(),
"drained.json written: the node will remove a tree whose only copy of dirty.txt is on it"
);
}
#[tokio::test]
async fn two_writers_publish_without_waiting_for_each_other() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "a/one.txt", "from A");
write(dir_b.path(), "b/one.txt", "from B");
let (ra, rb) = tokio::join!(a.run_barrier(), b.run_barrier());
let (ra, rb) = (ra.expect("A's barrier"), rb.expect("B's barrier"));
assert_eq!(ra.uploaded, vec!["a/one.txt"]);
assert_eq!(rb.uploaded, vec!["b/one.txt"]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("a/one.txt") && m.entries.contains_key("b/one.txt"), "{:?}", m.entries.keys());
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released && cell.handoff.is_none() && cell.waiters.is_empty(), "{cell:?}");
assert_eq!(cell.epoch, 2, "one epoch per barrier");
assert!(a.lease.is_none() && b.lease.is_none());
}
#[tokio::test]
async fn disjoint_edits_cross_at_the_next_consume() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "a/one.txt", "from A");
write(dir_b.path(), "b/one.txt", "from B");
a.run_barrier().await.unwrap();
let rb = b.run_barrier().await.unwrap();
assert_eq!(rb.foreign_queued, 1, "B's merge did not preserve A's entry as foreign");
assert!(read(dir_b.path(), "a/one.txt").is_none(), "the merge alone must not touch B's tree");
let rb2 = b.run_barrier().await.unwrap();
assert_eq!(rb2.consumed, 1, "B's next consume did not integrate A's file");
assert_eq!(read(dir_b.path(), "a/one.txt").as_deref(), Some("from A"));
let ra2 = a.run_barrier().await.unwrap();
assert_eq!(ra2.foreign_queued, 1);
let ra3 = a.run_barrier().await.unwrap();
assert_eq!(ra3.consumed, 1);
assert_eq!(read(dir_a.path(), "b/one.txt").as_deref(), Some("from B"));
}
#[tokio::test]
async fn a_same_path_edit_is_preserved_never_lost() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"));
write(dir_a.path(), "x.txt", "A's edit");
backdate_baseline(&a, "x.txt");
write(dir_b.path(), "x.txt", "B's edit, longer");
backdate_baseline(&b, "x.txt");
a.run_barrier().await.unwrap();
let rb = b.run_barrier().await.unwrap();
assert!(rb.parked.is_empty(), "B's upload parked instead of preserving: {rb:?}");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited = &m.entries["x.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"B's edit, longer", "the later commit is current");
let rec = b
.state
.load_conflicts()
.unwrap()
.into_iter()
.find(|c| c.path == "x.txt" && c.kind.starts_with("upload-412-preserved"))
.expect("B wrote no record of the version it superseded");
let preserved = rec.preserved_key.expect("the record names no preserved key");
let (_, kept) = store.get_whole(&preserved, None).await.unwrap();
assert_eq!(&kept[..], b"A's edit", "A's bytes are not where the record says");
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(read(dir_a.path(), "x.txt").as_deref(), Some("B's edit, longer"));
}
#[tokio::test]
async fn only_the_queue_head_is_told_to_poll_fast() {
let store = Arc::new(MemoryStore::new());
let dirs: Vec<_> = (0..3).map(|_| tempfile::tempdir().unwrap()).collect();
let mut a = syncer(&store, dirs[0].path()).await;
let mut b = syncer(&store, dirs[1].path()).await;
let mut c = syncer(&store, dirs[2].path()).await;
let next = |o: lease::ClaimOutcome| match o {
lease::ClaimOutcome::Waiting { next, .. } => Some(next),
lease::ClaimOutcome::Claimed(_) => None,
};
assert!(claim_until_held(&mut a, 1).await);
assert_eq!(next(lease::claim_step(&mut b, true).await.unwrap()), Some(true), "B heads the queue behind A");
assert_eq!(next(lease::claim_step(&mut c, true).await.unwrap()), Some(false), "C is second");
assert_eq!(next(lease::claim_step(&mut b, false).await.unwrap()), Some(true), "B still heads it on a re-poll");
lease::release(&mut a).await.unwrap();
assert_eq!(next(lease::claim_step(&mut c, false).await.unwrap()), Some(false));
assert_eq!(next(lease::claim_step(&mut b, false).await.unwrap()), None, "B claims its reservation");
assert_eq!(next(lease::claim_step(&mut c, false).await.unwrap()), Some(true), "C heads the queue behind B");
}
#[tokio::test]
async fn a_handoff_that_races_the_queue_still_hands_the_cell_on() {
for (races, conflicts) in [(2u32, 0u32), (0, 1), (3, 2)] {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&hooked, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
assert!(claim_until_held(&mut a, 1).await);
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }));
hooked.1.handoff_races.store(races, std::sync::atomic::Ordering::SeqCst);
hooked.1.handoff_conflicts.store(conflicts, std::sync::atomic::Ordering::SeqCst);
lease::release(&mut a).await.unwrap_or_else(|e| panic!("races {races}, conflicts {conflicts}: the release failed: {e}"));
let cell = inner.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(
cell.released,
"races {races}, conflicts {conflicts}: the release returned and the cell is still held by {}",
cell.holder_id
);
let b_id = lease::incarnation(&b).unwrap().holder_id;
assert_eq!(cell.handoff.as_deref(), Some(b_id.as_str()), "races {races}, conflicts {conflicts}: the head of the queue was not named");
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Claimed(_)));
}
}
#[tokio::test]
async fn the_ticket_hands_the_fence_to_the_queue_head() {
let store = Arc::new(MemoryStore::new());
let dirs: Vec<_> = (0..3).map(|_| tempfile::tempdir().unwrap()).collect();
let mut a = syncer(&store, dirs[0].path()).await;
let mut b = syncer(&store, dirs[1].path()).await;
let mut c = syncer(&store, dirs[2].path()).await;
let id = |sc: &Syncer| lease::incarnation(sc).unwrap().holder_id;
assert!(claim_until_held(&mut a, 1).await);
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }));
assert!(matches!(lease::claim_step(&mut c, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }));
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!(cell.waiters, vec![id(&b), id(&c)], "waiters queue in arrival order");
lease::release(&mut a).await.unwrap();
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released);
assert_eq!(cell.handoff.as_deref(), Some(id(&b).as_str()), "the handoff names the head");
assert_eq!(cell.waiters, vec![id(&c)]);
assert!(
matches!(lease::claim_step(&mut c, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }),
"a later waiter took a cell reserved for the queue head"
);
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Claimed(_)));
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!((cell.holder_id.as_str(), cell.released), (id(&b).as_str(), false));
assert_eq!(cell.waiters, vec![id(&c)], "the queue survives the handoff");
lease::release(&mut b).await.unwrap();
assert!(matches!(lease::claim_step(&mut c, true).await.unwrap(), lease::ClaimOutcome::Claimed(_)));
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!(cell.epoch, 3);
assert!(cell.waiters.is_empty() && cell.handoff.is_none());
}
#[tokio::test]
async fn a_dead_handoff_is_skipped_after_the_quiet_polls() {
let store = Arc::new(MemoryStore::new());
let dirs: Vec<_> = (0..3).map(|_| tempfile::tempdir().unwrap()).collect();
let mut a = syncer(&store, dirs[0].path()).await;
let mut b = syncer(&store, dirs[1].path()).await;
let mut c = syncer(&store, dirs[2].path()).await;
assert!(claim_until_held(&mut a, 1).await);
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }));
lease::release(&mut a).await.unwrap();
drop(b); assert!(!claim_until_held(&mut c, lease::HANDOFF_QUIET_POLLS).await, "C took a reservation that was not yet quiet");
assert!(claim_until_held(&mut c, 1).await, "C never took the dead reservation");
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!(cell.holder_id, lease::incarnation(&c).unwrap().holder_id);
assert!(cell.waiters.is_empty() && cell.handoff.is_none());
}
#[tokio::test]
async fn a_dead_holder_mid_commit_is_deposed_by_the_next_barrier() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
b.checkout().await.unwrap();
assert!(claim_until_held(&mut a, 1).await); write(dir_b.path(), "b.txt", "B publishes past a dead holder");
assert!(!claim_until_held(&mut b, 3).await, "the waiter deposed a holder below the quiet threshold");
let rb = b.run_barrier().await.expect("B's barrier deposes the dead holder and publishes");
assert_eq!(rb.uploaded, vec!["b.txt"]);
let err = lease::renew(&mut a).await.unwrap_err();
assert!(matches!(err, LeanError::Fenced(_)), "{err}");
write(dir_a.path(), "a.txt", "A, after being deposed");
a.run_barrier().await.expect("a deposed writer's next barrier claims again");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("a.txt") && m.entries.contains_key("b.txt"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_holder_deposed_mid_commit_abandons_the_barrier() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir_a = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir_a.path());
a.cfg.sentinel_min_interval_secs = 0;
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
write(dir_a.path(), "seed.txt", "seed");
a.run_barrier().await.unwrap();
let seq0 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
write(dir_a.path(), "work.txt", "v1");
touch_sentinel(dir_a.path(), control::PUBLISH, r#"{"nonce":"n1"}"#);
assert!(a.consume_sentinel(Verb::Publish).unwrap(), "fixture: the touch was not consumed");
assert!(matches!(a.sentinel_due().unwrap(), super::sentinel::Due::Ready), "fixture: not due");
let (rival_store, cfg) = (inner.clone(), a.cfg.clone());
hooked.before_put(&a.cfg.current_key(), move || {
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let key = cfg.epoch_key();
let seen = rival_store.epoch_read(&key).await.unwrap().unwrap();
assert!(!seen.released, "fixture: A is not holding at its CAS");
let l = rival_store.epoch_acquire(&key, "rival", Some(&seen)).await.unwrap();
manifest::rotate_for_takeover(rival_store.as_ref(), &cfg, l.epoch).await.unwrap();
})
});
});
let acks = a.sentinel_tick().await.expect("a fence is a retry, not an error the loop sees");
assert!(acks.is_empty(), "an ack was written for an abandoned barrier: {acks:?}");
assert!(a.lease.is_none());
assert!(a.load_pending(Verb::Publish).unwrap().is_some(), "the pending was dropped by the fence");
assert!(a.read_ack(Verb::Publish).is_none(), "an ack was written for an abandoned barrier");
let caps = a.read_capabilities().unwrap();
assert_eq!(caps.state, "live");
assert!(!caps.verbs.is_empty(), "the marker stopped advertising verbs");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("work.txt"), "the abandoned barrier's manifest landed");
assert_eq!(m.seq, seq0 + 1, "the rotation, and nothing after it");
let acks = a.sentinel_tick().await.expect("the retry");
assert_eq!(acks.len(), 1);
assert_eq!(acks[0].status, "ok");
assert!(acks[0].nonces.contains(&"n1".to_string()));
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("work.txt"));
}
#[tokio::test]
async fn readers_never_touch_the_cell() {
let store = Arc::new(MemoryStore::new());
let dir_w = tempfile::tempdir().unwrap();
let mut w = syncer(&store, dir_w.path()).await;
w.checkout().await.unwrap();
write(dir_w.path(), "f.txt", "published");
w.run_barrier().await.unwrap();
let epoch_before = store.epoch_read(&w.cfg.epoch_key()).await.unwrap().unwrap().epoch;
store.reset_op_counts();
let dir_r = tempfile::tempdir().unwrap();
let mut r = syncer(&store, dir_r.path()).await;
super::verbs::run_verb(&mut r, super::verbs::Step::Checkout).await.unwrap();
super::verbs::run_verb(&mut r, super::verbs::Step::Sync).await.unwrap();
assert_eq!(read(dir_r.path(), "f.txt").as_deref(), Some("published"));
let ops = store.op_counts();
assert!(
!ops.keys().any(|k| k.starts_with("epoch_") && *k != "epoch_read"),
"a reader wrote the cell: {ops:?}"
);
let cell = store.epoch_read(&w.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!(cell.epoch, epoch_before);
assert!(cell.released && cell.waiters.is_empty());
assert!(r.lease.is_none());
}
#[tokio::test]
async fn the_fence_costs_a_publishing_boundary_four_cell_requests_and_an_idle_one_none() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.floor_tick().await.unwrap();
store.reset_op_counts();
let idle = a.floor_tick().await.unwrap();
assert!(idle.no_change);
let ops = store.op_counts();
assert!(!ops.keys().any(|k| k.starts_with("epoch_")), "an idle tick touched the cell: {ops:?}");
assert_eq!(ops.values().sum::<u64>(), 2, "an idle tick is the inbox and the pointer: {ops:?}");
write(dir.path(), "f.txt", "v2");
backdate_baseline(&a, "f.txt");
store.reset_op_counts();
let busy = a.floor_tick().await.unwrap();
assert_eq!(busy.uploaded, 1);
let ops = store.op_counts();
assert_eq!(ops.get("epoch_read").copied(), Some(2), "{ops:?}");
assert_eq!(ops.get("epoch_acquire").copied(), Some(1), "{ops:?}");
assert_eq!(ops.get("epoch_handoff").copied(), Some(1), "{ops:?}");
assert_eq!(ops.get("epoch_renew"), None, "{ops:?}");
assert_eq!(ops.get("epoch_enqueue"), None, "{ops:?}");
}
#[tokio::test]
async fn a_pull_only_boundary_takes_no_fence_and_writes_nothing() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "gone.txt", "A deletes this");
a.floor_tick().await.unwrap();
b.checkout().await.unwrap();
b.floor_tick().await.unwrap();
write(dir_a.path(), "x.txt", "A's v2");
backdate_baseline(&a, "x.txt");
std::fs::remove_file(dir_a.path().join("gone.txt")).unwrap();
a.declared_barrier().await.unwrap();
let seq_a = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
store.reset_op_counts();
let pulled = b.floor_tick().await.unwrap();
let ops = store.op_counts();
eprintln!("a pull-only boundary: {ops:?}");
assert!(!ops.keys().any(|k| k.starts_with("epoch_")), "a pull-only boundary touched the cell: {ops:?}");
let writes: Vec<_> =
ops.keys().filter(|k| !matches!(**k, "get_whole" | "get_range" | "head" | "list")).collect();
assert!(writes.is_empty(), "a pull-only boundary wrote to the bucket: {ops:?}");
assert_eq!(pulled.seq, Some(seq_a), "{pulled:?}");
let queued = b.state.load_foreign_queue().unwrap();
assert!(queued.iter().any(|c| c.path == "x.txt" && c.etag.is_some()), "A's edit was not queued: {queued:?}");
assert!(queued.iter().any(|c| c.path == "gone.txt" && c.etag.is_none()), "A's delete was not queued: {queued:?}");
b.floor_tick().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's v2"));
assert_eq!(read(dir_b.path(), "gone.txt"), None);
let t = b.load_remote_seq();
assert_eq!(t.observed_seq, t.integrated_seq, "{t:?}");
assert_eq!(manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq, seq_a, "B installed a generation");
write(dir_b.path(), "b.txt", "B's own");
store.reset_op_counts();
let busy = b.floor_tick().await.unwrap();
assert_eq!(busy.uploaded, 1);
assert_eq!(store.op_counts().get("epoch_acquire").copied(), Some(1), "a publishing boundary skipped the fence");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_pull_only_boundary_inside_a_peers_commit_section_converges() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
b.cfg.claim_deadline_secs = 0;
a.checkout().await.unwrap();
write(dir_a.path(), "p.txt", "seed");
write(dir_a.path(), "q.txt", "A deletes this");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "p.txt", "A's v2");
backdate_baseline(&a, "p.txt");
std::fs::remove_file(dir_a.path().join("q.txt")).unwrap();
let b_slot = std::sync::Mutex::new(Some(b));
let b_done: Arc<std::sync::Mutex<Option<(Syncer, std::collections::BTreeMap<&'static str, u64>)>>> = Default::default();
let b_done_in = b_done.clone();
let (cfg, probe) = (a.cfg.clone(), inner.clone());
ha.before_delete(&a.cfg.file_key("q.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let m = manifest::load(probe.as_ref(), &cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("q.txt"), "fixture: A's CAS has not uncited q.txt");
let cell = probe.epoch_read(&cfg.epoch_key()).await.unwrap().unwrap();
assert!(!cell.released, "fixture: A is not inside its commit section");
probe.reset_op_counts();
b.floor_tick().await.expect("B's pull inside A's commit section");
*b_done_in.lock().unwrap() = Some((b, probe.op_counts()));
})
});
});
let ra = a.declared_barrier().await.expect("A's barrier");
assert!(ra.deleted.contains(&"q.txt".to_string()), "fixture: {ra:?}");
let (mut b, ops) = b_done.lock().unwrap().take().expect("fixture: A's GC never reached q.txt");
assert!(!ops.keys().any(|k| k.starts_with("epoch_")), "B's pull touched the cell: {ops:?}");
for _ in 0..3 {
b.floor_tick().await.expect("B's tick");
a.floor_tick().await.expect("A's tick");
}
assert_eq!(read(dir_b.path(), "p.txt").as_deref(), Some("A's v2"));
assert_eq!(read(dir_b.path(), "q.txt"), None, "A's delete never reached B");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("q.txt"), "the delete was undone");
assert_every_citation_resolves(&inner, &a.cfg, "after the pull inside A's commit").await;
let t = b.load_remote_seq();
assert_eq!(t.observed_seq, t.integrated_seq, "{t:?}");
}
#[tokio::test]
async fn a_claim_that_reaches_the_deadline_fails_the_barrier_and_the_retry_adopts_the_uploads() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.claim_deadline_secs = 0;
a.checkout().await.unwrap();
b.checkout().await.unwrap();
assert!(claim_until_held(&mut a, 1).await); write(dir_b.path(), "b.txt", "B's bytes");
let err = b.run_barrier().await.expect_err("the wait must give up at the deadline");
assert!(err.to_string().contains("publish fence"), "{err}");
assert!(b.lease.is_none());
let landed = store.head(&b.cfg.file_key("b.txt")).await.expect("the upload did not land before the claim");
assert!(manifest::load(store.as_ref(), &b.cfg).await.unwrap().is_none(), "a manifest was installed without the fence");
lease::release(&mut a).await.unwrap();
let r = b.run_barrier().await.expect("the retry");
assert_eq!(r.uploaded, vec!["b.txt"]);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["b.txt"].etag, landed.etag, "the retry re-sent the bytes instead of citing its own earlier PUT");
}
#[tokio::test]
async fn a_restarted_container_releases_the_fence_its_predecessor_left_held() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 1).await);
let held = a.lease.clone().unwrap();
drop(a); let mut a2 = syncer(&store, dir_a.path()).await; assert_eq!(lease::incarnation(&a2).unwrap().holder_id, held.holder_id);
lease::release_stale_own(&mut a2).await.unwrap();
let cell = store.epoch_read(&a2.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released, "the stale hold was not released");
assert_eq!(cell.epoch, held.epoch);
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 1).await, "the successor still had to wait");
}
fn barrier_on_thread(
mut sc: Syncer,
) -> std::thread::JoinHandle<(Syncer, crate::LeanResult<super::barrier::BarrierReport>)> {
std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.unwrap();
let r = rt.block_on(sc.run_barrier());
(sc, r)
})
}
async fn assert_every_citation_resolves(store: &Arc<MemoryStore>, cfg: &LeanConfig, when: &str) {
let m = manifest::load(store.as_ref(), cfg).await.unwrap().unwrap().manifest;
for (path, e) in &m.entries {
if let Err(err) = store.get_whole(&e.key, Some(&e.etag)).await {
panic!("{when}: seq {} cites {path} at {} but the object is gone: {err}", m.seq, e.etag);
}
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_peer_upload_between_the_gc_head_and_its_delete_is_not_deleted() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"));
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
let first = a.run_barrier().await.unwrap();
assert!(first.deleted.is_empty(), "fixture: the first absence was not withheld");
write(dir_b.path(), "x.txt", "B's edit, longer");
backdate_baseline(&b, "x.txt");
let key = a.cfg.file_key("x.txt");
let seed_etag = inner.head(&key).await.unwrap().etag;
let (parked_tx, parked_rx) = std::sync::mpsc::channel::<()>();
let (go_tx, go_rx) = std::sync::mpsc::channel::<()>();
let go_rx = std::sync::Mutex::new(go_rx);
let parked_tx = std::sync::Mutex::new(parked_tx);
ha.before_delete(&key, move || {
parked_tx.lock().unwrap().send(()).unwrap();
tokio::task::block_in_place(|| {
go_rx.lock().unwrap().recv_timeout(std::time::Duration::from_secs(20)).expect("never released")
});
});
let a_task = barrier_on_thread(a);
tokio::task::spawn_blocking(move || {
parked_rx.recv_timeout(std::time::Duration::from_secs(20)).expect("A never reached its GC delete")
})
.await
.unwrap();
let b_task = barrier_on_thread(b);
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20);
loop {
if inner.head(&key).await.map(|m| m.etag != seed_etag).unwrap_or(false) {
break;
}
assert!(std::time::Instant::now() < deadline, "B's upload never landed");
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
}
go_tx.send(()).unwrap();
let ((a, ra), (b, rb)) = tokio::task::spawn_blocking(move || {
(a_task.join().expect("A's thread"), b_task.join().expect("B's thread"))
})
.await
.unwrap();
ra.expect("A's barrier");
rb.expect("B's barrier");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited = m.entries.get("x.txt").expect("B's edit is not cited: a modify must beat the delete");
let (_, body) = inner
.get_whole(&cited.key, Some(&cited.etag))
.await
.expect("the manifest cites B's edit, and A's GC deleted the object under it");
assert_eq!(&body[..], b"B's edit, longer");
assert_every_citation_resolves(&inner, &b.cfg, "after both commits").await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_peer_upload_of_identical_bytes_before_the_gc_delete_is_not_deleted() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let hb = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = hooked_syncer(&hb, dir_b.path());
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"));
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
let first = a.run_barrier().await.unwrap();
assert!(first.deleted.is_empty(), "fixture: the first absence was not withheld");
std::fs::remove_file(dir_b.path().join("x.txt")).unwrap();
write(dir_b.path(), "x.txt", "seed");
backdate_baseline(&b, "x.txt");
let key = a.cfg.file_key("x.txt");
let (parked_tx, parked_rx) = std::sync::mpsc::channel::<()>();
let (go_tx, go_rx) = std::sync::mpsc::channel::<()>();
let (landed_tx, landed_rx) = std::sync::mpsc::channel::<()>();
let go_rx = std::sync::Mutex::new(go_rx);
let parked_tx = std::sync::Mutex::new(parked_tx);
let landed_tx = std::sync::Mutex::new(landed_tx);
ha.before_delete(&key, move || {
parked_tx.lock().unwrap().send(()).unwrap();
tokio::task::block_in_place(|| {
go_rx.lock().unwrap().recv_timeout(std::time::Duration::from_secs(20)).expect("never released")
});
});
hb.after_put(&key, move || landed_tx.lock().unwrap().send(()).unwrap());
let a_task = barrier_on_thread(a);
tokio::task::spawn_blocking(move || {
parked_rx.recv_timeout(std::time::Duration::from_secs(20)).expect("A never reached its GC delete")
})
.await
.unwrap();
let b_task = barrier_on_thread(b);
tokio::task::spawn_blocking(move || {
landed_rx
.recv_timeout(std::time::Duration::from_secs(20))
.expect("fixture: B never uploaded its rewrite of x.txt")
})
.await
.unwrap();
assert_eq!(
inner.get_whole(&key, None).await.unwrap().1.as_ref(),
b"seed",
"fixture: B's upload is not the same bytes"
);
go_tx.send(()).unwrap();
let ((a, ra), (b, rb)) = tokio::task::spawn_blocking(move || {
(a_task.join().expect("A's thread"), b_task.join().expect("B's thread"))
})
.await
.unwrap();
ra.expect("A's barrier");
let rb = rb.expect("B's barrier");
let _ = a;
assert_every_citation_resolves(&inner, &b.cfg, "after both commits").await;
assert!(rb.parked.contains(&"x.txt".to_string()), "fixture: B's upload was not withheld: {rb:?}");
let mut b = b;
b.run_barrier().await.expect("B's retry");
assert_every_citation_resolves(&inner, &b.cfg, "after B's retry").await;
let m = manifest::load(inner.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
let cited = m.entries.get("x.txt").expect("B's rewrite never reached the manifest");
let (_, body) = inner.get_whole(&cited.key, Some(&cited.etag)).await.expect("B's retry cites nothing");
assert_eq!(&body[..], b"seed");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_peer_put_over_an_identical_bytes_upload_is_not_cited_as_the_old_version() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let hb = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = hooked_syncer(&hb, dir_b.path());
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"));
std::fs::remove_file(dir_b.path().join("x.txt")).unwrap();
write(dir_b.path(), "x.txt", "seed");
backdate_baseline(&b, "x.txt");
write(dir_a.path(), "x.txt", "A's edit");
backdate_baseline(&a, "x.txt");
let key = a.cfg.file_key("x.txt");
let seed_etag = inner.head(&key).await.unwrap().etag;
let (b_landed_tx, b_landed_rx) = std::sync::mpsc::channel::<()>();
let (b_go_tx, b_go_rx) = std::sync::mpsc::channel::<()>();
let (a_landed_tx, a_landed_rx) = std::sync::mpsc::channel::<()>();
let (a_go_tx, a_go_rx) = std::sync::mpsc::channel::<()>();
let (b_landed_tx, b_go_rx) = (std::sync::Mutex::new(b_landed_tx), std::sync::Mutex::new(b_go_rx));
let (a_landed_tx, a_go_rx) = (std::sync::Mutex::new(a_landed_tx), std::sync::Mutex::new(a_go_rx));
hb.after_put(&key, move || {
b_landed_tx.lock().unwrap().send(()).unwrap();
tokio::task::block_in_place(|| {
b_go_rx.lock().unwrap().recv_timeout(std::time::Duration::from_secs(20)).expect("B never released")
});
});
ha.after_put(&key, move || {
a_landed_tx.lock().unwrap().send(()).unwrap();
tokio::task::block_in_place(|| {
a_go_rx.lock().unwrap().recv_timeout(std::time::Duration::from_secs(20)).expect("A never released")
});
});
let b_task = barrier_on_thread(b);
tokio::task::spawn_blocking(move || {
b_landed_rx
.recv_timeout(std::time::Duration::from_secs(20))
.expect("fixture: B never uploaded its rewrite of x.txt")
})
.await
.unwrap();
assert_eq!(inner.head(&key).await.unwrap().etag, seed_etag, "fixture: B's upload is not the same bytes");
let a_task = barrier_on_thread(a);
tokio::task::spawn_blocking(move || {
a_landed_rx
.recv_timeout(std::time::Duration::from_secs(20))
.expect("fixture: A's If-Match on the seed etag did not land over B's upload")
})
.await
.unwrap();
assert_eq!(inner.get_whole(&key, None).await.unwrap().1.as_ref(), b"A's edit");
a_go_tx.send(()).unwrap();
let (a, ra) = tokio::task::spawn_blocking(move || a_task.join().expect("A's thread")).await.unwrap();
ra.expect("A's barrier");
assert_every_citation_resolves(&inner, &a.cfg, "after A's commit").await;
b_go_tx.send(()).unwrap();
let (b, rb) = tokio::task::spawn_blocking(move || b_task.join().expect("B's thread")).await.unwrap();
let rb = rb.expect("B's barrier");
assert_every_citation_resolves(&inner, &b.cfg, "after B's commit").await;
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited = m.entries.get("x.txt").expect("A's edit is not cited");
let (_, body) = inner.get_whole(&cited.key, Some(&cited.etag)).await.expect("the manifest cites nothing");
assert_eq!(&body[..], b"A's edit");
assert!(rb.parked.contains(&"x.txt".to_string()), "fixture: B's upload was not withheld: {rb:?}");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn an_adopted_upload_deleted_by_the_peer_before_the_claim_is_not_cited() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "x.txt", "same");
backdate_baseline(&b, "x.txt");
b.run_barrier().await.unwrap();
write(dir_a.path(), "x.txt", "same");
backdate_baseline(&a, "x.txt");
std::fs::remove_file(dir_b.path().join("x.txt")).unwrap();
let first = b.run_barrier().await.unwrap();
assert!(first.deleted.is_empty(), "fixture: the first absence was not withheld");
let key = a.cfg.file_key("x.txt");
let b_slot = std::sync::Mutex::new(Some(b));
let b_done: Arc<std::sync::Mutex<Option<Syncer>>> = Arc::new(std::sync::Mutex::new(None));
let b_done_in = b_done.clone();
ha.after_head(&key, move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let r = b.run_barrier().await.expect("B's delete barrier");
assert!(r.deleted.contains(&"x.txt".to_string()), "fixture: B did not delete x.txt: {r:?}");
})
});
*b_done_in.lock().unwrap() = Some(b);
});
let ra = a.run_barrier().await.expect("A's barrier");
let b = b_done.lock().unwrap().take().expect("fixture: the adopt's HEAD never ran");
assert!(inner.head(&key).await.is_err(), "fixture: B's GC did not delete the object");
assert_every_citation_resolves(&inner, &a.cfg, "after A's commit").await;
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "A cited an object that is gone: {ra:?}");
let _ = b;
a.run_barrier().await.expect("A's retry");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited = m.entries.get("x.txt").expect("A's edit never reached the manifest");
let (_, body) = inner.get_whole(&cited.key, Some(&cited.etag)).await.expect("A's retry cites nothing");
assert_eq!(&body[..], b"same");
}
#[tokio::test]
async fn a_peers_change_reaches_the_writer_whose_merge_queued_it_even_if_the_peer_consumes_first() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "A's v2");
backdate_baseline(&a, "x.txt");
a.run_barrier().await.unwrap();
write(dir_b.path(), "z.txt", "z");
b.run_barrier().await.unwrap();
let queued = b.state.load_foreign_queue().unwrap();
assert!(
queued.iter().any(|c| c.path == "x.txt" && c.etag.is_some()),
"fixture: B's merge queued nothing for x.txt: {queued:?}"
);
let ib = super::inbox::load(store.as_ref(), &b.cfg).await.unwrap();
assert!(
!ib.doc.entries.iter().any(|e| e.path == "x.txt"),
"B's merge put its own queue in the shared inbox: {:?}",
ib.doc.entries
);
a.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's v2"), "A's change never reached B");
assert_every_citation_resolves(&store, &a.cfg, "after convergence").await;
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, body) = store.get_whole(&m.entries["x.txt"].key, Some(&m.entries["x.txt"].etag)).await.unwrap();
assert_eq!(&body[..], b"A's v2", "the manifest reverted A's change");
}
#[tokio::test]
async fn a_restart_between_the_cas_and_step_7_still_delivers_a_peers_change() {
restart_between_the_cas_and_step_7(false).await;
}
#[tokio::test]
async fn a_restart_between_the_cas_and_step_7_still_delivers_a_peers_delete() {
restart_between_the_cas_and_step_7(true).await;
}
async fn restart_between_the_cas_and_step_7(peer_deletes: bool) {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&inner, dir_a.path()).await;
let mut b = hooked_syncer(&hooked, dir_b.path());
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"));
write(dir_b.path(), "tmp.txt", "B's scratch");
b.run_barrier().await.unwrap();
if peer_deletes {
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
a.declared_barrier().await.unwrap();
} else {
write(dir_a.path(), "x.txt", "A's v2");
backdate_baseline(&a, "x.txt");
a.run_barrier().await.unwrap();
}
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.contains_key("x.txt"), !peer_deletes, "fixture: A's change never published");
let state_dir = b.cfg.state_dir();
let snap: Arc<std::sync::Mutex<Vec<(std::path::PathBuf, Option<Vec<u8>>)>>> = Default::default();
let snap_in = snap.clone();
hooked.before_delete(&b.cfg.file_key("tmp.txt"), move || {
*snap_in.lock().unwrap() = ["intent.json", "foreign-queue.json", "baseline.json"]
.iter()
.map(|f| {
let p = state_dir.join(f);
let bytes = std::fs::read(&p).ok();
(p, bytes)
})
.collect();
});
write(dir_b.path(), "z.txt", "z");
std::fs::remove_file(dir_b.path().join("tmp.txt")).unwrap();
b.declared_barrier().await.unwrap();
let taken = std::mem::take(&mut *snap.lock().unwrap());
assert!(!taken.is_empty(), "fixture: B's barrier never reached its GC delete");
assert!(
b.state.load_foreign_queue().unwrap().iter().any(|c| c.path == "x.txt" && c.etag.is_some() != peer_deletes),
"fixture: B's merge queued nothing for x.txt"
);
for (p, bytes) in taken {
match bytes {
Some(b) => std::fs::write(&p, b).unwrap(),
None => {
let _ = std::fs::remove_file(&p);
}
}
}
for _ in 0..3 {
b.run_barrier().await.unwrap();
}
if peer_deletes {
assert_eq!(read(dir_b.path(), "x.txt"), None, "a restart between B's CAS and its step 7 kept A's deleted file in B's tree");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "B re-published the file A deleted");
} else {
assert_eq!(
read(dir_b.path(), "x.txt").as_deref(),
Some("A's v2"),
"a restart between B's CAS and its step 7 lost A's change from B's tree for good"
);
}
assert!(b.state.load_intent().unwrap().installed_foreign.is_empty(), "the journal kept changes the queue took");
}
#[tokio::test]
async fn a_peers_delete_reaches_the_other_writers_tree() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"));
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "fixture: A's delete never published");
for _ in 0..3 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir_b.path(), "x.txt"), None, "A's delete never reached B's tree");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "B resurrected the path A deleted");
}
#[tokio::test]
async fn two_idle_writers_do_not_trade_empty_generations() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "from A");
a.run_barrier().await.unwrap();
for _ in 0..2 {
b.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
}
let settled = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
for _ in 0..4 {
b.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
}
let later = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert_eq!(later, settled, "idle writers kept installing generations: seq {settled} -> {later}");
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("from A"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_sync_does_not_advance_its_base_past_a_change_the_inbox_hid() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
a.run_barrier().await.unwrap();
hitl_write(&inner, &a.cfg, "x.txt", "hitl", "user").await.unwrap();
let key = a.cfg.file_key("x.txt");
let b_slot = std::sync::Mutex::new(Some(b));
let b_done: Arc<std::sync::Mutex<Option<Syncer>>> = Arc::new(std::sync::Mutex::new(None));
let b_done_in = b_done.clone();
let cfg = a.cfg.clone();
let probe = inner.clone();
ha.before_delete(&key, move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let m = manifest::load(probe.as_ref(), &cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "fixture: A's CAS has not uncited x.txt");
let ib = super::inbox::load(probe.as_ref(), &cfg).await.unwrap();
assert!(ib.doc.entries.iter().any(|e| e.path == "x.txt"), "fixture: the entry is gone");
b.sync_scoped(None).await.expect("B's sync");
})
});
*b_done_in.lock().unwrap() = Some(b);
});
let ra = a.run_barrier().await.expect("A's delete barrier");
assert!(ra.deleted.contains(&"x.txt".to_string()), "fixture: A did not publish the delete: {ra:?}");
let mut b = b_done.lock().unwrap().take().expect("fixture: A's GC never reached x.txt");
for _ in 0..3 {
b.run_barrier().await.expect("B's barrier");
}
let m = manifest::load(inner.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "the delete was undone");
assert_eq!(read(dir_b.path(), "x.txt"), None, "B's sync advanced its base past A's delete and kept the file");
assert_eq!(read(dir_b.path(), "keep.txt").as_deref(), Some("keep"));
}
#[tokio::test]
async fn a_gateway_write_reaches_every_writer_not_only_the_first_consumer() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "doc.md", "v1");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
hitl_write(&store, &a.cfg, "doc.md", "edited in the UI", "user@ui").await.unwrap();
hitl_write(&store, &a.cfg, "new-from-ui.md", "created in the UI", "user@ui").await.unwrap();
a.run_barrier().await.unwrap();
let ib = super::inbox::load(store.as_ref(), &a.cfg).await.unwrap();
assert!(ib.doc.entries.is_empty(), "fixture: A did not drop the consumed entries: {:?}", ib.doc.entries);
assert_eq!(read(dir_a.path(), "doc.md").as_deref(), Some("edited in the UI"));
b.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
assert_eq!(read(dir_b.path(), "doc.md").as_deref(), Some("edited in the UI"), "the UI edit never reached B");
assert_eq!(read(dir_b.path(), "new-from-ui.md").as_deref(), Some("created in the UI"), "the UI create never reached B");
assert_every_citation_resolves(&store, &a.cfg, "after both").await;
}
#[tokio::test]
async fn a_gateway_write_over_a_path_another_writer_edited_is_preserved_on_that_writer() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "doc.md", "v1");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "doc.md", "B's agent edit");
backdate_baseline(&b, "doc.md");
hitl_write(&store, &a.cfg, "doc.md", "edited in the UI", "user@ui").await.unwrap();
a.run_barrier().await.unwrap(); b.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, body) = store.get_whole(&m.entries["doc.md"].key, Some(&m.entries["doc.md"].etag)).await.unwrap();
assert_eq!(&body[..], b"B's agent edit", "B's later boundary is current");
let preserved: Vec<_> = b
.state
.load_conflicts()
.unwrap()
.into_iter()
.filter(|c| c.path == "doc.md" && c.preserved_key.is_some())
.collect();
let mut found_ui = false;
for c in &preserved {
let (_, kept) = store.get_whole(c.preserved_key.as_ref().unwrap(), None).await.unwrap();
found_ui |= &kept[..] == b"edited in the UI";
}
assert!(found_ui, "the UI's bytes are not preserved under any record on B: {preserved:?}");
assert_every_citation_resolves(&store, &a.cfg, "after both").await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_ui_write_over_an_uncited_upload_is_never_silently_lost() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
a.cfg.upload_fanout = 1; let b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "p1.txt", "seed-1");
write(dir_a.path(), "p2.txt", "seed-2");
a.run_barrier().await.unwrap();
let mut b = b;
b.checkout().await.unwrap();
write(dir_a.path(), "p1.txt", "A's edit of p1");
backdate_baseline(&a, "p1.txt");
write(dir_a.path(), "p2.txt", "A's edit of p2");
backdate_baseline(&a, "p2.txt");
let (store_ui, cfg_ui) = (inner.clone(), a.cfg.clone());
let b_slot = std::sync::Mutex::new(Some(b));
let b_done: Arc<std::sync::Mutex<Option<Syncer>>> = Arc::new(std::sync::Mutex::new(None));
let b_done_in = b_done.clone();
ha.before_put(&a.cfg.file_key("p2.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let cur = store_ui.head(&cfg_ui.file_key("p1.txt")).await.unwrap();
let (_, body) = store_ui.get_whole(&cfg_ui.file_key("p1.txt"), None).await.unwrap();
assert_eq!(&body[..], b"A's edit of p1", "fixture: A's upload of p1 has not landed: {cur:?}");
let _ = hitl_write(&store_ui, &cfg_ui, "p1.txt", "edited in the UI", "user@ui").await;
b.run_barrier().await.expect("B's barrier");
})
});
*b_done_in.lock().unwrap() = Some(b);
});
let ra = a.run_barrier().await;
let _b = b_done.lock().unwrap().take().expect("fixture: the hook never ran");
assert_every_citation_resolves(&inner, &a.cfg, "after A's commit").await;
let ui_accepted = {
let ib = super::inbox::load(inner.as_ref(), &a.cfg).await.unwrap();
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited = inner.get_whole(&m.entries["p1.txt"].key, Some(&m.entries["p1.txt"].etag)).await
.map(|(_, b)| &b[..] == b"edited in the UI").unwrap_or(false);
let tracked = ib.doc.entries.iter().any(|e| e.path == "p1.txt");
let mut preserved = false;
for o in inner.list(&format!("{}/.flint/lean/conflicts/", PREFIX)).await.unwrap() {
if let Ok((_, body)) = inner.get_whole(&o.key, None).await {
preserved |= &body[..] == b"edited in the UI";
}
}
(cited, tracked, preserved)
};
let ui_ever_landed = {
read(dir_b.path(), "p1.txt").as_deref() == Some("edited in the UI")
};
if ui_ever_landed {
assert!(
ui_accepted.0 || ui_accepted.1 || ui_accepted.2,
"the UI's acked write was silently lost: cited={} tracked={} preserved={} (A: {:?})",
ui_accepted.0, ui_accepted.1, ui_accepted.2, ra.as_ref().map(|r| r.uploaded.clone())
);
}
}
#[tokio::test]
async fn the_event_trace_reconstructs_a_two_writer_interleaving() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let buf = Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.cfg.event_trace = Some(crate::trace::Sink::Memory(buf.clone()));
b.cfg.event_trace = Some(crate::trace::Sink::Memory(buf.clone()));
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
write(dir_b.path(), "keep.txt", "B's edit");
backdate_baseline(&b, "keep.txt");
a.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
for _ in 0..3 {
a.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir_b.path(), "x.txt"), None, "fixture: A's delete never reached B");
assert_eq!(read(dir_a.path(), "keep.txt").as_deref(), Some("B's edit"), "fixture: B's edit never reached A");
let lines = buf.lock().unwrap().clone();
if let Ok(out) = std::env::var("FLINT_SYNC_TRACE_FIXTURE") {
std::fs::write(&out, lines.join("\n") + "\n").unwrap();
}
let evs: Vec<serde_json::Map<String, serde_json::Value>> = lines
.iter()
.map(|l| {
assert!(l.starts_with("{\"ts_ms\":"), "not the pinned prefix: {l}");
let v: serde_json::Value = serde_json::from_str(l).unwrap_or_else(|e| panic!("{e}: {l}"));
let m = v.as_object().unwrap().clone();
let head: Vec<&str> = m.keys().take(4).map(|k| k.as_str()).collect();
for k in ["ts_ms", "mono_ms", "holder", "ev"] {
assert!(m.contains_key(k), "{k} missing ({head:?}): {l}");
}
let pos: Vec<usize> = ["\"ts_ms\":", "\"mono_ms\":", "\"holder\":", "\"ev\":"]
.iter()
.map(|k| l.find(k).unwrap())
.collect();
assert!(pos.windows(2).all(|w| w[0] < w[1]), "prefix out of order: {l}");
m
})
.collect();
let ev = |e: &serde_json::Map<String, serde_json::Value>| e["ev"].as_str().unwrap().to_string();
let s = |e: &serde_json::Map<String, serde_json::Value>, k: &str| e.get(k).and_then(|v| v.as_str()).map(str::to_string);
let holders: std::collections::BTreeSet<String> =
evs.iter().filter(|e| ["claim", "cas", "gc", "release"].contains(&ev(e).as_str())).map(|e| {
s(e, "holder").unwrap_or_else(|| panic!("a commit-section event without a holder: {e:?}"))
}).collect();
assert_eq!(holders.len(), 2, "expected both writers in the trace: {holders:?}");
for e in evs.iter().filter(|e| ev(e) == "claim") {
let verdict = s(e, "verdict").unwrap();
assert!(["claimed", "waiting", "deadline"].contains(&verdict.as_str()), "claim verdict {verdict}: {e:?}");
if verdict == "claimed" {
let how = s(e, "how").unwrap_or_else(|| panic!("a claimed claim without how: {e:?}"));
assert!(
["fresh", "adopted-own", "deposed", "orphaned-own", "skipped-handoff", "released"].contains(&how.as_str()),
"claim how {how}: {e:?}"
);
assert!(e["epoch"].as_u64().is_some(), "claimed without an epoch: {e:?}");
}
}
for e in evs.iter().filter(|e| ev(e) == "cas") {
assert!(["ok", "lost"].contains(&s(e, "result").unwrap().as_str()), "cas result: {e:?}");
}
for e in evs.iter().filter(|e| ev(e) == "barrier_end") {
let req = e["requests"].as_object().unwrap_or_else(|| panic!("barrier_end without request counts: {e:?}"));
let keys: Vec<&str> = req.keys().map(|k| k.as_str()).collect();
let mut want = vec!["copy", "delete", "get", "head", "list", "multipart", "put"];
let mut got = keys.clone();
want.sort();
got.sort();
assert_eq!(got, want, "request counter keys: {e:?}");
}
let cas_ok_by: std::collections::BTreeSet<String> =
evs.iter().filter(|e| ev(e) == "cas" && s(e, "result").as_deref() == Some("ok")).filter_map(|e| s(e, "holder")).collect();
assert_eq!(cas_ok_by.len(), 2, "both writers must install at least once: {cas_ok_by:?}");
for h in &holders {
let mine: Vec<String> = evs
.iter()
.filter(|e| s(e, "holder").as_deref() == Some(h.as_str()))
.map(|e| match ev(e).as_str() {
"claim" => format!("claim:{}", s(e, "verdict").unwrap()),
"cas" => format!("cas:{}", s(e, "result").unwrap()),
other => other.to_string(),
})
.filter(|n| ["scan", "claim:claimed", "cas:ok", "release", "barrier_end"].contains(&n.as_str()))
.collect();
let joined = mine.join(" ");
assert!(
joined.contains("scan claim:claimed cas:ok release barrier_end"),
"{h}: no install in protocol order: {joined}"
);
}
let gc_x = evs
.iter()
.position(|e| ev(e) == "gc" && s(e, "path").as_deref() == Some("x.txt") && s(e, "result").as_deref() == Some("deleted"))
.unwrap_or_else(|| panic!("no gc deleted for x.txt in:\n{}", lines.join("\n")));
let tomb_x = evs
.iter()
.position(|e| ev(e) == "tombstone" && s(e, "path").as_deref() == Some("x.txt") && s(e, "action").as_deref() == Some("removed"))
.unwrap_or_else(|| panic!("no tombstone removed for x.txt in:\n{}", lines.join("\n")));
assert_ne!(s(&evs[gc_x], "holder"), s(&evs[tomb_x], "holder"), "the GC and the tombstone must be different writers");
assert!(gc_x < tomb_x, "the tombstone was emitted before the GC");
assert!(
evs[gc_x]["ts_ms"].as_u64().unwrap() <= evs[tomb_x]["ts_ms"].as_u64().unwrap(),
"wall clock puts the tombstone before the GC"
);
assert!(
evs.iter().any(|e| ev(e) == "consume" && s(e, "path").as_deref() == Some("keep.txt") && s(e, "from").as_deref() == Some("queue")),
"B's edit did not reach A through the local queue:\n{}",
lines.join("\n")
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "finding 10, open: a killed writer's uncited upload is never reconciled"]
async fn a_writer_killed_after_its_upload_does_not_leave_the_trees_diverged() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b, dir_c) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
a.cfg.upload_fanout = 1; let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "p1.txt", "seed-1");
write(dir_a.path(), "p2.txt", "seed-2");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "p1.txt", "A's unpublished edit");
backdate_baseline(&a, "p1.txt");
write(dir_a.path(), "p2.txt", "A's other edit");
backdate_baseline(&a, "p2.txt");
let key1 = a.cfg.file_key("p1.txt");
ha.before_put(&a.cfg.file_key("p2.txt"), || panic!("A is killed between its uploads"));
assert!(barrier_on_thread(a).join().is_err(), "fixture: A was not killed");
let (_, body) = inner.get_whole(&key1, None).await.unwrap();
assert_eq!(&body[..], b"A's unpublished edit", "fixture: A's upload of p1 never landed");
write(dir_b.path(), "b.txt", "B works elsewhere");
for _ in 0..4 {
b.run_barrier().await.unwrap();
}
let mut c = syncer(&inner, dir_c.path()).await;
c.checkout().await.unwrap();
assert_eq!(
read(dir_b.path(), "p1.txt"),
read(dir_c.path(), "p1.txt"),
"a live writer's tree and a fresh checkout disagree on a path whose key holds a killed writer's uncited upload"
);
}
const READ_OPS: &[&str] = &[
"get_whole", "get_range", "get_version", "head", "head_version", "list", "list_versions",
"list_uploads", "lifecycle_rules", "epoch_read", "presign_get",
];
fn writes_since_reset(store: &MemoryStore) -> Vec<(&'static str, u64)> {
store.op_counts().into_iter().filter(|(op, _)| !READ_OPS.contains(op)).collect()
}
async fn reader(store: &Arc<MemoryStore>, root: &std::path::Path) -> Syncer {
let mut sc = syncer(store, root).await;
sc.cfg.access = super::Access::Read;
sc
}
#[tokio::test]
async fn a_reader_follows_writers_and_the_inbox_without_one_store_write() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_r) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "A v1");
write(dir_a.path(), "gone.txt", "A deletes this");
a.floor_tick().await.unwrap();
store.reset_op_counts();
let mut r = reader(&store, dir_r.path()).await;
r.checkout().await.unwrap();
r.floor_tick().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a reader's checkout and first tick wrote to the bucket");
assert_eq!(read(dir_r.path(), "x.txt").as_deref(), Some("A v1"));
write(dir_a.path(), "x.txt", "A v2");
backdate_baseline(&a, "x.txt");
std::fs::remove_file(dir_a.path().join("gone.txt")).unwrap();
write(dir_a.path(), "new.txt", "A adds this");
a.declared_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "ui.txt", "from the UI", "reviewer").await.unwrap();
let seq_a = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
store.reset_op_counts();
r.floor_tick().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a reader's pull wrote to the bucket");
assert_eq!(read(dir_r.path(), "x.txt").as_deref(), Some("A v2"));
assert_eq!(read(dir_r.path(), "gone.txt"), None);
assert_eq!(read(dir_r.path(), "new.txt").as_deref(), Some("A adds this"));
assert_eq!(read(dir_r.path(), "ui.txt").as_deref(), Some("from the UI"), "the inbox write did not reach the reader");
let t = r.load_remote_seq();
assert_eq!((t.observed_seq, t.integrated_seq), (seq_a, seq_a), "{t:?}");
write(dir_r.path(), "x.txt", "the reader's local edit");
backdate_baseline(&r, "x.txt");
write(dir_r.path(), "local-only.txt", "never published");
write(dir_a.path(), "x.txt", "A v3");
backdate_baseline(&a, "x.txt");
a.declared_barrier().await.unwrap();
store.reset_op_counts();
r.floor_tick().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a reader with local changes wrote to the bucket");
assert_eq!(read(dir_r.path(), "x.txt").as_deref(), Some("the reader's local edit"));
assert!(store.head(&r.cfg.file_key("local-only.txt")).await.is_err(), "a reader published a local file");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("local-only.txt"));
let conflicts = r.state.load_conflicts().unwrap();
assert!(conflicts.iter().any(|c| c.path == "x.txt" && c.kind == "sync-dirty"), "{conflicts:?}");
}
#[tokio::test]
async fn an_idle_reader_tick_costs_the_inbox_and_the_pointer() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_r) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "A v1");
a.floor_tick().await.unwrap();
hitl_write(&store, &a.cfg, "ui.txt", "from the UI", "reviewer").await.unwrap();
let mut r = reader(&store, dir_r.path()).await;
r.checkout().await.unwrap();
r.floor_tick().await.unwrap();
store.reset_op_counts();
r.floor_tick().await.unwrap();
let ops = store.op_counts();
assert_eq!(ops.into_iter().collect::<Vec<_>>(), vec![("get_whole", 2)], "an idle reader tick is two GETs");
let t = r.load_remote_seq();
assert!(t.updated_unix > 0, "the ticker's heartbeat did not move on an idle tick: {t:?}");
hitl_write(&store, &a.cfg, "ui2.txt", "a second UI write", "reviewer").await.unwrap();
r.floor_tick().await.unwrap();
assert_eq!(read(dir_r.path(), "ui2.txt").as_deref(), Some("a second UI write"), "an inbox-only change never reached the reader");
}
#[tokio::test]
async fn a_publish_touch_on_a_reader_is_answered_refused_read_only() {
let store = Arc::new(MemoryStore::new());
let (dir_r, dir_w) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut r = reader(&store, dir_r.path()).await;
r.checkout().await.unwrap();
let posture = r.sentinel_preflight().unwrap();
r.write_capabilities(&posture).unwrap();
write(dir_r.path(), "f.txt", "a local file");
store.reset_op_counts();
touch_sentinel(dir_r.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
let acks = r.sentinel_tick().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a refused publish wrote to the bucket");
assert_eq!(acks.len(), 1, "{acks:?}");
assert_eq!(acks[0].status, "refused-read-only");
assert_eq!(acks[0].nonces, vec!["n-1".to_string()]);
assert!(acks[0].reason.as_deref().unwrap_or("").contains("read-only"), "{:?}", acks[0].reason);
assert!(r.load_pending(Verb::Publish).unwrap().is_none(), "the refused touch was left pending");
let on_disk = std::fs::read_to_string(dir_r.path().join(super::CONTROL_DIR).join(control::PUBLISH_ACK)).unwrap();
assert!(on_disk.contains("refused-read-only"), "{on_disk}");
assert!(store.head(&r.cfg.file_key("f.txt")).await.is_err());
r.cfg.sentinel_min_interval_secs = 0;
touch_sentinel(dir_r.path(), control::SYNC, r#"{"nonce":"s-1"}"#);
let acks = r.sentinel_tick().await.unwrap();
assert_eq!(acks.iter().map(|a| a.status.as_str()).collect::<Vec<_>>(), vec!["ok"]);
let mut w = syncer(&store, dir_w.path()).await;
w.checkout().await.unwrap();
let posture = w.sentinel_preflight().unwrap();
w.write_capabilities(&posture).unwrap();
write(dir_w.path(), "g.txt", "a writer's file");
touch_sentinel(dir_w.path(), control::PUBLISH, r#"{"nonce":"w-1"}"#);
let acks = w.sentinel_tick().await.unwrap();
assert_eq!(acks.iter().map(|a| a.status.as_str()).collect::<Vec<_>>(), vec!["ok"]);
}
#[tokio::test]
async fn a_barrier_on_a_reader_is_refused_before_any_write() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut r = reader(&store, dir.path()).await;
r.checkout().await.unwrap();
write(dir.path(), "f.txt", "would be an upload");
store.reset_op_counts();
for (name, out) in [
("run_barrier", r.run_barrier().await.map(|_| ())),
("cadence_barrier", r.cadence_barrier().await.map(|_| ())),
("declared_barrier", r.declared_barrier().await.map(|_| ())),
] {
assert!(matches!(out, Err(LeanError::Refused(_))), "{name} on a reader: {out:?}");
}
assert_eq!(store.total_ops(), 0, "a refused barrier sent requests: {:?}", store.op_counts());
assert!(store.head(&r.cfg.file_key("f.txt")).await.is_err());
}
#[tokio::test]
async fn one_shot_verbs_that_publish_or_fence_are_refused_on_a_reader() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut r = reader(&store, dir.path()).await;
r.checkout().await.unwrap();
store.reset_op_counts();
for step in [super::verbs::Step::Barrier, super::verbs::Step::Rescope(None)] {
let out = super::verbs::run_verb(&mut r, step).await;
assert!(matches!(out, Err(LeanError::Refused(_))), "{out:?}");
}
assert_eq!(writes_since_reset(&store), vec![], "a refused verb wrote: {:?}", store.op_counts());
assert!(super::verbs::run_verb(&mut r, super::verbs::Step::Sync).await.is_ok(), "sync is a reader's verb");
assert!(super::verbs::run_verb(&mut r, super::verbs::Step::Checkout).await.is_ok());
assert_eq!(writes_since_reset(&store), vec![]);
}
#[tokio::test]
async fn a_reader_drain_answers_what_it_owes_and_writes_nothing() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut r = reader(&store, dir.path()).await;
r.checkout().await.unwrap();
let posture = r.sentinel_preflight().unwrap();
r.write_capabilities(&posture).unwrap();
write(dir.path(), "f.txt", "local only");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"at-sigterm"}"#);
r.poll_sentinels().unwrap();
assert!(r.load_pending(Verb::Publish).unwrap().is_some());
store.reset_op_counts();
let acks = r.drain().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a reader's drain wrote to the bucket");
assert_eq!(acks.iter().map(|a| a.status.as_str()).collect::<Vec<_>>(), vec!["refused-read-only"]);
assert!(store.head(&r.cfg.file_key("f.txt")).await.is_err());
}
#[tokio::test]
async fn the_marker_says_read_access_and_the_guide_explains_it() {
let store = Arc::new(MemoryStore::new());
let (dir_r, dir_w) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut r = reader(&store, dir_r.path()).await;
r.checkout().await.unwrap();
let posture = r.sentinel_preflight().unwrap();
r.write_capabilities(&posture).unwrap();
let caps = r.read_capabilities().unwrap();
assert_eq!(caps.access, "read");
assert_eq!(caps.verbs, vec!["sync".to_string(), "remote-seq".to_string()], "a reader advertised {:?}", caps.verbs);
let mut w = syncer(&store, dir_w.path()).await;
w.checkout().await.unwrap();
let posture = w.sentinel_preflight().unwrap();
w.write_capabilities(&posture).unwrap();
let caps = w.read_capabilities().unwrap();
assert_eq!(caps.access, "readWrite");
assert!(caps.verbs.contains(&"publish".to_string()));
let old: control::Capabilities = serde_json::from_str(
r#"{"protocol":1,"verbs":[],"state":"live","sentinel_min_interval_secs":5,
"sentinel_hourly_budget":60,"syncer_version":"x","boot":{"holder_id":"h","boot_unix":0}}"#,
)
.unwrap();
assert_eq!(old.access, "readWrite");
for needle in ["\"access\": \"read\"", "refused-read-only", "EROFS"] {
assert!(control::AGENT_GUIDE_TEXT.contains(needle), "the guide never says {needle}");
}
}
#[test]
fn access_parses_exactly_what_the_plugin_writes() {
use super::Access;
assert_eq!(Access::parse("read"), Some(Access::Read));
assert_eq!(Access::parse("readWrite"), Some(Access::ReadWrite));
for bad in ["", "ro", "READ", "read-only", "readwrite", "rw"] {
assert_eq!(Access::parse(bad), None, "{bad:?} must not parse");
}
assert_eq!(Access::default(), Access::ReadWrite);
assert_eq!(Access::parse(Access::Read.as_str()), Some(Access::Read));
}