use std::sync::Arc;
use bytes::Bytes;
use flint_store::memory::MemoryStore;
use flint_store::{crc64_nvme, GenerationStamps, ObjectStore, PutCondition, StoreError};
use super::inbox;
use super::lease::{self, ClaimOutcome};
use super::manifest;
use super::state::SyncerState;
use super::{now_unix, LeanConfig, LeanError, Syncer};
const PREFIX: &str = "tenant/proj1";
pub(super) fn cfg_for(root: &std::path::Path) -> LeanConfig {
let mut c = LeanConfig::new(PREFIX, root);
instant_fence(&mut c);
c.retire_grace_secs = 0;
c
}
fn instant_fence(c: &mut LeanConfig) {
c.claim_poll_secs = 0;
c.claim_quiet_spacing_secs = 0;
c.claim_deadline_secs = 2;
}
fn cfg_single(root: &std::path::Path) -> LeanConfig {
let mut c = LeanConfig::new(PREFIX, root);
c.chunked = false;
instant_fence(&mut c);
c.retire_grace_secs = 0;
c
}
pub(super) async fn syncer(store: &Arc<MemoryStore>, root: &std::path::Path) -> Syncer {
let cfg = cfg_for(root);
let state = SyncerState::open(cfg.state_dir()).unwrap();
Syncer {
store: store.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
cell_written_at: None,
noted_not_regular: Default::default(),
}
}
async fn claim_until_held(sc: &mut Syncer, max_steps: u32) -> bool {
for _ in 0..max_steps {
match lease::claim_step(sc, true).await.unwrap() {
ClaimOutcome::Claimed(_) => return true,
ClaimOutcome::Waiting { .. } => {}
}
}
false
}
pub(super) fn write(root: &std::path::Path, rel: &str, content: &str) {
let p = root.join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, content).unwrap();
}
pub(super) async fn derive_pending(sc: &Syncer) -> bool {
let p = manifest::load_pointer(sc.store.as_ref(), &sc.cfg).await.unwrap().map(|p| p.etag);
let d = sc.state.load_baseline().unwrap().derived_etag;
let norm = |e: Option<String>| e.map(|e| e.trim_matches('"').to_string());
norm(p) != norm(d)
}
pub(super) fn read(root: &std::path::Path, rel: &str) -> Option<String> {
std::fs::read_to_string(root.join(rel)).ok()
}
pub(super) fn backdate_baseline(sc: &Syncer, rel: &str) {
let mut b = sc.state.load_baseline().unwrap();
if let Some(e) = b.entries.get_mut(rel) {
e.mtime_unix -= 10;
}
sc.state.save_baseline(&b).unwrap();
}
pub(super) async fn hitl_write(
store: &Arc<MemoryStore>,
cfg: &LeanConfig,
path: &str,
content: &str,
author: &str,
) -> Result<String, LeanError> {
hitl_write_at(store, cfg, path, content, author, now_unix()).await.map(|(etag, _)| etag)
}
pub(super) async fn hitl_write_at(
store: &Arc<MemoryStore>,
cfg: &LeanConfig,
path: &str,
content: &str,
_author: &str,
added_unix: u64,
) -> Result<(String, String), LeanError> {
let flush = super::ui_flush(added_unix);
let key = cfg.handle_key(path, &flush);
let body = Bytes::from(content.to_string());
let size = body.len() as u64;
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: flush.clone(),
boundary_source: None,
posix: None,
};
let meta = store.put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc).await?;
let entry_key = key.clone();
let etag = meta.etag.clone();
gateway_commit(store, cfg, &flush, move |doc| {
let was = doc.entries.get(path).cloned();
doc.tombstones.remove(path);
doc.entries.insert(
path.to_string(),
manifest::LeanEntry {
key: entry_key.clone(),
etag: etag.clone(),
crc64_b64: flint_store::crc64_to_b64(crc),
size,
mode: was.as_ref().map(|e| e.mode).unwrap_or(0o644),
mtime_unix: added_unix as i64,
generation: was.map(|e| e.generation).unwrap_or(0) + 1,
epoch: 0,
},
);
})
.await?;
Ok((meta.etag, key))
}
pub(super) async fn gateway_commit(
store: &Arc<MemoryStore>,
cfg: &LeanConfig,
flush: &str,
edit: impl Fn(&mut manifest::LeanManifest),
) -> Result<(), LeanError> {
manifest::commit_edit(store.as_ref(), cfg, flush, |_, doc| {
edit(doc);
Ok::<(), ()>(())
})
.await
.map_err(|e| match e {
manifest::EditError::Store(e) => e,
other => LeanError::State(format!("the gateway's commit did not land: {other:?}")),
})
}
pub(super) async fn cited_key(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str) -> String {
manifest::load(store.as_ref(), cfg)
.await
.unwrap()
.and_then(|l| l.manifest.entries.get(path).map(|e| e.key.clone()))
.unwrap_or_else(|| cfg.file_key(path))
}
pub(super) async fn handles_for(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str) -> Vec<String> {
let mut keys: Vec<String> = store
.list(&cfg.files_prefix())
.await
.unwrap()
.into_iter()
.map(|o| o.key)
.filter(|k| cfg.handle_parts(k).map(|(p, _)| p == path).unwrap_or(false))
.collect();
keys.sort();
keys
}
pub(super) async fn cited_bytes(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str) -> Option<Bytes> {
let l = manifest::load(store.as_ref(), cfg).await.unwrap()?;
let e = l.manifest.entries.get(path)?;
store.get_whole(&e.key, Some(&e.etag)).await.ok().map(|(_, b)| b)
}
pub(super) async fn object_at(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str) -> bool {
match manifest::load(store.as_ref(), cfg).await.unwrap().and_then(|l| l.manifest.entries.get(path).cloned()) {
Some(e) => store.head(&e.key).await.is_ok(),
None => false,
}
}
pub(super) async fn peer_publish(
store: &Arc<MemoryStore>,
cfg: &LeanConfig,
path: &str,
content: &str,
flush: &str,
) -> (String, String) {
let key = cfg.handle_key(path, flush);
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let loaded = manifest::load(store.as_ref(), cfg).await.unwrap();
let prev_gen =
loaded.as_ref().and_then(|l| l.manifest.entries.get(path)).map(|e| e.generation).unwrap_or(0);
let stamps = GenerationStamps {
generation: prev_gen + 1,
epoch: 0,
flush_uuid: flush.to_string(),
boundary_source: None,
posix: None,
};
let meta = store.put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc).await.unwrap();
let mut theirs = loaded.as_ref().map(|l| l.manifest.clone()).unwrap_or_default();
theirs.seq += 1;
theirs.entries.insert(
path.to_string(),
manifest::LeanEntry {
key: key.clone(),
etag: meta.etag.clone(),
crc64_b64: flint_store::crc64_to_b64(crc),
size: meta.size,
mode: 0o644,
mtime_unix: 0,
generation: prev_gen + 1,
epoch: 0,
},
);
let handle = loaded.as_ref().map(|l| l.handle());
let retired = loaded.as_ref().and_then(|l| l.manifest.entries.get(path)).map(|e| e.key.clone());
manifest::cas_write(store.as_ref(), cfg, &theirs, handle.as_ref(), 0, flush).await.unwrap();
if let Some(old) = retired {
if !theirs.entries.values().any(|e| e.key == old) {
store.delete(&old).await.unwrap();
}
}
(meta.etag, key)
}
#[tokio::test]
async fn checkout_publish_roundtrip_and_two_scan_delete() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap(); write(dir_a.path(), "src/main.rs", "fn main() {}");
write(dir_a.path(), "README.md", "hello");
let r = a.run_barrier().await.unwrap();
assert_eq!(r.uploaded.len(), 2);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(read(dir_b.path(), "src/main.rs").unwrap(), "fn main() {}");
std::fs::remove_file(dir_a.path().join("README.md")).unwrap();
let r1 = a.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty(), "first absence must not delete");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("README.md"));
let r2 = a.run_barrier().await.unwrap();
assert_eq!(r2.deleted, vec!["README.md".to_string()]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("README.md"));
assert!(!object_at(&store, &a.cfg, "README.md").await);
}
#[tokio::test]
async fn hitl_upload_survives_two_barriers_without_sync() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, "docs/upload.pdf", "user bytes", "dilip").await.unwrap();
write(dir.path(), "agent.txt", "agent work v2 — longer");
sc.run_barrier().await.unwrap();
sc.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "docs/upload.pdf").unwrap(), "user bytes");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("docs/upload.pdf"), "amputated!");
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "docs/upload.pdf").unwrap(), "user bytes");
}
async fn foreign_overwrite(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str, content: &str) {
let key = cited_key(store, cfg, path).await;
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "a-stranger".to_string(),
boundary_source: None,
posix: None,
};
store
.put_whole(&key, body, &PutCondition::Unconditional, &stamps, crc)
.await
.expect("the foreign write lands");
}
#[tokio::test]
async fn a_published_workspace_refuses_a_foreign_write_instead_of_adopting_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.sole_writer = true;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "README.md", "the real readme");
sc.run_barrier().await.unwrap();
foreign_overwrite(&store, &sc.cfg, "README.md", "FOREIGN BYTES").await;
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
let err = sc2.checkout().await.expect_err("a published workspace must refuse");
let msg = format!("{err}");
assert!(msg.contains("SOLE WRITER"), "the refusal must say why: {msg}");
assert!(
!msg.contains("recover-staged"),
"that remedy no longer exists; nothing was staged here: {msg}"
);
assert!(
read(dir2.path(), "README.md").is_none(),
"it materialized the foreign bytes anyway"
);
}
#[tokio::test]
async fn an_overwritten_handle_is_refused_by_checkout_in_every_workspace() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(!sc.cfg.sole_writer, "the default is an ordinary workspace");
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "README.md", "the real readme");
sc.run_barrier().await.unwrap();
foreign_overwrite(&store, &sc.cfg, "README.md", "bytes from outside").await;
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
let err = sc2.checkout().await.expect_err("an overwritten handle was adopted");
assert!(err.to_string().contains("immutable handle"), "{err}");
assert_eq!(read(dir2.path(), "README.md"), None, "uncited bytes were written");
}
#[tokio::test]
async fn the_published_flag_survives_the_pointer_round_trip() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.sole_writer = true;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "alpha");
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.sole_writer, "the pointer lost the flag");
write(dir.path(), "b.txt", "beta");
sc.run_barrier().await.unwrap();
let m2 = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m2.manifest.sole_writer, "the second publish dropped the flag");
}
#[tokio::test]
async fn a_published_mirror_does_not_probe_for_neighbours() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
store
.epoch_acquire(&format!("{PREFIX}/git/epoch"), "forge-1", None)
.await
.unwrap();
let seen = flint_store::layout::neighbours(
store.as_ref(),
&sc.cfg.prefix,
flint_store::layout::Writer::LeanWorkspace,
)
.await
.unwrap();
assert_eq!(seen.len(), 1, "the condition is there to be found");
assert!(!sc.cfg.sole_writer, "an ordinary workspace probes");
sc.cfg.sole_writer = true;
assert!(sc.cfg.sole_writer, "a published mirror does not");
}
#[tokio::test]
async fn ui_edit_vs_agent_edit_never_a_silent_winner() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "notes.md", "v1");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, "notes.md", "user version", "dilip").await.unwrap();
write(dir.path(), "notes.md", "agent version");
backdate_baseline(&sc, "notes.md");
sc.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "notes.md").unwrap(), "agent version");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let cited = &m.manifest.entries["notes.md"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"agent version");
let conflicts = sc.state.load_conflicts().unwrap();
let c = conflicts.iter().find(|c| c.kind.starts_with("commit-surfaced-foreign")).expect("conflict surfaced");
assert_eq!(c.path, "notes.md");
let preserved = c.preserved_key.as_ref().expect("foreign bytes preserved");
let (_, body) = store.get_whole(preserved, None).await.unwrap();
assert_eq!(&body[..], b"user version");
}
#[tokio::test]
async fn container_restart_never_resurrects_unpublished_delete() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "keep.txt", "keep");
write(dir.path(), "gone.txt", "delete me");
sc.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
drop(sc);
let mut sc = syncer(&store, dir.path()).await; let cr = sc.checkout().await.unwrap();
assert!(cr.resumed_live_tree, "marker present ⇒ live-tree row");
assert_eq!(cr.materialized, 0, "must not re-materialize");
assert!(read(dir.path(), "gone.txt").is_none(), "delete resurrected!");
assert!(claim_until_held(&mut sc, 1).await, "self-recognition must not wait");
sc.run_barrier().await.unwrap();
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("gone.txt"));
}
#[tokio::test]
async fn takeover_rotation_fences_the_straggler() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "from A");
a.run_barrier().await.unwrap();
let seq_before = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert!(claim_until_held(&mut a, 3).await, "a released cell is claimable at once");
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(
!claim_until_held(&mut b, 3).await,
"a fresh replacement must NOT claim instantly over a live-looking lease"
);
assert!(claim_until_held(&mut b, 10).await, "quiet polls exhausted ⇒ takeover");
let rotated = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap();
assert_eq!(rotated.manifest.seq, seq_before + 1, "rotation: seq++, content-identical");
assert_eq!(rotated.manifest.entries.len(), 1);
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "f.txt").unwrap(), "from A");
let err = lease::renew(&mut a).await.unwrap_err();
assert!(matches!(err, LeanError::Fenced(_)), "straggler must fence, got: {err:?}");
assert!(a.lease.is_none(), "a fenced holder must drop its lease");
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.seq, rotated.manifest.seq, "straggler CAS landed!");
let cited = &m.manifest.entries["f.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"from A", "straggler bytes reached a cited object");
write(dir_a.path(), "f.txt", "A, after the fence");
backdate_baseline(&a, "f.txt");
a.run_barrier().await.expect("a fenced writer's NEXT barrier publishes");
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap();
let cited = &m.manifest.entries["f.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"A, after the fence");
}
#[tokio::test]
async fn a_version_published_under_the_agents_edit_is_surfaced_at_the_commit_never_silently_overwritten() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "f.txt", "seed");
sc.run_barrier().await.unwrap();
let (peer_etag, peer_key) = peer_publish(&store, &sc.cfg, "f.txt", "peer bytes", "someone-else").await;
write(dir.path(), "f.txt", "agent bytes");
backdate_baseline(&sc, "f.txt");
let r = sc.run_barrier().await.unwrap();
assert!(r.parked.is_empty(), "parked with no way out: {:?}", r.parked);
assert_eq!(r.uploaded, vec!["f.txt".to_string()]);
assert_eq!(r.surfaced, vec!["f.txt".to_string()], "the commit did not surface what it published over");
assert_eq!(cited_bytes(&store, &sc.cfg, "f.txt").await.as_deref(), Some(&b"agent bytes"[..]));
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_ne!(m.manifest.entries["f.txt"].key, peer_key);
let rec = sc
.state
.load_conflicts()
.unwrap()
.into_iter()
.find(|c| c.kind.starts_with("commit-surfaced-foreign") && c.path == "f.txt")
.expect("no record surfaces the version published over");
assert_eq!(rec.foreign_etag, peer_etag);
let (_, kept) = store.get_whole(&rec.preserved_key.expect("not preserved"), None).await.unwrap();
assert_eq!(&kept[..], b"peer bytes", "the preserved copy is not the peer's version");
assert!(matches!(store.head(&peer_key).await, Err(StoreError::NotFound(_))), "the retired handle survived");
assert_eq!(handles_for(&store, &sc.cfg, "f.txt").await, vec![m.manifest.entries["f.txt"].key.clone()]);
assert_eq!(r.collected, 1, "the peer's handle was the one this commit retired: {r:?}");
}
#[tokio::test]
async fn a_delete_retires_only_the_handle_it_cited_never_a_ui_re_creation() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "doc.txt", "v1");
sc.run_barrier().await.unwrap();
let seed = cited_key(&store, &sc.cfg, "doc.txt").await;
std::fs::remove_file(dir.path().join("doc.txt")).unwrap();
sc.run_barrier().await.unwrap();
let (_, ui_key) = hitl_write_at(&store, &sc.cfg, "doc.txt", "user re-created", "dilip", now_unix()).await.unwrap();
let _ = &ui_key;
for _ in 0..4 {
sc.run_barrier().await.unwrap();
}
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("doc.txt"), "the agent's delete never published");
assert!(!m.manifest.entries.values().any(|e| e.key == seed), "the seed is still cited");
let conflicts = sc.state.load_conflicts().unwrap();
assert!(!conflicts.iter().any(|c| c.kind == "gc-skip"), "no guard, no skip: {conflicts:?}");
let rec = conflicts
.iter()
.find(|c| c.kind.starts_with("commit-deleted-over-theirs") && c.path == "doc.txt")
.expect("the UI write over the deleted path was not surfaced");
let (_, kept) = store.get_whole(rec.preserved_key.as_ref().unwrap(), None).await.unwrap();
assert_eq!(&kept[..], b"user re-created");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_commit_withholds_an_upload_a_peers_sweep_took_and_uploads_afresh_next_barrier() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "seed.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(dir_b.path(), "b.txt", "B publishes");
write(dir_a.path(), "x.txt", "A's upload");
let b_slot = std::sync::Mutex::new(Some(b));
let b_back: Arc<std::sync::Mutex<Option<Syncer>>> = Arc::new(std::sync::Mutex::new(None));
let b_back_in = b_back.clone();
ha.after_put(&a.cfg.file_key("x.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let rb = b.run_barrier().await.expect("B's barrier");
assert!(rb.swept >= 1, "fixture: B's sweep did not take A's in-flight upload: {rb:?}");
})
});
*b_back_in.lock().unwrap() = Some(b);
});
let ra = a.run_barrier().await.expect("A's barrier");
assert!(b_back.lock().unwrap().is_some(), "fixture: the hook never ran");
assert_eq!(ra.parked, vec!["x.txt".to_string()], "the commit cited a handle the sweep took: {ra:?}");
assert!(ra.uploaded.is_empty(), "{ra:?}");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "cited a collected handle");
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.kind.starts_with("upload-withheld")),
"the withheld upload left no record"
);
let ra2 = a.run_barrier().await.unwrap();
assert_eq!(ra2.uploaded, vec!["x.txt".to_string()], "{ra2:?}");
assert_eq!(cited_bytes(&inner, &a.cfg, "x.txt").await.as_deref(), Some(&b"A's upload"[..]));
}
async fn a_writers_sweep_inside_a_ui_saves_window(grace: bool) {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "seed");
a.run_barrier().await.unwrap();
if !grace {
a.cfg.untracked_grace_secs = 0;
}
a.state.save_orphan_sweep_at(now_unix() - a.cfg.untracked_sweep_secs - 1).unwrap();
let added = now_unix();
let flush = super::ui_flush(added);
let key = a.cfg.handle_key("ui.txt", &flush);
let body = Bytes::from_static(b"the reviewer's save");
let crc = crc64_nvme(&body);
let stamps =
GenerationStamps { generation: 0, epoch: 0, flush_uuid: flush.clone(), boundary_source: None, posix: None };
let meta = store.put_whole(&key, body.clone(), &PutCondition::IfNoneMatchAny, &stamps, crc).await.unwrap();
write(dir.path(), "a.txt", "A publishes");
let r = a.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["a.txt".to_string()], "fixture: the writer did not commit: {r:?}");
let (entry_key, etag) = (key.clone(), meta.etag.clone());
gateway_commit(&store, &a.cfg, &flush, move |doc| {
doc.entries.insert(
"ui.txt".to_string(),
manifest::LeanEntry {
key: entry_key.clone(),
etag: etag.clone(),
crc64_b64: flint_store::crc64_to_b64(crc),
size: body.len() as u64,
mode: 0o644,
mtime_unix: added as i64,
generation: 1,
epoch: 0,
},
);
})
.await
.unwrap();
let cited = cited_bytes(&store, &a.cfg, "ui.txt").await;
if grace {
assert_eq!(r.swept, 0, "the sweep took a UI save's handle inside its grace: {r:?}");
assert_eq!(cited.as_deref(), Some(&b"the reviewer's save"[..]), "the UI save cites a deleted handle");
} else {
assert_eq!(r.swept, 1, "fixture: with no grace the sweep did not take the save's handle: {r:?}");
assert!(cited.is_none(), "fixture: the control arm's citation is readable");
}
}
#[tokio::test]
async fn a_writers_orphan_sweep_spares_a_ui_save_between_its_put_and_its_commit() {
a_writers_sweep_inside_a_ui_saves_window(true).await;
}
#[tokio::test]
async fn without_the_grace_a_writers_sweep_leaves_a_ui_save_citing_nothing() {
a_writers_sweep_inside_a_ui_saves_window(false).await;
}
#[tokio::test]
async fn a_peers_sweep_never_runs_inside_a_holders_commit() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
a.checkout().await.unwrap();
write(dir_a.path(), "seed.txt", "seed");
a.run_barrier().await.unwrap();
syncer(&inner, dir_b.path()).await.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "A's upload");
let rb: Arc<std::sync::Mutex<Option<Result<super::barrier::BarrierReport, LeanError>>>> = Default::default();
{
let (inner, root_b, rb) = (inner.clone(), dir_b.path().to_path_buf(), rb.clone());
ha.after_head(&a.cfg.file_key("x.txt"), move || {
let (inner, root_b) = (inner.clone(), root_b.clone());
let r = std::thread::spawn(move || {
tokio::runtime::Runtime::new().unwrap().block_on(async {
let mut b = syncer(&inner, &root_b).await;
b.cfg.claim_quiet_spacing_secs = lease::QUIET_SPACING_SECS;
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(&root_b, "b.txt", "B publishes");
b.run_barrier().await
})
})
.join()
.unwrap();
*rb.lock().unwrap() = Some(r);
});
}
let ra = a.run_barrier().await.expect("A's barrier");
let rb = rb.lock().unwrap().take().expect("fixture: the hook never ran");
assert!(rb.is_err(), "fixture: B committed inside A's commit section: {rb:?}");
assert_eq!(ra.uploaded, vec!["x.txt".to_string()], "{ra:?}");
assert_eq!(
cited_bytes(&inner, &a.cfg, "x.txt").await.as_deref(),
Some(&b"A's upload"[..]),
"A's commit cites a handle a peer's sweep took inside it"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_peers_version_behind_a_withheld_upload_arrives_when_the_agent_drops_its_file() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "seed.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(dir_b.path(), "seed.txt", "seed");
backdate_baseline(&b, "seed.txt");
write(dir_a.path(), "x.txt", "A's draft");
let b_slot = std::sync::Mutex::new(Some(b));
let (store2, cfg_a) = (inner.clone(), a.cfg.clone());
ha.after_put(&a.cfg.file_key("x.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let rb = b.run_barrier().await.expect("B's barrier");
assert!(rb.swept >= 1, "fixture: B's sweep did not take A's in-flight upload: {rb:?}");
peer_publish(&store2, &cfg_a, "x.txt", "the peer's x", "peer").await;
})
});
});
let ra = a.run_barrier().await.expect("A's barrier");
assert_eq!(ra.parked, vec!["x.txt".to_string()], "fixture: the upload was not withheld: {ra:?}");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.keys().collect::<Vec<_>>(), ["seed.txt", "x.txt"], "fixture: something else changed");
assert_eq!(m.entries["seed.txt"].etag, a.state.load_baseline().unwrap().entries["seed.txt"].etag, "fixture: B's commit changed seed.txt's bytes");
assert_eq!(read(dir_a.path(), "x.txt").as_deref(), Some("A's draft"), "fixture");
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
for _ in 0..2 {
a.run_barrier().await.unwrap();
}
assert_eq!(read(dir_a.path(), "x.txt").as_deref(), Some("the peer's x"), "the peer's version never arrived");
}
#[tokio::test(flavor = "multi_thread")]
async fn a_peers_version_behind_a_withheld_edit_arrives_when_the_agent_reverts() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "seed.txt", "seed");
write(dir_a.path(), "x.txt", "x v1");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(dir_b.path(), "seed.txt", "seed");
backdate_baseline(&b, "seed.txt");
write(dir_a.path(), "x.txt", "A's edit");
let b_slot = std::sync::Mutex::new(Some(b));
let (store2, cfg_a) = (inner.clone(), a.cfg.clone());
ha.after_put(&a.cfg.file_key("x.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let rb = b.run_barrier().await.expect("B's barrier");
assert!(rb.swept >= 1, "fixture: B's sweep did not take A's in-flight upload: {rb:?}");
peer_publish(&store2, &cfg_a, "x.txt", "the peer's x", "peer").await;
})
});
});
let ra = a.run_barrier().await.expect("A's barrier");
assert_eq!(ra.parked, vec!["x.txt".to_string()], "fixture: the upload was not withheld: {ra:?}");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.keys().collect::<Vec<_>>(), ["seed.txt", "x.txt"], "fixture: something else changed");
assert_eq!(m.entries["seed.txt"].etag, a.state.load_baseline().unwrap().entries["seed.txt"].etag, "fixture: B's commit changed seed.txt's bytes");
assert_eq!(read(dir_a.path(), "x.txt").as_deref(), Some("A's edit"), "fixture");
restore_to_baseline(&a, "x.txt", "x v1");
for _ in 0..2 {
a.run_barrier().await.unwrap();
}
assert_eq!(read(dir_a.path(), "x.txt").as_deref(), Some("the peer's x"), "the peer's version never arrived");
}
#[tokio::test]
async fn a_ui_rename_over_a_dirty_source_keeps_both_versions_with_a_record() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
let seed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["x.txt"].etag.clone();
write(dir.path(), "x.txt", "the agent's edit");
backdate_baseline(&a, "x.txt");
hitl_rename(&store, &a.cfg, "x.txt", "y.txt", "dilip").await;
for _ in 0..3 {
a.run_barrier().await.unwrap();
}
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["y.txt"].etag, seed, "the rename's destination lost the moved version");
assert_eq!(read(dir.path(), "y.txt").as_deref(), Some("seed"));
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("the agent's edit"), "the agent's edit was touched");
assert!(m.entries.contains_key("x.txt"), "the agent's edit was never published");
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.path == "x.txt" && c.foreign_etag == seed && c.kind.starts_with("commit-recreated-deleted")),
"the edit overrode the rename's move with no record: {:?}",
a.state.load_conflicts().unwrap()
);
}
#[tokio::test]
async fn a_ui_rename_onto_a_destination_the_agent_took_preserves_the_moved_version() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
let seed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["x.txt"].etag.clone();
write(dir.path(), "y.txt", "the agent's own y");
hitl_rename(&store, &a.cfg, "x.txt", "y.txt", "dilip").await;
for _ in 0..3 {
a.run_barrier().await.unwrap();
}
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(read(dir.path(), "y.txt").as_deref(), Some("the agent's own y"), "the agent's file was touched");
assert!(!m.entries.contains_key("x.txt"), "the rename's source is cited again");
let rec = a
.state
.load_conflicts()
.unwrap()
.into_iter()
.find(|c| c.path == "y.txt" && c.foreign_etag == seed)
.expect("the moved version was lost with no record");
let (_, kept) = store.get_whole(rec.preserved_key.as_ref().expect("not preserved"), None).await.unwrap();
assert_eq!(&kept[..], b"seed");
}
#[tokio::test]
async fn a_writer_that_loses_several_cas_races_backs_off_and_installs() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&ha, dir.path());
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "seed");
a.run_barrier().await.unwrap();
write(dir.path(), "x.txt", "the writer's publish");
*ha.1.put_conflicts.lock().unwrap() = Some((a.cfg.current_key(), 6));
let t0 = std::time::Instant::now();
let r = a.run_barrier().await.expect("the barrier gave up on lost CAS races");
assert!(t0.elapsed() >= std::time::Duration::from_millis(630), "no backoff between the lost races: {:?}", t0.elapsed());
assert_eq!(r.uploaded, vec!["x.txt".to_string()], "{r:?}");
assert_eq!(ha.1.put_conflicts.lock().unwrap().as_ref().map(|(_, n)| *n), Some(0), "fixture: not every race was lost");
assert!(manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries.contains_key("x.txt"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_withheld_upload_republished_next_barrier_surfaces_the_version_it_never_integrated() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "p.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
let (ui_etag, ui_key) = hitl_write_at(&inner, &a.cfg, "p.txt", "the user's save", "dilip", now_unix()).await.unwrap();
b.run_barrier().await.unwrap();
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["p.txt"].key, ui_key, "fixture: B did not publish the UI write");
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(dir_b.path(), "b.txt", "B publishes");
write(dir_a.path(), "p.txt", "A's edit");
backdate_baseline(&a, "p.txt");
let b_slot = std::sync::Mutex::new(Some(b));
let b_back: Arc<std::sync::Mutex<Option<Syncer>>> = Arc::new(std::sync::Mutex::new(None));
let b_back_in = b_back.clone();
ha.after_put(&a.cfg.file_key("p.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let rb = b.run_barrier().await.expect("B's barrier");
assert!(rb.swept >= 1, "fixture: B's sweep did not take A's in-flight upload: {rb:?}");
})
});
*b_back_in.lock().unwrap() = Some(b);
});
let ra = a.run_barrier().await.unwrap();
assert!(b_back.lock().unwrap().is_some(), "fixture: the hook never ran");
assert_eq!(ra.parked, vec!["p.txt".to_string()], "fixture: the upload was not withheld: {ra:?}");
let ra2 = a.run_barrier().await.unwrap();
assert_eq!(ra2.uploaded, vec!["p.txt".to_string()], "fixture: {ra2:?}");
assert_eq!(cited_bytes(&inner, &a.cfg, "p.txt").await.as_deref(), Some(&b"A's edit"[..]));
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.path == "p.txt" && c.foreign_etag == ui_etag),
"A's edit replaced the user's acknowledged save with no record: {:?}",
a.state.load_conflicts().unwrap()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_withheld_upload_beside_a_published_one_surfaces_the_version_it_never_integrated() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "p.txt", "seed");
write(dir_a.path(), "q.txt", "seed q");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
let (ui_etag, ui_key) = hitl_write_at(&inner, &a.cfg, "p.txt", "the user's save", "dilip", now_unix()).await.unwrap();
b.run_barrier().await.unwrap();
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["p.txt"].key, ui_key, "fixture: B did not publish the UI write");
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(dir_b.path(), "b.txt", "B publishes");
write(dir_a.path(), "p.txt", "A's edit");
write(dir_a.path(), "q.txt", "A's q");
backdate_baseline(&a, "p.txt");
backdate_baseline(&a, "q.txt");
let b_slot = std::sync::Mutex::new(Some(b));
let b_back: Arc<std::sync::Mutex<Option<Syncer>>> = Arc::new(std::sync::Mutex::new(None));
let b_back_in = b_back.clone();
ha.after_put(&a.cfg.file_key("p.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let rb = b.run_barrier().await.expect("B's barrier");
assert!(rb.swept >= 1, "fixture: B's sweep did not take A's in-flight upload: {rb:?}");
})
});
*b_back_in.lock().unwrap() = Some(b);
});
let ra = a.run_barrier().await.unwrap();
assert!(b_back.lock().unwrap().is_some(), "fixture: the hook never ran");
assert_eq!(ra.parked, vec!["p.txt".to_string()], "fixture: p was not withheld: {ra:?}");
assert_eq!(ra.uploaded, vec!["q.txt".to_string()], "fixture: q did not publish beside it: {ra:?}");
let ra2 = a.run_barrier().await.unwrap();
assert_eq!(ra2.uploaded, vec!["p.txt".to_string()], "fixture: {ra2:?}");
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.path == "p.txt" && c.foreign_etag == ui_etag),
"A's edit replaced the user's acknowledged save with no record: {:?}",
a.state.load_conflicts().unwrap()
);
}
async fn l126_withheld_beside_a_cas(
ui: bool,
) -> (Arc<MemoryStore>, Arc<Hooked>, Syncer, tempfile::TempDir, tempfile::TempDir, String) {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
if ui {
write(dir_a.path(), "p.txt", "seed");
}
write(dir_a.path(), "q.txt", "seed q");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
let peer_etag = if ui {
let (etag, _) = hitl_write_at(&inner, &a.cfg, "p.txt", "the user's save", "dilip", now_unix()).await.unwrap();
b.run_barrier().await.unwrap();
etag
} else {
write(dir_b.path(), "p.txt", "B's p");
b.run_barrier().await.unwrap();
manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["p.txt"].etag.clone()
};
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(dir_b.path(), "b.txt", "B publishes");
write(dir_a.path(), "p.txt", "A's p");
write(dir_a.path(), "q.txt", "A's q");
backdate_baseline(&a, "p.txt");
backdate_baseline(&a, "q.txt");
let b_slot = std::sync::Mutex::new(Some(b));
let b_back: Arc<std::sync::Mutex<Option<Syncer>>> = Arc::new(std::sync::Mutex::new(None));
let b_back_in = b_back.clone();
ha.after_put(&a.cfg.file_key("p.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let rb = b.run_barrier().await.expect("B's barrier");
assert!(rb.swept >= 1, "fixture: B's sweep did not take A's in-flight upload: {rb:?}");
})
});
*b_back_in.lock().unwrap() = Some(b);
});
let ra = a.run_barrier().await.unwrap();
assert!(b_back.lock().unwrap().is_some(), "fixture: the hook never ran");
assert_eq!(ra.parked, vec!["p.txt".to_string()], "fixture: p was not withheld: {ra:?}");
assert_eq!(ra.uploaded, vec!["q.txt".to_string()], "fixture: q did not publish beside it: {ra:?}");
(inner, ha, a, dir_a, dir_b, peer_etag)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_withheld_new_file_beside_a_published_one_surfaces_the_peers_file() {
let (_inner, _ha, mut a, _da, _db, peer_etag) = l126_withheld_beside_a_cas(false).await;
let ra2 = a.run_barrier().await.unwrap();
assert_eq!(ra2.uploaded, vec!["p.txt".to_string()], "fixture: {ra2:?}");
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.path == "p.txt" && c.foreign_etag == peer_etag),
"A's new file replaced B's with no record: {:?}",
a.state.load_conflicts().unwrap()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_barrier_that_fails_before_its_cas_keeps_the_parked_set_for_the_retry() {
let (_inner, ha, mut a, _da, _db, peer_etag) = l126_withheld_beside_a_cas(true).await;
*ha.1.put_fail_containing.lock().unwrap() = Some(a.cfg.current_key());
assert!(a.run_barrier().await.is_err(), "fixture: the barrier's CAS did not fail");
*ha.1.put_fail_containing.lock().unwrap() = None;
let ra3 = a.run_barrier().await.unwrap();
assert_eq!(ra3.uploaded, vec!["p.txt".to_string()], "fixture: {ra3:?}");
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.path == "p.txt" && c.foreign_etag == peer_etag),
"A's edit replaced the user's acknowledged save with no record: {:?}",
a.state.load_conflicts().unwrap()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_withheld_new_file_republished_next_barrier_surfaces_the_peers_file() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "seed.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "p.txt", "B's p");
b.run_barrier().await.unwrap();
let b_etag = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["p.txt"].etag.clone();
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(dir_b.path(), "b.txt", "B publishes");
write(dir_a.path(), "p.txt", "A's p");
let b_slot = std::sync::Mutex::new(Some(b));
let b_back: Arc<std::sync::Mutex<Option<Syncer>>> = Arc::new(std::sync::Mutex::new(None));
let b_back_in = b_back.clone();
ha.after_put(&a.cfg.file_key("p.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let rb = b.run_barrier().await.expect("B's barrier");
assert!(rb.swept >= 1, "fixture: B's sweep did not take A's in-flight upload: {rb:?}");
})
});
*b_back_in.lock().unwrap() = Some(b);
});
let ra = a.run_barrier().await.unwrap();
assert!(b_back.lock().unwrap().is_some(), "fixture: the hook never ran");
assert_eq!(ra.parked, vec!["p.txt".to_string()], "fixture: the upload was not withheld: {ra:?}");
let ra2 = a.run_barrier().await.unwrap();
assert_eq!(ra2.uploaded, vec!["p.txt".to_string()], "fixture: {ra2:?}");
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.path == "p.txt" && c.foreign_etag == b_etag),
"A's new file replaced B's with no record: {:?}",
a.state.load_conflicts().unwrap()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_restarted_barrier_uploads_afresh_and_leaves_its_earlier_handles_to_the_sweep() {
let (inner, mut b, dir_a, dir_b, orphan) = killed_writer_orphan().await;
let mut a = syncer(&inner, dir_a.path()).await;
let ra = a.run_barrier().await.expect("A's retry");
assert_eq!(ra.uploaded, vec!["p1.txt".to_string(), "p2.txt".to_string()], "{ra:?}");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_ne!(m.entries["p1.txt"].key, orphan, "the retry cited its orphan instead of uploading afresh");
assert_eq!(cited_bytes(&inner, &a.cfg, "p1.txt").await.as_deref(), Some(&b"A's unpublished edit"[..]));
assert!(inner.head(&orphan).await.is_ok(), "the retry is not the collector of its predecessor");
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(dir_b.path(), "b2.txt", "B commits again");
let rb = b.run_barrier().await.unwrap();
assert_eq!(rb.swept, 1, "{rb:?}");
assert!(matches!(inner.head(&orphan).await, Err(StoreError::NotFound(_))), "the orphan survived the sweep");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_peers_in_flight_upload_is_untouched_by_anothers_collector() {
let inner = Arc::new(MemoryStore::new());
let hb = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&inner, dir_a.path()).await;
let mut b = hooked_syncer(&hb, dir_b.path());
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
let seed = cited_key(&inner, &a.cfg, "x.txt").await;
write(dir_b.path(), "x.txt", "seed!");
backdate_baseline(&b, "x.txt");
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
a.run_barrier().await.unwrap(); let a_slot = std::sync::Mutex::new(Some(a));
let a_back: Arc<std::sync::Mutex<Option<Syncer>>> = Arc::new(std::sync::Mutex::new(None));
let a_back_in = a_back.clone();
hb.after_put(&b.cfg.file_key("x.txt"), move || {
let mut a = a_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let ra = a.run_barrier().await.expect("A's delete barrier");
assert_eq!(ra.deleted, vec!["x.txt".to_string()], "fixture: A did not publish its delete: {ra:?}");
assert_eq!(ra.collected, 1, "fixture: A's collector took nothing: {ra:?}");
})
});
*a_back_in.lock().unwrap() = Some(a);
});
let rb = b.run_barrier().await.expect("B's barrier");
assert!(a_back.lock().unwrap().is_some(), "fixture: the hook never ran");
assert_eq!(rb.uploaded, vec!["x.txt".to_string()], "{rb:?}");
assert!(rb.parked.is_empty(), "A's collector took B's in-flight upload: {rb:?}");
assert_eq!(
cited_bytes(&inner, &b.cfg, "x.txt").await.as_deref(),
Some(&b"seed!"[..]),
"modify wins over the racing delete, and the bytes are B's"
);
assert!(matches!(inner.head(&seed).await, Err(StoreError::NotFound(_))), "A's collector did not take what A retired");
}
#[tokio::test]
async fn a_store_without_a_conditional_delete_collects_just_the_same() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
let seed = cited_key(&store, &a.cfg, "x.txt").await;
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
a.run_barrier().await.unwrap(); let ra = a.run_barrier().await.unwrap();
assert_eq!(ra.deleted, vec!["x.txt".to_string()]);
assert!(ra.leaked.is_empty(), "the collector gave way: {ra:?}");
assert_eq!(ra.collected, 1, "{ra:?}");
assert!(matches!(store.head(&seed).await, Err(StoreError::NotFound(_))), "the retired handle leaked");
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert!(read(dir_b.path(), "x.txt").is_none(), "the peer's delete did not reach B's tree");
assert_eq!(read(dir_b.path(), "keep.txt").as_deref(), Some("keep"));
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "the delete flapped back");
}
#[tokio::test]
async fn an_ingress_copy_a_writer_publishes_over_is_preserved_with_a_record() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.untracked_grace_secs = 0;
a.checkout().await.unwrap();
write(dir_a.path(), "p1.txt", "v1");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
let body = Bytes::from_static(b"outside bytes");
let stamps = GenerationStamps { generation: 0, epoch: 0, flush_uuid: "aws-cli".into(), boundary_source: None, posix: None };
let ingress = store
.put_whole(&b.cfg.file_key("p1.txt"), body.clone(), &PutCondition::Unconditional, &stamps, crc64_nvme(&body))
.await
.unwrap();
let tracked = b.track_untracked(now_unix() + 1).await.unwrap();
assert_eq!(tracked, vec!["p1.txt".to_string()], "fixture: the ingress object was not tracked");
assert_eq!(cited_bytes(&store, &b.cfg, "p1.txt").await.as_deref(), Some(&b"outside bytes"[..]), "the copy is not cited");
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir_b.path(), "p1.txt").as_deref(), Some("outside bytes"));
write(dir_a.path(), "p1.txt", "v2 by A's agent");
backdate_baseline(&a, "p1.txt");
let ra = a.run_barrier().await.unwrap();
assert_eq!(ra.surfaced, vec!["p1.txt".to_string()], "A published over the ingress copy with no record: {ra:?}");
assert_eq!(cited_bytes(&store, &b.cfg, "p1.txt").await.as_deref(), Some(&b"v2 by A's agent"[..]));
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir_b.path(), "p1.txt").as_deref(), Some("v2 by A's agent"), "B's tree did not follow the document");
let (at_key, _) = store.get_whole(&b.cfg.file_key("p1.txt"), None).await.unwrap();
assert_eq!(at_key.etag, ingress.etag, "the ingress object was touched");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_ui_write_landing_beside_an_in_flight_upload_is_preserved_never_lost() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir_a = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&ha, dir_a.path());
a.checkout().await.unwrap();
write(dir_a.path(), "p1.txt", "v1");
a.run_barrier().await.unwrap();
write(dir_a.path(), "p1.txt", "A's edit");
backdate_baseline(&a, "p1.txt");
let ui = Arc::new(std::sync::Mutex::new(None));
{
let (store, cfg, ui) = (inner.clone(), a.cfg.clone(), ui.clone());
ha.after_put(&a.cfg.file_key("p1.txt"), move || {
let (store, cfg) = (store.clone(), cfg.clone());
let landed = std::thread::spawn(move || {
tokio::runtime::Runtime::new()
.unwrap()
.block_on(hitl_write_at(&store, &cfg, "p1.txt", "the UI's version", "dilip", now_unix()))
.unwrap()
})
.join()
.unwrap();
*ui.lock().unwrap() = Some(landed);
});
}
let ra = a.run_barrier().await.unwrap();
ui.lock().unwrap().clone().expect("fixture: the UI write never landed mid-barrier");
assert_eq!(ra.uploaded, vec!["p1.txt".to_string()], "{ra:?}");
assert_eq!(ra.surfaced, vec!["p1.txt".to_string()], "A published over the UI's save with no record: {ra:?}");
assert_eq!(cited_bytes(&inner, &a.cfg, "p1.txt").await.as_deref(), Some(&b"A's edit"[..]));
let rec = a
.state
.load_conflicts()
.unwrap()
.into_iter()
.find(|c| c.kind.starts_with("commit-surfaced-foreign") && c.path == "p1.txt")
.expect("the UI write was dropped in silence");
let (_, kept) = inner.get_whole(rec.preserved_key.as_ref().expect("not preserved"), None).await.unwrap();
assert_eq!(&kept[..], b"the UI's version");
a.run_barrier().await.unwrap();
assert_eq!(read(dir_a.path(), "p1.txt").as_deref(), Some("A's edit"), "adopted over the published version");
}
#[tokio::test]
async fn an_agents_delete_over_a_peers_newer_version_lands_and_preserves_theirs() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
sc.run_barrier().await.unwrap();
let (their_etag, _) = peer_publish(&store, &sc.cfg, "shared.txt", "their v2", "other-writer").await;
std::fs::remove_file(dir.path().join("shared.txt")).unwrap();
let r = sc.declared_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["shared.txt".to_string()], "the delete lands in this barrier");
assert!(r.parked.is_empty(), "{:?}", r.parked);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("shared.txt"));
let c = sc
.state
.load_conflicts()
.unwrap()
.into_iter()
.find(|c| c.path == "shared.txt" && c.foreign_etag == their_etag)
.expect("the peer's version is named by a record");
let preserved = c.preserved_key.expect("and preserved");
let (_, body) = store.get_whole(&preserved, None).await.unwrap();
assert_eq!(&body[..], b"their v2");
assert!(read(dir.path(), "shared.txt").is_none(), "the tree keeps the agent's delete");
}
#[tokio::test]
async fn local_delete_meets_foreign_modify_and_preserves_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
sc.run_barrier().await.unwrap();
let (their_etag, _) = peer_publish(&store, &sc.cfg, "shared.txt", "their v2", "other-writer").await;
std::fs::remove_file(dir.path().join("shared.txt")).unwrap();
let r1 = sc.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty());
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries["shared.txt"].etag, their_etag, "foreign entry dropped!");
assert_eq!(cited_bytes(&store, &sc.cfg, "shared.txt").await.as_deref(), Some(&b"their v2"[..]));
let r2 = sc.run_barrier().await.unwrap();
assert_eq!(r2.deleted, vec!["shared.txt".to_string()]);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("shared.txt"));
let conflicts = sc.state.load_conflicts().unwrap();
let c = conflicts
.iter()
.find(|c| c.kind.starts_with("commit-deleted-over-theirs") && c.path == "shared.txt")
.expect("delete-vs-edit conflict must surface");
let preserved = c.preserved_key.as_ref().expect("foreign bytes preserved");
let (_, body) = store.get_whole(preserved, None).await.unwrap();
assert_eq!(&body[..], b"their v2", "the edit must stay recoverable after the delete wins");
}
#[tokio::test]
async fn large_file_publishes_via_compose_and_roundtrips() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.whole_put_max = 8; assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
let big_v1: String = (0..200).map(|i| format!("line {i}\n")).collect();
write(dir.path(), "model.bin", &big_v1);
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["model.bin".to_string()]);
assert!(r.deferred.is_empty());
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "model.bin").unwrap(), big_v1);
let big_v2 = format!("{big_v1}and more\n");
write(dir.path(), "model.bin", &big_v2);
backdate_baseline(&sc, "model.bin");
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["model.bin".to_string()]);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let cited = &m.manifest.entries["model.bin"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(String::from_utf8(body.to_vec()).unwrap(), big_v2);
}
#[tokio::test]
async fn second_syncer_on_one_tree_refuses() {
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let _held = SyncerState::open(cfg.state_dir()).unwrap();
let Err(err) = SyncerState::open(cfg.state_dir()) else {
panic!("second open over a held workspace must refuse");
};
assert!(matches!(err, LeanError::State(_)));
}
#[tokio::test]
async fn checkout_budget_refuses_before_first_byte() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "big.txt", "0123456789012345678901234567890123456789");
sc.run_barrier().await.unwrap();
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.cfg.max_bytes = 10;
let err = sc2.checkout().await.unwrap_err();
assert!(matches!(err, LeanError::Budget(_)));
assert!(!sc2.state.marker_present(), "budget refusal must not gate-open the agent");
assert!(read(dir2.path(), "big.txt").is_none());
}
#[tokio::test]
async fn an_all_rejected_sync_scope_is_refused_not_widened_to_the_whole_tree() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
let err = sc
.sync_scoped(Some(vec!["../escape".into(), "./here".into()]))
.await
.expect_err("an all-rejected scope must not silently become the whole tree");
let msg = format!("{err}");
assert!(
msg.contains("NONE survived validation"),
"the refusal must say WHY, so a typo is not read as a sync bug: {msg}"
);
sc.sync_scoped(Some(vec!["inputs".into()])).await.expect("a valid scope must still sync");
}
#[tokio::test]
async fn sync_scan_first_dirty_wins_clean_applies() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "shared.txt", "v1");
write(dir_a.path(), "mine.txt", "agent latest — NEVER destroy");
a.run_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "shared.txt", "user v2", "dilip").await.unwrap();
write(dir_a.path(), "mine.txt", "agent latest v2");
backdate_baseline(&a, "mine.txt");
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.entries.remove("mine.txt");
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "remote-delete")
.await
.unwrap();
let r = a.sync().await.unwrap();
assert_eq!(read(dir_a.path(), "shared.txt").unwrap(), "user v2");
assert!(r.applied.contains(&"shared.txt".to_string()));
assert_eq!(read(dir_a.path(), "mine.txt").unwrap(), "agent latest v2");
assert!(r.conflicts.contains(&"mine.txt".to_string()));
assert!(r.deleted.is_empty());
}
use super::control;
use super::sentinel::{Due, Verb};
pub(super) fn touch_sentinel(root: &std::path::Path, name: &str, body: &str) {
let dir = root.join(super::CONTROL_DIR);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join(name), body).unwrap();
}
fn control_exists(root: &std::path::Path, name: &str) -> bool {
root.join(super::CONTROL_DIR).join(name).exists()
}
#[tokio::test]
async fn flint_dir_never_scanned() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "real.txt", "data");
touch_sentinel(dir.path(), control::PUBLISH, "");
touch_sentinel(dir.path(), control::REMOTE_SEQ, "{}");
let scanned = super::scan::scan(dir.path()).unwrap();
assert!(scanned.contains_key("real.txt"));
assert!(
!scanned.keys().any(|k| k.starts_with(".flint")),
"the scan yielded a control path: {:?}",
scanned.keys().collect::<Vec<_>>()
);
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("real.txt"));
assert!(
!m.entries.keys().any(|k| k.starts_with(".flint")),
"a control path was CITED: {:?}",
m.entries.keys().collect::<Vec<_>>()
);
assert!(!object_at(&store, &a.cfg, ".flint/publish").await);
}
#[tokio::test]
async fn legacy_flint_citation_survives_upgrade() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "v1");
a.run_barrier().await.unwrap();
let key = a.cfg.file_key(".flint/legacy.txt");
let body = Bytes::from_static(b"legacy payload");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: "legacy".into(),
boundary_source: None,
posix: None,
};
let meta = store
.put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut m = loaded.manifest.clone();
m.seq += 1;
m.entries.insert(
".flint/legacy.txt".into(),
manifest::LeanEntry {
key: key.clone(),
etag: meta.etag.clone(),
crc64_b64: meta.crc64_b64.clone().unwrap(),
size: meta.size,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
},
);
let installed =
manifest::cas_write(store.as_ref(), &a.cfg, &m, Some(&loaded.handle()), 0, "legacy")
.await
.unwrap();
let mut b = a.state.load_baseline().unwrap();
b.entries.insert(
".flint/legacy.txt".into(),
super::state::BaselineEntry {
etag: meta.etag.clone(),
key: None,
generation: 1,
size: meta.size,
mtime_unix: 0,
mtime_nanos: None,
crc64_b64: meta.crc64_b64.clone(),
},
);
b.prev_scan.insert(".flint/legacy.txt".into());
b.seq = m.seq;
b.manifest_etag = Some(installed.etag.clone());
a.state.save_baseline(&b).unwrap();
assert!(store.head(&key).await.is_ok());
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
let after = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
after.entries.contains_key(".flint/legacy.txt"),
"an upgrade DELETED a legacy citation"
);
assert!(store.head(&key).await.is_ok(), "an upgrade GC'd the legacy object");
}
#[tokio::test]
async fn checkout_never_materializes_control_citation() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "real.txt", "v1");
a.run_barrier().await.unwrap();
let key = a.cfg.file_key(".flint/legacy.txt");
let body = Bytes::from_static(b"legacy");
let crc = crc64_nvme(&body);
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfNoneMatchAny,
&GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: "legacy".into(),
boundary_source: None,
posix: None,
},
crc,
)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut m = loaded.manifest.clone();
m.seq += 1;
m.entries.insert(
".flint/legacy.txt".into(),
manifest::LeanEntry {
key,
etag: meta.etag.clone(),
crc64_b64: meta.crc64_b64.clone().unwrap(),
size: meta.size,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
},
);
manifest::cas_write(store.as_ref(), &a.cfg, &m, Some(&loaded.handle()), 0, "legacy")
.await
.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let r = b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "real.txt").unwrap(), "v1");
assert!(
!dir_b.path().join(".flint/legacy.txt").exists(),
"checkout materialized into the reserved control namespace"
);
assert_eq!(r.refused, 1);
assert!(b
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.path == ".flint/legacy.txt" && c.kind.starts_with("checkout-refused")));
}
#[tokio::test]
async fn capabilities_written_on_live_tree_restart() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join(super::CONTROL_DIR).join(control::CAPABILITIES))
.unwrap();
assert!(a.state.marker_present());
assert!(!control_exists(dir.path(), control::CAPABILITIES));
let r = a.checkout().await.unwrap();
assert!(r.resumed_live_tree);
assert!(!control_exists(dir.path(), control::CAPABILITIES));
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
let caps = a.read_capabilities().unwrap();
assert_eq!(caps.protocol, super::SENTINEL_PROTOCOL);
assert_eq!(caps.state, "live");
assert!(caps.verbs.iter().any(|v| v == "publish"));
assert!(caps.verbs.iter().any(|v| v == "sync"));
}
#[tokio::test]
async fn agent_guide_names_every_advertised_verb() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
let caps = a.read_capabilities().unwrap();
assert!(!caps.verbs.is_empty(), "the fixture advertises verbs");
let guide = std::fs::read_to_string(dir.path().join(super::CONTROL_DIR).join(control::AGENT_GUIDE)).unwrap();
assert_eq!(guide, control::AGENT_GUIDE_TEXT);
for v in &caps.verbs {
let named = match v.as_str() {
"remote-seq" => ".flint/remote.seq",
other => &format!(".flint/{other}"),
};
assert!(guide.contains(named), "the guide never names the advertised verb {v} as {named}");
}
assert!(
guide.contains(&format!("sentinel protocol {}", super::SENTINEL_PROTOCOL)),
"the guide names a protocol other than the marker's {}",
super::SENTINEL_PROTOCOL
);
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("f.txt"));
assert!(m.manifest.entries.keys().all(|k| !k.starts_with(".flint/")), "the guide was published as data");
}
#[tokio::test]
async fn preexisting_flint_disables_sentinels() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
touch_sentinel(dir.path(), control::PUBLISH, "app-owned payload");
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
assert!(!posture.enabled);
assert_eq!(posture.reason.as_deref(), Some("preexisting-flint-paths"));
a.write_capabilities(&posture).unwrap();
let caps = a.read_capabilities().unwrap();
assert!(caps.verbs.is_empty());
assert_eq!(caps.reason.as_deref(), Some("preexisting-flint-paths"));
a.checkout().await.unwrap();
let acks = a.sentinel_tick().await.unwrap();
assert!(acks.is_empty());
assert_eq!(
std::fs::read_to_string(dir.path().join(".flint/publish")).unwrap(),
"app-owned payload"
);
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
let again = a.sentinel_preflight().unwrap();
assert!(!again.enabled);
}
pub(super) fn clear_min_interval(sc: &Syncer) {
let mut b = sc.load_budget().unwrap();
b.last_honor_unix = 0;
let bytes = serde_json::to_vec(&b).unwrap();
std::fs::write(sc.cfg.state_dir().join("sentinel-budget.json"), bytes).unwrap();
}
#[tokio::test]
async fn publish_sentinel_honored_and_acked() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
a.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &a.cfg).await.unwrap().map(|l| l.manifest.seq);
write(dir.path(), "model.json", "{}");
write(dir.path(), "model.json.index", "idx");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1","note":"step 1"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the sentinel was not honored");
let ack = &acks[0];
assert_eq!(ack.status, "ok");
assert_eq!(ack.boundary, "sentinel");
assert_eq!(ack.nonces, vec!["n-1".to_string()]);
assert!(ack.sentinel_mtime_unix_ns > 0);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("model.json"));
assert!(m.entries.contains_key("model.json.index"));
assert_eq!(ack.seq, Some(m.seq));
assert_ne!(before, Some(m.seq), "the manifest did not advance");
assert!(!control_exists(dir.path(), control::PUBLISH));
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
assert!(control_exists(dir.path(), control::PUBLISH_ACK));
assert_eq!(a.read_ack(Verb::Publish).unwrap().nonces, vec!["n-1".to_string()]);
}
#[tokio::test]
async fn a_publish_whose_delete_meets_a_peers_edit_is_ok_with_theirs_preserved() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir_a.path(), "shared.txt", "v1");
a.floor_tick().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "shared.txt", "B's v2");
backdate_baseline(&b, "shared.txt");
b.floor_tick().await.unwrap();
let theirs = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let b_etag = theirs.entries["shared.txt"].etag.clone();
assert_eq!(read(dir_a.path(), "shared.txt").as_deref(), Some("v1"), "fixture: B's edit already reached A");
std::fs::remove_file(dir_a.path().join("shared.txt")).unwrap();
touch_sentinel(dir_a.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(ack.seq, Some(m.seq), "fixture: the ack names a different seq than the one installed");
assert!(!m.entries.contains_key("shared.txt"), "the ack's seq still cites the deleted file: {ack:?}");
assert_eq!(ack.status, "ok", "{ack:?}");
assert!(ack.report.dropped.is_empty(), "{ack:?}");
let c = ack
.report
.conflicts
.iter()
.find(|c| c.kind.starts_with("commit-deleted-over-theirs") && c.path == "shared.txt")
.unwrap_or_else(|| panic!("no record of B's version on the ack: {ack:?}"));
assert_eq!(c.foreign_etag, b_etag);
let (_, body) = store.get_whole(c.preserved_key.as_ref().expect("B's edit preserved"), None).await.unwrap();
assert_eq!(&body[..], b"B's v2");
}
#[tokio::test]
async fn a_publish_whose_upload_was_swept_is_partial() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.whole_put_max = 8; let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
let big: String = (0..200).map(|i| format!("line {i}\n")).collect();
write(dir.path(), "model.bin", &big);
write(dir.path(), "notes.txt", "n");
store.inject_compose_swept();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(ack.seq, Some(m.seq), "fixture: the ack names a different seq than the one installed");
assert!(m.entries.contains_key("notes.txt"), "fixture: the boundary installed nothing");
assert!(!m.entries.contains_key("model.bin"), "fixture: the swept upload was cited anyway");
assert_eq!(ack.status, "partial", "the ack says model.bin is in seq {}, which does not cite it: {ack:?}", m.seq);
assert_eq!(ack.report.dropped, vec!["model.bin".to_string()]);
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-2"}"#);
a.poll_sentinels().unwrap();
clear_min_interval(&a);
let again = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
assert_eq!(again.status, "ok", "{again:?}");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(again.seq, Some(m.seq));
assert!(m.entries.contains_key("model.bin"), "the re-touch did not publish the swept file");
}
#[tokio::test]
async fn a_retried_honour_still_names_a_file_an_earlier_carry_deferred() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.whole_put_max = 8; let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
let big: String = (0..200).map(|i| format!("line {i}\n")).collect();
write(dir.path(), "model.bin", &big);
write(dir.path(), "notes.txt", "n");
store.inject_compose_swept();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
let squat = dir.path().join(super::CONTROL_DIR).join(control::PUBLISH_ACK);
std::fs::create_dir_all(squat.join("occupied")).unwrap();
clear_min_interval(&a);
assert!(a.honor_pending(Verb::Publish, false).await.is_err(), "fixture: the ack write did not fail");
std::fs::remove_dir_all(&squat).unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("notes.txt") && !m.entries.contains_key("model.bin"), "fixture: not the deferral");
std::fs::remove_file(dir.path().join("model.bin")).unwrap();
hitl_write(&store, &a.cfg, "model.bin", "the reviewer's model", "reviewer").await.unwrap();
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["model.bin"].crc64_b64, crc_of("the reviewer's model"), "fixture: the agent's model.bin was published");
assert_eq!(ack.status, "partial", "the ack says the declared model.bin is in seq {:?}: {ack:?}", ack.seq);
assert_eq!(ack.report.dropped, vec!["model.bin".to_string()], "{ack:?}");
}
#[tokio::test]
async fn an_honour_whose_only_upload_was_swept_is_partial_with_no_carrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.whole_put_max = 8;
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "s");
a.floor_tick().await.unwrap();
a.floor_tick().await.unwrap();
let big: String = (0..200).map(|i| format!("line {i}\n")).collect();
write(dir.path(), "model.bin", &big);
store.inject_compose_swept();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().expect("nothing was answered");
let carrier = a.state.load_intent().unwrap().carrier;
assert!(
carrier.as_ref().map_or(true, |c| !c.dropped.contains("model.bin")),
"fixture: a carrier names the deferral, so this does not isolate the barrier's own report: {carrier:?}"
);
assert_eq!(ack.status, "partial", "the ack says the swept model.bin is in seq {:?}: {ack:?}", ack.seq);
assert_eq!(ack.report.dropped, vec!["model.bin".to_string()], "{ack:?}");
}
#[tokio::test]
async fn a_drain_whose_upload_was_swept_does_not_attest() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.whole_put_max = 8; a.checkout().await.unwrap();
let big: String = (0..200).map(|i| format!("line {i}\n")).collect();
write(dir.path(), "model.bin", &big);
store.inject_compose_swept();
let drained = a.drain().await;
let cited = |m: Option<manifest::LeanManifest>| m.is_some_and(|m| m.entries.contains_key("model.bin"));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().map(|l| l.manifest);
assert!(!cited(m), "fixture: the swept upload was cited anyway");
assert!(drained.is_err(), "the drain attested a tree whose only copy of model.bin is on it: {drained:?}");
a.drain().await.expect("the retry did not drain");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().map(|l| l.manifest);
assert!(cited(m), "the retry did not publish model.bin");
}
#[tokio::test]
async fn a_drain_whose_delete_meets_a_peers_edit_attests_with_theirs_preserved() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "shared.txt", "v1");
a.floor_tick().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "shared.txt", "B's v2");
backdate_baseline(&b, "shared.txt");
b.floor_tick().await.unwrap();
std::fs::remove_file(dir_a.path().join("shared.txt")).unwrap();
a.drain().await.expect("the drain refused a delete that landed");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("shared.txt"), "the drain attested a tree whose delete is not in the bucket");
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.kind.starts_with("commit-deleted-over-theirs") && c.path == "shared.txt"),
"B's version was not preserved"
);
}
#[tokio::test]
async fn sentinel_ack_echoes_covered_nonces() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
for i in 0..5 {
touch_sentinel(dir.path(), control::PUBLISH, &format!(r#"{{"nonce":"n-{i}"}}"#));
a.poll_sentinels().unwrap();
}
let pending = a.load_pending(Verb::Publish).unwrap().unwrap();
assert_eq!(pending.nonces.len(), 5, "touches did not coalesce into one record");
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
for i in 0..5 {
assert!(
ack.nonces.contains(&format!("n-{i}")),
"nonce n-{i} was orphaned by coalescing: {:?}",
ack.nonces
);
}
assert!(ack.nonces.contains(&"n-2".to_string()));
}
#[tokio::test]
async fn min_interval_coalesces_into_one_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 3600; assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"first"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the first honor must be prompt");
let seq_after_first = acks[0].seq.unwrap();
write(dir.path(), "g.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"second"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert!(acks.is_empty(), "the min-interval did not hold the second honor");
assert_eq!(a.sentinel_due().unwrap(), Due::MinInterval);
assert_eq!(
a.load_pending(Verb::Publish).unwrap().unwrap().nonces,
vec!["second".to_string()]
);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.seq, seq_after_first, "a held honor still advanced the manifest");
let out = a.floor_tick().await.unwrap();
assert_eq!(out.acks.len(), 1);
assert_eq!(out.acks[0].nonces, vec!["second".to_string()]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("g.txt"));
}
#[tokio::test]
async fn budget_meters_bytes_not_calls() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.whole_put_max = 1024; a.cfg.sentinel_hourly_budget = 8; a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
let mut honors_large = 0;
let mut deferred_acks = 0;
for i in 0..6 {
write(dir.path(), "big.bin", &"x".repeat(4096 + i));
backdate_baseline(&a, "big.bin");
touch_sentinel(dir.path(), control::PUBLISH, &format!(r#"{{"nonce":"L{i}"}}"#));
let acks = a.sentinel_tick().await.unwrap();
honors_large += acks.len();
if acks.is_empty() && a.load_pending(Verb::Publish).unwrap().is_some() {
let out = a.floor_tick().await.unwrap();
deferred_acks += out
.acks
.iter()
.filter(|x| x.boundary == "sentinel-deferred")
.count();
}
}
assert!(
honors_large <= 3,
"a 4 KiB-over-ceiling storm was NOT throttled: {honors_large} prompt honors"
);
assert!(deferred_acks >= 1, "no ack was stamped sentinel-deferred");
assert_eq!(a.sentinel_due().unwrap(), Due::BudgetDeferred);
let store2 = Arc::new(MemoryStore::new());
let dir2 = tempfile::tempdir().unwrap();
let mut b = syncer(&store2, dir2.path()).await;
b.cfg.whole_put_max = 1024;
b.cfg.sentinel_hourly_budget = 8;
b.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut b, 3).await);
let posture = b.sentinel_preflight().unwrap();
b.write_capabilities(&posture).unwrap();
b.checkout().await.unwrap();
let mut honors_small = 0;
for i in 0..6 {
write(dir2.path(), "small.txt", &format!("v{i}"));
backdate_baseline(&b, "small.txt");
touch_sentinel(dir2.path(), control::PUBLISH, &format!(r#"{{"nonce":"S{i}"}}"#));
honors_small += b.sentinel_tick().await.unwrap().len();
}
assert_eq!(
honors_small, 6,
"the small-file storm was throttled — the meter is counting CALLS, not work"
);
assert!(b
.read_ack(Verb::Publish)
.map(|a| a.boundary == "sentinel")
.unwrap_or(false));
assert!(honors_small > honors_large);
}
#[tokio::test]
async fn no_diff_sentinel_honor_costs_no_budget() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_hourly_budget = 2;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
for i in 0..10 {
touch_sentinel(dir.path(), control::PUBLISH, &format!(r#"{{"nonce":"q{i}"}}"#));
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "a no-diff honor was throttled at touch {i}");
assert!(acks[0].report.no_change, "the tree was not actually quiet");
}
assert_eq!(a.load_budget().unwrap().spent(super::now_unix()), 0);
}
#[tokio::test]
async fn crash_between_consume_and_ack_reruns_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
a.run_barrier().await.unwrap();
let seq_before = a.state.load_baseline().unwrap().seq;
write(dir.path(), "late.txt", "written before the boundary");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"crashed"}"#);
a.poll_sentinels().unwrap();
assert!(a.load_pending(Verb::Publish).unwrap().is_some());
assert!(a.read_ack(Verb::Publish).is_none());
a.settle_pending_at_startup().await.unwrap();
let ack = a.read_ack(Verb::Publish).unwrap();
assert_eq!(ack.status, "ok");
assert!(ack.nonces.contains(&"crashed".to_string()));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(ack.seq, Some(m.seq));
assert!(m.seq > seq_before);
assert!(m.entries.contains_key("late.txt"));
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
}
#[tokio::test]
async fn restart_settles_pending_before_new_consume() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "a.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"old"}"#);
a.poll_sentinels().unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"new"}"#);
a.poll_sentinels().unwrap();
let pending = a.load_pending(Verb::Publish).unwrap().unwrap();
assert!(pending.nonces.contains(&"old".to_string()));
assert!(pending.nonces.contains(&"new".to_string()));
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
assert!(ack.nonces.contains(&"old".to_string()));
assert!(ack.nonces.contains(&"new".to_string()));
}
#[tokio::test]
async fn torn_pending_body_honored_as_bare_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"half-writ"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "a torn body wedged the verb");
assert_eq!(acks[0].status, "ok");
assert!(acks[0].nonces.is_empty());
assert!(acks[0].sentinel_mtime_unix_ns > 0, "the bare-touch mtime must still be covered");
assert!(a
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind == "sentinel-torn-body"));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("f.txt"));
}
#[tokio::test]
#[cfg(unix)]
async fn fifo_sentinel_skipped() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
let ctl = dir.path().join(super::CONTROL_DIR);
std::fs::create_dir_all(&ctl).unwrap();
let fifo = ctl.join(control::PUBLISH);
let cpath = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap();
let rc = unsafe { libc::mkfifo(cpath.as_ptr(), 0o644) };
assert_eq!(rc, 0, "could not create the FIFO fixture");
assert!(!std::fs::symlink_metadata(&fifo).unwrap().is_file());
let acks = a.sentinel_tick().await.unwrap();
assert!(acks.is_empty());
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
assert!(fifo.exists(), "the FIFO was consumed");
assert!(a
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind == "sentinel-not-regular-file"));
}
#[tokio::test]
async fn scoped_sync_preserves_out_of_scope_foreign_flow() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "inputs/data.txt", "v1");
write(dir_a.path(), "outputs/result.txt", "v1");
a.run_barrier().await.unwrap();
peer_publish(&store, &a.cfg, "inputs/data.txt", "foreign inputs v2", "sibling").await;
let (foreign_out_etag, _) =
peer_publish(&store, &a.cfg, "outputs/result.txt", "foreign outputs v2", "sibling").await;
assert_ne!(
foreign_out_etag,
a.state.load_baseline().unwrap().entries["outputs/result.txt"].etag
);
let r = a.sync_scoped(Some(vec!["inputs/".into()])).await.unwrap();
assert_eq!(read(dir_a.path(), "inputs/data.txt").unwrap(), "foreign inputs v2");
assert!(r.applied.contains(&"inputs/data.txt".to_string()));
assert_eq!(read(dir_a.path(), "outputs/result.txt").unwrap(), "v1");
assert!(r.out_of_scope_foreign >= 1);
assert_ne!(
a.state.load_baseline().unwrap().entries["outputs/result.txt"].etag,
foreign_out_etag,
"a scoped sync advanced the MERGE BASE for an out-of-scope path"
);
assert_eq!(r.seq, a.state.load_baseline().unwrap().seq);
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(
read(dir_a.path(), "outputs/result.txt").unwrap(),
"foreign outputs v2",
"the out-of-scope foreign change was LOST"
);
}
#[test]
fn scope_matches_on_component_boundary() {
let s = super::sync::Scope::new(&["in".to_string(), "inputs/".to_string()]);
assert!(s.covers("in"));
assert!(s.covers("in/x.txt"));
assert!(s.covers("inputs/a.txt"));
assert!(!s.covers("internal/secret.txt"));
assert!(!s.covers("inputsX/a.txt"));
let s = super::sync::Scope::new(&["../etc".to_string()]);
assert!(s.is_empty());
}
#[tokio::test]
#[cfg(unix)]
async fn write_file_atomic_refuses_symlink_escape() {
let outside = tempfile::tempdir().unwrap();
let secret = outside.path().join("secret.txt");
std::fs::write(&secret, "ORIGINAL CREDENTIALS").unwrap();
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
std::os::unix::fs::symlink(outside.path(), dir.path().join("inputs")).unwrap();
assert!(dir.path().join("inputs/secret.txt").exists());
let scanned = super::scan::scan(dir.path()).unwrap();
assert!(!scanned.keys().any(|k| k.starts_with("inputs")));
assert!(super::barrier::contained_path(dir.path(), "inputs/secret.txt").is_err());
assert!(super::barrier::contained_path(dir.path(), "../escape.txt").is_err());
assert!(super::barrier::contained_path(dir.path(), "/etc/passwd").is_err());
assert!(super::barrier::contained_path(dir.path(), ".flint/publish").is_err());
assert!(super::barrier::contained_path(dir.path(), "ok/nested/f.txt").is_ok());
hitl_write(&store, &a.cfg, "inputs/secret.txt", "ATTACKER PAYLOAD", "attacker")
.await
.unwrap();
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(
std::fs::read_to_string(&secret).unwrap(),
"ORIGINAL CREDENTIALS",
"the syncer wrote THROUGH a planted symlink, outside the workspace"
);
assert!(a
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind.starts_with("consume-refused-containment")));
}
#[tokio::test]
async fn sync_rehonor_no_phantom_conflicts() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "shared.txt", "foreign v2", "ci").await.unwrap();
let pre = a.state.load_baseline().unwrap();
let r = a.sync().await.unwrap();
assert!(r.applied.contains(&"shared.txt".to_string()));
let mut b = a.state.load_baseline().unwrap();
let stale = b.entries.get_mut("shared.txt").unwrap();
stale.etag = "\"stale-pre-sync-etag\"".into();
stale.mtime_unix -= 10;
stale.size = 1;
b.seq = pre.seq;
b.manifest_etag = pre.manifest_etag.clone();
a.state.save_baseline(&b).unwrap();
let scanned = super::scan::scan(dir.path()).unwrap();
let c = super::scan::classify(&scanned, &a.state.load_baseline().unwrap());
assert!(c.uploads.contains("shared.txt"), "the fixture is not actually dirty");
let r = a.sync().await.unwrap();
assert!(
!r.conflicts.contains(&"shared.txt".to_string()),
"a phantom conflict was reported for byte-identical content"
);
assert!(r.applied.contains(&"shared.txt".to_string()));
}
#[tokio::test]
async fn remote_seq_ticks_without_added_requests() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "v1");
a.floor_tick().await.unwrap();
let t0 = a.load_remote_seq();
assert!(t0.updated_unix > 0);
assert_eq!(t0.observed_seq, t0.integrated_seq, "no news, yet the ticker claims some");
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "foreign")
.await
.unwrap();
assert_eq!(t0.observed_seq, loaded.manifest.seq);
a.floor_tick().await.unwrap();
let t1 = a.load_remote_seq();
assert!(
t1.observed_seq >= theirs.seq,
"the ticker missed a foreign install: {} vs {}",
t1.observed_seq,
theirs.seq
);
assert!(t1.updated_unix >= t0.updated_unix);
}
#[tokio::test]
async fn remote_seq_reports_news_while_a_peers_change_waits_in_the_queue() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.floor_tick().await.unwrap();
b.checkout().await.unwrap();
b.floor_tick().await.unwrap();
let t0 = b.load_remote_seq();
assert_eq!(t0.observed_seq, t0.integrated_seq, "fixture: B has news before anyone wrote");
write(dir_b.path(), "z.txt", "z");
let (store2, cfg_a) = (store.clone(), a.cfg.clone());
super::barrier::CONSUME_WINDOW_HOOK.with(|h| {
*h.borrow_mut() = Some((
String::new(),
"before-scan",
Box::new(move || {
futures::executor::block_on(peer_publish(&store2, &cfg_a, "x.txt", "A's v2", "A"));
}),
))
});
b.floor_tick().await.unwrap();
assert!(super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()), "fixture: the window never ran");
assert!(derive_pending(&b).await, "fixture: B's merge left nothing owed for x.txt");
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"), "fixture: the change already reached B's tree");
let t1 = b.load_remote_seq();
assert!(
t1.observed_seq > t1.integrated_seq,
"B's tree lacks A's change, yet remote.seq says no news: {t1:?}"
);
b.floor_tick().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's v2"), "A's change never reached B on the floor");
let t2 = b.load_remote_seq();
assert_eq!(t2.observed_seq, t2.integrated_seq, "news that was integrated is still reported: {t2:?}");
}
#[tokio::test]
async fn sync_request_is_carried_never_executed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "agent bytes");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.entries.remove("keep.txt");
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "remote-delete")
.await
.unwrap();
let tree_before = std::fs::read_to_string(dir.path().join("keep.txt")).unwrap();
a.carry_sync_request(super::now_unix(), "ci@example").unwrap();
for _ in 0..3 {
let _ = a.sentinel_tick().await.unwrap();
}
let t = a.load_remote_seq();
assert_eq!(t.sync_requested_by.as_deref(), Some("ci@example"));
assert!(t.sync_requested_unix.is_some());
assert_eq!(
std::fs::read_to_string(dir.path().join("keep.txt")).unwrap(),
tree_before,
"the syncer acted on a remote's sync request"
);
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"mine"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1);
assert!(!dir.path().join("keep.txt").exists(), "the agent's own sync did not run");
}
#[tokio::test]
async fn a_stale_ack_never_retires_a_fresh_bare_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "w.txt", "one");
touch_sentinel(dir.path(), control::PUBLISH, "");
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the first bare touch was not honored");
let first = a.read_ack(Verb::Publish).expect("no ack from the first honor");
assert_eq!(first.status, "ok");
std::thread::sleep(std::time::Duration::from_millis(1100));
write(dir.path(), "w.txt", "two");
touch_sentinel(dir.path(), control::PUBLISH, "");
let path = dir.path().join(super::CONTROL_DIR).join(control::PUBLISH);
let backdated = std::time::SystemTime::UNIX_EPOCH
+ std::time::Duration::from_nanos((first.sentinel_mtime_unix_ns as u64).saturating_sub(5));
let f = std::fs::File::options().write(true).open(&path).unwrap();
f.set_times(std::fs::FileTimes::new().set_modified(backdated)).unwrap();
drop(f);
assert!(a.consume_sentinel(Verb::Publish).unwrap(), "the second touch was not consumed");
let pending = a.load_pending(Verb::Publish).unwrap().expect("no pending record");
assert!(pending.nonces.is_empty(), "the fixture needs a BARE touch");
assert!(
pending.consumed_mtime_unix_ns <= first.sentinel_mtime_unix_ns,
"the mtime did not go backwards — the fixture never armed"
);
assert!(
!a.ack_matches(Verb::Publish, &pending),
"a stale ack retired a fresh boundary — it will never run"
);
let ack = a
.honor_pending(Verb::Publish, true)
.await
.unwrap()
.expect("the second boundary never ran");
assert_eq!(ack.status, "ok");
let second = a.read_ack(Verb::Publish).expect("no ack from the second honor");
assert!(second.completed_unix > first.completed_unix, "the ack was not re-minted");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let e = &m.entries["w.txt"];
let (_, body) = store.get_whole(&e.key, Some(&e.etag)).await.unwrap();
assert_eq!(&body[..], b"two", "the second boundary was retired without publishing");
}
#[tokio::test]
async fn a_fifo_at_the_sentinel_path_never_wedges_the_poll_arm() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let ctl = dir.path().join(super::CONTROL_DIR);
std::fs::create_dir_all(&ctl).unwrap();
let path = ctl.join(control::PUBLISH);
let c = std::ffi::CString::new(path.to_str().unwrap()).unwrap();
assert_eq!(unsafe { libc::mkfifo(c.as_ptr(), 0o644) }, 0, "mkfifo failed");
assert!(
!std::fs::metadata(&path).unwrap().is_file(),
"the fixture never armed — that is not a FIFO"
);
let r = super::sentinel::read_bounded(&path);
assert!(r.is_err(), "a writer-less FIFO read as a body instead of erroring");
let target = ctl.join("elsewhere");
std::fs::write(&target, "not the agent's").unwrap();
let link = ctl.join("publish.link");
std::os::unix::fs::symlink(&target, &link).unwrap();
assert!(
super::sentinel::read_bounded(&link).is_err(),
"a symlink at the sentinel path was followed"
);
let ticked = tokio::time::timeout(std::time::Duration::from_secs(5), a.sentinel_tick()).await;
let acks = ticked.expect("THE POLL ARM WEDGED on a FIFO at the sentinel path").unwrap();
assert!(acks.is_empty(), "a FIFO was honored as a boundary");
let conflicts = a.state.load_conflicts().unwrap();
assert!(
conflicts.iter().any(|c| c.kind == "sentinel-not-regular-file"),
"the non-regular sentinel left no conflict record"
);
for _ in 0..5 {
a.sentinel_tick().await.unwrap();
}
let after = a.state.load_conflicts().unwrap();
assert_eq!(
after.iter().filter(|c| c.kind == "sentinel-not-regular-file").count(),
1,
"the standing condition was re-recorded on every poll tick"
);
std::fs::remove_file(&path).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, "");
a.sentinel_tick().await.unwrap();
let c = std::ffi::CString::new(path.to_str().unwrap()).unwrap();
assert_eq!(unsafe { libc::mkfifo(c.as_ptr(), 0o644) }, 0, "second mkfifo failed");
a.sentinel_tick().await.unwrap();
assert_eq!(
a.state.load_conflicts().unwrap().iter()
.filter(|c| c.kind == "sentinel-not-regular-file")
.count(),
2,
"a RECURRENCE of the condition went unrecorded — the latch never clears"
);
}
#[tokio::test]
async fn status_renders_without_claiming_the_lease() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "held.txt", "x");
a.floor_tick().await.unwrap();
let s = super::status_report(&cfg_for(dir.path())).unwrap();
assert!(s.gauges.is_some());
assert_eq!(s.capabilities.unwrap().state, "live");
assert!(s.baseline_seq >= 1, "the tick's boundary never reached the baseline status reads");
assert!(s.incarnation_epoch.is_some());
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released);
assert_eq!(
cell.epoch,
a.state.load_incarnation().unwrap().unwrap().epoch,
"status rotated the epoch it was supposed to observe"
);
}
#[tokio::test]
async fn a_sentinel_boundary_carries_a_delete_made_before_the_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "k");
write(dir.path(), "gone.txt", "v1");
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("gone.txt"), "fixture: never published");
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-del"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the sentinel was not honored");
assert_eq!(acks[0].status, "ok");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("gone.txt"),
"the ack said ok at seq {:?} while the manifest still cites a file the agent \
deleted before the touch (report.deleted = {})",
acks[0].seq,
acks[0].report.deleted
);
}
#[tokio::test]
async fn only_a_declared_barrier_confirms_a_first_absence() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "gone.txt", "v1");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
let r = a.run_barrier().await.unwrap();
assert_eq!(r.first_absence, vec!["gone.txt".to_string()]);
assert_eq!(r.absences_confirmed, 0);
assert!(r.deleted.is_empty(), "the cadence barrier published a first absence");
let mut b = a.state.load_baseline().unwrap();
b.prev_scan.insert("gone.txt".into());
a.state.save_baseline(&b).unwrap();
let r = a.declared_barrier().await.unwrap();
assert_eq!(r.absences_confirmed, 1, "the declared barrier did not confirm the absence");
assert!(r.first_absence.is_empty());
assert_eq!(r.deleted, vec!["gone.txt".to_string()]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("gone.txt"));
}
#[tokio::test]
async fn an_unreadable_path_is_not_confirmed_as_deleted() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
if unsafe { libc::geteuid() } == 0 {
eprintln!("SKIPPED: running as root, mode bits cannot induce EACCES");
return;
}
let locked = dir.path().join("locked");
std::fs::create_dir(&locked).unwrap();
std::fs::write(locked.join("f.txt"), b"still here").unwrap();
std::fs::set_permissions(&locked, std::os::unix::fs::PermissionsExt::from_mode(0o000)).unwrap();
let mut classified = super::scan::Classified::default();
classified.first_absence.insert("locked/f.txt".into());
let err = a
.confirm_absences(&mut classified)
.expect_err("an unreadable path must not confirm a deletion");
assert!(
format!("{err}").contains("refusing to publish a deletion"),
"the refusal must say what it refused: {err}"
);
assert!(
classified.deletes.is_empty(),
"NOTHING may be queued for deletion off an errno that is not NotFound: {:?}",
classified.deletes
);
std::fs::set_permissions(&locked, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
}
#[tokio::test]
async fn the_confirmation_never_deletes_a_path_the_walk_merely_missed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
write(dir.path(), "renamed.txt", "here all along");
let mut classified = super::scan::Classified::default();
classified.first_absence.insert("renamed.txt".into());
classified.first_absence.insert("truly-gone.txt".into());
let confirmed = a.confirm_absences(&mut classified).unwrap();
assert_eq!(confirmed, 1);
assert_eq!(
classified.deletes.iter().cloned().collect::<Vec<_>>(),
vec!["truly-gone.txt".to_string()]
);
assert!(
classified.first_absence.contains("renamed.txt"),
"the confirmation promoted a path that is on disk — the walk race would publish \
a delete of a live file"
);
}
#[tokio::test]
async fn the_drain_carries_a_delete_made_before_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "gone.txt", "v1");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
a.drain().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("gone.txt"),
"the drain left the delete withheld — the successor's checkout resurrects it"
);
}
#[tokio::test]
async fn a_crash_between_the_cas_and_step_7_never_makes_our_own_entry_foreign() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
write(dir.path(), "f.txt", "v2 is longer");
backdate_baseline(&a, "f.txt");
let stale = a.state.load_baseline().unwrap();
a.run_barrier().await.unwrap();
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
a.state.save_baseline(&stale).unwrap();
assert_ne!(
stale.entries.get("f.txt").map(|e| &e.etag),
Some(&installed.manifest.entries["f.txt"].etag),
"fixture: the baseline is not actually behind the install"
);
std::fs::remove_file(dir.path().join("f.txt")).unwrap();
let r = a.declared_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("f.txt"),
"our own install read as a foreign change and swallowed the agent's delete \
(deleted={:?}, foreign_queued={})",
r.deleted,
r.foreign_queued
);
}
#[tokio::test]
async fn a_pending_sync_at_sigterm_never_cancels_the_drains_own_boundary() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
a.run_barrier().await.unwrap();
write(dir.path(), "work.txt", "W1");
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"s-1"}"#);
a.poll_sentinels().unwrap();
assert!(a.load_pending(Verb::Sync).unwrap().is_some());
let acks = a.drain().await.unwrap();
assert_eq!(acks.len(), 1, "the drain did not settle the owed sync ack");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
m.entries.contains_key("work.txt"),
"the drain skipped its cite-everything pass because a SYNC ack carried a seq — \
every byte since the last boundary died with the emptyDir"
);
}
#[tokio::test]
async fn a_planted_temp_sibling_is_never_written_through() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let outside = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
a.run_barrier().await.unwrap();
let victims = ["consume-victim", "ticker-victim", "state-victim"];
for v in victims {
std::fs::write(outside.path().join(v), "ORIGINAL").unwrap();
}
std::fs::create_dir_all(dir.path().join("inputs")).unwrap();
std::os::unix::fs::symlink(
outside.path().join("consume-victim"),
dir.path().join("inputs/config.json.flint-sync-tmp"),
)
.unwrap();
std::fs::create_dir_all(dir.path().join(super::CONTROL_DIR)).unwrap();
std::os::unix::fs::symlink(
outside.path().join("ticker-victim"),
dir.path().join(super::CONTROL_DIR).join("remote.seq.tmp"),
)
.unwrap();
std::os::unix::fs::symlink(
outside.path().join("state-victim"),
a.cfg.state_dir().join("baseline.tmp"),
)
.unwrap();
hitl_write(&store, &a.cfg, "inputs/config.json", "REMOTE-BYTES", "ui").await.unwrap();
a.run_barrier().await.unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-tmp"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the fixture never exercised the control writer");
for v in victims {
assert_eq!(
std::fs::read_to_string(outside.path().join(v)).unwrap(),
"ORIGINAL",
"the syncer wrote through a planted temp sibling ({v}) — an arbitrary-file-write \
primitive outside the workspace, with the bucket credentials"
);
}
assert_eq!(read(dir.path(), "inputs/config.json").as_deref(), Some("REMOTE-BYTES"));
assert!(control_exists(dir.path(), "remote.seq"));
}
#[tokio::test]
async fn a_barrier_echoes_the_running_binary_into_the_lease_cell() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
let cited = a.run_barrier().await.unwrap().seq.unwrap();
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released, "a barrier must hand the cell on when it is done");
let echo: flint_store::LeaseEcho = serde_json::from_str(
cell.echo.as_deref().expect("the handoff carried no observed-state echo"),
)
.expect("the echo is not a LeaseEcho");
assert_eq!(echo.last_cited_seq, cited, "the echo names no citation");
assert_eq!(echo.protocol, super::SENTINEL_PROTOCOL);
assert!(!echo.syncer_version.is_empty(), "no version ⇒ no mixed-fleet tell");
}
#[tokio::test]
async fn a_uds_boundary_and_a_file_sentinel_coalesce_into_one_ack() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
a.cfg.sentinel_min_interval_secs = 0;
write(dir.path(), "work.txt", "W1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"file:1"}"#);
a.poll_sentinels().unwrap();
a.request_boundary("uds:1", Some("from the socket".into())).unwrap();
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the two doors produced {} barriers", acks.len());
let ack = &acks[0];
assert_eq!(ack.status, "ok");
for n in ["uds:1", "file:1"] {
assert!(
ack.nonces.iter().any(|x| x == n),
"the ack does not cover {n}: {:?}",
ack.nonces
);
}
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("work.txt"), "the coalesced boundary published nothing");
write(dir.path(), "work2.txt", "W2");
a.request_boundary("uds:2", None).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"file:2"}"#);
a.poll_sentinels().unwrap();
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1);
for n in ["uds:2", "file:2"] {
assert!(acks[0].nonces.iter().any(|x| x == n), "socket-first lost {n}");
}
}
#[tokio::test]
async fn a_gateway_boundary_request_becomes_a_pending_sentinel_with_no_conflict() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
inbox::gateway_request(
store.as_ref(),
&a.cfg,
inbox::RequestedVerb::Boundary,
"ci@example",
)
.await
.unwrap();
write(dir.path(), "work.txt", "W1");
a.read_cell_requests().await.unwrap();
let pending = a.load_pending(super::sentinel::Verb::Publish).unwrap();
let pending = pending.expect("the gateway request minted no pending sentinel");
assert!(
pending.nonces.iter().any(|n| n.contains("ci@example")),
"the requestor is not in the covered nonces: {:?}",
pending.nonces
);
assert_eq!(
std::fs::read_to_string(a.cfg.state_dir().join("conflicts.jsonl")).unwrap_or_default(),
"",
"the boundary request minted a conflict record"
);
let before = pending.nonces.len();
a.read_cell_requests().await.unwrap();
assert_eq!(
a.load_pending(super::sentinel::Verb::Publish).unwrap().unwrap().nonces.len(),
before,
"the same gateway request was consumed twice"
);
}
#[tokio::test]
async fn a_gateway_sync_request_is_carried_and_never_executed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "LOCAL");
a.run_barrier().await.unwrap();
let mut m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
m.manifest.entries.remove("keep.txt");
m.manifest.seq += 1;
manifest::cas_write(store.as_ref(), &a.cfg, &m.manifest, Some(&m.handle()), 9, "foreign")
.await
.unwrap();
let tree_before = read(dir.path(), "keep.txt");
inbox::gateway_request(store.as_ref(), &a.cfg, inbox::RequestedVerb::Sync, "ci@example")
.await
.unwrap();
a.read_cell_requests().await.unwrap();
assert_eq!(
read(dir.path(), "keep.txt"),
tree_before,
"the syncer EXECUTED a sync on a remote's say-so and rewrote the agent's tree"
);
assert!(tree_before.is_some(), "the fixture had nothing to lose");
let t: control::RemoteSeq = serde_json::from_slice(
&std::fs::read(dir.path().join(".flint/remote.seq")).unwrap(),
)
.unwrap();
assert!(t.sync_requested_unix.is_some(), "the request was not carried to the agent");
assert_eq!(t.sync_requested_by.as_deref(), Some("ci@example"));
assert!(a.load_pending(super::sentinel::Verb::Publish).unwrap().is_none());
}
#[test]
fn every_gauges_field_reaches_exactly_one_metric() {
let g = super::Gauges {
rpo_secs: 11,
withheld_reason: Some("parked-412".into()),
sentinel_budget_remaining: 44,
last_boundary: Some(super::gauges::LastBoundary {
source: "sentinel".into(),
seq: 66,
unix: 1_756_000_000,
}),
updated_unix: 1_756_000_100,
last_durable_unix: 1_756_000_050,
auth_paused_since_unix: Some(1_756_000_010),
};
let json = serde_json::to_value(&g).unwrap();
let fields: Vec<String> = json.as_object().unwrap().keys().cloned().collect();
assert!(fields.len() >= 7, "the fixture did not populate the struct: {fields:?}");
for f in &fields {
assert!(
super::metrics::COVERED_FIELDS.contains(&f.as_str()),
"gauges field {f:?} reaches no metric — an operator reading /metrics cannot see \
what an operator reading gauges.json can"
);
}
for f in super::metrics::COVERED_FIELDS {
assert!(
fields.iter().any(|x| x == f),
"metric table names {f:?}, which is not a gauges field any more"
);
}
let text = super::metrics::render(
&g,
&super::metrics::Labels { workspace: "proj1".into(), namespace: "agents".into() },
);
for (name, want) in [
("flint_lean_rpo_seconds", 11u64),
("flint_lean_sentinel_budget_remaining", 44),
("flint_lean_last_boundary_seq", 66),
("flint_lean_withheld_reason", 1), ("flint_lean_last_boundary_source", 1), ] {
let line = text
.lines()
.find(|l| l.starts_with(name) && !l.starts_with('#'))
.unwrap_or_else(|| panic!("no series for {name}"));
let got: u64 = line.rsplit(' ').next().unwrap().parse().unwrap();
assert_eq!(got, want, "{name} disagrees with the gauges it renders");
}
}
#[test]
fn the_label_key_set_is_exactly_workspace_and_namespace() {
let g = super::Gauges { rpo_secs: 1, ..Default::default() };
let text = super::metrics::render(
&g,
&super::metrics::Labels { workspace: "proj1".into(), namespace: "agents".into() },
);
let mut series = 0;
for line in text.lines().filter(|l| !l.starts_with('#')) {
let labels = line
.split_once('{')
.and_then(|(_, r)| r.split_once('}'))
.map(|(l, _)| l)
.unwrap_or_else(|| panic!("unlabelled series: {line}"));
let keys: Vec<&str> =
labels.split(',').map(|kv| kv.split_once('=').unwrap().0).collect();
assert_eq!(
keys,
vec!["workspace", "namespace"],
"series carries label keys beyond the fixed set: {line}"
);
series += 1;
}
assert!(series >= 8, "the renderer emitted almost nothing: {series}");
}
#[tokio::test]
async fn a_scrape_costs_no_bucket_request() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "work.txt", "W1");
a.run_barrier().await.unwrap();
let g = a.write_gauges(None).unwrap();
let before = store.list("").await.unwrap().len();
for _ in 0..25 {
let text = super::metrics::render(
&g,
&super::metrics::Labels { workspace: "w".into(), namespace: "n".into() },
);
assert!(text.contains("flint_lean_rpo_seconds"));
}
assert_eq!(
store.list("").await.unwrap().len(),
before,
"25 scrapes changed the bucket"
);
}
#[tokio::test]
async fn status_reports_a_standing_pending_sentinel() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let before = super::status_report(&a.cfg).unwrap();
assert!(before.pending_sentinels.is_empty(), "the fixture started with one pending");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"s-1"}"#);
a.poll_sentinels().unwrap();
assert!(
a.load_pending(super::sentinel::Verb::Publish).unwrap().is_some(),
"the fixture consumed nothing — the status field would be empty either way"
);
let r = super::status_report(&a.cfg).unwrap();
assert_eq!(
r.pending_sentinels,
vec!["publish".to_string()],
"status cannot see the pending record it exists to report"
);
}
#[tokio::test]
async fn a_resumed_checkout_never_adopts_a_stale_generation() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let dir2 = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "big.bin", "GENERATION-ONE");
a.run_barrier().await.unwrap();
write(dir2.path(), "big.bin", "GENERATION-ONE");
write(dir.path(), "big.bin", "GENERATION-TWO-IS-LONGER");
a.run_barrier().await.unwrap();
let cited = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert_eq!(
cited.manifest.entries["big.bin"].size,
"GENERATION-TWO-IS-LONGER".len() as u64,
"the fixture never advanced the manifest"
);
lease::release(&mut a).await.unwrap();
drop(a);
let mut b = syncer(&store, dir2.path()).await;
assert!(claim_until_held(&mut b, 8).await);
assert!(
!b.state.marker_present(),
"the fixture left a completion marker — this is the RESUME row"
);
b.checkout().await.unwrap();
let local = read(dir2.path(), "big.bin").unwrap();
assert_eq!(
local, "GENERATION-TWO-IS-LONGER",
"the resumed checkout adopted the STALE generation: the workspace holds {local:?} while \
the manifest cites the newer bytes, and the baseline claims they are the same — so the \
scan will never upload it and a sync will never re-fetch it"
);
}
#[tokio::test]
async fn every_boundary_says_which_clock_installed_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "one");
sc.floor_tick().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("a.txt"), "the fixture published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("cadence"),
"a cadence boundary does not say it was cadence"
);
write(dir.path(), "b.txt", "two");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n1"}"#);
sc.sentinel_tick().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("b.txt"), "the sentinel published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("sentinel"),
"a sentinel boundary does not say it was a sentinel"
);
write(dir.path(), "c.txt", "three");
sc.drain().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("c.txt"), "the drain published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("drain"),
"a drain boundary does not say it was a drain"
);
}
#[tokio::test]
async fn a_drained_sentinel_names_the_same_clock_in_the_ack_and_in_the_bucket() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "late.txt", "written before the drain");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"owed"}"#);
sc.poll_sentinels().unwrap();
assert!(
sc.load_pending(Verb::Publish).unwrap().is_some(),
"no pending record — the drain would not take the owed-ack arm"
);
let acks = sc.drain().await.unwrap();
let ack = acks.iter().find(|a| a.nonces.iter().any(|n| n == "owed")).expect("owed ack unsettled");
assert_eq!(ack.boundary, "drain");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("late.txt"), "the drain published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("drain"),
"the ack says drain and the bucket says something else"
);
}
#[tokio::test]
async fn sync_refuses_to_clobber_a_locally_dirty_path_and_says_so() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "dirty.txt", "published v1");
write(dir.path(), "clean.txt", "published v1");
a.run_barrier().await.unwrap();
peer_publish(&store, &a.cfg, "dirty.txt", "foreign bytes for dirty", "sibling").await;
peer_publish(&store, &a.cfg, "clean.txt", "foreign bytes for clean", "sibling").await;
write(dir.path(), "dirty.txt", "the agent's own unpublished work, longer");
let r = a.sync().await.unwrap();
assert_eq!(
read(dir.path(), "clean.txt").unwrap(),
"foreign bytes for clean",
"the control path was not applied — this sync did nothing at all"
);
assert!(r.applied.contains(&"clean.txt".to_string()));
assert_ne!(
read(dir.path(), "dirty.txt").unwrap(),
"foreign bytes for dirty",
"local and remote bytes are identical — nothing was in conflict"
);
assert_eq!(
read(dir.path(), "dirty.txt").unwrap(),
"the agent's own unpublished work, longer",
"sync CLOBBERED work the agent had not published"
);
assert!(
r.conflicts.contains(&"dirty.txt".to_string()),
"the conflict was not surfaced in the report: {:?}",
r.conflicts
);
let conflicts = a.state.load_conflicts().unwrap();
assert!(
conflicts.iter().any(|c| c.kind == "sync-dirty" && c.path == "dirty.txt"),
"no sync-dirty record was written: {conflicts:?}"
);
}
#[test]
fn merge_names_the_deletes_mine_publishes_over() {
fn e(etag: &str) -> manifest::LeanEntry {
manifest::LeanEntry {
key: "tenant/proj1/files/p.txt".into(),
etag: etag.into(),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 3,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
}
}
let base: std::collections::BTreeMap<String, String> =
[("gone.txt", "e1"), ("tomb.txt", "e1"), ("kept.txt", "e1")].iter().map(|(p, t)| (p.to_string(), t.to_string())).collect();
let mut theirs = manifest::LeanManifest::default();
theirs.entries.insert("kept.txt".into(), e("e1"));
theirs.tombstones.insert("tomb.txt".into(), manifest::Tombstone { etag: "e2".into(), seq: 3 });
let upserts: std::collections::BTreeMap<String, manifest::LeanEntry> =
["gone.txt", "tomb.txt", "kept.txt", "new.txt"].iter().map(|p| (p.to_string(), e("mine"))).collect();
let out = manifest::merge(&base, &theirs, &upserts, &Default::default(), &Default::default());
let mut got = out.recreated.clone();
got.sort();
assert_eq!(got, vec![("gone.txt".to_string(), "e1".to_string()), ("tomb.txt".to_string(), "e2".to_string())]);
}
#[tokio::test]
async fn merge_applies_a_local_delete_and_names_the_version_it_deleted_over() {
fn entry(etag: &str) -> manifest::LeanEntry {
manifest::LeanEntry {
key: "tenant/proj1/files/p.txt".into(),
etag: etag.into(),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 3,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
}
}
use manifest::DeleteOutcome::{Applied, OverTheirs, Parked};
#[allow(clippy::type_complexity)]
let cases: Vec<(&str, Option<&str>, Option<&str>, bool, bool, manifest::DeleteOutcome, Option<&str>)> = vec![
("theirs unchanged since our base", Some("e1"), Some("e1"), false, false, Applied, None),
("FOREIGN MODIFY under our delete", Some("e2"), Some("e1"), false, false, OverTheirs, Some("e2")),
("FOREIGN ADD under our delete", Some("e2"), None, false, false, OverTheirs, Some("e2")),
("absent from both", None, None, false, false, Applied, None),
("already deleted by someone else", None, Some("e1"), false, false, Applied, None),
("parked: never resolved this pass", Some("e1"), Some("e1"), true, true, Parked, None),
];
for (name, theirs_etag, base_etag, is_parked, must_survive, expected, named) in cases {
let mut theirs = manifest::LeanManifest::default();
if let Some(e) = theirs_etag {
theirs.entries.insert("p.txt".to_string(), entry(e));
}
let mut base = std::collections::BTreeMap::new();
if let Some(b) = base_etag {
base.insert("p.txt".to_string(), b.to_string());
}
let mut deletes = std::collections::BTreeSet::new();
deletes.insert("p.txt".to_string());
let mut parked = std::collections::BTreeSet::new();
if is_parked {
parked.insert("p.txt".to_string());
}
assert_eq!(theirs.entries.contains_key("p.txt"), theirs_etag.is_some(), "{name}");
assert_eq!(base.contains_key("p.txt"), base_etag.is_some(), "{name}");
let out = manifest::merge(&base, &theirs, &Default::default(), &deletes, &parked);
assert_eq!(out.deletes.get("p.txt"), Some(&expected), "{name}: reported outcome");
assert_eq!(out.doc.entries.contains_key("p.txt"), must_survive, "{name}: the entry");
let over: Vec<&str> = out.over_theirs.iter().map(|(_, e)| e.etag.as_str()).collect();
assert_eq!(over, named.into_iter().collect::<Vec<_>>(), "{name}: the version named over");
if !must_survive && theirs_etag.is_some() {
assert_eq!(
out.doc.tombstones.get("p.txt").map(|t| t.etag.as_str()),
theirs_etag,
"{name}: the tombstone names what the delete retired"
);
}
}
}
#[tokio::test]
async fn a_resumed_checkout_adopts_identical_bytes_and_refetches_same_size_impostors() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "same.txt", "these bytes never move");
write(dir.path(), "impostor.txt", "AAAA");
a.run_barrier().await.unwrap();
peer_publish(&store, &a.cfg, "impostor.txt", "BBBB", "sibling").await;
let theirs = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
theirs.entries["impostor.txt"].size,
std::fs::metadata(dir.path().join("impostor.txt")).unwrap().len(),
"the impostor differs in SIZE — this leg would pass without any crc at all"
);
assert_eq!(read(dir.path(), "impostor.txt").unwrap(), "AAAA");
std::fs::remove_file(dir.path().join(".flint-sync/checkout-complete")).unwrap();
let r = a.checkout().await.unwrap();
assert!(
r.skipped_present >= 1,
"nothing was adopted — the leg degenerates into 'checkout re-fetches everything'"
);
assert_eq!(
read(dir.path(), "same.txt").unwrap(),
"these bytes never move",
"an identical file was not left alone"
);
assert_eq!(
read(dir.path(), "impostor.txt").unwrap(),
"BBBB",
"checkout ADOPTED a same-size file whose bytes are not the cited bytes — \
the baseline now attests content the workspace does not hold"
);
}
#[tokio::test]
async fn a_takeover_rotation_carries_the_boundary_stamp_with_the_document() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
sc.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
manifest::cas_write_stamped(
store.as_ref(),
&sc.cfg,
&loaded.manifest,
Some(&loaded.handle()),
1,
"u-cited",
Some("sentinel"),
)
.await
.unwrap();
let lp = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(
lp.pointer.boundary_source.as_deref(),
Some("sentinel"),
"precondition: the cited manifest names its clock on the pointer"
);
manifest::rotate_for_takeover(store.as_ref(), &sc.cfg, 2).await.unwrap().unwrap();
let lp = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let doc = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(
lp.pointer.boundary_source.as_deref(),
Some("sentinel"),
"rotation must not drop the boundary stamp: a pointer reader would report an \
unknown clock for a workspace whose document still says `sentinel`"
);
assert_eq!(
lp.pointer.boundary_source, doc.manifest.boundary_source,
"the pointer and the document a reader assembles from it must never disagree"
);
assert_eq!(lp.pointer.seq, doc.manifest.seq, "the pointer's seq IS the document's seq");
}
fn entry(name: &str) -> manifest::LeanEntry {
manifest::LeanEntry {
key: format!("{PREFIX}/files/{name}"),
etag: "e".into(),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 1,
mode: 0o100644,
mtime_unix: 0,
generation: 1,
epoch: 1,
}
}
#[tokio::test]
async fn a_publish_writes_an_immutable_generation_and_a_pointer_that_names_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 1;
m.entries.insert("a.txt".into(), entry("x"));
manifest::cas_write(store.as_ref(), &cfg, &m, None, 3, "first").await.unwrap();
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(lp.pointer.seq, 1);
assert_eq!(lp.pointer.entries_seq, Some(1));
assert_eq!(lp.pointer.entries_key, lp.pointer.entries_key.clone());
assert!(store.head(lp.pointer.entries_key.as_deref().unwrap()).await.is_ok());
assert!(store.head(&cfg.manifest_key()).await.is_err());
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(loaded.manifest.seq, 1);
assert!(loaded.manifest.entries.contains_key("a.txt"));
assert!(!loaded.handle().legacy);
}
#[tokio::test]
async fn a_takeover_rotation_does_not_touch_the_entries_object() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 4;
for i in 0..50 {
m.entries.insert(format!("f{i:03}.txt"), entry("x"));
}
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "seed").await.unwrap();
let gen4 = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
let before = store.head(&gen4).await.unwrap();
let (rotated, _) = manifest::rotate_for_takeover(store.as_ref(), &cfg, 2).await.unwrap().unwrap();
assert_eq!(rotated.seq, 5, "rotation must bump the generation");
let after = store.head(&gen4).await.unwrap();
assert_eq!(before.etag, after.etag, "the entries object was rewritten by a rotation");
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(lp.pointer.seq, 5);
assert_eq!(lp.pointer.entries_seq, Some(4), "entries_seq must NOT move — a follower reads it to skip the GET");
assert_eq!(lp.pointer.entries_key.as_deref(), Some(gen4.as_str()));
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(loaded.manifest.entries.len(), 50);
assert_eq!(loaded.manifest.seq, 5, "the POINTER is the authority for seq, not the entries object");
}
#[tokio::test]
async fn migration_installs_the_pointer_and_poisons_the_legacy_key() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let mut old = manifest::LeanManifest::default();
old.seq = 7;
old.entries.insert("kept.txt".into(), entry("x"));
let bytes = old.to_bytes();
let crc = flint_store::crc64_nvme(&bytes);
let stamps = flint_store::GenerationStamps {
generation: 7,
epoch: 1,
flush_uuid: "legacy".into(),
boundary_source: None,
posix: None,
};
store
.put_whole(&cfg.manifest_key(), bytes.into(), &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert!(loaded.handle().legacy, "an un-migrated workspace must be recognised as legacy");
assert_eq!(loaded.manifest.seq, 7);
let mut next = loaded.manifest.clone();
next.seq += 1;
next.entries.insert("added.txt".into(), entry("x"));
manifest::cas_write(store.as_ref(), &cfg, &next, Some(&loaded.handle()), 2, "migrate")
.await
.unwrap();
let after = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert!(!after.handle().legacy);
assert_eq!(after.manifest.entries.len(), 2, "migration must not lose entries");
let (_, poisoned) = store.get_whole(&cfg.manifest_key(), None).await.unwrap();
assert!(
manifest::LeanManifest::parse(&poisoned).is_err(),
"the legacy key still parses as a manifest — an old syncer would serve a stale project from it"
);
assert!(String::from_utf8_lossy(&poisoned).contains("upgrade flint-sync"));
}
#[tokio::test]
async fn a_pointer_naming_a_missing_generation_refuses_rather_than_reading_empty() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 2;
m.entries.insert("a.txt".into(), entry("x"));
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "seed").await.unwrap();
let gen2 = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
store.delete(&gen2).await.unwrap();
match manifest::load(store.as_ref(), &cfg).await {
Err(LeanError::State(msg)) => {
assert!(msg.contains("does not exist"), "unexpected message: {msg}");
}
Ok(None) => panic!("a broken pointer read as an EMPTY workspace — the next barrier would re-seed over it"),
Ok(Some(_)) => panic!("a broken pointer read as a LOADED manifest"),
Err(e) => panic!("expected a State refusal, got {e:?}"),
}
}
#[tokio::test]
async fn a_second_writer_at_the_same_generation_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 1;
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "a").await.unwrap();
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
let mut mine = loaded.manifest.clone();
mine.seq += 1;
manifest::cas_write(store.as_ref(), &cfg, &mine, Some(&loaded.handle()), 1, "b").await.unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
let err = manifest::cas_write(store.as_ref(), &cfg, &theirs, Some(&loaded.handle()), 1, "c")
.await
.unwrap_err();
assert!(
matches!(err, LeanError::Store(flint_store::StoreError::PreconditionFailed(_))),
"expected a precondition failure, got {err:?}"
);
}
#[tokio::test]
async fn superseded_generations_are_reaped_but_the_live_one_and_a_window_survive() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let prefix = format!("{PREFIX}/.flint/lean/manifests/");
let mut handle = None;
for seq in 1..=12u64 {
let mut m = manifest::LeanManifest::default();
m.seq = seq;
m.entries.insert("a.txt".into(), entry("a.txt"));
let meta = manifest::cas_write(
store.as_ref(),
&cfg,
&m,
handle.as_ref(),
1,
&format!("flush-{seq}"),
)
.await
.unwrap();
handle = Some(manifest::ManifestHandle { etag: meta.etag, legacy: false, prev_chunks: Vec::new(), prev_tombstones: None });
}
assert_eq!(store.list(&prefix).await.unwrap().len(), 12);
let removed = manifest::sweep_generations(store.as_ref(), &cfg).await.unwrap();
let left = store.list(&prefix).await.unwrap();
assert_eq!(left.len(), manifest::KEEP_GENERATIONS + 1);
assert_eq!(removed, 12 - (manifest::KEEP_GENERATIONS + 1));
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
assert!(
left.iter().any(|o| Some(&o.key) == lp.pointer.entries_key.as_ref()),
"the reaper deleted the generation the pointer names"
);
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(loaded.manifest.seq, 12);
assert!(loaded.manifest.entries.contains_key("a.txt"));
}
#[tokio::test]
async fn the_reaper_collects_an_orphan_from_a_crash_before_the_pointer_cas() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 1;
m.entries.insert("a.txt".into(), entry("a.txt"));
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "live").await.unwrap();
let live = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
let bytes = m.to_bytes();
let crc = flint_store::crc64_nvme(&bytes);
let stamps = flint_store::GenerationStamps {
generation: 99,
epoch: 1,
flush_uuid: "crashed".into(),
boundary_source: None,
posix: None,
};
let orphan = cfg.generation_key(99, "crashed");
store.put_whole(&orphan, bytes.into(), &PutCondition::IfNoneMatchAny, &stamps, crc).await.unwrap();
assert_eq!(
manifest::sweep_generations(store.as_ref(), &cfg).await.unwrap(),
0,
"the sweep reaped a generation above the pointer that could still be an in-flight publish"
);
assert!(store.head(&orphan).await.is_ok());
store.backdate_epoch(&orphan, manifest::ORPHAN_GRACE_SECS + 60);
assert_eq!(manifest::sweep_generations(store.as_ref(), &cfg).await.unwrap(), 1);
let left: Vec<String> = store
.list(&format!("{PREFIX}/.flint/lean/manifests/"))
.await
.unwrap()
.into_iter()
.map(|o| o.key)
.collect();
assert!(!left.contains(&orphan), "the orphan survived the sweep");
assert!(left.contains(&live), "the LIVE generation was reaped");
assert!(manifest::load(store.as_ref(), &cfg).await.unwrap().is_some());
}
#[tokio::test]
async fn a_rotation_reads_and_writes_no_generation_object() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 3;
for i in 0..200 {
m.entries.insert(format!("f{i:04}.txt"), entry("f"));
}
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "seed").await.unwrap();
let gen = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
let before = store.head(&gen).await.unwrap();
store.reset_op_counts();
manifest::rotate_for_takeover(store.as_ref(), &cfg, 2).await.unwrap().unwrap();
let ops = store.op_counts();
assert!(
store.total_ops() <= 3,
"a rotation should be a couple of small requests, not {ops:?}"
);
let after = store.head(&gen).await.unwrap();
assert_eq!(before.etag, after.etag);
assert_eq!(
store.list(&format!("{PREFIX}/.flint/lean/manifests/")).await.unwrap().len(),
1,
"a rotation wrote a new generation object — it must reuse the standing one"
);
}
struct AuthRefusing {
inner: Arc<MemoryStore>,
refuse: std::sync::atomic::AtomicBool,
refuse_reads: std::sync::atomic::AtomicBool,
}
impl AuthRefusing {
fn new(inner: Arc<MemoryStore>) -> Self {
Self {
inner,
refuse: std::sync::atomic::AtomicBool::new(false),
refuse_reads: std::sync::atomic::AtomicBool::new(false),
}
}
fn set_refuse(&self, v: bool) {
self.refuse.store(v, std::sync::atomic::Ordering::SeqCst);
}
fn set_refuse_reads(&self, v: bool) {
self.refuse_reads.store(v, std::sync::atomic::Ordering::SeqCst);
}
}
#[async_trait::async_trait]
impl ObjectStore for AuthRefusing {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
if self.refuse_reads.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
if self.refuse_reads.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
if self.refuse_reads.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
if self.refuse_reads.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
if self.refuse.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_refused_credential_pauses_the_holder_without_fencing_it() {
let inner = Arc::new(MemoryStore::new());
let proxy = Arc::new(AuthRefusing::new(inner.clone()));
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store: proxy.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
cell_written_at: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_none(),
"a healthy holder started out reading as credential-paused"
);
proxy.set_refuse(true);
let e = lease::renew(&mut a).await.unwrap_err();
assert!(e.is_auth(), "a 403 renewal did not classify as a credential fault: {e}");
assert!(
!matches!(e, LeanError::Fenced(_)),
"a credential refusal self-fenced a live writer: {e}"
);
assert!(a.lease.is_some(), "the refusal dropped the lease");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_some(),
"the refusal left no local evidence at all"
);
let gp = a.cfg.state_dir().join("gauges.json");
let mut g: serde_json::Value = serde_json::from_slice(&std::fs::read(&gp).unwrap()).unwrap();
g["auth_paused_since_unix"] = serde_json::json!(1_000_u64);
std::fs::write(&gp, serde_json::to_vec(&g).unwrap()).unwrap();
lease::renew(&mut a).await.unwrap_err();
assert_eq!(
a.load_gauges().unwrap().auth_paused_since_unix,
Some(1_000),
"a second refusal slid the pause clock forward, so the gauge no \
longer answers the question it exists to answer"
);
a.write_gauges(None).unwrap();
assert_eq!(
a.load_gauges().unwrap().auth_paused_since_unix,
Some(1_000),
"an ordinary gauge tick erased the credential pause"
);
proxy.set_refuse(false);
lease::renew(&mut a).await.expect("renew after the credentials came back");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_none(),
"the pause outlived the credentials being restored"
);
}
#[tokio::test]
async fn an_idle_writer_records_and_clears_a_credential_pause_on_its_floor_tick() {
let inner = Arc::new(MemoryStore::new());
let proxy = Arc::new(AuthRefusing::new(inner.clone()));
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store: proxy.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
cell_written_at: None,
noted_not_regular: Default::default(),
};
a.checkout().await.unwrap();
a.floor_tick().await.expect("an idle tick with working credentials");
assert!(a.load_gauges().unwrap().auth_paused_since_unix.is_none(), "a healthy idle writer reads as paused");
proxy.set_refuse_reads(true);
let e = a.floor_tick().await.expect_err("fixture: the idle tick made no refused request");
assert!(e.is_auth(), "the refused tick did not classify as a credential fault: {e}");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_some(),
"an idle writer's refused tick left no local evidence of the pause"
);
proxy.set_refuse_reads(false);
a.floor_tick().await.expect("an idle tick after the credentials came back");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_none(),
"the pause outlived the credentials being restored on an idle writer"
);
}
#[test]
fn is_auth_reads_through_the_from_conversion() {
fn via_question_mark() -> super::LeanResult<()> {
Err(flint_store::StoreError::Auth("ExpiredToken".into()))?;
Ok(())
}
assert!(via_question_mark().unwrap_err().is_auth());
assert!(!LeanError::State("x".into()).is_auth());
assert!(!LeanError::Fenced("x".into()).is_auth());
let other: LeanError = flint_store::StoreError::Other("boom".into()).into();
assert!(!other.is_auth(), "an ordinary store error read as a credential fault");
let pf: LeanError = flint_store::StoreError::PreconditionFailed("412".into()).into();
assert!(!pf.is_auth(), "a deposal read as a credential fault");
}
fn chunks_of(keys: &[String]) -> Vec<Vec<String>> {
let refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect();
let cuts = super::chunk::chunk_ranges_with(&refs, 8, 2, 32);
let mut out = Vec::new();
let mut start = 0;
for c in cuts {
out.push(keys[start..c].to_vec());
start = c;
}
out
}
fn fixed_chunks_of(keys: &[String], n: usize) -> Vec<Vec<String>> {
keys.chunks(n).map(|c| c.to_vec()).collect()
}
fn changed_chunks(a: &[Vec<String>], b: &[Vec<String>]) -> usize {
let before: std::collections::HashSet<_> = a.iter().collect();
b.iter().filter(|c| !before.contains(c)).count()
}
#[test]
fn a_front_insert_rewrites_one_chunk_not_the_project() {
let base: Vec<String> = (0..300).map(|i| format!("src/f{i:04}.txt")).collect();
let mut inserted = base.clone();
inserted.insert(0, "src/a000-brand-new.txt".to_string());
assert!(inserted.windows(2).all(|w| w[0] < w[1]), "fixture is not sorted");
let c0 = chunks_of(&base);
let c1 = chunks_of(&inserted);
assert!(c0.len() > 4, "fixture produced {} chunks — too few to say anything", c0.len());
let cd = changed_chunks(&c0, &c1);
let f0 = fixed_chunks_of(&base, 8);
let f1 = fixed_chunks_of(&inserted, 8);
let fd = changed_chunks(&f0, &f1);
assert!(
fd >= f0.len(),
"the control did not exhibit the cascade it exists to demonstrate ({fd} of {} chunks \
changed), so the assertion below proves nothing about this fixture",
f0.len()
);
assert!(
cd <= 2,
"a front insert rewrote {cd} of {} content-defined chunks (fixed-count rewrote {fd}) — \
boundaries are moving with POSITION, which is the failure mode §3 disqualifies",
c0.len()
);
}
#[test]
fn an_edit_far_from_a_chunk_leaves_it_alone() {
let base: Vec<String> = (0..300).map(|i| format!("src/f{i:04}.txt")).collect();
let mut edited = base.clone();
edited.insert(150, "src/f0149-inserted.txt".to_string());
edited.sort();
let c0 = chunks_of(&base);
let c1 = chunks_of(&edited);
assert_eq!(c0.first(), c1.first(), "the FIRST chunk moved for a change in the middle");
assert_eq!(c0.last(), c1.last(), "the LAST chunk moved for a change in the middle");
assert!(
changed_chunks(&c0, &c1) <= 2,
"a middle insert rewrote {} chunks",
changed_chunks(&c0, &c1)
);
}
#[test]
fn chunk_sizing_respects_min_max_and_keeps_the_tail() {
let keys: Vec<String> = (0..500).map(|i| format!("k{i:05}")).collect();
let refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect();
let cuts = super::chunk::chunk_ranges_with(&refs, 8, 3, 20);
assert_eq!(*cuts.last().unwrap(), keys.len(), "the tail run was dropped");
let mut prev = 0;
for (i, c) in cuts.iter().enumerate() {
let len = c - prev;
assert!(len <= 20, "chunk {i} has {len} entries, over the max of 20");
if i + 1 < cuts.len() {
assert!(len >= 3, "chunk {i} has {len} entries, under the min of 3");
}
prev = *c;
}
let total: usize = cuts.iter().scan(0, |p, c| { let l = c - *p; *p = *c; Some(l) }).sum();
assert_eq!(total, keys.len(), "chunks do not partition the key stream");
}
#[test]
fn chunk_boundaries_are_stable_and_content_addresses_are_not_crc() {
let keys: Vec<String> = (0..64).map(|i| format!("src/f{i:03}.txt")).collect();
let refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect();
let a = super::chunk::chunk_ranges_with(&refs, 8, 2, 32);
let b = super::chunk::chunk_ranges_with(&refs, 8, 2, 32);
assert_eq!(a, b, "chunking is not deterministic");
assert!(!a.is_empty() && *a.last().unwrap() == 64);
let x = super::chunk::chunk_address(b"{\"a\":1}");
let y = super::chunk::chunk_address(b"{\"a\":2}");
assert_ne!(x, y, "two different chunk bodies share an address");
assert_eq!(x.len(), 32, "address is not 128 bits of hex");
assert_eq!(x, super::chunk::chunk_address(b"{\"a\":1}"), "address is not stable");
}
fn entry_at(k: &str, seq: u64) -> super::manifest::LeanEntry {
super::manifest::LeanEntry {
key: format!("files/{k}"),
etag: format!("e-{k}-{seq}"),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 10,
mode: 0o644,
mtime_unix: 1_700_000_000,
generation: seq,
epoch: 1,
}
}
fn manifest_of(n: usize, seq: u64) -> super::manifest::LeanManifest {
super::manifest::LeanManifest {
seq,
entries: (0..n).map(|i| (format!("src/f{i:05}.txt"), entry_at(&format!("f{i:05}"), seq))).collect(),
sole_writer: false,
boundary_source: None,
tombstones: std::collections::BTreeMap::new(),
}
}
#[tokio::test]
async fn a_three_file_publish_writes_chunks_proportional_to_the_change() {
let inner = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
let m0 = manifest_of(4000, 1);
inner.reset_op_counts();
let meta = manifest::cas_write_chunked(store.as_ref(), &cfg, &m0, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await
.expect("first chunked publish");
let first_puts = *inner.op_counts().get("put_whole").unwrap_or(&0);
let p0 = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer;
let chunks0 = match p0.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("the pointer is not chunked"),
};
assert!(chunks0.len() >= 4, "fixture made {} chunks — too few to measure", chunks0.len());
assert!(
first_puts as usize >= chunks0.len(),
"the first publish wrote {first_puts} objects for {} chunks — it cannot have written \
them all, so the comparison below is against nothing",
chunks0.len()
);
let mut m1 = m0.clone();
m1.seq = 2;
for i in [7usize, 1500, 3900] {
m1.entries.insert(format!("src/f{i:05}.txt"), entry_at(&format!("f{i:05}"), 99));
}
let h = super::manifest::ManifestHandle { etag: meta.etag.clone(), legacy: false, prev_chunks: Vec::new(), prev_tombstones: None };
inner.reset_op_counts();
manifest::cas_write_chunked(store.as_ref(), &cfg, &m1, Some(&h), &chunks0,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await
.expect("incremental chunked publish");
let puts = *inner.op_counts().get("put_whole").unwrap_or(&0);
eprintln!(
"chunked publish: {} chunks; full publish {first_puts} objects, 3-file publish {puts}",
chunks0.len()
);
assert!(
puts * 5 < first_puts,
"a three-file publish wrote {puts} objects against the full publish's {first_puts} \
over {} chunks — untouched chunks are being rewritten, which is O(entries) again",
chunks0.len()
);
assert!(puts <= 6, "a three-file publish wrote {puts} objects");
let back = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(back.manifest.entries.len(), 4000, "entries were lost across the chunked publish");
assert_eq!(back.manifest.entries, m1.entries, "the assembled manifest is not what was published");
}
#[tokio::test]
async fn a_missing_chunk_refuses_rather_than_shortening_the_manifest() {
let inner = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
let m = manifest_of(2000, 1);
manifest::cas_write_chunked(store.as_ref(), &cfg, &m, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None }).await.unwrap();
let p = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer;
let chunks = match p.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
assert!(chunks.len() >= 2);
store.delete(&cfg.chunk_key(&chunks[chunks.len() / 2].addr)).await.unwrap();
let msg = match manifest::load(store.as_ref(), &cfg).await {
Err(e) => e.to_string(),
Ok(m) => panic!(
"a manifest with a missing chunk LOADED, with {} entries",
m.map(|l| l.manifest.entries.len()).unwrap_or(0)
),
};
assert!(
msg.contains("hole"),
"a manifest with a missing chunk did not refuse; it said: {msg}"
);
}
#[test]
fn a_pointer_naming_both_layouts_or_neither_is_refused() {
let mut p = super::manifest::Pointer {
seq: 3,
entries_key: Some("k".into()),
entries_seq: Some(3),
chunks: Some(vec![]),
sole_writer: false,
boundary_source: None,
epoch: 1,
tombstones: None,
};
assert!(p.entries().unwrap_err().to_string().contains("BOTH"));
p.entries_key = None;
p.chunks = None;
assert!(p.entries().unwrap_err().to_string().contains("no entries at all"));
p.chunks = Some(vec![]);
assert!(matches!(p.entries().unwrap(), super::manifest::Entries::Chunked(c) if c.is_empty()));
}
#[test]
fn assemble_refuses_every_way_a_chunk_list_can_lie() {
let entries: std::collections::BTreeMap<String, super::manifest::LeanEntry> =
(0..40).map(|i| (format!("k{i:03}"), entry_at(&format!("k{i:03}"), 1))).collect();
let split = super::chunk::split_with(&entries, 4, 2, 8).unwrap();
assert!(split.len() >= 3, "fixture made {} chunks", split.len());
let refs: Vec<_> = split.iter().map(|(r, _)| r.clone()).collect();
let bodies: Vec<Vec<u8>> = split.iter().map(|(_, b)| b.clone()).collect();
let ok = super::chunk::assemble(&refs, &bodies).unwrap();
assert_eq!(ok, entries, "assemble does not round-trip its own split");
let must_fail = |r: Vec<super::chunk::ChunkRef>, b: Vec<Vec<u8>>, why: &str| {
assert!(
super::chunk::assemble(&r, &b).is_err(),
"assemble accepted a chunk list that {why}"
);
};
let mut r = refs.clone();
r[1].n += 1;
must_fail(r, bodies.clone(), "claims more entries than its body holds");
let mut b = bodies.clone();
b[1] = bodies[2].clone();
must_fail(refs.clone(), b, "returns a body that is not the addressed one");
let mut r = refs.clone();
let mut b = bodies.clone();
r.swap(0, 1);
b.swap(0, 1);
must_fail(r, b, "is not in increasing key order");
must_fail(refs.clone(), bodies[..bodies.len() - 1].to_vec(), "names more chunks than were fetched");
let mut r = refs.clone();
r[1].first = "zzz".into();
must_fail(r, bodies.clone(), "disagrees with its body about the first key");
}
struct SweepMidRead {
inner: Arc<MemoryStore>,
current_key: String,
doomed_key: String,
next_pointer: Vec<u8>,
move_pointer: bool,
fired: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl ObjectStore for SweepMidRead {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
if key == self.doomed_key
&& !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst)
{
if self.move_pointer {
let stamps = GenerationStamps {
generation: 9,
epoch: 1,
flush_uuid: "sweep".into(),
boundary_source: None,
posix: None,
};
let crc = crc64_nvme(&self.next_pointer);
self.inner
.put_whole(
&self.current_key,
Bytes::from(self.next_pointer.clone()),
&PutCondition::Unconditional,
&stamps,
crc,
)
.await?;
}
self.inner.delete(&self.doomed_key).await?;
return Err(flint_store::StoreError::NotFound(format!("swept: {key}")));
}
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_reader_swept_mid_read_restarts_onto_the_current_generation() {
for move_pointer in [true, false] {
let inner = Arc::new(MemoryStore::new());
let plain: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
let m1 = manifest_of(600, 1);
let meta1 =
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await
.unwrap();
let p1 = manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap().pointer;
let c1 = match p1.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new(), prev_tombstones: None };
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await
.unwrap();
let lp2 = manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap();
let ptr2_bytes = serde_json::to_vec(&lp2.pointer).unwrap();
let stamps = GenerationStamps {
generation: 1,
epoch: 1,
flush_uuid: "rewind".into(),
boundary_source: None,
posix: None,
};
let p1_bytes = serde_json::to_vec(&p1).unwrap();
let crc = crc64_nvme(&p1_bytes);
plain
.put_whole(&cfg.current_key(), Bytes::from(p1_bytes), &PutCondition::Unconditional, &stamps, crc)
.await
.unwrap();
let c2: Vec<String> = match lp2.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("not chunked"),
};
let doomed = c1
.iter()
.find(|r| !c2.contains(&r.addr))
.expect("generation 2 shares every chunk with generation 1 — nothing to sweep");
let store: Arc<dyn ObjectStore> = Arc::new(SweepMidRead {
inner: inner.clone(),
current_key: cfg.current_key(),
doomed_key: cfg.chunk_key(&doomed.addr),
next_pointer: ptr2_bytes.clone(),
move_pointer,
fired: std::sync::atomic::AtomicBool::new(false),
});
let got = manifest::load(store.as_ref(), &cfg).await;
if move_pointer {
let loaded = got.expect("a reader swept mid-read did not restart").unwrap();
assert_eq!(
loaded.manifest.entries, m2.entries,
"the restart did not land on the CURRENT generation — a reader that mixes \
generations is worse than one that fails"
);
assert_eq!(loaded.manifest.seq, 2);
} else {
let msg = match got {
Err(e) => e.to_string(),
Ok(_) => panic!(
"a chunk vanished under an UNCHANGED pointer and the load succeeded — \
that is a hole in the live manifest, not a race"
),
};
assert!(msg.contains("hole"), "wrong refusal: {msg}");
}
}
}
#[tokio::test]
async fn the_chunk_reaper_takes_orphans_and_nothing_else() {
let inner = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 0;
let m1 = manifest_of(600, 1);
let meta1 = manifest::cas_write_chunked(store.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await.unwrap();
let c1 = match manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap()
.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new(), prev_tombstones: None };
manifest::cas_write_chunked(store.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await.unwrap();
let live: Vec<String> = match manifest::load_pointer(store.as_ref(), &cfg).await.unwrap()
.unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("not chunked"),
};
let orphans: Vec<&super::chunk::ChunkRef> =
c1.iter().filter(|r| !live.contains(&r.addr)).collect();
assert!(!orphans.is_empty(), "generation 2 superseded no chunk — nothing to reap");
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
assert_eq!(n, orphans.len(), "the reaper took {n} chunks, expected {}", orphans.len());
for r in &orphans {
assert!(
store.head(&cfg.chunk_key(&r.addr)).await.is_err(),
"superseded chunk {} survived the sweep",
r.addr
);
}
for a in &live {
assert!(store.head(&cfg.chunk_key(a)).await.is_ok(), "the reaper took a LIVE chunk {a}");
}
let back = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(back.manifest.entries, m2.entries, "the sweep left the manifest short");
let meta2 = {
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
super::manifest::ManifestHandle { etag: lp.etag, legacy: false, prev_chunks: Vec::new(), prev_tombstones: None }
};
let mut m3 = m2.clone();
m3.seq = 3;
m3.entries.insert("src/f00300.txt".into(), entry_at("changed3", 43));
let c2: Vec<super::chunk::ChunkRef> = match manifest::load_pointer(store.as_ref(), &cfg)
.await.unwrap().unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
manifest::cas_write_chunked(store.as_ref(), &cfg, &m3, Some(&meta2), &c2,
manifest::PublishStamps { epoch: 1, flush_uuid: "u3", boundary_source: None })
.await.unwrap();
let before = store.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
cfg.orphan_grace_secs = 3600;
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
let after = store.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
assert_eq!(n, 0, "the reaper took {n} chunks that were inside the grace");
assert_eq!(before, after, "the chunk count moved despite the grace");
}
#[tokio::test]
async fn the_chunk_reaper_aborts_when_a_publish_lands_under_it() {
let inner = Arc::new(MemoryStore::new());
let plain: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 0;
let m1 = manifest_of(600, 1);
let meta1 = manifest::cas_write_chunked(plain.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await.unwrap();
let c1 = match manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap()
.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new(), prev_tombstones: None };
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await.unwrap();
let lp2 = manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap();
let mut moved = lp2.pointer.clone();
moved.seq = 99;
let moved_bytes = serde_json::to_vec(&moved).unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(PublishOnList {
inner: inner.clone(),
current_key: cfg.current_key(),
next_pointer: moved_bytes,
chunks_prefix: format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR),
fired: std::sync::atomic::AtomicBool::new(false),
});
let before = plain.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
let after = plain.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
assert_eq!(n, 0, "the reaper deleted {n} chunks after a publish landed under it");
assert_eq!(before, after, "chunks disappeared across an aborted pass");
}
struct PublishOnList {
inner: Arc<MemoryStore>,
current_key: String,
next_pointer: Vec<u8>,
chunks_prefix: String,
fired: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl ObjectStore for PublishOnList {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
let out = self.inner.list(prefix).await;
if prefix == self.chunks_prefix
&& !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst)
{
let stamps = GenerationStamps {
generation: 99,
epoch: 1,
flush_uuid: "raced".into(),
boundary_source: None,
posix: None,
};
let crc = crc64_nvme(&self.next_pointer);
self.inner
.put_whole(
&self.current_key,
Bytes::from(self.next_pointer.clone()),
&PutCondition::Unconditional,
&stamps,
crc,
)
.await?;
}
out
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_chunked_barrier_keeps_a_foreign_write_it_never_read() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunked = true;
sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..60 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
let p = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap().pointer;
let chunks = match p.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
super::manifest::Entries::Single { .. } => {
panic!("the barrier published a single generation object with cfg.chunked set")
}
};
assert!(
chunks.len() >= 3,
"PRECONDITION: {} chunk(s) — too few for a foreign write and a local one to land in \
different ones, which is the case that matters",
chunks.len()
);
hitl_write(&store, &sc.cfg, "src/f000.txt", "user version", "dilip").await.unwrap();
write(dir.path(), "src/f059.txt", "v2");
backdate_baseline(&sc, "src/f059.txt");
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(
m.manifest.entries.len(),
60,
"the chunked publish changed the entry count — a seam lost or duplicated a key"
);
let cited = &m.manifest.entries["src/f000.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(
&body[..],
b"user version",
"the chunked publish lost the foreign write — this is the splice failure the merge \
model refutes, reached through the barrier"
);
let mine = &m.manifest.entries["src/f059.txt"];
let (_, body) = store.get_whole(&mine.key, Some(&mine.etag)).await.unwrap();
assert_eq!(&body[..], b"v2", "the local edit did not land");
}
#[tokio::test]
async fn a_replaced_handle_outlives_its_retirement_by_the_retire_age() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.retire_grace_secs = 600;
sc.cfg.untracked_grace_secs = 0;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
write(dir.path(), "g.txt", "g1");
sc.run_barrier().await.unwrap();
let seen = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
write(dir.path(), "f.txt", "v2 by the writer");
backdate_baseline(&sc, "f.txt");
hitl_write(&store, &sc.cfg, "g.txt", "g2 by the ui", "ui").await.unwrap();
sc.state.save_orphan_sweep_at(now_unix() - sc.cfg.untracked_sweep_secs - 1).unwrap();
sc.run_barrier().await.unwrap();
for p in ["f.txt", "g.txt"] {
let e = &seen.entries[p];
assert!(
store.get_whole(&e.key, Some(&e.etag)).await.is_ok(),
"{p}: a reader of the document from before the commit found its handle gone"
);
}
sc.cfg.retire_grace_secs = 0;
write(dir.path(), "h.txt", "a later publish");
sc.state.save_orphan_sweep_at(now_unix() - sc.cfg.untracked_sweep_secs - 1).unwrap();
sc.run_barrier().await.unwrap();
for p in ["f.txt", "g.txt"] {
assert!(
matches!(store.head(&seen.entries[p].key).await, Err(StoreError::NotFound(_))),
"{p}: a retired handle outlived the retire age"
);
}
let logs = format!("{}/{}/retired/", sc.cfg.prefix, super::LEAN_DIR);
assert!(store.list(&logs).await.unwrap().is_empty(), "the retire logs were never reaped");
}
#[tokio::test]
async fn a_commit_never_retires_a_handle_the_installed_document_cites_at_another_path() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "p.txt", "shared bytes");
a.floor_tick().await.unwrap();
a.floor_tick().await.unwrap();
gateway_commit(&store, &a.cfg, &super::ui_flush(now_unix()), |doc| {
let e = doc.entries["p.txt"].clone();
doc.entries.insert("q.txt".to_string(), e);
})
.await
.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "q.txt").as_deref(), Some("shared bytes"), "fixture: the copy never reached the tree");
let h = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["p.txt"].key.clone();
a.cfg.retire_grace_secs = 3600;
std::fs::remove_file(dir.path().join("p.txt")).unwrap();
let r = a.declared_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["p.txt".to_string()], "fixture: the delete did not publish: {r:?}");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.get("q.txt").map(|e| e.key.as_str()), Some(h.as_str()), "fixture: q no longer cites the handle");
let logs = store.list(&manifest::retired_prefix(&a.cfg)).await.unwrap();
for o in &logs {
let (_, body) = store.get_whole(&o.key, None).await.unwrap();
let log: manifest::RetireLog = serde_json::from_slice(&body).unwrap();
assert!(!log.handles.contains(&h), "a commit retired {h}, which q.txt still cites: {log:?}");
}
assert_eq!(cited_bytes(&store, &a.cfg, "q.txt").await.as_deref(), Some(&b"shared bytes"[..]));
}
#[tokio::test]
async fn a_retire_log_naming_a_cited_handle_deletes_nothing() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "f.txt", "live");
sc.run_barrier().await.unwrap();
let cited = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest.entries["f.txt"].clone();
let log = manifest::RetireLog { at_unix: 1, handles: vec![cited.key.clone()], chunks: vec![] };
let key = format!("{}{:020}-planted.json", manifest::retired_prefix(&sc.cfg), 1);
let body = serde_json::to_vec(&log).unwrap();
let crc = crc64_nvme(&body);
let stamps = GenerationStamps { generation: 0, epoch: 0, flush_uuid: "planted".into(), boundary_source: None, posix: None };
store.put_whole(&key, body.into(), &PutCondition::IfNoneMatchAny, &stamps, crc).await.unwrap();
write(dir.path(), "g.txt", "a later publish, so a commit section runs");
sc.run_barrier().await.unwrap();
assert!(
store.get_whole(&cited.key, Some(&cited.etag)).await.is_ok(),
"a due retire log deleted a handle the document cites"
);
}
#[tokio::test]
async fn a_superseded_generation_outlives_its_supersession_by_the_retire_age() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut sc = Syncer {
store: store.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
cell_written_at: None,
noted_not_regular: Default::default(),
};
sc.cfg.retire_grace_secs = 600;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "f.txt", "v0");
sc.run_barrier().await.unwrap();
let first = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
for i in 1..=(manifest::KEEP_GENERATIONS + 3) {
write(dir.path(), "f.txt", &format!("v{i} {}", "x".repeat(i)));
backdate_baseline(&sc, "f.txt");
sc.run_barrier().await.unwrap();
}
assert!(
store.head(&first).await.is_ok(),
"a generation superseded seconds ago was reaped by count alone"
);
sc.cfg.retire_grace_secs = 0;
manifest::sweep_generations(store.as_ref(), &sc.cfg).await.unwrap();
assert!(
matches!(store.head(&first).await, Err(StoreError::NotFound(_))),
"past the retire age the count rule applies again"
);
}
#[tokio::test]
async fn a_superseded_chunk_outlives_its_retirement_by_the_retire_age() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunked = true;
sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
sc.cfg.orphan_grace_secs = 0;
sc.cfg.retire_grace_secs = 600;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..60 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
let chunks_of = |p: super::manifest::Pointer| match p.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect::<Vec<_>>(),
super::manifest::Entries::Single { .. } => panic!("not chunked"),
};
let before = chunks_of(manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap().pointer);
write(dir.path(), "src/f030.txt", "v2");
backdate_baseline(&sc, "src/f030.txt");
sc.run_barrier().await.unwrap();
let after = chunks_of(manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap().pointer);
let superseded: Vec<&String> = before.iter().filter(|a| !after.contains(a)).collect();
assert!(!superseded.is_empty(), "PRECONDITION: the commit superseded no chunk");
for a in &superseded {
assert!(
store.head(&sc.cfg.chunk_key(a)).await.is_ok(),
"a chunk the previous pointer names was reaped inside the retire age"
);
}
sc.cfg.retire_grace_secs = 0;
write(dir.path(), "src/f031.txt", "v2");
backdate_baseline(&sc, "src/f031.txt");
sc.run_barrier().await.unwrap();
for a in &superseded {
assert!(
matches!(store.head(&sc.cfg.chunk_key(a)).await, Err(StoreError::NotFound(_))),
"a superseded chunk outlived the retire age"
);
}
}
#[tokio::test]
async fn migrating_to_chunks_leaves_no_generation_objects_behind() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunked = false; sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
sc.cfg.orphan_grace_secs = 0;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..40 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
write(dir.path(), "src/f000.txt", "v2");
backdate_baseline(&sc, "src/f000.txt");
sc.run_barrier().await.unwrap();
let gens = format!("{}/{}/manifests/", sc.cfg.prefix, super::LEAN_DIR);
assert!(
!store.list(&gens).await.unwrap().is_empty(),
"PRECONDITION: no generation objects to migrate away from"
);
sc.cfg.chunked = true;
write(dir.path(), "src/f001.txt", "v2");
backdate_baseline(&sc, "src/f001.txt");
sc.run_barrier().await.unwrap();
let p = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap().pointer;
assert!(
matches!(p.entries().unwrap(), super::manifest::Entries::Chunked(_)),
"the workspace did not migrate to a chunk list"
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries.len(), 40, "entries were lost migrating to chunks");
manifest::sweep_generations(store.as_ref(), &sc.cfg).await.unwrap();
assert!(
store.list(&gens).await.unwrap().is_empty(),
"generation objects survived the migration: {:?}",
store.list(&gens).await.unwrap().iter().map(|o| &o.key).collect::<Vec<_>>()
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries.len(), 40, "the migration sweep broke the live manifest");
}
struct StaleListing {
inner: Arc<MemoryStore>,
}
#[async_trait::async_trait]
impl ObjectStore for StaleListing {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
let mut out = self.inner.list(prefix).await?;
for o in out.iter_mut() {
o.last_modified_unix = Some(0);
}
Ok(out)
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn the_chunk_reaper_judges_the_grace_now_not_when_it_listed() {
let inner = Arc::new(MemoryStore::new());
let plain: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 3600;
let m1 = manifest_of(600, 1);
let meta1 = manifest::cas_write_chunked(plain.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await.unwrap();
let c1 = match manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap()
.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new(), prev_tombstones: None };
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await.unwrap();
let prefix = format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR);
let before = plain.list(&prefix).await.unwrap().len();
let live: Vec<String> = match manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap()
.unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("not chunked"),
};
assert!(
c1.iter().any(|r| !live.contains(&r.addr)),
"PRECONDITION: nothing is unreferenced, so the reaper has no candidate to mis-judge"
);
let store: Arc<dyn ObjectStore> = Arc::new(StaleListing { inner: inner.clone() });
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
let after = plain.list(&prefix).await.unwrap().len();
assert_eq!(
n, 0,
"the reaper collected {n} chunk(s) on the LISTING's word that they were ancient — \
an adopt-rewrite landing after the listing is invisible to that, and rule 4 works \
by producing exactly such a rewrite"
);
assert_eq!(before, after, "chunks disappeared: {before} → {after}");
}
#[tokio::test]
async fn the_chunk_reaper_does_not_head_what_the_listing_shows_inside_the_grace() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 3600;
let mut m = manifest_of(600, 1);
let mut prev: Option<(super::manifest::ManifestHandle, Vec<super::chunk::ChunkRef>)> = None;
for seq in 1..=6u64 {
m.seq = seq;
m.entries.insert(format!("src/f{:05}.txt", seq * 97), entry_at(&format!("v{seq}"), seq));
let (h, c) = match &prev { Some((h, c)) => (Some(h), c.as_slice()), None => (None, &[][..]) };
manifest::cas_write_chunked(store.as_ref(), &cfg, &m, h, c,
manifest::PublishStamps { epoch: 1, flush_uuid: "u", boundary_source: None })
.await.unwrap();
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
let chunks = match lp.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
prev = Some((super::manifest::ManifestHandle { etag: lp.etag, legacy: false, prev_chunks: Vec::new(), prev_tombstones: None }, chunks));
}
let prefix = format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR);
let live: Vec<String> = prev.as_ref().unwrap().1.iter().map(|r| r.addr.clone()).collect();
let orphans = store.list(&prefix).await.unwrap().iter()
.filter(|o| !live.iter().any(|a| o.key.ends_with(a.as_str()))).count();
assert!(orphans >= 5, "PRECONDITION: five publishes left {orphans} unreferenced chunks");
store.reset_op_counts();
assert_eq!(manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap(), 0, "a chunk inside the grace was collected");
let heads = store.op_counts().get("head").copied().unwrap_or(0);
assert_eq!(heads, 0, "the reaper HEADed {heads} chunks the listing already showed inside the grace");
for o in store.list(&prefix).await.unwrap() {
store.backdate_epoch(&o.key, 3700);
}
store.reset_op_counts();
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
assert_eq!(n, orphans, "past the grace the reaper took {n} of {orphans}");
assert!(store.op_counts().get("head").copied().unwrap_or(0) >= orphans as u64, "a chunk was collected without a HEAD");
for a in &live {
assert!(store.head(&cfg.chunk_key(a)).await.is_ok(), "the reaper took a LIVE chunk {a}");
}
}
#[tokio::test]
async fn the_barrier_reaps_unreferenced_chunks_without_being_asked() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
sc.cfg.orphan_grace_secs = 0; assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..60 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
let prefix = format!("{}/{}/chunks/", sc.cfg.prefix, super::LEAN_DIR);
let after_first = store.list(&prefix).await.unwrap().len();
assert!(after_first >= 3, "PRECONDITION: {after_first} chunk(s) — too few to supersede any");
write(dir.path(), "src/f000.txt", "v2");
backdate_baseline(&sc, "src/f000.txt");
sc.run_barrier().await.unwrap();
let live: Vec<String> = match manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap()
.unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("the barrier did not publish a chunk list"),
};
let present: Vec<String> = store
.list(&prefix)
.await
.unwrap()
.into_iter()
.filter_map(|o| o.key.rsplit('/').next().map(|s| s.to_string()))
.collect();
let stray: Vec<&String> = present.iter().filter(|a| !live.contains(a)).collect();
assert!(
stray.is_empty(),
"the barrier left {} unreferenced chunk(s) in the bucket ({stray:?}) — the reaper is \
not reachable from a publish, which is the whole defect",
stray.len()
);
for a in &live {
assert!(present.contains(a), "the barrier's own sweep deleted live chunk {a}");
}
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries.len(), 60, "the sweep left the manifest short");
}
#[tokio::test]
async fn a_lost_renew_response_does_not_self_fence() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let key = a.cfg.epoch_key();
let stale = a.lease.clone().unwrap();
store.epoch_renew(&key, &stale, None).await.unwrap();
let landed = store.epoch_read(&key).await.unwrap().unwrap().token;
assert_ne!(landed, stale.token, "fixture: the renew did not move the token");
lease::renew(&mut a)
.await
.expect("a renew whose previous response was lost is not a deposal");
let cell = store.epoch_read(&key).await.unwrap().unwrap().token;
assert_eq!(
a.lease.as_ref().map(|l| l.token.as_str()),
Some(cell.as_str()),
"the holder did not adopt the cell's token after its lost renew"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 12).await, "quiet polls exhausted ⇒ takeover");
assert!(matches!(lease::renew(&mut a).await.unwrap_err(), LeanError::Fenced(_)));
assert!(a.lease.is_none(), "a fenced holder must drop its lease");
}
#[tokio::test]
async fn a_foreign_claim_refuses_the_syncer_before_it_claims() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
lease::verify_claim(&a).await.unwrap();
a.cfg.project_id = Some("team-a/p".into());
lease::verify_claim(&a).await.unwrap();
let key = a.cfg.claim_key();
let body = br#"{"project_id":"team-b/q","created_unix":1,"stamped_by":"op"}"#.to_vec();
let stamps = GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: "claim".into(),
boundary_source: None,
posix: None,
};
let crc = crc64_nvme(&body);
store
.put_whole(&key, Bytes::from(body), &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let err = lease::verify_claim(&a).await.unwrap_err();
assert!(
matches!(err, LeanError::Refused(_)),
"a foreign claim is a REFUSAL (exit EXIT_REFUSED, final for the delivery), not a \
state error a restart would retry: {err}"
);
let err = err.to_string();
assert!(
err.contains("team-b/q") && err.contains("team-a/p"),
"the refusal must name both projects: {err}"
);
assert!(a.lease.is_none());
a.cfg.project_id = Some("team-b/q".into());
lease::verify_claim(&a).await.unwrap();
}
#[test]
fn the_drain_attestation_is_written_by_the_drain_and_cleared_at_startup() {
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let st = SyncerState::open(cfg.state_dir()).unwrap();
assert!(!st.drained_path().exists(), "a fresh state dir must carry no attestation");
st.sync_tree().unwrap();
st.write_drained(Some(7), 1).unwrap();
let v: serde_json::Value = serde_json::from_slice(&std::fs::read(st.drained_path()).unwrap()).unwrap();
assert_eq!(v["seq"], 7);
assert_eq!(v["acks"], 1);
assert!(v["unix"].as_u64().unwrap() > 0);
st.clear_drained().unwrap();
assert!(!st.drained_path().exists());
st.clear_drained().unwrap(); }
#[tokio::test]
async fn ranged_checkout_is_byte_identical_and_parallel() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big: Vec<u8> = (0..5 * 1024 * 1024).map(|i| (i % 251) as u8).collect();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
write(dir_a.path(), "small.txt", "not ranged");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.range_get_min_bytes = 1024 * 1024;
b.cfg.range_get_chunk_bytes = 512 * 1024; b.cfg.range_get_parallelism = 4;
store.reset_peak_get_range_in_flight();
store.inject_get_range_delay_ms(5);
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(
std::fs::read(dir_b.path().join("weights.bin")).unwrap(),
big,
"ranged materialisation must be byte-identical"
);
assert_eq!(read(dir_b.path(), "small.txt").unwrap(), "not ranged");
let peak = store.peak_get_range_in_flight();
assert!(peak > 1, "ranges must overlap; peak in flight was {peak}");
}
fn ranged_default_tree() -> Vec<u8> {
(0..20 * 1024 * 1024).map(|i| (i % 251) as u8).collect()
}
#[tokio::test]
async fn ranged_checkout_on_by_default_uses_ranges() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big = ranged_default_tree();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
write(dir_a.path(), "small.txt", "under the threshold");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert_eq!(
b.cfg.range_get_min_bytes,
8 * 1024 * 1024,
"the shipped default moved to 8 MiB on the 2026-09-10 runcr drill"
);
store.reset_peak_get_range_in_flight();
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(
std::fs::read(dir_b.path().join("weights.bin")).unwrap(),
big,
"the DEFAULT path must be byte-identical, not merely fast"
);
assert_eq!(read(dir_b.path(), "small.txt").unwrap(), "under the threshold");
assert!(
store.peak_get_range_in_flight() > 0,
"an object over the default threshold must take the ranged path"
);
assert_eq!(cr.ranged, 1, "exactly the over-threshold object, not both");
}
#[tokio::test]
async fn ranged_checkout_explicitly_disabled_uses_no_ranges() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big = ranged_default_tree();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
write(dir_a.path(), "small.txt", "under the threshold");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.range_get_min_bytes = 0;
store.reset_peak_get_range_in_flight();
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(
std::fs::read(dir_b.path().join("weights.bin")).unwrap(),
big,
"the whole-object arm must materialise the same bytes"
);
assert_eq!(
store.peak_get_range_in_flight(),
0,
"the whole-object arm must issue no ranged reads"
);
assert_eq!(cr.ranged, 0);
}
async fn scoped_fixture(store: &Arc<MemoryStore>) -> tempfile::TempDir {
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "inputs/wanted.txt", "the file the agent edits");
write(dir_a.path(), "inputs/also.txt", "and this one");
for i in 0..6 {
write(dir_a.path(), &format!("outputs/big-{i}.bin"), &"x".repeat(4096));
}
a.run_barrier().await.unwrap();
dir_a
}
#[tokio::test]
async fn a_scoped_checkout_materialises_only_the_admitted_set() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(cr.materialized, 2, "only the admitted citations");
assert_eq!(cr.out_of_scope, 6, "and the declined count says the scope DID something");
assert_eq!(cr.scope.as_deref(), Some(&["inputs".to_string()][..]));
assert!(read(dir_b.path(), "inputs/wanted.txt").is_some());
assert!(
read(dir_b.path(), "outputs/big-0.bin").is_none(),
"an out-of-scope citation must not reach the tree"
);
let base = b.state.load_baseline().unwrap();
assert_eq!(
base.entries.keys().cloned().collect::<Vec<_>>(),
vec!["inputs/also.txt".to_string(), "inputs/wanted.txt".to_string()],
"an unadmitted path must never be cited"
);
assert_eq!(b.state.load_scope().unwrap().as_deref(), Some(&["inputs".to_string()][..]));
}
#[tokio::test]
async fn what_a_scoped_checkout_never_materialised_it_can_never_delete() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert!(claim_until_held(&mut b, 12).await, "quiet polls exhausted ⇒ takeover");
let r1 = b.run_barrier().await.unwrap();
let r2 = b.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty() && r2.deleted.is_empty(), "{:?} {:?}", r1.deleted, r2.deleted);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
m.entries.len(),
8,
"the six unadmitted citations survive a scoped workspace's barriers: {:?}",
m.entries.keys().collect::<Vec<_>>()
);
}
#[tokio::test]
async fn a_scoped_checkout_budget_is_computed_over_the_admitted_set() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_whole = tempfile::tempdir().unwrap();
let mut whole = syncer(&store, dir_whole.path()).await;
whole.cfg.max_bytes = 1024;
let err = whole.checkout().await.expect_err("the whole tree is over this budget");
assert!(matches!(err, LeanError::Budget(_)), "{err}");
let dir_scoped = tempfile::tempdir().unwrap();
let mut scoped = syncer(&store, dir_scoped.path()).await;
scoped.cfg.max_bytes = 1024;
let cr = scoped
.checkout_scoped(Some(vec!["inputs".into()]))
.await
.expect("a budget is a promise about what THIS checkout writes");
assert_eq!(cr.materialized, 2);
}
#[tokio::test]
async fn an_all_rejected_checkout_scope_is_refused_not_widened() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let err = b
.checkout_scoped(Some(vec!["../escape".into(), "./here".into()]))
.await
.expect_err("every entry malformed ⇒ an EMPTY scope ⇒ the whole manifest");
assert!(format!("{err}").contains("WHOLE MANIFEST"), "{err}");
assert!(
read(dir_b.path(), "outputs/big-0.bin").is_none(),
"the refusal must happen BEFORE the first byte"
);
assert!(!b.state.marker_present(), "and must not open the agent-start gate");
}
#[tokio::test]
async fn a_live_tree_refuses_a_checkout_whose_scope_disagrees() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
let err = b.checkout().await.expect_err("scoped tree, unscoped request");
let msg = format!("{err}");
assert!(msg.contains("UNSCOPED"), "the error must name BOTH sets: {msg}");
assert!(msg.contains("inputs"), "the error must name BOTH sets: {msg}");
let dir_c = tempfile::tempdir().unwrap();
let mut c = syncer(&store, dir_c.path()).await;
c.checkout().await.unwrap();
let err = c
.checkout_scoped(Some(vec!["inputs".into()]))
.await
.expect_err("unscoped tree, scoped request");
assert!(format!("{err}").contains("cannot change the admitted set"), "{err}");
let r = b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert!(r.resumed_live_tree);
}
#[tokio::test]
async fn a_scoped_checkout_owes_nothing_the_scope_declined() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 2, "the CITATIONS are scoped");
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert!(read(dir_b.path(), "outputs/big-0.bin").is_none(), "a barrier fetched a declined citation");
let base = b.state.load_baseline().unwrap();
assert_eq!(base.entries.len(), 2);
assert!(!derive_pending(&b).await, "the scope's declined citations read as owed");
}
#[tokio::test]
async fn a_sync_never_fetches_what_the_scope_declined() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
a.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
a.sync().await.unwrap();
assert!(
read(dir_a.path(), "outputs/big-0.bin").is_none(),
"a whole-tree sync must leave the declined citations declined"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
b.state.save_scope(None).unwrap();
b.sync().await.unwrap();
assert!(
read(dir_b.path(), "outputs/big-0.bin").is_some(),
"THE CLIFF: a citation the baseline does not hold reads as owed, so \
without the scope every declined citation comes back one sync later"
);
let base = b.state.load_baseline().unwrap();
assert_eq!(base.entries.len(), 8, "and the scope is gone entirely");
}
#[tokio::test]
async fn a_scoped_tree_never_receives_a_peers_change_outside_its_scope() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
peer_publish(&store, &b.cfg, "outputs/big-0.bin", "a peer's new version", "peer").await;
peer_publish(&store, &b.cfg, "outputs/new.bin", "a peer's new file", "peer").await;
peer_publish(&store, &b.cfg, "inputs/wanted.txt", "a peer's edit in scope", "peer").await;
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert!(read(dir_b.path(), "outputs/big-0.bin").is_none(), "a declined path arrived");
assert!(read(dir_b.path(), "outputs/new.bin").is_none(), "a file outside the scope arrived");
assert_eq!(read(dir_b.path(), "inputs/wanted.txt").as_deref(), Some("a peer's edit in scope"));
}
#[tokio::test]
async fn an_unreadable_scope_is_not_read_as_unscoped() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
if unsafe { libc::geteuid() } == 0 {
eprintln!("SKIPPED: running as root, mode bits cannot induce EACCES");
return;
}
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
let sd = b.cfg.state_dir();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o000)).unwrap();
let got = b.state.load_scope();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
let err = got.expect_err("EACCES must not read as 'unscoped'");
assert!(format!("{err}").contains("refusing to"), "{err}");
}
#[tokio::test]
async fn a_whole_tree_checkout_clears_a_stale_scope() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.state.save_scope(Some(&["inputs".to_string()])).unwrap();
assert!(!b.state.marker_present());
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 8, "the whole manifest, not the stale scope's share");
assert_eq!(b.state.load_scope().unwrap(), None, "and the stale scope is GONE");
}
fn bulky_conflict(i: usize) -> super::state::ConflictRecord {
super::state::ConflictRecord {
path: format!("{}/{}", "p".repeat(900), i),
foreign_etag: format!("\"etag-{i}\""),
preserved_key: None,
kind: "test-bulk".into(),
at_unix: 1_700_000_000 + i as u64,
}
}
#[tokio::test]
async fn the_conflict_log_rotates_instead_of_growing_without_bound() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
for i in 0..2000 {
a.state.append_conflict(&bulky_conflict(i)).unwrap();
}
let sd = a.cfg.state_dir();
let live = std::fs::metadata(sd.join("conflicts.jsonl")).unwrap().len();
assert!(
std::fs::metadata(sd.join("conflicts.1.jsonl")).is_ok(),
"past the cap the live log must have rotated"
);
assert!(live <= 1 << 20, "the LIVE log is what gets parsed on every read: {live}");
let all = a.state.load_conflicts().unwrap();
assert_eq!(all.len(), 2000, "a single rotation loses nothing");
assert_eq!(a.state.conflicts_dropped().unwrap(), 0);
assert_eq!(all[0].at_unix, 1_700_000_000);
assert_eq!(all[1999].at_unix, 1_700_000_000 + 1999);
}
#[tokio::test]
async fn a_rotation_never_shortens_what_a_reader_has_already_counted() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
let mut prev = 0usize;
let mut saw_rotation = false;
for i in 0..2000 {
a.state.append_conflict(&bulky_conflict(i)).unwrap();
let n = a.state.load_conflicts().unwrap().len();
assert!(n >= prev, "the log SHRANK at append {i}: {prev} -> {n}");
if std::fs::metadata(a.cfg.state_dir().join("conflicts.1.jsonl")).is_ok() {
saw_rotation = true;
}
prev = n;
}
assert!(saw_rotation, "the run must actually cross a rotation or it proves nothing");
}
#[tokio::test]
async fn a_second_rotation_reports_what_it_dropped() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
for i in 0..4000 {
a.state.append_conflict(&bulky_conflict(i)).unwrap();
}
let dropped = a.state.conflicts_dropped().unwrap();
let held = a.state.load_conflicts().unwrap().len();
assert!(dropped > 0, "a second rotation DID drop records; the count must say so");
assert_eq!(
dropped as usize + held,
4000,
"dropped + held must account for every record: {dropped} + {held}"
);
}
#[tokio::test]
async fn an_unreadable_conflict_log_is_not_reported_as_empty() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
if unsafe { libc::geteuid() } == 0 {
eprintln!("SKIPPED: running as root, mode bits cannot induce EACCES");
return;
}
a.state.append_conflict(&bulky_conflict(1)).unwrap();
let sd = a.cfg.state_dir();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o000)).unwrap();
let got = a.state.load_conflicts();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
let err = got.expect_err("an unreadable log must not read as 'no conflicts'");
assert!(format!("{err}").contains("refusing to report"), "{err}");
}
#[test]
fn a_shortened_conflict_log_never_reports_an_empty_ack() {
use super::sentinel::conflicts_since;
let rec = |i: usize| bulky_conflict(i);
let grew = vec![rec(0), rec(1), rec(2)];
assert_eq!(conflicts_since(1, 0, grew.clone()).len(), 2);
assert_eq!(conflicts_since(3, 0, grew.clone()).len(), 0, "grew by nothing");
let shrank = vec![rec(7), rec(8)];
assert_eq!(
conflicts_since(5, 0, shrank.clone()).len(),
2,
"a shortened log must not yield an empty conflict set"
);
}
struct ComposeWithoutChecksum(Arc<MemoryStore>);
#[async_trait::async_trait]
impl ObjectStore for ComposeWithoutChecksum {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
let mut m = self.0.compose_generation(spec).await?;
m.crc64_b64 = None;
Ok(m)
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
let mut out = self.0.list(prefix).await?;
for o in out.iter_mut() {
o.last_modified_unix = Some(0);
}
Ok(out)
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.0.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.0.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
async fn epoch_handoff(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<()> {
self.0.epoch_handoff(key, lease, echo).await
}
async fn epoch_enqueue(
&self,
key: &str,
observed: &flint_store::EpochState,
holder_id: &str,
) -> flint_store::StoreResult<flint_store::EpochState> {
self.0.epoch_enqueue(key, observed, holder_id).await
}
}
#[tokio::test]
async fn a_compose_that_reports_no_checksum_refuses_instead_of_citing() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(ComposeWithoutChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let cfg = {
let mut c = cfg_for(dir.path());
c.whole_put_max = 4096;
c
};
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store,
cfg,
state,
lease: None,
cell_written_at: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
std::fs::write(dir.path().join("big.bin"), vec![7u8; 20_000]).unwrap();
let err = a
.run_barrier()
.await
.expect_err("a publish with no checksum must refuse, not cite");
assert!(
format!("{err}").contains("refusing to cite bytes nothing vouched for"),
"{err}"
);
let m = manifest::load(mem.as_ref(), &a.cfg).await.unwrap();
assert!(
m.map(|l| l.manifest.entries.is_empty()).unwrap_or(true),
"a refused publish must leave no citation"
);
}
async fn rescope_fixture(store: &Arc<MemoryStore>) -> (tempfile::TempDir, Syncer) {
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(store, dir.path()).await;
b.checkout().await.unwrap();
assert!(claim_until_held(&mut b, 12).await, "quiet polls exhausted ⇒ takeover");
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 8);
(dir, b)
}
#[tokio::test]
async fn a_narrow_unwatches_without_publishing_a_single_deletion() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
let r = b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(r.uncited, 6, "six citations should have left the held set");
assert_eq!(r.unlinked, 6, "and six files the tree");
assert!(r.kept_dirty.is_empty());
assert!(read(dir.path(), "outputs/big-0.bin").is_none(), "the narrow did not unlink");
assert!(read(dir.path(), "inputs/wanted.txt").is_some(), "the narrow took an admitted path");
assert_eq!(b.state.load_scope().unwrap().as_deref(), Some(&["inputs".to_string()][..]));
let r1 = b.run_barrier().await.unwrap();
let r2 = b.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty() && r2.deleted.is_empty(), "{:?} {:?}", r1.deleted, r2.deleted);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 8, "a narrow published a deletion: {:?}", m.entries.keys());
}
#[tokio::test]
async fn the_same_unlink_without_the_uncite_publishes_deletions() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
for i in 0..6 {
std::fs::remove_file(dir.path().join(format!("outputs/big-{i}.bin"))).unwrap();
}
let r1 = b.run_barrier().await.unwrap();
let r2 = b.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty(), "first absence is not delete-eligible: {:?}", r1.deleted);
assert_eq!(r2.deleted.len(), 6, "the delete rule did not bite: {:?}", r2.deleted);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 2, "the bucket should have lost six objects here");
}
#[tokio::test]
async fn a_widen_materialises_and_cites_what_it_newly_admits() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert!(read(dir.path(), "outputs/big-0.bin").is_none());
let r = b.rescope(Some(vec!["inputs".into(), "outputs".into()])).await.unwrap();
assert_eq!(r.materialized, 6, "widen did not fetch the newly admitted set");
assert_eq!(r.already_held, 2, "and must not refetch what it already holds");
assert_eq!(r.uncited, 0);
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "widen did not materialise");
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 8);
let r1 = b.run_barrier().await.unwrap();
assert!(r1.uploaded.is_empty(), "widen re-uploaded its own fetch: {:?}", r1.uploaded);
}
#[tokio::test]
async fn a_narrow_owes_nothing_it_dropped() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (_dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
let base = b.state.load_baseline().unwrap();
assert_eq!(base.entries.len(), 2, "the HELD set narrows");
assert!(!base.entries.contains_key("outputs/big-0.bin"));
assert!(!base.prev_scan.contains("outputs/big-0.bin"), "prev_scan must drop with entries");
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 2, "a barrier fetched a dropped path back");
}
#[tokio::test]
async fn a_crash_between_the_intent_and_the_unlink_converges() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
drop: (0..6).map(|i| format!("outputs/big-{i}.bin")).collect(),
held: Default::default(),
})
.unwrap();
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "nothing applied yet");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed, "the barrier ran over a half-applied rescope");
assert!(r1.deleted.is_empty(), "{:?}", r1.deleted);
let r2 = b.run_barrier().await.unwrap();
assert!(!r2.rescope_replayed, "the replay must clear its own intent");
assert!(r2.deleted.is_empty(), "{:?}", r2.deleted);
assert!(read(dir.path(), "outputs/big-0.bin").is_none(), "the replay did not finish the job");
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 8, "converging cost the bucket objects");
}
#[tokio::test]
async fn a_crash_between_the_uncite_and_the_unlink_does_not_re_cite() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
let mut base = b.state.load_baseline().unwrap();
let drop: Vec<String> = (0..6).map(|i| format!("outputs/big-{i}.bin")).collect();
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
held: drop.iter().map(|p| (p.clone(), base.entries[p].clone())).collect(),
drop,
})
.unwrap();
for i in 0..6 {
let p = format!("outputs/big-{i}.bin");
base.entries.remove(&p);
base.prev_scan.remove(&p);
}
b.state.save_baseline(&base).unwrap();
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "the files are still on disk");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed);
assert!(
r1.uploaded.is_empty(),
"the barrier re-cited what the narrow dropped: {:?}",
r1.uploaded
);
assert!(read(dir.path(), "outputs/big-0.bin").is_none());
assert_eq!(
b.state.load_baseline().unwrap().entries.len(),
2,
"the narrow was undone by its own recovery"
);
}
#[tokio::test]
async fn a_narrow_stopped_before_its_last_unlink_has_uncited_everything_first() {
use futures::FutureExt;
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
let dropped: Vec<String> = (0..6).map(|i| format!("outputs/big-{i}.bin")).collect();
let (state_dir, gone) = (b.cfg.state_dir(), dropped.clone());
super::barrier::CONSUME_WINDOW_HOOK.with(|h| {
*h.borrow_mut() = Some((
"outputs/big-5.bin".into(),
"rescope-before-unlink",
Box::new(move || {
let raw = std::fs::read(state_dir.join("baseline.json")).unwrap();
let durable: serde_json::Value = serde_json::from_slice(&raw).unwrap();
let still: Vec<&String> = gone.iter().filter(|p| durable["entries"].get(p.as_str()).is_some()).collect();
assert!(durable["entries"].get("inputs/wanted.txt").is_some(), "fixture: not the baseline file");
assert!(still.is_empty(), "a narrow unlinked before its uncite was durable: still cited {still:?}");
panic!("the crash");
}),
))
});
let crashed = std::panic::AssertUnwindSafe(b.rescope(Some(vec!["inputs".into()]))).catch_unwind().await;
let msg = crashed.err().and_then(|e| e.downcast_ref::<&str>().map(|s| s.to_string()));
assert_eq!(msg.as_deref(), Some("the crash"), "fixture: the narrow did not stop in the window");
drop(b);
assert!(read(dir.path(), "outputs/big-0.bin").is_none(), "fixture: no unlink ran before the crash");
assert!(read(dir.path(), "outputs/big-5.bin").is_some(), "fixture: the last file is not on disk");
let mut b = syncer(&store, dir.path()).await;
b.state.clear_scope_intent().unwrap();
for _ in 0..2 {
let r = b.run_barrier().await.unwrap();
assert!(r.deleted.is_empty(), "the narrow's leftovers were published as DELETES: {r:?}");
}
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
for p in &dropped {
assert!(m.entries.contains_key(p.as_str()), "{p} left the bucket: a narrow deleted it");
}
}
#[tokio::test]
async fn a_touch_after_a_restart_mid_narrow_never_declares_the_leftover() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, b) = rescope_fixture(&store).await;
hitl_remove(&store, &b.cfg, "outputs/big-0.bin", "reviewer").await;
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("outputs/big-0.bin"), "fixture: the UI delete did not land");
let mut base = b.state.load_baseline().unwrap();
let dropped: Vec<String> = (0..6).map(|i| format!("outputs/big-{i}.bin")).collect();
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
held: dropped.iter().map(|p| (p.clone(), base.entries[p].clone())).collect(),
drop: dropped,
})
.unwrap();
for i in 0..6 {
let p = format!("outputs/big-{i}.bin");
base.entries.remove(&p);
base.prev_scan.remove(&p);
}
b.state.save_baseline(&base).unwrap();
drop(b);
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "fixture: the leftover is not on disk");
let mut b = syncer(&store, dir.path()).await;
let posture = b.sentinel_preflight().unwrap();
b.write_capabilities(&posture).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
b.poll_sentinels().unwrap();
clear_min_interval(&b);
let ack = b.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("outputs/big-0.bin"),
"the barrier re-published the leftover over the UI's delete: {ack:?}"
);
assert!(read(dir.path(), "outputs/big-0.bin").is_none(), "the replay did not unlink the leftover");
assert!(b.state.load_scope_intent().unwrap().is_none(), "the replay did not finish the narrow");
assert_eq!(ack.report.uploaded, 0, "the barrier carried a leftover: {ack:?}");
assert_eq!((ack.status.as_str(), ack.report.dropped.len()), ("ok", 0), "{ack:?}");
}
#[tokio::test]
async fn a_rescope_refuses_to_drop_a_path_with_unpublished_changes() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
write(dir.path(), "outputs/big-0.bin", "the user's unpublished edit");
let err = b.rescope(Some(vec!["inputs".into()])).await.unwrap_err();
match err {
LeanError::State(m) => assert!(m.contains("outputs/big-0.bin"), "{m}"),
e => panic!("wrong error: {e:?}"),
}
assert!(b.state.load_scope().unwrap().is_none(), "a refused rescope changed the scope");
assert!(b.state.load_scope_intent().unwrap().is_none(), "a refused rescope left an intent");
assert_eq!(
read(dir.path(), "outputs/big-0.bin").as_deref(),
Some("the user's unpublished edit"),
"a refused rescope touched the tree"
);
b.run_barrier().await.unwrap();
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
}
#[tokio::test]
async fn a_replay_keeps_a_dirty_path_instead_of_wedging_the_barrier() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
drop: (0..6).map(|i| format!("outputs/big-{i}.bin")).collect(),
held: Default::default(),
})
.unwrap();
write(dir.path(), "outputs/big-0.bin", "edited after the intent landed");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed);
assert!(b.state.load_scope_intent().unwrap().is_none(), "the replay did not converge");
assert_eq!(
read(dir.path(), "outputs/big-0.bin").as_deref(),
Some("edited after the intent landed")
);
assert!(b.state.load_baseline().unwrap().entries.contains_key("outputs/big-0.bin"));
assert!(read(dir.path(), "outputs/big-1.bin").is_none(), "the clean ones still left");
assert!(
b.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.path == "outputs/big-0.bin" && c.kind == "rescope-kept-locally-dirty"),
"a kept path must be surfaced, not silently retained"
);
}
#[tokio::test]
async fn a_narrow_never_unlinks_bytes_the_agent_wrote_after_the_uncite() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
let root = dir.path().to_path_buf();
super::barrier::CONSUME_WINDOW_HOOK.with(|h| {
*h.borrow_mut() = Some((
"outputs/big-0.bin".into(),
"rescope-before-unlink",
Box::new(move || write(&root, "outputs/big-0.bin", "the agent's edit after the uncite")),
))
});
let r = b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert!(super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()), "fixture: the window never ran");
assert_eq!(
read(dir.path(), "outputs/big-0.bin").as_deref(),
Some("the agent's edit after the uncite"),
"the narrow unlinked the agent's write"
);
assert!(read(dir.path(), "outputs/big-1.bin").is_none(), "the clean ones still left");
assert!(r.kept_dirty.contains(&"outputs/big-0.bin".to_string()), "{:?}", r.kept_dirty);
assert!(
b.state.load_conflicts().unwrap().iter().any(|c| c.path == "outputs/big-0.bin"),
"a kept path must be surfaced"
);
let r1 = b.run_barrier().await.unwrap();
assert!(r1.uploaded.iter().any(|p| p == "outputs/big-0.bin"), "{:?}", r1.uploaded);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 8, "{:?}", m.entries.keys());
}
#[tokio::test]
async fn a_replay_never_unlinks_what_the_agent_wrote_while_the_syncer_was_down() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
let mut base = b.state.load_baseline().unwrap();
let drop: Vec<String> = (0..6).map(|i| format!("outputs/big-{i}.bin")).collect();
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
held: drop.iter().map(|p| (p.clone(), base.entries[p].clone())).collect(),
drop: drop.clone(),
})
.unwrap();
for p in &drop {
base.entries.remove(p);
base.prev_scan.remove(p);
}
b.state.save_baseline(&base).unwrap();
write(dir.path(), "outputs/big-0.bin", "written while the syncer was down");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed);
assert_eq!(
read(dir.path(), "outputs/big-0.bin").as_deref(),
Some("written while the syncer was down"),
"the replay unlinked the agent's write"
);
assert!(read(dir.path(), "outputs/big-1.bin").is_none(), "the leftovers still left");
let r2 = b.run_barrier().await.unwrap();
let up: Vec<&String> = r1.uploaded.iter().chain(r2.uploaded.iter()).collect();
assert_eq!(up, vec!["outputs/big-0.bin"], "{up:?}");
}
#[tokio::test]
async fn an_all_rejected_rescope_is_refused_not_widened() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (_dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
let err = b.rescope(Some(vec!["../escape".into(), "./here".into()])).await.unwrap_err();
assert!(matches!(err, LeanError::State(_)), "{err:?}");
assert_eq!(
b.state.load_scope().unwrap().as_deref(),
Some(&["inputs".to_string()][..]),
"a refused rescope widened the workspace"
);
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 2);
}
#[tokio::test]
async fn a_rescope_before_any_checkout_is_refused() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir.path()).await;
let err = b.rescope(Some(vec!["inputs".into()])).await.unwrap_err();
match err {
LeanError::State(m) => assert!(m.contains("checkout"), "{m}"),
e => panic!("wrong error: {e:?}"),
}
}
#[tokio::test]
async fn a_rescope_to_none_holds_everything_and_clears_the_scope() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 2);
let r = b.rescope(None).await.unwrap();
assert_eq!(r.materialized, 6);
assert!(b.state.load_scope().unwrap().is_none(), "the scope document must be REMOVED");
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 8);
assert!(read(dir.path(), "outputs/big-5.bin").is_some());
}
#[tokio::test]
async fn a_checkout_does_not_wait_out_a_standing_lease() {
let store = Arc::new(MemoryStore::new());
let pdir = tempfile::tempdir().unwrap();
let mut publisher = syncer(&store, pdir.path()).await;
assert!(claim_until_held(&mut publisher, 3).await);
publisher.checkout().await.unwrap();
write(pdir.path(), "README.md", "the real readme");
write(pdir.path(), "src/main.rs", "fn main() {}");
publisher.run_barrier().await.unwrap();
assert!(claim_until_held(&mut publisher, 3).await);
let held = publisher.lease.clone().expect("the publisher holds a lease");
let rdir = tempfile::tempdir().unwrap();
let mut reader = syncer(&store, rdir.path()).await;
let done = tokio::time::timeout(
std::time::Duration::from_secs(2),
super::verbs::run_verb(&mut reader, super::verbs::Step::Checkout),
)
.await;
match done {
Err(_) => panic!(
"the checkout verb waited on the publisher's lease — it has no business \
holding the publish fence"
),
Ok(r) => r.expect("the checkout itself must succeed"),
}
assert_eq!(read(rdir.path(), "README.md").unwrap(), "the real readme");
assert_eq!(read(rdir.path(), "src/main.rs").unwrap(), "fn main() {}");
let cell = store.epoch_read(&reader.cfg.epoch_key()).await.unwrap().expect("the cell stands");
assert_eq!(cell.holder_id, held.holder_id, "the reader deposed the publisher");
assert_eq!(cell.epoch, held.epoch, "the reader moved the epoch");
assert!(!cell.released, "the reader released someone else's lease");
assert!(reader.lease.is_none(), "the reader came away holding a lease");
}
#[tokio::test]
async fn a_checkout_issues_no_write_to_the_bucket() {
let store = Arc::new(MemoryStore::new());
let pdir = tempfile::tempdir().unwrap();
let mut publisher = syncer(&store, pdir.path()).await;
assert!(claim_until_held(&mut publisher, 3).await);
publisher.checkout().await.unwrap();
for i in 0..8 {
write(pdir.path(), &format!("f{i}.txt"), &format!("body {i}"));
}
publisher.run_barrier().await.unwrap();
let rdir = tempfile::tempdir().unwrap();
let mut reader = syncer(&store, rdir.path()).await;
store.reset_op_counts();
super::verbs::run_verb(&mut reader, super::verbs::Step::Checkout).await.unwrap();
let ops = store.op_counts();
for op in [
"put_whole",
"delete",
"delete_version",
"copy_object",
"compose_generation",
"epoch_acquire",
"epoch_renew",
"epoch_release",
"abort_upload",
] {
assert_eq!(
ops.get(op).copied().unwrap_or(0),
0,
"checkout called {op} — it is no longer a read, and `read_only_then` is no \
longer justified. Full shape: {ops:?}"
);
}
assert!(ops.values().sum::<u64>() > 0, "the checkout made no requests at all: {ops:?}");
assert_eq!(reader.state.load_baseline().unwrap().entries.len(), 8);
}
struct PublishMidCheckout {
inner: Arc<MemoryStore>,
cfg: LeanConfig,
trigger_key: String,
trigger_path: String,
fired: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl ObjectStore for PublishMidCheckout {
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
if key == self.trigger_key
&& !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst)
{
let loaded = manifest::load(self.inner.as_ref(), &self.cfg)
.await
.expect("the publisher reads the standing manifest")
.expect("there is one");
let body = Bytes::from_static(b"the publisher's next generation");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 2,
epoch: 1,
flush_uuid: "the-publisher".into(),
boundary_source: None,
posix: None,
};
let new_key = self.cfg.handle_key(&self.trigger_path, "the-publisher");
let meta = self
.inner
.put_whole(&new_key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
.await?;
let mut m = loaded.manifest.clone();
m.seq += 1;
let e = m.entries.get_mut(&self.trigger_path).expect("the path is cited");
let retired = std::mem::replace(&mut e.key, new_key);
e.etag = meta.etag;
e.crc64_b64 = flint_store::crc64_to_b64(crc);
e.size = meta.size;
e.generation = 2;
manifest::cas_write(
self.inner.as_ref(),
&self.cfg,
&m,
Some(&loaded.handle()),
1,
"the-publisher",
)
.await
.expect("the publish lands");
self.inner.delete(&retired).await?;
}
self.inner.get_whole(key, if_match).await
}
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.inner.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_publish_that_lands_mid_checkout_is_re_resolved_not_blamed_on_a_stranger() {
let inner = Arc::new(MemoryStore::new());
let pdir = tempfile::tempdir().unwrap();
let mut publisher = syncer(&inner, pdir.path()).await;
publisher.cfg.sole_writer = true;
assert!(claim_until_held(&mut publisher, 3).await);
publisher.checkout().await.unwrap();
write(pdir.path(), "README.md", "generation one");
publisher.run_barrier().await.unwrap();
let rdir = tempfile::tempdir().unwrap();
let mut reader = syncer(&inner, rdir.path()).await;
let racing = Arc::new(PublishMidCheckout {
inner: inner.clone(),
cfg: reader.cfg.clone(),
trigger_key: cited_key(&inner, &reader.cfg, "README.md").await,
trigger_path: "README.md".to_string(),
fired: std::sync::atomic::AtomicBool::new(false),
});
reader.store = racing.clone() as Arc<dyn ObjectStore>;
let r = reader.checkout().await.expect("a reader one generation behind re-resolves and delivers");
assert!(racing.fired.load(std::sync::atomic::Ordering::SeqCst), "fixture: the publish never landed mid-checkout");
assert_eq!(r.reresolved, 1, "the pass was not re-run against the moved pointer: {r:?}");
assert_eq!(
read(rdir.path(), "README.md").as_deref(),
Some("the publisher's next generation"),
"the reader delivered a generation the pointer no longer names"
);
let b = reader.state.load_baseline().unwrap();
assert_eq!(b.seq, 2, "the baseline names the generation the tree holds");
assert!(reader.state.load_conflicts().unwrap().is_empty(), "nothing to surface: the publisher moved it");
}
#[test]
fn siblings_racing_to_create_one_parent_do_not_refuse_each_other() {
let dir = tempfile::tempdir().unwrap();
for round in 0..50 {
let root = dir.path().join(format!("r{round}"));
std::fs::create_dir(&root).unwrap();
let go = Arc::new(std::sync::Barrier::new(8));
let hs: Vec<_> = (0..8)
.map(|i| {
let root = root.clone();
let go = go.clone();
std::thread::spawn(move || {
go.wait();
super::barrier::contained_path(&root, &format!("new/deeper/f-{i}"))
})
})
.collect();
for h in hs {
h.join().unwrap().expect("a sibling's mkdir is not a containment refusal");
}
assert!(root.join("new/deeper").is_dir());
}
}
#[test]
fn the_temp_retry_paths_replace_a_leftover_and_recreate_a_vanished_parent() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::create_dir_all(root.join("d")).unwrap();
std::fs::write(root.join("d/f.txt.flint-sync-tmp"), b"STALE").unwrap();
let st = super::barrier::write_file_atomic(&root.join("d/f.txt"), b"NEW", Some(0o644)).unwrap();
assert_eq!(std::fs::read(root.join("d/f.txt")).unwrap(), b"NEW");
assert_eq!(st.len(), 3, "the returned metadata is the written file's, not the leftover's");
assert!(!root.join("d/f.txt.flint-sync-tmp").exists(), "the temp name is gone after the rename");
let outside = tempfile::tempdir().unwrap();
std::fs::write(outside.path().join("victim"), b"ORIGINAL").unwrap();
std::os::unix::fs::symlink(outside.path().join("victim"), root.join("d/g.txt.flint-sync-tmp"))
.unwrap();
super::barrier::write_file_atomic(&root.join("d/g.txt"), b"NEW", None).unwrap();
assert_eq!(std::fs::read(outside.path().join("victim")).unwrap(), b"ORIGINAL");
assert_eq!(std::fs::read(root.join("d/g.txt")).unwrap(), b"NEW");
let (target, present) = super::barrier::contained_path_stat(root, "gone/h.txt").unwrap();
assert!(present.is_none(), "the walk reports an absent final component as None");
std::fs::remove_dir(root.join("gone")).unwrap();
super::barrier::write_file_atomic(&target, b"NEW", None).unwrap();
assert_eq!(std::fs::read(root.join("gone/h.txt")).unwrap(), b"NEW");
let (_, present) = super::barrier::contained_path_stat(root, "gone/h.txt").unwrap();
assert_eq!(present.map(|m| m.len()), Some(3));
let missing = root.join("nostate/x.json");
let err = super::safefs::write_via_tmp(&missing, &root.join("nostate/x.json.tmp"), b"{}", None)
.unwrap_err();
assert!(err.to_string().contains("not exclusively creatable"), "{err}");
assert!(!root.join("nostate").exists());
}
#[tokio::test]
async fn a_fresh_fetch_whose_bytes_do_not_match_the_manifest_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "model/config.json", "{\"layers\": 12, \"hidden\": 768}");
write(dir_a.path(), "README.md", "intact");
a.run_barrier().await.unwrap();
store.inject_corrupt_body(&cited_key(&store, &a.cfg, "model/config.json").await, |b| b[10] ^= 0x01);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let err = b.checkout().await.expect_err("a corrupt fresh fetch must refuse");
let msg = err.to_string();
assert!(msg.contains("CRC-64") && msg.contains("model/config.json"), "{msg}");
assert!(
!dir_b.path().join("model/config.json").exists(),
"nothing may be written for a body that fails the check"
);
assert!(!dir_b.path().join("model/config.json.flint-sync-tmp").exists());
}
#[tokio::test]
async fn a_ranged_fetch_whose_bytes_do_not_match_the_manifest_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big: Vec<u8> = (0..5 * 1024 * 1024).map(|i| (i % 251) as u8).collect();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
a.run_barrier().await.unwrap();
store.inject_corrupt_body(&cited_key(&store, &a.cfg, "weights.bin").await, |b| {
let i = b.len() / 2 + 777;
b[i] ^= 0x01;
});
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.range_get_min_bytes = 1024 * 1024;
b.cfg.range_get_chunk_bytes = 512 * 1024; b.cfg.range_get_parallelism = 4;
let err = b.checkout().await.expect_err("a corrupt ranged fetch must refuse");
let msg = err.to_string();
assert!(msg.contains("CRC-64") && msg.contains("10 ranges"), "{msg}");
assert!(!dir_b.path().join("weights.bin").exists(), "a wrong object must never be renamed into place");
assert!(!dir_b.path().join("weights.bin.flint-sync-tmp").exists(), "the temp file is cleaned up");
}
struct AttestsNoChecksum(Arc<MemoryStore>);
fn unattested(mut m: flint_store::ObjectMeta) -> flint_store::ObjectMeta {
m.crc64_b64 = None;
m
}
#[async_trait::async_trait]
impl ObjectStore for AttestsNoChecksum {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await.map(unattested)
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.put_whole(key, body, cond, stamps, crc).await.map(unattested)
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head(key).await.map(unattested)
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_whole(key, if_match).await.map(|(m, b)| (unattested(m), b))
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.0.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.0.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
self.0.delete_if_match(key, etag).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await.map(unattested)
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await.map(|(m, b)| (unattested(m), b))
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
async fn epoch_handoff(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<()> {
self.0.epoch_handoff(key, lease, echo).await
}
async fn epoch_enqueue(
&self,
key: &str,
observed: &flint_store::EpochState,
holder_id: &str,
) -> flint_store::StoreResult<flint_store::EpochState> {
self.0.epoch_enqueue(key, observed, holder_id).await
}
}
fn syncer_on(store: Arc<dyn ObjectStore>, root: &std::path::Path) -> Syncer {
let cfg = cfg_for(root);
let state = SyncerState::open(cfg.state_dir()).unwrap();
Syncer { store, cfg, state, lease: None, cell_written_at: None, noted_not_regular: Default::default() }
}
fn crc_of(content: &str) -> String {
flint_store::crc64_to_b64(crc64_nvme(content.as_bytes()))
}
async fn assert_attests_nothing(mem: &Arc<MemoryStore>, store: &Arc<dyn ObjectStore>, key: &str) {
assert!(mem.head(key).await.unwrap().crc64_b64.is_some(), "the fixture object has a CRC");
assert!(store.head(key).await.unwrap().crc64_b64.is_none(), "HEAD attests a checksum");
let (m, _) = store.get_whole(key, None).await.unwrap();
assert!(m.crc64_b64.is_none(), "GET attests a checksum");
}
#[tokio::test]
async fn a_hitl_upload_on_a_backend_that_attests_no_checksum_is_cited_with_the_bytes_crc() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(AttestsNoChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer_on(store.clone(), dir.path());
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
let key = sc.cfg.file_key("docs/upload.pdf");
let body = Bytes::from_static(b"user bytes");
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "gateway-old".into(),
boundary_source: None,
posix: None,
};
let meta = mem
.put_whole(&key, body.clone(), &PutCondition::IfNoneMatchAny, &stamps, crc64_nvme(&body))
.await
.unwrap();
let (etag, k) = (meta.etag.clone(), key.clone());
gateway_commit(&mem, &sc.cfg, "gateway-old", move |doc| {
doc.entries.insert("docs/upload.pdf".into(), manifest::LeanEntry {
key: k.clone(), etag: etag.clone(), crc64_b64: crc_of("user bytes"), size: 10,
mode: 0o644, mtime_unix: 0, generation: 1, epoch: 0,
});
})
.await
.unwrap();
assert_attests_nothing(&mem, &store, &key).await;
write(dir.path(), "agent.txt", "agent work v2 — longer");
sc.run_barrier().await.unwrap();
sc.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "docs/upload.pdf").unwrap(), "user bytes");
let b = sc.state.load_baseline().unwrap();
assert_eq!(
b.entries["docs/upload.pdf"].crc64_b64.as_deref(),
Some(crc_of("user bytes").as_str()),
"the baseline carries the CRC of the bytes the consume wrote"
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
let e = m.entries.get("docs/upload.pdf").expect("amputated!");
assert_eq!(e.crc64_b64, crc_of("user bytes"), "the citation carries the bytes' CRC");
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer_on(store.clone(), dir2.path());
sc2.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "docs/upload.pdf").unwrap(), "user bytes");
}
#[tokio::test]
async fn sync_on_a_backend_that_attests_no_checksum_still_sees_identical_bytes() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(AttestsNoChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let mut a = syncer_on(store.clone(), dir.path());
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
let (_, ui_key) = hitl_write_at(&mem, &a.cfg, "shared.txt", "foreign v2", "ci", now_unix()).await.unwrap();
assert_attests_nothing(&mem, &store, &ui_key).await;
let pre = a.state.load_baseline().unwrap();
let r = a.sync().await.unwrap();
assert!(r.applied.contains(&"shared.txt".to_string()));
let mut b = a.state.load_baseline().unwrap();
let stale = b.entries.get_mut("shared.txt").unwrap();
stale.etag = "\"stale-pre-sync-etag\"".into();
stale.mtime_unix -= 10;
stale.size = 1;
b.seq = pre.seq;
b.manifest_etag = pre.manifest_etag.clone();
a.state.save_baseline(&b).unwrap();
let scanned = super::scan::scan(dir.path()).unwrap();
let c = super::scan::classify(&scanned, &a.state.load_baseline().unwrap());
assert!(c.uploads.contains("shared.txt"), "the fixture is not actually dirty");
let r = a.sync().await.unwrap();
assert!(
!r.conflicts.contains(&"shared.txt".to_string()),
"a phantom conflict was reported for byte-identical content"
);
assert!(r.applied.contains(&"shared.txt".to_string()));
}
#[tokio::test]
async fn a_consume_whose_bytes_do_not_match_the_writers_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
let (_, ui_key) =
hitl_write_at(&store, &sc.cfg, "docs/upload.pdf", "user bytes", "dilip", now_unix()).await.unwrap();
store.inject_corrupt_body(&ui_key, |b| b[3] ^= 0x40);
sc.run_barrier().await.unwrap();
assert!(read(dir.path(), "docs/upload.pdf").is_none(), "corrupt bytes were written");
assert!(
sc.state.load_conflicts().unwrap().iter().any(|c| c.kind.starts_with("consume-refused-checksum")),
"the refusal was silent"
);
assert!(derive_pending(&sc).await, "a refused change must stay owed, not be consumed away");
let n = sc.state.load_conflicts().unwrap().len();
sc.run_barrier().await.unwrap();
assert!(read(dir.path(), "docs/upload.pdf").is_none());
assert!(sc.state.load_conflicts().unwrap().len() > n, "the refusal did not repeat");
}
#[tokio::test]
async fn a_sync_whose_bytes_do_not_match_the_manifest_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "published bytes");
a.run_barrier().await.unwrap();
store.inject_corrupt_body(&cited_key(&store, &a.cfg, "f.txt").await, |bytes| bytes[0] ^= 0x01);
let err = b.sync().await.expect_err("a sync of corrupt bytes must refuse");
assert!(err.to_string().contains("refusing to apply it"), "{err}");
assert!(read(dir_b.path(), "f.txt").is_none(), "corrupt bytes were written");
}
pub(super) async fn hitl_remove(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str, _author: &str) {
gateway_commit(store, cfg, &super::ui_flush(now_unix()), |doc| {
let gone = doc.entries.remove(path).expect("the gateway deletes only what is cited");
doc.tombstones.insert(path.to_string(), manifest::Tombstone { etag: gone.etag, seq: doc.seq });
})
.await
.unwrap();
}
#[tokio::test]
async fn a_ui_delete_whose_cas_landed_behind_a_lost_response_is_answered_committed() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&inner, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "a.txt", "published");
a.run_barrier().await.unwrap();
let before = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
hooked.put_lands_then_412(&a.cfg.current_key());
let r = manifest::commit_edit(hooked.as_ref(), &a.cfg, &super::ui_flush(now_unix()), |_, doc| {
let Some(gone) = doc.entries.remove("a.txt") else {
return Err("NoSuchFile(a.txt)");
};
doc.tombstones.insert("a.txt".to_string(), manifest::Tombstone { etag: gone.etag, seq: doc.seq });
Ok(())
})
.await;
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("a.txt") && m.seq == before + 1, "fixture: the delete did not commit once");
assert!(r.is_ok(), "the delete committed at seq {} and the UI was told it failed: {:?}", m.seq, r.err());
}
pub(super) async fn gateway_current(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str) -> Option<String> {
manifest::load(store.as_ref(), cfg).await.unwrap().and_then(|m| m.manifest.entries.get(path).map(|e| e.etag.clone()))
}
pub(super) async fn hitl_rename(store: &Arc<MemoryStore>, cfg: &LeanConfig, from: &str, to: &str, _author: &str) {
gateway_commit(store, cfg, &super::ui_flush(now_unix()), |doc| {
let entry = doc.entries.remove(from).expect("the source is cited");
assert!(!doc.entries.contains_key(to), "the gateway refuses a destination that exists");
doc.tombstones.insert(from.to_string(), manifest::Tombstone { etag: entry.etag.clone(), seq: doc.seq });
doc.tombstones.remove(to);
doc.entries.insert(to.to_string(), entry);
})
.await
.unwrap();
}
#[tokio::test]
async fn a_failed_unlink_of_a_queued_deletion_is_retried_and_never_republishes() {
if unsafe { libc::geteuid() } == 0 {
eprintln!("skipped: root ignores directory permissions");
return;
}
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "locked/f.txt", "keep");
sc.run_barrier().await.unwrap();
use std::os::unix::fs::PermissionsExt;
let locked = dir.path().join("locked");
hitl_remove(&store, &sc.cfg, "locked/f.txt", "dilip").await;
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o555)).unwrap();
for _ in 0..3 {
sc.run_barrier().await.unwrap();
}
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap();
assert_eq!(read(dir.path(), "locked/f.txt").as_deref(), Some("keep"), "fixture: the unlink did not fail");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("locked/f.txt"), "a failed unlink re-published the deleted file");
for _ in 0..2 {
sc.run_barrier().await.unwrap();
}
assert!(read(dir.path(), "locked/f.txt").is_none(), "the deletion was dropped, not retried");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("locked/f.txt"));
}
#[tokio::test]
async fn an_invalid_sync_scope_is_acked_refused_and_never_wedges_publish() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"s1","scope":[]}"#);
a.sentinel_tick().await.expect("an invalid scope is the agent's error, not the syncer's");
let ack = a.read_ack(Verb::Sync).expect("no sync ack: the agent waits forever");
assert_eq!(ack.status, "refused-scope");
assert!(ack.nonces.contains(&"s1".to_string()), "the refusal does not name the agent's nonce");
assert!(
ack.reason.as_deref().unwrap_or("").contains("scope"),
"the refusal must say WHY: {:?}",
ack.reason
);
assert!(
a.load_pending(Verb::Sync).unwrap().is_none(),
"the pending record stood: every later tick repeats the same error"
);
write(dir.path(), "b.txt", "b");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"p1"}"#);
a.sentinel_tick().await.unwrap();
let ack = a.read_ack(Verb::Publish).expect("no publish ack after a refused sync");
assert_eq!(ack.status, "ok");
assert!(ack.nonces.contains(&"p1".to_string()));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("b.txt"), "the boundary the agent declared never reached the bucket");
}
#[tokio::test]
async fn a_refused_sync_pending_does_not_stop_the_cadence_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"s1","scope":["../escape"]}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
write(dir.path(), "c.txt", "c");
a.floor_tick().await.expect("a refused pending must not fail the floor tick");
assert_eq!(a.read_ack(Verb::Sync).map(|k| k.status), Some("refused-scope".into()));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("c.txt"), "the cadence barrier never ran behind the refused sync");
}
#[tokio::test]
async fn an_unscoped_sync_touch_widens_a_coalesced_scope_to_the_whole_tree() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"b","scope":["x/"]}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"a"}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
let p = a.load_pending(Verb::Sync).unwrap().unwrap();
assert!(p.scope.is_none(), "an unscoped touch was narrowed to {:?}", p.scope);
assert_eq!(p.nonces, vec!["b".to_string(), "a".to_string()]);
a.sentinel_tick().await.unwrap();
assert!(a.load_pending(Verb::Sync).unwrap().is_none());
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"c"}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"d","scope":["y/"]}"#);
assert!(a.consume_sentinel(Verb::Sync).unwrap());
let p = a.load_pending(Verb::Sync).unwrap().unwrap();
assert!(p.scope.is_none(), "a later scoped touch narrowed a whole-tree request to {:?}", p.scope);
}
#[tokio::test]
async fn a_publish_ack_carries_the_boundarys_conflict_records() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
write(dir.path(), "shared.txt", "the agent's longer edit");
hitl_write(&store, &a.cfg, "shared.txt", "UI", "ui").await.unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"p1"}"#);
a.sentinel_tick().await.unwrap();
let ack = a.read_ack(Verb::Publish).expect("no publish ack");
assert_eq!(ack.status, "ok");
assert!(
ack.report.conflicts.iter().any(|c| c.path == "shared.txt" && c.kind.starts_with("commit-surfaced-foreign")),
"the publish ack carries no conflict record for the path the agent won: {:?}",
ack.report.conflicts
);
}
#[tokio::test]
async fn a_same_size_rewrite_within_the_scan_second_is_still_published() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "status.json", r#"{"ok":1}"#);
write(dir.path(), "other.txt", "untouched");
a.run_barrier().await.unwrap();
let sec = a.state.load_baseline().unwrap().entries["status.json"].mtime_unix;
write(dir.path(), "status.json", r#"{"ok":2}"#);
let f = std::fs::OpenOptions::new().write(true).open(dir.path().join("status.json")).unwrap();
f.set_modified(std::time::UNIX_EPOCH + std::time::Duration::new(sec as u64, 500_000_000)).unwrap();
drop(f);
let r = a.run_barrier().await.unwrap();
assert!(
r.uploaded.contains(&"status.json".to_string()),
"the rewrite was invisible: uploaded={:?} no_change={}",
r.uploaded,
r.no_change
);
assert!(!r.uploaded.contains(&"other.txt".to_string()), "nanosecond compare re-uploads everything");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, body) = store.get_whole(&m.entries["status.json"].key, Some(&m.entries["status.json"].etag)).await.unwrap();
assert_eq!(&body[..], br#"{"ok":2}"#);
}
async fn a_collected_predecessor_costs_the_upload_nothing(answer_412: bool, compose: bool) {
let store = if answer_412 {
Arc::new(MemoryStore::new().with_if_match_missing_as_412())
} else {
Arc::new(MemoryStore::new())
};
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
if compose {
a.cfg.whole_put_max = 4; }
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "x.txt", "v1 body");
a.run_barrier().await.unwrap();
let v1 = cited_key(&store, &a.cfg, "x.txt").await;
store.delete(&v1).await.unwrap();
assert!(handles_for(&store, &a.cfg, "x.txt").await.is_empty(), "fixture: the predecessor is gone");
write(dir.path(), "x.txt", "v2, longer than before");
let r = a.run_barrier().await.expect("a collected predecessor failed the whole barrier");
assert!(r.uploaded.contains(&"x.txt".to_string()), "the edit was not published: {r:?}");
assert_eq!(cited_bytes(&store, &a.cfg, "x.txt").await.as_deref(), Some(&b"v2, longer than before"[..]));
let cited = cited_key(&store, &a.cfg, "x.txt").await;
assert_ne!(cited, v1);
assert!(a.cfg.handle_parts(&cited).is_some(), "not a handle: {cited}");
assert!(r.parked.is_empty() && r.deferred.is_empty(), "{r:?}");
}
#[tokio::test]
async fn a_collected_predecessor_costs_the_upload_nothing_on_s3s_404() {
a_collected_predecessor_costs_the_upload_nothing(false, false).await;
}
#[tokio::test]
async fn a_collected_predecessor_costs_the_upload_nothing_on_ozones_412() {
a_collected_predecessor_costs_the_upload_nothing(true, false).await;
}
#[tokio::test]
async fn a_collected_predecessor_costs_the_upload_nothing_through_compose() {
a_collected_predecessor_costs_the_upload_nothing(false, true).await;
}
#[test]
fn containment_refuses_the_state_directory_too() {
let dir = tempfile::tempdir().unwrap();
super::barrier::contained_path(dir.path(), ".flint/x").expect_err("the control namespace must be refused");
super::barrier::contained_path(dir.path(), ".flint-sync/scope-intent.json")
.expect_err("the state directory must be refused");
super::barrier::contained_path(dir.path(), ".flint-sync").expect_err("the state directory itself");
super::barrier::contained_path(dir.path(), "ok/x").expect("an ordinary path");
super::barrier::contained_path(dir.path(), ".flint-syncer/notes").expect("a look-alike is not the state dir");
}
#[tokio::test]
async fn an_orphaned_consume_temp_is_never_published() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "a.txt", "real");
write(dir.path(), "a.txt.flint-sync-tmp", "torn consume leftover");
let r = a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("a.txt"));
assert!(
!m.entries.contains_key("a.txt.flint-sync-tmp") && !r.uploaded.iter().any(|p| p.ends_with(".flint-sync-tmp")),
"a crash-orphaned temp was published as data: {:?}",
r.uploaded
);
}
#[tokio::test]
async fn adopting_a_lost_renew_token_moves_the_cell() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let key = a.cfg.epoch_key();
let stale = a.lease.clone().unwrap();
store.epoch_renew(&key, &stale, None).await.unwrap();
let landed = store.epoch_read(&key).await.unwrap().unwrap().token;
lease::renew(&mut a).await.expect("a lost renew response is not a deposal");
let now = store.epoch_read(&key).await.unwrap().unwrap().token;
assert_ne!(now, landed, "the adoption wrote nothing: a challenger's quiet count keeps climbing");
assert_eq!(a.lease.as_ref().map(|l| l.token.as_str()), Some(now.as_str()), "the holder's token is not the cell's");
let cell = store.epoch_read(&key).await.unwrap().unwrap();
assert_eq!(cell.holder_id, a.lease.as_ref().unwrap().holder_id);
assert!(!cell.released);
}
type Hook = Box<dyn Fn() + Send + Sync>;
#[derive(Default)]
pub(super) struct Hooks {
before_put: std::sync::Mutex<Option<(String, Hook)>>,
after_put: std::sync::Mutex<Option<(String, Hook)>>,
before_get_return: std::sync::Mutex<Option<(String, Hook)>>,
before_delete: std::sync::Mutex<Option<(String, Hook)>>,
after_head: std::sync::Mutex<Option<(String, Hook)>>,
before_compose: std::sync::Mutex<Option<(String, Hook)>>,
compose_err_once: std::sync::atomic::AtomicBool,
acquire_err_once: std::sync::atomic::AtomicBool,
copy_fail: std::sync::atomic::AtomicBool,
put_fail_containing: std::sync::Mutex<Option<String>>,
handoff_races: std::sync::atomic::AtomicU32,
handoff_conflicts: std::sync::atomic::AtomicU32,
put_conflicts: std::sync::Mutex<Option<(String, u32)>>,
put_lands_then_412: std::sync::Mutex<Option<String>>,
}
pub(super) struct Hooked(pub(super) Arc<MemoryStore>, pub(super) Hooks);
impl Hooked {
pub(super) fn put_lands_then_412(&self, suffix: &str) {
*self.1.put_lands_then_412.lock().unwrap() = Some(suffix.to_string());
}
fn before_put(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.before_put.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
pub(super) fn after_put(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.after_put.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
fn before_compose(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.before_compose.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
fn before_get_return(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.before_get_return.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
pub(super) fn before_delete(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.before_delete.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
pub(super) fn after_head(&self, key: &str, f: impl Fn() + Send + Sync + 'static) {
*self.1.after_head.lock().unwrap() = Some((key.to_string(), Box::new(f)));
}
}
fn hook_matches(key: &str, suffix: &str) -> bool {
if key.ends_with(suffix) {
return true;
}
key.rsplit_once('@').map(|(path, _)| path.ends_with(suffix)).unwrap_or(false)
}
fn take_hook(slot: &std::sync::Mutex<Option<(String, Hook)>>, key: &str) -> Option<Hook> {
let mut g = slot.lock().unwrap();
if g.as_ref().map(|(suf, _)| hook_matches(key, suf)).unwrap_or(false) {
g.take().map(|(_, h)| h)
} else {
None
}
}
pub(super) fn hooked_syncer(store: &Arc<Hooked>, root: &std::path::Path) -> Syncer {
let cfg = cfg_for(root);
Syncer {
store: store.clone() as Arc<dyn ObjectStore>,
state: SyncerState::open(cfg.state_dir()).unwrap(),
cfg,
lease: None,
cell_written_at: None,
noted_not_regular: Default::default(),
}
}
#[async_trait::async_trait]
impl ObjectStore for Hooked {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
if self.1.copy_fail.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Other("injected: preserve copy failed".into()));
}
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
if let Some(h) = take_hook(&self.1.before_put, key) {
h();
}
if let Some(needle) = self.1.put_fail_containing.lock().unwrap().as_deref() {
if key.contains(needle) {
return Err(flint_store::StoreError::Other(format!("injected: put refused for {key}")));
}
}
{
let mut g = self.1.put_conflicts.lock().unwrap();
if let Some((suffix, n)) = g.as_mut() {
if *n > 0 && key.ends_with(suffix.as_str()) {
*n -= 1;
return Err(flint_store::StoreError::Conflict(format!(
"injected: ConditionalRequestConflict on {key}"
)));
}
}
}
let r = self.0.put_whole(key, body, cond, stamps, crc).await;
if r.is_ok() {
if let Some(h) = take_hook(&self.1.after_put, key) {
h();
}
}
if r.is_ok() {
let mut g = self.1.put_lands_then_412.lock().unwrap();
if g.as_deref().is_some_and(|s| key.ends_with(s)) {
*g = None;
return Err(flint_store::StoreError::PreconditionFailed(format!(
"injected: the retry of a PUT that landed on {key}"
)));
}
}
r
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
if let Some(h) = take_hook(&self.1.before_compose, spec.key) {
h();
}
let r = self.0.compose_generation(spec).await;
if self.1.compose_err_once.swap(false, std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Other("injected: torn compose response".into()));
}
r
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
let r = self.0.head(key).await;
if let Some(h) = take_hook(&self.1.after_head, key) {
h();
}
r
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
let r = self.0.get_whole(key, if_match).await;
if let Some(h) = take_hook(&self.1.before_get_return, key) {
h();
}
r
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.0.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
if let Some(h) = take_hook(&self.1.before_delete, key) {
h();
}
self.0.delete(key).await
}
async fn delete_if_match(&self, key: &str, etag: &str) -> flint_store::StoreResult<()> {
if let Some(h) = take_hook(&self.1.before_delete, key) {
h();
}
self.0.delete_if_match(key, etag).await
}
async fn delete_many(&self, keys: &[String]) -> flint_store::StoreResult<flint_store::DeleteManyReport> {
for key in keys {
if let Some(h) = take_hook(&self.1.before_delete, key) {
h();
}
}
self.0.delete_many(keys).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
let r = self.0.epoch_acquire(key, holder, observed).await;
if self.1.acquire_err_once.swap(false, std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Other("injected: lost acquire response".into()));
}
r
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
async fn epoch_handoff(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<()> {
use std::sync::atomic::Ordering::SeqCst;
if self.1.handoff_conflicts.fetch_update(SeqCst, SeqCst, |n| n.checked_sub(1)).is_ok() {
return Err(flint_store::StoreError::Conflict("injected: ConditionalRequestConflict".into()));
}
if let Ok(left) = self.1.handoff_races.fetch_update(SeqCst, SeqCst, |n| n.checked_sub(1)) {
let state = self.0.epoch_read(key).await?.expect("a cell to race");
self.0.epoch_enqueue(key, &state, &format!("synthetic-waiter-{left}")).await?;
}
self.0.epoch_handoff(key, lease, echo).await
}
async fn epoch_enqueue(
&self,
key: &str,
observed: &flint_store::EpochState,
holder_id: &str,
) -> flint_store::StoreResult<flint_store::EpochState> {
self.0.epoch_enqueue(key, observed, holder_id).await
}
}
#[tokio::test]
async fn the_manifest_cites_the_uploaded_length_not_the_scanned_one() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
a.cfg.whole_put_max = 1 << 20;
a.cfg.range_get_min_bytes = 1 << 20;
a.cfg.range_get_chunk_bytes = 1 << 20;
a.checkout().await.unwrap();
let path = dir.path().join("ckpt.bin");
std::fs::write(&path, vec![7u8; 4 << 20]).unwrap();
write(dir.path(), "aaa-first.txt", "uploads before ckpt.bin");
a.cfg.upload_fanout = 1;
let grow = path.clone();
hooked.before_put(&a.cfg.file_key("aaa-first.txt"), move || {
use std::io::Write;
std::fs::OpenOptions::new().append(true).open(&grow).unwrap().write_all(&vec![9u8; 2 << 20]).unwrap();
});
a.run_barrier().await.unwrap();
let head = inner.head(&cited_key(&inner, &a.cfg, "ckpt.bin").await).await.unwrap();
assert_eq!(head.size, 6 << 20, "fixture: the upload did not carry the grown file");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
m.entries["ckpt.bin"].size, head.size,
"the manifest cites a length the object does not have"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = hooked_syncer(&hooked, dir_b.path());
b.cfg.whole_put_max = 1 << 20;
b.cfg.range_get_min_bytes = 1 << 20;
b.cfg.range_get_chunk_bytes = 1 << 20;
assert!(claim_until_held(&mut b, 12).await, "takeover");
b.checkout().await.expect("the successor cannot check out the boundary: the workspace is unrecoverable");
assert_eq!(std::fs::metadata(dir_b.path().join("ckpt.bin")).unwrap().len(), 6 << 20);
}
#[tokio::test]
async fn a_consume_never_overwrites_a_write_that_landed_during_its_fetch() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
hitl_write(&inner, &a.cfg, "shared.txt", "UI", "ui").await.unwrap();
let late = dir.path().join("shared.txt");
hooked.before_get_return(&a.cfg.file_key("shared.txt"), move || {
std::fs::write(&late, "AGENT-LATE, a different length").unwrap();
});
a.run_barrier().await.unwrap();
assert_eq!(
read(dir.path(), "shared.txt").as_deref(),
Some("AGENT-LATE, a different length"),
"the agent's write was overwritten by a consume that checked the path before its fetch"
);
let recs = a.state.load_conflicts().unwrap();
assert!(
recs.iter().any(|c| c.path == "shared.txt" && c.kind.starts_with("commit-surfaced-foreign") && c.preserved_key.is_some()),
"no record names the foreign version that lost: {recs:?}"
);
}
#[tokio::test]
async fn the_upload_refuses_a_symlink_swapped_in_after_the_scan() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
a.checkout().await.unwrap();
let outside = tempfile::tempdir().unwrap();
std::fs::write(outside.path().join("secret"), "SECRET").unwrap();
write(dir.path(), "cfg.yaml", "harmless");
write(dir.path(), "aaa-first.txt", "uploads before cfg.yaml");
a.cfg.upload_fanout = 1;
let p = dir.path().join("cfg.yaml");
let target = outside.path().join("secret");
hooked.before_put(&a.cfg.file_key("aaa-first.txt"), move || {
std::fs::remove_file(&p).unwrap();
std::os::unix::fs::symlink(&target, &p).unwrap();
});
let r = a.run_barrier().await.expect("a planted symlink must not fail the barrier");
assert!(!r.uploaded.contains(&"cfg.yaml".to_string()), "the link's target was published: {r:?}");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("cfg.yaml"), "the manifest cites the link's target");
assert!(inner.head(&a.cfg.file_key("cfg.yaml")).await.is_err(), "the outside bytes reached the bucket");
let recs = a.state.load_conflicts().unwrap();
assert!(recs.iter().any(|c| c.path == "cfg.yaml"), "nothing recorded the refusal: {recs:?}");
}
#[tokio::test]
async fn a_lost_acquire_response_still_rotates() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir_a = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir_a.path());
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "x");
a.run_barrier().await.unwrap();
let seq0 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert!(claim_until_held(&mut a, 3).await);
let dir_b = tempfile::tempdir().unwrap();
let mut b = hooked_syncer(&hooked, dir_b.path());
loop {
match lease::claim_step(&mut b, true).await.unwrap() {
lease::ClaimOutcome::Waiting { quiet_polls, .. } if quiet_polls >= 5 => break,
lease::ClaimOutcome::Waiting { .. } => {}
lease::ClaimOutcome::Claimed(_) => panic!("fixture: claimed before the threshold"),
}
}
hooked.1.acquire_err_once.store(true, std::sync::atomic::Ordering::SeqCst);
assert!(lease::claim_step(&mut b, true).await.is_err(), "fixture: the lost response did not surface");
let cell = inner.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
let b_id = b.state.load_incarnation().unwrap().unwrap().holder_id;
assert_eq!(cell.holder_id, b_id, "fixture: the acquire did not land");
let outcome = lease::claim_step(&mut b, true).await.unwrap();
assert!(matches!(outcome, lease::ClaimOutcome::Claimed(_)), "the retry did not claim");
let seq1 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert!(
seq1 > seq0,
"a lost acquire response skipped the takeover rotation (seq {seq0} → {seq1}): a straggler's CAS still matches"
);
}
#[tokio::test]
async fn two_writers_publish_without_waiting_for_each_other() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "a/one.txt", "from A");
write(dir_b.path(), "b/one.txt", "from B");
let (ra, rb) = tokio::join!(a.run_barrier(), b.run_barrier());
let (ra, rb) = (ra.expect("A's barrier"), rb.expect("B's barrier"));
assert_eq!(ra.uploaded, vec!["a/one.txt"]);
assert_eq!(rb.uploaded, vec!["b/one.txt"]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("a/one.txt") && m.entries.contains_key("b/one.txt"), "{:?}", m.entries.keys());
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released && cell.handoff.is_none() && cell.waiters.is_empty(), "{cell:?}");
assert_eq!(cell.epoch, 2, "one epoch per barrier");
assert!(a.lease.is_none() && b.lease.is_none());
}
#[tokio::test]
async fn disjoint_edits_cross_at_the_next_consume() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "a/one.txt", "from A");
write(dir_b.path(), "b/one.txt", "from B");
a.run_barrier().await.unwrap();
let rb = b.run_barrier().await.unwrap();
assert_eq!(rb.consumed, 1, "B's consume did not integrate A's file");
assert_eq!(read(dir_b.path(), "a/one.txt").as_deref(), Some("from A"));
assert!(rb.uploaded.contains(&"b/one.txt".to_string()), "B's own edit did not publish: {rb:?}");
let ra2 = a.run_barrier().await.unwrap();
assert_eq!(ra2.consumed, 1, "A's consume did not integrate B's file");
assert_eq!(read(dir_a.path(), "b/one.txt").as_deref(), Some("from B"));
}
#[tokio::test]
async fn a_same_path_edit_is_preserved_never_lost() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"));
write(dir_a.path(), "x.txt", "A's edit");
backdate_baseline(&a, "x.txt");
write(dir_b.path(), "x.txt", "B's edit, longer");
backdate_baseline(&b, "x.txt");
a.run_barrier().await.unwrap();
let rb = b.run_barrier().await.unwrap();
assert!(rb.parked.is_empty(), "B's upload parked instead of preserving: {rb:?}");
assert_eq!(rb.surfaced, vec!["x.txt".to_string()], "B's commit did not surface what it published over: {rb:?}");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited = &m.entries["x.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"B's edit, longer", "the later commit is current");
let rec = b
.state
.load_conflicts()
.unwrap()
.into_iter()
.find(|c| c.path == "x.txt" && c.kind.starts_with("commit-surfaced-foreign"))
.expect("B wrote no record of the version it superseded");
let preserved = rec.preserved_key.expect("the record names no preserved key");
let (_, kept) = store.get_whole(&preserved, None).await.unwrap();
assert_eq!(&kept[..], b"A's edit", "A's bytes are not where the record says");
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(read(dir_a.path(), "x.txt").as_deref(), Some("B's edit, longer"));
}
#[tokio::test]
async fn only_the_queue_head_is_told_to_poll_fast() {
let store = Arc::new(MemoryStore::new());
let dirs: Vec<_> = (0..3).map(|_| tempfile::tempdir().unwrap()).collect();
let mut a = syncer(&store, dirs[0].path()).await;
let mut b = syncer(&store, dirs[1].path()).await;
let mut c = syncer(&store, dirs[2].path()).await;
let next = |o: lease::ClaimOutcome| match o {
lease::ClaimOutcome::Waiting { next, .. } => Some(next),
lease::ClaimOutcome::Claimed(_) => None,
};
assert!(claim_until_held(&mut a, 1).await);
assert_eq!(next(lease::claim_step(&mut b, true).await.unwrap()), Some(true), "B heads the queue behind A");
assert_eq!(next(lease::claim_step(&mut c, true).await.unwrap()), Some(false), "C is second");
assert_eq!(next(lease::claim_step(&mut b, false).await.unwrap()), Some(true), "B still heads it on a re-poll");
lease::release(&mut a).await.unwrap();
assert_eq!(next(lease::claim_step(&mut c, false).await.unwrap()), Some(false));
assert_eq!(next(lease::claim_step(&mut b, false).await.unwrap()), None, "B claims its reservation");
assert_eq!(next(lease::claim_step(&mut c, false).await.unwrap()), Some(true), "C heads the queue behind B");
}
#[tokio::test]
async fn a_deposed_holders_release_never_frees_its_successors_lease() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut a, 1).await);
assert!(claim_until_held(&mut b, 12).await, "fixture: B never deposed A");
let key = a.cfg.epoch_key();
let held = store.epoch_read(&key).await.unwrap().unwrap();
assert_eq!(held.holder_id, b.lease.as_ref().unwrap().holder_id, "fixture: the cell is not B's");
assert!(a.lease.is_some(), "fixture: A no longer believes it holds the cell");
lease::release(&mut a).await.unwrap();
let after = store.epoch_read(&key).await.unwrap().unwrap();
assert_eq!(after.holder_id, held.holder_id, "A's release moved B's cell: {after:?}");
assert!(!after.released && after.handoff.is_none(), "A's release freed B's lease: {after:?}");
assert_eq!((after.epoch, &after.token), (held.epoch, &held.token), "A's release wrote B's cell: {after:?}");
lease::renew(&mut b).await.expect("B's lease survives its predecessor's release");
}
#[tokio::test]
async fn a_handoff_that_races_the_queue_still_hands_the_cell_on() {
for (races, conflicts) in [(2u32, 0u32), (0, 1), (3, 2)] {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&hooked, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
assert!(claim_until_held(&mut a, 1).await);
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }));
hooked.1.handoff_races.store(races, std::sync::atomic::Ordering::SeqCst);
hooked.1.handoff_conflicts.store(conflicts, std::sync::atomic::Ordering::SeqCst);
lease::release(&mut a).await.unwrap_or_else(|e| panic!("races {races}, conflicts {conflicts}: the release failed: {e}"));
let cell = inner.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(
cell.released,
"races {races}, conflicts {conflicts}: the release returned and the cell is still held by {}",
cell.holder_id
);
let b_id = lease::incarnation(&b).unwrap().holder_id;
assert_eq!(cell.handoff.as_deref(), Some(b_id.as_str()), "races {races}, conflicts {conflicts}: the head of the queue was not named");
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Claimed(_)));
}
}
#[tokio::test]
async fn the_ticket_hands_the_fence_to_the_queue_head() {
let store = Arc::new(MemoryStore::new());
let dirs: Vec<_> = (0..3).map(|_| tempfile::tempdir().unwrap()).collect();
let mut a = syncer(&store, dirs[0].path()).await;
let mut b = syncer(&store, dirs[1].path()).await;
let mut c = syncer(&store, dirs[2].path()).await;
let id = |sc: &Syncer| lease::incarnation(sc).unwrap().holder_id;
assert!(claim_until_held(&mut a, 1).await);
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }));
assert!(matches!(lease::claim_step(&mut c, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }));
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!(cell.waiters, vec![id(&b), id(&c)], "waiters queue in arrival order");
lease::release(&mut a).await.unwrap();
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released);
assert_eq!(cell.handoff.as_deref(), Some(id(&b).as_str()), "the handoff names the head");
assert_eq!(cell.waiters, vec![id(&c)]);
assert!(
matches!(lease::claim_step(&mut c, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }),
"a later waiter took a cell reserved for the queue head"
);
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Claimed(_)));
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!((cell.holder_id.as_str(), cell.released), (id(&b).as_str(), false));
assert_eq!(cell.waiters, vec![id(&c)], "the queue survives the handoff");
lease::release(&mut b).await.unwrap();
assert!(matches!(lease::claim_step(&mut c, true).await.unwrap(), lease::ClaimOutcome::Claimed(_)));
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!(cell.epoch, 3);
assert!(cell.waiters.is_empty() && cell.handoff.is_none());
}
#[tokio::test]
async fn a_dead_handoff_is_skipped_after_the_quiet_polls() {
let store = Arc::new(MemoryStore::new());
let dirs: Vec<_> = (0..3).map(|_| tempfile::tempdir().unwrap()).collect();
let mut a = syncer(&store, dirs[0].path()).await;
let mut b = syncer(&store, dirs[1].path()).await;
let mut c = syncer(&store, dirs[2].path()).await;
assert!(claim_until_held(&mut a, 1).await);
assert!(matches!(lease::claim_step(&mut b, true).await.unwrap(), lease::ClaimOutcome::Waiting { .. }));
lease::release(&mut a).await.unwrap();
drop(b); assert!(!claim_until_held(&mut c, lease::HANDOFF_QUIET_POLLS).await, "C took a reservation that was not yet quiet");
assert!(claim_until_held(&mut c, 1).await, "C never took the dead reservation");
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!(cell.holder_id, lease::incarnation(&c).unwrap().holder_id);
assert!(cell.waiters.is_empty() && cell.handoff.is_none());
}
#[tokio::test]
async fn a_dead_holder_mid_commit_is_deposed_by_the_next_barrier() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
b.checkout().await.unwrap();
assert!(claim_until_held(&mut a, 1).await); write(dir_b.path(), "b.txt", "B publishes past a dead holder");
assert!(!claim_until_held(&mut b, 3).await, "the waiter deposed a holder below the quiet threshold");
let rb = b.run_barrier().await.expect("B's barrier deposes the dead holder and publishes");
assert_eq!(rb.uploaded, vec!["b.txt"]);
let err = lease::renew(&mut a).await.unwrap_err();
assert!(matches!(err, LeanError::Fenced(_)), "{err}");
write(dir_a.path(), "a.txt", "A, after being deposed");
a.run_barrier().await.expect("a deposed writer's next barrier claims again");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("a.txt") && m.entries.contains_key("b.txt"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_holder_deposed_mid_commit_abandons_the_barrier() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir_a = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir_a.path());
a.cfg.sentinel_min_interval_secs = 0;
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
write(dir_a.path(), "seed.txt", "seed");
a.run_barrier().await.unwrap();
let seq0 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
write(dir_a.path(), "work.txt", "v1");
touch_sentinel(dir_a.path(), control::PUBLISH, r#"{"nonce":"n1"}"#);
assert!(a.consume_sentinel(Verb::Publish).unwrap(), "fixture: the touch was not consumed");
assert!(matches!(a.sentinel_due().unwrap(), super::sentinel::Due::Ready), "fixture: not due");
let (rival_store, cfg) = (inner.clone(), a.cfg.clone());
hooked.before_put(&a.cfg.current_key(), move || {
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let key = cfg.epoch_key();
let seen = rival_store.epoch_read(&key).await.unwrap().unwrap();
assert!(!seen.released, "fixture: A is not holding at its CAS");
let l = rival_store.epoch_acquire(&key, "rival", Some(&seen)).await.unwrap();
manifest::rotate_for_takeover(rival_store.as_ref(), &cfg, l.epoch).await.unwrap();
})
});
});
let acks = a.sentinel_tick().await.expect("a fence is a retry, not an error the loop sees");
assert!(acks.is_empty(), "an ack was written for an abandoned barrier: {acks:?}");
assert!(a.lease.is_none());
assert!(a.load_pending(Verb::Publish).unwrap().is_some(), "the pending was dropped by the fence");
assert!(a.read_ack(Verb::Publish).is_none(), "an ack was written for an abandoned barrier");
let caps = a.read_capabilities().unwrap();
assert_eq!(caps.state, "live");
assert!(!caps.verbs.is_empty(), "the marker stopped advertising verbs");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("work.txt"), "the abandoned barrier's manifest landed");
assert_eq!(m.seq, seq0 + 1, "the rotation, and nothing after it");
let acks = a.sentinel_tick().await.expect("the retry");
assert_eq!(acks.len(), 1);
assert_eq!(acks[0].status, "ok");
assert!(acks[0].nonces.contains(&"n1".to_string()));
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("work.txt"));
}
#[tokio::test]
async fn readers_never_touch_the_cell() {
let store = Arc::new(MemoryStore::new());
let dir_w = tempfile::tempdir().unwrap();
let mut w = syncer(&store, dir_w.path()).await;
w.checkout().await.unwrap();
write(dir_w.path(), "f.txt", "published");
w.run_barrier().await.unwrap();
let epoch_before = store.epoch_read(&w.cfg.epoch_key()).await.unwrap().unwrap().epoch;
store.reset_op_counts();
let dir_r = tempfile::tempdir().unwrap();
let mut r = syncer(&store, dir_r.path()).await;
super::verbs::run_verb(&mut r, super::verbs::Step::Checkout).await.unwrap();
super::verbs::run_verb(&mut r, super::verbs::Step::Sync).await.unwrap();
assert_eq!(read(dir_r.path(), "f.txt").as_deref(), Some("published"));
let ops = store.op_counts();
assert!(
!ops.keys().any(|k| k.starts_with("epoch_") && *k != "epoch_read"),
"a reader wrote the cell: {ops:?}"
);
let cell = store.epoch_read(&w.cfg.epoch_key()).await.unwrap().unwrap();
assert_eq!(cell.epoch, epoch_before);
assert!(cell.released && cell.waiters.is_empty());
assert!(r.lease.is_none());
}
#[tokio::test]
async fn the_fence_costs_a_publishing_boundary_four_cell_requests_and_an_idle_one_none() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.claim_quiet_spacing_secs = lease::QUIET_SPACING_SECS;
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.floor_tick().await.unwrap();
store.reset_op_counts();
let idle = a.floor_tick().await.unwrap();
assert!(idle.no_change);
let ops = store.op_counts();
assert!(!ops.keys().any(|k| k.starts_with("epoch_")), "an idle tick touched the cell: {ops:?}");
assert_eq!(ops.values().sum::<u64>(), 2, "an idle tick is the inbox and the pointer: {ops:?}");
write(dir.path(), "f.txt", "v2");
backdate_baseline(&a, "f.txt");
store.reset_op_counts();
let busy = a.floor_tick().await.unwrap();
assert_eq!(busy.uploaded, 1);
let ops = store.op_counts();
assert_eq!(ops.get("epoch_read").copied(), Some(2), "{ops:?}");
assert_eq!(ops.get("epoch_acquire").copied(), Some(1), "{ops:?}");
assert_eq!(ops.get("epoch_handoff").copied(), Some(1), "{ops:?}");
assert_eq!(ops.get("epoch_renew"), None, "{ops:?}");
assert_eq!(ops.get("epoch_enqueue"), None, "{ops:?}");
}
#[tokio::test]
async fn an_idle_tick_on_a_never_published_workspace_reads_the_pointer_once() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
let first = a.floor_tick().await.unwrap();
assert!(first.no_change, "fixture: an empty tree publishes nothing");
store.reset_op_counts();
let idle = a.floor_tick().await.unwrap();
assert!(idle.no_change);
let ops = store.op_counts();
assert!(
matches!(store.get_whole(&a.cfg.current_key(), None).await, Err(StoreError::NotFound(_))),
"fixture: the workspace has never published"
);
assert!(!ops.keys().any(|k| k.starts_with("epoch_")), "an idle tick touched the cell: {ops:?}");
assert_eq!(ops.get("head").copied().unwrap_or(0), 0, "the legacy key was HEADed after a GET already said it is absent: {ops:?}");
assert_eq!(
ops.get("get_whole").copied().unwrap_or(0),
3,
"an idle never-published tick is the inbox, the pointer and the legacy key, once each: {ops:?}"
);
assert_eq!(ops.values().sum::<u64>(), 3, "{ops:?}");
}
#[tokio::test]
async fn a_pull_only_boundary_takes_no_fence_and_writes_nothing() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "gone.txt", "A deletes this");
a.floor_tick().await.unwrap();
b.checkout().await.unwrap();
b.floor_tick().await.unwrap();
write(dir_a.path(), "x.txt", "A's v2");
backdate_baseline(&a, "x.txt");
std::fs::remove_file(dir_a.path().join("gone.txt")).unwrap();
a.declared_barrier().await.unwrap();
let seq_a = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
store.reset_op_counts();
let pulled = b.floor_tick().await.unwrap();
let ops = store.op_counts();
eprintln!("a pull-only boundary: {ops:?}");
assert!(!ops.keys().any(|k| k.starts_with("epoch_")), "a pull-only boundary touched the cell: {ops:?}");
let writes: Vec<_> =
ops.keys().filter(|k| !matches!(**k, "get_whole" | "get_range" | "head" | "list")).collect();
assert!(writes.is_empty(), "a pull-only boundary wrote to the bucket: {ops:?}");
assert_eq!(pulled.seq, Some(seq_a), "{pulled:?}");
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's v2"), "A's edit was not taken");
assert_eq!(read(dir_b.path(), "gone.txt"), None, "A's delete was not taken");
b.floor_tick().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's v2"));
assert_eq!(read(dir_b.path(), "gone.txt"), None);
let t = b.load_remote_seq();
assert_eq!(t.observed_seq, t.integrated_seq, "{t:?}");
assert_eq!(manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq, seq_a, "B installed a generation");
write(dir_b.path(), "b.txt", "B's own");
store.reset_op_counts();
let busy = b.floor_tick().await.unwrap();
assert_eq!(busy.uploaded, 1);
assert_eq!(store.op_counts().get("epoch_acquire").copied(), Some(1), "a publishing boundary skipped the fence");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_pull_only_boundary_inside_a_peers_commit_section_converges() {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
let mut b = syncer(&inner, dir_b.path()).await;
b.cfg.claim_deadline_secs = 0;
a.checkout().await.unwrap();
write(dir_a.path(), "p.txt", "seed");
write(dir_a.path(), "q.txt", "A deletes this");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "p.txt", "A's v2");
backdate_baseline(&a, "p.txt");
std::fs::remove_file(dir_a.path().join("q.txt")).unwrap();
let b_slot = std::sync::Mutex::new(Some(b));
let b_done: Arc<std::sync::Mutex<Option<(Syncer, std::collections::BTreeMap<&'static str, u64>)>>> = Default::default();
let b_done_in = b_done.clone();
let (cfg, probe) = (a.cfg.clone(), inner.clone());
ha.before_delete(&a.cfg.file_key("q.txt"), move || {
let mut b = b_slot.lock().unwrap().take().expect("hook ran twice");
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(async {
let m = manifest::load(probe.as_ref(), &cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("q.txt"), "fixture: A's CAS has not uncited q.txt");
let cell = probe.epoch_read(&cfg.epoch_key()).await.unwrap().unwrap();
assert!(!cell.released, "fixture: A is not inside its commit section");
probe.reset_op_counts();
b.floor_tick().await.expect("B's pull inside A's commit section");
*b_done_in.lock().unwrap() = Some((b, probe.op_counts()));
})
});
});
let ra = a.declared_barrier().await.expect("A's barrier");
assert!(ra.deleted.contains(&"q.txt".to_string()), "fixture: {ra:?}");
let (mut b, ops) = b_done.lock().unwrap().take().expect("fixture: A's GC never reached q.txt");
assert!(!ops.keys().any(|k| k.starts_with("epoch_")), "B's pull touched the cell: {ops:?}");
for _ in 0..3 {
b.floor_tick().await.expect("B's tick");
a.floor_tick().await.expect("A's tick");
}
assert_eq!(read(dir_b.path(), "p.txt").as_deref(), Some("A's v2"));
assert_eq!(read(dir_b.path(), "q.txt"), None, "A's delete never reached B");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("q.txt"), "the delete was undone");
assert_every_citation_resolves(&inner, &a.cfg, "after the pull inside A's commit").await;
let t = b.load_remote_seq();
assert_eq!(t.observed_seq, t.integrated_seq, "{t:?}");
}
#[tokio::test]
async fn a_restarted_container_releases_the_fence_its_predecessor_left_held() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 1).await);
let held = a.lease.clone().unwrap();
drop(a); let mut a2 = syncer(&store, dir_a.path()).await; assert_eq!(lease::incarnation(&a2).unwrap().holder_id, held.holder_id);
lease::release_stale_own(&mut a2).await.unwrap();
let cell = store.epoch_read(&a2.cfg.epoch_key()).await.unwrap().unwrap();
assert!(cell.released, "the stale hold was not released");
assert_eq!(cell.epoch, held.epoch);
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 1).await, "the successor still had to wait");
}
fn barrier_on_thread(
mut sc: Syncer,
) -> std::thread::JoinHandle<(Syncer, crate::LeanResult<super::barrier::BarrierReport>)> {
std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.unwrap();
let r = rt.block_on(sc.run_barrier());
(sc, r)
})
}
async fn assert_every_citation_resolves(store: &Arc<MemoryStore>, cfg: &LeanConfig, when: &str) {
let m = manifest::load(store.as_ref(), cfg).await.unwrap().unwrap().manifest;
for (path, e) in &m.entries {
if let Err(err) = store.get_whole(&e.key, Some(&e.etag)).await {
panic!("{when}: seq {} cites {path} at {} but the object is gone: {err}", m.seq, e.etag);
}
}
}
#[tokio::test]
async fn a_peers_change_reaches_the_writer_whose_merge_queued_it_even_if_the_peer_consumes_first() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "A's v2");
backdate_baseline(&a, "x.txt");
a.run_barrier().await.unwrap();
write(dir_b.path(), "z.txt", "z");
b.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's v2"), "A's change never reached B");
assert_every_citation_resolves(&store, &a.cfg, "after convergence").await;
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, body) = store.get_whole(&m.entries["x.txt"].key, Some(&m.entries["x.txt"].etag)).await.unwrap();
assert_eq!(&body[..], b"A's v2", "the manifest reverted A's change");
}
#[tokio::test]
async fn a_queued_peer_change_never_lands_over_a_newer_version_a_sync_fetched() {
queued_change_behind_a_sync(true).await;
}
#[tokio::test]
async fn a_queued_peer_change_a_sync_overtook_is_not_reported_missing() {
queued_change_behind_a_sync(false).await;
}
pub(super) fn land_delete_during_next_barrier(store: &Arc<MemoryStore>, sc: &Syncer, path: &'static str) {
let (store, cfg) = (store.clone(), sc.cfg.clone());
super::barrier::CONSUME_WINDOW_HOOK.with(|h| {
*h.borrow_mut() = Some((
String::new(),
"before-scan",
Box::new(move || futures::executor::block_on(hitl_remove(&store, &cfg, path, "A"))),
))
});
}
pub(super) fn land_during_next_barrier(store: &Arc<MemoryStore>, sc: &Syncer, path: &'static str, content: &'static str) {
let (store, cfg) = (store.clone(), sc.cfg.clone());
super::barrier::CONSUME_WINDOW_HOOK.with(|h| {
*h.borrow_mut() = Some((
String::new(),
"before-scan",
Box::new(move || {
futures::executor::block_on(peer_publish(&store, &cfg, path, content, "A"));
}),
))
});
}
async fn queued_change_behind_a_sync(v2_still_at_its_handle: bool) {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "z.txt", "z");
land_during_next_barrier(&store, &b, "x.txt", "A's v2");
b.run_barrier().await.unwrap();
assert!(super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()), "fixture: the window never ran");
assert!(derive_pending(&b).await, "fixture: B's merge left nothing owed for x.txt");
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"), "fixture: v2 already reached B's tree");
let v2 = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["x.txt"].clone();
let v2_key = v2.key.clone();
let (_, v2_body) = store.get_whole(&v2_key, Some(&v2.etag)).await.unwrap();
write(dir_a.path(), "x.txt", "A's v3");
backdate_baseline(&a, "x.txt");
a.run_barrier().await.unwrap();
if v2_still_at_its_handle {
if store.head(&v2_key).await.is_err() {
let crc = crc64_nvme(&v2_body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "a-before-its-collector".to_string(),
boundary_source: None,
posix: None,
};
store
.put_whole(&v2_key, v2_body.clone(), &PutCondition::Unconditional, &stamps, crc)
.await
.unwrap();
}
assert_eq!(
store.head(&v2_key).await.unwrap().etag,
v2.etag.clone(),
"fixture: v2's handle does not attest v2"
);
}
b.sync().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's v3"), "fixture: B's sync did not fetch v3");
let conflicts_before = b.state.load_conflicts().unwrap().len();
b.run_barrier().await.unwrap();
assert_eq!(
read(dir_b.path(), "x.txt").as_deref(),
Some("A's v3"),
"B's consume wrote the queued v2 over the v3 its sync fetched"
);
b.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, body) = store.get_whole(&m.entries["x.txt"].key, Some(&m.entries["x.txt"].etag)).await.unwrap();
assert_eq!(&body[..], b"A's v3", "the manifest reverted A's v3");
let new_conflicts: Vec<_> = b.state.load_conflicts().unwrap().into_iter().skip(conflicts_before).collect();
assert!(
new_conflicts.is_empty(),
"a queued change the sync overtook was surfaced as a conflict: {new_conflicts:?}"
);
}
#[tokio::test]
async fn a_restart_between_the_cas_and_step_7_still_delivers_a_peers_change() {
restart_between_the_cas_and_step_7(false).await;
}
#[tokio::test]
async fn a_restart_between_the_cas_and_step_7_still_delivers_a_peers_delete() {
restart_between_the_cas_and_step_7(true).await;
}
#[derive(Clone, Copy, PartialEq)]
enum AfterRestart {
Nothing,
ScopedSyncElsewhere,
PeerV3ThenSync,
UiRenamesTheCarriedChange,
}
#[tokio::test]
async fn a_change_the_journal_carries_past_a_restart_never_lands_over_a_newer_sync() {
restart_between_the_cas_and_step_7_then(false, AfterRestart::PeerV3ThenSync).await;
}
#[tokio::test]
async fn a_scoped_sync_elsewhere_after_a_restart_still_delivers_a_peers_change() {
restart_between_the_cas_and_step_7_then(false, AfterRestart::ScopedSyncElsewhere).await;
}
#[tokio::test]
async fn a_ui_rename_after_a_restart_between_the_cas_and_step_7_keeps_the_file() {
restart_between_the_cas_and_step_7_then(false, AfterRestart::UiRenamesTheCarriedChange).await;
}
async fn restart_between_the_cas_and_step_7(peer_deletes: bool) {
restart_between_the_cas_and_step_7_then(peer_deletes, AfterRestart::Nothing).await;
}
async fn restart_between_the_cas_and_step_7_then(peer_deletes: bool, then: AfterRestart) {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&inner, dir_a.path()).await;
let mut b = hooked_syncer(&hooked, dir_b.path());
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"));
write(dir_b.path(), "tmp.txt", "B's scratch");
b.run_barrier().await.unwrap();
if peer_deletes {
land_delete_during_next_barrier(&inner, &b, "x.txt");
} else {
land_during_next_barrier(&inner, &b, "x.txt", "A's v2");
}
let state_dir = b.cfg.state_dir();
let snap: Arc<std::sync::Mutex<Vec<(std::path::PathBuf, Option<Vec<u8>>)>>> = Default::default();
let snap_in = snap.clone();
hooked.before_delete(&b.cfg.file_key("tmp.txt"), move || {
*snap_in.lock().unwrap() = ["intent.json", "baseline.json"]
.iter()
.map(|f| {
let p = state_dir.join(f);
let bytes = std::fs::read(&p).ok();
(p, bytes)
})
.collect();
});
write(dir_b.path(), "z.txt", "z");
std::fs::remove_file(dir_b.path().join("tmp.txt")).unwrap();
b.declared_barrier().await.unwrap();
let taken = std::mem::take(&mut *snap.lock().unwrap());
assert!(!taken.is_empty(), "fixture: B's barrier never reached its GC delete");
assert!(super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()), "fixture: the window never ran");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.contains_key("x.txt"), !peer_deletes, "fixture: A's change never landed");
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"), "fixture: A's change already reached B's tree");
for (p, bytes) in taken {
match bytes {
Some(b) => std::fs::write(&p, b).unwrap(),
None => {
let _ = std::fs::remove_file(&p);
}
}
}
match then {
AfterRestart::Nothing => {}
AfterRestart::ScopedSyncElsewhere => {
b.sync_scoped(Some(vec!["keep.txt".into()])).await.unwrap();
}
AfterRestart::PeerV3ThenSync => {
let v2 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["x.txt"].clone();
let v2_key = v2.key.clone();
let (_, v2_body) = inner.get_whole(&v2_key, Some(&v2.etag)).await.unwrap();
write(dir_a.path(), "x.txt", "A's v3");
backdate_baseline(&a, "x.txt");
a.run_barrier().await.unwrap();
if inner.head(&v2_key).await.is_err() {
let crc = crc64_nvme(&v2_body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "a-before-its-collector".to_string(),
boundary_source: None,
posix: None,
};
inner.put_whole(&v2_key, v2_body, &PutCondition::Unconditional, &stamps, crc).await.unwrap();
}
b.sync().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's v3"), "fixture: B's sync did not fetch v3");
}
AfterRestart::UiRenamesTheCarriedChange => {
let before = manifest::load(inner.as_ref(), &b.cfg).await.unwrap().unwrap();
let v2 = before.manifest.entries.get("x.txt").cloned().expect("fixture: x.txt is not cited");
assert_ne!(
b.state.load_baseline().unwrap().entries.get("x.txt").map(|e| &e.etag),
Some(&v2.etag),
"fixture: B's baseline already holds v2"
);
hitl_rename(&inner, &b.cfg, "x.txt", "w.txt", "dilip").await;
let r = b.run_barrier().await.unwrap();
let m = manifest::load(inner.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "fixture: the rename's source was not removed ({r:?})");
assert_eq!(
m.entries.get("w.txt").map(|e| e.key.as_str()),
Some(v2.key.as_str()),
"the acked rename's handle is cited at neither name ({r:?})"
);
assert!(inner.head(&v2.key).await.is_ok(), "the collector took the renamed file's handle ({r:?})");
}
}
for _ in 0..3 {
b.run_barrier().await.unwrap();
}
if then == AfterRestart::PeerV3ThenSync {
assert_eq!(
read(dir_b.path(), "x.txt").as_deref(),
Some("A's v3"),
"the change the journal carried past the restart landed over the v3 B's sync fetched"
);
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, body) = inner.get_whole(&m.entries["x.txt"].key, Some(&m.entries["x.txt"].etag)).await.unwrap();
assert_eq!(&body[..], b"A's v3", "the manifest reverted A's v3");
} else if then == AfterRestart::UiRenamesTheCarriedChange {
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "the rename's source is still cited");
let w = m.entries.get("w.txt").unwrap_or_else(|| panic!("the renamed file is cited nowhere: {:?}", m.entries.keys()));
let (_, body) = inner.get_whole(&w.key, Some(&w.etag)).await.unwrap();
assert_eq!(&body[..], b"A's v2", "the rename's destination cites the wrong bytes");
assert_eq!(read(dir_b.path(), "w.txt").as_deref(), Some("A's v2"), "the renamed file left B's tree");
assert_eq!(read(dir_b.path(), "x.txt"), None, "the rename's source stayed in B's tree");
} else if peer_deletes {
assert_eq!(read(dir_b.path(), "x.txt"), None, "a restart between B's CAS and its step 7 kept A's deleted file in B's tree");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "B re-published the file A deleted");
} else {
assert_eq!(
read(dir_b.path(), "x.txt").as_deref(),
Some("A's v2"),
"a restart between B's CAS and its step 7 lost A's change from B's tree for good"
);
}
assert!(!derive_pending(&b).await, "B still owes a change after three barriers");
}
#[tokio::test]
async fn a_restart_after_the_cas_converges_on_its_own_upload_without_republishing() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
write(dir.path(), "f.txt", "v2 is longer");
backdate_baseline(&a, "f.txt");
let stale = a.state.load_baseline().unwrap();
a.run_barrier().await.unwrap();
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
a.state.save_baseline(&stale).unwrap();
let conflicts_before = a.state.load_conflicts().unwrap().len();
let r = a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.seq, installed.seq, "the converged upload was published again: {r:?}");
assert_eq!(m.entries["f.txt"].key, installed.entries["f.txt"].key);
let new: Vec<_> = a.state.load_conflicts().unwrap().into_iter().skip(conflicts_before).collect();
assert!(new.is_empty(), "its own version was recorded as a stranger's: {new:?}");
assert_eq!(a.state.load_baseline().unwrap().entries["f.txt"].etag, installed.entries["f.txt"].etag);
}
#[tokio::test]
async fn a_pull_only_barrier_leaves_a_change_it_saw_owed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir.path()).await;
b.checkout().await.unwrap();
write(dir.path(), "x.txt", "seed");
write(dir.path(), "gone.txt", "the agent deletes this");
b.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
land_during_next_barrier(&store, &b, "x.txt", "A's v2");
let r = b.run_barrier().await.unwrap();
assert!(super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()), "fixture: the window never ran");
assert!(r.uploaded.is_empty() && r.deleted.is_empty(), "fixture: not a pull-only barrier: {r:?}");
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("seed"), "fixture: the change already reached the tree");
write(dir.path(), "gone.txt", "the agent deletes this");
backdate_baseline(&b, "gone.txt");
b.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("A's v2"), "the change the pull-only barrier saw was never taken");
}
fn restore_to_baseline(sc: &Syncer, rel: &str, content: &str) {
write(&sc.cfg.root, rel, content);
let e = sc.state.load_baseline().unwrap().entries[rel].clone();
let at = std::time::UNIX_EPOCH
+ std::time::Duration::new(e.mtime_unix as u64, e.mtime_nanos.unwrap_or(0));
std::fs::File::options().write(true).open(sc.cfg.root.join(rel)).unwrap().set_modified(at).unwrap();
assert!(!super::barrier::local_dirty(&sc.cfg.root.join(rel), Some(&e)), "fixture: still dirty");
}
#[tokio::test]
async fn a_peers_file_skipped_under_an_agents_new_one_arrives_when_the_agent_drops_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir.path()).await;
b.checkout().await.unwrap();
write(dir.path(), "x.txt", "the agent's draft, never published");
peer_publish(&store, &b.cfg, "x.txt", "the peer's file", "peer").await;
b.consume_owed().await.unwrap();
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("the agent's draft, never published"), "fixture: the consume took it");
std::fs::remove_file(dir.path().join("x.txt")).unwrap();
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("the peer's file"), "the peer's file never arrived");
}
#[tokio::test]
async fn a_peers_delete_skipped_under_an_agents_edit_lands_when_the_agent_reverts() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir.path()).await;
b.checkout().await.unwrap();
write(dir.path(), "x.txt", "seed");
b.run_barrier().await.unwrap();
write(dir.path(), "x.txt", "the agent's edit");
hitl_remove(&store, &b.cfg, "x.txt", "peer").await;
b.consume_owed().await.unwrap();
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("the agent's edit"), "fixture: the consume removed it");
restore_to_baseline(&b, "x.txt", "seed");
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir.path(), "x.txt"), None, "the peer's delete never landed");
}
#[tokio::test]
async fn a_peers_version_whose_write_lost_to_the_agent_arrives_when_the_agent_reverts() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir.path()).await;
b.checkout().await.unwrap();
write(dir.path(), "x.txt", "seed");
b.run_barrier().await.unwrap();
peer_publish(&store, &b.cfg, "x.txt", "the peer's v2", "peer").await;
let root = b.cfg.root.clone();
super::barrier::CONSUME_WINDOW_HOOK.with(|h| {
*h.borrow_mut() = Some((
"x.txt".into(),
"before-write",
Box::new(move || {
std::fs::write(root.join("x.txt"), "the agent's edit").unwrap();
}),
))
});
b.consume_owed().await.unwrap();
assert!(super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()), "fixture: the window never ran");
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("the agent's edit"), "fixture: the consume overwrote it");
restore_to_baseline(&b, "x.txt", "seed");
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("the peer's v2"), "the peer's version never arrived");
}
#[tokio::test]
async fn a_peers_delete_reaches_the_other_writers_tree() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"));
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "fixture: A's delete never published");
for _ in 0..3 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir_b.path(), "x.txt"), None, "A's delete never reached B's tree");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "B resurrected the path A deleted");
}
#[tokio::test]
async fn my_delete_meeting_a_peers_edit_wins_at_once_with_the_edit_preserved() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "x.txt", "B's edit");
backdate_baseline(&b, "x.txt");
b.run_barrier().await.unwrap();
let v2 = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest.entries["x.txt"].clone();
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
a.declared_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "A's delete never won: {:?}", m.entries.get("x.txt"));
let records = a.state.load_conflicts().unwrap();
assert!(
records.iter().any(|r| r.path == "x.txt" && r.foreign_etag.trim_matches('"') == v2.etag.trim_matches('"')),
"B's edit was not preserved with a record: {records:?}"
);
for _ in 0..3 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir_b.path(), "x.txt"), None, "A's delete never reached B's tree");
}
#[tokio::test]
async fn my_edit_meeting_a_peers_delete_recreates_the_path() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
std::fs::remove_file(dir_b.path().join("x.txt")).unwrap();
b.declared_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "fixture: B's delete never published");
write(dir_a.path(), "x.txt", "A's edit");
backdate_baseline(&a, "x.txt");
for _ in 0..3 {
a.run_barrier().await.unwrap();
}
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let x = m.entries.get("x.txt").unwrap_or_else(|| panic!("A's edit lost to B's delete: {:?}", m.entries.keys()));
let (_, body) = store.get_whole(&x.key, Some(&x.etag)).await.unwrap();
assert_eq!(&body[..], b"A's edit");
let seed_etag = format!("\"{:x}\"", <sha2::Sha256 as sha2::Digest>::digest(b"seed"));
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.path == "x.txt" && c.foreign_etag == seed_etag && c.kind.starts_with("commit-recreated-deleted")),
"A's edit overrode B's delete with no record: {:?}",
a.state.load_conflicts().unwrap()
);
assert_eq!(read(dir_a.path(), "x.txt").as_deref(), Some("A's edit"));
for _ in 0..3 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("A's edit"), "B's tree never took the re-created path");
}
#[tokio::test]
async fn a_ui_write_over_a_peers_delete_survives_the_queued_tombstone() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b, dir_c) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "z.txt", "z");
land_delete_during_next_barrier(&store, &b, "x.txt");
b.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("x.txt"), "fixture: the delete never landed");
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("seed"), "fixture: the deletion already reached B's tree");
assert!(derive_pending(&b).await, "fixture: B owes nothing for x.txt");
hitl_write(&store, &a.cfg, "x.txt", "from the UI", "reviewer").await.unwrap();
b.run_barrier().await.unwrap();
for _ in 0..2 {
a.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
}
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let mut c = syncer(&store, dir_c.path()).await;
c.checkout().await.unwrap();
assert_eq!(
read(dir_c.path(), "x.txt").as_deref(),
Some("from the UI"),
"the acked UI write is tracked by nothing: manifest cites x.txt {:?}, A's tree {:?}, B's tree {:?}",
m.entries.get("x.txt").map(|e| e.etag.clone()),
read(dir_a.path(), "x.txt"),
read(dir_b.path(), "x.txt"),
);
}
#[tokio::test]
async fn two_idle_writers_do_not_trade_empty_generations() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
b.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "from A");
a.run_barrier().await.unwrap();
for _ in 0..2 {
b.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
}
let settled = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
for _ in 0..4 {
b.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
}
let later = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert_eq!(later, settled, "idle writers kept installing generations: seq {settled} -> {later}");
assert_eq!(read(dir_b.path(), "x.txt").as_deref(), Some("from A"));
}
#[tokio::test]
async fn a_gateway_write_reaches_every_writer_not_only_the_first_consumer() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "doc.md", "v1");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
hitl_write(&store, &a.cfg, "doc.md", "edited in the UI", "user@ui").await.unwrap();
hitl_write(&store, &a.cfg, "new-from-ui.md", "created in the UI", "user@ui").await.unwrap();
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(read(dir_a.path(), "doc.md").as_deref(), Some("edited in the UI"));
b.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
assert_eq!(read(dir_b.path(), "doc.md").as_deref(), Some("edited in the UI"), "the UI edit never reached B");
assert_eq!(read(dir_b.path(), "new-from-ui.md").as_deref(), Some("created in the UI"), "the UI create never reached B");
assert_every_citation_resolves(&store, &a.cfg, "after both").await;
}
#[tokio::test]
async fn a_gateway_write_over_a_path_another_writer_edited_is_preserved_on_that_writer() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "doc.md", "v1");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
write(dir_b.path(), "doc.md", "B's agent edit");
backdate_baseline(&b, "doc.md");
hitl_write(&store, &a.cfg, "doc.md", "edited in the UI", "user@ui").await.unwrap();
a.run_barrier().await.unwrap(); b.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let (_, body) = store.get_whole(&m.entries["doc.md"].key, Some(&m.entries["doc.md"].etag)).await.unwrap();
assert_eq!(&body[..], b"B's agent edit", "B's later boundary is current");
let preserved: Vec<_> = b
.state
.load_conflicts()
.unwrap()
.into_iter()
.filter(|c| c.path == "doc.md" && c.preserved_key.is_some())
.collect();
let mut found_ui = false;
for c in &preserved {
let (_, kept) = store.get_whole(c.preserved_key.as_ref().unwrap(), None).await.unwrap();
found_ui |= &kept[..] == b"edited in the UI";
}
assert!(found_ui, "the UI's bytes are not preserved under any record on B: {preserved:?}");
assert_every_citation_resolves(&store, &a.cfg, "after both").await;
}
#[tokio::test]
async fn the_event_trace_reconstructs_a_two_writer_interleaving() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let buf = Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let mut a = syncer(&store, dir_a.path()).await;
let mut b = syncer(&store, dir_b.path()).await;
a.cfg.event_trace = Some(crate::trace::Sink::Memory(buf.clone()));
b.cfg.event_trace = Some(crate::trace::Sink::Memory(buf.clone()));
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "seed");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
std::fs::remove_file(dir_a.path().join("x.txt")).unwrap();
write(dir_b.path(), "keep.txt", "B's edit");
backdate_baseline(&b, "keep.txt");
a.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
for _ in 0..3 {
a.run_barrier().await.unwrap();
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir_b.path(), "x.txt"), None, "fixture: A's delete never reached B");
assert_eq!(read(dir_a.path(), "keep.txt").as_deref(), Some("B's edit"), "fixture: B's edit never reached A");
let lines = buf.lock().unwrap().clone();
if let Ok(out) = std::env::var("FLINT_SYNC_TRACE_FIXTURE") {
std::fs::write(&out, lines.join("\n") + "\n").unwrap();
}
let evs: Vec<serde_json::Map<String, serde_json::Value>> = lines
.iter()
.map(|l| {
assert!(l.starts_with("{\"ts_ms\":"), "not the pinned prefix: {l}");
let v: serde_json::Value = serde_json::from_str(l).unwrap_or_else(|e| panic!("{e}: {l}"));
let m = v.as_object().unwrap().clone();
let head: Vec<&str> = m.keys().take(4).map(|k| k.as_str()).collect();
for k in ["ts_ms", "mono_ms", "holder", "ev"] {
assert!(m.contains_key(k), "{k} missing ({head:?}): {l}");
}
let pos: Vec<usize> = ["\"ts_ms\":", "\"mono_ms\":", "\"holder\":", "\"ev\":"]
.iter()
.map(|k| l.find(k).unwrap())
.collect();
assert!(pos.windows(2).all(|w| w[0] < w[1]), "prefix out of order: {l}");
m
})
.collect();
let ev = |e: &serde_json::Map<String, serde_json::Value>| e["ev"].as_str().unwrap().to_string();
let s = |e: &serde_json::Map<String, serde_json::Value>, k: &str| e.get(k).and_then(|v| v.as_str()).map(str::to_string);
let holders: std::collections::BTreeSet<String> =
evs.iter().filter(|e| ["claim", "cas", "gc", "release"].contains(&ev(e).as_str())).map(|e| {
s(e, "holder").unwrap_or_else(|| panic!("a commit-section event without a holder: {e:?}"))
}).collect();
assert_eq!(holders.len(), 2, "expected both writers in the trace: {holders:?}");
for e in evs.iter().filter(|e| ev(e) == "claim") {
let verdict = s(e, "verdict").unwrap();
assert!(["claimed", "waiting", "deadline"].contains(&verdict.as_str()), "claim verdict {verdict}: {e:?}");
if verdict == "claimed" {
let how = s(e, "how").unwrap_or_else(|| panic!("a claimed claim without how: {e:?}"));
assert!(
["fresh", "adopted-own", "deposed", "orphaned-own", "skipped-handoff", "released"].contains(&how.as_str()),
"claim how {how}: {e:?}"
);
assert!(e["epoch"].as_u64().is_some(), "claimed without an epoch: {e:?}");
}
}
for e in evs.iter().filter(|e| ev(e) == "cas") {
assert!(["ok", "lost"].contains(&s(e, "result").unwrap().as_str()), "cas result: {e:?}");
}
for e in evs.iter().filter(|e| ev(e) == "barrier_end") {
let req = e["requests"].as_object().unwrap_or_else(|| panic!("barrier_end without request counts: {e:?}"));
let keys: Vec<&str> = req.keys().map(|k| k.as_str()).collect();
let mut want = vec!["copy", "delete", "get", "head", "list", "multipart", "put"];
let mut got = keys.clone();
want.sort();
got.sort();
assert_eq!(got, want, "request counter keys: {e:?}");
}
let cas_ok_by: std::collections::BTreeSet<String> =
evs.iter().filter(|e| ev(e) == "cas" && s(e, "result").as_deref() == Some("ok")).filter_map(|e| s(e, "holder")).collect();
assert_eq!(cas_ok_by.len(), 2, "both writers must install at least once: {cas_ok_by:?}");
for h in &holders {
let mine: Vec<String> = evs
.iter()
.filter(|e| s(e, "holder").as_deref() == Some(h.as_str()))
.map(|e| match ev(e).as_str() {
"claim" => format!("claim:{}", s(e, "verdict").unwrap()),
"cas" => format!("cas:{}", s(e, "result").unwrap()),
other => other.to_string(),
})
.filter(|n| ["scan", "claim:claimed", "cas:ok", "release", "barrier_end"].contains(&n.as_str()))
.collect();
let joined = mine.join(" ");
assert!(
joined.contains("scan claim:claimed cas:ok release barrier_end"),
"{h}: no install in protocol order: {joined}"
);
}
let gc_x = evs
.iter()
.position(|e| ev(e) == "gc" && s(e, "path").as_deref() == Some("x.txt") && s(e, "result").as_deref() == Some("retired"))
.unwrap_or_else(|| panic!("no gc retired for x.txt in:\n{}", lines.join("\n")));
let tomb_x = evs
.iter()
.position(|e| ev(e) == "tombstone" && s(e, "path").as_deref() == Some("x.txt") && s(e, "action").as_deref() == Some("removed"))
.unwrap_or_else(|| panic!("no tombstone removed for x.txt in:\n{}", lines.join("\n")));
assert_ne!(s(&evs[gc_x], "holder"), s(&evs[tomb_x], "holder"), "the GC and the tombstone must be different writers");
assert!(gc_x < tomb_x, "the tombstone was emitted before the GC");
assert!(
evs[gc_x]["ts_ms"].as_u64().unwrap() <= evs[tomb_x]["ts_ms"].as_u64().unwrap(),
"wall clock puts the tombstone before the GC"
);
assert!(
evs.iter().any(|e| ev(e) == "consume" && s(e, "path").as_deref() == Some("keep.txt") && s(e, "from").as_deref() == Some("manifest")),
"B's edit did not reach A through the consume:\n{}",
lines.join("\n")
);
}
async fn killed_writer_orphan() -> (Arc<MemoryStore>, Syncer, tempfile::TempDir, tempfile::TempDir, String) {
let inner = Arc::new(MemoryStore::new());
let ha = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&ha, dir_a.path());
a.cfg.upload_fanout = 1; let mut b = syncer(&inner, dir_b.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "p1.txt", "seed-1");
write(dir_a.path(), "p2.txt", "seed-2");
a.run_barrier().await.unwrap();
b.checkout().await.unwrap();
let seed1 = cited_key(&inner, &a.cfg, "p1.txt").await;
write(dir_a.path(), "p1.txt", "A's unpublished edit");
backdate_baseline(&a, "p1.txt");
write(dir_a.path(), "p2.txt", "A's other edit");
backdate_baseline(&a, "p2.txt");
let cfg = a.cfg.clone();
ha.before_put(&a.cfg.file_key("p2.txt"), || panic!("A is killed between its uploads"));
assert!(barrier_on_thread(a).join().is_err(), "fixture: A was not killed");
let handles = handles_for(&inner, &cfg, "p1.txt").await;
let orphan = handles.iter().find(|k| **k != seed1).cloned().expect("fixture: A's upload of p1 never landed");
let (_, body) = inner.get_whole(&orphan, None).await.unwrap();
assert_eq!(&body[..], b"A's unpublished edit", "fixture: not A's upload");
write(dir_b.path(), "b.txt", "B works elsewhere");
for _ in 0..4 {
b.run_barrier().await.unwrap();
}
assert_eq!(read(dir_b.path(), "p1.txt").as_deref(), Some("seed-1"), "fixture: B already has A's edit");
(inner, b, dir_a, dir_b, orphan)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn a_writer_killed_after_its_upload_does_not_leave_the_trees_diverged() {
let (inner, mut b, _dir_a, dir_b, orphan) = killed_writer_orphan().await;
let dir_c = tempfile::tempdir().unwrap();
let mut c = syncer(&inner, dir_c.path()).await;
c.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "p1.txt").as_deref(), Some("seed-1"));
assert_eq!(read(dir_c.path(), "p1.txt").as_deref(), Some("seed-1"), "a fresh checkout read the orphan");
assert!(inner.head(&orphan).await.is_ok(), "fixture: the orphan is still there");
let tracked = b.track_untracked(now_unix() + b.cfg.untracked_grace_secs).await.unwrap();
assert!(tracked.is_empty(), "the ingress sweep tracked a lost writer's handle: {tracked:?}");
b.cfg.untracked_grace_secs = 0;
b.state.save_orphan_sweep_at(now_unix() - b.cfg.untracked_sweep_secs - 1).unwrap();
write(dir_b.path(), "b2.txt", "B publishes again");
let r = b.run_barrier().await.unwrap();
assert_eq!(r.swept, 1, "the orphan sweep did not collect the killed writer's handle: {r:?}");
assert!(matches!(inner.head(&orphan).await, Err(StoreError::NotFound(_))), "the orphan survived the sweep");
let m = manifest::load(inner.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(cited_bytes(&inner, &b.cfg, "p1.txt").await.as_deref(), Some(&b"seed-1"[..]), "the manifest adopted the orphan");
assert_eq!(m.entries["p1.txt"].key, cited_key(&inner, &b.cfg, "p1.txt").await);
assert_eq!(read(dir_b.path(), "p1.txt").as_deref(), Some("seed-1"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn the_floor_sweeps_for_ingress_objects_only_past_the_grace_and_when_due() {
let (inner, mut b, _dir_a, dir_b, _orphan) = killed_writer_orphan().await;
let body = Bytes::from_static(b"an outside writer's bytes");
let stamps = GenerationStamps { generation: 0, epoch: 0, flush_uuid: "aws-cli".into(), boundary_source: None, posix: None };
let ingress = inner
.put_whole(&b.cfg.file_key("p1.txt"), body.clone(), &PutCondition::Unconditional, &stamps, crc64_nvme(&body))
.await
.unwrap();
assert_eq!(b.track_untracked(now_unix()).await.unwrap(), Vec::<String>::new(), "tracked inside the grace");
b.cfg.untracked_grace_secs = 0;
b.cfg.untracked_sweep_secs = 3600;
b.floor_tick().await.unwrap();
assert_eq!(read(dir_b.path(), "p1.txt").as_deref(), Some("seed-1"), "the first tick swept instead of starting the clock");
assert!(b.state.load_untracked_sweep_at().unwrap() > 0, "the first tick did not start the clock");
b.state.save_untracked_sweep_at(now_unix() - 3601).unwrap();
b.floor_tick().await.unwrap();
for _ in 0..2 {
b.run_barrier().await.unwrap();
}
assert_eq!(
read(dir_b.path(), "p1.txt").as_deref(),
Some("an outside writer's bytes"),
"a due floor did not track and consume the ingress object"
);
let m = manifest::load(inner.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
let cited = &m.entries["p1.txt"];
assert_eq!(b.cfg.handle_parts(&cited.key).map(|(_, f)| f), Some(super::untracked::ingress_flush(&ingress.etag).as_str()),
"the manifest cites something other than the ingress object's adopted copy: {}", cited.key);
let (at_key, _) = inner.get_whole(&b.cfg.file_key("p1.txt"), None).await.unwrap();
assert_eq!(at_key.etag, ingress.etag, "the ingress object was touched");
assert_eq!(b.track_untracked(now_unix()).await.unwrap(), Vec::<String>::new(), "adopted twice");
}
const READ_OPS: &[&str] = &[
"get_whole", "get_range", "get_version", "head", "head_version", "list", "list_versions",
"list_uploads", "lifecycle_rules", "epoch_read", "presign_get",
];
fn writes_since_reset(store: &MemoryStore) -> Vec<(&'static str, u64)> {
store.op_counts().into_iter().filter(|(op, _)| !READ_OPS.contains(op)).collect()
}
async fn reader(store: &Arc<MemoryStore>, root: &std::path::Path) -> Syncer {
let mut sc = syncer(store, root).await;
sc.cfg.access = super::Access::Read;
sc
}
#[tokio::test]
async fn a_reader_follows_writers_and_the_inbox_without_one_store_write() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_r) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "A v1");
write(dir_a.path(), "gone.txt", "A deletes this");
a.floor_tick().await.unwrap();
store.reset_op_counts();
let mut r = reader(&store, dir_r.path()).await;
r.checkout().await.unwrap();
r.floor_tick().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a reader's checkout and first tick wrote to the bucket");
assert_eq!(read(dir_r.path(), "x.txt").as_deref(), Some("A v1"));
write(dir_a.path(), "x.txt", "A v2");
backdate_baseline(&a, "x.txt");
std::fs::remove_file(dir_a.path().join("gone.txt")).unwrap();
write(dir_a.path(), "new.txt", "A adds this");
a.declared_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "ui.txt", "from the UI", "reviewer").await.unwrap();
let seq_a = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
store.reset_op_counts();
r.floor_tick().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a reader's pull wrote to the bucket");
assert_eq!(read(dir_r.path(), "x.txt").as_deref(), Some("A v2"));
assert_eq!(read(dir_r.path(), "gone.txt"), None);
assert_eq!(read(dir_r.path(), "new.txt").as_deref(), Some("A adds this"));
assert_eq!(read(dir_r.path(), "ui.txt").as_deref(), Some("from the UI"), "the inbox write did not reach the reader");
let t = r.load_remote_seq();
assert_eq!((t.observed_seq, t.integrated_seq), (seq_a, seq_a), "{t:?}");
write(dir_r.path(), "x.txt", "the reader's local edit");
backdate_baseline(&r, "x.txt");
write(dir_r.path(), "local-only.txt", "never published");
write(dir_a.path(), "x.txt", "A v3");
backdate_baseline(&a, "x.txt");
a.declared_barrier().await.unwrap();
store.reset_op_counts();
r.floor_tick().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a reader with local changes wrote to the bucket");
assert_eq!(read(dir_r.path(), "x.txt").as_deref(), Some("the reader's local edit"));
assert!(!object_at(&store, &r.cfg, "local-only.txt").await, "a reader published a local file");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("local-only.txt"));
let conflicts = r.state.load_conflicts().unwrap();
assert!(conflicts.iter().any(|c| c.path == "x.txt" && c.kind == "sync-dirty"), "{conflicts:?}");
}
#[tokio::test]
async fn an_idle_reader_tick_costs_the_inbox_and_the_pointer() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_r) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "A v1");
a.floor_tick().await.unwrap();
hitl_write(&store, &a.cfg, "ui.txt", "from the UI", "reviewer").await.unwrap();
let mut r = reader(&store, dir_r.path()).await;
r.checkout().await.unwrap();
r.floor_tick().await.unwrap();
store.reset_op_counts();
r.floor_tick().await.unwrap();
let ops = store.op_counts();
assert_eq!(ops.into_iter().collect::<Vec<_>>(), vec![("get_whole", 2)], "an idle reader tick is two GETs");
let t = r.load_remote_seq();
assert!(t.updated_unix > 0, "the ticker's heartbeat did not move on an idle tick: {t:?}");
hitl_write(&store, &a.cfg, "ui2.txt", "a second UI write", "reviewer").await.unwrap();
r.floor_tick().await.unwrap();
assert_eq!(read(dir_r.path(), "ui2.txt").as_deref(), Some("a second UI write"), "an inbox-only change never reached the reader");
}
#[tokio::test]
async fn a_publish_touch_on_a_reader_is_answered_refused_read_only() {
let store = Arc::new(MemoryStore::new());
let (dir_r, dir_w) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut r = reader(&store, dir_r.path()).await;
r.checkout().await.unwrap();
let posture = r.sentinel_preflight().unwrap();
r.write_capabilities(&posture).unwrap();
write(dir_r.path(), "f.txt", "a local file");
store.reset_op_counts();
touch_sentinel(dir_r.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
let acks = r.sentinel_tick().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a refused publish wrote to the bucket");
assert_eq!(acks.len(), 1, "{acks:?}");
assert_eq!(acks[0].status, "refused-read-only");
assert_eq!(acks[0].nonces, vec!["n-1".to_string()]);
assert!(acks[0].reason.as_deref().unwrap_or("").contains("read-only"), "{:?}", acks[0].reason);
assert!(r.load_pending(Verb::Publish).unwrap().is_none(), "the refused touch was left pending");
let on_disk = std::fs::read_to_string(dir_r.path().join(super::CONTROL_DIR).join(control::PUBLISH_ACK)).unwrap();
assert!(on_disk.contains("refused-read-only"), "{on_disk}");
assert!(!object_at(&store, &r.cfg, "f.txt").await);
r.cfg.sentinel_min_interval_secs = 0;
touch_sentinel(dir_r.path(), control::SYNC, r#"{"nonce":"s-1"}"#);
let acks = r.sentinel_tick().await.unwrap();
assert_eq!(acks.iter().map(|a| a.status.as_str()).collect::<Vec<_>>(), vec!["ok"]);
let mut w = syncer(&store, dir_w.path()).await;
w.checkout().await.unwrap();
let posture = w.sentinel_preflight().unwrap();
w.write_capabilities(&posture).unwrap();
write(dir_w.path(), "g.txt", "a writer's file");
touch_sentinel(dir_w.path(), control::PUBLISH, r#"{"nonce":"w-1"}"#);
let acks = w.sentinel_tick().await.unwrap();
assert_eq!(acks.iter().map(|a| a.status.as_str()).collect::<Vec<_>>(), vec!["ok"]);
}
#[tokio::test]
async fn a_barrier_on_a_reader_is_refused_before_any_write() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut r = reader(&store, dir.path()).await;
r.checkout().await.unwrap();
write(dir.path(), "f.txt", "would be an upload");
store.reset_op_counts();
for (name, out) in [
("run_barrier", r.run_barrier().await.map(|_| ())),
("cadence_barrier", r.cadence_barrier().await.map(|_| ())),
("declared_barrier", r.declared_barrier().await.map(|_| ())),
] {
assert!(matches!(out, Err(LeanError::Refused(_))), "{name} on a reader: {out:?}");
}
assert_eq!(store.total_ops(), 0, "a refused barrier sent requests: {:?}", store.op_counts());
assert!(!object_at(&store, &r.cfg, "f.txt").await);
}
#[tokio::test]
async fn one_shot_verbs_that_publish_or_fence_are_refused_on_a_reader() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut r = reader(&store, dir.path()).await;
r.checkout().await.unwrap();
store.reset_op_counts();
for step in [super::verbs::Step::Barrier, super::verbs::Step::Rescope(None)] {
let out = super::verbs::run_verb(&mut r, step).await;
assert!(matches!(out, Err(LeanError::Refused(_))), "{out:?}");
}
assert_eq!(writes_since_reset(&store), vec![], "a refused verb wrote: {:?}", store.op_counts());
assert!(super::verbs::run_verb(&mut r, super::verbs::Step::Sync).await.is_ok(), "sync is a reader's verb");
assert!(super::verbs::run_verb(&mut r, super::verbs::Step::Checkout).await.is_ok());
assert_eq!(writes_since_reset(&store), vec![]);
}
#[tokio::test]
async fn a_reader_drain_answers_what_it_owes_and_writes_nothing() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut r = reader(&store, dir.path()).await;
r.checkout().await.unwrap();
let posture = r.sentinel_preflight().unwrap();
r.write_capabilities(&posture).unwrap();
write(dir.path(), "f.txt", "local only");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"at-sigterm"}"#);
r.poll_sentinels().unwrap();
assert!(r.load_pending(Verb::Publish).unwrap().is_some());
store.reset_op_counts();
let acks = r.drain().await.unwrap();
assert_eq!(writes_since_reset(&store), vec![], "a reader's drain wrote to the bucket");
assert_eq!(acks.iter().map(|a| a.status.as_str()).collect::<Vec<_>>(), vec!["refused-read-only"]);
assert!(!object_at(&store, &r.cfg, "f.txt").await);
}
#[tokio::test]
async fn the_marker_says_read_access_and_the_guide_explains_it() {
let store = Arc::new(MemoryStore::new());
let (dir_r, dir_w) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut r = reader(&store, dir_r.path()).await;
r.checkout().await.unwrap();
let posture = r.sentinel_preflight().unwrap();
r.write_capabilities(&posture).unwrap();
let caps = r.read_capabilities().unwrap();
assert_eq!(caps.access, "read");
assert_eq!(caps.verbs, vec!["sync".to_string(), "remote-seq".to_string()], "a reader advertised {:?}", caps.verbs);
let mut w = syncer(&store, dir_w.path()).await;
w.checkout().await.unwrap();
let posture = w.sentinel_preflight().unwrap();
w.write_capabilities(&posture).unwrap();
let caps = w.read_capabilities().unwrap();
assert_eq!(caps.access, "readWrite");
assert!(caps.verbs.contains(&"publish".to_string()));
let old: control::Capabilities = serde_json::from_str(
r#"{"protocol":1,"verbs":[],"state":"live","sentinel_min_interval_secs":5,
"sentinel_hourly_budget":60,"syncer_version":"x","boot":{"holder_id":"h","boot_unix":0}}"#,
)
.unwrap();
assert_eq!(old.access, "readWrite");
for needle in ["\"access\": \"read\"", "refused-read-only", "EROFS"] {
assert!(control::AGENT_GUIDE_TEXT.contains(needle), "the guide never says {needle}");
}
}
#[test]
fn access_parses_exactly_what_the_plugin_writes() {
use super::Access;
assert_eq!(Access::parse("read"), Some(Access::Read));
assert_eq!(Access::parse("readWrite"), Some(Access::ReadWrite));
for bad in ["", "ro", "READ", "read-only", "readwrite", "rw"] {
assert_eq!(Access::parse(bad), None, "{bad:?} must not parse");
}
assert_eq!(Access::default(), Access::ReadWrite);
assert_eq!(Access::parse(Access::Read.as_str()), Some(Access::Read));
}
#[tokio::test]
async fn a_consumed_entry_named_like_a_consume_temp_is_surfaced_never_collected() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
let path = "notes/report.flint-sync-tmp";
let key = sc.cfg.file_key(path);
let body = Bytes::from_static(b"user bytes");
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "gateway-old".into(),
boundary_source: None,
posix: None,
};
let meta = store
.put_whole(&key, body.clone(), &PutCondition::IfNoneMatchAny, &stamps, crc64_nvme(&body))
.await
.unwrap();
let (etag, crc) = (meta.etag.clone(), crc_of("user bytes"));
let k = key.clone();
gateway_commit(&store, &sc.cfg, "gateway-old", move |doc| {
doc.entries.insert(path.into(), manifest::LeanEntry {
key: k.clone(), etag: etag.clone(), crc64_b64: crc.clone(), size: 10,
mode: 0o644, mtime_unix: 0, generation: 1, epoch: 0,
});
})
.await
.unwrap();
for _ in 0..3 {
sc.run_barrier().await.unwrap();
}
assert!(
store.head(&key).await.is_ok(),
"the acked write's object was collected: the consume materialised a name the walk cannot see, \
the repair cited it as a first absence, and the next barrier deleted it"
);
assert!(read(dir.path(), path).is_none(), "a reserved name was materialised into the tree");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key(path), "the walk's blind spot became a delete of the citation");
let recs = std::fs::read_to_string(sc.cfg.state_dir().join("conflicts.jsonl")).unwrap_or_default();
assert!(
recs.contains("consume-refused-containment") && recs.contains(path),
"nothing recorded the refusal: {recs}"
);
}
async fn h10_declared_edit(store: &Arc<MemoryStore>, dir_a: &std::path::Path, dir_b: &std::path::Path) -> (Syncer, Syncer) {
let mut a = syncer(store, dir_a).await;
let mut b = syncer(store, dir_b).await;
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir_a, "shared.txt", "v1");
write(dir_a, "gone.txt", "published, then deleted by A's agent before the touch");
a.floor_tick().await.unwrap();
a.floor_tick().await.unwrap();
b.checkout().await.unwrap();
write(dir_a, "shared.txt", "A's declared v2");
backdate_baseline(&a, "shared.txt");
std::fs::remove_file(dir_a.join("gone.txt")).unwrap();
touch_sentinel(dir_a, control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
assert!(a.load_pending(Verb::Publish).unwrap().is_some(), "fixture: the sentinel was not consumed");
(a, b)
}
async fn h10_assert_carried(store: &Arc<MemoryStore>, a: &Syncer) {
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["shared.txt"].crc64_b64, crc_of("A's declared v2"), "fixture: the declared edit was not installed");
assert!(!m.entries.contains_key("gone.txt"), "the install withheld the delete the agent declared");
}
async fn h10_peer_deletes(store: &Arc<MemoryStore>, b: &mut Syncer, dir_b: &std::path::Path) -> u64 {
b.floor_tick().await.unwrap();
b.floor_tick().await.unwrap();
assert_eq!(read(dir_b, "shared.txt").as_deref(), Some("A's declared v2"), "fixture: A's edit never reached B");
std::fs::remove_file(dir_b.join("shared.txt")).unwrap();
b.declared_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("shared.txt"), "fixture: B's delete did not publish");
m.seq
}
fn h10_assert_ack_is_honest(_dir_a: &std::path::Path, ack: &super::sentinel::Ack, peer_seq: u64) {
assert!(ack.seq >= Some(peer_seq), "fixture: the ack names a document older than B's delete: {ack:?}");
assert_eq!(
ack.status, "partial",
"the ack says the declared boundary is in seq {:?}, which no longer cites shared.txt: {ack:?}",
ack.seq
);
assert_eq!(ack.report.dropped, vec!["shared.txt".to_string()], "{ack:?}");
}
#[tokio::test]
async fn a_restart_between_the_install_and_the_ack_does_not_ack_a_peers_later_delete_as_the_declaration() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let (mut a, mut b) = h10_declared_edit(&store, dir_a.path(), dir_b.path()).await;
a.declared_barrier_as("sentinel").await.unwrap();
h10_assert_carried(&store, &a).await;
assert!(a.read_ack(Verb::Publish).is_none(), "fixture: an ack was written");
drop(a);
let peer_seq = h10_peer_deletes(&store, &mut b, dir_b.path()).await;
let mut a = syncer(&store, dir_a.path()).await;
a.settle_pending_at_startup().await.unwrap();
let ack = a.read_ack(Verb::Publish).expect("the restart answered nothing");
h10_assert_ack_is_honest(dir_a.path(), &ack, peer_seq);
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
}
#[tokio::test]
async fn an_ack_write_that_failed_is_not_answered_later_with_a_peers_delete_as_the_declaration() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let (mut a, mut b) = h10_declared_edit(&store, dir_a.path(), dir_b.path()).await;
let squat = dir_a.path().join(super::CONTROL_DIR).join(control::PUBLISH_ACK);
let _ = std::fs::remove_file(&squat);
std::fs::create_dir_all(squat.join("occupied")).unwrap();
clear_min_interval(&a);
assert!(a.honor_pending(Verb::Publish, false).await.is_err(), "fixture: the ack write did not fail");
h10_assert_carried(&store, &a).await;
assert!(a.load_pending(Verb::Publish).unwrap().is_some(), "fixture: the pending was retired");
std::fs::remove_dir_all(&squat).unwrap();
let peer_seq = h10_peer_deletes(&store, &mut b, dir_b.path()).await;
let ack = a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
h10_assert_ack_is_honest(dir_a.path(), &ack, peer_seq);
}
#[tokio::test]
async fn the_floors_cadence_barrier_after_a_failed_honor_carries_the_whole_declaration() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let (mut a, mut b) = h10_declared_edit(&store, dir_a.path(), dir_b.path()).await;
a.cadence_barrier().await.unwrap();
h10_assert_carried(&store, &a).await;
let carried = manifest::load_pointer(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert_eq!(
carried.pointer.boundary_source.as_deref(),
Some("sentinel-deferred"),
"the install is stamped with the clock that published the declaration"
);
let peer_seq = h10_peer_deletes(&store, &mut b, dir_b.path()).await;
let ack = a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
h10_assert_ack_is_honest(dir_a.path(), &ack, peer_seq);
}
#[tokio::test]
async fn a_second_touch_does_not_orphan_the_carrier_of_the_first() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let (mut a, mut b) = h10_declared_edit(&store, dir_a.path(), dir_b.path()).await;
let first = a.load_pending(Verb::Publish).unwrap().expect("fixture: no pending").id;
let squat = dir_a.path().join(super::CONTROL_DIR).join(control::PUBLISH_ACK);
let _ = std::fs::remove_file(&squat);
std::fs::create_dir_all(squat.join("occupied")).unwrap();
clear_min_interval(&a);
assert!(a.honor_pending(Verb::Publish, false).await.is_err(), "fixture: the ack write did not fail");
h10_assert_carried(&store, &a).await;
std::fs::remove_dir_all(&squat).unwrap();
let peer_seq = h10_peer_deletes(&store, &mut b, dir_b.path()).await;
touch_sentinel(dir_a.path(), control::PUBLISH, r#"{"nonce":"n-2"}"#);
a.poll_sentinels().unwrap();
let pending = a.load_pending(Verb::Publish).unwrap().expect("fixture: the second touch was not folded");
assert_eq!(pending.nonces, vec!["n-1".to_string(), "n-2".to_string()], "fixture: the touches did not coalesce");
assert_eq!(pending.id, first, "a fold minted a new record id: the first touch's carrier no longer matches");
let ack = a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
assert_eq!(ack.nonces, vec!["n-1".to_string(), "n-2".to_string()], "{ack:?}");
h10_assert_ack_is_honest(dir_a.path(), &ack, peer_seq);
}
#[tokio::test]
async fn a_barrier_carries_the_gateway_request_it_reads() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "s");
a.floor_tick().await.unwrap();
a.floor_tick().await.unwrap();
write(dir.path(), "work.txt", "the declared bytes");
inbox::gateway_request(store.as_ref(), &a.cfg, inbox::RequestedVerb::Boundary, "ci@example").await.unwrap();
a.cadence_barrier().await.unwrap();
assert!(a.load_pending(Verb::Publish).unwrap().is_some(), "fixture: the request was not folded");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("work.txt"), "fixture: the barrier did not publish the file");
assert_eq!(
manifest::load_pointer(store.as_ref(), &a.cfg).await.unwrap().unwrap().pointer.boundary_source.as_deref(),
Some("sentinel-deferred"),
"the barrier that read the request did not carry it"
);
hitl_remove(&store, &a.cfg, "work.txt", "reviewer").await;
let ack = a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
assert_eq!(ack.status, "partial", "the ack says the declared file is in seq {:?}, which no longer cites it: {ack:?}", ack.seq);
assert_eq!(ack.report.dropped, vec!["work.txt".to_string()], "{ack:?}");
}
#[tokio::test]
async fn a_carried_delete_then_upload_names_a_later_delete_of_the_upload() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.floor_tick().await.unwrap();
a.floor_tick().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("f.txt"), "fixture: v1 was not published");
let squat = dir.path().join(super::CONTROL_DIR).join(control::PUBLISH_ACK);
let fail_honor = |a: &Syncer| {
let _ = std::fs::remove_file(&squat);
std::fs::create_dir_all(squat.join("occupied")).unwrap();
clear_min_interval(a);
};
std::fs::remove_file(dir.path().join("f.txt")).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
fail_honor(&a);
assert!(a.honor_pending(Verb::Publish, false).await.is_err(), "fixture: the ack write did not fail");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("f.txt"), "fixture: the declared delete was not carried");
std::fs::remove_dir_all(&squat).unwrap();
write(dir.path(), "f.txt", "v2, the declared bytes");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-2"}"#);
a.poll_sentinels().unwrap();
fail_honor(&a);
assert!(a.honor_pending(Verb::Publish, false).await.is_err(), "fixture: the second ack write did not fail");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("f.txt"), "fixture: the re-created file was not carried");
std::fs::remove_dir_all(&squat).unwrap();
hitl_remove(&store, &a.cfg, "f.txt", "reviewer").await;
let ack = a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
assert_eq!(ack.status, "partial", "the ack says the declared file is in seq {:?}, which no longer cites it: {ack:?}", ack.seq);
assert_eq!(ack.report.dropped, vec!["f.txt".to_string()], "{ack:?}");
}
#[tokio::test]
async fn a_carried_upload_the_ui_replaced_by_a_rename_is_named() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
write(dir.path(), "g.txt", "an older, unrelated file");
a.floor_tick().await.unwrap();
a.floor_tick().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("f.txt") && m.entries.contains_key("g.txt"), "fixture: the files were not published");
let squat = dir.path().join(super::CONTROL_DIR).join(control::PUBLISH_ACK);
write(dir.path(), "f.txt", "v2, the declared bytes");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
std::fs::create_dir_all(squat.join("occupied")).unwrap();
clear_min_interval(&a);
assert!(a.honor_pending(Verb::Publish, false).await.is_err(), "fixture: the ack write did not fail");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let published = m.entries.get("f.txt").expect("fixture: the declared file was not carried").crc64_b64.clone();
std::fs::remove_dir_all(&squat).unwrap();
hitl_remove(&store, &a.cfg, "f.txt", "reviewer").await;
hitl_rename(&store, &a.cfg, "g.txt", "f.txt", "reviewer").await;
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_ne!(m.entries["f.txt"].crc64_b64, published, "fixture: the path still cites the declared bytes");
let ack = a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
assert_eq!(ack.status, "partial", "the ack says the declared file is in seq {:?}, which cites other bytes there: {ack:?}", ack.seq);
assert_eq!(ack.report.dropped, vec!["f.txt".to_string()], "{ack:?}");
}
async fn a_published_file_renamed_away_across_a_restart(
other_dirty: bool,
restart: bool,
) -> (super::sentinel::Ack, manifest::LeanManifest) {
let (inner, dir, mut a, cfg) = renamed_away_after_an_install(restart).await;
if other_dirty {
write(dir.path(), "r.txt", "another declared file");
}
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
hitl_rename(&inner, &cfg, "q.txt", "p.txt", "reviewer").await;
land_delete_during_next_barrier(&inner, &a, "p.txt");
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
assert!(super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()), "fixture: the window never ran");
let m = manifest::load(inner.as_ref(), &cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("p.txt"), "fixture: the UI delete did not land");
(ack, m)
}
async fn renamed_away_after_an_install(restart: bool) -> (Arc<MemoryStore>, tempfile::TempDir, Syncer, LeanConfig) {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "p.txt", "v1");
write(dir.path(), "tmp.txt", "scratch");
a.floor_tick().await.unwrap();
a.floor_tick().await.unwrap();
write(dir.path(), "p.txt", "v2, the agent's bytes");
backdate_baseline(&a, "p.txt");
std::fs::remove_file(dir.path().join("tmp.txt")).unwrap();
let state_dir = a.cfg.state_dir();
let snap: Arc<std::sync::Mutex<Vec<(std::path::PathBuf, Option<Vec<u8>>)>>> = Default::default();
let snap_in = snap.clone();
hooked.before_delete(&a.cfg.file_key("tmp.txt"), move || {
*snap_in.lock().unwrap() = ["intent.json", "baseline.json"]
.iter()
.map(|f| {
let p = state_dir.join(f);
let bytes = std::fs::read(&p).ok();
(p, bytes)
})
.collect();
});
a.declared_barrier().await.unwrap();
let taken = std::mem::take(&mut *snap.lock().unwrap());
assert!(!taken.is_empty(), "fixture: the install never reached its GC delete");
let cfg = a.cfg.clone();
let m = manifest::load(inner.as_ref(), &cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["p.txt"].crc64_b64, crc_of("v2, the agent's bytes"), "fixture: v2 was not installed");
if restart {
for (p, bytes) in taken {
match bytes {
Some(b) => std::fs::write(&p, b).unwrap(),
None => {
let _ = std::fs::remove_file(&p);
}
}
}
}
drop(a);
hitl_rename(&inner, &cfg, "p.txt", "q.txt", "reviewer").await;
let mut a = syncer(&inner, dir.path()).await;
a.settle_pending_at_startup().await.unwrap();
let stale = a.state.load_baseline().unwrap().entries.get("p.txt").map(|e| e.etag.clone());
assert_eq!(stale != Some(m.entries["p.txt"].etag.clone()), restart, "fixture: the baseline after the restart");
(inner, dir, a, cfg)
}
#[tokio::test]
async fn a_restart_before_step_7_does_not_change_the_ack_of_a_renamed_away_file() {
let (ack, _) = a_published_file_renamed_away_across_a_restart(false, true).await;
let (control, _) = a_published_file_renamed_away_across_a_restart(false, false).await;
assert_eq!((ack.status.as_str(), &ack.report.dropped), (control.status.as_str(), &control.report.dropped), "{ack:?}");
assert_eq!(ack.status, "ok", "{ack:?}");
}
#[tokio::test]
async fn a_restart_before_step_7_does_not_declare_a_renamed_away_file_an_install_merges_past() {
let (ack, m) = a_published_file_renamed_away_across_a_restart(true, true).await;
let (control, _) = a_published_file_renamed_away_across_a_restart(true, false).await;
assert_eq!(ack.seq, Some(m.seq), "fixture: the ack does not name the install past the delete: {ack:?}");
assert_eq!((ack.status.as_str(), &ack.report.dropped), (control.status.as_str(), &control.report.dropped), "{ack:?}");
assert_eq!(ack.status, "ok", "{ack:?}");
}
#[tokio::test]
async fn a_restart_before_step_7_republishes_a_renamed_away_file_and_names_its_later_delete() {
let (inner, dir, mut a, cfg) = renamed_away_after_an_install(true).await;
let squat = dir.path().join(super::CONTROL_DIR).join(control::PUBLISH_ACK);
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
std::fs::create_dir_all(squat.join("occupied")).unwrap();
clear_min_interval(&a);
assert!(a.honor_pending(Verb::Publish, false).await.is_err(), "fixture: the ack write did not fail");
std::fs::remove_dir_all(&squat).unwrap();
let m = manifest::load(inner.as_ref(), &cfg).await.unwrap().unwrap().manifest;
let v2 = crc_of("v2, the agent's bytes");
assert_eq!(m.entries.get("p.txt").map(|e| &e.crc64_b64), Some(&v2), "the barrier did not re-publish p.txt");
assert_eq!(m.entries.get("q.txt").map(|e| &e.crc64_b64), Some(&v2), "fixture: the UI's rename was lost");
let carried = a.state.load_intent().unwrap().carrier.map(|c| c.paths).unwrap_or_default();
assert!(carried.contains("p.txt"), "the barrier did not carry p.txt: {carried:?}");
hitl_remove(&inner, &cfg, "p.txt", "reviewer").await;
let ack = a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered");
assert_eq!(ack.status, "partial", "the ack says the carried file is in seq {:?}, which no longer cites it: {ack:?}", ack.seq);
assert_eq!(ack.report.dropped, vec!["p.txt".to_string()], "{ack:?}");
}
async fn a_forced_honour_names_the_clock_it_stamped(restart: bool) {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "s");
a.floor_tick().await.unwrap();
a.floor_tick().await.unwrap();
write(dir.path(), "work.txt", "the declared bytes");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1"}"#);
a.poll_sentinels().unwrap();
clear_min_interval(&a);
let ack = if restart {
drop(a);
let mut a = syncer(&store, dir.path()).await;
a.settle_pending_at_startup().await.unwrap();
a.read_ack(Verb::Publish).expect("the restart answered nothing")
} else {
a.honor_pending(Verb::Publish, true).await.unwrap().expect("nothing was answered")
};
let p = manifest::load_pointer(store.as_ref(), &cfg_for(dir.path())).await.unwrap().unwrap();
let m = manifest::load(store.as_ref(), &cfg_for(dir.path())).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("work.txt"), "fixture: the honour installed nothing: {ack:?}");
assert_eq!(ack.seq, Some(p.pointer.seq), "the ack names a seq its barrier did not install: {ack:?}");
assert_eq!(
Some(ack.boundary.as_str()),
p.pointer.boundary_source.as_deref(),
"the ack and the bucket name different clocks for one install: {ack:?}"
);
assert_eq!(ack.boundary, "sentinel-deferred", "{ack:?}");
}
#[tokio::test]
async fn a_forced_honour_names_the_clock_its_install_stamped() {
a_forced_honour_names_the_clock_it_stamped(false).await;
}
#[tokio::test]
async fn a_startup_settle_names_the_clock_its_install_stamped() {
a_forced_honour_names_the_clock_it_stamped(true).await;
}
#[tokio::test]
async fn the_drains_publish_ack_names_the_drain_clock_in_its_one_write() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "w.txt", "drained");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"d-1"}"#);
a.poll_sentinels().unwrap();
let ack = a.honor_pending_as(Verb::Publish, true, Some("drain")).await.unwrap().expect("nothing was answered");
let on_disk = a.read_ack(Verb::Publish).expect("no ack was written");
let pointer = manifest::load_pointer(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert_eq!(pointer.pointer.boundary_source.as_deref(), Some("drain"), "fixture: the drain's barrier was not stamped drain");
assert_eq!(on_disk.boundary, "drain", "the ack names a clock the bucket does not: {on_disk:?}");
assert_eq!(ack.boundary, "drain", "{ack:?}");
}
#[tokio::test]
async fn a_verb_request_retries_a_409_on_the_cell() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let st: &dyn ObjectStore = hooked.as_ref();
*hooked.1.put_conflicts.lock().unwrap() = Some((cfg.inbox_key(), 1));
inbox::gateway_request(st, &cfg, inbox::RequestedVerb::Boundary, "ui")
.await
.expect("a 409 on the cell was an error, not a lost race");
assert_eq!(hooked.1.put_conflicts.lock().unwrap().as_ref().map(|x| x.1), Some(0), "fixture: no 409 was injected");
assert!(inbox::load(inner.as_ref(), &cfg).await.unwrap().doc.boundary_request.is_some());
}
async fn h7_fixture(path: &str) -> (Arc<MemoryStore>, tempfile::TempDir, Syncer) {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.checkout().await.unwrap();
write(dir.path(), path, "v0");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, path, "the ui's version", "ui").await.unwrap();
assert_eq!(read(dir.path(), path).as_deref(), Some("v0"), "fixture: the UI write already reached the tree");
(store, dir, sc)
}
fn h7_hook(stage: &'static str, root: &std::path::Path, path: &str, agent_bytes: &'static str) {
let target = root.join(path);
super::barrier::CONSUME_WINDOW_HOOK.with(|h| {
*h.borrow_mut() = Some((
path.to_string(),
stage,
Box::new(move || std::fs::write(&target, agent_bytes).unwrap()),
))
});
}
#[tokio::test]
async fn the_consume_never_overwrites_an_agent_write_made_while_it_writes() {
let (store, dir, mut sc) = h7_fixture("h7-a.txt").await;
h7_hook("before-write", dir.path(), "h7-a.txt", "the agent's write, inside the consume's window");
sc.run_barrier().await.unwrap();
assert!(
super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()),
"fixture: the consume never reached its write window"
);
assert_eq!(
read(dir.path(), "h7-a.txt").as_deref(),
Some("the agent's write, inside the consume's window"),
"the consume overwrote an agent write it never saw"
);
let c = sc.state.load_conflicts().unwrap();
let rec = c
.iter()
.find(|r| r.path == "h7-a.txt" && r.kind.starts_with("commit-surfaced-foreign"))
.expect("no record of the UI's version");
let (_, body) = store.get_whole(rec.preserved_key.as_ref().unwrap(), None).await.unwrap();
assert_eq!(&body[..], b"the ui's version");
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["h7-a.txt"].crc64_b64, crc_of("the agent's write, inside the consume's window"));
}
#[tokio::test]
async fn an_agent_write_just_after_the_consume_is_published() {
let (store, dir, mut sc) = h7_fixture("h7-b.txt").await;
h7_hook("after-rename", dir.path(), "h7-b.txt", "the agent's write, after the rename, much longer");
sc.run_barrier().await.unwrap();
assert!(
super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()),
"fixture: the consume never reached its write window"
);
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
m.entries["h7-b.txt"].crc64_b64,
crc_of("the agent's write, after the rename, much longer"),
"an agent write made just after the consume was recorded as the consumed version and never published"
);
}
#[tokio::test]
async fn the_upload_refuses_a_directory_swapped_for_a_symlink_after_the_scan() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
a.checkout().await.unwrap();
let outside = tempfile::tempdir().unwrap();
std::fs::write(outside.path().join("cfg.yaml"), "SECRET, outside the workspace").unwrap();
write(dir.path(), "d/cfg.yaml", "harmless");
write(dir.path(), "aaa-first.txt", "uploads before d/cfg.yaml");
a.cfg.upload_fanout = 1;
let (d, bak, target) = (dir.path().join("d"), dir.path().join("d.bak"), outside.path().to_path_buf());
hooked.before_put(&a.cfg.file_key("aaa-first.txt"), move || {
std::fs::rename(&d, &bak).unwrap();
std::os::unix::fs::symlink(&target, &d).unwrap();
});
let r = a.run_barrier().await.expect("a planted directory link must not fail the barrier");
assert!(dir.path().join("d.bak").exists(), "fixture: the swap never ran");
assert!(!r.uploaded.contains(&"d/cfg.yaml".to_string()), "the link's target was published: {r:?}");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("d/cfg.yaml"), "the manifest cites a file outside the workspace");
assert!(inner.head(&a.cfg.file_key("d/cfg.yaml")).await.is_err(), "the outside bytes reached the bucket");
let recs = a.state.load_conflicts().unwrap();
assert!(recs.iter().any(|c| c.path == "d/cfg.yaml"), "nothing recorded the refusal: {recs:?}");
}
#[tokio::test]
async fn the_compose_path_reads_the_descriptor_it_was_given_not_the_path_it_was_named() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let dir = tempfile::tempdir().unwrap();
let mut a = hooked_syncer(&hooked, dir.path());
a.checkout().await.unwrap();
let outside = tempfile::tempdir().unwrap();
let mine = "harmless bytes, inside the workspace";
let theirs = format!("{:!<1$}", "SECRET, outside the workspace", mine.len());
assert_eq!(theirs.len(), mine.len());
std::fs::write(outside.path().join("cfg.yaml"), &theirs).unwrap();
write(dir.path(), "d/cfg.yaml", mine);
a.cfg.upload_fanout = 1;
a.cfg.whole_put_max = 8; let (d, bak, target) =
(dir.path().join("d"), dir.path().join("d.bak"), outside.path().to_path_buf());
hooked.before_compose(&a.cfg.file_key("d/cfg.yaml"), move || {
std::fs::rename(&d, &bak).unwrap();
std::os::unix::fs::symlink(&target, &d).unwrap();
});
let r = a.run_barrier().await.expect("a planted directory link must not fail the barrier");
assert!(dir.path().join("d.bak").exists(), "fixture: the swap never ran");
assert!(r.uploaded.contains(&"d/cfg.yaml".to_string()), "fixture: the compose path never ran: {r:?}");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let key = m.entries.get("d/cfg.yaml").expect("the file was not published at all").key.clone();
let (_, body) = inner.get_whole(&key, None).await.unwrap();
assert_eq!(
std::str::from_utf8(&body).unwrap(),
mine,
"the compose path published the link target's bytes: it reopened the path instead of \
reading the descriptor the syncer opened"
);
}
#[tokio::test]
async fn a_holder_deposed_during_its_commit_heads_does_not_install_onto_its_successors_document() {
deposed_during_its_commit_heads(false).await;
}
#[tokio::test]
async fn a_holder_deposed_during_its_commit_heads_is_fenced_by_the_load_not_by_a_renew() {
deposed_during_its_commit_heads(true).await;
}
async fn deposed_during_its_commit_heads(shipped_spacing: bool) {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&hooked, dir_a.path());
a.checkout().await.unwrap();
write(dir_a.path(), "base.txt", "x");
a.run_barrier().await.unwrap();
syncer(&inner, dir_b.path()).await.checkout().await.unwrap();
if shipped_spacing {
a.cfg.claim_quiet_spacing_secs = lease::QUIET_SPACING_SECS;
}
write(dir_a.path(), "a.txt", "A's upload");
{
let (inner, root_b) = (inner.clone(), dir_b.path().to_path_buf());
hooked.after_head(&a.cfg.file_key("a.txt"), move || {
let (inner, root_b) = (inner.clone(), root_b.clone());
std::thread::spawn(move || {
tokio::runtime::Runtime::new().unwrap().block_on(async {
let mut b = syncer(&inner, &root_b).await;
assert!(claim_until_held(&mut b, 10).await, "fixture: B never deposed A");
write(&root_b, "b.txt", "B's commit");
b.run_barrier().await.unwrap();
})
})
.join()
.unwrap();
});
}
let before = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
let r = a.run_barrier().await;
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("b.txt"), "fixture: B's commit did not land (seq {before} -> {})", m.seq);
assert!(
!m.entries.contains_key("a.txt"),
"A, deposed during its commit, installed onto its successor's rotated document: {r:?}"
);
assert!(matches!(r, Err(LeanError::Fenced(_))), "a deposed holder's barrier must end fenced: {r:?}");
}
#[tokio::test]
async fn a_holder_renews_once_its_cell_has_stood_still_half_the_takeover_threshold() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.claim_quiet_spacing_secs = lease::QUIET_SPACING_SECS;
let key = a.cfg.epoch_key();
lease::renew_if_due(&mut a).await.expect("no lease is nothing to renew");
assert!(store.epoch_read(&key).await.unwrap().is_none(), "a renew without a lease wrote the cell");
assert!(claim_until_held(&mut a, 3).await);
let at_claim = store.epoch_read(&key).await.unwrap().unwrap().token;
lease::renew_if_due(&mut a).await.unwrap();
assert_eq!(
store.epoch_read(&key).await.unwrap().unwrap().token,
at_claim,
"renewed a cell written a moment ago"
);
a.cell_written_at = std::time::Instant::now().checked_sub(std::time::Duration::from_secs(31));
lease::renew_if_due(&mut a).await.unwrap();
assert_ne!(
store.epoch_read(&key).await.unwrap().unwrap().token,
at_claim,
"a cell that stood still 31 s of a 60 s takeover threshold was not renewed"
);
assert!(a.cell_written_at.unwrap().elapsed() < std::time::Duration::from_secs(5), "the renew did not restart the clock");
}
#[tokio::test]
async fn a_commit_section_renews_its_cell_before_the_cas_and_before_it_deletes() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
let buf = Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
a.cfg.event_trace = Some(crate::trace::Sink::Memory(buf.clone()));
write(dir.path(), "f.txt", "v2");
a.run_barrier().await.unwrap();
let evs: Vec<String> = buf
.lock()
.unwrap()
.iter()
.map(|l| serde_json::from_str::<serde_json::Value>(l).unwrap()["ev"].as_str().unwrap_or("").to_string())
.collect();
let at = |name: &str| evs.iter().position(|e| e == name).unwrap_or_else(|| panic!("no {name}: {evs:?}"));
let (claim, cas, release) = (at("claim"), at("cas"), at("release"));
assert!(evs[claim..cas].iter().any(|e| e == "renew"), "no renew between the claim and the CAS: {evs:?}");
assert!(evs[cas..release].iter().any(|e| e == "renew"), "no renew between the CAS and the release: {evs:?}");
}
#[tokio::test]
async fn a_holder_deposed_after_its_cas_deletes_nothing_under_its_successor() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&hooked, dir_a.path());
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
syncer(&inner, dir_b.path()).await.checkout().await.unwrap();
let v1 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["f.txt"].key.clone();
let deposed = Arc::new(std::sync::atomic::AtomicBool::new(false));
{
let (inner, root_b, deposed) = (inner.clone(), dir_b.path().to_path_buf(), deposed.clone());
hooked.before_delete(&v1, move || {
let (inner, root_b) = (inner.clone(), root_b.clone());
std::thread::spawn(move || {
tokio::runtime::Runtime::new().unwrap().block_on(async {
let mut b = syncer(&inner, &root_b).await;
assert!(claim_until_held(&mut b, 10).await, "fixture: B never deposed A");
})
})
.join()
.unwrap();
deposed.store(true, std::sync::atomic::Ordering::SeqCst);
});
}
write(dir_a.path(), "f.txt", "v2");
let mut r = a.run_barrier().await;
if !deposed.load(std::sync::atomic::Ordering::SeqCst) {
write(dir_a.path(), "f.txt", "v3");
r = a.run_barrier().await;
}
assert!(deposed.load(std::sync::atomic::Ordering::SeqCst), "fixture: A's collector never reached v1's handle");
assert!(r.is_ok(), "the publish landed before the deposal and stands: {r:?}");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_ne!(m.entries["f.txt"].key, v1, "fixture: A's CAS did not land");
assert!(a.lease.is_none(), "A still believes it holds the cell its successor took");
let logs = inner.list(&manifest::retired_prefix(&a.cfg)).await.unwrap();
assert!(!logs.is_empty(), "A, deposed, went on deleting retire logs under its successor's cell");
}
#[tokio::test]
async fn a_holder_deposed_during_one_sweep_batch_deletes_no_further_batch() {
const LOST: &str = "5f0c1d2e-0000-4000-8000-00000000dead";
let n = super::untracked::DELETE_RENEW_BATCH + 100;
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&hooked, dir_a.path());
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
syncer(&inner, dir_b.path()).await.checkout().await.unwrap();
let stamps = GenerationStamps { generation: 0, epoch: 0, flush_uuid: LOST.into(), boundary_source: None, posix: None };
let keys: Vec<String> = (0..n).map(|i| a.cfg.handle_key(&format!("o/{i:04}.bin"), LOST)).collect();
for k in &keys {
let body = Bytes::from_static(b"o");
inner.put_whole(k, body.clone(), &PutCondition::IfNoneMatchAny, &stamps, crc64_nvme(&body)).await.unwrap();
}
a.cfg.untracked_grace_secs = 0;
a.state.save_orphan_sweep_at(now_unix() - a.cfg.untracked_sweep_secs - 1).unwrap();
let deposed = Arc::new(std::sync::atomic::AtomicBool::new(false));
{
let (inner, root_b, deposed) = (inner.clone(), dir_b.path().to_path_buf(), deposed.clone());
hooked.before_delete(&keys[0], move || {
let (inner, root_b) = (inner.clone(), root_b.clone());
std::thread::spawn(move || {
tokio::runtime::Runtime::new().unwrap().block_on(async {
let mut b = syncer(&inner, &root_b).await;
assert!(claim_until_held(&mut b, 10).await, "fixture: B never deposed A");
})
})
.join()
.unwrap();
deposed.store(true, std::sync::atomic::Ordering::SeqCst);
});
}
a.cfg.orphan_grace_secs = 0;
let chunks = format!("{}/{}/chunks/", a.cfg.prefix, super::LEAN_DIR);
let superseded: Vec<String> = inner.list(&chunks).await.unwrap().into_iter().map(|o| o.key).collect();
assert!(!superseded.is_empty(), "fixture: the workspace is not chunked");
write(dir_a.path(), "f.txt", "v2");
let r = a.run_barrier().await;
assert!(deposed.load(std::sync::atomic::Ordering::SeqCst), "fixture: A's sweep never reached its first batch: {r:?}");
for k in &superseded {
assert!(inner.head(k).await.is_ok(), "A, deposed, reaped chunk {k} under its successor's cell: {r:?}");
}
let left = inner.list(&a.cfg.files_prefix()).await.unwrap().into_iter().filter(|o| o.key.contains(LOST)).count();
assert!(left < n, "fixture: the first batch was not deleted ({left} of {n} left)");
assert!(
left >= n - super::untracked::DELETE_RENEW_BATCH,
"A, deposed during its first batch, went on deleting under its successor's cell ({left} of {n} left): {r:?}"
);
}
#[tokio::test]
async fn a_restart_between_a_deposal_and_its_rotation_still_rotates() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b, dir_c) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&hooked, dir_a.path());
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "x");
a.run_barrier().await.unwrap();
assert!(claim_until_held(&mut a, 3).await);
let straggler = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut b = hooked_syncer(&hooked, dir_b.path());
loop {
match lease::claim_step(&mut b, true).await.unwrap() {
lease::ClaimOutcome::Waiting { quiet_polls, .. } if quiet_polls >= 5 => break,
lease::ClaimOutcome::Waiting { .. } => {}
lease::ClaimOutcome::Claimed(_) => panic!("fixture: claimed before the threshold"),
}
}
hooked.1.acquire_err_once.store(true, std::sync::atomic::Ordering::SeqCst);
assert!(lease::claim_step(&mut b, true).await.is_err(), "fixture: the acquire's response was not lost");
let b_id = b.state.load_incarnation().unwrap().unwrap().holder_id;
assert_eq!(inner.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap().holder_id, b_id, "fixture: B's acquire did not land");
drop(b);
let mut b2 = hooked_syncer(&hooked, dir_b.path());
lease::release_stale_own(&mut b2).await.unwrap();
let mut c = syncer(&inner, dir_c.path()).await;
assert!(claim_until_held(&mut c, 3).await, "fixture: C could not claim the released cell");
let mut mine = straggler.manifest.clone();
mine.seq += 1;
let r = manifest::cas_write(inner.as_ref(), &a.cfg, &mine, Some(&straggler.handle()), 1, "straggler").await;
assert!(
matches!(r, Err(LeanError::Store(flint_store::StoreError::PreconditionFailed(_)))),
"a deposed holder's CAS landed: the restart released the deposal without its rotation ({:?})",
r.map(|m| m.etag)
);
}
#[tokio::test]
async fn a_takeover_rotation_that_meets_a_409_retries_it() {
let inner = Arc::new(MemoryStore::new());
let hooked = Arc::new(Hooked(inner.clone(), Hooks::default()));
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = hooked_syncer(&hooked, dir_a.path());
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "x");
a.run_barrier().await.unwrap();
let seq0 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert!(claim_until_held(&mut a, 3).await);
let mut b = hooked_syncer(&hooked, dir_b.path());
loop {
match lease::claim_step(&mut b, true).await.unwrap() {
lease::ClaimOutcome::Waiting { quiet_polls, .. } if quiet_polls >= 5 => break,
lease::ClaimOutcome::Waiting { .. } => {}
lease::ClaimOutcome::Claimed(_) => panic!("fixture: claimed before the threshold"),
}
}
*hooked.1.put_conflicts.lock().unwrap() = Some((b.cfg.current_key(), 1));
let outcome = lease::claim_step(&mut b, true).await;
assert_eq!(hooked.1.put_conflicts.lock().unwrap().as_ref().map(|x| x.1), Some(0), "fixture: no 409 was injected");
assert!(
matches!(outcome, Ok(lease::ClaimOutcome::Claimed(_))),
"a 409 on the takeover rotation failed a claim whose acquire had landed: {:?}",
outcome.err()
);
let seq1 = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
assert!(seq1 > seq0, "the takeover did not rotate");
}
async fn h7_sync_fixture(path: &str) -> (Arc<MemoryStore>, tempfile::TempDir, Syncer) {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.checkout().await.unwrap();
write(dir.path(), path, "v0");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, path, "the remote version", "ui").await.unwrap();
(store, dir, sc)
}
#[tokio::test]
async fn sync_never_overwrites_an_agent_write_made_while_it_writes() {
let (_store, dir, mut sc) = h7_sync_fixture("h7-s1.txt").await;
h7_hook("before-write", dir.path(), "h7-s1.txt", "the agent's write, during the sync");
let r = sc.sync().await.unwrap();
assert!(
super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()),
"fixture: the sync never reached its write window"
);
assert_eq!(
read(dir.path(), "h7-s1.txt").as_deref(),
Some("the agent's write, during the sync"),
"sync overwrote an agent write it never saw"
);
assert!(r.conflicts.contains(&"h7-s1.txt".to_string()), "{r:?}");
}
#[tokio::test]
async fn an_agent_write_just_after_a_sync_applies_is_published() {
let (store, dir, mut sc) = h7_sync_fixture("h7-s2.txt").await;
h7_hook("after-rename", dir.path(), "h7-s2.txt", "the agent's write, after the sync's rename, longer");
sc.sync().await.unwrap();
assert!(
super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()),
"fixture: the sync never reached its write window"
);
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
m.entries["h7-s2.txt"].crc64_b64,
crc_of("the agent's write, after the sync's rename, longer"),
"an agent write made just after the sync applied was recorded as the synced version and never published"
);
}
#[tokio::test]
async fn sync_never_deletes_an_agent_write_made_after_its_scan() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.checkout().await.unwrap();
write(dir.path(), "h7-s3.txt", "v0");
sc.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.entries.remove("h7-s3.txt");
manifest::cas_write(store.as_ref(), &sc.cfg, &theirs, Some(&loaded.handle()), 0, "remote-delete").await.unwrap();
h7_hook("before-delete", dir.path(), "h7-s3.txt", "the agent's write, after the sync's scan");
let r = sc.sync().await.unwrap();
assert!(
super::barrier::CONSUME_WINDOW_HOOK.with(|h| h.borrow().is_none()),
"fixture: the sync never reached its delete"
);
assert_eq!(
read(dir.path(), "h7-s3.txt").as_deref(),
Some("the agent's write, after the sync's scan"),
"sync applied a remote delete over an agent write made after its scan"
);
assert!(r.conflicts.contains(&"h7-s3.txt".to_string()), "{r:?}");
}
#[tokio::test]
async fn a_published_file_replaced_by_a_directory_never_leaves_a_document_no_checkout_can_hold() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "a", "a file, published");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir_a.path().join("a")).unwrap();
write(dir_a.path(), "a/x", "now a directory");
let r = a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited: Vec<&String> = m.entries.keys().collect();
let clash: Vec<(&String, &String)> = cited
.iter()
.flat_map(|p| cited.iter().map(move |q| (*p, *q)))
.filter(|(p, q)| p != q && q.starts_with(&format!("{p}/")))
.collect();
assert!(
clash.is_empty(),
"the document cites a path and something under it, which no filesystem can hold: \
{clash:?} (barrier: {r:?})"
);
let mut b = syncer(&store, dir_b.path()).await;
b.checkout().await.expect("a fresh checkout could not materialise the published document");
assert_eq!(read(dir_b.path(), "keep.txt").as_deref(), Some("keep"));
let r2 = a.run_barrier().await.unwrap();
assert_eq!(r2.deleted, vec!["a".to_string()], "the absence never confirmed: {r2:?}");
assert_eq!(r2.uploaded, vec!["a/x".to_string()], "the withheld path never published: {r2:?}");
let m2 = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m2.entries.contains_key("a"), "the file citation outlived its file");
assert!(m2.entries.contains_key("a/x"));
let dir_c = tempfile::tempdir().unwrap();
syncer(&store, dir_c.path()).await.checkout().await.unwrap();
assert_eq!(read(dir_c.path(), "a/x").as_deref(), Some("now a directory"));
}
#[tokio::test]
async fn a_published_directory_replaced_by_a_file_never_leaves_a_document_no_checkout_can_hold() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_c) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "a/x", "a directory, published");
a.run_barrier().await.unwrap();
std::fs::remove_dir_all(dir_a.path().join("a")).unwrap();
write(dir_a.path(), "a", "now a file");
let r = a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!(m.entries.contains_key("a") && m.entries.contains_key("a/x")),
"the document cites a file and a path inside it: {:?} ({r:?})",
m.entries.keys().collect::<Vec<_>>()
);
assert_eq!(r.parked, vec!["a".to_string()], "the parent did not yield: {r:?}");
let r2 = a.run_barrier().await.unwrap();
assert_eq!(r2.deleted, vec!["a/x".to_string()], "{r2:?}");
assert_eq!(r2.uploaded, vec!["a".to_string()], "{r2:?}");
syncer(&store, dir_c.path()).await.checkout().await.unwrap();
assert_eq!(read(dir_c.path(), "a").as_deref(), Some("now a file"));
}
#[tokio::test]
async fn a_ui_delete_then_a_write_under_that_name_turns_the_file_into_a_directory() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "build", "a file, published");
a.run_barrier().await.unwrap();
hitl_remove(&store, &a.cfg, "build", "human").await;
hitl_write(&store, &a.cfg, "build/log.txt", "now a directory", "human").await.unwrap();
for _ in 0..4 {
a.run_barrier().await.unwrap();
}
let recs = a.state.load_conflicts().unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
read(dir.path(), "build/log.txt").as_deref(),
Some("now a directory"),
"the acked UI write never reached the tree; records: {:?}",
recs.iter().map(|c| (&c.path, &c.kind)).collect::<Vec<_>>()
);
assert!(!m.entries.contains_key("build"), "the deleted file is still cited");
assert!(m.entries.contains_key("build/log.txt"), "the UI's write was never published: {m:?}");
}
#[tokio::test]
async fn sync_turns_a_file_into_a_directory_in_one_pass() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "build", "a file, published");
write(dir_a.path(), "keep.txt", "keep");
a.run_barrier().await.unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "build").as_deref(), Some("a file, published"));
std::fs::remove_file(dir_a.path().join("build")).unwrap();
write(dir_a.path(), "build/log.txt", "now a directory");
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
let r = b.sync().await.unwrap();
assert_eq!(
read(dir_b.path(), "build/log.txt").as_deref(),
Some("now a directory"),
"sync refused the add and then deleted the file: {r:?}"
);
assert!(dir_b.path().join("build").is_dir(), "the file was not replaced by the directory");
}
fn path_clashes_quadratic(cited: &std::collections::BTreeMap<String, ()>, upserts: &std::collections::BTreeMap<String, ()>) -> Vec<String> {
let cited: Vec<&String> = cited.keys().collect();
let mut mine: Vec<String> = vec![];
for p in &cited {
let pfx = format!("{p}/");
if cited.iter().any(|q| q.starts_with(&pfx)) {
let child = cited.iter().find(|q| q.starts_with(&pfx) && upserts.contains_key(**q)).map(|q| (*q).clone());
if upserts.contains_key(*p) {
mine.push((*p).clone());
} else if let Some(c) = child {
mine.push(c);
}
}
}
mine.sort();
mine.dedup();
mine
}
#[test]
fn the_clash_check_answers_as_the_every_pair_scan_did() {
let names = ["a", "a-b", "a.txt", "a/x", "a/x/y", "a/z", "ab", "ab/c", "b", "b0", "b/c"];
let n = names.len();
for cited_bits in 0u32..(1 << n) {
let cited: std::collections::BTreeMap<String, ()> =
(0..n).filter(|i| cited_bits & (1 << i) != 0).map(|i| (names[i].to_string(), ())).collect();
let keys: Vec<&String> = cited.keys().collect();
if cited_bits % 5 != 0 {
continue;
}
for up_bits in 0u32..(1 << keys.len()) {
let upserts: std::collections::BTreeMap<String, ()> = keys
.iter()
.enumerate()
.filter(|(i, _)| up_bits & (1 << i) != 0)
.map(|(_, k)| ((*k).clone(), ()))
.collect();
assert_eq!(
super::barrier::path_clashes(&cited, &upserts),
path_clashes_quadratic(&cited, &upserts),
"cited {keys:?}, upserts {:?}",
upserts.keys().collect::<Vec<_>>()
);
}
}
}
#[test]
fn the_clash_check_is_not_quadratic_in_the_document() {
let cited: std::collections::BTreeMap<String, ()> =
(0..30_000).map(|i| (format!("d{:04}/f{i:07}.txt", i / 1000), ())).collect();
let upserts: std::collections::BTreeMap<String, ()> = [("d0015/f0015000.txt".to_string(), ())].into();
let t = std::time::Instant::now();
assert!(super::barrier::path_clashes(&cited, &upserts).is_empty());
assert!(t.elapsed() < std::time::Duration::from_secs(5), "took {:?}", t.elapsed());
}
#[tokio::test]
async fn a_busy_agents_idle_tick_after_its_own_publish_is_the_pointer_alone() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
for i in 0..20 {
write(dir.path(), &format!("f{i}.txt"), "seed");
}
a.run_barrier().await.unwrap();
for v in ["v2", "v3"] {
write(dir.path(), "f1.txt", v);
backdate_baseline(&a, "f1.txt");
let r = a.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["f1.txt".to_string()], "fixture: {r:?}");
}
store.reset_op_counts();
a.run_barrier().await.unwrap();
let gets = store.op_counts().get("get_whole").copied().unwrap_or(0);
assert_eq!(gets, 2, "an idle tick after its own publish re-derived: {:?}", store.op_counts());
}
#[tokio::test]
async fn a_skipped_path_this_barrier_publishes_leaves_the_next_tick_cheap() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
peer_publish(&store, &a.cfg, "x.txt", "the peer's v2", "peer").await;
write(dir.path(), "x.txt", "the agent's edit");
backdate_baseline(&a, "x.txt");
a.consume_owed().await.unwrap();
assert!(a.state.load_baseline().unwrap().skipped.contains("x.txt"), "fixture: x was not skipped");
let r = a.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["x.txt".to_string()], "fixture: {r:?}");
store.reset_op_counts();
a.run_barrier().await.unwrap();
let gets = store.op_counts().get("get_whole").copied().unwrap_or(0);
assert_eq!(gets, 2, "the published path still read as the agent's pending work: {:?}", store.op_counts());
}
#[tokio::test]
async fn a_path_a_sync_declined_as_dirty_arrives_when_the_agent_reverts() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.checkout().await.unwrap();
write(dir.path(), "x.txt", "seed");
a.run_barrier().await.unwrap();
peer_publish(&store, &a.cfg, "x.txt", "the peer's v2", "peer").await;
write(dir.path(), "x.txt", "the agent's edit");
let r = a.sync().await.unwrap();
assert_eq!(r.conflicts, vec!["x.txt".to_string()], "fixture: the sync took x: {r:?}");
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("the agent's edit"), "fixture");
restore_to_baseline(&a, "x.txt", "seed");
for _ in 0..2 {
a.run_barrier().await.unwrap();
}
assert_eq!(read(dir.path(), "x.txt").as_deref(), Some("the peer's v2"), "the peer's version never arrived");
}
#[tokio::test]
async fn a_sync_against_an_absent_document_deletes_nothing() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep/a.txt", "published a");
write(dir.path(), "b.txt", "published b");
a.run_barrier().await.unwrap();
assert!(!a.state.load_baseline().unwrap().entries.is_empty());
for o in store.list(PREFIX).await.unwrap() {
store.delete(&o.key).await.unwrap();
}
assert!(manifest::load(store.as_ref(), &a.cfg).await.unwrap().is_none());
let r = a.sync().await;
assert!(r.is_err(), "a sync with no document must refuse, not treat it as empty: {r:?}");
assert_eq!(read(dir.path(), "keep/a.txt").as_deref(), Some("published a"));
assert_eq!(read(dir.path(), "b.txt").as_deref(), Some("published b"));
}
#[tokio::test]
async fn a_path_a_sync_could_not_fetch_stays_owed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "shared.txt", "user v2", "dilip").await.unwrap();
let key = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["shared.txt"].key.clone();
store.delete(&key).await.unwrap();
a.sync().await.unwrap();
assert_eq!(read(dir.path(), "shared.txt").as_deref(), Some("v1"), "nothing was fetchable");
for v in store.list_versions(&key).await.unwrap() {
if v.key == key && v.is_delete_marker {
store.delete_version(&key, &v.version_id).await.unwrap();
}
}
a.consume_owed().await.unwrap();
assert_eq!(read(dir.path(), "shared.txt").as_deref(), Some("user v2"), "the path the sync could not take was never owed again");
}
#[tokio::test]
async fn a_path_a_sync_could_not_write_stays_owed() {
use std::os::unix::fs::PermissionsExt;
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "sub/f.txt", "v1");
a.run_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "sub/f.txt", "user v2", "dilip").await.unwrap();
let sub = dir.path().join("sub");
std::fs::set_permissions(&sub, std::fs::Permissions::from_mode(0o555)).unwrap();
let r = a.sync().await.unwrap();
std::fs::set_permissions(&sub, std::fs::Permissions::from_mode(0o755)).unwrap();
assert!(r.conflicts.contains(&"sub/f.txt".to_string()), "the write did not fail: {r:?}");
assert_eq!(read(dir.path(), "sub/f.txt").as_deref(), Some("v1"));
a.consume_owed().await.unwrap();
assert_eq!(read(dir.path(), "sub/f.txt").as_deref(), Some("user v2"), "the path the sync could not write was never owed again");
}
#[tokio::test]
async fn a_reader_retries_what_its_last_sync_left_owed() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_r) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "x.txt", "A v1");
a.floor_tick().await.unwrap();
let mut r = reader(&store, dir_r.path()).await;
r.checkout().await.unwrap();
r.floor_tick().await.unwrap();
assert_eq!(read(dir_r.path(), "x.txt").as_deref(), Some("A v1"));
write(dir_a.path(), "x.txt", "A v2");
backdate_baseline(&a, "x.txt");
a.declared_barrier().await.unwrap();
let key = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries["x.txt"].key.clone();
store.delete(&key).await.unwrap();
r.floor_tick().await.unwrap();
assert_eq!(read(dir_r.path(), "x.txt").as_deref(), Some("A v1"), "nothing was fetchable");
for v in store.list_versions(&key).await.unwrap() {
if v.key == key && v.is_delete_marker {
store.delete_version(&key, &v.version_id).await.unwrap();
}
}
r.floor_tick().await.unwrap();
assert_eq!(read(dir_r.path(), "x.txt").as_deref(), Some("A v2"), "the reader never retried the path its sync left");
}
#[tokio::test]
async fn a_widen_never_overwrites_a_file_the_agent_made_at_an_admitted_path() {
let store = Arc::new(MemoryStore::new());
let (dir_a, dir_b) = (tempfile::tempdir().unwrap(), tempfile::tempdir().unwrap());
let mut a = syncer(&store, dir_a.path()).await;
a.checkout().await.unwrap();
write(dir_a.path(), "inputs/in.txt", "input");
write(dir_a.path(), "outputs/o.txt", "A's published output");
a.run_barrier().await.unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(read(dir_b.path(), "outputs/o.txt"), None);
write(dir_b.path(), "outputs/o.txt", "B's unpublished work");
let r = b.rescope(None).await.unwrap();
assert_eq!(read(dir_b.path(), "outputs/o.txt").as_deref(), Some("B's unpublished work"), "the widen overwrote the agent's file");
let recs = b.state.load_conflicts().unwrap();
assert!(recs.iter().any(|c| c.path == "outputs/o.txt"), "no record names the kept file: {r:?} {recs:?}");
let a_etag = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest.entries["outputs/o.txt"].etag.clone();
assert!(claim_until_held(&mut b, 3).await);
b.run_barrier().await.unwrap();
let doc = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_ne!(doc.entries["outputs/o.txt"].etag, a_etag, "B's work was not published");
let recs = b.state.load_conflicts().unwrap();
assert!(
recs.iter().any(|c| c.path == "outputs/o.txt" && c.foreign_etag == a_etag && c.preserved_key.is_some()),
"A's version was replaced with no record preserving it: {recs:?}"
);
}
#[tokio::test]
async fn an_ack_names_every_record_its_operation_produced_across_a_second_rotation() {
use super::sentinel::conflicts_since;
let store = Arc::new(MemoryStore::new());
for (case, rotated, extra) in [
("first drop, as many as it drops", 1, 0usize),
("first drop, more than it drops", 1, 7),
("a later drop, as many as it drops", 2, 0),
("a later drop, more than it drops", 2, 7),
] {
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
let sd = a.cfg.state_dir();
let mut rotations = 0;
let mut i = 0usize;
while rotations < rotated {
let had = std::fs::metadata(sd.join("conflicts.jsonl")).map(|m| m.len()).unwrap_or(0);
a.state.append_conflict(&bulky_conflict(i)).unwrap();
i += 1;
let has = std::fs::metadata(sd.join("conflicts.jsonl")).map(|m| m.len()).unwrap_or(0);
if has < had {
rotations += 1;
}
}
for n in 0..5 {
a.state.append_conflict(&bulky_conflict(500_000 + n)).unwrap();
}
let dropped_before = a.state.conflicts_dropped().unwrap();
assert_eq!(dropped_before > 0, rotated > 1, "{case}: the fixture's starting point");
let generation =
std::fs::read_to_string(sd.join("conflicts.1.jsonl")).unwrap().lines().filter(|l| !l.trim().is_empty()).count();
assert!(generation > 100, "{case}: the fixture needs a real generation: {generation}");
let before = a.state.conflicts_appended().unwrap();
let k = generation + extra;
let made: Vec<String> = (0..k)
.map(|n| {
let r = bulky_conflict(1_000_000 + n);
a.state.append_conflict(&r).unwrap();
r.path
})
.collect();
let dropped = a.state.conflicts_dropped().unwrap();
assert!(dropped > dropped_before, "{case}: the run must cross a rotation that drops");
let got: Vec<String> =
conflicts_since(before, dropped, a.state.load_conflicts().unwrap()).into_iter().map(|r| r.path).collect();
assert_eq!(got.len(), made.len(), "{case}: the ack must name all {k} records the operation produced");
assert_eq!(got, made, "{case}: and exactly those, oldest first");
}
}
#[test]
fn a_log_reset_under_a_reader_over_reports() {
use super::sentinel::conflicts_since;
let survivors = vec![bulky_conflict(7), bulky_conflict(8)];
assert_eq!(conflicts_since(50, 0, survivors).len(), 2);
}
#[tokio::test]
async fn a_touch_stranded_by_a_failed_fold_is_folded_before_the_next_consume() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
let pending = a.cfg.state_dir().join(Verb::Publish.pending_name());
let staging = a.cfg.state_dir().join("publish.consumed");
let sentinel = dir.path().join(super::CONTROL_DIR).join(control::PUBLISH);
std::fs::create_dir_all(&pending).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n1"}"#);
assert!(a.consume_sentinel(Verb::Publish).is_err(), "the fixture must make the fold fail");
assert!(staging.exists(), "the failed fold leaves the touch in staging");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n2"}"#);
assert!(a.consume_sentinel(Verb::Publish).is_err());
assert!(sentinel.exists(), "a touch the syncer cannot fold stays where the agent put it");
let staged = std::fs::read_to_string(&staging).unwrap();
assert!(staged.contains("n1"), "the stranded touch was overwritten: {staged}");
std::fs::remove_dir(&pending).unwrap();
assert!(a.consume_sentinel(Verb::Publish).unwrap(), "the touch was not consumed");
assert!(!staging.exists(), "staging is cleared once folded");
assert!(!sentinel.exists());
let nonces = a.load_pending(Verb::Publish).unwrap().expect("no pending record").nonces;
assert_eq!(nonces, vec!["n1".to_string(), "n2".to_string()], "every touch's nonce must reach the ack");
}
#[tokio::test]
async fn a_stranded_touch_is_folded_on_the_next_poll_without_another_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
let pending = a.cfg.state_dir().join(Verb::Publish.pending_name());
let staging = a.cfg.state_dir().join("publish.consumed");
std::fs::create_dir_all(&pending).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n1"}"#);
assert!(a.poll_sentinels().is_err(), "the fixture must make the fold fail");
assert!(staging.exists());
std::fs::remove_dir(&pending).unwrap();
let consumed = a.poll_sentinels().unwrap();
assert_eq!(consumed, vec![Verb::Publish], "the stranded touch counts as this poll's consume");
assert!(!staging.exists());
let nonces = a.load_pending(Verb::Publish).unwrap().expect("no pending record").nonces;
assert_eq!(nonces, vec!["n1".to_string()]);
}