use std::sync::Arc;
use bytes::Bytes;
use flint_store::memory::MemoryStore;
use flint_store::{crc64_nvme, GenerationStamps, ObjectStore, PutCondition};
use super::inbox::{self, InboxEntry};
use super::lease::{self, ClaimOutcome};
use super::manifest;
use super::state::SyncerState;
use super::{now_unix, LeanConfig, LeanError, Syncer};
const PREFIX: &str = "tenant/proj1";
fn cfg_for(root: &std::path::Path) -> LeanConfig {
LeanConfig::new(PREFIX, root)
}
fn cfg_single(root: &std::path::Path) -> LeanConfig {
let mut c = LeanConfig::new(PREFIX, root);
c.chunked = false;
c
}
async fn syncer(store: &Arc<MemoryStore>, root: &std::path::Path) -> Syncer {
let cfg = cfg_for(root);
let state = SyncerState::open(cfg.state_dir()).unwrap();
Syncer {
store: store.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
noted_not_regular: Default::default(),
}
}
async fn claim_until_held(sc: &mut Syncer, max_steps: u32) -> bool {
for _ in 0..max_steps {
match lease::claim_step(sc).await.unwrap() {
ClaimOutcome::Claimed(_) => return true,
ClaimOutcome::Waiting { .. } => {}
}
}
false
}
fn write(root: &std::path::Path, rel: &str, content: &str) {
let p = root.join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, content).unwrap();
}
fn read(root: &std::path::Path, rel: &str) -> Option<String> {
std::fs::read_to_string(root.join(rel)).ok()
}
fn backdate_baseline(sc: &Syncer, rel: &str) {
let mut b = sc.state.load_baseline().unwrap();
if let Some(e) = b.entries.get_mut(rel) {
e.mtime_unix -= 10;
}
sc.state.save_baseline(&b).unwrap();
}
async fn hitl_write(
store: &Arc<MemoryStore>,
cfg: &LeanConfig,
path: &str,
content: &str,
author: &str,
) -> Result<String, LeanError> {
let key = cfg.file_key(path);
let cond = match store.head(&key).await {
Ok(meta) => PutCondition::IfMatch(meta.etag),
Err(_) => PutCondition::IfNoneMatchAny,
};
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: format!("gateway-{author}"),
boundary_source: None,
posix: None,
};
let meta = store.put_whole(&key, body, &cond, &stamps, crc).await?;
inbox::gateway_append(
store.as_ref(),
cfg,
InboxEntry {
path: path.to_string(),
etag: meta.etag.clone(),
author: author.to_string(),
added_unix: now_unix(),
crc64_b64: Some(flint_store::crc64_to_b64(crc)),
},
)
.await?;
Ok(meta.etag)
}
#[tokio::test]
async fn checkout_publish_roundtrip_and_two_scan_delete() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap(); write(dir_a.path(), "src/main.rs", "fn main() {}");
write(dir_a.path(), "README.md", "hello");
let r = a.run_barrier().await.unwrap();
assert_eq!(r.uploaded.len(), 2);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(read(dir_b.path(), "src/main.rs").unwrap(), "fn main() {}");
std::fs::remove_file(dir_a.path().join("README.md")).unwrap();
let r1 = a.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty(), "first absence must not delete");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("README.md"));
let r2 = a.run_barrier().await.unwrap();
assert_eq!(r2.deleted, vec!["README.md".to_string()]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("README.md"));
assert!(store.head(&a.cfg.file_key("README.md")).await.is_err());
}
#[tokio::test]
async fn hitl_upload_survives_two_barriers_without_sync() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, "docs/upload.pdf", "user bytes", "dilip").await.unwrap();
write(dir.path(), "agent.txt", "agent work v2 — longer");
sc.run_barrier().await.unwrap();
sc.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "docs/upload.pdf").unwrap(), "user bytes");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("docs/upload.pdf"), "amputated!");
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "docs/upload.pdf").unwrap(), "user bytes");
}
async fn foreign_overwrite(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str, content: &str) {
let key = cfg.file_key(path);
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "a-stranger".to_string(),
boundary_source: None,
posix: None,
};
store
.put_whole(&key, body, &PutCondition::Unconditional, &stamps, crc)
.await
.expect("the foreign write lands");
}
#[tokio::test]
async fn a_published_workspace_refuses_a_foreign_write_instead_of_adopting_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.sole_writer = true;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "README.md", "the real readme");
sc.run_barrier().await.unwrap();
foreign_overwrite(&store, &sc.cfg, "README.md", "FOREIGN BYTES").await;
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
let err = sc2.checkout().await.expect_err("a published workspace must refuse");
let msg = format!("{err}");
assert!(msg.contains("SOLE WRITER"), "the refusal must say why: {msg}");
assert!(
!msg.contains("recover-staged"),
"that is the gated lane's advice; nothing was staged here: {msg}"
);
assert!(
read(dir2.path(), "README.md").is_none(),
"it materialized the foreign bytes anyway"
);
}
#[tokio::test]
async fn an_ordinary_workspace_still_adopts_bytes_that_moved_past_the_manifest() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(!sc.cfg.sole_writer, "the default is an ordinary workspace");
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "README.md", "the real readme");
sc.run_barrier().await.unwrap();
foreign_overwrite(&store, &sc.cfg, "README.md", "newer human bytes").await;
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.checkout().await.expect("an ordinary workspace adopts");
assert_eq!(read(dir2.path(), "README.md").unwrap(), "newer human bytes");
}
#[tokio::test]
async fn the_published_flag_survives_the_pointer_round_trip() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.sole_writer = true;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "alpha");
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.sole_writer, "the pointer lost the flag");
write(dir.path(), "b.txt", "beta");
sc.run_barrier().await.unwrap();
let m2 = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m2.manifest.sole_writer, "the second publish dropped the flag");
}
#[tokio::test]
async fn a_published_mirror_does_not_probe_for_neighbours() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
store
.epoch_acquire(&format!("{PREFIX}/git/epoch"), "forge-1", None)
.await
.unwrap();
let seen = flint_store::layout::neighbours(
store.as_ref(),
&sc.cfg.prefix,
flint_store::layout::Writer::LeanWorkspace,
)
.await
.unwrap();
assert_eq!(seen.len(), 1, "the condition is there to be found");
assert!(!sc.cfg.sole_writer, "an ordinary workspace probes");
sc.cfg.sole_writer = true;
assert!(sc.cfg.sole_writer, "a published mirror does not");
}
#[tokio::test]
async fn ui_edit_vs_agent_edit_never_a_silent_winner() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "notes.md", "v1");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, "notes.md", "user version", "dilip").await.unwrap();
write(dir.path(), "notes.md", "agent version");
backdate_baseline(&sc, "notes.md");
sc.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "notes.md").unwrap(), "agent version");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let cited = &m.manifest.entries["notes.md"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"agent version");
let conflicts = sc.state.load_conflicts().unwrap();
let c = conflicts.iter().find(|c| c.kind == "consume-dirty").expect("conflict surfaced");
assert_eq!(c.path, "notes.md");
let preserved = c.preserved_key.as_ref().expect("foreign bytes preserved");
let (_, body) = store.get_whole(preserved, None).await.unwrap();
assert_eq!(&body[..], b"user version");
}
#[tokio::test]
async fn container_restart_never_resurrects_unpublished_delete() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "keep.txt", "keep");
write(dir.path(), "gone.txt", "delete me");
sc.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
drop(sc);
let mut sc = syncer(&store, dir.path()).await; let cr = sc.checkout().await.unwrap();
assert!(cr.resumed_live_tree, "marker present ⇒ live-tree row");
assert_eq!(cr.materialized, 0, "must not re-materialize");
assert!(read(dir.path(), "gone.txt").is_none(), "delete resurrected!");
assert!(claim_until_held(&mut sc, 1).await, "self-recognition must not wait");
sc.run_barrier().await.unwrap();
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("gone.txt"));
}
#[tokio::test]
async fn takeover_rotation_fences_the_straggler() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "from A");
a.run_barrier().await.unwrap();
let seq_before = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(
!claim_until_held(&mut b, 3).await,
"a fresh replacement must NOT claim instantly over a live-looking lease"
);
assert!(claim_until_held(&mut b, 10).await, "quiet polls exhausted ⇒ takeover");
let rotated = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap();
assert_eq!(rotated.manifest.seq, seq_before + 1, "rotation: seq++, content-identical");
assert_eq!(rotated.manifest.entries.len(), 1);
b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "f.txt").unwrap(), "from A");
write(dir_a.path(), "f.txt", "stale straggler bytes");
backdate_baseline(&a, "f.txt");
let err = a.run_barrier().await.unwrap_err();
assert!(matches!(err, LeanError::Fenced(_)), "straggler must fence, got: {err:?}");
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.seq, rotated.manifest.seq, "straggler CAS landed!");
let cited = &m.manifest.entries["f.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(&body[..], b"from A", "straggler bytes reached a cited object");
}
#[tokio::test]
async fn adopt_own_412_converges_without_conflict() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "big.bin", "payload");
let crashed_uuid = "crashed-barrier-uuid".to_string();
let mut intent = sc.state.load_intent().unwrap();
intent.flush_uuid = crashed_uuid.clone();
intent.keys = vec![sc.cfg.file_key("big.bin")];
sc.state.save_intent(&intent).unwrap();
sc.state.clear_intent_keys().unwrap(); let body = Bytes::from("payload");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 1,
epoch: 1,
flush_uuid: crashed_uuid,
boundary_source: None,
posix: None,
};
store
.put_whole(&sc.cfg.file_key("big.bin"), body, &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["big.bin".to_string()]);
assert!(r.parked.is_empty());
assert!(sc.state.load_conflicts().unwrap().is_empty(), "AdoptOwn must not conflict");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("big.bin"));
}
#[tokio::test]
async fn foreign_412_parks_never_overwrites() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
let body = Bytes::from("foreign bytes");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 9,
epoch: 0,
flush_uuid: "someone-else".into(),
boundary_source: None,
posix: None,
};
let foreign =
store.put_whole(&sc.cfg.file_key("f.txt"), body, &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
write(dir.path(), "f.txt", "agent bytes");
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.parked, vec!["f.txt".to_string()]);
assert!(r.uploaded.is_empty());
let (meta, body) = store.get_whole(&sc.cfg.file_key("f.txt"), None).await.unwrap();
assert_eq!(meta.etag, foreign.etag);
assert_eq!(&body[..], b"foreign bytes");
let conflicts = sc.state.load_conflicts().unwrap();
assert!(conflicts.iter().any(|c| c.kind == "upload-412-parked" && c.path == "f.txt"));
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("f.txt"));
}
#[tokio::test]
async fn gc_refuses_unrecognized_etag() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "doc.txt", "v1");
sc.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("doc.txt")).unwrap();
sc.run_barrier().await.unwrap();
let body = Bytes::from("user re-created");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "gateway-late".into(),
boundary_source: None,
posix: None,
};
let cur = store.head(&sc.cfg.file_key("doc.txt")).await.unwrap();
store
.put_whole(
&sc.cfg.file_key("doc.txt"),
body,
&PutCondition::IfMatch(cur.etag),
&stamps,
crc,
)
.await
.unwrap();
let r = sc.run_barrier().await.unwrap();
assert!(r.deleted.is_empty(), "GC deleted a foreign re-create!");
let (_, body) = store.get_whole(&sc.cfg.file_key("doc.txt"), None).await.unwrap();
assert_eq!(&body[..], b"user re-created");
let conflicts = sc.state.load_conflicts().unwrap();
assert!(conflicts.iter().any(|c| c.kind == "gc-skip" && c.path == "doc.txt"));
}
#[tokio::test]
async fn local_delete_loses_to_foreign_modify() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
sc.run_barrier().await.unwrap();
let body = Bytes::from("their v2");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 2,
epoch: 0,
flush_uuid: "other-writer".into(),
boundary_source: None,
posix: None,
};
let cur = store.head(&sc.cfg.file_key("shared.txt")).await.unwrap();
let newmeta = store
.put_whole(&sc.cfg.file_key("shared.txt"), body, &PutCondition::IfMatch(cur.etag), &stamps, crc)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.entries.get_mut("shared.txt").unwrap().etag = newmeta.etag.clone();
manifest::cas_write(store.as_ref(), &sc.cfg, &theirs, Some(&loaded.handle()), 0, "other-writer")
.await
.unwrap();
std::fs::remove_file(dir.path().join("shared.txt")).unwrap();
let r1 = sc.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty());
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries["shared.txt"].etag, newmeta.etag, "foreign entry dropped!");
let (_, body) = store.get_whole(&sc.cfg.file_key("shared.txt"), None).await.unwrap();
assert_eq!(&body[..], b"their v2");
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.entries.iter().any(|e| e.path == "shared.txt" && e.etag == newmeta.etag));
let r2 = sc.run_barrier().await.unwrap();
assert_eq!(r2.deleted, vec!["shared.txt".to_string()]);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("shared.txt"));
let conflicts = sc.state.load_conflicts().unwrap();
let c = conflicts
.iter()
.find(|c| c.kind == "consume-dirty" && c.path == "shared.txt")
.expect("delete-vs-edit conflict must surface");
let preserved = c.preserved_key.as_ref().expect("foreign bytes preserved");
let (_, body) = store.get_whole(preserved, None).await.unwrap();
assert_eq!(&body[..], b"their v2", "the edit must stay recoverable after the delete wins");
}
#[tokio::test]
async fn window_refuses_hitl_and_expiry_unwedges() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let sc = syncer(&store, dir.path()).await;
let entry = |p: &str| InboxEntry {
path: p.into(),
etag: "e".into(),
author: "dilip".into(),
added_unix: now_unix(),
crc64_b64: None,
};
inbox::open_window(store.as_ref(), &sc.cfg, 1, now_unix() + 300).await.unwrap();
let err = inbox::gateway_append(store.as_ref(), &sc.cfg, entry("a.txt")).await.unwrap_err();
assert!(matches!(err, LeanError::State(_)), "live window must refuse");
inbox::clear_window(store.as_ref(), &sc.cfg, 1, &[]).await.unwrap();
inbox::gateway_append(store.as_ref(), &sc.cfg, entry("a.txt")).await.unwrap();
inbox::open_window(store.as_ref(), &sc.cfg, 1, now_unix() - 10).await.unwrap();
inbox::gateway_append(store.as_ref(), &sc.cfg, entry("b.txt")).await.unwrap();
}
#[tokio::test]
async fn large_file_publishes_via_compose_and_roundtrips() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.whole_put_max = 8; assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
let big_v1: String = (0..200).map(|i| format!("line {i}\n")).collect();
write(dir.path(), "model.bin", &big_v1);
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["model.bin".to_string()]);
assert!(r.deferred.is_empty());
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "model.bin").unwrap(), big_v1);
let big_v2 = format!("{big_v1}and more\n");
write(dir.path(), "model.bin", &big_v2);
backdate_baseline(&sc, "model.bin");
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.uploaded, vec!["model.bin".to_string()]);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let cited = &m.manifest.entries["model.bin"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(String::from_utf8(body.to_vec()).unwrap(), big_v2);
}
#[tokio::test]
async fn second_syncer_on_one_tree_refuses() {
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let _held = SyncerState::open(cfg.state_dir()).unwrap();
let Err(err) = SyncerState::open(cfg.state_dir()) else {
panic!("second open over a held workspace must refuse");
};
assert!(matches!(err, LeanError::State(_)));
}
#[tokio::test]
async fn checkout_budget_refuses_before_first_byte() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "big.txt", "0123456789012345678901234567890123456789");
sc.run_barrier().await.unwrap();
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer(&store, dir2.path()).await;
sc2.cfg.max_bytes = 10;
let err = sc2.checkout().await.unwrap_err();
assert!(matches!(err, LeanError::Budget(_)));
assert!(!sc2.state.marker_present(), "budget refusal must not gate-open the agent");
assert!(read(dir2.path(), "big.txt").is_none());
}
#[tokio::test]
async fn an_all_rejected_sync_scope_is_refused_not_widened_to_the_whole_tree() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
let err = sc
.sync_scoped(Some(vec!["../escape".into(), "./here".into()]))
.await
.expect_err("an all-rejected scope must not silently become the whole tree");
let msg = format!("{err}");
assert!(
msg.contains("NONE survived validation"),
"the refusal must say WHY, so a typo is not read as a sync bug: {msg}"
);
sc.sync_scoped(Some(vec!["inputs".into()])).await.expect("a valid scope must still sync");
}
#[tokio::test]
async fn sync_scan_first_dirty_wins_clean_applies() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "shared.txt", "v1");
write(dir_a.path(), "mine.txt", "agent latest — NEVER destroy");
a.run_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "shared.txt", "user v2", "dilip").await.unwrap();
write(dir_a.path(), "mine.txt", "agent latest v2");
backdate_baseline(&a, "mine.txt");
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.entries.remove("mine.txt");
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "remote-delete")
.await
.unwrap();
let r = a.sync().await.unwrap();
assert_eq!(read(dir_a.path(), "shared.txt").unwrap(), "user v2");
assert!(r.applied.contains(&"shared.txt".to_string()));
assert_eq!(read(dir_a.path(), "mine.txt").unwrap(), "agent latest v2");
assert!(r.conflicts.contains(&"mine.txt".to_string()));
assert!(r.deleted.is_empty());
}
use super::control;
use super::sentinel::{Due, Verb};
fn touch_sentinel(root: &std::path::Path, name: &str, body: &str) {
let dir = root.join(super::CONTROL_DIR);
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join(name), body).unwrap();
}
fn control_exists(root: &std::path::Path, name: &str) -> bool {
root.join(super::CONTROL_DIR).join(name).exists()
}
#[tokio::test]
async fn flint_dir_never_scanned() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "real.txt", "data");
touch_sentinel(dir.path(), control::PUBLISH, "");
touch_sentinel(dir.path(), control::REMOTE_SEQ, "{}");
let scanned = super::scan::scan(dir.path()).unwrap();
assert!(scanned.contains_key("real.txt"));
assert!(
!scanned.keys().any(|k| k.starts_with(".flint")),
"the scan yielded a control path: {:?}",
scanned.keys().collect::<Vec<_>>()
);
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("real.txt"));
assert!(
!m.entries.keys().any(|k| k.starts_with(".flint")),
"a control path was CITED: {:?}",
m.entries.keys().collect::<Vec<_>>()
);
assert!(store.head(&a.cfg.file_key(".flint/publish")).await.is_err());
}
#[tokio::test]
async fn legacy_flint_citation_survives_upgrade() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "v1");
a.run_barrier().await.unwrap();
let key = a.cfg.file_key(".flint/legacy.txt");
let body = Bytes::from_static(b"legacy payload");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: "legacy".into(),
boundary_source: None,
posix: None,
};
let meta = store
.put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut m = loaded.manifest.clone();
m.seq += 1;
m.entries.insert(
".flint/legacy.txt".into(),
manifest::LeanEntry {
key: key.clone(),
etag: meta.etag.clone(),
crc64_b64: meta.crc64_b64.clone().unwrap(),
size: meta.size,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
version_id: meta.version_id.clone(),
},
);
let installed =
manifest::cas_write(store.as_ref(), &a.cfg, &m, Some(&loaded.handle()), 0, "legacy")
.await
.unwrap();
let mut b = a.state.load_baseline().unwrap();
b.entries.insert(
".flint/legacy.txt".into(),
super::state::BaselineEntry {
etag: meta.etag.clone(),
generation: 1,
size: meta.size,
mtime_unix: 0,
version_id: None,
crc64_b64: meta.crc64_b64.clone(),
},
);
b.inst_base.insert(".flint/legacy.txt".into(), meta.etag.clone());
b.prev_scan.insert(".flint/legacy.txt".into());
b.seq = m.seq;
b.manifest_etag = Some(installed.etag.clone());
a.state.save_baseline(&b).unwrap();
assert!(store.head(&key).await.is_ok());
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
let after = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
after.entries.contains_key(".flint/legacy.txt"),
"an upgrade DELETED a legacy citation"
);
assert!(store.head(&key).await.is_ok(), "an upgrade GC'd the legacy object");
}
#[tokio::test]
async fn checkout_never_materializes_control_citation() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "real.txt", "v1");
a.run_barrier().await.unwrap();
let key = a.cfg.file_key(".flint/legacy.txt");
let body = Bytes::from_static(b"legacy");
let crc = crc64_nvme(&body);
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfNoneMatchAny,
&GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: "legacy".into(),
boundary_source: None,
posix: None,
},
crc,
)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut m = loaded.manifest.clone();
m.seq += 1;
m.entries.insert(
".flint/legacy.txt".into(),
manifest::LeanEntry {
key,
etag: meta.etag.clone(),
crc64_b64: meta.crc64_b64.clone().unwrap(),
size: meta.size,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
version_id: meta.version_id.clone(),
},
);
manifest::cas_write(store.as_ref(), &a.cfg, &m, Some(&loaded.handle()), 0, "legacy")
.await
.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let r = b.checkout().await.unwrap();
assert_eq!(read(dir_b.path(), "real.txt").unwrap(), "v1");
assert!(
!dir_b.path().join(".flint/legacy.txt").exists(),
"checkout materialized into the reserved control namespace"
);
assert_eq!(r.refused, 1);
assert!(b
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.path == ".flint/legacy.txt" && c.kind.starts_with("checkout-refused")));
}
#[tokio::test]
async fn capabilities_written_on_live_tree_restart() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join(super::CONTROL_DIR).join(control::CAPABILITIES))
.unwrap();
assert!(a.state.marker_present());
assert!(!control_exists(dir.path(), control::CAPABILITIES));
let r = a.checkout().await.unwrap();
assert!(r.resumed_live_tree);
assert!(!control_exists(dir.path(), control::CAPABILITIES));
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
let caps = a.read_capabilities().unwrap();
assert_eq!(caps.protocol, super::SENTINEL_PROTOCOL);
assert_eq!(caps.state, "live");
assert!(caps.verbs.iter().any(|v| v == "publish"));
assert!(caps.verbs.iter().any(|v| v == "sync"));
assert_eq!(caps.boundary_mode, "hybrid");
}
#[tokio::test]
async fn preexisting_flint_disables_sentinels() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
touch_sentinel(dir.path(), control::PUBLISH, "app-owned payload");
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
assert!(!posture.enabled);
assert_eq!(posture.reason.as_deref(), Some("preexisting-flint-paths"));
a.write_capabilities(&posture, false).unwrap();
let caps = a.read_capabilities().unwrap();
assert!(caps.verbs.is_empty());
assert_eq!(caps.reason.as_deref(), Some("preexisting-flint-paths"));
a.checkout().await.unwrap();
let acks = a.sentinel_tick().await.unwrap();
assert!(acks.is_empty());
assert_eq!(
std::fs::read_to_string(dir.path().join(".flint/publish")).unwrap(),
"app-owned payload"
);
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
let again = a.sentinel_preflight().unwrap();
assert!(!again.enabled);
}
fn clear_min_interval(sc: &Syncer) {
let mut b = sc.load_budget().unwrap();
b.last_honor_unix = 0;
let bytes = serde_json::to_vec(&b).unwrap();
std::fs::write(sc.cfg.state_dir().join("sentinel-budget.json"), bytes).unwrap();
}
#[tokio::test]
async fn publish_sentinel_honored_and_acked() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
a.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &a.cfg).await.unwrap().map(|l| l.manifest.seq);
write(dir.path(), "model.json", "{}");
write(dir.path(), "model.json.index", "idx");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-1","note":"step 1"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the sentinel was not honored");
let ack = &acks[0];
assert_eq!(ack.status, "ok");
assert_eq!(ack.boundary, "sentinel");
assert_eq!(ack.nonces, vec!["n-1".to_string()]);
assert!(ack.sentinel_mtime_unix_ns > 0);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("model.json"));
assert!(m.entries.contains_key("model.json.index"));
assert_eq!(ack.seq, Some(m.seq));
assert_ne!(before, Some(m.seq), "the manifest did not advance");
assert!(!control_exists(dir.path(), control::PUBLISH));
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
assert!(control_exists(dir.path(), control::PUBLISH_ACK));
assert_eq!(a.read_ack(Verb::Publish).unwrap().nonces, vec!["n-1".to_string()]);
}
#[tokio::test]
async fn sentinel_ack_echoes_covered_nonces() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
for i in 0..5 {
touch_sentinel(dir.path(), control::PUBLISH, &format!(r#"{{"nonce":"n-{i}"}}"#));
a.poll_sentinels().unwrap();
}
let pending = a.load_pending(Verb::Publish).unwrap().unwrap();
assert_eq!(pending.nonces.len(), 5, "touches did not coalesce into one record");
clear_min_interval(&a);
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
for i in 0..5 {
assert!(
ack.nonces.contains(&format!("n-{i}")),
"nonce n-{i} was orphaned by coalescing: {:?}",
ack.nonces
);
}
assert!(ack.nonces.contains(&"n-2".to_string()));
}
#[tokio::test]
async fn min_interval_coalesces_into_one_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 3600; assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"first"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the first honor must be prompt");
let seq_after_first = acks[0].seq.unwrap();
write(dir.path(), "g.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"second"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert!(acks.is_empty(), "the min-interval did not hold the second honor");
assert_eq!(a.sentinel_due().unwrap(), Due::MinInterval);
assert_eq!(
a.load_pending(Verb::Publish).unwrap().unwrap().nonces,
vec!["second".to_string()]
);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.seq, seq_after_first, "a held honor still advanced the manifest");
let out = a.floor_tick().await.unwrap();
assert_eq!(out.acks.len(), 1);
assert_eq!(out.acks[0].nonces, vec!["second".to_string()]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("g.txt"));
}
#[tokio::test]
async fn budget_meters_bytes_not_calls() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.whole_put_max = 1024; a.cfg.sentinel_hourly_budget = 8; a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
let mut honors_large = 0;
let mut deferred_acks = 0;
for i in 0..6 {
write(dir.path(), "big.bin", &"x".repeat(4096 + i));
backdate_baseline(&a, "big.bin");
touch_sentinel(dir.path(), control::PUBLISH, &format!(r#"{{"nonce":"L{i}"}}"#));
let acks = a.sentinel_tick().await.unwrap();
honors_large += acks.len();
if acks.is_empty() && a.load_pending(Verb::Publish).unwrap().is_some() {
let out = a.floor_tick().await.unwrap();
deferred_acks += out
.acks
.iter()
.filter(|x| x.boundary == "sentinel-deferred")
.count();
}
}
assert!(
honors_large <= 3,
"a 4 KiB-over-ceiling storm was NOT throttled: {honors_large} prompt honors"
);
assert!(deferred_acks >= 1, "no ack was stamped sentinel-deferred");
assert_eq!(a.sentinel_due().unwrap(), Due::BudgetDeferred);
let store2 = Arc::new(MemoryStore::new());
let dir2 = tempfile::tempdir().unwrap();
let mut b = syncer(&store2, dir2.path()).await;
b.cfg.whole_put_max = 1024;
b.cfg.sentinel_hourly_budget = 8;
b.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut b, 3).await);
let posture = b.sentinel_preflight().unwrap();
b.write_capabilities(&posture, false).unwrap();
b.checkout().await.unwrap();
let mut honors_small = 0;
for i in 0..6 {
write(dir2.path(), "small.txt", &format!("v{i}"));
backdate_baseline(&b, "small.txt");
touch_sentinel(dir2.path(), control::PUBLISH, &format!(r#"{{"nonce":"S{i}"}}"#));
honors_small += b.sentinel_tick().await.unwrap().len();
}
assert_eq!(
honors_small, 6,
"the small-file storm was throttled — the meter is counting CALLS, not work"
);
assert!(b
.read_ack(Verb::Publish)
.map(|a| a.boundary == "sentinel")
.unwrap_or(false));
assert!(honors_small > honors_large);
}
#[tokio::test]
async fn no_diff_sentinel_honor_costs_no_budget() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_hourly_budget = 2;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
for i in 0..10 {
touch_sentinel(dir.path(), control::PUBLISH, &format!(r#"{{"nonce":"q{i}"}}"#));
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "a no-diff honor was throttled at touch {i}");
assert!(acks[0].report.no_change, "the tree was not actually quiet");
}
assert_eq!(a.load_budget().unwrap().spent(super::now_unix()), 0);
}
#[tokio::test]
async fn crash_between_consume_and_ack_reruns_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
a.run_barrier().await.unwrap();
let seq_before = a.state.load_baseline().unwrap().seq;
write(dir.path(), "late.txt", "written before the boundary");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"crashed"}"#);
a.poll_sentinels().unwrap();
assert!(a.load_pending(Verb::Publish).unwrap().is_some());
assert!(a.read_ack(Verb::Publish).is_none());
a.settle_pending_at_startup().await.unwrap();
let ack = a.read_ack(Verb::Publish).unwrap();
assert_eq!(ack.status, "ok");
assert!(ack.nonces.contains(&"crashed".to_string()));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(ack.seq, Some(m.seq));
assert!(m.seq > seq_before);
assert!(m.entries.contains_key("late.txt"));
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
}
#[tokio::test]
async fn restart_settles_pending_before_new_consume() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
a.cfg.sentinel_min_interval_secs = 0;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "a.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"old"}"#);
a.poll_sentinels().unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"new"}"#);
a.poll_sentinels().unwrap();
let pending = a.load_pending(Verb::Publish).unwrap().unwrap();
assert!(pending.nonces.contains(&"old".to_string()));
assert!(pending.nonces.contains(&"new".to_string()));
let ack = a.honor_pending(Verb::Publish, false).await.unwrap().unwrap();
assert!(ack.nonces.contains(&"old".to_string()));
assert!(ack.nonces.contains(&"new".to_string()));
}
#[tokio::test]
async fn torn_pending_body_honored_as_bare_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"half-writ"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "a torn body wedged the verb");
assert_eq!(acks[0].status, "ok");
assert!(acks[0].nonces.is_empty());
assert!(acks[0].sentinel_mtime_unix_ns > 0, "the bare-touch mtime must still be covered");
assert!(a
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind == "sentinel-torn-body"));
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("f.txt"));
}
#[tokio::test]
#[cfg(unix)]
async fn fifo_sentinel_skipped() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
let ctl = dir.path().join(super::CONTROL_DIR);
std::fs::create_dir_all(&ctl).unwrap();
let fifo = ctl.join(control::PUBLISH);
let cpath = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap();
let rc = unsafe { libc::mkfifo(cpath.as_ptr(), 0o644) };
assert_eq!(rc, 0, "could not create the FIFO fixture");
assert!(!std::fs::symlink_metadata(&fifo).unwrap().is_file());
let acks = a.sentinel_tick().await.unwrap();
assert!(acks.is_empty());
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
assert!(fifo.exists(), "the FIFO was consumed");
assert!(a
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind == "sentinel-not-regular-file"));
}
#[tokio::test]
async fn fenced_honor_writes_refused_ack() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
write(dir_a.path(), "g.txt", "v1");
touch_sentinel(dir_a.path(), control::PUBLISH, r#"{"nonce":"stranded"}"#);
a.poll_sentinels().unwrap();
assert!(a.load_pending(Verb::Publish).unwrap().is_some());
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 12).await);
let our_epoch = a.lease.as_ref().unwrap().epoch;
let their_epoch = b.lease.as_ref().unwrap().epoch;
assert!(their_epoch > our_epoch, "no takeover happened");
let err = a.sentinel_tick().await.unwrap_err();
assert!(matches!(err, LeanError::Fenced(_)));
let ack = a.read_ack(Verb::Publish).unwrap();
assert_eq!(ack.status, "refused-fenced");
assert!(ack.nonces.contains(&"stranded".to_string()));
assert_eq!(ack.observed_epoch, Some(their_epoch));
assert!(ack.seq.is_none());
assert!(a.load_pending(Verb::Publish).unwrap().is_none());
let caps = a.read_capabilities().unwrap();
assert_eq!(caps.state, "fenced");
assert!(caps.verbs.is_empty());
}
#[tokio::test]
async fn fenced_sync_honor_refused_and_tree_unmutated() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
write(dir_a.path(), "shared.txt", "zombie view");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 12).await);
b.checkout().await.unwrap();
write(dir_b.path(), "shared.txt", "successor view");
backdate_baseline(&b, "shared.txt");
b.run_barrier().await.unwrap();
let before = std::fs::read_to_string(dir_a.path().join("shared.txt")).unwrap();
touch_sentinel(dir_a.path(), control::SYNC, r#"{"nonce":"zombie-sync"}"#);
let err = a.sentinel_tick().await.unwrap_err();
assert!(matches!(err, LeanError::Fenced(_)));
let ack = a.read_ack(Verb::Sync).unwrap();
assert_eq!(ack.status, "refused-fenced");
assert_eq!(
std::fs::read_to_string(dir_a.path().join("shared.txt")).unwrap(),
before,
"a fenced sync honor MUTATED the zombie's tree"
);
assert_ne!(before, "successor view", "the fixture never diverged");
}
#[tokio::test]
async fn deposed_syncer_fails_the_explicit_epoch_check() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
a.verify_not_deposed_pub().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 12).await);
assert!(b.lease.as_ref().unwrap().epoch > a.lease.as_ref().unwrap().epoch);
assert!(matches!(
a.verify_not_deposed_pub().await.unwrap_err(),
LeanError::Fenced(_)
));
}
#[tokio::test]
async fn scoped_sync_preserves_out_of_scope_foreign_flow() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "inputs/data.txt", "v1");
write(dir_a.path(), "outputs/result.txt", "v1");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
for (path, content) in
[("inputs/data.txt", "foreign inputs v2"), ("outputs/result.txt", "foreign outputs v2")]
{
let key = a.cfg.file_key(path);
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let cur = store.head(&key).await.unwrap();
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfMatch(cur.etag),
&GenerationStamps {
generation: 2,
epoch: 0,
flush_uuid: "sibling".into(),
boundary_source: None,
posix: None,
},
crc,
)
.await
.unwrap();
let e = theirs.entries.get_mut(path).unwrap();
e.etag = meta.etag.clone();
e.crc64_b64 = meta.crc64_b64.clone().unwrap();
e.size = meta.size;
e.generation = 2;
}
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "sibling")
.await
.unwrap();
let foreign_out_etag = theirs.entries["outputs/result.txt"].etag.clone();
assert_ne!(
foreign_out_etag,
a.state.load_baseline().unwrap().inst_base["outputs/result.txt"]
);
let r = a.sync_scoped(Some(vec!["inputs/".into()])).await.unwrap();
assert_eq!(read(dir_a.path(), "inputs/data.txt").unwrap(), "foreign inputs v2");
assert!(r.applied.contains(&"inputs/data.txt".to_string()));
assert_eq!(read(dir_a.path(), "outputs/result.txt").unwrap(), "v1");
assert!(r.out_of_scope_foreign >= 1);
assert_ne!(
a.state.load_baseline().unwrap().inst_base["outputs/result.txt"],
foreign_out_etag,
"a scoped sync advanced the MERGE BASE for an out-of-scope path"
);
assert_eq!(r.seq, a.state.load_baseline().unwrap().seq);
a.run_barrier().await.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(
read(dir_a.path(), "outputs/result.txt").unwrap(),
"foreign outputs v2",
"the out-of-scope foreign change was LOST from the inbox flow"
);
}
#[test]
fn scope_matches_on_component_boundary() {
let s = super::sync::Scope::new(&["in".to_string(), "inputs/".to_string()]);
assert!(s.covers("in"));
assert!(s.covers("in/x.txt"));
assert!(s.covers("inputs/a.txt"));
assert!(!s.covers("internal/secret.txt"));
assert!(!s.covers("inputsX/a.txt"));
let s = super::sync::Scope::new(&["../etc".to_string()]);
assert!(s.is_empty());
}
#[tokio::test]
#[cfg(unix)]
async fn write_file_atomic_refuses_symlink_escape() {
let outside = tempfile::tempdir().unwrap();
let secret = outside.path().join("secret.txt");
std::fs::write(&secret, "ORIGINAL CREDENTIALS").unwrap();
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
std::os::unix::fs::symlink(outside.path(), dir.path().join("inputs")).unwrap();
assert!(dir.path().join("inputs/secret.txt").exists());
let scanned = super::scan::scan(dir.path()).unwrap();
assert!(!scanned.keys().any(|k| k.starts_with("inputs")));
assert!(super::barrier::contained_path(dir.path(), "inputs/secret.txt").is_err());
assert!(super::barrier::contained_path(dir.path(), "../escape.txt").is_err());
assert!(super::barrier::contained_path(dir.path(), "/etc/passwd").is_err());
assert!(super::barrier::contained_path(dir.path(), ".flint/publish").is_err());
assert!(super::barrier::contained_path(dir.path(), "ok/nested/f.txt").is_ok());
hitl_write(&store, &a.cfg, "inputs/secret.txt", "ATTACKER PAYLOAD", "attacker")
.await
.unwrap();
a.run_barrier().await.unwrap();
assert_eq!(
std::fs::read_to_string(&secret).unwrap(),
"ORIGINAL CREDENTIALS",
"the syncer wrote THROUGH a planted symlink, outside the workspace"
);
assert!(a
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind.starts_with("consume-refused-containment")));
}
#[tokio::test]
async fn sync_rehonor_no_phantom_conflicts() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
hitl_write(&store, &a.cfg, "shared.txt", "foreign v2", "ci").await.unwrap();
let r = a.sync().await.unwrap();
assert!(r.applied.contains(&"shared.txt".to_string()));
let mut b = a.state.load_baseline().unwrap();
let stale = b.entries.get_mut("shared.txt").unwrap();
stale.etag = "\"stale-pre-sync-etag\"".into();
stale.mtime_unix -= 10;
stale.size = 1;
a.state.save_baseline(&b).unwrap();
let scanned = super::scan::scan(dir.path()).unwrap();
let c = super::scan::classify(&scanned, &a.state.load_baseline().unwrap());
assert!(c.uploads.contains("shared.txt"), "the fixture is not actually dirty");
let r = a.sync().await.unwrap();
assert!(
!r.conflicts.contains(&"shared.txt".to_string()),
"a phantom conflict was reported for byte-identical content"
);
assert!(r.applied.contains(&"shared.txt".to_string()));
}
#[tokio::test]
async fn remote_seq_ticks_without_added_requests() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "v1");
a.floor_tick().await.unwrap();
let t0 = a.load_remote_seq();
assert!(t0.updated_unix > 0);
assert_eq!(t0.observed_seq, t0.integrated_seq, "no news, yet the ticker claims some");
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "foreign")
.await
.unwrap();
assert_eq!(t0.observed_seq, loaded.manifest.seq);
a.floor_tick().await.unwrap();
let t1 = a.load_remote_seq();
assert!(
t1.observed_seq >= theirs.seq,
"the ticker missed a foreign install: {} vs {}",
t1.observed_seq,
theirs.seq
);
assert!(t1.updated_unix >= t0.updated_unix);
}
#[tokio::test]
async fn sync_request_is_carried_never_executed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "agent bytes");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.entries.remove("keep.txt");
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "remote-delete")
.await
.unwrap();
let tree_before = std::fs::read_to_string(dir.path().join("keep.txt")).unwrap();
a.carry_sync_request(super::now_unix(), "ci@example").unwrap();
for _ in 0..3 {
let _ = a.sentinel_tick().await.unwrap();
}
let t = a.load_remote_seq();
assert_eq!(t.sync_requested_by.as_deref(), Some("ci@example"));
assert!(t.sync_requested_unix.is_some());
assert_eq!(
std::fs::read_to_string(dir.path().join("keep.txt")).unwrap(),
tree_before,
"the syncer acted on a remote's sync request"
);
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"mine"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1);
assert!(!dir.path().join("keep.txt").exists(), "the agent's own sync did not run");
}
#[tokio::test]
async fn a_restarted_claimant_that_can_never_honor_says_so_instead_of_stranding() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
touch_sentinel(dir_a.path(), control::PUBLISH, r#"{"nonce":"task-42"}"#);
assert!(a.consume_sentinel(Verb::Publish).unwrap(), "the touch was not consumed");
assert!(a.load_pending(Verb::Publish).unwrap().is_some(), "no pending record to owe");
assert!(a.read_ack(Verb::Publish).is_none(), "an ack already exists — fixture not armed");
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 10).await, "quiet polls exhausted ⇒ takeover");
let outcome = lease::claim_step(&mut a).await.unwrap();
assert!(
matches!(outcome, lease::ClaimOutcome::Waiting { .. }),
"the fixture never armed — A claimed over a live successor"
);
let answered = a.refuse_what_this_incarnation_can_never_honor().await.unwrap();
assert!(answered, "nothing was owed — the fixture never armed");
let ack = a.read_ack(Verb::Publish).expect("the agent is still stranded: no ack");
assert_eq!(ack.status, "refused-fenced");
assert!(ack.nonces.contains(&"task-42".to_string()), "the ack does not cover the agent's nonce");
assert_eq!(ack.observed_epoch, Some(b.lease.as_ref().unwrap().epoch), "the ack names no fencer");
let caps: serde_json::Value = serde_json::from_str(
&std::fs::read_to_string(dir_a.path().join(".flint").join("capabilities.json")).unwrap(),
)
.unwrap();
assert_eq!(caps["state"], "fenced", "capabilities.json still says the zombie is live");
}
#[tokio::test]
async fn a_healthy_replacement_waiting_out_quiet_polls_is_not_marked_fenced() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let posture = b.sentinel_preflight().unwrap();
b.write_capabilities(&posture, false).unwrap();
assert!(
matches!(lease::claim_step(&mut b).await.unwrap(), lease::ClaimOutcome::Waiting { .. }),
"the fixture never armed — B claimed instantly"
);
let answered = b.refuse_what_this_incarnation_can_never_honor().await.unwrap();
assert!(!answered, "a healthy replacement marked itself fenced on the way up");
assert!(b.read_ack(Verb::Publish).is_none(), "a refused ack appeared with nothing owed");
let caps: serde_json::Value = serde_json::from_str(
&std::fs::read_to_string(dir_b.path().join(".flint").join("capabilities.json")).unwrap(),
)
.unwrap();
assert_ne!(caps["state"], "fenced", "a healthy replacement was marked fenced");
assert!(claim_until_held(&mut b, 10).await, "the refusal path blocked a legitimate takeover");
}
#[tokio::test]
async fn a_stale_ack_never_retires_a_fresh_bare_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "w.txt", "one");
touch_sentinel(dir.path(), control::PUBLISH, "");
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the first bare touch was not honored");
let first = a.read_ack(Verb::Publish).expect("no ack from the first honor");
assert_eq!(first.status, "ok");
std::thread::sleep(std::time::Duration::from_millis(1100));
write(dir.path(), "w.txt", "two");
touch_sentinel(dir.path(), control::PUBLISH, "");
let path = dir.path().join(super::CONTROL_DIR).join(control::PUBLISH);
let backdated = std::time::SystemTime::UNIX_EPOCH
+ std::time::Duration::from_nanos((first.sentinel_mtime_unix_ns as u64).saturating_sub(5));
let f = std::fs::File::options().write(true).open(&path).unwrap();
f.set_times(std::fs::FileTimes::new().set_modified(backdated)).unwrap();
drop(f);
assert!(a.consume_sentinel(Verb::Publish).unwrap(), "the second touch was not consumed");
let pending = a.load_pending(Verb::Publish).unwrap().expect("no pending record");
assert!(pending.nonces.is_empty(), "the fixture needs a BARE touch");
assert!(
pending.consumed_mtime_unix_ns <= first.sentinel_mtime_unix_ns,
"the mtime did not go backwards — the fixture never armed"
);
assert!(
!a.ack_matches(Verb::Publish, &pending),
"a stale ack retired a fresh boundary — it will never run"
);
let ack = a
.honor_pending(Verb::Publish, true)
.await
.unwrap()
.expect("the second boundary never ran");
assert_eq!(ack.status, "ok");
let second = a.read_ack(Verb::Publish).expect("no ack from the second honor");
assert!(second.completed_unix > first.completed_unix, "the ack was not re-minted");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let e = &m.entries["w.txt"];
let (_, body) = store.get_whole(&e.key, Some(&e.etag)).await.unwrap();
assert_eq!(&body[..], b"two", "the second boundary was retired without publishing");
}
#[tokio::test]
async fn a_fifo_at_the_sentinel_path_never_wedges_the_poll_arm() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let ctl = dir.path().join(super::CONTROL_DIR);
std::fs::create_dir_all(&ctl).unwrap();
let path = ctl.join(control::PUBLISH);
let c = std::ffi::CString::new(path.to_str().unwrap()).unwrap();
assert_eq!(unsafe { libc::mkfifo(c.as_ptr(), 0o644) }, 0, "mkfifo failed");
assert!(
!std::fs::metadata(&path).unwrap().is_file(),
"the fixture never armed — that is not a FIFO"
);
let r = super::sentinel::read_bounded(&path);
assert!(r.is_err(), "a writer-less FIFO read as a body instead of erroring");
let target = ctl.join("elsewhere");
std::fs::write(&target, "not the agent's").unwrap();
let link = ctl.join("publish.link");
std::os::unix::fs::symlink(&target, &link).unwrap();
assert!(
super::sentinel::read_bounded(&link).is_err(),
"a symlink at the sentinel path was followed"
);
let ticked = tokio::time::timeout(std::time::Duration::from_secs(5), a.sentinel_tick()).await;
let acks = ticked.expect("THE POLL ARM WEDGED on a FIFO at the sentinel path").unwrap();
assert!(acks.is_empty(), "a FIFO was honored as a boundary");
let conflicts = a.state.load_conflicts().unwrap();
assert!(
conflicts.iter().any(|c| c.kind == "sentinel-not-regular-file"),
"the non-regular sentinel left no conflict record"
);
for _ in 0..5 {
a.sentinel_tick().await.unwrap();
}
let after = a.state.load_conflicts().unwrap();
assert_eq!(
after.iter().filter(|c| c.kind == "sentinel-not-regular-file").count(),
1,
"the standing condition was re-recorded on every poll tick"
);
std::fs::remove_file(&path).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, "");
a.sentinel_tick().await.unwrap();
let c = std::ffi::CString::new(path.to_str().unwrap()).unwrap();
assert_eq!(unsafe { libc::mkfifo(c.as_ptr(), 0o644) }, 0, "second mkfifo failed");
a.sentinel_tick().await.unwrap();
assert_eq!(
a.state.load_conflicts().unwrap().iter()
.filter(|c| c.kind == "sentinel-not-regular-file")
.count(),
2,
"a RECURRENCE of the condition went unrecorded — the latch never clears"
);
}
#[tokio::test]
async fn a_gated_no_diff_honor_still_names_the_boundary_it_is_satisfied_by() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "w.txt", "ONE");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-nodiff"}"#);
assert!(a.consume_sentinel(Verb::Publish).unwrap());
let ack = a
.honor_pending(Verb::Publish, true)
.await
.unwrap()
.expect("the no-diff honor produced no ack at all");
assert_eq!(ack.status, "ok");
assert_eq!(ack.report.uploaded, 0, "the fixture published something — not a no-diff honor");
let seq = ack.seq.expect("an ok ack with seq: null — §1.2's recipe has nothing to confirm");
assert_eq!(seq, installed, "the ack named a boundary that is not the standing one");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.seq, seq, "the ack's seq is not the manifest's");
}
use super::gated::CitationSource;
fn gated(sc: &mut Syncer) {
sc.cfg.boundary_mode = super::BoundaryMode::Gated;
sc.cfg.visibility_lag_bound_secs = Some(3600); sc.cfg.quiesce_bound_secs = 3600;
}
#[tokio::test]
async fn a_deposed_stragglers_reaper_never_deletes_the_successors_cited_version() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir_a.path(), "w.txt", "G1-FROM-A");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let a_installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let key = a.cfg.file_key("w.txt");
let v1 = a_installed.entries["w.txt"].version_id.clone().unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 10).await, "quiet polls exhausted ⇒ takeover");
b.checkout().await.unwrap();
gated(&mut b);
write(dir_b.path(), "w.txt", "G2-CITED-BY-B");
b.upload_lane().await.unwrap();
b.citation_pass(CitationSource::Sentinel).await.unwrap();
let b_installed = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
let v2 = b_installed.entries["w.txt"].version_id.clone().unwrap();
assert_ne!(v2, v1, "B cited the same version A did — the fixture never armed");
write(dir_b.path(), "w.txt", "G3-STAGED-BY-B");
backdate_baseline(&b, "w.txt");
b.upload_lane().await.unwrap();
let versions = store.list_versions(&key).await.unwrap();
let cited_by_b = versions
.iter()
.find(|v| v.version_id == v2)
.expect("B's cited version vanished before the reaper ran");
assert!(!cited_by_b.is_current, "B's cited version is still current — the kill zone is empty");
assert!(
versions.iter().any(|v| v.is_current && v.version_id != v2 && v.version_id != v1),
"nothing newer than B's citation is current — the fixture never armed"
);
let stage_a = a.load_stage().unwrap();
let mut upserts = std::collections::BTreeMap::new();
upserts.insert("w.txt".to_string(), a_installed.entries["w.txt"].clone());
let mut report = Default::default();
let _ = a.reclaim_superseded(&upserts, &a_installed, &stage_a, &mut report).await;
let (_, body) = store
.get_version(&key, &v2)
.await
.expect("the straggler's reaper DELETED the successor's cited version");
assert_eq!(&body[..], b"G2-CITED-BY-B", "the successor's cited bytes changed");
let cited = &b_installed.entries["w.txt"];
let (_, via_manifest) = store.get_version(&cited.key, cited.version_id.as_ref().unwrap()).await
.expect("the successor's manifest dangles after the straggler reaped");
assert_eq!(&via_manifest[..], b"G2-CITED-BY-B");
}
#[tokio::test]
async fn the_reaper_reclaims_only_the_version_its_own_record_names() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "n.txt", "V1");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let key = a.cfg.file_key("n.txt");
let v1 = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.entries
["n.txt"]
.version_id
.clone()
.unwrap();
write(dir.path(), "n.txt", "V2");
backdate_baseline(&a, "n.txt");
a.upload_lane().await.unwrap();
let r = a.citation_pass(CitationSource::Sentinel).await.unwrap();
assert_eq!(r.versions_reclaimed, 1, "the reaper stopped reclaiming anything at all");
assert!(
store.get_version(&key, &v1).await.is_err(),
"the version this workspace superseded was not reclaimed"
);
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let v2 = installed.entries["n.txt"].version_id.clone().unwrap();
let mut unnameable = vec![];
for (n, body) in [(98u64, "UNNAMEABLE-A"), (99, "UNNAMEABLE-B")] {
let b = Bytes::from(body);
let meta = store
.put_whole(
&key,
b.clone(),
&PutCondition::IfMatch(store.head(&key).await.unwrap().etag),
&GenerationStamps {
generation: n,
epoch: 0,
flush_uuid: "crash-remnant".into(),
boundary_source: None,
posix: None,
},
crc64_nvme(&b),
)
.await
.unwrap();
unnameable.push(meta.version_id.clone().expect("versioned store"));
}
let (orphan_noncurrent, orphan_vid) = (unnameable[0].clone(), unnameable[1].clone());
let vs = store.list_versions(&key).await.unwrap();
assert!(
vs.iter().any(|v| v.version_id == orphan_noncurrent && !v.is_current),
"the unnameable remnant is current — the old rule would have skipped it anyway"
);
let mut stage = a.load_stage().unwrap();
let mut pe = super::gated::PendingEntry {
key: key.clone(),
etag: installed.entries["n.txt"].etag.clone(),
crc64_b64: installed.entries["n.txt"].crc64_b64.clone(),
size: 2,
mode: 0o644,
mtime_unix: 0,
generation: 3,
epoch: 1,
version_id: None,
base_version_id: Some(v2.clone()),
staged_unix: 0,
};
pe.version_id = Some(orphan_vid.clone());
stage.entries.insert("n.txt".to_string(), pe);
let mut keep_manifest = installed.clone();
keep_manifest.entries.get_mut("n.txt").unwrap().version_id = Some(orphan_vid.clone());
let mut upserts = std::collections::BTreeMap::new();
upserts.insert("n.txt".to_string(), keep_manifest.entries["n.txt"].clone());
let mut report = Default::default();
a.reclaim_superseded(&upserts, &keep_manifest, &stage, &mut report).await.unwrap();
assert_eq!(report.versions_reclaimed, 1, "the named superseded version was not reclaimed");
assert!(store.get_version(&key, &v2).await.is_err(), "the named version survived");
store
.get_version(&key, &orphan_noncurrent)
.await
.expect("the reaper deleted a NONCURRENT version its own record never named");
store
.get_version(&key, &orphan_vid)
.await
.expect("the reaper deleted a version its own record never named");
}
#[tokio::test]
async fn staging_put_never_destroys_the_cited_version() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "model.json", "BOUNDARY 1");
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited = m.entries["model.json"].clone();
let cited_vid = cited.version_id.clone().expect("a versioned store cites a version");
gated(&mut a);
write(dir.path(), "model.json", "MID-LOGICAL-CHANGE");
backdate_baseline(&a, "model.json");
let lane = a.upload_lane().await.unwrap();
assert_eq!(lane.staged, vec!["model.json".to_string()]);
let key = a.cfg.file_key("model.json");
let (_, current) = store.get_whole(&key, None).await.unwrap();
assert_eq!(¤t[..], b"MID-LOGICAL-CHANGE");
let m2 = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m2.seq, m.seq, "the upload lane advanced the manifest");
let (_, still) = store.get_version(&key, &cited_vid).await.unwrap();
assert_eq!(&still[..], b"BOUNDARY 1", "an in-place staging PUT destroyed cited data");
}
#[tokio::test]
async fn pinned_reads_never_adopts_current() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir_a.path(), "model.json", "A1");
write(dir_a.path(), "model.json.index", "B1");
a.upload_lane().await.unwrap();
let cite = a.citation_pass(CitationSource::Sentinel).await.unwrap();
assert_eq!(cite.cited, 2);
let cited = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(cited.pinned_reads, "a gated citation did not stamp pinned_reads");
assert_eq!(cited.boundary_source.as_deref(), Some("sentinel"));
write(dir_a.path(), "model.json", "A2-MID-CHANGE");
backdate_baseline(&a, "model.json");
a.upload_lane().await.unwrap();
let key = a.cfg.file_key("model.json");
let (_, raw) = store.get_whole(&key, None).await.unwrap();
assert_eq!(&raw[..], b"A2-MID-CHANGE", "nothing was actually staged");
let now = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(now.seq, cited.seq);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let r = b.checkout().await.unwrap();
assert_eq!(r.materialized, 2);
assert_eq!(
read(dir_b.path(), "model.json").unwrap(),
"A1",
"a gated checkout S3-WINS-ADOPTED its own staged bytes"
);
assert_eq!(read(dir_b.path(), "model.json.index").unwrap(), "B1");
}
#[tokio::test]
async fn raw_key_reader_sees_uncited_bytes() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "f.txt", "CITED");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
write(dir.path(), "f.txt", "UNCITED");
backdate_baseline(&a, "f.txt");
a.upload_lane().await.unwrap();
let (_, raw) = store.get_whole(&a.cfg.file_key("f.txt"), None).await.unwrap();
assert_eq!(&raw[..], b"UNCITED");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let vid = m.entries["f.txt"].version_id.clone().unwrap();
let (_, cited) = store.get_version(&a.cfg.file_key("f.txt"), &vid).await.unwrap();
assert_eq!(&cited[..], b"CITED");
}
#[tokio::test]
async fn quiet_staged_file_is_not_restaged() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "quiet.txt", "v1");
let first = a.upload_lane().await.unwrap();
assert_eq!(first.staged, vec!["quiet.txt".to_string()], "nothing staged on the first tick");
for tick in 0..5 {
let r = a.upload_lane().await.unwrap();
assert!(
r.staged.is_empty(),
"tick {tick} re-staged a quiet file: the stage-diff base is the baseline alone"
);
}
assert_eq!(store.version_count(&a.cfg.file_key("quiet.txt")), 1);
}
#[tokio::test]
async fn hitl_admitted_between_citations() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "seed.txt", "v1");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
for i in 0..3 {
write(dir.path(), &format!("churn{i}.txt"), "x");
let lane = a.upload_lane().await.unwrap();
assert!(!lane.staged.is_empty(), "no lane tick happened at {i}");
assert!(a.citation_due(false).unwrap().is_none());
hitl_write(&store, &a.cfg, &format!("hitl{i}.txt"), "user bytes", "dilip")
.await
.unwrap_or_else(|e| panic!("HITL refused between citations at tick {i}: {e}"));
}
}
#[tokio::test]
async fn quiescence_fires_and_is_not_the_lag_cap() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
a.cfg.quiesce_bound_secs = 0; a.cfg.visibility_lag_bound_secs = Some(3600);
write(dir.path(), "one.txt", "written once, then silence");
a.upload_lane().await.unwrap();
assert!(a.citation_due(false).unwrap().is_none(), "quiescence fired before any stability");
a.upload_lane().await.unwrap();
let source = a.citation_due(false).unwrap();
assert_eq!(
source,
Some(CitationSource::Quiescence),
"quiescence did not fire — got {source:?} (dead-code shape if forced-lag-cap)"
);
let r = a.citation_pass(source.unwrap()).await.unwrap();
assert_eq!(r.source.as_deref(), Some("quiescence"));
let lp = manifest::load_pointer(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert_eq!(lp.pointer.boundary_source.as_deref(), Some("quiescence"));
}
#[tokio::test]
async fn lag_cap_forces_citation_and_stamps_the_source() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
a.cfg.visibility_lag_bound_secs = Some(0); a.cfg.quiesce_bound_secs = 3600;
let mut wrote = 0;
for i in 0..3 {
write(dir.path(), &format!("hot{i}.txt"), &format!("v{i}"));
wrote += 1;
a.upload_lane().await.unwrap();
}
assert_eq!(wrote, 3, "the writer was not writing");
let source = a.citation_due(false).unwrap();
assert_eq!(source, Some(CitationSource::ForcedLagCap));
let r = a.citation_pass(source.unwrap()).await.unwrap();
assert_eq!(r.source.as_deref(), Some("forced-lag-cap"));
let lp = manifest::load_pointer(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert_eq!(
lp.pointer.boundary_source.as_deref(),
Some("forced-lag-cap"),
"the forced source is not readable from the bucket pointer alone"
);
}
#[tokio::test]
async fn version_reclamation_returns_to_one_per_key() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
let key = a.cfg.file_key("churn.txt");
write(dir.path(), "churn.txt", "g1");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
assert_eq!(store.version_count(&key), 1);
for g in 2..6 {
write(dir.path(), "churn.txt", &format!("g{g}"));
backdate_baseline(&a, "churn.txt");
a.upload_lane().await.unwrap();
assert!(
store.version_count(&key) >= 2,
"generation {g} did not create a second version"
);
a.citation_pass(CitationSource::Sentinel).await.unwrap();
assert_eq!(
store.version_count(&key),
1,
"exact per-citation GC did not drain generation {g}"
);
}
assert!(store.expire_noncurrent(86_400).is_empty());
}
#[tokio::test]
async fn dangling_citation_refuses_rather_than_serving_a_hole() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "abandoned.txt", "CITED WORK");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
write(dir.path(), "abandoned.txt", "UNCITED WORK");
backdate_baseline(&a, "abandoned.txt");
a.upload_lane().await.unwrap();
let key = a.cfg.file_key("abandoned.txt");
assert_eq!(store.version_count(&key), 2);
let reaped = store.expire_noncurrent(0);
assert!(
reaped.iter().any(|(k, _)| k == &key),
"the backstop reaped nothing — the endgame fixture never armed"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let err = b.checkout().await.unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("recover-staged") && msg.contains("version"),
"checkout did not refuse loudly on a dangling citation: {msg}"
);
assert!(
!dir_b.path().join("abandoned.txt").exists(),
"checkout served a partial tree past a dangling citation"
);
let (_, survivor) = store.get_whole(&key, None).await.unwrap();
assert_eq!(&survivor[..], b"UNCITED WORK");
}
#[tokio::test]
async fn versioning_conformance_probe_passes_on_a_versioned_store() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.versioning_conformance().await.expect("a versioned store must pass the probe");
}
#[tokio::test]
async fn gated_floor_tick_stages_without_citing() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "model.json", "MID-LOGICAL-CHANGE");
let out = a.floor_tick().await.unwrap();
let key = a.cfg.file_key("model.json");
let (_, current) = store.get_whole(&key, None).await.unwrap();
assert_eq!(¤t[..], b"MID-LOGICAL-CHANGE", "the gated floor tick published nothing");
assert!(
a.load_stage().unwrap().entries.contains_key("model.json"),
"the floor tick did not run the upload lane"
);
let cited = manifest::load(store.as_ref(), &a.cfg)
.await
.unwrap()
.map(|l| l.manifest.entries.contains_key("model.json"))
.unwrap_or(false);
assert!(!cited, "the gated floor tick advanced the manifest — visibility is not gated");
assert_eq!(out.seq, None, "a staging-only tick reported a citation seq");
}
#[tokio::test]
async fn gated_floor_tick_cites_at_the_lag_cap() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
a.cfg.visibility_lag_bound_secs = Some(0);
write(dir.path(), "model.json", "COHERENT");
let out = a.floor_tick().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("model.json"), "the lag cap did not force a citation");
assert!(m.pinned_reads, "a gated citation did not stamp pinned_reads");
assert_eq!(m.boundary_source.as_deref(), Some("forced-lag-cap"));
assert_eq!(out.seq, Some(m.seq));
assert!(
a.load_stage().unwrap().entries.is_empty(),
"the citation left the staged set uncleared"
);
}
#[tokio::test]
async fn gated_publish_sentinel_cites_the_whole_stage() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "model.json", "A1");
a.upload_lane().await.unwrap();
write(dir.path(), "model.json.index", "B1");
a.upload_lane().await.unwrap();
assert_eq!(a.load_stage().unwrap().entries.len(), 2);
touch_sentinel(dir.path(), super::control::PUBLISH, r#"{"nonce":"n-1"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the sentinel was not honored");
assert_eq!(acks[0].status, "ok");
assert_eq!(acks[0].boundary, "sentinel");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("model.json") && m.entries.contains_key("model.json.index"));
assert_eq!(m.boundary_source.as_deref(), Some("sentinel"));
assert!(m.pinned_reads);
assert_eq!(acks[0].seq, Some(m.seq), "the ack did not name the boundary it installed");
}
#[tokio::test]
async fn gated_drain_cites_the_staged_versions_in_place() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "a.txt", "A1");
write(dir.path(), "b.txt", "B1");
a.upload_lane().await.unwrap();
let stage = a.load_stage().unwrap();
let staged_a = stage.entries["a.txt"].version_id.clone().unwrap();
let staged_b = stage.entries["b.txt"].version_id.clone().unwrap();
a.drain().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.boundary_source.as_deref(), Some("drain"));
assert!(m.pinned_reads, "the last boundary of a gated workspace was not pinned");
assert_eq!(
m.entries["a.txt"].version_id.as_deref(),
Some(staged_a.as_str()),
"the drain cited a version the lane had not staged — it moved data"
);
assert_eq!(m.entries["b.txt"].version_id.as_deref(), Some(staged_b.as_str()));
assert!(
a.load_stage().unwrap().entries.is_empty(),
"the drain left staged work in the pending record"
);
}
#[tokio::test]
async fn versioning_conformance_survives_a_leftover_probe_object() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let key = format!("{}/{}/probe/versioning", a.cfg.prefix, super::LEAN_DIR);
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "leftover".into(),
boundary_source: None,
posix: None,
};
store
.put_whole(
&key,
Bytes::from_static(b"crashed-probe"),
&PutCondition::IfNoneMatchAny,
&stamps,
crc64_nvme(b"crashed-probe"),
)
.await
.unwrap();
a.versioning_conformance()
.await
.expect("a leftover probe object wedged the conformance probe");
assert!(store.get_whole(&key, None).await.is_err(), "the probe did not clean up");
}
struct VersionStripping(Arc<MemoryStore>);
#[async_trait::async_trait]
impl ObjectStore for VersionStripping {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
let mut m = self.0.put_whole(key, body, cond, stamps, crc).await?;
m.version_id = None;
Ok(m)
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.0.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.0.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn gated_startup_refuses_a_version_stripping_backend() {
let inner = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store: Arc::new(VersionStripping(inner.clone())) as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
a.gated_startup_check().await.expect("the startup gate fired outside gated mode");
gated(&mut a);
let err = a.gated_startup_check().await.unwrap_err().to_string();
assert!(
err.contains("versioning conformance"),
"gated mode started over a stripping proxy: {err}"
);
}
#[tokio::test]
async fn recover_staged_uses_the_durable_summary_and_falls_back_without_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "cited.txt", "V1");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
write(dir.path(), "quiet.txt", "CITED-AND-THEN-QUIET");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
write(dir.path(), "cited.txt", "V2-UNCITED");
backdate_baseline(&a, "cited.txt");
write(dir.path(), "brand-new.txt", "NEW-UNCITED");
a.upload_lane().await.unwrap();
let okey = a.orphans_key();
let (_, obytes) = store.get_whole(&okey, None).await.expect("no orphan summary");
let odoc: serde_json::Value = serde_json::from_slice(&obytes).unwrap();
let named: Vec<String> = odoc["candidates"]
.as_array()
.unwrap()
.iter()
.map(|c| c["path"].as_str().unwrap().to_string())
.collect();
assert!(!named.is_empty(), "the summary names nothing");
assert!(
!named.contains(&"quiet.txt".to_string()),
"the summary names the quiet path — the narrowing gap cannot be reproduced"
);
drop(a);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.boundary_mode = super::BoundaryMode::Gated;
b.cfg.visibility_lag_bound_secs = Some(3600);
assert!(claim_until_held(&mut b, 12).await);
let r = b.recover_staged().await.unwrap();
assert!(r.from_summary, "recovery took the expensive path with a good summary on hand");
assert_eq!(r.recited.len(), 2, "the cheap path recovered less: {:?}", r.recited);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("brand-new.txt"), "the cheap path missed the new path");
assert!(
r.unrecoverable.is_empty(),
"recovery called a cited path unrecoverable: {:?}",
r.unrecoverable
);
assert!(
m.entries.contains_key("quiet.txt"),
"the narrowed recovery dropped a cited-but-not-staged path"
);
let q = &m.entries["quiet.txt"];
let (_, qb) = store.get_version(&q.key, q.version_id.as_ref().unwrap()).await.unwrap();
assert_eq!(&qb[..], b"CITED-AND-THEN-QUIET");
let dir_c = tempfile::tempdir().unwrap();
let mut c = syncer(&store, dir_c.path()).await;
c.cfg.boundary_mode = super::BoundaryMode::Gated;
c.cfg.visibility_lag_bound_secs = Some(3600);
assert!(claim_until_held(&mut c, 12).await);
c.checkout().await.unwrap();
write(dir_c.path(), "later.txt", "UNCITED-AGAIN");
c.upload_lane().await.unwrap();
store.delete(&okey).await.unwrap(); let r2 = c.recover_staged().await.unwrap();
assert!(!r2.from_summary, "reported the cheap path with no summary present");
assert!(
r2.recited.iter().any(|p| p == "later.txt"),
"the fallback missed uncited work: {:?}",
r2.recited
);
}
#[tokio::test]
async fn recover_staged_recites_uncited_work_forward() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "cited.txt", "V1");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let cited_seq = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
write(dir.path(), "cited.txt", "V2-UNCITED");
backdate_baseline(&a, "cited.txt");
write(dir.path(), "brand-new.txt", "NEW-UNCITED");
a.upload_lane().await.unwrap();
drop(a);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.boundary_mode = super::BoundaryMode::Gated;
b.cfg.visibility_lag_bound_secs = Some(3600);
assert!(claim_until_held(&mut b, 12).await);
let before = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert!(!before.entries.contains_key("brand-new.txt"));
let r = b.recover_staged().await.unwrap();
assert_eq!(r.recited.len(), 2, "recovery re-cited {:?}", r.recited);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.boundary_source.as_deref(), Some("recovered"));
assert!(m.seq > cited_seq);
assert!(m.entries.contains_key("brand-new.txt"));
let dir_c = tempfile::tempdir().unwrap();
let mut c = syncer(&store, dir_c.path()).await;
c.checkout().await.unwrap();
assert_eq!(read(dir_c.path(), "cited.txt").as_deref(), Some("V2-UNCITED"));
assert_eq!(read(dir_c.path(), "brand-new.txt").as_deref(), Some("NEW-UNCITED"));
assert!(
b.state.load_conflicts().unwrap().iter().any(|c| c.kind == "recovered-staged"),
"recovery re-cited foreign-generation bytes without a conflict record"
);
}
#[tokio::test]
async fn recover_staged_repairs_a_dangling_citation() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "abandoned.txt", "CITED WORK");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
write(dir.path(), "abandoned.txt", "UNCITED WORK");
backdate_baseline(&a, "abandoned.txt");
a.upload_lane().await.unwrap();
let key = a.cfg.file_key("abandoned.txt");
assert!(!store.expire_noncurrent(0).is_empty(), "the endgame fixture never armed");
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(b.checkout().await.is_err());
let dir_r = tempfile::tempdir().unwrap();
let mut r = syncer(&store, dir_r.path()).await;
r.cfg.boundary_mode = super::BoundaryMode::Gated;
r.cfg.visibility_lag_bound_secs = Some(3600);
assert!(claim_until_held(&mut r, 12).await);
let rep = r.recover_staged().await.unwrap();
assert_eq!(rep.dangling.len(), 1, "the dangling citation was not recognized");
let dir_c = tempfile::tempdir().unwrap();
let mut c = syncer(&store, dir_c.path()).await;
c.checkout().await.expect("checkout still refuses after recover-staged");
assert_eq!(read(dir_c.path(), "abandoned.txt").as_deref(), Some("UNCITED WORK"));
let _ = key;
}
#[tokio::test]
async fn recover_staged_is_a_no_op_when_nothing_is_uncited() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "quiet.txt", "V1");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let seq = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest.seq;
let r = a.recover_staged().await.unwrap();
assert!(r.recited.is_empty() && r.dangling.is_empty());
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.seq, seq, "a no-op recovery still advanced the manifest");
}
#[tokio::test]
async fn gauges_name_the_reason_visibility_is_withheld() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "big.bin", "STAGED BUT UNCITED");
a.floor_tick().await.unwrap();
let g = a.load_gauges().unwrap();
assert_eq!(g.state, "live");
assert_eq!(g.boundary_mode, "gated");
assert_eq!(g.staged_uncited_count, 1);
assert_eq!(g.staged_uncited_bytes, "STAGED BUT UNCITED".len() as u64);
assert_eq!(
g.withheld_reason.as_deref(),
Some("quiesce-pending"),
"a withheld tick did not name its reason"
);
assert!(g.last_boundary.is_none(), "nothing was cited, yet a boundary is claimed");
assert!(g.cited_noncurrent_age_max_secs < 5);
assert_eq!(g.sentinel_budget_remaining, a.cfg.sentinel_hourly_budget);
}
#[tokio::test]
async fn gauges_count_forced_citations_and_name_the_last_boundary() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
a.cfg.visibility_lag_bound_secs = Some(0);
for g in 1..4 {
write(dir.path(), "churn.txt", &format!("g{g}"));
backdate_baseline(&a, "churn.txt");
a.floor_tick().await.unwrap();
}
let g = a.load_gauges().unwrap();
assert_eq!(g.forced_citation_count, 3, "forced citations were not counted");
let lb = g.last_boundary.expect("a citation installed, but no boundary recorded");
assert_eq!(lb.source, "forced-lag-cap");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(lb.seq, m.seq);
assert_eq!(g.staged_uncited_count, 0);
assert_eq!(g.withheld_reason, None);
}
#[tokio::test]
async fn a_fenced_syncer_gauges_itself_fenced() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "work.txt", "v1");
a.floor_tick().await.unwrap();
assert_eq!(a.load_gauges().unwrap().state, "live");
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 12).await);
write(dir_a.path(), "work.txt", "v2");
backdate_baseline(&a, "work.txt");
let err = a.floor_tick().await.unwrap_err();
assert!(matches!(err, LeanError::Fenced(_)), "the straggler was not fenced: {err}");
assert_eq!(a.read_capabilities().unwrap().state, "fenced");
assert_eq!(
a.load_gauges().unwrap().state,
"fenced",
"capabilities say fenced but the gauges still say live"
);
}
#[tokio::test]
async fn status_renders_without_claiming_the_lease() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
gated(&mut a);
a.checkout().await.unwrap();
write(dir.path(), "held.txt", "x");
a.floor_tick().await.unwrap();
let s = super::status_report(&cfg_for(dir.path())).unwrap();
assert_eq!(s.gauges.unwrap().staged_uncited_count, 1);
assert_eq!(s.capabilities.unwrap().boundary_mode, "gated");
assert_eq!(s.pending_stage_entries, 1);
assert!(s.incarnation_epoch.is_some());
assert_eq!(
store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap().epoch,
a.lease.as_ref().unwrap().epoch,
"status rotated the epoch it was supposed to observe"
);
}
#[tokio::test]
async fn citation_never_reaps_a_hitl_write_that_landed_mid_stage() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "shared.txt", "AGENT V1");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
write(dir.path(), "shared.txt", "AGENT V2");
backdate_baseline(&a, "shared.txt");
a.upload_lane().await.unwrap();
let staged = a.load_stage().unwrap().entries["shared.txt"].version_id.clone().unwrap();
hitl_write(&store, &a.cfg, "shared.txt", "USER EDIT", "ui@example").await.unwrap();
let key = a.cfg.file_key("shared.txt");
let (hitl_meta, _) = store.get_whole(&key, None).await.unwrap();
let hitl_version = hitl_meta.version_id.clone().unwrap();
assert_ne!(hitl_version, staged, "the fixture never armed: no foreign version landed");
let cite = a.citation_pass(CitationSource::Sentinel).await.unwrap();
let (_, still) = store
.get_version(&key, &hitl_version)
.await
.expect("the citation reaped the user's CURRENT version");
assert_eq!(&still[..], b"USER EDIT");
let (_, current) = store.get_whole(&key, None).await.unwrap();
assert_eq!(¤t[..], b"USER EDIT");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_ne!(
m.entries["shared.txt"].version_id.as_deref(),
Some(staged.as_str()),
"the citation cited bytes that PREDATE the user's write, over it"
);
assert!(
a.state.load_conflicts().unwrap().iter().any(|c| c.kind.contains("hitl-inflight")),
"the dropped path was not surfaced"
);
assert_eq!(cite.dropped_inflight, vec!["shared.txt".to_string()]);
assert!(cite.dropped_stale_base.is_empty());
let ib = inbox::load(store.as_ref(), &a.cfg).await.unwrap();
assert!(ib.doc.entries.iter().any(|e| e.path == "shared.txt"));
}
#[tokio::test]
async fn the_reaper_never_takes_an_out_of_band_writers_current_version() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "shared.txt", "AGENT V1");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
write(dir.path(), "shared.txt", "AGENT V2");
backdate_baseline(&a, "shared.txt");
a.upload_lane().await.unwrap();
let key = a.cfg.file_key("shared.txt");
let (cur, _) = store.get_whole(&key, None).await.unwrap();
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "out-of-band".into(),
boundary_source: None,
posix: None,
};
let foreign = store
.put_whole(
&key,
Bytes::from_static(b"OUT OF BAND"),
&PutCondition::IfMatch(cur.etag.clone()),
&stamps,
crc64_nvme(b"OUT OF BAND"),
)
.await
.unwrap();
let foreign_version = foreign.version_id.clone().unwrap();
assert!(inbox::load(store.as_ref(), &a.cfg).await.unwrap().doc.entries.is_empty());
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let (_, still) = store
.get_version(&key, &foreign_version)
.await
.expect("the reaper deleted an out-of-band writer's CURRENT version");
assert_eq!(&still[..], b"OUT OF BAND");
}
#[tokio::test]
async fn a_sentinel_boundary_carries_a_delete_made_before_the_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "k");
write(dir.path(), "gone.txt", "v1");
a.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("gone.txt"), "fixture: never published");
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-del"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the sentinel was not honored");
assert_eq!(acks[0].status, "ok");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("gone.txt"),
"the ack said ok at seq {:?} while the manifest still cites a file the agent \
deleted before the touch (report.deleted = {})",
acks[0].seq,
acks[0].report.deleted
);
}
#[tokio::test]
async fn only_a_declared_barrier_confirms_a_first_absence() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "gone.txt", "v1");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
let r = a.run_barrier().await.unwrap();
assert_eq!(r.first_absence, vec!["gone.txt".to_string()]);
assert_eq!(r.absences_confirmed, 0);
assert!(r.deleted.is_empty(), "the cadence barrier published a first absence");
let mut b = a.state.load_baseline().unwrap();
b.prev_scan.insert("gone.txt".into());
a.state.save_baseline(&b).unwrap();
let r = a.declared_barrier().await.unwrap();
assert_eq!(r.absences_confirmed, 1, "the declared barrier did not confirm the absence");
assert!(r.first_absence.is_empty());
assert_eq!(r.deleted, vec!["gone.txt".to_string()]);
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("gone.txt"));
}
#[tokio::test]
async fn an_unreadable_path_is_not_confirmed_as_deleted() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
if unsafe { libc::geteuid() } == 0 {
eprintln!("SKIPPED: running as root, mode bits cannot induce EACCES");
return;
}
let locked = dir.path().join("locked");
std::fs::create_dir(&locked).unwrap();
std::fs::write(locked.join("f.txt"), b"still here").unwrap();
std::fs::set_permissions(&locked, std::os::unix::fs::PermissionsExt::from_mode(0o000)).unwrap();
let mut classified = super::scan::Classified::default();
classified.first_absence.insert("locked/f.txt".into());
let err = a
.confirm_absences(&mut classified)
.expect_err("an unreadable path must not confirm a deletion");
assert!(
format!("{err}").contains("refusing to publish a deletion"),
"the refusal must say what it refused: {err}"
);
assert!(
classified.deletes.is_empty(),
"NOTHING may be queued for deletion off an errno that is not NotFound: {:?}",
classified.deletes
);
std::fs::set_permissions(&locked, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
}
#[tokio::test]
async fn the_confirmation_never_deletes_a_path_the_walk_merely_missed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
write(dir.path(), "renamed.txt", "here all along");
let mut classified = super::scan::Classified::default();
classified.first_absence.insert("renamed.txt".into());
classified.first_absence.insert("truly-gone.txt".into());
let confirmed = a.confirm_absences(&mut classified).unwrap();
assert_eq!(confirmed, 1);
assert_eq!(
classified.deletes.iter().cloned().collect::<Vec<_>>(),
vec!["truly-gone.txt".to_string()]
);
assert!(
classified.first_absence.contains("renamed.txt"),
"the confirmation promoted a path that is on disk — the walk race would publish \
a delete of a live file"
);
}
#[tokio::test]
async fn a_gated_sentinel_boundary_carries_a_delete_made_before_the_touch() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
gated(&mut a);
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
write(dir.path(), "gone.txt", "v1");
a.gated_tick(true).await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("gone.txt"), "fixture: never cited");
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-del"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1);
assert_eq!(acks[0].status, "ok");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("gone.txt"),
"the gated citation acked a boundary that still cites a file the agent deleted \
before the touch"
);
}
#[tokio::test]
async fn the_drain_carries_a_delete_made_before_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "gone.txt", "v1");
a.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
a.drain().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("gone.txt"),
"the drain left the delete withheld — the successor's checkout resurrects it"
);
}
#[tokio::test]
async fn the_heartbeat_arm_settles_owed_acks_when_it_finds_the_fence() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let posture = a.sentinel_preflight().unwrap();
a.write_capabilities(&posture, false).unwrap();
a.checkout().await.unwrap();
touch_sentinel(dir_a.path(), control::PUBLISH, r#"{"nonce":"pub-stranded"}"#);
touch_sentinel(dir_a.path(), control::SYNC, r#"{"nonce":"sync-stranded"}"#);
a.poll_sentinels().unwrap();
assert!(a.load_pending(Verb::Publish).unwrap().is_some());
assert!(a.load_pending(Verb::Sync).unwrap().is_some());
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 12).await);
let their_epoch = b.lease.as_ref().unwrap().epoch;
assert!(their_epoch > a.lease.as_ref().unwrap().epoch, "no takeover happened");
let err = a.heartbeat_tick().await.unwrap_err();
assert!(matches!(err, LeanError::Fenced(_)));
for (verb, nonce) in [(Verb::Publish, "pub-stranded"), (Verb::Sync, "sync-stranded")] {
let ack = a
.read_ack(verb)
.unwrap_or_else(|| panic!("{verb:?} was stranded: the heartbeat exited unsettled"));
assert_eq!(ack.status, "refused-fenced");
assert!(ack.nonces.contains(&nonce.to_string()));
assert_eq!(ack.observed_epoch, Some(their_epoch));
assert!(a.load_pending(verb).unwrap().is_none());
}
let caps = a.read_capabilities().unwrap();
assert_eq!(caps.state, "fenced", "the marker still advertises a zombie as live");
assert!(caps.verbs.is_empty());
}
#[tokio::test]
async fn a_crash_between_the_cas_and_step_7_never_makes_our_own_entry_foreign() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.run_barrier().await.unwrap();
write(dir.path(), "f.txt", "v2 is longer");
backdate_baseline(&a, "f.txt");
let stale = a.state.load_baseline().unwrap();
a.run_barrier().await.unwrap();
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
a.state.save_baseline(&stale).unwrap();
assert_ne!(
stale.inst_base.get("f.txt"),
Some(&installed.manifest.entries["f.txt"].etag),
"fixture: the merge base is not actually behind the install"
);
std::fs::remove_file(dir.path().join("f.txt")).unwrap();
let r = a.declared_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("f.txt"),
"our own install read as a foreign change and swallowed the agent's delete \
(deleted={:?}, foreign_queued={})",
r.deleted,
r.foreign_queued
);
let ib = inbox::load(store.as_ref(), &a.cfg).await.unwrap();
assert!(
!ib.doc.entries.iter().any(|e| e.path == "f.txt"),
"a phantom foreign entry was queued for a path only this workspace ever wrote"
);
}
#[tokio::test]
async fn a_gated_citation_never_reads_its_own_boundary_as_foreign() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
gated(&mut a);
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.gated_tick(true).await.unwrap();
write(dir.path(), "f.txt", "v2 is longer");
backdate_baseline(&a, "f.txt");
let stale = a.state.load_baseline().unwrap();
a.gated_tick(true).await.unwrap();
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert_eq!(
installed.manifest.entries["f.txt"].size,
"v2 is longer".len() as u64,
"fixture: the second citation never landed"
);
a.state.save_baseline(&stale).unwrap();
assert_ne!(
stale.inst_base.get("f.txt"),
Some(&installed.manifest.entries["f.txt"].etag),
"fixture: the merge base is not actually behind the citation"
);
std::fs::remove_file(dir.path().join("f.txt")).unwrap();
a.declared_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("f.txt"),
"the citation read its own boundary as foreign and withheld the agent's delete"
);
}
#[tokio::test]
async fn a_pending_sync_at_sigterm_never_cancels_the_drains_own_boundary() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
a.run_barrier().await.unwrap();
write(dir.path(), "work.txt", "W1");
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"s-1"}"#);
a.poll_sentinels().unwrap();
assert!(a.load_pending(Verb::Sync).unwrap().is_some());
let acks = a.drain().await.unwrap();
assert_eq!(acks.len(), 1, "the drain did not settle the owed sync ack");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
m.entries.contains_key("work.txt"),
"the drain skipped its cite-everything pass because a SYNC ack carried a seq — \
every byte since the last boundary died with the emptyDir"
);
}
#[tokio::test]
async fn a_gated_drain_cites_its_stage_even_when_a_sync_was_owed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
a.run_barrier().await.unwrap();
gated(&mut a);
write(dir.path(), "ckpt.bin", "C1");
a.upload_lane().await.unwrap();
assert_eq!(a.load_stage().unwrap().entries.len(), 1);
touch_sentinel(dir.path(), control::SYNC, r#"{"nonce":"s-2"}"#);
a.poll_sentinels().unwrap();
a.drain().await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
m.entries.contains_key("ckpt.bin"),
"the gated drain left durable work uncited because a sync ack carried a seq"
);
assert!(
a.load_stage().unwrap().entries.is_empty(),
"the drain left staged work in the pending record"
);
}
#[tokio::test]
async fn a_recreated_file_survives_its_own_withheld_tombstone() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "ckpt.bin", "v1");
a.run_barrier().await.unwrap();
gated(&mut a);
std::fs::remove_file(dir.path().join("ckpt.bin")).unwrap();
a.upload_lane().await.unwrap();
a.upload_lane().await.unwrap();
assert!(
a.load_stage().unwrap().withheld_deletes.contains("ckpt.bin"),
"the fixture never armed the tombstone"
);
write(dir.path(), "ckpt.bin", "v2-the-replacement");
a.upload_lane().await.unwrap();
assert!(
a.load_stage().unwrap().entries.contains_key("ckpt.bin"),
"the fixture never staged the replacement"
);
a.citation_pass(CitationSource::ForcedLagCap).await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
m.entries.contains_key("ckpt.bin"),
"the stale tombstone amputated a file that exists on disk and was staged this pass"
);
let cited = m.entries["ckpt.bin"].version_id.clone().unwrap();
let (_, body) = store.get_version(&a.cfg.file_key("ckpt.bin"), &cited).await.unwrap();
assert_eq!(
&body[..],
b"v2-the-replacement",
"the boundary cited the pre-rotation checkpoint"
);
}
#[tokio::test]
async fn a_planted_temp_sibling_is_never_written_through() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let outside = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
a.run_barrier().await.unwrap();
let victims = ["consume-victim", "ticker-victim", "state-victim"];
for v in victims {
std::fs::write(outside.path().join(v), "ORIGINAL").unwrap();
}
std::fs::create_dir_all(dir.path().join("inputs")).unwrap();
std::os::unix::fs::symlink(
outside.path().join("consume-victim"),
dir.path().join("inputs/config.json.flint-sync-tmp"),
)
.unwrap();
std::fs::create_dir_all(dir.path().join(super::CONTROL_DIR)).unwrap();
std::os::unix::fs::symlink(
outside.path().join("ticker-victim"),
dir.path().join(super::CONTROL_DIR).join("remote.seq.tmp"),
)
.unwrap();
std::os::unix::fs::symlink(
outside.path().join("state-victim"),
a.cfg.state_dir().join("baseline.tmp"),
)
.unwrap();
hitl_write(&store, &a.cfg, "inputs/config.json", "REMOTE-BYTES", "ui").await.unwrap();
a.run_barrier().await.unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n-tmp"}"#);
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the fixture never exercised the control writer");
for v in victims {
assert_eq!(
std::fs::read_to_string(outside.path().join(v)).unwrap(),
"ORIGINAL",
"the syncer wrote through a planted temp sibling ({v}) — an arbitrary-file-write \
primitive outside the workspace, with the bucket credentials"
);
}
assert_eq!(read(dir.path(), "inputs/config.json").as_deref(), Some("REMOTE-BYTES"));
assert!(control_exists(dir.path(), "remote.seq"));
}
#[tokio::test]
async fn a_consumed_hitl_write_is_re_cited_in_gated_mode() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "AGENT-V1");
a.run_barrier().await.unwrap();
gated(&mut a);
hitl_write(&store, &a.cfg, "shared.txt", "UI-EDIT", "ui").await.unwrap();
a.upload_lane().await.unwrap();
assert_eq!(
read(dir.path(), "shared.txt").as_deref(),
Some("UI-EDIT"),
"the fixture never got as far as the consume"
);
assert!(
a.load_stage().unwrap().entries.is_empty(),
"the fixture staged the path, so it never exercises the repair gap"
);
let mut stage = a.load_stage().unwrap();
stage.last_citation_unix -= 3600;
a.save_stage(&stage).unwrap();
let due = a.citation_due(false).unwrap();
assert!(due.is_some(), "no citation is ever due for a consumed HITL write");
a.citation_pass(due.unwrap()).await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let e = m.entries.get("shared.txt").expect("the boundary dropped the path entirely");
let key = a.cfg.file_key("shared.txt");
let (_, body) = match &e.version_id {
Some(v) => store.get_version(&key, v).await.unwrap(),
None => store.get_whole(&key, Some(&e.etag)).await.unwrap(),
};
assert_eq!(
&body[..],
b"UI-EDIT",
"the manifest still cites the pre-HITL version — under pinned_reads the acked write \
is invisible to every checkout, sibling sync and DR re-materialization"
);
}
#[tokio::test]
async fn a_pinned_manifest_never_adopts_current_for_a_legacy_entry() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "legacy.txt", "L1");
write(dir_a.path(), "other.txt", "O1");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut m = loaded.manifest.clone();
m.entries.get_mut("legacy.txt").unwrap().version_id = None;
manifest::cas_write(store.as_ref(), &a.cfg, &m, Some(&loaded.handle()), 1, "legacy-writer")
.await
.unwrap();
gated(&mut a);
write(dir_a.path(), "other.txt", "O2");
backdate_baseline(&a, "other.txt");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::ForcedLagCap).await.unwrap();
write(dir_a.path(), "legacy.txt", "L2-MID-CHANGE");
backdate_baseline(&a, "legacy.txt");
a.upload_lane().await.unwrap();
let key = a.cfg.file_key("legacy.txt");
let (_, raw) = store.get_whole(&key, None).await.unwrap();
assert_eq!(&raw[..], b"L2-MID-CHANGE", "nothing was actually staged");
let now = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(now.pinned_reads, "the fixture never got a pinned manifest");
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout().await.unwrap();
assert_eq!(
read(dir_b.path(), "legacy.txt").as_deref(),
Some("L1"),
"a pinned checkout S3-wins-adopted uncited mid-change bytes for a legacy entry"
);
assert_eq!(read(dir_b.path(), "other.txt").as_deref(), Some("O2"));
}
#[tokio::test]
async fn an_unresolvable_legacy_citation_refuses_rather_than_adopts() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "legacy.txt", "L1");
write(dir_a.path(), "other.txt", "O1");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut m = loaded.manifest.clone();
let v1 = m.entries["legacy.txt"].version_id.clone().unwrap();
m.entries.get_mut("legacy.txt").unwrap().version_id = None;
manifest::cas_write(store.as_ref(), &a.cfg, &m, Some(&loaded.handle()), 1, "legacy-writer")
.await
.unwrap();
let key = a.cfg.file_key("legacy.txt");
let body = Bytes::from_static(b"FOREIGN-CURRENT");
let crc = crc64_nvme(&body);
store
.put_whole(
&key,
body,
&PutCondition::IfMatch(m.entries["legacy.txt"].etag.clone()),
&GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "foreign".into(),
boundary_source: None,
posix: None,
},
crc,
)
.await
.unwrap();
store.delete_version(&key, &v1).await.unwrap();
gated(&mut a);
write(dir_a.path(), "other.txt", "O2");
backdate_baseline(&a, "other.txt");
a.upload_lane().await.unwrap();
let cite = a.citation_pass(CitationSource::ForcedLagCap).await.unwrap();
assert!(
!cite.backfilled.contains(&"legacy.txt".to_string()),
"the fixture left the entry resolvable, so it never reaches the reader rule"
);
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(installed.pinned_reads);
assert!(
installed.entries["legacy.txt"].version_id.is_none(),
"the fixture never produced an unresolvable legacy entry"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let err = b.checkout().await;
assert!(
err.is_err(),
"a pinned checkout adopted the current version for an unresolvable legacy citation"
);
assert_eq!(
read(dir_b.path(), "legacy.txt"),
None,
"the uncited foreign bytes were materialized anyway"
);
}
#[tokio::test]
async fn a_gated_ack_never_claims_a_path_the_citation_dropped() {
let pending = super::sentinel::PendingSentinel {
verb: "publish".into(),
consumed_mtime_unix_ns: 7,
consumed_at: 0,
nonces: vec!["n-1".into()],
note: None,
scope: None,
torn: false,
};
let lane = super::gated::LaneReport {
staged: vec!["p".into(), "q".into()],
staged_bytes: 2,
..Default::default()
};
let clean = super::gated::CitationReport { seq: Some(9), cited: 2, ..Default::default() };
let ok = super::sentinel::gated_ack(&pending, false, &lane, &clean, Some("e".into()));
assert_eq!(ok.status, "ok");
assert!(ok.report.dropped.is_empty());
let dropped = super::gated::CitationReport {
seq: Some(9),
cited: 1,
dropped_inflight: vec!["p".into()],
..Default::default()
};
let partial = super::sentinel::gated_ack(&pending, false, &lane, &dropped, Some("e".into()));
assert_ne!(
partial.status, "ok",
"the ack claimed a coherent point that omits a path the agent declared"
);
assert_eq!(partial.report.dropped, vec!["p".to_string()]);
assert_eq!(partial.nonces, vec!["n-1".to_string()], "the agent must still be answered");
let equal = super::gated::CitationReport {
seq: Some(9),
cited: 1,
dropped_stale_base: vec!["p".into()],
..Default::default()
};
let still_ok = super::sentinel::gated_ack(&pending, false, &lane, &equal, Some("e".into()));
assert_eq!(still_ok.status, "ok");
}
#[tokio::test]
async fn a_delete_cancels_the_version_the_lane_had_already_staged() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "ckpt.bin", "V1");
a.run_barrier().await.unwrap();
gated(&mut a);
write(dir.path(), "ckpt.bin", "V2-the-next-one");
backdate_baseline(&a, "ckpt.bin");
write(dir.path(), "scratch.tmp", "TEMP");
a.upload_lane().await.unwrap();
let stage = a.load_stage().unwrap();
let staged_a = stage.entries["ckpt.bin"].version_id.clone().unwrap();
let staged_b = stage.entries["scratch.tmp"].version_id.clone().unwrap();
std::fs::remove_file(dir.path().join("ckpt.bin")).unwrap();
std::fs::remove_file(dir.path().join("scratch.tmp")).unwrap();
a.upload_lane().await.unwrap();
a.upload_lane().await.unwrap();
let stage = a.load_stage().unwrap();
assert!(
stage.withheld_deletes.contains("ckpt.bin"),
"the fixture never confirmed the cited path's absence"
);
assert!(
!stage.entries.contains_key("ckpt.bin"),
"the lane kept a staged version for a path it has since seen deleted"
);
assert!(
!stage.entries.contains_key("scratch.tmp"),
"a staged-but-never-cited path survived its own deletion — classify cannot see it"
);
a.citation_pass(CitationSource::ForcedLagCap).await.unwrap();
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(
!m.entries.contains_key("ckpt.bin"),
"the boundary cited a file the agent had deleted before it"
);
assert!(
!m.entries.contains_key("scratch.tmp"),
"the boundary cited a scratch file that never existed at any coherent point"
);
for (path, v) in [("ckpt.bin", staged_a), ("scratch.tmp", staged_b)] {
assert!(
store.get_version(&a.cfg.file_key(path), &v).await.is_err(),
"the cancelled staging left an uncited version behind ({path})"
);
}
}
#[tokio::test]
async fn a_heartbeat_echoes_the_running_mode_into_the_lease_cell() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
let cited = a.run_barrier().await.unwrap().seq.unwrap();
gated(&mut a);
write(dir.path(), "ckpt.bin", "C1");
a.upload_lane().await.unwrap();
let staged = a.load_stage().unwrap().entries.len() as u64;
assert_eq!(staged, 1, "the fixture staged nothing — the echo's count would be 0 == 0");
a.heartbeat_tick().await.unwrap();
let cell = store.epoch_read(&a.cfg.epoch_key()).await.unwrap().unwrap();
let echo: flint_store::LeaseEcho = serde_json::from_str(
cell.echo.as_deref().expect("the heartbeat carried no observed-state echo"),
)
.expect("the echo is not a LeaseEcho");
assert_eq!(echo.active_boundary_mode, "gated", "the echo reports the spec, not the run");
assert_eq!(echo.staged_uncited_count, staged, "the gated exposure is not echoed");
assert_eq!(echo.last_cited_seq, cited, "the echo names no citation");
assert_eq!(echo.protocol, super::SENTINEL_PROTOCOL);
assert!(!echo.syncer_version.is_empty(), "no version ⇒ no mixed-fleet tell");
}
#[tokio::test]
async fn a_version_stripping_proxy_refuses_gated_startup() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
gated(&mut a);
a.gated_startup_check().await.expect("a conformant store must pass the probe");
store.strip_version_ids(true);
let err = a.gated_startup_check().await.expect_err("a stripping proxy was accepted");
let msg = err.to_string();
assert!(
msg.contains("x-amz-version-id"),
"the refusal must name the header a proxy operator can fix: {msg}"
);
}
#[tokio::test]
async fn uncited_work_is_surfaced_durably_and_cleared_by_its_citation() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "seed.txt", "S1");
a.run_barrier().await.unwrap();
gated(&mut a);
let key = a.orphans_key();
write(dir.path(), "ckpt.bin", "C1");
a.upload_lane().await.unwrap();
assert_eq!(a.load_stage().unwrap().entries.len(), 1, "the fixture staged nothing");
let (_, body) = store.get_whole(&key, None).await.expect("no durable orphan summary");
let doc: super::gated::OrphanDoc = serde_json::from_slice(&body).unwrap();
assert_eq!(doc.candidates.len(), 1);
assert_eq!(doc.candidates[0].path, "ckpt.bin");
assert!(
doc.candidates[0].version_id.is_some(),
"a candidate with no version id cannot be recovered by name"
);
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let (_, body) = store.get_whole(&key, None).await.unwrap();
let doc: super::gated::OrphanDoc = serde_json::from_slice(&body).unwrap();
assert!(
doc.candidates.is_empty(),
"the summary still names work the citation made visible: {:?}",
doc.candidates
);
}
#[tokio::test]
async fn an_unchanged_orphan_set_costs_no_request() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "ckpt.bin", "C1");
a.upload_lane().await.unwrap();
let first = store.head(&a.orphans_key()).await.unwrap().etag;
a.upload_lane().await.unwrap();
let versions = store
.list_versions(&a.orphans_key())
.await
.unwrap()
.into_iter()
.filter(|v| v.key == a.orphans_key())
.count();
assert_eq!(versions, 1, "an unchanged candidate set re-wrote the summary");
assert_eq!(store.head(&a.orphans_key()).await.unwrap().etag, first);
}
#[tokio::test]
async fn a_uds_boundary_and_a_file_sentinel_coalesce_into_one_ack() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
a.cfg.sentinel_min_interval_secs = 0;
write(dir.path(), "work.txt", "W1");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"file:1"}"#);
a.poll_sentinels().unwrap();
a.request_boundary("uds:1", Some("from the socket".into())).unwrap();
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1, "the two doors produced {} barriers", acks.len());
let ack = &acks[0];
assert_eq!(ack.status, "ok");
for n in ["uds:1", "file:1"] {
assert!(
ack.nonces.iter().any(|x| x == n),
"the ack does not cover {n}: {:?}",
ack.nonces
);
}
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("work.txt"), "the coalesced boundary published nothing");
write(dir.path(), "work2.txt", "W2");
a.request_boundary("uds:2", None).unwrap();
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"file:2"}"#);
a.poll_sentinels().unwrap();
let acks = a.sentinel_tick().await.unwrap();
assert_eq!(acks.len(), 1);
for n in ["uds:2", "file:2"] {
assert!(acks[0].nonces.iter().any(|x| x == n), "socket-first lost {n}");
}
}
#[tokio::test]
async fn a_gateway_boundary_request_becomes_a_pending_sentinel_with_no_conflict() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
inbox::gateway_request(
store.as_ref(),
&a.cfg,
inbox::RequestedVerb::Boundary,
"ci@example",
)
.await
.unwrap();
write(dir.path(), "work.txt", "W1");
a.consume_inbox().await.unwrap();
let pending = a.load_pending(super::sentinel::Verb::Publish).unwrap();
let pending = pending.expect("the gateway request minted no pending sentinel");
assert!(
pending.nonces.iter().any(|n| n.contains("ci@example")),
"the requestor is not in the covered nonces: {:?}",
pending.nonces
);
assert_eq!(
std::fs::read_to_string(a.cfg.state_dir().join("conflicts.jsonl")).unwrap_or_default(),
"",
"the boundary request minted a conflict record"
);
let before = pending.nonces.len();
a.consume_inbox().await.unwrap();
assert_eq!(
a.load_pending(super::sentinel::Verb::Publish).unwrap().unwrap().nonces.len(),
before,
"the same gateway request was consumed twice"
);
}
#[tokio::test]
async fn a_gateway_sync_request_is_carried_and_never_executed() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "keep.txt", "LOCAL");
a.run_barrier().await.unwrap();
let mut m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
m.manifest.entries.remove("keep.txt");
m.manifest.seq += 1;
manifest::cas_write(store.as_ref(), &a.cfg, &m.manifest, Some(&m.handle()), 9, "foreign")
.await
.unwrap();
let tree_before = read(dir.path(), "keep.txt");
inbox::gateway_request(store.as_ref(), &a.cfg, inbox::RequestedVerb::Sync, "ci@example")
.await
.unwrap();
a.consume_inbox().await.unwrap();
assert_eq!(
read(dir.path(), "keep.txt"),
tree_before,
"the syncer EXECUTED a sync on a remote's say-so and rewrote the agent's tree"
);
assert!(tree_before.is_some(), "the fixture had nothing to lose");
let t: control::RemoteSeq = serde_json::from_slice(
&std::fs::read(dir.path().join(".flint/remote.seq")).unwrap(),
)
.unwrap();
assert!(t.sync_requested_unix.is_some(), "the request was not carried to the agent");
assert_eq!(t.sync_requested_by.as_deref(), Some("ci@example"));
assert!(a.load_pending(super::sentinel::Verb::Publish).unwrap().is_none());
}
#[test]
fn every_gauges_field_reaches_exactly_one_metric() {
let g = super::Gauges {
state: "live".into(),
boundary_mode: "gated".into(),
rpo_secs: 11,
visibility_lag_secs: 22,
staged_uncited_count: 3,
staged_uncited_bytes: 4096,
cited_noncurrent_age_max_secs: 33,
withheld_reason: Some("awaiting-boundary".into()),
sentinel_budget_remaining: 44,
forced_citation_count: 5,
last_boundary: Some(super::gauges::LastBoundary {
source: "quiescence".into(),
seq: 66,
unix: 1_756_000_000,
}),
updated_unix: 1_756_000_100,
last_durable_unix: 1_756_000_050,
auth_paused_since_unix: Some(1_756_000_010),
};
let json = serde_json::to_value(&g).unwrap();
let fields: Vec<String> = json.as_object().unwrap().keys().cloned().collect();
assert!(fields.len() >= 13, "the fixture did not populate the struct: {fields:?}");
for f in &fields {
assert!(
super::metrics::COVERED_FIELDS.contains(&f.as_str()),
"gauges field {f:?} reaches no metric — an operator reading /metrics cannot see \
what an operator reading gauges.json can"
);
}
for f in super::metrics::COVERED_FIELDS {
assert!(
fields.iter().any(|x| x == f),
"metric table names {f:?}, which is not a gauges field any more"
);
}
let text = super::metrics::render(
&g,
&super::metrics::Labels { workspace: "proj1".into(), namespace: "agents".into() },
);
for (name, want) in [
("flint_lean_rpo_seconds", 11u64),
("flint_lean_visibility_lag_seconds", 22),
("flint_lean_staged_uncited_objects", 3),
("flint_lean_staged_uncited_bytes", 4096),
("flint_lean_cited_noncurrent_age_max_seconds", 33),
("flint_lean_sentinel_budget_remaining", 44),
("flint_lean_forced_citations_total", 5),
("flint_lean_last_boundary_seq", 66),
("flint_lean_boundary_mode", 2), ("flint_lean_withheld_reason", 2), ("flint_lean_last_boundary_source", 2), ("flint_lean_fenced", 0),
] {
let line = text
.lines()
.find(|l| l.starts_with(name) && !l.starts_with('#'))
.unwrap_or_else(|| panic!("no series for {name}"));
let got: u64 = line.rsplit(' ').next().unwrap().parse().unwrap();
assert_eq!(got, want, "{name} disagrees with the gauges it renders");
}
}
#[test]
fn the_label_key_set_is_exactly_workspace_and_namespace() {
let g = super::Gauges { state: "live".into(), boundary_mode: "hybrid".into(), ..Default::default() };
let text = super::metrics::render(
&g,
&super::metrics::Labels { workspace: "proj1".into(), namespace: "agents".into() },
);
let mut series = 0;
for line in text.lines().filter(|l| !l.starts_with('#')) {
let labels = line
.split_once('{')
.and_then(|(_, r)| r.split_once('}'))
.map(|(l, _)| l)
.unwrap_or_else(|| panic!("unlabelled series: {line}"));
let keys: Vec<&str> =
labels.split(',').map(|kv| kv.split_once('=').unwrap().0).collect();
assert_eq!(
keys,
vec!["workspace", "namespace"],
"series carries label keys beyond the fixed set: {line}"
);
series += 1;
}
assert!(series >= 13, "the renderer emitted almost nothing: {series}");
}
#[tokio::test]
async fn a_scrape_costs_no_bucket_request() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "work.txt", "W1");
a.run_barrier().await.unwrap();
let g = a.write_gauges(false, None).unwrap();
let before = store.list("").await.unwrap().len();
for _ in 0..25 {
let text = super::metrics::render(
&g,
&super::metrics::Labels { workspace: "w".into(), namespace: "n".into() },
);
assert!(text.contains("flint_lean_rpo_seconds"));
}
assert_eq!(
store.list("").await.unwrap().len(),
before,
"25 scrapes changed the bucket"
);
}
#[tokio::test]
async fn status_reports_a_standing_pending_sentinel() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let before = super::status_report(&a.cfg).unwrap();
assert!(before.pending_sentinels.is_empty(), "the fixture started with one pending");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"s-1"}"#);
a.poll_sentinels().unwrap();
assert!(
a.load_pending(super::sentinel::Verb::Publish).unwrap().is_some(),
"the fixture consumed nothing — the status field would be empty either way"
);
let r = super::status_report(&a.cfg).unwrap();
assert_eq!(
r.pending_sentinels,
vec!["publish".to_string()],
"status cannot see the pending record it exists to report"
);
}
#[tokio::test]
async fn a_resumed_checkout_never_adopts_a_stale_generation() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let dir2 = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "big.bin", "GENERATION-ONE");
a.run_barrier().await.unwrap();
write(dir2.path(), "big.bin", "GENERATION-ONE");
write(dir.path(), "big.bin", "GENERATION-TWO-IS-LONGER");
a.run_barrier().await.unwrap();
let cited = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert_eq!(
cited.manifest.entries["big.bin"].size,
"GENERATION-TWO-IS-LONGER".len() as u64,
"the fixture never advanced the manifest"
);
lease::release(&mut a).await.unwrap();
drop(a);
let mut b = syncer(&store, dir2.path()).await;
assert!(claim_until_held(&mut b, 8).await);
assert!(
!b.state.marker_present(),
"the fixture left a completion marker — this is the RESUME row"
);
b.checkout().await.unwrap();
let local = read(dir2.path(), "big.bin").unwrap();
assert_eq!(
local, "GENERATION-TWO-IS-LONGER",
"the resumed checkout adopted the STALE generation: the workspace holds {local:?} while \
the manifest cites the newer bytes, and the baseline claims they are the same — so the \
scan will never upload it and a sync will never re-fetch it"
);
}
#[tokio::test]
async fn every_boundary_says_which_clock_installed_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "one");
sc.floor_tick().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("a.txt"), "the fixture published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("cadence"),
"a cadence boundary does not say it was cadence"
);
write(dir.path(), "b.txt", "two");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"n1"}"#);
sc.sentinel_tick().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("b.txt"), "the sentinel published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("sentinel"),
"a sentinel boundary does not say it was a sentinel"
);
write(dir.path(), "c.txt", "three");
sc.drain().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("c.txt"), "the drain published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("drain"),
"a drain boundary does not say it was a drain"
);
}
#[tokio::test]
async fn a_drained_sentinel_names_the_same_clock_in_the_ack_and_in_the_bucket() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "late.txt", "written before the drain");
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"owed"}"#);
sc.poll_sentinels().unwrap();
assert!(
sc.load_pending(Verb::Publish).unwrap().is_some(),
"no pending record — the drain would not take the owed-ack arm"
);
let acks = sc.drain().await.unwrap();
let ack = acks.iter().find(|a| a.nonces.iter().any(|n| n == "owed")).expect("owed ack unsettled");
assert_eq!(ack.boundary, "drain");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("late.txt"), "the drain published nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("drain"),
"the ack says drain and the bucket says something else"
);
}
#[tokio::test]
async fn a_gated_drain_names_the_same_clock_in_the_ack_and_in_the_bucket() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
gated(&mut sc);
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "staged.txt", "staged before the drain");
sc.upload_lane().await.unwrap();
assert!(
!sc.load_stage().unwrap().entries.is_empty(),
"nothing staged uncited — the gated drain would have nothing to do"
);
touch_sentinel(dir.path(), control::PUBLISH, r#"{"nonce":"owed"}"#);
sc.poll_sentinels().unwrap();
assert!(sc.load_pending(Verb::Publish).unwrap().is_some());
let acks = sc.drain().await.unwrap();
let ack = acks.iter().find(|a| a.nonces.iter().any(|n| n == "owed")).expect("owed ack unsettled");
assert_eq!(ack.boundary, "drain");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(m.entries.contains_key("staged.txt"), "the gated drain cited nothing");
assert_eq!(
m.boundary_source.as_deref(),
Some("drain"),
"the gated drain's ack says drain and the bucket says something else"
);
}
#[tokio::test]
async fn a_long_barrier_does_not_starve_the_lease_renewal() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..600 {
write(dir.path(), &format!("f{i:04}.txt"), &format!("body-{i}"));
}
let key = sc.cfg.epoch_key();
store.backdate_epoch(&key, 120);
let before = store.epoch_read(&key).await.unwrap().unwrap().last_renew_unix;
sc.run_barrier().await.unwrap();
let after = store.epoch_read(&key).await.unwrap().unwrap().last_renew_unix;
assert!(
after > before,
"the barrier ran to completion without renewing an overdue lease \
(last_renew {before:?} -> {after:?}); a barrier longer than the \
60 s takeover window would hand the lease to a standby while this \
writer was perfectly alive"
);
}
#[tokio::test]
async fn a_stage_that_outlived_its_citation_must_not_reap_the_cited_version() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
gated(&mut a);
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
a.upload_lane().await.unwrap();
let pre_clear = a.load_stage().unwrap();
let staged_vid = pre_clear.entries["f.txt"].version_id.clone().unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited_vid = installed.entries["f.txt"].version_id.clone().unwrap();
assert_eq!(
cited_vid, staged_vid,
"precondition: the citation named the version the lane staged"
);
a.save_stage(&pre_clear).unwrap();
write(dir.path(), "f.txt", "v2 — the agent keeps working");
let lane = a.upload_lane().await.unwrap();
assert!(
lane.staged.contains(&"f.txt".to_string()),
"the second lane pass never re-staged f.txt (staged={:?} parked={:?}) — \
this leg never reached the reclaim branch",
lane.staged, lane.parked
);
assert_eq!(
lane.superseded_recorded, 1,
"the lane did not reach the branch under test at all — it neither recorded \
nor reclaimed, so this leg proves nothing"
);
let still = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
still.entries["f.txt"].version_id.as_ref(),
Some(&cited_vid),
"precondition: no citation ran, so the installed boundary still cites v1"
);
store
.get_version(&a.cfg.file_key("f.txt"), &cited_vid)
.await
.expect("the lane reaped the version the INSTALLED MANIFEST CITES");
}
#[tokio::test]
async fn the_drain_never_reclaims_a_recorded_version_the_boundary_still_cites() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
gated(&mut a);
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "m.txt", "generation one");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let installed = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
let cited = installed.entries["m.txt"].version_id.clone().unwrap();
write(dir.path(), "m.txt", "generation two, longer than the first");
a.upload_lane().await.unwrap();
let mut stage = a.load_stage().unwrap();
stage.pending_reclaims.insert("m.txt".to_string(), vec![cited.clone()]);
let versions = store.list_versions(&a.cfg.file_key("m.txt")).await.unwrap();
let v = versions
.iter()
.find(|v| v.version_id == cited)
.expect("the cited version is not in the listing — the pass would skip it anyway");
assert!(
!v.is_current,
"the cited version is still CURRENT, so `is_current` protects it and this leg \
cannot isolate the `keep` guard"
);
assert!(
installed.entries.contains_key("m.txt"),
"the boundary does not cite this path, so `keep` is None and the guard is moot"
);
let upserts = std::collections::BTreeMap::new();
let mut report = Default::default();
a.reclaim_superseded(&upserts, &installed, &stage, &mut report).await.unwrap();
assert_eq!(
report.versions_reclaimed, 0,
"the drain reclaimed a version the installed boundary CITES"
);
store
.get_version(&a.cfg.file_key("m.txt"), &cited)
.await
.expect("the drain deleted the version the installed manifest CITES");
}
#[tokio::test]
async fn sync_refuses_to_clobber_a_locally_dirty_path_and_says_so() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "dirty.txt", "published v1");
write(dir.path(), "clean.txt", "published v1");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
for (path, content) in
[("dirty.txt", "foreign bytes for dirty"), ("clean.txt", "foreign bytes for clean")]
{
let key = a.cfg.file_key(path);
let body = Bytes::from(content.to_string());
let crc = crc64_nvme(&body);
let cur = store.head(&key).await.unwrap();
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfMatch(cur.etag),
&GenerationStamps {
generation: 2,
epoch: 0,
flush_uuid: "sibling".into(),
boundary_source: None,
posix: None,
},
crc,
)
.await
.unwrap();
let e = theirs.entries.get_mut(path).unwrap();
e.etag = meta.etag.clone();
e.crc64_b64 = meta.crc64_b64.clone().unwrap();
e.size = meta.size;
e.generation = 2;
}
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "sibling")
.await
.unwrap();
write(dir.path(), "dirty.txt", "the agent's own unpublished work, longer");
let r = a.sync().await.unwrap();
assert_eq!(
read(dir.path(), "clean.txt").unwrap(),
"foreign bytes for clean",
"the control path was not applied — this sync did nothing at all"
);
assert!(r.applied.contains(&"clean.txt".to_string()));
assert_ne!(
read(dir.path(), "dirty.txt").unwrap(),
"foreign bytes for dirty",
"local and remote bytes are identical — nothing was in conflict"
);
assert_eq!(
read(dir.path(), "dirty.txt").unwrap(),
"the agent's own unpublished work, longer",
"sync CLOBBERED work the agent had not published"
);
assert!(
r.conflicts.contains(&"dirty.txt".to_string()),
"the conflict was not surfaced in the report: {:?}",
r.conflicts
);
let conflicts = a.state.load_conflicts().unwrap();
assert!(
conflicts.iter().any(|c| c.kind == "sync-dirty" && c.path == "dirty.txt"),
"no sync-dirty record was written: {conflicts:?}"
);
}
#[tokio::test]
async fn merge_applies_a_local_delete_only_where_theirs_is_unchanged() {
fn entry(etag: &str) -> manifest::LeanEntry {
manifest::LeanEntry {
key: "tenant/proj1/files/p.txt".into(),
etag: etag.into(),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 3,
mode: 0o644,
mtime_unix: 0,
generation: 1,
epoch: 0,
version_id: None,
}
}
let cases: Vec<(&str, Option<&str>, Option<&str>, bool, bool)> = vec![
("theirs unchanged since our base", Some("e1"), Some("e1"), false, false),
("FOREIGN MODIFY under our delete", Some("e2"), Some("e1"), false, true),
("FOREIGN ADD under our delete", Some("e2"), None, false, true),
("absent from both", None, None, false, false),
("already deleted by someone else", None, Some("e1"), false, false),
("parked: never resolved this pass", Some("e1"), Some("e1"), true, true),
];
for (name, theirs_etag, base_etag, is_parked, must_survive) in cases {
let mut theirs = manifest::LeanManifest::default();
if let Some(e) = theirs_etag {
theirs.entries.insert("p.txt".to_string(), entry(e));
}
let mut base = std::collections::BTreeMap::new();
if let Some(b) = base_etag {
base.insert("p.txt".to_string(), b.to_string());
}
let mut deletes = std::collections::BTreeSet::new();
deletes.insert("p.txt".to_string());
let mut parked = std::collections::BTreeSet::new();
if is_parked {
parked.insert("p.txt".to_string());
}
assert_eq!(theirs.entries.contains_key("p.txt"), theirs_etag.is_some(), "{name}");
assert_eq!(base.contains_key("p.txt"), base_etag.is_some(), "{name}");
let (merged, _foreign) =
manifest::merge(&base, &theirs, &Default::default(), &deletes, &parked);
assert_eq!(
merged.entries.contains_key("p.txt"),
must_survive,
"{name}: our local delete {} the entry",
if must_survive { "destroyed" } else { "failed to remove" }
);
if must_survive {
if let Some(e) = theirs_etag {
assert_eq!(merged.entries["p.txt"].etag, e, "{name}: wrong bytes survived");
}
}
}
}
#[tokio::test]
async fn a_resumed_checkout_adopts_identical_bytes_and_refetches_same_size_impostors() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "same.txt", "these bytes never move");
write(dir.path(), "impostor.txt", "AAAA");
a.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
let key = a.cfg.file_key("impostor.txt");
let body = Bytes::from_static(b"BBBB");
let meta = store
.put_whole(
&key,
body,
&PutCondition::IfMatch(store.head(&key).await.unwrap().etag),
&GenerationStamps {
generation: 2,
epoch: 0,
flush_uuid: "sibling".into(),
boundary_source: None,
posix: None,
},
crc64_nvme(&Bytes::from_static(b"BBBB")),
)
.await
.unwrap();
let e = theirs.entries.get_mut("impostor.txt").unwrap();
e.etag = meta.etag.clone();
e.crc64_b64 = meta.crc64_b64.clone().unwrap();
e.size = meta.size;
e.generation = 2;
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "sibling")
.await
.unwrap();
assert_eq!(
theirs.entries["impostor.txt"].size,
std::fs::metadata(dir.path().join("impostor.txt")).unwrap().len(),
"the impostor differs in SIZE — this leg would pass without any crc at all"
);
assert_eq!(read(dir.path(), "impostor.txt").unwrap(), "AAAA");
std::fs::remove_file(dir.path().join(".flint-sync/checkout-complete")).unwrap();
let r = a.checkout().await.unwrap();
assert!(
r.skipped_present >= 1,
"nothing was adopted — the leg degenerates into 'checkout re-fetches everything'"
);
assert_eq!(
read(dir.path(), "same.txt").unwrap(),
"these bytes never move",
"an identical file was not left alone"
);
assert_eq!(
read(dir.path(), "impostor.txt").unwrap(),
"BBBB",
"checkout ADOPTED a same-size file whose bytes are not the cited bytes — \
the baseline now attests content the workspace does not hold"
);
}
#[tokio::test]
async fn sync_under_pinned_reads_resolves_the_cited_version_not_the_current_one() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
gated(&mut a);
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "d.txt", "cited generation one");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let key = a.cfg.file_key("d.txt");
let loaded = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(loaded.manifest.pinned_reads, "the gated citation did not pin reads");
let v2_body = Bytes::from_static(b"cited generation TWO");
let v2 = store
.put_whole(
&key,
v2_body.clone(),
&PutCondition::IfMatch(store.head(&key).await.unwrap().etag),
&GenerationStamps {
generation: 2,
epoch: 0,
flush_uuid: "sibling".into(),
boundary_source: None,
posix: None,
},
crc64_nvme(&v2_body),
)
.await
.unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
theirs.pinned_reads = true;
{
let e = theirs.entries.get_mut("d.txt").unwrap();
e.etag = v2.etag.clone();
e.crc64_b64 = v2.crc64_b64.clone().unwrap();
e.size = v2.size;
e.generation = 2;
e.version_id = v2.version_id.clone();
}
manifest::cas_write(store.as_ref(), &a.cfg, &theirs, Some(&loaded.handle()), 0, "sibling")
.await
.unwrap();
let v3_body = Bytes::from_static(b"uncited generation three, staged");
let v3 = store
.put_whole(
&key,
v3_body.clone(),
&PutCondition::IfMatch(v2.etag.clone()),
&GenerationStamps {
generation: 3,
epoch: 0,
flush_uuid: "sibling".into(),
boundary_source: None,
posix: None,
},
crc64_nvme(&v3_body),
)
.await
.unwrap();
let versions = store.list_versions(&key).await.unwrap();
let cited_vid = theirs.entries["d.txt"].version_id.clone().unwrap();
assert!(
versions.iter().any(|v| v.version_id == v3.version_id.clone().unwrap() && v.is_current),
"the uncited generation is not current — the two reads would agree"
);
assert_ne!(cited_vid, v3.version_id.clone().unwrap());
let r = a.sync().await.unwrap();
assert!(
r.applied.contains(&"d.txt".to_string()),
"sync applied nothing — it took the 412 path and left the workspace behind: {r:?}"
);
assert_eq!(
read(dir.path(), "d.txt").unwrap(),
"cited generation TWO",
"sync did not land the CITED bytes"
);
assert_ne!(
read(dir.path(), "d.txt").unwrap(),
"uncited generation three, staged",
"sync served the CURRENT version over the cited one — D13's promise inverted"
);
}
#[tokio::test]
async fn measure_backlog_cap_footprint() {
for hot in [1usize, 10, 100] {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
gated(&mut a);
a.cfg.visibility_lag_bound_secs = Some(86_400);
a.cfg.quiesce_bound_secs = 86_400;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
const TICKS: usize = 12;
let mut lane_ops = 0u64;
let mut lane_shape = Default::default();
let mut recorded_at = vec![];
for t in 0..TICKS {
for h in 0..hot {
write(dir.path(), &format!("hot{h:03}.txt"), &"x".repeat(8 + t * 3 + h));
}
store.reset_op_counts();
a.upload_lane().await.unwrap();
lane_ops += store.total_ops();
if t == TICKS - 1 {
lane_shape = store.op_counts();
}
let st = a.load_stage().unwrap();
let rec: usize = st.pending_reclaims.values().map(|v| v.len()).sum();
recorded_at.push((st.entries.len(), rec));
}
store.reset_op_counts();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let cite_ops = store.total_ops();
let cite_shape = store.op_counts();
let (entries, rec_last) = *recorded_at.last().unwrap();
let rate = rec_last as f64 / (TICKS - 1) as f64;
let cap = 5000f64; let old_ticks = if (entries as f64) >= cap { 1.0 } else { f64::INFINITY };
let new_ticks = if rate > 0.0 { (cap - entries as f64) / rate } else { f64::INFINITY };
eprintln!(
"\nhot paths = {hot}\n \
stage entries after {TICKS} ticks : {entries}\n \
recorded reclaims : {rec_last} (measured {rate:.2}/tick)\n \
lane ops/tick : {:.1} {:?}\n \
citation ops : {cite_ops} {:?}\n \
ticks to forced citation, OLD : {}\n \
ticks to forced citation, NEW : {:.0}",
lane_ops as f64 / TICKS as f64,
lane_shape,
cite_shape,
if old_ticks.is_finite() { "1".to_string() } else { "never".to_string() },
new_ticks,
);
}
}
#[tokio::test]
async fn a_takeover_rotation_carries_the_boundary_stamp_with_the_document() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "f.txt", "v1");
sc.run_barrier().await.unwrap();
let loaded = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
manifest::cas_write_stamped(
store.as_ref(),
&sc.cfg,
&loaded.manifest,
Some(&loaded.handle()),
1,
"u-cited",
Some("sentinel"),
)
.await
.unwrap();
let lp = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(
lp.pointer.boundary_source.as_deref(),
Some("sentinel"),
"precondition: the cited manifest names its clock on the pointer"
);
manifest::rotate_for_takeover(store.as_ref(), &sc.cfg, 2).await.unwrap().unwrap();
let lp = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let doc = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(
lp.pointer.boundary_source.as_deref(),
Some("sentinel"),
"rotation must not drop the boundary stamp: a pointer reader would report an \
unknown clock for a workspace whose document still says `sentinel`"
);
assert_eq!(
lp.pointer.boundary_source, doc.manifest.boundary_source,
"the pointer and the document a reader assembles from it must never disagree"
);
assert_eq!(lp.pointer.seq, doc.manifest.seq, "the pointer's seq IS the document's seq");
}
fn entry(name: &str) -> manifest::LeanEntry {
manifest::LeanEntry {
key: format!("{PREFIX}/files/{name}"),
etag: "e".into(),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 1,
mode: 0o100644,
mtime_unix: 0,
generation: 1,
epoch: 1,
version_id: None,
}
}
#[tokio::test]
async fn a_publish_writes_an_immutable_generation_and_a_pointer_that_names_it() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 1;
m.entries.insert("a.txt".into(), entry("x"));
manifest::cas_write(store.as_ref(), &cfg, &m, None, 3, "first").await.unwrap();
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(lp.pointer.seq, 1);
assert_eq!(lp.pointer.entries_seq, Some(1));
assert_eq!(lp.pointer.entries_key, lp.pointer.entries_key.clone());
assert!(store.head(lp.pointer.entries_key.as_deref().unwrap()).await.is_ok());
assert!(store.head(&cfg.manifest_key()).await.is_err());
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(loaded.manifest.seq, 1);
assert!(loaded.manifest.entries.contains_key("a.txt"));
assert!(!loaded.handle().legacy);
}
#[tokio::test]
async fn a_takeover_rotation_does_not_touch_the_entries_object() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 4;
for i in 0..50 {
m.entries.insert(format!("f{i:03}.txt"), entry("x"));
}
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "seed").await.unwrap();
let gen4 = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
let before = store.head(&gen4).await.unwrap();
let (rotated, _) = manifest::rotate_for_takeover(store.as_ref(), &cfg, 2).await.unwrap().unwrap();
assert_eq!(rotated.seq, 5, "rotation must bump the generation");
let after = store.head(&gen4).await.unwrap();
assert_eq!(before.etag, after.etag, "the entries object was rewritten by a rotation");
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(lp.pointer.seq, 5);
assert_eq!(lp.pointer.entries_seq, Some(4), "entries_seq must NOT move — a follower reads it to skip the GET");
assert_eq!(lp.pointer.entries_key.as_deref(), Some(gen4.as_str()));
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(loaded.manifest.entries.len(), 50);
assert_eq!(loaded.manifest.seq, 5, "the POINTER is the authority for seq, not the entries object");
}
#[tokio::test]
async fn migration_installs_the_pointer_and_poisons_the_legacy_key() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let mut old = manifest::LeanManifest::default();
old.seq = 7;
old.entries.insert("kept.txt".into(), entry("x"));
let bytes = old.to_bytes();
let crc = flint_store::crc64_nvme(&bytes);
let stamps = flint_store::GenerationStamps {
generation: 7,
epoch: 1,
flush_uuid: "legacy".into(),
boundary_source: None,
posix: None,
};
store
.put_whole(&cfg.manifest_key(), bytes.into(), &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert!(loaded.handle().legacy, "an un-migrated workspace must be recognised as legacy");
assert_eq!(loaded.manifest.seq, 7);
let mut next = loaded.manifest.clone();
next.seq += 1;
next.entries.insert("added.txt".into(), entry("x"));
manifest::cas_write(store.as_ref(), &cfg, &next, Some(&loaded.handle()), 2, "migrate")
.await
.unwrap();
let after = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert!(!after.handle().legacy);
assert_eq!(after.manifest.entries.len(), 2, "migration must not lose entries");
let (_, poisoned) = store.get_whole(&cfg.manifest_key(), None).await.unwrap();
assert!(
manifest::LeanManifest::parse(&poisoned).is_err(),
"the legacy key still parses as a manifest — an old syncer would serve a stale project from it"
);
assert!(String::from_utf8_lossy(&poisoned).contains("upgrade flint-sync"));
}
#[tokio::test]
async fn a_pointer_naming_a_missing_generation_refuses_rather_than_reading_empty() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 2;
m.entries.insert("a.txt".into(), entry("x"));
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "seed").await.unwrap();
let gen2 = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
store.delete(&gen2).await.unwrap();
match manifest::load(store.as_ref(), &cfg).await {
Err(LeanError::State(msg)) => {
assert!(msg.contains("does not exist"), "unexpected message: {msg}");
}
Ok(None) => panic!("a broken pointer read as an EMPTY workspace — the next barrier would re-seed over it"),
Ok(Some(_)) => panic!("a broken pointer read as a LOADED manifest"),
Err(e) => panic!("expected a State refusal, got {e:?}"),
}
}
#[tokio::test]
async fn a_second_writer_at_the_same_generation_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 1;
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "a").await.unwrap();
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
let mut mine = loaded.manifest.clone();
mine.seq += 1;
manifest::cas_write(store.as_ref(), &cfg, &mine, Some(&loaded.handle()), 1, "b").await.unwrap();
let mut theirs = loaded.manifest.clone();
theirs.seq += 1;
let err = manifest::cas_write(store.as_ref(), &cfg, &theirs, Some(&loaded.handle()), 1, "c")
.await
.unwrap_err();
assert!(
matches!(err, LeanError::Store(flint_store::StoreError::PreconditionFailed(_))),
"expected a precondition failure, got {err:?}"
);
}
#[tokio::test]
async fn superseded_generations_are_reaped_but_the_live_one_and_a_window_survive() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let prefix = format!("{PREFIX}/.flint/lean/manifests/");
let mut handle = None;
for seq in 1..=12u64 {
let mut m = manifest::LeanManifest::default();
m.seq = seq;
m.entries.insert("a.txt".into(), entry("a.txt"));
let meta = manifest::cas_write(
store.as_ref(),
&cfg,
&m,
handle.as_ref(),
1,
&format!("flush-{seq}"),
)
.await
.unwrap();
handle = Some(manifest::ManifestHandle { etag: meta.etag, legacy: false, prev_chunks: Vec::new() });
}
assert_eq!(store.list(&prefix).await.unwrap().len(), 12);
let removed = manifest::sweep_generations(store.as_ref(), &cfg).await.unwrap();
let left = store.list(&prefix).await.unwrap();
assert_eq!(left.len(), manifest::KEEP_GENERATIONS + 1);
assert_eq!(removed, 12 - (manifest::KEEP_GENERATIONS + 1));
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
assert!(
left.iter().any(|o| Some(&o.key) == lp.pointer.entries_key.as_ref()),
"the reaper deleted the generation the pointer names"
);
let loaded = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(loaded.manifest.seq, 12);
assert!(loaded.manifest.entries.contains_key("a.txt"));
}
#[tokio::test]
async fn the_reaper_collects_an_orphan_from_a_crash_before_the_pointer_cas() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 1;
m.entries.insert("a.txt".into(), entry("a.txt"));
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "live").await.unwrap();
let live = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
let bytes = m.to_bytes();
let crc = flint_store::crc64_nvme(&bytes);
let stamps = flint_store::GenerationStamps {
generation: 99,
epoch: 1,
flush_uuid: "crashed".into(),
boundary_source: None,
posix: None,
};
let orphan = cfg.generation_key(99, "crashed");
store.put_whole(&orphan, bytes.into(), &PutCondition::IfNoneMatchAny, &stamps, crc).await.unwrap();
assert_eq!(
manifest::sweep_generations(store.as_ref(), &cfg).await.unwrap(),
0,
"the sweep reaped a generation above the pointer that could still be an in-flight publish"
);
assert!(store.head(&orphan).await.is_ok());
store.backdate_epoch(&orphan, manifest::ORPHAN_GRACE_SECS + 60);
assert_eq!(manifest::sweep_generations(store.as_ref(), &cfg).await.unwrap(), 1);
let left: Vec<String> = store
.list(&format!("{PREFIX}/.flint/lean/manifests/"))
.await
.unwrap()
.into_iter()
.map(|o| o.key)
.collect();
assert!(!left.contains(&orphan), "the orphan survived the sweep");
assert!(left.contains(&live), "the LIVE generation was reaped");
assert!(manifest::load(store.as_ref(), &cfg).await.unwrap().is_some());
}
#[tokio::test]
async fn a_rotation_reads_and_writes_no_generation_object() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_single(dir.path());
let mut m = manifest::LeanManifest::default();
m.seq = 3;
for i in 0..200 {
m.entries.insert(format!("f{i:04}.txt"), entry("f"));
}
manifest::cas_write(store.as_ref(), &cfg, &m, None, 1, "seed").await.unwrap();
let gen = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer.entries_key.unwrap();
let before = store.head(&gen).await.unwrap();
store.reset_op_counts();
manifest::rotate_for_takeover(store.as_ref(), &cfg, 2).await.unwrap().unwrap();
let ops = store.op_counts();
assert!(
store.total_ops() <= 3,
"a rotation should be a couple of small requests, not {ops:?}"
);
let after = store.head(&gen).await.unwrap();
assert_eq!(before.etag, after.etag);
assert_eq!(
store.list(&format!("{PREFIX}/.flint/lean/manifests/")).await.unwrap().len(),
1,
"a rotation wrote a new generation object — it must reuse the standing one"
);
}
struct AuthRefusing {
inner: Arc<MemoryStore>,
refuse: std::sync::atomic::AtomicBool,
}
impl AuthRefusing {
fn new(inner: Arc<MemoryStore>) -> Self {
Self { inner, refuse: std::sync::atomic::AtomicBool::new(false) }
}
fn set_refuse(&self, v: bool) {
self.refuse.store(v, std::sync::atomic::Ordering::SeqCst);
}
}
#[async_trait::async_trait]
impl ObjectStore for AuthRefusing {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
if self.refuse.load(std::sync::atomic::Ordering::SeqCst) {
return Err(flint_store::StoreError::Auth(
"ExpiredToken: the security token included in the request is expired".into(),
));
}
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_refused_credential_pauses_the_holder_without_fencing_it() {
let inner = Arc::new(MemoryStore::new());
let proxy = Arc::new(AuthRefusing::new(inner.clone()));
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store: proxy.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_none(),
"a healthy holder started out reading as credential-paused"
);
proxy.set_refuse(true);
let e = lease::renew(&mut a).await.unwrap_err();
assert!(e.is_auth(), "a 403 renewal did not classify as a credential fault: {e}");
assert!(
!matches!(e, LeanError::Fenced(_)),
"a credential refusal self-fenced a live writer: {e}"
);
assert!(a.lease.is_some(), "the refusal dropped the lease");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_some(),
"the refusal left no local evidence at all"
);
let gp = a.cfg.state_dir().join("gauges.json");
let mut g: serde_json::Value = serde_json::from_slice(&std::fs::read(&gp).unwrap()).unwrap();
g["auth_paused_since_unix"] = serde_json::json!(1_000_u64);
std::fs::write(&gp, serde_json::to_vec(&g).unwrap()).unwrap();
lease::renew(&mut a).await.unwrap_err();
assert_eq!(
a.load_gauges().unwrap().auth_paused_since_unix,
Some(1_000),
"a second refusal slid the pause clock forward, so the gauge no \
longer answers the question it exists to answer"
);
a.write_gauges(false, None).unwrap();
assert_eq!(
a.load_gauges().unwrap().auth_paused_since_unix,
Some(1_000),
"an ordinary gauge tick erased the credential pause"
);
proxy.set_refuse(false);
lease::renew(&mut a).await.expect("renew after the credentials came back");
assert!(
a.load_gauges().unwrap().auth_paused_since_unix.is_none(),
"the pause outlived the credentials being restored"
);
}
#[test]
fn is_auth_reads_through_the_from_conversion() {
fn via_question_mark() -> super::LeanResult<()> {
Err(flint_store::StoreError::Auth("ExpiredToken".into()))?;
Ok(())
}
assert!(via_question_mark().unwrap_err().is_auth());
assert!(!LeanError::State("x".into()).is_auth());
assert!(!LeanError::Fenced("x".into()).is_auth());
let other: LeanError = flint_store::StoreError::Other("boom".into()).into();
assert!(!other.is_auth(), "an ordinary store error read as a credential fault");
let pf: LeanError = flint_store::StoreError::PreconditionFailed("412".into()).into();
assert!(!pf.is_auth(), "a deposal read as a credential fault");
}
fn chunks_of(keys: &[String]) -> Vec<Vec<String>> {
let refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect();
let cuts = super::chunk::chunk_ranges_with(&refs, 8, 2, 32);
let mut out = Vec::new();
let mut start = 0;
for c in cuts {
out.push(keys[start..c].to_vec());
start = c;
}
out
}
fn fixed_chunks_of(keys: &[String], n: usize) -> Vec<Vec<String>> {
keys.chunks(n).map(|c| c.to_vec()).collect()
}
fn changed_chunks(a: &[Vec<String>], b: &[Vec<String>]) -> usize {
let before: std::collections::HashSet<_> = a.iter().collect();
b.iter().filter(|c| !before.contains(c)).count()
}
#[test]
fn a_front_insert_rewrites_one_chunk_not_the_project() {
let base: Vec<String> = (0..300).map(|i| format!("src/f{i:04}.txt")).collect();
let mut inserted = base.clone();
inserted.insert(0, "src/a000-brand-new.txt".to_string());
assert!(inserted.windows(2).all(|w| w[0] < w[1]), "fixture is not sorted");
let c0 = chunks_of(&base);
let c1 = chunks_of(&inserted);
assert!(c0.len() > 4, "fixture produced {} chunks — too few to say anything", c0.len());
let cd = changed_chunks(&c0, &c1);
let f0 = fixed_chunks_of(&base, 8);
let f1 = fixed_chunks_of(&inserted, 8);
let fd = changed_chunks(&f0, &f1);
assert!(
fd >= f0.len(),
"the control did not exhibit the cascade it exists to demonstrate ({fd} of {} chunks \
changed), so the assertion below proves nothing about this fixture",
f0.len()
);
assert!(
cd <= 2,
"a front insert rewrote {cd} of {} content-defined chunks (fixed-count rewrote {fd}) — \
boundaries are moving with POSITION, which is the failure mode §3 disqualifies",
c0.len()
);
}
#[test]
fn an_edit_far_from_a_chunk_leaves_it_alone() {
let base: Vec<String> = (0..300).map(|i| format!("src/f{i:04}.txt")).collect();
let mut edited = base.clone();
edited.insert(150, "src/f0149-inserted.txt".to_string());
edited.sort();
let c0 = chunks_of(&base);
let c1 = chunks_of(&edited);
assert_eq!(c0.first(), c1.first(), "the FIRST chunk moved for a change in the middle");
assert_eq!(c0.last(), c1.last(), "the LAST chunk moved for a change in the middle");
assert!(
changed_chunks(&c0, &c1) <= 2,
"a middle insert rewrote {} chunks",
changed_chunks(&c0, &c1)
);
}
#[test]
fn chunk_sizing_respects_min_max_and_keeps_the_tail() {
let keys: Vec<String> = (0..500).map(|i| format!("k{i:05}")).collect();
let refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect();
let cuts = super::chunk::chunk_ranges_with(&refs, 8, 3, 20);
assert_eq!(*cuts.last().unwrap(), keys.len(), "the tail run was dropped");
let mut prev = 0;
for (i, c) in cuts.iter().enumerate() {
let len = c - prev;
assert!(len <= 20, "chunk {i} has {len} entries, over the max of 20");
if i + 1 < cuts.len() {
assert!(len >= 3, "chunk {i} has {len} entries, under the min of 3");
}
prev = *c;
}
let total: usize = cuts.iter().scan(0, |p, c| { let l = c - *p; *p = *c; Some(l) }).sum();
assert_eq!(total, keys.len(), "chunks do not partition the key stream");
}
#[test]
fn chunk_boundaries_are_stable_and_content_addresses_are_not_crc() {
let keys: Vec<String> = (0..64).map(|i| format!("src/f{i:03}.txt")).collect();
let refs: Vec<&str> = keys.iter().map(|s| s.as_str()).collect();
let a = super::chunk::chunk_ranges_with(&refs, 8, 2, 32);
let b = super::chunk::chunk_ranges_with(&refs, 8, 2, 32);
assert_eq!(a, b, "chunking is not deterministic");
assert!(!a.is_empty() && *a.last().unwrap() == 64);
let x = super::chunk::chunk_address(b"{\"a\":1}");
let y = super::chunk::chunk_address(b"{\"a\":2}");
assert_ne!(x, y, "two different chunk bodies share an address");
assert_eq!(x.len(), 32, "address is not 128 bits of hex");
assert_eq!(x, super::chunk::chunk_address(b"{\"a\":1}"), "address is not stable");
}
fn entry_at(k: &str, seq: u64) -> super::manifest::LeanEntry {
super::manifest::LeanEntry {
key: format!("files/{k}"),
etag: format!("e-{k}-{seq}"),
crc64_b64: "AAAAAAAAAAA=".into(),
size: 10,
mode: 0o644,
mtime_unix: 1_700_000_000,
generation: seq,
epoch: 1,
version_id: None,
}
}
fn manifest_of(n: usize, seq: u64) -> super::manifest::LeanManifest {
super::manifest::LeanManifest {
seq,
entries: (0..n).map(|i| (format!("src/f{i:05}.txt"), entry_at(&format!("f{i:05}"), seq))).collect(),
pinned_reads: false,
sole_writer: false,
boundary_source: None,
}
}
#[tokio::test]
async fn a_three_file_publish_writes_chunks_proportional_to_the_change() {
let inner = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
let m0 = manifest_of(4000, 1);
inner.reset_op_counts();
let meta = manifest::cas_write_chunked(store.as_ref(), &cfg, &m0, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await
.expect("first chunked publish");
let first_puts = *inner.op_counts().get("put_whole").unwrap_or(&0);
let p0 = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer;
let chunks0 = match p0.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("the pointer is not chunked"),
};
assert!(chunks0.len() >= 4, "fixture made {} chunks — too few to measure", chunks0.len());
assert!(
first_puts as usize >= chunks0.len(),
"the first publish wrote {first_puts} objects for {} chunks — it cannot have written \
them all, so the comparison below is against nothing",
chunks0.len()
);
let mut m1 = m0.clone();
m1.seq = 2;
for i in [7usize, 1500, 3900] {
m1.entries.insert(format!("src/f{i:05}.txt"), entry_at(&format!("f{i:05}"), 99));
}
let h = super::manifest::ManifestHandle { etag: meta.etag.clone(), legacy: false, prev_chunks: Vec::new() };
inner.reset_op_counts();
manifest::cas_write_chunked(store.as_ref(), &cfg, &m1, Some(&h), &chunks0,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await
.expect("incremental chunked publish");
let puts = *inner.op_counts().get("put_whole").unwrap_or(&0);
eprintln!(
"chunked publish: {} chunks; full publish {first_puts} objects, 3-file publish {puts}",
chunks0.len()
);
assert!(
puts * 5 < first_puts,
"a three-file publish wrote {puts} objects against the full publish's {first_puts} \
over {} chunks — untouched chunks are being rewritten, which is O(entries) again",
chunks0.len()
);
assert!(puts <= 6, "a three-file publish wrote {puts} objects");
let back = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(back.manifest.entries.len(), 4000, "entries were lost across the chunked publish");
assert_eq!(back.manifest.entries, m1.entries, "the assembled manifest is not what was published");
}
#[tokio::test]
async fn a_missing_chunk_refuses_rather_than_shortening_the_manifest() {
let inner = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
let m = manifest_of(2000, 1);
manifest::cas_write_chunked(store.as_ref(), &cfg, &m, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None }).await.unwrap();
let p = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap().pointer;
let chunks = match p.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
assert!(chunks.len() >= 2);
store.delete(&cfg.chunk_key(&chunks[chunks.len() / 2].addr)).await.unwrap();
let msg = match manifest::load(store.as_ref(), &cfg).await {
Err(e) => e.to_string(),
Ok(m) => panic!(
"a manifest with a missing chunk LOADED, with {} entries",
m.map(|l| l.manifest.entries.len()).unwrap_or(0)
),
};
assert!(
msg.contains("hole"),
"a manifest with a missing chunk did not refuse; it said: {msg}"
);
}
#[test]
fn a_pointer_naming_both_layouts_or_neither_is_refused() {
let mut p = super::manifest::Pointer {
seq: 3,
entries_key: Some("k".into()),
entries_seq: Some(3),
chunks: Some(vec![]),
pinned_reads: false,
sole_writer: false,
boundary_source: None,
epoch: 1,
};
assert!(p.entries().unwrap_err().to_string().contains("BOTH"));
p.entries_key = None;
p.chunks = None;
assert!(p.entries().unwrap_err().to_string().contains("no entries at all"));
p.chunks = Some(vec![]);
assert!(matches!(p.entries().unwrap(), super::manifest::Entries::Chunked(c) if c.is_empty()));
}
#[test]
fn assemble_refuses_every_way_a_chunk_list_can_lie() {
let entries: std::collections::BTreeMap<String, super::manifest::LeanEntry> =
(0..40).map(|i| (format!("k{i:03}"), entry_at(&format!("k{i:03}"), 1))).collect();
let split = super::chunk::split_with(&entries, 4, 2, 8).unwrap();
assert!(split.len() >= 3, "fixture made {} chunks", split.len());
let refs: Vec<_> = split.iter().map(|(r, _)| r.clone()).collect();
let bodies: Vec<Vec<u8>> = split.iter().map(|(_, b)| b.clone()).collect();
let ok = super::chunk::assemble(&refs, &bodies).unwrap();
assert_eq!(ok, entries, "assemble does not round-trip its own split");
let must_fail = |r: Vec<super::chunk::ChunkRef>, b: Vec<Vec<u8>>, why: &str| {
assert!(
super::chunk::assemble(&r, &b).is_err(),
"assemble accepted a chunk list that {why}"
);
};
let mut r = refs.clone();
r[1].n += 1;
must_fail(r, bodies.clone(), "claims more entries than its body holds");
let mut b = bodies.clone();
b[1] = bodies[2].clone();
must_fail(refs.clone(), b, "returns a body that is not the addressed one");
let mut r = refs.clone();
let mut b = bodies.clone();
r.swap(0, 1);
b.swap(0, 1);
must_fail(r, b, "is not in increasing key order");
must_fail(refs.clone(), bodies[..bodies.len() - 1].to_vec(), "names more chunks than were fetched");
let mut r = refs.clone();
r[1].first = "zzz".into();
must_fail(r, bodies.clone(), "disagrees with its body about the first key");
}
struct SweepMidRead {
inner: Arc<MemoryStore>,
current_key: String,
doomed_key: String,
next_pointer: Vec<u8>,
move_pointer: bool,
fired: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl ObjectStore for SweepMidRead {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
if key == self.doomed_key
&& !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst)
{
if self.move_pointer {
let stamps = GenerationStamps {
generation: 9,
epoch: 1,
flush_uuid: "sweep".into(),
boundary_source: None,
posix: None,
};
let crc = crc64_nvme(&self.next_pointer);
self.inner
.put_whole(
&self.current_key,
Bytes::from(self.next_pointer.clone()),
&PutCondition::Unconditional,
&stamps,
crc,
)
.await?;
}
self.inner.delete(&self.doomed_key).await?;
return Err(flint_store::StoreError::NotFound(format!("swept: {key}")));
}
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_reader_swept_mid_read_restarts_onto_the_current_generation() {
for move_pointer in [true, false] {
let inner = Arc::new(MemoryStore::new());
let plain: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
let m1 = manifest_of(600, 1);
let meta1 =
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await
.unwrap();
let p1 = manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap().pointer;
let c1 = match p1.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new() };
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await
.unwrap();
let lp2 = manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap();
let ptr2_bytes = serde_json::to_vec(&lp2.pointer).unwrap();
let stamps = GenerationStamps {
generation: 1,
epoch: 1,
flush_uuid: "rewind".into(),
boundary_source: None,
posix: None,
};
let p1_bytes = serde_json::to_vec(&p1).unwrap();
let crc = crc64_nvme(&p1_bytes);
plain
.put_whole(&cfg.current_key(), Bytes::from(p1_bytes), &PutCondition::Unconditional, &stamps, crc)
.await
.unwrap();
let c2: Vec<String> = match lp2.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("not chunked"),
};
let doomed = c1
.iter()
.find(|r| !c2.contains(&r.addr))
.expect("generation 2 shares every chunk with generation 1 — nothing to sweep");
let store: Arc<dyn ObjectStore> = Arc::new(SweepMidRead {
inner: inner.clone(),
current_key: cfg.current_key(),
doomed_key: cfg.chunk_key(&doomed.addr),
next_pointer: ptr2_bytes.clone(),
move_pointer,
fired: std::sync::atomic::AtomicBool::new(false),
});
let got = manifest::load(store.as_ref(), &cfg).await;
if move_pointer {
let loaded = got.expect("a reader swept mid-read did not restart").unwrap();
assert_eq!(
loaded.manifest.entries, m2.entries,
"the restart did not land on the CURRENT generation — a reader that mixes \
generations is worse than one that fails"
);
assert_eq!(loaded.manifest.seq, 2);
} else {
let msg = match got {
Err(e) => e.to_string(),
Ok(_) => panic!(
"a chunk vanished under an UNCHANGED pointer and the load succeeded — \
that is a hole in the live manifest, not a race"
),
};
assert!(msg.contains("hole"), "wrong refusal: {msg}");
}
}
}
#[tokio::test]
async fn the_chunk_reaper_takes_orphans_and_nothing_else() {
let inner = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 0;
let m1 = manifest_of(600, 1);
let meta1 = manifest::cas_write_chunked(store.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await.unwrap();
let c1 = match manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap()
.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new() };
manifest::cas_write_chunked(store.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await.unwrap();
let live: Vec<String> = match manifest::load_pointer(store.as_ref(), &cfg).await.unwrap()
.unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("not chunked"),
};
let orphans: Vec<&super::chunk::ChunkRef> =
c1.iter().filter(|r| !live.contains(&r.addr)).collect();
assert!(!orphans.is_empty(), "generation 2 superseded no chunk — nothing to reap");
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
assert_eq!(n, orphans.len(), "the reaper took {n} chunks, expected {}", orphans.len());
for r in &orphans {
assert!(
store.head(&cfg.chunk_key(&r.addr)).await.is_err(),
"superseded chunk {} survived the sweep",
r.addr
);
}
for a in &live {
assert!(store.head(&cfg.chunk_key(a)).await.is_ok(), "the reaper took a LIVE chunk {a}");
}
let back = manifest::load(store.as_ref(), &cfg).await.unwrap().unwrap();
assert_eq!(back.manifest.entries, m2.entries, "the sweep left the manifest short");
let meta2 = {
let lp = manifest::load_pointer(store.as_ref(), &cfg).await.unwrap().unwrap();
super::manifest::ManifestHandle { etag: lp.etag, legacy: false, prev_chunks: Vec::new() }
};
let mut m3 = m2.clone();
m3.seq = 3;
m3.entries.insert("src/f00300.txt".into(), entry_at("changed3", 43));
let c2: Vec<super::chunk::ChunkRef> = match manifest::load_pointer(store.as_ref(), &cfg)
.await.unwrap().unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
manifest::cas_write_chunked(store.as_ref(), &cfg, &m3, Some(&meta2), &c2,
manifest::PublishStamps { epoch: 1, flush_uuid: "u3", boundary_source: None })
.await.unwrap();
let before = store.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
cfg.orphan_grace_secs = 3600;
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
let after = store.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
assert_eq!(n, 0, "the reaper took {n} chunks that were inside the grace");
assert_eq!(before, after, "the chunk count moved despite the grace");
}
#[tokio::test]
async fn the_chunk_reaper_aborts_when_a_publish_lands_under_it() {
let inner = Arc::new(MemoryStore::new());
let plain: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 0;
let m1 = manifest_of(600, 1);
let meta1 = manifest::cas_write_chunked(plain.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await.unwrap();
let c1 = match manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap()
.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new() };
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await.unwrap();
let lp2 = manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap();
let mut moved = lp2.pointer.clone();
moved.seq = 99;
let moved_bytes = serde_json::to_vec(&moved).unwrap();
let store: Arc<dyn ObjectStore> = Arc::new(PublishOnList {
inner: inner.clone(),
current_key: cfg.current_key(),
next_pointer: moved_bytes,
chunks_prefix: format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR),
fired: std::sync::atomic::AtomicBool::new(false),
});
let before = plain.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
let after = plain.list(&format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR)).await.unwrap().len();
assert_eq!(n, 0, "the reaper deleted {n} chunks after a publish landed under it");
assert_eq!(before, after, "chunks disappeared across an aborted pass");
}
struct PublishOnList {
inner: Arc<MemoryStore>,
current_key: String,
next_pointer: Vec<u8>,
chunks_prefix: String,
fired: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl ObjectStore for PublishOnList {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
let out = self.inner.list(prefix).await;
if prefix == self.chunks_prefix
&& !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst)
{
let stamps = GenerationStamps {
generation: 99,
epoch: 1,
flush_uuid: "raced".into(),
boundary_source: None,
posix: None,
};
let crc = crc64_nvme(&self.next_pointer);
self.inner
.put_whole(
&self.current_key,
Bytes::from(self.next_pointer.clone()),
&PutCondition::Unconditional,
&stamps,
crc,
)
.await?;
}
out
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_chunked_barrier_keeps_a_foreign_write_it_never_read() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunked = true;
sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..60 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
let p = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap().pointer;
let chunks = match p.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
super::manifest::Entries::Single { .. } => {
panic!("the barrier published a single generation object with cfg.chunked set")
}
};
assert!(
chunks.len() >= 3,
"PRECONDITION: {} chunk(s) — too few for a foreign write and a local one to land in \
different ones, which is the case that matters",
chunks.len()
);
hitl_write(&store, &sc.cfg, "src/f000.txt", "user version", "dilip").await.unwrap();
write(dir.path(), "src/f059.txt", "v2");
backdate_baseline(&sc, "src/f059.txt");
sc.run_barrier().await.unwrap();
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(
m.manifest.entries.len(),
60,
"the chunked publish changed the entry count — a seam lost or duplicated a key"
);
let cited = &m.manifest.entries["src/f000.txt"];
let (_, body) = store.get_whole(&cited.key, Some(&cited.etag)).await.unwrap();
assert_eq!(
&body[..],
b"user version",
"the chunked publish lost the foreign write — this is the splice failure the merge \
model refutes, reached through the barrier"
);
let mine = &m.manifest.entries["src/f059.txt"];
let (_, body) = store.get_whole(&mine.key, Some(&mine.etag)).await.unwrap();
assert_eq!(&body[..], b"v2", "the local edit did not land");
}
#[tokio::test]
async fn migrating_to_chunks_leaves_no_generation_objects_behind() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunked = false; sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
sc.cfg.orphan_grace_secs = 0;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..40 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
write(dir.path(), "src/f000.txt", "v2");
backdate_baseline(&sc, "src/f000.txt");
sc.run_barrier().await.unwrap();
let gens = format!("{}/{}/manifests/", sc.cfg.prefix, super::LEAN_DIR);
assert!(
!store.list(&gens).await.unwrap().is_empty(),
"PRECONDITION: no generation objects to migrate away from"
);
sc.cfg.chunked = true;
write(dir.path(), "src/f001.txt", "v2");
backdate_baseline(&sc, "src/f001.txt");
sc.run_barrier().await.unwrap();
let p = manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap().unwrap().pointer;
assert!(
matches!(p.entries().unwrap(), super::manifest::Entries::Chunked(_)),
"the workspace did not migrate to a chunk list"
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries.len(), 40, "entries were lost migrating to chunks");
manifest::sweep_generations(store.as_ref(), &sc.cfg).await.unwrap();
assert!(
store.list(&gens).await.unwrap().is_empty(),
"generation objects survived the migration: {:?}",
store.list(&gens).await.unwrap().iter().map(|o| &o.key).collect::<Vec<_>>()
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries.len(), 40, "the migration sweep broke the live manifest");
}
struct StaleListing {
inner: Arc<MemoryStore>,
}
#[async_trait::async_trait]
impl ObjectStore for StaleListing {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
let mut out = self.inner.list(prefix).await?;
for o in out.iter_mut() {
o.last_modified_unix = Some(0);
}
Ok(out)
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn the_chunk_reaper_judges_the_grace_now_not_when_it_listed() {
let inner = Arc::new(MemoryStore::new());
let plain: Arc<dyn ObjectStore> = inner.clone();
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.chunk_target = 64;
cfg.chunk_min = 16;
cfg.chunk_max = 256;
cfg.orphan_grace_secs = 3600;
let m1 = manifest_of(600, 1);
let meta1 = manifest::cas_write_chunked(plain.as_ref(), &cfg, &m1, None, &[],
manifest::PublishStamps { epoch: 1, flush_uuid: "u1", boundary_source: None })
.await.unwrap();
let c1 = match manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap().unwrap()
.pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.to_vec(),
_ => panic!("not chunked"),
};
let mut m2 = m1.clone();
m2.seq = 2;
m2.entries.insert("src/f00003.txt".into(), entry_at("changed", 42));
let h = super::manifest::ManifestHandle { etag: meta1.etag.clone(), legacy: false, prev_chunks: Vec::new() };
manifest::cas_write_chunked(plain.as_ref(), &cfg, &m2, Some(&h), &c1,
manifest::PublishStamps { epoch: 1, flush_uuid: "u2", boundary_source: None })
.await.unwrap();
let prefix = format!("{}/{}/chunks/", cfg.prefix, super::LEAN_DIR);
let before = plain.list(&prefix).await.unwrap().len();
let live: Vec<String> = match manifest::load_pointer(plain.as_ref(), &cfg).await.unwrap()
.unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("not chunked"),
};
assert!(
c1.iter().any(|r| !live.contains(&r.addr)),
"PRECONDITION: nothing is unreferenced, so the reaper has no candidate to mis-judge"
);
let store: Arc<dyn ObjectStore> = Arc::new(StaleListing { inner: inner.clone() });
let n = manifest::sweep_chunks(store.as_ref(), &cfg).await.unwrap();
let after = plain.list(&prefix).await.unwrap().len();
assert_eq!(
n, 0,
"the reaper collected {n} chunk(s) on the LISTING's word that they were ancient — \
an adopt-rewrite landing after the listing is invisible to that, and rule 4 works \
by producing exactly such a rewrite"
);
assert_eq!(before, after, "chunks disappeared: {before} → {after}");
}
#[tokio::test]
async fn the_barrier_reaps_unreferenced_chunks_without_being_asked() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
sc.cfg.chunk_target = 8;
sc.cfg.chunk_min = 2;
sc.cfg.chunk_max = 32;
sc.cfg.orphan_grace_secs = 0; assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
for i in 0..60 {
write(dir.path(), &format!("src/f{i:03}.txt"), "v1");
}
sc.run_barrier().await.unwrap();
let prefix = format!("{}/{}/chunks/", sc.cfg.prefix, super::LEAN_DIR);
let after_first = store.list(&prefix).await.unwrap().len();
assert!(after_first >= 3, "PRECONDITION: {after_first} chunk(s) — too few to supersede any");
write(dir.path(), "src/f000.txt", "v2");
backdate_baseline(&sc, "src/f000.txt");
sc.run_barrier().await.unwrap();
let live: Vec<String> = match manifest::load_pointer(store.as_ref(), &sc.cfg).await.unwrap()
.unwrap().pointer.entries().unwrap() {
super::manifest::Entries::Chunked(c) => c.iter().map(|r| r.addr.clone()).collect(),
_ => panic!("the barrier did not publish a chunk list"),
};
let present: Vec<String> = store
.list(&prefix)
.await
.unwrap()
.into_iter()
.filter_map(|o| o.key.rsplit('/').next().map(|s| s.to_string()))
.collect();
let stray: Vec<&String> = present.iter().filter(|a| !live.contains(a)).collect();
assert!(
stray.is_empty(),
"the barrier left {} unreferenced chunk(s) in the bucket ({stray:?}) — the reaper is \
not reachable from a publish, which is the whole defect",
stray.len()
);
for a in &live {
assert!(present.contains(a), "the barrier's own sweep deleted live chunk {a}");
}
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries.len(), 60, "the sweep left the manifest short");
}
struct DeposeOnNthPut {
inner: Arc<MemoryStore>,
epoch_key: String,
at: usize,
puts: std::sync::atomic::AtomicUsize,
deposed: std::sync::atomic::AtomicBool,
puts_after: std::sync::atomic::AtomicUsize,
}
#[async_trait::async_trait]
impl ObjectStore for DeposeOnNthPut {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
use std::sync::atomic::Ordering::SeqCst;
let n = self.puts.fetch_add(1, SeqCst) + 1;
if self.deposed.load(SeqCst) {
self.puts_after.fetch_add(1, SeqCst);
}
let r = self.inner.put_whole(key, body, cond, stamps, crc).await;
if n == self.at && !self.deposed.swap(true, SeqCst) {
let state = self.inner.epoch_read(&self.epoch_key).await?.expect("a held cell");
self.inner.epoch_acquire(&self.epoch_key, "lean-usurper", Some(&state)).await?;
}
r
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_deposed_writer_stops_at_the_next_chunk_boundary() {
let inner = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut cfg = cfg_for(dir.path());
cfg.upload_fanout = 1; let hooked = Arc::new(DeposeOnNthPut {
inner: inner.clone(),
epoch_key: cfg.epoch_key(),
at: 4,
puts: Default::default(),
deposed: Default::default(),
puts_after: Default::default(),
});
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store: hooked.clone() as Arc<dyn ObjectStore>,
cfg,
state,
lease: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
for i in 0..40 {
write(dir.path(), &format!("f{i:02}.txt"), &format!("body-{i}"));
}
let err = a
.run_barrier()
.await
.expect_err("a writer deposed mid-barrier completed its barrier");
assert!(matches!(err, LeanError::Fenced(_)), "expected Fenced, got: {err}");
use std::sync::atomic::Ordering::SeqCst;
assert!(hooked.deposed.load(SeqCst), "the fixture never deposed the writer — this run proves nothing");
let after = hooked.puts_after.load(SeqCst);
assert!(
after > 0,
"no PUT landed after the deposal: the barrier's entry check caught it, not the chunk boundary"
);
assert!(
after < 16,
"the deposed writer issued {after} more data PUTs after its takeover — it ran past the \
chunk boundary (16 files at fanout 1) instead of fencing there"
);
assert!(
manifest::load(inner.as_ref(), &a.cfg).await.unwrap().is_none(),
"the straggler installed a manifest after being deposed"
);
}
#[tokio::test]
async fn a_lost_renew_response_does_not_self_fence() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut a, 3).await);
let key = a.cfg.epoch_key();
let stale = a.lease.clone().unwrap();
store.epoch_renew(&key, &stale, None).await.unwrap();
let landed = store.epoch_read(&key).await.unwrap().unwrap().token;
assert_ne!(landed, stale.token, "fixture: the renew did not move the token");
lease::renew(&mut a)
.await
.expect("a renew whose previous response was lost is not a deposal");
assert_eq!(
a.lease.as_ref().map(|l| l.token.as_str()),
Some(landed.as_str()),
"the holder did not adopt the token its own landed renew minted"
);
a.verify_not_deposed_pub().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert!(claim_until_held(&mut b, 12).await, "quiet polls exhausted ⇒ takeover");
assert!(matches!(lease::renew(&mut a).await.unwrap_err(), LeanError::Fenced(_)));
assert!(a.lease.is_none(), "a fenced holder must drop its lease");
}
#[tokio::test]
async fn a_foreign_claim_refuses_the_syncer_before_it_claims() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir.path()).await;
lease::verify_claim(&a).await.unwrap();
a.cfg.project_id = Some("team-a/p".into());
lease::verify_claim(&a).await.unwrap();
let key = a.cfg.claim_key();
let body = br#"{"project_id":"team-b/q","created_unix":1,"stamped_by":"op"}"#.to_vec();
let stamps = GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: "claim".into(),
boundary_source: None,
posix: None,
};
let crc = crc64_nvme(&body);
store
.put_whole(&key, Bytes::from(body), &PutCondition::IfNoneMatchAny, &stamps, crc)
.await
.unwrap();
let err = lease::verify_claim(&a).await.unwrap_err();
assert!(
matches!(err, LeanError::Refused(_)),
"a foreign claim is a REFUSAL (exit EXIT_REFUSED, final for the delivery), not a \
state error a restart would retry: {err}"
);
let err = err.to_string();
assert!(
err.contains("team-b/q") && err.contains("team-a/p"),
"the refusal must name both projects: {err}"
);
assert!(a.lease.is_none());
a.cfg.project_id = Some("team-b/q".into());
lease::verify_claim(&a).await.unwrap();
}
#[test]
fn the_drain_attestation_is_written_by_the_drain_and_cleared_at_startup() {
let dir = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir.path());
let st = SyncerState::open(cfg.state_dir()).unwrap();
assert!(!st.drained_path().exists(), "a fresh state dir must carry no attestation");
st.sync_tree().unwrap();
st.write_drained(Some(7), 1).unwrap();
let v: serde_json::Value = serde_json::from_slice(&std::fs::read(st.drained_path()).unwrap()).unwrap();
assert_eq!(v["seq"], 7);
assert_eq!(v["acks"], 1);
assert!(v["unix"].as_u64().unwrap() > 0);
st.clear_drained().unwrap();
assert!(!st.drained_path().exists());
st.clear_drained().unwrap(); }
#[tokio::test]
async fn ranged_checkout_is_byte_identical_and_parallel() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big: Vec<u8> = (0..5 * 1024 * 1024).map(|i| (i % 251) as u8).collect();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
write(dir_a.path(), "small.txt", "not ranged");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.range_get_min_bytes = 1024 * 1024;
b.cfg.range_get_chunk_bytes = 512 * 1024; b.cfg.range_get_parallelism = 4;
store.reset_peak_get_range_in_flight();
store.inject_get_range_delay_ms(5);
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(
std::fs::read(dir_b.path().join("weights.bin")).unwrap(),
big,
"ranged materialisation must be byte-identical"
);
assert_eq!(read(dir_b.path(), "small.txt").unwrap(), "not ranged");
let peak = store.peak_get_range_in_flight();
assert!(peak > 1, "ranges must overlap; peak in flight was {peak}");
}
fn ranged_default_tree() -> Vec<u8> {
(0..20 * 1024 * 1024).map(|i| (i % 251) as u8).collect()
}
#[tokio::test]
async fn ranged_checkout_on_by_default_uses_ranges() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big = ranged_default_tree();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
write(dir_a.path(), "small.txt", "under the threshold");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
assert_eq!(
b.cfg.range_get_min_bytes,
8 * 1024 * 1024,
"the shipped default moved to 8 MiB on the 2026-09-10 runcr drill"
);
store.reset_peak_get_range_in_flight();
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(
std::fs::read(dir_b.path().join("weights.bin")).unwrap(),
big,
"the DEFAULT path must be byte-identical, not merely fast"
);
assert_eq!(read(dir_b.path(), "small.txt").unwrap(), "under the threshold");
assert!(
store.peak_get_range_in_flight() > 0,
"an object over the default threshold must take the ranged path"
);
assert_eq!(cr.ranged, 1, "exactly the over-threshold object, not both");
}
#[tokio::test]
async fn ranged_checkout_explicitly_disabled_uses_no_ranges() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big = ranged_default_tree();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
write(dir_a.path(), "small.txt", "under the threshold");
a.run_barrier().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.range_get_min_bytes = 0;
store.reset_peak_get_range_in_flight();
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 2);
assert_eq!(
std::fs::read(dir_b.path().join("weights.bin")).unwrap(),
big,
"the whole-object arm must materialise the same bytes"
);
assert_eq!(
store.peak_get_range_in_flight(),
0,
"the whole-object arm must issue no ranged reads"
);
assert_eq!(cr.ranged, 0);
}
async fn scoped_fixture(store: &Arc<MemoryStore>) -> tempfile::TempDir {
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "inputs/wanted.txt", "the file the agent edits");
write(dir_a.path(), "inputs/also.txt", "and this one");
for i in 0..6 {
write(dir_a.path(), &format!("outputs/big-{i}.bin"), &"x".repeat(4096));
}
a.run_barrier().await.unwrap();
dir_a
}
#[tokio::test]
async fn a_scoped_checkout_materialises_only_the_admitted_set() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(cr.materialized, 2, "only the admitted citations");
assert_eq!(cr.out_of_scope, 6, "and the declined count says the scope DID something");
assert_eq!(cr.scope.as_deref(), Some(&["inputs".to_string()][..]));
assert!(read(dir_b.path(), "inputs/wanted.txt").is_some());
assert!(
read(dir_b.path(), "outputs/big-0.bin").is_none(),
"an out-of-scope citation must not reach the tree"
);
let base = b.state.load_baseline().unwrap();
assert_eq!(
base.entries.keys().cloned().collect::<Vec<_>>(),
vec!["inputs/also.txt".to_string(), "inputs/wanted.txt".to_string()],
"an unadmitted path must never be cited"
);
assert_eq!(b.state.load_scope().unwrap().as_deref(), Some(&["inputs".to_string()][..]));
}
#[tokio::test]
async fn what_a_scoped_checkout_never_materialised_it_can_never_delete() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert!(claim_until_held(&mut b, 12).await, "quiet polls exhausted ⇒ takeover");
let r1 = b.run_barrier().await.unwrap();
let r2 = b.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty() && r2.deleted.is_empty(), "{:?} {:?}", r1.deleted, r2.deleted);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(
m.entries.len(),
8,
"the six unadmitted citations survive a scoped workspace's barriers: {:?}",
m.entries.keys().collect::<Vec<_>>()
);
}
#[tokio::test]
async fn a_scoped_checkout_budget_is_computed_over_the_admitted_set() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_whole = tempfile::tempdir().unwrap();
let mut whole = syncer(&store, dir_whole.path()).await;
whole.cfg.max_bytes = 1024;
let err = whole.checkout().await.expect_err("the whole tree is over this budget");
assert!(matches!(err, LeanError::Budget(_)), "{err}");
let dir_scoped = tempfile::tempdir().unwrap();
let mut scoped = syncer(&store, dir_scoped.path()).await;
scoped.cfg.max_bytes = 1024;
let cr = scoped
.checkout_scoped(Some(vec!["inputs".into()]))
.await
.expect("a budget is a promise about what THIS checkout writes");
assert_eq!(cr.materialized, 2);
}
#[tokio::test]
async fn an_all_rejected_checkout_scope_is_refused_not_widened() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let err = b
.checkout_scoped(Some(vec!["../escape".into(), "./here".into()]))
.await
.expect_err("every entry malformed ⇒ an EMPTY scope ⇒ the whole manifest");
assert!(format!("{err}").contains("WHOLE MANIFEST"), "{err}");
assert!(
read(dir_b.path(), "outputs/big-0.bin").is_none(),
"the refusal must happen BEFORE the first byte"
);
assert!(!b.state.marker_present(), "and must not open the agent-start gate");
}
#[tokio::test]
async fn a_live_tree_refuses_a_checkout_whose_scope_disagrees() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
let err = b.checkout().await.expect_err("scoped tree, unscoped request");
let msg = format!("{err}");
assert!(msg.contains("UNSCOPED"), "the error must name BOTH sets: {msg}");
assert!(msg.contains("inputs"), "the error must name BOTH sets: {msg}");
let dir_c = tempfile::tempdir().unwrap();
let mut c = syncer(&store, dir_c.path()).await;
c.checkout().await.unwrap();
let err = c
.checkout_scoped(Some(vec!["inputs".into()]))
.await
.expect_err("unscoped tree, scoped request");
assert!(format!("{err}").contains("cannot change the admitted set"), "{err}");
let r = b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
assert!(r.resumed_live_tree);
}
#[tokio::test]
async fn a_scoped_checkout_leaves_the_merge_base_whole() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
let base = b.state.load_baseline().unwrap();
assert_eq!(base.entries.len(), 2, "the CITATIONS are scoped");
assert_eq!(
base.inst_base.len(),
8,
"the MERGE BASE is not: {:?}",
base.inst_base.keys().collect::<Vec<_>>()
);
}
#[tokio::test]
async fn a_narrowed_merge_base_makes_every_unadmitted_citation_foreign() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
a.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
a.sync().await.unwrap();
assert!(
read(dir_a.path(), "outputs/big-0.bin").is_none(),
"a whole-tree sync must leave the declined citations declined"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
{
let mut base = b.state.load_baseline().unwrap();
base.inst_base.retain(|p, _| p.starts_with("inputs/"));
b.state.save_baseline(&base).unwrap();
}
b.sync().await.unwrap();
assert!(
read(dir_b.path(), "outputs/big-0.bin").is_some(),
"THE CLIFF: absent from the merge base reads as CHANGED \
(manifest.rs `unwrap_or(true)`, sync.rs `unwrap_or(false)`), so every \
declined citation comes back one sync later"
);
let base = b.state.load_baseline().unwrap();
assert_eq!(base.entries.len(), 8, "and the scope is gone entirely");
}
#[tokio::test]
async fn an_unreadable_scope_is_not_read_as_unscoped() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
if unsafe { libc::geteuid() } == 0 {
eprintln!("SKIPPED: running as root, mode bits cannot induce EACCES");
return;
}
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout_scoped(Some(vec!["inputs".into()])).await.unwrap();
let sd = b.cfg.state_dir();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o000)).unwrap();
let got = b.state.load_scope();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
let err = got.expect_err("EACCES must not read as 'unscoped'");
assert!(format!("{err}").contains("refusing to"), "{err}");
}
#[tokio::test]
async fn a_whole_tree_checkout_clears_a_stale_scope() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.state.save_scope(Some(&["inputs".to_string()])).unwrap();
assert!(!b.state.marker_present());
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 8, "the whole manifest, not the stale scope's share");
assert_eq!(b.state.load_scope().unwrap(), None, "and the stale scope is GONE");
}
fn bulky_conflict(i: usize) -> super::state::ConflictRecord {
super::state::ConflictRecord {
path: format!("{}/{}", "p".repeat(900), i),
foreign_etag: format!("\"etag-{i}\""),
preserved_key: None,
kind: "test-bulk".into(),
at_unix: 1_700_000_000 + i as u64,
}
}
#[tokio::test]
async fn the_conflict_log_rotates_instead_of_growing_without_bound() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
for i in 0..2000 {
a.state.append_conflict(&bulky_conflict(i)).unwrap();
}
let sd = a.cfg.state_dir();
let live = std::fs::metadata(sd.join("conflicts.jsonl")).unwrap().len();
assert!(
std::fs::metadata(sd.join("conflicts.1.jsonl")).is_ok(),
"past the cap the live log must have rotated"
);
assert!(live <= 1 << 20, "the LIVE log is what gets parsed on every read: {live}");
let all = a.state.load_conflicts().unwrap();
assert_eq!(all.len(), 2000, "a single rotation loses nothing");
assert_eq!(a.state.conflicts_dropped().unwrap(), 0);
assert_eq!(all[0].at_unix, 1_700_000_000);
assert_eq!(all[1999].at_unix, 1_700_000_000 + 1999);
}
#[tokio::test]
async fn a_rotation_never_shortens_what_a_reader_has_already_counted() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
let mut prev = 0usize;
let mut saw_rotation = false;
for i in 0..2000 {
a.state.append_conflict(&bulky_conflict(i)).unwrap();
let n = a.state.load_conflicts().unwrap().len();
assert!(n >= prev, "the log SHRANK at append {i}: {prev} -> {n}");
if std::fs::metadata(a.cfg.state_dir().join("conflicts.1.jsonl")).is_ok() {
saw_rotation = true;
}
prev = n;
}
assert!(saw_rotation, "the run must actually cross a rotation or it proves nothing");
}
#[tokio::test]
async fn a_second_rotation_reports_what_it_dropped() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
for i in 0..4000 {
a.state.append_conflict(&bulky_conflict(i)).unwrap();
}
let dropped = a.state.conflicts_dropped().unwrap();
let held = a.state.load_conflicts().unwrap().len();
assert!(dropped > 0, "a second rotation DID drop records; the count must say so");
assert_eq!(
dropped as usize + held,
4000,
"dropped + held must account for every record: {dropped} + {held}"
);
}
#[tokio::test]
async fn an_unreadable_conflict_log_is_not_reported_as_empty() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let a = syncer(&store, dir.path()).await;
if unsafe { libc::geteuid() } == 0 {
eprintln!("SKIPPED: running as root, mode bits cannot induce EACCES");
return;
}
a.state.append_conflict(&bulky_conflict(1)).unwrap();
let sd = a.cfg.state_dir();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o000)).unwrap();
let got = a.state.load_conflicts();
std::fs::set_permissions(&sd, std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap();
let err = got.expect_err("an unreadable log must not read as 'no conflicts'");
assert!(format!("{err}").contains("refusing to report"), "{err}");
}
#[test]
fn a_shortened_conflict_log_never_reports_an_empty_ack() {
use super::sentinel::conflicts_since;
let rec = |i: usize| bulky_conflict(i);
let grew = vec![rec(0), rec(1), rec(2)];
assert_eq!(conflicts_since(1, grew.clone()).len(), 2);
assert_eq!(conflicts_since(3, grew.clone()).len(), 0, "grew by nothing");
let shrank = vec![rec(7), rec(8)];
assert_eq!(
conflicts_since(5, shrank.clone()).len(),
2,
"a shortened log must not yield an empty conflict set"
);
}
struct ComposeWithoutChecksum(Arc<MemoryStore>);
#[async_trait::async_trait]
impl ObjectStore for ComposeWithoutChecksum {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
let mut m = self.0.compose_generation(spec).await?;
m.crc64_b64 = None;
Ok(m)
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_whole(key, if_match).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
let mut out = self.0.list(prefix).await?;
for o in out.iter_mut() {
o.last_modified_unix = Some(0);
}
Ok(out)
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.0.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_compose_that_reports_no_checksum_refuses_instead_of_citing() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(ComposeWithoutChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let cfg = {
let mut c = cfg_for(dir.path());
c.whole_put_max = 4096;
c
};
let state = SyncerState::open(cfg.state_dir()).unwrap();
let mut a = Syncer {
store,
cfg,
state,
lease: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
std::fs::write(dir.path().join("big.bin"), vec![7u8; 20_000]).unwrap();
let err = a
.run_barrier()
.await
.expect_err("a publish with no checksum must refuse, not cite");
assert!(
format!("{err}").contains("refusing to cite bytes nothing vouched for"),
"{err}"
);
let m = manifest::load(mem.as_ref(), &a.cfg).await.unwrap();
assert!(
m.map(|l| l.manifest.entries.is_empty()).unwrap_or(true),
"a refused publish must leave no citation"
);
}
async fn rescope_fixture(store: &Arc<MemoryStore>) -> (tempfile::TempDir, Syncer) {
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(store, dir.path()).await;
b.checkout().await.unwrap();
assert!(claim_until_held(&mut b, 12).await, "quiet polls exhausted ⇒ takeover");
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 8);
(dir, b)
}
#[tokio::test]
async fn a_narrow_unwatches_without_publishing_a_single_deletion() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
let r = b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(r.uncited, 6, "six citations should have left the held set");
assert_eq!(r.unlinked, 6, "and six files the tree");
assert!(r.kept_dirty.is_empty());
assert!(read(dir.path(), "outputs/big-0.bin").is_none(), "the narrow did not unlink");
assert!(read(dir.path(), "inputs/wanted.txt").is_some(), "the narrow took an admitted path");
assert_eq!(b.state.load_scope().unwrap().as_deref(), Some(&["inputs".to_string()][..]));
let r1 = b.run_barrier().await.unwrap();
let r2 = b.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty() && r2.deleted.is_empty(), "{:?} {:?}", r1.deleted, r2.deleted);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 8, "a narrow published a deletion: {:?}", m.entries.keys());
}
#[tokio::test]
async fn the_same_unlink_without_the_uncite_publishes_deletions() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
for i in 0..6 {
std::fs::remove_file(dir.path().join(format!("outputs/big-{i}.bin"))).unwrap();
}
let r1 = b.run_barrier().await.unwrap();
let r2 = b.run_barrier().await.unwrap();
assert!(r1.deleted.is_empty(), "first absence is not delete-eligible: {:?}", r1.deleted);
assert_eq!(r2.deleted.len(), 6, "the delete rule did not bite: {:?}", r2.deleted);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 2, "the bucket should have lost six objects here");
}
#[tokio::test]
async fn a_widen_materialises_and_cites_what_it_newly_admits() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert!(read(dir.path(), "outputs/big-0.bin").is_none());
let r = b.rescope(Some(vec!["inputs".into(), "outputs".into()])).await.unwrap();
assert_eq!(r.materialized, 6, "widen did not fetch the newly admitted set");
assert_eq!(r.already_held, 2, "and must not refetch what it already holds");
assert_eq!(r.uncited, 0);
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "widen did not materialise");
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 8);
let r1 = b.run_barrier().await.unwrap();
assert!(r1.uploaded.is_empty(), "widen re-uploaded its own fetch: {:?}", r1.uploaded);
}
#[tokio::test]
async fn a_narrow_leaves_the_merge_base_whole() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (_dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
let base = b.state.load_baseline().unwrap();
assert_eq!(base.entries.len(), 2, "the HELD set narrows");
assert_eq!(base.inst_base.len(), 8, "the MERGE BASE does not");
assert!(base.inst_base.contains_key("outputs/big-0.bin"));
assert!(!base.entries.contains_key("outputs/big-0.bin"));
assert!(!base.prev_scan.contains("outputs/big-0.bin"), "prev_scan must drop with entries");
}
#[tokio::test]
async fn a_crash_between_the_intent_and_the_unlink_converges() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
drop: (0..6).map(|i| format!("outputs/big-{i}.bin")).collect(),
})
.unwrap();
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "nothing applied yet");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed, "the barrier ran over a half-applied rescope");
assert!(r1.deleted.is_empty(), "{:?}", r1.deleted);
let r2 = b.run_barrier().await.unwrap();
assert!(!r2.rescope_replayed, "the replay must clear its own intent");
assert!(r2.deleted.is_empty(), "{:?}", r2.deleted);
assert!(read(dir.path(), "outputs/big-0.bin").is_none(), "the replay did not finish the job");
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries.len(), 8, "converging cost the bucket objects");
}
#[tokio::test]
async fn a_crash_between_the_uncite_and_the_unlink_does_not_re_cite() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
drop: (0..6).map(|i| format!("outputs/big-{i}.bin")).collect(),
})
.unwrap();
let mut base = b.state.load_baseline().unwrap();
for i in 0..6 {
let p = format!("outputs/big-{i}.bin");
base.entries.remove(&p);
base.prev_scan.remove(&p);
}
b.state.save_baseline(&base).unwrap();
assert!(read(dir.path(), "outputs/big-0.bin").is_some(), "the files are still on disk");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed);
assert!(
r1.uploaded.is_empty(),
"the barrier re-cited what the narrow dropped: {:?}",
r1.uploaded
);
assert!(read(dir.path(), "outputs/big-0.bin").is_none());
assert_eq!(
b.state.load_baseline().unwrap().entries.len(),
2,
"the narrow was undone by its own recovery"
);
}
#[tokio::test]
async fn a_rescope_refuses_to_drop_a_path_with_unpublished_changes() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
write(dir.path(), "outputs/big-0.bin", "the user's unpublished edit");
let err = b.rescope(Some(vec!["inputs".into()])).await.unwrap_err();
match err {
LeanError::State(m) => assert!(m.contains("outputs/big-0.bin"), "{m}"),
e => panic!("wrong error: {e:?}"),
}
assert!(b.state.load_scope().unwrap().is_none(), "a refused rescope changed the scope");
assert!(b.state.load_scope_intent().unwrap().is_none(), "a refused rescope left an intent");
assert_eq!(
read(dir.path(), "outputs/big-0.bin").as_deref(),
Some("the user's unpublished edit"),
"a refused rescope touched the tree"
);
b.run_barrier().await.unwrap();
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
}
#[tokio::test]
async fn a_replay_keeps_a_dirty_path_instead_of_wedging_the_barrier() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.state
.save_scope_intent(&super::state::ScopeIntent {
target: Some(vec!["inputs".into()]),
drop: (0..6).map(|i| format!("outputs/big-{i}.bin")).collect(),
})
.unwrap();
write(dir.path(), "outputs/big-0.bin", "edited after the intent landed");
let r1 = b.run_barrier().await.unwrap();
assert!(r1.rescope_replayed);
assert!(b.state.load_scope_intent().unwrap().is_none(), "the replay did not converge");
assert_eq!(
read(dir.path(), "outputs/big-0.bin").as_deref(),
Some("edited after the intent landed")
);
assert!(b.state.load_baseline().unwrap().entries.contains_key("outputs/big-0.bin"));
assert!(read(dir.path(), "outputs/big-1.bin").is_none(), "the clean ones still left");
assert!(
b.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.path == "outputs/big-0.bin" && c.kind == "rescope-kept-locally-dirty"),
"a kept path must be surfaced, not silently retained"
);
}
#[tokio::test]
async fn an_all_rejected_rescope_is_refused_not_widened() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (_dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
let err = b.rescope(Some(vec!["../escape".into(), "./here".into()])).await.unwrap_err();
assert!(matches!(err, LeanError::State(_)), "{err:?}");
assert_eq!(
b.state.load_scope().unwrap().as_deref(),
Some(&["inputs".to_string()][..]),
"a refused rescope widened the workspace"
);
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 2);
}
#[tokio::test]
async fn a_rescope_before_any_checkout_is_refused() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let dir = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir.path()).await;
let err = b.rescope(Some(vec!["inputs".into()])).await.unwrap_err();
match err {
LeanError::State(m) => assert!(m.contains("checkout"), "{m}"),
e => panic!("wrong error: {e:?}"),
}
}
#[tokio::test]
async fn a_rescope_to_none_holds_everything_and_clears_the_scope() {
let store = Arc::new(MemoryStore::new());
let _keep = scoped_fixture(&store).await;
let (dir, mut b) = rescope_fixture(&store).await;
b.rescope(Some(vec!["inputs".into()])).await.unwrap();
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 2);
let r = b.rescope(None).await.unwrap();
assert_eq!(r.materialized, 6);
assert!(b.state.load_scope().unwrap().is_none(), "the scope document must be REMOVED");
assert_eq!(b.state.load_baseline().unwrap().entries.len(), 8);
assert!(read(dir.path(), "outputs/big-5.bin").is_some());
}
#[tokio::test]
async fn a_checkout_does_not_wait_out_a_standing_lease() {
let store = Arc::new(MemoryStore::new());
let pdir = tempfile::tempdir().unwrap();
let mut publisher = syncer(&store, pdir.path()).await;
assert!(claim_until_held(&mut publisher, 3).await);
publisher.checkout().await.unwrap();
write(pdir.path(), "README.md", "the real readme");
write(pdir.path(), "src/main.rs", "fn main() {}");
publisher.run_barrier().await.unwrap();
let held = publisher.lease.clone().expect("the publisher holds a lease");
let rdir = tempfile::tempdir().unwrap();
let mut reader = syncer(&store, rdir.path()).await;
let done = tokio::time::timeout(
std::time::Duration::from_secs(2),
super::verbs::run_verb(&mut reader, super::verbs::Step::Checkout),
)
.await;
match done {
Err(_) => panic!(
"the checkout verb waited on the publisher's lease — it has no business \
holding the publish fence"
),
Ok(r) => r.expect("the checkout itself must succeed"),
}
assert_eq!(read(rdir.path(), "README.md").unwrap(), "the real readme");
assert_eq!(read(rdir.path(), "src/main.rs").unwrap(), "fn main() {}");
let cell = store.epoch_read(&reader.cfg.epoch_key()).await.unwrap().expect("the cell stands");
assert_eq!(cell.holder_id, held.holder_id, "the reader deposed the publisher");
assert_eq!(cell.epoch, held.epoch, "the reader moved the epoch");
assert!(!cell.released, "the reader released someone else's lease");
assert!(reader.lease.is_none(), "the reader came away holding a lease");
}
#[tokio::test]
async fn a_checkout_issues_no_write_to_the_bucket() {
let store = Arc::new(MemoryStore::new());
let pdir = tempfile::tempdir().unwrap();
let mut publisher = syncer(&store, pdir.path()).await;
assert!(claim_until_held(&mut publisher, 3).await);
publisher.checkout().await.unwrap();
for i in 0..8 {
write(pdir.path(), &format!("f{i}.txt"), &format!("body {i}"));
}
publisher.run_barrier().await.unwrap();
let rdir = tempfile::tempdir().unwrap();
let mut reader = syncer(&store, rdir.path()).await;
store.reset_op_counts();
super::verbs::run_verb(&mut reader, super::verbs::Step::Checkout).await.unwrap();
let ops = store.op_counts();
for op in [
"put_whole",
"delete",
"delete_version",
"copy_object",
"compose_generation",
"epoch_acquire",
"epoch_renew",
"epoch_release",
"abort_upload",
] {
assert_eq!(
ops.get(op).copied().unwrap_or(0),
0,
"checkout called {op} — it is no longer a read, and `read_only_then` is no \
longer justified. Full shape: {ops:?}"
);
}
assert!(ops.values().sum::<u64>() > 0, "the checkout made no requests at all: {ops:?}");
assert_eq!(reader.state.load_baseline().unwrap().entries.len(), 8);
}
struct PublishMidCheckout {
inner: Arc<MemoryStore>,
cfg: LeanConfig,
trigger_key: String,
trigger_path: String,
fired: std::sync::atomic::AtomicBool,
}
#[async_trait::async_trait]
impl ObjectStore for PublishMidCheckout {
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
if key == self.trigger_key
&& !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst)
{
let loaded = manifest::load(self.inner.as_ref(), &self.cfg)
.await
.expect("the publisher reads the standing manifest")
.expect("there is one");
let body = Bytes::from_static(b"the publisher's next generation");
let crc = crc64_nvme(&body);
let stamps = GenerationStamps {
generation: 1,
epoch: 1,
flush_uuid: "the-publisher".into(),
boundary_source: None,
posix: None,
};
let meta = self
.inner
.put_whole(&self.trigger_key, body, &PutCondition::Unconditional, &stamps, crc)
.await?;
let mut m = loaded.manifest.clone();
m.seq += 1;
m.entries.get_mut(&self.trigger_path).expect("the path is cited").etag = meta.etag;
manifest::cas_write(
self.inner.as_ref(),
&self.cfg,
&m,
Some(&loaded.handle()),
1,
"the-publisher",
)
.await
.expect("the publish lands");
}
self.inner.get_whole(key, if_match).await
}
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head(key).await
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.inner.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.inner.min_part_size()
}
fn max_parts(&self) -> usize {
self.inner.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.inner.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.inner.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.inner.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.inner.get_version(key, v).await
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.inner.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.inner.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.inner.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.inner.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.inner.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.inner.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.inner.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.inner.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_publish_that_lands_mid_checkout_names_the_publisher_not_a_stranger() {
let inner = Arc::new(MemoryStore::new());
let pdir = tempfile::tempdir().unwrap();
let mut publisher = syncer(&inner, pdir.path()).await;
publisher.cfg.sole_writer = true;
assert!(claim_until_held(&mut publisher, 3).await);
publisher.checkout().await.unwrap();
write(pdir.path(), "README.md", "generation one");
publisher.run_barrier().await.unwrap();
let rdir = tempfile::tempdir().unwrap();
let mut reader = syncer(&inner, rdir.path()).await;
let racing = Arc::new(PublishMidCheckout {
inner: inner.clone(),
cfg: reader.cfg.clone(),
trigger_key: reader.cfg.file_key("README.md"),
trigger_path: "README.md".to_string(),
fired: std::sync::atomic::AtomicBool::new(false),
});
reader.store = racing as Arc<dyn ObjectStore>;
let err = reader.checkout().await.expect_err("the citation it loaded is one generation stale");
let msg = format!("{err}");
assert!(
msg.contains("its publisher published while the checkout was running"),
"the reader was not told who moved it: {msg}"
);
assert!(msg.contains("seq 1") && msg.contains("seq 2"), "name both generations: {msg}");
assert!(
msg.contains("Re-run checkout"),
"a reader one generation behind has a remedy; say it: {msg}"
);
assert!(msg.contains("SOLE WRITER"), "the underlying refusal must survive: {msg}");
assert!(
read(rdir.path(), "README.md").is_none(),
"it materialized a file it could not verify"
);
}
#[test]
fn siblings_racing_to_create_one_parent_do_not_refuse_each_other() {
let dir = tempfile::tempdir().unwrap();
for round in 0..50 {
let root = dir.path().join(format!("r{round}"));
std::fs::create_dir(&root).unwrap();
let go = Arc::new(std::sync::Barrier::new(8));
let hs: Vec<_> = (0..8)
.map(|i| {
let root = root.clone();
let go = go.clone();
std::thread::spawn(move || {
go.wait();
super::barrier::contained_path(&root, &format!("new/deeper/f-{i}"))
})
})
.collect();
for h in hs {
h.join().unwrap().expect("a sibling's mkdir is not a containment refusal");
}
assert!(root.join("new/deeper").is_dir());
}
}
#[test]
fn the_temp_retry_paths_replace_a_leftover_and_recreate_a_vanished_parent() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::create_dir_all(root.join("d")).unwrap();
std::fs::write(root.join("d/f.txt.flint-sync-tmp"), b"STALE").unwrap();
let st = super::barrier::write_file_atomic(&root.join("d/f.txt"), b"NEW", Some(0o644)).unwrap();
assert_eq!(std::fs::read(root.join("d/f.txt")).unwrap(), b"NEW");
assert_eq!(st.len(), 3, "the returned metadata is the written file's, not the leftover's");
assert!(!root.join("d/f.txt.flint-sync-tmp").exists(), "the temp name is gone after the rename");
let outside = tempfile::tempdir().unwrap();
std::fs::write(outside.path().join("victim"), b"ORIGINAL").unwrap();
std::os::unix::fs::symlink(outside.path().join("victim"), root.join("d/g.txt.flint-sync-tmp"))
.unwrap();
super::barrier::write_file_atomic(&root.join("d/g.txt"), b"NEW", None).unwrap();
assert_eq!(std::fs::read(outside.path().join("victim")).unwrap(), b"ORIGINAL");
assert_eq!(std::fs::read(root.join("d/g.txt")).unwrap(), b"NEW");
let (target, present) = super::barrier::contained_path_stat(root, "gone/h.txt").unwrap();
assert!(present.is_none(), "the walk reports an absent final component as None");
std::fs::remove_dir(root.join("gone")).unwrap();
super::barrier::write_file_atomic(&target, b"NEW", None).unwrap();
assert_eq!(std::fs::read(root.join("gone/h.txt")).unwrap(), b"NEW");
let (_, present) = super::barrier::contained_path_stat(root, "gone/h.txt").unwrap();
assert_eq!(present.map(|m| m.len()), Some(3));
let missing = root.join("nostate/x.json");
let err = super::safefs::write_via_tmp(&missing, &root.join("nostate/x.json.tmp"), b"{}", None)
.unwrap_err();
assert!(err.to_string().contains("not exclusively creatable"), "{err}");
assert!(!root.join("nostate").exists());
}
#[tokio::test]
async fn a_fresh_fetch_whose_bytes_do_not_match_the_manifest_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "model/config.json", "{\"layers\": 12, \"hidden\": 768}");
write(dir_a.path(), "README.md", "intact");
a.run_barrier().await.unwrap();
store.inject_corrupt_body(&a.cfg.file_key("model/config.json"), |b| b[10] ^= 0x01);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let err = b.checkout().await.expect_err("a corrupt fresh fetch must refuse");
let msg = err.to_string();
assert!(msg.contains("CRC-64") && msg.contains("model/config.json"), "{msg}");
assert!(
!dir_b.path().join("model/config.json").exists(),
"nothing may be written for a body that fails the check"
);
assert!(!dir_b.path().join("model/config.json.flint-sync-tmp").exists());
}
#[tokio::test]
async fn a_ranged_fetch_whose_bytes_do_not_match_the_manifest_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let big: Vec<u8> = (0..5 * 1024 * 1024).map(|i| (i % 251) as u8).collect();
std::fs::write(dir_a.path().join("weights.bin"), &big).unwrap();
a.run_barrier().await.unwrap();
store.inject_corrupt_body(&a.cfg.file_key("weights.bin"), |b| {
let i = b.len() / 2 + 777;
b[i] ^= 0x01;
});
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.cfg.range_get_min_bytes = 1024 * 1024;
b.cfg.range_get_chunk_bytes = 512 * 1024; b.cfg.range_get_parallelism = 4;
let err = b.checkout().await.expect_err("a corrupt ranged fetch must refuse");
let msg = err.to_string();
assert!(msg.contains("CRC-64") && msg.contains("10 ranges"), "{msg}");
assert!(!dir_b.path().join("weights.bin").exists(), "a wrong object must never be renamed into place");
assert!(!dir_b.path().join("weights.bin.flint-sync-tmp").exists(), "the temp file is cleaned up");
}
struct AttestsNoChecksum(Arc<MemoryStore>);
fn unattested(mut m: flint_store::ObjectMeta) -> flint_store::ObjectMeta {
m.crc64_b64 = None;
m
}
#[async_trait::async_trait]
impl ObjectStore for AttestsNoChecksum {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await.map(unattested)
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.put_whole(key, body, cond, stamps, crc).await.map(unattested)
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head(key).await.map(unattested)
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_whole(key, if_match).await.map(|(m, b)| (unattested(m), b))
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.0.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.0.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await.map(unattested)
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await.map(|(m, b)| (unattested(m), b))
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
}
fn syncer_on(store: Arc<dyn ObjectStore>, root: &std::path::Path) -> Syncer {
let cfg = cfg_for(root);
let state = SyncerState::open(cfg.state_dir()).unwrap();
Syncer { store, cfg, state, lease: None, noted_not_regular: Default::default() }
}
fn crc_of(content: &str) -> String {
flint_store::crc64_to_b64(crc64_nvme(content.as_bytes()))
}
async fn assert_attests_nothing(mem: &Arc<MemoryStore>, store: &Arc<dyn ObjectStore>, key: &str) {
assert!(mem.head(key).await.unwrap().crc64_b64.is_some(), "the fixture object has a CRC");
assert!(store.head(key).await.unwrap().crc64_b64.is_none(), "HEAD attests a checksum");
let (m, _) = store.get_whole(key, None).await.unwrap();
assert!(m.crc64_b64.is_none(), "GET attests a checksum");
}
#[tokio::test]
async fn a_hitl_upload_on_a_backend_that_attests_no_checksum_is_cited_with_the_bytes_crc() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(AttestsNoChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer_on(store.clone(), dir.path());
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
let key = sc.cfg.file_key("docs/upload.pdf");
let body = Bytes::from_static(b"user bytes");
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "gateway-old".into(),
boundary_source: None,
posix: None,
};
let meta = mem
.put_whole(&key, body.clone(), &PutCondition::IfNoneMatchAny, &stamps, crc64_nvme(&body))
.await
.unwrap();
inbox::gateway_append(
store.as_ref(),
&sc.cfg,
InboxEntry {
path: "docs/upload.pdf".into(),
etag: meta.etag.clone(),
author: "dilip".into(),
added_unix: now_unix(),
crc64_b64: None,
},
)
.await
.unwrap();
assert_attests_nothing(&mem, &store, &key).await;
write(dir.path(), "agent.txt", "agent work v2 — longer");
sc.run_barrier().await.unwrap();
sc.run_barrier().await.unwrap();
assert_eq!(read(dir.path(), "docs/upload.pdf").unwrap(), "user bytes");
let b = sc.state.load_baseline().unwrap();
assert_eq!(
b.entries["docs/upload.pdf"].crc64_b64.as_deref(),
Some(crc_of("user bytes").as_str()),
"the baseline carries the CRC of the bytes the consume wrote"
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
let e = m.entries.get("docs/upload.pdf").expect("amputated!");
assert_eq!(e.crc64_b64, crc_of("user bytes"), "the repair cited the bytes' CRC");
let dir2 = tempfile::tempdir().unwrap();
let mut sc2 = syncer_on(store.clone(), dir2.path());
sc2.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "docs/upload.pdf").unwrap(), "user bytes");
}
#[tokio::test]
async fn recover_staged_on_a_backend_that_attests_no_checksum_hashes_what_it_recites() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(AttestsNoChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let mut a = syncer_on(store.clone(), dir.path());
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "cited.txt", "V1");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
write(dir.path(), "cited.txt", "V2-UNCITED");
backdate_baseline(&a, "cited.txt");
write(dir.path(), "brand-new.txt", "NEW-UNCITED");
a.upload_lane().await.unwrap();
let key = a.cfg.file_key("brand-new.txt");
drop(a);
assert_attests_nothing(&mem, &store, &key).await;
let vs = store.list_versions(&key).await.unwrap();
let cur = vs.iter().find(|v| v.is_current).unwrap();
assert!(
store.head_version(&key, &cur.version_id).await.unwrap().crc64_b64.is_none(),
"the versioned HEAD attests a checksum"
);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer_on(store.clone(), dir_b.path());
b.cfg.boundary_mode = super::BoundaryMode::Gated;
b.cfg.visibility_lag_bound_secs = Some(3600);
assert!(claim_until_held(&mut b, 12).await);
let r = b.recover_staged().await.unwrap();
assert_eq!(r.recited.len(), 2, "recovery re-cited {:?}", r.recited);
let m = manifest::load(store.as_ref(), &b.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["brand-new.txt"].crc64_b64, crc_of("NEW-UNCITED"));
assert_eq!(m.entries["cited.txt"].crc64_b64, crc_of("V2-UNCITED"));
let dir_c = tempfile::tempdir().unwrap();
let mut c = syncer_on(store.clone(), dir_c.path());
c.checkout().await.unwrap();
assert_eq!(read(dir_c.path(), "cited.txt").as_deref(), Some("V2-UNCITED"));
assert_eq!(read(dir_c.path(), "brand-new.txt").as_deref(), Some("NEW-UNCITED"));
}
#[tokio::test]
async fn sync_on_a_backend_that_attests_no_checksum_still_sees_identical_bytes() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(AttestsNoChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let mut a = syncer_on(store.clone(), dir.path());
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
a.run_barrier().await.unwrap();
hitl_write(&mem, &a.cfg, "shared.txt", "foreign v2", "ci").await.unwrap();
assert_attests_nothing(&mem, &store, &a.cfg.file_key("shared.txt")).await;
let r = a.sync().await.unwrap();
assert!(r.applied.contains(&"shared.txt".to_string()));
let mut b = a.state.load_baseline().unwrap();
let stale = b.entries.get_mut("shared.txt").unwrap();
stale.etag = "\"stale-pre-sync-etag\"".into();
stale.mtime_unix -= 10;
stale.size = 1;
a.state.save_baseline(&b).unwrap();
let scanned = super::scan::scan(dir.path()).unwrap();
let c = super::scan::classify(&scanned, &a.state.load_baseline().unwrap());
assert!(c.uploads.contains("shared.txt"), "the fixture is not actually dirty");
let r = a.sync().await.unwrap();
assert!(
!r.conflicts.contains(&"shared.txt".to_string()),
"a phantom conflict was reported for byte-identical content"
);
assert!(r.applied.contains(&"shared.txt".to_string()));
}
#[tokio::test]
async fn a_consume_whose_bytes_do_not_match_the_writers_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "agent.txt", "agent work");
sc.run_barrier().await.unwrap();
hitl_write(&store, &sc.cfg, "docs/upload.pdf", "user bytes", "dilip").await.unwrap();
store.inject_corrupt_body(&sc.cfg.file_key("docs/upload.pdf"), |b| b[3] ^= 0x40);
sc.run_barrier().await.unwrap();
assert!(read(dir.path(), "docs/upload.pdf").is_none(), "corrupt bytes were written");
assert!(
sc.state.load_conflicts().unwrap().iter().any(|c| c.kind.starts_with("consume-refused-checksum")),
"the refusal was silent"
);
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(
ib.doc.entries.iter().any(|e| e.path == "docs/upload.pdf"),
"a refused entry must stay in the inbox, not be consumed away"
);
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest;
assert!(!m.entries.contains_key("docs/upload.pdf"), "refused bytes were cited");
}
#[tokio::test]
async fn a_sync_whose_bytes_do_not_match_the_manifest_crc_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir_a = tempfile::tempdir().unwrap();
let mut a = syncer(&store, dir_a.path()).await;
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
b.checkout().await.unwrap();
write(dir_a.path(), "f.txt", "published bytes");
a.run_barrier().await.unwrap();
store.inject_corrupt_body(&a.cfg.file_key("f.txt"), |bytes| bytes[0] ^= 0x01);
let err = b.sync().await.expect_err("a sync of corrupt bytes must refuse");
assert!(err.to_string().contains("refusing to apply it"), "{err}");
assert!(read(dir_b.path(), "f.txt").is_none(), "corrupt bytes were written");
}
#[tokio::test]
async fn a_gated_citation_repair_on_a_backend_that_attests_no_checksum_cites_the_bytes_crc() {
let mem = Arc::new(MemoryStore::new());
let store: Arc<dyn ObjectStore> = Arc::new(AttestsNoChecksum(mem.clone()));
let dir = tempfile::tempdir().unwrap();
let mut a = syncer_on(store.clone(), dir.path());
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
gated(&mut a);
write(dir.path(), "agent.txt", "agent work");
a.upload_lane().await.unwrap();
a.citation_pass(CitationSource::Sentinel).await.unwrap();
let key = a.cfg.file_key("docs/upload.pdf");
let body = Bytes::from_static(b"user bytes");
let stamps = GenerationStamps {
generation: 0,
epoch: 0,
flush_uuid: "gateway-old".into(),
boundary_source: None,
posix: None,
};
let meta = mem
.put_whole(&key, body.clone(), &PutCondition::IfNoneMatchAny, &stamps, crc64_nvme(&body))
.await
.unwrap();
inbox::gateway_append(
store.as_ref(),
&a.cfg,
InboxEntry {
path: "docs/upload.pdf".into(),
etag: meta.etag.clone(),
author: "dilip".into(),
added_unix: now_unix(),
crc64_b64: None,
},
)
.await
.unwrap();
assert_attests_nothing(&mem, &store, &key).await;
a.upload_lane().await.unwrap();
assert_eq!(read(dir.path(), "docs/upload.pdf").unwrap(), "user bytes");
let r = a.citation_pass(CitationSource::Sentinel).await.unwrap();
assert!(r.repaired.contains(&"docs/upload.pdf".to_string()), "not repaired: {r:?}");
let m = manifest::load(store.as_ref(), &a.cfg).await.unwrap().unwrap().manifest;
assert_eq!(m.entries["docs/upload.pdf"].crc64_b64, crc_of("user bytes"));
let dir2 = tempfile::tempdir().unwrap();
let mut b = syncer_on(store.clone(), dir2.path());
b.checkout().await.unwrap();
assert_eq!(read(dir2.path(), "docs/upload.pdf").unwrap(), "user bytes");
}
async fn hitl_remove(store: &Arc<MemoryStore>, cfg: &LeanConfig, path: &str, author: &str) {
inbox::gateway_remove(
store.as_ref(),
cfg,
vec![inbox::Removal {
path: path.to_string(),
author: author.to_string(),
requested_unix: now_unix(),
moved_to: None,
refused: None,
}],
)
.await
.unwrap();
}
async fn hitl_rename(store: &Arc<MemoryStore>, cfg: &LeanConfig, from: &str, to: &str, author: &str) {
let m = manifest::load(store.as_ref(), cfg).await.unwrap().unwrap();
let src = m.manifest.entries.get(from).expect("source is cited");
let stamps = GenerationStamps {
generation: 1,
epoch: 0,
flush_uuid: format!("gateway-rename-{author}"),
boundary_source: None,
posix: None,
};
let dst = store
.copy_object(
&cfg.file_key(from),
Some(&src.etag),
&cfg.file_key(to),
&PutCondition::IfNoneMatchAny,
&stamps,
)
.await
.unwrap();
inbox::gateway_rename(
store.as_ref(),
cfg,
vec![InboxEntry {
path: to.to_string(),
etag: dst.etag,
author: author.to_string(),
added_unix: now_unix(),
crc64_b64: Some(src.crc64_b64.clone()),
}],
vec![inbox::Removal {
path: from.to_string(),
author: author.to_string(),
requested_unix: now_unix(),
moved_to: Some(to.to_string()),
refused: None,
}],
)
.await
.unwrap();
}
#[tokio::test]
async fn a_declared_removal_is_cited_out_in_one_barrier() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "README.md", "hello");
write(dir.path(), "src/main.rs", "fn main() {}");
sc.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest.seq;
hitl_remove(&store, &sc.cfg, "README.md", "dilip").await;
assert!(store.head(&sc.cfg.file_key("README.md")).await.is_ok());
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.pending_removal("README.md").is_some());
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.removed, vec!["README.md".to_string()], "declared this barrier");
assert_eq!(r.deleted, vec!["README.md".to_string()], "and GC'd this barrier, not next");
assert!(r.first_absence.is_empty(), "a declaration is not a first absence");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.seq, before + 1, "exactly one generation");
assert!(!m.manifest.entries.contains_key("README.md"));
assert!(m.manifest.entries.contains_key("src/main.rs"));
assert!(store.head(&sc.cfg.file_key("README.md")).await.is_err(), "object GC'd");
assert!(read(dir.path(), "README.md").is_none(), "unlinked from the agent's tree");
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.removals.is_empty(), "settled out of the cell");
assert!(!sc.state.load_baseline().unwrap().entries.contains_key("README.md"));
let r2 = sc.run_barrier().await.unwrap();
assert!(r2.no_change, "{r2:?}");
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 1);
assert!(read(dir_b.path(), "README.md").is_none());
}
#[tokio::test]
async fn a_declared_removal_of_a_dirty_path_applies_nothing_and_is_refused() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "shared.txt", "v1");
sc.run_barrier().await.unwrap();
write(dir.path(), "shared.txt", "the agent's unpublished v2");
backdate_baseline(&sc, "shared.txt");
hitl_remove(&store, &sc.cfg, "shared.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
assert!(r.removed.is_empty(), "nothing declared");
assert!(r.deleted.is_empty(), "nothing deleted");
assert_eq!(r.removals_refused, 1);
assert_eq!(read(dir.path(), "shared.txt").unwrap(), "the agent's unpublished v2");
assert_eq!(r.uploaded, vec!["shared.txt".to_string()], "the agent's edit publishes");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let (_, body) = store.get_whole(&sc.cfg.file_key("shared.txt"), None).await.unwrap();
assert_eq!(&body[..], b"the agent's unpublished v2");
assert!(m.manifest.entries.contains_key("shared.txt"));
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.pending_removal("shared.txt").is_none(), "not pending any more");
let rm = ib.doc.removals.iter().find(|r| r.path == "shared.txt").expect("kept, annotated");
let refusal = rm.refused.as_ref().expect("refused");
assert_eq!(refusal.kind, "removal-refused-dirty");
assert!(refusal.message.contains("dilip"), "names who asked: {}", refusal.message);
assert!(refusal.message.contains("kept"), "{}", refusal.message);
assert!(
sc.state.load_conflicts().unwrap().iter().any(|c| c.kind.starts_with("removal-refused-dirty")),
"the pod-side record too"
);
for _ in 0..2 {
let r = sc.run_barrier().await.unwrap();
assert!(r.deleted.is_empty() && r.removed.is_empty(), "{r:?}");
}
assert_eq!(read(dir.path(), "shared.txt").unwrap(), "the agent's unpublished v2");
assert!(store.head(&sc.cfg.file_key("shared.txt")).await.is_ok());
hitl_remove(&store, &sc.cfg, "shared.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["shared.txt".to_string()]);
assert!(inbox::load(store.as_ref(), &sc.cfg).await.unwrap().doc.removals.is_empty());
}
#[tokio::test]
async fn a_failed_unlink_publishes_no_deletion() {
if unsafe { libc::geteuid() } == 0 {
eprintln!("skipped: root ignores directory permissions");
return;
}
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "locked/f.txt", "keep");
sc.run_barrier().await.unwrap();
use std::os::unix::fs::PermissionsExt;
let locked = dir.path().join("locked");
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o555)).unwrap();
hitl_remove(&store, &sc.cfg, "locked/f.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap();
assert!(r.removed.is_empty() && r.deleted.is_empty(), "{r:?}");
assert_eq!(read(dir.path(), "locked/f.txt").unwrap(), "keep");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("locked/f.txt"), "still cited");
assert!(store.head(&sc.cfg.file_key("locked/f.txt")).await.is_ok(), "object untouched");
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.pending_removal("locked/f.txt").is_some(), "deferred, not refused");
assert!(sc
.state
.load_conflicts()
.unwrap()
.iter()
.any(|c| c.kind.starts_with("removal-unlink-failed")));
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["locked/f.txt".to_string()]);
assert!(read(dir.path(), "locked/f.txt").is_none());
}
#[tokio::test]
async fn a_rename_rides_one_manifest_generation() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "notes/a.txt", "the same bytes");
sc.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
let src_crc = before.manifest.entries["notes/a.txt"].crc64_b64.clone();
hitl_rename(&store, &sc.cfg, "notes/a.txt", "docs/b.txt", "dilip").await;
assert!(store.head(&sc.cfg.file_key("docs/b.txt")).await.is_ok());
assert!(store.head(&sc.cfg.file_key("notes/a.txt")).await.is_ok());
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert_eq!(ib.doc.entries.len(), 1);
assert_eq!(ib.doc.removals.len(), 1);
assert_eq!(ib.doc.removals[0].moved_to.as_deref(), Some("docs/b.txt"));
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.consumed, 1);
assert_eq!(r.removed, vec!["notes/a.txt".to_string()]);
assert_eq!(r.deleted, vec!["notes/a.txt".to_string()]);
let after = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(after.manifest.seq, before.manifest.seq + 1, "ONE generation carries both halves");
assert!(!after.manifest.entries.contains_key("notes/a.txt"));
let dst = &after.manifest.entries["docs/b.txt"];
assert_eq!(dst.crc64_b64, src_crc, "the same bytes, attested");
assert!(store.head(&sc.cfg.file_key("notes/a.txt")).await.is_err(), "source GC'd");
assert_eq!(read(dir.path(), "docs/b.txt").unwrap(), "the same bytes");
assert!(read(dir.path(), "notes/a.txt").is_none());
assert!(inbox::load(store.as_ref(), &sc.cfg).await.unwrap().doc.removals.is_empty());
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&store, dir_b.path()).await;
let cr = b.checkout().await.unwrap();
assert_eq!(cr.materialized, 1);
assert_eq!(read(dir_b.path(), "docs/b.txt").unwrap(), "the same bytes");
}
#[tokio::test]
async fn a_rename_interrupted_before_the_source_unlink_re_applies() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "bytes");
sc.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest.seq;
hitl_rename(&store, &sc.cfg, "a.txt", "b.txt", "dilip").await;
sc.consume_inbox().await.unwrap();
assert_eq!(read(dir.path(), "b.txt").unwrap(), "bytes");
assert_eq!(read(dir.path(), "a.txt").unwrap(), "bytes", "the source is still there");
drop(sc);
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.removed, vec!["a.txt".to_string()]);
let after = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(after.manifest.seq, before + 1, "still one generation");
assert!(after.manifest.entries.contains_key("b.txt"));
assert!(!after.manifest.entries.contains_key("a.txt"));
assert!(read(dir.path(), "a.txt").is_none());
assert_eq!(read(dir.path(), "b.txt").unwrap(), "bytes");
}
#[tokio::test]
async fn a_journalled_declared_delete_survives_a_crash() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "gone.txt", "bytes");
write(dir.path(), "kept.txt", "bytes");
sc.run_barrier().await.unwrap();
let before = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap().manifest.seq;
std::fs::remove_file(dir.path().join("gone.txt")).unwrap();
let mut intent = sc.state.load_intent().unwrap();
intent.declared_deletes = vec!["gone.txt".to_string()];
sc.state.save_intent(&intent).unwrap();
drop(sc);
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["gone.txt".to_string()], "one barrier, from the journal");
assert!(r.first_absence.is_empty());
let after = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert_eq!(after.manifest.seq, before + 1);
assert!(!after.manifest.entries.contains_key("gone.txt"));
assert!(after.manifest.entries.contains_key("kept.txt"));
assert!(sc.state.load_intent().unwrap().declared_deletes.is_empty(), "cleared with the keys");
}
#[tokio::test]
async fn a_declared_removal_of_an_already_absent_path_declares_as_it_stands() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "both.txt", "bytes");
sc.run_barrier().await.unwrap();
std::fs::remove_file(dir.path().join("both.txt")).unwrap();
hitl_remove(&store, &sc.cfg, "both.txt", "dilip").await;
hitl_remove(&store, &sc.cfg, "never/there.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.deleted, vec!["both.txt".to_string()]);
assert!(r.first_absence.is_empty());
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
assert!(ib.doc.removals.is_empty(), "both settled: {:?}", ib.doc.removals);
}
#[tokio::test]
async fn a_rename_whose_destination_the_agent_took_keeps_the_source() {
let store = Arc::new(MemoryStore::new());
let dir = tempfile::tempdir().unwrap();
let mut sc = syncer(&store, dir.path()).await;
assert!(claim_until_held(&mut sc, 3).await);
sc.checkout().await.unwrap();
write(dir.path(), "a.txt", "the user's bytes");
sc.run_barrier().await.unwrap();
write(dir.path(), "b.txt", "the agent's unpublished file");
hitl_rename(&store, &sc.cfg, "a.txt", "b.txt", "dilip").await;
let r = sc.run_barrier().await.unwrap();
assert_eq!(r.consumed, 1, "the destination entry is consumed (as a conflict)");
assert_eq!(r.removals_refused, 1);
assert!(r.removed.is_empty() && r.deleted.is_empty(), "{r:?}");
assert_eq!(read(dir.path(), "a.txt").unwrap(), "the user's bytes", "the source stays");
assert_eq!(read(dir.path(), "b.txt").unwrap(), "the agent's unpublished file");
let m = manifest::load(store.as_ref(), &sc.cfg).await.unwrap().unwrap();
assert!(m.manifest.entries.contains_key("a.txt"), "still cited");
let (_, b) = store.get_whole(&sc.cfg.file_key("b.txt"), None).await.unwrap();
assert_eq!(&b[..], b"the agent's unpublished file", "the agent's version publishes");
let ib = inbox::load(store.as_ref(), &sc.cfg).await.unwrap();
let rm = ib.doc.removals.iter().find(|r| r.path == "a.txt").expect("refused, kept");
assert_eq!(rm.refused.as_ref().unwrap().kind, "removal-refused-destination-conflict");
let conflicts = sc.state.load_conflicts().unwrap();
let c = conflicts.iter().find(|c| c.kind == "consume-dirty" && c.path == "b.txt").unwrap();
let (_, kept) = store.get_whole(c.preserved_key.as_ref().unwrap(), None).await.unwrap();
assert_eq!(&kept[..], b"the user's bytes", "the moved bytes are preserved too");
}
struct FailPutTo(Arc<MemoryStore>, String, std::sync::atomic::AtomicBool);
impl FailPutTo {
fn arm(&self) {
self.2.store(true, std::sync::atomic::Ordering::SeqCst);
}
}
#[async_trait::async_trait]
impl ObjectStore for FailPutTo {
async fn copy_object(
&self,
src_key: &str,
src_if_match: Option<&str>,
dst_key: &str,
condition: &PutCondition,
stamps: &GenerationStamps,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
}
async fn put_whole(
&self,
key: &str,
body: Bytes,
cond: &PutCondition,
stamps: &GenerationStamps,
crc: u64,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
if self.2.load(std::sync::atomic::Ordering::SeqCst) && key.ends_with(&self.1) {
return Err(flint_store::StoreError::Other(format!("injected failure on {key}")));
}
self.0.put_whole(key, body, cond, stamps, crc).await
}
async fn compose_generation(
&self,
spec: &flint_store::ComposeSpec<'_>,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.compose_generation(spec).await
}
async fn head(&self, key: &str) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head(key).await
}
async fn get_whole(
&self,
key: &str,
if_match: Option<&str>,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_whole(key, if_match).await.map(|(m, b)| (unattested(m), b))
}
async fn get_range(
&self,
key: &str,
off: u64,
len: u64,
if_match: &str,
) -> flint_store::StoreResult<Bytes> {
self.0.get_range(key, off, len, if_match).await
}
fn min_part_size(&self) -> u64 {
self.0.min_part_size()
}
fn max_parts(&self) -> usize {
self.0.max_parts()
}
async fn list(&self, prefix: &str) -> flint_store::StoreResult<Vec<flint_store::ListedObject>> {
self.0.list(prefix).await
}
async fn delete(&self, key: &str) -> flint_store::StoreResult<()> {
self.0.delete(key).await
}
async fn head_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<flint_store::ObjectMeta> {
self.0.head_version(key, v).await
}
async fn get_version(
&self,
key: &str,
v: &str,
) -> flint_store::StoreResult<(flint_store::ObjectMeta, Bytes)> {
self.0.get_version(key, v).await.map(|(m, b)| (unattested(m), b))
}
async fn delete_version(&self, key: &str, v: &str) -> flint_store::StoreResult<()> {
self.0.delete_version(key, v).await
}
async fn list_versions(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::ListedVersion>> {
self.0.list_versions(prefix).await
}
async fn list_uploads(
&self,
prefix: &str,
) -> flint_store::StoreResult<Vec<flint_store::PendingUpload>> {
self.0.list_uploads(prefix).await
}
async fn abort_upload(&self, key: &str, id: &str) -> flint_store::StoreResult<()> {
self.0.abort_upload(key, id).await
}
async fn bootstrap(
&self,
prefix: &str,
) -> flint_store::StoreResult<flint_store::BootstrapReport> {
self.0.bootstrap(prefix).await
}
async fn epoch_read(
&self,
key: &str,
) -> flint_store::StoreResult<Option<flint_store::EpochState>> {
self.0.epoch_read(key).await
}
async fn epoch_acquire(
&self,
key: &str,
holder: &str,
observed: Option<&flint_store::EpochState>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_acquire(key, holder, observed).await
}
async fn epoch_renew(
&self,
key: &str,
lease: &flint_store::EpochLease,
echo: Option<&str>,
) -> flint_store::StoreResult<flint_store::EpochLease> {
self.0.epoch_renew(key, lease, echo).await
}
async fn epoch_release(
&self,
key: &str,
lease: &flint_store::EpochLease,
) -> flint_store::StoreResult<()> {
self.0.epoch_release(key, lease).await
}
}
#[tokio::test]
async fn a_consumed_hitl_write_survives_pod_replacement_before_the_cas() {
let inner = Arc::new(MemoryStore::new());
let failing = Arc::new(FailPutTo(
inner.clone(),
"/current".into(),
std::sync::atomic::AtomicBool::new(false),
));
let dir_a = tempfile::tempdir().unwrap();
let cfg = cfg_for(dir_a.path());
let mut a = Syncer {
store: failing.clone() as Arc<dyn ObjectStore>,
state: SyncerState::open(cfg.state_dir()).unwrap(),
cfg,
lease: None,
noted_not_regular: Default::default(),
};
assert!(claim_until_held(&mut a, 3).await);
a.checkout().await.unwrap();
write(dir_a.path(), "base.txt", "published");
a.run_barrier().await.unwrap();
let etag = hitl_write(&inner, &a.cfg, "ui/upload.txt", "acked to the user", "dilip").await.unwrap();
failing.arm();
let err = a.run_barrier().await.unwrap_err();
assert!(err.to_string().contains("injected"), "{err}");
assert_eq!(read(dir_a.path(), "ui/upload.txt").unwrap(), "acked to the user", "consumed into A's tree");
let m = manifest::load(inner.as_ref(), &a.cfg).await.unwrap().unwrap();
assert!(!m.manifest.entries.contains_key("ui/upload.txt"), "and not cited");
drop(a);
drop(dir_a);
let dir_b = tempfile::tempdir().unwrap();
let mut b = syncer(&inner, dir_b.path()).await;
assert!(!claim_until_held(&mut b, 3).await);
assert!(claim_until_held(&mut b, 10).await, "takeover");
b.checkout().await.unwrap();
assert!(read(dir_b.path(), "ui/upload.txt").is_none(), "not cited, so not materialised");
let r = b.run_barrier().await.unwrap();
assert_eq!(r.consumed, 1, "the entry was still in the cell for the successor");
assert_eq!(read(dir_b.path(), "ui/upload.txt").unwrap(), "acked to the user");
let m = manifest::load(inner.as_ref(), &b.cfg).await.unwrap().unwrap();
assert_eq!(m.manifest.entries["ui/upload.txt"].etag, etag, "cited by the successor");
assert!(inbox::load(inner.as_ref(), &b.cfg).await.unwrap().doc.entries.is_empty(), "and then dropped");
}