flint_lean/barrier.rs
1//! The publish barrier (plan §2.1, seven steps) — the machine
2//! `lean/formal/LeanSubtree.tla` checks. Order is load-bearing:
3//! consume → scan → intent/window → uploads → manifest CAS (merge) →
4//! GC deletes LAST → baseline rewrite.
5
6use std::collections::{BTreeMap, BTreeSet};
7use std::path::Path;
8
9use bytes::Bytes;
10
11use flint_store::{
12 crc64_nvme, crc64_to_b64, GenerationStamps, ObjectStore, PosixStamps, PutCondition, RequestCounts, StoreError,
13};
14
15use super::inbox;
16use super::docview::{CommitDocs, DocView, Opened};
17use super::manifest::{self, LeanEntry};
18use super::scan;
19use super::state::{BaselineEntry, Carrier, ConflictRecord, IntentJournal};
20use super::{now_unix, LeanError, LeanResult, Syncer};
21
22/// The last gateway request this workspace acted on, per verb. A
23/// watermark rather than a consumed queue — see `note_verb_requests`.
24#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
25pub struct VerbWatermark {
26 #[serde(default)]
27 pub boundary_unix: u64,
28 #[serde(default)]
29 pub sync_unix: u64,
30}
31
32#[derive(Debug, Default)]
33pub struct BarrierReport {
34 pub seq: Option<u64>,
35 pub uploaded: Vec<String>,
36 pub deleted: Vec<String>,
37 /// Paths a DECLARED removal (delete/rename design) cited out this
38 /// barrier — a subset of `deleted` once the GC has run.
39 pub removed: Vec<String>,
40 /// Declared removals this barrier REFUSED (locally dirty, not a
41 /// file, uncontainable); the cell carries the reason.
42 pub removals_refused: usize,
43 pub parked: Vec<String>,
44 pub consumed: usize,
45 pub foreign_queued: usize,
46 /// Paths whose transfer drifted or was swept mid-compose: nothing
47 /// published, nothing advanced; the next scan re-queues them.
48 pub deferred: Vec<String>,
49 /// Paths the syncer may not read (`scan::Scan::held`): a directory
50 /// it may not list (with its trailing `/`), an entry it may not
51 /// lstat, a file it may not open, an absence it may not confirm; and
52 /// a file or directory whose name is not UTF-8, which no key names.
53 /// Nothing at or under one was published or deleted, and the rest of
54 /// the tree was. A boundary that holds anything is not complete: a
55 /// sentinel's ack is `partial` and names them, and the drain refuses
56 /// to attest it.
57 pub held: Vec<String>,
58 /// Paths whose object the collector LEFT BEHIND because this store
59 /// does not enforce `If-Match` on DELETE (`conformance.rs`). The
60 /// delete still reached the boundary; only the object survives.
61 ///
62 /// NOT added to `deleted`, and so NOT dropped from the merge base:
63 /// the rule step 7 follows is "drop the entry once the key no longer
64 /// holds our bytes", which is why the absent and replaced-absent
65 /// arms drop it and the skipped-etag arm does not. A leaked key
66 /// still holds them. The model names that rule
67 /// `BaselineKeepsUncollected` and it is what ships, so a leak
68 /// follows the arm that already exists rather than inventing a
69 /// third; the cost is that the collector re-offers the path (one
70 /// HEAD) at every later barrier, exactly as a skipped one is.
71 pub leaked: Vec<String>,
72 /// Paths this barrier published OVER a version it never integrated;
73 /// each has a conflict record naming the preserved copy (R7).
74 pub surfaced: Vec<String>,
75 /// Retired handles the batch delete took.
76 pub collected: usize,
77 /// Orphan handles the sweep took, when it was due.
78 pub swept: usize,
79 pub no_change: bool,
80 /// Bytes this barrier actually published — the input to the
81 /// sentinel work meter (boundary-verbs plan D3.1). Metering work
82 /// rather than calls is what keeps a sentinel storm from
83 /// un-coalescing a hot large file's republish: a counted budget
84 /// charges a 2 GiB checkpoint the same one unit as a 4 KiB file.
85 pub published_bytes: u64,
86 /// The manifest ETag this barrier installed (the ack's CAS token).
87 pub manifest_etag: Option<String>,
88 /// What the manifest HEAD/install told us the bucket is at, for the
89 /// news ticker (D5) — free, off requests the barrier already made.
90 pub observed_seq: Option<u64>,
91 pub observed_etag: Option<String>,
92 /// Paths this barrier saw absent for the FIRST time: their deletes
93 /// are withheld to the next scan (the rename-vs-walk guard). On a
94 /// declared boundary a non-empty set means the confirmation lstat
95 /// found them back on disk — the race the guard exists for.
96 pub first_absence: Vec<String>,
97 /// First-absence paths a DECLARED boundary confirmed gone by direct
98 /// lstat and published as ordinary deletes (`confirm_absences`).
99 pub absences_confirmed: usize,
100 /// This barrier finished a rescope a crash had left half-applied
101 /// before it did anything else. Reported so a run that looks like
102 /// an ordinary barrier can be told from one that had to converge a
103 /// workspace first.
104 pub rescope_replayed: bool,
105 /// Where this barrier's time went, step by step ([`StepTiming`]).
106 pub steps: Vec<StepTiming>,
107}
108
109/// One step of a barrier and what it cost: wall time, and the requests
110/// the store sent during it (`None` from a store that does not count).
111/// A step the barrier entered more than once — each CAS attempt's load,
112/// fence, merge and CAS — is one entry, summed, in the order it first
113/// ran.
114#[derive(Debug, Clone, Default, serde::Serialize)]
115pub struct StepTiming {
116 pub step: &'static str,
117 pub us: u64,
118 pub requests: Option<RequestCounts>,
119}
120
121/// Times a barrier's steps. Each `lap` charges everything since the
122/// previous one to the step it names, so the steps tile the barrier:
123/// their times sum to its wall time, and their requests to everything
124/// the store sent for it.
125struct StepClock {
126 store: std::sync::Arc<dyn ObjectStore>,
127 at: std::time::Instant,
128 counted: Option<RequestCounts>,
129 steps: Vec<StepTiming>,
130}
131
132impl StepClock {
133 fn start(store: std::sync::Arc<dyn ObjectStore>) -> Self {
134 let counted = store.request_counts();
135 Self { store, at: std::time::Instant::now(), counted, steps: vec![] }
136 }
137
138 fn lap(&mut self, step: &'static str) {
139 let now = std::time::Instant::now();
140 let counted = self.store.request_counts();
141 let us = now.duration_since(self.at).as_micros() as u64;
142 let sent = match (self.counted, counted) {
143 (Some(a), Some(b)) => Some(counts_between(a, b)),
144 _ => None,
145 };
146 match self.steps.iter_mut().find(|s| s.step == step) {
147 Some(s) => {
148 s.us += us;
149 s.requests = match (s.requests, sent) {
150 (Some(a), Some(b)) => Some(counts_sum(a, b)),
151 _ => None,
152 };
153 }
154 None => self.steps.push(StepTiming { step, us, requests: sent }),
155 }
156 self.at = now;
157 self.counted = counted;
158 }
159}
160
161fn counts_between(a: RequestCounts, b: RequestCounts) -> RequestCounts {
162 RequestCounts {
163 get: b.get.saturating_sub(a.get),
164 head: b.head.saturating_sub(a.head),
165 put: b.put.saturating_sub(a.put),
166 copy: b.copy.saturating_sub(a.copy),
167 delete: b.delete.saturating_sub(a.delete),
168 list: b.list.saturating_sub(a.list),
169 multipart: b.multipart.saturating_sub(a.multipart),
170 }
171}
172
173fn counts_sum(a: RequestCounts, b: RequestCounts) -> RequestCounts {
174 RequestCounts {
175 get: a.get + b.get,
176 head: a.head + b.head,
177 put: a.put + b.put,
178 copy: a.copy + b.copy,
179 delete: a.delete + b.delete,
180 list: a.list + b.list,
181 multipart: a.multipart + b.multipart,
182 }
183}
184
185/// Upload waves per chunk. The chunk is `fanout * this`, so a wave
186/// still saturates fan-out and the sync point between chunks is rare.
187const UPLOAD_CHUNK_WAVES: usize = 16;
188
189
190
191/// What a consume's read of the document established, for the barrier's
192/// no-change fast path (which must not pay for the same read twice).
193#[derive(Debug, Clone, PartialEq, Eq)]
194pub enum Seen {
195 /// The pointer at this (seq, etag).
196 Pointer(u64, String),
197 /// Neither the pointer nor the legacy object exists: nothing has ever
198 /// been published here. Both reads happened in this pass.
199 Nothing,
200 /// A legacy single-object document (no pointer); the fast path asks.
201 Legacy,
202}
203
204impl Syncer {
205 /// Verify the cell still names us at OUR epoch; anything else is a
206 /// fence. Read-verify before the manifest CAS (the per-request
207 /// validation the gateway will also enforce).
208 async fn verify_not_deposed(&self) -> LeanResult<()> {
209 let lease = self
210 .lease
211 .as_ref()
212 .ok_or_else(|| LeanError::State("no lease".into()))?;
213 match self.store.epoch_read(&self.cfg.epoch_key()).await? {
214 Some(state) if state.epoch == lease.epoch && state.holder_id == lease.holder_id => {
215 Ok(())
216 }
217 Some(state) => Err(LeanError::Fenced(format!(
218 "cell at epoch {} holder {} (we are epoch {})",
219 state.epoch, state.holder_id, lease.epoch
220 ))),
221 None => Err(LeanError::Fenced("epoch cell vanished".into())),
222 }
223 }
224
225 /// Step 1 (P1-lite, 2026-09-25): what the tree is OWED, derived from
226 /// the document and taken in one pass. A path is owed where the
227 /// document differs from the baseline — which is also the merge base —
228 /// and the tree is clean there; only a path the tree HOLDS or its
229 /// scope covers (what the scope declined stays declined). Nothing is
230 /// stored between deriving it and taking it, so nothing can be
231 /// overtaken: the queue this replaces needed a prune for that (L-123).
232 /// A DIRTY path is the agent's newer work and is not owed: it
233 /// publishes, and the merge records what it publishes over (R7, the
234 /// delete override). Unless its bytes already ARE the document's —
235 /// a restart between this writer's CAS and step 7 leaves exactly
236 /// that — and then the baseline simply follows (content convergence).
237 /// Returns how many paths it took, and what its read of the document
238 /// established ([`Seen`]) — the barrier's fast path needs exactly that,
239 /// and must not pay a second GET for it (an idle tick is the cell and
240 /// the pointer).
241 pub async fn consume_owed(&mut self) -> LeanResult<(usize, Seen)> {
242 // Taken, not copied (lever D): every return below hands it back,
243 // and one that does not costs the next load a parse, nothing more.
244 let mut baseline = self.state.take_baseline()?;
245 // THE CHEAP PATH (scan trigger). Only three things make a path
246 // newly owed: the document moved (the pointer is not the one last
247 // derived against — a peer, a UI save, or this tree's own commit
248 // over a document that had moved), the baseline moved (only this
249 // tree's own commit, which moves the pointer too), or the agent
250 // backed out of a path the last derive skipped as its work. So: one
251 // small GET, a stat per skipped path, never the entries.
252 //
253 // The pointer read here is handed to the load below rather than
254 // read again: a moved pointer needs the document behind THAT read,
255 // and an absent one sends the load straight to the legacy key.
256 let mut read: Option<Option<manifest::LoadedPointer>> = None;
257 if let Some(d) = baseline.derived_etag.clone() {
258 let lp = manifest::load_pointer(self.store.as_ref(), &self.cfg).await?;
259 if let Some(p) = &lp {
260 let still_theirs = baseline.skipped.iter().all(|path| {
261 check_contained(&self.cfg.root, path).is_ok()
262 && local_dirty(&self.cfg.root.join(path), baseline.entries.get(path))
263 });
264 if d == p.etag && still_theirs {
265 // The document this consume derived against: a replay of
266 // several writers' traces (formal/trace/storm2conf.py)
267 // places the consume after the install that wrote it.
268 self.trace("derive", serde_json::json!({"seq": p.pointer.seq, "cheap": true}));
269 self.state.commit_baseline(baseline)?;
270 return Ok((0, Seen::Pointer(p.pointer.seq, p.etag.clone())));
271 }
272 }
273 read = Some(lp);
274 }
275 let Some(current) = manifest::load_from(self.store.as_ref(), &self.cfg, read).await? else {
276 // Neither the pointer nor the legacy object exists: nothing
277 // has ever been published here, and this pass read both.
278 self.state.commit_baseline(baseline)?;
279 return Ok((0, Seen::Nothing));
280 };
281 let doc = ¤t.manifest;
282 self.trace("derive", serde_json::json!({"seq": doc.seq, "cheap": false}));
283 // H10: whatever this barrier does next — the fast path included —
284 // the ack names this document or a later one.
285 self.note_carrier_uncited(doc)?;
286 let scope = self.state.load_scope()?.map(|v| super::sync::Scope::new(&v));
287 let same = |a: &str, b: &str| a.trim_matches('"') == b.trim_matches('"');
288 let mut taken = 0usize;
289 // Something owed could not be taken now (a fetch that lost to a
290 // newer document, an unreadable path, a failed write): the next
291 // consume derives again even if the pointer does not move.
292 let mut left = false;
293 // What is not owed because the agent is working on it: owed the
294 // moment it backs out, which the cheap path checks.
295 let mut skipped: BTreeSet<String> = BTreeSet::new();
296
297 // Deletions first: a document that turned the file `build` into
298 // the directory `build/log.txt` owes both, and the file must go
299 // before the directory can be made.
300 let gone: Vec<String> =
301 baseline.entries.keys().filter(|p| !doc.entries.contains_key(*p)).cloned().collect();
302 for path in gone {
303 if check_contained(&self.cfg.root, &path).is_err() {
304 // Never materializable here, so nothing to remove.
305 baseline.entries.remove(&path);
306 baseline.prev_scan.remove(&path);
307 continue;
308 }
309 let local = self.cfg.root.join(&path);
310 let be = baseline.entries.get(&path).cloned();
311 let record = |kind: String| ConflictRecord {
312 path: path.clone(),
313 foreign_etag: String::new(),
314 preserved_key: None,
315 kind,
316 at_unix: now_unix(),
317 };
318 match std::fs::symlink_metadata(&local) {
319 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
320 self.trace("tombstone", serde_json::json!({"path": path, "action": "absent"}));
321 }
322 Err(e) => {
323 // Unreadable is not absent.
324 self.trace("tombstone", serde_json::json!({"path": path, "action": "deferred"}));
325 self.state.append_conflict(&record(format!(
326 "consume-foreign-delete-deferred: cannot stat the path: {e}"
327 )))?;
328 left = true;
329 continue;
330 }
331 Ok(m) if !m.is_file() || local_dirty(&local, be.as_ref()) => {
332 // The agent's newer work: not owed. It publishes over
333 // the delete, with a record (`commit-recreated-deleted`).
334 // Owed again if the agent backs out.
335 self.trace("tombstone", serde_json::json!({"path": path, "action": "kept-dirty"}));
336 skipped.insert(path.clone());
337 continue;
338 }
339 Ok(_) => {
340 if let Err(e) = std::fs::remove_file(&local) {
341 self.state.append_conflict(&record(format!(
342 "consume-foreign-delete-failed (will retry): {e}"
343 )))?;
344 left = true;
345 continue;
346 }
347 // CONFIRM: only NotFound is absence.
348 if !matches!(
349 std::fs::symlink_metadata(&local),
350 Err(ref e) if e.kind() == std::io::ErrorKind::NotFound
351 ) {
352 self.state.append_conflict(&record(
353 "consume-foreign-delete-unconfirmed (will retry)".into(),
354 ))?;
355 left = true;
356 continue;
357 }
358 self.trace("tombstone", serde_json::json!({"path": path, "action": "removed"}));
359 taken += 1;
360 }
361 }
362 baseline.entries.remove(&path);
363 baseline.prev_scan.remove(&path);
364 }
365
366 for (path, e) in &doc.entries {
367 let be = baseline.entries.get(path).cloned();
368 if be.as_ref().is_some_and(|b| same(&b.etag, &e.etag) && b.cites_mode(e.mode)) {
369 continue;
370 }
371 let held = be.is_some() || scope.as_ref().is_none_or(|s| s.covers(path));
372 if !held {
373 continue;
374 }
375 if let Err(err) = check_contained(&self.cfg.root, path) {
376 // A path this tree can never safely materialise: surfaced,
377 // and not left owed — nothing here will ever take it.
378 self.state.append_conflict(&ConflictRecord {
379 path: path.clone(),
380 foreign_etag: e.etag.clone(),
381 preserved_key: None,
382 kind: format!("consume-refused-containment: {err}"),
383 at_unix: now_unix(),
384 })?;
385 self.trace("consume", serde_json::json!({"path": path, "etag": e.etag, "from": "manifest", "action": "refused"}));
386 continue;
387 }
388 let local = self.cfg.root.join(path);
389 if local_dirty(&local, be.as_ref()) {
390 // CONTENT CONVERGENCE: the tree's bytes are the document's.
391 let converged = std::fs::symlink_metadata(&local).is_ok_and(|m| m.is_file() && m.len() == e.size)
392 && file_crc(&self.cfg.root, path).is_ok_and(|c| same(&crc64_to_b64(c), &e.crc64_b64));
393 if converged {
394 let st = std::fs::metadata(&local)?;
395 baseline.entries.insert(
396 path.clone(),
397 BaselineEntry {
398 etag: e.etag.clone(),
399 key: Some(e.key.clone()),
400 generation: e.generation,
401 size: st.len(),
402 mtime_unix: mtime_of(&st),
403 mtime_nanos: Some(mtime_nanos_of(&st)),
404 // The DOCUMENT's mode, as every entry records:
405 // the bytes converged, the mode may not have,
406 // and the tree's left here would never be
407 // published.
408 mode: Some(e.mode & scan::PERM_BITS),
409 racy: super::barrier::racy_stat(&st),
410 change: Some(super::scan::ChangeStamp::of(&st)),
411 crc64_b64: Some(e.crc64_b64.clone()),
412 },
413 );
414 baseline.prev_scan.insert(path.clone());
415 self.trace("consume", serde_json::json!({"path": path, "etag": e.etag, "from": "manifest", "action": "converged"}));
416 taken += 1;
417 } else {
418 // The agent's work, not owed — until it backs out.
419 skipped.insert(path.clone());
420 }
421 continue;
422 }
423 let (meta, body) = match self.store.get_whole(&e.key, Some(&e.etag)).await {
424 Ok(got) => got,
425 // A newer document replaced it (and its collector took the
426 // handle): owed again against that one.
427 Err(StoreError::NotFound(_)) | Err(StoreError::PreconditionFailed(_)) => {
428 self.trace("consume", serde_json::json!({"path": path, "etag": e.etag, "from": "manifest", "action": "missing"}));
429 left = true;
430 continue;
431 }
432 Err(err) => return Err(err.into()),
433 };
434 // VERIFIED before it is written, as checkout's fetch is: against
435 // the CRC the document cites, else the backend's attestation.
436 let got = crc64_to_b64(crc64_nvme(&body));
437 let want = Some(e.crc64_b64.clone()).filter(|c| !c.is_empty()).or_else(|| meta.crc64_b64.clone());
438 if let Some(want) = want {
439 if want != got {
440 self.state.append_conflict(&ConflictRecord {
441 path: path.clone(),
442 foreign_etag: e.etag.clone(),
443 preserved_key: None,
444 kind: format!(
445 "consume-refused-checksum: etag {} is cited with CRC-64 {want} but the \
446 bytes fetched under it hash to {got}",
447 e.etag
448 ),
449 at_unix: now_unix(),
450 })?;
451 self.trace("consume", serde_json::json!({"path": path, "etag": e.etag, "from": "manifest", "action": "refused-checksum"}));
452 left = true;
453 continue;
454 }
455 }
456 // The mode the DOCUMENT cites, as checkout applies — not the
457 // object's stamp, which a gateway write does not carry (its
458 // file came out 0644 here and its cited mode in a checkout).
459 let mode = Some(e.mode);
460 consume_window("before-write", path);
461 // H7: the licence to overwrite is re-checked with the temp
462 // written, immediately before the rename; a write since is the
463 // agent's, and it stays (and publishes).
464 let still_clean = || !local_dirty(&local, be.as_ref());
465 let st = match contained_path(&self.cfg.root, path)
466 .and_then(|target| {
467 if self.cfg.tree_reserve {
468 super::safefs::leave_the_reserve(&self.cfg.root, path, body.len() as u64, self.cfg.tree_room_for_tests)?;
469 }
470 write_file_atomic_if(&target, &body, mode, &still_clean)
471 })
472 {
473 Ok(Some(st)) => st,
474 // The agent wrote first: as a dirty path above.
475 Ok(None) => {
476 skipped.insert(path.clone());
477 continue;
478 }
479 Err(err) => {
480 // TRANSIENT (a full disk is not a planted symlink): the
481 // path stays owed and the record repeats.
482 self.state.append_conflict(&ConflictRecord {
483 path: path.clone(),
484 foreign_etag: e.etag.clone(),
485 preserved_key: None,
486 kind: format!("consume-write-failed (will retry): {err}"),
487 at_unix: now_unix(),
488 })?;
489 self.trace("consume", serde_json::json!({"path": path, "etag": e.etag, "from": "manifest", "action": "deferred"}));
490 left = true;
491 continue;
492 }
493 };
494 consume_window("after-rename", path);
495 // The baseline records the inode this consume WROTE (H7).
496 let stamps = GenerationStamps::from_meta(&meta.meta);
497 baseline.entries.insert(
498 path.clone(),
499 BaselineEntry {
500 etag: meta.etag.clone(),
501 key: Some(e.key.clone()),
502 generation: stamps.map(|s| s.generation).unwrap_or(e.generation),
503 size: st.len(),
504 mtime_unix: mtime_of(&st),
505 mtime_nanos: Some(mtime_nanos_of(&st)),
506 mode: Some(e.mode & scan::PERM_BITS),
507 racy: super::barrier::racy_stat(&st),
508 change: Some(super::scan::ChangeStamp::of(&st)),
509 crc64_b64: Some(got),
510 },
511 );
512 baseline.prev_scan.insert(path.clone());
513 self.trace("consume", serde_json::json!({"path": path, "etag": e.etag, "from": "manifest", "action": "adopted"}));
514 taken += 1;
515 }
516 baseline.derived_etag = (!left).then(|| current.etag.clone());
517 baseline.skipped.sync_to(&skipped);
518 if !left {
519 // Everything owed is taken: the tree is integrated with this
520 // document (its dirty paths are the agent's, about to publish).
521 baseline.seq = doc.seq;
522 baseline.manifest_etag = Some(current.etag.clone());
523 }
524 // The files this baseline vouches for reached the disk before the
525 // baseline does (audit 2026-09-03, finding 9).
526 self.state.sync_tree()?;
527 self.state.commit_baseline(baseline)?;
528 // The pointer as this load saw it (a legacy single object has none,
529 // and the fast path asks the bucket itself).
530 let seen = match ¤t.pointer {
531 Some(_) => Seen::Pointer(doc.seq, current.etag.clone()),
532 None => Seen::Legacy,
533 };
534 Ok((taken, seen))
535 }
536
537 /// The cell's verb requests alone (§2.5), as a barrier's step 1 reads
538 /// them: no consume, no fetch.
539 pub async fn read_cell_requests(&mut self) -> LeanResult<()> {
540 let doc = inbox::load(self.store.as_ref(), &self.cfg).await?.doc;
541 self.note_verb_requests(&doc)
542 }
543
544 /// Act on the inbox document's two request fields (§2.5, D14).
545 ///
546 /// The asymmetry is deliberate and is about blast radius, not
547 /// principle: a boundary request is PERFORMED (it publishes what is
548 /// already on disk and mutates nothing local), a sync request is
549 /// CARRIED into the ticker as advisory news and the agent decides.
550 ///
551 /// Both are watermarked by `requested_unix` rather than consumed by
552 /// a clearing CAS. The field is idempotent state, so a repeat is a
553 /// no-op and a lost clear cannot double-publish; and not writing
554 /// means the doors cost no request at all, which is what §2.5
555 /// promises.
556 fn note_verb_requests(&mut self, doc: &inbox::InboxDoc) -> LeanResult<()> {
557 let mut mark = self.load_verb_watermark()?;
558 if let Some(r) = &doc.sync_request {
559 if r.requested_unix > mark.sync_unix {
560 self.carry_sync_request(r.requested_unix, &r.requestor)?;
561 mark.sync_unix = r.requested_unix;
562 }
563 }
564 if let Some(r) = &doc.boundary_request {
565 if r.requested_unix > mark.boundary_unix {
566 self.request_boundary(
567 &format!("gw:{}:{}", r.requested_unix, r.requestor),
568 Some(format!("boundary requested by {}", r.requestor)),
569 )?;
570 mark.boundary_unix = r.requested_unix;
571 }
572 }
573 self.save_verb_watermark(&mark)
574 }
575
576 fn verb_watermark_path(&self) -> std::path::PathBuf {
577 self.cfg.state_dir().join("verb-requests.json")
578 }
579
580 pub(crate) fn load_verb_watermark(&self) -> LeanResult<VerbWatermark> {
581 let p = self.verb_watermark_path();
582 if !p.exists() {
583 return Ok(VerbWatermark::default());
584 }
585 Ok(std::fs::read(&p)
586 .ok()
587 .and_then(|b| serde_json::from_slice(&b).ok())
588 .unwrap_or_default())
589 }
590
591 fn save_verb_watermark(&self, m: &VerbWatermark) -> LeanResult<()> {
592 let bytes = serde_json::to_vec_pretty(m)
593 .map_err(|e| LeanError::State(format!("verb watermark: {e}")))?;
594 super::control::write_atomic(&self.verb_watermark_path(), &bytes)
595 }
596
597 /// The consume's dirty case: the agent's local version wins; the
598 /// FOREIGN bytes are preserved first (a conflict record keeps both
599 /// recoverable), then the recognized ETag advances so the eventual
600 /// publish supersedes them KNOWINGLY.
601 /// Preserve the bytes at `key` (a handle, or an ingress object) under
602 /// `conflicts/<uuid>/<path>`, by a server-side copy guarded on `etag`:
603 /// no bytes cross this process, and a 10 GB checkpoint is preserved
604 /// in one request. The copy is what a conflict record names, and what
605 /// outlives the handle once the collector or the orphan sweep takes it.
606 async fn preserve_conflict_copy(&self, path: &str, key: &str, etag: &str) -> LeanResult<String> {
607 let dst = self.cfg.conflict_key(&uuid::Uuid::new_v4().to_string(), path);
608 let stamps = GenerationStamps {
609 generation: 0,
610 epoch: self
611 .lease
612 .as_ref()
613 .map(|l| l.epoch)
614 .or_else(|| self.state.load_incarnation().ok().flatten().map(|i| i.epoch))
615 .unwrap_or(0),
616 flush_uuid: "conflict-preserve".into(),
617 boundary_source: None,
618 posix: None,
619 };
620 self.store
621 .copy_object(key, Some(etag), &dst, &PutCondition::IfNoneMatchAny, &stamps)
622 .await?;
623 Ok(dst)
624 }
625
626 /// Take the SECOND absence observation now, by direct `lstat`.
627 ///
628 /// `scan::classify` withholds a path's delete until absence has
629 /// survived two consecutive scans — the rename-vs-walk race guard
630 /// (`lib.rs:37`). The hazard that rule names is the WALK missing a
631 /// file renamed under it, not deletion itself. On the cadence path
632 /// the second walk arrives at the next floor tick and nobody has
633 /// been promised otherwise. On a DECLARED boundary it cannot wait:
634 /// the ack would claim a coherent point (D1 — "everything
635 /// ordered-before T") while the manifest still cites a file the
636 /// agent removed before it touched the sentinel, and it would say
637 /// so with `report.deleted: 0` and `status: "ok"`.
638 ///
639 /// A direct `lstat` is exactly what the rename-vs-walk guard asks
640 /// for and is immune to the walk race by construction, so the
641 /// second observation costs one syscall per transiently-absent
642 /// path — never a second full pass. The cadence path is unchanged
643 /// and still waits for the second walk.
644 pub(crate) fn confirm_absences(
645 &self,
646 classified: &mut scan::Classified,
647 held: &mut scan::Scan,
648 ) -> LeanResult<usize> {
649 if classified.first_absence.is_empty() {
650 return Ok(0);
651 }
652 let mut confirmed = 0;
653 for path in std::mem::take(&mut classified.first_absence) {
654 // Only a path that cannot be there is absence: nothing at it
655 // (ENOENT), or a component above it that is not a directory
656 // (ENOTDIR, ELOOP), which the walk would not have descended
657 // either (`scan::name_error`, the walk's rule). Every other
658 // errno — EACCES on a parent, EIO, EMFILE — used to read as
659 // "the agent deleted it", and this is the oracle that
660 // PUBLISHES the delete and then DELETES THE OBJECT. An
661 // unreadable path is the one case where guessing is
662 // unrecoverable, so it is never a delete: EACCES or EPERM holds
663 // it, as the walk holds what it may not read
664 // (`scan::Scan::held`), and every other errno fails the
665 // barrier, as it fails the walk. Two call sites, one syscall:
666 // they must not have opposite error policies.
667 let gone = match std::fs::symlink_metadata(self.cfg.root.join(&path)) {
668 Err(e) => match scan::name_error(&e) {
669 scan::NameError::Gone | scan::NameError::Replaced => true,
670 scan::NameError::Refused => {
671 held.hold(path, scan::Hold::File);
672 continue;
673 }
674 scan::NameError::Other => {
675 classified.first_absence.insert(path.clone());
676 return Err(LeanError::State(format!(
677 "cannot confirm the absence of {path:?}: {e} — refusing to publish \
678 a deletion on an unreadable path"
679 )));
680 }
681 },
682 Ok(_) => false,
683 };
684 if gone {
685 classified.deletes.insert(path);
686 confirmed += 1;
687 } else {
688 // Back on disk: the rename-vs-walk race, caught exactly
689 // as the rule intends. Still only a first absence.
690 classified.first_absence.insert(path);
691 }
692 }
693 Ok(confirmed)
694 }
695
696 /// Steps 2–7. `barrier` = one full publish cycle (the cadence arm).
697 pub async fn run_barrier(&mut self) -> LeanResult<BarrierReport> {
698 self.barrier_inner(false, "manual").await
699 }
700
701 /// The floor tick's barrier. Stamped `cadence` so the bucket can
702 /// answer "which clock installed this boundary?" without the ack —
703 /// the ack is a LOCAL file, and the fleet-visible question is asked
704 /// by operators and by the gateway's `/status`, neither of which
705 /// can see it.
706 pub async fn cadence_barrier(&mut self) -> LeanResult<BarrierReport> {
707 self.barrier_inner(false, "cadence").await
708 }
709
710 /// A barrier whose result somebody is going to ACK: a sentinel
711 /// honor (D1) or the preStop drain (D10). Identical to
712 /// `run_barrier` except that it confirms first-absence paths rather
713 /// than acking a boundary that withholds them.
714 pub async fn declared_barrier(&mut self) -> LeanResult<BarrierReport> {
715 self.barrier_inner(true, "sentinel").await
716 }
717
718 /// `declared_barrier` under an explicit provenance stamp — the
719 /// drain and the budget-deferred honor are the same barrier
720 /// reporting a different clock.
721 pub async fn declared_barrier_as(&mut self, source: &str) -> LeanResult<BarrierReport> {
722 self.barrier_inner(true, source).await
723 }
724
725 async fn barrier_inner(&mut self, declared: bool, source: &str) -> LeanResult<BarrierReport> {
726 // Every publishing path ends here — cadence, sentinel, drain, the
727 // one-shot verb — so a reader is refused here before the first
728 // request, whichever of them a caller reached. The callers that
729 // owe an answer (the publish sentinel, the drain) give it before
730 // this; this is the backstop for the ones that do not.
731 self.refuse_if_read("barrier")?;
732 let mut clock = StepClock::start(self.store.clone());
733 // Review 2026-09-18, H10: a barrier that begins while a publish
734 // declaration stands CARRIES it, whoever called — the floor's
735 // cadence barrier after a failed honor as much as the honor itself
736 // (the model's `DSet` and `InstallSource` read the live pending,
737 // not the caller). So it confirms first absences as a declared
738 // barrier does, stamps the deferred clock, and journals itself
739 // with its CAS as the install the ack names (`IntentJournal::
740 // carrier`). A record written before the id field has none and
741 // carries nothing.
742 // Step 1's first half: the cell's verb requests (§2.5's doors ride
743 // this GET), read BEFORE the carry is decided. Read after it, a
744 // gateway `boundary_request` folded into the pending here was not
745 // carried by the barrier that read it: its install journalled no
746 // carrier, and a peer's delete of a path it published, before the
747 // honour, was answered `ok` (L-135, 2026-10-14). A failure here
748 // must never fail the barrier: the request is idempotent state and
749 // the next tick re-reads it.
750 let inbox_doc = inbox::load(self.store.as_ref(), &self.cfg).await?.doc;
751 if let Err(e) = self.note_verb_requests(&inbox_doc) {
752 eprintln!("flint-sync: verb request not consumed (retrying next tick): {e}");
753 }
754 let carries: Option<String> = self
755 .load_pending(super::sentinel::Verb::Publish)
756 .ok()
757 .flatten()
758 .map(|p| p.id)
759 .filter(|id| !id.is_empty());
760 let declared = declared || carries.is_some();
761 let source = if carries.is_some() && source == "cadence" { "sentinel-deferred" } else { source };
762 clock.lap("cell");
763 // No lease is held until the commit section (design 2026-09-13
764 // §4): the consume, the scan and the uploads below run with the
765 // cell at rest, guarded by each object's own etag. What the
766 // uploads are STAMPED with is the last epoch this incarnation
767 // held — informational, like every epoch stamp on an object; the
768 // manifest entries carry the commit section's real epoch.
769 let epoch_hint = self.state.load_incarnation()?.map(|i| i.epoch).unwrap_or(0);
770 let mut report = BarrierReport::default();
771 let barrier_started = std::time::Instant::now();
772 self.trace("barrier_start", serde_json::json!({"source": source, "declared": declared}));
773
774 // STEP 0: finish any rescope a crash left half-applied, before
775 // the scan reads the tree (scoped-read design §4.3). This is the
776 // gate that makes the narrow verb safe at all: a half-applied
777 // narrow is a tree whose files and whose citations disagree, and
778 // `classify` reads that disagreement as an upload or as a
779 // DELETE depending on which half landed. Replay is idempotent,
780 // so the cost on the normal path is one `stat` of a file that
781 // is not there.
782 if let Some(r) = self.replay_scope_intent().await? {
783 report.rescope_replayed = true;
784 eprintln!(
785 "flint-sync: finished an interrupted rescope before the barrier \
786 (uncited {}, unlinked {}, materialised {})",
787 r.uncited, r.unlinked, r.materialized
788 );
789 }
790 clock.lap("rescope_replay");
791
792 // Step 1's second half: what the tree is owed. A UI save, delete or
793 // rename COMMITS (P2), so it reaches the tree as a peer's change
794 // does: derived, not queued.
795 let (consumed, seen_pointer) = self.consume_owed().await?;
796 report.consumed = consumed;
797 clock.lap("consume");
798 // The agent runs alongside: what it does to the tree between the
799 // removal pass above and the scan below is the model's interleaving
800 // of a local edit between Consume and Scan (the rename world's
801 // tenth box run reached its counterexample through it). A test
802 // window, like the consume's (H7); nothing in it calls the store.
803 consume_window("before-scan", "");
804 // Step 2: scan-diff against the persisted baseline, taken from the
805 // store and handed back at each end of the barrier (lever D).
806 let mut baseline = self.state.take_baseline()?;
807 clock.lap("baseline_load");
808 let mut scanned = scan::scan_with(&self.cfg.root, self.cfg.scan_threads)?;
809 // A path too long for a store key has none either (L-145): held,
810 // as a name that is not UTF-8 is, before anything reads the scan.
811 // Its PUT used to fail on S3 at every barrier.
812 let unkeyed: Vec<String> = scanned.entries.keys().filter(|p| !self.cfg.key_fits(p)).cloned().collect();
813 for p in unkeyed {
814 scanned.entries.remove(&p);
815 scanned.hold(p, scan::Hold::File);
816 }
817 clock.lap("walk");
818 let mut classified = scan::classify(&scanned, &baseline);
819 // An unsure entry the hash vouched for records the change stamp it
820 // was hashed under and whether it is still racy; written only where
821 // that changed, so an idle tick writes nothing.
822 for v in scan::confirm_racy(&self.cfg.root, &scanned, &baseline, &mut classified) {
823 let stale = baseline.entries.get(&v.path).is_some_and(|e| e.racy != v.racy || e.change != Some(v.change));
824 if stale {
825 if let Some(e) = baseline.entries.get_mut(&v.path) {
826 e.racy = v.racy;
827 e.change = Some(v.change);
828 }
829 }
830 }
831 // A file the syncer may not read is held, not uploaded
832 // (`Scan::held`). Asked here, before the idle check, so a standing
833 // `chmod 000` leaves an otherwise idle tick idle.
834 let unreadable: Vec<String> =
835 classified.uploads.iter().filter(|p| !scan::may_read(&self.cfg.root, p)).cloned().collect();
836 for p in unreadable {
837 classified.uploads.remove(&p);
838 scanned.hold(p, scan::Hold::File);
839 }
840 if declared {
841 report.absences_confirmed = self.confirm_absences(&mut classified, &mut scanned)?;
842 }
843 report.first_absence = classified.first_absence.iter().cloned().collect();
844 report.held = scanned.held.iter().cloned().collect();
845 if !report.held.is_empty() {
846 note_held(&report.held);
847 }
848 clock.lap("classify");
849
850 // Skip-on-no-diff: nothing local, and the bucket manifest where the
851 // consume left it.
852 if classified.uploads.is_empty()
853 && classified.deletes.is_empty()
854 && classified.first_absence.is_empty()
855 {
856 // Read the POINTER, never the entries: the 0b rig measured
857 // the idle tick at 1M entries as 27 s / 1.3 GiB — dominated
858 // by fetching and parsing a 264 MiB document just to read
859 // `seq`. The pointer is a few hundred bytes and carries the
860 // seq in its body, so a GET of it costs one round trip and
861 // answers strictly more than a HEAD of the old object did.
862 //
863 // Legacy workspaces (no pointer yet) keep the HEAD they had.
864 let pointer = match seen_pointer {
865 // The consume just read it: no second GET.
866 Seen::Pointer(seq, etag) => Some(Some((seq, etag))),
867 // The consume read the pointer AND the legacy key and found
868 // neither: decided below with no request at all.
869 Seen::Nothing => None,
870 Seen::Legacy => Some(
871 manifest::load_pointer(self.store.as_ref(), &self.cfg).await?.map(|l| (l.pointer.seq, l.etag)),
872 ),
873 };
874 let unchanged = match pointer {
875 // Nothing published: the answer the 404 arm reaches, read
876 // once in this pass instead of three times (B8 trace,
877 // 2026-10-03: three pointer GETs and a legacy HEAD a tick).
878 None => baseline.manifest_etag.is_none(),
879 Some(Some((seq, etag))) => {
880 // The news ticker rides this request for free — D5's
881 // "zero added bucket requests" is still literal.
882 report.observed_seq = Some(seq);
883 report.observed_etag = Some(etag.clone());
884 baseline.manifest_etag.as_deref() == Some(etag.as_str())
885 }
886 Some(None) => match self.store.head(&self.cfg.manifest_key()).await {
887 Ok(meta) => {
888 report.observed_seq =
889 GenerationStamps::from_meta(&meta.meta).map(|s| s.generation);
890 report.observed_etag = Some(meta.etag.clone());
891 baseline.manifest_etag.as_deref() == Some(meta.etag.as_str())
892 }
893 Err(StoreError::NotFound(_)) => baseline.manifest_etag.is_none(),
894 Err(e) => return Err(e.into()),
895 },
896 };
897 if unchanged {
898 report.no_change = true;
899 report.seq = Some(baseline.seq);
900 // prev_scan still advances (the two-scan rule's clock), and
901 // a racy flag the hash cleared is kept (unsaved, every later
902 // barrier would hash that file again). Handed back either
903 // way: the store writes what changed, and nothing when
904 // nothing did. `sync_to` walks both sorted sides and copies
905 // only the paths that came or went.
906 baseline.prev_scan.sync_to(scanned.present(&baseline.entries));
907 self.state.commit_baseline(baseline)?;
908 clock.lap("idle_check");
909 report.steps = clock.steps;
910 self.trace_barrier_end(&report, barrier_started);
911 return Ok(report);
912 }
913 }
914
915 // Step 3: intent journal, then the window (the commitment
916 // point: the same CAS drops the consumed entries).
917 let flush_uuid = uuid::Uuid::new_v4().to_string();
918 self.trace("scan", serde_json::json!({"flush": flush_uuid, "uploads": classified.uploads.len(),
919 "deletes": classified.deletes.len(), "first_absence": classified.first_absence.len()}));
920 let mut intent = self.state.load_intent()?;
921 intent = IntentJournal { flush_uuid: flush_uuid.clone(), carrier: intent.carrier.clone() };
922 self.state.save_intent(&intent)?;
923 clock.lap("intent");
924 // No window (P2): a UI save commits without the lease, and the
925 // uploads below race it the way two writers race each other —
926 // the commit's CAS decides, and the loser is preserved and
927 // recorded.
928
929 // Step 4: guarded uploads, fanned out under a bounded window
930 // (each key's guard chain is independent; the 412 policy and
931 // conflict records are applied to the collected results below,
932 // in deterministic path order).
933 let mut upserts: BTreeMap<String, LeanEntry> = BTreeMap::new();
934 let mut parked: BTreeSet<String> = BTreeSet::new();
935 // Citations whose etag was observed with no lease held (adopted
936 // uploads, citation repairs): re-read inside the commit section.
937 let mut observed: BTreeSet<String> = BTreeSet::new();
938 let mut new_baseline_entries: BTreeMap<String, BaselineEntry> = BTreeMap::new();
939 let outcomes: Vec<(String, LeanResult<UploadOutcome>)> = {
940 use futures::stream::{self, StreamExt};
941 // CHUNKED into waves, so a wave's outcomes are collected
942 // before the next is fanned out. No lease is held here
943 // (design 2026-09-13 §4), so nothing renews or fences
944 // between chunks any more — the between-chunk fence that
945 // stopped a deposed straggler's PUTs went with the straggler:
946 // a PUT outside the lease is one legitimate writer's PUT,
947 // guarded by If-Match, and a collision is preserved and
948 // superseded, never silently overwritten.
949 let mut outcomes: Vec<(String, LeanResult<UploadOutcome>)> = Vec::new();
950 // NOT cfg.fanout. That knob was raised to 128 for the READ
951 // path, and the read-side measurement (2.5x on 20k small
952 // files) says nothing about this one, so uploads keep the
953 // value they were measured at until someone measures them.
954 // Memory is no longer the reason they are separate: this
955 // path is byte-bounded too now, by the store's gate
956 // (`with_upload_inflight_max_bytes`), which charges every
957 // whole body below and every multipart part inside the
958 // store from before its read until its PUT returns.
959 let fanout = self.cfg.upload_fanout.max(1);
960 let chunk = fanout.saturating_mul(UPLOAD_CHUNK_WAVES).max(1);
961 let all: Vec<&String> = classified.uploads.iter().collect();
962 for group in all.chunks(chunk) {
963 {
964 let this: &Syncer = &*self;
965 let mut part: Vec<(String, LeanResult<UploadOutcome>)> =
966 stream::iter(group.iter().map(|path| {
967 let path = *path;
968 let scanned_entry = &scanned.entries[path];
969 let base = baseline.entries.get(path);
970 let flush_uuid = &flush_uuid;
971 async move {
972 let r = this
973 .upload_one(path, scanned_entry, base, epoch_hint, flush_uuid)
974 .await;
975 (path.clone(), r)
976 }
977 }))
978 .buffer_unordered(fanout)
979 .collect()
980 .await;
981 outcomes.append(&mut part);
982 }
983 }
984 outcomes
985 };
986 let mut outcomes = outcomes;
987 outcomes.sort_by(|a, b| a.0.cmp(&b.0));
988 for (path, outcome) in outcomes {
989 let path = &path;
990 match outcome? {
991 UploadOutcome::Published { entry, baseline_entry, adopted } => {
992 self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "etag": entry.etag,
993 "key": entry.key, "outcome": if adopted { "adopted" } else { "put" }}));
994 if adopted {
995 observed.insert(path.clone());
996 }
997 report.published_bytes += entry.size;
998 upserts.insert(path.clone(), entry);
999 new_baseline_entries.insert(path.clone(), baseline_entry);
1000 report.uploaded.push(path.clone());
1001 }
1002 UploadOutcome::Deferred => {
1003 self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "outcome": "deferred"}));
1004 report.deferred.push(path.clone());
1005 }
1006 UploadOutcome::Unreadable => {
1007 // Made unreadable since `may_read` asked: held, as it
1008 // would have been then.
1009 self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "outcome": "held"}));
1010 classified.uploads.remove(path);
1011 scanned.hold(path.clone(), scan::Hold::File);
1012 report.held.push(path.clone());
1013 note_held(std::slice::from_ref(path));
1014 }
1015 }
1016 }
1017 clock.lap("upload");
1018
1019 // A PULL-ONLY boundary: nothing uploaded or deleted, so the merge
1020 // can only add nothing — and then the commit section writes nothing
1021 // to the bucket (no CAS, no GC) and what remains is local: note what
1022 // theirs owes this tree and take its seq. That needs no
1023 // fence and no window; claiming for it queued every such writer
1024 // behind the publishing ones (65 of 191 claims in the writers drill).
1025 // A merge that does add something (a mirror flag to restamp, say)
1026 // falls through to the commit section.
1027 if classified.uploads.is_empty()
1028 && classified.deletes.is_empty()
1029 && upserts.is_empty()
1030 && observed.is_empty()
1031 {
1032 let opened = if self.cfg.chunk_view {
1033 DocView::open(self.store.clone(), &self.cfg, None).await?
1034 } else {
1035 Opened::Full(manifest::load(self.store.as_ref(), &self.cfg).await?)
1036 };
1037 // When the merge adds nothing: the document's seq and etag, its
1038 // entries at the carried paths, and the foreign count.
1039 let pulled: Option<(u64, String, manifest::LeanManifest, usize)> = match opened {
1040 Opened::Full(current) => {
1041 let m = self.merge_onto(
1042 current.as_ref(),
1043 &merge_base(&baseline),
1044 &upserts,
1045 &classified.deletes,
1046 &parked,
1047 &flush_uuid,
1048 );
1049 match m.expected.as_ref().filter(|_| m.adds_nothing()).map(|h| h.etag.clone()) {
1050 Some(etag) => Some((m.theirs.seq, etag, m.theirs, m.foreign.len())),
1051 None => None,
1052 }
1053 }
1054 Opened::View(mut v) => {
1055 // Lever G: no upload and no delete, so the merge adds
1056 // nothing unless it restamps the mirror flag.
1057 let full = if self.cfg.event_trace.is_some() || self.cfg.g_shadow {
1058 let doc = v.assemble().await?;
1059 Some(self.merge_onto(
1060 Some(&doc),
1061 &merge_base(&baseline),
1062 &upserts,
1063 &classified.deletes,
1064 &parked,
1065 &flush_uuid,
1066 ))
1067 } else {
1068 None
1069 };
1070 let adds_nothing = self.cfg.sole_writer == v.pointer.sole_writer;
1071 if let Some(f) = full.as_ref().filter(|_| self.cfg.g_shadow) {
1072 assert_eq!(adds_nothing, f.adds_nothing(), "lever G: a pull-only adds-nothing is not the whole-document merge's");
1073 }
1074 if adds_nothing {
1075 let carried = self.carried_paths()?;
1076 v.prefetch(carried.iter().map(|p| p.as_str())).await?;
1077 let mut entries = BTreeMap::new();
1078 for p in &carried {
1079 if let Some(e) = v.get(p).await? {
1080 entries.insert(p.clone(), e);
1081 }
1082 }
1083 let at = manifest::LeanManifest { seq: v.seq(), entries, ..Default::default() };
1084 Some((v.seq(), v.etag.clone(), at, full.map(|f| f.foreign.len()).unwrap_or(0)))
1085 } else {
1086 None
1087 }
1088 }
1089 };
1090 if let Some((seq, etag, theirs_at, foreign)) = pulled {
1091 // What theirs changed at a path this tree holds is owed: the
1092 // next consume derives it, because theirs is not the document
1093 // it last derived against (or is, and there is nothing).
1094 self.note_carrier_uncited(&theirs_at)?;
1095 report.seq = Some(seq);
1096 report.no_change = true;
1097 report.manifest_etag = Some(etag.clone());
1098 report.observed_seq = Some(seq);
1099 report.observed_etag = Some(etag.clone());
1100 report.foreign_queued = foreign;
1101 baseline.seq = seq;
1102 baseline.manifest_etag = Some(etag.clone());
1103 baseline.prev_scan.sync_to(scanned.present(&baseline.entries));
1104 self.state.commit_baseline(baseline)?;
1105 self.state.clear_intent_keys()?;
1106 clock.lap("pull_only");
1107 report.steps = clock.steps;
1108 self.trace_barrier_end(&report, barrier_started);
1109 return Ok(report);
1110 }
1111 clock.lap("pull_only");
1112 }
1113
1114 // THE COMMIT SECTION (design 2026-09-13 §4). Everything above ran
1115 // with no lease: the uploads are durable and etag-guarded, and
1116 // nothing is cited yet. Claim the cell now — for the merge, the
1117 // CAS, the deletes and the baseline — and hand it to the next
1118 // waiter when done. This is the only stretch two writers of one
1119 // workspace serialise on, and it is small: one pointer CAS and
1120 // the guarded deletes, milliseconds to seconds, never the upload
1121 // of a checkpoint.
1122 let held = super::lease::claim(self).await?;
1123 let epoch = held.epoch;
1124 eprintln!("flint-sync: publish fence held (epoch {epoch})");
1125 clock.lap("claim");
1126 if self.cfg.drill_hold_commit_secs > 0 {
1127 eprintln!(
1128 "flint-sync: DRILL: holding the fence for {}s inside the commit section",
1129 self.cfg.drill_hold_commit_secs
1130 );
1131 tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_commit_secs)).await;
1132 clock.lap("drill_hold");
1133 }
1134 let commit: LeanResult<()> = async {
1135 // The entries carry the epoch the commit HOLDS, not the hint
1136 // the uploads were stamped with.
1137 for e in upserts.values_mut() {
1138 e.epoch = epoch;
1139 }
1140 // Step 5: the manifest CAS (three-way merge; bounded retries).
1141 // The cell is read after each manifest load below, not here: a
1142 // read here preceded the HEAD fan-out and the window, and a
1143 // holder deposed after it CASed onto its successor's document
1144 // (review 2026-09-18, H2).
1145 // EVERY citation this barrier adds was read or written with no
1146 // lease held. Another writer's commit may since have uncited
1147 // the path and collected the object: its GC deletes by the etag
1148 // it integrated. That etag names an ADOPTED object (the model's
1149 // LeanBarrierLeaseAdoptBlind) — and, since an S3 whole-PUT etag
1150 // is the MD5 of the bytes, it also names this barrier's own PUT
1151 // of IDENTICAL bytes (finding 13, the writers drill's A3: a
1152 // same-content rewrite re-uploaded, the peer's GC deleted it,
1153 // the commit cited a hole). Collection runs only inside a
1154 // commit section, so a re-read HERE, holding the fence, cannot
1155 // be overtaken before this CAS — held, that is, until the read
1156 // of the cell after the load: a deposal before it is seen
1157 // there, and after it the successor's rotation moves the
1158 // pointer the CAS expects (H2). Whatever is gone or replaced is
1159 // withheld: parked, recorded, and left dirty for the next
1160 // barrier to publish with a PUT of its own.
1161 let heads: Vec<(String, LeanResult<bool>)> = {
1162 use futures::stream::{self, StreamExt};
1163 let this: &Syncer = &*self;
1164 stream::iter(upserts.iter().map(|(path, cited)| async move {
1165 let still = match this.store.head(&cited.key).await {
1166 Ok(meta) => Ok(meta.etag == cited.etag),
1167 Err(StoreError::NotFound(_)) => Ok(false),
1168 Err(e) => Err(e.into()),
1169 };
1170 (path.clone(), still)
1171 }))
1172 .buffer_unordered(self.cfg.upload_fanout.max(1))
1173 .collect()
1174 .await
1175 };
1176 let mut heads = heads;
1177 heads.sort_by(|a, b| a.0.cmp(&b.0));
1178 for (path, still) in heads {
1179 let still_there = still?;
1180 let was_observed = observed.contains(&path);
1181 let cited = &upserts[&path];
1182 self.trace("observed", serde_json::json!({"flush": flush_uuid, "path": path, "etag": cited.etag, "still": still_there, "own_put": !was_observed}));
1183 if still_there {
1184 continue;
1185 }
1186 let gone = upserts.remove(&path).expect("looked up above");
1187 new_baseline_entries.remove(&path);
1188 report.uploaded.retain(|p| p != &path);
1189 report.published_bytes = report.published_bytes.saturating_sub(gone.size);
1190 // ADOPTED bytes that are the baseline's own (a file touched
1191 // without a change, re-uploaded by a restarted barrier under
1192 // the same flush) that went before the commit: nothing new
1193 // is lost, so the path is not parked, and the next consume
1194 // takes the manifest's version. (Once this was the citation
1195 // repair's arm; storm drill S0, 2026-09-15, showed parking
1196 // it kept a writer on withheld bytes for good.)
1197 let integrated = was_observed
1198 && baseline.entries.get(&path).is_some_and(|be| be.etag == gone.etag);
1199 if integrated {
1200 self.state.append_conflict(&ConflictRecord {
1201 path: path.clone(),
1202 foreign_etag: gone.etag,
1203 preserved_key: None,
1204 kind: "repair-withheld: the object this barrier re-cited was replaced or \
1205 collected before its commit; nothing cited, and the next consume \
1206 installs the manifest's version"
1207 .into(),
1208 at_unix: now_unix(),
1209 })?;
1210 continue;
1211 }
1212 parked.insert(path.clone());
1213 let kind = if was_observed {
1214 "adopt-withheld: the object this barrier found already at its key was \
1215 replaced or collected before its commit; nothing cited, the path stays \
1216 dirty and publishes next barrier"
1217 } else {
1218 "upload-withheld: the object this barrier PUT was replaced or collected \
1219 before its commit (a peer's GC recognizes the etag of identical bytes); \
1220 nothing cited, the path stays dirty and publishes next barrier"
1221 };
1222 self.state.append_conflict(&ConflictRecord {
1223 path: path.clone(),
1224 foreign_etag: gone.etag,
1225 preserved_key: None,
1226 kind: kind.into(),
1227 at_unix: now_unix(),
1228 })?;
1229 report.parked.push(path.clone());
1230 }
1231 clock.lap("verify");
1232 let mut foreign_entries: Vec<(String, LeanEntry)> = vec![];
1233 // Set when the merge added nothing to the document: nothing is
1234 // installed, and theirs becomes the merge base as it stands.
1235 let mut installed_nothing = false;
1236 let mut attempt = 0;
1237 // The last value carried out: the adoptions this install
1238 // declined for a move, whose entries stay in the cell.
1239 let (mut docs, installed_etag, overridden, recreated, over_theirs, replaced_etag) = loop {
1240 attempt += 1;
1241 if attempt > CAS_ATTEMPTS {
1242 return Err(LeanError::State(format!(
1243 "manifest CAS lost {CAS_ATTEMPTS} merge races — refusing this barrier"
1244 )));
1245 }
1246 // Lever G (latency plan §12): the document chunk by chunk,
1247 // unless it is on another layout or the view is off.
1248 let opened = if self.cfg.chunk_view {
1249 DocView::open(self.store.clone(), &self.cfg, None).await?
1250 } else {
1251 Opened::Full(manifest::load(self.store.as_ref(), &self.cfg).await?)
1252 };
1253 clock.lap("load");
1254 // Review 2026-09-18, H2: THE fence — the cell read AFTER the
1255 // load this CAS merges onto. The read at the top of step 5
1256 // comes before the HEAD fan-out and the window, and a holder
1257 // deposed in between loaded its successor's document,
1258 // rotation included, and CASed onto it: the rotation fences
1259 // only a load that PRECEDES it. Read here, the order closes
1260 // both ways: deposed before this load, and this read sees
1261 // it; deposed after, and the successor's rotation has moved
1262 // the pointer this CAS expects.
1263 //
1264 // Review 2026-09-18, M3: renew first if the cell has stood
1265 // still for half the takeover threshold. Nothing else moves
1266 // the token between the claim and the end of the section, so
1267 // a live holder whose verify, merge or CAS retries ran past
1268 // it was deposed by a waiter, and two such writers could
1269 // depose each other every floor.
1270 super::lease::renew_if_due(self).await?;
1271 self.verify_not_deposed().await?;
1272 clock.lap("fence");
1273 let mut theirs = match opened {
1274 Opened::View(v) => v,
1275 Opened::Full(current) => {
1276 // The baseline IS the merge base (P1-lite).
1277 let base = merge_base(&baseline);
1278 let MergeOnto { theirs, expected, merged, foreign, overridden, recreated, over_theirs } = self.merge_onto(
1279 current.as_ref(),
1280 &base,
1281 &upserts,
1282 &classified.deletes,
1283 &parked,
1284 &flush_uuid,
1285 );
1286 // A DOCUMENT MUST BE MATERIALISABLE. No filesystem can hold
1287 // a file `a` and a file `a/x`, so no document may cite
1288 // both — a fresh checkout would have to create a directory
1289 // where its own manifest says a file goes. The agent makes
1290 // exactly that shape by replacing a published file with a
1291 // directory (or the reverse): the scan sees the old path
1292 // absent and the new one new, the two-scan guard withholds a
1293 // FIRST absence, and the upload lands beside the citation it
1294 // was meant to replace (review 2026-09-18, M5's agent-side
1295 // twin). Judged here because this is where the document's
1296 // final shape is known — theirs, my upserts and the deletes
1297 // that actually applied. Mine is the side that yields: the
1298 // path stays dirty and publishes at the next barrier, by
1299 // which time the withheld absence is confirmed and its
1300 // citation gone. This re-merges without spending a CAS
1301 // attempt; the clash is decided, not raced.
1302 let clash = path_clashes(&merged.entries, &upserts);
1303 if !clash.is_empty() {
1304 self.withhold_clashes(clash, &mut upserts, &mut new_baseline_entries, &mut report, &mut parked, &flush_uuid)?;
1305 clock.lap("merge");
1306 attempt -= 1;
1307 continue;
1308 }
1309 clock.lap("merge");
1310 // Nothing of ours changes the document — a barrier that only
1311 // found the manifest moved by another writer. Installing it
1312 // anyway was an empty generation, and the peer's next tick
1313 // then found the manifest moved and did the same: two idle
1314 // writers traded generations (and cell claims) for as long
1315 // as both ran. Theirs becomes the merge base as it stands.
1316 if let Some(handle) = expected.as_ref() {
1317 if merged.entries == theirs.entries && merged.sole_writer == theirs.sole_writer {
1318 foreign_entries = foreign;
1319 installed_nothing = true;
1320 break (
1321 CommitDocs::Full { installed: theirs.clone(), theirs },
1322 handle.etag.clone(),
1323 vec![],
1324 vec![],
1325 vec![],
1326 Some(handle.etag.clone()),
1327 );
1328 }
1329 }
1330 match manifest::cas_write_retiring(
1331 self.store.as_ref(),
1332 &self.cfg,
1333 &merged,
1334 Some(&theirs),
1335 expected.as_ref(),
1336 epoch,
1337 &flush_uuid,
1338 Some(source),
1339 )
1340 .await
1341 {
1342 Ok(meta) => {
1343 // Before the deletes and before step 7: this is the
1344 // only record that survives a crash in that window.
1345 self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1346 "expected": expected.as_ref().map(|h| h.etag.clone()), "etag": meta.etag, "result": "ok"}));
1347 // H10: what this install carried of the standing
1348 // declaration survives a restart, for the ack.
1349 if let Some(id) = carries.as_ref() {
1350 note_carried(&mut intent, id, &classified, &upserts, &report, |p| merged.entries.contains_key(p));
1351 }
1352 self.state.save_intent(&intent)?;
1353 clock.lap("cas");
1354 foreign_entries = foreign;
1355 let replaced = expected.as_ref().map(|h| h.etag.clone());
1356 break (CommitDocs::Full { theirs, installed: merged }, meta.etag, overridden, recreated, over_theirs, replaced);
1357 }
1358 Err(LeanError::Store(StoreError::PreconditionFailed(_)))
1359 | Err(LeanError::Store(StoreError::Conflict(_))) => {
1360 self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1361 "expected": expected.as_ref().map(|h| h.etag.clone()), "result": "lost"}));
1362 clock.lap("cas");
1363 // A rotation is exactly this 412, and re-merging past
1364 // it would be the straggler install: the next
1365 // attempt's read after its load is the fence.
1366 // Back off first: under P2 a UI save never waits on
1367 // the lease, so the race is usually lost to saves,
1368 // and the gap between two of them is the way in (G2).
1369 tokio::time::sleep(cas_backoff(attempt)).await;
1370 clock.lap("backoff");
1371 continue;
1372 }
1373 Err(e) => return Err(e),
1374 }
1375 }
1376 };
1377 // Lever G (latency plan §12): the merge, the clash check, the
1378 // re-chunk and the retire set read only the chunks the
1379 // commit's paths fall in, and the new chunk list re-encodes
1380 // only what the change and its cascade reach.
1381 let paths: Vec<String> = upserts.keys().chain(classified.deletes.iter()).cloned().collect();
1382 theirs.prefetch(paths.iter().map(|p| p.as_str())).await?;
1383 let mut at: BTreeMap<String, LeanEntry> = BTreeMap::new();
1384 for p in &paths {
1385 if let Some(e) = theirs.get(p).await? {
1386 at.insert(p.clone(), e);
1387 }
1388 }
1389 let base_of = |p: &str| baseline.entries.get(p).map(|e| e.etag.as_str());
1390 let mc = manifest::merge_changes(
1391 &base_of,
1392 &at,
1393 &theirs.tombstones,
1394 theirs.seq(),
1395 &upserts,
1396 &classified.deletes,
1397 &parked,
1398 );
1399 // The event trace's merge record (its foreign and gone counts
1400 // are about every path) and the shadow check read the whole
1401 // documents.
1402 let full = if self.cfg.event_trace.is_some() || self.cfg.g_shadow {
1403 let doc = theirs.assemble().await?;
1404 Some(self.merge_onto(Some(&doc), &merge_base(&baseline), &upserts, &classified.deletes, &parked, &flush_uuid))
1405 } else {
1406 None
1407 };
1408 let clash = clashes_view(&mut theirs, &mc.overlay, &upserts).await?;
1409 let shadow = full.as_ref().filter(|_| self.cfg.g_shadow);
1410 if let Some(f) = shadow {
1411 shadow_merge(f, &mc, &clash, &upserts);
1412 }
1413 if !clash.is_empty() {
1414 self.withhold_clashes(clash, &mut upserts, &mut new_baseline_entries, &mut report, &mut parked, &flush_uuid)?;
1415 clock.lap("merge");
1416 attempt -= 1;
1417 continue;
1418 }
1419 clock.lap("merge");
1420 let expected = theirs.handle();
1421 // Nothing of ours changes the document: the full path's
1422 // check, over the paths the overlay names.
1423 let adds_nothing = self.cfg.sole_writer == theirs.pointer.sole_writer
1424 && mc.overlay.iter().all(|(p, v)| at.get(p) == v.as_ref());
1425 if let Some(f) = shadow {
1426 assert_eq!(adds_nothing, f.adds_nothing(), "lever G: adds-nothing is not the whole-document merge's");
1427 }
1428 if adds_nothing {
1429 foreign_entries = full.map(|f| f.foreign).unwrap_or_default();
1430 installed_nothing = true;
1431 let (seq, tombstones) = (theirs.seq(), theirs.tombstones.clone());
1432 let (sole_writer, boundary_source) = (theirs.pointer.sole_writer, theirs.pointer.boundary_source.clone());
1433 break (
1434 CommitDocs::View { theirs, overlay: BTreeMap::new(), retired: vec![], seq, tombstones, sole_writer, boundary_source },
1435 expected.etag.clone(),
1436 vec![],
1437 vec![],
1438 vec![],
1439 Some(expected.etag),
1440 );
1441 }
1442 let chunks = theirs.rechunk(&mc.overlay).await?;
1443 let retired = theirs.retired(&mc.overlay).await?;
1444 if let Some(f) = shadow {
1445 shadow_write(f, &chunks, &retired, &self.cfg);
1446 }
1447 let doc = manifest::ChunkedDoc {
1448 seq: mc.seq,
1449 sole_writer: self.cfg.sole_writer,
1450 boundary_source: Some(source.to_string()),
1451 tombstones: &mc.tombstones,
1452 };
1453 match manifest::cas_write_view(
1454 self.store.as_ref(),
1455 &self.cfg,
1456 &chunks,
1457 &doc,
1458 &expected,
1459 epoch,
1460 &flush_uuid,
1461 retired.clone(),
1462 )
1463 .await
1464 {
1465 Ok(meta) => {
1466 // Before the deletes and before step 7: this is the
1467 // only record that survives a crash in that window.
1468 self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": mc.seq,
1469 "expected": Some(expected.etag.clone()), "etag": meta.etag, "result": "ok"}));
1470 if let Some(id) = carries.as_ref() {
1471 note_carried(&mut intent, id, &classified, &upserts, &report, |p| match mc.overlay.get(p) {
1472 Some(v) => v.is_some(),
1473 None => at.contains_key(p),
1474 });
1475 }
1476 self.state.save_intent(&intent)?;
1477 clock.lap("cas");
1478 foreign_entries = full.map(|f| f.foreign).unwrap_or_default();
1479 let live: std::collections::HashSet<&str> = chunks.iter().map(|(r, _)| r.addr.as_str()).collect();
1480 theirs.retain_aliases(&live);
1481 let replaced = Some(expected.etag.clone());
1482 break (
1483 CommitDocs::View {
1484 theirs,
1485 overlay: mc.overlay,
1486 retired,
1487 seq: mc.seq,
1488 tombstones: mc.tombstones,
1489 sole_writer: self.cfg.sole_writer,
1490 boundary_source: Some(source.to_string()),
1491 },
1492 meta.etag,
1493 mc.overridden,
1494 mc.recreated,
1495 mc.over_theirs,
1496 replaced,
1497 );
1498 }
1499 Err(LeanError::Store(StoreError::PreconditionFailed(_)))
1500 | Err(LeanError::Store(StoreError::Conflict(_))) => {
1501 self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": mc.seq,
1502 "expected": Some(expected.etag.clone()), "result": "lost"}));
1503 clock.lap("cas");
1504 // As the full path: the next attempt's read after its
1505 // load is the fence, after a backoff (G2).
1506 tokio::time::sleep(cas_backoff(attempt)).await;
1507 clock.lap("backoff");
1508 continue;
1509 }
1510 Err(e) => return Err(e),
1511 }
1512 };
1513 let installed_seq = docs.installed_seq();
1514 report.seq = Some(installed_seq);
1515 report.no_change = installed_nothing;
1516 // A fused install IS a coherent point, and cadence/hybrid have
1517 // exactly one source. The gauges must not report "no boundary
1518 // ever" on a workspace that publishes every minute. A barrier
1519 // that installed nothing marks no boundary of its own.
1520 if !installed_nothing {
1521 self.note_boundary("cadence", installed_seq)?;
1522 }
1523 report.manifest_etag = Some(installed_etag.clone());
1524 report.observed_seq = Some(installed_seq);
1525 report.observed_etag = Some(installed_etag.clone());
1526 report.foreign_queued = foreign_entries.len();
1527
1528 // Step 6: RETIREMENT and collection (design 2026-09-19, R3 and
1529 // R7). What the document this CAS merged onto cited and the
1530 // installed document does not — a modified path's old handle,
1531 // a deleted path's — is retired, minus every handle the
1532 // installed document still cites at another path (a rename
1533 // cites the source's handle at the destination) and every
1534 // handle an inbox entry names (a rename whose destination is
1535 // still only in the cell). A retired handle can never be cited
1536 // again: nothing discovers handles from paths, and the merge
1537 // carries forward only what the base cites. So the collector
1538 // deletes it UNCONDITIONALLY, in one batch — no HEAD, no
1539 // If-Match, no fence, no renew (a straggler deposed after its
1540 // CAS has an exact retired set too; the model's
1541 // `LeanImmutableStragglerGC`). The HEAD-then-If-Match dance,
1542 // the collector that gave way on a store without a conditional
1543 // DELETE (review H1's leak) and the C2 renew-before-delete
1544 // clock all went with the slot.
1545 //
1546 // And what this barrier published OVER without ever integrating
1547 // it — a peer's publish, a UI write consumed elsewhere — is
1548 // preserved by a server-side copy and a record BEFORE the batch
1549 // takes the handle. The slot's 412 caught that at the PUT and
1550 // `Upload412Preserves` did the same; a fresh handle has no slot
1551 // to fail on, so the CAS, the one place the citation is read,
1552 // takes the decision (R7; the model's first HitlOverAny run
1553 // under handles cited an acked write over blind without it).
1554 let retired: Vec<String> = docs.retired();
1555 clock.lap("retire_set");
1556 for (path, was) in &overridden {
1557 let integrated = baseline
1558 .entries
1559 .get(path)
1560 .is_some_and(|be| be.key.as_deref() == Some(was.key.as_str()));
1561 if integrated || parked.contains(path) {
1562 continue;
1563 }
1564 let preserved = self.preserve_conflict_copy(path, &was.key, &was.etag).await?;
1565 self.state.append_conflict(&ConflictRecord {
1566 path: path.clone(),
1567 foreign_etag: was.etag.clone(),
1568 preserved_key: Some(preserved),
1569 kind: "commit-surfaced-foreign: this barrier published over a version it never \
1570 integrated (another writer's publish, or a UI write consumed elsewhere); \
1571 that version is preserved under this record and superseded knowingly"
1572 .into(),
1573 at_unix: now_unix(),
1574 })?;
1575 report.surfaced.push(path.clone());
1576 self.trace("surface", serde_json::json!({"flush": flush_uuid, "path": path, "etag": was.etag, "key": was.key}));
1577 }
1578 for (path, deleted) in &recreated {
1579 self.state.append_conflict(&ConflictRecord {
1580 path: path.clone(),
1581 foreign_etag: deleted.clone(),
1582 preserved_key: None,
1583 kind: "commit-recreated-deleted: this barrier published over a DELETE it never \
1584 integrated (another writer's, or the UI's); mine wins, the path is cited \
1585 again with this tree's bytes, and the deleted version is named here"
1586 .into(),
1587 at_unix: now_unix(),
1588 })?;
1589 report.surfaced.push(path.clone());
1590 self.trace("surface", serde_json::json!({"flush": flush_uuid, "path": path, "etag": deleted, "deleted": true}));
1591 }
1592 // M3: what this barrier's DELETE removed that it never
1593 // integrated — theirs, changed since the baseline — is preserved
1594 // before the batch below takes the handle, and named.
1595 for (path, was) in &over_theirs {
1596 let preserved = self.preserve_conflict_copy(path, &was.key, &was.etag).await?;
1597 self.state.append_conflict(&ConflictRecord {
1598 path: path.clone(),
1599 foreign_etag: was.etag.clone(),
1600 preserved_key: Some(preserved),
1601 kind: "commit-deleted-over-theirs: this barrier deleted a path whose version it \
1602 never integrated (another writer's publish, or a UI save); the delete \
1603 stands, and that version is preserved under this record"
1604 .into(),
1605 at_unix: now_unix(),
1606 })?;
1607 report.surfaced.push(path.clone());
1608 self.trace("surface", serde_json::json!({"flush": flush_uuid, "path": path, "etag": was.etag, "key": was.key, "over_delete": true}));
1609 }
1610 for path in &classified.deletes {
1611 if docs.installed_get(path).await?.is_some() {
1612 // Never expected: since M3 a delete applies over theirs,
1613 // and only uploads are parked, so every delete leaves the
1614 // document. Were one still cited it is not in this
1615 // boundary — PARKED, which the ack names as not carried —
1616 // and the next scan classifies it again. Traced like
1617 // every outcome: a replay owes a GC step for every path
1618 // in the delete set (W4 phase 2, R3-S2).
1619 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "result": "still-cited"}));
1620 report.parked.push(path.clone());
1621 } else {
1622 let key = docs.theirs_get(path).await?.map(|e| e.key);
1623 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "key": key, "result": "retired"}));
1624 report.deleted.push(path.clone());
1625 }
1626 }
1627 clock.lap("surface");
1628 if self.cfg.drill_hold_gc_secs > 0 && !retired.is_empty() {
1629 eprintln!(
1630 "flint-sync: DRILL: holding the GC for {}s before the batch delete",
1631 self.cfg.drill_hold_gc_secs
1632 );
1633 self.trace("drill_hold", serde_json::json!({"flush": flush_uuid, "where": "gc",
1634 "secs": self.cfg.drill_hold_gc_secs}));
1635 tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_gc_secs)).await;
1636 clock.lap("drill_hold");
1637 }
1638 // THE COLLECTION PASS (latency plan 2026-10-23, lever C): the
1639 // retire ledger, the reap, the orphan sweep, the generation and
1640 // chunk sweeps, run together and only when
1641 // `collect_every_secs` has passed since this writer last ran
1642 // them — not on every publish, where they were three LISTs and
1643 // four GETs under the lease. Still in the commit section, under
1644 // the cell (DP2): only WHEN the collector runs moves, and the
1645 // model's reap and sweeps are steps a commit section may take
1646 // or not. Skipping one costs storage for the interval, never a
1647 // citation: nothing is deleted while it is skipped.
1648 let collect_at = now_unix();
1649 let collect = self.cfg.collect_every_secs == 0 || {
1650 let last = self.state.load_collect_at();
1651 last == 0 || collect_at.saturating_sub(last) >= self.cfg.collect_every_secs
1652 };
1653 // RETIRE-AGE G (M1): what this commit retired was LOGGED with its
1654 // CAS (`manifest::cas_write_retiring`), as the gateway's commits
1655 // log theirs. It is collected once the retirement is
1656 // `retire_grace_secs` old — at once when that is 0 — by whichever
1657 // writer's collection pass comes next; until then the sweeps spare
1658 // it, so a reader that loaded the document before this CAS can
1659 // still fetch it. No ledger, no sweep: a leak, never a deletion
1660 // a lagging reader meets.
1661 let ledger = match collect {
1662 false => None,
1663 true => match self.retire_ledger(collect_at).await {
1664 Ok(l) => Some(l),
1665 Err(e) => {
1666 eprintln!("flint-sync: could not read the retire logs ({e}); no sweep this time");
1667 None
1668 }
1669 },
1670 };
1671 if collect {
1672 clock.lap("retire_ledger");
1673 }
1674 // The collection pass reads the whole installed document (lever G
1675 // assembles it from the view only on the commits that run it).
1676 let installed_full = match &ledger {
1677 Some(_) => Some(docs.installed_full().await?),
1678 None => None,
1679 };
1680 let whole = || installed_full.as_deref().expect("assembled for the collection pass");
1681 if let Some(l) = &ledger {
1682 match self.reap_retired(l, whole(), &flush_uuid).await {
1683 Ok(n) => report.collected = n,
1684 Err(e) => eprintln!("flint-sync: retired handles not collected (next time): {e}"),
1685 }
1686 }
1687 if collect {
1688 clock.lap("reap");
1689 }
1690 if !retired.is_empty() {
1691 self.trace("gc", serde_json::json!({"flush": flush_uuid, "retired": retired.len(),
1692 "collected": report.collected}));
1693 }
1694 // The orphan sweep (R4): a handle nothing cites, no young retire
1695 // log names, older than its grace — left by a writer that never
1696 // committed. Inside the commit section, under the cell, so the
1697 // commit's re-read of its own uploads above and this CAS are
1698 // race-free against it; never this barrier's own handles.
1699 if let Some(l) = ledger.as_ref().filter(|_| self.lease.is_some()) {
1700 match self.sweep_orphans_if_due(whole(), &flush_uuid, now_unix(), &l.spared).await {
1701 Ok(n) => report.swept = n,
1702 Err(e) => eprintln!("flint-sync: orphan sweep (retrying next time it is due): {e}"),
1703 }
1704 }
1705 drop(installed_full);
1706 if collect {
1707 clock.lap("orphan_sweep");
1708 }
1709
1710 // Step 7: the baseline follows what this barrier published. What
1711 // theirs changed is owed, and the next consume derives it: the
1712 // pointer now names a document it never derived against. UNLESS
1713 // this CAS replaced exactly the document the consume derived
1714 // against — then the installed one is that plus this tree's own
1715 // changes, nothing new is owed, and the next tick stays cheap
1716 // (a busy agent's publishes would otherwise each cost a full
1717 // derive). The paths published or deleted here are no longer the
1718 // agent's pending work.
1719 let same = |a: &str, b: &str| a.trim_matches('"') == b.trim_matches('"');
1720 // No document replaced ("") is the empty workspace's derive.
1721 let over_derived = matches!(&baseline.derived_etag,
1722 Some(d) if same(replaced_etag.as_deref().unwrap_or(""), d));
1723 if over_derived {
1724 baseline.derived_etag = Some(installed_etag.clone());
1725 for path in new_baseline_entries.keys().chain(&report.deleted) {
1726 baseline.skipped.remove(path);
1727 }
1728 }
1729 for (path, be) in new_baseline_entries {
1730 baseline.entries.insert(path, be);
1731 }
1732 for path in &report.deleted {
1733 baseline.entries.remove(path);
1734 }
1735 let carried = self.carried_paths()?;
1736 let carried_at = docs.installed_at(&carried).await?;
1737 self.note_carrier_uncited(&carried_at)?;
1738 baseline.seq = installed_seq;
1739 baseline.manifest_etag = Some(installed_etag);
1740 baseline.prev_scan.sync_to(scanned.present(&baseline.entries));
1741 self.state.commit_baseline(baseline)?;
1742 self.state.clear_intent_keys()?;
1743 clock.lap("baseline");
1744 // The commit section's end (the model's `Finish`). No window is
1745 // written: under P2 a UI save never waits on one (G1).
1746 self.trace("window_clear", serde_json::json!({"flush": flush_uuid}));
1747 // The rest of the collection pass (see above).
1748 if collect {
1749 // Reap superseded generations. Immutable metadata that is never
1750 // collected is a leak that grows by a whole manifest per
1751 // publish, and this also collects the orphan a crash between the
1752 // entries PUT and the pointer CAS leaves behind. Best effort by
1753 // design: a publish that succeeded is not un-done by a failure
1754 // to tidy up after it.
1755 // A deposal after the CAS stops every deletion after it (M3):
1756 // what is left is a leak the next holder's sweeps take, never a
1757 // deletion under a successor's cell. The reap and the orphan
1758 // sweep renew between their batches (`delete_many_renewing`);
1759 // these two renew once before they start.
1760 let may_delete = self.lease.is_some() && super::lease::renew_if_due(self).await.is_ok();
1761 let swept_generations = match may_delete {
1762 false => Ok(0),
1763 true => manifest::sweep_generations(self.store.as_ref(), &self.cfg).await,
1764 };
1765 let swept_generations = match swept_generations {
1766 Ok(0) => 0,
1767 Ok(n) => {
1768 eprintln!("flint-sync: reaped {n} superseded manifest generation(s)");
1769 n
1770 }
1771 Err(e) => {
1772 eprintln!("flint-sync: generation sweep: {e}");
1773 0
1774 }
1775 };
1776 self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "generations", "removed": swept_generations}));
1777 clock.lap("generation_sweep");
1778 // And the CHUNK reaper, same reason, same best-effort terms.
1779 // `sweep_chunks` returns 0 without a request on a workspace that
1780 // is not chunked, so this costs nothing until the layout moves.
1781 //
1782 // It was written to four model-established rules, guarded by six
1783 // mutation configs, and called from nothing but its own tests —
1784 // so with chunking on, every publish left its superseded chunks
1785 // in the bucket forever. The rules ran in the model and in the
1786 // suite and never once in production. `the_barrier_reaps...`
1787 // below is the test that asks whether this line exists at all.
1788 let may_delete = may_delete && self.lease.is_some() && super::lease::renew_if_due(self).await.is_ok();
1789 let swept_chunks = match ledger.as_ref() {
1790 _ if !may_delete => Ok(0),
1791 None => Ok(0),
1792 Some(l) => manifest::sweep_chunks_sparing(self.store.as_ref(), &self.cfg, &l.spared).await,
1793 };
1794 let swept_chunks = match swept_chunks {
1795 Ok(0) => 0,
1796 Ok(n) => {
1797 eprintln!("flint-sync: reaped {n} unreferenced manifest chunk(s)");
1798 n
1799 }
1800 Err(e) => {
1801 eprintln!("flint-sync: chunk sweep: {e}");
1802 0
1803 }
1804 };
1805 self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "chunks", "removed": swept_chunks}));
1806 clock.lap("chunk_sweep");
1807 // The pass ran: the next is due `collect_every_secs` from
1808 // now. A ledger that could not be read collected nothing,
1809 // so the next commit tries again.
1810 if ledger.is_some() {
1811 if let Err(e) = self.state.save_collect_at(collect_at) {
1812 eprintln!("flint-sync: could not record the collection pass ({e}); the next commit runs it again");
1813 }
1814 }
1815 }
1816 Ok(())
1817 }
1818 .await;
1819 // The section's documents (theirs, the merge, the install) are
1820 // freed as its block ends: at a million entries that is time the
1821 // lease is held for, so it is its own step rather than release's.
1822 clock.lap("free");
1823 // Hand the cell on whatever happened — unless it is no longer
1824 // ours to hand on: a fence means a successor holds it, and the
1825 // barrier is simply abandoned (its manifest never installed; its
1826 // uploads stand, and the next barrier adopts them by flush_uuid).
1827 match &commit {
1828 Err(LeanError::Fenced(m)) => {
1829 self.trace("fence", serde_json::json!({"where": "commit", "detail": m}));
1830 self.lease = None
1831 }
1832 _ => {
1833 if let Err(e) = super::lease::release(self).await {
1834 eprintln!(
1835 "flint-sync: the publish fence could not be released ({e}); a waiter \
1836 deposes it after the quiet threshold"
1837 );
1838 }
1839 }
1840 }
1841 clock.lap("release");
1842 report.steps = clock.steps;
1843 commit?;
1844 self.trace_barrier_end(&report, barrier_started);
1845 Ok(report)
1846 }
1847
1848 fn trace_barrier_end(&self, report: &BarrierReport, started: std::time::Instant) {
1849 if self.cfg.event_trace.is_none() {
1850 return;
1851 }
1852 self.trace(
1853 "barrier_end",
1854 serde_json::json!({
1855 "seq": report.seq, "uploaded": report.uploaded.len(), "deleted": report.deleted.len(),
1856 "parked": report.parked.len(),
1857 "leaked": report.leaked.len(),
1858 "consumed": report.consumed, "no_change": report.no_change,
1859 "ms": started.elapsed().as_millis() as u64, "requests": self.trace_requests(),
1860 "steps": report.steps,
1861 }),
1862 );
1863 }
1864
1865 /// The > whole_put_max path: contiguous `PartSource::Local` chunks
1866 /// through `compose_generation` (streaming multipart; the store
1867 /// aborts its partial assembly on every failure path). The CRC is
1868 /// computed by a streaming pass first; a writer racing the compose
1869 /// fails server-side validation and the path DEFERS to the next
1870 /// barrier — publish-possibly-torn is put_whole's documented
1871 /// dilemma, not this path's.
1872 #[allow(clippy::too_many_arguments)]
1873 #[allow(clippy::too_many_arguments)]
1874 async fn upload_compose(
1875 &self,
1876 path: &str,
1877 key: &str,
1878 // The ONE descriptor `upload_one` opened by a walk that follows
1879 // no link at any depth. The store reads its parts from this and
1880 // never reopens the path (review 2026-09-18, H5's residual).
1881 file: std::sync::Arc<std::fs::File>,
1882 size: u64,
1883 scanned: &scan::ScanEntry,
1884 stamps: GenerationStamps,
1885 generation: u64,
1886 epoch: u64,
1887 ) -> LeanResult<UploadOutcome> {
1888 // NO PRE-PASS. The store accumulates the full-object CRC from
1889 // the parts as it reads them for upload, so this file is read
1890 // ONCE (the 2026-09-10 door drill measured the second read at
1891 // roughly 11 s of the `mixed` workload's 24.3 s).
1892 // Part grid: within [min_part_size, ...], at most max_parts,
1893 // contiguous from 0.
1894 let min_part = self.store.min_part_size().max(1);
1895 let max_parts = self.store.max_parts().max(1) as u64;
1896 let mut chunk = self.cfg.whole_put_max.max(min_part);
1897 let need = size.div_ceil(chunk);
1898 if need > max_parts {
1899 chunk = size.div_ceil(max_parts).div_ceil(min_part) * min_part;
1900 }
1901 let mut parts = vec![];
1902 let mut off = 0u64;
1903 while off < size {
1904 let len = chunk.min(size - off);
1905 parts.push(flint_store::PartSource::Local { offset: off, len });
1906 off += len;
1907 }
1908
1909
1910 // A fresh handle: the Complete lands on a key nobody else writes
1911 // (design 2026-09-19, R1). The one thing that can already sit
1912 // there is this barrier's OWN Complete whose response was lost —
1913 // a torn response — and the recogniser is the bytes' checksum:
1914 // the same, cite them (re-read under the fence first, like every
1915 // citation); different, nothing minted this key but us, so the
1916 // barrier refuses rather than guess. The 412 arms this replaced
1917 // — adopt-own through the crash journal, preserve-a-foreign-
1918 // version, park — arbitrated a slot that no longer exists.
1919 let spec = flint_store::ComposeSpec {
1920 progress: None,
1921 key,
1922 local: Some(file),
1923 parts: parts.clone(),
1924 base_key: None,
1925 base_etag: None,
1926 condition: PutCondition::IfNoneMatchAny,
1927 stamps: stamps.clone(),
1928 crc64: None,
1929 };
1930 match self.store.compose_generation(&spec).await {
1931 Ok(meta) => {
1932 // The store computed it; refuse to cite bytes nothing
1933 // vouched for rather than invent a number for the
1934 // manifest. A backend that returns no checksum has not
1935 // validated the publish server-side either.
1936 let crc = meta
1937 .crc64_b64
1938 .as_deref()
1939 .and_then(flint_store::crc64_from_b64)
1940 .ok_or_else(|| {
1941 LeanError::State(format!(
1942 "compose of {key} returned no full-object checksum — refusing to \
1943 cite bytes nothing vouched for"
1944 ))
1945 })?;
1946 Ok(UploadOutcome::published(
1947 path, key.to_string(), meta.etag, crc, size, scanned, generation, epoch,
1948 ))
1949 }
1950 Err(StoreError::ChecksumMismatch(_)) | Err(StoreError::NoSuchUpload(_)) => {
1951 // Drift mid-compose, or the operator sweep aborted us:
1952 // nothing published; the next barrier re-queues.
1953 Ok(UploadOutcome::Deferred)
1954 }
1955 Err(StoreError::PreconditionFailed(_)) => {
1956 // The torn-response recogniser compares OUR bytes' checksum
1957 // against what landed, and without the pre-pass we do not
1958 // have one — so the file is read HERE, on the rare recovery
1959 // path, instead of on every publish.
1960 let crc = file_crc(&self.cfg.root, path)?;
1961 let head = self.store.head(key).await?;
1962 if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
1963 let g = GenerationStamps::from_meta(&head.meta).map(|s| s.generation).unwrap_or(generation);
1964 return Ok(UploadOutcome::adopted(
1965 path, key.to_string(), head.etag, crc, head.size, scanned, g, epoch,
1966 ));
1967 }
1968 Err(LeanError::State(format!(
1969 "handle {key} already holds different bytes (etag {}): a handle is minted once \
1970 per flush, so something other than this barrier wrote it; refusing",
1971 head.etag
1972 )))
1973 }
1974 Err(e) => Err(e.into()),
1975 }
1976 }
1977
1978
1979 async fn upload_one(
1980 &self,
1981 path: &str,
1982 scanned: &scan::ScanEntry,
1983 base: Option<&BaselineEntry>,
1984 epoch: u64,
1985 flush_uuid: &str,
1986 ) -> LeanResult<UploadOutcome> {
1987 let key = self.cfg.handle_key(path, flush_uuid);
1988 let local_path = self.cfg.root.join(path);
1989 let generation = base.map(|b| b.generation + 1).unwrap_or(1);
1990 // Review 2026-09-12, atomicity-6: the scan skipped symlinks; this
1991 // read followed them. A regular file swapped for a symlink between
1992 // the two published the link's TARGET — a file outside the
1993 // workspace, the syncer's own /proc/self/environ included. Every
1994 // stat here is an lstat and the read opens O_NOFOLLOW; a path that
1995 // is no longer a regular file is refused, recorded, and deferred
1996 // (the next scan skips it, and the two-scan rule retires it).
1997 //
1998 // Review 2026-09-18, H5: and not only the last component. A
1999 // DIRECTORY on the path swapped for a link (`mv d d.bak; ln -s
2000 // /proc/self d`) made `d/environ` a regular file outside the
2001 // workspace; the lstat and the O_NOFOLLOW open guarded the last
2002 // component only. The file is opened by a walk that follows no
2003 // link at any depth, and everything below — the stamps, the
2004 // size, the bytes — comes from that one descriptor.
2005 consume_window("before-upload", path);
2006 let file = match super::safefs::open_beneath_nofollow(&self.cfg.root, path) {
2007 Ok(f) => f,
2008 Err(e) => match scan::name_error(&e) {
2009 // Removed since the walk saw it (L-144): nothing to publish,
2010 // and the next scan reads it as a first absence. Not a
2011 // refusal, so no record.
2012 scan::NameError::Gone => return Ok(UploadOutcome::Deferred),
2013 scan::NameError::Replaced => {
2014 self.state.append_conflict(&ConflictRecord {
2015 path: path.to_string(),
2016 foreign_etag: String::new(),
2017 preserved_key: None,
2018 kind: format!(
2019 "upload-refused-not-regular: no longer a regular file inside the workspace after \
2020 the scan (a symlink or special file replaced it or a directory above it: {e}); \
2021 nothing published"
2022 ),
2023 at_unix: now_unix(),
2024 })?;
2025 return Ok(UploadOutcome::Deferred);
2026 }
2027 scan::NameError::Refused => return Ok(UploadOutcome::Unreadable),
2028 scan::NameError::Other => return Err(LeanError::State(format!("stat {}: {e}", local_path.display()))),
2029 },
2030 };
2031 let lmeta = file
2032 .metadata()
2033 .map_err(|e| LeanError::State(format!("stat {}: {e}", local_path.display())))?;
2034 let posix = Some(PosixStamps::from_metadata(&lmeta));
2035 let stamps = GenerationStamps {
2036 generation,
2037 epoch,
2038 flush_uuid: flush_uuid.to_string(),
2039 boundary_source: None,
2040 posix,
2041 };
2042 let size = lmeta.len();
2043 if size > self.cfg.whole_put_max {
2044 // Streaming multipart compose: put_whole is never fed past
2045 // whole_put_max (unbounded memory + S3's 5 GiB wall).
2046 return self
2047 .upload_compose(path, &key, std::sync::Arc::new(file), size, scanned, stamps, generation, epoch)
2048 .await;
2049 }
2050 // The upload byte gate — the store's own, shared with the compose
2051 // arm above, which charges its parts inside the store. This body
2052 // is charged from before it is read (the read is the allocation)
2053 // until its PUT, retried or not, has returned: the permit lives
2054 // to the end of this function. `None` = a store with no gate.
2055 let _upload_permit = match self.store.upload_gate() {
2056 Some(g) => Some(g.acquire(size).await),
2057 None => None,
2058 };
2059 // `Bytes::from(Vec<u8>)` TAKES OWNERSHIP without copying, so the
2060 // old `Bytes::from(body.clone())` was a full memcpy of every
2061 // published file body — bought solely to leave `body` intact for
2062 // the 412 retry below, which is reached almost never. Build the
2063 // `Bytes` once; the retry gets a refcount clone.
2064 let body = Bytes::from({
2065 use std::io::Read;
2066 let mut v = Vec::with_capacity(size as usize);
2067 (&file).read_to_end(&mut v)?;
2068 v
2069 });
2070 let uploaded_len = body.len() as u64;
2071 let crc = crc64_nvme(&body);
2072 // A fresh handle (design 2026-09-19, R1): no If-Match, because
2073 // nothing else ever writes this key. The one thing that can sit
2074 // there is this barrier's OWN PUT whose response was lost, and
2075 // the bytes' checksum recognises it: the same, cite it (re-read
2076 // under the fence first, like every citation); different, nothing
2077 // minted this key but us, so refuse rather than guess. The 412
2078 // policy this replaced — adopt-own, preserve-then-supersede a
2079 // foreign version, park, the 404-on-If-Match create arm —
2080 // arbitrated a slot that no longer exists; a foreign version is
2081 // now met at the CAS, the one place a citation is read (R7).
2082 match self
2083 .store
2084 .put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
2085 .await
2086 {
2087 Ok(meta) => Ok(UploadOutcome::published(
2088 path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
2089 )),
2090 Err(StoreError::PreconditionFailed(_)) => {
2091 let head = self.store.head(&key).await?;
2092 if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
2093 let g = GenerationStamps::from_meta(&head.meta).map(|s| s.generation).unwrap_or(generation);
2094 return Ok(UploadOutcome::adopted(
2095 path, key, head.etag, crc, head.size, scanned, g, epoch,
2096 ));
2097 }
2098 Err(LeanError::State(format!(
2099 "handle {key} already holds different bytes (etag {}): a handle is minted once \
2100 per flush, so something other than this barrier wrote it; refusing",
2101 head.etag
2102 )))
2103 }
2104 Err(e) => Err(e.into()),
2105 }
2106 }
2107}
2108
2109enum UploadOutcome {
2110 /// `adopted`: the etag was OBSERVED at the handle (bytes already
2111 /// there, no PUT of ours produced them — a torn response), so the
2112 /// commit section re-reads it before citing, as it does every
2113 /// citation.
2114 Published { entry: LeanEntry, baseline_entry: BaselineEntry, adopted: bool },
2115 /// The source drifted mid-transfer (checksum refused server-side)
2116 /// or the assembly was swept: publish nothing, advance nothing —
2117 /// the next scan re-queues the path.
2118 Deferred,
2119 /// The syncer may not open the file (EACCES or EPERM): held
2120 /// (`scan::Scan::held`), and nothing advanced, so the next scan asks
2121 /// again.
2122 Unreadable,
2123}
2124
2125impl UploadOutcome {
2126 #[allow(clippy::too_many_arguments)]
2127 fn published(
2128 path: &str,
2129 key: String,
2130 etag: String,
2131 crc: u64,
2132 uploaded_len: u64,
2133 scanned: &scan::ScanEntry,
2134 generation: u64,
2135 epoch: u64,
2136 ) -> UploadOutcome {
2137 let _ = path;
2138 UploadOutcome::Published {
2139 adopted: false,
2140 entry: LeanEntry {
2141 key: key.clone(),
2142 etag: etag.clone(),
2143 crc64_b64: crc64_to_b64(crc),
2144 // The length the object HAS (review 2026-09-12,
2145 // atomicity-1): the scanned size was cited while the
2146 // body was read fresh, so a file that grew between the
2147 // two was cited short and every fresh checkout of it
2148 // failed its CRC fold — the successor never started.
2149 size: uploaded_len,
2150 mode: scanned.mode,
2151 mtime_unix: scanned.mtime_unix,
2152 generation,
2153 epoch,
2154 },
2155 baseline_entry: BaselineEntry {
2156 etag,
2157 key: Some(key),
2158 generation,
2159 // The PRE-read stat: if the agent wrote during our read,
2160 // the next scan sees the drift and re-queues (the
2161 // re-stat/re-queue valve).
2162 size: scanned.size,
2163 mtime_unix: scanned.mtime_unix,
2164 mtime_nanos: Some(scanned.mtime_nanos),
2165 // The mode the entry above cites.
2166 mode: Some(scanned.mode & scan::PERM_BITS),
2167 crc64_b64: Some(crc64_to_b64(crc)),
2168 // Measured from the SCAN's stat, which precedes the read:
2169 // an upload can take longer than the window, and the
2170 // moment that matters is when the bytes were read.
2171 // From its later stamp (`scan::later`).
2172 racy: {
2173 let (sec, nanos) = scan::later(scanned.mtime_unix, scanned.mtime_nanos, Some(&scanned.change));
2174 scan::racy(sec, nanos, scanned.seen_nanos)
2175 },
2176 // The scan's, with the stat above.
2177 change: Some(scanned.change),
2178 },
2179 }
2180 }
2181}
2182
2183impl UploadOutcome {
2184 /// `published`, for bytes this barrier found already at the key and
2185 /// cites without a PUT of its own.
2186 #[allow(clippy::too_many_arguments)]
2187 fn adopted(
2188 path: &str,
2189 key: String,
2190 etag: String,
2191 crc: u64,
2192 uploaded_len: u64,
2193 scanned: &scan::ScanEntry,
2194 generation: u64,
2195 epoch: u64,
2196 ) -> UploadOutcome {
2197 match UploadOutcome::published(path, key, etag, crc, uploaded_len, scanned, generation, epoch) {
2198 UploadOutcome::Published { entry, baseline_entry, .. } => {
2199 UploadOutcome::Published { entry, baseline_entry, adopted: true }
2200 }
2201 other => other,
2202 }
2203 }
2204}
2205
2206/// The paths THIS barrier must withhold because the document would cite a
2207/// file and something under it (a published file replaced by a directory,
2208/// or the reverse). Whichever side is this barrier's upsert yields; if both
2209/// are, so does the parent.
2210///
2211/// Every key under `p/` sits in one contiguous run of the sorted map, so a
2212/// range lookup finds them: O(N log N). The first version scanned every
2213/// key for every key, and a 200k-path commit spent 80 s here (bench
2214/// 2026-09-25). The run does NOT start right after `p`: `p-b` and `p.txt`
2215/// sort between `p` and `p/`.
2216pub(super) fn path_clashes<V, U>(
2217 cited: &BTreeMap<String, V>,
2218 upserts: &BTreeMap<String, U>,
2219) -> Vec<String> {
2220 let mut mine: Vec<String> = vec![];
2221 for p in cited.keys() {
2222 let pfx = format!("{p}/");
2223 let mut under = cited.range::<str, _>((std::ops::Bound::Included(pfx.as_str()), std::ops::Bound::Unbounded))
2224 .map(|(k, _)| k)
2225 .take_while(|q| q.starts_with(&pfx))
2226 .peekable();
2227 if under.peek().is_none() {
2228 continue;
2229 }
2230 if upserts.contains_key(p) {
2231 mine.push(p.clone());
2232 } else if let Some(c) = under.find(|q| upserts.contains_key(*q)) {
2233 mine.push(c.clone());
2234 }
2235 }
2236 mine.sort();
2237 mine.dedup();
2238 mine
2239}
2240
2241/// `path_clashes` over a view and the overlay a merge made of it (lever G,
2242/// latency plan §12): the same answer, from each upsert's own neighbourhood
2243/// rather than every cited path. An upsert with anything cited under it is
2244/// mine to withhold; so, for each cited ancestor that is not an upsert, is
2245/// the first upsert under it (the whole-document rule takes one a round,
2246/// and the caller re-merges for the rest).
2247pub(super) async fn clashes_view(
2248 theirs: &mut DocView,
2249 overlay: &BTreeMap<String, Option<LeanEntry>>,
2250 upserts: &BTreeMap<String, LeanEntry>,
2251) -> LeanResult<Vec<String>> {
2252 async fn cites(theirs: &mut DocView, overlay: &BTreeMap<String, Option<LeanEntry>>, p: &str) -> LeanResult<bool> {
2253 match overlay.get(p) {
2254 Some(v) => Ok(v.is_some()),
2255 None => Ok(theirs.get(p).await?.is_some()),
2256 }
2257 }
2258 let under = |pfx: &str| {
2259 overlay
2260 .range::<str, _>((std::ops::Bound::Included(pfx), std::ops::Bound::Unbounded))
2261 .take_while(|(k, _)| k.starts_with(pfx))
2262 .any(|(_, v)| v.is_some())
2263 };
2264 let mut mine: Vec<String> = vec![];
2265 for u in upserts.keys() {
2266 if !cites(theirs, overlay, u).await? {
2267 continue;
2268 }
2269 let pfx = format!("{u}/");
2270 if under(&pfx) || theirs.first_under(&pfx, &|k| overlay.contains_key(k)).await?.is_some() {
2271 mine.push(u.clone());
2272 }
2273 let mut cut = 0;
2274 while let Some(i) = u[cut..].find('/') {
2275 let a = &u[..cut + i];
2276 cut += i + 1;
2277 if upserts.contains_key(a) || !cites(theirs, overlay, a).await? {
2278 continue;
2279 }
2280 let apfx = format!("{a}/");
2281 let first = upserts
2282 .range::<str, _>((std::ops::Bound::Included(apfx.as_str()), std::ops::Bound::Unbounded))
2283 .take_while(|(k, _)| k.starts_with(&apfx))
2284 .map(|(k, _)| k)
2285 .find(|k| matches!(overlay.get(k.as_str()), Some(Some(_))));
2286 if let Some(q) = first {
2287 mine.push(q.clone());
2288 }
2289 }
2290 }
2291 mine.sort();
2292 mine.dedup();
2293 Ok(mine)
2294}
2295
2296/// The shadow check of a chunk-view merge (tests; latency plan §12,
2297/// "Verification"): the overlay over theirs is the whole-document merge,
2298/// and so are the tombstones, every per-path decision and the clashes.
2299fn shadow_merge(
2300 full: &MergeOnto,
2301 mc: &manifest::MergedChanges,
2302 clash: &[String],
2303 upserts: &BTreeMap<String, LeanEntry>,
2304) {
2305 let mut applied = full.theirs.entries.clone();
2306 for (p, v) in &mc.overlay {
2307 match v {
2308 Some(e) => {
2309 applied.insert(p.clone(), e.clone());
2310 }
2311 None => {
2312 applied.remove(p);
2313 }
2314 }
2315 }
2316 assert!(applied == full.merged.entries, "lever G: the overlay over theirs is not the whole-document merge");
2317 assert_eq!(mc.seq, full.merged.seq, "lever G: the merged seq");
2318 assert_eq!(mc.tombstones, full.merged.tombstones, "lever G: the merged tombstones");
2319 assert_eq!(mc.overridden, full.overridden, "lever G: overridden");
2320 assert_eq!(mc.recreated, full.recreated, "lever G: recreated");
2321 assert_eq!(mc.over_theirs, full.over_theirs, "lever G: over_theirs");
2322 assert_eq!(clash, path_clashes(&full.merged.entries, upserts).as_slice(), "lever G: the path clashes");
2323}
2324
2325/// The shadow check of a chunk-view write: the chunk list is `split_with`
2326/// of the merged entries, ref for ref and byte for byte, and the retire set
2327/// is the whole-document one.
2328fn shadow_write(full: &MergeOnto, chunks: &[(super::chunk::ChunkRef, Option<Vec<u8>>)], retired: &[String], cfg: &super::LeanConfig) {
2329 let want = super::chunk::split_with(&full.merged.entries, cfg.chunk_target, cfg.chunk_min, cfg.chunk_max)
2330 .expect("the shadow split");
2331 let got: Vec<&super::chunk::ChunkRef> = chunks.iter().map(|(r, _)| r).collect();
2332 let want_refs: Vec<&super::chunk::ChunkRef> = want.iter().map(|(r, _)| r).collect();
2333 assert_eq!(got, want_refs, "lever G: the re-chunk is not split_with of the merged entries");
2334 for ((r, body), (_, w)) in chunks.iter().zip(&want) {
2335 if let Some(b) = body {
2336 assert!(b == w, "lever G: chunk {} encodes differently from split_with", r.addr);
2337 }
2338 }
2339 let whole = CommitDocs::Full { theirs: full.theirs.clone(), installed: full.merged.clone() }.retired();
2340 assert_eq!(retired, whole.as_slice(), "lever G: the retire set");
2341}
2342
2343/// H10: what an install carried of the standing declaration, kept for the
2344/// ack: the paths it published and deleted, the bytes it published, and
2345/// what it dropped. `cites(p)`: the installed document cites `p`. Both
2346/// commit paths, the whole document's and lever G's view, record here.
2347fn note_carried(
2348 intent: &mut IntentJournal,
2349 id: &String,
2350 classified: &scan::Classified,
2351 upserts: &BTreeMap<String, LeanEntry>,
2352 report: &BarrierReport,
2353 cites: impl Fn(&str) -> bool,
2354) {
2355 let mut c = intent
2356 .carrier
2357 .take()
2358 .filter(|c| &c.pending_id == id)
2359 .unwrap_or_else(|| Carrier { pending_id: id.clone(), ..Default::default() });
2360 c.paths.extend(classified.uploads.iter().chain(&classified.deletes).cloned());
2361 // L-136: an earlier carry's delete that this one
2362 // uploads is an upload again, so a peer's delete
2363 // of it is named (`note_carrier_uncited`).
2364 // L-137: the bytes it published, so a peer's
2365 // replacement of them is named too.
2366 for p in &classified.uploads {
2367 c.deletes.remove(p);
2368 match upserts.get(p) {
2369 Some(e) => c.published.insert(p.clone(), e.crc64_b64.clone()),
2370 None => c.published.remove(p),
2371 };
2372 }
2373 for p in &classified.deletes {
2374 c.published.remove(p);
2375 }
2376 c.deletes.extend(classified.deletes.iter().cloned());
2377 // `honor_publish`'s dropped set, decided here:
2378 // parks and deferrals are settled before the
2379 // CAS, and an outranked delete is one the merged
2380 // document still cites. Since M3 no delete is
2381 // outranked (only uploads park, and the merge
2382 // removes every unparked delete: `merge_applies_
2383 // a_local_delete_and_names_the_version_it_
2384 // deleted_over`), so the second clause is a
2385 // backstop no barrier reaches.
2386 c.dropped.extend(report.parked.iter().chain(&report.deferred).chain(&report.held).cloned());
2387 c.dropped.extend(classified.deletes.iter().filter(|p| cites(p.as_str())).cloned());
2388 intent.carrier = Some(c);
2389}
2390
2391/// Says what a barrier held, on every barrier that holds anything: until
2392/// the tree's owner makes it readable (or renames a name that is not
2393/// UTF-8), nothing there reaches the bucket and no boundary is complete,
2394/// and the operator has to be told which paths. Ten at most; the report
2395/// carries them all.
2396fn note_held(held: &[String]) {
2397 const SHOWN: usize = 10;
2398 let more = held.len().saturating_sub(SHOWN);
2399 eprintln!(
2400 "flint-sync: held {} path(s) the syncer may not read (EACCES or EPERM) or name (not UTF-8, \
2401 shown with U+FFFD; or too long for a store key): {:?}{} — nothing at or \
2402 under them is published or deleted until they are readable and named in UTF-8 within the key \
2403 limit; the rest of the tree is",
2404 held.len(),
2405 &held[..held.len().min(SHOWN)],
2406 if more > 0 { format!(" and {more} more") } else { String::new() },
2407 );
2408}
2409
2410pub(super) fn local_dirty(local: &Path, base: Option<&BaselineEntry>) -> bool {
2411 match (std::fs::metadata(local), base) {
2412 (Err(_), None) => false, // both absent: clean
2413 (Err(_), Some(_)) => true, // locally deleted vs baseline
2414 (Ok(_), None) => true, // local exists, never published
2415 (Ok(m), Some(b)) => {
2416 scan::stat_changed(b, m.len(), mtime_of(&m), mtime_nanos_of(&m), mode_of(&m))
2417 // UNSURE: an unchanged stat recorded inside the window
2418 // (`scan::racy`), or one whose change stamp moved
2419 // (`scan::stamp_moved`), is believed only if the bytes
2420 // agree.
2421 || ((b.racy || scan::stamp_moved(b, &scan::ChangeStamp::of(&m))) && racy_bytes_differ(local, b))
2422 }
2423 }
2424}
2425
2426/// `local_dirty`'s content check for a racy entry: dirty unless `local`
2427/// is a regular file (not a link) whose CRC is the one the baseline
2428/// integrated. Anything that cannot be read to tell is dirty.
2429fn racy_bytes_differ(local: &Path, b: &BaselineEntry) -> bool {
2430 use std::io::Read;
2431 if !std::fs::symlink_metadata(local).is_ok_and(|m| m.is_file()) {
2432 return true;
2433 }
2434 let Ok(mut f) = std::fs::File::open(local) else { return true };
2435 let mut crc = flint_store::Crc64Nvme::new();
2436 let mut buf = vec![0u8; 1 << 20];
2437 loop {
2438 match f.read(&mut buf) {
2439 Ok(0) => break,
2440 Ok(n) => crc.update(&buf[..n]),
2441 Err(_) => return true,
2442 }
2443 }
2444 b.crc64_b64.as_deref() != Some(crc64_to_b64(crc.finalize()).as_str())
2445}
2446
2447pub(super) fn file_crc(root: &Path, rel: &str) -> LeanResult<u64> {
2448 use std::io::Read;
2449 let mut crc = flint_store::Crc64Nvme::new();
2450 // No link anywhere on the path (H5): this hashes what a 412 recovery
2451 // is about to call "already there".
2452 let mut f = super::safefs::open_beneath_nofollow(root, rel)?;
2453 let mut buf = vec![0u8; 4 << 20];
2454 loop {
2455 let n = f.read(&mut buf)?;
2456 if n == 0 {
2457 break;
2458 }
2459 crc.update(&buf[..n]);
2460 }
2461 Ok(crc.finalize())
2462}
2463
2464/// Whether a stat taken just now is racy (`scan::racy`): every site that
2465/// records a baseline entry from a fresh stat of bytes it just read or
2466/// wrote stamps the entry with this.
2467pub(super) fn racy_stat(m: &std::fs::Metadata) -> bool {
2468 let (sec, nanos) = scan::later(mtime_of(m), mtime_nanos_of(m), Some(&scan::ChangeStamp::of(m)));
2469 scan::racy(sec, nanos, scan::now_nanos())
2470}
2471
2472pub(super) fn mtime_nanos_of(m: &std::fs::Metadata) -> u32 {
2473 m.modified()
2474 .ok()
2475 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2476 .map(|d| d.subsec_nanos())
2477 .unwrap_or(0)
2478}
2479
2480pub(super) fn mtime_of(m: &std::fs::Metadata) -> i64 {
2481 m.modified()
2482 .ok()
2483 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2484 .map(|d| d.as_secs() as i64)
2485 .unwrap_or(0)
2486}
2487
2488/// A file's `st_mode`, as the walk reports it (`scan::stat_changed`).
2489pub(super) fn mode_of(m: &std::fs::Metadata) -> u32 {
2490 #[cfg(unix)]
2491 {
2492 use std::os::unix::fs::MetadataExt;
2493 m.mode()
2494 }
2495 #[cfg(not(unix))]
2496 {
2497 let _ = m;
2498 0o644
2499 }
2500}
2501
2502/// Validate `rel` under `root` WITHOUT creating anything — for callers
2503/// that must decide whether a path is writable before doing any work
2504/// (inbox consume refuses a refused path outright rather than routing it
2505/// through conflict-preserve, which would cost a GET+PUT and leave the
2506/// path in the baseline).
2507pub(super) fn check_contained(root: &Path, rel: &str) -> LeanResult<()> {
2508 resolve_contained(root, rel, false).map(|_| ())
2509}
2510
2511/// Resolve `rel` under `root`, refusing escapes, creating missing parent
2512/// directories. Walks the parent chain component by component: a
2513/// component that EXISTS as a symlink is a refusal (never followed).
2514pub(super) fn contained_path(root: &Path, rel: &str) -> LeanResult<std::path::PathBuf> {
2515 resolve_contained(root, rel, true).map(|(p, _)| p)
2516}
2517
2518/// As `contained_path`, and also what the walk already learned about
2519/// the FINAL component: its `lstat` if it exists, `None` if it does
2520/// not. The walk stats every component to refuse a symlink; a caller
2521/// that then asked `exists()` and `metadata()` paid two more stats for
2522/// an answer it was already holding (checkout's resume check did, on
2523/// every one of 20,000 files).
2524pub(super) fn contained_path_stat(
2525 root: &Path,
2526 rel: &str,
2527) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2528 resolve_contained(root, rel, true)
2529}
2530
2531fn resolve_contained(
2532 root: &Path,
2533 rel: &str,
2534 create_dirs: bool,
2535) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2536 use std::path::Component;
2537 let refuse = |why: &str| {
2538 Err(LeanError::State(format!(
2539 "refusing write to {rel:?}: {why} (containment)"
2540 )))
2541 };
2542 if rel.is_empty() {
2543 return refuse("empty path");
2544 }
2545 if super::scan::is_control_path(rel) {
2546 // The reserved namespace is the syncer's own; a citation or
2547 // inbox entry naming it is surfaced, never materialized (D0.3).
2548 return refuse("reserved control namespace");
2549 }
2550 if rel.split('/').any(|seg| seg.ends_with(super::scan::TMP_SUFFIX)) {
2551 // Review 2026-09-18, C1: the walk skips this suffix at every
2552 // depth (the consume's own temp sibling, atomicity-7), so a file
2553 // materialised under it is absent from every scan — cited once
2554 // by the repair as a first absence, then classified a delete and
2555 // its object collected. Surfaced and never materialised, like the
2556 // control namespace; the gateway refuses the name at the door.
2557 return refuse("a name reserved for the syncer's temporary files");
2558 }
2559 let relp = Path::new(rel);
2560 let mut cur = root.to_path_buf();
2561 let mut comps: Vec<&std::ffi::OsStr> = vec![];
2562 for c in relp.components() {
2563 match c {
2564 Component::Normal(n) => comps.push(n),
2565 Component::CurDir => {}
2566 Component::ParentDir => return refuse("`..` component"),
2567 Component::RootDir | Component::Prefix(_) => return refuse("absolute path"),
2568 }
2569 }
2570 if comps.is_empty() {
2571 return refuse("no path components");
2572 }
2573 let last = comps.len() - 1;
2574 let mut final_meta = None;
2575 for (i, c) in comps.iter().enumerate() {
2576 cur.push(c);
2577 match std::fs::symlink_metadata(&cur) {
2578 Ok(m) if m.file_type().is_symlink() => {
2579 return refuse("path traverses a symlink");
2580 }
2581 Ok(m) if i < last && !m.is_dir() => {
2582 return refuse("parent component is not a directory");
2583 }
2584 Ok(m) => {
2585 if i == last {
2586 final_meta = Some(m);
2587 }
2588 }
2589 Err(_) if i < last => {
2590 if create_dirs {
2591 if let Err(e) = std::fs::create_dir(&cur) {
2592 // A SIBLING may have created it between the stat
2593 // above and this mkdir: fetches run on their own
2594 // tasks, so the first files of one directory
2595 // race here, and the loser's EEXIST is not a
2596 // containment refusal. It is fine only if what
2597 // exists now is a plain directory — a symlink
2598 // that appeared in the window is refused exactly
2599 // as one the stat found.
2600 match std::fs::symlink_metadata(&cur) {
2601 Ok(m) if m.file_type().is_symlink() => {
2602 return refuse("path traverses a symlink");
2603 }
2604 Ok(m) if m.is_dir() => {}
2605 _ => {
2606 return Err(LeanError::State(format!(
2607 "mkdir {}: {e}",
2608 cur.display()
2609 )))
2610 }
2611 }
2612 }
2613 }
2614 }
2615 Err(_) => {}
2616 }
2617 }
2618 Ok((cur, final_meta))
2619}
2620
2621/// Write `bytes` to `path` through an exclusive temp sibling and a
2622/// rename. The parent already exists — containment created it — and
2623/// one that vanished since is recreated on the retry path in `safefs`,
2624/// so the common path pays no `mkdir` and no `stat` for it. Returns
2625/// the written file's metadata.
2626pub(super) fn write_file_atomic(
2627 path: &Path,
2628 bytes: &[u8],
2629 mode: Option<u32>,
2630) -> LeanResult<std::fs::Metadata> {
2631 super::safefs::check_parent(path)?;
2632 // NOT with_extension(): that REPLACES the final extension, so
2633 // "a.txt" and "a.md" would collide on one tmp name.
2634 let tmp = path.with_file_name(format!(
2635 "{}.flint-sync-tmp",
2636 path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default()
2637 ));
2638 // The temp sibling is computed AFTER containment ran, so the walk
2639 // never saw it: it needs its own refusal, not a plain write.
2640 // FAST (no per-file fsync): materialisations are made durable by
2641 // `sync_tree` before the marker or baseline that vouches for them.
2642 super::safefs::write_via_tmp_fast(path, &tmp, bytes, mode)
2643}
2644
2645/// `write_file_atomic`, renamed over `path` only if `proceed()` holds once
2646/// the temp is written (`safefs::write_via_tmp_fast_if`).
2647pub(super) fn write_file_atomic_if(
2648 path: &Path,
2649 bytes: &[u8],
2650 mode: Option<u32>,
2651 proceed: &dyn Fn() -> bool,
2652) -> LeanResult<Option<std::fs::Metadata>> {
2653 super::safefs::check_parent(path)?;
2654 let tmp = path.with_file_name(format!(
2655 "{}.flint-sync-tmp",
2656 path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default()
2657 ));
2658 super::safefs::write_via_tmp_fast_if(path, &tmp, bytes, mode, proceed)
2659}
2660
2661// Test-only: run a closure inside a write window for one path —
2662// `"before-write"` (the licence to overwrite checked, the temp not yet
2663// written), `"after-rename"` (before the baseline records the write), or
2664// `"before-delete"` (`sync`'s remote-delete arm). The consume and `sync`
2665// both open them. Nothing in either window calls the store, so the hooked double
2666// cannot reach them (review 2026-09-18, H7).
2667#[cfg(test)]
2668thread_local! {
2669 pub(super) static CONSUME_WINDOW_HOOK: std::cell::RefCell<Option<(String, &'static str, Box<dyn Fn()>)>> =
2670 std::cell::RefCell::new(None);
2671}
2672
2673#[cfg(test)]
2674pub(super) fn consume_window(stage: &'static str, path: &str) {
2675 let hook = CONSUME_WINDOW_HOOK.with(|h| {
2676 let mut h = h.borrow_mut();
2677 if matches!(&*h, Some((p, s, _)) if p == path && *s == stage) {
2678 h.take().map(|(_, _, f)| f)
2679 } else {
2680 None
2681 }
2682 });
2683 if let Some(f) = hook {
2684 f();
2685 }
2686}
2687
2688#[cfg(not(test))]
2689#[inline(always)]
2690pub(super) fn consume_window(_: &'static str, _: &str) {}
2691
2692/// How many merge races a commit may lose before its barrier gives up (the
2693/// next tick tries again). Under P2 the UI saves without the lease (G1), so
2694/// the races are lost to saves, and backing off finds the gaps between them.
2695const CAS_ATTEMPTS: u32 = 12;
2696
2697/// The wait after the `attempt`-th lost race: 10 ms, doubling, capped at
2698/// 640 ms, plus up to half again of jitter from the clock's nanoseconds, so
2699/// a writer and a stream of saves do not fall into lockstep.
2700fn cas_backoff(attempt: u32) -> std::time::Duration {
2701 let base = 10u64 << attempt.saturating_sub(1).min(6);
2702 let nanos = std::time::SystemTime::now()
2703 .duration_since(std::time::UNIX_EPOCH)
2704 .map(|d| d.subsec_nanos() as u64)
2705 .unwrap_or(0);
2706 std::time::Duration::from_millis(base + nanos % (base / 2 + 1))
2707}
2708
2709
2710/// The merge base, P1-lite: the baseline's own citations. A tree holds no
2711/// version the document never cited (P2), so what it integrated IS what it
2712/// merges from; one base serves the merge and every reader of its outcome
2713/// (M5, L-124).
2714fn merge_base(baseline: &super::state::Baseline) -> BTreeMap<String, String> {
2715 baseline.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect()
2716}
2717
2718/// One three-way merge of a barrier's changes onto the bucket's manifest.
2719struct MergeOnto {
2720 theirs: manifest::LeanManifest,
2721 /// The CAS handle over `theirs`; `None` when the bucket has no manifest.
2722 expected: Option<manifest::ManifestHandle>,
2723 merged: manifest::LeanManifest,
2724 /// Other writers' changes since the merge base that the tree lacks.
2725 foreign: Vec<(String, LeanEntry)>,
2726 /// Theirs' version at a path this barrier uploaded, changed since the
2727 /// merge base: what mine publishes OVER. The caller surfaces those it
2728 /// never integrated (design 2026-09-19, R7).
2729 overridden: Vec<(String, LeanEntry)>,
2730 /// Theirs' DELETES this barrier's uploads publish over (`merge`).
2731 recreated: Vec<(String, String)>,
2732 /// Theirs' version at a path this barrier DELETES, changed since the
2733 /// merge base (M3): the caller preserves and records each.
2734 over_theirs: Vec<(String, LeanEntry)>,
2735}
2736
2737impl MergeOnto {
2738 /// The merge adds nothing to the document: installing it would be an
2739 /// empty generation.
2740 fn adds_nothing(&self) -> bool {
2741 self.merged.entries == self.theirs.entries && self.merged.sole_writer == self.theirs.sole_writer
2742 }
2743}
2744
2745impl Syncer {
2746 /// H10: the carried uploads the document this barrier ended at does not
2747 /// cite (a peer deleted them since). The ack names that document, so it
2748 /// reads this; no request is made for it.
2749 /// The paths the standing carrier record names (H10): all that
2750 /// `note_carrier_uncited` reads of a document.
2751 fn carried_paths(&self) -> LeanResult<Vec<String>> {
2752 Ok(self.state.load_intent()?.carrier.map(|c| c.paths.into_iter().collect()).unwrap_or_default())
2753 }
2754
2755 /// A DOCUMENT MUST BE MATERIALISABLE: each upload `clash` names is
2756 /// withheld this barrier, recorded, and stays dirty for the next (the
2757 /// caller's comment at `path_clashes`). Both commit paths withhold here.
2758 #[allow(clippy::too_many_arguments)]
2759 fn withhold_clashes(
2760 &self,
2761 clash: Vec<String>,
2762 upserts: &mut BTreeMap<String, LeanEntry>,
2763 new_baseline_entries: &mut BTreeMap<String, BaselineEntry>,
2764 report: &mut BarrierReport,
2765 parked: &mut BTreeSet<String>,
2766 flush_uuid: &str,
2767 ) -> LeanResult<()> {
2768 for path in clash {
2769 let gone = upserts.remove(&path).expect("named from upserts above");
2770 new_baseline_entries.remove(&path);
2771 report.uploaded.retain(|p| p != &path);
2772 report.published_bytes = report.published_bytes.saturating_sub(gone.size);
2773 parked.insert(path.clone());
2774 report.parked.push(path.clone());
2775 self.state.append_conflict(&ConflictRecord {
2776 path: path.clone(),
2777 foreign_etag: gone.etag,
2778 preserved_key: None,
2779 kind: "upload-withheld-path-clash: the document still cites a path \
2780 this one would sit inside (or around) — a published file \
2781 replaced by a directory, or the reverse. No workspace can \
2782 hold both, so nothing is cited here until that citation \
2783 goes; the path stays dirty and publishes next barrier"
2784 .into(),
2785 at_unix: now_unix(),
2786 })?;
2787 self.trace("path_clash", serde_json::json!({"flush": flush_uuid, "path": path}));
2788 }
2789 Ok(())
2790 }
2791
2792 fn note_carrier_uncited(&self, doc: &manifest::LeanManifest) -> LeanResult<()> {
2793 let mut intent = self.state.load_intent()?;
2794 let Some(c) = intent.carrier.as_mut() else { return Ok(()) };
2795 let uncited: BTreeSet<String> = c
2796 .paths
2797 .iter()
2798 .filter(|p| {
2799 !c.deletes.contains(*p)
2800 && match doc.entries.get(p.as_str()) {
2801 None => true,
2802 // L-137: other bytes at the path are not the upload
2803 // this writer carried.
2804 Some(e) => c.published.get(p.as_str()).is_some_and(|crc| *crc != e.crc64_b64),
2805 }
2806 })
2807 .cloned()
2808 .collect();
2809 if uncited != c.uncited {
2810 c.uncited = uncited;
2811 self.state.save_intent(&intent)?;
2812 }
2813 Ok(())
2814 }
2815
2816 fn merge_onto(
2817 &self,
2818 current: Option<&manifest::LoadedManifest>,
2819 // `merge_base`'s answer, which the caller also gives every other
2820 // reader of this merge's outcome.
2821 base: &BTreeMap<String, String>,
2822 upserts: &BTreeMap<String, LeanEntry>,
2823 deletes: &BTreeSet<String>,
2824 parked: &BTreeSet<String>,
2825 flush_uuid: &str,
2826 ) -> MergeOnto {
2827 let (theirs, expected) = match current {
2828 // The handle carries the LAYOUT as well as the tag: a
2829 // workspace still on the legacy single object CASes a
2830 // pointer that must not exist yet, not one it read.
2831 Some(l) => (l.manifest.clone(), Some(l.handle())),
2832 None => (Default::default(), None),
2833 };
2834 let manifest::Merged { doc: mut merged, foreign, overridden, deletes: outcomes, recreated, over_theirs } =
2835 manifest::merge(base, &theirs, upserts, deletes, parked);
2836 // The deletes theirs outranked, named: the trace's record of the
2837 // merge's per-path decision (`TraceCore.tla` checks it against
2838 // `Install`'s `Foreign(s, p)`).
2839 // Since M3 no delete is outranked; the trace keeps the field (empty)
2840 // and names the deletes that went over theirs.
2841 let outranked: Vec<&String> = vec![];
2842 let deleted_over: Vec<&String> =
2843 outcomes.iter().filter(|(_, o)| **o == manifest::DeleteOutcome::OverTheirs).map(|(p, _)| p).collect();
2844 // Deletions another writer made since this workspace's
2845 // merge base: in the base, gone from theirs, and not this
2846 // barrier's own upsert, delete or park. `merge` has no use
2847 // for them — theirs already lacks the path — but the TREE
2848 // does: they reach it through the local queue, as the
2849 // foreign upserts do.
2850 let gone: Vec<(String, String)> = base
2851 .iter()
2852 .filter(|(p, _)| {
2853 !theirs.entries.contains_key(*p)
2854 && !upserts.contains_key(*p)
2855 && !deletes.contains(*p)
2856 && !parked.contains(*p)
2857 })
2858 .map(|(p, retired)| (p.clone(), retired.clone()))
2859 .collect();
2860 // `merge` clears it; the installing pass owns it. A mirror
2861 // is a property of how this workspace is DEPLOYED, so it
2862 // comes from config on every publish rather than being
2863 // inherited from whatever wrote last.
2864 merged.sole_writer = self.cfg.sole_writer;
2865 self.trace("merge", serde_json::json!({"flush": flush_uuid, "theirs_seq": theirs.seq, "upserts": upserts.len(),
2866 "deletes": deletes.len(), "foreign": foreign.len(), "gone": gone.len(), "overridden": overridden.len(),
2867 "outranked": outranked, "over_theirs": deleted_over, "adds_nothing": merged.entries == theirs.entries}));
2868 MergeOnto { theirs, expected, merged, foreign, overridden, recreated, over_theirs }
2869 }
2870}