Skip to main content

flint_lean/
barrier.rs

1//! The publish barrier (plan §2.1, seven steps) — the machine
2//! `lean/formal/LeanSubtree.tla` checks. Order is load-bearing:
3//! consume → scan → intent/window → uploads → manifest CAS (merge) →
4//! GC deletes LAST → baseline rewrite.
5
6use std::collections::{BTreeMap, BTreeSet};
7use std::path::Path;
8
9use bytes::Bytes;
10
11use flint_store::{
12    EpochState,
13    crc64_nvme, crc64_to_b64, GenerationStamps, PosixStamps, PutCondition, StoreError,
14};
15
16use super::inbox::{self, InboxDoc, InboxEntry, Refusal, Removal};
17use super::manifest::{self, LeanEntry};
18use super::scan;
19use super::state::{BaselineEntry, ConflictRecord, ForeignChange, IntentJournal};
20use super::{now_unix, LeanError, LeanResult, Syncer};
21
22/// The last gateway request this workspace acted on, per verb. A
23/// watermark rather than a consumed queue — see `note_verb_requests`.
24#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
25pub struct VerbWatermark {
26    #[serde(default)]
27    pub boundary_unix: u64,
28    #[serde(default)]
29    pub sync_unix: u64,
30}
31
32#[derive(Debug, Default)]
33pub struct BarrierReport {
34    pub seq: Option<u64>,
35    pub uploaded: Vec<String>,
36    pub deleted: Vec<String>,
37    /// Paths a DECLARED removal (delete/rename design) cited out this
38    /// barrier — a subset of `deleted` once the GC has run.
39    pub removed: Vec<String>,
40    /// Declared removals this barrier REFUSED (locally dirty, not a
41    /// file, uncontainable); the cell carries the reason.
42    pub removals_refused: usize,
43    pub parked: Vec<String>,
44    pub consumed: usize,
45    pub foreign_queued: usize,
46    /// Paths whose transfer drifted or was swept mid-compose: nothing
47    /// published, nothing advanced; the next scan re-queues them.
48    pub deferred: Vec<String>,
49    pub no_change: bool,
50    /// Bytes this barrier actually published — the input to the
51    /// sentinel work meter (boundary-verbs plan D3.1). Metering work
52    /// rather than calls is what keeps a sentinel storm from
53    /// un-coalescing a hot large file's republish: a counted budget
54    /// charges a 2 GiB checkpoint the same one unit as a 4 KiB file.
55    pub published_bytes: u64,
56    /// The manifest ETag this barrier installed (the ack's CAS token).
57    pub manifest_etag: Option<String>,
58    /// What the manifest HEAD/install told us the bucket is at, for the
59    /// news ticker (D5) — free, off requests the barrier already made.
60    pub observed_seq: Option<u64>,
61    pub observed_etag: Option<String>,
62    /// Paths this barrier saw absent for the FIRST time: their deletes
63    /// are withheld to the next scan (the rename-vs-walk guard). On a
64    /// declared boundary a non-empty set means the confirmation lstat
65    /// found them back on disk — the race the guard exists for.
66    pub first_absence: Vec<String>,
67    /// First-absence paths a DECLARED boundary confirmed gone by direct
68    /// lstat and published as ordinary deletes (`confirm_absences`).
69    pub absences_confirmed: usize,
70    /// This barrier finished a rescope a crash had left half-applied
71    /// before it did anything else. Reported so a run that looks like
72    /// an ordinary barrier can be told from one that had to converge a
73    /// workspace first.
74    pub rescope_replayed: bool,
75}
76
77/// What a pass over the cell's DECLARED removals did (`apply_removals`).
78#[derive(Debug, Default)]
79pub struct RemovalPass {
80    /// Performed: the local file is gone (or already was) and the path
81    /// is in `declared`. Dropped from the cell with the window clear —
82    /// AFTER the manifest CAS — so a listing keeps hiding the path
83    /// until the manifest stops citing it.
84    pub applied: Vec<Removal>,
85    /// Refused for good, `refused` filled in: the cell is told at once
86    /// and the removal is never retried.
87    pub refused: Vec<Removal>,
88    /// Left in the cell for the next barrier: a rename whose
89    /// destination is not integrated yet, or an unlink that failed for
90    /// a transient reason.
91    pub deferred: usize,
92    /// The paths this barrier cites OUT: `applied`, plus any a crashed
93    /// earlier barrier had unlinked and journalled.
94    pub declared: BTreeSet<String>,
95}
96
97/// Upload waves per chunk. The chunk is `fanout * this`, so a wave
98/// still saturates fan-out and the sync point between chunks is rare.
99const UPLOAD_CHUNK_WAVES: usize = 16;
100
101/// How stale the lease may get inside a barrier before it is renewed:
102/// comfortably below the 6-poll (~60 s) takeover window, so a holder
103/// busy in a long commit section never reads as quiet to a waiter.
104const RENEW_WITHIN_SECS: u64 = 20;
105
106impl Syncer {
107    /// Verify the cell still names us at OUR epoch; anything else is a
108    /// fence. Read-verify before the manifest CAS (the per-request
109    /// validation the gateway will also enforce).
110    async fn verify_not_deposed(&self) -> LeanResult<()> {
111        let lease = self
112            .lease
113            .as_ref()
114            .ok_or_else(|| LeanError::State("no lease".into()))?;
115        match self.store.epoch_read(&self.cfg.epoch_key()).await? {
116            Some(state) if state.epoch == lease.epoch && state.holder_id == lease.holder_id => {
117                Ok(())
118            }
119            Some(state) => Err(LeanError::Fenced(format!(
120                "cell at epoch {} holder {} (we are epoch {})",
121                state.epoch, state.holder_id, lease.epoch
122            ))),
123            None => Err(LeanError::Fenced("epoch cell vanished".into())),
124        }
125    }
126
127    /// Step 1: consume the inbox. A barrier NEVER runs against an
128    /// unconsumed inbox — this is what makes HITL uploads structurally
129    /// un-amputatable. Returns the entries integrated (dropped from the
130    /// cell at the window-open CAS).
131    pub async fn consume_inbox(&mut self) -> LeanResult<Vec<InboxEntry>> {
132        let loaded = inbox::load(self.store.as_ref(), &self.cfg).await?;
133        self.consume_inbox_doc(&loaded.doc).await
134    }
135
136    /// `consume_inbox` over a cell the caller has already read — the
137    /// barrier reads it once and hands the same document to
138    /// `apply_removals`, so the removal pass adds no request.
139    pub async fn consume_inbox_doc(&mut self, doc: &InboxDoc) -> LeanResult<Vec<InboxEntry>> {
140        Ok(self.consume_counted(doc).await?.0)
141    }
142
143    /// `consume_inbox_doc`, plus how many changes from this writer's LOCAL
144    /// queue it settled. Those leave no entry in the cell to drop, so they
145    /// are not in the returned list — but they are consumptions, and the
146    /// report (and the ack) count them.
147    async fn consume_counted(&mut self, doc: &InboxDoc) -> LeanResult<(Vec<InboxEntry>, usize)> {
148        // §2.5's layered doors ride the inbox GET this function already
149        // pays for: promptness is one tick, and the added request count
150        // is ZERO. A failure here must never fail the consume — the
151        // request is idempotent state and the next tick re-reads it.
152        if let Err(e) = self.note_verb_requests(doc) {
153            eprintln!("flint-sync: verb request not consumed (retrying next tick): {e}");
154        }
155        // Indices into `entries`: the writer-local queue's upserts first,
156        // then the shared inbox's. Both run the same rules below.
157        let mut consumed: Vec<usize> = vec![];
158        let mut baseline = self.state.load_baseline()?;
159        // An install whose step 7 a restart cut off left the other writers'
160        // changes it carried in the intent journal (`installed_foreign`).
161        let requeued = self.state.requeue_installed_foreign()?;
162        if requeued > 0 {
163            self.trace("queue", serde_json::json!({"requeued_from_intent": requeued}));
164        }
165        // The writer-local queue: changes other writers made that this
166        // workspace's own merges carried into the manifest but not yet
167        // into the tree (`state::ForeignChange` says why it is not the
168        // shared inbox any more).
169        let queue = self.state.load_foreign_queue()?;
170        let queued: Vec<InboxEntry> = queue
171            .iter()
172            .filter_map(|c| {
173                c.etag.as_ref().map(|etag| InboxEntry {
174                    path: c.path.clone(),
175                    etag: etag.clone(),
176                    author: "merge-preserved".into(),
177                    added_unix: 0,
178                    crc64_b64: c.crc64_b64.clone(),
179                })
180            })
181            .collect();
182        let n_queued = queued.len();
183        let entries: Vec<InboxEntry> = queued.into_iter().chain(doc.entries.iter().cloned()).collect();
184        for (idx, entry) in entries.iter().enumerate() {
185            let key = self.cfg.file_key(&entry.path);
186            // Containment BEFORE anything else: a path we could never
187            // safely materialize must be surfaced and dropped, not
188            // routed through the locally-dirty branch — which would
189            // "preserve" it with a GET+PUT and leave a baseline entry
190            // for a path the scanner can never see (the planted-symlink
191            // shape: `inputs -> /root/.aws` reads as locally-present,
192            // therefore dirty, therefore a conflict-preserve of someone
193            // else's file).
194            if let Err(e) = check_contained(&self.cfg.root, &entry.path) {
195                self.state.append_conflict(&ConflictRecord {
196                    path: entry.path.clone(),
197                    foreign_etag: entry.etag.clone(),
198                    preserved_key: None,
199                    kind: format!("consume-refused-containment: {e}"),
200                    at_unix: now_unix(),
201                })?;
202                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused"}));
203                consumed.push(idx);
204                continue;
205            }
206            // Already integrated (a crashed earlier consume): idempotent.
207            if baseline.entries.get(&entry.path).map(|b| b.etag == entry.etag).unwrap_or(false) {
208                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "already"}));
209                consumed.push(idx);
210                continue;
211            }
212            let head = match self.store.head(&key).await {
213                Ok(m) => m,
214                Err(StoreError::NotFound(_)) => {
215                    // The object vanished under a tracked entry: surface
216                    // it — the bytes are NOT recoverable from here.
217                    self.state.append_conflict(&ConflictRecord {
218                        path: entry.path.clone(),
219                        foreign_etag: entry.etag.clone(),
220                        preserved_key: None,
221                        kind: "consume-object-missing".into(),
222                        at_unix: now_unix(),
223                    })?;
224                    self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "missing"}));
225                    consumed.push(idx);
226                    continue;
227                }
228                Err(e) => return Err(e.into()),
229            };
230            if head.etag != entry.etag {
231                // Superseded by a newer write (its own inbox entry
232                // follows, or it is the syncer's): drop.
233                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "superseded"}));
234                consumed.push(idx);
235                continue;
236            }
237            let local_path = self.cfg.root.join(&entry.path);
238            let mut dirty = local_dirty(&local_path, baseline.entries.get(&entry.path));
239            // Review 2026-09-12, atomicity-3 / inbox-2: the fetch is a
240            // network round trip and the agent may write the path inside
241            // it. The stat that licenses the adopt is repeated AFTER the
242            // fetch, before anything touches the file: a write in that
243            // window makes this the dirty case — the foreign bytes are
244            // preserved and a record names them, the agent's stay.
245            let mut fetched = None;
246            if !dirty {
247                let got = self.store.get_whole(&key, Some(&entry.etag)).await.map_err(|e| match e {
248                    StoreError::PreconditionFailed(m) => {
249                        StoreError::Other(format!("consume raced a newer write: {m}"))
250                    }
251                    other => other,
252                })?;
253                if local_dirty(&local_path, baseline.entries.get(&entry.path)) {
254                    dirty = true;
255                } else {
256                    fetched = Some(got);
257                }
258            }
259            if dirty {
260                // Locally-dirty wins; preserve the FOREIGN bytes first
261                // (a conflict record must keep both versions
262                // recoverable), then advance the recognized ETag so our
263                // eventual publish supersedes it KNOWINGLY.
264                let preserved = self.preserve_conflict_copy(&entry.path, &head.etag).await?;
265                self.state.append_conflict(&ConflictRecord {
266                    path: entry.path.clone(),
267                    foreign_etag: entry.etag.clone(),
268                    preserved_key: Some(preserved),
269                    kind: "consume-dirty".into(),
270                    at_unix: now_unix(),
271                })?;
272                let stamps = GenerationStamps::from_meta(&head.meta);
273                baseline.entries.insert(
274                    entry.path.clone(),
275                    BaselineEntry {
276                        etag: head.etag.clone(),
277                        generation: stamps.map(|s| s.generation).unwrap_or(0),
278                        // Deliberately NOT the local file's stat: the
279                        // path must still scan as dirty so the local
280                        // version publishes.
281                        size: u64::MAX,
282                        mtime_unix: 0,
283                        mtime_nanos: None,
284                        // The sentinel's bytes are the LOCAL edit; the
285                        // publish that supersedes hashes them itself.
286                        crc64_b64: None,
287                    },
288                );
289                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "dirty-preserved"}));
290            } else {
291                // Clean, re-checked after the fetch: adopt the foreign
292                // content into the tree.
293                let (meta, body) = fetched.expect("fetched while clean");
294                let mode = PosixStamps::from_meta(&meta.meta).map(|p| p.mode);
295                // VERIFIED before it is written, as checkout's fresh
296                // fetch is: against the writer's CRC when the inbox
297                // entry carries one (the gateway hashes what it sent),
298                // else the backend's attestation when it offers one.
299                // Ozone offers none, so the inbox's is what a HITL
300                // upload gets checked against there. What the baseline
301                // records — and the next citation repair cites — is
302                // OURS, over the bytes actually written.
303                let got = crc64_to_b64(crc64_nvme(&body));
304                let want = entry.crc64_b64.clone().or_else(|| meta.crc64_b64.clone());
305                if let Some(want) = want {
306                    if want != got {
307                        // NOT consumed: the entry stays in the cell and
308                        // the record repeats — a permanent, visible
309                        // contradiction is the right failure for
310                        // bytes nobody vouches for.
311                        self.state.append_conflict(&ConflictRecord {
312                            path: entry.path.clone(),
313                            foreign_etag: entry.etag.clone(),
314                            preserved_key: None,
315                            kind: format!(
316                                "consume-refused-checksum: etag {} is attested with CRC-64 \
317                                 {want} but the bytes fetched under it hash to {got}",
318                                entry.etag
319                            ),
320                            at_unix: now_unix(),
321                        })?;
322                        self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused-checksum"}));
323                        continue;
324                    }
325                }
326                // CONTAINMENT and I/O are split, because the answers are
327                // opposite. Containment was already decided above by
328                // `check_contained`, so a refusal reaching here is a
329                // path that only became unsafe once parents were
330                // created — still a permanent property of the path, so
331                // it is surfaced and DROPPED, exactly as above.
332                //
333                // Everything else — ENOSPC, EACCES, EIO, a read-only
334                // filesystem — is TRANSIENT, and dropping the entry for
335                // one is silent permanent loss: the foreign bytes stay
336                // in the bucket, the workspace never adopts them, and
337                // nothing ever re-offers the entry. It used to be
338                // recorded as `consume-refused-containment`, which sent
339                // whoever read it looking for a hostile path that was
340                // never there. A full disk is not a planted symlink.
341                let target = match contained_path(&self.cfg.root, &entry.path) {
342                    Ok(t) => t,
343                    Err(e) => {
344                        self.state.append_conflict(&ConflictRecord {
345                            path: entry.path.clone(),
346                            foreign_etag: entry.etag.clone(),
347                            preserved_key: None,
348                            kind: format!("consume-refused-containment: {e}"),
349                            at_unix: now_unix(),
350                        })?;
351                        self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused"}));
352                        consumed.push(idx);
353                        continue;
354                    }
355                };
356                if let Err(e) = write_file_atomic(&target, &body, mode) {
357                    // NOT consumed: the entry stays in the cell so the
358                    // next barrier retries it. A visible conflict record
359                    // that repeats is a far better failure than one
360                    // silent drop.
361                    self.state.append_conflict(&ConflictRecord {
362                        path: entry.path.clone(),
363                        foreign_etag: entry.etag.clone(),
364                        preserved_key: None,
365                        kind: format!("consume-write-failed (will retry): {e}"),
366                        at_unix: now_unix(),
367                    })?;
368                    self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "deferred"}));
369                    continue;
370                }
371                let st = std::fs::metadata(&local_path)?;
372                let stamps = GenerationStamps::from_meta(&meta.meta);
373                baseline.entries.insert(
374                    entry.path.clone(),
375                    BaselineEntry {
376                        etag: meta.etag.clone(),
377                        generation: stamps.map(|s| s.generation).unwrap_or(0),
378                        size: st.len(),
379                        mtime_unix: mtime_of(&st),
380                        mtime_nanos: Some(mtime_nanos_of(&st)),
381                        crc64_b64: Some(got),
382                    },
383                );
384                baseline.prev_scan.insert(entry.path.clone());
385                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "adopted"}));
386            }
387            consumed.push(idx);
388        }
389        let mut settled: BTreeSet<String> =
390            consumed.iter().filter(|i| **i < n_queued).map(|i| entries[*i].path.clone()).collect();
391
392        // A queued DELETION: another writer removed the path and this
393        // workspace's merge cited that. A clean local copy goes; a dirty
394        // one is the agent's newer work and stays — it publishes, and a
395        // modify beats a delete at the merge. Never a declared removal of
396        // our own: the deletion is already in the manifest.
397        for change in queue.iter().filter(|c| c.etag.is_none()) {
398            if check_contained(&self.cfg.root, &change.path).is_err() {
399                // Never materializable here, so nothing to remove.
400                settled.insert(change.path.clone());
401                continue;
402            }
403            let local = self.cfg.root.join(&change.path);
404            let be = baseline.entries.get(&change.path).cloned();
405            let record = |kind: String| ConflictRecord {
406                path: change.path.clone(),
407                foreign_etag: String::new(),
408                preserved_key: None,
409                kind,
410                at_unix: now_unix(),
411            };
412            match std::fs::symlink_metadata(&local) {
413                Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
414                    self.trace("tombstone", serde_json::json!({"path": change.path, "action": "absent"}));
415                }
416                Err(e) => {
417                    // Unreadable is not absent: keep it queued.
418                    self.trace("tombstone", serde_json::json!({"path": change.path, "action": "deferred"}));
419                    self.state.append_conflict(&record(format!(
420                        "consume-foreign-delete-deferred: cannot stat the path: {e}"
421                    )))?;
422                    continue;
423                }
424                Ok(m) if !m.is_file() || be.is_none() || local_dirty(&local, be.as_ref()) => {
425                    self.state.append_conflict(&record(
426                        "consume-foreign-delete-vs-dirty: another writer deleted this path; the \
427                         local version has unpublished changes, so it stays and publishes"
428                            .into(),
429                    ))?;
430                    self.trace("tombstone", serde_json::json!({"path": change.path, "action": "kept-dirty"}));
431                    settled.insert(change.path.clone());
432                    continue;
433                }
434                Ok(_) => {
435                    if let Err(e) = std::fs::remove_file(&local) {
436                        self.state.append_conflict(&record(format!(
437                            "consume-foreign-delete-failed (will retry): {e}"
438                        )))?;
439                        continue;
440                    }
441                    // CONFIRM: only NotFound is absence.
442                    if !matches!(
443                        std::fs::symlink_metadata(&local),
444                        Err(ref e) if e.kind() == std::io::ErrorKind::NotFound
445                    ) {
446                        self.state.append_conflict(&record(
447                            "consume-foreign-delete-unconfirmed (will retry)".into(),
448                        ))?;
449                        continue;
450                    }
451                    self.trace("tombstone", serde_json::json!({"path": change.path, "action": "removed"}));
452                }
453            }
454            baseline.entries.remove(&change.path);
455            baseline.prev_scan.remove(&change.path);
456            settled.insert(change.path.clone());
457        }
458
459        // The files this baseline vouches for reached the disk before
460        // the baseline does (audit 2026-09-03, finding 9).
461        self.state.sync_tree()?;
462        self.state.save_baseline(&baseline)?;
463        // After the baseline: a crash between the two re-applies a
464        // settled change, and both arms are idempotent (an upsert the
465        // baseline holds is "already integrated"; a deletion with nothing
466        // on disk only drops a baseline entry that is already gone).
467        if !settled.is_empty() {
468            let remaining: Vec<_> =
469                queue.into_iter().filter(|c| !settled.contains(&c.path)).collect();
470            self.state.save_foreign_queue(&remaining)?;
471        }
472        // Only the SHARED inbox's entries leave the cell at the window
473        // clear; the local queue was settled above.
474        let n_settled = settled.len();
475        let shared = consumed.into_iter().filter(|i| *i >= n_queued).map(|i| entries[i].clone()).collect();
476        Ok((shared, n_settled))
477    }
478
479    /// Step 1b: perform the DECLARED removals (delete/rename design
480    /// §4-§6). Runs after `consume_inbox_doc`, so a rename's destination
481    /// is in the tree before its source leaves it — §5: create first,
482    /// removal second; a partial application leaves an EXTRA file,
483    /// never a missing one.
484    ///
485    /// Per removal, in order:
486    /// - the path must be containable, as a consume's must;
487    /// - a rename WAITS until its destination is integrated (in the
488    ///   baseline): a consume that deferred the destination defers the
489    ///   removal with it;
490    /// - a locally-DIRTY path — unpublished edits, or a file the agent
491    ///   created there — is REFUSED, nothing applied, the agent's work
492    ///   untouched. The same rule a consume applies to a HITL write
493    ///   over dirty bytes, and the reason a declared removal cannot be
494    ///   "just a delete". Refused for good: the cell is told, and a
495    ///   retry that waited for the agent to publish would delete the
496    ///   very edit the refusal protected;
497    /// - a clean file is unlinked, and the unlink is CONFIRMED by lstat
498    ///   before the path is declared — an unlink that did not take
499    ///   publishes no deletion (`confirm_absences`' rule, kept);
500    /// - a path already absent locally and known to the baseline or the
501    ///   merge base is declared as it stands: the agent got there
502    ///   first, or this tree never held it (a scoped workspace).
503    ///
504    /// A declared path skips the two-scan guard (§4): that guard
505    /// protects against absence INFERRED by a walk, and a declaration
506    /// is not an inference. That is what lets a rename's two halves
507    /// ride one manifest generation.
508    pub async fn apply_removals(&mut self, doc: &InboxDoc) -> LeanResult<RemovalPass> {
509        let mut pass = RemovalPass::default();
510        let baseline = self.state.load_baseline()?;
511        // A crashed earlier barrier that had unlinked and journalled but
512        // not installed: its declarations are still deletions with a
513        // recorded basis — unless the agent has since put a file back.
514        for p in self.state.load_intent()?.declared_deletes {
515            let gone = matches!(
516                std::fs::symlink_metadata(self.cfg.root.join(&p)),
517                Err(ref e) if e.kind() == std::io::ErrorKind::NotFound
518            );
519            if gone && (baseline.entries.contains_key(&p) || baseline.inst_base.contains_key(&p)) {
520                pass.declared.insert(p);
521            }
522        }
523        for r in &doc.removals {
524            if r.refused.is_some() {
525                continue;
526            }
527            let refusal = |kind: &str, message: String| Removal {
528                refused: Some(Refusal { kind: kind.into(), message, at_unix: now_unix() }),
529                ..r.clone()
530            };
531            let mut refuse = |kind: &str, message: String| -> LeanResult<()> {
532                self.state.append_conflict(&ConflictRecord {
533                    path: r.path.clone(),
534                    foreign_etag: String::new(),
535                    preserved_key: None,
536                    kind: format!("{kind}: {message}"),
537                    at_unix: now_unix(),
538                })?;
539                pass.refused.push(refusal(kind, message));
540                Ok(())
541            };
542            if let Err(e) = check_contained(&self.cfg.root, &r.path) {
543                refuse("removal-refused-containment", e.to_string())?;
544                continue;
545            }
546            if let Some(dst) = &r.moved_to {
547                match baseline.entries.get(dst) {
548                    // Not integrated yet (a consume deferred it): wait.
549                    None => {
550                        pass.deferred += 1;
551                        continue;
552                    }
553                    // Integrated as a CONFLICT: the agent had an
554                    // unpublished file at the destination, its version
555                    // won, and the moved bytes are preserved under
556                    // `conflicts/`. Deleting the source now would leave
557                    // the user's file nowhere in the tree — so the move
558                    // is refused and the source stays. The sentinel is
559                    // the consume-dirty baseline entry's `u64::MAX`.
560                    Some(b) if b.size == u64::MAX => {
561                        refuse(
562                            "removal-refused-destination-conflict",
563                            format!(
564                                "{} was not moved to {dst}: the agent has an unpublished                                  file at the destination and it wins; the moved bytes are                                  preserved under the conflict record and the source is kept",
565                                r.path
566                            ),
567                        )?;
568                        continue;
569                    }
570                    Some(_) => {}
571                }
572            }
573            let local = self.cfg.root.join(&r.path);
574            let be = baseline.entries.get(&r.path);
575            match std::fs::symlink_metadata(&local) {
576                Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
577                    if be.is_some() || baseline.inst_base.contains_key(&r.path) {
578                        pass.declared.insert(r.path.clone());
579                    }
580                    // Nothing here and nothing cited: a removal of a path
581                    // that was never there, or that a sibling already
582                    // removed. Applied as a no-op.
583                    pass.applied.push(r.clone());
584                }
585                Err(e) => {
586                    // Unreadable is not absent (`confirm_absences`' rule):
587                    // fail closed, and say so where the pod-side reader
588                    // looks. Transient, so deferred rather than refused.
589                    self.state.append_conflict(&ConflictRecord {
590                        path: r.path.clone(),
591                        foreign_etag: String::new(),
592                        preserved_key: None,
593                        kind: format!("removal-deferred: cannot stat the path: {e}"),
594                        at_unix: now_unix(),
595                    })?;
596                    pass.deferred += 1;
597                }
598                Ok(m) if !m.is_file() => {
599                    refuse(
600                        "removal-refused-not-a-file",
601                        format!("{} is not a regular file in the workspace", r.path),
602                    )?;
603                }
604                Ok(_) => {
605                    if local_dirty(&local, be) {
606                        refuse(
607                            "removal-refused-dirty",
608                            format!(
609                                "{} has unpublished local changes; the removal {} asked for                                  was not applied and the agent's work is kept — publish or                                  discard the local version, then ask again",
610                                r.path, r.author
611                            ),
612                        )?;
613                        continue;
614                    }
615                    if let Err(e) = std::fs::remove_file(&local) {
616                        self.state.append_conflict(&ConflictRecord {
617                            path: r.path.clone(),
618                            foreign_etag: String::new(),
619                            preserved_key: None,
620                            kind: format!("removal-unlink-failed (will retry): {e}"),
621                            at_unix: now_unix(),
622                        })?;
623                        pass.deferred += 1;
624                        continue;
625                    }
626                    // CONFIRM: only NotFound is absence.
627                    match std::fs::symlink_metadata(&local) {
628                        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
629                            pass.declared.insert(r.path.clone());
630                            pass.applied.push(r.clone());
631                        }
632                        _ => {
633                            self.state.append_conflict(&ConflictRecord {
634                                path: r.path.clone(),
635                                foreign_etag: String::new(),
636                                preserved_key: None,
637                                kind: "removal-unlink-unconfirmed (will retry)".into(),
638                                at_unix: now_unix(),
639                            })?;
640                            pass.deferred += 1;
641                        }
642                    }
643                }
644            }
645        }
646        Ok(pass)
647    }
648
649    /// Act on the inbox document's two request fields (§2.5, D14).
650    ///
651    /// The asymmetry is deliberate and is about blast radius, not
652    /// principle: a boundary request is PERFORMED (it publishes what is
653    /// already on disk and mutates nothing local), a sync request is
654    /// CARRIED into the ticker as advisory news and the agent decides.
655    ///
656    /// Both are watermarked by `requested_unix` rather than consumed by
657    /// a clearing CAS. The field is idempotent state, so a repeat is a
658    /// no-op and a lost clear cannot double-publish; and not writing
659    /// means the doors cost no request at all, which is what §2.5
660    /// promises.
661    fn note_verb_requests(&mut self, doc: &inbox::InboxDoc) -> LeanResult<()> {
662        let mut mark = self.load_verb_watermark()?;
663        if let Some(r) = &doc.sync_request {
664            if r.requested_unix > mark.sync_unix {
665                self.carry_sync_request(r.requested_unix, &r.requestor)?;
666                mark.sync_unix = r.requested_unix;
667            }
668        }
669        if let Some(r) = &doc.boundary_request {
670            if r.requested_unix > mark.boundary_unix {
671                self.request_boundary(
672                    &format!("gw:{}:{}", r.requested_unix, r.requestor),
673                    Some(format!("boundary requested by {}", r.requestor)),
674                )?;
675                mark.boundary_unix = r.requested_unix;
676            }
677        }
678        self.save_verb_watermark(&mark)
679    }
680
681    fn verb_watermark_path(&self) -> std::path::PathBuf {
682        self.cfg.state_dir().join("verb-requests.json")
683    }
684
685    pub(crate) fn load_verb_watermark(&self) -> LeanResult<VerbWatermark> {
686        let p = self.verb_watermark_path();
687        if !p.exists() {
688            return Ok(VerbWatermark::default());
689        }
690        Ok(std::fs::read(&p)
691            .ok()
692            .and_then(|b| serde_json::from_slice(&b).ok())
693            .unwrap_or_default())
694    }
695
696    fn save_verb_watermark(&self, m: &VerbWatermark) -> LeanResult<()> {
697        let bytes = serde_json::to_vec_pretty(m)
698            .map_err(|e| LeanError::State(format!("verb watermark: {e}")))?;
699        super::control::write_atomic(&self.verb_watermark_path(), &bytes)
700    }
701
702    async fn preserve_conflict_copy(&self, path: &str, etag: &str) -> LeanResult<String> {
703        let key = self.cfg.file_key(path);
704        let dst = self.cfg.conflict_key(&uuid::Uuid::new_v4().to_string(), path);
705        // v1: client-side copy (GET + guarded PUT). The server-side
706        // CopyObject lever is the designed v2 optimization for large
707        // files.
708        let (meta, body) = self.store.get_whole(&key, Some(etag)).await?;
709        let crc = crc64_nvme(&body);
710        let stamps = GenerationStamps {
711            generation: 0,
712            epoch: self
713                .lease
714                .as_ref()
715                .map(|l| l.epoch)
716                .or_else(|| self.state.load_incarnation().ok().flatten().map(|i| i.epoch))
717                .unwrap_or(0),
718            flush_uuid: "conflict-preserve".into(),
719            boundary_source: None,
720            posix: PosixStamps::from_meta(&meta.meta),
721        };
722        self.store
723            .put_whole(&dst, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
724            .await?;
725        Ok(dst)
726    }
727
728    /// Take the SECOND absence observation now, by direct `lstat`.
729    ///
730    /// `scan::classify` withholds a path's delete until absence has
731    /// survived two consecutive scans — the rename-vs-walk race guard
732    /// (`lib.rs:37`). The hazard that rule names is the WALK missing a
733    /// file renamed under it, not deletion itself. On the cadence path
734    /// the second walk arrives at the next floor tick and nobody has
735    /// been promised otherwise. On a DECLARED boundary it cannot wait:
736    /// the ack would claim a coherent point (D1 — "everything
737    /// ordered-before T") while the manifest still cites a file the
738    /// agent removed before it touched the sentinel, and it would say
739    /// so with `report.deleted: 0` and `status: "ok"`.
740    ///
741    /// A direct `lstat` is exactly what the rename-vs-walk guard asks
742    /// for and is immune to the walk race by construction, so the
743    /// second observation costs one syscall per transiently-absent
744    /// path — never a second full pass. The cadence path is unchanged
745    /// and still waits for the second walk.
746    pub(crate) fn confirm_absences(&self, classified: &mut scan::Classified) -> LeanResult<usize> {
747        if classified.first_absence.is_empty() {
748            return Ok(0);
749        }
750        let mut confirmed = 0;
751        for path in std::mem::take(&mut classified.first_absence) {
752            // ONLY NotFound is absence. Every other errno — EACCES on a
753            // parent, EIO, EMFILE, ELOOP — used to read as "the agent
754            // deleted it", and this is the oracle that PUBLISHES the
755            // delete and then DELETES THE OBJECT. An unreadable file is
756            // the one case where guessing is unrecoverable, so it fails
757            // CLOSED, exactly as the walk does at `scan.rs`'s
758            // `symlink_metadata(&path)?`. Two call sites, one syscall:
759            // they must not have opposite error policies.
760            let gone = match std::fs::symlink_metadata(self.cfg.root.join(&path)) {
761                Err(e) if e.kind() == std::io::ErrorKind::NotFound => true,
762                Err(e) => {
763                    classified.first_absence.insert(path.clone());
764                    return Err(LeanError::State(format!(
765                        "cannot confirm the absence of {path:?}: {e} — refusing to publish \
766                         a deletion on an unreadable path"
767                    )));
768                }
769                Ok(_) => false,
770            };
771            if gone {
772                classified.deletes.insert(path);
773                confirmed += 1;
774            } else {
775                // Back on disk: the rename-vs-walk race, caught exactly
776                // as the rule intends. Still only a first absence.
777                classified.first_absence.insert(path);
778            }
779        }
780        Ok(confirmed)
781    }
782
783    /// Renew the lease if it has gone stale MID-BARRIER, and hand back
784    /// the cell as it was read so the CALLER can fence on it.
785    ///
786    /// Costs nothing on a short barrier: the renewal is skipped unless
787    /// it was already due, so the request count is unchanged — this
788    /// moves WHEN the renewal can happen, not how often.
789    ///
790    /// What this does NOT do, stated because the previous comment
791    /// claimed it did: it never raises `Fenced` for a deposed holder.
792    /// `lease::renew` can only 412 when the cell is still at OUR epoch
793    /// with a token we do not hold; a cell at a FOREIGN epoch is
794    /// skipped here (renewing it would be nonsense), so a deposed
795    /// straggler passed straight through. The 2026-09-03 audit found
796    /// the cadence barrier relying on exactly that phantom fence
797    /// between upload chunks and completing every remaining data PUT
798    /// after a takeover. The cell is returned so the caller can compare
799    /// it against its lease — the cadence barrier uses the read it
800    /// already paid for rather than a separate fence. `None` when there is
801    /// no lease or the read failed: neither is a fence, and the
802    /// ordinary renewal arm will try again.
803    pub(crate) async fn renew_if_due(&mut self) -> LeanResult<Option<EpochState>> {
804        let Some(lease) = self.lease.clone() else { return Ok(None) };
805        let state = match self.store.epoch_read(&self.cfg.epoch_key()).await {
806            Ok(Some(s)) => s,
807            _ => return Ok(None),
808        };
809        let now = super::now_unix();
810        let fresh = state
811            .last_renew_unix
812            .map(|t| now.saturating_sub(t) < RENEW_WITHIN_SECS)
813            .unwrap_or(false);
814        if fresh || state.epoch != lease.epoch {
815            // Not due, or not ours to renew. The caller decides what a
816            // foreign cell means; this function only knows it cannot
817            // renew one.
818            return Ok(Some(state));
819        }
820        super::lease::renew(self).await?;
821        Ok(Some(state))
822    }
823
824    /// The between-chunk fence of the cadence barrier: a cell that no
825    /// longer names this holder at this epoch stops the upload set
826    /// HERE, before the next chunk's PUTs. Every one of those PUTs
827    /// carries If-Match on a baseline etag that still matches — the
828    /// successor has published nothing yet — so without this a deposed
829    /// straggler overwrites the cited generation of every key it still
830    /// had to upload, and every reader's S3-wins arm then adopts the
831    /// uncited bytes silently (audit 2026-09-03, finding 1).
832    fn fence_on_cell(&self, cell: &EpochState) -> LeanResult<()> {
833        let Some(lease) = self.lease.as_ref() else {
834            return Err(LeanError::Fenced("lease dropped mid-barrier".into()));
835        };
836        if cell.epoch != lease.epoch || cell.holder_id != lease.holder_id {
837            return Err(LeanError::Fenced(format!(
838                "deposed between upload chunks: cell at epoch {} holder {} (we are epoch {})",
839                cell.epoch, cell.holder_id, lease.epoch
840            )));
841        }
842        Ok(())
843    }
844
845    /// Steps 2–7. `barrier` = one full publish cycle (the cadence arm).
846    pub async fn run_barrier(&mut self) -> LeanResult<BarrierReport> {
847        self.barrier_inner(false, "manual").await
848    }
849
850    /// The floor tick's barrier. Stamped `cadence` so the bucket can
851    /// answer "which clock installed this boundary?" without the ack —
852    /// the ack is a LOCAL file, and the fleet-visible question is asked
853    /// by operators and by the gateway's `/status`, neither of which
854    /// can see it.
855    pub async fn cadence_barrier(&mut self) -> LeanResult<BarrierReport> {
856        self.barrier_inner(false, "cadence").await
857    }
858
859    /// A barrier whose result somebody is going to ACK: a sentinel
860    /// honor (D1) or the preStop drain (D10). Identical to
861    /// `run_barrier` except that it confirms first-absence paths rather
862    /// than acking a boundary that withholds them.
863    pub async fn declared_barrier(&mut self) -> LeanResult<BarrierReport> {
864        self.barrier_inner(true, "sentinel").await
865    }
866
867    /// `declared_barrier` under an explicit provenance stamp — the
868    /// drain and the budget-deferred honor are the same barrier
869    /// reporting a different clock.
870    pub async fn declared_barrier_as(&mut self, source: &str) -> LeanResult<BarrierReport> {
871        self.barrier_inner(true, source).await
872    }
873
874    async fn barrier_inner(&mut self, declared: bool, source: &str) -> LeanResult<BarrierReport> {
875        // No lease is held until the commit section (design 2026-09-13
876        // §4): the consume, the scan and the uploads below run with the
877        // cell at rest, guarded by each object's own etag. What the
878        // uploads are STAMPED with is the last epoch this incarnation
879        // held — informational, like every epoch stamp on an object; the
880        // manifest entries carry the commit section's real epoch.
881        let epoch_hint = self.state.load_incarnation()?.map(|i| i.epoch).unwrap_or(0);
882        let mut report = BarrierReport::default();
883        let barrier_started = std::time::Instant::now();
884        self.trace("barrier_start", serde_json::json!({"source": source, "declared": declared}));
885
886        // STEP 0: finish any rescope a crash left half-applied, before
887        // the scan reads the tree (scoped-read design §4.3). This is the
888        // gate that makes the narrow verb safe at all: a half-applied
889        // narrow is a tree whose files and whose citations disagree, and
890        // `classify` reads that disagreement as an upload or as a
891        // DELETE depending on which half landed. Replay is idempotent,
892        // so the cost on the normal path is one `stat` of a file that
893        // is not there.
894        if let Some(r) = self.replay_scope_intent().await? {
895            report.rescope_replayed = true;
896            eprintln!(
897                "flint-sync: finished an interrupted rescope before the barrier \
898                 (uncited {}, unlinked {}, materialised {})",
899                r.uncited, r.unlinked, r.materialized
900            );
901        }
902
903        // Step 1: the cell, read ONCE — HITL entries into the tree, then
904        // the DECLARED removals (delete/rename design §4-§6).
905        let inbox_doc = inbox::load(self.store.as_ref(), &self.cfg).await?.doc;
906        let (consumed, settled_locally) = self.consume_counted(&inbox_doc).await?;
907        report.consumed = consumed.len() + settled_locally;
908        let removals = self.apply_removals(&inbox_doc).await?;
909        report.removed = removals.declared.iter().cloned().collect();
910        report.removals_refused = removals.refused.len();
911        // A refusal is settled NOW, whatever else this barrier does: the
912        // cell is where the human who asked reads the answer, and a
913        // refused removal is never retried, so saying so before the
914        // no-diff return below loses nothing.
915        inbox::settle_removals(self.store.as_ref(), &self.cfg, epoch_hint, &[], &removals.refused)
916            .await?;
917
918        // Step 2: scan-diff against the persisted baseline.
919        let mut baseline = self.state.load_baseline()?;
920        let scanned = scan::scan(&self.cfg.root)?;
921        let mut classified = scan::classify(&scanned, &baseline);
922        if declared {
923            report.absences_confirmed = self.confirm_absences(&mut classified)?;
924        }
925        // A declared removal is a deletion with its basis RECORDED, so
926        // it skips the two-scan guard the walk needs and goes straight
927        // into the delete set — which is what lets a rename's two halves
928        // ride ONE manifest generation (§4).
929        for p in &removals.declared {
930            classified.first_absence.remove(p);
931            if classified.uploads.contains(p) {
932                // The agent re-created the path between the unlink and
933                // the scan. The declaration named the OLD file, which is
934                // gone from the tree (and, for a rename, moved); what is
935                // here now is a new file the agent wrote, and it
936                // publishes as one — its PUT supersedes the old object
937                // under the same key. Deleting it would un-cite the
938                // agent's work for a barrier and GC-skip its object; the
939                // model's install gives the upload the same precedence.
940                continue;
941            }
942            classified.deletes.insert(p.clone());
943        }
944        report.first_absence = classified.first_absence.iter().cloned().collect();
945
946        // Skip-on-no-diff: nothing local, nothing consumed, no pending
947        // citation repair, and the bucket manifest where we left it.
948        let repairs_pending = baseline
949            .entries
950            .iter()
951            .any(|(p, be)| be.size != u64::MAX && baseline.inst_base.get(p) != Some(&be.etag));
952        if classified.uploads.is_empty()
953            && classified.deletes.is_empty()
954            && consumed.is_empty()
955            && classified.first_absence.is_empty()
956            && removals.applied.is_empty()
957            && !repairs_pending
958        {
959            // Read the POINTER, never the entries: the 0b rig measured
960            // the idle tick at 1M entries as 27 s / 1.3 GiB — dominated
961            // by fetching and parsing a 264 MiB document just to read
962            // `seq`. The pointer is a few hundred bytes and carries the
963            // seq in its body, so a GET of it costs one round trip and
964            // answers strictly more than a HEAD of the old object did.
965            //
966            // Legacy workspaces (no pointer yet) keep the HEAD they had.
967            let unchanged = match manifest::load_pointer(self.store.as_ref(), &self.cfg).await? {
968                Some(manifest::LoadedPointer { pointer: p, etag, .. }) => {
969                    // The news ticker rides this request for free — D5's
970                    // "zero added bucket requests" is still literal.
971                    report.observed_seq = Some(p.seq);
972                    report.observed_etag = Some(etag.clone());
973                    baseline.manifest_etag.as_deref() == Some(etag.as_str())
974                }
975                None => match self.store.head(&self.cfg.manifest_key()).await {
976                    Ok(meta) => {
977                        report.observed_seq =
978                            GenerationStamps::from_meta(&meta.meta).map(|s| s.generation);
979                        report.observed_etag = Some(meta.etag.clone());
980                        baseline.manifest_etag.as_deref() == Some(meta.etag.as_str())
981                    }
982                    Err(StoreError::NotFound(_)) => baseline.manifest_etag.is_none(),
983                    Err(e) => return Err(e.into()),
984                },
985            };
986            if unchanged {
987                report.no_change = true;
988                report.seq = Some(baseline.seq);
989                // prev_scan still advances (the two-scan rule's clock) —
990                // but the baseline document (hundreds of MB at 1M
991                // entries) is rewritten only when the scan SET moved.
992                // Compare without building the set: both sides iterate
993                // sorted, and the allocation this used to make was the
994                // size of the very document it was trying not to write.
995                if scanned.len() != baseline.prev_scan.len()
996                    || !scanned.keys().eq(baseline.prev_scan.iter())
997                {
998                    baseline.prev_scan = scanned.keys().cloned().collect();
999                    self.state.save_baseline(&baseline)?;
1000                }
1001                self.trace_barrier_end(&report, barrier_started);
1002                return Ok(report);
1003            }
1004        }
1005
1006        // Step 3: intent journal, then the window (the commitment
1007        // point: the same CAS drops the consumed entries).
1008        let flush_uuid = uuid::Uuid::new_v4().to_string();
1009        self.trace("scan", serde_json::json!({"flush": flush_uuid, "uploads": classified.uploads.len(),
1010            "deletes": classified.deletes.len(), "first_absence": classified.first_absence.len()}));
1011        let mut intent = self.state.load_intent()?;
1012        let prior_uuids = {
1013            let mut v = intent.recent_uuids.clone();
1014            if !intent.flush_uuid.is_empty() {
1015                v.push(intent.flush_uuid.clone());
1016            }
1017            v
1018        };
1019        // Carried forward, not dropped: it stays true until we install
1020        // again, and the merge below is what reads it.
1021        let prev_installed = intent.installed_etag.clone();
1022        intent = IntentJournal {
1023            flush_uuid: flush_uuid.clone(),
1024            keys: classified.uploads.iter().map(|p| self.cfg.file_key(p)).collect(),
1025            recent_uuids: prior_uuids.clone(),
1026            installed_etag: prev_installed.clone(),
1027            installed_foreign: intent.installed_foreign.clone(),
1028            declared_deletes: removals.declared.iter().cloned().collect(),
1029        };
1030        self.state.save_intent(&intent)?;
1031        // The window is NOT opened here any more. It is the bucket's
1032        // "a barrier is committing" sign for HITL writers, and it opens
1033        // in the commit section, under the lease, at the epoch the
1034        // commit really holds. The uploads below race a HITL write the
1035        // way two writers race each other: If-Match decides, the loser
1036        // is preserved and recorded (`LeanNoWindowHolds` proves safety
1037        // never depended on the window).
1038        //
1039        // The consumed entries are not dropped here either — "durably
1040        // in the baseline" is true of a container restart and false of
1041        // a pod REPLACEMENT: the emptyDir goes with the pod. They leave
1042        // the cell with the window clear, after the manifest cites
1043        // them; a successor that finds them re-consumes, idempotently.
1044
1045        // Step 4: guarded uploads, fanned out under a bounded window
1046        // (each key's guard chain is independent; the 412 policy and
1047        // conflict records are applied to the collected results below,
1048        // in deterministic path order).
1049        let mut upserts: BTreeMap<String, LeanEntry> = BTreeMap::new();
1050        let mut parked: BTreeSet<String> = BTreeSet::new();
1051        // Citations whose etag was observed with no lease held (adopted
1052        // uploads, citation repairs): re-read inside the commit section.
1053        let mut observed: BTreeSet<String> = BTreeSet::new();
1054        let mut new_baseline_entries: BTreeMap<String, BaselineEntry> = BTreeMap::new();
1055        let outcomes: Vec<(String, LeanResult<UploadOutcome>)> = {
1056            use futures::stream::{self, StreamExt};
1057            // CHUNKED into waves, so a wave's outcomes are collected
1058            // before the next is fanned out. No lease is held here
1059            // (design 2026-09-13 §4), so nothing renews or fences
1060            // between chunks any more — the between-chunk fence that
1061            // stopped a deposed straggler's PUTs went with the straggler:
1062            // a PUT outside the lease is one legitimate writer's PUT,
1063            // guarded by If-Match, and a collision is preserved and
1064            // superseded, never silently overwritten.
1065            let mut outcomes: Vec<(String, LeanResult<UploadOutcome>)> = Vec::new();
1066            // NOT cfg.fanout. That knob was raised to 128 for the READ
1067            // path, and the read-side measurement (2.5x on 20k small
1068            // files) says nothing about this one, so uploads keep the
1069            // value they were measured at until someone measures them.
1070            // Memory is no longer the reason they are separate: this
1071            // path is byte-bounded too now, by the store's gate
1072            // (`with_upload_inflight_max_bytes`), which charges every
1073            // whole body below and every multipart part inside the
1074            // store from before its read until its PUT returns.
1075            let fanout = self.cfg.upload_fanout.max(1);
1076            let chunk = fanout.saturating_mul(UPLOAD_CHUNK_WAVES).max(1);
1077            let all: Vec<&String> = classified.uploads.iter().collect();
1078            for group in all.chunks(chunk) {
1079                {
1080                    let this: &Syncer = &*self;
1081                    let mut part: Vec<(String, LeanResult<UploadOutcome>)> =
1082                        stream::iter(group.iter().map(|path| {
1083                            let path = *path;
1084                            let scanned_entry = &scanned[path];
1085                            let base = baseline.entries.get(path);
1086                            let flush_uuid = &flush_uuid;
1087                            let prior_uuids = &prior_uuids;
1088                            async move {
1089                                let r = this
1090                                    .upload_one(
1091                                        path, scanned_entry, base, epoch_hint, flush_uuid,
1092                                        prior_uuids,
1093                                    )
1094                                    .await;
1095                                (path.clone(), r)
1096                            }
1097                        }))
1098                        .buffer_unordered(fanout)
1099                        .collect()
1100                        .await;
1101                    outcomes.append(&mut part);
1102                }
1103            }
1104            outcomes
1105        };
1106        let mut outcomes = outcomes;
1107        outcomes.sort_by(|a, b| a.0.cmp(&b.0));
1108        for (path, outcome) in outcomes {
1109            let path = &path;
1110            match outcome? {
1111                UploadOutcome::Published { entry, baseline_entry, adopted } => {
1112                    self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "etag": entry.etag,
1113                        "outcome": if adopted { "adopted" } else { "put" }}));
1114                    if adopted {
1115                        observed.insert(path.clone());
1116                    }
1117                    report.published_bytes += entry.size;
1118                    upserts.insert(path.clone(), entry);
1119                    new_baseline_entries.insert(path.clone(), baseline_entry);
1120                    report.uploaded.push(path.clone());
1121                }
1122                UploadOutcome::Parked { foreign_etag } => {
1123                    self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "etag": foreign_etag, "outcome": "parked"}));
1124                    parked.insert(path.clone());
1125                    self.state.append_conflict(&ConflictRecord {
1126                        path: path.clone(),
1127                        foreign_etag,
1128                        preserved_key: None, // parking preserves in place
1129                        kind: "upload-412-parked".into(),
1130                        at_unix: now_unix(),
1131                    })?;
1132                    report.parked.push(path.clone());
1133                }
1134                UploadOutcome::Deferred => {
1135                    self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "outcome": "deferred"}));
1136                    report.deferred.push(path.clone());
1137                }
1138            }
1139        }
1140
1141        // Citation repairs: paths whose integrated object (the
1142        // baseline) differs from the manifest's citation — consumed
1143        // HITL adoptions and checkout's S3-wins arm. No bytes move; the
1144        // manifest re-cites what this syncer already integrated.
1145        // Without this, an adopted upload is clean-vs-baseline, never
1146        // enters the upload set, and the manifest silently drops it —
1147        // the battery's amputation leg caught exactly that.
1148        let repair_candidates: Vec<String> = baseline
1149            .entries
1150            .iter()
1151            .filter(|(path, be)| {
1152                !classified.uploads.contains(*path)
1153                    && !classified.deletes.contains(*path)
1154                    && be.size != u64::MAX // consume-dirty sentinel: publishes via upload
1155                    && baseline.inst_base.get(*path) != Some(&be.etag)
1156            })
1157            .map(|(path, _)| path.clone())
1158            .collect();
1159        for path in repair_candidates {
1160            let key = self.cfg.file_key(&path);
1161            let be = baseline.entries[&path].clone();
1162            match self.store.head(&key).await {
1163                Ok(meta) if meta.etag == be.etag => {
1164                    let crc = repair_crc(&path, &be, &meta)?;
1165                    let stamps = GenerationStamps::from_meta(&meta.meta);
1166                    let scan_entry = scanned.get(&path);
1167                    upserts.insert(
1168                        path.clone(),
1169                        LeanEntry {
1170                            key,
1171                            etag: meta.etag.clone(),
1172                            crc64_b64: crc,
1173                            size: meta.size,
1174                            mode: stamps
1175                                .as_ref()
1176                                .and_then(|s| s.posix)
1177                                .map(|p| p.mode)
1178                                .or(scan_entry.map(|s| s.mode))
1179                                .unwrap_or(0o644),
1180                            mtime_unix: scan_entry.map(|s| s.mtime_unix).unwrap_or(0),
1181                            generation: stamps.map(|s| s.generation).unwrap_or(be.generation),
1182                            epoch: epoch_hint,
1183                        },
1184                    );
1185                    observed.insert(path.clone());
1186                }
1187                // Moved again or gone: the next consume reconciles it.
1188                _ => {}
1189            }
1190        }
1191
1192        // A PULL-ONLY boundary: nothing uploaded, deleted, consumed, removed
1193        // or re-cited, so the merge can only add nothing — and then the
1194        // commit section writes nothing to the bucket (no CAS, no GC, no
1195        // inbox entry to drop) and what remains is local: queue the other
1196        // writers' changes and take theirs as the merge base. That needs no
1197        // fence and no window; claiming for it queued every such writer
1198        // behind the publishing ones (65 of 191 claims in the writers drill).
1199        // A merge that does add something (a mirror flag to restamp, say)
1200        // falls through to the commit section.
1201        if classified.uploads.is_empty()
1202            && classified.deletes.is_empty()
1203            && upserts.is_empty()
1204            && consumed.is_empty()
1205            && removals.applied.is_empty()
1206            && observed.is_empty()
1207        {
1208            let current = manifest::load(self.store.as_ref(), &self.cfg).await?;
1209            let m = self.merge_onto(
1210                current.as_ref(),
1211                prev_installed.as_deref(),
1212                &baseline.inst_base,
1213                &upserts,
1214                &classified.deletes,
1215                &parked,
1216                &flush_uuid,
1217            );
1218            if let Some(handle) = m.expected.as_ref().filter(|_| m.adds_nothing()) {
1219                if !m.foreign.is_empty() || !m.gone.is_empty() {
1220                    self.state.queue_foreign(&foreign_changes(&m.foreign, &m.gone))?;
1221                    self.trace("queue", serde_json::json!({"flush": flush_uuid, "upserts": m.foreign.len(), "tombstones": m.gone.len(), "fence": false}));
1222                }
1223                report.seq = Some(m.theirs.seq);
1224                report.no_change = true;
1225                report.manifest_etag = Some(handle.etag.clone());
1226                report.observed_seq = Some(m.theirs.seq);
1227                report.observed_etag = Some(handle.etag.clone());
1228                report.foreign_queued = m.foreign.len();
1229                baseline.inst_base = m.theirs.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
1230                baseline.seq = m.theirs.seq;
1231                baseline.manifest_etag = Some(handle.etag.clone());
1232                baseline.prev_scan = scanned.keys().cloned().collect();
1233                self.state.save_baseline(&baseline)?;
1234                self.state.clear_intent_keys()?;
1235                self.trace_barrier_end(&report, barrier_started);
1236                return Ok(report);
1237            }
1238        }
1239
1240        // THE COMMIT SECTION (design 2026-09-13 §4). Everything above ran
1241        // with no lease: the uploads are durable and etag-guarded, and
1242        // nothing is cited yet. Claim the cell now — for the merge, the
1243        // CAS, the deletes and the baseline — and hand it to the next
1244        // waiter when done. This is the only stretch two writers of one
1245        // workspace serialise on, and it is small: one pointer CAS and
1246        // the guarded deletes, milliseconds to seconds, never the upload
1247        // of a checkpoint.
1248        let held = super::lease::claim(self).await?;
1249        let epoch = held.epoch;
1250        eprintln!("flint-sync: publish fence held (epoch {epoch})");
1251        if self.cfg.drill_hold_commit_secs > 0 {
1252            eprintln!(
1253                "flint-sync: DRILL: holding the fence for {}s inside the commit section",
1254                self.cfg.drill_hold_commit_secs
1255            );
1256            tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_commit_secs)).await;
1257        }
1258        let commit: LeanResult<()> = async {
1259            // The entries carry the epoch the commit HOLDS, not the hint
1260            // the uploads were stamped with.
1261            for e in upserts.values_mut() {
1262                e.epoch = epoch;
1263            }
1264            // Step 5: the manifest CAS (three-way merge; bounded retries).
1265            self.verify_not_deposed().await?;
1266            // EVERY citation this barrier adds was read or written with no
1267            // lease held. Another writer's commit may since have uncited
1268            // the path and collected the object: its GC deletes by the etag
1269            // it integrated. That etag names an ADOPTED object (the model's
1270            // LeanBarrierLeaseAdoptBlind) — and, since an S3 whole-PUT etag
1271            // is the MD5 of the bytes, it also names this barrier's own PUT
1272            // of IDENTICAL bytes (finding 13, the writers drill's A3: a
1273            // same-content rewrite re-uploaded, the peer's GC deleted it,
1274            // the commit cited a hole). Collection runs only inside a
1275            // commit section, so a re-read HERE, holding the fence, cannot
1276            // be overtaken before this CAS. Whatever is gone or replaced is
1277            // withheld: parked, recorded, and left dirty for the next
1278            // barrier to publish with a PUT of its own.
1279            let heads: Vec<(String, LeanResult<bool>)> = {
1280                use futures::stream::{self, StreamExt};
1281                let this: &Syncer = &*self;
1282                stream::iter(upserts.iter().map(|(path, cited)| async move {
1283                    let still = match this.store.head(&cited.key).await {
1284                        Ok(meta) => Ok(meta.etag == cited.etag),
1285                        Err(StoreError::NotFound(_)) => Ok(false),
1286                        Err(e) => Err(e.into()),
1287                    };
1288                    (path.clone(), still)
1289                }))
1290                .buffer_unordered(self.cfg.upload_fanout.max(1))
1291                .collect()
1292                .await
1293            };
1294            let mut heads = heads;
1295            heads.sort_by(|a, b| a.0.cmp(&b.0));
1296            for (path, still) in heads {
1297                let still_there = still?;
1298                let was_observed = observed.contains(&path);
1299                let cited = &upserts[&path];
1300                self.trace("observed", serde_json::json!({"flush": flush_uuid, "path": path, "etag": cited.etag, "still": still_there, "own_put": !was_observed}));
1301                if still_there {
1302                    continue;
1303                }
1304                let gone = upserts.remove(&path).expect("looked up above");
1305                new_baseline_entries.remove(&path);
1306                report.uploaded.retain(|p| p != &path);
1307                report.published_bytes = report.published_bytes.saturating_sub(gone.size);
1308                parked.insert(path.clone());
1309                let kind = if was_observed {
1310                    "adopt-withheld: the object this barrier found already at its key was \
1311                     replaced or collected before its commit; nothing cited, the path stays \
1312                     dirty and publishes next barrier"
1313                } else {
1314                    "upload-withheld: the object this barrier PUT was replaced or collected \
1315                     before its commit (a peer's GC recognizes the etag of identical bytes); \
1316                     nothing cited, the path stays dirty and publishes next barrier"
1317                };
1318                self.state.append_conflict(&ConflictRecord {
1319                    path: path.clone(),
1320                    foreign_etag: gone.etag,
1321                    preserved_key: None,
1322                    kind: kind.into(),
1323                    at_unix: now_unix(),
1324                })?;
1325                report.parked.push(path.clone());
1326            }
1327            // The window: the bucket-visible "a barrier is committing" sign
1328            // HITL writers wait on, at the epoch this commit holds.
1329            let deadline = now_unix() + self.cfg.window_slack_secs;
1330            inbox::open_window(self.store.as_ref(), &self.cfg, epoch, deadline).await?;
1331            let mut foreign_entries: Vec<(String, LeanEntry)> = vec![];
1332            let mut foreign_gone: Vec<String> = vec![];
1333            // Set when the merge added nothing to the document: nothing is
1334            // installed, and theirs becomes the merge base as it stands.
1335            let mut installed_nothing = false;
1336            let mut attempt = 0;
1337            let (installed, installed_etag) = loop {
1338                attempt += 1;
1339                if attempt > 4 {
1340                    return Err(LeanError::State(
1341                        "manifest CAS lost 4 merge races — refusing this barrier".into(),
1342                    ));
1343                }
1344                let current = manifest::load(self.store.as_ref(), &self.cfg).await?;
1345                let MergeOnto { theirs, expected, merged, foreign, gone } = self.merge_onto(
1346                    current.as_ref(),
1347                    prev_installed.as_deref(),
1348                    &baseline.inst_base,
1349                    &upserts,
1350                    &classified.deletes,
1351                    &parked,
1352                    &flush_uuid,
1353                );
1354                // Nothing of ours changes the document — a barrier that only
1355                // found the manifest moved by another writer. Installing it
1356                // anyway was an empty generation, and the peer's next tick
1357                // then found the manifest moved and did the same: two idle
1358                // writers traded generations (and cell claims) for as long
1359                // as both ran. Theirs becomes the merge base as it stands.
1360                if let Some(handle) = expected.as_ref() {
1361                    if merged.entries == theirs.entries && merged.sole_writer == theirs.sole_writer {
1362                        foreign_entries = foreign;
1363                        foreign_gone = gone;
1364                        installed_nothing = true;
1365                        break (theirs, handle.etag.clone());
1366                    }
1367                }
1368                match manifest::cas_write_stamped(
1369                    self.store.as_ref(),
1370                    &self.cfg,
1371                    &merged,
1372                    expected.as_ref(),
1373                    epoch,
1374                    &flush_uuid,
1375                    Some(source),
1376                )
1377                .await
1378                {
1379                    Ok(meta) => {
1380                        // Before the deletes and before step 7: this is the
1381                        // only record that survives a crash in that window.
1382                        self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1383                            "expected": expected.as_ref().map(|h| h.etag.clone()), "etag": meta.etag, "result": "ok"}));
1384                        intent.installed_etag = Some(meta.etag.clone());
1385                        intent.installed_foreign = foreign_changes(&foreign, &gone);
1386                        self.state.save_intent(&intent)?;
1387                        foreign_entries = foreign;
1388                        foreign_gone = gone;
1389                        break (merged, meta.etag);
1390                    }
1391                    Err(LeanError::Store(StoreError::PreconditionFailed(_)))
1392                    | Err(LeanError::Store(StoreError::Conflict(_))) => {
1393                        self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1394                            "expected": expected.as_ref().map(|h| h.etag.clone()), "result": "lost"}));
1395                        // Re-verify the cell before retrying: a rotation is
1396                        // exactly this 412, and re-merging past it would be
1397                        // the straggler install.
1398                        self.verify_not_deposed().await?;
1399                        continue;
1400                    }
1401                    Err(e) => return Err(e),
1402                }
1403            };
1404            report.seq = Some(installed.seq);
1405            report.no_change = installed_nothing;
1406            // A fused install IS a coherent point, and cadence/hybrid have
1407            // exactly one source. The gauges must not report "no boundary
1408            // ever" on a workspace that publishes every minute. A barrier
1409            // that installed nothing marks no boundary of its own.
1410            if !installed_nothing {
1411                self.note_boundary("cadence", installed.seq)?;
1412            }
1413            report.manifest_etag = Some(installed_etag.clone());
1414            report.observed_seq = Some(installed.seq);
1415            report.observed_etag = Some(installed_etag.clone());
1416            report.foreign_queued = foreign_entries.len();
1417
1418            // Step 6: deletes LAST — GC of keys the NEW manifest no longer
1419            // references, HEAD-guarded on the recognized ETag.
1420            let mut swept = 0usize;
1421            let mut gc_held = false;
1422            for path in &classified.deletes {
1423                // A mass delete is the one long stretch of the commit
1424                // section: keep the token moving so a waiter does not count
1425                // a live holder dead, and fence on what the read returns.
1426                swept += 1;
1427                if swept % 200 == 0 {
1428                    if let Some(cell) = self.renew_if_due().await? {
1429                        self.fence_on_cell(&cell)?;
1430                    }
1431                }
1432                if installed.entries.contains_key(path) {
1433                    continue; // delete/modify resolved foreign-wins: not garbage
1434                }
1435                let key = self.cfg.file_key(path);
1436                let recognized = baseline.entries.get(path).map(|b| b.etag.clone()).or_else(|| {
1437                    // A DECLARED removal of a path this tree never held (a
1438                    // scoped workspace's out-of-scope citation): what the
1439                    // declaration named is the object the manifest cited
1440                    // when this barrier began, and that is the merge base.
1441                    removals
1442                        .declared
1443                        .contains(path)
1444                        .then(|| baseline.inst_base.get(path).cloned())
1445                        .flatten()
1446                });
1447                // The HEAD answers most paths in one request (gone, or an
1448                // etag we do not recognize). The DELETE itself carries
1449                // If-Match on the recognized etag: another writer's
1450                // upload holds no lease and can replace the object
1451                // between the two requests, and an unconditional delete
1452                // then removed bytes that writer's commit cites (the
1453                // model's LeanBarrierLeaseGCUnconditional).
1454                let unrecognized = match self.store.head(&key).await {
1455                    Err(StoreError::NotFound(_)) => {
1456                        self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": null, "result": "absent"}));
1457                        report.deleted.push(path.clone());
1458                        continue;
1459                    }
1460                    Ok(meta) if Some(&meta.etag) == recognized.as_ref() => {
1461                        if self.cfg.drill_hold_gc_secs > 0 && !gc_held {
1462                            gc_held = true;
1463                            eprintln!(
1464                                "flint-sync: DRILL: holding the GC for {}s between the HEAD of {path} and its DELETE",
1465                                self.cfg.drill_hold_gc_secs
1466                            );
1467                            self.trace("drill_hold", serde_json::json!({"flush": flush_uuid, "where": "gc", "path": path,
1468                                "secs": self.cfg.drill_hold_gc_secs}));
1469                            tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_gc_secs)).await;
1470                        }
1471                        match self.store.delete_if_match(&key, &meta.etag).await {
1472                            Ok(()) | Err(StoreError::NotFound(_)) => {
1473                                self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": meta.etag, "result": "deleted"}));
1474                                report.deleted.push(path.clone());
1475                                continue;
1476                            }
1477                            // Replaced between the HEAD and the DELETE:
1478                            // whatever is there now is not ours to
1479                            // collect. Name it, if it still exists.
1480                            Err(StoreError::PreconditionFailed(_)) => match self.store.head(&key).await {
1481                                Ok(now) => now.etag,
1482                                Err(StoreError::NotFound(_)) => {
1483                                    self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": meta.etag, "result": "replaced-absent"}));
1484                                    report.deleted.push(path.clone());
1485                                    continue;
1486                                }
1487                                Err(e) => return Err(e.into()),
1488                            },
1489                            Err(e) => return Err(e.into()),
1490                        }
1491                    }
1492                    Ok(meta) => meta.etag,
1493                    Err(e) => return Err(e.into()),
1494                };
1495                // An ETag this syncer does not recognize is NEVER deleted
1496                // (a HITL re-create landed after our CAS, or another
1497                // writer's upload).
1498                self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": unrecognized,
1499                    "recognized": recognized, "result": "skip"}));
1500                self.state.append_conflict(&ConflictRecord {
1501                    path: path.clone(),
1502                    foreign_etag: unrecognized,
1503                    preserved_key: None,
1504                    kind: "gc-skip".into(),
1505                    at_unix: now_unix(),
1506                })?;
1507            }
1508
1509            // Step 7: the foreign queue, then the baseline rewrite, intent
1510            // clear and window clear.
1511            //
1512            // Other writers' changes go to THIS writer's queue BEFORE the
1513            // merge base below moves past them: a crash between the two
1514            // re-applies them idempotently, where the other order would
1515            // leave a base that claims changes the tree never received.
1516            if !foreign_entries.is_empty() || !foreign_gone.is_empty() {
1517                self.state.queue_foreign(&foreign_changes(&foreign_entries, &foreign_gone))?;
1518                self.trace("queue", serde_json::json!({"flush": flush_uuid, "upserts": foreign_entries.len(), "tombstones": foreign_gone.len()}));
1519            }
1520            for (path, be) in new_baseline_entries {
1521                baseline.entries.insert(path, be);
1522            }
1523            for path in &report.deleted {
1524                baseline.entries.remove(path);
1525            }
1526            baseline.inst_base =
1527                installed.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
1528            baseline.seq = installed.seq;
1529            baseline.manifest_etag = Some(installed_etag);
1530            baseline.prev_scan = scanned.keys().cloned().collect();
1531            self.state.save_baseline(&baseline)?;
1532            self.state.clear_intent_keys()?;
1533            // No `merge-preserved` entries into the SHARED inbox any more:
1534            // they are this writer's, and sit in its local queue.
1535            inbox::clear_window_settling(
1536                self.store.as_ref(),
1537                &self.cfg,
1538                epoch,
1539                &[],
1540                &consumed,
1541                &removals.applied,
1542            )
1543            .await?;
1544            self.trace("window_clear", serde_json::json!({"flush": flush_uuid}));
1545            // Reap superseded generations. Immutable metadata that is never
1546            // collected is a leak that grows by a whole manifest per
1547            // publish, and this also collects the orphan a crash between the
1548            // entries PUT and the pointer CAS leaves behind. Best effort by
1549            // design: a publish that succeeded is not un-done by a failure
1550            // to tidy up after it.
1551            let swept_generations = match manifest::sweep_generations(self.store.as_ref(), &self.cfg).await {
1552                Ok(0) => 0,
1553                Ok(n) => {
1554                    eprintln!("flint-sync: reaped {n} superseded manifest generation(s)");
1555                    n
1556                }
1557                Err(e) => {
1558                    eprintln!("flint-sync: generation sweep: {e}");
1559                    0
1560                }
1561            };
1562            self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "generations", "removed": swept_generations}));
1563            // And the CHUNK reaper, same reason, same best-effort terms.
1564            // `sweep_chunks` returns 0 without a request on a workspace that
1565            // is not chunked, so this costs nothing until the layout moves.
1566            //
1567            // It was written to four model-established rules, guarded by six
1568            // mutation configs, and called from nothing but its own tests —
1569            // so with chunking on, every publish left its superseded chunks
1570            // in the bucket forever. The rules ran in the model and in the
1571            // suite and never once in production. `the_barrier_reaps...`
1572            // below is the test that asks whether this line exists at all.
1573            let swept_chunks = match manifest::sweep_chunks(self.store.as_ref(), &self.cfg).await {
1574                Ok(0) => 0,
1575                Ok(n) => {
1576                    eprintln!("flint-sync: reaped {n} unreferenced manifest chunk(s)");
1577                    n
1578                }
1579                Err(e) => {
1580                    eprintln!("flint-sync: chunk sweep: {e}");
1581                    0
1582                }
1583            };
1584            self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "chunks", "removed": swept_chunks}));
1585            Ok(())
1586        }
1587        .await;
1588        // Hand the cell on whatever happened — unless it is no longer
1589        // ours to hand on: a fence means a successor holds it, and the
1590        // barrier is simply abandoned (its manifest never installed; its
1591        // uploads stand, and the next barrier adopts them by flush_uuid).
1592        match &commit {
1593            Err(LeanError::Fenced(m)) => {
1594                self.trace("fence", serde_json::json!({"where": "commit", "detail": m}));
1595                self.lease = None
1596            }
1597            _ => {
1598                if let Err(e) = super::lease::release(self).await {
1599                    eprintln!(
1600                        "flint-sync: the publish fence could not be released ({e}); a waiter \
1601                         deposes it after the quiet threshold"
1602                    );
1603                }
1604            }
1605        }
1606        commit?;
1607        self.trace_barrier_end(&report, barrier_started);
1608        Ok(report)
1609    }
1610
1611    fn trace_barrier_end(&self, report: &BarrierReport, started: std::time::Instant) {
1612        if self.cfg.event_trace.is_none() {
1613            return;
1614        }
1615        self.trace(
1616            "barrier_end",
1617            serde_json::json!({
1618                "seq": report.seq, "uploaded": report.uploaded.len(), "deleted": report.deleted.len(),
1619                "parked": report.parked.len(), "consumed": report.consumed, "no_change": report.no_change,
1620                "ms": started.elapsed().as_millis() as u64, "requests": self.trace_requests(),
1621            }),
1622        );
1623    }
1624
1625    /// The > whole_put_max path: contiguous `PartSource::Local` chunks
1626    /// through `compose_generation` (streaming multipart; the store
1627    /// aborts its partial assembly on every failure path). The CRC is
1628    /// computed by a streaming pass first; a writer racing the compose
1629    /// fails server-side validation and the path DEFERS to the next
1630    /// barrier — publish-possibly-torn is put_whole's documented
1631    /// dilemma, not this path's.
1632    #[allow(clippy::too_many_arguments)]
1633    #[allow(clippy::too_many_arguments)]
1634    async fn upload_compose(
1635        &self,
1636        path: &str,
1637        key: &str,
1638        local_path: &Path,
1639        size: u64,
1640        scanned: &scan::ScanEntry,
1641        condition: PutCondition,
1642        stamps: GenerationStamps,
1643        generation: u64,
1644        epoch: u64,
1645        prior_uuids: &[String],
1646    ) -> LeanResult<UploadOutcome> {
1647        // NO PRE-PASS. The store accumulates the full-object CRC from
1648        // the parts as it reads them for upload, so this file is read
1649        // ONCE (the 2026-09-10 door drill measured the second read at
1650        // roughly 11 s of the `mixed` workload's 24.3 s).
1651        // Part grid: within [min_part_size, ...], at most max_parts,
1652        // contiguous from 0.
1653        let min_part = self.store.min_part_size().max(1);
1654        let max_parts = self.store.max_parts().max(1) as u64;
1655        let mut chunk = self.cfg.whole_put_max.max(min_part);
1656        let need = size.div_ceil(chunk);
1657        if need > max_parts {
1658            chunk = size.div_ceil(max_parts).div_ceil(min_part) * min_part;
1659        }
1660        let mut parts = vec![];
1661        let mut off = 0u64;
1662        while off < size {
1663            let len = chunk.min(size - off);
1664            parts.push(flint_store::PartSource::Local { offset: off, len });
1665            off += len;
1666        }
1667
1668
1669        // At most three attempts: the caller's condition; then once more
1670        // If-Match on what a 412 showed us — our own torn Complete
1671        // (recognized through the crash journal, review 2026-09-12
1672        // atomicity-2), or a foreign version now PRESERVED (inbox-1) —
1673        // or a create if the base object vanished (inbox-5). A 412 on
1674        // the retry means the path is being written under us: park.
1675        let mut condition = condition;
1676        for attempt in 0..3u32 {
1677            let spec = flint_store::ComposeSpec {
1678                progress: None,
1679                key,
1680                local_path,
1681                parts: parts.clone(),
1682                base_key: None,
1683                base_etag: None,
1684                condition: condition.clone(),
1685                stamps: stamps.clone(),
1686                crc64: None,
1687            };
1688            match self.store.compose_generation(&spec).await {
1689                Ok(meta) => {
1690                    // The store computed it; refuse to cite bytes nothing
1691                    // vouched for rather than invent a number for the
1692                    // manifest. A backend that returns no checksum has not
1693                    // validated the publish server-side either.
1694                    let crc = meta
1695                        .crc64_b64
1696                        .as_deref()
1697                        .and_then(flint_store::crc64_from_b64)
1698                        .ok_or_else(|| {
1699                            LeanError::State(format!(
1700                                "compose of {key} returned no full-object checksum — refusing to \
1701                                 cite bytes nothing vouched for"
1702                            ))
1703                        })?;
1704                    return Ok(UploadOutcome::published(
1705                        path, key.to_string(), meta.etag, crc, size, scanned, generation, epoch,
1706                    ));
1707                }
1708                Err(StoreError::ChecksumMismatch(_)) | Err(StoreError::NoSuchUpload(_)) => {
1709                    // Drift mid-compose, or the operator sweep aborted us:
1710                    // nothing published; the next barrier re-queues.
1711                    return Ok(UploadOutcome::Deferred);
1712                }
1713                // S3's answer to If-Match on a missing key (see upload_one):
1714                // the base vanished, and a vanished base is a create.
1715                Err(StoreError::NotFound(_)) if matches!(condition, PutCondition::IfMatch(_)) => {
1716                    condition = PutCondition::IfNoneMatchAny;
1717                    continue;
1718                }
1719                Err(StoreError::PreconditionFailed(_)) => {
1720                    // The AdoptOwn recognizer compares OUR bytes' checksum
1721                    // against what landed, and without the pre-pass we do
1722                    // not have one — so the file is read HERE, on the rare
1723                    // recovery path, instead of on every publish.
1724                    let crc = file_crc(local_path)?;
1725                    let head = match self.store.head(key).await {
1726                        Ok(h) => h,
1727                        Err(StoreError::NotFound(_)) => {
1728                            condition = PutCondition::IfNoneMatchAny;
1729                            continue;
1730                        }
1731                        Err(e) => return Err(e.into()),
1732                    };
1733                    let head_stamps = GenerationStamps::from_meta(&head.meta);
1734                    if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
1735                        // These bytes are already there (a torn Complete):
1736                        // cite them — re-read under the fence first.
1737                        let g = head_stamps.map(|s| s.generation).unwrap_or(generation);
1738                        return Ok(UploadOutcome::adopted(
1739                            path, key.to_string(), head.etag, crc, head.size, scanned, g, epoch,
1740                        ));
1741                    }
1742                    if attempt >= 1 {
1743                        return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1744                    }
1745                    let own = head_stamps
1746                        .as_ref()
1747                        .map(|s| s.flush_uuid == stamps.flush_uuid || prior_uuids.contains(&s.flush_uuid))
1748                        .unwrap_or(false);
1749                    if !own && self.preserve_foreign_412(path, &head).await.is_none() {
1750                        return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1751                    }
1752                    condition = PutCondition::IfMatch(head.etag);
1753                }
1754                Err(e) => return Err(e.into()),
1755            }
1756        }
1757        Ok(UploadOutcome::Deferred)
1758    }
1759
1760
1761    async fn upload_one(
1762        &self,
1763        path: &str,
1764        scanned: &scan::ScanEntry,
1765        base: Option<&BaselineEntry>,
1766        epoch: u64,
1767        flush_uuid: &str,
1768        prior_uuids: &[String],
1769    ) -> LeanResult<UploadOutcome> {
1770        let key = self.cfg.file_key(path);
1771        let local_path = self.cfg.root.join(path);
1772        let generation = base.map(|b| b.generation + 1).unwrap_or(1);
1773        // Review 2026-09-12, atomicity-6: the scan skipped symlinks; this
1774        // read followed them. A regular file swapped for a symlink between
1775        // the two published the link's TARGET — a file outside the
1776        // workspace, the syncer's own /proc/self/environ included. Every
1777        // stat here is an lstat and the read opens O_NOFOLLOW; a path that
1778        // is no longer a regular file is refused, recorded, and deferred
1779        // (the next scan skips it, and the two-scan rule retires it).
1780        let lmeta = std::fs::symlink_metadata(&local_path)
1781            .map_err(|e| LeanError::State(format!("stat {}: {e}", local_path.display())))?;
1782        if !lmeta.is_file() {
1783            self.state.append_conflict(&ConflictRecord {
1784                path: path.to_string(),
1785                foreign_etag: String::new(),
1786                preserved_key: None,
1787                kind: "upload-refused-not-regular: no longer a regular file after the scan (a symlink \
1788                       or special file replaced it); nothing published"
1789                    .into(),
1790                at_unix: now_unix(),
1791            })?;
1792            return Ok(UploadOutcome::Deferred);
1793        }
1794        let posix = Some(PosixStamps::from_metadata(&lmeta));
1795        let stamps = GenerationStamps {
1796            generation,
1797            epoch,
1798            flush_uuid: flush_uuid.to_string(),
1799            boundary_source: None,
1800            posix,
1801        };
1802        let condition = match base {
1803            Some(b) => PutCondition::IfMatch(b.etag.clone()),
1804            None => PutCondition::IfNoneMatchAny,
1805        };
1806        let size = lmeta.len();
1807        if size > self.cfg.whole_put_max {
1808            // Streaming multipart compose: put_whole is never fed past
1809            // whole_put_max (unbounded memory + S3's 5 GiB wall).
1810            return self
1811                .upload_compose(
1812                    path, &key, &local_path, size, scanned, condition, stamps, generation, epoch,
1813                    prior_uuids,
1814                )
1815                .await;
1816        }
1817        // The upload byte gate — the store's own, shared with the compose
1818        // arm above, which charges its parts inside the store. This body
1819        // is charged from before it is read (the read is the allocation)
1820        // until its PUT, retried or not, has returned: the permit lives
1821        // to the end of this function. `None` = a store with no gate.
1822        let _upload_permit = match self.store.upload_gate() {
1823            Some(g) => Some(g.acquire(size).await),
1824            None => None,
1825        };
1826        // `Bytes::from(Vec<u8>)` TAKES OWNERSHIP without copying, so the
1827        // old `Bytes::from(body.clone())` was a full memcpy of every
1828        // published file body — bought solely to leave `body` intact for
1829        // the 412 retry below, which is reached almost never. Build the
1830        // `Bytes` once; the retry gets a refcount clone.
1831        let body = Bytes::from(read_nofollow(&local_path)?);
1832        let uploaded_len = body.len() as u64;
1833        let crc = crc64_nvme(&body);
1834        match self
1835            .store
1836            .put_whole(&key, body.clone(), &condition, &stamps, crc)
1837            .await
1838        {
1839            Ok(meta) => Ok(UploadOutcome::published(
1840                path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1841            )),
1842            // S3 answers If-Match on a key that no longer exists with 404
1843            // NoSuchKey, not 412 (Ozone answers 412, and so did the
1844            // double): the base is gone — a peer's GC collected it, or a
1845            // bucket-level delete. The same rule as the 412 arm's vanished
1846            // HEAD below: a vanished base is a create. Without this arm the
1847            // rule never ran on S3, and a writer whose edited path a peer
1848            // deleted failed EVERY barrier (drill host leg H2, 2026-09-13).
1849            Err(StoreError::NotFound(_)) if matches!(condition, PutCondition::IfMatch(_)) => {
1850                let meta = self
1851                    .store
1852                    .put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
1853                    .await?;
1854                Ok(UploadOutcome::published(
1855                    path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1856                ))
1857            }
1858            Err(StoreError::PreconditionFailed(_)) => {
1859                // The 412 policy: my own crashed/torn PUT ⇒ adopt; a
1860                // foreign version ⇒ the consume-dirty rule, at upload time
1861                // (preserve it, then supersede it knowingly). NEVER the
1862                // inherited LOCAL-WINS overwrite, and never a park with
1863                // no way out.
1864                let head = match self.store.head(&key).await {
1865                    Ok(h) => h,
1866                    Err(StoreError::NotFound(_)) => {
1867                        // Review 2026-09-12, inbox-5: the base object is
1868                        // gone (a bucket-level delete, or our own GC after
1869                        // a crash between the CAS and the baseline
1870                        // rewrite). Failing the barrier here failed EVERY
1871                        // barrier, forever. A vanished base is a create.
1872                        let meta = self
1873                            .store
1874                            .put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
1875                            .await?;
1876                        return Ok(UploadOutcome::published(
1877                            path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1878                        ));
1879                    }
1880                    Err(e) => return Err(e.into()),
1881                };
1882                let head_stamps = GenerationStamps::from_meta(&head.meta);
1883                let own = head_stamps
1884                    .as_ref()
1885                    .map(|s| s.flush_uuid == flush_uuid || prior_uuids.contains(&s.flush_uuid))
1886                    .unwrap_or(false);
1887                if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
1888                    // Bytes already there (a torn response, ours or a
1889                    // foreign write of the same content): cite it — re-read
1890                    // under the fence first.
1891                    let g = head_stamps.map(|s| s.generation).unwrap_or(generation);
1892                    return Ok(UploadOutcome::adopted(
1893                        path, key, head.etag, crc, head.size, scanned, g, epoch,
1894                    ));
1895                }
1896                if !own && self.preserve_foreign_412(path, &head).await.is_none() {
1897                    return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1898                }
1899                // Our earlier PUT (older content), or a foreign version
1900                // now preserved: supersede it knowingly, If-Match on what
1901                // we saw. A second 412 means the path is being written
1902                // under us right now; that one parks.
1903                let meta = match self
1904                    .store
1905                    .put_whole(&key, body, &PutCondition::IfMatch(head.etag.clone()), &stamps, crc)
1906                    .await
1907                {
1908                    Ok(m) => m,
1909                    // 412 (a writer landed between) or 404 (a GC removed
1910                    // it between the HEAD and this PUT): written under us
1911                    // right now. Park; the next barrier retries.
1912                    Err(StoreError::PreconditionFailed(_)) | Err(StoreError::NotFound(_)) => {
1913                        return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1914                    }
1915                    Err(e) => return Err(e.into()),
1916                };
1917                Ok(UploadOutcome::published(
1918                    path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1919                ))
1920            }
1921            Err(e) => Err(e.into()),
1922        }
1923    }
1924
1925    /// A 412 against a version this syncer did not write, met at upload
1926    /// time. The contract's rule for a foreign write to a path the agent
1927    /// modified is the consume-dirty rule — the agent's version wins and
1928    /// the foreign bytes are preserved — and a "park" was that case with
1929    /// no resolution: nothing ever un-parked, and every ack said `ok`
1930    /// (review 2026-09-12, inbox-1). Preserve, record, and let the caller
1931    /// supersede. `None` = the preserve failed; the caller parks, and the
1932    /// boundary is `partial`.
1933    async fn preserve_foreign_412(&self, path: &str, head: &flint_store::ObjectMeta) -> Option<()> {
1934        let preserved = match self.preserve_conflict_copy(path, &head.etag).await {
1935            Ok(k) => k,
1936            Err(e) => {
1937                eprintln!("flint-sync: {path}: foreign version {} could not be preserved ({e}); parking", head.etag);
1938                return None;
1939            }
1940        };
1941        self.state
1942            .append_conflict(&ConflictRecord {
1943                path: path.to_string(),
1944                foreign_etag: head.etag.clone(),
1945                preserved_key: Some(preserved),
1946                kind: "upload-412-preserved".into(),
1947                at_unix: now_unix(),
1948            })
1949            .ok()?;
1950        Some(())
1951    }
1952}
1953
1954/// Read a workspace file for upload without following a symlink at the
1955/// final component, and refuse anything that is not a regular file once
1956/// open (review 2026-09-12, atomicity-6).
1957fn read_nofollow(p: &Path) -> std::io::Result<Vec<u8>> {
1958    use std::io::Read;
1959    use std::os::unix::fs::OpenOptionsExt;
1960    let mut f = std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NOFOLLOW).open(p)?;
1961    if !f.metadata()?.is_file() {
1962        return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "not a regular file"));
1963    }
1964    let mut v = Vec::new();
1965    f.read_to_end(&mut v)?;
1966    Ok(v)
1967}
1968
1969enum UploadOutcome {
1970    /// `adopted`: the etag was OBSERVED at the key (bytes already there,
1971    /// no PUT of ours produced it), so the commit section must re-read it
1972    /// before citing — see `verify_observed_citations` in the barrier.
1973    Published { entry: LeanEntry, baseline_entry: BaselineEntry, adopted: bool },
1974    Parked { foreign_etag: String },
1975    /// The source drifted mid-transfer (checksum refused server-side)
1976    /// or the assembly was swept: publish nothing, advance nothing —
1977    /// the next scan re-queues the path.
1978    Deferred,
1979}
1980
1981impl UploadOutcome {
1982    #[allow(clippy::too_many_arguments)]
1983    fn published(
1984        path: &str,
1985        key: String,
1986        etag: String,
1987        crc: u64,
1988        uploaded_len: u64,
1989        scanned: &scan::ScanEntry,
1990        generation: u64,
1991        epoch: u64,
1992    ) -> UploadOutcome {
1993        let _ = path;
1994        UploadOutcome::Published {
1995            adopted: false,
1996            entry: LeanEntry {
1997                key,
1998                etag: etag.clone(),
1999                crc64_b64: crc64_to_b64(crc),
2000                // The length the object HAS (review 2026-09-12,
2001                // atomicity-1): the scanned size was cited while the
2002                // body was read fresh, so a file that grew between the
2003                // two was cited short and every fresh checkout of it
2004                // failed its CRC fold — the successor never started.
2005                size: uploaded_len,
2006                mode: scanned.mode,
2007                mtime_unix: scanned.mtime_unix,
2008                generation,
2009                epoch,
2010            },
2011            baseline_entry: BaselineEntry {
2012                etag,
2013                generation,
2014                // The PRE-read stat: if the agent wrote during our read,
2015                // the next scan sees the drift and re-queues (the
2016                // re-stat/re-queue valve).
2017                size: scanned.size,
2018                mtime_unix: scanned.mtime_unix,
2019                mtime_nanos: Some(scanned.mtime_nanos),
2020                crc64_b64: Some(crc64_to_b64(crc)),
2021            },
2022        }
2023    }
2024}
2025
2026impl UploadOutcome {
2027    /// `published`, for bytes this barrier found already at the key and
2028    /// cites without a PUT of its own.
2029    #[allow(clippy::too_many_arguments)]
2030    fn adopted(
2031        path: &str,
2032        key: String,
2033        etag: String,
2034        crc: u64,
2035        uploaded_len: u64,
2036        scanned: &scan::ScanEntry,
2037        generation: u64,
2038        epoch: u64,
2039    ) -> UploadOutcome {
2040        match UploadOutcome::published(path, key, etag, crc, uploaded_len, scanned, generation, epoch) {
2041            UploadOutcome::Published { entry, baseline_entry, .. } => {
2042                UploadOutcome::Published { entry, baseline_entry, adopted: true }
2043            }
2044            other => other,
2045        }
2046    }
2047}
2048
2049fn local_dirty(local: &Path, base: Option<&BaselineEntry>) -> bool {
2050    match (std::fs::metadata(local), base) {
2051        (Err(_), None) => false,                   // both absent: clean
2052        (Err(_), Some(_)) => true,                 // locally deleted vs baseline
2053        (Ok(_), None) => true,                     // local exists, never published
2054        (Ok(m), Some(b)) => {
2055            scan::stat_changed(b.size, b.mtime_unix, b.mtime_nanos, m.len(), mtime_of(&m), mtime_nanos_of(&m))
2056        }
2057    }
2058}
2059
2060/// Full-object CRC-64/NVME of a local file, streamed.
2061///
2062/// Reached only by the 412 recovery path: the ordinary publish gets its
2063/// checksum from the store, which computes one while uploading.
2064/// The CRC a citation repair cites: the bytes this syncer integrated,
2065/// as the baseline recorded them when it wrote or uploaded them. The
2066/// HEAD's own checksum is a cross-check when the backend offers one,
2067/// never the source — Ozone offers none, a HITL uploader may have sent
2068/// none, and the manifest must carry a CRC either way, because every
2069/// reader now verifies a fresh fetch against it.
2070///
2071/// A baseline entry with no CRC is refused loudly rather than cited
2072/// bare: the only entry built without one is the consume-dirty sentinel,
2073/// which the candidate filters exclude, so reaching this is a defect.
2074/// A HEAD that attests a DIFFERENT value is refused the same way — the
2075/// workspace holds bytes the object does not, and citing either would
2076/// make one reader or another refuse the path forever.
2077pub(crate) fn repair_crc(
2078    path: &str,
2079    be: &BaselineEntry,
2080    head: &flint_store::ObjectMeta,
2081) -> LeanResult<String> {
2082    let Some(ours) = be.crc64_b64.clone() else {
2083        return Err(LeanError::State(format!(
2084            "citation repair of {path}: the baseline records no CRC for the bytes it \
2085             integrated at etag {} — refusing to cite bytes nothing hashed",
2086            be.etag
2087        )));
2088    };
2089    if let Some(theirs) = head.crc64_b64.as_deref() {
2090        if theirs != ours {
2091            return Err(LeanError::State(format!(
2092                "citation repair of {path}: the store attests CRC-64 {theirs} for etag {} but \
2093                 the bytes this workspace integrated under that etag hash to {ours} — the \
2094                 workspace holds bytes the object does not; refusing to cite either",
2095                be.etag
2096            )));
2097        }
2098    }
2099    Ok(ours)
2100}
2101
2102fn file_crc(local_path: &Path) -> LeanResult<u64> {
2103    use std::io::Read;
2104    let mut crc = flint_store::Crc64Nvme::new();
2105    let mut f = std::fs::File::open(local_path)?;
2106    let mut buf = vec![0u8; 4 << 20];
2107    loop {
2108        let n = f.read(&mut buf)?;
2109        if n == 0 {
2110            break;
2111        }
2112        crc.update(&buf[..n]);
2113    }
2114    Ok(crc.finalize())
2115}
2116
2117pub(super) fn mtime_nanos_of(m: &std::fs::Metadata) -> u32 {
2118    m.modified()
2119        .ok()
2120        .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2121        .map(|d| d.subsec_nanos())
2122        .unwrap_or(0)
2123}
2124
2125pub(super) fn mtime_of(m: &std::fs::Metadata) -> i64 {
2126    m.modified()
2127        .ok()
2128        .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2129        .map(|d| d.as_secs() as i64)
2130        .unwrap_or(0)
2131}
2132
2133/// Containment-safe workspace write (boundary-verbs plan §2.2 security
2134/// gate). `rel` is a workspace-relative path; the target may not escape
2135/// the root, traverse a symlink component, or land in the reserved
2136/// control namespace.
2137///
2138/// The hazard this closes is pre-existing and reachable from three
2139/// callers (checkout, inbox consume, sync): `write_file_atomic` did
2140/// `create_dir_all(parent)` + write with no `O_NOFOLLOW` and no
2141/// root-containment check, while the scanner SKIPS symlinks — so a
2142/// planted symlink is invisible to the syncer. An unprivileged app
2143/// that plants `inputs -> /root/.aws`, lands an object at
2144/// `inputs/<path>` and drops a scoped sync turns the credential-holding
2145/// syncer into an on-demand arbitrary-file-write primitive outside the
2146/// workspace.
2147pub(super) fn write_file_atomic_in(
2148    root: &Path,
2149    rel: &str,
2150    bytes: &[u8],
2151    mode: Option<u32>,
2152) -> LeanResult<()> {
2153    let target = contained_path(root, rel)?;
2154    write_file_atomic(&target, bytes, mode).map(|_| ())
2155}
2156
2157/// Validate `rel` under `root` WITHOUT creating anything — for callers
2158/// that must decide whether a path is writable before doing any work
2159/// (inbox consume refuses a refused path outright rather than routing it
2160/// through conflict-preserve, which would cost a GET+PUT and leave the
2161/// path in the baseline).
2162pub(super) fn check_contained(root: &Path, rel: &str) -> LeanResult<()> {
2163    resolve_contained(root, rel, false).map(|_| ())
2164}
2165
2166/// Resolve `rel` under `root`, refusing escapes, creating missing parent
2167/// directories. Walks the parent chain component by component: a
2168/// component that EXISTS as a symlink is a refusal (never followed).
2169pub(super) fn contained_path(root: &Path, rel: &str) -> LeanResult<std::path::PathBuf> {
2170    resolve_contained(root, rel, true).map(|(p, _)| p)
2171}
2172
2173/// As `contained_path`, and also what the walk already learned about
2174/// the FINAL component: its `lstat` if it exists, `None` if it does
2175/// not. The walk stats every component to refuse a symlink; a caller
2176/// that then asked `exists()` and `metadata()` paid two more stats for
2177/// an answer it was already holding (checkout's resume check did, on
2178/// every one of 20,000 files).
2179pub(super) fn contained_path_stat(
2180    root: &Path,
2181    rel: &str,
2182) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2183    resolve_contained(root, rel, true)
2184}
2185
2186fn resolve_contained(
2187    root: &Path,
2188    rel: &str,
2189    create_dirs: bool,
2190) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2191    use std::path::Component;
2192    let refuse = |why: &str| {
2193        Err(LeanError::State(format!(
2194            "refusing write to {rel:?}: {why} (containment)"
2195        )))
2196    };
2197    if rel.is_empty() {
2198        return refuse("empty path");
2199    }
2200    if super::scan::is_control_path(rel) {
2201        // The reserved namespace is the syncer's own; a citation or
2202        // inbox entry naming it is surfaced, never materialized (D0.3).
2203        return refuse("reserved control namespace");
2204    }
2205    let relp = Path::new(rel);
2206    let mut cur = root.to_path_buf();
2207    let mut comps: Vec<&std::ffi::OsStr> = vec![];
2208    for c in relp.components() {
2209        match c {
2210            Component::Normal(n) => comps.push(n),
2211            Component::CurDir => {}
2212            Component::ParentDir => return refuse("`..` component"),
2213            Component::RootDir | Component::Prefix(_) => return refuse("absolute path"),
2214        }
2215    }
2216    if comps.is_empty() {
2217        return refuse("no path components");
2218    }
2219    let last = comps.len() - 1;
2220    let mut final_meta = None;
2221    for (i, c) in comps.iter().enumerate() {
2222        cur.push(c);
2223        match std::fs::symlink_metadata(&cur) {
2224            Ok(m) if m.file_type().is_symlink() => {
2225                return refuse("path traverses a symlink");
2226            }
2227            Ok(m) if i < last && !m.is_dir() => {
2228                return refuse("parent component is not a directory");
2229            }
2230            Ok(m) => {
2231                if i == last {
2232                    final_meta = Some(m);
2233                }
2234            }
2235            Err(_) if i < last => {
2236                if create_dirs {
2237                    if let Err(e) = std::fs::create_dir(&cur) {
2238                        // A SIBLING may have created it between the stat
2239                        // above and this mkdir: fetches run on their own
2240                        // tasks, so the first files of one directory
2241                        // race here, and the loser's EEXIST is not a
2242                        // containment refusal. It is fine only if what
2243                        // exists now is a plain directory — a symlink
2244                        // that appeared in the window is refused exactly
2245                        // as one the stat found.
2246                        match std::fs::symlink_metadata(&cur) {
2247                            Ok(m) if m.file_type().is_symlink() => {
2248                                return refuse("path traverses a symlink");
2249                            }
2250                            Ok(m) if m.is_dir() => {}
2251                            _ => {
2252                                return Err(LeanError::State(format!(
2253                                    "mkdir {}: {e}",
2254                                    cur.display()
2255                                )))
2256                            }
2257                        }
2258                    }
2259                }
2260            }
2261            Err(_) => {}
2262        }
2263    }
2264    Ok((cur, final_meta))
2265}
2266
2267/// Write `bytes` to `path` through an exclusive temp sibling and a
2268/// rename. The parent already exists — containment created it — and
2269/// one that vanished since is recreated on the retry path in `safefs`,
2270/// so the common path pays no `mkdir` and no `stat` for it. Returns
2271/// the written file's metadata.
2272pub(super) fn write_file_atomic(
2273    path: &Path,
2274    bytes: &[u8],
2275    mode: Option<u32>,
2276) -> LeanResult<std::fs::Metadata> {
2277    super::safefs::check_parent(path)?;
2278    // NOT with_extension(): that REPLACES the final extension, so
2279    // "a.txt" and "a.md" would collide on one tmp name.
2280    let tmp = path.with_file_name(format!(
2281        "{}.flint-sync-tmp",
2282        path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default()
2283    ));
2284    // The temp sibling is computed AFTER containment ran, so the walk
2285    // never saw it: it needs its own refusal, not a plain write.
2286    // FAST (no per-file fsync): materialisations are made durable by
2287    // `sync_tree` before the marker or baseline that vouches for them.
2288    super::safefs::write_via_tmp_fast(path, &tmp, bytes, mode)
2289}
2290
2291/// The queue form of a merge's foreign upserts and deletions.
2292fn foreign_changes(entries: &[(String, LeanEntry)], gone: &[String]) -> Vec<ForeignChange> {
2293    entries
2294        .iter()
2295        .map(|(path, e)| ForeignChange {
2296            path: path.clone(),
2297            etag: Some(e.etag.clone()),
2298            crc64_b64: Some(e.crc64_b64.clone()),
2299        })
2300        .chain(gone.iter().map(|path| ForeignChange { path: path.clone(), etag: None, crc64_b64: None }))
2301        .collect()
2302}
2303
2304/// One three-way merge of a barrier's changes onto the bucket's manifest.
2305struct MergeOnto {
2306    theirs: manifest::LeanManifest,
2307    /// The CAS handle over `theirs`; `None` when the bucket has no manifest.
2308    expected: Option<manifest::ManifestHandle>,
2309    merged: manifest::LeanManifest,
2310    /// Other writers' changes since the merge base that the tree lacks.
2311    foreign: Vec<(String, LeanEntry)>,
2312    /// Other writers' deletions since the merge base.
2313    gone: Vec<String>,
2314}
2315
2316impl MergeOnto {
2317    /// The merge adds nothing to the document: installing it would be an
2318    /// empty generation.
2319    fn adds_nothing(&self) -> bool {
2320        self.merged.entries == self.theirs.entries && self.merged.sole_writer == self.theirs.sole_writer
2321    }
2322}
2323
2324impl Syncer {
2325    #[allow(clippy::too_many_arguments)]
2326    fn merge_onto(
2327        &self,
2328        current: Option<&manifest::LoadedManifest>,
2329        prev_installed: Option<&str>,
2330        inst_base: &BTreeMap<String, String>,
2331        upserts: &BTreeMap<String, LeanEntry>,
2332        deletes: &BTreeSet<String>,
2333        parked: &BTreeSet<String>,
2334        flush_uuid: &str,
2335    ) -> MergeOnto {
2336        let (theirs, expected) = match current {
2337            // The handle carries the LAYOUT as well as the tag: a
2338            // workspace still on the legacy single object CASes a
2339            // pointer that must not exist yet, not one it read.
2340            Some(l) => (l.manifest.clone(), Some(l.handle())),
2341            None => (Default::default(), None),
2342        };
2343        // If the bucket is still at the document THIS workspace
2344        // installed, that document IS the merge base — whatever the
2345        // persisted one says. Step 7 rewrites the merge base after
2346        // the CAS, so a restart in between leaves it behind a
2347        // document we wrote, and every entry in it would read as
2348        // somebody else's change. See `IntentJournal::installed_etag`.
2349        let own_base: BTreeMap<String, String>;
2350        let base: &BTreeMap<String, String> =
2351            if prev_installed.is_some() && prev_installed == expected.as_ref().map(|h| h.etag.as_str()) {
2352                own_base = theirs.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
2353                &own_base
2354            } else {
2355                inst_base
2356            };
2357        let (mut merged, foreign) = manifest::merge(base, &theirs, upserts, deletes, parked);
2358        // Deletions another writer made since this workspace's
2359        // merge base: in the base, gone from theirs, and not this
2360        // barrier's own upsert, delete or park. `merge` has no use
2361        // for them — theirs already lacks the path — but the TREE
2362        // does: they reach it through the local queue, as the
2363        // foreign upserts do.
2364        let gone: Vec<String> = base
2365            .keys()
2366            .filter(|p| {
2367                !theirs.entries.contains_key(*p)
2368                    && !upserts.contains_key(*p)
2369                    && !deletes.contains(*p)
2370                    && !parked.contains(*p)
2371            })
2372            .cloned()
2373            .collect();
2374        // `merge` clears it; the installing pass owns it. A mirror
2375        // is a property of how this workspace is DEPLOYED, so it
2376        // comes from config on every publish rather than being
2377        // inherited from whatever wrote last.
2378        merged.sole_writer = self.cfg.sole_writer;
2379        self.trace("merge", serde_json::json!({"flush": flush_uuid, "theirs_seq": theirs.seq, "upserts": upserts.len(),
2380            "deletes": deletes.len(), "foreign": foreign.len(), "gone": gone.len(),
2381            "adds_nothing": merged.entries == theirs.entries}));
2382        MergeOnto { theirs, expected, merged, foreign, gone }
2383    }
2384}