Skip to main content

flint_lean/
barrier.rs

1//! The publish barrier (plan §2.1, seven steps) — the machine
2//! `lean/formal/LeanSubtree.tla` checks. Order is load-bearing:
3//! consume → scan → intent/window → uploads → manifest CAS (merge) →
4//! GC deletes LAST → baseline rewrite.
5
6use std::collections::{BTreeMap, BTreeSet};
7use std::path::Path;
8
9use bytes::Bytes;
10
11use flint_store::{
12    EpochState,
13    crc64_nvme, crc64_to_b64, GenerationStamps, PosixStamps, PutCondition, StoreError,
14};
15
16use super::inbox::{self, InboxDoc, InboxEntry, Refusal, Removal};
17use super::manifest::{self, LeanEntry};
18use super::scan;
19use super::state::{BaselineEntry, ConflictRecord, ForeignChange, IntentJournal};
20use super::{now_unix, LeanError, LeanResult, Syncer};
21
22/// The last gateway request this workspace acted on, per verb. A
23/// watermark rather than a consumed queue — see `note_verb_requests`.
24#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
25pub struct VerbWatermark {
26    #[serde(default)]
27    pub boundary_unix: u64,
28    #[serde(default)]
29    pub sync_unix: u64,
30}
31
32#[derive(Debug, Default)]
33pub struct BarrierReport {
34    pub seq: Option<u64>,
35    pub uploaded: Vec<String>,
36    pub deleted: Vec<String>,
37    /// Paths a DECLARED removal (delete/rename design) cited out this
38    /// barrier — a subset of `deleted` once the GC has run.
39    pub removed: Vec<String>,
40    /// Declared removals this barrier REFUSED (locally dirty, not a
41    /// file, uncontainable); the cell carries the reason.
42    pub removals_refused: usize,
43    pub parked: Vec<String>,
44    pub consumed: usize,
45    pub foreign_queued: usize,
46    /// Paths whose transfer drifted or was swept mid-compose: nothing
47    /// published, nothing advanced; the next scan re-queues them.
48    pub deferred: Vec<String>,
49    /// Local deletes another writer's change outranked (delete/modify
50    /// resolves foreign-wins): the installed seq still cites them, and the
51    /// next consume reconciles each against the local delete.
52    pub outranked: Vec<String>,
53    pub no_change: bool,
54    /// Bytes this barrier actually published — the input to the
55    /// sentinel work meter (boundary-verbs plan D3.1). Metering work
56    /// rather than calls is what keeps a sentinel storm from
57    /// un-coalescing a hot large file's republish: a counted budget
58    /// charges a 2 GiB checkpoint the same one unit as a 4 KiB file.
59    pub published_bytes: u64,
60    /// The manifest ETag this barrier installed (the ack's CAS token).
61    pub manifest_etag: Option<String>,
62    /// What the manifest HEAD/install told us the bucket is at, for the
63    /// news ticker (D5) — free, off requests the barrier already made.
64    pub observed_seq: Option<u64>,
65    pub observed_etag: Option<String>,
66    /// Paths this barrier saw absent for the FIRST time: their deletes
67    /// are withheld to the next scan (the rename-vs-walk guard). On a
68    /// declared boundary a non-empty set means the confirmation lstat
69    /// found them back on disk — the race the guard exists for.
70    pub first_absence: Vec<String>,
71    /// First-absence paths a DECLARED boundary confirmed gone by direct
72    /// lstat and published as ordinary deletes (`confirm_absences`).
73    pub absences_confirmed: usize,
74    /// This barrier finished a rescope a crash had left half-applied
75    /// before it did anything else. Reported so a run that looks like
76    /// an ordinary barrier can be told from one that had to converge a
77    /// workspace first.
78    pub rescope_replayed: bool,
79}
80
81/// What a pass over the cell's DECLARED removals did (`apply_removals`).
82#[derive(Debug, Default)]
83pub struct RemovalPass {
84    /// Performed: the local file is gone (or already was) and the path
85    /// is in `declared`. Dropped from the cell with the window clear —
86    /// AFTER the manifest CAS — so a listing keeps hiding the path
87    /// until the manifest stops citing it.
88    pub applied: Vec<Removal>,
89    /// Refused for good, `refused` filled in: the cell is told at once
90    /// and the removal is never retried.
91    pub refused: Vec<Removal>,
92    /// Left in the cell for the next barrier: a rename whose
93    /// destination is not integrated yet, or an unlink that failed for
94    /// a transient reason.
95    pub deferred: usize,
96    /// The paths this barrier cites OUT: `applied`, plus any a crashed
97    /// earlier barrier had unlinked and journalled.
98    pub declared: BTreeSet<String>,
99}
100
101/// Upload waves per chunk. The chunk is `fanout * this`, so a wave
102/// still saturates fan-out and the sync point between chunks is rare.
103const UPLOAD_CHUNK_WAVES: usize = 16;
104
105/// How stale the lease may get inside a barrier before it is renewed:
106/// comfortably below the 6-poll (~60 s) takeover window, so a holder
107/// busy in a long commit section never reads as quiet to a waiter.
108const RENEW_WITHIN_SECS: u64 = 20;
109
110impl Syncer {
111    /// Verify the cell still names us at OUR epoch; anything else is a
112    /// fence. Read-verify before the manifest CAS (the per-request
113    /// validation the gateway will also enforce).
114    async fn verify_not_deposed(&self) -> LeanResult<()> {
115        let lease = self
116            .lease
117            .as_ref()
118            .ok_or_else(|| LeanError::State("no lease".into()))?;
119        match self.store.epoch_read(&self.cfg.epoch_key()).await? {
120            Some(state) if state.epoch == lease.epoch && state.holder_id == lease.holder_id => {
121                Ok(())
122            }
123            Some(state) => Err(LeanError::Fenced(format!(
124                "cell at epoch {} holder {} (we are epoch {})",
125                state.epoch, state.holder_id, lease.epoch
126            ))),
127            None => Err(LeanError::Fenced("epoch cell vanished".into())),
128        }
129    }
130
131    /// Step 1: consume the inbox. A barrier NEVER runs against an
132    /// unconsumed inbox — this is what makes HITL uploads structurally
133    /// un-amputatable. Returns the entries integrated (dropped from the
134    /// cell at the window-open CAS).
135    pub async fn consume_inbox(&mut self) -> LeanResult<Vec<InboxEntry>> {
136        let loaded = inbox::load(self.store.as_ref(), &self.cfg).await?;
137        self.consume_inbox_doc(&loaded.doc).await
138    }
139
140    /// `consume_inbox` over a cell the caller has already read — the
141    /// barrier reads it once and hands the same document to
142    /// `apply_removals`, so the removal pass adds no request.
143    pub async fn consume_inbox_doc(&mut self, doc: &InboxDoc) -> LeanResult<Vec<InboxEntry>> {
144        Ok(self.consume_counted(doc).await?.0)
145    }
146
147    /// `consume_inbox_doc`, plus how many changes from this writer's LOCAL
148    /// queue it settled. Those leave no entry in the cell to drop, so they
149    /// are not in the returned list — but they are consumptions, and the
150    /// report (and the ack) count them.
151    async fn consume_counted(&mut self, doc: &InboxDoc) -> LeanResult<(Vec<InboxEntry>, usize)> {
152        // §2.5's layered doors ride the inbox GET this function already
153        // pays for: promptness is one tick, and the added request count
154        // is ZERO. A failure here must never fail the consume — the
155        // request is idempotent state and the next tick re-reads it.
156        if let Err(e) = self.note_verb_requests(doc) {
157            eprintln!("flint-sync: verb request not consumed (retrying next tick): {e}");
158        }
159        // Indices into `entries`: the writer-local queue's upserts first,
160        // then the shared inbox's. Both run the same rules below.
161        let mut consumed: Vec<usize> = vec![];
162        let mut baseline = self.state.load_baseline()?;
163        // An install whose step 7 a restart cut off left the other writers'
164        // changes it carried in the intent journal (`installed_foreign`).
165        let requeued = self.state.requeue_installed_foreign()?;
166        if requeued > 0 {
167            self.trace("queue", serde_json::json!({"requeued_from_intent": requeued}));
168        }
169        // The writer-local queue: changes other writers made that this
170        // workspace's own merges carried into the manifest but not yet
171        // into the tree (`state::ForeignChange` says why it is not the
172        // shared inbox any more).
173        let queue = self.state.load_foreign_queue()?;
174        let queued: Vec<InboxEntry> = queue
175            .iter()
176            .filter_map(|c| {
177                c.etag.as_ref().map(|etag| InboxEntry {
178                    path: c.path.clone(),
179                    etag: etag.clone(),
180                    author: "merge-preserved".into(),
181                    added_unix: 0,
182                    crc64_b64: c.crc64_b64.clone(),
183                })
184            })
185            .collect();
186        let n_queued = queued.len();
187        let entries: Vec<InboxEntry> = queued.into_iter().chain(doc.entries.iter().cloned()).collect();
188        for (idx, entry) in entries.iter().enumerate() {
189            let key = self.cfg.file_key(&entry.path);
190            // Containment BEFORE anything else: a path we could never
191            // safely materialize must be surfaced and dropped, not
192            // routed through the locally-dirty branch — which would
193            // "preserve" it with a GET+PUT and leave a baseline entry
194            // for a path the scanner can never see (the planted-symlink
195            // shape: `inputs -> /root/.aws` reads as locally-present,
196            // therefore dirty, therefore a conflict-preserve of someone
197            // else's file).
198            if let Err(e) = check_contained(&self.cfg.root, &entry.path) {
199                self.state.append_conflict(&ConflictRecord {
200                    path: entry.path.clone(),
201                    foreign_etag: entry.etag.clone(),
202                    preserved_key: None,
203                    kind: format!("consume-refused-containment: {e}"),
204                    at_unix: now_unix(),
205                })?;
206                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused"}));
207                consumed.push(idx);
208                continue;
209            }
210            // Already integrated (a crashed earlier consume): idempotent.
211            if baseline.entries.get(&entry.path).map(|b| b.etag == entry.etag).unwrap_or(false) {
212                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "already"}));
213                consumed.push(idx);
214                continue;
215            }
216            let head = match self.store.head(&key).await {
217                Ok(m) => m,
218                Err(StoreError::NotFound(_)) => {
219                    // The object vanished under a tracked entry: surface
220                    // it — the bytes are NOT recoverable from here.
221                    self.state.append_conflict(&ConflictRecord {
222                        path: entry.path.clone(),
223                        foreign_etag: entry.etag.clone(),
224                        preserved_key: None,
225                        kind: "consume-object-missing".into(),
226                        at_unix: now_unix(),
227                    })?;
228                    self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "missing"}));
229                    consumed.push(idx);
230                    continue;
231                }
232                Err(e) => return Err(e.into()),
233            };
234            if head.etag != entry.etag {
235                // Superseded by a newer write (its own inbox entry
236                // follows, or it is the syncer's): drop.
237                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "superseded"}));
238                consumed.push(idx);
239                continue;
240            }
241            let local_path = self.cfg.root.join(&entry.path);
242            let mut dirty = local_dirty(&local_path, baseline.entries.get(&entry.path));
243            // Review 2026-09-12, atomicity-3 / inbox-2: the fetch is a
244            // network round trip and the agent may write the path inside
245            // it. The stat that licenses the adopt is repeated AFTER the
246            // fetch, before anything touches the file: a write in that
247            // window makes this the dirty case — the foreign bytes are
248            // preserved and a record names them, the agent's stay.
249            let mut fetched = None;
250            if !dirty {
251                let got = self.store.get_whole(&key, Some(&entry.etag)).await.map_err(|e| match e {
252                    StoreError::PreconditionFailed(m) => {
253                        StoreError::Other(format!("consume raced a newer write: {m}"))
254                    }
255                    other => other,
256                })?;
257                if local_dirty(&local_path, baseline.entries.get(&entry.path)) {
258                    dirty = true;
259                } else {
260                    fetched = Some(got);
261                }
262            }
263            if dirty {
264                // Locally-dirty wins; preserve the FOREIGN bytes first
265                // (a conflict record must keep both versions
266                // recoverable), then advance the recognized ETag so our
267                // eventual publish supersedes it KNOWINGLY.
268                let preserved = self.preserve_conflict_copy(&entry.path, &head.etag).await?;
269                self.state.append_conflict(&ConflictRecord {
270                    path: entry.path.clone(),
271                    foreign_etag: entry.etag.clone(),
272                    preserved_key: Some(preserved),
273                    kind: "consume-dirty".into(),
274                    at_unix: now_unix(),
275                })?;
276                let stamps = GenerationStamps::from_meta(&head.meta);
277                baseline.entries.insert(
278                    entry.path.clone(),
279                    BaselineEntry {
280                        etag: head.etag.clone(),
281                        generation: stamps.map(|s| s.generation).unwrap_or(0),
282                        // Deliberately NOT the local file's stat: the
283                        // path must still scan as dirty so the local
284                        // version publishes.
285                        size: u64::MAX,
286                        mtime_unix: 0,
287                        mtime_nanos: None,
288                        // The sentinel's bytes are the LOCAL edit; the
289                        // publish that supersedes hashes them itself.
290                        crc64_b64: None,
291                    },
292                );
293                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "dirty-preserved"}));
294            } else {
295                // Clean, re-checked after the fetch: adopt the foreign
296                // content into the tree.
297                let (meta, body) = fetched.expect("fetched while clean");
298                let mode = PosixStamps::from_meta(&meta.meta).map(|p| p.mode);
299                // VERIFIED before it is written, as checkout's fresh
300                // fetch is: against the writer's CRC when the inbox
301                // entry carries one (the gateway hashes what it sent),
302                // else the backend's attestation when it offers one.
303                // Ozone offers none, so the inbox's is what a HITL
304                // upload gets checked against there. What the baseline
305                // records — and the next citation repair cites — is
306                // OURS, over the bytes actually written.
307                let got = crc64_to_b64(crc64_nvme(&body));
308                let want = entry.crc64_b64.clone().or_else(|| meta.crc64_b64.clone());
309                if let Some(want) = want {
310                    if want != got {
311                        // NOT consumed: the entry stays in the cell and
312                        // the record repeats — a permanent, visible
313                        // contradiction is the right failure for
314                        // bytes nobody vouches for.
315                        self.state.append_conflict(&ConflictRecord {
316                            path: entry.path.clone(),
317                            foreign_etag: entry.etag.clone(),
318                            preserved_key: None,
319                            kind: format!(
320                                "consume-refused-checksum: etag {} is attested with CRC-64 \
321                                 {want} but the bytes fetched under it hash to {got}",
322                                entry.etag
323                            ),
324                            at_unix: now_unix(),
325                        })?;
326                        self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused-checksum"}));
327                        continue;
328                    }
329                }
330                // CONTAINMENT and I/O are split, because the answers are
331                // opposite. Containment was already decided above by
332                // `check_contained`, so a refusal reaching here is a
333                // path that only became unsafe once parents were
334                // created — still a permanent property of the path, so
335                // it is surfaced and DROPPED, exactly as above.
336                //
337                // Everything else — ENOSPC, EACCES, EIO, a read-only
338                // filesystem — is TRANSIENT, and dropping the entry for
339                // one is silent permanent loss: the foreign bytes stay
340                // in the bucket, the workspace never adopts them, and
341                // nothing ever re-offers the entry. It used to be
342                // recorded as `consume-refused-containment`, which sent
343                // whoever read it looking for a hostile path that was
344                // never there. A full disk is not a planted symlink.
345                let target = match contained_path(&self.cfg.root, &entry.path) {
346                    Ok(t) => t,
347                    Err(e) => {
348                        self.state.append_conflict(&ConflictRecord {
349                            path: entry.path.clone(),
350                            foreign_etag: entry.etag.clone(),
351                            preserved_key: None,
352                            kind: format!("consume-refused-containment: {e}"),
353                            at_unix: now_unix(),
354                        })?;
355                        self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused"}));
356                        consumed.push(idx);
357                        continue;
358                    }
359                };
360                if let Err(e) = write_file_atomic(&target, &body, mode) {
361                    // NOT consumed: the entry stays in the cell so the
362                    // next barrier retries it. A visible conflict record
363                    // that repeats is a far better failure than one
364                    // silent drop.
365                    self.state.append_conflict(&ConflictRecord {
366                        path: entry.path.clone(),
367                        foreign_etag: entry.etag.clone(),
368                        preserved_key: None,
369                        kind: format!("consume-write-failed (will retry): {e}"),
370                        at_unix: now_unix(),
371                    })?;
372                    self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "deferred"}));
373                    continue;
374                }
375                let st = std::fs::metadata(&local_path)?;
376                let stamps = GenerationStamps::from_meta(&meta.meta);
377                baseline.entries.insert(
378                    entry.path.clone(),
379                    BaselineEntry {
380                        etag: meta.etag.clone(),
381                        generation: stamps.map(|s| s.generation).unwrap_or(0),
382                        size: st.len(),
383                        mtime_unix: mtime_of(&st),
384                        mtime_nanos: Some(mtime_nanos_of(&st)),
385                        crc64_b64: Some(got),
386                    },
387                );
388                baseline.prev_scan.insert(entry.path.clone());
389                self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "adopted"}));
390            }
391            consumed.push(idx);
392        }
393        let mut settled: BTreeSet<String> =
394            consumed.iter().filter(|i| **i < n_queued).map(|i| entries[*i].path.clone()).collect();
395
396        // A queued DELETION: another writer removed the path and this
397        // workspace's merge cited that. A clean local copy goes; a dirty
398        // one is the agent's newer work and stays — it publishes, and a
399        // modify beats a delete at the merge. Never a declared removal of
400        // our own: the deletion is already in the manifest.
401        for change in queue.iter().filter(|c| c.etag.is_none()) {
402            if check_contained(&self.cfg.root, &change.path).is_err() {
403                // Never materializable here, so nothing to remove.
404                settled.insert(change.path.clone());
405                continue;
406            }
407            let local = self.cfg.root.join(&change.path);
408            // A key that holds an object has SUPERSEDED the deletion: a
409            // newer write — a UI write, another writer's re-create — landed
410            // after the merge that queued it, and the upserts above already
411            // drop an entry whose etag the key no longer holds. Applying it
412            // anyway removes the newer bytes from the tree, which this very
413            // consume may just have adopted — and the window clear then
414            // drops their inbox entry, leaving an acked write cited and
415            // tracked by nothing (found by the formal model with this queue
416            // modelled, 2026-09-15). Only a file still on disk needs the
417            // look: an absent one settles as it always did.
418            if std::fs::symlink_metadata(&local).is_ok() {
419                match self.store.head(&self.cfg.file_key(&change.path)).await {
420                    Ok(_) => {
421                        self.trace("tombstone", serde_json::json!({"path": change.path, "action": "superseded"}));
422                        settled.insert(change.path.clone());
423                        continue;
424                    }
425                    Err(StoreError::NotFound(_)) => {}
426                    Err(e) => return Err(e.into()),
427                }
428            }
429            let be = baseline.entries.get(&change.path).cloned();
430            let record = |kind: String| ConflictRecord {
431                path: change.path.clone(),
432                foreign_etag: String::new(),
433                preserved_key: None,
434                kind,
435                at_unix: now_unix(),
436            };
437            match std::fs::symlink_metadata(&local) {
438                Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
439                    self.trace("tombstone", serde_json::json!({"path": change.path, "action": "absent"}));
440                }
441                Err(e) => {
442                    // Unreadable is not absent: keep it queued.
443                    self.trace("tombstone", serde_json::json!({"path": change.path, "action": "deferred"}));
444                    self.state.append_conflict(&record(format!(
445                        "consume-foreign-delete-deferred: cannot stat the path: {e}"
446                    )))?;
447                    continue;
448                }
449                Ok(m) if !m.is_file() || be.is_none() || local_dirty(&local, be.as_ref()) => {
450                    self.state.append_conflict(&record(
451                        "consume-foreign-delete-vs-dirty: another writer deleted this path; the \
452                         local version has unpublished changes, so it stays and publishes"
453                            .into(),
454                    ))?;
455                    self.trace("tombstone", serde_json::json!({"path": change.path, "action": "kept-dirty"}));
456                    settled.insert(change.path.clone());
457                    continue;
458                }
459                Ok(_) => {
460                    if let Err(e) = std::fs::remove_file(&local) {
461                        self.state.append_conflict(&record(format!(
462                            "consume-foreign-delete-failed (will retry): {e}"
463                        )))?;
464                        continue;
465                    }
466                    // CONFIRM: only NotFound is absence.
467                    if !matches!(
468                        std::fs::symlink_metadata(&local),
469                        Err(ref e) if e.kind() == std::io::ErrorKind::NotFound
470                    ) {
471                        self.state.append_conflict(&record(
472                            "consume-foreign-delete-unconfirmed (will retry)".into(),
473                        ))?;
474                        continue;
475                    }
476                    self.trace("tombstone", serde_json::json!({"path": change.path, "action": "removed"}));
477                }
478            }
479            baseline.entries.remove(&change.path);
480            baseline.prev_scan.remove(&change.path);
481            settled.insert(change.path.clone());
482        }
483
484        // The files this baseline vouches for reached the disk before
485        // the baseline does (audit 2026-09-03, finding 9).
486        self.state.sync_tree()?;
487        self.state.save_baseline(&baseline)?;
488        // After the baseline: a crash between the two re-applies a
489        // settled change, and both arms are idempotent (an upsert the
490        // baseline holds is "already integrated"; a deletion with nothing
491        // on disk only drops a baseline entry that is already gone).
492        if !settled.is_empty() {
493            let remaining: Vec<_> =
494                queue.into_iter().filter(|c| !settled.contains(&c.path)).collect();
495            self.state.save_foreign_queue(&remaining)?;
496        }
497        // Only the SHARED inbox's entries leave the cell at the window
498        // clear; the local queue was settled above.
499        let n_settled = settled.len();
500        let shared = consumed.into_iter().filter(|i| *i >= n_queued).map(|i| entries[i].clone()).collect();
501        Ok((shared, n_settled))
502    }
503
504    /// Step 1b: perform the DECLARED removals (delete/rename design
505    /// §4-§6). Runs after `consume_inbox_doc`, so a rename's destination
506    /// is in the tree before its source leaves it — §5: create first,
507    /// removal second; a partial application leaves an EXTRA file,
508    /// never a missing one.
509    ///
510    /// Per removal, in order:
511    /// - the path must be containable, as a consume's must;
512    /// - a rename WAITS until its destination is integrated (in the
513    ///   baseline): a consume that deferred the destination defers the
514    ///   removal with it;
515    /// - a locally-DIRTY path — unpublished edits, or a file the agent
516    ///   created there — is REFUSED, nothing applied, the agent's work
517    ///   untouched. The same rule a consume applies to a HITL write
518    ///   over dirty bytes, and the reason a declared removal cannot be
519    ///   "just a delete". Refused for good: the cell is told, and a
520    ///   retry that waited for the agent to publish would delete the
521    ///   very edit the refusal protected;
522    /// - a clean file is unlinked, and the unlink is CONFIRMED by lstat
523    ///   before the path is declared — an unlink that did not take
524    ///   publishes no deletion (`confirm_absences`' rule, kept);
525    /// - a path already absent locally and known to the baseline or the
526    ///   merge base is declared as it stands: the agent got there
527    ///   first, or this tree never held it (a scoped workspace).
528    ///
529    /// A declared path skips the two-scan guard (§4): that guard
530    /// protects against absence INFERRED by a walk, and a declaration
531    /// is not an inference. That is what lets a rename's two halves
532    /// ride one manifest generation.
533    pub async fn apply_removals(&mut self, doc: &InboxDoc) -> LeanResult<RemovalPass> {
534        let mut pass = RemovalPass::default();
535        let baseline = self.state.load_baseline()?;
536        // A crashed earlier barrier that had unlinked and journalled but
537        // not installed: its declarations are still deletions with a
538        // recorded basis — unless the agent has since put a file back.
539        for p in self.state.load_intent()?.declared_deletes {
540            let gone = matches!(
541                std::fs::symlink_metadata(self.cfg.root.join(&p)),
542                Err(ref e) if e.kind() == std::io::ErrorKind::NotFound
543            );
544            if gone && (baseline.entries.contains_key(&p) || baseline.inst_base.contains_key(&p)) {
545                pass.declared.insert(p);
546            }
547        }
548        for r in &doc.removals {
549            if r.refused.is_some() {
550                continue;
551            }
552            let refusal = |kind: &str, message: String| Removal {
553                refused: Some(Refusal { kind: kind.into(), message, at_unix: now_unix() }),
554                ..r.clone()
555            };
556            let mut refuse = |kind: &str, message: String| -> LeanResult<()> {
557                self.state.append_conflict(&ConflictRecord {
558                    path: r.path.clone(),
559                    foreign_etag: String::new(),
560                    preserved_key: None,
561                    kind: format!("{kind}: {message}"),
562                    at_unix: now_unix(),
563                })?;
564                pass.refused.push(refusal(kind, message));
565                Ok(())
566            };
567            if let Err(e) = check_contained(&self.cfg.root, &r.path) {
568                refuse("removal-refused-containment", e.to_string())?;
569                continue;
570            }
571            if let Some(dst) = &r.moved_to {
572                match baseline.entries.get(dst) {
573                    // Not integrated yet (a consume deferred it): wait.
574                    None => {
575                        pass.deferred += 1;
576                        continue;
577                    }
578                    // Integrated as a CONFLICT: the agent had an
579                    // unpublished file at the destination, its version
580                    // won, and the moved bytes are preserved under
581                    // `conflicts/`. Deleting the source now would leave
582                    // the user's file nowhere in the tree — so the move
583                    // is refused and the source stays. The sentinel is
584                    // the consume-dirty baseline entry's `u64::MAX`.
585                    Some(b) if b.size == u64::MAX => {
586                        refuse(
587                            "removal-refused-destination-conflict",
588                            format!(
589                                "{} was not moved to {dst}: the agent has an unpublished                                  file at the destination and it wins; the moved bytes are                                  preserved under the conflict record and the source is kept",
590                                r.path
591                            ),
592                        )?;
593                        continue;
594                    }
595                    Some(_) => {}
596                }
597            }
598            let local = self.cfg.root.join(&r.path);
599            let be = baseline.entries.get(&r.path);
600            match std::fs::symlink_metadata(&local) {
601                Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
602                    if be.is_some() || baseline.inst_base.contains_key(&r.path) {
603                        pass.declared.insert(r.path.clone());
604                    }
605                    // Nothing here and nothing cited: a removal of a path
606                    // that was never there, or that a sibling already
607                    // removed. Applied as a no-op.
608                    pass.applied.push(r.clone());
609                }
610                Err(e) => {
611                    // Unreadable is not absent (`confirm_absences`' rule):
612                    // fail closed, and say so where the pod-side reader
613                    // looks. Transient, so deferred rather than refused.
614                    self.state.append_conflict(&ConflictRecord {
615                        path: r.path.clone(),
616                        foreign_etag: String::new(),
617                        preserved_key: None,
618                        kind: format!("removal-deferred: cannot stat the path: {e}"),
619                        at_unix: now_unix(),
620                    })?;
621                    pass.deferred += 1;
622                }
623                Ok(m) if !m.is_file() => {
624                    refuse(
625                        "removal-refused-not-a-file",
626                        format!("{} is not a regular file in the workspace", r.path),
627                    )?;
628                }
629                Ok(_) => {
630                    if local_dirty(&local, be) {
631                        refuse(
632                            "removal-refused-dirty",
633                            format!(
634                                "{} has unpublished local changes; the removal {} asked for                                  was not applied and the agent's work is kept — publish or                                  discard the local version, then ask again",
635                                r.path, r.author
636                            ),
637                        )?;
638                        continue;
639                    }
640                    if let Err(e) = std::fs::remove_file(&local) {
641                        self.state.append_conflict(&ConflictRecord {
642                            path: r.path.clone(),
643                            foreign_etag: String::new(),
644                            preserved_key: None,
645                            kind: format!("removal-unlink-failed (will retry): {e}"),
646                            at_unix: now_unix(),
647                        })?;
648                        pass.deferred += 1;
649                        continue;
650                    }
651                    // CONFIRM: only NotFound is absence.
652                    match std::fs::symlink_metadata(&local) {
653                        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
654                            pass.declared.insert(r.path.clone());
655                            pass.applied.push(r.clone());
656                        }
657                        _ => {
658                            self.state.append_conflict(&ConflictRecord {
659                                path: r.path.clone(),
660                                foreign_etag: String::new(),
661                                preserved_key: None,
662                                kind: "removal-unlink-unconfirmed (will retry)".into(),
663                                at_unix: now_unix(),
664                            })?;
665                            pass.deferred += 1;
666                        }
667                    }
668                }
669            }
670        }
671        Ok(pass)
672    }
673
674    /// Act on the inbox document's two request fields (§2.5, D14).
675    ///
676    /// The asymmetry is deliberate and is about blast radius, not
677    /// principle: a boundary request is PERFORMED (it publishes what is
678    /// already on disk and mutates nothing local), a sync request is
679    /// CARRIED into the ticker as advisory news and the agent decides.
680    ///
681    /// Both are watermarked by `requested_unix` rather than consumed by
682    /// a clearing CAS. The field is idempotent state, so a repeat is a
683    /// no-op and a lost clear cannot double-publish; and not writing
684    /// means the doors cost no request at all, which is what §2.5
685    /// promises.
686    fn note_verb_requests(&mut self, doc: &inbox::InboxDoc) -> LeanResult<()> {
687        let mut mark = self.load_verb_watermark()?;
688        if let Some(r) = &doc.sync_request {
689            if r.requested_unix > mark.sync_unix {
690                self.carry_sync_request(r.requested_unix, &r.requestor)?;
691                mark.sync_unix = r.requested_unix;
692            }
693        }
694        if let Some(r) = &doc.boundary_request {
695            if r.requested_unix > mark.boundary_unix {
696                self.request_boundary(
697                    &format!("gw:{}:{}", r.requested_unix, r.requestor),
698                    Some(format!("boundary requested by {}", r.requestor)),
699                )?;
700                mark.boundary_unix = r.requested_unix;
701            }
702        }
703        self.save_verb_watermark(&mark)
704    }
705
706    fn verb_watermark_path(&self) -> std::path::PathBuf {
707        self.cfg.state_dir().join("verb-requests.json")
708    }
709
710    pub(crate) fn load_verb_watermark(&self) -> LeanResult<VerbWatermark> {
711        let p = self.verb_watermark_path();
712        if !p.exists() {
713            return Ok(VerbWatermark::default());
714        }
715        Ok(std::fs::read(&p)
716            .ok()
717            .and_then(|b| serde_json::from_slice(&b).ok())
718            .unwrap_or_default())
719    }
720
721    fn save_verb_watermark(&self, m: &VerbWatermark) -> LeanResult<()> {
722        let bytes = serde_json::to_vec_pretty(m)
723            .map_err(|e| LeanError::State(format!("verb watermark: {e}")))?;
724        super::control::write_atomic(&self.verb_watermark_path(), &bytes)
725    }
726
727    async fn preserve_conflict_copy(&self, path: &str, etag: &str) -> LeanResult<String> {
728        let key = self.cfg.file_key(path);
729        let dst = self.cfg.conflict_key(&uuid::Uuid::new_v4().to_string(), path);
730        // v1: client-side copy (GET + guarded PUT). The server-side
731        // CopyObject lever is the designed v2 optimization for large
732        // files.
733        let (meta, body) = self.store.get_whole(&key, Some(etag)).await?;
734        let crc = crc64_nvme(&body);
735        let stamps = GenerationStamps {
736            generation: 0,
737            epoch: self
738                .lease
739                .as_ref()
740                .map(|l| l.epoch)
741                .or_else(|| self.state.load_incarnation().ok().flatten().map(|i| i.epoch))
742                .unwrap_or(0),
743            flush_uuid: "conflict-preserve".into(),
744            boundary_source: None,
745            posix: PosixStamps::from_meta(&meta.meta),
746        };
747        self.store
748            .put_whole(&dst, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
749            .await?;
750        Ok(dst)
751    }
752
753    /// Take the SECOND absence observation now, by direct `lstat`.
754    ///
755    /// `scan::classify` withholds a path's delete until absence has
756    /// survived two consecutive scans — the rename-vs-walk race guard
757    /// (`lib.rs:37`). The hazard that rule names is the WALK missing a
758    /// file renamed under it, not deletion itself. On the cadence path
759    /// the second walk arrives at the next floor tick and nobody has
760    /// been promised otherwise. On a DECLARED boundary it cannot wait:
761    /// the ack would claim a coherent point (D1 — "everything
762    /// ordered-before T") while the manifest still cites a file the
763    /// agent removed before it touched the sentinel, and it would say
764    /// so with `report.deleted: 0` and `status: "ok"`.
765    ///
766    /// A direct `lstat` is exactly what the rename-vs-walk guard asks
767    /// for and is immune to the walk race by construction, so the
768    /// second observation costs one syscall per transiently-absent
769    /// path — never a second full pass. The cadence path is unchanged
770    /// and still waits for the second walk.
771    pub(crate) fn confirm_absences(&self, classified: &mut scan::Classified) -> LeanResult<usize> {
772        if classified.first_absence.is_empty() {
773            return Ok(0);
774        }
775        let mut confirmed = 0;
776        for path in std::mem::take(&mut classified.first_absence) {
777            // ONLY NotFound is absence. Every other errno — EACCES on a
778            // parent, EIO, EMFILE, ELOOP — used to read as "the agent
779            // deleted it", and this is the oracle that PUBLISHES the
780            // delete and then DELETES THE OBJECT. An unreadable file is
781            // the one case where guessing is unrecoverable, so it fails
782            // CLOSED, exactly as the walk does at `scan.rs`'s
783            // `symlink_metadata(&path)?`. Two call sites, one syscall:
784            // they must not have opposite error policies.
785            let gone = match std::fs::symlink_metadata(self.cfg.root.join(&path)) {
786                Err(e) if e.kind() == std::io::ErrorKind::NotFound => true,
787                Err(e) => {
788                    classified.first_absence.insert(path.clone());
789                    return Err(LeanError::State(format!(
790                        "cannot confirm the absence of {path:?}: {e} — refusing to publish \
791                         a deletion on an unreadable path"
792                    )));
793                }
794                Ok(_) => false,
795            };
796            if gone {
797                classified.deletes.insert(path);
798                confirmed += 1;
799            } else {
800                // Back on disk: the rename-vs-walk race, caught exactly
801                // as the rule intends. Still only a first absence.
802                classified.first_absence.insert(path);
803            }
804        }
805        Ok(confirmed)
806    }
807
808    /// Renew the lease if it has gone stale MID-BARRIER, and hand back
809    /// the cell as it was read so the CALLER can fence on it.
810    ///
811    /// Costs nothing on a short barrier: the renewal is skipped unless
812    /// it was already due, so the request count is unchanged — this
813    /// moves WHEN the renewal can happen, not how often.
814    ///
815    /// What this does NOT do, stated because the previous comment
816    /// claimed it did: it never raises `Fenced` for a deposed holder.
817    /// `lease::renew` can only 412 when the cell is still at OUR epoch
818    /// with a token we do not hold; a cell at a FOREIGN epoch is
819    /// skipped here (renewing it would be nonsense), so a deposed
820    /// straggler passed straight through. The 2026-09-03 audit found
821    /// the cadence barrier relying on exactly that phantom fence
822    /// between upload chunks and completing every remaining data PUT
823    /// after a takeover. The cell is returned so the caller can compare
824    /// it against its lease — the cadence barrier uses the read it
825    /// already paid for rather than a separate fence. `None` when there is
826    /// no lease or the read failed: neither is a fence, and the
827    /// ordinary renewal arm will try again.
828    pub(crate) async fn renew_if_due(&mut self) -> LeanResult<Option<EpochState>> {
829        let Some(lease) = self.lease.clone() else { return Ok(None) };
830        let state = match self.store.epoch_read(&self.cfg.epoch_key()).await {
831            Ok(Some(s)) => s,
832            _ => return Ok(None),
833        };
834        let now = super::now_unix();
835        let fresh = state
836            .last_renew_unix
837            .map(|t| now.saturating_sub(t) < RENEW_WITHIN_SECS)
838            .unwrap_or(false);
839        if fresh || state.epoch != lease.epoch {
840            // Not due, or not ours to renew. The caller decides what a
841            // foreign cell means; this function only knows it cannot
842            // renew one.
843            return Ok(Some(state));
844        }
845        super::lease::renew(self).await?;
846        Ok(Some(state))
847    }
848
849    /// The between-chunk fence of the cadence barrier: a cell that no
850    /// longer names this holder at this epoch stops the upload set
851    /// HERE, before the next chunk's PUTs. Every one of those PUTs
852    /// carries If-Match on a baseline etag that still matches — the
853    /// successor has published nothing yet — so without this a deposed
854    /// straggler overwrites the cited generation of every key it still
855    /// had to upload, and every reader's S3-wins arm then adopts the
856    /// uncited bytes silently (audit 2026-09-03, finding 1).
857    fn fence_on_cell(&self, cell: &EpochState) -> LeanResult<()> {
858        let Some(lease) = self.lease.as_ref() else {
859            return Err(LeanError::Fenced("lease dropped mid-barrier".into()));
860        };
861        if cell.epoch != lease.epoch || cell.holder_id != lease.holder_id {
862            return Err(LeanError::Fenced(format!(
863                "deposed between upload chunks: cell at epoch {} holder {} (we are epoch {})",
864                cell.epoch, cell.holder_id, lease.epoch
865            )));
866        }
867        Ok(())
868    }
869
870    /// Steps 2–7. `barrier` = one full publish cycle (the cadence arm).
871    pub async fn run_barrier(&mut self) -> LeanResult<BarrierReport> {
872        self.barrier_inner(false, "manual").await
873    }
874
875    /// The floor tick's barrier. Stamped `cadence` so the bucket can
876    /// answer "which clock installed this boundary?" without the ack —
877    /// the ack is a LOCAL file, and the fleet-visible question is asked
878    /// by operators and by the gateway's `/status`, neither of which
879    /// can see it.
880    pub async fn cadence_barrier(&mut self) -> LeanResult<BarrierReport> {
881        self.barrier_inner(false, "cadence").await
882    }
883
884    /// A barrier whose result somebody is going to ACK: a sentinel
885    /// honor (D1) or the preStop drain (D10). Identical to
886    /// `run_barrier` except that it confirms first-absence paths rather
887    /// than acking a boundary that withholds them.
888    pub async fn declared_barrier(&mut self) -> LeanResult<BarrierReport> {
889        self.barrier_inner(true, "sentinel").await
890    }
891
892    /// `declared_barrier` under an explicit provenance stamp — the
893    /// drain and the budget-deferred honor are the same barrier
894    /// reporting a different clock.
895    pub async fn declared_barrier_as(&mut self, source: &str) -> LeanResult<BarrierReport> {
896        self.barrier_inner(true, source).await
897    }
898
899    async fn barrier_inner(&mut self, declared: bool, source: &str) -> LeanResult<BarrierReport> {
900        // Every publishing path ends here — cadence, sentinel, drain, the
901        // one-shot verb — so a reader is refused here before the first
902        // request, whichever of them a caller reached. The callers that
903        // owe an answer (the publish sentinel, the drain) give it before
904        // this; this is the backstop for the ones that do not.
905        self.refuse_if_read("barrier")?;
906        // No lease is held until the commit section (design 2026-09-13
907        // §4): the consume, the scan and the uploads below run with the
908        // cell at rest, guarded by each object's own etag. What the
909        // uploads are STAMPED with is the last epoch this incarnation
910        // held — informational, like every epoch stamp on an object; the
911        // manifest entries carry the commit section's real epoch.
912        let epoch_hint = self.state.load_incarnation()?.map(|i| i.epoch).unwrap_or(0);
913        let mut report = BarrierReport::default();
914        let barrier_started = std::time::Instant::now();
915        self.trace("barrier_start", serde_json::json!({"source": source, "declared": declared}));
916
917        // STEP 0: finish any rescope a crash left half-applied, before
918        // the scan reads the tree (scoped-read design §4.3). This is the
919        // gate that makes the narrow verb safe at all: a half-applied
920        // narrow is a tree whose files and whose citations disagree, and
921        // `classify` reads that disagreement as an upload or as a
922        // DELETE depending on which half landed. Replay is idempotent,
923        // so the cost on the normal path is one `stat` of a file that
924        // is not there.
925        if let Some(r) = self.replay_scope_intent().await? {
926            report.rescope_replayed = true;
927            eprintln!(
928                "flint-sync: finished an interrupted rescope before the barrier \
929                 (uncited {}, unlinked {}, materialised {})",
930                r.uncited, r.unlinked, r.materialized
931            );
932        }
933
934        // Step 1: the cell, read ONCE — HITL entries into the tree, then
935        // the DECLARED removals (delete/rename design §4-§6).
936        let inbox_doc = inbox::load(self.store.as_ref(), &self.cfg).await?.doc;
937        let (consumed, settled_locally) = self.consume_counted(&inbox_doc).await?;
938        report.consumed = consumed.len() + settled_locally;
939        let removals = self.apply_removals(&inbox_doc).await?;
940        report.removed = removals.declared.iter().cloned().collect();
941        report.removals_refused = removals.refused.len();
942        // A refusal is settled NOW, whatever else this barrier does: the
943        // cell is where the human who asked reads the answer, and a
944        // refused removal is never retried, so saying so before the
945        // no-diff return below loses nothing.
946        inbox::settle_removals(self.store.as_ref(), &self.cfg, epoch_hint, &[], &removals.refused)
947            .await?;
948
949        // Step 2: scan-diff against the persisted baseline.
950        let mut baseline = self.state.load_baseline()?;
951        let scanned = scan::scan(&self.cfg.root)?;
952        let mut classified = scan::classify(&scanned, &baseline);
953        if declared {
954            report.absences_confirmed = self.confirm_absences(&mut classified)?;
955        }
956        // A declared removal is a deletion with its basis RECORDED, so
957        // it skips the two-scan guard the walk needs and goes straight
958        // into the delete set — which is what lets a rename's two halves
959        // ride ONE manifest generation (§4).
960        for p in &removals.declared {
961            classified.first_absence.remove(p);
962            if classified.uploads.contains(p) {
963                // The agent re-created the path between the unlink and
964                // the scan. The declaration named the OLD file, which is
965                // gone from the tree (and, for a rename, moved); what is
966                // here now is a new file the agent wrote, and it
967                // publishes as one — its PUT supersedes the old object
968                // under the same key. Deleting it would un-cite the
969                // agent's work for a barrier and GC-skip its object; the
970                // model's install gives the upload the same precedence.
971                continue;
972            }
973            classified.deletes.insert(p.clone());
974        }
975        report.first_absence = classified.first_absence.iter().cloned().collect();
976
977        // Skip-on-no-diff: nothing local, nothing consumed, no pending
978        // citation repair, and the bucket manifest where we left it.
979        let repairs_pending = baseline
980            .entries
981            .iter()
982            .any(|(p, be)| be.size != u64::MAX && baseline.inst_base.get(p) != Some(&be.etag));
983        if classified.uploads.is_empty()
984            && classified.deletes.is_empty()
985            && consumed.is_empty()
986            && classified.first_absence.is_empty()
987            && removals.applied.is_empty()
988            && !repairs_pending
989        {
990            // Read the POINTER, never the entries: the 0b rig measured
991            // the idle tick at 1M entries as 27 s / 1.3 GiB — dominated
992            // by fetching and parsing a 264 MiB document just to read
993            // `seq`. The pointer is a few hundred bytes and carries the
994            // seq in its body, so a GET of it costs one round trip and
995            // answers strictly more than a HEAD of the old object did.
996            //
997            // Legacy workspaces (no pointer yet) keep the HEAD they had.
998            let unchanged = match manifest::load_pointer(self.store.as_ref(), &self.cfg).await? {
999                Some(manifest::LoadedPointer { pointer: p, etag, .. }) => {
1000                    // The news ticker rides this request for free — D5's
1001                    // "zero added bucket requests" is still literal.
1002                    report.observed_seq = Some(p.seq);
1003                    report.observed_etag = Some(etag.clone());
1004                    baseline.manifest_etag.as_deref() == Some(etag.as_str())
1005                }
1006                None => match self.store.head(&self.cfg.manifest_key()).await {
1007                    Ok(meta) => {
1008                        report.observed_seq =
1009                            GenerationStamps::from_meta(&meta.meta).map(|s| s.generation);
1010                        report.observed_etag = Some(meta.etag.clone());
1011                        baseline.manifest_etag.as_deref() == Some(meta.etag.as_str())
1012                    }
1013                    Err(StoreError::NotFound(_)) => baseline.manifest_etag.is_none(),
1014                    Err(e) => return Err(e.into()),
1015                },
1016            };
1017            if unchanged {
1018                report.no_change = true;
1019                report.seq = Some(baseline.seq);
1020                // prev_scan still advances (the two-scan rule's clock) —
1021                // but the baseline document (hundreds of MB at 1M
1022                // entries) is rewritten only when the scan SET moved.
1023                // Compare without building the set: both sides iterate
1024                // sorted, and the allocation this used to make was the
1025                // size of the very document it was trying not to write.
1026                if scanned.len() != baseline.prev_scan.len()
1027                    || !scanned.keys().eq(baseline.prev_scan.iter())
1028                {
1029                    baseline.prev_scan = scanned.keys().cloned().collect();
1030                    self.state.save_baseline(&baseline)?;
1031                }
1032                self.trace_barrier_end(&report, barrier_started);
1033                return Ok(report);
1034            }
1035        }
1036
1037        // Step 3: intent journal, then the window (the commitment
1038        // point: the same CAS drops the consumed entries).
1039        let flush_uuid = uuid::Uuid::new_v4().to_string();
1040        self.trace("scan", serde_json::json!({"flush": flush_uuid, "uploads": classified.uploads.len(),
1041            "deletes": classified.deletes.len(), "first_absence": classified.first_absence.len()}));
1042        let mut intent = self.state.load_intent()?;
1043        let prior_uuids = {
1044            let mut v = intent.recent_uuids.clone();
1045            if !intent.flush_uuid.is_empty() {
1046                v.push(intent.flush_uuid.clone());
1047            }
1048            v
1049        };
1050        // Carried forward, not dropped: it stays true until we install
1051        // again, and the merge below is what reads it.
1052        let prev_installed = intent.installed_etag.clone();
1053        intent = IntentJournal {
1054            flush_uuid: flush_uuid.clone(),
1055            keys: classified.uploads.iter().map(|p| self.cfg.file_key(p)).collect(),
1056            recent_uuids: prior_uuids.clone(),
1057            installed_etag: prev_installed.clone(),
1058            installed_foreign: intent.installed_foreign.clone(),
1059            declared_deletes: removals.declared.iter().cloned().collect(),
1060        };
1061        self.state.save_intent(&intent)?;
1062        // The window is NOT opened here any more. It is the bucket's
1063        // "a barrier is committing" sign for HITL writers, and it opens
1064        // in the commit section, under the lease, at the epoch the
1065        // commit really holds. The uploads below race a HITL write the
1066        // way two writers race each other: If-Match decides, the loser
1067        // is preserved and recorded (`LeanNoWindowHolds` proves safety
1068        // never depended on the window).
1069        //
1070        // The consumed entries are not dropped here either — "durably
1071        // in the baseline" is true of a container restart and false of
1072        // a pod REPLACEMENT: the emptyDir goes with the pod. They leave
1073        // the cell with the window clear, after the manifest cites
1074        // them; a successor that finds them re-consumes, idempotently.
1075
1076        // Step 4: guarded uploads, fanned out under a bounded window
1077        // (each key's guard chain is independent; the 412 policy and
1078        // conflict records are applied to the collected results below,
1079        // in deterministic path order).
1080        let mut upserts: BTreeMap<String, LeanEntry> = BTreeMap::new();
1081        let mut parked: BTreeSet<String> = BTreeSet::new();
1082        // Citations whose etag was observed with no lease held (adopted
1083        // uploads, citation repairs): re-read inside the commit section.
1084        let mut observed: BTreeSet<String> = BTreeSet::new();
1085        let mut new_baseline_entries: BTreeMap<String, BaselineEntry> = BTreeMap::new();
1086        let outcomes: Vec<(String, LeanResult<UploadOutcome>)> = {
1087            use futures::stream::{self, StreamExt};
1088            // CHUNKED into waves, so a wave's outcomes are collected
1089            // before the next is fanned out. No lease is held here
1090            // (design 2026-09-13 §4), so nothing renews or fences
1091            // between chunks any more — the between-chunk fence that
1092            // stopped a deposed straggler's PUTs went with the straggler:
1093            // a PUT outside the lease is one legitimate writer's PUT,
1094            // guarded by If-Match, and a collision is preserved and
1095            // superseded, never silently overwritten.
1096            let mut outcomes: Vec<(String, LeanResult<UploadOutcome>)> = Vec::new();
1097            // NOT cfg.fanout. That knob was raised to 128 for the READ
1098            // path, and the read-side measurement (2.5x on 20k small
1099            // files) says nothing about this one, so uploads keep the
1100            // value they were measured at until someone measures them.
1101            // Memory is no longer the reason they are separate: this
1102            // path is byte-bounded too now, by the store's gate
1103            // (`with_upload_inflight_max_bytes`), which charges every
1104            // whole body below and every multipart part inside the
1105            // store from before its read until its PUT returns.
1106            let fanout = self.cfg.upload_fanout.max(1);
1107            let chunk = fanout.saturating_mul(UPLOAD_CHUNK_WAVES).max(1);
1108            let all: Vec<&String> = classified.uploads.iter().collect();
1109            for group in all.chunks(chunk) {
1110                {
1111                    let this: &Syncer = &*self;
1112                    let mut part: Vec<(String, LeanResult<UploadOutcome>)> =
1113                        stream::iter(group.iter().map(|path| {
1114                            let path = *path;
1115                            let scanned_entry = &scanned[path];
1116                            let base = baseline.entries.get(path);
1117                            let flush_uuid = &flush_uuid;
1118                            let prior_uuids = &prior_uuids;
1119                            async move {
1120                                let r = this
1121                                    .upload_one(
1122                                        path, scanned_entry, base, epoch_hint, flush_uuid,
1123                                        prior_uuids,
1124                                    )
1125                                    .await;
1126                                (path.clone(), r)
1127                            }
1128                        }))
1129                        .buffer_unordered(fanout)
1130                        .collect()
1131                        .await;
1132                    outcomes.append(&mut part);
1133                }
1134            }
1135            outcomes
1136        };
1137        let mut outcomes = outcomes;
1138        outcomes.sort_by(|a, b| a.0.cmp(&b.0));
1139        for (path, outcome) in outcomes {
1140            let path = &path;
1141            match outcome? {
1142                UploadOutcome::Published { entry, baseline_entry, adopted } => {
1143                    self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "etag": entry.etag,
1144                        "outcome": if adopted { "adopted" } else { "put" }}));
1145                    if adopted {
1146                        observed.insert(path.clone());
1147                    }
1148                    report.published_bytes += entry.size;
1149                    upserts.insert(path.clone(), entry);
1150                    new_baseline_entries.insert(path.clone(), baseline_entry);
1151                    report.uploaded.push(path.clone());
1152                }
1153                UploadOutcome::Parked { foreign_etag } => {
1154                    self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "etag": foreign_etag, "outcome": "parked"}));
1155                    parked.insert(path.clone());
1156                    self.state.append_conflict(&ConflictRecord {
1157                        path: path.clone(),
1158                        foreign_etag,
1159                        preserved_key: None, // parking preserves in place
1160                        kind: "upload-412-parked".into(),
1161                        at_unix: now_unix(),
1162                    })?;
1163                    report.parked.push(path.clone());
1164                }
1165                UploadOutcome::Deferred => {
1166                    self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "outcome": "deferred"}));
1167                    report.deferred.push(path.clone());
1168                }
1169            }
1170        }
1171
1172        // Citation repairs: paths whose integrated object (the
1173        // baseline) differs from the manifest's citation — consumed
1174        // HITL adoptions and checkout's S3-wins arm. No bytes move; the
1175        // manifest re-cites what this syncer already integrated.
1176        // Without this, an adopted upload is clean-vs-baseline, never
1177        // enters the upload set, and the manifest silently drops it —
1178        // the battery's amputation leg caught exactly that.
1179        let repair_candidates: Vec<String> = baseline
1180            .entries
1181            .iter()
1182            .filter(|(path, be)| {
1183                !classified.uploads.contains(*path)
1184                    && !classified.deletes.contains(*path)
1185                    && be.size != u64::MAX // consume-dirty sentinel: publishes via upload
1186                    && baseline.inst_base.get(*path) != Some(&be.etag)
1187            })
1188            .map(|(path, _)| path.clone())
1189            .collect();
1190        for path in repair_candidates {
1191            let key = self.cfg.file_key(&path);
1192            let be = baseline.entries[&path].clone();
1193            match self.store.head(&key).await {
1194                Ok(meta) if meta.etag == be.etag => {
1195                    let crc = repair_crc(&path, &be, &meta)?;
1196                    let stamps = GenerationStamps::from_meta(&meta.meta);
1197                    let scan_entry = scanned.get(&path);
1198                    upserts.insert(
1199                        path.clone(),
1200                        LeanEntry {
1201                            key,
1202                            etag: meta.etag.clone(),
1203                            crc64_b64: crc,
1204                            size: meta.size,
1205                            mode: stamps
1206                                .as_ref()
1207                                .and_then(|s| s.posix)
1208                                .map(|p| p.mode)
1209                                .or(scan_entry.map(|s| s.mode))
1210                                .unwrap_or(0o644),
1211                            mtime_unix: scan_entry.map(|s| s.mtime_unix).unwrap_or(0),
1212                            generation: stamps.map(|s| s.generation).unwrap_or(be.generation),
1213                            epoch: epoch_hint,
1214                        },
1215                    );
1216                    observed.insert(path.clone());
1217                }
1218                // Moved again or gone: the next consume reconciles it.
1219                _ => {}
1220            }
1221        }
1222
1223        // A PULL-ONLY boundary: nothing uploaded, deleted, consumed, removed
1224        // or re-cited, so the merge can only add nothing — and then the
1225        // commit section writes nothing to the bucket (no CAS, no GC, no
1226        // inbox entry to drop) and what remains is local: queue the other
1227        // writers' changes and take theirs as the merge base. That needs no
1228        // fence and no window; claiming for it queued every such writer
1229        // behind the publishing ones (65 of 191 claims in the writers drill).
1230        // A merge that does add something (a mirror flag to restamp, say)
1231        // falls through to the commit section.
1232        if classified.uploads.is_empty()
1233            && classified.deletes.is_empty()
1234            && upserts.is_empty()
1235            && consumed.is_empty()
1236            && removals.applied.is_empty()
1237            && observed.is_empty()
1238        {
1239            let current = manifest::load(self.store.as_ref(), &self.cfg).await?;
1240            let m = self.merge_onto(
1241                current.as_ref(),
1242                prev_installed.as_deref(),
1243                &baseline.inst_base,
1244                &upserts,
1245                &classified.deletes,
1246                &parked,
1247                &flush_uuid,
1248            );
1249            if let Some(handle) = m.expected.as_ref().filter(|_| m.adds_nothing()) {
1250                if !m.foreign.is_empty() || !m.gone.is_empty() {
1251                    self.state.queue_foreign(&foreign_changes(&m.foreign, &m.gone))?;
1252                    self.trace("queue", serde_json::json!({"flush": flush_uuid, "upserts": m.foreign.len(), "tombstones": m.gone.len(), "fence": false}));
1253                }
1254                report.seq = Some(m.theirs.seq);
1255                report.no_change = true;
1256                report.manifest_etag = Some(handle.etag.clone());
1257                report.observed_seq = Some(m.theirs.seq);
1258                report.observed_etag = Some(handle.etag.clone());
1259                report.foreign_queued = m.foreign.len();
1260                baseline.inst_base = m.theirs.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
1261                baseline.seq = m.theirs.seq;
1262                baseline.manifest_etag = Some(handle.etag.clone());
1263                baseline.prev_scan = scanned.keys().cloned().collect();
1264                self.state.save_baseline(&baseline)?;
1265                self.state.clear_intent_keys()?;
1266                self.trace_barrier_end(&report, barrier_started);
1267                return Ok(report);
1268            }
1269        }
1270
1271        // THE COMMIT SECTION (design 2026-09-13 §4). Everything above ran
1272        // with no lease: the uploads are durable and etag-guarded, and
1273        // nothing is cited yet. Claim the cell now — for the merge, the
1274        // CAS, the deletes and the baseline — and hand it to the next
1275        // waiter when done. This is the only stretch two writers of one
1276        // workspace serialise on, and it is small: one pointer CAS and
1277        // the guarded deletes, milliseconds to seconds, never the upload
1278        // of a checkpoint.
1279        let held = super::lease::claim(self).await?;
1280        let epoch = held.epoch;
1281        eprintln!("flint-sync: publish fence held (epoch {epoch})");
1282        if self.cfg.drill_hold_commit_secs > 0 {
1283            eprintln!(
1284                "flint-sync: DRILL: holding the fence for {}s inside the commit section",
1285                self.cfg.drill_hold_commit_secs
1286            );
1287            tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_commit_secs)).await;
1288        }
1289        let commit: LeanResult<()> = async {
1290            // The entries carry the epoch the commit HOLDS, not the hint
1291            // the uploads were stamped with.
1292            for e in upserts.values_mut() {
1293                e.epoch = epoch;
1294            }
1295            // Step 5: the manifest CAS (three-way merge; bounded retries).
1296            self.verify_not_deposed().await?;
1297            // EVERY citation this barrier adds was read or written with no
1298            // lease held. Another writer's commit may since have uncited
1299            // the path and collected the object: its GC deletes by the etag
1300            // it integrated. That etag names an ADOPTED object (the model's
1301            // LeanBarrierLeaseAdoptBlind) — and, since an S3 whole-PUT etag
1302            // is the MD5 of the bytes, it also names this barrier's own PUT
1303            // of IDENTICAL bytes (finding 13, the writers drill's A3: a
1304            // same-content rewrite re-uploaded, the peer's GC deleted it,
1305            // the commit cited a hole). Collection runs only inside a
1306            // commit section, so a re-read HERE, holding the fence, cannot
1307            // be overtaken before this CAS. Whatever is gone or replaced is
1308            // withheld: parked, recorded, and left dirty for the next
1309            // barrier to publish with a PUT of its own.
1310            let heads: Vec<(String, LeanResult<bool>)> = {
1311                use futures::stream::{self, StreamExt};
1312                let this: &Syncer = &*self;
1313                stream::iter(upserts.iter().map(|(path, cited)| async move {
1314                    let still = match this.store.head(&cited.key).await {
1315                        Ok(meta) => Ok(meta.etag == cited.etag),
1316                        Err(StoreError::NotFound(_)) => Ok(false),
1317                        Err(e) => Err(e.into()),
1318                    };
1319                    (path.clone(), still)
1320                }))
1321                .buffer_unordered(self.cfg.upload_fanout.max(1))
1322                .collect()
1323                .await
1324            };
1325            let mut heads = heads;
1326            heads.sort_by(|a, b| a.0.cmp(&b.0));
1327            for (path, still) in heads {
1328                let still_there = still?;
1329                let was_observed = observed.contains(&path);
1330                let cited = &upserts[&path];
1331                self.trace("observed", serde_json::json!({"flush": flush_uuid, "path": path, "etag": cited.etag, "still": still_there, "own_put": !was_observed}));
1332                if still_there {
1333                    continue;
1334                }
1335                let gone = upserts.remove(&path).expect("looked up above");
1336                new_baseline_entries.remove(&path);
1337                report.uploaded.retain(|p| p != &path);
1338                report.published_bytes = report.published_bytes.saturating_sub(gone.size);
1339                // A citation REPAIR re-cites bytes the tree already holds (a
1340                // consumed UI write, checkout's S3-wins arm): the baseline
1341                // names this etag and the scan sees the path clean, so no
1342                // later barrier publishes it. Parking it kept the merge from
1343                // queueing the version that replaced it while the merge base
1344                // moved past that version, and the tree kept the withheld
1345                // bytes for good (storm drill S0, 2026-09-15: one writer of
1346                // six diverged). Not parked, the merge queues the manifest's
1347                // version and the next consume installs it.
1348                let integrated = was_observed
1349                    && baseline
1350                        .entries
1351                        .get(&path)
1352                        .is_some_and(|be| be.size != u64::MAX && be.etag == gone.etag);
1353                if integrated {
1354                    self.state.append_conflict(&ConflictRecord {
1355                        path: path.clone(),
1356                        foreign_etag: gone.etag,
1357                        preserved_key: None,
1358                        kind: "repair-withheld: the object this barrier re-cited was replaced or \
1359                               collected before its commit; nothing cited, and the next consume \
1360                               installs the manifest's version"
1361                            .into(),
1362                        at_unix: now_unix(),
1363                    })?;
1364                    continue;
1365                }
1366                parked.insert(path.clone());
1367                let kind = if was_observed {
1368                    "adopt-withheld: the object this barrier found already at its key was \
1369                     replaced or collected before its commit; nothing cited, the path stays \
1370                     dirty and publishes next barrier"
1371                } else {
1372                    "upload-withheld: the object this barrier PUT was replaced or collected \
1373                     before its commit (a peer's GC recognizes the etag of identical bytes); \
1374                     nothing cited, the path stays dirty and publishes next barrier"
1375                };
1376                self.state.append_conflict(&ConflictRecord {
1377                    path: path.clone(),
1378                    foreign_etag: gone.etag,
1379                    preserved_key: None,
1380                    kind: kind.into(),
1381                    at_unix: now_unix(),
1382                })?;
1383                report.parked.push(path.clone());
1384            }
1385            // The window: the bucket-visible "a barrier is committing" sign
1386            // HITL writers wait on, at the epoch this commit holds.
1387            let deadline = now_unix() + self.cfg.window_slack_secs;
1388            inbox::open_window(self.store.as_ref(), &self.cfg, epoch, deadline).await?;
1389            let mut foreign_entries: Vec<(String, LeanEntry)> = vec![];
1390            let mut foreign_gone: Vec<String> = vec![];
1391            // Set when the merge added nothing to the document: nothing is
1392            // installed, and theirs becomes the merge base as it stands.
1393            let mut installed_nothing = false;
1394            let mut attempt = 0;
1395            let (installed, installed_etag) = loop {
1396                attempt += 1;
1397                if attempt > 4 {
1398                    return Err(LeanError::State(
1399                        "manifest CAS lost 4 merge races — refusing this barrier".into(),
1400                    ));
1401                }
1402                let current = manifest::load(self.store.as_ref(), &self.cfg).await?;
1403                let MergeOnto { theirs, expected, merged, foreign, gone } = self.merge_onto(
1404                    current.as_ref(),
1405                    prev_installed.as_deref(),
1406                    &baseline.inst_base,
1407                    &upserts,
1408                    &classified.deletes,
1409                    &parked,
1410                    &flush_uuid,
1411                );
1412                // Nothing of ours changes the document — a barrier that only
1413                // found the manifest moved by another writer. Installing it
1414                // anyway was an empty generation, and the peer's next tick
1415                // then found the manifest moved and did the same: two idle
1416                // writers traded generations (and cell claims) for as long
1417                // as both ran. Theirs becomes the merge base as it stands.
1418                if let Some(handle) = expected.as_ref() {
1419                    if merged.entries == theirs.entries && merged.sole_writer == theirs.sole_writer {
1420                        foreign_entries = foreign;
1421                        foreign_gone = gone;
1422                        installed_nothing = true;
1423                        break (theirs, handle.etag.clone());
1424                    }
1425                }
1426                match manifest::cas_write_stamped(
1427                    self.store.as_ref(),
1428                    &self.cfg,
1429                    &merged,
1430                    expected.as_ref(),
1431                    epoch,
1432                    &flush_uuid,
1433                    Some(source),
1434                )
1435                .await
1436                {
1437                    Ok(meta) => {
1438                        // Before the deletes and before step 7: this is the
1439                        // only record that survives a crash in that window.
1440                        self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1441                            "expected": expected.as_ref().map(|h| h.etag.clone()), "etag": meta.etag, "result": "ok"}));
1442                        intent.installed_etag = Some(meta.etag.clone());
1443                        intent.installed_foreign = foreign_changes(&foreign, &gone);
1444                        self.state.save_intent(&intent)?;
1445                        foreign_entries = foreign;
1446                        foreign_gone = gone;
1447                        break (merged, meta.etag);
1448                    }
1449                    Err(LeanError::Store(StoreError::PreconditionFailed(_)))
1450                    | Err(LeanError::Store(StoreError::Conflict(_))) => {
1451                        self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1452                            "expected": expected.as_ref().map(|h| h.etag.clone()), "result": "lost"}));
1453                        // Re-verify the cell before retrying: a rotation is
1454                        // exactly this 412, and re-merging past it would be
1455                        // the straggler install.
1456                        self.verify_not_deposed().await?;
1457                        continue;
1458                    }
1459                    Err(e) => return Err(e),
1460                }
1461            };
1462            report.seq = Some(installed.seq);
1463            report.no_change = installed_nothing;
1464            // A fused install IS a coherent point, and cadence/hybrid have
1465            // exactly one source. The gauges must not report "no boundary
1466            // ever" on a workspace that publishes every minute. A barrier
1467            // that installed nothing marks no boundary of its own.
1468            if !installed_nothing {
1469                self.note_boundary("cadence", installed.seq)?;
1470            }
1471            report.manifest_etag = Some(installed_etag.clone());
1472            report.observed_seq = Some(installed.seq);
1473            report.observed_etag = Some(installed_etag.clone());
1474            report.foreign_queued = foreign_entries.len();
1475
1476            // Step 6: deletes LAST — GC of keys the NEW manifest no longer
1477            // references, HEAD-guarded on the recognized ETag.
1478            let mut swept = 0usize;
1479            let mut gc_held = false;
1480            for path in &classified.deletes {
1481                // A mass delete is the one long stretch of the commit
1482                // section: keep the token moving so a waiter does not count
1483                // a live holder dead, and fence on what the read returns.
1484                swept += 1;
1485                if swept % 200 == 0 {
1486                    if let Some(cell) = self.renew_if_due().await? {
1487                        self.fence_on_cell(&cell)?;
1488                    }
1489                }
1490                if installed.entries.contains_key(path) {
1491                    // delete/modify resolved foreign-wins: not garbage, and
1492                    // not in this boundary either.
1493                    report.outranked.push(path.clone());
1494                    continue;
1495                }
1496                let key = self.cfg.file_key(path);
1497                let recognized = baseline.entries.get(path).map(|b| b.etag.clone()).or_else(|| {
1498                    // A DECLARED removal of a path this tree never held (a
1499                    // scoped workspace's out-of-scope citation): what the
1500                    // declaration named is the object the manifest cited
1501                    // when this barrier began, and that is the merge base.
1502                    removals
1503                        .declared
1504                        .contains(path)
1505                        .then(|| baseline.inst_base.get(path).cloned())
1506                        .flatten()
1507                });
1508                // The HEAD answers most paths in one request (gone, or an
1509                // etag we do not recognize). The DELETE itself carries
1510                // If-Match on the recognized etag: another writer's
1511                // upload holds no lease and can replace the object
1512                // between the two requests, and an unconditional delete
1513                // then removed bytes that writer's commit cites (the
1514                // model's LeanBarrierLeaseGCUnconditional).
1515                let unrecognized = match self.store.head(&key).await {
1516                    Err(StoreError::NotFound(_)) => {
1517                        self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": null, "result": "absent"}));
1518                        report.deleted.push(path.clone());
1519                        continue;
1520                    }
1521                    Ok(meta) if Some(&meta.etag) == recognized.as_ref() => {
1522                        if self.cfg.drill_hold_gc_secs > 0 && !gc_held {
1523                            gc_held = true;
1524                            eprintln!(
1525                                "flint-sync: DRILL: holding the GC for {}s between the HEAD of {path} and its DELETE",
1526                                self.cfg.drill_hold_gc_secs
1527                            );
1528                            self.trace("drill_hold", serde_json::json!({"flush": flush_uuid, "where": "gc", "path": path,
1529                                "secs": self.cfg.drill_hold_gc_secs}));
1530                            tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_gc_secs)).await;
1531                        }
1532                        match self.store.delete_if_match(&key, &meta.etag).await {
1533                            Ok(()) | Err(StoreError::NotFound(_)) => {
1534                                self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": meta.etag, "result": "deleted"}));
1535                                report.deleted.push(path.clone());
1536                                continue;
1537                            }
1538                            // Replaced between the HEAD and the DELETE:
1539                            // whatever is there now is not ours to
1540                            // collect. Name it, if it still exists.
1541                            Err(StoreError::PreconditionFailed(_)) => match self.store.head(&key).await {
1542                                Ok(now) => now.etag,
1543                                Err(StoreError::NotFound(_)) => {
1544                                    self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": meta.etag, "result": "replaced-absent"}));
1545                                    report.deleted.push(path.clone());
1546                                    continue;
1547                                }
1548                                Err(e) => return Err(e.into()),
1549                            },
1550                            Err(e) => return Err(e.into()),
1551                        }
1552                    }
1553                    Ok(meta) => meta.etag,
1554                    Err(e) => return Err(e.into()),
1555                };
1556                // An ETag this syncer does not recognize is NEVER deleted
1557                // (a HITL re-create landed after our CAS, or another
1558                // writer's upload).
1559                self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": unrecognized,
1560                    "recognized": recognized, "result": "skip"}));
1561                self.state.append_conflict(&ConflictRecord {
1562                    path: path.clone(),
1563                    foreign_etag: unrecognized,
1564                    preserved_key: None,
1565                    kind: "gc-skip".into(),
1566                    at_unix: now_unix(),
1567                })?;
1568            }
1569
1570            // Step 7: the foreign queue, then the baseline rewrite, intent
1571            // clear and window clear.
1572            //
1573            // Other writers' changes go to THIS writer's queue BEFORE the
1574            // merge base below moves past them: a crash between the two
1575            // re-applies them idempotently, where the other order would
1576            // leave a base that claims changes the tree never received.
1577            if !foreign_entries.is_empty() || !foreign_gone.is_empty() {
1578                self.state.queue_foreign(&foreign_changes(&foreign_entries, &foreign_gone))?;
1579                self.trace("queue", serde_json::json!({"flush": flush_uuid, "upserts": foreign_entries.len(), "tombstones": foreign_gone.len()}));
1580            }
1581            for (path, be) in new_baseline_entries {
1582                baseline.entries.insert(path, be);
1583            }
1584            for path in &report.deleted {
1585                baseline.entries.remove(path);
1586            }
1587            baseline.inst_base =
1588                installed.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
1589            baseline.seq = installed.seq;
1590            baseline.manifest_etag = Some(installed_etag);
1591            baseline.prev_scan = scanned.keys().cloned().collect();
1592            self.state.save_baseline(&baseline)?;
1593            self.state.clear_intent_keys()?;
1594            // No `merge-preserved` entries into the SHARED inbox any more:
1595            // they are this writer's, and sit in its local queue.
1596            inbox::clear_window_settling(
1597                self.store.as_ref(),
1598                &self.cfg,
1599                epoch,
1600                &[],
1601                &consumed,
1602                &removals.applied,
1603            )
1604            .await?;
1605            self.trace("window_clear", serde_json::json!({"flush": flush_uuid}));
1606            // Reap superseded generations. Immutable metadata that is never
1607            // collected is a leak that grows by a whole manifest per
1608            // publish, and this also collects the orphan a crash between the
1609            // entries PUT and the pointer CAS leaves behind. Best effort by
1610            // design: a publish that succeeded is not un-done by a failure
1611            // to tidy up after it.
1612            let swept_generations = match manifest::sweep_generations(self.store.as_ref(), &self.cfg).await {
1613                Ok(0) => 0,
1614                Ok(n) => {
1615                    eprintln!("flint-sync: reaped {n} superseded manifest generation(s)");
1616                    n
1617                }
1618                Err(e) => {
1619                    eprintln!("flint-sync: generation sweep: {e}");
1620                    0
1621                }
1622            };
1623            self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "generations", "removed": swept_generations}));
1624            // And the CHUNK reaper, same reason, same best-effort terms.
1625            // `sweep_chunks` returns 0 without a request on a workspace that
1626            // is not chunked, so this costs nothing until the layout moves.
1627            //
1628            // It was written to four model-established rules, guarded by six
1629            // mutation configs, and called from nothing but its own tests —
1630            // so with chunking on, every publish left its superseded chunks
1631            // in the bucket forever. The rules ran in the model and in the
1632            // suite and never once in production. `the_barrier_reaps...`
1633            // below is the test that asks whether this line exists at all.
1634            let swept_chunks = match manifest::sweep_chunks(self.store.as_ref(), &self.cfg).await {
1635                Ok(0) => 0,
1636                Ok(n) => {
1637                    eprintln!("flint-sync: reaped {n} unreferenced manifest chunk(s)");
1638                    n
1639                }
1640                Err(e) => {
1641                    eprintln!("flint-sync: chunk sweep: {e}");
1642                    0
1643                }
1644            };
1645            self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "chunks", "removed": swept_chunks}));
1646            Ok(())
1647        }
1648        .await;
1649        // Hand the cell on whatever happened — unless it is no longer
1650        // ours to hand on: a fence means a successor holds it, and the
1651        // barrier is simply abandoned (its manifest never installed; its
1652        // uploads stand, and the next barrier adopts them by flush_uuid).
1653        match &commit {
1654            Err(LeanError::Fenced(m)) => {
1655                self.trace("fence", serde_json::json!({"where": "commit", "detail": m}));
1656                self.lease = None
1657            }
1658            _ => {
1659                if let Err(e) = super::lease::release(self).await {
1660                    eprintln!(
1661                        "flint-sync: the publish fence could not be released ({e}); a waiter \
1662                         deposes it after the quiet threshold"
1663                    );
1664                }
1665            }
1666        }
1667        commit?;
1668        self.trace_barrier_end(&report, barrier_started);
1669        Ok(report)
1670    }
1671
1672    fn trace_barrier_end(&self, report: &BarrierReport, started: std::time::Instant) {
1673        if self.cfg.event_trace.is_none() {
1674            return;
1675        }
1676        self.trace(
1677            "barrier_end",
1678            serde_json::json!({
1679                "seq": report.seq, "uploaded": report.uploaded.len(), "deleted": report.deleted.len(),
1680                "parked": report.parked.len(), "outranked": report.outranked.len(),
1681                "consumed": report.consumed, "no_change": report.no_change,
1682                "ms": started.elapsed().as_millis() as u64, "requests": self.trace_requests(),
1683            }),
1684        );
1685    }
1686
1687    /// The > whole_put_max path: contiguous `PartSource::Local` chunks
1688    /// through `compose_generation` (streaming multipart; the store
1689    /// aborts its partial assembly on every failure path). The CRC is
1690    /// computed by a streaming pass first; a writer racing the compose
1691    /// fails server-side validation and the path DEFERS to the next
1692    /// barrier — publish-possibly-torn is put_whole's documented
1693    /// dilemma, not this path's.
1694    #[allow(clippy::too_many_arguments)]
1695    #[allow(clippy::too_many_arguments)]
1696    async fn upload_compose(
1697        &self,
1698        path: &str,
1699        key: &str,
1700        local_path: &Path,
1701        size: u64,
1702        scanned: &scan::ScanEntry,
1703        condition: PutCondition,
1704        stamps: GenerationStamps,
1705        generation: u64,
1706        epoch: u64,
1707        prior_uuids: &[String],
1708    ) -> LeanResult<UploadOutcome> {
1709        // NO PRE-PASS. The store accumulates the full-object CRC from
1710        // the parts as it reads them for upload, so this file is read
1711        // ONCE (the 2026-09-10 door drill measured the second read at
1712        // roughly 11 s of the `mixed` workload's 24.3 s).
1713        // Part grid: within [min_part_size, ...], at most max_parts,
1714        // contiguous from 0.
1715        let min_part = self.store.min_part_size().max(1);
1716        let max_parts = self.store.max_parts().max(1) as u64;
1717        let mut chunk = self.cfg.whole_put_max.max(min_part);
1718        let need = size.div_ceil(chunk);
1719        if need > max_parts {
1720            chunk = size.div_ceil(max_parts).div_ceil(min_part) * min_part;
1721        }
1722        let mut parts = vec![];
1723        let mut off = 0u64;
1724        while off < size {
1725            let len = chunk.min(size - off);
1726            parts.push(flint_store::PartSource::Local { offset: off, len });
1727            off += len;
1728        }
1729
1730
1731        // At most three attempts: the caller's condition; then once more
1732        // If-Match on what a 412 showed us — our own torn Complete
1733        // (recognized through the crash journal, review 2026-09-12
1734        // atomicity-2), or a foreign version now PRESERVED (inbox-1) —
1735        // or a create if the base object vanished (inbox-5). A 412 on
1736        // the retry means the path is being written under us: park.
1737        let mut condition = condition;
1738        for attempt in 0..3u32 {
1739            let spec = flint_store::ComposeSpec {
1740                progress: None,
1741                key,
1742                local_path,
1743                parts: parts.clone(),
1744                base_key: None,
1745                base_etag: None,
1746                condition: condition.clone(),
1747                stamps: stamps.clone(),
1748                crc64: None,
1749            };
1750            match self.store.compose_generation(&spec).await {
1751                Ok(meta) => {
1752                    // The store computed it; refuse to cite bytes nothing
1753                    // vouched for rather than invent a number for the
1754                    // manifest. A backend that returns no checksum has not
1755                    // validated the publish server-side either.
1756                    let crc = meta
1757                        .crc64_b64
1758                        .as_deref()
1759                        .and_then(flint_store::crc64_from_b64)
1760                        .ok_or_else(|| {
1761                            LeanError::State(format!(
1762                                "compose of {key} returned no full-object checksum — refusing to \
1763                                 cite bytes nothing vouched for"
1764                            ))
1765                        })?;
1766                    return Ok(UploadOutcome::published(
1767                        path, key.to_string(), meta.etag, crc, size, scanned, generation, epoch,
1768                    ));
1769                }
1770                Err(StoreError::ChecksumMismatch(_)) | Err(StoreError::NoSuchUpload(_)) => {
1771                    // Drift mid-compose, or the operator sweep aborted us:
1772                    // nothing published; the next barrier re-queues.
1773                    return Ok(UploadOutcome::Deferred);
1774                }
1775                // S3's answer to If-Match on a missing key (see upload_one):
1776                // the base vanished, and a vanished base is a create.
1777                Err(StoreError::NotFound(_)) if matches!(condition, PutCondition::IfMatch(_)) => {
1778                    condition = PutCondition::IfNoneMatchAny;
1779                    continue;
1780                }
1781                Err(StoreError::PreconditionFailed(_)) => {
1782                    // The AdoptOwn recognizer compares OUR bytes' checksum
1783                    // against what landed, and without the pre-pass we do
1784                    // not have one — so the file is read HERE, on the rare
1785                    // recovery path, instead of on every publish.
1786                    let crc = file_crc(local_path)?;
1787                    let head = match self.store.head(key).await {
1788                        Ok(h) => h,
1789                        Err(StoreError::NotFound(_)) => {
1790                            condition = PutCondition::IfNoneMatchAny;
1791                            continue;
1792                        }
1793                        Err(e) => return Err(e.into()),
1794                    };
1795                    let head_stamps = GenerationStamps::from_meta(&head.meta);
1796                    if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
1797                        // These bytes are already there (a torn Complete):
1798                        // cite them — re-read under the fence first.
1799                        let g = head_stamps.map(|s| s.generation).unwrap_or(generation);
1800                        return Ok(UploadOutcome::adopted(
1801                            path, key.to_string(), head.etag, crc, head.size, scanned, g, epoch,
1802                        ));
1803                    }
1804                    if attempt >= 1 {
1805                        return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1806                    }
1807                    let own = head_stamps
1808                        .as_ref()
1809                        .map(|s| s.flush_uuid == stamps.flush_uuid || prior_uuids.contains(&s.flush_uuid))
1810                        .unwrap_or(false);
1811                    if !own && self.preserve_foreign_412(path, &head).await.is_none() {
1812                        return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1813                    }
1814                    condition = PutCondition::IfMatch(head.etag);
1815                }
1816                Err(e) => return Err(e.into()),
1817            }
1818        }
1819        Ok(UploadOutcome::Deferred)
1820    }
1821
1822
1823    async fn upload_one(
1824        &self,
1825        path: &str,
1826        scanned: &scan::ScanEntry,
1827        base: Option<&BaselineEntry>,
1828        epoch: u64,
1829        flush_uuid: &str,
1830        prior_uuids: &[String],
1831    ) -> LeanResult<UploadOutcome> {
1832        let key = self.cfg.file_key(path);
1833        let local_path = self.cfg.root.join(path);
1834        let generation = base.map(|b| b.generation + 1).unwrap_or(1);
1835        // Review 2026-09-12, atomicity-6: the scan skipped symlinks; this
1836        // read followed them. A regular file swapped for a symlink between
1837        // the two published the link's TARGET — a file outside the
1838        // workspace, the syncer's own /proc/self/environ included. Every
1839        // stat here is an lstat and the read opens O_NOFOLLOW; a path that
1840        // is no longer a regular file is refused, recorded, and deferred
1841        // (the next scan skips it, and the two-scan rule retires it).
1842        let lmeta = std::fs::symlink_metadata(&local_path)
1843            .map_err(|e| LeanError::State(format!("stat {}: {e}", local_path.display())))?;
1844        if !lmeta.is_file() {
1845            self.state.append_conflict(&ConflictRecord {
1846                path: path.to_string(),
1847                foreign_etag: String::new(),
1848                preserved_key: None,
1849                kind: "upload-refused-not-regular: no longer a regular file after the scan (a symlink \
1850                       or special file replaced it); nothing published"
1851                    .into(),
1852                at_unix: now_unix(),
1853            })?;
1854            return Ok(UploadOutcome::Deferred);
1855        }
1856        let posix = Some(PosixStamps::from_metadata(&lmeta));
1857        let stamps = GenerationStamps {
1858            generation,
1859            epoch,
1860            flush_uuid: flush_uuid.to_string(),
1861            boundary_source: None,
1862            posix,
1863        };
1864        let condition = match base {
1865            Some(b) => PutCondition::IfMatch(b.etag.clone()),
1866            None => PutCondition::IfNoneMatchAny,
1867        };
1868        let size = lmeta.len();
1869        if size > self.cfg.whole_put_max {
1870            // Streaming multipart compose: put_whole is never fed past
1871            // whole_put_max (unbounded memory + S3's 5 GiB wall).
1872            return self
1873                .upload_compose(
1874                    path, &key, &local_path, size, scanned, condition, stamps, generation, epoch,
1875                    prior_uuids,
1876                )
1877                .await;
1878        }
1879        // The upload byte gate — the store's own, shared with the compose
1880        // arm above, which charges its parts inside the store. This body
1881        // is charged from before it is read (the read is the allocation)
1882        // until its PUT, retried or not, has returned: the permit lives
1883        // to the end of this function. `None` = a store with no gate.
1884        let _upload_permit = match self.store.upload_gate() {
1885            Some(g) => Some(g.acquire(size).await),
1886            None => None,
1887        };
1888        // `Bytes::from(Vec<u8>)` TAKES OWNERSHIP without copying, so the
1889        // old `Bytes::from(body.clone())` was a full memcpy of every
1890        // published file body — bought solely to leave `body` intact for
1891        // the 412 retry below, which is reached almost never. Build the
1892        // `Bytes` once; the retry gets a refcount clone.
1893        let body = Bytes::from(read_nofollow(&local_path)?);
1894        let uploaded_len = body.len() as u64;
1895        let crc = crc64_nvme(&body);
1896        match self
1897            .store
1898            .put_whole(&key, body.clone(), &condition, &stamps, crc)
1899            .await
1900        {
1901            Ok(meta) => Ok(UploadOutcome::published(
1902                path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1903            )),
1904            // S3 answers If-Match on a key that no longer exists with 404
1905            // NoSuchKey, not 412 (Ozone answers 412, and so did the
1906            // double): the base is gone — a peer's GC collected it, or a
1907            // bucket-level delete. The same rule as the 412 arm's vanished
1908            // HEAD below: a vanished base is a create. Without this arm the
1909            // rule never ran on S3, and a writer whose edited path a peer
1910            // deleted failed EVERY barrier (drill host leg H2, 2026-09-13).
1911            Err(StoreError::NotFound(_)) if matches!(condition, PutCondition::IfMatch(_)) => {
1912                let meta = self
1913                    .store
1914                    .put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
1915                    .await?;
1916                Ok(UploadOutcome::published(
1917                    path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1918                ))
1919            }
1920            Err(StoreError::PreconditionFailed(_)) => {
1921                // The 412 policy: my own crashed/torn PUT ⇒ adopt; a
1922                // foreign version ⇒ the consume-dirty rule, at upload time
1923                // (preserve it, then supersede it knowingly). NEVER the
1924                // inherited LOCAL-WINS overwrite, and never a park with
1925                // no way out.
1926                let head = match self.store.head(&key).await {
1927                    Ok(h) => h,
1928                    Err(StoreError::NotFound(_)) => {
1929                        // Review 2026-09-12, inbox-5: the base object is
1930                        // gone (a bucket-level delete, or our own GC after
1931                        // a crash between the CAS and the baseline
1932                        // rewrite). Failing the barrier here failed EVERY
1933                        // barrier, forever. A vanished base is a create.
1934                        let meta = self
1935                            .store
1936                            .put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
1937                            .await?;
1938                        return Ok(UploadOutcome::published(
1939                            path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1940                        ));
1941                    }
1942                    Err(e) => return Err(e.into()),
1943                };
1944                let head_stamps = GenerationStamps::from_meta(&head.meta);
1945                let own = head_stamps
1946                    .as_ref()
1947                    .map(|s| s.flush_uuid == flush_uuid || prior_uuids.contains(&s.flush_uuid))
1948                    .unwrap_or(false);
1949                if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
1950                    // Bytes already there (a torn response, ours or a
1951                    // foreign write of the same content): cite it — re-read
1952                    // under the fence first.
1953                    let g = head_stamps.map(|s| s.generation).unwrap_or(generation);
1954                    return Ok(UploadOutcome::adopted(
1955                        path, key, head.etag, crc, head.size, scanned, g, epoch,
1956                    ));
1957                }
1958                if !own && self.preserve_foreign_412(path, &head).await.is_none() {
1959                    return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1960                }
1961                // Our earlier PUT (older content), or a foreign version
1962                // now preserved: supersede it knowingly, If-Match on what
1963                // we saw. A second 412 means the path is being written
1964                // under us right now; that one parks.
1965                let meta = match self
1966                    .store
1967                    .put_whole(&key, body, &PutCondition::IfMatch(head.etag.clone()), &stamps, crc)
1968                    .await
1969                {
1970                    Ok(m) => m,
1971                    // 412 (a writer landed between) or 404 (a GC removed
1972                    // it between the HEAD and this PUT): written under us
1973                    // right now. Park; the next barrier retries.
1974                    Err(StoreError::PreconditionFailed(_)) | Err(StoreError::NotFound(_)) => {
1975                        return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1976                    }
1977                    Err(e) => return Err(e.into()),
1978                };
1979                Ok(UploadOutcome::published(
1980                    path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1981                ))
1982            }
1983            Err(e) => Err(e.into()),
1984        }
1985    }
1986
1987    /// A 412 against a version this syncer did not write, met at upload
1988    /// time. The contract's rule for a foreign write to a path the agent
1989    /// modified is the consume-dirty rule — the agent's version wins and
1990    /// the foreign bytes are preserved — and a "park" was that case with
1991    /// no resolution: nothing ever un-parked, and every ack said `ok`
1992    /// (review 2026-09-12, inbox-1). Preserve, record, and let the caller
1993    /// supersede. `None` = the preserve failed; the caller parks, and the
1994    /// boundary is `partial`.
1995    async fn preserve_foreign_412(&self, path: &str, head: &flint_store::ObjectMeta) -> Option<()> {
1996        let preserved = match self.preserve_conflict_copy(path, &head.etag).await {
1997            Ok(k) => k,
1998            Err(e) => {
1999                eprintln!("flint-sync: {path}: foreign version {} could not be preserved ({e}); parking", head.etag);
2000                return None;
2001            }
2002        };
2003        self.state
2004            .append_conflict(&ConflictRecord {
2005                path: path.to_string(),
2006                foreign_etag: head.etag.clone(),
2007                preserved_key: Some(preserved),
2008                kind: "upload-412-preserved".into(),
2009                at_unix: now_unix(),
2010            })
2011            .ok()?;
2012        Some(())
2013    }
2014}
2015
2016/// Read a workspace file for upload without following a symlink at the
2017/// final component, and refuse anything that is not a regular file once
2018/// open (review 2026-09-12, atomicity-6).
2019fn read_nofollow(p: &Path) -> std::io::Result<Vec<u8>> {
2020    use std::io::Read;
2021    use std::os::unix::fs::OpenOptionsExt;
2022    let mut f = std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NOFOLLOW).open(p)?;
2023    if !f.metadata()?.is_file() {
2024        return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "not a regular file"));
2025    }
2026    let mut v = Vec::new();
2027    f.read_to_end(&mut v)?;
2028    Ok(v)
2029}
2030
2031enum UploadOutcome {
2032    /// `adopted`: the etag was OBSERVED at the key (bytes already there,
2033    /// no PUT of ours produced it), so the commit section must re-read it
2034    /// before citing — see `verify_observed_citations` in the barrier.
2035    Published { entry: LeanEntry, baseline_entry: BaselineEntry, adopted: bool },
2036    Parked { foreign_etag: String },
2037    /// The source drifted mid-transfer (checksum refused server-side)
2038    /// or the assembly was swept: publish nothing, advance nothing —
2039    /// the next scan re-queues the path.
2040    Deferred,
2041}
2042
2043impl UploadOutcome {
2044    #[allow(clippy::too_many_arguments)]
2045    fn published(
2046        path: &str,
2047        key: String,
2048        etag: String,
2049        crc: u64,
2050        uploaded_len: u64,
2051        scanned: &scan::ScanEntry,
2052        generation: u64,
2053        epoch: u64,
2054    ) -> UploadOutcome {
2055        let _ = path;
2056        UploadOutcome::Published {
2057            adopted: false,
2058            entry: LeanEntry {
2059                key,
2060                etag: etag.clone(),
2061                crc64_b64: crc64_to_b64(crc),
2062                // The length the object HAS (review 2026-09-12,
2063                // atomicity-1): the scanned size was cited while the
2064                // body was read fresh, so a file that grew between the
2065                // two was cited short and every fresh checkout of it
2066                // failed its CRC fold — the successor never started.
2067                size: uploaded_len,
2068                mode: scanned.mode,
2069                mtime_unix: scanned.mtime_unix,
2070                generation,
2071                epoch,
2072            },
2073            baseline_entry: BaselineEntry {
2074                etag,
2075                generation,
2076                // The PRE-read stat: if the agent wrote during our read,
2077                // the next scan sees the drift and re-queues (the
2078                // re-stat/re-queue valve).
2079                size: scanned.size,
2080                mtime_unix: scanned.mtime_unix,
2081                mtime_nanos: Some(scanned.mtime_nanos),
2082                crc64_b64: Some(crc64_to_b64(crc)),
2083            },
2084        }
2085    }
2086}
2087
2088impl UploadOutcome {
2089    /// `published`, for bytes this barrier found already at the key and
2090    /// cites without a PUT of its own.
2091    #[allow(clippy::too_many_arguments)]
2092    fn adopted(
2093        path: &str,
2094        key: String,
2095        etag: String,
2096        crc: u64,
2097        uploaded_len: u64,
2098        scanned: &scan::ScanEntry,
2099        generation: u64,
2100        epoch: u64,
2101    ) -> UploadOutcome {
2102        match UploadOutcome::published(path, key, etag, crc, uploaded_len, scanned, generation, epoch) {
2103            UploadOutcome::Published { entry, baseline_entry, .. } => {
2104                UploadOutcome::Published { entry, baseline_entry, adopted: true }
2105            }
2106            other => other,
2107        }
2108    }
2109}
2110
2111fn local_dirty(local: &Path, base: Option<&BaselineEntry>) -> bool {
2112    match (std::fs::metadata(local), base) {
2113        (Err(_), None) => false,                   // both absent: clean
2114        (Err(_), Some(_)) => true,                 // locally deleted vs baseline
2115        (Ok(_), None) => true,                     // local exists, never published
2116        (Ok(m), Some(b)) => {
2117            scan::stat_changed(b.size, b.mtime_unix, b.mtime_nanos, m.len(), mtime_of(&m), mtime_nanos_of(&m))
2118        }
2119    }
2120}
2121
2122/// Full-object CRC-64/NVME of a local file, streamed.
2123///
2124/// Reached only by the 412 recovery path: the ordinary publish gets its
2125/// checksum from the store, which computes one while uploading.
2126/// The CRC a citation repair cites: the bytes this syncer integrated,
2127/// as the baseline recorded them when it wrote or uploaded them. The
2128/// HEAD's own checksum is a cross-check when the backend offers one,
2129/// never the source — Ozone offers none, a HITL uploader may have sent
2130/// none, and the manifest must carry a CRC either way, because every
2131/// reader now verifies a fresh fetch against it.
2132///
2133/// A baseline entry with no CRC is refused loudly rather than cited
2134/// bare: the only entry built without one is the consume-dirty sentinel,
2135/// which the candidate filters exclude, so reaching this is a defect.
2136/// A HEAD that attests a DIFFERENT value is refused the same way — the
2137/// workspace holds bytes the object does not, and citing either would
2138/// make one reader or another refuse the path forever.
2139pub(crate) fn repair_crc(
2140    path: &str,
2141    be: &BaselineEntry,
2142    head: &flint_store::ObjectMeta,
2143) -> LeanResult<String> {
2144    let Some(ours) = be.crc64_b64.clone() else {
2145        return Err(LeanError::State(format!(
2146            "citation repair of {path}: the baseline records no CRC for the bytes it \
2147             integrated at etag {} — refusing to cite bytes nothing hashed",
2148            be.etag
2149        )));
2150    };
2151    if let Some(theirs) = head.crc64_b64.as_deref() {
2152        if theirs != ours {
2153            return Err(LeanError::State(format!(
2154                "citation repair of {path}: the store attests CRC-64 {theirs} for etag {} but \
2155                 the bytes this workspace integrated under that etag hash to {ours} — the \
2156                 workspace holds bytes the object does not; refusing to cite either",
2157                be.etag
2158            )));
2159        }
2160    }
2161    Ok(ours)
2162}
2163
2164fn file_crc(local_path: &Path) -> LeanResult<u64> {
2165    use std::io::Read;
2166    let mut crc = flint_store::Crc64Nvme::new();
2167    let mut f = std::fs::File::open(local_path)?;
2168    let mut buf = vec![0u8; 4 << 20];
2169    loop {
2170        let n = f.read(&mut buf)?;
2171        if n == 0 {
2172            break;
2173        }
2174        crc.update(&buf[..n]);
2175    }
2176    Ok(crc.finalize())
2177}
2178
2179pub(super) fn mtime_nanos_of(m: &std::fs::Metadata) -> u32 {
2180    m.modified()
2181        .ok()
2182        .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2183        .map(|d| d.subsec_nanos())
2184        .unwrap_or(0)
2185}
2186
2187pub(super) fn mtime_of(m: &std::fs::Metadata) -> i64 {
2188    m.modified()
2189        .ok()
2190        .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2191        .map(|d| d.as_secs() as i64)
2192        .unwrap_or(0)
2193}
2194
2195/// Containment-safe workspace write (boundary-verbs plan §2.2 security
2196/// gate). `rel` is a workspace-relative path; the target may not escape
2197/// the root, traverse a symlink component, or land in the reserved
2198/// control namespace.
2199///
2200/// The hazard this closes is pre-existing and reachable from three
2201/// callers (checkout, inbox consume, sync): `write_file_atomic` did
2202/// `create_dir_all(parent)` + write with no `O_NOFOLLOW` and no
2203/// root-containment check, while the scanner SKIPS symlinks — so a
2204/// planted symlink is invisible to the syncer. An unprivileged app
2205/// that plants `inputs -> /root/.aws`, lands an object at
2206/// `inputs/<path>` and drops a scoped sync turns the credential-holding
2207/// syncer into an on-demand arbitrary-file-write primitive outside the
2208/// workspace.
2209pub(super) fn write_file_atomic_in(
2210    root: &Path,
2211    rel: &str,
2212    bytes: &[u8],
2213    mode: Option<u32>,
2214) -> LeanResult<()> {
2215    let target = contained_path(root, rel)?;
2216    write_file_atomic(&target, bytes, mode).map(|_| ())
2217}
2218
2219/// Validate `rel` under `root` WITHOUT creating anything — for callers
2220/// that must decide whether a path is writable before doing any work
2221/// (inbox consume refuses a refused path outright rather than routing it
2222/// through conflict-preserve, which would cost a GET+PUT and leave the
2223/// path in the baseline).
2224pub(super) fn check_contained(root: &Path, rel: &str) -> LeanResult<()> {
2225    resolve_contained(root, rel, false).map(|_| ())
2226}
2227
2228/// Resolve `rel` under `root`, refusing escapes, creating missing parent
2229/// directories. Walks the parent chain component by component: a
2230/// component that EXISTS as a symlink is a refusal (never followed).
2231pub(super) fn contained_path(root: &Path, rel: &str) -> LeanResult<std::path::PathBuf> {
2232    resolve_contained(root, rel, true).map(|(p, _)| p)
2233}
2234
2235/// As `contained_path`, and also what the walk already learned about
2236/// the FINAL component: its `lstat` if it exists, `None` if it does
2237/// not. The walk stats every component to refuse a symlink; a caller
2238/// that then asked `exists()` and `metadata()` paid two more stats for
2239/// an answer it was already holding (checkout's resume check did, on
2240/// every one of 20,000 files).
2241pub(super) fn contained_path_stat(
2242    root: &Path,
2243    rel: &str,
2244) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2245    resolve_contained(root, rel, true)
2246}
2247
2248fn resolve_contained(
2249    root: &Path,
2250    rel: &str,
2251    create_dirs: bool,
2252) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2253    use std::path::Component;
2254    let refuse = |why: &str| {
2255        Err(LeanError::State(format!(
2256            "refusing write to {rel:?}: {why} (containment)"
2257        )))
2258    };
2259    if rel.is_empty() {
2260        return refuse("empty path");
2261    }
2262    if super::scan::is_control_path(rel) {
2263        // The reserved namespace is the syncer's own; a citation or
2264        // inbox entry naming it is surfaced, never materialized (D0.3).
2265        return refuse("reserved control namespace");
2266    }
2267    let relp = Path::new(rel);
2268    let mut cur = root.to_path_buf();
2269    let mut comps: Vec<&std::ffi::OsStr> = vec![];
2270    for c in relp.components() {
2271        match c {
2272            Component::Normal(n) => comps.push(n),
2273            Component::CurDir => {}
2274            Component::ParentDir => return refuse("`..` component"),
2275            Component::RootDir | Component::Prefix(_) => return refuse("absolute path"),
2276        }
2277    }
2278    if comps.is_empty() {
2279        return refuse("no path components");
2280    }
2281    let last = comps.len() - 1;
2282    let mut final_meta = None;
2283    for (i, c) in comps.iter().enumerate() {
2284        cur.push(c);
2285        match std::fs::symlink_metadata(&cur) {
2286            Ok(m) if m.file_type().is_symlink() => {
2287                return refuse("path traverses a symlink");
2288            }
2289            Ok(m) if i < last && !m.is_dir() => {
2290                return refuse("parent component is not a directory");
2291            }
2292            Ok(m) => {
2293                if i == last {
2294                    final_meta = Some(m);
2295                }
2296            }
2297            Err(_) if i < last => {
2298                if create_dirs {
2299                    if let Err(e) = std::fs::create_dir(&cur) {
2300                        // A SIBLING may have created it between the stat
2301                        // above and this mkdir: fetches run on their own
2302                        // tasks, so the first files of one directory
2303                        // race here, and the loser's EEXIST is not a
2304                        // containment refusal. It is fine only if what
2305                        // exists now is a plain directory — a symlink
2306                        // that appeared in the window is refused exactly
2307                        // as one the stat found.
2308                        match std::fs::symlink_metadata(&cur) {
2309                            Ok(m) if m.file_type().is_symlink() => {
2310                                return refuse("path traverses a symlink");
2311                            }
2312                            Ok(m) if m.is_dir() => {}
2313                            _ => {
2314                                return Err(LeanError::State(format!(
2315                                    "mkdir {}: {e}",
2316                                    cur.display()
2317                                )))
2318                            }
2319                        }
2320                    }
2321                }
2322            }
2323            Err(_) => {}
2324        }
2325    }
2326    Ok((cur, final_meta))
2327}
2328
2329/// Write `bytes` to `path` through an exclusive temp sibling and a
2330/// rename. The parent already exists — containment created it — and
2331/// one that vanished since is recreated on the retry path in `safefs`,
2332/// so the common path pays no `mkdir` and no `stat` for it. Returns
2333/// the written file's metadata.
2334pub(super) fn write_file_atomic(
2335    path: &Path,
2336    bytes: &[u8],
2337    mode: Option<u32>,
2338) -> LeanResult<std::fs::Metadata> {
2339    super::safefs::check_parent(path)?;
2340    // NOT with_extension(): that REPLACES the final extension, so
2341    // "a.txt" and "a.md" would collide on one tmp name.
2342    let tmp = path.with_file_name(format!(
2343        "{}.flint-sync-tmp",
2344        path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default()
2345    ));
2346    // The temp sibling is computed AFTER containment ran, so the walk
2347    // never saw it: it needs its own refusal, not a plain write.
2348    // FAST (no per-file fsync): materialisations are made durable by
2349    // `sync_tree` before the marker or baseline that vouches for them.
2350    super::safefs::write_via_tmp_fast(path, &tmp, bytes, mode)
2351}
2352
2353/// The queue form of a merge's foreign upserts and deletions.
2354fn foreign_changes(entries: &[(String, LeanEntry)], gone: &[String]) -> Vec<ForeignChange> {
2355    entries
2356        .iter()
2357        .map(|(path, e)| ForeignChange {
2358            path: path.clone(),
2359            etag: Some(e.etag.clone()),
2360            crc64_b64: Some(e.crc64_b64.clone()),
2361        })
2362        .chain(gone.iter().map(|path| ForeignChange { path: path.clone(), etag: None, crc64_b64: None }))
2363        .collect()
2364}
2365
2366/// One three-way merge of a barrier's changes onto the bucket's manifest.
2367struct MergeOnto {
2368    theirs: manifest::LeanManifest,
2369    /// The CAS handle over `theirs`; `None` when the bucket has no manifest.
2370    expected: Option<manifest::ManifestHandle>,
2371    merged: manifest::LeanManifest,
2372    /// Other writers' changes since the merge base that the tree lacks.
2373    foreign: Vec<(String, LeanEntry)>,
2374    /// Other writers' deletions since the merge base.
2375    gone: Vec<String>,
2376}
2377
2378impl MergeOnto {
2379    /// The merge adds nothing to the document: installing it would be an
2380    /// empty generation.
2381    fn adds_nothing(&self) -> bool {
2382        self.merged.entries == self.theirs.entries && self.merged.sole_writer == self.theirs.sole_writer
2383    }
2384}
2385
2386impl Syncer {
2387    #[allow(clippy::too_many_arguments)]
2388    fn merge_onto(
2389        &self,
2390        current: Option<&manifest::LoadedManifest>,
2391        prev_installed: Option<&str>,
2392        inst_base: &BTreeMap<String, String>,
2393        upserts: &BTreeMap<String, LeanEntry>,
2394        deletes: &BTreeSet<String>,
2395        parked: &BTreeSet<String>,
2396        flush_uuid: &str,
2397    ) -> MergeOnto {
2398        let (theirs, expected) = match current {
2399            // The handle carries the LAYOUT as well as the tag: a
2400            // workspace still on the legacy single object CASes a
2401            // pointer that must not exist yet, not one it read.
2402            Some(l) => (l.manifest.clone(), Some(l.handle())),
2403            None => (Default::default(), None),
2404        };
2405        // If the bucket is still at the document THIS workspace
2406        // installed, that document IS the merge base — whatever the
2407        // persisted one says. Step 7 rewrites the merge base after
2408        // the CAS, so a restart in between leaves it behind a
2409        // document we wrote, and every entry in it would read as
2410        // somebody else's change. See `IntentJournal::installed_etag`.
2411        let own_base: BTreeMap<String, String>;
2412        let base: &BTreeMap<String, String> =
2413            if prev_installed.is_some() && prev_installed == expected.as_ref().map(|h| h.etag.as_str()) {
2414                own_base = theirs.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
2415                &own_base
2416            } else {
2417                inst_base
2418            };
2419        let (mut merged, foreign) = manifest::merge(base, &theirs, upserts, deletes, parked);
2420        // Deletions another writer made since this workspace's
2421        // merge base: in the base, gone from theirs, and not this
2422        // barrier's own upsert, delete or park. `merge` has no use
2423        // for them — theirs already lacks the path — but the TREE
2424        // does: they reach it through the local queue, as the
2425        // foreign upserts do.
2426        let gone: Vec<String> = base
2427            .keys()
2428            .filter(|p| {
2429                !theirs.entries.contains_key(*p)
2430                    && !upserts.contains_key(*p)
2431                    && !deletes.contains(*p)
2432                    && !parked.contains(*p)
2433            })
2434            .cloned()
2435            .collect();
2436        // `merge` clears it; the installing pass owns it. A mirror
2437        // is a property of how this workspace is DEPLOYED, so it
2438        // comes from config on every publish rather than being
2439        // inherited from whatever wrote last.
2440        merged.sole_writer = self.cfg.sole_writer;
2441        self.trace("merge", serde_json::json!({"flush": flush_uuid, "theirs_seq": theirs.seq, "upserts": upserts.len(),
2442            "deletes": deletes.len(), "foreign": foreign.len(), "gone": gone.len(),
2443            "adds_nothing": merged.entries == theirs.entries}));
2444        MergeOnto { theirs, expected, merged, foreign, gone }
2445    }
2446}