flint_lean/barrier.rs
1//! The publish barrier (plan §2.1, seven steps) — the machine
2//! `lean/formal/LeanSubtree.tla` checks. Order is load-bearing:
3//! consume → scan → intent/window → uploads → manifest CAS (merge) →
4//! GC deletes LAST → baseline rewrite.
5
6use std::collections::{BTreeMap, BTreeSet};
7use std::path::Path;
8
9use bytes::Bytes;
10
11use flint_store::{
12 EpochState,
13 crc64_nvme, crc64_to_b64, GenerationStamps, PosixStamps, PutCondition, StoreError,
14};
15
16use super::inbox::{self, InboxDoc, InboxEntry, Refusal, Removal};
17use super::manifest::{self, LeanEntry};
18use super::scan;
19use super::state::{BaselineEntry, ConflictRecord, ForeignChange, IntentJournal};
20use super::{now_unix, LeanError, LeanResult, Syncer};
21
22/// The last gateway request this workspace acted on, per verb. A
23/// watermark rather than a consumed queue — see `note_verb_requests`.
24#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
25pub struct VerbWatermark {
26 #[serde(default)]
27 pub boundary_unix: u64,
28 #[serde(default)]
29 pub sync_unix: u64,
30}
31
32#[derive(Debug, Default)]
33pub struct BarrierReport {
34 pub seq: Option<u64>,
35 pub uploaded: Vec<String>,
36 pub deleted: Vec<String>,
37 /// Paths a DECLARED removal (delete/rename design) cited out this
38 /// barrier — a subset of `deleted` once the GC has run.
39 pub removed: Vec<String>,
40 /// Declared removals this barrier REFUSED (locally dirty, not a
41 /// file, uncontainable); the cell carries the reason.
42 pub removals_refused: usize,
43 pub parked: Vec<String>,
44 pub consumed: usize,
45 pub foreign_queued: usize,
46 /// Paths whose transfer drifted or was swept mid-compose: nothing
47 /// published, nothing advanced; the next scan re-queues them.
48 pub deferred: Vec<String>,
49 /// Local deletes another writer's change outranked (delete/modify
50 /// resolves foreign-wins): the installed seq still cites them, and the
51 /// next consume reconciles each against the local delete.
52 pub outranked: Vec<String>,
53 pub no_change: bool,
54 /// Bytes this barrier actually published — the input to the
55 /// sentinel work meter (boundary-verbs plan D3.1). Metering work
56 /// rather than calls is what keeps a sentinel storm from
57 /// un-coalescing a hot large file's republish: a counted budget
58 /// charges a 2 GiB checkpoint the same one unit as a 4 KiB file.
59 pub published_bytes: u64,
60 /// The manifest ETag this barrier installed (the ack's CAS token).
61 pub manifest_etag: Option<String>,
62 /// What the manifest HEAD/install told us the bucket is at, for the
63 /// news ticker (D5) — free, off requests the barrier already made.
64 pub observed_seq: Option<u64>,
65 pub observed_etag: Option<String>,
66 /// Paths this barrier saw absent for the FIRST time: their deletes
67 /// are withheld to the next scan (the rename-vs-walk guard). On a
68 /// declared boundary a non-empty set means the confirmation lstat
69 /// found them back on disk — the race the guard exists for.
70 pub first_absence: Vec<String>,
71 /// First-absence paths a DECLARED boundary confirmed gone by direct
72 /// lstat and published as ordinary deletes (`confirm_absences`).
73 pub absences_confirmed: usize,
74 /// This barrier finished a rescope a crash had left half-applied
75 /// before it did anything else. Reported so a run that looks like
76 /// an ordinary barrier can be told from one that had to converge a
77 /// workspace first.
78 pub rescope_replayed: bool,
79}
80
81/// What a pass over the cell's DECLARED removals did (`apply_removals`).
82#[derive(Debug, Default)]
83pub struct RemovalPass {
84 /// Performed: the local file is gone (or already was) and the path
85 /// is in `declared`. Dropped from the cell with the window clear —
86 /// AFTER the manifest CAS — so a listing keeps hiding the path
87 /// until the manifest stops citing it.
88 pub applied: Vec<Removal>,
89 /// Refused for good, `refused` filled in: the cell is told at once
90 /// and the removal is never retried.
91 pub refused: Vec<Removal>,
92 /// Left in the cell for the next barrier: a rename whose
93 /// destination is not integrated yet, or an unlink that failed for
94 /// a transient reason.
95 pub deferred: usize,
96 /// The paths this barrier cites OUT: `applied`, plus any a crashed
97 /// earlier barrier had unlinked and journalled.
98 pub declared: BTreeSet<String>,
99}
100
101/// Upload waves per chunk. The chunk is `fanout * this`, so a wave
102/// still saturates fan-out and the sync point between chunks is rare.
103const UPLOAD_CHUNK_WAVES: usize = 16;
104
105/// How stale the lease may get inside a barrier before it is renewed:
106/// comfortably below the 6-poll (~60 s) takeover window, so a holder
107/// busy in a long commit section never reads as quiet to a waiter.
108const RENEW_WITHIN_SECS: u64 = 20;
109
110impl Syncer {
111 /// Verify the cell still names us at OUR epoch; anything else is a
112 /// fence. Read-verify before the manifest CAS (the per-request
113 /// validation the gateway will also enforce).
114 async fn verify_not_deposed(&self) -> LeanResult<()> {
115 let lease = self
116 .lease
117 .as_ref()
118 .ok_or_else(|| LeanError::State("no lease".into()))?;
119 match self.store.epoch_read(&self.cfg.epoch_key()).await? {
120 Some(state) if state.epoch == lease.epoch && state.holder_id == lease.holder_id => {
121 Ok(())
122 }
123 Some(state) => Err(LeanError::Fenced(format!(
124 "cell at epoch {} holder {} (we are epoch {})",
125 state.epoch, state.holder_id, lease.epoch
126 ))),
127 None => Err(LeanError::Fenced("epoch cell vanished".into())),
128 }
129 }
130
131 /// Step 1: consume the inbox. A barrier NEVER runs against an
132 /// unconsumed inbox — this is what makes HITL uploads structurally
133 /// un-amputatable. Returns the entries integrated (dropped from the
134 /// cell at the window-open CAS).
135 pub async fn consume_inbox(&mut self) -> LeanResult<Vec<InboxEntry>> {
136 let loaded = inbox::load(self.store.as_ref(), &self.cfg).await?;
137 self.consume_inbox_doc(&loaded.doc).await
138 }
139
140 /// `consume_inbox` over a cell the caller has already read — the
141 /// barrier reads it once and hands the same document to
142 /// `apply_removals`, so the removal pass adds no request.
143 pub async fn consume_inbox_doc(&mut self, doc: &InboxDoc) -> LeanResult<Vec<InboxEntry>> {
144 Ok(self.consume_counted(doc).await?.0)
145 }
146
147 /// `consume_inbox_doc`, plus how many changes from this writer's LOCAL
148 /// queue it settled. Those leave no entry in the cell to drop, so they
149 /// are not in the returned list — but they are consumptions, and the
150 /// report (and the ack) count them.
151 async fn consume_counted(&mut self, doc: &InboxDoc) -> LeanResult<(Vec<InboxEntry>, usize)> {
152 // §2.5's layered doors ride the inbox GET this function already
153 // pays for: promptness is one tick, and the added request count
154 // is ZERO. A failure here must never fail the consume — the
155 // request is idempotent state and the next tick re-reads it.
156 if let Err(e) = self.note_verb_requests(doc) {
157 eprintln!("flint-sync: verb request not consumed (retrying next tick): {e}");
158 }
159 // Indices into `entries`: the writer-local queue's upserts first,
160 // then the shared inbox's. Both run the same rules below.
161 let mut consumed: Vec<usize> = vec![];
162 let mut baseline = self.state.load_baseline()?;
163 // An install whose step 7 a restart cut off left the other writers'
164 // changes it carried in the intent journal (`installed_foreign`).
165 let requeued = self.state.requeue_installed_foreign()?;
166 if requeued > 0 {
167 self.trace("queue", serde_json::json!({"requeued_from_intent": requeued}));
168 }
169 // The writer-local queue: changes other writers made that this
170 // workspace's own merges carried into the manifest but not yet
171 // into the tree (`state::ForeignChange` says why it is not the
172 // shared inbox any more).
173 let queue = self.state.load_foreign_queue()?;
174 let queued: Vec<InboxEntry> = queue
175 .iter()
176 .filter_map(|c| {
177 c.etag.as_ref().map(|etag| InboxEntry {
178 path: c.path.clone(),
179 etag: etag.clone(),
180 author: "merge-preserved".into(),
181 added_unix: 0,
182 crc64_b64: c.crc64_b64.clone(),
183 })
184 })
185 .collect();
186 let n_queued = queued.len();
187 let entries: Vec<InboxEntry> = queued.into_iter().chain(doc.entries.iter().cloned()).collect();
188 for (idx, entry) in entries.iter().enumerate() {
189 let key = self.cfg.file_key(&entry.path);
190 // Containment BEFORE anything else: a path we could never
191 // safely materialize must be surfaced and dropped, not
192 // routed through the locally-dirty branch — which would
193 // "preserve" it with a GET+PUT and leave a baseline entry
194 // for a path the scanner can never see (the planted-symlink
195 // shape: `inputs -> /root/.aws` reads as locally-present,
196 // therefore dirty, therefore a conflict-preserve of someone
197 // else's file).
198 if let Err(e) = check_contained(&self.cfg.root, &entry.path) {
199 self.state.append_conflict(&ConflictRecord {
200 path: entry.path.clone(),
201 foreign_etag: entry.etag.clone(),
202 preserved_key: None,
203 kind: format!("consume-refused-containment: {e}"),
204 at_unix: now_unix(),
205 })?;
206 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused"}));
207 consumed.push(idx);
208 continue;
209 }
210 // Already integrated (a crashed earlier consume): idempotent.
211 if baseline.entries.get(&entry.path).map(|b| b.etag == entry.etag).unwrap_or(false) {
212 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "already"}));
213 consumed.push(idx);
214 continue;
215 }
216 let head = match self.store.head(&key).await {
217 Ok(m) => m,
218 Err(StoreError::NotFound(_)) => {
219 // The object vanished under a tracked entry: surface
220 // it — the bytes are NOT recoverable from here.
221 self.state.append_conflict(&ConflictRecord {
222 path: entry.path.clone(),
223 foreign_etag: entry.etag.clone(),
224 preserved_key: None,
225 kind: "consume-object-missing".into(),
226 at_unix: now_unix(),
227 })?;
228 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "missing"}));
229 consumed.push(idx);
230 continue;
231 }
232 Err(e) => return Err(e.into()),
233 };
234 if head.etag != entry.etag {
235 // Superseded by a newer write (its own inbox entry
236 // follows, or it is the syncer's): drop.
237 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "superseded"}));
238 consumed.push(idx);
239 continue;
240 }
241 let local_path = self.cfg.root.join(&entry.path);
242 let mut dirty = local_dirty(&local_path, baseline.entries.get(&entry.path));
243 // Review 2026-09-12, atomicity-3 / inbox-2: the fetch is a
244 // network round trip and the agent may write the path inside
245 // it. The stat that licenses the adopt is repeated AFTER the
246 // fetch, before anything touches the file: a write in that
247 // window makes this the dirty case — the foreign bytes are
248 // preserved and a record names them, the agent's stay.
249 let mut fetched = None;
250 if !dirty {
251 let got = self.store.get_whole(&key, Some(&entry.etag)).await.map_err(|e| match e {
252 StoreError::PreconditionFailed(m) => {
253 StoreError::Other(format!("consume raced a newer write: {m}"))
254 }
255 other => other,
256 })?;
257 if local_dirty(&local_path, baseline.entries.get(&entry.path)) {
258 dirty = true;
259 } else {
260 fetched = Some(got);
261 }
262 }
263 if dirty {
264 // Locally-dirty wins; preserve the FOREIGN bytes first
265 // (a conflict record must keep both versions
266 // recoverable), then advance the recognized ETag so our
267 // eventual publish supersedes it KNOWINGLY.
268 let preserved = self.preserve_conflict_copy(&entry.path, &head.etag).await?;
269 self.state.append_conflict(&ConflictRecord {
270 path: entry.path.clone(),
271 foreign_etag: entry.etag.clone(),
272 preserved_key: Some(preserved),
273 kind: "consume-dirty".into(),
274 at_unix: now_unix(),
275 })?;
276 let stamps = GenerationStamps::from_meta(&head.meta);
277 baseline.entries.insert(
278 entry.path.clone(),
279 BaselineEntry {
280 etag: head.etag.clone(),
281 generation: stamps.map(|s| s.generation).unwrap_or(0),
282 // Deliberately NOT the local file's stat: the
283 // path must still scan as dirty so the local
284 // version publishes.
285 size: u64::MAX,
286 mtime_unix: 0,
287 mtime_nanos: None,
288 // The sentinel's bytes are the LOCAL edit; the
289 // publish that supersedes hashes them itself.
290 crc64_b64: None,
291 },
292 );
293 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "dirty-preserved"}));
294 } else {
295 // Clean, re-checked after the fetch: adopt the foreign
296 // content into the tree.
297 let (meta, body) = fetched.expect("fetched while clean");
298 let mode = PosixStamps::from_meta(&meta.meta).map(|p| p.mode);
299 // VERIFIED before it is written, as checkout's fresh
300 // fetch is: against the writer's CRC when the inbox
301 // entry carries one (the gateway hashes what it sent),
302 // else the backend's attestation when it offers one.
303 // Ozone offers none, so the inbox's is what a HITL
304 // upload gets checked against there. What the baseline
305 // records — and the next citation repair cites — is
306 // OURS, over the bytes actually written.
307 let got = crc64_to_b64(crc64_nvme(&body));
308 let want = entry.crc64_b64.clone().or_else(|| meta.crc64_b64.clone());
309 if let Some(want) = want {
310 if want != got {
311 // NOT consumed: the entry stays in the cell and
312 // the record repeats — a permanent, visible
313 // contradiction is the right failure for
314 // bytes nobody vouches for.
315 self.state.append_conflict(&ConflictRecord {
316 path: entry.path.clone(),
317 foreign_etag: entry.etag.clone(),
318 preserved_key: None,
319 kind: format!(
320 "consume-refused-checksum: etag {} is attested with CRC-64 \
321 {want} but the bytes fetched under it hash to {got}",
322 entry.etag
323 ),
324 at_unix: now_unix(),
325 })?;
326 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused-checksum"}));
327 continue;
328 }
329 }
330 // CONTAINMENT and I/O are split, because the answers are
331 // opposite. Containment was already decided above by
332 // `check_contained`, so a refusal reaching here is a
333 // path that only became unsafe once parents were
334 // created — still a permanent property of the path, so
335 // it is surfaced and DROPPED, exactly as above.
336 //
337 // Everything else — ENOSPC, EACCES, EIO, a read-only
338 // filesystem — is TRANSIENT, and dropping the entry for
339 // one is silent permanent loss: the foreign bytes stay
340 // in the bucket, the workspace never adopts them, and
341 // nothing ever re-offers the entry. It used to be
342 // recorded as `consume-refused-containment`, which sent
343 // whoever read it looking for a hostile path that was
344 // never there. A full disk is not a planted symlink.
345 let target = match contained_path(&self.cfg.root, &entry.path) {
346 Ok(t) => t,
347 Err(e) => {
348 self.state.append_conflict(&ConflictRecord {
349 path: entry.path.clone(),
350 foreign_etag: entry.etag.clone(),
351 preserved_key: None,
352 kind: format!("consume-refused-containment: {e}"),
353 at_unix: now_unix(),
354 })?;
355 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused"}));
356 consumed.push(idx);
357 continue;
358 }
359 };
360 if let Err(e) = write_file_atomic(&target, &body, mode) {
361 // NOT consumed: the entry stays in the cell so the
362 // next barrier retries it. A visible conflict record
363 // that repeats is a far better failure than one
364 // silent drop.
365 self.state.append_conflict(&ConflictRecord {
366 path: entry.path.clone(),
367 foreign_etag: entry.etag.clone(),
368 preserved_key: None,
369 kind: format!("consume-write-failed (will retry): {e}"),
370 at_unix: now_unix(),
371 })?;
372 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "deferred"}));
373 continue;
374 }
375 let st = std::fs::metadata(&local_path)?;
376 let stamps = GenerationStamps::from_meta(&meta.meta);
377 baseline.entries.insert(
378 entry.path.clone(),
379 BaselineEntry {
380 etag: meta.etag.clone(),
381 generation: stamps.map(|s| s.generation).unwrap_or(0),
382 size: st.len(),
383 mtime_unix: mtime_of(&st),
384 mtime_nanos: Some(mtime_nanos_of(&st)),
385 crc64_b64: Some(got),
386 },
387 );
388 baseline.prev_scan.insert(entry.path.clone());
389 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "adopted"}));
390 }
391 consumed.push(idx);
392 }
393 let mut settled: BTreeSet<String> =
394 consumed.iter().filter(|i| **i < n_queued).map(|i| entries[*i].path.clone()).collect();
395
396 // A queued DELETION: another writer removed the path and this
397 // workspace's merge cited that. A clean local copy goes; a dirty
398 // one is the agent's newer work and stays — it publishes, and a
399 // modify beats a delete at the merge. Never a declared removal of
400 // our own: the deletion is already in the manifest.
401 for change in queue.iter().filter(|c| c.etag.is_none()) {
402 if check_contained(&self.cfg.root, &change.path).is_err() {
403 // Never materializable here, so nothing to remove.
404 settled.insert(change.path.clone());
405 continue;
406 }
407 let local = self.cfg.root.join(&change.path);
408 // A key that holds an object has SUPERSEDED the deletion: a
409 // newer write — a UI write, another writer's re-create — landed
410 // after the merge that queued it, and the upserts above already
411 // drop an entry whose etag the key no longer holds. Applying it
412 // anyway removes the newer bytes from the tree, which this very
413 // consume may just have adopted — and the window clear then
414 // drops their inbox entry, leaving an acked write cited and
415 // tracked by nothing (found by the formal model with this queue
416 // modelled, 2026-09-15). Only a file still on disk needs the
417 // look: an absent one settles as it always did.
418 if std::fs::symlink_metadata(&local).is_ok() {
419 match self.store.head(&self.cfg.file_key(&change.path)).await {
420 Ok(_) => {
421 self.trace("tombstone", serde_json::json!({"path": change.path, "action": "superseded"}));
422 settled.insert(change.path.clone());
423 continue;
424 }
425 Err(StoreError::NotFound(_)) => {}
426 Err(e) => return Err(e.into()),
427 }
428 }
429 let be = baseline.entries.get(&change.path).cloned();
430 let record = |kind: String| ConflictRecord {
431 path: change.path.clone(),
432 foreign_etag: String::new(),
433 preserved_key: None,
434 kind,
435 at_unix: now_unix(),
436 };
437 match std::fs::symlink_metadata(&local) {
438 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
439 self.trace("tombstone", serde_json::json!({"path": change.path, "action": "absent"}));
440 }
441 Err(e) => {
442 // Unreadable is not absent: keep it queued.
443 self.trace("tombstone", serde_json::json!({"path": change.path, "action": "deferred"}));
444 self.state.append_conflict(&record(format!(
445 "consume-foreign-delete-deferred: cannot stat the path: {e}"
446 )))?;
447 continue;
448 }
449 Ok(m) if !m.is_file() || be.is_none() || local_dirty(&local, be.as_ref()) => {
450 self.state.append_conflict(&record(
451 "consume-foreign-delete-vs-dirty: another writer deleted this path; the \
452 local version has unpublished changes, so it stays and publishes"
453 .into(),
454 ))?;
455 self.trace("tombstone", serde_json::json!({"path": change.path, "action": "kept-dirty"}));
456 settled.insert(change.path.clone());
457 continue;
458 }
459 Ok(_) => {
460 if let Err(e) = std::fs::remove_file(&local) {
461 self.state.append_conflict(&record(format!(
462 "consume-foreign-delete-failed (will retry): {e}"
463 )))?;
464 continue;
465 }
466 // CONFIRM: only NotFound is absence.
467 if !matches!(
468 std::fs::symlink_metadata(&local),
469 Err(ref e) if e.kind() == std::io::ErrorKind::NotFound
470 ) {
471 self.state.append_conflict(&record(
472 "consume-foreign-delete-unconfirmed (will retry)".into(),
473 ))?;
474 continue;
475 }
476 self.trace("tombstone", serde_json::json!({"path": change.path, "action": "removed"}));
477 }
478 }
479 baseline.entries.remove(&change.path);
480 baseline.prev_scan.remove(&change.path);
481 settled.insert(change.path.clone());
482 }
483
484 // The files this baseline vouches for reached the disk before
485 // the baseline does (audit 2026-09-03, finding 9).
486 self.state.sync_tree()?;
487 self.state.save_baseline(&baseline)?;
488 // After the baseline: a crash between the two re-applies a
489 // settled change, and both arms are idempotent (an upsert the
490 // baseline holds is "already integrated"; a deletion with nothing
491 // on disk only drops a baseline entry that is already gone).
492 if !settled.is_empty() {
493 let remaining: Vec<_> =
494 queue.into_iter().filter(|c| !settled.contains(&c.path)).collect();
495 self.state.save_foreign_queue(&remaining)?;
496 }
497 // Only the SHARED inbox's entries leave the cell at the window
498 // clear; the local queue was settled above.
499 let n_settled = settled.len();
500 let shared = consumed.into_iter().filter(|i| *i >= n_queued).map(|i| entries[i].clone()).collect();
501 Ok((shared, n_settled))
502 }
503
504 /// Step 1b: perform the DECLARED removals (delete/rename design
505 /// §4-§6). Runs after `consume_inbox_doc`, so a rename's destination
506 /// is in the tree before its source leaves it — §5: create first,
507 /// removal second; a partial application leaves an EXTRA file,
508 /// never a missing one.
509 ///
510 /// Per removal, in order:
511 /// - the path must be containable, as a consume's must;
512 /// - a rename WAITS until its destination is integrated (in the
513 /// baseline): a consume that deferred the destination defers the
514 /// removal with it;
515 /// - a locally-DIRTY path — unpublished edits, or a file the agent
516 /// created there — is REFUSED, nothing applied, the agent's work
517 /// untouched. The same rule a consume applies to a HITL write
518 /// over dirty bytes, and the reason a declared removal cannot be
519 /// "just a delete". Refused for good: the cell is told, and a
520 /// retry that waited for the agent to publish would delete the
521 /// very edit the refusal protected;
522 /// - a clean file is unlinked, and the unlink is CONFIRMED by lstat
523 /// before the path is declared — an unlink that did not take
524 /// publishes no deletion (`confirm_absences`' rule, kept);
525 /// - a path already absent locally and known to the baseline or the
526 /// merge base is declared as it stands: the agent got there
527 /// first, or this tree never held it (a scoped workspace).
528 ///
529 /// A declared path skips the two-scan guard (§4): that guard
530 /// protects against absence INFERRED by a walk, and a declaration
531 /// is not an inference. That is what lets a rename's two halves
532 /// ride one manifest generation.
533 pub async fn apply_removals(&mut self, doc: &InboxDoc) -> LeanResult<RemovalPass> {
534 let mut pass = RemovalPass::default();
535 let baseline = self.state.load_baseline()?;
536 // A crashed earlier barrier that had unlinked and journalled but
537 // not installed: its declarations are still deletions with a
538 // recorded basis — unless the agent has since put a file back.
539 for p in self.state.load_intent()?.declared_deletes {
540 let gone = matches!(
541 std::fs::symlink_metadata(self.cfg.root.join(&p)),
542 Err(ref e) if e.kind() == std::io::ErrorKind::NotFound
543 );
544 if gone && (baseline.entries.contains_key(&p) || baseline.inst_base.contains_key(&p)) {
545 pass.declared.insert(p);
546 }
547 }
548 for r in &doc.removals {
549 if r.refused.is_some() {
550 continue;
551 }
552 let refusal = |kind: &str, message: String| Removal {
553 refused: Some(Refusal { kind: kind.into(), message, at_unix: now_unix() }),
554 ..r.clone()
555 };
556 let mut refuse = |kind: &str, message: String| -> LeanResult<()> {
557 self.state.append_conflict(&ConflictRecord {
558 path: r.path.clone(),
559 foreign_etag: String::new(),
560 preserved_key: None,
561 kind: format!("{kind}: {message}"),
562 at_unix: now_unix(),
563 })?;
564 pass.refused.push(refusal(kind, message));
565 Ok(())
566 };
567 if let Err(e) = check_contained(&self.cfg.root, &r.path) {
568 refuse("removal-refused-containment", e.to_string())?;
569 continue;
570 }
571 if let Some(dst) = &r.moved_to {
572 match baseline.entries.get(dst) {
573 // Not integrated yet (a consume deferred it): wait.
574 None => {
575 pass.deferred += 1;
576 continue;
577 }
578 // Integrated as a CONFLICT: the agent had an
579 // unpublished file at the destination, its version
580 // won, and the moved bytes are preserved under
581 // `conflicts/`. Deleting the source now would leave
582 // the user's file nowhere in the tree — so the move
583 // is refused and the source stays. The sentinel is
584 // the consume-dirty baseline entry's `u64::MAX`.
585 Some(b) if b.size == u64::MAX => {
586 refuse(
587 "removal-refused-destination-conflict",
588 format!(
589 "{} was not moved to {dst}: the agent has an unpublished file at the destination and it wins; the moved bytes are preserved under the conflict record and the source is kept",
590 r.path
591 ),
592 )?;
593 continue;
594 }
595 Some(_) => {}
596 }
597 }
598 let local = self.cfg.root.join(&r.path);
599 let be = baseline.entries.get(&r.path);
600 match std::fs::symlink_metadata(&local) {
601 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
602 if be.is_some() || baseline.inst_base.contains_key(&r.path) {
603 pass.declared.insert(r.path.clone());
604 }
605 // Nothing here and nothing cited: a removal of a path
606 // that was never there, or that a sibling already
607 // removed. Applied as a no-op.
608 pass.applied.push(r.clone());
609 }
610 Err(e) => {
611 // Unreadable is not absent (`confirm_absences`' rule):
612 // fail closed, and say so where the pod-side reader
613 // looks. Transient, so deferred rather than refused.
614 self.state.append_conflict(&ConflictRecord {
615 path: r.path.clone(),
616 foreign_etag: String::new(),
617 preserved_key: None,
618 kind: format!("removal-deferred: cannot stat the path: {e}"),
619 at_unix: now_unix(),
620 })?;
621 pass.deferred += 1;
622 }
623 Ok(m) if !m.is_file() => {
624 refuse(
625 "removal-refused-not-a-file",
626 format!("{} is not a regular file in the workspace", r.path),
627 )?;
628 }
629 Ok(_) => {
630 if local_dirty(&local, be) {
631 refuse(
632 "removal-refused-dirty",
633 format!(
634 "{} has unpublished local changes; the removal {} asked for was not applied and the agent's work is kept — publish or discard the local version, then ask again",
635 r.path, r.author
636 ),
637 )?;
638 continue;
639 }
640 if let Err(e) = std::fs::remove_file(&local) {
641 self.state.append_conflict(&ConflictRecord {
642 path: r.path.clone(),
643 foreign_etag: String::new(),
644 preserved_key: None,
645 kind: format!("removal-unlink-failed (will retry): {e}"),
646 at_unix: now_unix(),
647 })?;
648 pass.deferred += 1;
649 continue;
650 }
651 // CONFIRM: only NotFound is absence.
652 match std::fs::symlink_metadata(&local) {
653 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
654 pass.declared.insert(r.path.clone());
655 pass.applied.push(r.clone());
656 }
657 _ => {
658 self.state.append_conflict(&ConflictRecord {
659 path: r.path.clone(),
660 foreign_etag: String::new(),
661 preserved_key: None,
662 kind: "removal-unlink-unconfirmed (will retry)".into(),
663 at_unix: now_unix(),
664 })?;
665 pass.deferred += 1;
666 }
667 }
668 }
669 }
670 }
671 Ok(pass)
672 }
673
674 /// Act on the inbox document's two request fields (§2.5, D14).
675 ///
676 /// The asymmetry is deliberate and is about blast radius, not
677 /// principle: a boundary request is PERFORMED (it publishes what is
678 /// already on disk and mutates nothing local), a sync request is
679 /// CARRIED into the ticker as advisory news and the agent decides.
680 ///
681 /// Both are watermarked by `requested_unix` rather than consumed by
682 /// a clearing CAS. The field is idempotent state, so a repeat is a
683 /// no-op and a lost clear cannot double-publish; and not writing
684 /// means the doors cost no request at all, which is what §2.5
685 /// promises.
686 fn note_verb_requests(&mut self, doc: &inbox::InboxDoc) -> LeanResult<()> {
687 let mut mark = self.load_verb_watermark()?;
688 if let Some(r) = &doc.sync_request {
689 if r.requested_unix > mark.sync_unix {
690 self.carry_sync_request(r.requested_unix, &r.requestor)?;
691 mark.sync_unix = r.requested_unix;
692 }
693 }
694 if let Some(r) = &doc.boundary_request {
695 if r.requested_unix > mark.boundary_unix {
696 self.request_boundary(
697 &format!("gw:{}:{}", r.requested_unix, r.requestor),
698 Some(format!("boundary requested by {}", r.requestor)),
699 )?;
700 mark.boundary_unix = r.requested_unix;
701 }
702 }
703 self.save_verb_watermark(&mark)
704 }
705
706 fn verb_watermark_path(&self) -> std::path::PathBuf {
707 self.cfg.state_dir().join("verb-requests.json")
708 }
709
710 pub(crate) fn load_verb_watermark(&self) -> LeanResult<VerbWatermark> {
711 let p = self.verb_watermark_path();
712 if !p.exists() {
713 return Ok(VerbWatermark::default());
714 }
715 Ok(std::fs::read(&p)
716 .ok()
717 .and_then(|b| serde_json::from_slice(&b).ok())
718 .unwrap_or_default())
719 }
720
721 fn save_verb_watermark(&self, m: &VerbWatermark) -> LeanResult<()> {
722 let bytes = serde_json::to_vec_pretty(m)
723 .map_err(|e| LeanError::State(format!("verb watermark: {e}")))?;
724 super::control::write_atomic(&self.verb_watermark_path(), &bytes)
725 }
726
727 async fn preserve_conflict_copy(&self, path: &str, etag: &str) -> LeanResult<String> {
728 let key = self.cfg.file_key(path);
729 let dst = self.cfg.conflict_key(&uuid::Uuid::new_v4().to_string(), path);
730 // v1: client-side copy (GET + guarded PUT). The server-side
731 // CopyObject lever is the designed v2 optimization for large
732 // files.
733 let (meta, body) = self.store.get_whole(&key, Some(etag)).await?;
734 let crc = crc64_nvme(&body);
735 let stamps = GenerationStamps {
736 generation: 0,
737 epoch: self
738 .lease
739 .as_ref()
740 .map(|l| l.epoch)
741 .or_else(|| self.state.load_incarnation().ok().flatten().map(|i| i.epoch))
742 .unwrap_or(0),
743 flush_uuid: "conflict-preserve".into(),
744 boundary_source: None,
745 posix: PosixStamps::from_meta(&meta.meta),
746 };
747 self.store
748 .put_whole(&dst, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
749 .await?;
750 Ok(dst)
751 }
752
753 /// Take the SECOND absence observation now, by direct `lstat`.
754 ///
755 /// `scan::classify` withholds a path's delete until absence has
756 /// survived two consecutive scans — the rename-vs-walk race guard
757 /// (`lib.rs:37`). The hazard that rule names is the WALK missing a
758 /// file renamed under it, not deletion itself. On the cadence path
759 /// the second walk arrives at the next floor tick and nobody has
760 /// been promised otherwise. On a DECLARED boundary it cannot wait:
761 /// the ack would claim a coherent point (D1 — "everything
762 /// ordered-before T") while the manifest still cites a file the
763 /// agent removed before it touched the sentinel, and it would say
764 /// so with `report.deleted: 0` and `status: "ok"`.
765 ///
766 /// A direct `lstat` is exactly what the rename-vs-walk guard asks
767 /// for and is immune to the walk race by construction, so the
768 /// second observation costs one syscall per transiently-absent
769 /// path — never a second full pass. The cadence path is unchanged
770 /// and still waits for the second walk.
771 pub(crate) fn confirm_absences(&self, classified: &mut scan::Classified) -> LeanResult<usize> {
772 if classified.first_absence.is_empty() {
773 return Ok(0);
774 }
775 let mut confirmed = 0;
776 for path in std::mem::take(&mut classified.first_absence) {
777 // ONLY NotFound is absence. Every other errno — EACCES on a
778 // parent, EIO, EMFILE, ELOOP — used to read as "the agent
779 // deleted it", and this is the oracle that PUBLISHES the
780 // delete and then DELETES THE OBJECT. An unreadable file is
781 // the one case where guessing is unrecoverable, so it fails
782 // CLOSED, exactly as the walk does at `scan.rs`'s
783 // `symlink_metadata(&path)?`. Two call sites, one syscall:
784 // they must not have opposite error policies.
785 let gone = match std::fs::symlink_metadata(self.cfg.root.join(&path)) {
786 Err(e) if e.kind() == std::io::ErrorKind::NotFound => true,
787 Err(e) => {
788 classified.first_absence.insert(path.clone());
789 return Err(LeanError::State(format!(
790 "cannot confirm the absence of {path:?}: {e} — refusing to publish \
791 a deletion on an unreadable path"
792 )));
793 }
794 Ok(_) => false,
795 };
796 if gone {
797 classified.deletes.insert(path);
798 confirmed += 1;
799 } else {
800 // Back on disk: the rename-vs-walk race, caught exactly
801 // as the rule intends. Still only a first absence.
802 classified.first_absence.insert(path);
803 }
804 }
805 Ok(confirmed)
806 }
807
808 /// Renew the lease if it has gone stale MID-BARRIER, and hand back
809 /// the cell as it was read so the CALLER can fence on it.
810 ///
811 /// Costs nothing on a short barrier: the renewal is skipped unless
812 /// it was already due, so the request count is unchanged — this
813 /// moves WHEN the renewal can happen, not how often.
814 ///
815 /// What this does NOT do, stated because the previous comment
816 /// claimed it did: it never raises `Fenced` for a deposed holder.
817 /// `lease::renew` can only 412 when the cell is still at OUR epoch
818 /// with a token we do not hold; a cell at a FOREIGN epoch is
819 /// skipped here (renewing it would be nonsense), so a deposed
820 /// straggler passed straight through. The 2026-09-03 audit found
821 /// the cadence barrier relying on exactly that phantom fence
822 /// between upload chunks and completing every remaining data PUT
823 /// after a takeover. The cell is returned so the caller can compare
824 /// it against its lease — the cadence barrier uses the read it
825 /// already paid for rather than a separate fence. `None` when there is
826 /// no lease or the read failed: neither is a fence, and the
827 /// ordinary renewal arm will try again.
828 pub(crate) async fn renew_if_due(&mut self) -> LeanResult<Option<EpochState>> {
829 let Some(lease) = self.lease.clone() else { return Ok(None) };
830 let state = match self.store.epoch_read(&self.cfg.epoch_key()).await {
831 Ok(Some(s)) => s,
832 _ => return Ok(None),
833 };
834 let now = super::now_unix();
835 let fresh = state
836 .last_renew_unix
837 .map(|t| now.saturating_sub(t) < RENEW_WITHIN_SECS)
838 .unwrap_or(false);
839 if fresh || state.epoch != lease.epoch {
840 // Not due, or not ours to renew. The caller decides what a
841 // foreign cell means; this function only knows it cannot
842 // renew one.
843 return Ok(Some(state));
844 }
845 super::lease::renew(self).await?;
846 Ok(Some(state))
847 }
848
849 /// The between-chunk fence of the cadence barrier: a cell that no
850 /// longer names this holder at this epoch stops the upload set
851 /// HERE, before the next chunk's PUTs. Every one of those PUTs
852 /// carries If-Match on a baseline etag that still matches — the
853 /// successor has published nothing yet — so without this a deposed
854 /// straggler overwrites the cited generation of every key it still
855 /// had to upload, and every reader's S3-wins arm then adopts the
856 /// uncited bytes silently (audit 2026-09-03, finding 1).
857 fn fence_on_cell(&self, cell: &EpochState) -> LeanResult<()> {
858 let Some(lease) = self.lease.as_ref() else {
859 return Err(LeanError::Fenced("lease dropped mid-barrier".into()));
860 };
861 if cell.epoch != lease.epoch || cell.holder_id != lease.holder_id {
862 return Err(LeanError::Fenced(format!(
863 "deposed between upload chunks: cell at epoch {} holder {} (we are epoch {})",
864 cell.epoch, cell.holder_id, lease.epoch
865 )));
866 }
867 Ok(())
868 }
869
870 /// Steps 2–7. `barrier` = one full publish cycle (the cadence arm).
871 pub async fn run_barrier(&mut self) -> LeanResult<BarrierReport> {
872 self.barrier_inner(false, "manual").await
873 }
874
875 /// The floor tick's barrier. Stamped `cadence` so the bucket can
876 /// answer "which clock installed this boundary?" without the ack —
877 /// the ack is a LOCAL file, and the fleet-visible question is asked
878 /// by operators and by the gateway's `/status`, neither of which
879 /// can see it.
880 pub async fn cadence_barrier(&mut self) -> LeanResult<BarrierReport> {
881 self.barrier_inner(false, "cadence").await
882 }
883
884 /// A barrier whose result somebody is going to ACK: a sentinel
885 /// honor (D1) or the preStop drain (D10). Identical to
886 /// `run_barrier` except that it confirms first-absence paths rather
887 /// than acking a boundary that withholds them.
888 pub async fn declared_barrier(&mut self) -> LeanResult<BarrierReport> {
889 self.barrier_inner(true, "sentinel").await
890 }
891
892 /// `declared_barrier` under an explicit provenance stamp — the
893 /// drain and the budget-deferred honor are the same barrier
894 /// reporting a different clock.
895 pub async fn declared_barrier_as(&mut self, source: &str) -> LeanResult<BarrierReport> {
896 self.barrier_inner(true, source).await
897 }
898
899 async fn barrier_inner(&mut self, declared: bool, source: &str) -> LeanResult<BarrierReport> {
900 // Every publishing path ends here — cadence, sentinel, drain, the
901 // one-shot verb — so a reader is refused here before the first
902 // request, whichever of them a caller reached. The callers that
903 // owe an answer (the publish sentinel, the drain) give it before
904 // this; this is the backstop for the ones that do not.
905 self.refuse_if_read("barrier")?;
906 // No lease is held until the commit section (design 2026-09-13
907 // §4): the consume, the scan and the uploads below run with the
908 // cell at rest, guarded by each object's own etag. What the
909 // uploads are STAMPED with is the last epoch this incarnation
910 // held — informational, like every epoch stamp on an object; the
911 // manifest entries carry the commit section's real epoch.
912 let epoch_hint = self.state.load_incarnation()?.map(|i| i.epoch).unwrap_or(0);
913 let mut report = BarrierReport::default();
914 let barrier_started = std::time::Instant::now();
915 self.trace("barrier_start", serde_json::json!({"source": source, "declared": declared}));
916
917 // STEP 0: finish any rescope a crash left half-applied, before
918 // the scan reads the tree (scoped-read design §4.3). This is the
919 // gate that makes the narrow verb safe at all: a half-applied
920 // narrow is a tree whose files and whose citations disagree, and
921 // `classify` reads that disagreement as an upload or as a
922 // DELETE depending on which half landed. Replay is idempotent,
923 // so the cost on the normal path is one `stat` of a file that
924 // is not there.
925 if let Some(r) = self.replay_scope_intent().await? {
926 report.rescope_replayed = true;
927 eprintln!(
928 "flint-sync: finished an interrupted rescope before the barrier \
929 (uncited {}, unlinked {}, materialised {})",
930 r.uncited, r.unlinked, r.materialized
931 );
932 }
933
934 // Step 1: the cell, read ONCE — HITL entries into the tree, then
935 // the DECLARED removals (delete/rename design §4-§6).
936 let inbox_doc = inbox::load(self.store.as_ref(), &self.cfg).await?.doc;
937 let (consumed, settled_locally) = self.consume_counted(&inbox_doc).await?;
938 report.consumed = consumed.len() + settled_locally;
939 let removals = self.apply_removals(&inbox_doc).await?;
940 report.removed = removals.declared.iter().cloned().collect();
941 report.removals_refused = removals.refused.len();
942 // A refusal is settled NOW, whatever else this barrier does: the
943 // cell is where the human who asked reads the answer, and a
944 // refused removal is never retried, so saying so before the
945 // no-diff return below loses nothing.
946 inbox::settle_removals(self.store.as_ref(), &self.cfg, epoch_hint, &[], &removals.refused)
947 .await?;
948
949 // Step 2: scan-diff against the persisted baseline.
950 let mut baseline = self.state.load_baseline()?;
951 let scanned = scan::scan(&self.cfg.root)?;
952 let mut classified = scan::classify(&scanned, &baseline);
953 if declared {
954 report.absences_confirmed = self.confirm_absences(&mut classified)?;
955 }
956 // A declared removal is a deletion with its basis RECORDED, so
957 // it skips the two-scan guard the walk needs and goes straight
958 // into the delete set — which is what lets a rename's two halves
959 // ride ONE manifest generation (§4).
960 for p in &removals.declared {
961 classified.first_absence.remove(p);
962 if classified.uploads.contains(p) {
963 // The agent re-created the path between the unlink and
964 // the scan. The declaration named the OLD file, which is
965 // gone from the tree (and, for a rename, moved); what is
966 // here now is a new file the agent wrote, and it
967 // publishes as one — its PUT supersedes the old object
968 // under the same key. Deleting it would un-cite the
969 // agent's work for a barrier and GC-skip its object; the
970 // model's install gives the upload the same precedence.
971 continue;
972 }
973 classified.deletes.insert(p.clone());
974 }
975 report.first_absence = classified.first_absence.iter().cloned().collect();
976
977 // Skip-on-no-diff: nothing local, nothing consumed, no pending
978 // citation repair, and the bucket manifest where we left it.
979 let repairs_pending = baseline
980 .entries
981 .iter()
982 .any(|(p, be)| be.size != u64::MAX && baseline.inst_base.get(p) != Some(&be.etag));
983 if classified.uploads.is_empty()
984 && classified.deletes.is_empty()
985 && consumed.is_empty()
986 && classified.first_absence.is_empty()
987 && removals.applied.is_empty()
988 && !repairs_pending
989 {
990 // Read the POINTER, never the entries: the 0b rig measured
991 // the idle tick at 1M entries as 27 s / 1.3 GiB — dominated
992 // by fetching and parsing a 264 MiB document just to read
993 // `seq`. The pointer is a few hundred bytes and carries the
994 // seq in its body, so a GET of it costs one round trip and
995 // answers strictly more than a HEAD of the old object did.
996 //
997 // Legacy workspaces (no pointer yet) keep the HEAD they had.
998 let unchanged = match manifest::load_pointer(self.store.as_ref(), &self.cfg).await? {
999 Some(manifest::LoadedPointer { pointer: p, etag, .. }) => {
1000 // The news ticker rides this request for free — D5's
1001 // "zero added bucket requests" is still literal.
1002 report.observed_seq = Some(p.seq);
1003 report.observed_etag = Some(etag.clone());
1004 baseline.manifest_etag.as_deref() == Some(etag.as_str())
1005 }
1006 None => match self.store.head(&self.cfg.manifest_key()).await {
1007 Ok(meta) => {
1008 report.observed_seq =
1009 GenerationStamps::from_meta(&meta.meta).map(|s| s.generation);
1010 report.observed_etag = Some(meta.etag.clone());
1011 baseline.manifest_etag.as_deref() == Some(meta.etag.as_str())
1012 }
1013 Err(StoreError::NotFound(_)) => baseline.manifest_etag.is_none(),
1014 Err(e) => return Err(e.into()),
1015 },
1016 };
1017 if unchanged {
1018 report.no_change = true;
1019 report.seq = Some(baseline.seq);
1020 // prev_scan still advances (the two-scan rule's clock) —
1021 // but the baseline document (hundreds of MB at 1M
1022 // entries) is rewritten only when the scan SET moved.
1023 // Compare without building the set: both sides iterate
1024 // sorted, and the allocation this used to make was the
1025 // size of the very document it was trying not to write.
1026 if scanned.len() != baseline.prev_scan.len()
1027 || !scanned.keys().eq(baseline.prev_scan.iter())
1028 {
1029 baseline.prev_scan = scanned.keys().cloned().collect();
1030 self.state.save_baseline(&baseline)?;
1031 }
1032 self.trace_barrier_end(&report, barrier_started);
1033 return Ok(report);
1034 }
1035 }
1036
1037 // Step 3: intent journal, then the window (the commitment
1038 // point: the same CAS drops the consumed entries).
1039 let flush_uuid = uuid::Uuid::new_v4().to_string();
1040 self.trace("scan", serde_json::json!({"flush": flush_uuid, "uploads": classified.uploads.len(),
1041 "deletes": classified.deletes.len(), "first_absence": classified.first_absence.len()}));
1042 let mut intent = self.state.load_intent()?;
1043 let prior_uuids = {
1044 let mut v = intent.recent_uuids.clone();
1045 if !intent.flush_uuid.is_empty() {
1046 v.push(intent.flush_uuid.clone());
1047 }
1048 v
1049 };
1050 // Carried forward, not dropped: it stays true until we install
1051 // again, and the merge below is what reads it.
1052 let prev_installed = intent.installed_etag.clone();
1053 intent = IntentJournal {
1054 flush_uuid: flush_uuid.clone(),
1055 keys: classified.uploads.iter().map(|p| self.cfg.file_key(p)).collect(),
1056 recent_uuids: prior_uuids.clone(),
1057 installed_etag: prev_installed.clone(),
1058 installed_foreign: intent.installed_foreign.clone(),
1059 declared_deletes: removals.declared.iter().cloned().collect(),
1060 };
1061 self.state.save_intent(&intent)?;
1062 // The window is NOT opened here any more. It is the bucket's
1063 // "a barrier is committing" sign for HITL writers, and it opens
1064 // in the commit section, under the lease, at the epoch the
1065 // commit really holds. The uploads below race a HITL write the
1066 // way two writers race each other: If-Match decides, the loser
1067 // is preserved and recorded (`LeanNoWindowHolds` proves safety
1068 // never depended on the window).
1069 //
1070 // The consumed entries are not dropped here either — "durably
1071 // in the baseline" is true of a container restart and false of
1072 // a pod REPLACEMENT: the emptyDir goes with the pod. They leave
1073 // the cell with the window clear, after the manifest cites
1074 // them; a successor that finds them re-consumes, idempotently.
1075
1076 // Step 4: guarded uploads, fanned out under a bounded window
1077 // (each key's guard chain is independent; the 412 policy and
1078 // conflict records are applied to the collected results below,
1079 // in deterministic path order).
1080 let mut upserts: BTreeMap<String, LeanEntry> = BTreeMap::new();
1081 let mut parked: BTreeSet<String> = BTreeSet::new();
1082 // Citations whose etag was observed with no lease held (adopted
1083 // uploads, citation repairs): re-read inside the commit section.
1084 let mut observed: BTreeSet<String> = BTreeSet::new();
1085 let mut new_baseline_entries: BTreeMap<String, BaselineEntry> = BTreeMap::new();
1086 let outcomes: Vec<(String, LeanResult<UploadOutcome>)> = {
1087 use futures::stream::{self, StreamExt};
1088 // CHUNKED into waves, so a wave's outcomes are collected
1089 // before the next is fanned out. No lease is held here
1090 // (design 2026-09-13 §4), so nothing renews or fences
1091 // between chunks any more — the between-chunk fence that
1092 // stopped a deposed straggler's PUTs went with the straggler:
1093 // a PUT outside the lease is one legitimate writer's PUT,
1094 // guarded by If-Match, and a collision is preserved and
1095 // superseded, never silently overwritten.
1096 let mut outcomes: Vec<(String, LeanResult<UploadOutcome>)> = Vec::new();
1097 // NOT cfg.fanout. That knob was raised to 128 for the READ
1098 // path, and the read-side measurement (2.5x on 20k small
1099 // files) says nothing about this one, so uploads keep the
1100 // value they were measured at until someone measures them.
1101 // Memory is no longer the reason they are separate: this
1102 // path is byte-bounded too now, by the store's gate
1103 // (`with_upload_inflight_max_bytes`), which charges every
1104 // whole body below and every multipart part inside the
1105 // store from before its read until its PUT returns.
1106 let fanout = self.cfg.upload_fanout.max(1);
1107 let chunk = fanout.saturating_mul(UPLOAD_CHUNK_WAVES).max(1);
1108 let all: Vec<&String> = classified.uploads.iter().collect();
1109 for group in all.chunks(chunk) {
1110 {
1111 let this: &Syncer = &*self;
1112 let mut part: Vec<(String, LeanResult<UploadOutcome>)> =
1113 stream::iter(group.iter().map(|path| {
1114 let path = *path;
1115 let scanned_entry = &scanned[path];
1116 let base = baseline.entries.get(path);
1117 let flush_uuid = &flush_uuid;
1118 let prior_uuids = &prior_uuids;
1119 async move {
1120 let r = this
1121 .upload_one(
1122 path, scanned_entry, base, epoch_hint, flush_uuid,
1123 prior_uuids,
1124 )
1125 .await;
1126 (path.clone(), r)
1127 }
1128 }))
1129 .buffer_unordered(fanout)
1130 .collect()
1131 .await;
1132 outcomes.append(&mut part);
1133 }
1134 }
1135 outcomes
1136 };
1137 let mut outcomes = outcomes;
1138 outcomes.sort_by(|a, b| a.0.cmp(&b.0));
1139 for (path, outcome) in outcomes {
1140 let path = &path;
1141 match outcome? {
1142 UploadOutcome::Published { entry, baseline_entry, adopted } => {
1143 self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "etag": entry.etag,
1144 "outcome": if adopted { "adopted" } else { "put" }}));
1145 if adopted {
1146 observed.insert(path.clone());
1147 }
1148 report.published_bytes += entry.size;
1149 upserts.insert(path.clone(), entry);
1150 new_baseline_entries.insert(path.clone(), baseline_entry);
1151 report.uploaded.push(path.clone());
1152 }
1153 UploadOutcome::Parked { foreign_etag } => {
1154 self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "etag": foreign_etag, "outcome": "parked"}));
1155 parked.insert(path.clone());
1156 self.state.append_conflict(&ConflictRecord {
1157 path: path.clone(),
1158 foreign_etag,
1159 preserved_key: None, // parking preserves in place
1160 kind: "upload-412-parked".into(),
1161 at_unix: now_unix(),
1162 })?;
1163 report.parked.push(path.clone());
1164 }
1165 UploadOutcome::Deferred => {
1166 self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "outcome": "deferred"}));
1167 report.deferred.push(path.clone());
1168 }
1169 }
1170 }
1171
1172 // Citation repairs: paths whose integrated object (the
1173 // baseline) differs from the manifest's citation — consumed
1174 // HITL adoptions and checkout's S3-wins arm. No bytes move; the
1175 // manifest re-cites what this syncer already integrated.
1176 // Without this, an adopted upload is clean-vs-baseline, never
1177 // enters the upload set, and the manifest silently drops it —
1178 // the battery's amputation leg caught exactly that.
1179 let repair_candidates: Vec<String> = baseline
1180 .entries
1181 .iter()
1182 .filter(|(path, be)| {
1183 !classified.uploads.contains(*path)
1184 && !classified.deletes.contains(*path)
1185 && be.size != u64::MAX // consume-dirty sentinel: publishes via upload
1186 && baseline.inst_base.get(*path) != Some(&be.etag)
1187 })
1188 .map(|(path, _)| path.clone())
1189 .collect();
1190 for path in repair_candidates {
1191 let key = self.cfg.file_key(&path);
1192 let be = baseline.entries[&path].clone();
1193 match self.store.head(&key).await {
1194 Ok(meta) if meta.etag == be.etag => {
1195 let crc = repair_crc(&path, &be, &meta)?;
1196 let stamps = GenerationStamps::from_meta(&meta.meta);
1197 let scan_entry = scanned.get(&path);
1198 upserts.insert(
1199 path.clone(),
1200 LeanEntry {
1201 key,
1202 etag: meta.etag.clone(),
1203 crc64_b64: crc,
1204 size: meta.size,
1205 mode: stamps
1206 .as_ref()
1207 .and_then(|s| s.posix)
1208 .map(|p| p.mode)
1209 .or(scan_entry.map(|s| s.mode))
1210 .unwrap_or(0o644),
1211 mtime_unix: scan_entry.map(|s| s.mtime_unix).unwrap_or(0),
1212 generation: stamps.map(|s| s.generation).unwrap_or(be.generation),
1213 epoch: epoch_hint,
1214 },
1215 );
1216 observed.insert(path.clone());
1217 }
1218 // Moved again or gone: the next consume reconciles it.
1219 _ => {}
1220 }
1221 }
1222
1223 // A PULL-ONLY boundary: nothing uploaded, deleted, consumed, removed
1224 // or re-cited, so the merge can only add nothing — and then the
1225 // commit section writes nothing to the bucket (no CAS, no GC, no
1226 // inbox entry to drop) and what remains is local: queue the other
1227 // writers' changes and take theirs as the merge base. That needs no
1228 // fence and no window; claiming for it queued every such writer
1229 // behind the publishing ones (65 of 191 claims in the writers drill).
1230 // A merge that does add something (a mirror flag to restamp, say)
1231 // falls through to the commit section.
1232 if classified.uploads.is_empty()
1233 && classified.deletes.is_empty()
1234 && upserts.is_empty()
1235 && consumed.is_empty()
1236 && removals.applied.is_empty()
1237 && observed.is_empty()
1238 {
1239 let current = manifest::load(self.store.as_ref(), &self.cfg).await?;
1240 let m = self.merge_onto(
1241 current.as_ref(),
1242 prev_installed.as_deref(),
1243 &baseline.inst_base,
1244 &upserts,
1245 &classified.deletes,
1246 &parked,
1247 &flush_uuid,
1248 );
1249 if let Some(handle) = m.expected.as_ref().filter(|_| m.adds_nothing()) {
1250 if !m.foreign.is_empty() || !m.gone.is_empty() {
1251 self.state.queue_foreign(&foreign_changes(&m.foreign, &m.gone))?;
1252 self.trace("queue", serde_json::json!({"flush": flush_uuid, "upserts": m.foreign.len(), "tombstones": m.gone.len(), "fence": false}));
1253 }
1254 report.seq = Some(m.theirs.seq);
1255 report.no_change = true;
1256 report.manifest_etag = Some(handle.etag.clone());
1257 report.observed_seq = Some(m.theirs.seq);
1258 report.observed_etag = Some(handle.etag.clone());
1259 report.foreign_queued = m.foreign.len();
1260 baseline.inst_base = m.theirs.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
1261 baseline.seq = m.theirs.seq;
1262 baseline.manifest_etag = Some(handle.etag.clone());
1263 baseline.prev_scan = scanned.keys().cloned().collect();
1264 self.state.save_baseline(&baseline)?;
1265 self.state.clear_intent_keys()?;
1266 self.trace_barrier_end(&report, barrier_started);
1267 return Ok(report);
1268 }
1269 }
1270
1271 // THE COMMIT SECTION (design 2026-09-13 §4). Everything above ran
1272 // with no lease: the uploads are durable and etag-guarded, and
1273 // nothing is cited yet. Claim the cell now — for the merge, the
1274 // CAS, the deletes and the baseline — and hand it to the next
1275 // waiter when done. This is the only stretch two writers of one
1276 // workspace serialise on, and it is small: one pointer CAS and
1277 // the guarded deletes, milliseconds to seconds, never the upload
1278 // of a checkpoint.
1279 let held = super::lease::claim(self).await?;
1280 let epoch = held.epoch;
1281 eprintln!("flint-sync: publish fence held (epoch {epoch})");
1282 if self.cfg.drill_hold_commit_secs > 0 {
1283 eprintln!(
1284 "flint-sync: DRILL: holding the fence for {}s inside the commit section",
1285 self.cfg.drill_hold_commit_secs
1286 );
1287 tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_commit_secs)).await;
1288 }
1289 let commit: LeanResult<()> = async {
1290 // The entries carry the epoch the commit HOLDS, not the hint
1291 // the uploads were stamped with.
1292 for e in upserts.values_mut() {
1293 e.epoch = epoch;
1294 }
1295 // Step 5: the manifest CAS (three-way merge; bounded retries).
1296 self.verify_not_deposed().await?;
1297 // EVERY citation this barrier adds was read or written with no
1298 // lease held. Another writer's commit may since have uncited
1299 // the path and collected the object: its GC deletes by the etag
1300 // it integrated. That etag names an ADOPTED object (the model's
1301 // LeanBarrierLeaseAdoptBlind) — and, since an S3 whole-PUT etag
1302 // is the MD5 of the bytes, it also names this barrier's own PUT
1303 // of IDENTICAL bytes (finding 13, the writers drill's A3: a
1304 // same-content rewrite re-uploaded, the peer's GC deleted it,
1305 // the commit cited a hole). Collection runs only inside a
1306 // commit section, so a re-read HERE, holding the fence, cannot
1307 // be overtaken before this CAS. Whatever is gone or replaced is
1308 // withheld: parked, recorded, and left dirty for the next
1309 // barrier to publish with a PUT of its own.
1310 let heads: Vec<(String, LeanResult<bool>)> = {
1311 use futures::stream::{self, StreamExt};
1312 let this: &Syncer = &*self;
1313 stream::iter(upserts.iter().map(|(path, cited)| async move {
1314 let still = match this.store.head(&cited.key).await {
1315 Ok(meta) => Ok(meta.etag == cited.etag),
1316 Err(StoreError::NotFound(_)) => Ok(false),
1317 Err(e) => Err(e.into()),
1318 };
1319 (path.clone(), still)
1320 }))
1321 .buffer_unordered(self.cfg.upload_fanout.max(1))
1322 .collect()
1323 .await
1324 };
1325 let mut heads = heads;
1326 heads.sort_by(|a, b| a.0.cmp(&b.0));
1327 for (path, still) in heads {
1328 let still_there = still?;
1329 let was_observed = observed.contains(&path);
1330 let cited = &upserts[&path];
1331 self.trace("observed", serde_json::json!({"flush": flush_uuid, "path": path, "etag": cited.etag, "still": still_there, "own_put": !was_observed}));
1332 if still_there {
1333 continue;
1334 }
1335 let gone = upserts.remove(&path).expect("looked up above");
1336 new_baseline_entries.remove(&path);
1337 report.uploaded.retain(|p| p != &path);
1338 report.published_bytes = report.published_bytes.saturating_sub(gone.size);
1339 // A citation REPAIR re-cites bytes the tree already holds (a
1340 // consumed UI write, checkout's S3-wins arm): the baseline
1341 // names this etag and the scan sees the path clean, so no
1342 // later barrier publishes it. Parking it kept the merge from
1343 // queueing the version that replaced it while the merge base
1344 // moved past that version, and the tree kept the withheld
1345 // bytes for good (storm drill S0, 2026-09-15: one writer of
1346 // six diverged). Not parked, the merge queues the manifest's
1347 // version and the next consume installs it.
1348 let integrated = was_observed
1349 && baseline
1350 .entries
1351 .get(&path)
1352 .is_some_and(|be| be.size != u64::MAX && be.etag == gone.etag);
1353 if integrated {
1354 self.state.append_conflict(&ConflictRecord {
1355 path: path.clone(),
1356 foreign_etag: gone.etag,
1357 preserved_key: None,
1358 kind: "repair-withheld: the object this barrier re-cited was replaced or \
1359 collected before its commit; nothing cited, and the next consume \
1360 installs the manifest's version"
1361 .into(),
1362 at_unix: now_unix(),
1363 })?;
1364 continue;
1365 }
1366 parked.insert(path.clone());
1367 let kind = if was_observed {
1368 "adopt-withheld: the object this barrier found already at its key was \
1369 replaced or collected before its commit; nothing cited, the path stays \
1370 dirty and publishes next barrier"
1371 } else {
1372 "upload-withheld: the object this barrier PUT was replaced or collected \
1373 before its commit (a peer's GC recognizes the etag of identical bytes); \
1374 nothing cited, the path stays dirty and publishes next barrier"
1375 };
1376 self.state.append_conflict(&ConflictRecord {
1377 path: path.clone(),
1378 foreign_etag: gone.etag,
1379 preserved_key: None,
1380 kind: kind.into(),
1381 at_unix: now_unix(),
1382 })?;
1383 report.parked.push(path.clone());
1384 }
1385 // The window: the bucket-visible "a barrier is committing" sign
1386 // HITL writers wait on, at the epoch this commit holds.
1387 let deadline = now_unix() + self.cfg.window_slack_secs;
1388 inbox::open_window(self.store.as_ref(), &self.cfg, epoch, deadline).await?;
1389 let mut foreign_entries: Vec<(String, LeanEntry)> = vec![];
1390 let mut foreign_gone: Vec<String> = vec![];
1391 // Set when the merge added nothing to the document: nothing is
1392 // installed, and theirs becomes the merge base as it stands.
1393 let mut installed_nothing = false;
1394 let mut attempt = 0;
1395 let (installed, installed_etag) = loop {
1396 attempt += 1;
1397 if attempt > 4 {
1398 return Err(LeanError::State(
1399 "manifest CAS lost 4 merge races — refusing this barrier".into(),
1400 ));
1401 }
1402 let current = manifest::load(self.store.as_ref(), &self.cfg).await?;
1403 let MergeOnto { theirs, expected, merged, foreign, gone } = self.merge_onto(
1404 current.as_ref(),
1405 prev_installed.as_deref(),
1406 &baseline.inst_base,
1407 &upserts,
1408 &classified.deletes,
1409 &parked,
1410 &flush_uuid,
1411 );
1412 // Nothing of ours changes the document — a barrier that only
1413 // found the manifest moved by another writer. Installing it
1414 // anyway was an empty generation, and the peer's next tick
1415 // then found the manifest moved and did the same: two idle
1416 // writers traded generations (and cell claims) for as long
1417 // as both ran. Theirs becomes the merge base as it stands.
1418 if let Some(handle) = expected.as_ref() {
1419 if merged.entries == theirs.entries && merged.sole_writer == theirs.sole_writer {
1420 foreign_entries = foreign;
1421 foreign_gone = gone;
1422 installed_nothing = true;
1423 break (theirs, handle.etag.clone());
1424 }
1425 }
1426 match manifest::cas_write_stamped(
1427 self.store.as_ref(),
1428 &self.cfg,
1429 &merged,
1430 expected.as_ref(),
1431 epoch,
1432 &flush_uuid,
1433 Some(source),
1434 )
1435 .await
1436 {
1437 Ok(meta) => {
1438 // Before the deletes and before step 7: this is the
1439 // only record that survives a crash in that window.
1440 self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1441 "expected": expected.as_ref().map(|h| h.etag.clone()), "etag": meta.etag, "result": "ok"}));
1442 intent.installed_etag = Some(meta.etag.clone());
1443 intent.installed_foreign = foreign_changes(&foreign, &gone);
1444 self.state.save_intent(&intent)?;
1445 foreign_entries = foreign;
1446 foreign_gone = gone;
1447 break (merged, meta.etag);
1448 }
1449 Err(LeanError::Store(StoreError::PreconditionFailed(_)))
1450 | Err(LeanError::Store(StoreError::Conflict(_))) => {
1451 self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1452 "expected": expected.as_ref().map(|h| h.etag.clone()), "result": "lost"}));
1453 // Re-verify the cell before retrying: a rotation is
1454 // exactly this 412, and re-merging past it would be
1455 // the straggler install.
1456 self.verify_not_deposed().await?;
1457 continue;
1458 }
1459 Err(e) => return Err(e),
1460 }
1461 };
1462 report.seq = Some(installed.seq);
1463 report.no_change = installed_nothing;
1464 // A fused install IS a coherent point, and cadence/hybrid have
1465 // exactly one source. The gauges must not report "no boundary
1466 // ever" on a workspace that publishes every minute. A barrier
1467 // that installed nothing marks no boundary of its own.
1468 if !installed_nothing {
1469 self.note_boundary("cadence", installed.seq)?;
1470 }
1471 report.manifest_etag = Some(installed_etag.clone());
1472 report.observed_seq = Some(installed.seq);
1473 report.observed_etag = Some(installed_etag.clone());
1474 report.foreign_queued = foreign_entries.len();
1475
1476 // Step 6: deletes LAST — GC of keys the NEW manifest no longer
1477 // references, HEAD-guarded on the recognized ETag.
1478 let mut swept = 0usize;
1479 let mut gc_held = false;
1480 for path in &classified.deletes {
1481 // A mass delete is the one long stretch of the commit
1482 // section: keep the token moving so a waiter does not count
1483 // a live holder dead, and fence on what the read returns.
1484 swept += 1;
1485 if swept % 200 == 0 {
1486 if let Some(cell) = self.renew_if_due().await? {
1487 self.fence_on_cell(&cell)?;
1488 }
1489 }
1490 if installed.entries.contains_key(path) {
1491 // delete/modify resolved foreign-wins: not garbage, and
1492 // not in this boundary either.
1493 report.outranked.push(path.clone());
1494 continue;
1495 }
1496 let key = self.cfg.file_key(path);
1497 let recognized = baseline.entries.get(path).map(|b| b.etag.clone()).or_else(|| {
1498 // A DECLARED removal of a path this tree never held (a
1499 // scoped workspace's out-of-scope citation): what the
1500 // declaration named is the object the manifest cited
1501 // when this barrier began, and that is the merge base.
1502 removals
1503 .declared
1504 .contains(path)
1505 .then(|| baseline.inst_base.get(path).cloned())
1506 .flatten()
1507 });
1508 // The HEAD answers most paths in one request (gone, or an
1509 // etag we do not recognize). The DELETE itself carries
1510 // If-Match on the recognized etag: another writer's
1511 // upload holds no lease and can replace the object
1512 // between the two requests, and an unconditional delete
1513 // then removed bytes that writer's commit cites (the
1514 // model's LeanBarrierLeaseGCUnconditional).
1515 let unrecognized = match self.store.head(&key).await {
1516 Err(StoreError::NotFound(_)) => {
1517 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": null, "result": "absent"}));
1518 report.deleted.push(path.clone());
1519 continue;
1520 }
1521 Ok(meta) if Some(&meta.etag) == recognized.as_ref() => {
1522 if self.cfg.drill_hold_gc_secs > 0 && !gc_held {
1523 gc_held = true;
1524 eprintln!(
1525 "flint-sync: DRILL: holding the GC for {}s between the HEAD of {path} and its DELETE",
1526 self.cfg.drill_hold_gc_secs
1527 );
1528 self.trace("drill_hold", serde_json::json!({"flush": flush_uuid, "where": "gc", "path": path,
1529 "secs": self.cfg.drill_hold_gc_secs}));
1530 tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_gc_secs)).await;
1531 }
1532 match self.store.delete_if_match(&key, &meta.etag).await {
1533 Ok(()) | Err(StoreError::NotFound(_)) => {
1534 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": meta.etag, "result": "deleted"}));
1535 report.deleted.push(path.clone());
1536 continue;
1537 }
1538 // Replaced between the HEAD and the DELETE:
1539 // whatever is there now is not ours to
1540 // collect. Name it, if it still exists.
1541 Err(StoreError::PreconditionFailed(_)) => match self.store.head(&key).await {
1542 Ok(now) => now.etag,
1543 Err(StoreError::NotFound(_)) => {
1544 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": meta.etag, "result": "replaced-absent"}));
1545 report.deleted.push(path.clone());
1546 continue;
1547 }
1548 Err(e) => return Err(e.into()),
1549 },
1550 Err(e) => return Err(e.into()),
1551 }
1552 }
1553 Ok(meta) => meta.etag,
1554 Err(e) => return Err(e.into()),
1555 };
1556 // An ETag this syncer does not recognize is NEVER deleted
1557 // (a HITL re-create landed after our CAS, or another
1558 // writer's upload).
1559 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": unrecognized,
1560 "recognized": recognized, "result": "skip"}));
1561 self.state.append_conflict(&ConflictRecord {
1562 path: path.clone(),
1563 foreign_etag: unrecognized,
1564 preserved_key: None,
1565 kind: "gc-skip".into(),
1566 at_unix: now_unix(),
1567 })?;
1568 }
1569
1570 // Step 7: the foreign queue, then the baseline rewrite, intent
1571 // clear and window clear.
1572 //
1573 // Other writers' changes go to THIS writer's queue BEFORE the
1574 // merge base below moves past them: a crash between the two
1575 // re-applies them idempotently, where the other order would
1576 // leave a base that claims changes the tree never received.
1577 if !foreign_entries.is_empty() || !foreign_gone.is_empty() {
1578 self.state.queue_foreign(&foreign_changes(&foreign_entries, &foreign_gone))?;
1579 self.trace("queue", serde_json::json!({"flush": flush_uuid, "upserts": foreign_entries.len(), "tombstones": foreign_gone.len()}));
1580 }
1581 for (path, be) in new_baseline_entries {
1582 baseline.entries.insert(path, be);
1583 }
1584 for path in &report.deleted {
1585 baseline.entries.remove(path);
1586 }
1587 baseline.inst_base =
1588 installed.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
1589 baseline.seq = installed.seq;
1590 baseline.manifest_etag = Some(installed_etag);
1591 baseline.prev_scan = scanned.keys().cloned().collect();
1592 self.state.save_baseline(&baseline)?;
1593 self.state.clear_intent_keys()?;
1594 // No `merge-preserved` entries into the SHARED inbox any more:
1595 // they are this writer's, and sit in its local queue.
1596 inbox::clear_window_settling(
1597 self.store.as_ref(),
1598 &self.cfg,
1599 epoch,
1600 &[],
1601 &consumed,
1602 &removals.applied,
1603 )
1604 .await?;
1605 self.trace("window_clear", serde_json::json!({"flush": flush_uuid}));
1606 // Reap superseded generations. Immutable metadata that is never
1607 // collected is a leak that grows by a whole manifest per
1608 // publish, and this also collects the orphan a crash between the
1609 // entries PUT and the pointer CAS leaves behind. Best effort by
1610 // design: a publish that succeeded is not un-done by a failure
1611 // to tidy up after it.
1612 let swept_generations = match manifest::sweep_generations(self.store.as_ref(), &self.cfg).await {
1613 Ok(0) => 0,
1614 Ok(n) => {
1615 eprintln!("flint-sync: reaped {n} superseded manifest generation(s)");
1616 n
1617 }
1618 Err(e) => {
1619 eprintln!("flint-sync: generation sweep: {e}");
1620 0
1621 }
1622 };
1623 self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "generations", "removed": swept_generations}));
1624 // And the CHUNK reaper, same reason, same best-effort terms.
1625 // `sweep_chunks` returns 0 without a request on a workspace that
1626 // is not chunked, so this costs nothing until the layout moves.
1627 //
1628 // It was written to four model-established rules, guarded by six
1629 // mutation configs, and called from nothing but its own tests —
1630 // so with chunking on, every publish left its superseded chunks
1631 // in the bucket forever. The rules ran in the model and in the
1632 // suite and never once in production. `the_barrier_reaps...`
1633 // below is the test that asks whether this line exists at all.
1634 let swept_chunks = match manifest::sweep_chunks(self.store.as_ref(), &self.cfg).await {
1635 Ok(0) => 0,
1636 Ok(n) => {
1637 eprintln!("flint-sync: reaped {n} unreferenced manifest chunk(s)");
1638 n
1639 }
1640 Err(e) => {
1641 eprintln!("flint-sync: chunk sweep: {e}");
1642 0
1643 }
1644 };
1645 self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "chunks", "removed": swept_chunks}));
1646 Ok(())
1647 }
1648 .await;
1649 // Hand the cell on whatever happened — unless it is no longer
1650 // ours to hand on: a fence means a successor holds it, and the
1651 // barrier is simply abandoned (its manifest never installed; its
1652 // uploads stand, and the next barrier adopts them by flush_uuid).
1653 match &commit {
1654 Err(LeanError::Fenced(m)) => {
1655 self.trace("fence", serde_json::json!({"where": "commit", "detail": m}));
1656 self.lease = None
1657 }
1658 _ => {
1659 if let Err(e) = super::lease::release(self).await {
1660 eprintln!(
1661 "flint-sync: the publish fence could not be released ({e}); a waiter \
1662 deposes it after the quiet threshold"
1663 );
1664 }
1665 }
1666 }
1667 commit?;
1668 self.trace_barrier_end(&report, barrier_started);
1669 Ok(report)
1670 }
1671
1672 fn trace_barrier_end(&self, report: &BarrierReport, started: std::time::Instant) {
1673 if self.cfg.event_trace.is_none() {
1674 return;
1675 }
1676 self.trace(
1677 "barrier_end",
1678 serde_json::json!({
1679 "seq": report.seq, "uploaded": report.uploaded.len(), "deleted": report.deleted.len(),
1680 "parked": report.parked.len(), "outranked": report.outranked.len(),
1681 "consumed": report.consumed, "no_change": report.no_change,
1682 "ms": started.elapsed().as_millis() as u64, "requests": self.trace_requests(),
1683 }),
1684 );
1685 }
1686
1687 /// The > whole_put_max path: contiguous `PartSource::Local` chunks
1688 /// through `compose_generation` (streaming multipart; the store
1689 /// aborts its partial assembly on every failure path). The CRC is
1690 /// computed by a streaming pass first; a writer racing the compose
1691 /// fails server-side validation and the path DEFERS to the next
1692 /// barrier — publish-possibly-torn is put_whole's documented
1693 /// dilemma, not this path's.
1694 #[allow(clippy::too_many_arguments)]
1695 #[allow(clippy::too_many_arguments)]
1696 async fn upload_compose(
1697 &self,
1698 path: &str,
1699 key: &str,
1700 local_path: &Path,
1701 size: u64,
1702 scanned: &scan::ScanEntry,
1703 condition: PutCondition,
1704 stamps: GenerationStamps,
1705 generation: u64,
1706 epoch: u64,
1707 prior_uuids: &[String],
1708 ) -> LeanResult<UploadOutcome> {
1709 // NO PRE-PASS. The store accumulates the full-object CRC from
1710 // the parts as it reads them for upload, so this file is read
1711 // ONCE (the 2026-09-10 door drill measured the second read at
1712 // roughly 11 s of the `mixed` workload's 24.3 s).
1713 // Part grid: within [min_part_size, ...], at most max_parts,
1714 // contiguous from 0.
1715 let min_part = self.store.min_part_size().max(1);
1716 let max_parts = self.store.max_parts().max(1) as u64;
1717 let mut chunk = self.cfg.whole_put_max.max(min_part);
1718 let need = size.div_ceil(chunk);
1719 if need > max_parts {
1720 chunk = size.div_ceil(max_parts).div_ceil(min_part) * min_part;
1721 }
1722 let mut parts = vec![];
1723 let mut off = 0u64;
1724 while off < size {
1725 let len = chunk.min(size - off);
1726 parts.push(flint_store::PartSource::Local { offset: off, len });
1727 off += len;
1728 }
1729
1730
1731 // At most three attempts: the caller's condition; then once more
1732 // If-Match on what a 412 showed us — our own torn Complete
1733 // (recognized through the crash journal, review 2026-09-12
1734 // atomicity-2), or a foreign version now PRESERVED (inbox-1) —
1735 // or a create if the base object vanished (inbox-5). A 412 on
1736 // the retry means the path is being written under us: park.
1737 let mut condition = condition;
1738 for attempt in 0..3u32 {
1739 let spec = flint_store::ComposeSpec {
1740 progress: None,
1741 key,
1742 local_path,
1743 parts: parts.clone(),
1744 base_key: None,
1745 base_etag: None,
1746 condition: condition.clone(),
1747 stamps: stamps.clone(),
1748 crc64: None,
1749 };
1750 match self.store.compose_generation(&spec).await {
1751 Ok(meta) => {
1752 // The store computed it; refuse to cite bytes nothing
1753 // vouched for rather than invent a number for the
1754 // manifest. A backend that returns no checksum has not
1755 // validated the publish server-side either.
1756 let crc = meta
1757 .crc64_b64
1758 .as_deref()
1759 .and_then(flint_store::crc64_from_b64)
1760 .ok_or_else(|| {
1761 LeanError::State(format!(
1762 "compose of {key} returned no full-object checksum — refusing to \
1763 cite bytes nothing vouched for"
1764 ))
1765 })?;
1766 return Ok(UploadOutcome::published(
1767 path, key.to_string(), meta.etag, crc, size, scanned, generation, epoch,
1768 ));
1769 }
1770 Err(StoreError::ChecksumMismatch(_)) | Err(StoreError::NoSuchUpload(_)) => {
1771 // Drift mid-compose, or the operator sweep aborted us:
1772 // nothing published; the next barrier re-queues.
1773 return Ok(UploadOutcome::Deferred);
1774 }
1775 // S3's answer to If-Match on a missing key (see upload_one):
1776 // the base vanished, and a vanished base is a create.
1777 Err(StoreError::NotFound(_)) if matches!(condition, PutCondition::IfMatch(_)) => {
1778 condition = PutCondition::IfNoneMatchAny;
1779 continue;
1780 }
1781 Err(StoreError::PreconditionFailed(_)) => {
1782 // The AdoptOwn recognizer compares OUR bytes' checksum
1783 // against what landed, and without the pre-pass we do
1784 // not have one — so the file is read HERE, on the rare
1785 // recovery path, instead of on every publish.
1786 let crc = file_crc(local_path)?;
1787 let head = match self.store.head(key).await {
1788 Ok(h) => h,
1789 Err(StoreError::NotFound(_)) => {
1790 condition = PutCondition::IfNoneMatchAny;
1791 continue;
1792 }
1793 Err(e) => return Err(e.into()),
1794 };
1795 let head_stamps = GenerationStamps::from_meta(&head.meta);
1796 if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
1797 // These bytes are already there (a torn Complete):
1798 // cite them — re-read under the fence first.
1799 let g = head_stamps.map(|s| s.generation).unwrap_or(generation);
1800 return Ok(UploadOutcome::adopted(
1801 path, key.to_string(), head.etag, crc, head.size, scanned, g, epoch,
1802 ));
1803 }
1804 if attempt >= 1 {
1805 return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1806 }
1807 let own = head_stamps
1808 .as_ref()
1809 .map(|s| s.flush_uuid == stamps.flush_uuid || prior_uuids.contains(&s.flush_uuid))
1810 .unwrap_or(false);
1811 if !own && self.preserve_foreign_412(path, &head).await.is_none() {
1812 return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1813 }
1814 condition = PutCondition::IfMatch(head.etag);
1815 }
1816 Err(e) => return Err(e.into()),
1817 }
1818 }
1819 Ok(UploadOutcome::Deferred)
1820 }
1821
1822
1823 async fn upload_one(
1824 &self,
1825 path: &str,
1826 scanned: &scan::ScanEntry,
1827 base: Option<&BaselineEntry>,
1828 epoch: u64,
1829 flush_uuid: &str,
1830 prior_uuids: &[String],
1831 ) -> LeanResult<UploadOutcome> {
1832 let key = self.cfg.file_key(path);
1833 let local_path = self.cfg.root.join(path);
1834 let generation = base.map(|b| b.generation + 1).unwrap_or(1);
1835 // Review 2026-09-12, atomicity-6: the scan skipped symlinks; this
1836 // read followed them. A regular file swapped for a symlink between
1837 // the two published the link's TARGET — a file outside the
1838 // workspace, the syncer's own /proc/self/environ included. Every
1839 // stat here is an lstat and the read opens O_NOFOLLOW; a path that
1840 // is no longer a regular file is refused, recorded, and deferred
1841 // (the next scan skips it, and the two-scan rule retires it).
1842 let lmeta = std::fs::symlink_metadata(&local_path)
1843 .map_err(|e| LeanError::State(format!("stat {}: {e}", local_path.display())))?;
1844 if !lmeta.is_file() {
1845 self.state.append_conflict(&ConflictRecord {
1846 path: path.to_string(),
1847 foreign_etag: String::new(),
1848 preserved_key: None,
1849 kind: "upload-refused-not-regular: no longer a regular file after the scan (a symlink \
1850 or special file replaced it); nothing published"
1851 .into(),
1852 at_unix: now_unix(),
1853 })?;
1854 return Ok(UploadOutcome::Deferred);
1855 }
1856 let posix = Some(PosixStamps::from_metadata(&lmeta));
1857 let stamps = GenerationStamps {
1858 generation,
1859 epoch,
1860 flush_uuid: flush_uuid.to_string(),
1861 boundary_source: None,
1862 posix,
1863 };
1864 let condition = match base {
1865 Some(b) => PutCondition::IfMatch(b.etag.clone()),
1866 None => PutCondition::IfNoneMatchAny,
1867 };
1868 let size = lmeta.len();
1869 if size > self.cfg.whole_put_max {
1870 // Streaming multipart compose: put_whole is never fed past
1871 // whole_put_max (unbounded memory + S3's 5 GiB wall).
1872 return self
1873 .upload_compose(
1874 path, &key, &local_path, size, scanned, condition, stamps, generation, epoch,
1875 prior_uuids,
1876 )
1877 .await;
1878 }
1879 // The upload byte gate — the store's own, shared with the compose
1880 // arm above, which charges its parts inside the store. This body
1881 // is charged from before it is read (the read is the allocation)
1882 // until its PUT, retried or not, has returned: the permit lives
1883 // to the end of this function. `None` = a store with no gate.
1884 let _upload_permit = match self.store.upload_gate() {
1885 Some(g) => Some(g.acquire(size).await),
1886 None => None,
1887 };
1888 // `Bytes::from(Vec<u8>)` TAKES OWNERSHIP without copying, so the
1889 // old `Bytes::from(body.clone())` was a full memcpy of every
1890 // published file body — bought solely to leave `body` intact for
1891 // the 412 retry below, which is reached almost never. Build the
1892 // `Bytes` once; the retry gets a refcount clone.
1893 let body = Bytes::from(read_nofollow(&local_path)?);
1894 let uploaded_len = body.len() as u64;
1895 let crc = crc64_nvme(&body);
1896 match self
1897 .store
1898 .put_whole(&key, body.clone(), &condition, &stamps, crc)
1899 .await
1900 {
1901 Ok(meta) => Ok(UploadOutcome::published(
1902 path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1903 )),
1904 // S3 answers If-Match on a key that no longer exists with 404
1905 // NoSuchKey, not 412 (Ozone answers 412, and so did the
1906 // double): the base is gone — a peer's GC collected it, or a
1907 // bucket-level delete. The same rule as the 412 arm's vanished
1908 // HEAD below: a vanished base is a create. Without this arm the
1909 // rule never ran on S3, and a writer whose edited path a peer
1910 // deleted failed EVERY barrier (drill host leg H2, 2026-09-13).
1911 Err(StoreError::NotFound(_)) if matches!(condition, PutCondition::IfMatch(_)) => {
1912 let meta = self
1913 .store
1914 .put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
1915 .await?;
1916 Ok(UploadOutcome::published(
1917 path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1918 ))
1919 }
1920 Err(StoreError::PreconditionFailed(_)) => {
1921 // The 412 policy: my own crashed/torn PUT ⇒ adopt; a
1922 // foreign version ⇒ the consume-dirty rule, at upload time
1923 // (preserve it, then supersede it knowingly). NEVER the
1924 // inherited LOCAL-WINS overwrite, and never a park with
1925 // no way out.
1926 let head = match self.store.head(&key).await {
1927 Ok(h) => h,
1928 Err(StoreError::NotFound(_)) => {
1929 // Review 2026-09-12, inbox-5: the base object is
1930 // gone (a bucket-level delete, or our own GC after
1931 // a crash between the CAS and the baseline
1932 // rewrite). Failing the barrier here failed EVERY
1933 // barrier, forever. A vanished base is a create.
1934 let meta = self
1935 .store
1936 .put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
1937 .await?;
1938 return Ok(UploadOutcome::published(
1939 path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1940 ));
1941 }
1942 Err(e) => return Err(e.into()),
1943 };
1944 let head_stamps = GenerationStamps::from_meta(&head.meta);
1945 let own = head_stamps
1946 .as_ref()
1947 .map(|s| s.flush_uuid == flush_uuid || prior_uuids.contains(&s.flush_uuid))
1948 .unwrap_or(false);
1949 if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
1950 // Bytes already there (a torn response, ours or a
1951 // foreign write of the same content): cite it — re-read
1952 // under the fence first.
1953 let g = head_stamps.map(|s| s.generation).unwrap_or(generation);
1954 return Ok(UploadOutcome::adopted(
1955 path, key, head.etag, crc, head.size, scanned, g, epoch,
1956 ));
1957 }
1958 if !own && self.preserve_foreign_412(path, &head).await.is_none() {
1959 return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1960 }
1961 // Our earlier PUT (older content), or a foreign version
1962 // now preserved: supersede it knowingly, If-Match on what
1963 // we saw. A second 412 means the path is being written
1964 // under us right now; that one parks.
1965 let meta = match self
1966 .store
1967 .put_whole(&key, body, &PutCondition::IfMatch(head.etag.clone()), &stamps, crc)
1968 .await
1969 {
1970 Ok(m) => m,
1971 // 412 (a writer landed between) or 404 (a GC removed
1972 // it between the HEAD and this PUT): written under us
1973 // right now. Park; the next barrier retries.
1974 Err(StoreError::PreconditionFailed(_)) | Err(StoreError::NotFound(_)) => {
1975 return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1976 }
1977 Err(e) => return Err(e.into()),
1978 };
1979 Ok(UploadOutcome::published(
1980 path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1981 ))
1982 }
1983 Err(e) => Err(e.into()),
1984 }
1985 }
1986
1987 /// A 412 against a version this syncer did not write, met at upload
1988 /// time. The contract's rule for a foreign write to a path the agent
1989 /// modified is the consume-dirty rule — the agent's version wins and
1990 /// the foreign bytes are preserved — and a "park" was that case with
1991 /// no resolution: nothing ever un-parked, and every ack said `ok`
1992 /// (review 2026-09-12, inbox-1). Preserve, record, and let the caller
1993 /// supersede. `None` = the preserve failed; the caller parks, and the
1994 /// boundary is `partial`.
1995 async fn preserve_foreign_412(&self, path: &str, head: &flint_store::ObjectMeta) -> Option<()> {
1996 let preserved = match self.preserve_conflict_copy(path, &head.etag).await {
1997 Ok(k) => k,
1998 Err(e) => {
1999 eprintln!("flint-sync: {path}: foreign version {} could not be preserved ({e}); parking", head.etag);
2000 return None;
2001 }
2002 };
2003 self.state
2004 .append_conflict(&ConflictRecord {
2005 path: path.to_string(),
2006 foreign_etag: head.etag.clone(),
2007 preserved_key: Some(preserved),
2008 kind: "upload-412-preserved".into(),
2009 at_unix: now_unix(),
2010 })
2011 .ok()?;
2012 Some(())
2013 }
2014}
2015
2016/// Read a workspace file for upload without following a symlink at the
2017/// final component, and refuse anything that is not a regular file once
2018/// open (review 2026-09-12, atomicity-6).
2019fn read_nofollow(p: &Path) -> std::io::Result<Vec<u8>> {
2020 use std::io::Read;
2021 use std::os::unix::fs::OpenOptionsExt;
2022 let mut f = std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NOFOLLOW).open(p)?;
2023 if !f.metadata()?.is_file() {
2024 return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "not a regular file"));
2025 }
2026 let mut v = Vec::new();
2027 f.read_to_end(&mut v)?;
2028 Ok(v)
2029}
2030
2031enum UploadOutcome {
2032 /// `adopted`: the etag was OBSERVED at the key (bytes already there,
2033 /// no PUT of ours produced it), so the commit section must re-read it
2034 /// before citing — see `verify_observed_citations` in the barrier.
2035 Published { entry: LeanEntry, baseline_entry: BaselineEntry, adopted: bool },
2036 Parked { foreign_etag: String },
2037 /// The source drifted mid-transfer (checksum refused server-side)
2038 /// or the assembly was swept: publish nothing, advance nothing —
2039 /// the next scan re-queues the path.
2040 Deferred,
2041}
2042
2043impl UploadOutcome {
2044 #[allow(clippy::too_many_arguments)]
2045 fn published(
2046 path: &str,
2047 key: String,
2048 etag: String,
2049 crc: u64,
2050 uploaded_len: u64,
2051 scanned: &scan::ScanEntry,
2052 generation: u64,
2053 epoch: u64,
2054 ) -> UploadOutcome {
2055 let _ = path;
2056 UploadOutcome::Published {
2057 adopted: false,
2058 entry: LeanEntry {
2059 key,
2060 etag: etag.clone(),
2061 crc64_b64: crc64_to_b64(crc),
2062 // The length the object HAS (review 2026-09-12,
2063 // atomicity-1): the scanned size was cited while the
2064 // body was read fresh, so a file that grew between the
2065 // two was cited short and every fresh checkout of it
2066 // failed its CRC fold — the successor never started.
2067 size: uploaded_len,
2068 mode: scanned.mode,
2069 mtime_unix: scanned.mtime_unix,
2070 generation,
2071 epoch,
2072 },
2073 baseline_entry: BaselineEntry {
2074 etag,
2075 generation,
2076 // The PRE-read stat: if the agent wrote during our read,
2077 // the next scan sees the drift and re-queues (the
2078 // re-stat/re-queue valve).
2079 size: scanned.size,
2080 mtime_unix: scanned.mtime_unix,
2081 mtime_nanos: Some(scanned.mtime_nanos),
2082 crc64_b64: Some(crc64_to_b64(crc)),
2083 },
2084 }
2085 }
2086}
2087
2088impl UploadOutcome {
2089 /// `published`, for bytes this barrier found already at the key and
2090 /// cites without a PUT of its own.
2091 #[allow(clippy::too_many_arguments)]
2092 fn adopted(
2093 path: &str,
2094 key: String,
2095 etag: String,
2096 crc: u64,
2097 uploaded_len: u64,
2098 scanned: &scan::ScanEntry,
2099 generation: u64,
2100 epoch: u64,
2101 ) -> UploadOutcome {
2102 match UploadOutcome::published(path, key, etag, crc, uploaded_len, scanned, generation, epoch) {
2103 UploadOutcome::Published { entry, baseline_entry, .. } => {
2104 UploadOutcome::Published { entry, baseline_entry, adopted: true }
2105 }
2106 other => other,
2107 }
2108 }
2109}
2110
2111fn local_dirty(local: &Path, base: Option<&BaselineEntry>) -> bool {
2112 match (std::fs::metadata(local), base) {
2113 (Err(_), None) => false, // both absent: clean
2114 (Err(_), Some(_)) => true, // locally deleted vs baseline
2115 (Ok(_), None) => true, // local exists, never published
2116 (Ok(m), Some(b)) => {
2117 scan::stat_changed(b.size, b.mtime_unix, b.mtime_nanos, m.len(), mtime_of(&m), mtime_nanos_of(&m))
2118 }
2119 }
2120}
2121
2122/// Full-object CRC-64/NVME of a local file, streamed.
2123///
2124/// Reached only by the 412 recovery path: the ordinary publish gets its
2125/// checksum from the store, which computes one while uploading.
2126/// The CRC a citation repair cites: the bytes this syncer integrated,
2127/// as the baseline recorded them when it wrote or uploaded them. The
2128/// HEAD's own checksum is a cross-check when the backend offers one,
2129/// never the source — Ozone offers none, a HITL uploader may have sent
2130/// none, and the manifest must carry a CRC either way, because every
2131/// reader now verifies a fresh fetch against it.
2132///
2133/// A baseline entry with no CRC is refused loudly rather than cited
2134/// bare: the only entry built without one is the consume-dirty sentinel,
2135/// which the candidate filters exclude, so reaching this is a defect.
2136/// A HEAD that attests a DIFFERENT value is refused the same way — the
2137/// workspace holds bytes the object does not, and citing either would
2138/// make one reader or another refuse the path forever.
2139pub(crate) fn repair_crc(
2140 path: &str,
2141 be: &BaselineEntry,
2142 head: &flint_store::ObjectMeta,
2143) -> LeanResult<String> {
2144 let Some(ours) = be.crc64_b64.clone() else {
2145 return Err(LeanError::State(format!(
2146 "citation repair of {path}: the baseline records no CRC for the bytes it \
2147 integrated at etag {} — refusing to cite bytes nothing hashed",
2148 be.etag
2149 )));
2150 };
2151 if let Some(theirs) = head.crc64_b64.as_deref() {
2152 if theirs != ours {
2153 return Err(LeanError::State(format!(
2154 "citation repair of {path}: the store attests CRC-64 {theirs} for etag {} but \
2155 the bytes this workspace integrated under that etag hash to {ours} — the \
2156 workspace holds bytes the object does not; refusing to cite either",
2157 be.etag
2158 )));
2159 }
2160 }
2161 Ok(ours)
2162}
2163
2164fn file_crc(local_path: &Path) -> LeanResult<u64> {
2165 use std::io::Read;
2166 let mut crc = flint_store::Crc64Nvme::new();
2167 let mut f = std::fs::File::open(local_path)?;
2168 let mut buf = vec![0u8; 4 << 20];
2169 loop {
2170 let n = f.read(&mut buf)?;
2171 if n == 0 {
2172 break;
2173 }
2174 crc.update(&buf[..n]);
2175 }
2176 Ok(crc.finalize())
2177}
2178
2179pub(super) fn mtime_nanos_of(m: &std::fs::Metadata) -> u32 {
2180 m.modified()
2181 .ok()
2182 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2183 .map(|d| d.subsec_nanos())
2184 .unwrap_or(0)
2185}
2186
2187pub(super) fn mtime_of(m: &std::fs::Metadata) -> i64 {
2188 m.modified()
2189 .ok()
2190 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2191 .map(|d| d.as_secs() as i64)
2192 .unwrap_or(0)
2193}
2194
2195/// Containment-safe workspace write (boundary-verbs plan §2.2 security
2196/// gate). `rel` is a workspace-relative path; the target may not escape
2197/// the root, traverse a symlink component, or land in the reserved
2198/// control namespace.
2199///
2200/// The hazard this closes is pre-existing and reachable from three
2201/// callers (checkout, inbox consume, sync): `write_file_atomic` did
2202/// `create_dir_all(parent)` + write with no `O_NOFOLLOW` and no
2203/// root-containment check, while the scanner SKIPS symlinks — so a
2204/// planted symlink is invisible to the syncer. An unprivileged app
2205/// that plants `inputs -> /root/.aws`, lands an object at
2206/// `inputs/<path>` and drops a scoped sync turns the credential-holding
2207/// syncer into an on-demand arbitrary-file-write primitive outside the
2208/// workspace.
2209pub(super) fn write_file_atomic_in(
2210 root: &Path,
2211 rel: &str,
2212 bytes: &[u8],
2213 mode: Option<u32>,
2214) -> LeanResult<()> {
2215 let target = contained_path(root, rel)?;
2216 write_file_atomic(&target, bytes, mode).map(|_| ())
2217}
2218
2219/// Validate `rel` under `root` WITHOUT creating anything — for callers
2220/// that must decide whether a path is writable before doing any work
2221/// (inbox consume refuses a refused path outright rather than routing it
2222/// through conflict-preserve, which would cost a GET+PUT and leave the
2223/// path in the baseline).
2224pub(super) fn check_contained(root: &Path, rel: &str) -> LeanResult<()> {
2225 resolve_contained(root, rel, false).map(|_| ())
2226}
2227
2228/// Resolve `rel` under `root`, refusing escapes, creating missing parent
2229/// directories. Walks the parent chain component by component: a
2230/// component that EXISTS as a symlink is a refusal (never followed).
2231pub(super) fn contained_path(root: &Path, rel: &str) -> LeanResult<std::path::PathBuf> {
2232 resolve_contained(root, rel, true).map(|(p, _)| p)
2233}
2234
2235/// As `contained_path`, and also what the walk already learned about
2236/// the FINAL component: its `lstat` if it exists, `None` if it does
2237/// not. The walk stats every component to refuse a symlink; a caller
2238/// that then asked `exists()` and `metadata()` paid two more stats for
2239/// an answer it was already holding (checkout's resume check did, on
2240/// every one of 20,000 files).
2241pub(super) fn contained_path_stat(
2242 root: &Path,
2243 rel: &str,
2244) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2245 resolve_contained(root, rel, true)
2246}
2247
2248fn resolve_contained(
2249 root: &Path,
2250 rel: &str,
2251 create_dirs: bool,
2252) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2253 use std::path::Component;
2254 let refuse = |why: &str| {
2255 Err(LeanError::State(format!(
2256 "refusing write to {rel:?}: {why} (containment)"
2257 )))
2258 };
2259 if rel.is_empty() {
2260 return refuse("empty path");
2261 }
2262 if super::scan::is_control_path(rel) {
2263 // The reserved namespace is the syncer's own; a citation or
2264 // inbox entry naming it is surfaced, never materialized (D0.3).
2265 return refuse("reserved control namespace");
2266 }
2267 let relp = Path::new(rel);
2268 let mut cur = root.to_path_buf();
2269 let mut comps: Vec<&std::ffi::OsStr> = vec![];
2270 for c in relp.components() {
2271 match c {
2272 Component::Normal(n) => comps.push(n),
2273 Component::CurDir => {}
2274 Component::ParentDir => return refuse("`..` component"),
2275 Component::RootDir | Component::Prefix(_) => return refuse("absolute path"),
2276 }
2277 }
2278 if comps.is_empty() {
2279 return refuse("no path components");
2280 }
2281 let last = comps.len() - 1;
2282 let mut final_meta = None;
2283 for (i, c) in comps.iter().enumerate() {
2284 cur.push(c);
2285 match std::fs::symlink_metadata(&cur) {
2286 Ok(m) if m.file_type().is_symlink() => {
2287 return refuse("path traverses a symlink");
2288 }
2289 Ok(m) if i < last && !m.is_dir() => {
2290 return refuse("parent component is not a directory");
2291 }
2292 Ok(m) => {
2293 if i == last {
2294 final_meta = Some(m);
2295 }
2296 }
2297 Err(_) if i < last => {
2298 if create_dirs {
2299 if let Err(e) = std::fs::create_dir(&cur) {
2300 // A SIBLING may have created it between the stat
2301 // above and this mkdir: fetches run on their own
2302 // tasks, so the first files of one directory
2303 // race here, and the loser's EEXIST is not a
2304 // containment refusal. It is fine only if what
2305 // exists now is a plain directory — a symlink
2306 // that appeared in the window is refused exactly
2307 // as one the stat found.
2308 match std::fs::symlink_metadata(&cur) {
2309 Ok(m) if m.file_type().is_symlink() => {
2310 return refuse("path traverses a symlink");
2311 }
2312 Ok(m) if m.is_dir() => {}
2313 _ => {
2314 return Err(LeanError::State(format!(
2315 "mkdir {}: {e}",
2316 cur.display()
2317 )))
2318 }
2319 }
2320 }
2321 }
2322 }
2323 Err(_) => {}
2324 }
2325 }
2326 Ok((cur, final_meta))
2327}
2328
2329/// Write `bytes` to `path` through an exclusive temp sibling and a
2330/// rename. The parent already exists — containment created it — and
2331/// one that vanished since is recreated on the retry path in `safefs`,
2332/// so the common path pays no `mkdir` and no `stat` for it. Returns
2333/// the written file's metadata.
2334pub(super) fn write_file_atomic(
2335 path: &Path,
2336 bytes: &[u8],
2337 mode: Option<u32>,
2338) -> LeanResult<std::fs::Metadata> {
2339 super::safefs::check_parent(path)?;
2340 // NOT with_extension(): that REPLACES the final extension, so
2341 // "a.txt" and "a.md" would collide on one tmp name.
2342 let tmp = path.with_file_name(format!(
2343 "{}.flint-sync-tmp",
2344 path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default()
2345 ));
2346 // The temp sibling is computed AFTER containment ran, so the walk
2347 // never saw it: it needs its own refusal, not a plain write.
2348 // FAST (no per-file fsync): materialisations are made durable by
2349 // `sync_tree` before the marker or baseline that vouches for them.
2350 super::safefs::write_via_tmp_fast(path, &tmp, bytes, mode)
2351}
2352
2353/// The queue form of a merge's foreign upserts and deletions.
2354fn foreign_changes(entries: &[(String, LeanEntry)], gone: &[String]) -> Vec<ForeignChange> {
2355 entries
2356 .iter()
2357 .map(|(path, e)| ForeignChange {
2358 path: path.clone(),
2359 etag: Some(e.etag.clone()),
2360 crc64_b64: Some(e.crc64_b64.clone()),
2361 })
2362 .chain(gone.iter().map(|path| ForeignChange { path: path.clone(), etag: None, crc64_b64: None }))
2363 .collect()
2364}
2365
2366/// One three-way merge of a barrier's changes onto the bucket's manifest.
2367struct MergeOnto {
2368 theirs: manifest::LeanManifest,
2369 /// The CAS handle over `theirs`; `None` when the bucket has no manifest.
2370 expected: Option<manifest::ManifestHandle>,
2371 merged: manifest::LeanManifest,
2372 /// Other writers' changes since the merge base that the tree lacks.
2373 foreign: Vec<(String, LeanEntry)>,
2374 /// Other writers' deletions since the merge base.
2375 gone: Vec<String>,
2376}
2377
2378impl MergeOnto {
2379 /// The merge adds nothing to the document: installing it would be an
2380 /// empty generation.
2381 fn adds_nothing(&self) -> bool {
2382 self.merged.entries == self.theirs.entries && self.merged.sole_writer == self.theirs.sole_writer
2383 }
2384}
2385
2386impl Syncer {
2387 #[allow(clippy::too_many_arguments)]
2388 fn merge_onto(
2389 &self,
2390 current: Option<&manifest::LoadedManifest>,
2391 prev_installed: Option<&str>,
2392 inst_base: &BTreeMap<String, String>,
2393 upserts: &BTreeMap<String, LeanEntry>,
2394 deletes: &BTreeSet<String>,
2395 parked: &BTreeSet<String>,
2396 flush_uuid: &str,
2397 ) -> MergeOnto {
2398 let (theirs, expected) = match current {
2399 // The handle carries the LAYOUT as well as the tag: a
2400 // workspace still on the legacy single object CASes a
2401 // pointer that must not exist yet, not one it read.
2402 Some(l) => (l.manifest.clone(), Some(l.handle())),
2403 None => (Default::default(), None),
2404 };
2405 // If the bucket is still at the document THIS workspace
2406 // installed, that document IS the merge base — whatever the
2407 // persisted one says. Step 7 rewrites the merge base after
2408 // the CAS, so a restart in between leaves it behind a
2409 // document we wrote, and every entry in it would read as
2410 // somebody else's change. See `IntentJournal::installed_etag`.
2411 let own_base: BTreeMap<String, String>;
2412 let base: &BTreeMap<String, String> =
2413 if prev_installed.is_some() && prev_installed == expected.as_ref().map(|h| h.etag.as_str()) {
2414 own_base = theirs.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
2415 &own_base
2416 } else {
2417 inst_base
2418 };
2419 let (mut merged, foreign) = manifest::merge(base, &theirs, upserts, deletes, parked);
2420 // Deletions another writer made since this workspace's
2421 // merge base: in the base, gone from theirs, and not this
2422 // barrier's own upsert, delete or park. `merge` has no use
2423 // for them — theirs already lacks the path — but the TREE
2424 // does: they reach it through the local queue, as the
2425 // foreign upserts do.
2426 let gone: Vec<String> = base
2427 .keys()
2428 .filter(|p| {
2429 !theirs.entries.contains_key(*p)
2430 && !upserts.contains_key(*p)
2431 && !deletes.contains(*p)
2432 && !parked.contains(*p)
2433 })
2434 .cloned()
2435 .collect();
2436 // `merge` clears it; the installing pass owns it. A mirror
2437 // is a property of how this workspace is DEPLOYED, so it
2438 // comes from config on every publish rather than being
2439 // inherited from whatever wrote last.
2440 merged.sole_writer = self.cfg.sole_writer;
2441 self.trace("merge", serde_json::json!({"flush": flush_uuid, "theirs_seq": theirs.seq, "upserts": upserts.len(),
2442 "deletes": deletes.len(), "foreign": foreign.len(), "gone": gone.len(),
2443 "adds_nothing": merged.entries == theirs.entries}));
2444 MergeOnto { theirs, expected, merged, foreign, gone }
2445 }
2446}