flint_lean/barrier.rs
1//! The publish barrier (plan §2.1, seven steps) — the machine
2//! `lean/formal/LeanSubtree.tla` checks. Order is load-bearing:
3//! consume → scan → intent/window → uploads → manifest CAS (merge) →
4//! GC deletes LAST → baseline rewrite.
5
6use std::collections::{BTreeMap, BTreeSet};
7use std::path::Path;
8
9use bytes::Bytes;
10
11use flint_store::{
12 EpochState,
13 crc64_nvme, crc64_to_b64, GenerationStamps, PosixStamps, PutCondition, StoreError,
14};
15
16use super::inbox::{self, InboxDoc, InboxEntry, Refusal, Removal};
17use super::manifest::{self, LeanEntry};
18use super::scan;
19use super::state::{BaselineEntry, ConflictRecord, ForeignChange, IntentJournal};
20use super::{now_unix, LeanError, LeanResult, Syncer};
21
22/// The last gateway request this workspace acted on, per verb. A
23/// watermark rather than a consumed queue — see `note_verb_requests`.
24#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
25pub struct VerbWatermark {
26 #[serde(default)]
27 pub boundary_unix: u64,
28 #[serde(default)]
29 pub sync_unix: u64,
30}
31
32#[derive(Debug, Default)]
33pub struct BarrierReport {
34 pub seq: Option<u64>,
35 pub uploaded: Vec<String>,
36 pub deleted: Vec<String>,
37 /// Paths a DECLARED removal (delete/rename design) cited out this
38 /// barrier — a subset of `deleted` once the GC has run.
39 pub removed: Vec<String>,
40 /// Declared removals this barrier REFUSED (locally dirty, not a
41 /// file, uncontainable); the cell carries the reason.
42 pub removals_refused: usize,
43 pub parked: Vec<String>,
44 pub consumed: usize,
45 pub foreign_queued: usize,
46 /// Paths whose transfer drifted or was swept mid-compose: nothing
47 /// published, nothing advanced; the next scan re-queues them.
48 pub deferred: Vec<String>,
49 /// Local deletes another writer's change outranked (delete/modify
50 /// resolves foreign-wins): the installed seq still cites them, and the
51 /// next consume reconciles each against the local delete.
52 pub outranked: Vec<String>,
53 pub no_change: bool,
54 /// Bytes this barrier actually published — the input to the
55 /// sentinel work meter (boundary-verbs plan D3.1). Metering work
56 /// rather than calls is what keeps a sentinel storm from
57 /// un-coalescing a hot large file's republish: a counted budget
58 /// charges a 2 GiB checkpoint the same one unit as a 4 KiB file.
59 pub published_bytes: u64,
60 /// The manifest ETag this barrier installed (the ack's CAS token).
61 pub manifest_etag: Option<String>,
62 /// What the manifest HEAD/install told us the bucket is at, for the
63 /// news ticker (D5) — free, off requests the barrier already made.
64 pub observed_seq: Option<u64>,
65 pub observed_etag: Option<String>,
66 /// Paths this barrier saw absent for the FIRST time: their deletes
67 /// are withheld to the next scan (the rename-vs-walk guard). On a
68 /// declared boundary a non-empty set means the confirmation lstat
69 /// found them back on disk — the race the guard exists for.
70 pub first_absence: Vec<String>,
71 /// First-absence paths a DECLARED boundary confirmed gone by direct
72 /// lstat and published as ordinary deletes (`confirm_absences`).
73 pub absences_confirmed: usize,
74 /// This barrier finished a rescope a crash had left half-applied
75 /// before it did anything else. Reported so a run that looks like
76 /// an ordinary barrier can be told from one that had to converge a
77 /// workspace first.
78 pub rescope_replayed: bool,
79}
80
81/// What a pass over the cell's DECLARED removals did (`apply_removals`).
82#[derive(Debug, Default)]
83pub struct RemovalPass {
84 /// Performed: the local file is gone (or already was) and the path
85 /// is in `declared`. Dropped from the cell with the window clear —
86 /// AFTER the manifest CAS — so a listing keeps hiding the path
87 /// until the manifest stops citing it.
88 pub applied: Vec<Removal>,
89 /// Refused for good, `refused` filled in: the cell is told at once
90 /// and the removal is never retried.
91 pub refused: Vec<Removal>,
92 /// Left in the cell for the next barrier: a rename whose
93 /// destination is not integrated yet, or an unlink that failed for
94 /// a transient reason.
95 pub deferred: usize,
96 /// The paths this barrier cites OUT: `applied`, plus any a crashed
97 /// earlier barrier had unlinked and journalled.
98 pub declared: BTreeSet<String>,
99}
100
101/// Upload waves per chunk. The chunk is `fanout * this`, so a wave
102/// still saturates fan-out and the sync point between chunks is rare.
103const UPLOAD_CHUNK_WAVES: usize = 16;
104
105/// How stale the lease may get inside a barrier before it is renewed:
106/// comfortably below the 6-poll (~60 s) takeover window, so a holder
107/// busy in a long commit section never reads as quiet to a waiter.
108const RENEW_WITHIN_SECS: u64 = 20;
109
110impl Syncer {
111 /// Verify the cell still names us at OUR epoch; anything else is a
112 /// fence. Read-verify before the manifest CAS (the per-request
113 /// validation the gateway will also enforce).
114 async fn verify_not_deposed(&self) -> LeanResult<()> {
115 let lease = self
116 .lease
117 .as_ref()
118 .ok_or_else(|| LeanError::State("no lease".into()))?;
119 match self.store.epoch_read(&self.cfg.epoch_key()).await? {
120 Some(state) if state.epoch == lease.epoch && state.holder_id == lease.holder_id => {
121 Ok(())
122 }
123 Some(state) => Err(LeanError::Fenced(format!(
124 "cell at epoch {} holder {} (we are epoch {})",
125 state.epoch, state.holder_id, lease.epoch
126 ))),
127 None => Err(LeanError::Fenced("epoch cell vanished".into())),
128 }
129 }
130
131 /// Step 1: consume the inbox. A barrier NEVER runs against an
132 /// unconsumed inbox — this is what makes HITL uploads structurally
133 /// un-amputatable. Returns the entries integrated (dropped from the
134 /// cell at the window-open CAS).
135 pub async fn consume_inbox(&mut self) -> LeanResult<Vec<InboxEntry>> {
136 let loaded = inbox::load(self.store.as_ref(), &self.cfg).await?;
137 self.consume_inbox_doc(&loaded.doc).await
138 }
139
140 /// `consume_inbox` over a cell the caller has already read — the
141 /// barrier reads it once and hands the same document to
142 /// `apply_removals`, so the removal pass adds no request.
143 pub async fn consume_inbox_doc(&mut self, doc: &InboxDoc) -> LeanResult<Vec<InboxEntry>> {
144 Ok(self.consume_counted(doc).await?.0)
145 }
146
147 /// `consume_inbox_doc`, plus how many changes from this writer's LOCAL
148 /// queue it settled. Those leave no entry in the cell to drop, so they
149 /// are not in the returned list — but they are consumptions, and the
150 /// report (and the ack) count them.
151 async fn consume_counted(&mut self, doc: &InboxDoc) -> LeanResult<(Vec<InboxEntry>, usize)> {
152 // §2.5's layered doors ride the inbox GET this function already
153 // pays for: promptness is one tick, and the added request count
154 // is ZERO. A failure here must never fail the consume — the
155 // request is idempotent state and the next tick re-reads it.
156 if let Err(e) = self.note_verb_requests(doc) {
157 eprintln!("flint-sync: verb request not consumed (retrying next tick): {e}");
158 }
159 // Indices into `entries`: the writer-local queue's upserts first,
160 // then the shared inbox's. Both run the same rules below.
161 let mut consumed: Vec<usize> = vec![];
162 let mut baseline = self.state.load_baseline()?;
163 // An install whose step 7 a restart cut off left the other writers'
164 // changes it carried in the intent journal (`installed_foreign`).
165 let requeued = self.state.requeue_installed_foreign()?;
166 if requeued > 0 {
167 self.trace("queue", serde_json::json!({"requeued_from_intent": requeued}));
168 }
169 // The writer-local queue: changes other writers made that this
170 // workspace's own merges carried into the manifest but not yet
171 // into the tree (`state::ForeignChange` says why it is not the
172 // shared inbox any more).
173 let queue = self.state.load_foreign_queue()?;
174 let queued: Vec<InboxEntry> = queue
175 .iter()
176 .filter_map(|c| {
177 c.etag.as_ref().map(|etag| InboxEntry {
178 path: c.path.clone(),
179 etag: etag.clone(),
180 author: "merge-preserved".into(),
181 added_unix: 0,
182 crc64_b64: c.crc64_b64.clone(),
183 })
184 })
185 .collect();
186 let n_queued = queued.len();
187 let entries: Vec<InboxEntry> = queued.into_iter().chain(doc.entries.iter().cloned()).collect();
188 for (idx, entry) in entries.iter().enumerate() {
189 let key = self.cfg.file_key(&entry.path);
190 // Containment BEFORE anything else: a path we could never
191 // safely materialize must be surfaced and dropped, not
192 // routed through the locally-dirty branch — which would
193 // "preserve" it with a GET+PUT and leave a baseline entry
194 // for a path the scanner can never see (the planted-symlink
195 // shape: `inputs -> /root/.aws` reads as locally-present,
196 // therefore dirty, therefore a conflict-preserve of someone
197 // else's file).
198 if let Err(e) = check_contained(&self.cfg.root, &entry.path) {
199 self.state.append_conflict(&ConflictRecord {
200 path: entry.path.clone(),
201 foreign_etag: entry.etag.clone(),
202 preserved_key: None,
203 kind: format!("consume-refused-containment: {e}"),
204 at_unix: now_unix(),
205 })?;
206 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused"}));
207 consumed.push(idx);
208 continue;
209 }
210 // Already integrated (a crashed earlier consume): idempotent.
211 if baseline.entries.get(&entry.path).map(|b| b.etag == entry.etag).unwrap_or(false) {
212 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "already"}));
213 consumed.push(idx);
214 continue;
215 }
216 let head = match self.store.head(&key).await {
217 Ok(m) => m,
218 Err(StoreError::NotFound(_)) => {
219 // The object vanished under a tracked entry: surface
220 // it — the bytes are NOT recoverable from here.
221 self.state.append_conflict(&ConflictRecord {
222 path: entry.path.clone(),
223 foreign_etag: entry.etag.clone(),
224 preserved_key: None,
225 kind: "consume-object-missing".into(),
226 at_unix: now_unix(),
227 })?;
228 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "missing"}));
229 consumed.push(idx);
230 continue;
231 }
232 Err(e) => return Err(e.into()),
233 };
234 if head.etag != entry.etag {
235 // Superseded by a newer write (its own inbox entry
236 // follows, or it is the syncer's): drop.
237 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "superseded"}));
238 consumed.push(idx);
239 continue;
240 }
241 let local_path = self.cfg.root.join(&entry.path);
242 let mut dirty = local_dirty(&local_path, baseline.entries.get(&entry.path));
243 // Review 2026-09-12, atomicity-3 / inbox-2: the fetch is a
244 // network round trip and the agent may write the path inside
245 // it. The stat that licenses the adopt is repeated AFTER the
246 // fetch, before anything touches the file: a write in that
247 // window makes this the dirty case — the foreign bytes are
248 // preserved and a record names them, the agent's stay.
249 let mut fetched = None;
250 if !dirty {
251 let got = self.store.get_whole(&key, Some(&entry.etag)).await.map_err(|e| match e {
252 StoreError::PreconditionFailed(m) => {
253 StoreError::Other(format!("consume raced a newer write: {m}"))
254 }
255 other => other,
256 })?;
257 if local_dirty(&local_path, baseline.entries.get(&entry.path)) {
258 dirty = true;
259 } else {
260 fetched = Some(got);
261 }
262 }
263 if dirty {
264 // Locally-dirty wins; preserve the FOREIGN bytes first
265 // (a conflict record must keep both versions
266 // recoverable), then advance the recognized ETag so our
267 // eventual publish supersedes it KNOWINGLY.
268 let preserved = self.preserve_conflict_copy(&entry.path, &head.etag).await?;
269 self.state.append_conflict(&ConflictRecord {
270 path: entry.path.clone(),
271 foreign_etag: entry.etag.clone(),
272 preserved_key: Some(preserved),
273 kind: "consume-dirty".into(),
274 at_unix: now_unix(),
275 })?;
276 let stamps = GenerationStamps::from_meta(&head.meta);
277 baseline.entries.insert(
278 entry.path.clone(),
279 BaselineEntry {
280 etag: head.etag.clone(),
281 generation: stamps.map(|s| s.generation).unwrap_or(0),
282 // Deliberately NOT the local file's stat: the
283 // path must still scan as dirty so the local
284 // version publishes.
285 size: u64::MAX,
286 mtime_unix: 0,
287 mtime_nanos: None,
288 // The sentinel's bytes are the LOCAL edit; the
289 // publish that supersedes hashes them itself.
290 crc64_b64: None,
291 },
292 );
293 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "dirty-preserved"}));
294 } else {
295 // Clean, re-checked after the fetch: adopt the foreign
296 // content into the tree.
297 let (meta, body) = fetched.expect("fetched while clean");
298 let mode = PosixStamps::from_meta(&meta.meta).map(|p| p.mode);
299 // VERIFIED before it is written, as checkout's fresh
300 // fetch is: against the writer's CRC when the inbox
301 // entry carries one (the gateway hashes what it sent),
302 // else the backend's attestation when it offers one.
303 // Ozone offers none, so the inbox's is what a HITL
304 // upload gets checked against there. What the baseline
305 // records — and the next citation repair cites — is
306 // OURS, over the bytes actually written.
307 let got = crc64_to_b64(crc64_nvme(&body));
308 let want = entry.crc64_b64.clone().or_else(|| meta.crc64_b64.clone());
309 if let Some(want) = want {
310 if want != got {
311 // NOT consumed: the entry stays in the cell and
312 // the record repeats — a permanent, visible
313 // contradiction is the right failure for
314 // bytes nobody vouches for.
315 self.state.append_conflict(&ConflictRecord {
316 path: entry.path.clone(),
317 foreign_etag: entry.etag.clone(),
318 preserved_key: None,
319 kind: format!(
320 "consume-refused-checksum: etag {} is attested with CRC-64 \
321 {want} but the bytes fetched under it hash to {got}",
322 entry.etag
323 ),
324 at_unix: now_unix(),
325 })?;
326 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused-checksum"}));
327 continue;
328 }
329 }
330 // CONTAINMENT and I/O are split, because the answers are
331 // opposite. Containment was already decided above by
332 // `check_contained`, so a refusal reaching here is a
333 // path that only became unsafe once parents were
334 // created — still a permanent property of the path, so
335 // it is surfaced and DROPPED, exactly as above.
336 //
337 // Everything else — ENOSPC, EACCES, EIO, a read-only
338 // filesystem — is TRANSIENT, and dropping the entry for
339 // one is silent permanent loss: the foreign bytes stay
340 // in the bucket, the workspace never adopts them, and
341 // nothing ever re-offers the entry. It used to be
342 // recorded as `consume-refused-containment`, which sent
343 // whoever read it looking for a hostile path that was
344 // never there. A full disk is not a planted symlink.
345 let target = match contained_path(&self.cfg.root, &entry.path) {
346 Ok(t) => t,
347 Err(e) => {
348 self.state.append_conflict(&ConflictRecord {
349 path: entry.path.clone(),
350 foreign_etag: entry.etag.clone(),
351 preserved_key: None,
352 kind: format!("consume-refused-containment: {e}"),
353 at_unix: now_unix(),
354 })?;
355 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "refused"}));
356 consumed.push(idx);
357 continue;
358 }
359 };
360 if let Err(e) = write_file_atomic(&target, &body, mode) {
361 // NOT consumed: the entry stays in the cell so the
362 // next barrier retries it. A visible conflict record
363 // that repeats is a far better failure than one
364 // silent drop.
365 self.state.append_conflict(&ConflictRecord {
366 path: entry.path.clone(),
367 foreign_etag: entry.etag.clone(),
368 preserved_key: None,
369 kind: format!("consume-write-failed (will retry): {e}"),
370 at_unix: now_unix(),
371 })?;
372 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "deferred"}));
373 continue;
374 }
375 let st = std::fs::metadata(&local_path)?;
376 let stamps = GenerationStamps::from_meta(&meta.meta);
377 baseline.entries.insert(
378 entry.path.clone(),
379 BaselineEntry {
380 etag: meta.etag.clone(),
381 generation: stamps.map(|s| s.generation).unwrap_or(0),
382 size: st.len(),
383 mtime_unix: mtime_of(&st),
384 mtime_nanos: Some(mtime_nanos_of(&st)),
385 crc64_b64: Some(got),
386 },
387 );
388 baseline.prev_scan.insert(entry.path.clone());
389 self.trace("consume", serde_json::json!({"path": entry.path, "etag": entry.etag, "from": if idx < n_queued { "queue" } else { "inbox" }, "action": "adopted"}));
390 }
391 consumed.push(idx);
392 }
393 let mut settled: BTreeSet<String> =
394 consumed.iter().filter(|i| **i < n_queued).map(|i| entries[*i].path.clone()).collect();
395
396 // A queued DELETION: another writer removed the path and this
397 // workspace's merge cited that. A clean local copy goes; a dirty
398 // one is the agent's newer work and stays — it publishes, and a
399 // modify beats a delete at the merge. Never a declared removal of
400 // our own: the deletion is already in the manifest.
401 for change in queue.iter().filter(|c| c.etag.is_none()) {
402 if check_contained(&self.cfg.root, &change.path).is_err() {
403 // Never materializable here, so nothing to remove.
404 settled.insert(change.path.clone());
405 continue;
406 }
407 let local = self.cfg.root.join(&change.path);
408 let be = baseline.entries.get(&change.path).cloned();
409 let record = |kind: String| ConflictRecord {
410 path: change.path.clone(),
411 foreign_etag: String::new(),
412 preserved_key: None,
413 kind,
414 at_unix: now_unix(),
415 };
416 match std::fs::symlink_metadata(&local) {
417 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
418 self.trace("tombstone", serde_json::json!({"path": change.path, "action": "absent"}));
419 }
420 Err(e) => {
421 // Unreadable is not absent: keep it queued.
422 self.trace("tombstone", serde_json::json!({"path": change.path, "action": "deferred"}));
423 self.state.append_conflict(&record(format!(
424 "consume-foreign-delete-deferred: cannot stat the path: {e}"
425 )))?;
426 continue;
427 }
428 Ok(m) if !m.is_file() || be.is_none() || local_dirty(&local, be.as_ref()) => {
429 self.state.append_conflict(&record(
430 "consume-foreign-delete-vs-dirty: another writer deleted this path; the \
431 local version has unpublished changes, so it stays and publishes"
432 .into(),
433 ))?;
434 self.trace("tombstone", serde_json::json!({"path": change.path, "action": "kept-dirty"}));
435 settled.insert(change.path.clone());
436 continue;
437 }
438 Ok(_) => {
439 if let Err(e) = std::fs::remove_file(&local) {
440 self.state.append_conflict(&record(format!(
441 "consume-foreign-delete-failed (will retry): {e}"
442 )))?;
443 continue;
444 }
445 // CONFIRM: only NotFound is absence.
446 if !matches!(
447 std::fs::symlink_metadata(&local),
448 Err(ref e) if e.kind() == std::io::ErrorKind::NotFound
449 ) {
450 self.state.append_conflict(&record(
451 "consume-foreign-delete-unconfirmed (will retry)".into(),
452 ))?;
453 continue;
454 }
455 self.trace("tombstone", serde_json::json!({"path": change.path, "action": "removed"}));
456 }
457 }
458 baseline.entries.remove(&change.path);
459 baseline.prev_scan.remove(&change.path);
460 settled.insert(change.path.clone());
461 }
462
463 // The files this baseline vouches for reached the disk before
464 // the baseline does (audit 2026-09-03, finding 9).
465 self.state.sync_tree()?;
466 self.state.save_baseline(&baseline)?;
467 // After the baseline: a crash between the two re-applies a
468 // settled change, and both arms are idempotent (an upsert the
469 // baseline holds is "already integrated"; a deletion with nothing
470 // on disk only drops a baseline entry that is already gone).
471 if !settled.is_empty() {
472 let remaining: Vec<_> =
473 queue.into_iter().filter(|c| !settled.contains(&c.path)).collect();
474 self.state.save_foreign_queue(&remaining)?;
475 }
476 // Only the SHARED inbox's entries leave the cell at the window
477 // clear; the local queue was settled above.
478 let n_settled = settled.len();
479 let shared = consumed.into_iter().filter(|i| *i >= n_queued).map(|i| entries[i].clone()).collect();
480 Ok((shared, n_settled))
481 }
482
483 /// Step 1b: perform the DECLARED removals (delete/rename design
484 /// §4-§6). Runs after `consume_inbox_doc`, so a rename's destination
485 /// is in the tree before its source leaves it — §5: create first,
486 /// removal second; a partial application leaves an EXTRA file,
487 /// never a missing one.
488 ///
489 /// Per removal, in order:
490 /// - the path must be containable, as a consume's must;
491 /// - a rename WAITS until its destination is integrated (in the
492 /// baseline): a consume that deferred the destination defers the
493 /// removal with it;
494 /// - a locally-DIRTY path — unpublished edits, or a file the agent
495 /// created there — is REFUSED, nothing applied, the agent's work
496 /// untouched. The same rule a consume applies to a HITL write
497 /// over dirty bytes, and the reason a declared removal cannot be
498 /// "just a delete". Refused for good: the cell is told, and a
499 /// retry that waited for the agent to publish would delete the
500 /// very edit the refusal protected;
501 /// - a clean file is unlinked, and the unlink is CONFIRMED by lstat
502 /// before the path is declared — an unlink that did not take
503 /// publishes no deletion (`confirm_absences`' rule, kept);
504 /// - a path already absent locally and known to the baseline or the
505 /// merge base is declared as it stands: the agent got there
506 /// first, or this tree never held it (a scoped workspace).
507 ///
508 /// A declared path skips the two-scan guard (§4): that guard
509 /// protects against absence INFERRED by a walk, and a declaration
510 /// is not an inference. That is what lets a rename's two halves
511 /// ride one manifest generation.
512 pub async fn apply_removals(&mut self, doc: &InboxDoc) -> LeanResult<RemovalPass> {
513 let mut pass = RemovalPass::default();
514 let baseline = self.state.load_baseline()?;
515 // A crashed earlier barrier that had unlinked and journalled but
516 // not installed: its declarations are still deletions with a
517 // recorded basis — unless the agent has since put a file back.
518 for p in self.state.load_intent()?.declared_deletes {
519 let gone = matches!(
520 std::fs::symlink_metadata(self.cfg.root.join(&p)),
521 Err(ref e) if e.kind() == std::io::ErrorKind::NotFound
522 );
523 if gone && (baseline.entries.contains_key(&p) || baseline.inst_base.contains_key(&p)) {
524 pass.declared.insert(p);
525 }
526 }
527 for r in &doc.removals {
528 if r.refused.is_some() {
529 continue;
530 }
531 let refusal = |kind: &str, message: String| Removal {
532 refused: Some(Refusal { kind: kind.into(), message, at_unix: now_unix() }),
533 ..r.clone()
534 };
535 let mut refuse = |kind: &str, message: String| -> LeanResult<()> {
536 self.state.append_conflict(&ConflictRecord {
537 path: r.path.clone(),
538 foreign_etag: String::new(),
539 preserved_key: None,
540 kind: format!("{kind}: {message}"),
541 at_unix: now_unix(),
542 })?;
543 pass.refused.push(refusal(kind, message));
544 Ok(())
545 };
546 if let Err(e) = check_contained(&self.cfg.root, &r.path) {
547 refuse("removal-refused-containment", e.to_string())?;
548 continue;
549 }
550 if let Some(dst) = &r.moved_to {
551 match baseline.entries.get(dst) {
552 // Not integrated yet (a consume deferred it): wait.
553 None => {
554 pass.deferred += 1;
555 continue;
556 }
557 // Integrated as a CONFLICT: the agent had an
558 // unpublished file at the destination, its version
559 // won, and the moved bytes are preserved under
560 // `conflicts/`. Deleting the source now would leave
561 // the user's file nowhere in the tree — so the move
562 // is refused and the source stays. The sentinel is
563 // the consume-dirty baseline entry's `u64::MAX`.
564 Some(b) if b.size == u64::MAX => {
565 refuse(
566 "removal-refused-destination-conflict",
567 format!(
568 "{} was not moved to {dst}: the agent has an unpublished file at the destination and it wins; the moved bytes are preserved under the conflict record and the source is kept",
569 r.path
570 ),
571 )?;
572 continue;
573 }
574 Some(_) => {}
575 }
576 }
577 let local = self.cfg.root.join(&r.path);
578 let be = baseline.entries.get(&r.path);
579 match std::fs::symlink_metadata(&local) {
580 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
581 if be.is_some() || baseline.inst_base.contains_key(&r.path) {
582 pass.declared.insert(r.path.clone());
583 }
584 // Nothing here and nothing cited: a removal of a path
585 // that was never there, or that a sibling already
586 // removed. Applied as a no-op.
587 pass.applied.push(r.clone());
588 }
589 Err(e) => {
590 // Unreadable is not absent (`confirm_absences`' rule):
591 // fail closed, and say so where the pod-side reader
592 // looks. Transient, so deferred rather than refused.
593 self.state.append_conflict(&ConflictRecord {
594 path: r.path.clone(),
595 foreign_etag: String::new(),
596 preserved_key: None,
597 kind: format!("removal-deferred: cannot stat the path: {e}"),
598 at_unix: now_unix(),
599 })?;
600 pass.deferred += 1;
601 }
602 Ok(m) if !m.is_file() => {
603 refuse(
604 "removal-refused-not-a-file",
605 format!("{} is not a regular file in the workspace", r.path),
606 )?;
607 }
608 Ok(_) => {
609 if local_dirty(&local, be) {
610 refuse(
611 "removal-refused-dirty",
612 format!(
613 "{} has unpublished local changes; the removal {} asked for was not applied and the agent's work is kept — publish or discard the local version, then ask again",
614 r.path, r.author
615 ),
616 )?;
617 continue;
618 }
619 if let Err(e) = std::fs::remove_file(&local) {
620 self.state.append_conflict(&ConflictRecord {
621 path: r.path.clone(),
622 foreign_etag: String::new(),
623 preserved_key: None,
624 kind: format!("removal-unlink-failed (will retry): {e}"),
625 at_unix: now_unix(),
626 })?;
627 pass.deferred += 1;
628 continue;
629 }
630 // CONFIRM: only NotFound is absence.
631 match std::fs::symlink_metadata(&local) {
632 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
633 pass.declared.insert(r.path.clone());
634 pass.applied.push(r.clone());
635 }
636 _ => {
637 self.state.append_conflict(&ConflictRecord {
638 path: r.path.clone(),
639 foreign_etag: String::new(),
640 preserved_key: None,
641 kind: "removal-unlink-unconfirmed (will retry)".into(),
642 at_unix: now_unix(),
643 })?;
644 pass.deferred += 1;
645 }
646 }
647 }
648 }
649 }
650 Ok(pass)
651 }
652
653 /// Act on the inbox document's two request fields (§2.5, D14).
654 ///
655 /// The asymmetry is deliberate and is about blast radius, not
656 /// principle: a boundary request is PERFORMED (it publishes what is
657 /// already on disk and mutates nothing local), a sync request is
658 /// CARRIED into the ticker as advisory news and the agent decides.
659 ///
660 /// Both are watermarked by `requested_unix` rather than consumed by
661 /// a clearing CAS. The field is idempotent state, so a repeat is a
662 /// no-op and a lost clear cannot double-publish; and not writing
663 /// means the doors cost no request at all, which is what §2.5
664 /// promises.
665 fn note_verb_requests(&mut self, doc: &inbox::InboxDoc) -> LeanResult<()> {
666 let mut mark = self.load_verb_watermark()?;
667 if let Some(r) = &doc.sync_request {
668 if r.requested_unix > mark.sync_unix {
669 self.carry_sync_request(r.requested_unix, &r.requestor)?;
670 mark.sync_unix = r.requested_unix;
671 }
672 }
673 if let Some(r) = &doc.boundary_request {
674 if r.requested_unix > mark.boundary_unix {
675 self.request_boundary(
676 &format!("gw:{}:{}", r.requested_unix, r.requestor),
677 Some(format!("boundary requested by {}", r.requestor)),
678 )?;
679 mark.boundary_unix = r.requested_unix;
680 }
681 }
682 self.save_verb_watermark(&mark)
683 }
684
685 fn verb_watermark_path(&self) -> std::path::PathBuf {
686 self.cfg.state_dir().join("verb-requests.json")
687 }
688
689 pub(crate) fn load_verb_watermark(&self) -> LeanResult<VerbWatermark> {
690 let p = self.verb_watermark_path();
691 if !p.exists() {
692 return Ok(VerbWatermark::default());
693 }
694 Ok(std::fs::read(&p)
695 .ok()
696 .and_then(|b| serde_json::from_slice(&b).ok())
697 .unwrap_or_default())
698 }
699
700 fn save_verb_watermark(&self, m: &VerbWatermark) -> LeanResult<()> {
701 let bytes = serde_json::to_vec_pretty(m)
702 .map_err(|e| LeanError::State(format!("verb watermark: {e}")))?;
703 super::control::write_atomic(&self.verb_watermark_path(), &bytes)
704 }
705
706 async fn preserve_conflict_copy(&self, path: &str, etag: &str) -> LeanResult<String> {
707 let key = self.cfg.file_key(path);
708 let dst = self.cfg.conflict_key(&uuid::Uuid::new_v4().to_string(), path);
709 // v1: client-side copy (GET + guarded PUT). The server-side
710 // CopyObject lever is the designed v2 optimization for large
711 // files.
712 let (meta, body) = self.store.get_whole(&key, Some(etag)).await?;
713 let crc = crc64_nvme(&body);
714 let stamps = GenerationStamps {
715 generation: 0,
716 epoch: self
717 .lease
718 .as_ref()
719 .map(|l| l.epoch)
720 .or_else(|| self.state.load_incarnation().ok().flatten().map(|i| i.epoch))
721 .unwrap_or(0),
722 flush_uuid: "conflict-preserve".into(),
723 boundary_source: None,
724 posix: PosixStamps::from_meta(&meta.meta),
725 };
726 self.store
727 .put_whole(&dst, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
728 .await?;
729 Ok(dst)
730 }
731
732 /// Take the SECOND absence observation now, by direct `lstat`.
733 ///
734 /// `scan::classify` withholds a path's delete until absence has
735 /// survived two consecutive scans — the rename-vs-walk race guard
736 /// (`lib.rs:37`). The hazard that rule names is the WALK missing a
737 /// file renamed under it, not deletion itself. On the cadence path
738 /// the second walk arrives at the next floor tick and nobody has
739 /// been promised otherwise. On a DECLARED boundary it cannot wait:
740 /// the ack would claim a coherent point (D1 — "everything
741 /// ordered-before T") while the manifest still cites a file the
742 /// agent removed before it touched the sentinel, and it would say
743 /// so with `report.deleted: 0` and `status: "ok"`.
744 ///
745 /// A direct `lstat` is exactly what the rename-vs-walk guard asks
746 /// for and is immune to the walk race by construction, so the
747 /// second observation costs one syscall per transiently-absent
748 /// path — never a second full pass. The cadence path is unchanged
749 /// and still waits for the second walk.
750 pub(crate) fn confirm_absences(&self, classified: &mut scan::Classified) -> LeanResult<usize> {
751 if classified.first_absence.is_empty() {
752 return Ok(0);
753 }
754 let mut confirmed = 0;
755 for path in std::mem::take(&mut classified.first_absence) {
756 // ONLY NotFound is absence. Every other errno — EACCES on a
757 // parent, EIO, EMFILE, ELOOP — used to read as "the agent
758 // deleted it", and this is the oracle that PUBLISHES the
759 // delete and then DELETES THE OBJECT. An unreadable file is
760 // the one case where guessing is unrecoverable, so it fails
761 // CLOSED, exactly as the walk does at `scan.rs`'s
762 // `symlink_metadata(&path)?`. Two call sites, one syscall:
763 // they must not have opposite error policies.
764 let gone = match std::fs::symlink_metadata(self.cfg.root.join(&path)) {
765 Err(e) if e.kind() == std::io::ErrorKind::NotFound => true,
766 Err(e) => {
767 classified.first_absence.insert(path.clone());
768 return Err(LeanError::State(format!(
769 "cannot confirm the absence of {path:?}: {e} — refusing to publish \
770 a deletion on an unreadable path"
771 )));
772 }
773 Ok(_) => false,
774 };
775 if gone {
776 classified.deletes.insert(path);
777 confirmed += 1;
778 } else {
779 // Back on disk: the rename-vs-walk race, caught exactly
780 // as the rule intends. Still only a first absence.
781 classified.first_absence.insert(path);
782 }
783 }
784 Ok(confirmed)
785 }
786
787 /// Renew the lease if it has gone stale MID-BARRIER, and hand back
788 /// the cell as it was read so the CALLER can fence on it.
789 ///
790 /// Costs nothing on a short barrier: the renewal is skipped unless
791 /// it was already due, so the request count is unchanged — this
792 /// moves WHEN the renewal can happen, not how often.
793 ///
794 /// What this does NOT do, stated because the previous comment
795 /// claimed it did: it never raises `Fenced` for a deposed holder.
796 /// `lease::renew` can only 412 when the cell is still at OUR epoch
797 /// with a token we do not hold; a cell at a FOREIGN epoch is
798 /// skipped here (renewing it would be nonsense), so a deposed
799 /// straggler passed straight through. The 2026-09-03 audit found
800 /// the cadence barrier relying on exactly that phantom fence
801 /// between upload chunks and completing every remaining data PUT
802 /// after a takeover. The cell is returned so the caller can compare
803 /// it against its lease — the cadence barrier uses the read it
804 /// already paid for rather than a separate fence. `None` when there is
805 /// no lease or the read failed: neither is a fence, and the
806 /// ordinary renewal arm will try again.
807 pub(crate) async fn renew_if_due(&mut self) -> LeanResult<Option<EpochState>> {
808 let Some(lease) = self.lease.clone() else { return Ok(None) };
809 let state = match self.store.epoch_read(&self.cfg.epoch_key()).await {
810 Ok(Some(s)) => s,
811 _ => return Ok(None),
812 };
813 let now = super::now_unix();
814 let fresh = state
815 .last_renew_unix
816 .map(|t| now.saturating_sub(t) < RENEW_WITHIN_SECS)
817 .unwrap_or(false);
818 if fresh || state.epoch != lease.epoch {
819 // Not due, or not ours to renew. The caller decides what a
820 // foreign cell means; this function only knows it cannot
821 // renew one.
822 return Ok(Some(state));
823 }
824 super::lease::renew(self).await?;
825 Ok(Some(state))
826 }
827
828 /// The between-chunk fence of the cadence barrier: a cell that no
829 /// longer names this holder at this epoch stops the upload set
830 /// HERE, before the next chunk's PUTs. Every one of those PUTs
831 /// carries If-Match on a baseline etag that still matches — the
832 /// successor has published nothing yet — so without this a deposed
833 /// straggler overwrites the cited generation of every key it still
834 /// had to upload, and every reader's S3-wins arm then adopts the
835 /// uncited bytes silently (audit 2026-09-03, finding 1).
836 fn fence_on_cell(&self, cell: &EpochState) -> LeanResult<()> {
837 let Some(lease) = self.lease.as_ref() else {
838 return Err(LeanError::Fenced("lease dropped mid-barrier".into()));
839 };
840 if cell.epoch != lease.epoch || cell.holder_id != lease.holder_id {
841 return Err(LeanError::Fenced(format!(
842 "deposed between upload chunks: cell at epoch {} holder {} (we are epoch {})",
843 cell.epoch, cell.holder_id, lease.epoch
844 )));
845 }
846 Ok(())
847 }
848
849 /// Steps 2–7. `barrier` = one full publish cycle (the cadence arm).
850 pub async fn run_barrier(&mut self) -> LeanResult<BarrierReport> {
851 self.barrier_inner(false, "manual").await
852 }
853
854 /// The floor tick's barrier. Stamped `cadence` so the bucket can
855 /// answer "which clock installed this boundary?" without the ack —
856 /// the ack is a LOCAL file, and the fleet-visible question is asked
857 /// by operators and by the gateway's `/status`, neither of which
858 /// can see it.
859 pub async fn cadence_barrier(&mut self) -> LeanResult<BarrierReport> {
860 self.barrier_inner(false, "cadence").await
861 }
862
863 /// A barrier whose result somebody is going to ACK: a sentinel
864 /// honor (D1) or the preStop drain (D10). Identical to
865 /// `run_barrier` except that it confirms first-absence paths rather
866 /// than acking a boundary that withholds them.
867 pub async fn declared_barrier(&mut self) -> LeanResult<BarrierReport> {
868 self.barrier_inner(true, "sentinel").await
869 }
870
871 /// `declared_barrier` under an explicit provenance stamp — the
872 /// drain and the budget-deferred honor are the same barrier
873 /// reporting a different clock.
874 pub async fn declared_barrier_as(&mut self, source: &str) -> LeanResult<BarrierReport> {
875 self.barrier_inner(true, source).await
876 }
877
878 async fn barrier_inner(&mut self, declared: bool, source: &str) -> LeanResult<BarrierReport> {
879 // Every publishing path ends here — cadence, sentinel, drain, the
880 // one-shot verb — so a reader is refused here before the first
881 // request, whichever of them a caller reached. The callers that
882 // owe an answer (the publish sentinel, the drain) give it before
883 // this; this is the backstop for the ones that do not.
884 self.refuse_if_read("barrier")?;
885 // No lease is held until the commit section (design 2026-09-13
886 // §4): the consume, the scan and the uploads below run with the
887 // cell at rest, guarded by each object's own etag. What the
888 // uploads are STAMPED with is the last epoch this incarnation
889 // held — informational, like every epoch stamp on an object; the
890 // manifest entries carry the commit section's real epoch.
891 let epoch_hint = self.state.load_incarnation()?.map(|i| i.epoch).unwrap_or(0);
892 let mut report = BarrierReport::default();
893 let barrier_started = std::time::Instant::now();
894 self.trace("barrier_start", serde_json::json!({"source": source, "declared": declared}));
895
896 // STEP 0: finish any rescope a crash left half-applied, before
897 // the scan reads the tree (scoped-read design §4.3). This is the
898 // gate that makes the narrow verb safe at all: a half-applied
899 // narrow is a tree whose files and whose citations disagree, and
900 // `classify` reads that disagreement as an upload or as a
901 // DELETE depending on which half landed. Replay is idempotent,
902 // so the cost on the normal path is one `stat` of a file that
903 // is not there.
904 if let Some(r) = self.replay_scope_intent().await? {
905 report.rescope_replayed = true;
906 eprintln!(
907 "flint-sync: finished an interrupted rescope before the barrier \
908 (uncited {}, unlinked {}, materialised {})",
909 r.uncited, r.unlinked, r.materialized
910 );
911 }
912
913 // Step 1: the cell, read ONCE — HITL entries into the tree, then
914 // the DECLARED removals (delete/rename design §4-§6).
915 let inbox_doc = inbox::load(self.store.as_ref(), &self.cfg).await?.doc;
916 let (consumed, settled_locally) = self.consume_counted(&inbox_doc).await?;
917 report.consumed = consumed.len() + settled_locally;
918 let removals = self.apply_removals(&inbox_doc).await?;
919 report.removed = removals.declared.iter().cloned().collect();
920 report.removals_refused = removals.refused.len();
921 // A refusal is settled NOW, whatever else this barrier does: the
922 // cell is where the human who asked reads the answer, and a
923 // refused removal is never retried, so saying so before the
924 // no-diff return below loses nothing.
925 inbox::settle_removals(self.store.as_ref(), &self.cfg, epoch_hint, &[], &removals.refused)
926 .await?;
927
928 // Step 2: scan-diff against the persisted baseline.
929 let mut baseline = self.state.load_baseline()?;
930 let scanned = scan::scan(&self.cfg.root)?;
931 let mut classified = scan::classify(&scanned, &baseline);
932 if declared {
933 report.absences_confirmed = self.confirm_absences(&mut classified)?;
934 }
935 // A declared removal is a deletion with its basis RECORDED, so
936 // it skips the two-scan guard the walk needs and goes straight
937 // into the delete set — which is what lets a rename's two halves
938 // ride ONE manifest generation (§4).
939 for p in &removals.declared {
940 classified.first_absence.remove(p);
941 if classified.uploads.contains(p) {
942 // The agent re-created the path between the unlink and
943 // the scan. The declaration named the OLD file, which is
944 // gone from the tree (and, for a rename, moved); what is
945 // here now is a new file the agent wrote, and it
946 // publishes as one — its PUT supersedes the old object
947 // under the same key. Deleting it would un-cite the
948 // agent's work for a barrier and GC-skip its object; the
949 // model's install gives the upload the same precedence.
950 continue;
951 }
952 classified.deletes.insert(p.clone());
953 }
954 report.first_absence = classified.first_absence.iter().cloned().collect();
955
956 // Skip-on-no-diff: nothing local, nothing consumed, no pending
957 // citation repair, and the bucket manifest where we left it.
958 let repairs_pending = baseline
959 .entries
960 .iter()
961 .any(|(p, be)| be.size != u64::MAX && baseline.inst_base.get(p) != Some(&be.etag));
962 if classified.uploads.is_empty()
963 && classified.deletes.is_empty()
964 && consumed.is_empty()
965 && classified.first_absence.is_empty()
966 && removals.applied.is_empty()
967 && !repairs_pending
968 {
969 // Read the POINTER, never the entries: the 0b rig measured
970 // the idle tick at 1M entries as 27 s / 1.3 GiB — dominated
971 // by fetching and parsing a 264 MiB document just to read
972 // `seq`. The pointer is a few hundred bytes and carries the
973 // seq in its body, so a GET of it costs one round trip and
974 // answers strictly more than a HEAD of the old object did.
975 //
976 // Legacy workspaces (no pointer yet) keep the HEAD they had.
977 let unchanged = match manifest::load_pointer(self.store.as_ref(), &self.cfg).await? {
978 Some(manifest::LoadedPointer { pointer: p, etag, .. }) => {
979 // The news ticker rides this request for free — D5's
980 // "zero added bucket requests" is still literal.
981 report.observed_seq = Some(p.seq);
982 report.observed_etag = Some(etag.clone());
983 baseline.manifest_etag.as_deref() == Some(etag.as_str())
984 }
985 None => match self.store.head(&self.cfg.manifest_key()).await {
986 Ok(meta) => {
987 report.observed_seq =
988 GenerationStamps::from_meta(&meta.meta).map(|s| s.generation);
989 report.observed_etag = Some(meta.etag.clone());
990 baseline.manifest_etag.as_deref() == Some(meta.etag.as_str())
991 }
992 Err(StoreError::NotFound(_)) => baseline.manifest_etag.is_none(),
993 Err(e) => return Err(e.into()),
994 },
995 };
996 if unchanged {
997 report.no_change = true;
998 report.seq = Some(baseline.seq);
999 // prev_scan still advances (the two-scan rule's clock) —
1000 // but the baseline document (hundreds of MB at 1M
1001 // entries) is rewritten only when the scan SET moved.
1002 // Compare without building the set: both sides iterate
1003 // sorted, and the allocation this used to make was the
1004 // size of the very document it was trying not to write.
1005 if scanned.len() != baseline.prev_scan.len()
1006 || !scanned.keys().eq(baseline.prev_scan.iter())
1007 {
1008 baseline.prev_scan = scanned.keys().cloned().collect();
1009 self.state.save_baseline(&baseline)?;
1010 }
1011 self.trace_barrier_end(&report, barrier_started);
1012 return Ok(report);
1013 }
1014 }
1015
1016 // Step 3: intent journal, then the window (the commitment
1017 // point: the same CAS drops the consumed entries).
1018 let flush_uuid = uuid::Uuid::new_v4().to_string();
1019 self.trace("scan", serde_json::json!({"flush": flush_uuid, "uploads": classified.uploads.len(),
1020 "deletes": classified.deletes.len(), "first_absence": classified.first_absence.len()}));
1021 let mut intent = self.state.load_intent()?;
1022 let prior_uuids = {
1023 let mut v = intent.recent_uuids.clone();
1024 if !intent.flush_uuid.is_empty() {
1025 v.push(intent.flush_uuid.clone());
1026 }
1027 v
1028 };
1029 // Carried forward, not dropped: it stays true until we install
1030 // again, and the merge below is what reads it.
1031 let prev_installed = intent.installed_etag.clone();
1032 intent = IntentJournal {
1033 flush_uuid: flush_uuid.clone(),
1034 keys: classified.uploads.iter().map(|p| self.cfg.file_key(p)).collect(),
1035 recent_uuids: prior_uuids.clone(),
1036 installed_etag: prev_installed.clone(),
1037 installed_foreign: intent.installed_foreign.clone(),
1038 declared_deletes: removals.declared.iter().cloned().collect(),
1039 };
1040 self.state.save_intent(&intent)?;
1041 // The window is NOT opened here any more. It is the bucket's
1042 // "a barrier is committing" sign for HITL writers, and it opens
1043 // in the commit section, under the lease, at the epoch the
1044 // commit really holds. The uploads below race a HITL write the
1045 // way two writers race each other: If-Match decides, the loser
1046 // is preserved and recorded (`LeanNoWindowHolds` proves safety
1047 // never depended on the window).
1048 //
1049 // The consumed entries are not dropped here either — "durably
1050 // in the baseline" is true of a container restart and false of
1051 // a pod REPLACEMENT: the emptyDir goes with the pod. They leave
1052 // the cell with the window clear, after the manifest cites
1053 // them; a successor that finds them re-consumes, idempotently.
1054
1055 // Step 4: guarded uploads, fanned out under a bounded window
1056 // (each key's guard chain is independent; the 412 policy and
1057 // conflict records are applied to the collected results below,
1058 // in deterministic path order).
1059 let mut upserts: BTreeMap<String, LeanEntry> = BTreeMap::new();
1060 let mut parked: BTreeSet<String> = BTreeSet::new();
1061 // Citations whose etag was observed with no lease held (adopted
1062 // uploads, citation repairs): re-read inside the commit section.
1063 let mut observed: BTreeSet<String> = BTreeSet::new();
1064 let mut new_baseline_entries: BTreeMap<String, BaselineEntry> = BTreeMap::new();
1065 let outcomes: Vec<(String, LeanResult<UploadOutcome>)> = {
1066 use futures::stream::{self, StreamExt};
1067 // CHUNKED into waves, so a wave's outcomes are collected
1068 // before the next is fanned out. No lease is held here
1069 // (design 2026-09-13 §4), so nothing renews or fences
1070 // between chunks any more — the between-chunk fence that
1071 // stopped a deposed straggler's PUTs went with the straggler:
1072 // a PUT outside the lease is one legitimate writer's PUT,
1073 // guarded by If-Match, and a collision is preserved and
1074 // superseded, never silently overwritten.
1075 let mut outcomes: Vec<(String, LeanResult<UploadOutcome>)> = Vec::new();
1076 // NOT cfg.fanout. That knob was raised to 128 for the READ
1077 // path, and the read-side measurement (2.5x on 20k small
1078 // files) says nothing about this one, so uploads keep the
1079 // value they were measured at until someone measures them.
1080 // Memory is no longer the reason they are separate: this
1081 // path is byte-bounded too now, by the store's gate
1082 // (`with_upload_inflight_max_bytes`), which charges every
1083 // whole body below and every multipart part inside the
1084 // store from before its read until its PUT returns.
1085 let fanout = self.cfg.upload_fanout.max(1);
1086 let chunk = fanout.saturating_mul(UPLOAD_CHUNK_WAVES).max(1);
1087 let all: Vec<&String> = classified.uploads.iter().collect();
1088 for group in all.chunks(chunk) {
1089 {
1090 let this: &Syncer = &*self;
1091 let mut part: Vec<(String, LeanResult<UploadOutcome>)> =
1092 stream::iter(group.iter().map(|path| {
1093 let path = *path;
1094 let scanned_entry = &scanned[path];
1095 let base = baseline.entries.get(path);
1096 let flush_uuid = &flush_uuid;
1097 let prior_uuids = &prior_uuids;
1098 async move {
1099 let r = this
1100 .upload_one(
1101 path, scanned_entry, base, epoch_hint, flush_uuid,
1102 prior_uuids,
1103 )
1104 .await;
1105 (path.clone(), r)
1106 }
1107 }))
1108 .buffer_unordered(fanout)
1109 .collect()
1110 .await;
1111 outcomes.append(&mut part);
1112 }
1113 }
1114 outcomes
1115 };
1116 let mut outcomes = outcomes;
1117 outcomes.sort_by(|a, b| a.0.cmp(&b.0));
1118 for (path, outcome) in outcomes {
1119 let path = &path;
1120 match outcome? {
1121 UploadOutcome::Published { entry, baseline_entry, adopted } => {
1122 self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "etag": entry.etag,
1123 "outcome": if adopted { "adopted" } else { "put" }}));
1124 if adopted {
1125 observed.insert(path.clone());
1126 }
1127 report.published_bytes += entry.size;
1128 upserts.insert(path.clone(), entry);
1129 new_baseline_entries.insert(path.clone(), baseline_entry);
1130 report.uploaded.push(path.clone());
1131 }
1132 UploadOutcome::Parked { foreign_etag } => {
1133 self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "etag": foreign_etag, "outcome": "parked"}));
1134 parked.insert(path.clone());
1135 self.state.append_conflict(&ConflictRecord {
1136 path: path.clone(),
1137 foreign_etag,
1138 preserved_key: None, // parking preserves in place
1139 kind: "upload-412-parked".into(),
1140 at_unix: now_unix(),
1141 })?;
1142 report.parked.push(path.clone());
1143 }
1144 UploadOutcome::Deferred => {
1145 self.trace("upload", serde_json::json!({"flush": flush_uuid, "path": path, "outcome": "deferred"}));
1146 report.deferred.push(path.clone());
1147 }
1148 }
1149 }
1150
1151 // Citation repairs: paths whose integrated object (the
1152 // baseline) differs from the manifest's citation — consumed
1153 // HITL adoptions and checkout's S3-wins arm. No bytes move; the
1154 // manifest re-cites what this syncer already integrated.
1155 // Without this, an adopted upload is clean-vs-baseline, never
1156 // enters the upload set, and the manifest silently drops it —
1157 // the battery's amputation leg caught exactly that.
1158 let repair_candidates: Vec<String> = baseline
1159 .entries
1160 .iter()
1161 .filter(|(path, be)| {
1162 !classified.uploads.contains(*path)
1163 && !classified.deletes.contains(*path)
1164 && be.size != u64::MAX // consume-dirty sentinel: publishes via upload
1165 && baseline.inst_base.get(*path) != Some(&be.etag)
1166 })
1167 .map(|(path, _)| path.clone())
1168 .collect();
1169 for path in repair_candidates {
1170 let key = self.cfg.file_key(&path);
1171 let be = baseline.entries[&path].clone();
1172 match self.store.head(&key).await {
1173 Ok(meta) if meta.etag == be.etag => {
1174 let crc = repair_crc(&path, &be, &meta)?;
1175 let stamps = GenerationStamps::from_meta(&meta.meta);
1176 let scan_entry = scanned.get(&path);
1177 upserts.insert(
1178 path.clone(),
1179 LeanEntry {
1180 key,
1181 etag: meta.etag.clone(),
1182 crc64_b64: crc,
1183 size: meta.size,
1184 mode: stamps
1185 .as_ref()
1186 .and_then(|s| s.posix)
1187 .map(|p| p.mode)
1188 .or(scan_entry.map(|s| s.mode))
1189 .unwrap_or(0o644),
1190 mtime_unix: scan_entry.map(|s| s.mtime_unix).unwrap_or(0),
1191 generation: stamps.map(|s| s.generation).unwrap_or(be.generation),
1192 epoch: epoch_hint,
1193 },
1194 );
1195 observed.insert(path.clone());
1196 }
1197 // Moved again or gone: the next consume reconciles it.
1198 _ => {}
1199 }
1200 }
1201
1202 // A PULL-ONLY boundary: nothing uploaded, deleted, consumed, removed
1203 // or re-cited, so the merge can only add nothing — and then the
1204 // commit section writes nothing to the bucket (no CAS, no GC, no
1205 // inbox entry to drop) and what remains is local: queue the other
1206 // writers' changes and take theirs as the merge base. That needs no
1207 // fence and no window; claiming for it queued every such writer
1208 // behind the publishing ones (65 of 191 claims in the writers drill).
1209 // A merge that does add something (a mirror flag to restamp, say)
1210 // falls through to the commit section.
1211 if classified.uploads.is_empty()
1212 && classified.deletes.is_empty()
1213 && upserts.is_empty()
1214 && consumed.is_empty()
1215 && removals.applied.is_empty()
1216 && observed.is_empty()
1217 {
1218 let current = manifest::load(self.store.as_ref(), &self.cfg).await?;
1219 let m = self.merge_onto(
1220 current.as_ref(),
1221 prev_installed.as_deref(),
1222 &baseline.inst_base,
1223 &upserts,
1224 &classified.deletes,
1225 &parked,
1226 &flush_uuid,
1227 );
1228 if let Some(handle) = m.expected.as_ref().filter(|_| m.adds_nothing()) {
1229 if !m.foreign.is_empty() || !m.gone.is_empty() {
1230 self.state.queue_foreign(&foreign_changes(&m.foreign, &m.gone))?;
1231 self.trace("queue", serde_json::json!({"flush": flush_uuid, "upserts": m.foreign.len(), "tombstones": m.gone.len(), "fence": false}));
1232 }
1233 report.seq = Some(m.theirs.seq);
1234 report.no_change = true;
1235 report.manifest_etag = Some(handle.etag.clone());
1236 report.observed_seq = Some(m.theirs.seq);
1237 report.observed_etag = Some(handle.etag.clone());
1238 report.foreign_queued = m.foreign.len();
1239 baseline.inst_base = m.theirs.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
1240 baseline.seq = m.theirs.seq;
1241 baseline.manifest_etag = Some(handle.etag.clone());
1242 baseline.prev_scan = scanned.keys().cloned().collect();
1243 self.state.save_baseline(&baseline)?;
1244 self.state.clear_intent_keys()?;
1245 self.trace_barrier_end(&report, barrier_started);
1246 return Ok(report);
1247 }
1248 }
1249
1250 // THE COMMIT SECTION (design 2026-09-13 §4). Everything above ran
1251 // with no lease: the uploads are durable and etag-guarded, and
1252 // nothing is cited yet. Claim the cell now — for the merge, the
1253 // CAS, the deletes and the baseline — and hand it to the next
1254 // waiter when done. This is the only stretch two writers of one
1255 // workspace serialise on, and it is small: one pointer CAS and
1256 // the guarded deletes, milliseconds to seconds, never the upload
1257 // of a checkpoint.
1258 let held = super::lease::claim(self).await?;
1259 let epoch = held.epoch;
1260 eprintln!("flint-sync: publish fence held (epoch {epoch})");
1261 if self.cfg.drill_hold_commit_secs > 0 {
1262 eprintln!(
1263 "flint-sync: DRILL: holding the fence for {}s inside the commit section",
1264 self.cfg.drill_hold_commit_secs
1265 );
1266 tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_commit_secs)).await;
1267 }
1268 let commit: LeanResult<()> = async {
1269 // The entries carry the epoch the commit HOLDS, not the hint
1270 // the uploads were stamped with.
1271 for e in upserts.values_mut() {
1272 e.epoch = epoch;
1273 }
1274 // Step 5: the manifest CAS (three-way merge; bounded retries).
1275 self.verify_not_deposed().await?;
1276 // EVERY citation this barrier adds was read or written with no
1277 // lease held. Another writer's commit may since have uncited
1278 // the path and collected the object: its GC deletes by the etag
1279 // it integrated. That etag names an ADOPTED object (the model's
1280 // LeanBarrierLeaseAdoptBlind) — and, since an S3 whole-PUT etag
1281 // is the MD5 of the bytes, it also names this barrier's own PUT
1282 // of IDENTICAL bytes (finding 13, the writers drill's A3: a
1283 // same-content rewrite re-uploaded, the peer's GC deleted it,
1284 // the commit cited a hole). Collection runs only inside a
1285 // commit section, so a re-read HERE, holding the fence, cannot
1286 // be overtaken before this CAS. Whatever is gone or replaced is
1287 // withheld: parked, recorded, and left dirty for the next
1288 // barrier to publish with a PUT of its own.
1289 let heads: Vec<(String, LeanResult<bool>)> = {
1290 use futures::stream::{self, StreamExt};
1291 let this: &Syncer = &*self;
1292 stream::iter(upserts.iter().map(|(path, cited)| async move {
1293 let still = match this.store.head(&cited.key).await {
1294 Ok(meta) => Ok(meta.etag == cited.etag),
1295 Err(StoreError::NotFound(_)) => Ok(false),
1296 Err(e) => Err(e.into()),
1297 };
1298 (path.clone(), still)
1299 }))
1300 .buffer_unordered(self.cfg.upload_fanout.max(1))
1301 .collect()
1302 .await
1303 };
1304 let mut heads = heads;
1305 heads.sort_by(|a, b| a.0.cmp(&b.0));
1306 for (path, still) in heads {
1307 let still_there = still?;
1308 let was_observed = observed.contains(&path);
1309 let cited = &upserts[&path];
1310 self.trace("observed", serde_json::json!({"flush": flush_uuid, "path": path, "etag": cited.etag, "still": still_there, "own_put": !was_observed}));
1311 if still_there {
1312 continue;
1313 }
1314 let gone = upserts.remove(&path).expect("looked up above");
1315 new_baseline_entries.remove(&path);
1316 report.uploaded.retain(|p| p != &path);
1317 report.published_bytes = report.published_bytes.saturating_sub(gone.size);
1318 parked.insert(path.clone());
1319 let kind = if was_observed {
1320 "adopt-withheld: the object this barrier found already at its key was \
1321 replaced or collected before its commit; nothing cited, the path stays \
1322 dirty and publishes next barrier"
1323 } else {
1324 "upload-withheld: the object this barrier PUT was replaced or collected \
1325 before its commit (a peer's GC recognizes the etag of identical bytes); \
1326 nothing cited, the path stays dirty and publishes next barrier"
1327 };
1328 self.state.append_conflict(&ConflictRecord {
1329 path: path.clone(),
1330 foreign_etag: gone.etag,
1331 preserved_key: None,
1332 kind: kind.into(),
1333 at_unix: now_unix(),
1334 })?;
1335 report.parked.push(path.clone());
1336 }
1337 // The window: the bucket-visible "a barrier is committing" sign
1338 // HITL writers wait on, at the epoch this commit holds.
1339 let deadline = now_unix() + self.cfg.window_slack_secs;
1340 inbox::open_window(self.store.as_ref(), &self.cfg, epoch, deadline).await?;
1341 let mut foreign_entries: Vec<(String, LeanEntry)> = vec![];
1342 let mut foreign_gone: Vec<String> = vec![];
1343 // Set when the merge added nothing to the document: nothing is
1344 // installed, and theirs becomes the merge base as it stands.
1345 let mut installed_nothing = false;
1346 let mut attempt = 0;
1347 let (installed, installed_etag) = loop {
1348 attempt += 1;
1349 if attempt > 4 {
1350 return Err(LeanError::State(
1351 "manifest CAS lost 4 merge races — refusing this barrier".into(),
1352 ));
1353 }
1354 let current = manifest::load(self.store.as_ref(), &self.cfg).await?;
1355 let MergeOnto { theirs, expected, merged, foreign, gone } = self.merge_onto(
1356 current.as_ref(),
1357 prev_installed.as_deref(),
1358 &baseline.inst_base,
1359 &upserts,
1360 &classified.deletes,
1361 &parked,
1362 &flush_uuid,
1363 );
1364 // Nothing of ours changes the document — a barrier that only
1365 // found the manifest moved by another writer. Installing it
1366 // anyway was an empty generation, and the peer's next tick
1367 // then found the manifest moved and did the same: two idle
1368 // writers traded generations (and cell claims) for as long
1369 // as both ran. Theirs becomes the merge base as it stands.
1370 if let Some(handle) = expected.as_ref() {
1371 if merged.entries == theirs.entries && merged.sole_writer == theirs.sole_writer {
1372 foreign_entries = foreign;
1373 foreign_gone = gone;
1374 installed_nothing = true;
1375 break (theirs, handle.etag.clone());
1376 }
1377 }
1378 match manifest::cas_write_stamped(
1379 self.store.as_ref(),
1380 &self.cfg,
1381 &merged,
1382 expected.as_ref(),
1383 epoch,
1384 &flush_uuid,
1385 Some(source),
1386 )
1387 .await
1388 {
1389 Ok(meta) => {
1390 // Before the deletes and before step 7: this is the
1391 // only record that survives a crash in that window.
1392 self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1393 "expected": expected.as_ref().map(|h| h.etag.clone()), "etag": meta.etag, "result": "ok"}));
1394 intent.installed_etag = Some(meta.etag.clone());
1395 intent.installed_foreign = foreign_changes(&foreign, &gone);
1396 self.state.save_intent(&intent)?;
1397 foreign_entries = foreign;
1398 foreign_gone = gone;
1399 break (merged, meta.etag);
1400 }
1401 Err(LeanError::Store(StoreError::PreconditionFailed(_)))
1402 | Err(LeanError::Store(StoreError::Conflict(_))) => {
1403 self.trace("cas", serde_json::json!({"flush": flush_uuid, "seq": merged.seq,
1404 "expected": expected.as_ref().map(|h| h.etag.clone()), "result": "lost"}));
1405 // Re-verify the cell before retrying: a rotation is
1406 // exactly this 412, and re-merging past it would be
1407 // the straggler install.
1408 self.verify_not_deposed().await?;
1409 continue;
1410 }
1411 Err(e) => return Err(e),
1412 }
1413 };
1414 report.seq = Some(installed.seq);
1415 report.no_change = installed_nothing;
1416 // A fused install IS a coherent point, and cadence/hybrid have
1417 // exactly one source. The gauges must not report "no boundary
1418 // ever" on a workspace that publishes every minute. A barrier
1419 // that installed nothing marks no boundary of its own.
1420 if !installed_nothing {
1421 self.note_boundary("cadence", installed.seq)?;
1422 }
1423 report.manifest_etag = Some(installed_etag.clone());
1424 report.observed_seq = Some(installed.seq);
1425 report.observed_etag = Some(installed_etag.clone());
1426 report.foreign_queued = foreign_entries.len();
1427
1428 // Step 6: deletes LAST — GC of keys the NEW manifest no longer
1429 // references, HEAD-guarded on the recognized ETag.
1430 let mut swept = 0usize;
1431 let mut gc_held = false;
1432 for path in &classified.deletes {
1433 // A mass delete is the one long stretch of the commit
1434 // section: keep the token moving so a waiter does not count
1435 // a live holder dead, and fence on what the read returns.
1436 swept += 1;
1437 if swept % 200 == 0 {
1438 if let Some(cell) = self.renew_if_due().await? {
1439 self.fence_on_cell(&cell)?;
1440 }
1441 }
1442 if installed.entries.contains_key(path) {
1443 // delete/modify resolved foreign-wins: not garbage, and
1444 // not in this boundary either.
1445 report.outranked.push(path.clone());
1446 continue;
1447 }
1448 let key = self.cfg.file_key(path);
1449 let recognized = baseline.entries.get(path).map(|b| b.etag.clone()).or_else(|| {
1450 // A DECLARED removal of a path this tree never held (a
1451 // scoped workspace's out-of-scope citation): what the
1452 // declaration named is the object the manifest cited
1453 // when this barrier began, and that is the merge base.
1454 removals
1455 .declared
1456 .contains(path)
1457 .then(|| baseline.inst_base.get(path).cloned())
1458 .flatten()
1459 });
1460 // The HEAD answers most paths in one request (gone, or an
1461 // etag we do not recognize). The DELETE itself carries
1462 // If-Match on the recognized etag: another writer's
1463 // upload holds no lease and can replace the object
1464 // between the two requests, and an unconditional delete
1465 // then removed bytes that writer's commit cites (the
1466 // model's LeanBarrierLeaseGCUnconditional).
1467 let unrecognized = match self.store.head(&key).await {
1468 Err(StoreError::NotFound(_)) => {
1469 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": null, "result": "absent"}));
1470 report.deleted.push(path.clone());
1471 continue;
1472 }
1473 Ok(meta) if Some(&meta.etag) == recognized.as_ref() => {
1474 if self.cfg.drill_hold_gc_secs > 0 && !gc_held {
1475 gc_held = true;
1476 eprintln!(
1477 "flint-sync: DRILL: holding the GC for {}s between the HEAD of {path} and its DELETE",
1478 self.cfg.drill_hold_gc_secs
1479 );
1480 self.trace("drill_hold", serde_json::json!({"flush": flush_uuid, "where": "gc", "path": path,
1481 "secs": self.cfg.drill_hold_gc_secs}));
1482 tokio::time::sleep(std::time::Duration::from_secs(self.cfg.drill_hold_gc_secs)).await;
1483 }
1484 match self.store.delete_if_match(&key, &meta.etag).await {
1485 Ok(()) | Err(StoreError::NotFound(_)) => {
1486 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": meta.etag, "result": "deleted"}));
1487 report.deleted.push(path.clone());
1488 continue;
1489 }
1490 // Replaced between the HEAD and the DELETE:
1491 // whatever is there now is not ours to
1492 // collect. Name it, if it still exists.
1493 Err(StoreError::PreconditionFailed(_)) => match self.store.head(&key).await {
1494 Ok(now) => now.etag,
1495 Err(StoreError::NotFound(_)) => {
1496 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": meta.etag, "result": "replaced-absent"}));
1497 report.deleted.push(path.clone());
1498 continue;
1499 }
1500 Err(e) => return Err(e.into()),
1501 },
1502 Err(e) => return Err(e.into()),
1503 }
1504 }
1505 Ok(meta) => meta.etag,
1506 Err(e) => return Err(e.into()),
1507 };
1508 // An ETag this syncer does not recognize is NEVER deleted
1509 // (a HITL re-create landed after our CAS, or another
1510 // writer's upload).
1511 self.trace("gc", serde_json::json!({"flush": flush_uuid, "path": path, "head": unrecognized,
1512 "recognized": recognized, "result": "skip"}));
1513 self.state.append_conflict(&ConflictRecord {
1514 path: path.clone(),
1515 foreign_etag: unrecognized,
1516 preserved_key: None,
1517 kind: "gc-skip".into(),
1518 at_unix: now_unix(),
1519 })?;
1520 }
1521
1522 // Step 7: the foreign queue, then the baseline rewrite, intent
1523 // clear and window clear.
1524 //
1525 // Other writers' changes go to THIS writer's queue BEFORE the
1526 // merge base below moves past them: a crash between the two
1527 // re-applies them idempotently, where the other order would
1528 // leave a base that claims changes the tree never received.
1529 if !foreign_entries.is_empty() || !foreign_gone.is_empty() {
1530 self.state.queue_foreign(&foreign_changes(&foreign_entries, &foreign_gone))?;
1531 self.trace("queue", serde_json::json!({"flush": flush_uuid, "upserts": foreign_entries.len(), "tombstones": foreign_gone.len()}));
1532 }
1533 for (path, be) in new_baseline_entries {
1534 baseline.entries.insert(path, be);
1535 }
1536 for path in &report.deleted {
1537 baseline.entries.remove(path);
1538 }
1539 baseline.inst_base =
1540 installed.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
1541 baseline.seq = installed.seq;
1542 baseline.manifest_etag = Some(installed_etag);
1543 baseline.prev_scan = scanned.keys().cloned().collect();
1544 self.state.save_baseline(&baseline)?;
1545 self.state.clear_intent_keys()?;
1546 // No `merge-preserved` entries into the SHARED inbox any more:
1547 // they are this writer's, and sit in its local queue.
1548 inbox::clear_window_settling(
1549 self.store.as_ref(),
1550 &self.cfg,
1551 epoch,
1552 &[],
1553 &consumed,
1554 &removals.applied,
1555 )
1556 .await?;
1557 self.trace("window_clear", serde_json::json!({"flush": flush_uuid}));
1558 // Reap superseded generations. Immutable metadata that is never
1559 // collected is a leak that grows by a whole manifest per
1560 // publish, and this also collects the orphan a crash between the
1561 // entries PUT and the pointer CAS leaves behind. Best effort by
1562 // design: a publish that succeeded is not un-done by a failure
1563 // to tidy up after it.
1564 let swept_generations = match manifest::sweep_generations(self.store.as_ref(), &self.cfg).await {
1565 Ok(0) => 0,
1566 Ok(n) => {
1567 eprintln!("flint-sync: reaped {n} superseded manifest generation(s)");
1568 n
1569 }
1570 Err(e) => {
1571 eprintln!("flint-sync: generation sweep: {e}");
1572 0
1573 }
1574 };
1575 self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "generations", "removed": swept_generations}));
1576 // And the CHUNK reaper, same reason, same best-effort terms.
1577 // `sweep_chunks` returns 0 without a request on a workspace that
1578 // is not chunked, so this costs nothing until the layout moves.
1579 //
1580 // It was written to four model-established rules, guarded by six
1581 // mutation configs, and called from nothing but its own tests —
1582 // so with chunking on, every publish left its superseded chunks
1583 // in the bucket forever. The rules ran in the model and in the
1584 // suite and never once in production. `the_barrier_reaps...`
1585 // below is the test that asks whether this line exists at all.
1586 let swept_chunks = match manifest::sweep_chunks(self.store.as_ref(), &self.cfg).await {
1587 Ok(0) => 0,
1588 Ok(n) => {
1589 eprintln!("flint-sync: reaped {n} unreferenced manifest chunk(s)");
1590 n
1591 }
1592 Err(e) => {
1593 eprintln!("flint-sync: chunk sweep: {e}");
1594 0
1595 }
1596 };
1597 self.trace("sweep", serde_json::json!({"flush": flush_uuid, "what": "chunks", "removed": swept_chunks}));
1598 Ok(())
1599 }
1600 .await;
1601 // Hand the cell on whatever happened — unless it is no longer
1602 // ours to hand on: a fence means a successor holds it, and the
1603 // barrier is simply abandoned (its manifest never installed; its
1604 // uploads stand, and the next barrier adopts them by flush_uuid).
1605 match &commit {
1606 Err(LeanError::Fenced(m)) => {
1607 self.trace("fence", serde_json::json!({"where": "commit", "detail": m}));
1608 self.lease = None
1609 }
1610 _ => {
1611 if let Err(e) = super::lease::release(self).await {
1612 eprintln!(
1613 "flint-sync: the publish fence could not be released ({e}); a waiter \
1614 deposes it after the quiet threshold"
1615 );
1616 }
1617 }
1618 }
1619 commit?;
1620 self.trace_barrier_end(&report, barrier_started);
1621 Ok(report)
1622 }
1623
1624 fn trace_barrier_end(&self, report: &BarrierReport, started: std::time::Instant) {
1625 if self.cfg.event_trace.is_none() {
1626 return;
1627 }
1628 self.trace(
1629 "barrier_end",
1630 serde_json::json!({
1631 "seq": report.seq, "uploaded": report.uploaded.len(), "deleted": report.deleted.len(),
1632 "parked": report.parked.len(), "outranked": report.outranked.len(),
1633 "consumed": report.consumed, "no_change": report.no_change,
1634 "ms": started.elapsed().as_millis() as u64, "requests": self.trace_requests(),
1635 }),
1636 );
1637 }
1638
1639 /// The > whole_put_max path: contiguous `PartSource::Local` chunks
1640 /// through `compose_generation` (streaming multipart; the store
1641 /// aborts its partial assembly on every failure path). The CRC is
1642 /// computed by a streaming pass first; a writer racing the compose
1643 /// fails server-side validation and the path DEFERS to the next
1644 /// barrier — publish-possibly-torn is put_whole's documented
1645 /// dilemma, not this path's.
1646 #[allow(clippy::too_many_arguments)]
1647 #[allow(clippy::too_many_arguments)]
1648 async fn upload_compose(
1649 &self,
1650 path: &str,
1651 key: &str,
1652 local_path: &Path,
1653 size: u64,
1654 scanned: &scan::ScanEntry,
1655 condition: PutCondition,
1656 stamps: GenerationStamps,
1657 generation: u64,
1658 epoch: u64,
1659 prior_uuids: &[String],
1660 ) -> LeanResult<UploadOutcome> {
1661 // NO PRE-PASS. The store accumulates the full-object CRC from
1662 // the parts as it reads them for upload, so this file is read
1663 // ONCE (the 2026-09-10 door drill measured the second read at
1664 // roughly 11 s of the `mixed` workload's 24.3 s).
1665 // Part grid: within [min_part_size, ...], at most max_parts,
1666 // contiguous from 0.
1667 let min_part = self.store.min_part_size().max(1);
1668 let max_parts = self.store.max_parts().max(1) as u64;
1669 let mut chunk = self.cfg.whole_put_max.max(min_part);
1670 let need = size.div_ceil(chunk);
1671 if need > max_parts {
1672 chunk = size.div_ceil(max_parts).div_ceil(min_part) * min_part;
1673 }
1674 let mut parts = vec![];
1675 let mut off = 0u64;
1676 while off < size {
1677 let len = chunk.min(size - off);
1678 parts.push(flint_store::PartSource::Local { offset: off, len });
1679 off += len;
1680 }
1681
1682
1683 // At most three attempts: the caller's condition; then once more
1684 // If-Match on what a 412 showed us — our own torn Complete
1685 // (recognized through the crash journal, review 2026-09-12
1686 // atomicity-2), or a foreign version now PRESERVED (inbox-1) —
1687 // or a create if the base object vanished (inbox-5). A 412 on
1688 // the retry means the path is being written under us: park.
1689 let mut condition = condition;
1690 for attempt in 0..3u32 {
1691 let spec = flint_store::ComposeSpec {
1692 progress: None,
1693 key,
1694 local_path,
1695 parts: parts.clone(),
1696 base_key: None,
1697 base_etag: None,
1698 condition: condition.clone(),
1699 stamps: stamps.clone(),
1700 crc64: None,
1701 };
1702 match self.store.compose_generation(&spec).await {
1703 Ok(meta) => {
1704 // The store computed it; refuse to cite bytes nothing
1705 // vouched for rather than invent a number for the
1706 // manifest. A backend that returns no checksum has not
1707 // validated the publish server-side either.
1708 let crc = meta
1709 .crc64_b64
1710 .as_deref()
1711 .and_then(flint_store::crc64_from_b64)
1712 .ok_or_else(|| {
1713 LeanError::State(format!(
1714 "compose of {key} returned no full-object checksum — refusing to \
1715 cite bytes nothing vouched for"
1716 ))
1717 })?;
1718 return Ok(UploadOutcome::published(
1719 path, key.to_string(), meta.etag, crc, size, scanned, generation, epoch,
1720 ));
1721 }
1722 Err(StoreError::ChecksumMismatch(_)) | Err(StoreError::NoSuchUpload(_)) => {
1723 // Drift mid-compose, or the operator sweep aborted us:
1724 // nothing published; the next barrier re-queues.
1725 return Ok(UploadOutcome::Deferred);
1726 }
1727 // S3's answer to If-Match on a missing key (see upload_one):
1728 // the base vanished, and a vanished base is a create.
1729 Err(StoreError::NotFound(_)) if matches!(condition, PutCondition::IfMatch(_)) => {
1730 condition = PutCondition::IfNoneMatchAny;
1731 continue;
1732 }
1733 Err(StoreError::PreconditionFailed(_)) => {
1734 // The AdoptOwn recognizer compares OUR bytes' checksum
1735 // against what landed, and without the pre-pass we do
1736 // not have one — so the file is read HERE, on the rare
1737 // recovery path, instead of on every publish.
1738 let crc = file_crc(local_path)?;
1739 let head = match self.store.head(key).await {
1740 Ok(h) => h,
1741 Err(StoreError::NotFound(_)) => {
1742 condition = PutCondition::IfNoneMatchAny;
1743 continue;
1744 }
1745 Err(e) => return Err(e.into()),
1746 };
1747 let head_stamps = GenerationStamps::from_meta(&head.meta);
1748 if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
1749 // These bytes are already there (a torn Complete):
1750 // cite them — re-read under the fence first.
1751 let g = head_stamps.map(|s| s.generation).unwrap_or(generation);
1752 return Ok(UploadOutcome::adopted(
1753 path, key.to_string(), head.etag, crc, head.size, scanned, g, epoch,
1754 ));
1755 }
1756 if attempt >= 1 {
1757 return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1758 }
1759 let own = head_stamps
1760 .as_ref()
1761 .map(|s| s.flush_uuid == stamps.flush_uuid || prior_uuids.contains(&s.flush_uuid))
1762 .unwrap_or(false);
1763 if !own && self.preserve_foreign_412(path, &head).await.is_none() {
1764 return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1765 }
1766 condition = PutCondition::IfMatch(head.etag);
1767 }
1768 Err(e) => return Err(e.into()),
1769 }
1770 }
1771 Ok(UploadOutcome::Deferred)
1772 }
1773
1774
1775 async fn upload_one(
1776 &self,
1777 path: &str,
1778 scanned: &scan::ScanEntry,
1779 base: Option<&BaselineEntry>,
1780 epoch: u64,
1781 flush_uuid: &str,
1782 prior_uuids: &[String],
1783 ) -> LeanResult<UploadOutcome> {
1784 let key = self.cfg.file_key(path);
1785 let local_path = self.cfg.root.join(path);
1786 let generation = base.map(|b| b.generation + 1).unwrap_or(1);
1787 // Review 2026-09-12, atomicity-6: the scan skipped symlinks; this
1788 // read followed them. A regular file swapped for a symlink between
1789 // the two published the link's TARGET — a file outside the
1790 // workspace, the syncer's own /proc/self/environ included. Every
1791 // stat here is an lstat and the read opens O_NOFOLLOW; a path that
1792 // is no longer a regular file is refused, recorded, and deferred
1793 // (the next scan skips it, and the two-scan rule retires it).
1794 let lmeta = std::fs::symlink_metadata(&local_path)
1795 .map_err(|e| LeanError::State(format!("stat {}: {e}", local_path.display())))?;
1796 if !lmeta.is_file() {
1797 self.state.append_conflict(&ConflictRecord {
1798 path: path.to_string(),
1799 foreign_etag: String::new(),
1800 preserved_key: None,
1801 kind: "upload-refused-not-regular: no longer a regular file after the scan (a symlink \
1802 or special file replaced it); nothing published"
1803 .into(),
1804 at_unix: now_unix(),
1805 })?;
1806 return Ok(UploadOutcome::Deferred);
1807 }
1808 let posix = Some(PosixStamps::from_metadata(&lmeta));
1809 let stamps = GenerationStamps {
1810 generation,
1811 epoch,
1812 flush_uuid: flush_uuid.to_string(),
1813 boundary_source: None,
1814 posix,
1815 };
1816 let condition = match base {
1817 Some(b) => PutCondition::IfMatch(b.etag.clone()),
1818 None => PutCondition::IfNoneMatchAny,
1819 };
1820 let size = lmeta.len();
1821 if size > self.cfg.whole_put_max {
1822 // Streaming multipart compose: put_whole is never fed past
1823 // whole_put_max (unbounded memory + S3's 5 GiB wall).
1824 return self
1825 .upload_compose(
1826 path, &key, &local_path, size, scanned, condition, stamps, generation, epoch,
1827 prior_uuids,
1828 )
1829 .await;
1830 }
1831 // The upload byte gate — the store's own, shared with the compose
1832 // arm above, which charges its parts inside the store. This body
1833 // is charged from before it is read (the read is the allocation)
1834 // until its PUT, retried or not, has returned: the permit lives
1835 // to the end of this function. `None` = a store with no gate.
1836 let _upload_permit = match self.store.upload_gate() {
1837 Some(g) => Some(g.acquire(size).await),
1838 None => None,
1839 };
1840 // `Bytes::from(Vec<u8>)` TAKES OWNERSHIP without copying, so the
1841 // old `Bytes::from(body.clone())` was a full memcpy of every
1842 // published file body — bought solely to leave `body` intact for
1843 // the 412 retry below, which is reached almost never. Build the
1844 // `Bytes` once; the retry gets a refcount clone.
1845 let body = Bytes::from(read_nofollow(&local_path)?);
1846 let uploaded_len = body.len() as u64;
1847 let crc = crc64_nvme(&body);
1848 match self
1849 .store
1850 .put_whole(&key, body.clone(), &condition, &stamps, crc)
1851 .await
1852 {
1853 Ok(meta) => Ok(UploadOutcome::published(
1854 path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1855 )),
1856 // S3 answers If-Match on a key that no longer exists with 404
1857 // NoSuchKey, not 412 (Ozone answers 412, and so did the
1858 // double): the base is gone — a peer's GC collected it, or a
1859 // bucket-level delete. The same rule as the 412 arm's vanished
1860 // HEAD below: a vanished base is a create. Without this arm the
1861 // rule never ran on S3, and a writer whose edited path a peer
1862 // deleted failed EVERY barrier (drill host leg H2, 2026-09-13).
1863 Err(StoreError::NotFound(_)) if matches!(condition, PutCondition::IfMatch(_)) => {
1864 let meta = self
1865 .store
1866 .put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
1867 .await?;
1868 Ok(UploadOutcome::published(
1869 path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1870 ))
1871 }
1872 Err(StoreError::PreconditionFailed(_)) => {
1873 // The 412 policy: my own crashed/torn PUT ⇒ adopt; a
1874 // foreign version ⇒ the consume-dirty rule, at upload time
1875 // (preserve it, then supersede it knowingly). NEVER the
1876 // inherited LOCAL-WINS overwrite, and never a park with
1877 // no way out.
1878 let head = match self.store.head(&key).await {
1879 Ok(h) => h,
1880 Err(StoreError::NotFound(_)) => {
1881 // Review 2026-09-12, inbox-5: the base object is
1882 // gone (a bucket-level delete, or our own GC after
1883 // a crash between the CAS and the baseline
1884 // rewrite). Failing the barrier here failed EVERY
1885 // barrier, forever. A vanished base is a create.
1886 let meta = self
1887 .store
1888 .put_whole(&key, body, &PutCondition::IfNoneMatchAny, &stamps, crc)
1889 .await?;
1890 return Ok(UploadOutcome::published(
1891 path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1892 ));
1893 }
1894 Err(e) => return Err(e.into()),
1895 };
1896 let head_stamps = GenerationStamps::from_meta(&head.meta);
1897 let own = head_stamps
1898 .as_ref()
1899 .map(|s| s.flush_uuid == flush_uuid || prior_uuids.contains(&s.flush_uuid))
1900 .unwrap_or(false);
1901 if head.crc64_b64.as_deref() == Some(crc64_to_b64(crc).as_str()) {
1902 // Bytes already there (a torn response, ours or a
1903 // foreign write of the same content): cite it — re-read
1904 // under the fence first.
1905 let g = head_stamps.map(|s| s.generation).unwrap_or(generation);
1906 return Ok(UploadOutcome::adopted(
1907 path, key, head.etag, crc, head.size, scanned, g, epoch,
1908 ));
1909 }
1910 if !own && self.preserve_foreign_412(path, &head).await.is_none() {
1911 return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1912 }
1913 // Our earlier PUT (older content), or a foreign version
1914 // now preserved: supersede it knowingly, If-Match on what
1915 // we saw. A second 412 means the path is being written
1916 // under us right now; that one parks.
1917 let meta = match self
1918 .store
1919 .put_whole(&key, body, &PutCondition::IfMatch(head.etag.clone()), &stamps, crc)
1920 .await
1921 {
1922 Ok(m) => m,
1923 // 412 (a writer landed between) or 404 (a GC removed
1924 // it between the HEAD and this PUT): written under us
1925 // right now. Park; the next barrier retries.
1926 Err(StoreError::PreconditionFailed(_)) | Err(StoreError::NotFound(_)) => {
1927 return Ok(UploadOutcome::Parked { foreign_etag: head.etag });
1928 }
1929 Err(e) => return Err(e.into()),
1930 };
1931 Ok(UploadOutcome::published(
1932 path, key, meta.etag, crc, uploaded_len, scanned, generation, epoch,
1933 ))
1934 }
1935 Err(e) => Err(e.into()),
1936 }
1937 }
1938
1939 /// A 412 against a version this syncer did not write, met at upload
1940 /// time. The contract's rule for a foreign write to a path the agent
1941 /// modified is the consume-dirty rule — the agent's version wins and
1942 /// the foreign bytes are preserved — and a "park" was that case with
1943 /// no resolution: nothing ever un-parked, and every ack said `ok`
1944 /// (review 2026-09-12, inbox-1). Preserve, record, and let the caller
1945 /// supersede. `None` = the preserve failed; the caller parks, and the
1946 /// boundary is `partial`.
1947 async fn preserve_foreign_412(&self, path: &str, head: &flint_store::ObjectMeta) -> Option<()> {
1948 let preserved = match self.preserve_conflict_copy(path, &head.etag).await {
1949 Ok(k) => k,
1950 Err(e) => {
1951 eprintln!("flint-sync: {path}: foreign version {} could not be preserved ({e}); parking", head.etag);
1952 return None;
1953 }
1954 };
1955 self.state
1956 .append_conflict(&ConflictRecord {
1957 path: path.to_string(),
1958 foreign_etag: head.etag.clone(),
1959 preserved_key: Some(preserved),
1960 kind: "upload-412-preserved".into(),
1961 at_unix: now_unix(),
1962 })
1963 .ok()?;
1964 Some(())
1965 }
1966}
1967
1968/// Read a workspace file for upload without following a symlink at the
1969/// final component, and refuse anything that is not a regular file once
1970/// open (review 2026-09-12, atomicity-6).
1971fn read_nofollow(p: &Path) -> std::io::Result<Vec<u8>> {
1972 use std::io::Read;
1973 use std::os::unix::fs::OpenOptionsExt;
1974 let mut f = std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NOFOLLOW).open(p)?;
1975 if !f.metadata()?.is_file() {
1976 return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "not a regular file"));
1977 }
1978 let mut v = Vec::new();
1979 f.read_to_end(&mut v)?;
1980 Ok(v)
1981}
1982
1983enum UploadOutcome {
1984 /// `adopted`: the etag was OBSERVED at the key (bytes already there,
1985 /// no PUT of ours produced it), so the commit section must re-read it
1986 /// before citing — see `verify_observed_citations` in the barrier.
1987 Published { entry: LeanEntry, baseline_entry: BaselineEntry, adopted: bool },
1988 Parked { foreign_etag: String },
1989 /// The source drifted mid-transfer (checksum refused server-side)
1990 /// or the assembly was swept: publish nothing, advance nothing —
1991 /// the next scan re-queues the path.
1992 Deferred,
1993}
1994
1995impl UploadOutcome {
1996 #[allow(clippy::too_many_arguments)]
1997 fn published(
1998 path: &str,
1999 key: String,
2000 etag: String,
2001 crc: u64,
2002 uploaded_len: u64,
2003 scanned: &scan::ScanEntry,
2004 generation: u64,
2005 epoch: u64,
2006 ) -> UploadOutcome {
2007 let _ = path;
2008 UploadOutcome::Published {
2009 adopted: false,
2010 entry: LeanEntry {
2011 key,
2012 etag: etag.clone(),
2013 crc64_b64: crc64_to_b64(crc),
2014 // The length the object HAS (review 2026-09-12,
2015 // atomicity-1): the scanned size was cited while the
2016 // body was read fresh, so a file that grew between the
2017 // two was cited short and every fresh checkout of it
2018 // failed its CRC fold — the successor never started.
2019 size: uploaded_len,
2020 mode: scanned.mode,
2021 mtime_unix: scanned.mtime_unix,
2022 generation,
2023 epoch,
2024 },
2025 baseline_entry: BaselineEntry {
2026 etag,
2027 generation,
2028 // The PRE-read stat: if the agent wrote during our read,
2029 // the next scan sees the drift and re-queues (the
2030 // re-stat/re-queue valve).
2031 size: scanned.size,
2032 mtime_unix: scanned.mtime_unix,
2033 mtime_nanos: Some(scanned.mtime_nanos),
2034 crc64_b64: Some(crc64_to_b64(crc)),
2035 },
2036 }
2037 }
2038}
2039
2040impl UploadOutcome {
2041 /// `published`, for bytes this barrier found already at the key and
2042 /// cites without a PUT of its own.
2043 #[allow(clippy::too_many_arguments)]
2044 fn adopted(
2045 path: &str,
2046 key: String,
2047 etag: String,
2048 crc: u64,
2049 uploaded_len: u64,
2050 scanned: &scan::ScanEntry,
2051 generation: u64,
2052 epoch: u64,
2053 ) -> UploadOutcome {
2054 match UploadOutcome::published(path, key, etag, crc, uploaded_len, scanned, generation, epoch) {
2055 UploadOutcome::Published { entry, baseline_entry, .. } => {
2056 UploadOutcome::Published { entry, baseline_entry, adopted: true }
2057 }
2058 other => other,
2059 }
2060 }
2061}
2062
2063fn local_dirty(local: &Path, base: Option<&BaselineEntry>) -> bool {
2064 match (std::fs::metadata(local), base) {
2065 (Err(_), None) => false, // both absent: clean
2066 (Err(_), Some(_)) => true, // locally deleted vs baseline
2067 (Ok(_), None) => true, // local exists, never published
2068 (Ok(m), Some(b)) => {
2069 scan::stat_changed(b.size, b.mtime_unix, b.mtime_nanos, m.len(), mtime_of(&m), mtime_nanos_of(&m))
2070 }
2071 }
2072}
2073
2074/// Full-object CRC-64/NVME of a local file, streamed.
2075///
2076/// Reached only by the 412 recovery path: the ordinary publish gets its
2077/// checksum from the store, which computes one while uploading.
2078/// The CRC a citation repair cites: the bytes this syncer integrated,
2079/// as the baseline recorded them when it wrote or uploaded them. The
2080/// HEAD's own checksum is a cross-check when the backend offers one,
2081/// never the source — Ozone offers none, a HITL uploader may have sent
2082/// none, and the manifest must carry a CRC either way, because every
2083/// reader now verifies a fresh fetch against it.
2084///
2085/// A baseline entry with no CRC is refused loudly rather than cited
2086/// bare: the only entry built without one is the consume-dirty sentinel,
2087/// which the candidate filters exclude, so reaching this is a defect.
2088/// A HEAD that attests a DIFFERENT value is refused the same way — the
2089/// workspace holds bytes the object does not, and citing either would
2090/// make one reader or another refuse the path forever.
2091pub(crate) fn repair_crc(
2092 path: &str,
2093 be: &BaselineEntry,
2094 head: &flint_store::ObjectMeta,
2095) -> LeanResult<String> {
2096 let Some(ours) = be.crc64_b64.clone() else {
2097 return Err(LeanError::State(format!(
2098 "citation repair of {path}: the baseline records no CRC for the bytes it \
2099 integrated at etag {} — refusing to cite bytes nothing hashed",
2100 be.etag
2101 )));
2102 };
2103 if let Some(theirs) = head.crc64_b64.as_deref() {
2104 if theirs != ours {
2105 return Err(LeanError::State(format!(
2106 "citation repair of {path}: the store attests CRC-64 {theirs} for etag {} but \
2107 the bytes this workspace integrated under that etag hash to {ours} — the \
2108 workspace holds bytes the object does not; refusing to cite either",
2109 be.etag
2110 )));
2111 }
2112 }
2113 Ok(ours)
2114}
2115
2116fn file_crc(local_path: &Path) -> LeanResult<u64> {
2117 use std::io::Read;
2118 let mut crc = flint_store::Crc64Nvme::new();
2119 let mut f = std::fs::File::open(local_path)?;
2120 let mut buf = vec![0u8; 4 << 20];
2121 loop {
2122 let n = f.read(&mut buf)?;
2123 if n == 0 {
2124 break;
2125 }
2126 crc.update(&buf[..n]);
2127 }
2128 Ok(crc.finalize())
2129}
2130
2131pub(super) fn mtime_nanos_of(m: &std::fs::Metadata) -> u32 {
2132 m.modified()
2133 .ok()
2134 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2135 .map(|d| d.subsec_nanos())
2136 .unwrap_or(0)
2137}
2138
2139pub(super) fn mtime_of(m: &std::fs::Metadata) -> i64 {
2140 m.modified()
2141 .ok()
2142 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2143 .map(|d| d.as_secs() as i64)
2144 .unwrap_or(0)
2145}
2146
2147/// Containment-safe workspace write (boundary-verbs plan §2.2 security
2148/// gate). `rel` is a workspace-relative path; the target may not escape
2149/// the root, traverse a symlink component, or land in the reserved
2150/// control namespace.
2151///
2152/// The hazard this closes is pre-existing and reachable from three
2153/// callers (checkout, inbox consume, sync): `write_file_atomic` did
2154/// `create_dir_all(parent)` + write with no `O_NOFOLLOW` and no
2155/// root-containment check, while the scanner SKIPS symlinks — so a
2156/// planted symlink is invisible to the syncer. An unprivileged app
2157/// that plants `inputs -> /root/.aws`, lands an object at
2158/// `inputs/<path>` and drops a scoped sync turns the credential-holding
2159/// syncer into an on-demand arbitrary-file-write primitive outside the
2160/// workspace.
2161pub(super) fn write_file_atomic_in(
2162 root: &Path,
2163 rel: &str,
2164 bytes: &[u8],
2165 mode: Option<u32>,
2166) -> LeanResult<()> {
2167 let target = contained_path(root, rel)?;
2168 write_file_atomic(&target, bytes, mode).map(|_| ())
2169}
2170
2171/// Validate `rel` under `root` WITHOUT creating anything — for callers
2172/// that must decide whether a path is writable before doing any work
2173/// (inbox consume refuses a refused path outright rather than routing it
2174/// through conflict-preserve, which would cost a GET+PUT and leave the
2175/// path in the baseline).
2176pub(super) fn check_contained(root: &Path, rel: &str) -> LeanResult<()> {
2177 resolve_contained(root, rel, false).map(|_| ())
2178}
2179
2180/// Resolve `rel` under `root`, refusing escapes, creating missing parent
2181/// directories. Walks the parent chain component by component: a
2182/// component that EXISTS as a symlink is a refusal (never followed).
2183pub(super) fn contained_path(root: &Path, rel: &str) -> LeanResult<std::path::PathBuf> {
2184 resolve_contained(root, rel, true).map(|(p, _)| p)
2185}
2186
2187/// As `contained_path`, and also what the walk already learned about
2188/// the FINAL component: its `lstat` if it exists, `None` if it does
2189/// not. The walk stats every component to refuse a symlink; a caller
2190/// that then asked `exists()` and `metadata()` paid two more stats for
2191/// an answer it was already holding (checkout's resume check did, on
2192/// every one of 20,000 files).
2193pub(super) fn contained_path_stat(
2194 root: &Path,
2195 rel: &str,
2196) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2197 resolve_contained(root, rel, true)
2198}
2199
2200fn resolve_contained(
2201 root: &Path,
2202 rel: &str,
2203 create_dirs: bool,
2204) -> LeanResult<(std::path::PathBuf, Option<std::fs::Metadata>)> {
2205 use std::path::Component;
2206 let refuse = |why: &str| {
2207 Err(LeanError::State(format!(
2208 "refusing write to {rel:?}: {why} (containment)"
2209 )))
2210 };
2211 if rel.is_empty() {
2212 return refuse("empty path");
2213 }
2214 if super::scan::is_control_path(rel) {
2215 // The reserved namespace is the syncer's own; a citation or
2216 // inbox entry naming it is surfaced, never materialized (D0.3).
2217 return refuse("reserved control namespace");
2218 }
2219 let relp = Path::new(rel);
2220 let mut cur = root.to_path_buf();
2221 let mut comps: Vec<&std::ffi::OsStr> = vec![];
2222 for c in relp.components() {
2223 match c {
2224 Component::Normal(n) => comps.push(n),
2225 Component::CurDir => {}
2226 Component::ParentDir => return refuse("`..` component"),
2227 Component::RootDir | Component::Prefix(_) => return refuse("absolute path"),
2228 }
2229 }
2230 if comps.is_empty() {
2231 return refuse("no path components");
2232 }
2233 let last = comps.len() - 1;
2234 let mut final_meta = None;
2235 for (i, c) in comps.iter().enumerate() {
2236 cur.push(c);
2237 match std::fs::symlink_metadata(&cur) {
2238 Ok(m) if m.file_type().is_symlink() => {
2239 return refuse("path traverses a symlink");
2240 }
2241 Ok(m) if i < last && !m.is_dir() => {
2242 return refuse("parent component is not a directory");
2243 }
2244 Ok(m) => {
2245 if i == last {
2246 final_meta = Some(m);
2247 }
2248 }
2249 Err(_) if i < last => {
2250 if create_dirs {
2251 if let Err(e) = std::fs::create_dir(&cur) {
2252 // A SIBLING may have created it between the stat
2253 // above and this mkdir: fetches run on their own
2254 // tasks, so the first files of one directory
2255 // race here, and the loser's EEXIST is not a
2256 // containment refusal. It is fine only if what
2257 // exists now is a plain directory — a symlink
2258 // that appeared in the window is refused exactly
2259 // as one the stat found.
2260 match std::fs::symlink_metadata(&cur) {
2261 Ok(m) if m.file_type().is_symlink() => {
2262 return refuse("path traverses a symlink");
2263 }
2264 Ok(m) if m.is_dir() => {}
2265 _ => {
2266 return Err(LeanError::State(format!(
2267 "mkdir {}: {e}",
2268 cur.display()
2269 )))
2270 }
2271 }
2272 }
2273 }
2274 }
2275 Err(_) => {}
2276 }
2277 }
2278 Ok((cur, final_meta))
2279}
2280
2281/// Write `bytes` to `path` through an exclusive temp sibling and a
2282/// rename. The parent already exists — containment created it — and
2283/// one that vanished since is recreated on the retry path in `safefs`,
2284/// so the common path pays no `mkdir` and no `stat` for it. Returns
2285/// the written file's metadata.
2286pub(super) fn write_file_atomic(
2287 path: &Path,
2288 bytes: &[u8],
2289 mode: Option<u32>,
2290) -> LeanResult<std::fs::Metadata> {
2291 super::safefs::check_parent(path)?;
2292 // NOT with_extension(): that REPLACES the final extension, so
2293 // "a.txt" and "a.md" would collide on one tmp name.
2294 let tmp = path.with_file_name(format!(
2295 "{}.flint-sync-tmp",
2296 path.file_name().map(|n| n.to_string_lossy()).unwrap_or_default()
2297 ));
2298 // The temp sibling is computed AFTER containment ran, so the walk
2299 // never saw it: it needs its own refusal, not a plain write.
2300 // FAST (no per-file fsync): materialisations are made durable by
2301 // `sync_tree` before the marker or baseline that vouches for them.
2302 super::safefs::write_via_tmp_fast(path, &tmp, bytes, mode)
2303}
2304
2305/// The queue form of a merge's foreign upserts and deletions.
2306fn foreign_changes(entries: &[(String, LeanEntry)], gone: &[String]) -> Vec<ForeignChange> {
2307 entries
2308 .iter()
2309 .map(|(path, e)| ForeignChange {
2310 path: path.clone(),
2311 etag: Some(e.etag.clone()),
2312 crc64_b64: Some(e.crc64_b64.clone()),
2313 })
2314 .chain(gone.iter().map(|path| ForeignChange { path: path.clone(), etag: None, crc64_b64: None }))
2315 .collect()
2316}
2317
2318/// One three-way merge of a barrier's changes onto the bucket's manifest.
2319struct MergeOnto {
2320 theirs: manifest::LeanManifest,
2321 /// The CAS handle over `theirs`; `None` when the bucket has no manifest.
2322 expected: Option<manifest::ManifestHandle>,
2323 merged: manifest::LeanManifest,
2324 /// Other writers' changes since the merge base that the tree lacks.
2325 foreign: Vec<(String, LeanEntry)>,
2326 /// Other writers' deletions since the merge base.
2327 gone: Vec<String>,
2328}
2329
2330impl MergeOnto {
2331 /// The merge adds nothing to the document: installing it would be an
2332 /// empty generation.
2333 fn adds_nothing(&self) -> bool {
2334 self.merged.entries == self.theirs.entries && self.merged.sole_writer == self.theirs.sole_writer
2335 }
2336}
2337
2338impl Syncer {
2339 #[allow(clippy::too_many_arguments)]
2340 fn merge_onto(
2341 &self,
2342 current: Option<&manifest::LoadedManifest>,
2343 prev_installed: Option<&str>,
2344 inst_base: &BTreeMap<String, String>,
2345 upserts: &BTreeMap<String, LeanEntry>,
2346 deletes: &BTreeSet<String>,
2347 parked: &BTreeSet<String>,
2348 flush_uuid: &str,
2349 ) -> MergeOnto {
2350 let (theirs, expected) = match current {
2351 // The handle carries the LAYOUT as well as the tag: a
2352 // workspace still on the legacy single object CASes a
2353 // pointer that must not exist yet, not one it read.
2354 Some(l) => (l.manifest.clone(), Some(l.handle())),
2355 None => (Default::default(), None),
2356 };
2357 // If the bucket is still at the document THIS workspace
2358 // installed, that document IS the merge base — whatever the
2359 // persisted one says. Step 7 rewrites the merge base after
2360 // the CAS, so a restart in between leaves it behind a
2361 // document we wrote, and every entry in it would read as
2362 // somebody else's change. See `IntentJournal::installed_etag`.
2363 let own_base: BTreeMap<String, String>;
2364 let base: &BTreeMap<String, String> =
2365 if prev_installed.is_some() && prev_installed == expected.as_ref().map(|h| h.etag.as_str()) {
2366 own_base = theirs.entries.iter().map(|(p, e)| (p.clone(), e.etag.clone())).collect();
2367 &own_base
2368 } else {
2369 inst_base
2370 };
2371 let (mut merged, foreign) = manifest::merge(base, &theirs, upserts, deletes, parked);
2372 // Deletions another writer made since this workspace's
2373 // merge base: in the base, gone from theirs, and not this
2374 // barrier's own upsert, delete or park. `merge` has no use
2375 // for them — theirs already lacks the path — but the TREE
2376 // does: they reach it through the local queue, as the
2377 // foreign upserts do.
2378 let gone: Vec<String> = base
2379 .keys()
2380 .filter(|p| {
2381 !theirs.entries.contains_key(*p)
2382 && !upserts.contains_key(*p)
2383 && !deletes.contains(*p)
2384 && !parked.contains(*p)
2385 })
2386 .cloned()
2387 .collect();
2388 // `merge` clears it; the installing pass owns it. A mirror
2389 // is a property of how this workspace is DEPLOYED, so it
2390 // comes from config on every publish rather than being
2391 // inherited from whatever wrote last.
2392 merged.sole_writer = self.cfg.sole_writer;
2393 self.trace("merge", serde_json::json!({"flush": flush_uuid, "theirs_seq": theirs.seq, "upserts": upserts.len(),
2394 "deletes": deletes.len(), "foreign": foreign.len(), "gone": gone.len(),
2395 "adds_nothing": merged.entries == theirs.entries}));
2396 MergeOnto { theirs, expected, merged, foreign, gone }
2397 }
2398}