car_sync/session.rs
1//! The device-side sync session — the pump that drives a [`DeviceLog`] +
2//! [`OplogJournal`] pair against a [`Relay`] (slice B3 of
3//! `docs/proposals/multi-device-sync.md`).
4//!
5//! [`SyncSession::pump`] is one reconciliation round, sequenced so the
6//! binding contracts from B1/B4 hold **by construction**, not by caller
7//! discipline:
8//!
9//! 1. **Push** — only ops read from (or appended through) the journal are
10//! ever handed to the relay, and the journal is **fsync'd**
11//! ([`OplogJournal::sync`]) before the push, so **an op is journal-durable
12//! before it is transmitted** (B1 MUST — `flush` is only the page cache;
13//! a power loss in the writeback window would otherwise lose a
14//! transmitted op and re-mint its seq into a permanent Fork). A crash
15//! that loses the push cursor is harmless — a re-push dedups relay-side
16//! on `op_id`.
17//! 2. **Pull** — `pull(since = my per-device seq frontier)`, then
18//! **verify before fold** (B1 MUST): the union of held + pulled ops
19//! must pass [`verify_log`] (or [`verify_anchored`] against the
20//! session's base checkpoint) before anything is folded or journaled.
21//! 3. **Fold durably, then ack** — verified remote ops are appended to the
22//! journal, **advancing `self.ops` in lockstep** (each op enters
23//! `self.ops` the instant its append succeeds, so a mid-loop failure
24//! leaves `self.ops == journal` and the retry re-pull filters the
25//! already-journaled ops instead of duplicating them — a duplicate line
26//! would `DuplicateSeq`-brick the next open). The journal is then fsync'd
27//! and only then is `ack` sent. The ack value is *derived from the
28//! journal-held ops* — there is no API to ack anything else, so **acking
29//! merely-received (un-journaled) state is impossible by construction**
30//! (B4 MUST). A crash between the fold and the ack leaves the relay's ack
31//! table behind — the safe direction: GC can't drop what we haven't
32//! acked, and the next pump re-acks.
33//!
34//! Idempotence: re-running `pump` after any crash point re-pushes
35//! (relay dedups by `op_id`), re-pulls (already-held ops are filtered by
36//! `op_id`; re-folding is a no-op), and re-acks (monotone) — the whole
37//! round is retry-safe.
38//!
39//! **Cold bootstrap / straggler re-entry** ([`SyncSession::bootstrap`] /
40//! [`SyncSession::rebase`]): when the relay has GC'd past the session's
41//! frontier (`RelayError::FrontierTruncated`) or the device is brand new,
42//! the path is exactly the proposal's — `checkpoint_get()` → `pull(since =
43//! checkpoint frontier)` → verify → journal rewritten as checkpoint-anchored
44//! tail (`truncate_to`, stamping the truncation marker so the naive
45//! `load`/`resume` stays fenced — B4 contract 5) → **`resume_anchored`**,
46//! never `DeviceLog::resume`. Local ops **not covered** by the checkpoint —
47//! including a returning straggler's never-pushed writes — are carried into
48//! the rebased tail and pushed on the next pump: re-entry is lossless (see
49//! the relay module docs for why this is safe under seq-based frontiers).
50
51use crate::checkpoint::{resume_anchored, AnchorError, Checkpoint, CheckpointError};
52use crate::crypto::{CryptoError, Envelope, SyncKeyProvider};
53use crate::fold::{fold_onto, state_hash, FoldedRecord, SyncState};
54use crate::journal::OplogJournal;
55use crate::lease::{Intent, IntentStatus, LeaseCoordinator, LeaseError};
56use crate::oplog::{verify_log, ChainError, DeviceLog, Hlc, OpRecord, Scope, Surface, WallClock};
57use crate::relay::{checkpoint_frontier, frontier_of, Frontier, Relay, RelayError};
58use serde_json::Value;
59use std::fmt;
60use std::path::{Path, PathBuf};
61use std::sync::Arc;
62
63/// A sync-session failure.
64#[derive(Debug)]
65pub enum SessionError {
66 Io(std::io::Error),
67 Chain(ChainError),
68 Anchor(AnchorError),
69 Relay(RelayError),
70 Checkpoint(CheckpointError),
71 /// The journal carries a truncation marker naming a checkpoint that is
72 /// not present in the session's checkpoint directory — resume is
73 /// impossible without it (fetch it from the relay and retry).
74 MissingCheckpoint {
75 checkpoint_hash: String,
76 },
77 /// The relay served a "latest" checkpoint that does not cover the
78 /// session's current base — rebasing onto it would silently lose state.
79 CheckpointRegression {
80 held: String,
81 offered: String,
82 },
83 /// A lease-coordinator call failed (B5) — used by the best-effort local
84 /// gate [`SyncSession::record_intent_if_current`].
85 Lease(LeaseError),
86 /// Encrypting an op payload / decrypting a folded op failed (B6 E2E).
87 Crypto(CryptoError),
88}
89
90impl fmt::Display for SessionError {
91 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
92 match self {
93 SessionError::Io(e) => write!(f, "sync session io error: {e}"),
94 SessionError::Chain(e) => write!(f, "sync session chain error: {e}"),
95 SessionError::Anchor(e) => write!(f, "sync session anchor error: {e}"),
96 SessionError::Relay(e) => write!(f, "sync session relay error: {e}"),
97 SessionError::Checkpoint(e) => write!(f, "sync session checkpoint error: {e}"),
98 SessionError::MissingCheckpoint { checkpoint_hash } => write!(
99 f,
100 "journal is truncated below checkpoint {checkpoint_hash}, which is not in the \
101 session checkpoint directory — fetch it (relay checkpoint_get) and retry"
102 ),
103 SessionError::CheckpointRegression { held, offered } => write!(
104 f,
105 "relay's latest checkpoint {offered} does not cover the session's base {held} — \
106 refusing to rebase onto it (state would be lost)"
107 ),
108 SessionError::Lease(e) => write!(f, "sync session lease error: {e}"),
109 SessionError::Crypto(e) => write!(f, "sync session crypto error: {e}"),
110 }
111 }
112}
113
114impl std::error::Error for SessionError {}
115
116impl From<std::io::Error> for SessionError {
117 fn from(e: std::io::Error) -> Self {
118 SessionError::Io(e)
119 }
120}
121impl From<ChainError> for SessionError {
122 fn from(e: ChainError) -> Self {
123 SessionError::Chain(e)
124 }
125}
126impl From<AnchorError> for SessionError {
127 fn from(e: AnchorError) -> Self {
128 SessionError::Anchor(e)
129 }
130}
131impl From<RelayError> for SessionError {
132 fn from(e: RelayError) -> Self {
133 SessionError::Relay(e)
134 }
135}
136impl From<CheckpointError> for SessionError {
137 fn from(e: CheckpointError) -> Self {
138 SessionError::Checkpoint(e)
139 }
140}
141impl From<LeaseError> for SessionError {
142 fn from(e: LeaseError) -> Self {
143 SessionError::Lease(e)
144 }
145}
146
147/// What one [`SyncSession::pump`] round did.
148#[derive(Debug, Clone, Default, PartialEq)]
149pub struct PumpReport {
150 /// Own ops newly admitted by the relay.
151 pub pushed: usize,
152 /// Own ops the relay already held.
153 pub push_deduped: usize,
154 /// Remote ops pulled, verified, journaled, and folded this round.
155 pub folded: usize,
156 /// The fold frontier acked (derived from journal-held ops), if any.
157 pub acked: Option<Hlc>,
158}
159
160/// A device's live sync endpoint: its append chain, its durable journal,
161/// and the pump. See the module docs for the contract sequencing.
162pub struct SyncSession {
163 device_id: String,
164 device: DeviceLog,
165 journal: OplogJournal,
166 checkpoint_dir: PathBuf,
167 wall: WallClock,
168 /// Every op the journal holds (the tail, when `base` is set).
169 ops: Vec<OpRecord>,
170 /// The checkpoint the journal is anchored on, when truncated.
171 base: Option<Checkpoint>,
172 /// In-memory push cursor (own max seq pushed). Deliberately NOT
173 /// persisted: losing it in a crash only causes a deduped re-push.
174 pushed_through: Option<u64>,
175 /// E2E key provider (B6). When `Some`, op payloads are encrypted at
176 /// [`SyncSession::append`] (so the op is ciphertext-native — `op_id`/chain
177 /// cover the ciphertext, and a remote relay stores only ciphertext) and
178 /// decrypted at [`SyncSession::state`], **after** the chain verifies and
179 /// **before** the fold groups them. `None` = cleartext (local `FsRelay`).
180 key_provider: Option<Arc<dyn SyncKeyProvider>>,
181}
182
183impl fmt::Debug for SyncSession {
184 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
185 f.debug_struct("SyncSession")
186 .field("device_id", &self.device_id)
187 .field("ops", &self.ops.len())
188 .field("base", &self.base.as_ref().map(|c| &c.checkpoint_hash))
189 .field("pushed_through", &self.pushed_through)
190 .finish_non_exhaustive()
191 }
192}
193
194impl SyncSession {
195 /// Open a session over an existing (possibly empty, possibly truncated)
196 /// journal. A truncated journal resumes **only** through its covering
197 /// checkpoint (`resume_anchored` — B4 contract): the checkpoint file
198 /// must be present in `checkpoint_dir` under its content address.
199 pub fn open(
200 device_id: impl Into<String>,
201 journal_path: &Path,
202 checkpoint_dir: &Path,
203 wall: WallClock,
204 ) -> Result<Self, SessionError> {
205 let device_id = device_id.into();
206 let (marker, ops) = OplogJournal::load_with_marker(journal_path)?;
207 let journal = OplogJournal::open(journal_path)?;
208 let (device, base) = match marker {
209 Some(marker) => {
210 let path =
211 checkpoint_dir.join(format!("{}.checkpoint.json", marker.checkpoint_hash));
212 if !path.exists() {
213 return Err(SessionError::MissingCheckpoint {
214 checkpoint_hash: marker.checkpoint_hash,
215 });
216 }
217 let checkpoint = Checkpoint::load(&path)?;
218 let device = resume_anchored(device_id.clone(), &checkpoint, &ops)?;
219 (device, Some(checkpoint))
220 }
221 None => (DeviceLog::resume(device_id.clone(), &ops)?, None),
222 };
223 let mut session = Self {
224 device_id,
225 device,
226 journal,
227 checkpoint_dir: checkpoint_dir.to_path_buf(),
228 wall,
229 ops,
230 base,
231 pushed_through: None,
232 key_provider: None,
233 };
234 session.device.set_wall_clock(session.wall.clone());
235 Ok(session)
236 }
237
238 /// Turn on E2E: op payloads are encrypted at [`SyncSession::append`] and
239 /// decrypted at [`SyncSession::state`], keyed per scope audience by
240 /// `provider` (a login-derived [`crate::crypto::DerivedKeyProvider`] in the
241 /// daemon). Builder so [`SyncSession::open`]'s signature stays stable for the
242 /// local (cleartext `FsRelay`) path. NOTE: with a *remote* relay, checkpoint
243 /// state is still cleartext-folded — do not push checkpoints to an untrusted
244 /// relay until per-scope checkpoint encryption lands (crypto.rs follow-up).
245 pub fn with_key_provider(mut self, provider: Arc<dyn SyncKeyProvider>) -> Self {
246 self.key_provider = Some(provider);
247 self
248 }
249
250 /// Open + immediately [`SyncSession::rebase`] onto the relay's latest
251 /// checkpoint — the cold-device / returning-straggler entry point
252 /// (proposal §"Cold / new device bootstrap"). With no relay checkpoint
253 /// (young account) this degrades to a plain open; the first `pump`
254 /// replays from genesis.
255 pub fn bootstrap(
256 device_id: impl Into<String>,
257 journal_path: &Path,
258 checkpoint_dir: &Path,
259 relay: &mut dyn Relay,
260 wall: WallClock,
261 ) -> Result<Self, SessionError> {
262 let mut session = Self::open(device_id, journal_path, checkpoint_dir, wall)?;
263 session.rebase(relay)?;
264 Ok(session)
265 }
266
267 /// Record a local mutation: stamp (hybrid clock), **journal (flushed)**,
268 /// then hold for push — the op is journal-durable before it can ever be
269 /// transmitted (B1 MUST). If the journal write fails the device chain
270 /// is rolled back (rebuilt from the durable ops), so the failed op can
271 /// never leave a hole for the next append to chain onto.
272 pub fn append(
273 &mut self,
274 scope: Scope,
275 surface: Surface,
276 payload: Value,
277 ) -> Result<OpRecord, SessionError> {
278 // E2E: encrypt the payload BEFORE building the op, so `op_id`/the chain
279 // cover the ciphertext and a remote relay never sees cleartext. The
280 // cipher is chosen by the op's scope audience (personal vs org key).
281 let payload = match &self.key_provider {
282 Some(provider) => provider
283 .cipher_for(&scope)
284 .encrypt(&payload)
285 .map_err(SessionError::Crypto)?,
286 None => payload,
287 };
288 let op = self.device.append(scope, surface, payload);
289 if let Err(e) = self.journal.append(&op) {
290 self.rebuild_device()?;
291 return Err(SessionError::Io(e));
292 }
293 self.ops.push(op.clone());
294 Ok(op)
295 }
296
297 /// The fence-independent **committed-run idempotency oracle** (B5) over
298 /// this session's folded state (checkpoint base + journal tail): "has
299 /// `run_id` already committed for `agent_id`?" Keep-all and immune to epoch
300 /// bumps AND compaction, so it is the CORRECT idempotency lookup — unlike
301 /// [`crate::fold::SyncState::intent`], which is the fenced "who holds now"
302 /// view and can read `None`/pending for a run that actually committed. A B6
303 /// dispatch fence performs exactly this read before an external side effect.
304 pub fn committed_run(&self, agent_id: &str, run_id: &str) -> Option<FoldedRecord> {
305 self.state().committed_run(agent_id, run_id).cloned()
306 }
307
308 /// Record a leased execution [`Intent`] (B5) — journal-durable exactly
309 /// like any [`SyncSession::append`], so it is durable before it can be
310 /// transmitted. The **partitioned / ungated** path: a zombie that cannot
311 /// reach the coordinator still records here, and the fold converges the
312 /// ledger deterministically.
313 ///
314 /// **Terminal guard (C3):** a committed run is terminal. A `pending`/`failed`
315 /// write for a run already committed (per the fence-independent
316 /// [`SyncSession::committed_run`] oracle) is a **no-op** (`Ok(None)`), so a
317 /// failed-over holder writing `pending` before checking cannot revert the
318 /// committed ledger.
319 ///
320 /// This converges the **ledger**; it is NOT the exactly-once execution gate.
321 /// The B6 dispatch fence — a linearizable "am I still epoch N?" plus this
322 /// committed-oracle read **before** the external effect — is what makes
323 /// execution single-shot.
324 pub fn record_intent(
325 &mut self,
326 scope: Scope,
327 intent: &Intent,
328 ) -> Result<Option<OpRecord>, SessionError> {
329 if intent.status != IntentStatus::Committed
330 && self
331 .committed_run(&intent.agent_id, &intent.run_id)
332 .is_some()
333 {
334 return Ok(None); // terminal: the run already committed — do not revert
335 }
336 Ok(Some(self.append(
337 scope,
338 Surface::Intent,
339 intent.payload(),
340 )?))
341 }
342
343 /// Record a leased [`Intent`] **only if the local epoch is still current**
344 /// — the best-effort local gate: a linearizable coordinator read confirms
345 /// this device still holds the lease at `intent.epoch` before the op is
346 /// journaled. Returns `Ok(None)` when the coordinator says this device is
347 /// no longer the holder at that epoch (skipping a write the fold would fence)
348 /// or when the run is already committed (the [`SyncSession::record_intent`]
349 /// terminal guard).
350 ///
351 /// This is a *liveness optimization*, NOT the safety gate: a partitioned
352 /// zombie that cannot reach the coordinator falls back to
353 /// [`SyncSession::record_intent`]. Exactly-once execution is the B6 dispatch
354 /// fence (this check races a pause-after-check — the Kleppmann residual).
355 pub fn record_intent_if_current(
356 &mut self,
357 scope: Scope,
358 intent: &Intent,
359 coordinator: &mut dyn LeaseCoordinator,
360 ) -> Result<Option<OpRecord>, SessionError> {
361 let current = coordinator.current(&intent.agent_id)?;
362 let still_ours = current
363 .as_ref()
364 .is_some_and(|l| l.holder == self.device_id && l.epoch == intent.epoch);
365 if !still_ours {
366 return Ok(None); // not the current holder — don't even write it
367 }
368 self.record_intent(scope, intent) // terminal-guarded
369 }
370
371 /// Rebuild the append chain from what is actually durable — the
372 /// journal-held ops (+ base anchor).
373 fn rebuild_device(&mut self) -> Result<(), SessionError> {
374 let mut device = match &self.base {
375 Some(checkpoint) => resume_anchored(self.device_id.clone(), checkpoint, &self.ops)?,
376 None => DeviceLog::resume(self.device_id.clone(), &self.ops)?,
377 };
378 device.set_wall_clock(self.wall.clone());
379 self.device = device;
380 Ok(())
381 }
382
383 /// The per-device seq cursor of everything this session holds
384 /// (checkpoint coverage + journal tail).
385 fn held_frontier(&self) -> Frontier {
386 let mut frontier = self
387 .base
388 .as_ref()
389 .map(checkpoint_frontier)
390 .unwrap_or_default();
391 for (device, seq) in frontier_of(&self.ops) {
392 let entry = frontier.entry(device).or_insert(seq);
393 if seq > *entry {
394 *entry = seq;
395 }
396 }
397 frontier
398 }
399
400 /// The fold frontier this session may ack: the max HLC across
401 /// journal-held ops and the base checkpoint's coverage — derived from
402 /// durable state ONLY, which is what makes ack-before-fold impossible.
403 fn ack_frontier(&self) -> Option<Hlc> {
404 let from_ops = self.ops.iter().map(|op| &op.hlc).max();
405 let from_base = self
406 .base
407 .as_ref()
408 .and_then(|c| c.frontier.values().map(|e| &e.hlc).max());
409 [from_ops, from_base].into_iter().flatten().max().cloned()
410 }
411
412 /// One reconciliation round: push journal-durable own ops → pull →
413 /// verify → journal the folds → ack. See the module docs for the
414 /// contract sequencing; retry-safe at every crash point.
415 ///
416 /// Returns [`RelayError::FrontierTruncated`] (wrapped) when the relay
417 /// has GC'd past this session's frontier — call
418 /// [`SyncSession::rebase`] and pump again.
419 pub fn pump(&mut self, relay: &mut dyn Relay) -> Result<PumpReport, SessionError> {
420 let mut report = PumpReport::default();
421
422 // 1. Push own journal-durable ops the relay may not have.
423 let mut own: Vec<OpRecord> = self
424 .ops
425 .iter()
426 .filter(|op| {
427 op.device_id == self.device_id
428 && self.pushed_through.is_none_or(|through| op.seq > through)
429 })
430 .cloned()
431 .collect();
432 own.sort_by_key(|op| op.seq);
433 if !own.is_empty() {
434 // Journal-durable BEFORE transmit (B1 MUST): the per-op appends
435 // only reached the OS page cache (flush, not fsync). One batch
436 // barrier here makes the whole own-op tail stable before any of
437 // it leaves the device — a power loss in the writeback window
438 // otherwise loses a transmitted op and re-mints its seq (a
439 // permanent Fork). fsync once, not per append.
440 self.journal.sync()?;
441 let outcome = relay.push(&self.device_id, &own)?;
442 report.pushed = outcome.accepted;
443 report.push_deduped = outcome.deduped;
444 self.pushed_through = own.last().map(|op| op.seq);
445 }
446
447 // 2. Pull everything above what we hold.
448 let pulled = relay.pull(&self.device_id, &self.held_frontier())?;
449
450 // 3. Filter already-held (idempotent re-pull) and VERIFY BEFORE FOLD.
451 let held: std::collections::BTreeSet<&str> =
452 self.ops.iter().map(|op| op.op_id.as_str()).collect();
453 let new_ops: Vec<OpRecord> = pulled
454 .ops
455 .into_iter()
456 .filter(|op| !held.contains(op.op_id.as_str()))
457 .collect();
458 if !new_ops.is_empty() {
459 let mut candidate = self.ops.clone();
460 candidate.extend(new_ops.iter().cloned());
461 match &self.base {
462 Some(checkpoint) => crate::checkpoint::verify_anchored(checkpoint, &candidate)?,
463 None => verify_log(&candidate)?,
464 }
465
466 // 4. Journal the folds — advancing self.ops IN LOCKSTEP with the
467 // journal. Recording each op into self.ops the instant its
468 // append succeeds is load-bearing for retry-safety: a mid-loop
469 // append failure then leaves self.ops == the journal, so the
470 // retry pump filters the already-journaled ops out of the
471 // re-pull instead of appending them a second time (a duplicate
472 // journal line = DuplicateSeq on the next open = a bricked
473 // device). `candidate` is discarded on failure.
474 drop(candidate);
475 for op in &new_ops {
476 self.journal.append(op)?;
477 self.ops.push(op.clone());
478 self.device.observe(&op.hlc);
479 report.folded += 1;
480 }
481 // Durable fold BEFORE ack (B4 MUST: ack asserts durably-folded
482 // state). One batch fsync over the folds just journaled.
483 self.journal.sync()?;
484 }
485
486 // 5. Ack — derived from journal-held state only, after it is
487 // durable. (B4 MUST: ack asserts durably-folded state.)
488 if let Some(frontier) = self.ack_frontier() {
489 relay.ack(&self.device_id, frontier.clone())?;
490 report.acked = Some(frontier);
491 }
492 Ok(report)
493 }
494
495 /// Re-anchor this session on the relay's latest checkpoint — the cold
496 /// bootstrap / post-eviction re-entry move. Returns `true` when a
497 /// rebase happened.
498 ///
499 /// Sequencing (each step durable before the next depends on it):
500 /// `checkpoint_get` → `pull(since = checkpoint frontier)` → merge in
501 /// every locally-held op the checkpoint does NOT cover (a returning
502 /// straggler's unpushed writes survive) → `verify_anchored` → save the
503 /// checkpoint into the session checkpoint dir → `truncate_to` (journal
504 /// rewritten as the anchored tail, truncation marker stamped) →
505 /// `resume_anchored`. The push cursor resets so the next `pump`
506 /// re-offers every own op in the tail (relay dedups the already-pushed).
507 pub fn rebase(&mut self, relay: &mut dyn Relay) -> Result<bool, SessionError> {
508 let Some(checkpoint) = relay.checkpoint_get()? else {
509 return Ok(false); // young account: genesis replay via pump
510 };
511 if let Some(base) = &self.base {
512 if base.checkpoint_hash == checkpoint.checkpoint_hash {
513 return Ok(false); // already anchored here
514 }
515 if !crate::relay::frontier_dominates(&checkpoint, base) {
516 return Err(SessionError::CheckpointRegression {
517 held: base.checkpoint_hash.clone(),
518 offered: checkpoint.checkpoint_hash.clone(),
519 });
520 }
521 }
522
523 let pulled = relay.pull(&self.device_id, &checkpoint_frontier(&checkpoint))?;
524
525 // Keep every held op the checkpoint does not cover — own unpushed
526 // writes AND foreign tails we already folded — deduped against the
527 // pull by op_id.
528 let mut tail: Vec<OpRecord> = pulled.ops;
529 let mut seen: std::collections::BTreeSet<String> =
530 tail.iter().map(|op| op.op_id.clone()).collect();
531 for op in &self.ops {
532 let covered = checkpoint
533 .frontier
534 .get(&op.device_id)
535 .is_some_and(|entry| op.seq <= entry.seq);
536 if !covered && seen.insert(op.op_id.clone()) {
537 tail.push(op.clone());
538 }
539 }
540 tail.sort_by(|a, b| (&a.hlc, &a.op_id).cmp(&(&b.hlc, &b.op_id)));
541
542 // Verify BEFORE any durable rewrite; then checkpoint durable FIRST,
543 // then the journal truncation that names it (B4 crash ordering).
544 crate::checkpoint::verify_anchored(&checkpoint, &tail)?;
545 checkpoint.save(&self.checkpoint_dir)?;
546 self.journal
547 .truncate_to(&tail, &checkpoint.checkpoint_hash)?;
548
549 let mut device = resume_anchored(self.device_id.clone(), &checkpoint, &tail)?;
550 device.set_wall_clock(self.wall.clone());
551 self.device = device;
552 self.ops = tail;
553 self.base = Some(checkpoint);
554 self.pushed_through = None;
555 Ok(true)
556 }
557
558 /// Compute a checkpoint at the relay's stable frontier from this
559 /// session's held ops and upload it — the device-computed snapshot the
560 /// proposal requires under E2E ("the relay holds ciphertext and cannot
561 /// fold"). Call **after** a `pump` (so held == relay-known and own ops
562 /// are pushed). Returns the uploaded checkpoint, or `None` when there
563 /// is no stable frontier, nothing below it, or this session is itself
564 /// anchored on a checkpoint (recompaction over a base is the same
565 /// later slice B4 deferred).
566 pub fn publish_checkpoint(
567 &mut self,
568 relay: &mut dyn Relay,
569 ) -> Result<Option<Checkpoint>, SessionError> {
570 // E2E (B6): the ops are ciphertext-native, so a checkpoint folded from
571 // them would key facts by ciphertext hash (the fold groups on
572 // `payload["id"]`, hidden under the envelope) — a peer loading it as its
573 // decrypt-before-fold base would then fold cleartext tail onto a
574 // ciphertext base and DIVERGE. Publishing a *cleartext* checkpoint would
575 // instead leak state to the relay. So under E2E we do not publish to the
576 // (untrusted) relay: the encrypted op log is retained and cold bootstrap
577 // replays it. Per-scope *encrypted* checkpoint push (which restores
578 // relay-side GC) is the documented follow-up; until then the Parslee
579 // server owns retention. Local journal compaction is unaffected.
580 if self.key_provider.is_some() {
581 return Ok(None);
582 }
583 if self.base.is_some() {
584 return Ok(None);
585 }
586 let Some(frontier) = relay.stable_frontier()? else {
587 return Ok(None);
588 };
589 let below: Vec<OpRecord> = self
590 .ops
591 .iter()
592 .filter(|op| op.hlc <= frontier)
593 .cloned()
594 .collect();
595 if below.is_empty() {
596 return Ok(None);
597 }
598 let checkpoint = Checkpoint::from_ops(&below)?;
599 relay.checkpoint_put(&self.device_id, &checkpoint)?;
600 Ok(Some(checkpoint))
601 }
602
603 /// The materialized state: `fold_onto(base checkpoint, journal tail)`.
604 ///
605 /// Under E2E ([`SyncSession::with_key_provider`]) the tail is ciphertext, so
606 /// each op's payload is decrypted here — after the chain has verified, before
607 /// the fold groups on `payload["id"]`/`fold_key`. Op identity stays the
608 /// cleartext-metadata `op_id`; only the payload is swapped.
609 pub fn state(&self) -> SyncState {
610 let base = self
611 .base
612 .as_ref()
613 .map(|c| c.state.clone())
614 .unwrap_or_default();
615 match &self.key_provider {
616 Some(provider) => {
617 let decrypted = self.decrypted_tail(provider.as_ref());
618 fold_onto(&base, &decrypted)
619 }
620 None => fold_onto(&base, &self.ops),
621 }
622 }
623
624 /// The journal tail with each E2E payload decrypted for folding. An op that
625 /// is not an envelope (cleartext, e.g. mixed rollout) is passed through; one
626 /// we hold no key for (a foreign org audience) keeps its envelope — the fold
627 /// then groups it by content hash, harmlessly, rather than mis-applying it.
628 fn decrypted_tail(&self, provider: &dyn SyncKeyProvider) -> Vec<OpRecord> {
629 self.ops
630 .iter()
631 .map(|op| {
632 if !Envelope::is_envelope(&op.payload) {
633 return op.clone();
634 }
635 match provider.cipher_for(&op.scope).decrypt(&op.payload) {
636 Ok(plaintext) => {
637 let mut cleartext = op.clone();
638 cleartext.payload = plaintext;
639 cleartext
640 }
641 Err(_) => op.clone(),
642 }
643 })
644 .collect()
645 }
646
647 /// [`state_hash`] of [`SyncSession::state`] — the divergence invariant
648 /// two synced devices must agree on.
649 pub fn state_hash(&self) -> String {
650 state_hash(&self.state())
651 }
652
653 pub fn device_id(&self) -> &str {
654 &self.device_id
655 }
656
657 /// The journal-held ops (the anchored tail, when a base is set).
658 pub fn ops(&self) -> &[OpRecord] {
659 &self.ops
660 }
661
662 /// The checkpoint this session's journal is anchored on, if truncated.
663 pub fn base(&self) -> Option<&Checkpoint> {
664 self.base.as_ref()
665 }
666}
667
668#[cfg(test)]
669mod tests {
670 use super::*;
671 use crate::relay::{
672 AckOutcome, GcReport, InMemoryRelay, PullResult, PushOutcome, RelayConfig, RosterEntry,
673 };
674 use serde_json::json;
675 use std::sync::atomic::{AtomicBool, Ordering};
676 use std::sync::Arc;
677
678 fn zero_wall() -> WallClock {
679 Arc::new(|| 0)
680 }
681
682 fn mem_relay() -> InMemoryRelay {
683 InMemoryRelay::new(RelayConfig::default(), zero_wall())
684 }
685
686 struct Dirs {
687 _tmp: tempfile::TempDir,
688 journal: std::path::PathBuf,
689 ckpts: std::path::PathBuf,
690 }
691
692 fn dirs() -> Dirs {
693 let tmp = tempfile::tempdir().unwrap();
694 let journal = tmp.path().join("oplog.jsonl");
695 let ckpts = tmp.path().join("checkpoints");
696 Dirs {
697 _tmp: tmp,
698 journal,
699 ckpts,
700 }
701 }
702
703 fn open(device: &str, d: &Dirs) -> SyncSession {
704 SyncSession::open(device, &d.journal, &d.ckpts, zero_wall()).unwrap()
705 }
706
707 #[test]
708 fn e2e_encrypts_payloads_on_the_wire_and_decrypts_for_the_fold() {
709 use crate::crypto::DerivedKeyProvider;
710 let provider: Arc<dyn SyncKeyProvider> =
711 Arc::new(DerivedKeyProvider::new(b"parslee-login-master".to_vec()));
712 let d = dirs();
713 let mut sess = open("mac", &d).with_key_provider(provider.clone());
714
715 let secret = json!({"id": "fact-1", "body": "slack bot token xoxb-SECRET"});
716 let op = sess
717 .append(Scope::Personal, Surface::Knowledge, secret.clone())
718 .unwrap();
719
720 // What a remote relay receives is ciphertext — never the cleartext.
721 assert!(
722 Envelope::is_envelope(&op.payload),
723 "the relay must only ever see an encrypted envelope"
724 );
725 assert_ne!(op.payload, secret);
726 // op_id covers the ciphertext (chain stays verifiable over what ships).
727 assert_eq!(sess.ops[0].op_id, op.op_id);
728
729 // A peer with the SAME login master recovers the cleartext.
730 let recovered = provider
731 .cipher_for(&Scope::Personal)
732 .decrypt(&op.payload)
733 .unwrap();
734 assert_eq!(recovered, secret);
735
736 // decrypt-before-fold runs without error (the fold sees cleartext).
737 let _ = sess.state();
738 }
739
740 #[test]
741 fn e2e_never_publishes_a_checkpoint_to_the_relay() {
742 // Under E2E a checkpoint would either leak cleartext or (folded from the
743 // ciphertext ops) form an inconsistent decrypt base — so publishing is
744 // guarded off. The relay must never receive one.
745 use crate::crypto::DerivedKeyProvider;
746 let provider: Arc<dyn SyncKeyProvider> =
747 Arc::new(DerivedKeyProvider::new(b"master".to_vec()));
748 let d = dirs();
749 let mut sess = open("mac", &d).with_key_provider(provider);
750 sess.append(
751 Scope::Personal,
752 Surface::Knowledge,
753 json!({"id": "f1", "body": "x"}),
754 )
755 .unwrap();
756 let mut relay = mem_relay();
757 assert!(
758 sess.publish_checkpoint(&mut relay).unwrap().is_none(),
759 "E2E must not publish a checkpoint to an untrusted relay"
760 );
761 }
762
763 #[test]
764 fn two_devices_converge_through_the_relay_across_all_tiers() {
765 let mut relay = mem_relay();
766 let (da, db) = (dirs(), dirs());
767 let mut a = open("mac-a", &da);
768 let mut b = open("mac-b", &db);
769
770 // Concurrent writes on every fold tier, including an LWW conflict
771 // and a routing observation multiset.
772 a.append(
773 Scope::Personal,
774 Surface::Knowledge,
775 json!({"id": "f1", "v": 1}),
776 )
777 .unwrap();
778 a.append(
779 Scope::Personal,
780 Surface::Declagent,
781 json!({"id": "milo", "owner": "a"}),
782 )
783 .unwrap();
784 a.append(Scope::Personal, Surface::Routing, json!({"sample": 1.0}))
785 .unwrap();
786 b.append(
787 Scope::Personal,
788 Surface::Knowledge,
789 json!({"id": "f2", "v": 2}),
790 )
791 .unwrap();
792 b.append(Scope::Personal, Surface::Routing, json!({"sample": 1.0}))
793 .unwrap();
794
795 a.pump(&mut relay).unwrap();
796 let rb = b.pump(&mut relay).unwrap();
797 assert_eq!(rb.folded, 3, "b folded a's three ops");
798 // b now writes causally AFTER folding a's registry record.
799 b.append(
800 Scope::Personal,
801 Surface::Declagent,
802 json!({"id": "milo", "owner": "b"}),
803 )
804 .unwrap();
805 b.pump(&mut relay).unwrap();
806 let ra = a.pump(&mut relay).unwrap();
807 assert_eq!(ra.folded, 3);
808
809 assert_eq!(
810 a.state_hash(),
811 b.state_hash(),
812 "divergence invariant: same hash"
813 );
814 let state = a.state();
815 assert_eq!(
816 state.registries[&Surface::Declagent.tag()]["id:milo"].payload["owner"],
817 json!("b"),
818 "LWW resolved by the hybrid clock's causal order"
819 );
820 assert_eq!(
821 state.log_entries(&Surface::Routing.tag()).len(),
822 2,
823 "the observation multiset survived transport"
824 );
825
826 // Idempotence: an extra pump on both sides is a complete no-op.
827 let ra = a.pump(&mut relay).unwrap();
828 let rb = b.pump(&mut relay).unwrap();
829 assert_eq!((ra.pushed, ra.folded), (0, 0));
830 assert_eq!((rb.pushed, rb.folded), (0, 0));
831 assert_eq!(a.state_hash(), b.state_hash());
832 }
833
834 #[test]
835 fn op_is_journal_durable_before_it_is_ever_transmitted() {
836 // Contract 1 (B1 MUST): append journals+flushes; the crash window
837 // between append and pump loses NOTHING and re-mints NO seq.
838 let mut relay = mem_relay();
839 let d = dirs();
840 {
841 let mut a = open("mac-a", &d);
842 a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
843 .unwrap();
844 // "crash" before any pump: session dropped, nothing transmitted.
845 }
846 let mut a = open("mac-a", &d);
847 // The op survived in the journal; resume did not re-mint its seq.
848 let next = a
849 .append(Scope::Personal, Surface::Knowledge, json!({"id": "f2"}))
850 .unwrap();
851 assert_eq!(next.seq, 1);
852 a.pump(&mut relay).unwrap();
853 assert_eq!(relay.pull("x", &Frontier::new()).unwrap().ops.len(), 2);
854 verify_log(a.ops()).unwrap();
855 }
856
857 #[test]
858 fn ack_is_derived_from_journal_held_state_only() {
859 // Contract 2 (B4 MUST), shown by construction: the acked frontier
860 // equals the max HLC of what is ON DISK in the journal — never of
861 // anything merely received.
862 let mut relay = mem_relay();
863 let (da, db) = (dirs(), dirs());
864 let mut a = open("mac-a", &da);
865 a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
866 .unwrap();
867 a.pump(&mut relay).unwrap();
868
869 let mut b = open("mac-b", &db);
870 let report = b.pump(&mut relay).unwrap();
871 let acked = report.acked.unwrap();
872
873 // Reload b's journal from disk: the ack is exactly its max stamp.
874 drop(b);
875 let on_disk = OplogJournal::load(&db.journal).unwrap();
876 assert_eq!(
877 acked,
878 on_disk.iter().map(|op| op.hlc.clone()).max().unwrap()
879 );
880 let roster: std::collections::BTreeMap<String, RosterEntry> = relay
881 .roster()
882 .unwrap()
883 .into_iter()
884 .map(|e| (e.device_id.clone(), e))
885 .collect();
886 assert_eq!(roster["mac-b"].acked.as_ref(), Some(&acked));
887 }
888
889 /// A relay wrapper that fails `ack` while the flag is set — the
890 /// crash/partition at the worst point of the pump (after the durable
891 /// fold, before the ack).
892 struct FlakyAckRelay<'a> {
893 inner: &'a mut dyn Relay,
894 fail_ack: Arc<AtomicBool>,
895 }
896
897 impl Relay for FlakyAckRelay<'_> {
898 fn register(&mut self, d: &str) -> Result<RosterEntry, RelayError> {
899 self.inner.register(d)
900 }
901 fn push(&mut self, d: &str, ops: &[OpRecord]) -> Result<PushOutcome, RelayError> {
902 self.inner.push(d, ops)
903 }
904 fn pull(&mut self, d: &str, since: &Frontier) -> Result<PullResult, RelayError> {
905 self.inner.pull(d, since)
906 }
907 fn ack(&mut self, d: &str, frontier: Hlc) -> Result<AckOutcome, RelayError> {
908 if self.fail_ack.load(Ordering::SeqCst) {
909 return Err(RelayError::Io(std::io::Error::other("network down")));
910 }
911 self.inner.ack(d, frontier)
912 }
913 fn checkpoint_put(&mut self, d: &str, c: &Checkpoint) -> Result<bool, RelayError> {
914 self.inner.checkpoint_put(d, c)
915 }
916 fn checkpoint_get(&mut self) -> Result<Option<Checkpoint>, RelayError> {
917 self.inner.checkpoint_get()
918 }
919 fn roster(&mut self) -> Result<Vec<RosterEntry>, RelayError> {
920 self.inner.roster()
921 }
922 fn stable_frontier(&mut self) -> Result<Option<Hlc>, RelayError> {
923 self.inner.stable_frontier()
924 }
925 fn gc(&mut self) -> Result<GcReport, RelayError> {
926 self.inner.gc()
927 }
928 }
929
930 #[test]
931 fn crash_mid_pump_is_idempotent_at_every_step() {
932 let mut relay = mem_relay();
933 let (da, db) = (dirs(), dirs());
934 let mut a = open("mac-a", &da);
935 a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
936 .unwrap();
937 a.pump(&mut relay).unwrap();
938
939 // --- Crash point 1: pulled but nothing device-side happened yet
940 // (transport delivered bytes; the process died before verify/fold).
941 // Nothing durable changed, the relay ack table is untouched → a
942 // fresh pump redoes everything.
943 let _ = relay.pull("mac-b", &Frontier::new()).unwrap();
944 let roster: std::collections::BTreeMap<String, RosterEntry> = relay
945 .roster()
946 .unwrap()
947 .into_iter()
948 .map(|e| (e.device_id.clone(), e))
949 .collect();
950 assert_eq!(
951 roster["mac-b"].acked, None,
952 "merely-received is never acked"
953 );
954
955 // --- Crash point 2: fold journaled durably, ack lost (network died
956 // between the fsync and the ack).
957 let fail = Arc::new(AtomicBool::new(true));
958 let mut b = open("mac-b", &db);
959 b.append(Scope::Personal, Surface::Knowledge, json!({"id": "f2"}))
960 .unwrap();
961 {
962 let mut flaky = FlakyAckRelay {
963 inner: &mut relay,
964 fail_ack: fail.clone(),
965 };
966 let err = b.pump(&mut flaky).unwrap_err();
967 assert!(matches!(err, SessionError::Relay(RelayError::Io(_))));
968 }
969 // The fold IS durable (journal has a's op)…
970 let on_disk = OplogJournal::load(&db.journal).unwrap();
971 assert_eq!(on_disk.len(), 2);
972 // …but the ack never landed — the SAFE direction: GC can't drop
973 // what b hasn't acked.
974 let roster: std::collections::BTreeMap<String, RosterEntry> = relay
975 .roster()
976 .unwrap()
977 .into_iter()
978 .map(|e| (e.device_id.clone(), e))
979 .collect();
980 assert_eq!(roster["mac-b"].acked, None);
981
982 // --- Crash point 3: session lost entirely (push cursor gone).
983 // Reopen from the journal and re-pump: re-push dedups by op_id,
984 // re-pull folds nothing new, the ack finally lands.
985 drop(b);
986 fail.store(false, Ordering::SeqCst);
987 let mut b = open("mac-b", &db);
988 let report = b.pump(&mut relay).unwrap();
989 assert_eq!(
990 report.folded, 0,
991 "re-pull re-fold is a no-op by op_id dedup"
992 );
993 assert_eq!(report.pushed, 0, "re-push deduped relay-side");
994 assert_eq!(report.push_deduped, 1);
995 assert!(report.acked.is_some());
996
997 a.pump(&mut relay).unwrap();
998 assert_eq!(
999 a.state_hash(),
1000 b.state_hash(),
1001 "convergence after every crash point"
1002 );
1003 }
1004
1005 #[test]
1006 fn cold_bootstrap_goes_through_resume_anchored_and_stays_fenced() {
1007 // Contract 5: a cold device bootstraps checkpoint-first; its
1008 // journal carries the truncation marker, so the naive
1009 // load()/DeviceLog::resume path stays a runtime error.
1010 let mut relay = mem_relay();
1011 let da = dirs();
1012 let mut a = open("mac-a", &da);
1013 for i in 0..4 {
1014 a.append(
1015 Scope::Personal,
1016 Surface::Knowledge,
1017 json!({"id": format!("f{i}"), "timestamp": i}),
1018 )
1019 .unwrap();
1020 }
1021 a.pump(&mut relay).unwrap();
1022 let ckpt = a.publish_checkpoint(&mut relay).unwrap().unwrap();
1023 relay.gc().unwrap();
1024
1025 // Fresh device: bootstrap = checkpoint_get + pull(since ckpt
1026 // frontier) + resume_anchored.
1027 let db = dirs();
1028 let mut b =
1029 SyncSession::bootstrap("mac-b", &db.journal, &db.ckpts, &mut relay, zero_wall())
1030 .unwrap();
1031 assert_eq!(b.base().unwrap().checkpoint_hash, ckpt.checkpoint_hash);
1032 b.pump(&mut relay).unwrap();
1033 assert_eq!(b.state_hash(), a.state_hash());
1034
1035 // The fences hold on the bootstrapped journal.
1036 let err = OplogJournal::load(&db.journal).unwrap_err();
1037 assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
1038 let (marker, tail) = OplogJournal::load_with_marker(&db.journal).unwrap();
1039 assert_eq!(marker.unwrap().checkpoint_hash, ckpt.checkpoint_hash);
1040 // b keeps working across a restart (open() resumes anchored)…
1041 drop(b);
1042 let mut b = open("mac-b", &db);
1043 let op = b
1044 .append(Scope::Personal, Surface::Knowledge, json!({"id": "from-b"}))
1045 .unwrap();
1046 assert!(
1047 op.hlc
1048 > tail.iter().map(|o| o.hlc.clone()).max().unwrap_or(Hlc {
1049 wall_ms: 0,
1050 counter: 0,
1051 device_id: String::new()
1052 })
1053 );
1054 b.pump(&mut relay).unwrap();
1055 a.pump(&mut relay).unwrap();
1056 assert_eq!(a.state_hash(), b.state_hash());
1057 }
1058
1059 #[test]
1060 fn partial_fold_failure_is_retry_safe_and_never_bricks_the_journal() {
1061 // Kernel-review BRICKED-DEVICE repro (mechanism, not just the
1062 // duplicated-line consequence): a mid-fold-loop journal append
1063 // failure must leave self.ops == the journal, so the retry filters
1064 // the already-journaled ops out of the re-pull instead of writing
1065 // them a SECOND time (a duplicate line = DuplicateSeq on the next
1066 // open = a permanently bricked device with no recovery API).
1067 let mut relay = mem_relay();
1068 let (da, db) = (dirs(), dirs());
1069 let mut a = open("mac-a", &da);
1070 a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
1071 .unwrap();
1072 a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f2"}))
1073 .unwrap();
1074 a.pump(&mut relay).unwrap();
1075
1076 let mut b = open("mac-b", &db);
1077 // Force the SECOND fold append to fail (ENOSPC-class).
1078 b.journal.fail_append_after = Some(1);
1079 let err = b.pump(&mut relay).unwrap_err();
1080 assert!(matches!(err, SessionError::Io(_)), "got {err:?}");
1081
1082 // Lockstep invariant: self.ops holds exactly what the journal holds
1083 // (the one op that appended before the failure), never more.
1084 assert_eq!(
1085 b.ops().len(),
1086 1,
1087 "only the successfully-journaled fold is in self.ops"
1088 );
1089 let on_disk = OplogJournal::load(&db.journal).unwrap();
1090 assert_eq!(on_disk.len(), 1);
1091 assert_eq!(on_disk[0].op_id, b.ops()[0].op_id);
1092
1093 // Retry: the seam auto-cleared, so this pump folds the remaining op.
1094 // Crucially it does NOT re-journal the first fold (filtered by op_id
1095 // from self.ops) — no duplicate line.
1096 let report = b.pump(&mut relay).unwrap();
1097 assert_eq!(
1098 report.folded, 1,
1099 "only the un-journaled op is folded on retry"
1100 );
1101 let on_disk = OplogJournal::load(&db.journal).unwrap();
1102 assert_eq!(
1103 on_disk.len(),
1104 2,
1105 "no duplicate line — the journal is not bricked"
1106 );
1107 verify_log(&on_disk).expect("no DuplicateSeq: the journal opens cleanly");
1108
1109 // The device is not bricked: it reopens and converges.
1110 drop(b);
1111 let b = open("mac-b", &db);
1112 assert_eq!(a.state_hash(), b.state_hash());
1113 }
1114
1115 #[test]
1116 fn append_failure_rolls_the_chain_back() {
1117 // Force a journal append failure by dropping the journal file's
1118 // directory out from under it is not portable; instead exercise the
1119 // rebuild path directly: rebuild_device must reproduce the exact
1120 // chain position after arbitrary appends.
1121 let d = dirs();
1122 let mut a = open("mac-a", &d);
1123 a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
1124 .unwrap();
1125 a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f2"}))
1126 .unwrap();
1127 let before = a.ops().to_vec();
1128 a.rebuild_device().unwrap();
1129 let next = a
1130 .append(Scope::Personal, Surface::Knowledge, json!({"id": "f3"}))
1131 .unwrap();
1132 assert_eq!(next.seq, 2);
1133 assert_eq!(next.prev.as_deref(), Some(before[1].op_id.as_str()));
1134 let mut all = before;
1135 all.push(next);
1136 verify_log(&all).unwrap();
1137 }
1138
1139 // ------------------------------------------------------------------
1140 // B5: execution lease + fencing, end-to-end through the relay.
1141 // ------------------------------------------------------------------
1142
1143 #[test]
1144 fn partition_both_write_intents_converge_to_the_higher_epoch_winner() {
1145 // The full arc: mac-a holds the lease (epoch 1) and fires a scheduled
1146 // run; it pauses past its TTL; mac-b STEALS the lease (epoch 2) and
1147 // fires the SAME run (same B7 deterministic run_id); mac-a, partitioned
1148 // from the coordinator, wrongly still believes it holds epoch 1 and
1149 // ALSO commits the run (a real zombie). After convergence the fold
1150 // fences the zombie deterministically — one ledger record, the
1151 // higher-epoch (mac-b) commit wins, even though the zombie's op has a
1152 // LATER HLC. No double-execution; both devices agree.
1153 use crate::lease::{InMemoryLeaseCoordinator, IntentStatus};
1154 use std::sync::atomic::AtomicU64;
1155
1156 let coord_t = Arc::new(AtomicU64::new(0));
1157 let reader = coord_t.clone();
1158 let coord_wall: WallClock = Arc::new(move || reader.load(Ordering::SeqCst));
1159 let mut coord = InMemoryLeaseCoordinator::new(coord_wall);
1160
1161 let mut relay = mem_relay();
1162 let (da, db) = (dirs(), dirs());
1163 let mut a = open("mac-a", &da);
1164 let mut b = open("mac-b", &db);
1165 let run = car_proto::deterministic_run_id("milo", "3am digest", "occurrence-1");
1166
1167 // 1. mac-a acquires (epoch 1) and records the run's pending intent
1168 // through the GATED path (coordinator confirms it still holds).
1169 let lease_a = coord.acquire("milo", "mac-a", 100).unwrap();
1170 assert_eq!(lease_a.epoch, 1);
1171 let recorded = a
1172 .record_intent_if_current(
1173 Scope::Personal,
1174 &Intent::new("milo", &run, 1, IntentStatus::Pending),
1175 &mut coord,
1176 )
1177 .unwrap();
1178 assert!(
1179 recorded.is_some(),
1180 "mac-a holds the lease → intent recorded"
1181 );
1182 a.pump(&mut relay).unwrap();
1183
1184 // 2. mac-b folds a's state, then — a's lid closed past the TTL —
1185 // STEALS the lease (epoch 2) and commits the same run.
1186 b.pump(&mut relay).unwrap();
1187 coord_t.store(200, Ordering::SeqCst); // past mac-a's 100ms TTL
1188 let lease_b = coord.acquire("milo", "mac-b", 100).unwrap();
1189 assert_eq!((lease_b.epoch, lease_b.holder.as_str()), (2, "mac-b"));
1190 b.record_intent_if_current(
1191 Scope::Personal,
1192 &Intent::new("milo", &run, 2, IntentStatus::Pending),
1193 &mut coord,
1194 )
1195 .unwrap()
1196 .expect("mac-b holds epoch 2");
1197 let b_commit = b
1198 .record_intent_if_current(
1199 Scope::Personal,
1200 &Intent::new("milo", &run, 2, IntentStatus::Committed),
1201 &mut coord,
1202 )
1203 .unwrap()
1204 .expect("mac-b holds epoch 2");
1205 b.pump(&mut relay).unwrap();
1206
1207 // 3. mac-a is PARTITIONED from the coordinator (never learns it lost)
1208 // but still syncs with the relay. It folds b's ops, then — a genuine
1209 // double-execution artifact — also commits the run via the UNGATED
1210 // path, stamping a LATER HLC than b's committed op. (A committed
1211 // write is allowed even for an already-committed run — idempotent;
1212 // the fold converges it. The terminal guard only no-ops a *pending*
1213 // write for an already-committed run — see the C3 session test.)
1214 a.pump(&mut relay).unwrap();
1215 let a_zombie = a
1216 .record_intent(
1217 Scope::Personal,
1218 &Intent::new("milo", &run, 1, IntentStatus::Committed),
1219 )
1220 .unwrap()
1221 .expect("a committed write is recorded (not terminal-guarded)");
1222 assert!(
1223 a_zombie.hlc > b_commit.hlc,
1224 "the zombie's write is later in HLC"
1225 );
1226 a.pump(&mut relay).unwrap();
1227 b.pump(&mut relay).unwrap();
1228
1229 // 4. Converge: both agree; the epoch-2 (mac-b) commit is the ledger
1230 // winner despite the zombie's later HLC (fencing beats the clock),
1231 // and the fence-independent oracle agrees.
1232 assert_eq!(a.state_hash(), b.state_hash(), "divergence invariant holds");
1233 let state = a.state();
1234 assert_eq!(state.intents["milo"].runs.len(), 1, "one who-holds record");
1235 assert_eq!(state.fencing_epoch("milo"), Some(2));
1236 let winner = state.intent("milo", &run).expect("run present");
1237 assert_eq!(
1238 winner.op_id, b_commit.op_id,
1239 "higher-epoch commit wins despite lower HLC"
1240 );
1241 let decoded = Intent::from_payload(&winner.payload).unwrap();
1242 assert_eq!(
1243 (decoded.epoch, decoded.status),
1244 (2, IntentStatus::Committed)
1245 );
1246 // The durable idempotency oracle also resolves to the epoch-2 commit,
1247 // on both devices — the read a B6 dispatch fence would perform.
1248 assert_eq!(a.committed_run("milo", &run).unwrap().op_id, b_commit.op_id);
1249 assert_eq!(b.committed_run("milo", &run).unwrap().op_id, b_commit.op_id);
1250 }
1251
1252 #[test]
1253 fn record_intent_if_current_gates_out_a_lost_holder() {
1254 // The best-effort local gate when the coordinator IS reachable: once
1255 // mac-a no longer holds the lease, the gated write records nothing
1256 // (saving an op the fold would only fence). This is a liveness
1257 // optimization — the sound gate remains the fold's epoch fence.
1258 use crate::lease::{InMemoryLeaseCoordinator, IntentStatus};
1259 let mut coord = InMemoryLeaseCoordinator::new(zero_wall());
1260 let d = dirs();
1261 let mut a = open("mac-a", &d);
1262 let run = "run-x";
1263
1264 coord.acquire("milo", "mac-a", 100).unwrap();
1265 let recorded = a
1266 .record_intent_if_current(
1267 Scope::Personal,
1268 &Intent::new("milo", run, 1, IntentStatus::Pending),
1269 &mut coord,
1270 )
1271 .unwrap();
1272 assert!(recorded.is_some());
1273 assert_eq!(a.ops().len(), 1);
1274
1275 // mac-a releases; mac-b acquires epoch 2. The coordinator now reports
1276 // mac-b as the holder → the gate refuses mac-a's write.
1277 coord.release("milo", "mac-a", 1).unwrap();
1278 coord.acquire("milo", "mac-b", 100).unwrap();
1279 let gated = a
1280 .record_intent_if_current(
1281 Scope::Personal,
1282 &Intent::new("milo", run, 1, IntentStatus::Committed),
1283 &mut coord,
1284 )
1285 .unwrap();
1286 assert!(
1287 gated.is_none(),
1288 "coordinator says mac-a lost → not recorded"
1289 );
1290 assert_eq!(a.ops().len(), 1, "nothing new journaled");
1291 }
1292
1293 #[test]
1294 fn c3_record_intent_no_ops_a_pending_for_an_already_committed_run() {
1295 // C3 write-side guard: once a run has committed (per the
1296 // fence-independent oracle), record_intent NO-OPs a later
1297 // pending/failed write for it — a failed-over holder that writes
1298 // `pending` before checking cannot revert the committed ledger. This is
1299 // the ungated path (no coordinator); the guard is a local oracle read.
1300 let mut relay = mem_relay();
1301 let (da, db) = (dirs(), dirs());
1302 let mut a = open("mac-a", &da);
1303 let mut b = open("mac-b", &db);
1304 let run = "run-nightly";
1305
1306 // mac-a commits the run at epoch 1 and syncs it to mac-b.
1307 a.record_intent(
1308 Scope::Personal,
1309 &Intent::new("milo", run, 1, IntentStatus::Committed),
1310 )
1311 .unwrap()
1312 .expect("first commit recorded");
1313 a.pump(&mut relay).unwrap();
1314 b.pump(&mut relay).unwrap();
1315 assert!(
1316 b.committed_run("milo", run).is_some(),
1317 "mac-b folded the commit"
1318 );
1319
1320 // mac-b fails over to epoch 2 and — before checking — tries to write the
1321 // run PENDING. The terminal guard no-ops it (the run already committed).
1322 let before = b.ops().len();
1323 let attempt = b
1324 .record_intent(
1325 Scope::Personal,
1326 &Intent::new("milo", run, 2, IntentStatus::Pending),
1327 )
1328 .unwrap();
1329 assert!(
1330 attempt.is_none(),
1331 "pending for an already-committed run is a no-op"
1332 );
1333 assert_eq!(b.ops().len(), before, "nothing journaled");
1334
1335 // The ledger stays committed on both sides after further sync.
1336 b.pump(&mut relay).unwrap();
1337 a.pump(&mut relay).unwrap();
1338 assert!(a.committed_run("milo", run).is_some());
1339 let decoded =
1340 Intent::from_payload(&b.state().intent("milo", run).unwrap().payload).unwrap();
1341 assert_eq!(
1342 decoded.status,
1343 IntentStatus::Committed,
1344 "not reverted to pending"
1345 );
1346 }
1347}