Skip to main content

car_sync/
session.rs

1//! The device-side sync session — the pump that drives a [`DeviceLog`] +
2//! [`OplogJournal`] pair against a [`Relay`] (slice B3 of
3//! `docs/proposals/multi-device-sync.md`).
4//!
5//! [`SyncSession::pump`] is one reconciliation round, sequenced so the
6//! binding contracts from B1/B4 hold **by construction**, not by caller
7//! discipline:
8//!
9//! 1. **Push** — only ops read from (or appended through) the journal are
10//!    ever handed to the relay, and the journal is **fsync'd**
11//!    ([`OplogJournal::sync`]) before the push, so **an op is journal-durable
12//!    before it is transmitted** (B1 MUST — `flush` is only the page cache;
13//!    a power loss in the writeback window would otherwise lose a
14//!    transmitted op and re-mint its seq into a permanent Fork). A crash
15//!    that loses the push cursor is harmless — a re-push dedups relay-side
16//!    on `op_id`.
17//! 2. **Pull** — `pull(since = my per-device seq frontier)`, then
18//!    **verify before fold** (B1 MUST): the union of held + pulled ops
19//!    must pass [`verify_log`] (or [`verify_anchored`] against the
20//!    session's base checkpoint) before anything is folded or journaled.
21//! 3. **Fold durably, then ack** — verified remote ops are appended to the
22//!    journal, **advancing `self.ops` in lockstep** (each op enters
23//!    `self.ops` the instant its append succeeds, so a mid-loop failure
24//!    leaves `self.ops == journal` and the retry re-pull filters the
25//!    already-journaled ops instead of duplicating them — a duplicate line
26//!    would `DuplicateSeq`-brick the next open). The journal is then fsync'd
27//!    and only then is `ack` sent. The ack value is *derived from the
28//!    journal-held ops* — there is no API to ack anything else, so **acking
29//!    merely-received (un-journaled) state is impossible by construction**
30//!    (B4 MUST). A crash between the fold and the ack leaves the relay's ack
31//!    table behind — the safe direction: GC can't drop what we haven't
32//!    acked, and the next pump re-acks.
33//!
34//! Idempotence: re-running `pump` after any crash point re-pushes
35//! (relay dedups by `op_id`), re-pulls (already-held ops are filtered by
36//! `op_id`; re-folding is a no-op), and re-acks (monotone) — the whole
37//! round is retry-safe.
38//!
39//! **Cold bootstrap / straggler re-entry** ([`SyncSession::bootstrap`] /
40//! [`SyncSession::rebase`]): when the relay has GC'd past the session's
41//! frontier (`RelayError::FrontierTruncated`) or the device is brand new,
42//! the path is exactly the proposal's — `checkpoint_get()` → `pull(since =
43//! checkpoint frontier)` → verify → journal rewritten as checkpoint-anchored
44//! tail (`truncate_to`, stamping the truncation marker so the naive
45//! `load`/`resume` stays fenced — B4 contract 5) → **`resume_anchored`**,
46//! never `DeviceLog::resume`. Local ops **not covered** by the checkpoint —
47//! including a returning straggler's never-pushed writes — are carried into
48//! the rebased tail and pushed on the next pump: re-entry is lossless (see
49//! the relay module docs for why this is safe under seq-based frontiers).
50
51use crate::checkpoint::{resume_anchored, AnchorError, Checkpoint, CheckpointError};
52use crate::crypto::{CryptoError, Envelope, SyncKeyProvider};
53use crate::fold::{fold_onto, state_hash, FoldedRecord, SyncState};
54use crate::journal::OplogJournal;
55use crate::lease::{Intent, IntentStatus, LeaseCoordinator, LeaseError};
56use crate::oplog::{verify_log, ChainError, DeviceLog, Hlc, OpRecord, Scope, Surface, WallClock};
57use crate::relay::{checkpoint_frontier, frontier_of, Frontier, Relay, RelayError};
58use serde_json::Value;
59use std::fmt;
60use std::path::{Path, PathBuf};
61use std::sync::Arc;
62
63/// A sync-session failure.
64#[derive(Debug)]
65pub enum SessionError {
66    Io(std::io::Error),
67    Chain(ChainError),
68    Anchor(AnchorError),
69    Relay(RelayError),
70    Checkpoint(CheckpointError),
71    /// The journal carries a truncation marker naming a checkpoint that is
72    /// not present in the session's checkpoint directory — resume is
73    /// impossible without it (fetch it from the relay and retry).
74    MissingCheckpoint {
75        checkpoint_hash: String,
76    },
77    /// The relay served a "latest" checkpoint that does not cover the
78    /// session's current base — rebasing onto it would silently lose state.
79    CheckpointRegression {
80        held: String,
81        offered: String,
82    },
83    /// A lease-coordinator call failed (B5) — used by the best-effort local
84    /// gate [`SyncSession::record_intent_if_current`].
85    Lease(LeaseError),
86    /// Encrypting an op payload / decrypting a folded op failed (B6 E2E).
87    Crypto(CryptoError),
88}
89
90impl fmt::Display for SessionError {
91    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
92        match self {
93            SessionError::Io(e) => write!(f, "sync session io error: {e}"),
94            SessionError::Chain(e) => write!(f, "sync session chain error: {e}"),
95            SessionError::Anchor(e) => write!(f, "sync session anchor error: {e}"),
96            SessionError::Relay(e) => write!(f, "sync session relay error: {e}"),
97            SessionError::Checkpoint(e) => write!(f, "sync session checkpoint error: {e}"),
98            SessionError::MissingCheckpoint { checkpoint_hash } => write!(
99                f,
100                "journal is truncated below checkpoint {checkpoint_hash}, which is not in the \
101                 session checkpoint directory — fetch it (relay checkpoint_get) and retry"
102            ),
103            SessionError::CheckpointRegression { held, offered } => write!(
104                f,
105                "relay's latest checkpoint {offered} does not cover the session's base {held} — \
106                 refusing to rebase onto it (state would be lost)"
107            ),
108            SessionError::Lease(e) => write!(f, "sync session lease error: {e}"),
109            SessionError::Crypto(e) => write!(f, "sync session crypto error: {e}"),
110        }
111    }
112}
113
114impl std::error::Error for SessionError {}
115
116impl From<std::io::Error> for SessionError {
117    fn from(e: std::io::Error) -> Self {
118        SessionError::Io(e)
119    }
120}
121impl From<ChainError> for SessionError {
122    fn from(e: ChainError) -> Self {
123        SessionError::Chain(e)
124    }
125}
126impl From<AnchorError> for SessionError {
127    fn from(e: AnchorError) -> Self {
128        SessionError::Anchor(e)
129    }
130}
131impl From<RelayError> for SessionError {
132    fn from(e: RelayError) -> Self {
133        SessionError::Relay(e)
134    }
135}
136impl From<CheckpointError> for SessionError {
137    fn from(e: CheckpointError) -> Self {
138        SessionError::Checkpoint(e)
139    }
140}
141impl From<LeaseError> for SessionError {
142    fn from(e: LeaseError) -> Self {
143        SessionError::Lease(e)
144    }
145}
146
147/// What one [`SyncSession::pump`] round did.
148#[derive(Debug, Clone, Default, PartialEq)]
149pub struct PumpReport {
150    /// Own ops newly admitted by the relay.
151    pub pushed: usize,
152    /// Own ops the relay already held.
153    pub push_deduped: usize,
154    /// Remote ops pulled, verified, journaled, and folded this round.
155    pub folded: usize,
156    /// The fold frontier acked (derived from journal-held ops), if any.
157    pub acked: Option<Hlc>,
158}
159
160/// A device's live sync endpoint: its append chain, its durable journal,
161/// and the pump. See the module docs for the contract sequencing.
162pub struct SyncSession {
163    device_id: String,
164    device: DeviceLog,
165    journal: OplogJournal,
166    checkpoint_dir: PathBuf,
167    wall: WallClock,
168    /// Every op the journal holds (the tail, when `base` is set).
169    ops: Vec<OpRecord>,
170    /// The checkpoint the journal is anchored on, when truncated.
171    base: Option<Checkpoint>,
172    /// In-memory push cursor (own max seq pushed). Deliberately NOT
173    /// persisted: losing it in a crash only causes a deduped re-push.
174    pushed_through: Option<u64>,
175    /// E2E key provider (B6). When `Some`, op payloads are encrypted at
176    /// [`SyncSession::append`] (so the op is ciphertext-native — `op_id`/chain
177    /// cover the ciphertext, and a remote relay stores only ciphertext) and
178    /// decrypted at [`SyncSession::state`], **after** the chain verifies and
179    /// **before** the fold groups them. `None` = cleartext (local `FsRelay`).
180    key_provider: Option<Arc<dyn SyncKeyProvider>>,
181}
182
183impl fmt::Debug for SyncSession {
184    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
185        f.debug_struct("SyncSession")
186            .field("device_id", &self.device_id)
187            .field("ops", &self.ops.len())
188            .field("base", &self.base.as_ref().map(|c| &c.checkpoint_hash))
189            .field("pushed_through", &self.pushed_through)
190            .finish_non_exhaustive()
191    }
192}
193
194impl SyncSession {
195    /// Open a session over an existing (possibly empty, possibly truncated)
196    /// journal. A truncated journal resumes **only** through its covering
197    /// checkpoint (`resume_anchored` — B4 contract): the checkpoint file
198    /// must be present in `checkpoint_dir` under its content address.
199    pub fn open(
200        device_id: impl Into<String>,
201        journal_path: &Path,
202        checkpoint_dir: &Path,
203        wall: WallClock,
204    ) -> Result<Self, SessionError> {
205        let device_id = device_id.into();
206        let (marker, ops) = OplogJournal::load_with_marker(journal_path)?;
207        let journal = OplogJournal::open(journal_path)?;
208        let (device, base) = match marker {
209            Some(marker) => {
210                let path =
211                    checkpoint_dir.join(format!("{}.checkpoint.json", marker.checkpoint_hash));
212                if !path.exists() {
213                    return Err(SessionError::MissingCheckpoint {
214                        checkpoint_hash: marker.checkpoint_hash,
215                    });
216                }
217                let checkpoint = Checkpoint::load(&path)?;
218                let device = resume_anchored(device_id.clone(), &checkpoint, &ops)?;
219                (device, Some(checkpoint))
220            }
221            None => (DeviceLog::resume(device_id.clone(), &ops)?, None),
222        };
223        let mut session = Self {
224            device_id,
225            device,
226            journal,
227            checkpoint_dir: checkpoint_dir.to_path_buf(),
228            wall,
229            ops,
230            base,
231            pushed_through: None,
232            key_provider: None,
233        };
234        session.device.set_wall_clock(session.wall.clone());
235        Ok(session)
236    }
237
238    /// Turn on E2E: op payloads are encrypted at [`SyncSession::append`] and
239    /// decrypted at [`SyncSession::state`], keyed per scope audience by
240    /// `provider` (a login-derived [`crate::crypto::DerivedKeyProvider`] in the
241    /// daemon). Builder so [`SyncSession::open`]'s signature stays stable for the
242    /// local (cleartext `FsRelay`) path. NOTE: with a *remote* relay, checkpoint
243    /// state is still cleartext-folded — do not push checkpoints to an untrusted
244    /// relay until per-scope checkpoint encryption lands (crypto.rs follow-up).
245    pub fn with_key_provider(mut self, provider: Arc<dyn SyncKeyProvider>) -> Self {
246        self.key_provider = Some(provider);
247        self
248    }
249
250    /// Open + immediately [`SyncSession::rebase`] onto the relay's latest
251    /// checkpoint — the cold-device / returning-straggler entry point
252    /// (proposal §"Cold / new device bootstrap"). With no relay checkpoint
253    /// (young account) this degrades to a plain open; the first `pump`
254    /// replays from genesis.
255    pub fn bootstrap(
256        device_id: impl Into<String>,
257        journal_path: &Path,
258        checkpoint_dir: &Path,
259        relay: &mut dyn Relay,
260        wall: WallClock,
261    ) -> Result<Self, SessionError> {
262        let mut session = Self::open(device_id, journal_path, checkpoint_dir, wall)?;
263        session.rebase(relay)?;
264        Ok(session)
265    }
266
267    /// Record a local mutation: stamp (hybrid clock), **journal (flushed)**,
268    /// then hold for push — the op is journal-durable before it can ever be
269    /// transmitted (B1 MUST). If the journal write fails the device chain
270    /// is rolled back (rebuilt from the durable ops), so the failed op can
271    /// never leave a hole for the next append to chain onto.
272    pub fn append(
273        &mut self,
274        scope: Scope,
275        surface: Surface,
276        payload: Value,
277    ) -> Result<OpRecord, SessionError> {
278        // E2E: encrypt the payload BEFORE building the op, so `op_id`/the chain
279        // cover the ciphertext and a remote relay never sees cleartext. The
280        // cipher is chosen by the op's scope audience (personal vs org key).
281        let payload = match &self.key_provider {
282            Some(provider) => provider
283                .cipher_for(&scope)
284                .encrypt(&payload)
285                .map_err(SessionError::Crypto)?,
286            None => payload,
287        };
288        let op = self.device.append(scope, surface, payload);
289        if let Err(e) = self.journal.append(&op) {
290            self.rebuild_device()?;
291            return Err(SessionError::Io(e));
292        }
293        self.ops.push(op.clone());
294        Ok(op)
295    }
296
297    /// The fence-independent **committed-run idempotency oracle** (B5) over
298    /// this session's folded state (checkpoint base + journal tail): "has
299    /// `run_id` already committed for `agent_id`?" Keep-all and immune to epoch
300    /// bumps AND compaction, so it is the CORRECT idempotency lookup — unlike
301    /// [`crate::fold::SyncState::intent`], which is the fenced "who holds now"
302    /// view and can read `None`/pending for a run that actually committed. A B6
303    /// dispatch fence performs exactly this read before an external side effect.
304    pub fn committed_run(&self, agent_id: &str, run_id: &str) -> Option<FoldedRecord> {
305        self.state().committed_run(agent_id, run_id).cloned()
306    }
307
308    /// Record a leased execution [`Intent`] (B5) — journal-durable exactly
309    /// like any [`SyncSession::append`], so it is durable before it can be
310    /// transmitted. The **partitioned / ungated** path: a zombie that cannot
311    /// reach the coordinator still records here, and the fold converges the
312    /// ledger deterministically.
313    ///
314    /// **Terminal guard (C3):** a committed run is terminal. A `pending`/`failed`
315    /// write for a run already committed (per the fence-independent
316    /// [`SyncSession::committed_run`] oracle) is a **no-op** (`Ok(None)`), so a
317    /// failed-over holder writing `pending` before checking cannot revert the
318    /// committed ledger.
319    ///
320    /// This converges the **ledger**; it is NOT the exactly-once execution gate.
321    /// The B6 dispatch fence — a linearizable "am I still epoch N?" plus this
322    /// committed-oracle read **before** the external effect — is what makes
323    /// execution single-shot.
324    pub fn record_intent(
325        &mut self,
326        scope: Scope,
327        intent: &Intent,
328    ) -> Result<Option<OpRecord>, SessionError> {
329        if intent.status != IntentStatus::Committed
330            && self
331                .committed_run(&intent.agent_id, &intent.run_id)
332                .is_some()
333        {
334            return Ok(None); // terminal: the run already committed — do not revert
335        }
336        Ok(Some(self.append(
337            scope,
338            Surface::Intent,
339            intent.payload(),
340        )?))
341    }
342
343    /// Record a leased [`Intent`] **only if the local epoch is still current**
344    /// — the best-effort local gate: a linearizable coordinator read confirms
345    /// this device still holds the lease at `intent.epoch` before the op is
346    /// journaled. Returns `Ok(None)` when the coordinator says this device is
347    /// no longer the holder at that epoch (skipping a write the fold would fence)
348    /// or when the run is already committed (the [`SyncSession::record_intent`]
349    /// terminal guard).
350    ///
351    /// This is a *liveness optimization*, NOT the safety gate: a partitioned
352    /// zombie that cannot reach the coordinator falls back to
353    /// [`SyncSession::record_intent`]. Exactly-once execution is the B6 dispatch
354    /// fence (this check races a pause-after-check — the Kleppmann residual).
355    pub fn record_intent_if_current(
356        &mut self,
357        scope: Scope,
358        intent: &Intent,
359        coordinator: &mut dyn LeaseCoordinator,
360    ) -> Result<Option<OpRecord>, SessionError> {
361        let current = coordinator.current(&intent.agent_id)?;
362        let still_ours = current
363            .as_ref()
364            .is_some_and(|l| l.holder == self.device_id && l.epoch == intent.epoch);
365        if !still_ours {
366            return Ok(None); // not the current holder — don't even write it
367        }
368        self.record_intent(scope, intent) // terminal-guarded
369    }
370
371    /// Rebuild the append chain from what is actually durable — the
372    /// journal-held ops (+ base anchor).
373    fn rebuild_device(&mut self) -> Result<(), SessionError> {
374        let mut device = match &self.base {
375            Some(checkpoint) => resume_anchored(self.device_id.clone(), checkpoint, &self.ops)?,
376            None => DeviceLog::resume(self.device_id.clone(), &self.ops)?,
377        };
378        device.set_wall_clock(self.wall.clone());
379        self.device = device;
380        Ok(())
381    }
382
383    /// The per-device seq cursor of everything this session holds
384    /// (checkpoint coverage + journal tail).
385    fn held_frontier(&self) -> Frontier {
386        let mut frontier = self
387            .base
388            .as_ref()
389            .map(checkpoint_frontier)
390            .unwrap_or_default();
391        for (device, seq) in frontier_of(&self.ops) {
392            let entry = frontier.entry(device).or_insert(seq);
393            if seq > *entry {
394                *entry = seq;
395            }
396        }
397        frontier
398    }
399
400    /// The fold frontier this session may ack: the max HLC across
401    /// journal-held ops and the base checkpoint's coverage — derived from
402    /// durable state ONLY, which is what makes ack-before-fold impossible.
403    fn ack_frontier(&self) -> Option<Hlc> {
404        let from_ops = self.ops.iter().map(|op| &op.hlc).max();
405        let from_base = self
406            .base
407            .as_ref()
408            .and_then(|c| c.frontier.values().map(|e| &e.hlc).max());
409        [from_ops, from_base].into_iter().flatten().max().cloned()
410    }
411
412    /// One reconciliation round: push journal-durable own ops → pull →
413    /// verify → journal the folds → ack. See the module docs for the
414    /// contract sequencing; retry-safe at every crash point.
415    ///
416    /// Returns [`RelayError::FrontierTruncated`] (wrapped) when the relay
417    /// has GC'd past this session's frontier — call
418    /// [`SyncSession::rebase`] and pump again.
419    pub fn pump(&mut self, relay: &mut dyn Relay) -> Result<PumpReport, SessionError> {
420        let mut report = PumpReport::default();
421
422        // 1. Push own journal-durable ops the relay may not have.
423        let mut own: Vec<OpRecord> = self
424            .ops
425            .iter()
426            .filter(|op| {
427                op.device_id == self.device_id
428                    && self.pushed_through.is_none_or(|through| op.seq > through)
429            })
430            .cloned()
431            .collect();
432        own.sort_by_key(|op| op.seq);
433        if !own.is_empty() {
434            // Journal-durable BEFORE transmit (B1 MUST): the per-op appends
435            // only reached the OS page cache (flush, not fsync). One batch
436            // barrier here makes the whole own-op tail stable before any of
437            // it leaves the device — a power loss in the writeback window
438            // otherwise loses a transmitted op and re-mints its seq (a
439            // permanent Fork). fsync once, not per append.
440            self.journal.sync()?;
441            let outcome = relay.push(&self.device_id, &own)?;
442            report.pushed = outcome.accepted;
443            report.push_deduped = outcome.deduped;
444            self.pushed_through = own.last().map(|op| op.seq);
445        }
446
447        // 2. Pull everything above what we hold.
448        let pulled = relay.pull(&self.device_id, &self.held_frontier())?;
449
450        // 3. Filter already-held (idempotent re-pull) and VERIFY BEFORE FOLD.
451        let held: std::collections::BTreeSet<&str> =
452            self.ops.iter().map(|op| op.op_id.as_str()).collect();
453        let new_ops: Vec<OpRecord> = pulled
454            .ops
455            .into_iter()
456            .filter(|op| !held.contains(op.op_id.as_str()))
457            .collect();
458        if !new_ops.is_empty() {
459            let mut candidate = self.ops.clone();
460            candidate.extend(new_ops.iter().cloned());
461            match &self.base {
462                Some(checkpoint) => crate::checkpoint::verify_anchored(checkpoint, &candidate)?,
463                None => verify_log(&candidate)?,
464            }
465
466            // 4. Journal the folds — advancing self.ops IN LOCKSTEP with the
467            // journal. Recording each op into self.ops the instant its
468            // append succeeds is load-bearing for retry-safety: a mid-loop
469            // append failure then leaves self.ops == the journal, so the
470            // retry pump filters the already-journaled ops out of the
471            // re-pull instead of appending them a second time (a duplicate
472            // journal line = DuplicateSeq on the next open = a bricked
473            // device). `candidate` is discarded on failure.
474            drop(candidate);
475            for op in &new_ops {
476                self.journal.append(op)?;
477                self.ops.push(op.clone());
478                self.device.observe(&op.hlc);
479                report.folded += 1;
480            }
481            // Durable fold BEFORE ack (B4 MUST: ack asserts durably-folded
482            // state). One batch fsync over the folds just journaled.
483            self.journal.sync()?;
484        }
485
486        // 5. Ack — derived from journal-held state only, after it is
487        // durable. (B4 MUST: ack asserts durably-folded state.)
488        if let Some(frontier) = self.ack_frontier() {
489            relay.ack(&self.device_id, frontier.clone())?;
490            report.acked = Some(frontier);
491        }
492        Ok(report)
493    }
494
495    /// Re-anchor this session on the relay's latest checkpoint — the cold
496    /// bootstrap / post-eviction re-entry move. Returns `true` when a
497    /// rebase happened.
498    ///
499    /// Sequencing (each step durable before the next depends on it):
500    /// `checkpoint_get` → `pull(since = checkpoint frontier)` → merge in
501    /// every locally-held op the checkpoint does NOT cover (a returning
502    /// straggler's unpushed writes survive) → `verify_anchored` → save the
503    /// checkpoint into the session checkpoint dir → `truncate_to` (journal
504    /// rewritten as the anchored tail, truncation marker stamped) →
505    /// `resume_anchored`. The push cursor resets so the next `pump`
506    /// re-offers every own op in the tail (relay dedups the already-pushed).
507    pub fn rebase(&mut self, relay: &mut dyn Relay) -> Result<bool, SessionError> {
508        let Some(checkpoint) = relay.checkpoint_get()? else {
509            return Ok(false); // young account: genesis replay via pump
510        };
511        if let Some(base) = &self.base {
512            if base.checkpoint_hash == checkpoint.checkpoint_hash {
513                return Ok(false); // already anchored here
514            }
515            if !crate::relay::frontier_dominates(&checkpoint, base) {
516                return Err(SessionError::CheckpointRegression {
517                    held: base.checkpoint_hash.clone(),
518                    offered: checkpoint.checkpoint_hash.clone(),
519                });
520            }
521        }
522
523        let pulled = relay.pull(&self.device_id, &checkpoint_frontier(&checkpoint))?;
524
525        // Keep every held op the checkpoint does not cover — own unpushed
526        // writes AND foreign tails we already folded — deduped against the
527        // pull by op_id.
528        let mut tail: Vec<OpRecord> = pulled.ops;
529        let mut seen: std::collections::BTreeSet<String> =
530            tail.iter().map(|op| op.op_id.clone()).collect();
531        for op in &self.ops {
532            let covered = checkpoint
533                .frontier
534                .get(&op.device_id)
535                .is_some_and(|entry| op.seq <= entry.seq);
536            if !covered && seen.insert(op.op_id.clone()) {
537                tail.push(op.clone());
538            }
539        }
540        tail.sort_by(|a, b| (&a.hlc, &a.op_id).cmp(&(&b.hlc, &b.op_id)));
541
542        // Verify BEFORE any durable rewrite; then checkpoint durable FIRST,
543        // then the journal truncation that names it (B4 crash ordering).
544        crate::checkpoint::verify_anchored(&checkpoint, &tail)?;
545        checkpoint.save(&self.checkpoint_dir)?;
546        self.journal
547            .truncate_to(&tail, &checkpoint.checkpoint_hash)?;
548
549        let mut device = resume_anchored(self.device_id.clone(), &checkpoint, &tail)?;
550        device.set_wall_clock(self.wall.clone());
551        self.device = device;
552        self.ops = tail;
553        self.base = Some(checkpoint);
554        self.pushed_through = None;
555        Ok(true)
556    }
557
558    /// Compute a checkpoint at the relay's stable frontier from this
559    /// session's held ops and upload it — the device-computed snapshot the
560    /// proposal requires under E2E ("the relay holds ciphertext and cannot
561    /// fold"). Call **after** a `pump` (so held == relay-known and own ops
562    /// are pushed). Returns the uploaded checkpoint, or `None` when there
563    /// is no stable frontier, nothing below it, or this session is itself
564    /// anchored on a checkpoint (recompaction over a base is the same
565    /// later slice B4 deferred).
566    pub fn publish_checkpoint(
567        &mut self,
568        relay: &mut dyn Relay,
569    ) -> Result<Option<Checkpoint>, SessionError> {
570        // E2E (B6): the ops are ciphertext-native, so a checkpoint folded from
571        // them would key facts by ciphertext hash (the fold groups on
572        // `payload["id"]`, hidden under the envelope) — a peer loading it as its
573        // decrypt-before-fold base would then fold cleartext tail onto a
574        // ciphertext base and DIVERGE. Publishing a *cleartext* checkpoint would
575        // instead leak state to the relay. So under E2E we do not publish to the
576        // (untrusted) relay: the encrypted op log is retained and cold bootstrap
577        // replays it. Per-scope *encrypted* checkpoint push (which restores
578        // relay-side GC) is the documented follow-up; until then the Parslee
579        // server owns retention. Local journal compaction is unaffected.
580        if self.key_provider.is_some() {
581            return Ok(None);
582        }
583        if self.base.is_some() {
584            return Ok(None);
585        }
586        let Some(frontier) = relay.stable_frontier()? else {
587            return Ok(None);
588        };
589        let below: Vec<OpRecord> = self
590            .ops
591            .iter()
592            .filter(|op| op.hlc <= frontier)
593            .cloned()
594            .collect();
595        if below.is_empty() {
596            return Ok(None);
597        }
598        let checkpoint = Checkpoint::from_ops(&below)?;
599        relay.checkpoint_put(&self.device_id, &checkpoint)?;
600        Ok(Some(checkpoint))
601    }
602
603    /// The materialized state: `fold_onto(base checkpoint, journal tail)`.
604    ///
605    /// Under E2E ([`SyncSession::with_key_provider`]) the tail is ciphertext, so
606    /// each op's payload is decrypted here — after the chain has verified, before
607    /// the fold groups on `payload["id"]`/`fold_key`. Op identity stays the
608    /// cleartext-metadata `op_id`; only the payload is swapped.
609    pub fn state(&self) -> SyncState {
610        let base = self
611            .base
612            .as_ref()
613            .map(|c| c.state.clone())
614            .unwrap_or_default();
615        match &self.key_provider {
616            Some(provider) => {
617                let decrypted = self.decrypted_tail(provider.as_ref());
618                fold_onto(&base, &decrypted)
619            }
620            None => fold_onto(&base, &self.ops),
621        }
622    }
623
624    /// The journal tail with each E2E payload decrypted for folding. An op that
625    /// is not an envelope (cleartext, e.g. mixed rollout) is passed through; one
626    /// we hold no key for (a foreign org audience) keeps its envelope — the fold
627    /// then groups it by content hash, harmlessly, rather than mis-applying it.
628    fn decrypted_tail(&self, provider: &dyn SyncKeyProvider) -> Vec<OpRecord> {
629        self.ops
630            .iter()
631            .map(|op| {
632                if !Envelope::is_envelope(&op.payload) {
633                    return op.clone();
634                }
635                match provider.cipher_for(&op.scope).decrypt(&op.payload) {
636                    Ok(plaintext) => {
637                        let mut cleartext = op.clone();
638                        cleartext.payload = plaintext;
639                        cleartext
640                    }
641                    Err(_) => op.clone(),
642                }
643            })
644            .collect()
645    }
646
647    /// [`state_hash`] of [`SyncSession::state`] — the divergence invariant
648    /// two synced devices must agree on.
649    pub fn state_hash(&self) -> String {
650        state_hash(&self.state())
651    }
652
653    pub fn device_id(&self) -> &str {
654        &self.device_id
655    }
656
657    /// The journal-held ops (the anchored tail, when a base is set).
658    pub fn ops(&self) -> &[OpRecord] {
659        &self.ops
660    }
661
662    /// The checkpoint this session's journal is anchored on, if truncated.
663    pub fn base(&self) -> Option<&Checkpoint> {
664        self.base.as_ref()
665    }
666}
667
668#[cfg(test)]
669mod tests {
670    use super::*;
671    use crate::relay::{
672        AckOutcome, GcReport, InMemoryRelay, PullResult, PushOutcome, RelayConfig, RosterEntry,
673    };
674    use serde_json::json;
675    use std::sync::atomic::{AtomicBool, Ordering};
676    use std::sync::Arc;
677
678    fn zero_wall() -> WallClock {
679        Arc::new(|| 0)
680    }
681
682    fn mem_relay() -> InMemoryRelay {
683        InMemoryRelay::new(RelayConfig::default(), zero_wall())
684    }
685
686    struct Dirs {
687        _tmp: tempfile::TempDir,
688        journal: std::path::PathBuf,
689        ckpts: std::path::PathBuf,
690    }
691
692    fn dirs() -> Dirs {
693        let tmp = tempfile::tempdir().unwrap();
694        let journal = tmp.path().join("oplog.jsonl");
695        let ckpts = tmp.path().join("checkpoints");
696        Dirs {
697            _tmp: tmp,
698            journal,
699            ckpts,
700        }
701    }
702
703    fn open(device: &str, d: &Dirs) -> SyncSession {
704        SyncSession::open(device, &d.journal, &d.ckpts, zero_wall()).unwrap()
705    }
706
707    #[test]
708    fn e2e_encrypts_payloads_on_the_wire_and_decrypts_for_the_fold() {
709        use crate::crypto::DerivedKeyProvider;
710        let provider: Arc<dyn SyncKeyProvider> =
711            Arc::new(DerivedKeyProvider::new(b"parslee-login-master".to_vec()));
712        let d = dirs();
713        let mut sess = open("mac", &d).with_key_provider(provider.clone());
714
715        let secret = json!({"id": "fact-1", "body": "slack bot token xoxb-SECRET"});
716        let op = sess
717            .append(Scope::Personal, Surface::Knowledge, secret.clone())
718            .unwrap();
719
720        // What a remote relay receives is ciphertext — never the cleartext.
721        assert!(
722            Envelope::is_envelope(&op.payload),
723            "the relay must only ever see an encrypted envelope"
724        );
725        assert_ne!(op.payload, secret);
726        // op_id covers the ciphertext (chain stays verifiable over what ships).
727        assert_eq!(sess.ops[0].op_id, op.op_id);
728
729        // A peer with the SAME login master recovers the cleartext.
730        let recovered = provider
731            .cipher_for(&Scope::Personal)
732            .decrypt(&op.payload)
733            .unwrap();
734        assert_eq!(recovered, secret);
735
736        // decrypt-before-fold runs without error (the fold sees cleartext).
737        let _ = sess.state();
738    }
739
740    #[test]
741    fn e2e_never_publishes_a_checkpoint_to_the_relay() {
742        // Under E2E a checkpoint would either leak cleartext or (folded from the
743        // ciphertext ops) form an inconsistent decrypt base — so publishing is
744        // guarded off. The relay must never receive one.
745        use crate::crypto::DerivedKeyProvider;
746        let provider: Arc<dyn SyncKeyProvider> =
747            Arc::new(DerivedKeyProvider::new(b"master".to_vec()));
748        let d = dirs();
749        let mut sess = open("mac", &d).with_key_provider(provider);
750        sess.append(
751            Scope::Personal,
752            Surface::Knowledge,
753            json!({"id": "f1", "body": "x"}),
754        )
755        .unwrap();
756        let mut relay = mem_relay();
757        assert!(
758            sess.publish_checkpoint(&mut relay).unwrap().is_none(),
759            "E2E must not publish a checkpoint to an untrusted relay"
760        );
761    }
762
763    #[test]
764    fn two_devices_converge_through_the_relay_across_all_tiers() {
765        let mut relay = mem_relay();
766        let (da, db) = (dirs(), dirs());
767        let mut a = open("mac-a", &da);
768        let mut b = open("mac-b", &db);
769
770        // Concurrent writes on every fold tier, including an LWW conflict
771        // and a routing observation multiset.
772        a.append(
773            Scope::Personal,
774            Surface::Knowledge,
775            json!({"id": "f1", "v": 1}),
776        )
777        .unwrap();
778        a.append(
779            Scope::Personal,
780            Surface::Declagent,
781            json!({"id": "milo", "owner": "a"}),
782        )
783        .unwrap();
784        a.append(Scope::Personal, Surface::Routing, json!({"sample": 1.0}))
785            .unwrap();
786        b.append(
787            Scope::Personal,
788            Surface::Knowledge,
789            json!({"id": "f2", "v": 2}),
790        )
791        .unwrap();
792        b.append(Scope::Personal, Surface::Routing, json!({"sample": 1.0}))
793            .unwrap();
794
795        a.pump(&mut relay).unwrap();
796        let rb = b.pump(&mut relay).unwrap();
797        assert_eq!(rb.folded, 3, "b folded a's three ops");
798        // b now writes causally AFTER folding a's registry record.
799        b.append(
800            Scope::Personal,
801            Surface::Declagent,
802            json!({"id": "milo", "owner": "b"}),
803        )
804        .unwrap();
805        b.pump(&mut relay).unwrap();
806        let ra = a.pump(&mut relay).unwrap();
807        assert_eq!(ra.folded, 3);
808
809        assert_eq!(
810            a.state_hash(),
811            b.state_hash(),
812            "divergence invariant: same hash"
813        );
814        let state = a.state();
815        assert_eq!(
816            state.registries[&Surface::Declagent.tag()]["id:milo"].payload["owner"],
817            json!("b"),
818            "LWW resolved by the hybrid clock's causal order"
819        );
820        assert_eq!(
821            state.log_entries(&Surface::Routing.tag()).len(),
822            2,
823            "the observation multiset survived transport"
824        );
825
826        // Idempotence: an extra pump on both sides is a complete no-op.
827        let ra = a.pump(&mut relay).unwrap();
828        let rb = b.pump(&mut relay).unwrap();
829        assert_eq!((ra.pushed, ra.folded), (0, 0));
830        assert_eq!((rb.pushed, rb.folded), (0, 0));
831        assert_eq!(a.state_hash(), b.state_hash());
832    }
833
834    #[test]
835    fn op_is_journal_durable_before_it_is_ever_transmitted() {
836        // Contract 1 (B1 MUST): append journals+flushes; the crash window
837        // between append and pump loses NOTHING and re-mints NO seq.
838        let mut relay = mem_relay();
839        let d = dirs();
840        {
841            let mut a = open("mac-a", &d);
842            a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
843                .unwrap();
844            // "crash" before any pump: session dropped, nothing transmitted.
845        }
846        let mut a = open("mac-a", &d);
847        // The op survived in the journal; resume did not re-mint its seq.
848        let next = a
849            .append(Scope::Personal, Surface::Knowledge, json!({"id": "f2"}))
850            .unwrap();
851        assert_eq!(next.seq, 1);
852        a.pump(&mut relay).unwrap();
853        assert_eq!(relay.pull("x", &Frontier::new()).unwrap().ops.len(), 2);
854        verify_log(a.ops()).unwrap();
855    }
856
857    #[test]
858    fn ack_is_derived_from_journal_held_state_only() {
859        // Contract 2 (B4 MUST), shown by construction: the acked frontier
860        // equals the max HLC of what is ON DISK in the journal — never of
861        // anything merely received.
862        let mut relay = mem_relay();
863        let (da, db) = (dirs(), dirs());
864        let mut a = open("mac-a", &da);
865        a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
866            .unwrap();
867        a.pump(&mut relay).unwrap();
868
869        let mut b = open("mac-b", &db);
870        let report = b.pump(&mut relay).unwrap();
871        let acked = report.acked.unwrap();
872
873        // Reload b's journal from disk: the ack is exactly its max stamp.
874        drop(b);
875        let on_disk = OplogJournal::load(&db.journal).unwrap();
876        assert_eq!(
877            acked,
878            on_disk.iter().map(|op| op.hlc.clone()).max().unwrap()
879        );
880        let roster: std::collections::BTreeMap<String, RosterEntry> = relay
881            .roster()
882            .unwrap()
883            .into_iter()
884            .map(|e| (e.device_id.clone(), e))
885            .collect();
886        assert_eq!(roster["mac-b"].acked.as_ref(), Some(&acked));
887    }
888
889    /// A relay wrapper that fails `ack` while the flag is set — the
890    /// crash/partition at the worst point of the pump (after the durable
891    /// fold, before the ack).
892    struct FlakyAckRelay<'a> {
893        inner: &'a mut dyn Relay,
894        fail_ack: Arc<AtomicBool>,
895    }
896
897    impl Relay for FlakyAckRelay<'_> {
898        fn register(&mut self, d: &str) -> Result<RosterEntry, RelayError> {
899            self.inner.register(d)
900        }
901        fn push(&mut self, d: &str, ops: &[OpRecord]) -> Result<PushOutcome, RelayError> {
902            self.inner.push(d, ops)
903        }
904        fn pull(&mut self, d: &str, since: &Frontier) -> Result<PullResult, RelayError> {
905            self.inner.pull(d, since)
906        }
907        fn ack(&mut self, d: &str, frontier: Hlc) -> Result<AckOutcome, RelayError> {
908            if self.fail_ack.load(Ordering::SeqCst) {
909                return Err(RelayError::Io(std::io::Error::other("network down")));
910            }
911            self.inner.ack(d, frontier)
912        }
913        fn checkpoint_put(&mut self, d: &str, c: &Checkpoint) -> Result<bool, RelayError> {
914            self.inner.checkpoint_put(d, c)
915        }
916        fn checkpoint_get(&mut self) -> Result<Option<Checkpoint>, RelayError> {
917            self.inner.checkpoint_get()
918        }
919        fn roster(&mut self) -> Result<Vec<RosterEntry>, RelayError> {
920            self.inner.roster()
921        }
922        fn stable_frontier(&mut self) -> Result<Option<Hlc>, RelayError> {
923            self.inner.stable_frontier()
924        }
925        fn gc(&mut self) -> Result<GcReport, RelayError> {
926            self.inner.gc()
927        }
928    }
929
930    #[test]
931    fn crash_mid_pump_is_idempotent_at_every_step() {
932        let mut relay = mem_relay();
933        let (da, db) = (dirs(), dirs());
934        let mut a = open("mac-a", &da);
935        a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
936            .unwrap();
937        a.pump(&mut relay).unwrap();
938
939        // --- Crash point 1: pulled but nothing device-side happened yet
940        // (transport delivered bytes; the process died before verify/fold).
941        // Nothing durable changed, the relay ack table is untouched → a
942        // fresh pump redoes everything.
943        let _ = relay.pull("mac-b", &Frontier::new()).unwrap();
944        let roster: std::collections::BTreeMap<String, RosterEntry> = relay
945            .roster()
946            .unwrap()
947            .into_iter()
948            .map(|e| (e.device_id.clone(), e))
949            .collect();
950        assert_eq!(
951            roster["mac-b"].acked, None,
952            "merely-received is never acked"
953        );
954
955        // --- Crash point 2: fold journaled durably, ack lost (network died
956        // between the fsync and the ack).
957        let fail = Arc::new(AtomicBool::new(true));
958        let mut b = open("mac-b", &db);
959        b.append(Scope::Personal, Surface::Knowledge, json!({"id": "f2"}))
960            .unwrap();
961        {
962            let mut flaky = FlakyAckRelay {
963                inner: &mut relay,
964                fail_ack: fail.clone(),
965            };
966            let err = b.pump(&mut flaky).unwrap_err();
967            assert!(matches!(err, SessionError::Relay(RelayError::Io(_))));
968        }
969        // The fold IS durable (journal has a's op)…
970        let on_disk = OplogJournal::load(&db.journal).unwrap();
971        assert_eq!(on_disk.len(), 2);
972        // …but the ack never landed — the SAFE direction: GC can't drop
973        // what b hasn't acked.
974        let roster: std::collections::BTreeMap<String, RosterEntry> = relay
975            .roster()
976            .unwrap()
977            .into_iter()
978            .map(|e| (e.device_id.clone(), e))
979            .collect();
980        assert_eq!(roster["mac-b"].acked, None);
981
982        // --- Crash point 3: session lost entirely (push cursor gone).
983        // Reopen from the journal and re-pump: re-push dedups by op_id,
984        // re-pull folds nothing new, the ack finally lands.
985        drop(b);
986        fail.store(false, Ordering::SeqCst);
987        let mut b = open("mac-b", &db);
988        let report = b.pump(&mut relay).unwrap();
989        assert_eq!(
990            report.folded, 0,
991            "re-pull re-fold is a no-op by op_id dedup"
992        );
993        assert_eq!(report.pushed, 0, "re-push deduped relay-side");
994        assert_eq!(report.push_deduped, 1);
995        assert!(report.acked.is_some());
996
997        a.pump(&mut relay).unwrap();
998        assert_eq!(
999            a.state_hash(),
1000            b.state_hash(),
1001            "convergence after every crash point"
1002        );
1003    }
1004
1005    #[test]
1006    fn cold_bootstrap_goes_through_resume_anchored_and_stays_fenced() {
1007        // Contract 5: a cold device bootstraps checkpoint-first; its
1008        // journal carries the truncation marker, so the naive
1009        // load()/DeviceLog::resume path stays a runtime error.
1010        let mut relay = mem_relay();
1011        let da = dirs();
1012        let mut a = open("mac-a", &da);
1013        for i in 0..4 {
1014            a.append(
1015                Scope::Personal,
1016                Surface::Knowledge,
1017                json!({"id": format!("f{i}"), "timestamp": i}),
1018            )
1019            .unwrap();
1020        }
1021        a.pump(&mut relay).unwrap();
1022        let ckpt = a.publish_checkpoint(&mut relay).unwrap().unwrap();
1023        relay.gc().unwrap();
1024
1025        // Fresh device: bootstrap = checkpoint_get + pull(since ckpt
1026        // frontier) + resume_anchored.
1027        let db = dirs();
1028        let mut b =
1029            SyncSession::bootstrap("mac-b", &db.journal, &db.ckpts, &mut relay, zero_wall())
1030                .unwrap();
1031        assert_eq!(b.base().unwrap().checkpoint_hash, ckpt.checkpoint_hash);
1032        b.pump(&mut relay).unwrap();
1033        assert_eq!(b.state_hash(), a.state_hash());
1034
1035        // The fences hold on the bootstrapped journal.
1036        let err = OplogJournal::load(&db.journal).unwrap_err();
1037        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
1038        let (marker, tail) = OplogJournal::load_with_marker(&db.journal).unwrap();
1039        assert_eq!(marker.unwrap().checkpoint_hash, ckpt.checkpoint_hash);
1040        // b keeps working across a restart (open() resumes anchored)…
1041        drop(b);
1042        let mut b = open("mac-b", &db);
1043        let op = b
1044            .append(Scope::Personal, Surface::Knowledge, json!({"id": "from-b"}))
1045            .unwrap();
1046        assert!(
1047            op.hlc
1048                > tail.iter().map(|o| o.hlc.clone()).max().unwrap_or(Hlc {
1049                    wall_ms: 0,
1050                    counter: 0,
1051                    device_id: String::new()
1052                })
1053        );
1054        b.pump(&mut relay).unwrap();
1055        a.pump(&mut relay).unwrap();
1056        assert_eq!(a.state_hash(), b.state_hash());
1057    }
1058
1059    #[test]
1060    fn partial_fold_failure_is_retry_safe_and_never_bricks_the_journal() {
1061        // Kernel-review BRICKED-DEVICE repro (mechanism, not just the
1062        // duplicated-line consequence): a mid-fold-loop journal append
1063        // failure must leave self.ops == the journal, so the retry filters
1064        // the already-journaled ops out of the re-pull instead of writing
1065        // them a SECOND time (a duplicate line = DuplicateSeq on the next
1066        // open = a permanently bricked device with no recovery API).
1067        let mut relay = mem_relay();
1068        let (da, db) = (dirs(), dirs());
1069        let mut a = open("mac-a", &da);
1070        a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
1071            .unwrap();
1072        a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f2"}))
1073            .unwrap();
1074        a.pump(&mut relay).unwrap();
1075
1076        let mut b = open("mac-b", &db);
1077        // Force the SECOND fold append to fail (ENOSPC-class).
1078        b.journal.fail_append_after = Some(1);
1079        let err = b.pump(&mut relay).unwrap_err();
1080        assert!(matches!(err, SessionError::Io(_)), "got {err:?}");
1081
1082        // Lockstep invariant: self.ops holds exactly what the journal holds
1083        // (the one op that appended before the failure), never more.
1084        assert_eq!(
1085            b.ops().len(),
1086            1,
1087            "only the successfully-journaled fold is in self.ops"
1088        );
1089        let on_disk = OplogJournal::load(&db.journal).unwrap();
1090        assert_eq!(on_disk.len(), 1);
1091        assert_eq!(on_disk[0].op_id, b.ops()[0].op_id);
1092
1093        // Retry: the seam auto-cleared, so this pump folds the remaining op.
1094        // Crucially it does NOT re-journal the first fold (filtered by op_id
1095        // from self.ops) — no duplicate line.
1096        let report = b.pump(&mut relay).unwrap();
1097        assert_eq!(
1098            report.folded, 1,
1099            "only the un-journaled op is folded on retry"
1100        );
1101        let on_disk = OplogJournal::load(&db.journal).unwrap();
1102        assert_eq!(
1103            on_disk.len(),
1104            2,
1105            "no duplicate line — the journal is not bricked"
1106        );
1107        verify_log(&on_disk).expect("no DuplicateSeq: the journal opens cleanly");
1108
1109        // The device is not bricked: it reopens and converges.
1110        drop(b);
1111        let b = open("mac-b", &db);
1112        assert_eq!(a.state_hash(), b.state_hash());
1113    }
1114
1115    #[test]
1116    fn append_failure_rolls_the_chain_back() {
1117        // Force a journal append failure by dropping the journal file's
1118        // directory out from under it is not portable; instead exercise the
1119        // rebuild path directly: rebuild_device must reproduce the exact
1120        // chain position after arbitrary appends.
1121        let d = dirs();
1122        let mut a = open("mac-a", &d);
1123        a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f1"}))
1124            .unwrap();
1125        a.append(Scope::Personal, Surface::Knowledge, json!({"id": "f2"}))
1126            .unwrap();
1127        let before = a.ops().to_vec();
1128        a.rebuild_device().unwrap();
1129        let next = a
1130            .append(Scope::Personal, Surface::Knowledge, json!({"id": "f3"}))
1131            .unwrap();
1132        assert_eq!(next.seq, 2);
1133        assert_eq!(next.prev.as_deref(), Some(before[1].op_id.as_str()));
1134        let mut all = before;
1135        all.push(next);
1136        verify_log(&all).unwrap();
1137    }
1138
1139    // ------------------------------------------------------------------
1140    // B5: execution lease + fencing, end-to-end through the relay.
1141    // ------------------------------------------------------------------
1142
1143    #[test]
1144    fn partition_both_write_intents_converge_to_the_higher_epoch_winner() {
1145        // The full arc: mac-a holds the lease (epoch 1) and fires a scheduled
1146        // run; it pauses past its TTL; mac-b STEALS the lease (epoch 2) and
1147        // fires the SAME run (same B7 deterministic run_id); mac-a, partitioned
1148        // from the coordinator, wrongly still believes it holds epoch 1 and
1149        // ALSO commits the run (a real zombie). After convergence the fold
1150        // fences the zombie deterministically — one ledger record, the
1151        // higher-epoch (mac-b) commit wins, even though the zombie's op has a
1152        // LATER HLC. No double-execution; both devices agree.
1153        use crate::lease::{InMemoryLeaseCoordinator, IntentStatus};
1154        use std::sync::atomic::AtomicU64;
1155
1156        let coord_t = Arc::new(AtomicU64::new(0));
1157        let reader = coord_t.clone();
1158        let coord_wall: WallClock = Arc::new(move || reader.load(Ordering::SeqCst));
1159        let mut coord = InMemoryLeaseCoordinator::new(coord_wall);
1160
1161        let mut relay = mem_relay();
1162        let (da, db) = (dirs(), dirs());
1163        let mut a = open("mac-a", &da);
1164        let mut b = open("mac-b", &db);
1165        let run = car_proto::deterministic_run_id("milo", "3am digest", "occurrence-1");
1166
1167        // 1. mac-a acquires (epoch 1) and records the run's pending intent
1168        //    through the GATED path (coordinator confirms it still holds).
1169        let lease_a = coord.acquire("milo", "mac-a", 100).unwrap();
1170        assert_eq!(lease_a.epoch, 1);
1171        let recorded = a
1172            .record_intent_if_current(
1173                Scope::Personal,
1174                &Intent::new("milo", &run, 1, IntentStatus::Pending),
1175                &mut coord,
1176            )
1177            .unwrap();
1178        assert!(
1179            recorded.is_some(),
1180            "mac-a holds the lease → intent recorded"
1181        );
1182        a.pump(&mut relay).unwrap();
1183
1184        // 2. mac-b folds a's state, then — a's lid closed past the TTL —
1185        //    STEALS the lease (epoch 2) and commits the same run.
1186        b.pump(&mut relay).unwrap();
1187        coord_t.store(200, Ordering::SeqCst); // past mac-a's 100ms TTL
1188        let lease_b = coord.acquire("milo", "mac-b", 100).unwrap();
1189        assert_eq!((lease_b.epoch, lease_b.holder.as_str()), (2, "mac-b"));
1190        b.record_intent_if_current(
1191            Scope::Personal,
1192            &Intent::new("milo", &run, 2, IntentStatus::Pending),
1193            &mut coord,
1194        )
1195        .unwrap()
1196        .expect("mac-b holds epoch 2");
1197        let b_commit = b
1198            .record_intent_if_current(
1199                Scope::Personal,
1200                &Intent::new("milo", &run, 2, IntentStatus::Committed),
1201                &mut coord,
1202            )
1203            .unwrap()
1204            .expect("mac-b holds epoch 2");
1205        b.pump(&mut relay).unwrap();
1206
1207        // 3. mac-a is PARTITIONED from the coordinator (never learns it lost)
1208        //    but still syncs with the relay. It folds b's ops, then — a genuine
1209        //    double-execution artifact — also commits the run via the UNGATED
1210        //    path, stamping a LATER HLC than b's committed op. (A committed
1211        //    write is allowed even for an already-committed run — idempotent;
1212        //    the fold converges it. The terminal guard only no-ops a *pending*
1213        //    write for an already-committed run — see the C3 session test.)
1214        a.pump(&mut relay).unwrap();
1215        let a_zombie = a
1216            .record_intent(
1217                Scope::Personal,
1218                &Intent::new("milo", &run, 1, IntentStatus::Committed),
1219            )
1220            .unwrap()
1221            .expect("a committed write is recorded (not terminal-guarded)");
1222        assert!(
1223            a_zombie.hlc > b_commit.hlc,
1224            "the zombie's write is later in HLC"
1225        );
1226        a.pump(&mut relay).unwrap();
1227        b.pump(&mut relay).unwrap();
1228
1229        // 4. Converge: both agree; the epoch-2 (mac-b) commit is the ledger
1230        //    winner despite the zombie's later HLC (fencing beats the clock),
1231        //    and the fence-independent oracle agrees.
1232        assert_eq!(a.state_hash(), b.state_hash(), "divergence invariant holds");
1233        let state = a.state();
1234        assert_eq!(state.intents["milo"].runs.len(), 1, "one who-holds record");
1235        assert_eq!(state.fencing_epoch("milo"), Some(2));
1236        let winner = state.intent("milo", &run).expect("run present");
1237        assert_eq!(
1238            winner.op_id, b_commit.op_id,
1239            "higher-epoch commit wins despite lower HLC"
1240        );
1241        let decoded = Intent::from_payload(&winner.payload).unwrap();
1242        assert_eq!(
1243            (decoded.epoch, decoded.status),
1244            (2, IntentStatus::Committed)
1245        );
1246        // The durable idempotency oracle also resolves to the epoch-2 commit,
1247        // on both devices — the read a B6 dispatch fence would perform.
1248        assert_eq!(a.committed_run("milo", &run).unwrap().op_id, b_commit.op_id);
1249        assert_eq!(b.committed_run("milo", &run).unwrap().op_id, b_commit.op_id);
1250    }
1251
1252    #[test]
1253    fn record_intent_if_current_gates_out_a_lost_holder() {
1254        // The best-effort local gate when the coordinator IS reachable: once
1255        // mac-a no longer holds the lease, the gated write records nothing
1256        // (saving an op the fold would only fence). This is a liveness
1257        // optimization — the sound gate remains the fold's epoch fence.
1258        use crate::lease::{InMemoryLeaseCoordinator, IntentStatus};
1259        let mut coord = InMemoryLeaseCoordinator::new(zero_wall());
1260        let d = dirs();
1261        let mut a = open("mac-a", &d);
1262        let run = "run-x";
1263
1264        coord.acquire("milo", "mac-a", 100).unwrap();
1265        let recorded = a
1266            .record_intent_if_current(
1267                Scope::Personal,
1268                &Intent::new("milo", run, 1, IntentStatus::Pending),
1269                &mut coord,
1270            )
1271            .unwrap();
1272        assert!(recorded.is_some());
1273        assert_eq!(a.ops().len(), 1);
1274
1275        // mac-a releases; mac-b acquires epoch 2. The coordinator now reports
1276        // mac-b as the holder → the gate refuses mac-a's write.
1277        coord.release("milo", "mac-a", 1).unwrap();
1278        coord.acquire("milo", "mac-b", 100).unwrap();
1279        let gated = a
1280            .record_intent_if_current(
1281                Scope::Personal,
1282                &Intent::new("milo", run, 1, IntentStatus::Committed),
1283                &mut coord,
1284            )
1285            .unwrap();
1286        assert!(
1287            gated.is_none(),
1288            "coordinator says mac-a lost → not recorded"
1289        );
1290        assert_eq!(a.ops().len(), 1, "nothing new journaled");
1291    }
1292
1293    #[test]
1294    fn c3_record_intent_no_ops_a_pending_for_an_already_committed_run() {
1295        // C3 write-side guard: once a run has committed (per the
1296        // fence-independent oracle), record_intent NO-OPs a later
1297        // pending/failed write for it — a failed-over holder that writes
1298        // `pending` before checking cannot revert the committed ledger. This is
1299        // the ungated path (no coordinator); the guard is a local oracle read.
1300        let mut relay = mem_relay();
1301        let (da, db) = (dirs(), dirs());
1302        let mut a = open("mac-a", &da);
1303        let mut b = open("mac-b", &db);
1304        let run = "run-nightly";
1305
1306        // mac-a commits the run at epoch 1 and syncs it to mac-b.
1307        a.record_intent(
1308            Scope::Personal,
1309            &Intent::new("milo", run, 1, IntentStatus::Committed),
1310        )
1311        .unwrap()
1312        .expect("first commit recorded");
1313        a.pump(&mut relay).unwrap();
1314        b.pump(&mut relay).unwrap();
1315        assert!(
1316            b.committed_run("milo", run).is_some(),
1317            "mac-b folded the commit"
1318        );
1319
1320        // mac-b fails over to epoch 2 and — before checking — tries to write the
1321        // run PENDING. The terminal guard no-ops it (the run already committed).
1322        let before = b.ops().len();
1323        let attempt = b
1324            .record_intent(
1325                Scope::Personal,
1326                &Intent::new("milo", run, 2, IntentStatus::Pending),
1327            )
1328            .unwrap();
1329        assert!(
1330            attempt.is_none(),
1331            "pending for an already-committed run is a no-op"
1332        );
1333        assert_eq!(b.ops().len(), before, "nothing journaled");
1334
1335        // The ledger stays committed on both sides after further sync.
1336        b.pump(&mut relay).unwrap();
1337        a.pump(&mut relay).unwrap();
1338        assert!(a.committed_run("milo", run).is_some());
1339        let decoded =
1340            Intent::from_payload(&b.state().intent("milo", run).unwrap().payload).unwrap();
1341        assert_eq!(
1342            decoded.status,
1343            IntentStatus::Committed,
1344            "not reverted to pending"
1345        );
1346    }
1347}