liminal_sdk/remote/participant/recovery.rs
1use liminal_protocol::client::{
2 DetachReplayRefusalReason, DetachTransportAttemptDecision, DetachTransportFate,
3 DetachTransportFateDecision, ExplicitReconnectAction, LostAuthorityKind,
4 LostOperationAuthorityDecision, LostReconnectAuthorityDecision, ProvedOnlineTransition,
5 ReconnectAttemptDecision, ReconnectAttemptFate, ReconnectAttemptFateDecision,
6 ReconnectAttemptFateRefusalReason, ReconnectAttemptRefusalReason, ReconnectPermitDecision,
7 RecoveredExpectedOperationDecision, RecoveredReconnectPermitDecision, record_attempt_fate,
8 record_explicit_reconnect, record_online_transition, recover_expected_operation,
9 recover_reconnect_permit, redeem_attempt, resolve_lost_operation_authority,
10 resolve_lost_reconnect_authority, transport_attempt_started, transport_fate,
11};
12use liminal_protocol::outcome::ReconnectState;
13use liminal_protocol::wire::{ClientRequest, DetachRequest, Generation, ServerValue};
14
15use super::{
16 OperationDurability, ParticipantResumeStore, RemoteOperationRecordOutcome,
17 RemoteOperationTransportFate, RemoteParticipantError, RemoteParticipantHandle,
18 RemoteParticipantInbound, RemoteParticipantOperation, RemoteParticipantSendOutcome,
19 RemoteReconnectPermit, RemoteReconnectPermitOutcome, persist_retaining, record_connection_fate,
20 record_operation_transport_fate, take_aggregate,
21};
22
23/// Result of releasing a committed cold-restored operation.
24#[derive(Debug)]
25pub enum RemoteExpectedOperationRecovery {
26 /// One unissued operation authority was recovered.
27 Recovered(RemoteParticipantOperation),
28 /// No recoverable operation exists.
29 NotAvailable {
30 /// Whether the retained operation had already been issued.
31 already_issued: bool,
32 },
33}
34
35/// Result of consuming operation-domain crash testimony.
36#[derive(Debug, PartialEq, Eq)]
37pub enum RemoteLostOperationResolution {
38 /// A non-detach operation was terminalized by serialized testimony.
39 Recorded {
40 /// Exact request whose authority was destroyed.
41 request: ClientRequest,
42 /// Closed testimony kind consumed by the crate.
43 testimony: LostAuthorityKind,
44 },
45 /// A detach was returned to parked replay by serialized testimony.
46 DetachParked {
47 /// Exact detach request retained for replay.
48 request: ClientRequest,
49 /// Closed testimony kind consumed by the crate.
50 testimony: LostAuthorityKind,
51 },
52 /// No operation-domain testimony was pending.
53 Refused {
54 /// Closed crate refusal reason.
55 reason: liminal_protocol::client::LostAuthorityResolutionRefusalReason,
56 },
57}
58
59/// Result of consuming reconnect-domain crash testimony.
60#[derive(Debug, PartialEq, Eq)]
61pub enum RemoteLostReconnectResolution {
62 /// Testimony parked reconnect state without minting replacement authority.
63 Recorded {
64 /// Closed testimony kind consumed by the crate.
65 testimony: LostAuthorityKind,
66 },
67 /// No reconnect-domain testimony was pending.
68 Refused {
69 /// Closed crate refusal reason.
70 reason: liminal_protocol::client::LostAuthorityResolutionRefusalReason,
71 },
72}
73
74/// Result of releasing a committed cold-restored reconnect permit.
75#[derive(Debug)]
76pub enum RemoteReconnectPermitRecovery {
77 /// One unissued permit was recovered.
78 Recovered(RemoteReconnectPermit),
79 /// No recoverable permit exists.
80 NotAvailable {
81 /// Current crate reconnect state.
82 state: ReconnectState,
83 },
84}
85
86/// Result of a real connection attempt redeemed from one crate permit.
87#[derive(Debug)]
88pub enum RemoteReconnectAttemptOutcome {
89 /// The real attempt connected and the crate recorded online state.
90 Connected {
91 /// Provenance assigned to the new established socket.
92 provenance: super::ParticipantResponseProvenance,
93 },
94 /// The real attempt failed and the crate parked without timer authority.
95 Failed {
96 /// Concrete socket failure.
97 error: crate::SdkError,
98 },
99 /// The crate refused permit redemption and returned it unchanged.
100 Refused {
101 /// Reusable unchanged permit.
102 permit: RemoteReconnectPermit,
103 /// Closed crate refusal reason.
104 reason: ReconnectAttemptRefusalReason,
105 },
106 /// The transport ran, but the crate retained the in-progress fate authority.
107 FateRefused {
108 /// Closed crate refusal reason.
109 reason: ReconnectAttemptFateRefusalReason,
110 /// Socket failure when the attempted fate was `Failed`.
111 error: Option<crate::SdkError>,
112 /// New socket provenance when the attempted fate was `Connected`.
113 provenance: Option<super::ParticipantResponseProvenance>,
114 },
115}
116
117/// Result of starting and sending a parked detach replay.
118#[derive(Debug)]
119pub enum RemoteDetachReplayOutcome {
120 /// A real transport send was attempted.
121 Send(RemoteParticipantSendOutcome),
122 /// The crate refused replay start without changing state.
123 Refused {
124 /// Closed crate replay refusal reason.
125 reason: DetachReplayRefusalReason,
126 },
127}
128
129/// Result of an explicit replay apply seam delegated to the crate.
130#[derive(Debug, PartialEq, Eq)]
131pub enum RemoteReplayApplyOutcome<T> {
132 /// The crate applied the transition.
133 Applied,
134 /// The crate retained state, correlation, and exact input.
135 Refused {
136 /// Exact refused input.
137 input: T,
138 /// Closed crate refusal reason.
139 reason: DetachReplayRefusalReason,
140 },
141}
142
143/// Why a lost credential attach could not be driven from retained testimony.
144#[derive(Clone, Copy, Debug, PartialEq, Eq)]
145pub enum LostCredentialAttachRefusalReason {
146 /// No operation-domain lost-authority testimony is pending at all.
147 NoPendingTestimony,
148 /// Testimony is pending, but the operation it testifies is not an issued
149 /// credential attach.
150 ///
151 /// The driver leaves it strictly alone rather than resolving it: a detach
152 /// keeps its own replay machinery and a tokenless operation keeps its typed
153 /// abandonment, and both of those paths need the take-once testimony this
154 /// driver would otherwise have spent to discover it did not own the case.
155 NotAnIssuedCredentialAttach,
156}
157
158/// Why a driven credential attach ended in an honest re-issue terminal.
159///
160/// This is an SDK-side classification of two wire answers, not a new wire
161/// value: both arms are read off responses the server already sends.
162#[derive(Clone, Copy, Debug, PartialEq, Eq)]
163pub enum CredentialAttachReissueReason {
164 /// The receipt window closed while provenance still explains the commit, so
165 /// the server can still state WHICH generation the lost commit produced.
166 ReceiptExpired(liminal_protocol::wire::ReceiptExpiryReason),
167 /// Provenance expired too. The server deliberately claims no commit proof,
168 /// so exact-old and unknown tokens are indistinguishable from here.
169 StaleOrUnknownReceipt,
170}
171
172/// Outcome of driving one lost issued credential attach to a server answer.
173///
174/// The three healing-or-terminal arms are the exhaustive server answers to a
175/// same-token re-presentation, and the remainder are pass-throughs that hand
176/// back exactly what the crate or the transport reported.
177#[derive(Debug)]
178pub enum RemoteCredentialAttachRecovery {
179 /// The server replayed its committed receipt and the crate applied it: the
180 /// ROTATED credential is now held, and the orphan is over.
181 ///
182 /// This is the designed healing window being spent. The value is the exact
183 /// replay the server sent — `Bound` when the receipt still names its origin
184 /// binding, `UnboundReceipt` when the tear killed the connection that held
185 /// it. Both carry the successor generation and the newly minted secret; the
186 /// difference is only whether the crate lands in `Bound` or `Detached`.
187 HealedFromReceipt {
188 /// Exact applied replay value.
189 value: ServerValue,
190 /// Connection/attempt that delivered it.
191 provenance: super::ParticipantResponseProvenance,
192 },
193 /// The attach had never committed, so the re-presentation committed it now.
194 ///
195 /// The kill landed in the window between the client's send and the server's
196 /// commit. Nothing was lost and nothing needed replaying.
197 CommittedFresh {
198 /// Exact applied `AttachBound` value.
199 value: ServerValue,
200 /// Connection/attempt that delivered it.
201 provenance: super::ParticipantResponseProvenance,
202 },
203 /// The committed outcome is permanently unanswerable; operator re-issue is
204 /// the cure.
205 ///
206 /// THE LOAD-BEARING TERMINAL. It is reached when the client was dead longer
207 /// than the receipt window the server could hold open, which is policy
208 /// (config-owned since #39) rather than failure. It is deliberately a state
209 /// of its own rather than a generic refusal, because it is the exact point
210 /// at which an embedder should dispose and re-enroll instead of retrying —
211 /// and no amount of retrying will ever change it.
212 ReissueRequired {
213 /// The generation the lost commit produced, when the server can still
214 /// prove it. `None` for `StaleOrUnknownReceipt`, which makes no commit
215 /// claim at all — the absence is the server's honesty, not a gap here.
216 result_generation: Option<Generation>,
217 /// The generation the identity is live at now.
218 current_generation: Generation,
219 /// Which of the two unanswerable classes this is.
220 reason: CredentialAttachReissueReason,
221 /// Exact applied server value.
222 value: ServerValue,
223 /// Connection/attempt that delivered it.
224 provenance: super::ParticipantResponseProvenance,
225 },
226 /// The crate applied some other correlated answer, carried verbatim.
227 ///
228 /// `StaleAuthority`, `ParticipantUnknown`, `Retired` and their kin arrive
229 /// here. The driver relabels nothing: an answer it does not classify is
230 /// handed over as the server sent it.
231 Answered {
232 /// Exact applied server value.
233 value: ServerValue,
234 /// Connection/attempt that delivered it.
235 provenance: super::ParticipantResponseProvenance,
236 },
237 /// The crate refused the answer and retained its correlation unchanged.
238 AnswerRefused {
239 /// Exact refused server value.
240 value: ServerValue,
241 /// Closed crate refusal reason.
242 reason: liminal_protocol::client::ClientInboundRefusalReason,
243 /// Connection/attempt that delivered it.
244 provenance: super::ParticipantResponseProvenance,
245 },
246 /// A push arrived where the correlated answer was owed.
247 ///
248 /// The delivery is handed back rather than dropped, and the live response
249 /// correlation is still held, so a caller may simply keep receiving: the
250 /// crate applies the answer whenever it does arrive.
251 PushedBeforeAnswer {
252 /// Exact pushed value.
253 value: liminal_protocol::wire::ServerPush,
254 /// Connection/attempt that delivered it.
255 provenance: super::ParticipantResponseProvenance,
256 },
257 /// No issued credential-attach testimony was pending; nothing was consumed.
258 NotPending {
259 /// Closed refusal reason.
260 reason: LostCredentialAttachRefusalReason,
261 },
262 /// The crate refused to re-record the retained envelope.
263 RerecordRefused {
264 /// Exact refused request.
265 request: ClientRequest,
266 /// Closed crate refusal reason.
267 reason: liminal_protocol::client::ClientOperationRecordRefusalReason,
268 },
269 /// The probe could not be written; both fates were delegated to the crate.
270 TransportLost {
271 /// Concrete socket failure.
272 error: crate::SdkError,
273 /// Crate-owned operation-fate result.
274 operation_fate: RemoteOperationTransportFate,
275 /// Crate-owned reconnect permit result.
276 reconnect: RemoteReconnectPermitOutcome,
277 },
278}
279
280/// Combined typed consequence of an established connection loss.
281#[derive(Debug)]
282pub struct RemoteTransportLossOutcome {
283 /// Operation-domain fate selected by crate rules.
284 pub operation_fate: RemoteOperationTransportFate,
285 /// Event-driven reconnect decision selected by crate rules.
286 pub reconnect: RemoteReconnectPermitOutcome,
287}
288
289impl<S: ParticipantResumeStore> RemoteParticipantHandle<S> {
290 /// Releases one unissued operation from committed cold-restored state.
291 ///
292 /// # Errors
293 ///
294 /// Returns [`RemoteParticipantError::StateUnavailable`] after a prior fatal
295 /// durability failure.
296 pub fn recover_expected_operation(
297 &self,
298 ) -> Result<RemoteExpectedOperationRecovery, RemoteParticipantError> {
299 let mut state = self.state.lock();
300 let aggregate = take_aggregate(&mut state)?;
301 match recover_expected_operation(aggregate) {
302 RecoveredExpectedOperationDecision::Recovered {
303 aggregate,
304 operation,
305 } => {
306 state.aggregate = Some(aggregate);
307 Ok(RemoteExpectedOperationRecovery::Recovered(
308 RemoteParticipantOperation {
309 operation,
310 durability: OperationDurability::WriteAhead,
311 },
312 ))
313 }
314 RecoveredExpectedOperationDecision::NotAvailable {
315 aggregate,
316 already_issued,
317 } => {
318 state.aggregate = Some(aggregate);
319 Ok(RemoteExpectedOperationRecovery::NotAvailable { already_issued })
320 }
321 }
322 }
323
324 /// Consumes operation-domain lost-authority testimony exactly once.
325 ///
326 /// # Errors
327 ///
328 /// Returns LPCR encode or storage failures while checkpointing the decision.
329 pub fn resolve_lost_operation_authority(
330 &self,
331 ) -> Result<RemoteLostOperationResolution, RemoteParticipantError> {
332 let mut state = self.state.lock();
333 let aggregate = take_aggregate(&mut state)?;
334 let outcome = match resolve_lost_operation_authority(aggregate) {
335 LostOperationAuthorityDecision::Recorded {
336 aggregate,
337 request,
338 testimony,
339 } => {
340 state.aggregate = Some(aggregate);
341 RemoteLostOperationResolution::Recorded {
342 request,
343 testimony: testimony.kind(),
344 }
345 }
346 LostOperationAuthorityDecision::DetachParked {
347 aggregate,
348 request,
349 testimony,
350 } => {
351 state.aggregate = Some(aggregate);
352 RemoteLostOperationResolution::DetachParked {
353 request,
354 testimony: testimony.kind(),
355 }
356 }
357 LostOperationAuthorityDecision::Refused { aggregate, reason } => {
358 state.aggregate = Some(aggregate);
359 RemoteLostOperationResolution::Refused { reason }
360 }
361 };
362 checkpoint_state(&mut state)?;
363 Ok(outcome)
364 }
365
366 /// Drives one lost issued credential attach back to a server answer.
367 ///
368 /// # The act this performs, and why it is lawful
369 ///
370 /// A client killed between issuing a `CredentialAttach` and consuming its
371 /// answer has lost the ONLY carrier of the rotated credential, because the
372 /// commit mints a fresh secret and advances the generation and the
373 /// `AttachBound` response is the sole place either value appears. The
374 /// server, however, holds the committed outcome inside a receipt window and
375 /// will replay it — including the rotation — to a re-presentation of the
376 /// SAME attempt token, verified against the receipt's own committed
377 /// presented secret, which is the invalidated OLD one. That is deliberate.
378 ///
379 /// So the healing act is to re-present the EXACT retained envelope: same
380 /// attach attempt token, same generation, same old secret. Nothing is
381 /// forged and nothing new is minted — this method re-records the envelope
382 /// the restore handed back, unchanged, and the crate admits it because the
383 /// retained binding still matches it. Token dedup makes it at-most-once, so
384 /// a re-presentation of a never-committed attach commits exactly once.
385 ///
386 /// # FIRST-ACT, by construction
387 ///
388 /// The probe is sent on THIS call, with no backoff, no timer, and no retry
389 /// loop in front of it. That is a hard requirement rather than a
390 /// performance preference: the receipt window is the healing window, it is
391 /// fixed at commit and never re-opens, and any delay this driver introduced
392 /// would be spent out of the window it exists to spend. A retry discipline
393 /// must never EXTEND the orphan (§0.16 A5 condition 2).
394 ///
395 /// # What it will not touch
396 ///
397 /// Testimony belonging to any other operation class is left entirely alone,
398 /// including its take-once atom — see
399 /// [`LostCredentialAttachRefusalReason::NotAnIssuedCredentialAttach`].
400 ///
401 /// # Errors
402 ///
403 /// Returns typed LPCR encode, storage, or state failures. Every socket and
404 /// server outcome is a typed arm of [`RemoteCredentialAttachRecovery`]
405 /// rather than an error.
406 pub fn recover_lost_credential_attach(
407 &self,
408 ) -> Result<RemoteCredentialAttachRecovery, RemoteParticipantError> {
409 // 1. Look before consuming. The take-once atom must survive a driver
410 // that turns out not to own this case.
411 match self.lost_credential_attach_pending()? {
412 PendingTestimonyVerdict::IssuedCredentialAttach => {}
413 PendingTestimonyVerdict::Nothing => {
414 return Ok(RemoteCredentialAttachRecovery::NotPending {
415 reason: LostCredentialAttachRefusalReason::NoPendingTestimony,
416 });
417 }
418 PendingTestimonyVerdict::OtherOperation => {
419 return Ok(RemoteCredentialAttachRecovery::NotPending {
420 reason: LostCredentialAttachRefusalReason::NotAnIssuedCredentialAttach,
421 });
422 }
423 }
424
425 // 2. Consume the testimony. The peek above proved this is an issued
426 // credential attach, so `Recorded` is the only reachable arm; the
427 // others fall through to a typed refusal rather than a panic.
428 let request = match self.resolve_lost_operation_authority()? {
429 RemoteLostOperationResolution::Recorded { request, .. } => request,
430 RemoteLostOperationResolution::DetachParked { .. }
431 | RemoteLostOperationResolution::Refused { .. } => {
432 return Ok(RemoteCredentialAttachRecovery::NotPending {
433 reason: LostCredentialAttachRefusalReason::NotAnIssuedCredentialAttach,
434 });
435 }
436 };
437
438 // 3. Re-record the EXACT retained envelope. Nothing is minted and
439 // nothing is advanced: same attempt token, same generation, same old
440 // secret. The crate admits it because the retained binding still
441 // matches it, and server-side token dedup keeps it at-most-once.
442 let operation = match self.record_operation(request)? {
443 RemoteOperationRecordOutcome::Recorded(operation)
444 | RemoteOperationRecordOutcome::Continuous(operation) => operation,
445 RemoteOperationRecordOutcome::Refused { request, reason } => {
446 return Ok(RemoteCredentialAttachRecovery::RerecordRefused { request, reason });
447 }
448 };
449
450 // 4. THE PROBE, on this call. No backoff, no timer, no retry loop: the
451 // receipt window is what this is spending, and a delay here would be
452 // spent out of it.
453 match self.send_operation(operation)? {
454 RemoteParticipantSendOutcome::Sent { .. } => {}
455 RemoteParticipantSendOutcome::TransportLost {
456 error,
457 operation_fate,
458 reconnect,
459 } => {
460 return Ok(RemoteCredentialAttachRecovery::TransportLost {
461 error,
462 operation_fate,
463 reconnect,
464 });
465 }
466 }
467
468 // 5. Apply the answer through the crate's ordinary inbound path, then
469 // classify what it applied. The application is the crate's; the
470 // classification below reads the applied value and invents nothing.
471 Ok(classify_recovery_answer(self.receive()?))
472 }
473
474 /// Borrows a copy of the retained credential-attach envelope this handle
475 /// would re-present, without consuming anything.
476 ///
477 /// `Some` means [`Self::recover_lost_credential_attach`] has work to do and
478 /// names exactly the envelope it will send. It is the honest way for an
479 /// embedder to ask "am I an orphan, and what is owed" before deciding to
480 /// drive, and for a test to prove the driver re-presents the retained bytes
481 /// rather than something it minted.
482 ///
483 /// # Errors
484 ///
485 /// Returns [`RemoteParticipantError::StateUnavailable`] after a prior fatal
486 /// durability failure.
487 pub fn peek_lost_credential_attach(
488 &self,
489 ) -> Result<Option<liminal_protocol::wire::CredentialAttachRequest>, RemoteParticipantError>
490 {
491 let mut state = self.state.lock();
492 let aggregate = take_aggregate(&mut state)?;
493 let retained = aggregate.lost_credential_attach().cloned();
494 state.aggregate = Some(aggregate);
495 Ok(retained)
496 }
497
498 /// Classifies the pending operation-domain testimony WITHOUT consuming it.
499 fn lost_credential_attach_pending(
500 &self,
501 ) -> Result<PendingTestimonyVerdict, RemoteParticipantError> {
502 let mut state = self.state.lock();
503 let aggregate = take_aggregate(&mut state)?;
504 let verdict = if aggregate.lost_credential_attach().is_some() {
505 PendingTestimonyVerdict::IssuedCredentialAttach
506 } else if aggregate.lost_operation_testimony().is_some() {
507 PendingTestimonyVerdict::OtherOperation
508 } else {
509 PendingTestimonyVerdict::Nothing
510 };
511 state.aggregate = Some(aggregate);
512 Ok(verdict)
513 }
514
515 /// Takes a durable tokenless abandonment so its exact request can be re-recorded.
516 ///
517 /// # Errors
518 ///
519 /// Returns LPCR encode or storage failures while durably recording the take.
520 pub fn take_restored_operation_abandonment(
521 &self,
522 ) -> Result<
523 Option<liminal_protocol::client::RestoredExpectedOperationAbandonment>,
524 RemoteParticipantError,
525 > {
526 let mut state = self.state.lock();
527 let mut aggregate = take_aggregate(&mut state)?;
528 let abandonment = aggregate.take_restored_operation_abandonment();
529 if abandonment.is_some() {
530 aggregate = persist_retaining(&mut state, aggregate)?;
531 }
532 state.aggregate = Some(aggregate);
533 Ok(abandonment)
534 }
535
536 /// Records established-connection fate and returns at most one reconnect permit.
537 ///
538 /// # Errors
539 ///
540 /// Returns LPCR encode or storage failures while checkpointing the event.
541 pub fn record_transport_fate(
542 &self,
543 ) -> Result<RemoteReconnectPermitOutcome, RemoteParticipantError> {
544 let mut state = self.state.lock();
545 record_connection_fate(&mut state)
546 }
547
548 /// Records a proved online transition as a crate fresh event.
549 ///
550 /// # Errors
551 ///
552 /// Returns LPCR encode or storage failures while checkpointing issued authority.
553 pub fn record_online_transition(
554 &self,
555 ) -> Result<RemoteReconnectPermitOutcome, RemoteParticipantError> {
556 self.record_fresh_reconnect(|aggregate| {
557 record_online_transition(aggregate, ProvedOnlineTransition::ProvedOnline)
558 })
559 }
560
561 /// Records explicit caller action as a crate fresh event, with no timer arm.
562 ///
563 /// # Errors
564 ///
565 /// Returns LPCR encode or storage failures while checkpointing issued authority.
566 pub fn record_explicit_reconnect(
567 &self,
568 ) -> Result<RemoteReconnectPermitOutcome, RemoteParticipantError> {
569 self.record_fresh_reconnect(|aggregate| {
570 record_explicit_reconnect(aggregate, ExplicitReconnectAction::ReconnectNow)
571 })
572 }
573
574 fn record_fresh_reconnect(
575 &self,
576 decide: impl FnOnce(
577 liminal_protocol::client::ClientParticipantAggregate,
578 ) -> ReconnectPermitDecision,
579 ) -> Result<RemoteReconnectPermitOutcome, RemoteParticipantError> {
580 let mut state = self.state.lock();
581 let aggregate = take_aggregate(&mut state)?;
582 let outcome = match decide(aggregate) {
583 ReconnectPermitDecision::Permitted {
584 aggregate,
585 permit,
586 result,
587 } => {
588 state.aggregate = Some(aggregate);
589 RemoteReconnectPermitOutcome::Permitted {
590 permit: RemoteReconnectPermit { permit },
591 result,
592 }
593 }
594 ReconnectPermitDecision::Refused(refusal) => {
595 let reason = refusal.reason();
596 let result = refusal.result();
597 let (aggregate, _) = refusal.into_parts();
598 state.aggregate = Some(aggregate);
599 RemoteReconnectPermitOutcome::Refused { reason, result }
600 }
601 };
602 checkpoint_state(&mut state)?;
603 Ok(outcome)
604 }
605
606 /// Releases one unissued reconnect permit from committed cold-restored state.
607 ///
608 /// # Errors
609 ///
610 /// Returns [`RemoteParticipantError::StateUnavailable`] after a prior fatal failure.
611 pub fn recover_reconnect_permit(
612 &self,
613 ) -> Result<RemoteReconnectPermitRecovery, RemoteParticipantError> {
614 let mut state = self.state.lock();
615 let aggregate = take_aggregate(&mut state)?;
616 match recover_reconnect_permit(aggregate) {
617 RecoveredReconnectPermitDecision::Recovered { aggregate, permit } => {
618 state.aggregate = Some(aggregate);
619 Ok(RemoteReconnectPermitRecovery::Recovered(
620 RemoteReconnectPermit { permit },
621 ))
622 }
623 RecoveredReconnectPermitDecision::NotAvailable {
624 aggregate,
625 state: value,
626 } => {
627 state.aggregate = Some(aggregate);
628 Ok(RemoteReconnectPermitRecovery::NotAvailable { state: value })
629 }
630 }
631 }
632
633 /// Consumes reconnect-domain lost-authority testimony exactly once.
634 ///
635 /// # Errors
636 ///
637 /// Returns LPCR encode or storage failures while checkpointing the resolution.
638 pub fn resolve_lost_reconnect_authority(
639 &self,
640 ) -> Result<RemoteLostReconnectResolution, RemoteParticipantError> {
641 let mut state = self.state.lock();
642 let aggregate = take_aggregate(&mut state)?;
643 let outcome = match resolve_lost_reconnect_authority(aggregate) {
644 LostReconnectAuthorityDecision::Recorded {
645 aggregate,
646 testimony,
647 } => {
648 state.aggregate = Some(aggregate);
649 RemoteLostReconnectResolution::Recorded {
650 testimony: testimony.kind(),
651 }
652 }
653 LostReconnectAuthorityDecision::Refused { aggregate, reason } => {
654 state.aggregate = Some(aggregate);
655 RemoteLostReconnectResolution::Refused { reason }
656 }
657 };
658 checkpoint_state(&mut state)?;
659 Ok(outcome)
660 }
661
662 /// Redeems one permit before opening one real transport connection.
663 ///
664 /// # Errors
665 ///
666 /// Returns LPCR encode or storage failures before or after the real attempt.
667 pub fn reconnect(
668 &self,
669 permit: RemoteReconnectPermit,
670 ) -> Result<RemoteReconnectAttemptOutcome, RemoteParticipantError> {
671 let mut state = self.state.lock();
672 let aggregate = take_aggregate(&mut state)?;
673 let (aggregate, attempt) = match redeem_attempt(aggregate, permit.permit) {
674 ReconnectAttemptDecision::Started { aggregate, attempt } => (aggregate, attempt),
675 ReconnectAttemptDecision::Refused {
676 aggregate,
677 permit,
678 reason,
679 } => {
680 state.aggregate = Some(aggregate);
681 return Ok(RemoteReconnectAttemptOutcome::Refused {
682 permit: RemoteReconnectPermit { permit },
683 reason,
684 });
685 }
686 };
687 let aggregate = persist_retaining(&mut state, aggregate)?;
688 state.aggregate = Some(aggregate);
689 drop(state);
690
691 let transport_result = self.transport.reconnect_participant(&self.server_address);
692 let fate = if transport_result.is_ok() {
693 ReconnectAttemptFate::Connected
694 } else {
695 ReconnectAttemptFate::Failed
696 };
697
698 let mut state = self.state.lock();
699 let aggregate = take_aggregate(&mut state)?;
700 match record_attempt_fate(aggregate, attempt, fate) {
701 ReconnectAttemptFateDecision::Recorded(aggregate) => {
702 let aggregate = persist_retaining(&mut state, aggregate)?;
703 state.aggregate = Some(aggregate);
704 match transport_result {
705 Ok(provenance) => Ok(RemoteReconnectAttemptOutcome::Connected { provenance }),
706 Err(error) => Ok(RemoteReconnectAttemptOutcome::Failed { error }),
707 }
708 }
709 ReconnectAttemptFateDecision::Refused {
710 aggregate,
711 attempt,
712 reason,
713 ..
714 } => {
715 state.aggregate = Some(aggregate);
716 state.reconnect_attempt = Some(attempt);
717 let (provenance, error) = match transport_result {
718 Ok(value) => (Some(value), None),
719 Err(value) => (None, Some(value)),
720 };
721 Ok(RemoteReconnectAttemptOutcome::FateRefused {
722 reason,
723 error,
724 provenance,
725 })
726 }
727 }
728 }
729
730 /// Records response and connection fates after an established transport loss.
731 ///
732 /// # Errors
733 ///
734 /// Returns LPCR encode or storage failures while checkpointing both decisions.
735 pub fn record_established_transport_loss(
736 &self,
737 ) -> Result<RemoteTransportLossOutcome, RemoteParticipantError> {
738 let mut state = self.state.lock();
739 let operation_fate = if let Some(correlation) = state.correlation.take() {
740 let aggregate = take_aggregate(&mut state)?;
741 record_operation_transport_fate(&mut state, aggregate, correlation)
742 } else {
743 RemoteOperationTransportFate::NotOutstanding
744 };
745 let reconnect = record_connection_fate(&mut state)?;
746 Ok(RemoteTransportLossOutcome {
747 operation_fate,
748 reconnect,
749 })
750 }
751
752 /// Starts and sends the exact parked detach replay selected by the crate.
753 ///
754 /// # Errors
755 ///
756 /// Returns LPCR, storage, or state failures. Socket failure is a typed send outcome.
757 pub fn replay_detach(&self) -> Result<RemoteDetachReplayOutcome, RemoteParticipantError> {
758 let mut state = self.state.lock();
759 let aggregate = take_aggregate(&mut state)?;
760 let (aggregate, attempt) = match transport_attempt_started(aggregate) {
761 DetachTransportAttemptDecision::Started { aggregate, attempt } => (aggregate, attempt),
762 DetachTransportAttemptDecision::Refused(refusal) => {
763 let reason = refusal.reason();
764 let (aggregate, ()) = refusal.into_parts();
765 state.aggregate = Some(aggregate);
766 return Ok(RemoteDetachReplayOutcome::Refused { reason });
767 }
768 };
769 let aggregate = persist_retaining(&mut state, aggregate)?;
770 let (request, correlation) = attempt.into_request();
771 let request = ClientRequest::Detach(DetachRequest {
772 conversation_id: request.conversation_id,
773 participant_id: request.participant_id,
774 capability_generation: request.capability_generation,
775 detach_attempt_token: request.detach_attempt_token,
776 });
777 match self
778 .transport
779 .send_participant(&self.server_address, &request)
780 {
781 Ok(provenance) => {
782 state.aggregate = Some(aggregate);
783 state.correlation = Some(correlation);
784 Ok(RemoteDetachReplayOutcome::Send(
785 RemoteParticipantSendOutcome::Sent { provenance },
786 ))
787 }
788 Err(error) => {
789 let operation_fate = match transport_fate(
790 aggregate,
791 correlation,
792 DetachTransportFate::ResponseUnavailable,
793 ) {
794 DetachTransportFateDecision::Parked(applied) => {
795 state.aggregate = Some(applied.into_aggregate());
796 RemoteOperationTransportFate::DetachParked
797 }
798 DetachTransportFateDecision::Refused(refusal) => {
799 let (aggregate, (correlation, _)) = refusal.into_parts();
800 state.aggregate = Some(aggregate);
801 state.correlation = Some(correlation);
802 RemoteOperationTransportFate::Refused {
803 reason: liminal_protocol::client::ExpectedOperationFateRefusalReason::DetachUsesReplayFate,
804 }
805 }
806 };
807 let reconnect = record_connection_fate(&mut state)?;
808 Ok(RemoteDetachReplayOutcome::Send(
809 RemoteParticipantSendOutcome::TransportLost {
810 error,
811 operation_fate,
812 reconnect,
813 },
814 ))
815 }
816 }
817 }
818}
819
820/// What the pending operation-domain testimony is, read without consuming it.
821///
822/// Three cases, kept distinct because the driver must act differently on each:
823/// drive it, decline it while leaving the atom for the path that owns it, or
824/// report that nothing is owed at all.
825#[derive(Clone, Copy, Debug, PartialEq, Eq)]
826enum PendingTestimonyVerdict {
827 /// No operation-domain testimony is pending.
828 Nothing,
829 /// An issued credential attach — this driver's case.
830 IssuedCredentialAttach,
831 /// Testimony for an operation class this driver does not own.
832 OtherOperation,
833}
834
835/// Classifies the answer to one recovery probe.
836///
837/// Every arm reads a value the crate has ALREADY applied (or refused). Nothing
838/// here re-derives a decision the crate made, and nothing is relabelled: the
839/// four classified arms are the four exhaustive server answers to a same-token
840/// re-presentation, and everything else is handed back verbatim.
841fn classify_recovery_answer(inbound: RemoteParticipantInbound) -> RemoteCredentialAttachRecovery {
842 let (value, provenance) = match inbound {
843 RemoteParticipantInbound::Applied { value, provenance } => (value, provenance),
844 RemoteParticipantInbound::Refused {
845 value,
846 reason,
847 provenance,
848 } => {
849 return RemoteCredentialAttachRecovery::AnswerRefused {
850 value,
851 reason,
852 provenance,
853 };
854 }
855 RemoteParticipantInbound::Push { value, provenance } => {
856 return RemoteCredentialAttachRecovery::PushedBeforeAnswer { value, provenance };
857 }
858 };
859 match &value {
860 // The committed outcome, replayed. `Bound` and `UnboundReceipt` differ
861 // only in whether the receipt still names a live origin binding; both
862 // carry the rotation, and the crate has already adopted it.
863 ServerValue::Bound(liminal_protocol::wire::ReceiptReplay::CredentialAttach(_))
864 | ServerValue::UnboundReceipt(liminal_protocol::wire::ReceiptReplay::CredentialAttach(_)) => {
865 RemoteCredentialAttachRecovery::HealedFromReceipt { value, provenance }
866 }
867 // Never committed before, committed now.
868 ServerValue::AttachBound(_) => {
869 RemoteCredentialAttachRecovery::CommittedFresh { value, provenance }
870 }
871 // Past the receipt window, inside provenance: the server can still name
872 // the generation the lost commit produced.
873 ServerValue::ReceiptExpired(liminal_protocol::wire::ReceiptExpired::CredentialAttach {
874 result_generation,
875 current_generation,
876 reason,
877 ..
878 }) => {
879 let (result_generation, current_generation, reason) = (
880 Some(*result_generation),
881 *current_generation,
882 CredentialAttachReissueReason::ReceiptExpired(*reason),
883 );
884 RemoteCredentialAttachRecovery::ReissueRequired {
885 result_generation,
886 current_generation,
887 reason,
888 value,
889 provenance,
890 }
891 }
892 // Past provenance: the server claims no commit proof, so no result
893 // generation is reported rather than one being inferred.
894 ServerValue::StaleOrUnknownReceipt(stale) => {
895 let current_generation = stale.current_generation;
896 RemoteCredentialAttachRecovery::ReissueRequired {
897 result_generation: None,
898 current_generation,
899 reason: CredentialAttachReissueReason::StaleOrUnknownReceipt,
900 value,
901 provenance,
902 }
903 }
904 _ => RemoteCredentialAttachRecovery::Answered { value, provenance },
905 }
906}
907
908fn checkpoint_state<S: ParticipantResumeStore>(
909 state: &mut super::RemoteParticipantState<S>,
910) -> Result<(), RemoteParticipantError> {
911 let aggregate = take_aggregate(state)?;
912 let aggregate = persist_retaining(state, aggregate)?;
913 state.aggregate = Some(aggregate);
914 Ok(())
915}