1mod recovery;
9mod replay_apply;
10
11pub use recovery::{
12 RemoteDetachReplayOutcome, RemoteExpectedOperationRecovery, RemoteLostOperationResolution,
13 RemoteLostReconnectResolution, RemoteReconnectAttemptOutcome, RemoteReconnectPermitRecovery,
14 RemoteReplayApplyOutcome, RemoteTransportLossOutcome,
15};
16
17use alloc::sync::Arc;
18use core::fmt;
19
20use liminal_protocol::client::{
21 ClientCorrelatedInboundDecision, ClientInboundDecision, ClientInboundRefusalReason,
22 ClientOperationRecordDecision, ClientOperationRecordRefusalReason, ClientParticipantAggregate,
23 ClientResponseCorrelation, ClientResumeRecord, ClientResumeRecordDecodeError,
24 ClientResumeRecordEncodeError, ClientResumeRestoreError, ExpectedOperationFateRefusalReason,
25 ExpectedOperationTransportFate, ExpectedParticipantOperation, ReconnectPermitDecision,
26 decide_correlated_inbound, decide_inbound, record_expected_operation_fate,
27 record_transport_fate,
28};
29use liminal_protocol::outcome::ReconnectDelayResult;
30use liminal_protocol::wire::{ClientRequest, ParticipantFrame, ServerPush, ServerValue};
31use spin::Mutex;
32
33use crate::SdkError;
34
35use super::protocol::{ParticipantTransportFrame, RemoteTransport};
36use super::{RemoteConfig, ServerAddress};
37
38pub trait ParticipantResumeStore: Send {
44 fn persist(&mut self, canonical_lpcr: &[u8]) -> Result<(), SdkError>;
50}
51
52#[derive(Clone, Copy, Debug, PartialEq, Eq)]
58pub struct ParticipantResponseProvenance {
59 connection_id: u64,
60 attempt_id: u64,
61}
62
63impl ParticipantResponseProvenance {
64 #[cfg(feature = "std")]
65 pub(super) const fn new(connection_id: u64, attempt_id: u64) -> Self {
66 Self {
67 connection_id,
68 attempt_id,
69 }
70 }
71
72 #[must_use]
74 pub const fn connection_id(self) -> u64 {
75 self.connection_id
76 }
77
78 #[must_use]
80 pub const fn attempt_id(self) -> u64 {
81 self.attempt_id
82 }
83}
84
85#[derive(Debug, thiserror::Error)]
87pub enum RemoteParticipantError {
88 #[error("participant state is unavailable after an unreleased durability failure")]
90 StateUnavailable,
91 #[error("client resume record encode failed: {0:?}")]
93 ResumeEncode(ClientResumeRecordEncodeError),
94 #[error("client resume record decode failed: {0:?}")]
96 ResumeDecode(ClientResumeRecordDecodeError),
97 #[error("client resume record restore failed: {0:?}")]
99 ResumeRestore(ClientResumeRestoreError),
100 #[error("client resume record persistence failed: {0}")]
102 Storage(SdkError),
103 #[error("participant transport failed: {0}")]
105 Transport(SdkError),
106 #[error("participant transport decoded a request in the client receive direction")]
108 InvalidInboundDirection,
109 #[error("no live participant response authority is held")]
111 ResponseAuthorityUnavailable,
112}
113
114#[derive(Debug)]
116pub struct RemoteParticipantOperation {
117 operation: ExpectedParticipantOperation,
118 durability: OperationDurability,
119}
120
121#[derive(Clone, Copy, Debug, PartialEq, Eq)]
122enum OperationDurability {
123 WriteAhead,
124 Continuous,
125}
126
127#[derive(Debug)]
129pub enum RemoteOperationRecordOutcome {
130 Recorded(RemoteParticipantOperation),
132 Continuous(RemoteParticipantOperation),
134 Refused {
136 request: ClientRequest,
138 reason: ClientOperationRecordRefusalReason,
140 },
141}
142
143#[derive(Debug, PartialEq, Eq)]
145pub enum RemoteOperationTransportFate {
146 Recorded {
148 request: ClientRequest,
150 },
151 DetachParked,
153 Refused {
155 reason: ExpectedOperationFateRefusalReason,
157 },
158 NotOutstanding,
160}
161
162#[derive(Debug)]
164pub struct RemoteReconnectPermit {
165 pub(super) permit: liminal_protocol::client::ReconnectAttemptPermit,
166}
167
168#[derive(Debug)]
170pub enum RemoteReconnectPermitOutcome {
171 Permitted {
173 permit: RemoteReconnectPermit,
175 result: ReconnectDelayResult,
177 },
178 Refused {
180 reason: liminal_protocol::client::ReconnectPermitRefusalReason,
182 result: ReconnectDelayResult,
184 },
185}
186
187#[derive(Debug)]
189pub enum RemoteParticipantSendOutcome {
190 Sent {
192 provenance: ParticipantResponseProvenance,
194 },
195 TransportLost {
197 error: SdkError,
199 operation_fate: RemoteOperationTransportFate,
201 reconnect: RemoteReconnectPermitOutcome,
203 },
204}
205
206#[derive(Debug)]
208pub enum RemoteParticipantInbound {
209 Applied {
211 value: ServerValue,
213 provenance: ParticipantResponseProvenance,
215 },
216 Refused {
218 value: ServerValue,
220 reason: ClientInboundRefusalReason,
222 provenance: ParticipantResponseProvenance,
224 },
225 Push {
227 value: ServerPush,
229 provenance: ParticipantResponseProvenance,
231 },
232}
233
234pub(super) struct RemoteParticipantState<S> {
235 pub(super) aggregate: Option<ClientParticipantAggregate>,
236 pub(super) correlation: Option<ClientResponseCorrelation>,
237 pub(super) reconnect_attempt: Option<liminal_protocol::client::ReconnectInProgressAttempt>,
238 pub(super) store: S,
239}
240
241pub struct RemoteParticipantHandle<S> {
247 pub(super) server_address: ServerAddress,
248 pub(super) transport: Arc<dyn RemoteTransport>,
249 pub(super) state: Mutex<RemoteParticipantState<S>>,
250}
251
252impl<S> fmt::Debug for RemoteParticipantHandle<S> {
253 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
254 formatter
255 .debug_struct("RemoteParticipantHandle")
256 .field("server_address", &self.server_address)
257 .finish_non_exhaustive()
258 }
259}
260
261impl<S: ParticipantResumeStore> RemoteParticipantHandle<S> {
262 pub fn new(config: &RemoteConfig, store: S) -> Result<Self, RemoteParticipantError> {
268 Self::from_aggregate(config, store, ClientParticipantAggregate::new())
269 }
270
271 pub fn restore(
277 config: &RemoteConfig,
278 store: S,
279 canonical_lpcr: &[u8],
280 ) -> Result<Self, RemoteParticipantError> {
281 let record = ClientResumeRecord::decode_canonical(canonical_lpcr)
282 .map_err(RemoteParticipantError::ResumeDecode)?;
283 let aggregate = record
284 .restore()
285 .map_err(RemoteParticipantError::ResumeRestore)?;
286 Self::from_aggregate(config, store, aggregate)
287 }
288
289 fn from_aggregate(
290 config: &RemoteConfig,
291 mut store: S,
292 aggregate: ClientParticipantAggregate,
293 ) -> Result<Self, RemoteParticipantError> {
294 persist(&mut store, &aggregate)?;
295 Ok(Self {
296 server_address: config.server_address.clone(),
297 transport: Arc::clone(&config.transport),
298 state: Mutex::new(RemoteParticipantState {
299 aggregate: Some(aggregate),
300 correlation: None,
301 reconnect_attempt: None,
302 store,
303 }),
304 })
305 }
306
307 pub fn record_operation(
314 &self,
315 request: ClientRequest,
316 ) -> Result<RemoteOperationRecordOutcome, RemoteParticipantError> {
317 let mut state = self.state.lock();
318 let aggregate = take_aggregate(&mut state)?;
319 match liminal_protocol::client::record_operation(aggregate, request) {
320 ClientOperationRecordDecision::Pending(pending) => {
321 let commit = pending.commit();
322 let record = commit
323 .resume_record()
324 .map_err(RemoteParticipantError::ResumeEncode)?;
325 state
326 .store
327 .persist(&record.encode_canonical())
328 .map_err(RemoteParticipantError::Storage)?;
329 let (aggregate, operation) = commit.into_parts();
330 state.aggregate = Some(aggregate);
331 Ok(RemoteOperationRecordOutcome::Recorded(
332 RemoteParticipantOperation {
333 operation,
334 durability: OperationDurability::WriteAhead,
335 },
336 ))
337 }
338 ClientOperationRecordDecision::Continuous(continuous) => {
339 let (aggregate, operation) = continuous.into_parts();
340 state.aggregate = Some(aggregate);
341 Ok(RemoteOperationRecordOutcome::Continuous(
342 RemoteParticipantOperation {
343 operation,
344 durability: OperationDurability::Continuous,
345 },
346 ))
347 }
348 ClientOperationRecordDecision::Refused(refusal) => {
349 let reason = refusal.reason();
350 let (aggregate, request) = refusal.into_parts();
351 state.aggregate = Some(aggregate);
352 Ok(RemoteOperationRecordOutcome::Refused { request, reason })
353 }
354 }
355 }
356
357 pub fn send_operation(
364 &self,
365 operation: RemoteParticipantOperation,
366 ) -> Result<RemoteParticipantSendOutcome, RemoteParticipantError> {
367 let mut state = self.state.lock();
368 let aggregate = take_aggregate(&mut state)?;
369 if operation.durability == OperationDurability::WriteAhead {
370 persist(&mut state.store, &aggregate)?;
371 }
372 let (request, correlation) = operation.operation.into_request();
373 match self
374 .transport
375 .send_participant(&self.server_address, &request)
376 {
377 Ok(provenance) => {
378 if operation.durability == OperationDurability::WriteAhead {
379 state.correlation = Some(correlation);
380 }
381 state.aggregate = Some(aggregate);
382 Ok(RemoteParticipantSendOutcome::Sent { provenance })
383 }
384 Err(error) => {
385 let operation_fate = if operation.durability == OperationDurability::WriteAhead {
386 record_operation_transport_fate(&mut state, aggregate, correlation)
387 } else {
388 state.aggregate = Some(aggregate);
389 RemoteOperationTransportFate::NotOutstanding
390 };
391 let reconnect = record_connection_fate(&mut state)?;
392 Ok(RemoteParticipantSendOutcome::TransportLost {
393 error,
394 operation_fate,
395 reconnect,
396 })
397 }
398 }
399 }
400
401 pub fn receive(&self) -> Result<RemoteParticipantInbound, RemoteParticipantError> {
407 let ParticipantTransportFrame { frame, provenance } = self
408 .transport
409 .receive_participant(&self.server_address)
410 .map_err(RemoteParticipantError::Transport)?;
411 match frame {
412 ParticipantFrame::ServerPush(value) => {
413 Ok(RemoteParticipantInbound::Push { value, provenance })
414 }
415 ParticipantFrame::ClientRequest(_) => {
416 Err(RemoteParticipantError::InvalidInboundDirection)
417 }
418 ParticipantFrame::ServerValue(value) => self.apply_inbound(value, provenance),
419 }
420 }
421
422 fn apply_inbound(
423 &self,
424 value: ServerValue,
425 provenance: ParticipantResponseProvenance,
426 ) -> Result<RemoteParticipantInbound, RemoteParticipantError> {
427 let mut state = self.state.lock();
428 let aggregate = take_aggregate(&mut state)?;
429 if let Some(correlation) = state.correlation.take() {
430 match decide_correlated_inbound(aggregate, value, correlation) {
431 ClientCorrelatedInboundDecision::Applied(applied) => {
432 let (aggregate, value) = applied.into_parts();
433 persist(&mut state.store, &aggregate)?;
434 state.aggregate = Some(aggregate);
435 Ok(RemoteParticipantInbound::Applied { value, provenance })
436 }
437 ClientCorrelatedInboundDecision::Refused(refusal) => {
438 let reason = refusal.reason();
439 let (aggregate, value, correlation) = refusal.into_parts();
440 state.aggregate = Some(aggregate);
441 state.correlation = Some(correlation);
442 Ok(RemoteParticipantInbound::Refused {
443 value,
444 reason,
445 provenance,
446 })
447 }
448 }
449 } else {
450 match decide_inbound(aggregate, value) {
451 ClientInboundDecision::Applied(applied) => {
452 let (aggregate, value) = applied.into_parts();
453 persist(&mut state.store, &aggregate)?;
454 state.aggregate = Some(aggregate);
455 Ok(RemoteParticipantInbound::Applied { value, provenance })
456 }
457 ClientInboundDecision::Refused(refusal) => {
458 let reason = refusal.reason();
459 let (aggregate, value) = refusal.into_parts();
460 state.aggregate = Some(aggregate);
461 Ok(RemoteParticipantInbound::Refused {
462 value,
463 reason,
464 provenance,
465 })
466 }
467 }
468 }
469 }
470}
471
472pub(super) fn take_aggregate<S>(
473 state: &mut RemoteParticipantState<S>,
474) -> Result<ClientParticipantAggregate, RemoteParticipantError> {
475 state
476 .aggregate
477 .take()
478 .ok_or(RemoteParticipantError::StateUnavailable)
479}
480
481pub(super) fn persist<S: ParticipantResumeStore>(
482 store: &mut S,
483 aggregate: &ClientParticipantAggregate,
484) -> Result<(), RemoteParticipantError> {
485 let record = aggregate
486 .resume_record()
487 .map_err(RemoteParticipantError::ResumeEncode)?;
488 store
489 .persist(&record.encode_canonical())
490 .map_err(RemoteParticipantError::Storage)
491}
492
493fn record_operation_transport_fate<S: ParticipantResumeStore>(
494 state: &mut RemoteParticipantState<S>,
495 aggregate: ClientParticipantAggregate,
496 correlation: ClientResponseCorrelation,
497) -> RemoteOperationTransportFate {
498 match record_expected_operation_fate(
499 aggregate,
500 correlation,
501 ExpectedOperationTransportFate::ResponseUnavailable,
502 ) {
503 liminal_protocol::client::ExpectedOperationFateDecision::Recorded {
504 aggregate,
505 request,
506 ..
507 } => {
508 state.aggregate = Some(aggregate);
509 RemoteOperationTransportFate::Recorded { request }
510 }
511 liminal_protocol::client::ExpectedOperationFateDecision::Refused {
512 aggregate,
513 correlation,
514 reason: ExpectedOperationFateRefusalReason::DetachUsesReplayFate,
515 ..
516 } => match liminal_protocol::client::transport_fate(
517 aggregate,
518 correlation,
519 liminal_protocol::client::DetachTransportFate::ResponseUnavailable,
520 ) {
521 liminal_protocol::client::DetachTransportFateDecision::Parked(applied) => {
522 state.aggregate = Some(applied.into_aggregate());
523 RemoteOperationTransportFate::DetachParked
524 }
525 liminal_protocol::client::DetachTransportFateDecision::Refused(refusal) => {
526 let (aggregate, (correlation, _)) = refusal.into_parts();
527 state.aggregate = Some(aggregate);
528 state.correlation = Some(correlation);
529 RemoteOperationTransportFate::Refused {
530 reason: ExpectedOperationFateRefusalReason::DetachUsesReplayFate,
531 }
532 }
533 },
534 liminal_protocol::client::ExpectedOperationFateDecision::Refused {
535 aggregate,
536 correlation,
537 reason,
538 ..
539 } => {
540 state.aggregate = Some(aggregate);
541 state.correlation = Some(correlation);
542 RemoteOperationTransportFate::Refused { reason }
543 }
544 }
545}
546
547pub(super) fn record_connection_fate<S: ParticipantResumeStore>(
548 state: &mut RemoteParticipantState<S>,
549) -> Result<RemoteReconnectPermitOutcome, RemoteParticipantError> {
550 let aggregate = take_aggregate(state)?;
551 let (aggregate, outcome) = match record_transport_fate(
552 aggregate,
553 liminal_protocol::client::EstablishedConnectionTransportFate::Lost,
554 ) {
555 ReconnectPermitDecision::Permitted {
556 aggregate,
557 permit,
558 result,
559 } => (
560 aggregate,
561 RemoteReconnectPermitOutcome::Permitted {
562 permit: RemoteReconnectPermit { permit },
563 result,
564 },
565 ),
566 ReconnectPermitDecision::Refused(refusal) => {
567 let reason = refusal.reason();
568 let result = refusal.result();
569 let (aggregate, _) = refusal.into_parts();
570 (
571 aggregate,
572 RemoteReconnectPermitOutcome::Refused { reason, result },
573 )
574 }
575 };
576 persist(&mut state.store, &aggregate)?;
577 state.aggregate = Some(aggregate);
578 Ok(outcome)
579}
580
581#[cfg(test)]
582mod tests;