1mod recovery;
9mod replay_apply;
10
11pub use recovery::{
12 RemoteDetachReplayOutcome, RemoteExpectedOperationRecovery, RemoteLostOperationResolution,
13 RemoteLostReconnectResolution, RemoteReconnectAttemptOutcome, RemoteReconnectPermitRecovery,
14 RemoteReplayApplyOutcome, RemoteTransportLossOutcome,
15};
16
17use alloc::sync::Arc;
18use core::fmt;
19
20use liminal_protocol::client::{
21 ClientCorrelatedInboundDecision, ClientInboundDecision, ClientInboundRefusalReason,
22 ClientOperationRecordDecision, ClientOperationRecordRefusalReason, ClientParticipantAggregate,
23 ClientResponseCorrelation, ClientResumeRecord, ClientResumeRecordDecodeError,
24 ClientResumeRecordEncodeError, ClientResumeRestoreError, ExpectedOperationFateRefusalReason,
25 ExpectedOperationTransportFate, ExpectedParticipantOperation, ReconnectPermitDecision,
26 decide_correlated_inbound, decide_inbound, record_expected_operation_fate,
27 record_transport_fate,
28};
29use liminal_protocol::outcome::ReconnectDelayResult;
30use liminal_protocol::wire::{
31 ClientRequest, DeliverySeq, ParticipantFrame, ServerPush, ServerValue,
32};
33use spin::Mutex;
34
35use crate::SdkError;
36
37use super::protocol::{ParticipantTransportFrame, RemoteTransport};
38use super::{RemoteConfig, ServerAddress};
39
40pub trait ParticipantResumeStore: Send {
46 fn persist(&mut self, canonical_lpcr: &[u8]) -> Result<(), SdkError>;
52}
53
54#[derive(Clone, Copy, Debug, PartialEq, Eq)]
60pub struct ParticipantResponseProvenance {
61 connection_id: u64,
62 attempt_id: u64,
63}
64
65impl ParticipantResponseProvenance {
66 #[cfg(feature = "std")]
67 pub(super) const fn new(connection_id: u64, attempt_id: u64) -> Self {
68 Self {
69 connection_id,
70 attempt_id,
71 }
72 }
73
74 #[must_use]
76 pub const fn connection_id(self) -> u64 {
77 self.connection_id
78 }
79
80 #[must_use]
82 pub const fn attempt_id(self) -> u64 {
83 self.attempt_id
84 }
85}
86
87#[derive(Debug, thiserror::Error)]
89pub enum RemoteParticipantError {
90 #[error("participant state is unavailable after an unreleased durability failure")]
92 StateUnavailable,
93 #[error("client resume record encode failed: {0:?}")]
95 ResumeEncode(ClientResumeRecordEncodeError),
96 #[error("client resume record decode failed: {0:?}")]
98 ResumeDecode(ClientResumeRecordDecodeError),
99 #[error("client resume record restore failed: {0:?}")]
101 ResumeRestore(ClientResumeRestoreError),
102 #[error("client resume record persistence failed: {0}")]
104 Storage(SdkError),
105 #[error("participant transport failed: {0}")]
107 Transport(SdkError),
108 #[error("participant transport decoded a request in the client receive direction")]
110 InvalidInboundDirection,
111 #[error("no live participant response authority is held")]
113 ResponseAuthorityUnavailable,
114}
115
116#[derive(Debug)]
118pub struct RemoteParticipantOperation {
119 operation: ExpectedParticipantOperation,
120 durability: OperationDurability,
121}
122
123#[derive(Clone, Copy, Debug, PartialEq, Eq)]
124enum OperationDurability {
125 WriteAhead,
126 Continuous,
127}
128
129#[derive(Debug)]
131pub enum RemoteOperationRecordOutcome {
132 Recorded(RemoteParticipantOperation),
134 Continuous(RemoteParticipantOperation),
136 Refused {
138 request: ClientRequest,
140 reason: ClientOperationRecordRefusalReason,
142 },
143}
144
145#[derive(Debug, PartialEq, Eq)]
147pub enum RemoteOperationTransportFate {
148 Recorded {
150 request: ClientRequest,
152 },
153 DetachParked,
155 Refused {
157 reason: ExpectedOperationFateRefusalReason,
159 },
160 NotOutstanding,
162}
163
164#[derive(Debug)]
166pub struct RemoteReconnectPermit {
167 pub(super) permit: liminal_protocol::client::ReconnectAttemptPermit,
168}
169
170#[derive(Debug)]
172pub enum RemoteReconnectPermitOutcome {
173 Permitted {
175 permit: RemoteReconnectPermit,
177 result: ReconnectDelayResult,
179 },
180 Refused {
182 reason: liminal_protocol::client::ReconnectPermitRefusalReason,
184 result: ReconnectDelayResult,
186 },
187}
188
189#[derive(Debug)]
191pub enum RemoteParticipantSendOutcome {
192 Sent {
194 provenance: ParticipantResponseProvenance,
196 },
197 TransportLost {
199 error: SdkError,
201 operation_fate: RemoteOperationTransportFate,
203 reconnect: RemoteReconnectPermitOutcome,
205 },
206}
207
208#[derive(Debug)]
210pub enum RemoteParticipantInbound {
211 Applied {
213 value: ServerValue,
215 provenance: ParticipantResponseProvenance,
217 },
218 Refused {
220 value: ServerValue,
222 reason: ClientInboundRefusalReason,
224 provenance: ParticipantResponseProvenance,
226 },
227 Push {
229 value: ServerPush,
231 provenance: ParticipantResponseProvenance,
233 },
234}
235
236impl RemoteParticipantInbound {
237 #[must_use]
253 pub const fn committed_delivery_seq(&self) -> Option<DeliverySeq> {
254 match self {
255 Self::Applied {
256 value: ServerValue::RecordCommitted(committed),
257 ..
258 } => Some(committed.delivery_seq()),
259 Self::Applied { .. } | Self::Refused { .. } | Self::Push { .. } => None,
260 }
261 }
262}
263
264pub(super) struct RemoteParticipantState<S> {
265 pub(super) aggregate: Option<ClientParticipantAggregate>,
266 pub(super) correlation: Option<ClientResponseCorrelation>,
267 pub(super) reconnect_attempt: Option<liminal_protocol::client::ReconnectInProgressAttempt>,
268 pub(super) store: S,
269}
270
271pub struct RemoteParticipantHandle<S> {
277 pub(super) server_address: ServerAddress,
278 pub(super) transport: Arc<dyn RemoteTransport>,
279 pub(super) state: Mutex<RemoteParticipantState<S>>,
280}
281
282impl<S> fmt::Debug for RemoteParticipantHandle<S> {
283 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
284 formatter
285 .debug_struct("RemoteParticipantHandle")
286 .field("server_address", &self.server_address)
287 .finish_non_exhaustive()
288 }
289}
290
291impl<S: ParticipantResumeStore> RemoteParticipantHandle<S> {
292 pub fn new(config: &RemoteConfig, store: S) -> Result<Self, RemoteParticipantError> {
298 Self::from_aggregate(config, store, ClientParticipantAggregate::new())
299 }
300
301 pub fn restore(
307 config: &RemoteConfig,
308 store: S,
309 canonical_lpcr: &[u8],
310 ) -> Result<Self, RemoteParticipantError> {
311 let record = ClientResumeRecord::decode_canonical(canonical_lpcr)
312 .map_err(RemoteParticipantError::ResumeDecode)?;
313 let aggregate = record
314 .restore()
315 .map_err(RemoteParticipantError::ResumeRestore)?;
316 Self::from_aggregate(config, store, aggregate)
317 }
318
319 fn from_aggregate(
320 config: &RemoteConfig,
321 mut store: S,
322 aggregate: ClientParticipantAggregate,
323 ) -> Result<Self, RemoteParticipantError> {
324 persist(&mut store, &aggregate)?;
325 Ok(Self {
326 server_address: config.server_address.clone(),
327 transport: Arc::clone(&config.transport),
328 state: Mutex::new(RemoteParticipantState {
329 aggregate: Some(aggregate),
330 correlation: None,
331 reconnect_attempt: None,
332 store,
333 }),
334 })
335 }
336
337 pub fn record_operation(
344 &self,
345 request: ClientRequest,
346 ) -> Result<RemoteOperationRecordOutcome, RemoteParticipantError> {
347 let mut state = self.state.lock();
348 let aggregate = take_aggregate(&mut state)?;
349 match liminal_protocol::client::record_operation(aggregate, request) {
350 ClientOperationRecordDecision::Pending(pending) => {
351 let commit = pending.commit();
352 let record = commit
353 .resume_record()
354 .map_err(RemoteParticipantError::ResumeEncode)?;
355 state
356 .store
357 .persist(&record.encode_canonical())
358 .map_err(RemoteParticipantError::Storage)?;
359 let (aggregate, operation) = commit.into_parts();
360 state.aggregate = Some(aggregate);
361 Ok(RemoteOperationRecordOutcome::Recorded(
362 RemoteParticipantOperation {
363 operation,
364 durability: OperationDurability::WriteAhead,
365 },
366 ))
367 }
368 ClientOperationRecordDecision::Continuous(continuous) => {
369 let (aggregate, operation) = continuous.into_parts();
370 state.aggregate = Some(aggregate);
371 Ok(RemoteOperationRecordOutcome::Continuous(
372 RemoteParticipantOperation {
373 operation,
374 durability: OperationDurability::Continuous,
375 },
376 ))
377 }
378 ClientOperationRecordDecision::Refused(refusal) => {
379 let reason = refusal.reason();
380 let (aggregate, request) = refusal.into_parts();
381 state.aggregate = Some(aggregate);
382 Ok(RemoteOperationRecordOutcome::Refused { request, reason })
383 }
384 }
385 }
386
387 pub fn send_operation(
394 &self,
395 operation: RemoteParticipantOperation,
396 ) -> Result<RemoteParticipantSendOutcome, RemoteParticipantError> {
397 let mut state = self.state.lock();
398 let aggregate = take_aggregate(&mut state)?;
399 if operation.durability == OperationDurability::WriteAhead {
400 persist(&mut state.store, &aggregate)?;
401 }
402 let (request, correlation) = operation.operation.into_request();
403 match self
404 .transport
405 .send_participant(&self.server_address, &request)
406 {
407 Ok(provenance) => {
408 if operation.durability == OperationDurability::WriteAhead {
409 state.correlation = Some(correlation);
410 }
411 state.aggregate = Some(aggregate);
412 Ok(RemoteParticipantSendOutcome::Sent { provenance })
413 }
414 Err(error) => {
415 let operation_fate = if operation.durability == OperationDurability::WriteAhead {
416 record_operation_transport_fate(&mut state, aggregate, correlation)
417 } else {
418 state.aggregate = Some(aggregate);
419 RemoteOperationTransportFate::NotOutstanding
420 };
421 let reconnect = record_connection_fate(&mut state)?;
422 Ok(RemoteParticipantSendOutcome::TransportLost {
423 error,
424 operation_fate,
425 reconnect,
426 })
427 }
428 }
429 }
430
431 pub fn receive(&self) -> Result<RemoteParticipantInbound, RemoteParticipantError> {
442 let ParticipantTransportFrame { frame, provenance } = self
443 .transport
444 .receive_participant(&self.server_address)
445 .map_err(RemoteParticipantError::Transport)?;
446 match frame {
447 ParticipantFrame::ServerPush(value) => {
448 Ok(RemoteParticipantInbound::Push { value, provenance })
449 }
450 ParticipantFrame::ClientRequest(_) => {
451 Err(RemoteParticipantError::InvalidInboundDirection)
452 }
453 ParticipantFrame::ServerValue(value) => self.apply_inbound(value, provenance),
454 }
455 }
456
457 fn apply_inbound(
458 &self,
459 value: ServerValue,
460 provenance: ParticipantResponseProvenance,
461 ) -> Result<RemoteParticipantInbound, RemoteParticipantError> {
462 let mut state = self.state.lock();
463 let aggregate = take_aggregate(&mut state)?;
464 if let Some(correlation) = state.correlation.take() {
465 match decide_correlated_inbound(aggregate, value, correlation) {
466 ClientCorrelatedInboundDecision::Applied(applied) => {
467 let (aggregate, value) = applied.into_parts();
468 persist(&mut state.store, &aggregate)?;
469 state.aggregate = Some(aggregate);
470 Ok(RemoteParticipantInbound::Applied { value, provenance })
471 }
472 ClientCorrelatedInboundDecision::Refused(refusal) => {
473 let reason = refusal.reason();
474 let (aggregate, value, correlation) = refusal.into_parts();
475 state.aggregate = Some(aggregate);
476 state.correlation = Some(correlation);
477 Ok(RemoteParticipantInbound::Refused {
478 value,
479 reason,
480 provenance,
481 })
482 }
483 }
484 } else {
485 match decide_inbound(aggregate, value) {
486 ClientInboundDecision::Applied(applied) => {
487 let (aggregate, value) = applied.into_parts();
488 persist(&mut state.store, &aggregate)?;
489 state.aggregate = Some(aggregate);
490 Ok(RemoteParticipantInbound::Applied { value, provenance })
491 }
492 ClientInboundDecision::Refused(refusal) => {
493 let reason = refusal.reason();
494 let (aggregate, value) = refusal.into_parts();
495 state.aggregate = Some(aggregate);
496 Ok(RemoteParticipantInbound::Refused {
497 value,
498 reason,
499 provenance,
500 })
501 }
502 }
503 }
504 }
505}
506
507pub(super) fn take_aggregate<S>(
508 state: &mut RemoteParticipantState<S>,
509) -> Result<ClientParticipantAggregate, RemoteParticipantError> {
510 state
511 .aggregate
512 .take()
513 .ok_or(RemoteParticipantError::StateUnavailable)
514}
515
516pub(super) fn persist<S: ParticipantResumeStore>(
517 store: &mut S,
518 aggregate: &ClientParticipantAggregate,
519) -> Result<(), RemoteParticipantError> {
520 let record = aggregate
521 .resume_record()
522 .map_err(RemoteParticipantError::ResumeEncode)?;
523 store
524 .persist(&record.encode_canonical())
525 .map_err(RemoteParticipantError::Storage)
526}
527
528fn record_operation_transport_fate<S: ParticipantResumeStore>(
529 state: &mut RemoteParticipantState<S>,
530 aggregate: ClientParticipantAggregate,
531 correlation: ClientResponseCorrelation,
532) -> RemoteOperationTransportFate {
533 match record_expected_operation_fate(
534 aggregate,
535 correlation,
536 ExpectedOperationTransportFate::ResponseUnavailable,
537 ) {
538 liminal_protocol::client::ExpectedOperationFateDecision::Recorded {
539 aggregate,
540 request,
541 ..
542 } => {
543 state.aggregate = Some(aggregate);
544 RemoteOperationTransportFate::Recorded { request }
545 }
546 liminal_protocol::client::ExpectedOperationFateDecision::Refused {
547 aggregate,
548 correlation,
549 reason: ExpectedOperationFateRefusalReason::DetachUsesReplayFate,
550 ..
551 } => match liminal_protocol::client::transport_fate(
552 aggregate,
553 correlation,
554 liminal_protocol::client::DetachTransportFate::ResponseUnavailable,
555 ) {
556 liminal_protocol::client::DetachTransportFateDecision::Parked(applied) => {
557 state.aggregate = Some(applied.into_aggregate());
558 RemoteOperationTransportFate::DetachParked
559 }
560 liminal_protocol::client::DetachTransportFateDecision::Refused(refusal) => {
561 let (aggregate, (correlation, _)) = refusal.into_parts();
562 state.aggregate = Some(aggregate);
563 state.correlation = Some(correlation);
564 RemoteOperationTransportFate::Refused {
565 reason: ExpectedOperationFateRefusalReason::DetachUsesReplayFate,
566 }
567 }
568 },
569 liminal_protocol::client::ExpectedOperationFateDecision::Refused {
570 aggregate,
571 correlation,
572 reason,
573 ..
574 } => {
575 state.aggregate = Some(aggregate);
576 state.correlation = Some(correlation);
577 RemoteOperationTransportFate::Refused { reason }
578 }
579 }
580}
581
582pub(super) fn record_connection_fate<S: ParticipantResumeStore>(
583 state: &mut RemoteParticipantState<S>,
584) -> Result<RemoteReconnectPermitOutcome, RemoteParticipantError> {
585 let aggregate = take_aggregate(state)?;
586 let (aggregate, outcome) = match record_transport_fate(
587 aggregate,
588 liminal_protocol::client::EstablishedConnectionTransportFate::Lost,
589 ) {
590 ReconnectPermitDecision::Permitted {
591 aggregate,
592 permit,
593 result,
594 } => (
595 aggregate,
596 RemoteReconnectPermitOutcome::Permitted {
597 permit: RemoteReconnectPermit { permit },
598 result,
599 },
600 ),
601 ReconnectPermitDecision::Refused(refusal) => {
602 let reason = refusal.reason();
603 let result = refusal.result();
604 let (aggregate, _) = refusal.into_parts();
605 (
606 aggregate,
607 RemoteReconnectPermitOutcome::Refused { reason, result },
608 )
609 }
610 };
611 persist(&mut state.store, &aggregate)?;
612 state.aggregate = Some(aggregate);
613 Ok(outcome)
614}
615
616#[cfg(test)]
617mod tests;