mod recovery;
mod replay_apply;
pub use recovery::{
RemoteDetachReplayOutcome, RemoteExpectedOperationRecovery, RemoteLostOperationResolution,
RemoteLostReconnectResolution, RemoteReconnectAttemptOutcome, RemoteReconnectPermitRecovery,
RemoteReplayApplyOutcome, RemoteTransportLossOutcome,
};
use alloc::sync::Arc;
use core::fmt;
use liminal_protocol::client::{
ClientCorrelatedInboundDecision, ClientInboundDecision, ClientInboundRefusalReason,
ClientOperationRecordDecision, ClientOperationRecordRefusalReason, ClientParticipantAggregate,
ClientResponseCorrelation, ClientResumeRecord, ClientResumeRecordDecodeError,
ClientResumeRecordEncodeError, ClientResumeRestoreError, ExpectedOperationFateRefusalReason,
ExpectedOperationTransportFate, ExpectedParticipantOperation, ReconnectPermitDecision,
decide_correlated_inbound, decide_inbound, record_expected_operation_fate,
record_transport_fate,
};
use liminal_protocol::outcome::ReconnectDelayResult;
use liminal_protocol::wire::{ClientRequest, ParticipantFrame, ServerPush, ServerValue};
use spin::Mutex;
use crate::SdkError;
use super::protocol::{ParticipantTransportFrame, RemoteTransport};
use super::{RemoteConfig, ServerAddress};
pub trait ParticipantResumeStore: Send {
fn persist(&mut self, canonical_lpcr: &[u8]) -> Result<(), SdkError>;
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ParticipantResponseProvenance {
connection_id: u64,
attempt_id: u64,
}
impl ParticipantResponseProvenance {
#[cfg(feature = "std")]
pub(super) const fn new(connection_id: u64, attempt_id: u64) -> Self {
Self {
connection_id,
attempt_id,
}
}
#[must_use]
pub const fn connection_id(self) -> u64 {
self.connection_id
}
#[must_use]
pub const fn attempt_id(self) -> u64 {
self.attempt_id
}
}
#[derive(Debug, thiserror::Error)]
pub enum RemoteParticipantError {
#[error("participant state is unavailable after an unreleased durability failure")]
StateUnavailable,
#[error("client resume record encode failed: {0:?}")]
ResumeEncode(ClientResumeRecordEncodeError),
#[error("client resume record decode failed: {0:?}")]
ResumeDecode(ClientResumeRecordDecodeError),
#[error("client resume record restore failed: {0:?}")]
ResumeRestore(ClientResumeRestoreError),
#[error("client resume record persistence failed: {0}")]
Storage(SdkError),
#[error("participant transport failed: {0}")]
Transport(SdkError),
#[error("participant transport decoded a request in the client receive direction")]
InvalidInboundDirection,
#[error("no live participant response authority is held")]
ResponseAuthorityUnavailable,
}
#[derive(Debug)]
pub struct RemoteParticipantOperation {
operation: ExpectedParticipantOperation,
durability: OperationDurability,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum OperationDurability {
WriteAhead,
Continuous,
}
#[derive(Debug)]
pub enum RemoteOperationRecordOutcome {
Recorded(RemoteParticipantOperation),
Continuous(RemoteParticipantOperation),
Refused {
request: ClientRequest,
reason: ClientOperationRecordRefusalReason,
},
}
#[derive(Debug, PartialEq, Eq)]
pub enum RemoteOperationTransportFate {
Recorded {
request: ClientRequest,
},
DetachParked,
Refused {
reason: ExpectedOperationFateRefusalReason,
},
NotOutstanding,
}
#[derive(Debug)]
pub struct RemoteReconnectPermit {
pub(super) permit: liminal_protocol::client::ReconnectAttemptPermit,
}
#[derive(Debug)]
pub enum RemoteReconnectPermitOutcome {
Permitted {
permit: RemoteReconnectPermit,
result: ReconnectDelayResult,
},
Refused {
reason: liminal_protocol::client::ReconnectPermitRefusalReason,
result: ReconnectDelayResult,
},
}
#[derive(Debug)]
pub enum RemoteParticipantSendOutcome {
Sent {
provenance: ParticipantResponseProvenance,
},
TransportLost {
error: SdkError,
operation_fate: RemoteOperationTransportFate,
reconnect: RemoteReconnectPermitOutcome,
},
}
#[derive(Debug)]
pub enum RemoteParticipantInbound {
Applied {
value: ServerValue,
provenance: ParticipantResponseProvenance,
},
Refused {
value: ServerValue,
reason: ClientInboundRefusalReason,
provenance: ParticipantResponseProvenance,
},
Push {
value: ServerPush,
provenance: ParticipantResponseProvenance,
},
}
pub(super) struct RemoteParticipantState<S> {
pub(super) aggregate: Option<ClientParticipantAggregate>,
pub(super) correlation: Option<ClientResponseCorrelation>,
pub(super) reconnect_attempt: Option<liminal_protocol::client::ReconnectInProgressAttempt>,
pub(super) store: S,
}
pub struct RemoteParticipantHandle<S> {
pub(super) server_address: ServerAddress,
pub(super) transport: Arc<dyn RemoteTransport>,
pub(super) state: Mutex<RemoteParticipantState<S>>,
}
impl<S> fmt::Debug for RemoteParticipantHandle<S> {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("RemoteParticipantHandle")
.field("server_address", &self.server_address)
.finish_non_exhaustive()
}
}
impl<S: ParticipantResumeStore> RemoteParticipantHandle<S> {
pub fn new(config: &RemoteConfig, store: S) -> Result<Self, RemoteParticipantError> {
Self::from_aggregate(config, store, ClientParticipantAggregate::new())
}
pub fn restore(
config: &RemoteConfig,
store: S,
canonical_lpcr: &[u8],
) -> Result<Self, RemoteParticipantError> {
let record = ClientResumeRecord::decode_canonical(canonical_lpcr)
.map_err(RemoteParticipantError::ResumeDecode)?;
let aggregate = record
.restore()
.map_err(RemoteParticipantError::ResumeRestore)?;
Self::from_aggregate(config, store, aggregate)
}
fn from_aggregate(
config: &RemoteConfig,
mut store: S,
aggregate: ClientParticipantAggregate,
) -> Result<Self, RemoteParticipantError> {
persist(&mut store, &aggregate)?;
Ok(Self {
server_address: config.server_address.clone(),
transport: Arc::clone(&config.transport),
state: Mutex::new(RemoteParticipantState {
aggregate: Some(aggregate),
correlation: None,
reconnect_attempt: None,
store,
}),
})
}
pub fn record_operation(
&self,
request: ClientRequest,
) -> Result<RemoteOperationRecordOutcome, RemoteParticipantError> {
let mut state = self.state.lock();
let aggregate = take_aggregate(&mut state)?;
match liminal_protocol::client::record_operation(aggregate, request) {
ClientOperationRecordDecision::Pending(pending) => {
let commit = pending.commit();
let record = commit
.resume_record()
.map_err(RemoteParticipantError::ResumeEncode)?;
state
.store
.persist(&record.encode_canonical())
.map_err(RemoteParticipantError::Storage)?;
let (aggregate, operation) = commit.into_parts();
state.aggregate = Some(aggregate);
Ok(RemoteOperationRecordOutcome::Recorded(
RemoteParticipantOperation {
operation,
durability: OperationDurability::WriteAhead,
},
))
}
ClientOperationRecordDecision::Continuous(continuous) => {
let (aggregate, operation) = continuous.into_parts();
state.aggregate = Some(aggregate);
Ok(RemoteOperationRecordOutcome::Continuous(
RemoteParticipantOperation {
operation,
durability: OperationDurability::Continuous,
},
))
}
ClientOperationRecordDecision::Refused(refusal) => {
let reason = refusal.reason();
let (aggregate, request) = refusal.into_parts();
state.aggregate = Some(aggregate);
Ok(RemoteOperationRecordOutcome::Refused { request, reason })
}
}
}
pub fn send_operation(
&self,
operation: RemoteParticipantOperation,
) -> Result<RemoteParticipantSendOutcome, RemoteParticipantError> {
let mut state = self.state.lock();
let aggregate = take_aggregate(&mut state)?;
if operation.durability == OperationDurability::WriteAhead {
persist(&mut state.store, &aggregate)?;
}
let (request, correlation) = operation.operation.into_request();
match self
.transport
.send_participant(&self.server_address, &request)
{
Ok(provenance) => {
if operation.durability == OperationDurability::WriteAhead {
state.correlation = Some(correlation);
}
state.aggregate = Some(aggregate);
Ok(RemoteParticipantSendOutcome::Sent { provenance })
}
Err(error) => {
let operation_fate = if operation.durability == OperationDurability::WriteAhead {
record_operation_transport_fate(&mut state, aggregate, correlation)
} else {
state.aggregate = Some(aggregate);
RemoteOperationTransportFate::NotOutstanding
};
let reconnect = record_connection_fate(&mut state)?;
Ok(RemoteParticipantSendOutcome::TransportLost {
error,
operation_fate,
reconnect,
})
}
}
}
pub fn receive(&self) -> Result<RemoteParticipantInbound, RemoteParticipantError> {
let ParticipantTransportFrame { frame, provenance } = self
.transport
.receive_participant(&self.server_address)
.map_err(RemoteParticipantError::Transport)?;
match frame {
ParticipantFrame::ServerPush(value) => {
Ok(RemoteParticipantInbound::Push { value, provenance })
}
ParticipantFrame::ClientRequest(_) => {
Err(RemoteParticipantError::InvalidInboundDirection)
}
ParticipantFrame::ServerValue(value) => self.apply_inbound(value, provenance),
}
}
fn apply_inbound(
&self,
value: ServerValue,
provenance: ParticipantResponseProvenance,
) -> Result<RemoteParticipantInbound, RemoteParticipantError> {
let mut state = self.state.lock();
let aggregate = take_aggregate(&mut state)?;
if let Some(correlation) = state.correlation.take() {
match decide_correlated_inbound(aggregate, value, correlation) {
ClientCorrelatedInboundDecision::Applied(applied) => {
let (aggregate, value) = applied.into_parts();
persist(&mut state.store, &aggregate)?;
state.aggregate = Some(aggregate);
Ok(RemoteParticipantInbound::Applied { value, provenance })
}
ClientCorrelatedInboundDecision::Refused(refusal) => {
let reason = refusal.reason();
let (aggregate, value, correlation) = refusal.into_parts();
state.aggregate = Some(aggregate);
state.correlation = Some(correlation);
Ok(RemoteParticipantInbound::Refused {
value,
reason,
provenance,
})
}
}
} else {
match decide_inbound(aggregate, value) {
ClientInboundDecision::Applied(applied) => {
let (aggregate, value) = applied.into_parts();
persist(&mut state.store, &aggregate)?;
state.aggregate = Some(aggregate);
Ok(RemoteParticipantInbound::Applied { value, provenance })
}
ClientInboundDecision::Refused(refusal) => {
let reason = refusal.reason();
let (aggregate, value) = refusal.into_parts();
state.aggregate = Some(aggregate);
Ok(RemoteParticipantInbound::Refused {
value,
reason,
provenance,
})
}
}
}
}
}
pub(super) fn take_aggregate<S>(
state: &mut RemoteParticipantState<S>,
) -> Result<ClientParticipantAggregate, RemoteParticipantError> {
state
.aggregate
.take()
.ok_or(RemoteParticipantError::StateUnavailable)
}
pub(super) fn persist<S: ParticipantResumeStore>(
store: &mut S,
aggregate: &ClientParticipantAggregate,
) -> Result<(), RemoteParticipantError> {
let record = aggregate
.resume_record()
.map_err(RemoteParticipantError::ResumeEncode)?;
store
.persist(&record.encode_canonical())
.map_err(RemoteParticipantError::Storage)
}
fn record_operation_transport_fate<S: ParticipantResumeStore>(
state: &mut RemoteParticipantState<S>,
aggregate: ClientParticipantAggregate,
correlation: ClientResponseCorrelation,
) -> RemoteOperationTransportFate {
match record_expected_operation_fate(
aggregate,
correlation,
ExpectedOperationTransportFate::ResponseUnavailable,
) {
liminal_protocol::client::ExpectedOperationFateDecision::Recorded {
aggregate,
request,
..
} => {
state.aggregate = Some(aggregate);
RemoteOperationTransportFate::Recorded { request }
}
liminal_protocol::client::ExpectedOperationFateDecision::Refused {
aggregate,
correlation,
reason: ExpectedOperationFateRefusalReason::DetachUsesReplayFate,
..
} => match liminal_protocol::client::transport_fate(
aggregate,
correlation,
liminal_protocol::client::DetachTransportFate::ResponseUnavailable,
) {
liminal_protocol::client::DetachTransportFateDecision::Parked(applied) => {
state.aggregate = Some(applied.into_aggregate());
RemoteOperationTransportFate::DetachParked
}
liminal_protocol::client::DetachTransportFateDecision::Refused(refusal) => {
let (aggregate, (correlation, _)) = refusal.into_parts();
state.aggregate = Some(aggregate);
state.correlation = Some(correlation);
RemoteOperationTransportFate::Refused {
reason: ExpectedOperationFateRefusalReason::DetachUsesReplayFate,
}
}
},
liminal_protocol::client::ExpectedOperationFateDecision::Refused {
aggregate,
correlation,
reason,
..
} => {
state.aggregate = Some(aggregate);
state.correlation = Some(correlation);
RemoteOperationTransportFate::Refused { reason }
}
}
}
pub(super) fn record_connection_fate<S: ParticipantResumeStore>(
state: &mut RemoteParticipantState<S>,
) -> Result<RemoteReconnectPermitOutcome, RemoteParticipantError> {
let aggregate = take_aggregate(state)?;
let (aggregate, outcome) = match record_transport_fate(
aggregate,
liminal_protocol::client::EstablishedConnectionTransportFate::Lost,
) {
ReconnectPermitDecision::Permitted {
aggregate,
permit,
result,
} => (
aggregate,
RemoteReconnectPermitOutcome::Permitted {
permit: RemoteReconnectPermit { permit },
result,
},
),
ReconnectPermitDecision::Refused(refusal) => {
let reason = refusal.reason();
let result = refusal.result();
let (aggregate, _) = refusal.into_parts();
(
aggregate,
RemoteReconnectPermitOutcome::Refused { reason, result },
)
}
};
persist(&mut state.store, &aggregate)?;
state.aggregate = Some(aggregate);
Ok(outcome)
}
#[cfg(test)]
mod tests;