use std::collections::{BTreeMap, HashMap};
use std::fmt;
use std::future::Future;
use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use futures_util::{SinkExt, StreamExt};
use serde_json::value::RawValue;
use tokio::sync::{Semaphore, broadcast, mpsc, oneshot, watch};
use crate::decode::{DecodeOptions, Repair};
use crate::engine::{
Answer, BootState, CallFailure, CallOptions, CallToken, ClockSample, CloseReason, Engine,
Input, Instant, MessageStore, Output, ProtocolViolation, Timer,
};
use crate::message::{ActionName, Confirmed, DecodeRequest, NoResponse, Request, Unconfirmed};
use crate::rpc::CallError;
use crate::types::{Identity, MessageId};
use crate::version::Version;
use super::TransportError;
use super::ws::{CloseCode, CloseFrame, Message, WebSocket, WsError};
pub type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
const MAX_CONCURRENT_HANDLERS: usize = 32;
pub use crate::engine::IncomingRequest;
pub trait Handler: Send + Sync + 'static {
fn on_request(
&self,
ctx: Ctx,
request: IncomingRequest,
) -> BoxFuture<'_, Result<Box<RawValue>, CallError>>;
fn on_event(&self, _ctx: &Ctx, _event: &Event) {}
}
pub struct NotImplemented;
impl Handler for NotImplemented {
fn on_request(
&self,
_ctx: Ctx,
request: IncomingRequest,
) -> BoxFuture<'_, Result<Box<RawValue>, CallError>> {
Box::pin(async move { Err(CallError::not_implemented(&request.action)) })
}
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum Event {
Connected {
version: Version,
compressed: bool,
},
Disconnected {
reason: String,
peer_close: Option<CloseFrame>,
},
BootState(BootState),
ClockSample(ClockSample),
Violation(ProtocolViolation),
ResultRejected {
id: MessageId,
error: CallError,
},
NetworkProfileSelected {
configuration_slot: i32,
url: String,
},
Reconnecting {
attempt: u32,
delay: Duration,
},
Repaired {
action: String,
id: MessageId,
repairs: Vec<Repair>,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct SessionState {
pub connected: bool,
pub version: Version,
pub boot: BootState,
pub queued: usize,
}
pub(crate) type CallAnswer = Result<(Answer, Option<MessageId>), CallFailure>;
pub(crate) enum Command {
Call {
action: String,
payload: Box<RawValue>,
options: CallOptions,
reply: oneshot::Sender<CallAnswer>,
},
Respond {
id: MessageId,
result: Result<Box<RawValue>, CallError>,
},
RejectResult {
id: MessageId,
error: CallError,
},
Shutdown {
deadline: Duration,
},
}
#[derive(Clone, Default)]
pub struct SessionContext(Option<Arc<dyn core::any::Any + Send + Sync>>);
impl SessionContext {
#[must_use]
pub const fn empty() -> Self {
Self(None)
}
#[must_use]
pub fn new<T: core::any::Any + Send + Sync>(value: T) -> Self {
Self(Some(Arc::new(value)))
}
#[must_use]
pub fn get<T: core::any::Any + Send + Sync>(&self) -> Option<&T> {
self.0.as_ref()?.downcast_ref::<T>()
}
#[must_use]
pub const fn is_empty(&self) -> bool {
self.0.is_none()
}
}
impl fmt::Debug for SessionContext {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_tuple("SessionContext")
.field(&if self.is_empty() { "empty" } else { "set" })
.finish()
}
}
pub(crate) struct Shared {
pub identity: Identity,
pub remote: Option<SocketAddr>,
pub session: SessionContext,
pub decode: DecodeOptions,
pub commands: mpsc::Sender<Command>,
pub events: broadcast::Sender<Event>,
pub state: watch::Receiver<SessionState>,
}
impl Shared {
pub(crate) fn report_repairs(&self, action: &str, id: &MessageId, repairs: Vec<Repair>) {
if repairs.is_empty() {
return;
}
#[cfg(feature = "tracing")]
for repair in &repairs {
tracing::debug!(
identity = %self.identity,
action,
pointer = %repair.pointer,
kind = repair.kind.as_str(),
"lenient decode repaired a member"
);
}
let _ = self.events.send(Event::Repaired {
action: action.to_owned(),
id: id.clone(),
repairs,
});
}
}
#[derive(Clone)]
pub struct Ctx {
shared: Arc<Shared>,
version: Version,
id: MessageId,
}
impl Ctx {
pub(crate) fn new(shared: Arc<Shared>, version: Version, id: MessageId) -> Self {
Self {
shared,
version,
id,
}
}
#[must_use]
pub fn identity(&self) -> &Identity {
&self.shared.identity
}
#[must_use]
pub fn remote_addr(&self) -> Option<SocketAddr> {
self.shared.remote
}
#[must_use]
pub fn session<T: core::any::Any + Send + Sync>(&self) -> Option<&T> {
self.shared.session.get::<T>()
}
#[must_use]
pub fn version(&self) -> Version {
self.version
}
#[must_use]
pub fn message_id(&self) -> &MessageId {
&self.id
}
#[must_use]
pub fn decode_options(&self) -> &DecodeOptions {
&self.shared.decode
}
pub fn decode<U: DecodeRequest>(&self, request: &IncomingRequest) -> Result<U, CallError> {
let action = U::Action::from_wire(&request.action)
.ok_or_else(|| CallError::not_implemented(&request.action))?;
let decoded = U::decode_reporting(action, &request.payload, &self.shared.decode)?;
self.shared
.report_repairs(&request.action, &request.id, decoded.repairs);
Ok(decoded.value)
}
#[must_use]
pub fn handle(&self) -> Handle {
Handle {
shared: self.shared.clone(),
}
}
pub fn reply<R: serde::Serialize>(&self, response: &R) -> Result<Box<RawValue>, CallError> {
serde_json::value::to_raw_value(response)
.map_err(|error| CallError::internal(format!("response is not serializable: {error}")))
}
}
#[derive(Clone)]
pub struct Handle {
shared: Arc<Shared>,
}
impl Handle {
pub(crate) fn new(shared: Arc<Shared>) -> Self {
Self { shared }
}
#[must_use]
pub fn session<T: core::any::Any + Send + Sync>(&self) -> Option<&T> {
self.shared.session.get::<T>()
}
#[must_use]
pub fn identity(&self) -> &Identity {
&self.shared.identity
}
#[must_use]
pub fn state(&self) -> SessionState {
*self.shared.state.borrow()
}
#[must_use]
pub fn events(&self) -> broadcast::Receiver<Event> {
self.shared.events.subscribe()
}
pub async fn wait_ready(&self) -> bool {
let mut state = self.shared.state.clone();
loop {
if state.borrow().connected && state.borrow().boot.allows_traffic() {
return true;
}
if state.changed().await.is_err() {
return false;
}
}
}
pub async fn call<R>(&self, request: R) -> Result<R::Response, CallFailure>
where
R: Request + Confirmed,
{
self.call_with(request, CallOptions::default()).await
}
pub async fn call_with<R>(
&self,
request: R,
options: CallOptions,
) -> Result<R::Response, CallFailure>
where
R: Request + Confirmed,
{
let payload = serde_json::value::to_raw_value(&request).map_err(|error| {
CallFailure::Rejected(CallError::internal(format!(
"request is not serializable: {error}"
)))
})?;
let (answer, id) = self
.call_raw_with_id(R::ACTION.as_str(), payload, options)
.await?;
let Some(raw) = answer.into_payload() else {
return Err(CallFailure::Rejected(CallError::internal(
"a CALL was completed as a SEND",
)));
};
match crate::decode::decode_payload_reporting::<R::Response>(&raw, &self.shared.decode) {
Ok(decoded) => {
if let Some(id) = &id {
self.shared
.report_repairs(R::ACTION.as_str(), id, decoded.repairs);
}
Ok(decoded.value)
}
Err(error) => {
let error = CallError::from(error);
if let Some(id) = id {
let _ = self
.shared
.commands
.send(Command::RejectResult {
id,
error: error.clone(),
})
.await;
}
Err(CallFailure::Rejected(error))
}
}
}
pub async fn send<R>(&self, request: R) -> Result<(), CallFailure>
where
R: Request<Response = NoResponse> + Unconfirmed,
{
self.send_with(request, CallOptions::default()).await
}
pub async fn send_with<R>(&self, request: R, options: CallOptions) -> Result<(), CallFailure>
where
R: Request<Response = NoResponse> + Unconfirmed,
{
let payload = serde_json::value::to_raw_value(&request).map_err(|error| {
CallFailure::Rejected(CallError::internal(format!(
"request is not serializable: {error}"
)))
})?;
self.call_raw_with_id(R::ACTION.as_str(), payload, options)
.await
.map(|_| ())
}
pub async fn call_raw(
&self,
action: &str,
payload: Box<RawValue>,
options: CallOptions,
) -> Result<Answer, CallFailure> {
self.call_raw_with_id(action, payload, options)
.await
.map(|(answer, _)| answer)
}
async fn call_raw_with_id(
&self,
action: &str,
payload: Box<RawValue>,
options: CallOptions,
) -> Result<(Answer, Option<MessageId>), CallFailure> {
let (reply, answer) = oneshot::channel();
let command = Command::Call {
action: action.to_owned(),
payload,
options,
reply,
};
if self.shared.commands.send(command).await.is_err() {
return Err(CallFailure::Disconnected);
}
let outcome = answer.await.unwrap_or(Err(CallFailure::Disconnected))?;
Ok(outcome)
}
pub async fn shutdown(&self, deadline: Duration) {
let _ = self
.shared
.commands
.send(Command::Shutdown { deadline })
.await;
let mut state = self.shared.state.clone();
while state.borrow().connected {
if state.changed().await.is_err() {
break;
}
}
}
}
pub(crate) enum Ended {
Disconnected(String),
Closed(CloseReason),
}
pub(crate) struct Driver<S: MessageStore> {
pub engine: Engine<S>,
pub shared: Arc<Shared>,
pub handler: Arc<dyn Handler>,
pub state: watch::Sender<SessionState>,
pub started: tokio::time::Instant,
pending: HashMap<CallToken, oneshot::Sender<CallAnswer>>,
timers: BTreeMap<Timer, Instant>,
slots: Arc<Semaphore>,
peer_close: Option<CloseFrame>,
pub store_poll: Option<Duration>,
}
#[derive(Clone, Copy, Debug)]
pub struct Keepalive {
pub interval: Option<Duration>,
pub timeout: Duration,
}
impl Default for Keepalive {
fn default() -> Self {
Self {
interval: Some(Duration::from_secs(60)),
timeout: Duration::from_secs(30),
}
}
}
impl Keepalive {
#[must_use]
pub fn every(interval: Duration) -> Self {
Self {
interval: Some(interval),
..Self::default()
}
}
#[must_use]
pub const fn disabled() -> Self {
Self {
interval: None,
timeout: Duration::ZERO,
}
}
#[must_use]
pub const fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
}
impl<S: MessageStore> Driver<S> {
pub fn new(
engine: Engine<S>,
shared: Arc<Shared>,
handler: Arc<dyn Handler>,
state: watch::Sender<SessionState>,
) -> Self {
Self {
engine,
shared,
handler,
state,
started: tokio::time::Instant::now(),
pending: HashMap::new(),
timers: BTreeMap::new(),
slots: Arc::new(Semaphore::new(MAX_CONCURRENT_HANDLERS)),
peer_close: None,
store_poll: None,
}
}
fn adopt_pending(&mut self) {
if let Err(error) = self.engine.adopt_pending(self.now()) {
let _ = self
.shared
.events
.send(Event::Violation(ProtocolViolation::StoreFailure { error }));
}
}
pub(crate) fn now(&self) -> Instant {
Instant::from_millis(u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX))
}
pub async fn run<St>(
&mut self,
socket: WebSocket<St>,
commands: &mut mpsc::Receiver<Command>,
version: Version,
keepalive: Keepalive,
) -> Ended
where
St: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send,
{
let compressed = socket.codec().is_compressed();
let (mut sink, mut stream) = socket.split();
self.peer_close = None;
self.engine.handle(self.now(), Input::Connected { version });
if self.store_poll.is_some() {
self.adopt_pending();
}
self.publish(true, version);
let _ = self.shared.events.send(Event::Connected {
version,
compressed,
});
let mut ping = keepalive.interval.map(|interval| {
let mut timer = tokio::time::interval(interval);
timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
timer
});
let mut store_poll = self.store_poll.map(|interval| {
let mut timer =
tokio::time::interval_at(tokio::time::Instant::now() + interval, interval);
timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
timer
});
let mut pong_due: Option<tokio::time::Instant> = None;
let ended = loop {
match self.flush(&mut sink).await {
Ok(Some(outcome)) => break outcome,
Ok(None) => {}
Err(error) => break Ended::Disconnected(error.to_string()),
}
let sleep = self.next_deadline();
tokio::select! {
command = commands.recv() => {
match command {
Some(command) => self.apply(command),
None => break Ended::Closed(CloseReason::Requested),
}
}
message = stream.next() => {
if matches!(message, Some(Ok(Message::Pong(_)))) {
pong_due = None;
}
if let Some(ended) = self.on_message(message, &mut sink).await {
break ended;
}
}
() = sleep => {
let now = self.now();
self.engine.handle(now, Input::Timeout);
self.timers.retain(|_, at| *at > now);
}
_ = async { ping.as_mut().expect("checked").tick().await }, if ping.is_some() => {
if pong_due.is_none() {
pong_due = Some(tokio::time::Instant::now() + keepalive.timeout);
}
if let Err(error) = sink.send(Message::Ping(Vec::new())).await {
break Ended::Disconnected(error.to_string());
}
}
_ = async { store_poll.as_mut().expect("checked").tick().await },
if store_poll.is_some() =>
{
self.adopt_pending();
}
() = async { tokio::time::sleep_until(pong_due.expect("checked")).await },
if pong_due.is_some() =>
{
break Ended::Disconnected(format!(
"no pong within {:?} of a WebSocket ping",
keepalive.timeout
));
}
}
};
self.engine.handle(self.now(), Input::Disconnected);
let _ = self.flush(&mut sink).await;
let reason = match &ended {
Ended::Disconnected(reason) => reason.clone(),
Ended::Closed(why) => format!("{why:?}"),
};
self.publish(false, version);
let _ = self.shared.events.send(Event::Disconnected {
reason,
peer_close: self.peer_close.take(),
});
if matches!(ended, Ended::Closed(_)) {
let _ = sink
.send(Message::Close(Some(CloseFrame::new(CloseCode::NORMAL, ""))))
.await;
}
let _ = sink.close().await;
ended
}
async fn on_message<Si>(
&mut self,
message: Option<Result<Message, WsError>>,
sink: &mut Si,
) -> Option<Ended>
where
Si: futures_util::Sink<Message, Error = WsError> + Unpin,
{
match message {
Some(Ok(Message::Text(text))) => {
self.engine.handle(self.now(), Input::Received(&text));
None
}
Some(Ok(Message::Ping(payload))) => {
sink.send(Message::Pong(payload))
.await
.err()
.map(|error| Ended::Disconnected(error.to_string()))
}
Some(Ok(Message::Pong(_) | Message::Binary(_))) => None,
Some(Ok(Message::Close(frame))) => {
let reason = frame.as_ref().map_or_else(
|| "peer closed".to_owned(),
|frame| format!("peer closed: {} {}", frame.code, frame.reason),
);
let echo = frame.as_ref().map(|frame| CloseFrame::new(frame.code, ""));
self.peer_close = frame;
let _ = sink.send(Message::Close(echo)).await;
Some(Ended::Disconnected(reason))
}
Some(Err(error)) => {
if let Some(code) = error.close_code() {
let _ = sink
.send(Message::Close(Some(CloseFrame::new(
code,
error.to_string(),
))))
.await;
}
Some(Ended::Disconnected(error.to_string()))
}
None => Some(Ended::Disconnected("stream ended".to_owned())),
}
}
pub fn abandon(&mut self, why: &CallFailure) {
for (_, reply) in self.pending.drain() {
let _ = reply.send(Err(why.clone()));
}
}
fn apply(&mut self, command: Command) {
match command {
Command::Call {
action,
payload,
options,
reply,
} => match self.engine.call_with(self.now(), &action, payload, options) {
Ok(token) => {
self.pending.insert(token, reply);
}
Err(error) => {
let _ = reply.send(Err(CallFailure::Rejected(CallError::internal(
error.to_string(),
))));
}
},
Command::RejectResult { id, error } => {
let _ = self.engine.reject_result(self.now(), &id, error);
}
Command::Respond { id, result } => {
let now = self.now();
let outcome = match result {
Ok(payload) => self.engine.respond(now, &id, &payload),
Err(error) => self.engine.respond_error(now, &id, error),
};
if let Err(error) = outcome {
let _ = self.shared.events.send(Event::Violation(
ProtocolViolation::UnexpectedResponse {
id: match error {
crate::engine::EngineError::NoSuchRequest(id) => id,
_ => id,
},
},
));
}
}
Command::Shutdown { deadline } => {
let now = self.now();
self.engine.shutdown(now, now.saturating_add(deadline));
}
}
}
async fn flush<Si>(&mut self, sink: &mut Si) -> Result<Option<Ended>, TransportError>
where
Si: futures_util::Sink<Message, Error = WsError> + Unpin,
{
let mut close = None;
let mut wrote = false;
while let Some(output) = self.engine.poll_output() {
match output {
Output::Transmit(text) => {
sink.feed(Message::Text(text)).await?;
wrote = true;
}
Output::Request(request) => self.dispatch(request),
Output::Outcome(outcome) => {
if let Some(reply) = self.pending.remove(&outcome.token) {
let id = outcome.id;
let _ = reply.send(outcome.result.map(|answer| (answer, id)));
}
}
Output::SetTimer { timer, at } => {
self.timers.insert(timer, at);
}
Output::ClearTimer(timer) => {
self.timers.remove(&timer);
}
Output::BootState(state) => {
let _ = self.shared.events.send(Event::BootState(state));
}
Output::ClockSample(sample) => {
let _ = self.shared.events.send(Event::ClockSample(sample));
}
Output::ResultRejected { id, error } => {
let _ = self.shared.events.send(Event::ResultRejected { id, error });
}
Output::Violation(violation) => {
let _ = self.shared.events.send(Event::Violation(violation));
}
Output::Close(reason) => close = Some(Ended::Closed(reason)),
}
}
if wrote {
sink.flush().await?;
}
let version = self.engine.version();
self.publish(self.engine.is_connected(), version);
Ok(close)
}
fn dispatch(&self, request: IncomingRequest) {
let handler = self.handler.clone();
let shared = self.shared.clone();
let slots = self.slots.clone();
let ctx = Ctx::new(shared.clone(), self.engine.version(), request.id.clone());
let is_send = request.kind == crate::message::MessageKind::Send;
let id = request.id.clone();
tokio::spawn(async move {
let Ok(_permit) = slots.acquire().await else {
return;
};
let result = handler.on_request(ctx, request).await;
if is_send {
return;
}
let _ = shared.commands.send(Command::Respond { id, result }).await;
});
}
fn publish(&self, connected: bool, version: Version) {
let next = SessionState {
connected,
version,
boot: self.engine.boot_state(),
queued: self.engine.queued(),
};
self.state.send_if_modified(|state| {
if *state == next {
false
} else {
*state = next;
true
}
});
}
fn next_deadline(&self) -> impl Future<Output = ()> + use<S> {
let next = self.timers.values().min().copied();
let started = self.started;
async move {
match next {
Some(at) => {
tokio::time::sleep_until(started + Duration::from_millis(at.as_millis())).await;
}
None => std::future::pending::<()>().await,
}
}
}
}