use std::collections::VecDeque;
use std::pin::Pin;
use std::time::Duration;
use bytes::Bytes;
use futures::SinkExt;
use rand::Rng;
use tokio::sync::{mpsc, oneshot, watch};
use tokio_util::sync::CancellationToken;
use crate::Protocol;
use crate::proto::resumption::{Command, ResumptionCodec, ResumptionMessage};
use crate::sync::{ChannelCommand, ChannelSink, ChannelStream};
use crate::util::{DecodedSink, EncodedStream};
#[derive(Debug, Eq, PartialEq, Clone)]
pub enum RequestError {
ResponsePending,
ChannelClosed,
ChannelTimeout,
}
impl std::fmt::Display for RequestError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "RequestError::{:?}", self)
}
}
#[derive(Debug, Eq, PartialEq, Clone)]
pub enum ResponseError {
ChannelNotFound,
ChannelClosed,
ChannelTimeout,
ResumeFailed,
}
impl std::fmt::Display for ResponseError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "ResponseError::{:?}", self)
}
}
pub struct ResumptionClient<T>
where
T: Protocol + 'static,
{
stream: ChannelStream<ResumptionMessage, EncodedStream<T>>,
sink: ChannelSink<ResumptionMessage>,
handle: ResumptionHandle,
}
impl<T> ResumptionClient<T>
where
T: Protocol + Send,
T::Message: Send,
T::SinkHalf: Send,
T::StreamHalf: Send,
T::Codec: Send,
{
pub fn new(next: T) -> Self {
Self::new_with(next, rand::rng().next_u64())
}
fn new_with(next: T, id: u64) -> Self {
let (stream, sink) = next.into_split();
let stream = EncodedStream::new(stream, T::codec());
let sink = Box::pin(DecodedSink::<T>::new(sink, T::codec()));
let (sink_tx_ctrl, ctrl_rx_sink) = mpsc::channel(1);
let (ctrl_tx_stream, stream_rx_ctrl) = mpsc::unbounded_channel();
let (request_tx, request_rx) = mpsc::channel(5);
let (stream_next_tx, stream_next_rx) = watch::channel(true);
let _ = ctrl_tx_stream.send(ResumptionMessage::from(Command::Start(id)));
let token = CancellationToken::new();
let controller = Controller {
rx: ctrl_rx_sink,
tx: Some(ctrl_tx_stream),
request_rx,
stream_open: token.clone(),
stream_next_tx,
id,
sink,
sink_closed: false,
pending_sink: VecDeque::new(),
pending_resume: None,
};
controller.into_background();
Self {
stream: ChannelStream::new(stream_rx_ctrl, stream, Some(stream_next_rx), Some(token)),
sink: ChannelSink::new(sink_tx_ctrl),
handle: ResumptionHandle::new(request_tx),
}
}
pub fn handle(&self) -> &ResumptionHandle {
&self.handle
}
}
impl<T: Protocol> Protocol for ResumptionClient<T> {
type Message = ResumptionMessage;
type Codec = ResumptionCodec;
type StreamHalf = ChannelStream<ResumptionMessage, EncodedStream<T>>;
type SinkHalf = ChannelSink<ResumptionMessage>;
fn into_split(self) -> (Self::StreamHalf, Self::SinkHalf) {
(self.stream, self.sink)
}
fn codec() -> Self::Codec {
ResumptionCodec::new()
}
}
pub struct ResumptionHandle {
request_tx: mpsc::Sender<oneshot::Sender<Result<u64, ResponseError>>>,
response_rx: Option<oneshot::Receiver<Result<u64, ResponseError>>>,
}
impl ResumptionHandle {
fn new(request_tx: mpsc::Sender<oneshot::Sender<Result<u64, ResponseError>>>) -> Self {
Self {
request_tx,
response_rx: None,
}
}
pub async fn resume_request(&mut self, timeout: Option<Duration>) -> Result<(), RequestError> {
if self.response_rx.is_some() {
return Err(RequestError::ResponsePending);
}
let (response_tx, response_rx) = oneshot::channel();
let timeout_dur = timeout.unwrap_or(Duration::MAX);
log::debug!("Sending resume request to server");
let result = match self.request_tx.send_timeout(response_tx, timeout_dur).await {
Ok(()) => {
self.response_rx = Some(response_rx);
Ok(())
}
Err(mpsc::error::SendTimeoutError::Closed(_)) => Err(RequestError::ChannelClosed),
Err(mpsc::error::SendTimeoutError::Timeout(_)) => Err(RequestError::ChannelTimeout),
};
log::debug!("Resume request result {result:?}");
result
}
pub async fn resume_response(
&mut self,
timeout: Option<Duration>,
) -> Result<u64, ResponseError> {
let Some(response_rx) = self.response_rx.as_mut() else {
return Err(ResponseError::ChannelNotFound);
};
let timeout_dur = timeout.unwrap_or(Duration::MAX);
log::debug!("Waiting for resume response");
let result = match tokio::time::timeout(timeout_dur, response_rx).await {
Ok(recv_result) => match recv_result {
Ok(resume_result) => resume_result,
Err(_) => Err(ResponseError::ChannelClosed),
},
Err(_) => Err(ResponseError::ChannelTimeout),
};
log::debug!("Resume response result {result:?}");
result
}
}
impl Clone for ResumptionHandle {
fn clone(&self) -> Self {
Self {
request_tx: self.request_tx.clone(),
response_rx: None,
}
}
}
struct Controller<T: Protocol> {
rx: mpsc::Receiver<ChannelCommand<ResumptionMessage>>,
tx: Option<mpsc::UnboundedSender<ResumptionMessage>>,
request_rx: mpsc::Receiver<oneshot::Sender<Result<u64, ResponseError>>>,
stream_open: CancellationToken,
stream_next_tx: watch::Sender<bool>,
id: u64,
sink: Pin<Box<DecodedSink<T>>>,
sink_closed: bool,
pending_sink: VecDeque<Bytes>,
pending_resume: Option<oneshot::Sender<Result<u64, ResponseError>>>,
}
#[derive(Debug, Eq, PartialEq, Clone)]
enum ControllerError {
StreamSendFailed,
StreamSenderDropped,
SinkSendFailed,
SinkFlushFailed,
SinkCloseFailed,
}
impl<T> Controller<T>
where
T: Protocol + Send + 'static,
T::Message: Send,
T::SinkHalf: Send,
T::StreamHalf: Send,
T::Codec: Send,
{
fn into_background(self) {
tokio::spawn(async move {
let id = self.id;
let result = self.run().await;
log::debug!("Controller({id}) finished: {result:?}");
result
});
}
async fn run(mut self) -> Result<(), ControllerError> {
loop {
log::trace!("Controller({:?}) select input", self.id);
tokio::select! {
biased;
result = async {
if let Some(chunk) = self.pending_sink.front() {
self.sink.send(chunk.clone()).await
} else {
std::future::pending().await
}
}, if !self.pending_sink.is_empty() => {
log::trace!("Controller({:?}) selected sink_send: {result:?}", self.id);
result.map_err(|_| ControllerError::SinkSendFailed)?;
self.pending_sink.pop_front();
}
result = self.request_rx.recv(),
if !(self.request_rx.is_closed() && self.request_rx.is_empty()) => {
log::trace!("Controller({:?}) selected resume_recv: {:?}", self.id, result.as_ref().map(|_| ()));
if let Some(reply_tx) = result {
self.resume(reply_tx)?;
} else {
log::trace!("Controller({:?}) resume_recv disconnected", self.id);
}
}
result = self.rx.recv(),
if !(self.rx.is_closed() && self.rx.is_empty())
&& (self.pending_sink.is_empty() || self.pending_resume.is_some()) => {
log::trace!("Controller({:?}) selected sink_recv: {result:?}", self.id);
if let Some(cmd) = result {
self.process_cmd(cmd).await?;
} else {
log::trace!("Controller({:?}) sink_recv disconnected", self.id);
}
}
_ = self.stream_open.cancelled(),
if self.tx.is_some() => {
self.tx.take();
}
else => return Ok(())
}
}
}
fn send_stream(&mut self, cmd: Command) -> Result<(), ControllerError> {
if let Some(sender) = self.tx.as_mut() {
sender
.send(ResumptionMessage::from(cmd))
.map_err(|_| ControllerError::StreamSendFailed)
} else {
Err(ControllerError::StreamSenderDropped)
}
}
fn set_stream_next(&mut self, new_val: bool) {
self.stream_next_tx.send_if_modified(|cur_val| {
if *cur_val != new_val {
*cur_val = new_val;
true } else {
false }
});
}
fn resume(
&mut self,
reply_tx: oneshot::Sender<Result<u64, ResponseError>>,
) -> Result<(), ControllerError> {
self.set_stream_next(false);
self.pending_resume = Some(reply_tx);
self.send_stream(Command::Resume(self.id))
}
async fn flush_sink(&mut self) -> Result<(), ControllerError> {
while let Some(chunk) = self.pending_sink.pop_front() {
self.sink
.send(chunk)
.await
.map_err(|_| ControllerError::SinkSendFailed)?;
}
self.sink
.flush()
.await
.map_err(|_| ControllerError::SinkFlushFailed)
}
async fn close_sink(&mut self) -> Result<(), ControllerError> {
let _ = self.flush_sink().await;
self.sink_closed = true;
self.sink
.close()
.await
.map_err(|_| ControllerError::SinkCloseFailed)
}
async fn process_cmd(
&mut self,
cmd: ChannelCommand<ResumptionMessage>,
) -> Result<(), ControllerError> {
match cmd {
ChannelCommand::Message(msg) => self.process_msg(msg),
ChannelCommand::ExplicitFlush => self.flush_sink().await,
ChannelCommand::ExplicitClose => self.close_sink().await,
}
}
fn process_msg(&mut self, msg: ResumptionMessage) -> Result<(), ControllerError> {
match msg.command {
Command::Start(_) => {}
Command::Resume(_) => {}
Command::ResumeOk(id) => {
self.set_stream_next(true);
if let Some(sender) = self.pending_resume.take() {
let _ = sender.send(Ok(id));
}
}
Command::ResumeErr(_id) => {
if let Some(sender) = self.pending_resume.take() {
let _ = sender.send(Err(ResponseError::ResumeFailed));
}
}
Command::Forward(payload) => self.pending_sink.push_back(payload.chunk),
}
Ok(())
}
}
#[cfg(test)]
pub mod tests {
use std::future::poll_fn;
use std::io;
use std::task::{Context, Poll, Waker};
use bytes::Bytes;
use futures::{FutureExt, SinkExt, StreamExt};
use tokio::task::yield_now;
use tokio_test::io::Builder;
use crate::proto::resumption::ResumptionClient;
use crate::proto::resumption::client::ResponseError;
use crate::proto::resumption::message::{Command, ResumptionMessage};
use crate::test::{TestCodec, TestIoProtocol};
const ID: u64 = 1;
fn encode_io(data: &'static str) -> Bytes {
TestCodec::encode_str(data)
}
fn forward_msg(data: &'static str) -> ResumptionMessage {
ResumptionMessage::from(Command::Forward(encode_io(data).into()))
}
async fn poll_next(
client: &mut ResumptionClient<TestIoProtocol>,
) -> Poll<Option<ResumptionMessage>> {
let mut fut = client.stream.next().boxed();
let mut cx = Context::from_waker(Waker::noop());
let mut f = poll_fn(move |cx| fut.poll_unpin(cx));
yield_now().await;
f.poll_unpin(&mut cx)
}
#[tokio::test]
async fn stream_data_before_id() {
let io = Builder::new().read(b"a").read(b"b").read(b"c").build();
let mut client = ResumptionClient::new(TestIoProtocol::from_mock(io));
let msg = client.stream.next().await.unwrap();
assert!(matches!(msg.command, Command::Start(_)));
yield_now().await;
for char in ["a", "b", "c"] {
assert_eq!(client.stream.next().await, Some(encode_io(char).into()));
yield_now().await;
}
assert_eq!(client.stream.next().await, None);
}
#[tokio::test]
async fn stream_data_after_id() {
let io = Builder::new().read(b"a").read(b"b").read(b"c").build();
let mut client = ResumptionClient::new(TestIoProtocol::from_mock(io));
let msg = client.stream.next().await.unwrap();
assert!(matches!(msg.command, Command::Start(_)));
yield_now().await;
for char in ["a", "b", "c"] {
assert_eq!(client.stream.next().await, Some(encode_io(char).into()));
yield_now().await;
}
assert_eq!(client.stream.next().await, None);
}
#[tokio::test]
async fn stream_data_after_sink_close() {
let io = Builder::new().read(b"a").read(b"b").read(b"c").build();
let mut client = ResumptionClient::new(TestIoProtocol::from_mock(io));
let msg = client.stream.next().await.unwrap();
assert!(matches!(msg.command, Command::Start(_)));
yield_now().await;
client.sink.close().await.unwrap();
yield_now().await;
for char in ["a", "b", "c"] {
assert_eq!(client.stream.next().await, Some(encode_io(char).into()));
yield_now().await;
}
assert_eq!(client.stream.next().await, None);
}
#[tokio::test]
async fn sink_data_before_id() {
let io = Builder::new().write(b"a").write(b"b").write(b"c").build();
let mut client = ResumptionClient::new(TestIoProtocol::from_mock(io));
for char in ["a", "b", "c"] {
assert!(client.sink.send(forward_msg(char)).await.is_ok());
yield_now().await;
}
}
#[tokio::test]
async fn sink_data_after_id() {
let io = Builder::new().write(b"a").write(b"b").write(b"c").build();
let mut client = ResumptionClient::new(TestIoProtocol::from_mock(io));
for char in ["a", "b", "c"] {
assert!(client.sink.send(forward_msg(char)).await.is_ok());
yield_now().await;
}
}
#[tokio::test]
async fn sink_data_after_stream_eof() {
let io = Builder::new()
.read_error(io::ErrorKind::UnexpectedEof.into())
.write(b"a")
.write(b"b")
.write(b"c")
.build();
let mut client = ResumptionClient::new(TestIoProtocol::from_mock(io));
let msg = client.stream.next().await.unwrap();
assert!(matches!(msg.command, Command::Start(_)));
yield_now().await;
assert_eq!(client.stream.next().await, None);
yield_now().await;
for char in ["a", "b", "c"] {
assert!(client.sink.send(forward_msg(char)).await.is_ok());
yield_now().await;
}
}
async fn resume_request_response(
id: u64,
simulate_resume_success: bool,
) -> (
ResumptionClient<TestIoProtocol>,
Poll<Result<u64, ResponseError>>,
) {
let io = Builder::new().read(b"a").read(b"b").build();
let proto = TestIoProtocol::from_mock(io);
let mut client = ResumptionClient::new_with(proto, id);
let mut handle = client.handle().clone();
let msg = client.stream.next().await.unwrap();
assert!(matches!(msg.command, Command::Start(_)));
assert_eq!(client.stream.next().await, Some(encode_io("a").into()));
assert_eq!(handle.resume_request(None).await, Ok(()));
let mut fut = handle.resume_response(None).boxed();
let mut cx = Context::from_waker(Waker::noop());
let mut f = poll_fn(move |cx| fut.poll_unpin(cx));
yield_now().await;
assert_eq!(f.poll_unpin(&mut cx), Poll::Pending);
let msg = client.stream.next().await.unwrap();
assert_eq!(msg.command, Command::Resume(id));
let response = if simulate_resume_success {
Command::ResumeOk(id)
} else {
Command::ResumeErr(id)
};
client.sink.send(response.into()).await.unwrap();
yield_now().await;
let result = f.poll_unpin(&mut cx);
(client, result)
}
#[tokio::test]
async fn resume_request_response_ok() {
let (mut client, result) = resume_request_response(ID, true).await;
assert_eq!(result, Poll::Ready(Ok(ID)));
assert_eq!(client.stream.next().await, Some(encode_io("b").into()));
assert_eq!(client.stream.next().await, None);
}
#[tokio::test]
async fn resume_request_response_err() {
let (mut client, result) = resume_request_response(ID, false).await;
assert_eq!(result, Poll::Ready(Err(ResponseError::ResumeFailed)));
assert_eq!(poll_next(&mut client).await, Poll::Pending);
}
}