use std::{
borrow::BorrowMut,
fmt::Debug,
future::Future,
io,
marker::PhantomData,
ops::Range,
pin::Pin,
sync::Arc,
task::{Context, Poll, ready},
};
use n0_error::{e, stack_error};
use n0_future::future::Boxed as BoxFuture;
use noq::ConnectionError;
pub use noq::VarInt;
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use smallvec::{SmallVec, smallvec};
use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
use tracing::{Instrument, debug, trace, warn};
#[cfg(feature = "iroh")]
pub(crate) mod iroh_impl;
pub(crate) mod noq_impl;
use crate::{
LocalSender, RequestError, RpcMessage, Service,
channel::{
SendError,
mpsc::{self, DynReceiver, DynSender},
none::NoSender,
oneshot,
},
util::{AsyncReadVarintExt, WriteVarintExt, now_or_never},
};
pub const MAX_MESSAGE_SIZE: u64 = 1024 * 1024 * 16;
#[derive(Debug, derive_more::Display, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum ErrorCode {
#[display("request aborted")]
RequestAborted,
#[display("request too large")]
RequestMaxSizeExceeded,
#[display("request does not encode")]
RequestEncodeFailed,
#[display("request does not decode")]
RequestDecodeFailed,
#[display("message too large")]
MessageMaxSizeExceeded,
#[display("message does not encode")]
MessageEncodeFailed,
#[display("message does not decode")]
MessageDecodeFailed,
#[display("message aborted")]
MessageAborted,
#[display("code {_0}")]
Other(VarInt),
}
impl ErrorCode {
const KNOWN: [Self; 8] = [
Self::RequestAborted,
Self::RequestMaxSizeExceeded,
Self::RequestEncodeFailed,
Self::RequestDecodeFailed,
Self::MessageMaxSizeExceeded,
Self::MessageEncodeFailed,
Self::MessageDecodeFailed,
Self::MessageAborted,
];
pub const fn to_code(self) -> VarInt {
match self {
Self::RequestAborted => VarInt::from_u32(1),
Self::RequestMaxSizeExceeded => VarInt::from_u32(2),
Self::RequestEncodeFailed => VarInt::from_u32(3),
Self::RequestDecodeFailed => VarInt::from_u32(4),
Self::MessageMaxSizeExceeded => VarInt::from_u32(5),
Self::MessageEncodeFailed => VarInt::from_u32(6),
Self::MessageDecodeFailed => VarInt::from_u32(7),
Self::MessageAborted => VarInt::from_u32(8),
Self::Other(code) => code,
}
}
pub fn from_code(code: VarInt) -> Option<Self> {
if code.into_inner() == 0 {
return None;
}
let known = Self::KNOWN.into_iter().find(|c| c.to_code() == code);
Some(known.unwrap_or(Self::Other(code)))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum ConnectionClose {
Remote {
code: Option<ErrorCode>,
reason: Vec<u8>,
},
Local,
}
impl ConnectionClose {
pub fn code(&self) -> Option<ErrorCode> {
match self {
Self::Remote { code, .. } => *code,
Self::Local => None,
}
}
pub(crate) fn from_error(err: &ConnectionError) -> Option<Self> {
match err {
ConnectionError::ApplicationClosed(close) => Some(Self::Remote {
code: ErrorCode::from_code(close.error_code),
reason: close.reason.to_vec(),
}),
ConnectionError::LocallyClosed => Some(Self::Local),
_ => None,
}
}
pub(crate) fn io_kind(&self) -> io::ErrorKind {
match self {
Self::Remote { .. } => io::ErrorKind::ConnectionAborted,
Self::Local => io::ErrorKind::NotConnected,
}
}
pub(crate) fn from_io(err: &io::Error) -> Option<Self> {
let err = err.get_ref()?;
if let Some(noq::ReadError::ConnectionLost(err)) = err.downcast_ref() {
Self::from_error(err)
} else if let Some(noq::WriteError::ConnectionLost(err)) = err.downcast_ref() {
Self::from_error(err)
} else {
None
}
}
}
impl std::fmt::Display for ConnectionClose {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Remote { code, reason } => {
write!(f, "closed by the peer")?;
if let Some(code) = code {
write!(f, " ({code})")?;
}
if !reason.is_empty() {
write!(f, ": {}", String::from_utf8_lossy(reason))?;
}
Ok(())
}
Self::Local => write!(f, "closed by this side"),
}
}
}
#[stack_error(derive, add_meta)]
#[non_exhaustive]
pub enum ReadRequestError {
#[error("Request too large")]
MaxSizeExceeded,
#[error("Request malformed")]
Malformed {
#[error(std_err)]
source: io::Error,
},
#[error("Request does not decode")]
DecodeFailed {
#[error(std_err)]
source: postcard::Error,
},
#[error("Connection {close}")]
ConnectionClosed { close: ConnectionClose },
#[error("Connection failed")]
ConnectionFailed {
#[error(std_err)]
source: ConnectionError,
},
}
impl ReadRequestError {
pub fn remote_code(&self) -> Option<ErrorCode> {
match self {
Self::ConnectionClosed { close, .. } => close.code(),
_ => None,
}
}
fn error_code(&self) -> Option<ErrorCode> {
match self {
Self::MaxSizeExceeded { .. } => Some(ErrorCode::RequestMaxSizeExceeded),
Self::Malformed { .. } | Self::DecodeFailed { .. } => {
Some(ErrorCode::RequestDecodeFailed)
}
Self::ConnectionClosed { .. } | Self::ConnectionFailed { .. } => None,
}
}
}
impl From<ReadRequestError> for io::Error {
fn from(err: ReadRequestError) -> Self {
match err {
ReadRequestError::ConnectionFailed { source, .. } => source.into(),
ReadRequestError::ConnectionClosed { ref close, .. } => {
let kind = close.io_kind();
io::Error::new(kind, err)
}
err => io::Error::new(io::ErrorKind::InvalidData, err),
}
}
}
#[stack_error(derive, add_meta)]
#[non_exhaustive]
pub enum RpcSendError {
#[error("Message too large")]
MaxSizeExceeded,
#[error("Message does not encode")]
EncodeFailed {
#[error(std_err)]
source: postcard::Error,
},
#[error("Receiver stopped the stream: {code}")]
Remote { code: ErrorCode },
#[error("Connection {close}")]
ConnectionClosed { close: ConnectionClose },
#[error("Sender closed after an earlier error")]
Closed,
#[error("Connection failed")]
ConnectionFailed {
#[error(std_err)]
source: io::Error,
},
}
impl RpcSendError {
pub fn remote_code(&self) -> Option<ErrorCode> {
match self {
Self::Remote { code, .. } => Some(*code),
Self::ConnectionClosed { close, .. } => close.code(),
_ => None,
}
}
}
impl From<RpcSendError> for io::Error {
fn from(e: RpcSendError) -> Self {
match e {
RpcSendError::ConnectionFailed { source, .. } => source,
RpcSendError::Remote { .. } => io::Error::new(io::ErrorKind::ConnectionReset, e),
RpcSendError::ConnectionClosed { ref close, .. } => {
let kind = close.io_kind();
io::Error::new(kind, e)
}
RpcSendError::Closed { .. } => io::Error::new(io::ErrorKind::BrokenPipe, e),
_ => io::Error::new(io::ErrorKind::InvalidData, e),
}
}
}
#[stack_error(derive, add_meta)]
#[non_exhaustive]
pub enum RpcRecvError {
#[error("Message too large")]
MaxSizeExceeded,
#[error("Message does not decode")]
DecodeFailed {
#[error(std_err)]
source: postcard::Error,
},
#[error("Sender reset the stream{}", code.map(|code| format!(": {code}")).unwrap_or_default())]
Remote { code: Option<ErrorCode> },
#[error("Connection {close}")]
ConnectionClosed { close: ConnectionClose },
#[error("Connection failed")]
ConnectionFailed {
#[error(std_err)]
source: io::Error,
},
}
impl RpcRecvError {
pub fn remote_code(&self) -> Option<ErrorCode> {
match self {
Self::Remote { code, .. } => *code,
Self::ConnectionClosed { close, .. } => close.code(),
_ => None,
}
}
}
impl From<io::Error> for RpcRecvError {
fn from(err: io::Error) -> Self {
if let Some(code) = reset_code(&err) {
return e!(RpcRecvError::Remote { code });
}
if let Some(close) = ConnectionClose::from_io(&err) {
return e!(RpcRecvError::ConnectionClosed { close });
}
e!(RpcRecvError::ConnectionFailed, err)
}
}
impl From<RpcRecvError> for io::Error {
fn from(e: RpcRecvError) -> Self {
match e {
RpcRecvError::ConnectionFailed { source, .. } => source,
RpcRecvError::Remote { .. } => io::Error::new(io::ErrorKind::ConnectionReset, e),
RpcRecvError::ConnectionClosed { ref close, .. } => {
let kind = close.io_kind();
io::Error::new(kind, e)
}
_ => io::Error::new(io::ErrorKind::InvalidData, e),
}
}
}
pub trait RemoteConnection: Send + Sync + Debug + 'static {
fn clone_boxed(&self) -> Box<dyn RemoteConnection>;
fn open_bi(
&self,
) -> BoxFuture<std::result::Result<(noq::SendStream, noq::RecvStream), RequestError>>;
fn zero_rtt_rejected(&self) -> BoxFuture<bool>;
}
#[derive(Debug)]
pub struct RemoteSender<S>(ResetOnDrop, noq::RecvStream, std::marker::PhantomData<S>);
#[derive(Debug)]
struct ResetOnDrop<S: BorrowMut<noq::SendStream> = noq::SendStream> {
send: Option<S>,
code: ErrorCode,
}
impl<S: BorrowMut<noq::SendStream>> ResetOnDrop<S> {
fn new(send: S, code: ErrorCode) -> Self {
Self {
send: Some(send),
code,
}
}
fn get_mut(&mut self) -> &mut noq::SendStream {
self.send
.as_mut()
.expect("only `into_inner` and `reset` take the stream")
.borrow_mut()
}
fn into_inner(mut self) -> S {
self.send
.take()
.expect("only `into_inner` and `reset` take the stream")
}
fn reset(&mut self, code: ErrorCode) {
if let Some(mut send) = self.send.take() {
send.borrow_mut().reset(code.to_code()).ok();
}
}
}
impl<S: BorrowMut<noq::SendStream>> Drop for ResetOnDrop<S> {
fn drop(&mut self) {
if let Some(send) = &mut self.send {
send.borrow_mut().reset(self.code.to_code()).ok();
}
}
}
const INLINE_FRAME_SIZE: usize = 128;
fn prepare_write<S: Service>(
msg: impl Into<S>,
span: &tracing::Span,
) -> Result<SmallVec<[u8; INLINE_FRAME_SIZE]>, EncodeError> {
let msg = msg.into();
let mut buf = SmallVec::new();
let res = if S::SPAN_PROPAGATION {
let span_ctx = crate::span_propagation::SpanContextCarrier::from_span_or_current(span);
encode(&(span_ctx, msg), &mut buf)
} else {
encode(&msg, &mut buf)
};
res?;
Ok(buf)
}
enum EncodeError {
MaxSizeExceeded,
Failed(postcard::Error),
}
fn encode(
value: &impl Serialize,
buf: &mut SmallVec<[u8; INLINE_FRAME_SIZE]>,
) -> Result<(), EncodeError> {
let size = postcard::experimental::serialized_size(value).map_err(EncodeError::Failed)? as u64;
if size > MAX_MESSAGE_SIZE {
return Err(EncodeError::MaxSizeExceeded);
}
buf.clear();
buf.write_varint_u64(size)
.expect("a buffer does not fail to write");
postcard::to_io(value, buf).map_err(EncodeError::Failed)?;
Ok(())
}
fn encode_message(
value: &impl Serialize,
buf: &mut SmallVec<[u8; INLINE_FRAME_SIZE]>,
send: &mut noq::SendStream,
) -> Result<(), SendError> {
encode(value, buf).map_err(|err| {
let (code, err) = match err {
EncodeError::MaxSizeExceeded => (
ErrorCode::MessageMaxSizeExceeded,
e!(RpcSendError::MaxSizeExceeded),
),
EncodeError::Failed(source) => (
ErrorCode::MessageEncodeFailed,
e!(RpcSendError::EncodeFailed, source),
),
};
send.reset(code.to_code()).ok();
err.into()
})
}
impl<S: Service> RemoteSender<S> {
pub fn new(send: noq::SendStream, recv: noq::RecvStream) -> Self {
Self(
ResetOnDrop::new(send, ErrorCode::RequestAborted),
recv,
PhantomData,
)
}
pub async fn write(
self,
msg: impl Into<S>,
) -> std::result::Result<(noq::SendStream, noq::RecvStream), SendError> {
let (send, recv, _span) = self.write_in_span(msg).await?;
Ok((send, recv))
}
pub(crate) async fn write_in_span(
mut self,
msg: impl Into<S>,
) -> std::result::Result<(noq::SendStream, noq::RecvStream, tracing::Span), SendError> {
let (span, buf) = self.prepare(msg)?;
let (send, recv) = self.write_raw(&buf).instrument(span.clone()).await?;
Ok((send, recv, span))
}
pub(crate) fn req_id(&self) -> u64 {
self.1.id().index()
}
pub(crate) fn prepare(
&mut self,
msg: impl Into<S>,
) -> std::result::Result<(tracing::Span, SmallVec<[u8; INLINE_FRAME_SIZE]>), SendError> {
let msg = msg.into();
let span = msg.new_request_span(self.req_id());
let buf = prepare_write::<S>(msg, &span).map_err(|err| {
let (code, err) = match err {
EncodeError::MaxSizeExceeded => (
ErrorCode::RequestMaxSizeExceeded,
e!(RpcSendError::MaxSizeExceeded),
),
EncodeError::Failed(source) => (
ErrorCode::RequestEncodeFailed,
e!(RpcSendError::EncodeFailed, source),
),
};
self.0.reset(code);
SendError::from(err)
})?;
Ok((span, buf))
}
pub(crate) async fn write_raw(
self,
buf: &[u8],
) -> std::result::Result<(noq::SendStream, noq::RecvStream), SendError> {
let RemoteSender(mut send, recv, _) = self;
send.get_mut().write_all(buf).await?;
debug!("request sent");
Ok((send.into_inner(), recv))
}
}
impl<T: DeserializeOwned> From<noq::RecvStream> for oneshot::Receiver<T> {
fn from(mut read: noq::RecvStream) -> Self {
let span = tracing::Span::current();
let fut = async move {
let Some(size) = read_size(&mut read).await? else {
return Err(e!(oneshot::RecvError::SenderClosed));
};
if size > MAX_MESSAGE_SIZE {
read.stop(ErrorCode::MessageMaxSizeExceeded.to_code()).ok();
return Err(e!(RpcRecvError::MaxSizeExceeded).into());
}
let mut buf = vec![0; size as usize];
AsyncReadExt::read_exact(&mut read, &mut buf).await?;
let msg: T = postcard::from_bytes(&buf).map_err(|err| {
read.stop(ErrorCode::MessageDecodeFailed.to_code()).ok();
oneshot::RecvError::from(e!(RpcRecvError::DecodeFailed, err))
})?;
span.in_scope(|| debug!("response received"));
Ok(msg)
};
oneshot::Receiver::from(|| fut)
}
}
fn read_error(err: &io::Error) -> Option<&noq::ReadError> {
err.get_ref()?.downcast_ref()
}
fn write_error(err: &io::Error) -> Option<&noq::WriteError> {
err.get_ref()?.downcast_ref()
}
impl SendError {
pub(crate) fn is_zero_rtt_rejected(&self) -> bool {
match self {
Self::Rpc {
source: RpcSendError::ConnectionFailed { source, .. },
..
} => matches!(write_error(source), Some(noq::WriteError::ZeroRttRejected)),
_ => false,
}
}
}
async fn read_size(recv: &mut noq::RecvStream) -> io::Result<Option<u64>> {
match recv.read_varint_u64().await {
Err(err) if err.kind() == io::ErrorKind::InvalidData && read_error(&err).is_none() => {
Ok(Some(u64::MAX))
}
res => res,
}
}
pub(crate) fn is_truncated(err: &io::Error) -> bool {
err.kind() == io::ErrorKind::UnexpectedEof && read_error(err).is_none()
}
pub(crate) fn reset_code(err: &io::Error) -> Option<Option<ErrorCode>> {
match read_error(err)? {
noq::ReadError::Reset(code) => Some(ErrorCode::from_code(*code)),
_ => None,
}
}
impl From<noq::RecvStream> for crate::channel::none::NoReceiver {
fn from(read: noq::RecvStream) -> Self {
drop(read);
Self
}
}
impl<T: RpcMessage> From<noq::RecvStream> for mpsc::Receiver<T> {
fn from(read: noq::RecvStream) -> Self {
mpsc::Receiver::from_boxed(Box::new(NoqReceiver {
recv: read,
span: tracing::Span::current(),
received: 0,
finished: false,
buf: smallvec![0; INLINE_FRAME_SIZE],
end: 0,
_marker: PhantomData,
}))
}
}
impl From<noq::SendStream> for NoSender {
fn from(write: noq::SendStream) -> Self {
let _ = write;
NoSender
}
}
impl<T: RpcMessage> From<noq::SendStream> for oneshot::Sender<T> {
fn from(writer: noq::SendStream) -> Self {
oneshot::Sender::from_boxed(Box::new(NoqOneshotSender {
writer,
span: tracing::Span::current(),
}))
}
}
#[derive(Debug)]
struct NoqOneshotSender {
writer: noq::SendStream,
span: tracing::Span,
}
impl<T: RpcMessage> oneshot::DynSender<T> for NoqOneshotSender {
fn send(self: Box<Self>, value: T) -> BoxFuture<Result<(), SendError>> {
let Self { mut writer, span } = *self;
Box::pin(async move {
let mut buf = SmallVec::new();
encode_message(&value, &mut buf, &mut writer)?;
let mut guard = ResetOnDrop::new(&mut writer, ErrorCode::MessageAborted);
let res = guard.get_mut().write_all(&buf).await;
guard.into_inner();
res?;
span.in_scope(|| debug!("response sent"));
Ok(())
})
}
fn is_rpc(&self) -> bool {
true
}
}
impl<T: RpcMessage> From<noq::SendStream> for mpsc::Sender<T> {
fn from(write: noq::SendStream) -> Self {
mpsc::Sender::from_boxed(Arc::new(NoqSender(tokio::sync::Mutex::new(
NoqSenderState::Open(NoqSenderInner {
send: write,
buffer: SmallVec::new(),
written: 0,
span: tracing::Span::current(),
sent: 0,
_marker: PhantomData,
}),
))))
}
}
struct NoqReceiver<T> {
recv: noq::RecvStream,
span: tracing::Span,
received: u64,
finished: bool,
buf: SmallVec<[u8; INLINE_FRAME_SIZE]>,
end: usize,
_marker: std::marker::PhantomData<T>,
}
#[derive(Debug, PartialEq)]
enum Frame {
Partial { frame_len: Option<usize> },
Whole { body: Range<usize> },
TooLarge,
}
#[derive(Debug, Serialize, Deserialize)]
enum MessageEnd {
Complete,
Aborted,
}
fn decode_frame<T: DeserializeOwned>(buf: &[u8]) -> postcard::Result<Option<T>> {
let (msg, rest) = match postcard::take_from_bytes(buf) {
Ok(decoded) => decoded,
Err(_) if ends_aborted(buf) => return Ok(None),
Err(err) => return Err(err),
};
let end = match rest {
[] => MessageEnd::Complete,
[_] => postcard::from_bytes(rest)?,
_ if ends_aborted(buf) => return Ok(None),
_ => return Err(postcard::Error::DeserializeBadEncoding),
};
Ok(match end {
MessageEnd::Complete => Some(msg),
MessageEnd::Aborted => None,
})
}
fn ends_aborted(buf: &[u8]) -> bool {
let last = buf.last().map(std::slice::from_ref).unwrap_or_default();
matches!(postcard::from_bytes(last), Ok(MessageEnd::Aborted))
}
fn parse_frame(buf: &[u8]) -> io::Result<Frame> {
let (size, rest) = match postcard::take_from_bytes::<u64>(buf) {
Ok(parsed) => parsed,
Err(postcard::Error::DeserializeUnexpectedEnd) => {
return Ok(Frame::Partial { frame_len: None });
}
Err(postcard::Error::DeserializeBadVarint) => return Ok(Frame::TooLarge),
Err(err) => return Err(io::Error::new(io::ErrorKind::InvalidData, err)),
};
let prefix = buf.len() - rest.len();
if size > MAX_MESSAGE_SIZE {
return Ok(Frame::TooLarge);
}
let end = prefix + size as usize;
if buf.len() < end {
return Ok(Frame::Partial {
frame_len: Some(end),
});
}
Ok(Frame::Whole { body: prefix..end })
}
impl<T> NoqReceiver<T> {
fn finish(&mut self) -> Option<T> {
if self.end > 0 {
self.span.in_scope(|| {
debug!("stream ended in the middle of a message, the send was cancelled")
});
}
self.end = 0;
if self.buf.spilled() {
self.buf.truncate(INLINE_FRAME_SIZE);
self.buf.shrink_to_fit();
}
if !self.finished {
self.finished = true;
let received = self.received;
self.span.in_scope(|| debug!(received, "stream finished"));
}
None
}
}
impl<T> Debug for NoqReceiver<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NoqReceiver").finish()
}
}
impl<T: RpcMessage> NoqReceiver<T> {
fn poll_recv_inner(
&mut self,
cx: &mut Context<'_>,
) -> Poll<Result<Option<T>, mpsc::RecvError>> {
loop {
match parse_frame(&self.buf[..self.end])? {
Frame::Whole { body } => {
let msg = decode_frame(&self.buf[body.clone()]);
self.buf.copy_within(body.end..self.end, 0);
self.end -= body.end;
if self.end == 0 && self.buf.spilled() {
self.buf.truncate(INLINE_FRAME_SIZE);
self.buf.shrink_to_fit();
}
let msg = msg.map_err(|err| {
self.recv
.stop(ErrorCode::MessageDecodeFailed.to_code())
.ok();
mpsc::RecvError::from(e!(RpcRecvError::DecodeFailed, err))
})?;
let Some(msg) = msg else {
self.span
.in_scope(|| debug!("dropped a message whose send was cancelled"));
continue;
};
self.received += 1;
if self.received == 1 {
self.span.in_scope(|| debug!("first message received"));
}
return Poll::Ready(Ok(Some(msg)));
}
Frame::TooLarge => {
self.recv
.stop(ErrorCode::MessageMaxSizeExceeded.to_code())
.ok();
return Poll::Ready(Err(e!(RpcRecvError::MaxSizeExceeded).into()));
}
Frame::Partial { frame_len } => {
if let Some(len) = frame_len
&& self.buf.len() < len
{
self.buf.resize(len, 0);
}
let mut read = ReadBuf::new(&mut self.buf[self.end..]);
ready!(Pin::new(&mut self.recv).poll_read(cx, &mut read))?;
let n = read.filled().len();
self.end += n;
if n == 0 {
return Poll::Ready(Ok(self.finish()));
}
}
}
}
}
}
impl<T: RpcMessage> DynReceiver<T> for NoqReceiver<T> {
fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Result<Option<T>, mpsc::RecvError>> {
if self.finished {
return Poll::Ready(Ok(None));
}
let result = self.poll_recv_inner(cx);
if matches!(&result, Poll::Ready(Err(_))) {
self.finished = true;
self.end = 0;
clear_frame(&mut self.buf);
}
result
}
}
impl<T> Drop for NoqReceiver<T> {
fn drop(&mut self) {}
}
fn clear_frame(buf: &mut SmallVec<[u8; INLINE_FRAME_SIZE]>) {
buf.clear();
if buf.spilled() {
buf.shrink_to_fit();
}
}
struct NoqSenderInner<T> {
send: noq::SendStream,
buffer: SmallVec<[u8; INLINE_FRAME_SIZE]>,
written: usize,
span: tracing::Span,
sent: u64,
_marker: std::marker::PhantomData<T>,
}
impl<T: RpcMessage> NoqSenderInner<T> {
fn record_sent(&mut self) {
self.sent += 1;
if self.sent == 1 {
self.span.in_scope(|| debug!("first message sent"));
}
}
async fn write_rest(&mut self) -> Result<(), SendError> {
while self.written < self.buffer.len() {
self.written += self.send.write(&self.buffer[self.written..]).await?;
}
clear_frame(&mut self.buffer);
self.written = 0;
Ok(())
}
async fn prepare(&mut self) -> Result<(), SendError> {
if self.written > 0 {
let last = self.buffer.len() - 1;
postcard::to_slice(&MessageEnd::Aborted, &mut self.buffer[last..])
.expect("the end of a message is one byte");
self.write_rest().await?;
}
clear_frame(&mut self.buffer);
Ok(())
}
fn encode(&mut self, value: &T) -> Result<(), SendError> {
let frame = (value, MessageEnd::Complete);
encode_message(&frame, &mut self.buffer, &mut self.send)
}
async fn send(&mut self, value: T) -> Result<(), SendError> {
self.prepare().await?;
self.encode(&value)?;
self.write_rest().await?;
self.record_sent();
Ok(())
}
async fn try_send(&mut self, value: T) -> Result<bool, SendError> {
let Some(res) = now_or_never(self.prepare()) else {
return Ok(false);
};
res?;
self.encode(&value)?;
let Some(n) = now_or_never(self.send.write(&self.buffer)) else {
clear_frame(&mut self.buffer);
return Ok(false);
};
self.written = n?;
self.write_rest().await?;
self.record_sent();
Ok(true)
}
}
#[allow(clippy::large_enum_variant)]
enum NoqSenderState<T> {
Open(NoqSenderInner<T>),
Closed(CloseReason),
}
#[derive(Debug, Clone)]
enum CloseReason {
ReceiverClosed,
Remote(ErrorCode),
ConnectionClosed(ConnectionClose),
Io(noq::WriteError),
Other,
}
impl CloseReason {
fn error(self) -> SendError {
match self {
Self::ReceiverClosed => e!(SendError::ReceiverClosed),
Self::Remote(code) => e!(RpcSendError::Remote { code }).into(),
Self::ConnectionClosed(close) => e!(RpcSendError::ConnectionClosed { close }).into(),
Self::Io(err) => e!(RpcSendError::ConnectionFailed, err.into()).into(),
Self::Other => e!(RpcSendError::Closed).into(),
}
}
}
impl<T: RpcMessage> NoqSenderState<T> {
async fn run<R>(
&mut self,
f: impl AsyncFnOnce(&mut NoqSenderInner<T>) -> Result<R, SendError>,
) -> Result<R, SendError> {
let sender = match self {
Self::Open(sender) => sender,
Self::Closed(reason) => return Err(reason.clone().error()),
};
let res = f(sender).await;
match &res {
Ok(_) => {}
Err(SendError::ReceiverClosed { .. }) => {
*self = Self::Closed(CloseReason::ReceiverClosed)
}
Err(SendError::Rpc { source, .. }) => {
*self = Self::Closed(match source {
RpcSendError::Remote { code, .. } => CloseReason::Remote(*code),
RpcSendError::ConnectionClosed { close, .. } => {
CloseReason::ConnectionClosed(close.clone())
}
RpcSendError::ConnectionFailed { source, .. } => match write_error(source) {
Some(err) => CloseReason::Io(err.clone()),
None => CloseReason::Other,
},
_ => CloseReason::Other,
})
}
}
res
}
}
struct NoqSender<T>(tokio::sync::Mutex<NoqSenderState<T>>);
impl<T> Drop for NoqSender<T> {
fn drop(&mut self) {
if let NoqSenderState::Open(sender) = self.0.get_mut() {
let sent = sender.sent;
sender.span.in_scope(|| debug!(sent, "stream finished"));
}
}
}
impl<T> Debug for NoqSender<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NoqSender").finish()
}
}
impl<T: RpcMessage> DynSender<T> for NoqSender<T> {
fn send(&self, value: T) -> Pin<Box<dyn Future<Output = Result<(), SendError>> + Send + '_>> {
Box::pin(async {
let mut state = self.0.lock().await;
state.run(async |sender| sender.send(value).await).await
})
}
fn try_send(
&self,
value: T,
) -> Pin<Box<dyn Future<Output = Result<bool, SendError>> + Send + '_>> {
Box::pin(async {
let mut state = self.0.lock().await;
state.run(async |sender| sender.try_send(value).await).await
})
}
fn closed(&self) -> Pin<Box<dyn Future<Output = ()> + Send + Sync + '_>> {
Box::pin(async {
let stopped = match &*self.0.lock().await {
NoqSenderState::Open(sender) => sender.send.stopped(),
NoqSenderState::Closed(_) => return,
};
stopped.await.ok();
})
}
fn is_rpc(&self) -> bool {
true
}
}
type HandlerFn<S> = Arc<
dyn Fn(S, noq::RecvStream, noq::SendStream) -> BoxFuture<std::result::Result<(), HandlerError>>
+ Send
+ Sync
+ 'static,
>;
#[derive(Debug)]
pub struct HandlerError {
inner: HandlerErrorInner,
}
#[derive(Debug)]
enum HandlerErrorInner {
CloseConnection { code: VarInt, reason: Vec<u8> },
}
impl std::error::Error for HandlerError {}
impl std::fmt::Display for HandlerError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self.inner {
HandlerErrorInner::CloseConnection { code, reason } => write!(
f,
"handler asked to close the connection with code {code}: {}",
String::from_utf8_lossy(reason)
),
}
}
}
impl HandlerError {
pub fn close_connection(code: VarInt, reason: impl AsRef<[u8]>) -> Self {
Self {
inner: HandlerErrorInner::CloseConnection {
code,
reason: reason.as_ref().to_vec(),
},
}
}
}
#[derive(derive_more::Debug)]
pub struct Handler<S> {
#[debug(skip)]
f: HandlerFn<S>,
skip_bad_requests: bool,
}
impl<S> Clone for Handler<S> {
fn clone(&self) -> Self {
Self {
f: self.f.clone(),
skip_bad_requests: self.skip_bad_requests,
}
}
}
impl<S: Service> Handler<S> {
pub fn raw<F, Fut>(f: F) -> Self
where
F: Fn(S, noq::RecvStream, noq::SendStream) -> Fut + Send + Sync + 'static,
Fut: Future<Output = std::result::Result<(), HandlerError>> + Send + 'static,
{
let f: HandlerFn<S> = Arc::new(move |msg, rx, tx| Box::pin(f(msg, rx, tx)));
Self {
f,
skip_bad_requests: false,
}
}
pub fn call(
&self,
request: S,
rx: noq::RecvStream,
tx: noq::SendStream,
) -> impl Future<Output = std::result::Result<(), HandlerError>> + Send + 'static {
(self.f)(request, rx, tx)
}
pub fn skip_bad_requests(mut self, skip: bool) -> Self {
self.skip_bad_requests = skip;
self
}
pub async fn handle_connection(
&self,
connection: &impl IncomingRemoteConnection,
) -> io::Result<()> {
loop {
let Some((msg, carrier, rx, tx)) = self.read_next(connection).await? else {
return Ok(());
};
let span = new_request_span(&msg, &rx, carrier.as_ref());
let fut = span
.in_scope(|| {
debug!("request received");
(self.f)(msg, rx, tx)
})
.instrument(span);
if let Err(err) = fut.await {
match err.inner {
HandlerErrorInner::CloseConnection { code, reason } => {
debug!(%code, "handler closed the connection");
connection.close(code, &reason);
return Ok(());
}
}
}
}
}
async fn read_next(
&self,
connection: &impl IncomingRemoteConnection,
) -> io::Result<Option<Request<S>>> {
loop {
match read_request_inner::<S>(connection).await {
Ok(request) => return Ok(request),
Err(err) => match err.error_code() {
None => return Err(err.into()), Some(_) if self.skip_bad_requests => debug!("skipped bad request: {err:#}"),
Some(code) => {
connection.close(code.to_code(), err.to_string().as_bytes());
return Err(err.into());
}
},
}
}
}
}
impl<S: RemoteService> Handler<S> {
pub fn from_sender(local_sender: impl Into<LocalSender<S>>) -> Self {
let local_sender = local_sender.into();
Self::sequential(move |msg| {
let local_sender = local_sender.clone();
async move {
local_sender.send_raw(msg).await.map_err(|err| {
warn!("handler stopped: {err:#}");
HandlerError::close_connection(VarInt::from_u32(0), b"")
})
}
})
}
pub fn sequential<F, Fut>(f: F) -> Self
where
F: Fn(S::Message) -> Fut + Send + Sync + 'static,
Fut: Future<Output = std::result::Result<(), HandlerError>> + Send + 'static,
{
let f = Arc::new(f);
Self::raw(move |msg, rx, tx| {
let f = f.clone();
async move { f(S::with_remote_channels(msg, rx, tx)).await }
})
}
}
impl<S: RemoteService> From<LocalSender<S>> for Handler<S> {
fn from(local_sender: LocalSender<S>) -> Self {
Self::from_sender(local_sender)
}
}
pub trait RemoteService: Service + Sized {
fn with_remote_channels(self, rx: noq::RecvStream, tx: noq::SendStream) -> Self::Message;
}
pub trait IncomingRemoteConnection: crate::sealed::Sealed {
fn accept_bi(
&self,
) -> impl Future<Output = Result<(noq::SendStream, noq::RecvStream), ConnectionError>> + Send;
fn close(&self, error_code: VarInt, reason: &[u8]);
}
pub async fn read_request<S: RemoteService>(
connection: &impl IncomingRemoteConnection,
) -> Result<Option<S::Message>, ReadRequestError> {
let Some((msg, carrier, rx, tx)) = read_request_inner::<S>(connection).await? else {
return Ok(None);
};
let span = new_request_span(&msg, &rx, carrier.as_ref());
Ok(Some(span.in_scope(|| {
debug!("request received");
S::with_remote_channels(msg, rx, tx)
})))
}
fn new_request_span<S: Service>(
msg: &S,
recv: &noq::RecvStream,
carrier: Option<&crate::span_propagation::SpanContextCarrier>,
) -> tracing::Span {
let span = msg.new_request_span(recv.id().index());
if let Some(carrier) = carrier {
carrier.set_parent_of(&span);
}
span
}
pub async fn read_request_raw<S: Service>(
connection: &impl IncomingRemoteConnection,
) -> Result<Option<(S, noq::RecvStream, noq::SendStream)>, ReadRequestError> {
Ok(read_request_inner::<S>(connection)
.await?
.map(|(msg, _carrier, rx, tx)| (msg, rx, tx)))
}
type Request<S> = (
S,
Option<crate::span_propagation::SpanContextCarrier>,
noq::RecvStream,
noq::SendStream,
);
async fn read_request_inner<S: Service>(
connection: &impl IncomingRemoteConnection,
) -> Result<Option<Request<S>>, ReadRequestError> {
loop {
let (mut send, mut recv) = match connection.accept_bi().await {
Ok(streams) => streams,
Err(ConnectionError::ApplicationClosed(cause))
if cause.error_code == VarInt::from_u32(0) =>
{
trace!("remote side closed connection {cause:?}");
return Ok(None);
}
Err(ConnectionError::LocallyClosed) => return Ok(None),
Err(cause) => match ConnectionClose::from_error(&cause) {
Some(close) => return Err(e!(ReadRequestError::ConnectionClosed { close })),
None => return Err(e!(ReadRequestError::ConnectionFailed, cause)),
},
};
let err = match read_request_frame::<S>(&mut recv).await {
ReadFrame::Request(carrier, msg) => return Ok(Some((msg, carrier, recv, send))),
ReadFrame::MaxSizeExceeded => e!(ReadRequestError::MaxSizeExceeded),
ReadFrame::Malformed(source) => e!(ReadRequestError::Malformed, source),
ReadFrame::DecodeFailed(source) => e!(ReadRequestError::DecodeFailed, source),
ReadFrame::Reset(reason) => {
debug!(?reason, "skipped request: the client reset its stream");
send.reset(ErrorCode::RequestAborted.to_code()).ok();
continue;
}
ReadFrame::ConnectionLost => continue,
};
if let Some(code) = err.error_code() {
recv.stop(code.to_code()).ok();
send.reset(code.to_code()).ok();
}
return Err(err);
}
}
enum ReadFrame<S> {
Request(Option<crate::span_propagation::SpanContextCarrier>, S),
MaxSizeExceeded,
Malformed(io::Error),
DecodeFailed(postcard::Error),
Reset(Option<ErrorCode>),
ConnectionLost,
}
impl<S> ReadFrame<S> {
fn from_read_err(err: &noq::ReadError) -> Self {
match err {
noq::ReadError::Reset(code) => Self::Reset(ErrorCode::from_code(*code)),
noq::ReadError::ConnectionLost(_) => Self::ConnectionLost,
noq::ReadError::ClosedStream | noq::ReadError::ZeroRttRejected => Self::Reset(None),
}
}
}
async fn read_request_frame<S: Service>(recv: &mut noq::RecvStream) -> ReadFrame<S> {
let size = match read_size(recv).await {
Ok(Some(size)) => size,
Ok(None) => {
return ReadFrame::Malformed(io::Error::new(
io::ErrorKind::UnexpectedEof,
"stream ended before the request",
));
}
Err(err) => {
return match read_error(&err) {
Some(read_err) => ReadFrame::from_read_err(read_err),
None => ReadFrame::Malformed(err),
};
}
};
if size > MAX_MESSAGE_SIZE {
return ReadFrame::MaxSizeExceeded;
}
let mut buf = vec![0; size as usize];
match recv.read_exact(&mut buf).await {
Ok(()) => {}
Err(noq::ReadExactError::ReadError(err)) => return ReadFrame::from_read_err(&err),
Err(err @ noq::ReadExactError::FinishedEarly(_)) => {
return ReadFrame::Malformed(io::Error::new(io::ErrorKind::UnexpectedEof, err));
}
}
let decoded = if S::SPAN_PROPAGATION {
postcard::from_bytes(&buf)
} else {
postcard::from_bytes(&buf).map(|msg| (None, msg))
};
match decoded {
Ok((carrier, msg)) => ReadFrame::Request(carrier, msg),
Err(err) => ReadFrame::DecodeFailed(err),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "noq_endpoint_setup")]
#[tokio::test]
#[n0_tracing_test::traced_test]
async fn mpsc_frame_error_ends_receiver() -> testresult::TestResult<()> {
use std::future::poll_fn;
let (server, cert) = crate::noq::make_server_endpoint("127.0.0.1:0".parse()?)?;
let client = crate::noq::make_client_endpoint("127.0.0.1:0".parse()?, &[cert.as_slice()])?;
let connecting = client.connect(server.local_addr()?, "localhost")?;
let accepting = async { server.accept().await.unwrap().await };
let (client_conn, server_conn) = tokio::try_join!(connecting, accepting)?;
for (bytes, expected_code) in [
(vec![1, 2, 1, 0], ErrorCode::MessageDecodeFailed),
(vec![0xff; 10], ErrorCode::MessageMaxSizeExceeded),
] {
let (mut send, _response) = client_conn.open_bi().await?;
send.write_all(&[0]).await?;
let (_response, recv) = server_conn.accept_bi().await?;
let mut receiver = NoqReceiver::<bool> {
recv,
span: tracing::Span::none(),
received: 0,
finished: false,
end: bytes.len(),
buf: SmallVec::from_vec(bytes),
_marker: PhantomData,
};
let err = poll_fn(|cx| receiver.poll_recv(cx)).await.unwrap_err();
match expected_code {
ErrorCode::MessageDecodeFailed => assert!(matches!(
err,
mpsc::RecvError::Rpc {
source: RpcRecvError::DecodeFailed { .. },
..
}
)),
ErrorCode::MessageMaxSizeExceeded => assert!(matches!(
err,
mpsc::RecvError::Rpc {
source: RpcRecvError::MaxSizeExceeded { .. },
..
}
)),
_ => unreachable!(),
}
for _ in 0..2 {
assert_eq!(poll_fn(|cx| receiver.poll_recv(cx)).await?, None);
}
assert_eq!(send.stopped().await?, Some(expected_code.to_code()));
}
Ok(())
}
#[test]
fn error_code_round_trips() {
for code in ErrorCode::KNOWN {
assert_eq!(ErrorCode::from_code(code.to_code()), Some(code));
}
for code in [16, 401, u64::from(u32::MAX) + 1, VarInt::MAX.into_inner()] {
let code = VarInt::from_u64(code).unwrap();
assert_eq!(ErrorCode::from_code(code), Some(ErrorCode::Other(code)));
}
assert_eq!(ErrorCode::from_code(VarInt::from_u32(0)), None);
}
fn take(buf: &mut Vec<u8>) -> io::Result<Result<Vec<u8>, Frame>> {
Ok(match parse_frame(buf)? {
Frame::Whole { body } => {
let bytes = buf[body.clone()].to_vec();
buf.drain(..body.end);
Ok(bytes)
}
other => Err(other),
})
}
fn frame(body: &[u8]) -> Vec<u8> {
use crate::util::WriteVarintExt;
let mut frame = Vec::new();
frame.write_varint_u64(body.len() as u64).unwrap();
frame.extend_from_slice(body);
frame
}
#[test]
fn parse_frame_waits_for_every_split() {
let body = vec![7u8; 300];
let frame = frame(&body);
let prefix = frame.len() - body.len();
for split in 0..frame.len() {
let mut buf = frame[..split].to_vec();
let frame_len = (split >= prefix).then_some(frame.len());
assert_eq!(parse_frame(&buf).unwrap(), Frame::Partial { frame_len });
buf.extend_from_slice(&frame[split..]);
assert_eq!(take(&mut buf).unwrap(), Ok(body.clone()));
assert!(buf.is_empty());
}
}
#[test]
fn parse_frame_leaves_the_next_frame() {
let mut buf = [frame(b"one"), frame(b""), frame(b"three")].concat();
assert_eq!(take(&mut buf).unwrap(), Ok(b"one".to_vec()));
assert_eq!(take(&mut buf).unwrap(), Ok(b"".to_vec()));
assert_eq!(take(&mut buf).unwrap(), Ok(b"three".to_vec()));
assert_eq!(
take(&mut buf).unwrap(),
Err(Frame::Partial { frame_len: None })
);
}
#[test]
fn parse_frame_rejects_oversized_and_overlong_lengths() {
use crate::util::WriteVarintExt;
let mut buf = Vec::new();
buf.write_varint_u64(MAX_MESSAGE_SIZE + 1).unwrap();
assert_eq!(parse_frame(&buf).unwrap(), Frame::TooLarge);
let max = [[0xFFu8; 9].as_slice(), &[0x01]].concat();
assert_eq!(parse_frame(&max).unwrap(), Frame::TooLarge);
let overflow = [[0xFFu8; 9].as_slice(), &[0x02]].concat();
let overlong = [0x80u8; 10];
for buf in [&overflow[..], &overlong[..]] {
assert_eq!(parse_frame(buf).unwrap(), Frame::TooLarge);
}
}
#[test]
fn connection_close_shows_code_and_reason() {
let close = |code, reason: &[u8]| {
ConnectionClose::Remote {
code,
reason: reason.to_vec(),
}
.to_string()
};
let code = Some(ErrorCode::Other(VarInt::from_u32(42)));
assert_eq!(close(None, b""), "closed by the peer");
assert_eq!(close(None, b"bye"), "closed by the peer: bye");
assert_eq!(close(code, b""), "closed by the peer (code 42)");
assert_eq!(
close(code, b"maintenance"),
"closed by the peer (code 42): maintenance"
);
assert_eq!(
close(Some(ErrorCode::RequestDecodeFailed), b""),
"closed by the peer (request does not decode)"
);
}
#[test]
fn connection_close_maps_to_one_io_kind() {
let remote = ConnectionClose::Remote {
code: None,
reason: Vec::new(),
};
for (close, kind) in [
(remote, io::ErrorKind::ConnectionAborted),
(ConnectionClose::Local, io::ErrorKind::NotConnected),
] {
let errors = [
io::Error::from(e!(RpcSendError::ConnectionClosed {
close: close.clone()
})),
io::Error::from(e!(RpcRecvError::ConnectionClosed {
close: close.clone()
})),
io::Error::from(e!(ReadRequestError::ConnectionClosed {
close: close.clone()
})),
io::Error::from(e!(crate::RequestError::ConnectionClosed {
close: close.clone()
})),
];
for err in errors {
assert_eq!(err.kind(), kind, "{err}");
}
}
}
#[test]
fn send_error_maps_stop_codes() {
let err = SendError::from(noq::WriteError::Stopped(
ErrorCode::RequestDecodeFailed.to_code(),
));
assert_eq!(err.remote_code(), Some(ErrorCode::RequestDecodeFailed));
assert_eq!(io::Error::from(err).kind(), io::ErrorKind::ConnectionReset);
let err = SendError::from(noq::WriteError::Stopped(0u32.into()));
assert!(matches!(err, SendError::ReceiverClosed { .. }), "{err:?}");
let err = SendError::from(noq::WriteError::Stopped(401u32.into()));
assert_eq!(err.remote_code(), Some(ErrorCode::Other(401u32.into())));
}
#[test]
fn send_error_detects_zero_rtt_rejection() {
assert!(SendError::from(noq::WriteError::ZeroRttRejected).is_zero_rtt_rejected());
assert!(!SendError::from(noq::WriteError::ClosedStream).is_zero_rtt_rejected());
assert!(!SendError::from(noq::WriteError::Stopped(16u32.into())).is_zero_rtt_rejected());
}
}