use std::{
fmt,
io::{self, Write},
mem,
panic::{self, AssertUnwindSafe},
sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError},
task::{Context, Poll, Waker},
thread::{self, JoinHandle},
time::{Duration, Instant},
};
use crate::{
framing::Framing,
sink::{Drain, Sink},
};
const KEEP: usize = 64 * 1024;
const WRITE_CHUNK: usize = 64 * 1024;
const DRAIN_POLL: Duration = Duration::from_millis(50);
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub struct QueueLimits {
pub max_bytes: usize,
pub max_messages: usize,
pub write_timeout: Option<Duration>,
pub close_timeout: Duration,
}
impl QueueLimits {
#[must_use]
pub const fn new() -> Self {
Self {
max_bytes: 64 * 1024 * 1024,
max_messages: 65_536,
write_timeout: Some(Duration::from_secs(60)),
close_timeout: Duration::from_secs(10),
}
}
#[must_use]
pub const fn with_max_bytes(mut self, bytes: usize) -> Self {
self.max_bytes = bytes;
self
}
#[must_use]
pub const fn with_max_messages(mut self, count: usize) -> Self {
self.max_messages = count;
self
}
#[must_use]
pub const fn with_write_timeout(mut self, timeout: Option<Duration>) -> Self {
self.write_timeout = timeout;
self
}
#[must_use]
pub const fn with_close_timeout(mut self, timeout: Duration) -> Self {
self.close_timeout = timeout;
self
}
}
impl Default for QueueLimits {
fn default() -> Self {
Self::new()
}
}
struct Core {
framing: Framing,
limits: QueueLimits,
state: Mutex<State>,
signal: Condvar,
}
struct State {
queued: Vec<u8>,
queued_messages: usize,
writing: usize,
writing_messages: usize,
progressed: Option<Instant>,
failed: Option<(io::ErrorKind, String)>,
closed: bool,
finished: bool,
waker: Option<Waker>,
idle: bool,
}
impl Core {
fn new(framing: Framing, limits: QueueLimits) -> Arc<Self> {
Arc::new(Self {
framing,
limits,
state: Mutex::new(State {
queued: Vec::new(),
queued_messages: 0,
writing: 0,
writing_messages: 0,
progressed: None,
failed: None,
closed: false,
finished: false,
waker: None,
idle: false,
}),
signal: Condvar::new(),
})
}
fn lock(&self) -> MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
fn push(&self, message: &str) -> io::Result<()> {
let mut state = self.lock();
state.check(&self.limits)?;
let bytes = state.queued.len() + state.writing;
let messages = state.queued_messages + state.writing_messages;
let before = state.queued.len();
self.framing.encode(message.as_bytes(), &mut state.queued);
let added = state.queued.len() - before;
let over = bytes.saturating_add(added) > self.limits.max_bytes
|| messages.saturating_add(1) > self.limits.max_messages;
if messages > 0 && over {
state.queued.truncate(before);
return Err(state.fail(
io::ErrorKind::Other,
"the output queue is full: the peer is not reading".into(),
));
}
state.queued_messages += 1;
let waker = state.waker.take();
let idle = std::mem::take(&mut state.idle);
drop(state);
if idle {
self.signal.notify_all();
}
if let Some(waker) = waker {
waker.wake();
}
Ok(())
}
fn check(&self) -> io::Result<()> {
self.lock().check(&self.limits)
}
fn close(&self) {
let waker = {
let mut state = self.lock();
state.closed = true;
state.waker.take()
};
self.signal.notify_all();
if let Some(waker) = waker {
waker.wake();
}
}
fn wait_drained(&self) -> bool {
let deadline = Instant::now().checked_add(self.limits.close_timeout);
let mut state = self.lock();
while !state.finished {
let now = Instant::now();
let late = deadline.is_some_and(|at| now >= at);
if late || state.stalled(&self.limits).is_some() {
return false;
}
let left = deadline.map_or(DRAIN_POLL, |at| (at - now).min(DRAIN_POLL));
state = self
.signal
.wait_timeout(state, left)
.unwrap_or_else(PoisonError::into_inner)
.0;
}
true
}
fn progress(&self) {
let mut state = self.lock();
if state.progressed.is_some() {
state.progressed = Some(Instant::now());
}
}
fn pending(&self) -> (usize, usize) {
let state = self.lock();
(
state.queued.len() + state.writing,
state.queued_messages + state.writing_messages,
)
}
}
impl State {
fn check(&mut self, limits: &QueueLimits) -> io::Result<()> {
if let Some((kind, message)) = &self.failed {
return Err(io::Error::new(*kind, message.clone()));
}
if let Some(limit) = self.stalled(limits) {
return Err(self.fail(
io::ErrorKind::TimedOut,
format!(
"a write to the peer made no progress for {limit:?}: the peer is not reading"
),
));
}
Ok(())
}
fn stalled(&self, limits: &QueueLimits) -> Option<Duration> {
let since = self.progressed?;
let limit = limits.write_timeout?;
(since.elapsed() > limit).then_some(limit)
}
fn fail(&mut self, kind: io::ErrorKind, message: String) -> io::Error {
let (kind, message) = self.failed.get_or_insert((kind, message)).clone();
self.queued = Vec::new();
self.queued_messages = 0;
io::Error::new(kind, message)
}
fn take(&mut self, batch: &mut Vec<u8>) {
mem::swap(&mut self.queued, batch);
self.writing = batch.len();
self.writing_messages = mem::take(&mut self.queued_messages);
self.progressed = Some(Instant::now());
}
fn written(&mut self) {
self.writing = 0;
self.writing_messages = 0;
self.progressed = None;
}
}
fn recycle(batch: &mut Vec<u8>) {
batch.clear();
if batch.capacity() > 2 * KEEP {
batch.shrink_to(KEEP);
}
}
pub struct Queued {
core: Arc<Core>,
writer: Option<JoinHandle<()>>,
}
impl Queued {
pub fn new<W: Write + Send + 'static>(writer: W, framing: Framing) -> io::Result<Self> {
Self::with_limits(writer, framing, QueueLimits::new())
}
pub fn with_limits<W: Write + Send + 'static>(
writer: W,
framing: Framing,
limits: QueueLimits,
) -> io::Result<Self> {
let core = Core::new(framing, limits);
let shared = Arc::clone(&core);
let writer = thread::Builder::new()
.name("rpc-io-writer".into())
.spawn(move || write_loop(&shared, writer))?;
Ok(Self {
core,
writer: Some(writer),
})
}
#[must_use]
pub fn framing(&self) -> Framing {
self.core.framing
}
#[must_use]
pub fn limits(&self) -> QueueLimits {
self.core.limits
}
#[must_use]
pub fn pending_bytes(&self) -> usize {
self.core.pending().0
}
#[must_use]
pub fn pending_messages(&self) -> usize {
self.core.pending().1
}
fn waiter(&mut self) -> Drain {
let core = Arc::clone(&self.core);
let writer = self.writer.take();
Drain::new(move || {
if core.wait_drained() {
if let Some(writer) = writer {
let _joined = writer.join();
}
}
})
}
}
impl Sink for Queued {
fn send(&mut self, message: &str) -> io::Result<()> {
self.core.push(message)
}
fn flush(&mut self) -> io::Result<()> {
self.core.check()
}
fn close(&mut self) -> io::Result<()> {
let result = self.core.check();
self.core.close();
result
}
fn drain(&mut self) -> Option<Drain> {
Some(self.waiter())
}
}
impl Drop for Queued {
fn drop(&mut self) {
self.core.close();
if self.writer.is_some() {
self.waiter().wait();
}
}
}
impl fmt::Debug for Queued {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let (bytes, messages) = self.core.pending();
f.debug_struct("Queued")
.field("framing", &self.core.framing)
.field("limits", &self.core.limits)
.field("pending_bytes", &bytes)
.field("pending_messages", &messages)
.finish_non_exhaustive()
}
}
fn write_loop<W: Write>(core: &Core, mut writer: W) {
let mut batch = Vec::new();
loop {
{
let mut state = core.lock();
loop {
if state.failed.is_some() || (state.queued.is_empty() && state.closed) {
state.finished = true;
drop(state);
core.signal.notify_all();
return;
}
if !state.queued.is_empty() {
break;
}
state.idle = true;
state = core
.signal
.wait(state)
.unwrap_or_else(PoisonError::into_inner);
state.idle = false;
}
state.take(&mut batch);
}
let written = panic::catch_unwind(AssertUnwindSafe(|| {
write_chunks(core, &mut writer, &batch)?;
writer.flush()
}))
.unwrap_or_else(|_| Err(io::Error::other("the writer panicked")));
{
let mut state = core.lock();
state.written();
if let Err(error) = written {
let _failed =
state.fail(error.kind(), format!("writing to the peer failed: {error}"));
}
}
core.signal.notify_all();
recycle(&mut batch);
}
}
fn write_chunks<W: Write>(core: &Core, writer: &mut W, batch: &[u8]) -> io::Result<()> {
let mut rest = batch;
while !rest.is_empty() {
let chunk = &rest[..rest.len().min(WRITE_CHUNK)];
match writer.write(chunk) {
Ok(0) => {
return Err(io::Error::new(
io::ErrorKind::WriteZero,
"the writer accepted no bytes",
));
}
Ok(n) => {
rest = &rest[n..];
core.progress();
}
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
Err(error) => return Err(error),
}
}
Ok(())
}
#[must_use]
pub fn outbox(framing: Framing, limits: QueueLimits) -> (OutboxSink, Outbox) {
let core = Core::new(framing, limits);
(
OutboxSink {
core: Arc::clone(&core),
},
Outbox {
core,
batch: Vec::new(),
at: 0,
},
)
}
pub struct OutboxSink {
core: Arc<Core>,
}
impl OutboxSink {
#[must_use]
pub fn pending_bytes(&self) -> usize {
self.core.pending().0
}
}
impl Sink for OutboxSink {
fn send(&mut self, message: &str) -> io::Result<()> {
self.core.push(message)
}
fn flush(&mut self) -> io::Result<()> {
self.core.check()
}
fn close(&mut self) -> io::Result<()> {
let result = self.core.check();
self.core.close();
result
}
}
impl Drop for OutboxSink {
fn drop(&mut self) {
self.core.close();
}
}
impl fmt::Debug for OutboxSink {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("OutboxSink")
.field("framing", &self.core.framing)
.field("pending_bytes", &self.pending_bytes())
.finish_non_exhaustive()
}
}
pub struct Outbox {
core: Arc<Core>,
batch: Vec<u8>,
at: usize,
}
impl Outbox {
pub fn poll_chunk(&mut self, cx: &mut Context<'_>) -> Poll<&[u8]> {
if self.at < self.batch.len() {
return Poll::Ready(&self.batch[self.at..]);
}
let mut state = self.core.lock();
if !self.batch.is_empty() {
state.written();
recycle(&mut self.batch);
self.at = 0;
}
if state.failed.is_some() {
state.finished = true;
return Poll::Ready(&[]);
}
if !state.queued.is_empty() {
state.take(&mut self.batch);
drop(state);
return Poll::Ready(&self.batch);
}
if state.closed {
state.finished = true;
return Poll::Ready(&[]);
}
if !state
.waker
.as_ref()
.is_some_and(|waker| waker.will_wake(cx.waker()))
{
state.waker = Some(cx.waker().clone());
}
Poll::Pending
}
pub fn consume(&mut self, n: usize) {
self.at = self.at.saturating_add(n).min(self.batch.len());
if n > 0 {
self.core.progress();
}
}
#[must_use]
pub fn pending_bytes(&self) -> usize {
self.core.pending().0
}
#[cfg(feature = "futures-io")]
pub(crate) fn current(&self) -> &[u8] {
&self.batch[self.at..]
}
#[cfg(feature = "futures-io")]
pub(crate) fn abandon(&mut self, error: io::Error) -> io::Error {
let _failed = self
.core
.lock()
.fail(error.kind(), format!("writing to the peer failed: {error}"));
error
}
}
impl Drop for Outbox {
fn drop(&mut self) {
let mut state = self.core.lock();
state.finished = true;
let _failed = state.fail(
io::ErrorKind::BrokenPipe,
"the outbox was dropped: nothing reaches the peer".into(),
);
drop(state);
self.core.signal.notify_all();
}
}
impl fmt::Debug for Outbox {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Outbox")
.field("framing", &self.core.framing)
.field("pending_bytes", &self.pending_bytes())
.finish_non_exhaustive()
}
}
#[cfg(test)]
#[allow(clippy::panic)]
mod tests {
use std::{
sync::mpsc,
time::{Duration, Instant},
};
use super::*;
struct Gate {
go: mpsc::Receiver<()>,
out: mpsc::Sender<Vec<u8>>,
}
impl Write for Gate {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if self.go.recv().is_err() {
return Err(io::Error::other("released"));
}
let _sent = self.out.send(buf.to_vec());
Ok(buf.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[test]
fn test_queue_overflow_fails_the_queue_but_admits_one_message() {
let (mut sink, _outbox) = outbox(
Framing::Newline,
QueueLimits::new().with_max_bytes(4).with_max_messages(10),
);
assert!(sink.send("123456").is_ok());
let error = sink.send("1").err().map(|e| e.to_string());
assert_eq!(
error.as_deref(),
Some("the output queue is full: the peer is not reading")
);
assert!(sink.flush().is_err());
assert!(sink.send("").is_err());
}
#[test]
fn test_message_limit() {
let (mut sink, _outbox) = outbox(Framing::Newline, QueueLimits::new().with_max_messages(2));
assert!(sink.send("1").is_ok() && sink.send("2").is_ok());
assert!(sink.send("3").is_err());
}
#[test]
fn test_outbox_stall_times_out_the_sender() {
let mut cx = Context::from_waker(Waker::noop());
let (mut sink, mut outbox) = outbox(
Framing::Newline,
QueueLimits::new().with_write_timeout(Some(Duration::from_millis(10))),
);
assert!(sink.send("1").is_ok());
assert!(outbox.poll_chunk(&mut cx).is_ready()); thread::sleep(Duration::from_millis(30));
let error = sink.send("2").err().map(|e| e.kind());
assert_eq!(error, Some(io::ErrorKind::TimedOut));
assert_eq!(outbox.poll_chunk(&mut cx), Poll::Ready(&b"1\n"[..]));
outbox.consume(2);
assert_eq!(outbox.poll_chunk(&mut cx), Poll::Ready(&b""[..]));
}
#[test]
fn test_dropped_outbox_fails_the_sink() {
let (mut sink, outbox) = outbox(Framing::Newline, QueueLimits::new());
drop(outbox);
assert_eq!(
sink.send("x").err().map(|e| e.kind()),
Some(io::ErrorKind::BrokenPipe)
);
assert!(sink.close().is_err());
}
#[test]
fn test_outbox_partial_consume_and_recycling() {
let mut cx = Context::from_waker(Waker::noop());
let (mut sink, mut outbox) = outbox(Framing::Newline, QueueLimits::new());
let big = "x".repeat(4 * KEEP);
assert!(sink.send(&big).is_ok());
let Poll::Ready(chunk) = outbox.poll_chunk(&mut cx) else {
panic!("nothing to write")
};
assert_eq!(chunk.len(), 4 * KEEP + 1);
outbox.consume(10);
let Poll::Ready(rest) = outbox.poll_chunk(&mut cx) else {
panic!("the rest is gone")
};
assert_eq!(rest.len(), 4 * KEEP + 1 - 10);
outbox.consume(usize::MAX);
assert!(outbox.poll_chunk(&mut cx).is_pending());
assert!(outbox.batch.capacity() <= 2 * KEEP);
assert_eq!(outbox.pending_bytes(), 0);
assert!(format!("{outbox:?} {sink:?}").contains("pending_bytes: 0"));
}
#[test]
fn test_queued_writes_in_order_off_the_sender_thread() {
let (go, release) = mpsc::channel();
let (out, written) = mpsc::channel();
let mut sink = match Queued::new(Gate { go: release, out }, Framing::Newline) {
Ok(sink) => sink,
Err(error) => panic!("{error}"),
};
let started = Instant::now();
for i in 0..100 {
assert!(sink.send(&i.to_string()).is_ok());
}
assert!(started.elapsed() < Duration::from_secs(1));
assert!(sink.pending_messages() > 0);
for _ in 0..100 {
let _go = go.send(());
}
let mut text = Vec::new();
while text.len() < 290 {
match written.recv_timeout(Duration::from_secs(5)) {
Ok(chunk) => text.extend(chunk),
Err(_) => panic!("the writer stopped"),
}
}
let expected: Vec<u8> = (0..100)
.flat_map(|i| format!("{i}\n").into_bytes())
.collect();
assert_eq!(text, expected);
drop(go);
assert!(sink.close().is_ok());
assert!(format!("{sink:?}").starts_with("Queued"));
}
#[test]
fn test_queued_reports_a_failed_write_and_a_stall() {
struct Broken;
impl Write for Broken {
fn write(&mut self, _: &[u8]) -> io::Result<usize> {
Err(io::Error::new(io::ErrorKind::BrokenPipe, "gone"))
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
let mut sink = match Queued::new(Broken, Framing::Newline) {
Ok(sink) => sink,
Err(error) => panic!("{error}"),
};
assert!(sink.send("x").is_ok());
let deadline = Instant::now() + Duration::from_secs(5);
while sink.flush().is_ok() {
assert!(Instant::now() < deadline, "the failure was not reported");
thread::sleep(Duration::from_millis(2));
}
assert_eq!(
sink.send("y").err().map(|e| e.kind()),
Some(io::ErrorKind::BrokenPipe)
);
let (_go, release) = mpsc::channel();
let (out, _written) = mpsc::channel();
let limits = QueueLimits::new().with_write_timeout(Some(Duration::from_millis(20)));
let mut sink =
match Queued::with_limits(Gate { go: release, out }, Framing::Newline, limits) {
Ok(sink) => sink,
Err(error) => panic!("{error}"),
};
assert!(sink.send("x").is_ok());
let deadline = Instant::now() + Duration::from_secs(30);
while sink.core.lock().progressed.is_none() {
assert!(Instant::now() < deadline, "the writer never took the batch");
thread::sleep(Duration::from_millis(1));
}
thread::sleep(Duration::from_millis(60));
assert_eq!(
sink.send("y").err().map(|e| e.kind()),
Some(io::ErrorKind::TimedOut)
);
let started = Instant::now();
assert!(sink.close().is_err());
assert!(started.elapsed() < Duration::from_secs(5));
}
#[test]
fn test_queued_contains_a_panicking_writer() {
struct Panics;
impl Write for Panics {
fn write(&mut self, _: &[u8]) -> io::Result<usize> {
panic!("deliberate: writer")
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
let mut sink = match Queued::new(Panics, Framing::Newline) {
Ok(sink) => sink,
Err(error) => panic!("{error}"),
};
assert!(sink.send("x").is_ok());
let deadline = Instant::now() + Duration::from_secs(5);
while sink.flush().is_ok() {
assert!(Instant::now() < deadline, "the panic was not reported");
thread::sleep(Duration::from_millis(2));
}
let error = sink.flush().err().map(|e| e.to_string());
assert_eq!(
error.as_deref(),
Some("writing to the peer failed: the writer panicked")
);
}
}