use crate::{
signal::Signal,
telemetry::metrics::{Metric, Registered},
Blob, BufMut, BufferPool, BufferPooler, Clock, Error, Handle, IoBufs, IoBufsMut, Metrics, Name,
Spawner, Storage, Supervisor,
};
use bytes::{Bytes, BytesMut};
use commonware_utils::{
channel::{fallible::OneshotExt, oneshot},
sync::Mutex,
};
use governor::clock::{Clock as GovernorClock, ReasonablyRealtime};
use rand::{TryCryptoRng, TryRng};
use std::{future::Future, mem, sync::Arc};
const DEFAULT_BUFFER_SIZE: usize = 64 * 1024;
pub struct Channel {
buffer: BytesMut,
waiter: Option<(usize, oneshot::Sender<Bytes>)>,
buffer_size: usize,
drain_waiter: Option<oneshot::Sender<()>>,
sink_alive: bool,
stream_alive: bool,
}
impl Channel {
pub fn init() -> (Sink, Stream) {
Self::init_with_buffer_size(DEFAULT_BUFFER_SIZE)
}
pub fn init_with_buffer_size(buffer_size: usize) -> (Sink, Stream) {
let channel = Arc::new(Mutex::new(Self {
buffer: BytesMut::new(),
waiter: None,
buffer_size,
drain_waiter: None,
sink_alive: true,
stream_alive: true,
}));
(
Sink {
channel: channel.clone(),
state: SinkState::Open,
},
Stream {
channel,
buffer: BytesMut::new(),
poisoned: false,
},
)
}
fn restore_front(&mut self, data: Bytes) {
if data.is_empty() {
return;
}
let mut restored = BytesMut::with_capacity(data.len() + self.buffer.len());
restored.extend_from_slice(&data);
restored.extend_from_slice(&self.buffer);
self.buffer = restored;
}
fn close_sink(&mut self) {
self.sink_alive = false;
self.waiter.take();
}
}
struct RecvWaiterGuard {
channel: Arc<Mutex<Channel>>,
active: bool,
}
impl RecvWaiterGuard {
const fn new(channel: Arc<Mutex<Channel>>) -> Self {
Self {
channel,
active: true,
}
}
const fn disarm(&mut self) {
self.active = false;
}
}
impl Drop for RecvWaiterGuard {
fn drop(&mut self) {
if !self.active {
return;
}
self.channel.lock().waiter.take();
}
}
pub struct Sink {
channel: Arc<Mutex<Channel>>,
state: SinkState,
}
enum SinkState {
Open,
Sending,
Closed,
}
impl Sink {
fn close(&mut self) {
if matches!(self.state, SinkState::Closed) {
return;
}
self.channel.lock().close_sink();
self.state = SinkState::Closed;
}
}
impl crate::Sink for Sink {
async fn send(&mut self, bufs: impl Into<IoBufs> + Send) -> Result<(), Error> {
match self.state {
SinkState::Open => {}
SinkState::Sending => {
self.close();
return Err(Error::Closed);
}
SinkState::Closed => return Err(Error::Closed),
}
let drain_recv = {
let mut channel = self.channel.lock();
if !channel.stream_alive {
channel.close_sink();
self.state = SinkState::Closed;
return Err(Error::SendFailed);
}
channel.buffer.put(bufs.into());
if channel
.waiter
.as_ref()
.is_some_and(|(requested, _)| *requested <= channel.buffer.len())
{
let (requested, os_send) = channel.waiter.take().unwrap();
let send_amount = channel.buffer.len().min(requested.max(channel.buffer_size));
let data = channel.buffer.split_to(send_amount).freeze();
if let Err(data) = os_send.send(data) {
channel.restore_front(data);
if !channel.stream_alive {
channel.close_sink();
self.state = SinkState::Closed;
return Err(Error::SendFailed);
}
}
}
if channel.buffer.len() > channel.buffer_size {
assert!(channel.drain_waiter.is_none());
let (os_send, os_recv) = oneshot::channel();
channel.drain_waiter = Some(os_send);
os_recv
} else {
return Ok(());
}
};
self.state = SinkState::Sending;
match drain_recv.await {
Ok(()) => {
self.state = SinkState::Open;
Ok(())
}
Err(_) => {
self.close();
Err(Error::SendFailed)
}
}
}
}
impl Drop for Sink {
fn drop(&mut self) {
self.close();
}
}
pub struct Stream {
channel: Arc<Mutex<Channel>>,
buffer: BytesMut,
poisoned: bool,
}
impl crate::Stream for Stream {
async fn recv(&mut self, len: usize) -> Result<IoBufs, Error> {
if self.poisoned {
return Err(Error::Closed);
}
let os_recv = {
let mut channel = self.channel.lock();
let target = len.max(channel.buffer_size);
let pull_amount = channel
.buffer
.len()
.min(target.saturating_sub(self.buffer.len()));
if pull_amount > 0 {
let data = channel.buffer.split_to(pull_amount);
self.buffer.extend_from_slice(&data);
if channel.buffer.len() <= channel.buffer_size {
if let Some(sender) = channel.drain_waiter.take() {
sender.send_lossy(());
}
}
}
if self.buffer.len() >= len {
return Ok(IoBufs::from(self.buffer.split_to(len).freeze()));
}
if !channel.sink_alive {
self.poisoned = true;
return Err(Error::RecvFailed);
}
let remaining = len - self.buffer.len();
assert!(channel.waiter.is_none());
let (os_send, os_recv) = oneshot::channel();
channel.waiter = Some((remaining, os_send));
os_recv
};
let mut waiter_guard = RecvWaiterGuard::new(self.channel.clone());
self.poisoned = true;
let data = match os_recv.await {
Ok(data) => {
waiter_guard.disarm();
self.poisoned = false;
data
}
Err(_) => {
waiter_guard.disarm();
return Err(Error::RecvFailed);
}
};
self.buffer.extend_from_slice(&data);
assert!(self.buffer.len() >= len);
Ok(IoBufs::from(self.buffer.split_to(len).freeze()))
}
fn peek(&self, max_len: usize) -> &[u8] {
let len = max_len.min(self.buffer.len());
&self.buffer[..len]
}
}
impl Drop for Stream {
fn drop(&mut self) {
let mut channel = self.channel.lock();
channel.stream_alive = false;
channel.drain_waiter.take();
}
}
pub struct DeferredSync {
pub release: oneshot::Sender<Result<(), Error>>,
pub blocked: oneshot::Receiver<()>,
}
#[derive(Clone, Default)]
pub struct PendingSyncs {
syncs: Arc<Mutex<Vec<DeferredSync>>>,
gate: Arc<Mutex<SyncGateState>>,
}
macro_rules! forward_context {
($wrapper:ident, $field:ident) => {
impl<E: Supervisor> Supervisor for $wrapper<E> {
fn name(&self) -> Name {
self.inner.name()
}
fn child(&self, label: &'static str) -> Self {
Self {
inner: self.inner.child(label),
$field: self.$field.clone(),
}
}
fn with_attribute(self, key: &'static str, value: impl std::fmt::Display) -> Self {
Self {
inner: self.inner.with_attribute(key, value),
$field: self.$field,
}
}
}
impl<E: Clock> Clock for $wrapper<E> {
fn current(&self) -> std::time::SystemTime {
self.inner.current()
}
fn sleep(
&self,
duration: std::time::Duration,
) -> impl Future<Output = ()> + Send + 'static {
self.inner.sleep(duration)
}
fn sleep_until(
&self,
deadline: std::time::SystemTime,
) -> impl Future<Output = ()> + Send + 'static {
self.inner.sleep_until(deadline)
}
}
impl<E: Clock> GovernorClock for $wrapper<E> {
type Instant = std::time::SystemTime;
fn now(&self) -> Self::Instant {
self.current()
}
}
impl<E: Clock> ReasonablyRealtime for $wrapper<E> {}
impl<E: Metrics> Metrics for $wrapper<E> {
fn register<N: Into<String>, H: Into<String>, M: Metric>(
&self,
name: N,
help: H,
metric: M,
) -> Registered<M> {
self.inner.register(name, help, metric)
}
fn encode(&self) -> String {
self.inner.encode()
}
}
impl<E: BufferPooler> BufferPooler for $wrapper<E> {
fn network_buffer_pool(&self) -> &BufferPool {
self.inner.network_buffer_pool()
}
fn storage_buffer_pool(&self) -> &BufferPool {
self.inner.storage_buffer_pool()
}
}
impl<E: TryRng> TryRng for $wrapper<E> {
type Error = E::Error;
fn try_next_u32(&mut self) -> Result<u32, Self::Error> {
self.inner.try_next_u32()
}
fn try_next_u64(&mut self) -> Result<u64, Self::Error> {
self.inner.try_next_u64()
}
fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Self::Error> {
self.inner.try_fill_bytes(dest)
}
}
impl<E: TryCryptoRng> TryCryptoRng for $wrapper<E> {}
};
}
#[derive(Clone)]
pub struct DelayedSyncContext<E> {
pub inner: E,
pub pending: PendingSyncs,
}
forward_context!(DelayedSyncContext, pending);
impl<E: Spawner> Spawner for DelayedSyncContext<E> {
fn shared(mut self, blocking: bool) -> Self {
self.inner = self.inner.shared(blocking);
self
}
fn dedicated(mut self) -> Self {
self.inner = self.inner.dedicated();
self
}
fn spawn<F, Fut, T>(self, f: F) -> Handle<T>
where
F: FnOnce(Self) -> Fut + Send + 'static,
Fut: Future<Output = T> + Send + 'static,
T: Send + 'static,
{
let pending = self.pending;
self.inner.spawn(move |inner| f(Self { inner, pending }))
}
async fn stop(self, value: i32, timeout: Option<std::time::Duration>) -> Result<(), Error> {
self.inner.stop(value, timeout).await
}
fn stopped(&self) -> Signal {
self.inner.stopped()
}
}
impl<E: Storage> Storage for DelayedSyncContext<E> {
type Blob = DelayedSyncBlob<E::Blob>;
async fn open_versioned(
&self,
partition: &str,
name: &[u8],
versions: std::ops::RangeInclusive<u16>,
) -> Result<(Self::Blob, u64, u16), Error> {
let (inner, len, version) = self.inner.open_versioned(partition, name, versions).await?;
Ok((
DelayedSyncBlob {
inner,
pending: self.pending.clone(),
},
len,
version,
))
}
async fn remove(&self, partition: &str, name: Option<&[u8]>) -> Result<(), Error> {
self.inner.remove(partition, name).await
}
async fn scan(&self, partition: &str) -> Result<Vec<Vec<u8>>, Error> {
self.inner.scan(partition).await
}
}
#[derive(Clone)]
pub struct DelayedSyncBlob<B> {
inner: B,
pending: PendingSyncs,
}
impl<B> DelayedSyncBlob<B> {
pub fn new(inner: B) -> (Self, PendingSyncs) {
let pending = PendingSyncs::default();
(
Self {
inner,
pending: pending.clone(),
},
pending,
)
}
}
impl<B: Blob> Blob for DelayedSyncBlob<B> {
async fn read_at_buf(
&self,
offset: u64,
len: usize,
bufs: impl Into<IoBufsMut> + Send,
) -> Result<IoBufsMut, Error> {
self.inner.read_at_buf(offset, len, bufs).await
}
async fn read_at(&self, offset: u64, len: usize) -> Result<IoBufsMut, Error> {
self.inner.read_at(offset, len).await
}
async fn write_at(&self, offset: u64, bufs: impl Into<IoBufs> + Send) -> Result<(), Error> {
self.inner.write_at(offset, bufs).await
}
async fn write_at_sync(
&self,
offset: u64,
bufs: impl Into<IoBufs> + Send,
) -> Result<(), Error> {
if !self.pending.tracking() {
return self.inner.write_at_sync(offset, bufs).await;
}
self.inner.write_at(offset, bufs).await?;
self.pending.wait().await?;
self.inner.sync().await
}
async fn resize(&self, len: u64) -> Result<(), Error> {
self.inner.resize(len).await
}
async fn sync(&self) -> Result<(), Error> {
self.pending.wait().await?;
self.inner.sync().await
}
async fn start_sync(&self) -> Handle<()> {
let inner = self.inner.clone();
let waiter = self
.pending
.observe()
.unwrap_or_else(|| self.pending.defer());
Handle::from_future(async move {
waiter.wait().await?;
inner.sync().await
})
}
}
pub fn next_pending_sync(pending: &PendingSyncs) -> DeferredSync {
let mut pending = pending.lock();
assert!(!pending.is_empty(), "no pending sync was started");
pending.remove(0)
}
pub fn release_next_pending_syncs(pending: &PendingSyncs, count: usize) {
let syncs = {
let mut pending = pending.lock();
assert!(
pending.len() >= count,
"not enough pending syncs: have {}, need {count}",
pending.len()
);
pending.drain(..count).collect::<Vec<_>>()
};
for sync in syncs {
let _ = sync.release.send(Ok(()));
}
}
pub fn release_pending_syncs(pending: &PendingSyncs) {
for sync in mem::take(&mut *pending.lock()) {
let _ = sync.release.send(Ok(()));
}
}
pub fn fail_pending_syncs(pending: &PendingSyncs) {
for sync in mem::take(&mut *pending.lock()) {
let err = std::io::Error::other("injected sync failure");
let _ = sync.release.send(Err(Error::Io(err.into())));
}
}
struct SyncWaiter {
entered: oneshot::Sender<()>,
release: oneshot::Receiver<Result<(), Error>>,
}
impl SyncWaiter {
async fn wait(self) -> Result<(), Error> {
self.entered.send_lossy(());
self.release.await.map_err(|_| Error::Closed)??;
Ok(())
}
}
#[derive(Default)]
struct SyncGateState {
tracking: bool,
calls: usize,
waiter: Option<SyncWaiter>,
}
impl PendingSyncs {
pub fn lock(&self) -> commonware_utils::sync::MutexGuard<'_, Vec<DeferredSync>> {
self.syncs.lock()
}
pub fn arm(&self) {
let mut state = self.gate.lock();
assert!(!state.tracking, "sync gate already armed");
assert!(state.waiter.is_none(), "sync gate already has a waiter");
state.tracking = true;
state.calls = 0;
state.waiter = Some(self.defer());
}
pub fn calls(&self) -> usize {
self.gate.lock().calls
}
fn tracking(&self) -> bool {
self.gate.lock().tracking
}
fn defer(&self) -> SyncWaiter {
let (release, release_rx) = oneshot::channel();
let (entered, blocked) = oneshot::channel();
self.syncs.lock().push(DeferredSync { release, blocked });
SyncWaiter {
entered,
release: release_rx,
}
}
fn observe(&self) -> Option<SyncWaiter> {
let mut state = self.gate.lock();
if !state.tracking {
return None;
}
state.calls += 1;
state.waiter.take()
}
async fn wait(&self) -> Result<(), Error> {
match self.observe() {
Some(waiter) => waiter.wait().await,
None => Ok(()),
}
}
}
#[derive(Clone)]
pub struct SyncFaultContext<E> {
pub inner: E,
pub fail_partition: String,
}
forward_context!(SyncFaultContext, fail_partition);
impl<E: Storage> Storage for SyncFaultContext<E> {
type Blob = SyncFaultBlob<E::Blob>;
async fn open_versioned(
&self,
partition: &str,
name: &[u8],
versions: std::ops::RangeInclusive<u16>,
) -> Result<(Self::Blob, u64, u16), Error> {
let (inner, len, version) = self.inner.open_versioned(partition, name, versions).await?;
Ok((
SyncFaultBlob {
inner,
faulty: partition == self.fail_partition,
},
len,
version,
))
}
async fn remove(&self, partition: &str, name: Option<&[u8]>) -> Result<(), Error> {
self.inner.remove(partition, name).await
}
async fn scan(&self, partition: &str) -> Result<Vec<Vec<u8>>, Error> {
self.inner.scan(partition).await
}
}
#[derive(Clone)]
pub struct SyncFaultBlob<B> {
inner: B,
faulty: bool,
}
impl<B: Blob> Blob for SyncFaultBlob<B> {
async fn read_at_buf(
&self,
offset: u64,
len: usize,
bufs: impl Into<IoBufsMut> + Send,
) -> Result<IoBufsMut, Error> {
self.inner.read_at_buf(offset, len, bufs).await
}
async fn read_at(&self, offset: u64, len: usize) -> Result<IoBufsMut, Error> {
self.inner.read_at(offset, len).await
}
async fn write_at(&self, offset: u64, bufs: impl Into<IoBufs> + Send) -> Result<(), Error> {
self.inner.write_at(offset, bufs).await
}
async fn write_at_sync(
&self,
offset: u64,
bufs: impl Into<IoBufs> + Send,
) -> Result<(), Error> {
self.inner.write_at_sync(offset, bufs).await
}
async fn resize(&self, len: u64) -> Result<(), Error> {
self.inner.resize(len).await
}
async fn sync(&self) -> Result<(), Error> {
if self.faulty {
let err = std::io::Error::other("injected partition sync fault");
return Err(Error::Io(err.into()));
}
self.inner.sync().await
}
async fn start_sync(&self) -> Handle<()> {
if self.faulty {
return Handle::ready(self.sync().await);
}
self.inner.start_sync().await
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{deterministic, Clock, Runner, Sink, Spawner, Stream};
use commonware_macros::select;
use std::{thread::sleep, time::Duration};
#[test]
fn test_send_recv() {
let (mut sink, mut stream) = Channel::init();
let data = b"hello world";
let executor = deterministic::Runner::default();
executor.start(|_| async move {
sink.send(data.as_slice()).await.unwrap();
let received = stream.recv(data.len()).await.unwrap();
assert_eq!(received.coalesce(), data);
});
}
#[test]
fn test_send_recv_partial_multiple() {
let (mut sink, mut stream) = Channel::init();
let data = b"hello";
let data2 = b" world";
let executor = deterministic::Runner::default();
executor.start(|_| async move {
sink.send(data.as_slice()).await.unwrap();
sink.send(data2.as_slice()).await.unwrap();
let received = stream.recv(5).await.unwrap();
assert_eq!(received.coalesce(), b"hello");
let received = stream.recv(5).await.unwrap();
assert_eq!(received.coalesce(), b" worl");
let received = stream.recv(1).await.unwrap();
assert_eq!(received.coalesce(), b"d");
});
}
#[test]
fn test_send_recv_async() {
let (mut sink, mut stream) = Channel::init();
let data = b"hello world";
let executor = deterministic::Runner::default();
executor.start(|_| async move {
let (received, _) = futures::try_join!(stream.recv(data.len()), async {
sleep(Duration::from_millis(50));
sink.send(data.as_slice()).await
})
.unwrap();
assert_eq!(received.coalesce(), data);
});
}
#[test]
fn test_recv_error_sink_dropped_while_waiting() {
let (sink, mut stream) = Channel::init();
let executor = deterministic::Runner::default();
executor.start(|context| async move {
futures::join!(
async {
let result = stream.recv(5).await;
assert!(matches!(result, Err(Error::RecvFailed)));
let result = stream.recv(5).await;
assert!(matches!(result, Err(Error::Closed)));
},
async {
context.sleep(Duration::from_millis(50)).await;
drop(sink);
}
);
});
}
#[test]
fn test_recv_error_sink_dropped_before_recv() {
let (sink, mut stream) = Channel::init();
drop(sink);
let executor = deterministic::Runner::default();
executor.start(|_| async move {
let result = stream.recv(5).await;
assert!(matches!(result, Err(Error::RecvFailed)));
let result = stream.recv(5).await;
assert!(matches!(result, Err(Error::Closed)));
});
}
#[test]
fn test_send_error_stream_dropped() {
let (mut sink, mut stream) = Channel::init();
let executor = deterministic::Runner::default();
executor.start(|context| async move {
assert!(sink.send(b"7 bytes".as_slice()).await.is_ok());
let handle = context.child("recv").spawn(|_| async move {
let _ = stream.recv(5).await;
let _ = stream.recv(5).await;
});
context.sleep(Duration::from_millis(50)).await;
handle.abort();
assert!(matches!(handle.await, Err(Error::Closed)));
let result = sink.send(b"hello world".as_slice()).await;
assert!(matches!(result, Err(Error::SendFailed)));
let result = sink.send(b"hello world".as_slice()).await;
assert!(matches!(result, Err(Error::Closed)));
});
}
#[test]
fn test_send_error_stream_dropped_before_send() {
let (mut sink, stream) = Channel::init();
drop(stream);
let executor = deterministic::Runner::default();
executor.start(|_| async move {
let result = sink.send(b"hello world".as_slice()).await;
assert!(matches!(result, Err(Error::SendFailed)));
let result = sink.send(b"hello world".as_slice()).await;
assert!(matches!(result, Err(Error::Closed)));
});
}
#[test]
fn test_recv_timeout() {
let (_sink, mut stream) = Channel::init();
let executor = deterministic::Runner::default();
executor.start(|context| async move {
select! {
v = stream.recv(5) => {
panic!("unexpected value: {v:?}");
},
_ = context.sleep(Duration::from_millis(100)) => "timeout",
};
});
}
#[test]
fn test_peek_empty() {
let (_sink, stream) = Channel::init();
assert!(stream.peek(10).is_empty());
}
#[test]
fn test_peek_after_partial_recv() {
let (mut sink, mut stream) = Channel::init();
let executor = deterministic::Runner::default();
executor.start(|_| async move {
sink.send(b"hello world".as_slice()).await.unwrap();
let received = stream.recv(5).await.unwrap();
assert_eq!(received.coalesce(), b"hello");
assert_eq!(stream.peek(100), b" world");
assert_eq!(stream.peek(3), b" wo");
assert_eq!(stream.peek(100), b" world");
let received = stream.recv(6).await.unwrap();
assert_eq!(received.coalesce(), b" world");
assert!(stream.peek(100).is_empty());
});
}
#[test]
fn test_peek_after_recv_wakeup() {
let (mut sink, mut stream) = Channel::init_with_buffer_size(64);
let executor = deterministic::Runner::default();
executor.start(|context| async move {
let (tx, rx) = oneshot::channel();
let recv_handle = context.child("recv").spawn(|_| async move {
let data = stream.recv(3).await.unwrap();
tx.send(stream).ok();
data
});
context.sleep(Duration::from_millis(10)).await;
sink.send(b"ABCDEFGHIJ".as_slice()).await.unwrap();
let received = recv_handle.await.unwrap();
assert_eq!(received.coalesce(), b"ABC");
let stream = rx.await.unwrap();
assert_eq!(stream.peek(100), b"DEFGHIJ");
});
}
#[test]
fn test_peek_multiple_sends() {
let (mut sink, mut stream) = Channel::init();
let executor = deterministic::Runner::default();
executor.start(|_| async move {
sink.send(b"aaa".as_slice()).await.unwrap();
sink.send(b"bbb".as_slice()).await.unwrap();
sink.send(b"ccc".as_slice()).await.unwrap();
let received = stream.recv(4).await.unwrap();
assert_eq!(received.coalesce(), b"aaab");
assert_eq!(stream.peek(100), b"bbccc");
});
}
#[test]
fn test_buffer_size_limit() {
let (mut sink, mut stream) = Channel::init_with_buffer_size(10);
let executor = deterministic::Runner::default();
executor.start(|context| async move {
let send_handle = context.child("sender").spawn(|_| async move {
sink.send(b"0123456789ABCDEF".as_slice()).await.unwrap();
sink
});
let received = stream.recv(2).await.unwrap();
assert_eq!(received.coalesce(), b"01");
assert_eq!(stream.peek(100), b"23456789");
let received = stream.recv(8).await.unwrap();
assert_eq!(received.coalesce(), b"23456789");
let received = stream.recv(2).await.unwrap();
assert_eq!(received.coalesce(), b"AB");
assert_eq!(stream.peek(100), b"CDEF");
send_handle.await.unwrap();
});
}
#[test]
fn test_recv_before_send() {
let (mut sink, mut stream) = Channel::init_with_buffer_size(10);
let executor = deterministic::Runner::default();
executor.start(|context| async move {
let recv_handle = context
.child("recv")
.spawn(|_| async move { stream.recv(3).await.unwrap() });
context.sleep(Duration::from_millis(10)).await;
sink.send(b"ABCDEFGHIJKLMNOP".as_slice()).await.unwrap();
let received = recv_handle.await.unwrap();
assert_eq!(received.coalesce(), b"ABC");
});
}
}