use std::{
any::Any,
collections::VecDeque,
fmt,
future::{self, Future},
io::{self, Read, Seek, SeekFrom, Write},
mem,
num::NonZeroUsize,
panic::{self, AssertUnwindSafe},
pin::Pin,
sync::{Arc, Mutex, MutexGuard, PoisonError},
task::{Context, Poll, Wake, Waker, ready},
};
use futures_core::Stream;
use futures_io::{AsyncRead, AsyncSeek, AsyncWrite};
use super::{BlockingWork, dispose, unblock};
pub struct Unblock<T> {
inner: Mutex<Inner<T>>,
}
impl<T> Unblock<T> {
pub fn new(io: T) -> Self {
Self::with_capacity(DEFAULT_CAPACITY, io)
}
pub fn with_capacity(cap: NonZeroUsize, io: T) -> Self {
Self {
inner: Mutex::new(Inner {
state: State::Idle(io),
read_ahead: ReadAhead::default(),
write_buf: Vec::new(),
write_error: None,
items: None,
waiters: Arc::default(),
cap,
}),
}
}
pub async fn get_mut(&mut self) -> &mut T {
let inner = self.inner();
future::poll_fn(|cx| inner.poll_idle(cx)).await;
inner.drop_ahead();
let State::Idle(handle) = &mut inner.state else {
unreachable!("the handle is back once no operation is in flight");
};
handle
}
pub async fn with_mut<R, F>(&mut self, op: F) -> R
where
T: Send + 'static,
F: FnOnce(&mut T) -> R + Send + 'static,
R: Send + 'static,
{
let inner = self.inner();
future::poll_fn(|cx| inner.poll_idle(cx)).await;
inner.drop_ahead();
inner.start(move |handle| Outcome::Ran(Box::new(op(handle))));
let Some(Outcome::Ran(value)) = future::poll_fn(|cx| inner.poll_job(cx)).await else {
unreachable!("the operation in flight is the one `with_mut` started");
};
match value.downcast() {
Ok(value) => *value,
Err(_) => unreachable!("`with_mut` hands back what its own operation returned"),
}
}
pub async fn into_inner(mut self) -> T {
let inner = self.inner();
future::poll_fn(|cx| inner.poll_idle(cx)).await;
let inner = self
.inner
.into_inner()
.unwrap_or_else(PoisonError::into_inner);
let State::Idle(handle) = inner.state else {
unreachable!("the handle is back once no operation is in flight");
};
handle
}
fn inner(&mut self) -> &mut Inner<T> {
let inner = self.inner.get_mut().unwrap_or_else(PoisonError::into_inner);
if let State::Lost = inner.state {
lost();
}
inner
}
}
impl<T> Unpin for Unblock<T> {}
impl<T> fmt::Debug for Unblock<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Unblock").finish_non_exhaustive()
}
}
impl<T> AsyncRead for Unblock<T>
where
T: Read + Send + 'static,
{
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut [u8],
) -> Poll<io::Result<usize>> {
let inner = self.get_mut().inner();
if buf.is_empty() {
return Poll::Ready(Ok(0));
}
loop {
if let Some(read) = inner.read_ahead.hand_out(buf) {
return Poll::Ready(read);
}
match ready!(inner.poll_job(cx)) {
Some(outcome) => inner.keep(outcome),
None => {
let mut ahead = inner.read_ahead.take_buf();
ahead.resize(inner.cap.get(), 0);
inner.start(move |handle| {
let read = uninterrupted(|| handle.read(&mut ahead));
Outcome::Read(ahead, read)
});
}
}
}
}
}
impl<T> AsyncWrite for Unblock<T>
where
T: Write + Send + 'static,
{
fn poll_write(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
let inner = self.get_mut().inner();
ready!(inner.poll_write_ready(cx))?;
if buf.is_empty() {
return Poll::Ready(Ok(0));
}
let len = buf.len().min(inner.cap.get());
let mut bytes = mem::take(&mut inner.write_buf);
bytes.extend_from_slice(&buf[..len]);
inner.start(move |handle| {
let written = handle.write_all(&bytes);
bytes.clear();
Outcome::Written(bytes, written)
});
Poll::Ready(Ok(len))
}
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
let inner = self.get_mut().inner();
loop {
match ready!(inner.poll_job(cx)) {
Some(Outcome::Flushed(flushed)) => return Poll::Ready(flushed),
Some(outcome) => inner.keep(outcome),
None => {
if let Some(error) = inner.write_error.take() {
return Poll::Ready(Err(error));
}
inner.start(|handle| Outcome::Flushed(uninterrupted(|| handle.flush())));
}
}
}
}
fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
self.poll_flush(cx)
}
}
impl<T> AsyncSeek for Unblock<T>
where
T: Seek + Send + 'static,
{
fn poll_seek(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
pos: SeekFrom,
) -> Poll<io::Result<u64>> {
let inner = self.get_mut().inner();
loop {
match ready!(inner.poll_job(cx)) {
Some(Outcome::Sought {
asked,
sought,
read_ahead,
}) if asked == pos => {
inner.settle_seek(&sought, read_ahead);
return Poll::Ready(sought);
}
Some(outcome) => inner.keep(outcome),
None => {
let to = match pos {
SeekFrom::Current(offset) => {
let Some(offset) = inner.read_ahead.offset_from_handle(offset) else {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::InvalidInput,
"seek from the current position out of the range of an i64",
)));
};
SeekFrom::Current(offset)
}
SeekFrom::Start(_) | SeekFrom::End(_) => pos,
};
let read_ahead = mem::take(&mut inner.read_ahead);
inner.start(move |handle| Outcome::Sought {
asked: pos,
sought: uninterrupted(|| handle.seek(to)),
read_ahead,
});
}
}
}
}
}
impl<T> Stream for Unblock<T>
where
T: Iterator + Send + 'static,
T::Item: Send + 'static,
{
type Item = T::Item;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<T::Item>> {
let inner = self.get_mut().inner();
loop {
if let Some(batch) = &mut inner.items {
let Some(batch) = batch.downcast_mut::<Batch<T::Item>>() else {
unreachable!("the items pulled ahead are the iterator's own");
};
if let Some(item) = batch.items.pop_front() {
return Poll::Ready(Some(item));
}
if mem::take(&mut batch.ended) {
return Poll::Ready(None);
}
}
match ready!(inner.poll_job(cx)) {
Some(outcome) => inner.keep(outcome),
None => {
let mut batch = match inner.items.take().map(|batch| batch.downcast()) {
Some(Ok(batch)) => batch,
Some(Err(_)) => {
unreachable!("the items pulled ahead are the iterator's own")
}
None => Box::new(Batch {
items: VecDeque::new(),
ended: false,
}),
};
let size = inner.cap.get().min(BATCH);
inner.start(move |iter| {
batch.pull(iter, size);
Outcome::Pulled(batch)
});
}
}
}
}
}
struct Inner<T> {
state: State<T>,
read_ahead: ReadAhead,
write_buf: Vec<u8>,
write_error: Option<io::Error>,
items: Option<Box<dyn Any + Send>>,
waiters: Arc<Waiters>,
cap: NonZeroUsize,
}
impl<T> Inner<T> {
fn poll_job(&mut self, cx: &mut Context<'_>) -> Poll<Option<Outcome>> {
let mut work = match mem::replace(&mut self.state, State::Lost) {
State::Busy(work) => work,
state @ State::Idle(_) => {
self.state = state;
return Poll::Ready(None);
}
State::Lost => lost(),
};
self.waiters.add(cx.waker());
let waker = Waker::from(self.waiters.clone());
let polled = panic::catch_unwind(AssertUnwindSafe(|| {
Pin::new(&mut work).poll(&mut Context::from_waker(&waker))
}));
match polled {
Ok(Poll::Pending) => {
self.state = State::Busy(work);
Poll::Pending
}
Ok(Poll::Ready(Done { handle, outcome })) => {
self.state = State::Idle(handle);
if let Some(panic) = self.waiters.wake_others(cx.waker()) {
self.keep(outcome);
panic::resume_unwind(panic);
}
Poll::Ready(Some(outcome))
}
Err(panic) => {
if let Some(waker_panic) = self.waiters.wake_others(cx.waker()) {
dispose(waker_panic);
}
panic::resume_unwind(panic)
}
}
}
fn poll_idle(&mut self, cx: &mut Context<'_>) -> Poll<()> {
while let Some(outcome) = ready!(self.poll_job(cx)) {
self.keep(outcome);
}
Poll::Ready(())
}
fn poll_write_ready(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
ready!(self.poll_idle(cx));
Poll::Ready(self.write_error.take().map_or(Ok(()), Err))
}
fn keep(&mut self, outcome: Outcome) {
match outcome {
Outcome::Read(buf, read) => self.read_ahead.fill(buf, read),
Outcome::Written(buf, written) => {
self.write_buf = buf;
if let Err(error) = written {
self.write_error = Some(error);
}
}
Outcome::Flushed(flushed) => {
if let Err(error) = flushed {
self.write_error = Some(error);
}
}
Outcome::Sought {
sought, read_ahead, ..
} => self.settle_seek(&sought, read_ahead),
Outcome::Pulled(batch) => self.items = Some(batch),
Outcome::Ran(_) => {}
}
}
fn settle_seek(&mut self, sought: &io::Result<u64>, mut read_ahead: ReadAhead) {
if sought.is_ok() {
read_ahead.clear();
}
self.read_ahead = read_ahead;
}
fn drop_ahead(&mut self) {
self.read_ahead.clear();
self.items = None;
}
fn start<F>(&mut self, op: F)
where
T: Send + 'static,
F: FnOnce(&mut T) -> Outcome + Send + 'static,
{
let State::Idle(mut handle) = mem::replace(&mut self.state, State::Lost) else {
unreachable!("a job starts only once the one before it is over");
};
self.state = State::Busy(unblock(move || {
let outcome = op(&mut handle);
Done { handle, outcome }
}));
}
}
enum State<T> {
Idle(T),
Busy(BlockingWork<Done<T>>),
Lost,
}
struct Done<T> {
handle: T,
outcome: Outcome,
}
enum Outcome {
Read(Vec<u8>, io::Result<usize>),
Written(Vec<u8>, io::Result<()>),
Flushed(io::Result<()>),
Sought {
asked: SeekFrom,
sought: io::Result<u64>,
read_ahead: ReadAhead,
},
Pulled(Box<dyn Any + Send>),
Ran(Box<dyn Any + Send>),
}
#[derive(Default)]
struct ReadAhead {
buf: Vec<u8>,
start: usize,
end: usize,
stop: Option<io::Result<()>>,
}
impl ReadAhead {
fn hand_out(&mut self, buf: &mut [u8]) -> Option<io::Result<usize>> {
let ahead = &self.buf[self.start..self.end];
if ahead.is_empty() {
return self.stop.take().map(|stop| stop.map(|()| 0));
}
let len = ahead.len().min(buf.len());
buf[..len].copy_from_slice(&ahead[..len]);
self.start += len;
Some(Ok(len))
}
fn take_buf(&mut self) -> Vec<u8> {
self.start = 0;
self.end = 0;
mem::take(&mut self.buf)
}
fn fill(&mut self, buf: Vec<u8>, read: io::Result<usize>) {
self.start = 0;
self.end = 0;
match read {
Ok(0) => self.stop = Some(Ok(())),
Ok(len) => self.end = len.min(buf.len()),
Err(error) => self.stop = Some(Err(error)),
}
self.buf = buf;
}
fn offset_from_handle(&self, offset: i64) -> Option<i64> {
offset.checked_sub(i64::try_from(self.end - self.start).ok()?)
}
fn clear(&mut self) {
self.start = 0;
self.end = 0;
self.stop = None;
}
}
struct Batch<I> {
items: VecDeque<I>,
ended: bool,
}
impl<I> Batch<I> {
fn pull<T>(&mut self, iter: &mut T, size: usize)
where
T: Iterator<Item = I>,
{
while self.items.len() < size {
let Some(item) = iter.next() else {
self.ended = true;
return;
};
self.items.push_back(item);
}
}
}
#[derive(Default)]
struct Waiters(Mutex<Vec<Waker>>);
impl Waiters {
fn add(&self, waker: &Waker) {
let mut wakers = self.wakers();
if !wakers.iter().any(|waiting| waiting.will_wake(waker)) {
wakers.push(waker.clone());
}
}
fn wake_others(&self, waker: &Waker) -> Option<Box<dyn Any + Send>> {
let wakers = mem::take(&mut *self.wakers());
wake_all(
wakers
.into_iter()
.filter(|waiting| !waiting.will_wake(waker)),
)
}
fn wakers(&self) -> MutexGuard<'_, Vec<Waker>> {
self.0.lock().unwrap_or_else(PoisonError::into_inner)
}
}
impl Wake for Waiters {
fn wake(self: Arc<Self>) {
self.wake_by_ref();
}
fn wake_by_ref(self: &Arc<Self>) {
let wakers = mem::take(&mut *self.wakers());
if let Some(panic) = wake_all(wakers) {
panic::resume_unwind(panic);
}
}
}
fn wake_all<I>(wakers: I) -> Option<Box<dyn Any + Send>>
where
I: IntoIterator<Item = Waker>,
{
let mut first_panic = None;
for waker in wakers {
if let Err(panic) = panic::catch_unwind(AssertUnwindSafe(|| waker.wake())) {
match first_panic {
None => first_panic = Some(panic),
Some(_) => dispose(panic),
}
}
}
first_panic
}
fn uninterrupted<F, R>(mut op: F) -> io::Result<R>
where
F: FnMut() -> io::Result<R>,
{
loop {
match op() {
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
outcome => return outcome,
}
}
}
fn lost() -> ! {
panic!("the handle of an `Unblock` was lost to a panic in an operation on it");
}
const DEFAULT_CAPACITY: NonZeroUsize = NonZeroUsize::new(8 * 1024).unwrap();
const BATCH: usize = 16;