use crate::{Error, Result, SuspensionReason, context, sync::wait::Wait};
use std::{
io,
os::fd::{AsRawFd, BorrowedFd, OwnedFd},
sync::{Arc, OnceLock},
};
const RUNNABLE_YIELD_LIMIT: usize = 128;
#[derive(Debug, Default)]
pub(super) struct ReadinessSource {
fd: OnceLock<Arc<OwnedFd>>,
}
impl ReadinessSource {
fn acquire(&self, fd: BorrowedFd<'_>) -> Result<Arc<OwnedFd>> {
if let Some(owned) = self.fd.get() {
return Ok(Arc::clone(owned));
}
let owned = Arc::new(checked(
"duplicate readiness descriptor",
fd,
fd.try_clone_to_owned(),
)?);
if self.fd.set(Arc::clone(&owned)).is_ok() {
Ok(owned)
} else {
Ok(Arc::clone(
self.fd.get().expect("readiness source initialized"),
))
}
}
}
pub(crate) fn checked<T>(
operation: &'static str,
fd: BorrowedFd<'_>,
result: io::Result<T>,
) -> Result<T> {
result.map_err(|error| Error::io(operation, format_args!("fd={}", fd.as_raw_fd()), error))
}
pub(super) fn attempt<T>(
fd: BorrowedFd<'_>,
reason: SuspensionReason,
mut operation: impl FnMut() -> io::Result<T>,
) -> Result<T> {
loop {
crate::checkpoint()?;
match operation() {
Ok(value) => return Ok(value),
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
return Err(Error::WouldBlock);
}
Err(error) => return Err(operation_error(reason, fd, error)),
}
}
}
pub(super) fn operation<T>(
fd: BorrowedFd<'_>,
interest: zio::Interest,
reason: SuspensionReason,
operation: impl FnMut() -> io::Result<T>,
) -> Result<T> {
operation_with_source(fd, None, interest, reason, operation)
}
pub(super) fn operation_cached<T>(
fd: BorrowedFd<'_>,
source: &ReadinessSource,
interest: zio::Interest,
reason: SuspensionReason,
operation: impl FnMut() -> io::Result<T>,
) -> Result<T> {
operation_with_source(fd, Some(source), interest, reason, operation)
}
fn operation_with_source<T>(
fd: BorrowedFd<'_>,
source: Option<&ReadinessSource>,
interest: zio::Interest,
reason: SuspensionReason,
mut operation: impl FnMut() -> io::Result<T>,
) -> Result<T> {
crate::checkpoint()?;
match operation() {
Ok(value) => Ok(value),
Err(error)
if !matches!(
error.kind(),
io::ErrorKind::Interrupted | io::ErrorKind::WouldBlock
) =>
{
Err(operation_error(reason, fd, error))
}
Err(error) => retry_operation(fd, source, interest, reason, operation, error),
}
}
fn retry_operation<T>(
fd: BorrowedFd<'_>,
source: Option<&ReadinessSource>,
interest: zio::Interest,
reason: SuspensionReason,
mut operation: impl FnMut() -> io::Result<T>,
mut error: io::Error,
) -> Result<T> {
let _reason = Wait::enter(reason)?;
let mut yielded = 0;
loop {
match error.kind() {
io::ErrorKind::Interrupted => crate::checkpoint()?,
io::ErrorKind::WouldBlock
if yielded < RUNNABLE_YIELD_LIMIT && context::carrier_has_runnable() =>
{
yielded += 1;
crate::yield_now()?;
}
io::ErrorKind::WouldBlock => {
wait_with_source(fd, source, interest)?;
crate::checkpoint()?;
}
_ => return Err(operation_error(reason, fd, error)),
}
error = match operation() {
Ok(value) => return Ok(value),
Err(error) => error,
};
}
}
#[cold]
fn operation_error(reason: SuspensionReason, fd: BorrowedFd<'_>, error: io::Error) -> Error {
Error::io(
match reason {
SuspensionReason::IoRead => "socket read",
SuspensionReason::IoWrite => "socket write",
SuspensionReason::IoAccept => "socket accept",
_ => "socket operation",
},
format_args!("fd={}", fd.as_raw_fd()),
error,
)
}
pub(super) fn wait(fd: BorrowedFd<'_>, interest: zio::Interest) -> Result<()> {
wait_with_source(fd, None, interest)
}
fn wait_with_source(
fd: BorrowedFd<'_>,
source: Option<&ReadinessSource>,
interest: zio::Interest,
) -> Result<()> {
let mounted = context::current().ok_or(Error::OutsideVThread)?;
let execution = mounted.execution()?;
let services = execution
.shared()
.services
.get()
.ok_or(Error::RuntimeStopped)?;
let parker = execution.readiness_parker()?;
let owned = source.map(|source| source.acquire(fd)).transpose()?;
parker.park_registered(move |token, wake| match owned {
Some(fd) => services
.reactor
.register_owned(fd, interest, token, wake.clone()),
None => services.reactor.register(fd, interest, token, wake.clone()),
})?;
execution.data.check()?;
services.reactor.check()
}
pub(super) fn read_exact(
mut read: impl FnMut(&mut [u8]) -> Result<usize>,
mut buffer: &mut [u8],
) -> Result<()> {
if buffer.is_empty() {
return crate::checkpoint();
}
while !buffer.is_empty() {
let count = read(buffer)?;
if count == 0 {
return Err(Error::io(
"socket read_exact",
"stream EOF",
io::Error::from(io::ErrorKind::UnexpectedEof),
));
}
buffer = &mut buffer[count..];
}
Ok(())
}
pub(super) fn write_all(
mut write: impl FnMut(&[u8]) -> Result<usize>,
mut buffer: &[u8],
) -> Result<()> {
if buffer.is_empty() {
return crate::checkpoint();
}
while !buffer.is_empty() {
let count = write(buffer)?;
if count == 0 {
return Err(Error::io(
"socket write_all",
"stream made no progress",
io::Error::from(io::ErrorKind::WriteZero),
));
}
buffer = &buffer[count..];
}
Ok(())
}
#[cfg(test)]
#[path = "io_test.rs"]
mod io_test;
#[cfg(test)]
#[path = "io_retry_test.rs"]
mod io_retry_test;