use std::cell::RefCell;
use std::collections::HashMap;
use std::io;
use std::marker::PhantomData;
use std::os::unix::io::{AsRawFd, RawFd};
use std::sync::{Arc, Mutex};
use crate::backend::{detect_backend, Backend};
use crate::error::{Result, SaferRingError};
use crate::future::WakerRegistry;
use crate::operation::tracker::OperationTracker;
use crate::operation::{Building, Operation};
use crate::safety::{CompletionChecker, OrphanTracker, SubmissionId};
#[cfg(target_os = "linux")]
use tokio::io::unix::AsyncFd;
#[cfg(target_os = "linux")]
#[derive(Debug)]
pub(super) struct RawFdWrapper(RawFd);
#[cfg(target_os = "linux")]
impl AsRawFd for RawFdWrapper {
fn as_raw_fd(&self) -> RawFd {
self.0
}
}
#[cfg(target_os = "linux")]
unsafe impl Send for RawFdWrapper {}
#[cfg(target_os = "linux")]
unsafe impl Sync for RawFdWrapper {}
pub mod batch_operations;
pub mod configuration;
pub mod fixed_operations;
pub mod io_operations;
pub mod network_operations;
pub mod safe_operations;
pub mod utility;
pub struct Ring<'ring> {
pub(super) backend: RefCell<Box<dyn Backend>>,
pub(super) phantom: PhantomData<&'ring ()>,
pub(super) operations: RefCell<OperationTracker<'ring>>,
pub(super) waker_registry: Arc<WakerRegistry>,
pub(super) orphan_tracker: Arc<Mutex<OrphanTracker>>,
pub(super) completion_cache: RefCell<HashMap<u64, io::Result<i32>>>,
#[cfg(target_os = "linux")]
pub(super) async_fd: Option<AsyncFd<RawFdWrapper>>,
#[cfg(not(target_os = "linux"))]
pub(super) async_fd: Option<()>, }
impl<'ring> std::fmt::Debug for Ring<'ring> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Ring")
.field("backend", &"<dyn Backend>")
.field("operations", &self.operations)
.finish()
}
}
impl<'ring> Ring<'ring> {
pub fn new(entries: u32) -> Result<Self> {
if entries == 0 {
return Err(SaferRingError::Io(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"Queue depth must be greater than 0",
)));
}
let backend = detect_backend(entries)?;
#[cfg(target_os = "linux")]
let async_fd = {
if tokio::runtime::Handle::try_current().is_ok() {
if let Some(io_uring_backend) = backend
.as_any()
.downcast_ref::<crate::backend::io_uring::IoUringBackend>(
) {
let fd = io_uring_backend.as_raw_fd();
AsyncFd::new(RawFdWrapper(fd)).ok()
} else {
None
}
} else {
None
}
};
#[cfg(not(target_os = "linux"))]
let async_fd = None;
Ok(Self {
backend: RefCell::new(backend),
phantom: PhantomData,
operations: RefCell::new(OperationTracker::new()),
waker_registry: Arc::new(WakerRegistry::new()),
orphan_tracker: Arc::new(Mutex::new(OrphanTracker::new())),
completion_cache: RefCell::new(HashMap::new()),
async_fd,
})
}
pub fn submit<'buf>(
&mut self,
operation: Operation<'ring, 'buf, Building>,
) -> Result<Operation<'ring, 'buf, crate::operation::Submitted>>
where
'buf: 'ring, {
operation.validate().map_err(|msg| {
SaferRingError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, msg))
})?;
let id = {
let mut tracker = self.operations.borrow_mut();
tracker.register_operation(operation.get_type(), operation.get_fd())
};
let submitted = operation.submit_with_id(id).map_err(|msg| {
self.operations.borrow_mut().complete_operation(id);
SaferRingError::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, msg))
})?;
self.submit_to_backend(&submitted)?;
Ok(submitted)
}
fn submit_to_backend<'buf>(
&mut self,
operation: &Operation<'ring, 'buf, crate::operation::Submitted>,
) -> Result<()> {
let (buffer_ptr, buffer_len) = match operation.buffer_info() {
Some((ptr, len)) => (ptr, len),
None => (std::ptr::null_mut(), 0), };
self.backend.borrow_mut().submit_operation(
operation.op_type(),
operation.fd(),
operation.offset(),
buffer_ptr,
buffer_len,
operation.id(),
)
}
}
impl<'ring> Drop for Ring<'ring> {
fn drop(&mut self) {
let tracker = self.operations.borrow();
let count = tracker.count();
if count > 0 {
let debug_info = tracker.debug_info();
drop(tracker);
let mut message = format!("Ring dropped with {count} operations in flight:\n");
for (id, op_type, fd) in debug_info {
message.push_str(&format!(" - Operation {id}: {op_type:?} on fd {fd}\n"));
}
message.push_str("All operations must complete before dropping the ring.");
panic!("{}", message);
}
}
}
unsafe impl<'ring> Send for Ring<'ring> {}
unsafe impl<'ring> Sync for Ring<'ring> {}
impl<'ring> CompletionChecker for Ring<'ring> {
fn try_complete_safe_operation(
&self,
submission_id: SubmissionId,
) -> Result<Option<std::io::Result<i32>>> {
{
let cache = self.completion_cache.borrow();
if let Some(cached_result) = cache.get(&submission_id) {
let result = match cached_result {
Ok(bytes) => Ok(*bytes),
Err(e) => Err(std::io::Error::new(e.kind(), format!("{e}"))),
};
drop(cache);
self.completion_cache.borrow_mut().remove(&submission_id);
return Ok(Some(result));
}
}
let completions = self.backend.borrow_mut().try_complete()?;
let mut target_result = None;
for (completed_id, result) in completions {
if completed_id == submission_id {
let result_for_return = match &result {
Ok(bytes) => Ok(*bytes),
Err(e) => Err(std::io::Error::new(e.kind(), format!("{e}"))),
};
let mut orphan_tracker = self.orphan_tracker.lock().unwrap();
if let Some((orphaned_buffer, _operation_result)) =
orphan_tracker.handle_completion(completed_id, result)
{
drop(orphaned_buffer);
} else {
target_result = Some(result_for_return);
}
drop(orphan_tracker);
} else {
let cached_result = match &result {
Ok(bytes) => Ok(*bytes),
Err(e) => Err(std::io::Error::new(e.kind(), format!("{e}"))),
};
self.completion_cache
.borrow_mut()
.insert(completed_id, cached_result);
let mut orphan_tracker = self.orphan_tracker.lock().unwrap();
if let Some((orphaned_buffer, _operation_result)) =
orphan_tracker.handle_completion(completed_id, result)
{
drop(orphaned_buffer);
self.completion_cache.borrow_mut().remove(&completed_id);
} else {
drop(orphan_tracker);
self.waker_registry.wake_operation(completed_id);
}
}
}
Ok(target_result)
}
fn supports_async_wait(&self) -> bool {
#[cfg(target_os = "linux")]
{
self.async_fd.is_some()
}
#[cfg(not(target_os = "linux"))]
{
false
}
}
}
impl<'ring> Ring<'ring> {
#[cfg(target_os = "linux")]
pub async fn await_completion(&mut self, submission_id: u64) -> Result<io::Result<i32>> {
loop {
if let Some(result) = self.try_complete_by_id(submission_id)? {
return Ok(result);
}
if let Some(async_fd) = &self.async_fd {
match async_fd.readable().await {
Ok(mut guard) => {
guard.clear_ready();
if let Some(result) = self.try_complete_by_id(submission_id)? {
return Ok(result);
}
}
Err(e) => {
return Err(SaferRingError::Io(e));
}
}
} else {
tokio::task::yield_now().await;
if let Some(result) = self.try_complete_by_id(submission_id)? {
return Ok(result);
}
}
}
}
#[cfg(not(target_os = "linux"))]
pub async fn await_completion(&mut self, submission_id: u64) -> Result<io::Result<i32>> {
loop {
if let Some(result) = self.try_complete_by_id(submission_id)? {
return Ok(result);
}
tokio::task::yield_now().await;
}
}
}