use crate::lockfree::{AtomicWakerSlot, MpmcStack, TreiberStack};
use std::ffi::CString;
use std::future::Future;
use std::io;
use std::os::unix::ffi::OsStrExt;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicI64, Ordering};
use std::task::{Context, Poll};
use std::thread::Thread;
use io_uring::{IoUring, opcode, squeue, types};
const PENDING: i64 = i64::MIN;
#[repr(align(64))]
struct OpState {
result: AtomicI64,
waker: AtomicWakerSlot,
}
impl OpState {
const fn fresh() -> Self {
Self {
result: AtomicI64::new(PENDING),
waker: AtomicWakerSlot::new(),
}
}
}
#[repr(align(64))]
struct SlotPool {
slots: Box<[OpState]>,
free: TreiberStack,
}
static RING_DEPTH: OnceLock<usize> = OnceLock::new();
static SLOT_POOL: OnceLock<SlotPool> = OnceLock::new();
fn slot_pool() -> &'static SlotPool {
SLOT_POOL.get_or_init(|| {
let depth = *RING_DEPTH.get_or_init(|| 256);
let mut slots = Vec::with_capacity(depth);
for _ in 0..depth {
slots.push(OpState::fresh());
}
let free = TreiberStack::new(depth);
for i in 0..depth as u32 {
free.push(i);
}
SlotPool {
slots: slots.into_boxed_slice(),
free,
}
})
}
#[repr(align(64))]
enum Slot {
Pooled(u32),
Heap(Box<OpState>),
}
fn acquire_slot() -> Slot {
let pool = slot_pool();
pool.free.pop().map_or_else(
|| Slot::Heap(Box::new(OpState::fresh())),
|idx| {
pool.slots[idx as usize]
.result
.store(PENDING, Ordering::Relaxed);
Slot::Pooled(idx)
},
)
}
struct SendEntry(squeue::Entry);
unsafe impl Send for SendEntry {}
#[repr(align(64))]
struct Ring {
pending: MpmcStack<SendEntry>,
worker: OnceLock<Thread>,
}
static RING: OnceLock<Ring> = OnceLock::new();
fn ring() -> &'static Ring {
RING.get_or_init(|| {
let r = Ring {
pending: MpmcStack::new(),
worker: OnceLock::new(),
};
let handle = std::thread::Builder::new()
.name("dtact-fs-uring".into())
.spawn(worker_loop)
.expect("failed to spawn dtact-fs-uring worker thread");
let _ = r.worker.set(handle.thread().clone());
r
})
}
pub fn init_fs(
_workers: usize,
ring_depth: u32,
_buffer_pool_size: usize,
_chunk_size: usize,
_pin_cpus: &[usize],
) {
let _ = RING_DEPTH.set(ring_depth.max(1) as usize);
let _ = slot_pool();
let _ = ring();
}
pub fn init(workers: usize) {
init_fs(workers, 256, 0, 0, &[]);
}
fn worker_loop() {
let mut io_uring = IoUring::new(256).expect("dtact-fs: IoUring::new failed");
let r = RING
.get()
.expect("ring() must be called before worker_loop starts");
loop {
if r.pending.is_empty() {
std::thread::park_timeout(std::time::Duration::from_millis(5));
continue;
}
let batch = r.pending.drain_all();
if batch.is_empty() {
continue;
}
{
let mut sq = io_uring.submission();
for entry in &batch {
unsafe {
let _ = sq.push(&entry.0);
}
}
sq.sync();
}
if let Err(e) = io_uring.submit_and_wait(batch.len()) {
eprintln!("dtact-fs-uring: submit_and_wait failed: {e}");
continue;
}
let mut cq = io_uring.completion();
cq.sync();
for cqe in &mut cq {
let user_data = cqe.user_data();
if user_data == 0 {
continue;
}
let state = unsafe { &*(user_data as *const OpState) };
let res = cqe.result();
state.result.store(i64::from(res), Ordering::Release);
state.waker.take_and_wake();
}
}
}
fn submit(entry: squeue::Entry) -> IoOp {
let slot = acquire_slot();
let ptr: *const OpState = match &slot {
Slot::Pooled(idx) => &raw const slot_pool().slots[*idx as usize],
Slot::Heap(b) => b.as_ref(),
};
let entry = entry.user_data(ptr as u64);
let r = ring();
r.pending.push(SendEntry(entry));
if let Some(t) = r.worker.get() {
t.unpark();
}
IoOp { slot }
}
struct IoOp {
slot: Slot,
}
impl IoOp {
#[inline]
fn state(&self) -> &OpState {
match &self.slot {
Slot::Pooled(idx) => &slot_pool().slots[*idx as usize],
Slot::Heap(b) => b,
}
}
}
impl Future for IoOp {
type Output = io::Result<i32>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<i32>> {
let r = self.state().result.load(Ordering::Acquire);
if r != PENDING {
return Poll::Ready(decode(r));
}
self.state().waker.register(cx.waker());
let r = self.state().result.load(Ordering::Acquire);
if r != PENDING {
return Poll::Ready(decode(r));
}
Poll::Pending
}
}
impl Drop for IoOp {
fn drop(&mut self) {
if let Slot::Pooled(idx) = self.slot {
let pool = slot_pool();
let done = pool.slots[idx as usize].result.load(Ordering::Acquire) != PENDING;
if done {
pool.free.push(idx);
}
}
}
}
#[inline]
fn try_pread(fd: i32, buf: &mut [u8], offset: u64) -> Option<io::Result<usize>> {
loop {
let n = unsafe {
libc::pread(
fd,
buf.as_mut_ptr().cast::<libc::c_void>(),
buf.len(),
offset as libc::off_t,
)
};
if n >= 0 {
return Some(Ok(n as usize));
}
let err = io::Error::last_os_error();
match err.raw_os_error() {
Some(libc::EINTR) => {}
Some(libc::EAGAIN) => return None,
_ => return Some(Err(err)),
}
}
}
#[inline]
fn try_pwrite(fd: i32, buf: &[u8], offset: u64) -> Option<io::Result<usize>> {
loop {
let n = unsafe {
libc::pwrite(
fd,
buf.as_ptr().cast::<libc::c_void>(),
buf.len(),
offset as libc::off_t,
)
};
if n >= 0 {
return Some(Ok(n as usize));
}
let err = io::Error::last_os_error();
match err.raw_os_error() {
Some(libc::EINTR) => {}
Some(libc::EAGAIN) => return None,
_ => return Some(Err(err)),
}
}
}
fn decode(res: i64) -> io::Result<i32> {
if res < 0 {
Err(io::Error::from_raw_os_error(-res as i32))
} else {
Ok(res as i32)
}
}
fn path_cstring(path: &Path) -> io::Result<CString> {
CString::new(path.as_os_str().as_bytes())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains a NUL byte"))
}
pub struct DtactFile {
fd: i32,
cursor: AtomicI64,
}
unsafe impl Send for DtactFile {}
unsafe impl Sync for DtactFile {}
async fn open_impl(path: &Path, flags: i32, mode: u32) -> io::Result<DtactFile> {
let cpath = path_cstring(path)?;
let cpath_ptr = cpath.as_ptr();
let entry = opcode::OpenAt::new(types::Fd(libc::AT_FDCWD), cpath_ptr)
.flags(flags)
.mode(mode)
.build();
let op = submit(entry);
let fd = op.await?;
drop(cpath);
Ok(DtactFile {
fd,
cursor: AtomicI64::new(0),
})
}
impl DtactFile {
pub async fn open(path: impl Into<PathBuf>) -> io::Result<Self> {
let path = path.into();
open_impl(&path, libc::O_RDONLY, 0).await
}
pub async fn create(path: impl Into<PathBuf>) -> io::Result<Self> {
let path = path.into();
open_impl(&path, libc::O_RDWR | libc::O_CREAT | libc::O_TRUNC, 0o644).await
}
pub async fn open_with(
path: impl Into<PathBuf>,
opts: std::fs::OpenOptions,
) -> io::Result<Self> {
use std::os::unix::io::IntoRawFd;
let path = path.into();
let file = opts.open(&path)?;
let fd = file.into_raw_fd();
Ok(Self {
fd,
cursor: AtomicI64::new(0),
})
}
pub async fn read(&self, mut buf: Vec<u8>) -> io::Result<(usize, Vec<u8>)> {
let offset = self.cursor.load(Ordering::Relaxed) as u64;
if let Some(res) = try_pread(self.fd, &mut buf, offset) {
let n = res?;
self.cursor.fetch_add(n as i64, Ordering::Relaxed);
return Ok((n, buf));
}
let entry = opcode::Read::new(types::Fd(self.fd), buf.as_mut_ptr(), buf.len() as u32)
.offset(offset)
.build();
let n = submit(entry).await?;
self.cursor.fetch_add(i64::from(n), Ordering::Relaxed);
Ok((n as usize, buf))
}
pub async fn write(&self, buf: Vec<u8>) -> io::Result<(usize, Vec<u8>)> {
let offset = self.cursor.load(Ordering::Relaxed) as u64;
if let Some(res) = try_pwrite(self.fd, &buf, offset) {
let n = res?;
self.cursor.fetch_add(n as i64, Ordering::Relaxed);
return Ok((n, buf));
}
let entry = opcode::Write::new(types::Fd(self.fd), buf.as_ptr(), buf.len() as u32)
.offset(offset)
.build();
let n = submit(entry).await?;
self.cursor.fetch_add(i64::from(n), Ordering::Relaxed);
Ok((n as usize, buf))
}
pub async fn read_at(&self, mut buf: Vec<u8>, offset: u64) -> io::Result<(usize, Vec<u8>)> {
if let Some(res) = try_pread(self.fd, &mut buf, offset) {
let n = res?;
return Ok((n, buf));
}
let entry = opcode::Read::new(types::Fd(self.fd), buf.as_mut_ptr(), buf.len() as u32)
.offset(offset)
.build();
let n = submit(entry).await?;
Ok((n as usize, buf))
}
pub async fn write_at(&self, buf: Vec<u8>, offset: u64) -> io::Result<(usize, Vec<u8>)> {
if let Some(res) = try_pwrite(self.fd, &buf, offset) {
let n = res?;
return Ok((n, buf));
}
let entry = opcode::Write::new(types::Fd(self.fd), buf.as_ptr(), buf.len() as u32)
.offset(offset)
.build();
let n = submit(entry).await?;
Ok((n as usize, buf))
}
pub async fn sync_all(&self) -> io::Result<()> {
let entry = opcode::Fsync::new(types::Fd(self.fd)).build();
submit(entry).await?;
Ok(())
}
pub async fn metadata(&self) -> io::Result<std::fs::Metadata> {
use std::os::unix::io::{AsRawFd, FromRawFd};
let file = unsafe { std::fs::File::from_raw_fd(self.fd) };
let file = std::mem::ManuallyDrop::new(file);
let meta = file.metadata();
let _ = file.as_raw_fd();
meta
}
pub async fn close(self) -> io::Result<()> {
let entry = opcode::Close::new(types::Fd(self.fd)).build();
submit(entry).await?;
std::mem::forget(self); Ok(())
}
}
impl Drop for DtactFile {
fn drop(&mut self) {
unsafe {
libc::close(self.fd);
}
}
}
pub async fn metadata(path: impl Into<PathBuf>) -> io::Result<std::fs::Metadata> {
let path = path.into();
std::fs::metadata(&path)
}
pub async fn read_dir(path: impl Into<PathBuf>) -> io::Result<Vec<std::fs::DirEntry>> {
let path: PathBuf = path.into();
std::fs::read_dir(&path)?.collect()
}
pub async fn create_dir_all(path: impl Into<PathBuf>) -> io::Result<()> {
let path = path.into();
std::fs::create_dir_all(&path)
}
pub async fn remove_file(path: impl Into<PathBuf>) -> io::Result<()> {
let path = path.into();
std::fs::remove_file(&path)
}
pub async fn canonicalize(path: impl Into<PathBuf>) -> io::Result<PathBuf> {
let path = path.into();
std::fs::canonicalize(&path)
}
pub async fn copy(from: impl Into<PathBuf>, to: impl Into<PathBuf>) -> io::Result<u64> {
let from = from.into();
let to = to.into();
std::fs::copy(&from, &to)
}
pub async fn create_dir(path: impl Into<PathBuf>) -> io::Result<()> {
let path = path.into();
std::fs::create_dir(&path)
}
pub async fn hard_link(src: impl Into<PathBuf>, dst: impl Into<PathBuf>) -> io::Result<()> {
let src = src.into();
let dst = dst.into();
std::fs::hard_link(&src, &dst)
}
pub async fn read(path: impl Into<PathBuf>) -> io::Result<Vec<u8>> {
let path = path.into();
std::fs::read(&path)
}
pub async fn read_link(path: impl Into<PathBuf>) -> io::Result<PathBuf> {
let path = path.into();
std::fs::read_link(&path)
}
pub async fn read_to_string(path: impl Into<PathBuf>) -> io::Result<String> {
let path = path.into();
std::fs::read_to_string(&path)
}
pub async fn remove_dir(path: impl Into<PathBuf>) -> io::Result<()> {
let path = path.into();
std::fs::remove_dir(&path)
}
pub async fn remove_dir_all(path: impl Into<PathBuf>) -> io::Result<()> {
let path = path.into();
std::fs::remove_dir_all(&path)
}
pub async fn rename(from: impl Into<PathBuf>, to: impl Into<PathBuf>) -> io::Result<()> {
let from = from.into();
let to = to.into();
std::fs::rename(&from, &to)
}
pub async fn set_permissions(
path: impl Into<PathBuf>,
perm: std::fs::Permissions,
) -> io::Result<()> {
let path = path.into();
std::fs::set_permissions(&path, perm)
}
pub async fn symlink(src: impl Into<PathBuf>, dst: impl Into<PathBuf>) -> io::Result<()> {
let src = src.into();
let dst = dst.into();
std::os::unix::fs::symlink(&src, &dst)
}
pub async fn symlink_metadata(path: impl Into<PathBuf>) -> io::Result<std::fs::Metadata> {
let path = path.into();
std::fs::symlink_metadata(&path)
}
pub async fn try_exists(path: impl Into<PathBuf>) -> io::Result<bool> {
let path = path.into();
std::fs::exists(&path)
}
pub async fn write(path: impl Into<PathBuf>, contents: impl AsRef<[u8]>) -> io::Result<()> {
let path = path.into();
std::fs::write(&path, contents)
}