use crate::lockfree::{AtomicWakerSlot, TreiberStack};
use std::future::Future;
use std::io;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::ptr;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicI64, Ordering};
use std::task::{Context, Poll};
use windows_sys::Win32::Foundation::{
CloseHandle, ERROR_HANDLE_EOF, ERROR_IO_PENDING, GENERIC_READ, GENERIC_WRITE, HANDLE,
INVALID_HANDLE_VALUE,
};
use windows_sys::Win32::Storage::FileSystem::{
CREATE_ALWAYS, CreateFileW, FILE_ATTRIBUTE_NORMAL, FILE_FLAG_OVERLAPPED, FILE_SHARE_READ,
FILE_SHARE_WRITE, FlushFileBuffers, GetFileSizeEx, OPEN_EXISTING, ReadFile, WriteFile,
};
use windows_sys::Win32::System::IO::{
CreateIoCompletionPort, GetQueuedCompletionStatusEx, OVERLAPPED, OVERLAPPED_ENTRY,
};
const WAKE_KEY: usize = 0;
const FILE_KEY: usize = 1;
struct Port {
handle: HANDLE,
}
unsafe impl Send for Port {}
unsafe impl Sync for Port {}
static PORT: OnceLock<Port> = OnceLock::new();
fn port() -> HANDLE {
let p = PORT.get_or_init(|| {
let handle = unsafe { CreateIoCompletionPort(INVALID_HANDLE_VALUE, ptr::null_mut(), 0, 1) };
assert!(!handle.is_null(), "dtact-fs: CreateIoCompletionPort failed");
std::thread::Builder::new()
.name("dtact-fs-iocp".into())
.spawn(worker_loop)
.expect("failed to spawn dtact-fs-iocp worker thread");
Port { handle }
});
p.handle
}
const PENDING: i64 = i64::MIN;
#[repr(C)]
struct OpState {
overlapped: OVERLAPPED,
result: AtomicI64,
waker: AtomicWakerSlot,
}
impl OpState {
const fn fresh() -> Self {
Self {
overlapped: unsafe { std::mem::zeroed() },
result: AtomicI64::new(PENDING),
waker: AtomicWakerSlot::new(),
}
}
}
unsafe impl Send for OpState {}
unsafe impl Sync for OpState {}
struct SlotPool {
slots: Box<[OpState]>,
free: TreiberStack,
}
static RING_DEPTH: OnceLock<usize> = OnceLock::new();
static SLOT_POOL: OnceLock<SlotPool> = OnceLock::new();
fn slot_pool() -> &'static SlotPool {
SLOT_POOL.get_or_init(|| {
let depth = *RING_DEPTH.get_or_init(|| 256);
let mut slots = Vec::with_capacity(depth);
for _ in 0..depth {
slots.push(OpState::fresh());
}
let free = TreiberStack::new(depth);
for i in 0..depth as u32 {
free.push(i);
}
SlotPool {
slots: slots.into_boxed_slice(),
free,
}
})
}
pub fn init_fs(
_workers: usize,
ring_depth: u32,
_buffer_pool_size: usize,
_chunk_size: usize,
_pin_cpus: &[usize],
) {
let _ = RING_DEPTH.set(ring_depth.max(1) as usize);
let _ = slot_pool();
let _ = port();
}
pub fn init(workers: usize) {
init_fs(workers, 256, 0, 0, &[]);
}
const fn encode_ok(n: usize) -> i64 {
n as i64
}
const fn encode_err(win32_code: u32) -> i64 {
-(win32_code as i64)
}
fn decode(result: i64) -> io::Result<usize> {
if result >= 0 {
Ok(result as usize)
} else {
Err(io::Error::from_raw_os_error(-result as i32))
}
}
fn worker_loop() {
let iocp = port();
let mut entries: [OVERLAPPED_ENTRY; 64] = unsafe { std::mem::zeroed() };
loop {
let mut removed: u32 = 0;
let ok = unsafe {
GetQueuedCompletionStatusEx(
iocp,
entries.as_mut_ptr(),
entries.len() as u32,
&raw mut removed,
u32::MAX,
0,
)
};
if ok == 0 {
continue;
}
for entry in &entries[..removed as usize] {
if entry.lpCompletionKey == WAKE_KEY {
continue;
}
let op_ptr = entry.lpOverlapped.cast::<OpState>();
if op_ptr.is_null() {
continue;
}
let op = unsafe { &*op_ptr };
let bytes = entry.dwNumberOfBytesTransferred as usize;
op.result.store(encode_ok(bytes), Ordering::Release);
op.waker.take_and_wake();
}
}
}
enum Slot {
Pooled(u32),
Heap(Box<OpState>),
}
fn acquire_slot() -> Slot {
let pool = slot_pool();
pool.free.pop().map_or_else(
|| Slot::Heap(Box::new(OpState::fresh())),
|idx| {
pool.slots[idx as usize]
.result
.store(PENDING, Ordering::Relaxed);
Slot::Pooled(idx)
},
)
}
struct IoOp {
slot: Slot,
}
impl IoOp {
#[inline]
fn state(&self) -> &OpState {
match &self.slot {
Slot::Pooled(idx) => &slot_pool().slots[*idx as usize],
Slot::Heap(b) => b,
}
}
#[inline]
fn overlapped_ptr(&self) -> *mut OVERLAPPED {
match &self.slot {
Slot::Pooled(idx) => (&raw const slot_pool().slots[*idx as usize])
.cast_mut()
.cast(),
Slot::Heap(b) => std::ptr::from_ref::<OpState>(b.as_ref()).cast_mut().cast(),
}
}
}
impl Future for IoOp {
type Output = io::Result<usize>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
let r = self.state().result.load(Ordering::Acquire);
if r != PENDING {
return Poll::Ready(decode(r));
}
self.state().waker.register(cx.waker());
let r = self.state().result.load(Ordering::Acquire);
if r != PENDING {
return Poll::Ready(decode(r));
}
Poll::Pending
}
}
impl Drop for IoOp {
fn drop(&mut self) {
if let Slot::Pooled(idx) = self.slot {
let pool = slot_pool();
let done = pool.slots[idx as usize].result.load(Ordering::Acquire) != PENDING;
if done {
pool.free.push(idx);
}
}
}
}
fn issue_read(handle: HANDLE, buf: &mut [u8], offset: u64) -> IoOp {
let slot = acquire_slot();
let op = IoOp { slot };
let ov_ptr = op.overlapped_ptr();
unsafe {
(*ov_ptr).Anonymous.Anonymous.Offset = offset as u32;
(*ov_ptr).Anonymous.Anonymous.OffsetHigh = (offset >> 32) as u32;
}
let ok = unsafe {
ReadFile(
handle,
buf.as_mut_ptr(),
buf.len() as u32,
ptr::null_mut(),
ov_ptr,
)
};
if ok == 0 {
let err = unsafe { windows_sys::Win32::Foundation::GetLastError() };
if err != ERROR_IO_PENDING {
let encoded = if err == ERROR_HANDLE_EOF {
encode_ok(0)
} else {
encode_err(err)
};
op.state().result.store(encoded, Ordering::Release);
}
}
op
}
fn issue_write(handle: HANDLE, buf: &[u8], offset: u64) -> IoOp {
let slot = acquire_slot();
let op = IoOp { slot };
let ov_ptr = op.overlapped_ptr();
unsafe {
(*ov_ptr).Anonymous.Anonymous.Offset = offset as u32;
(*ov_ptr).Anonymous.Anonymous.OffsetHigh = (offset >> 32) as u32;
}
let ok = unsafe {
WriteFile(
handle,
buf.as_ptr(),
buf.len() as u32,
ptr::null_mut(),
ov_ptr,
)
};
if ok == 0 {
let err = unsafe { windows_sys::Win32::Foundation::GetLastError() };
if err != ERROR_IO_PENDING {
op.state().result.store(encode_err(err), Ordering::Release);
}
}
op
}
fn to_wide(path: &Path) -> Vec<u16> {
use std::os::windows::ffi::OsStrExt;
path.as_os_str().encode_wide().chain(Some(0)).collect()
}
pub struct DtactFile {
handle: HANDLE,
cursor: AtomicI64,
}
unsafe impl Send for DtactFile {}
unsafe impl Sync for DtactFile {}
fn open_impl(path: &Path, disposition: u32, access: u32) -> io::Result<DtactFile> {
let wide = to_wide(path);
let handle = unsafe {
CreateFileW(
wide.as_ptr(),
access,
FILE_SHARE_READ | FILE_SHARE_WRITE,
ptr::null_mut(),
disposition,
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED,
ptr::null_mut(),
)
};
if handle == INVALID_HANDLE_VALUE {
return Err(io::Error::last_os_error());
}
let iocp = port();
let assoc = unsafe { CreateIoCompletionPort(handle, iocp, FILE_KEY, 0) };
if assoc.is_null() {
let e = io::Error::last_os_error();
unsafe { CloseHandle(handle) };
return Err(e);
}
Ok(DtactFile {
handle,
cursor: AtomicI64::new(0),
})
}
fn open_with_impl(path: &Path, opts: &std::fs::OpenOptions) -> io::Result<DtactFile> {
use std::os::windows::io::IntoRawHandle;
let file = opts.open(path)?;
let handle = file.into_raw_handle() as HANDLE;
let iocp = port();
let assoc = unsafe { CreateIoCompletionPort(handle, iocp, FILE_KEY, 0) };
if assoc.is_null() {
let e = io::Error::last_os_error();
unsafe { CloseHandle(handle) };
return Err(e);
}
Ok(DtactFile {
handle,
cursor: AtomicI64::new(0),
})
}
impl DtactFile {
pub fn open(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<Self>> {
let path = path.into();
std::future::ready(open_impl(&path, OPEN_EXISTING, GENERIC_READ))
}
pub fn create(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<Self>> {
let path = path.into();
std::future::ready(open_impl(
&path,
CREATE_ALWAYS,
GENERIC_READ | GENERIC_WRITE,
))
}
pub fn open_with(
path: impl Into<PathBuf>,
mut opts: std::fs::OpenOptions,
) -> impl Future<Output = io::Result<Self>> {
use std::os::windows::fs::OpenOptionsExt;
opts.custom_flags(FILE_FLAG_OVERLAPPED);
let path = path.into();
std::future::ready(open_with_impl(&path, &opts))
}
pub async fn read(&self, mut buf: Vec<u8>) -> io::Result<(usize, Vec<u8>)> {
let offset = self.cursor.load(Ordering::Relaxed) as u64;
let n = issue_read(self.handle, &mut buf, offset).await?;
self.cursor.fetch_add(n as i64, Ordering::Relaxed);
Ok((n, buf))
}
pub async fn write(&self, buf: Vec<u8>) -> io::Result<(usize, Vec<u8>)> {
let offset = self.cursor.load(Ordering::Relaxed) as u64;
let n = issue_write(self.handle, &buf, offset).await?;
self.cursor.fetch_add(n as i64, Ordering::Relaxed);
Ok((n, buf))
}
pub async fn read_at(&self, mut buf: Vec<u8>, offset: u64) -> io::Result<(usize, Vec<u8>)> {
let n = issue_read(self.handle, &mut buf, offset).await?;
Ok((n, buf))
}
pub async fn write_at(&self, buf: Vec<u8>, offset: u64) -> io::Result<(usize, Vec<u8>)> {
let n = issue_write(self.handle, &buf, offset).await?;
Ok((n, buf))
}
pub fn sync_all(&self) -> impl Future<Output = io::Result<()>> + '_ {
std::future::ready(self.sync_all_impl())
}
fn sync_all_impl(&self) -> io::Result<()> {
let ok = unsafe { FlushFileBuffers(self.handle) };
if ok == 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
pub fn metadata(&self) -> impl Future<Output = io::Result<std::fs::Metadata>> + '_ {
std::future::ready(self.metadata_impl())
}
fn metadata_impl(&self) -> io::Result<std::fs::Metadata> {
use std::os::windows::io::{AsRawHandle, FromRawHandle};
let file = unsafe { std::fs::File::from_raw_handle(self.handle.cast()) };
let file = std::mem::ManuallyDrop::new(file);
let meta = file.metadata();
let _ = file.as_raw_handle(); meta
}
pub fn close(self) -> impl Future<Output = io::Result<()>> {
std::future::ready(Ok(()))
}
pub fn len(&self) -> io::Result<u64> {
let mut size: i64 = 0;
let ok = unsafe { GetFileSizeEx(self.handle, &raw mut size) };
if ok == 0 {
return Err(io::Error::last_os_error());
}
Ok(size as u64)
}
pub fn is_empty(&self) -> io::Result<bool> {
self.len().map(|n| n == 0)
}
}
impl Drop for DtactFile {
fn drop(&mut self) {
unsafe {
CloseHandle(self.handle);
}
}
}
pub fn metadata(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<std::fs::Metadata>> {
let path = path.into();
std::future::ready(std::fs::metadata(&path))
}
pub fn read_dir(
path: impl Into<PathBuf>,
) -> impl Future<Output = io::Result<Vec<std::fs::DirEntry>>> {
let path: PathBuf = path.into();
std::future::ready(std::fs::read_dir(&path).and_then(Iterator::collect))
}
pub fn create_dir_all(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<()>> {
let path = path.into();
std::future::ready(std::fs::create_dir_all(&path))
}
pub fn remove_file(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<()>> {
let path = path.into();
std::future::ready(std::fs::remove_file(&path))
}
pub fn canonicalize(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<PathBuf>> {
let path = path.into();
std::future::ready(std::fs::canonicalize(&path))
}
pub fn copy(
from: impl Into<PathBuf>,
to: impl Into<PathBuf>,
) -> impl Future<Output = io::Result<u64>> {
let from = from.into();
let to = to.into();
std::future::ready(std::fs::copy(&from, &to))
}
pub fn create_dir(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<()>> {
let path = path.into();
std::future::ready(std::fs::create_dir(&path))
}
pub fn hard_link(
src: impl Into<PathBuf>,
dst: impl Into<PathBuf>,
) -> impl Future<Output = io::Result<()>> {
let src = src.into();
let dst = dst.into();
std::future::ready(std::fs::hard_link(&src, &dst))
}
pub fn read(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<Vec<u8>>> {
let path = path.into();
std::future::ready(std::fs::read(&path))
}
pub fn read_link(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<PathBuf>> {
let path = path.into();
std::future::ready(std::fs::read_link(&path))
}
pub fn read_to_string(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<String>> {
let path = path.into();
std::future::ready(std::fs::read_to_string(&path))
}
pub fn remove_dir(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<()>> {
let path = path.into();
std::future::ready(std::fs::remove_dir(&path))
}
pub fn remove_dir_all(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<()>> {
let path = path.into();
std::future::ready(std::fs::remove_dir_all(&path))
}
pub fn rename(
from: impl Into<PathBuf>,
to: impl Into<PathBuf>,
) -> impl Future<Output = io::Result<()>> {
let from = from.into();
let to = to.into();
std::future::ready(std::fs::rename(&from, &to))
}
pub fn set_permissions(
path: impl Into<PathBuf>,
perm: std::fs::Permissions,
) -> impl Future<Output = io::Result<()>> {
let path = path.into();
std::future::ready(std::fs::set_permissions(&path, perm))
}
pub fn symlink_dir(
src: impl Into<PathBuf>,
dst: impl Into<PathBuf>,
) -> impl Future<Output = io::Result<()>> {
let src = src.into();
let dst = dst.into();
std::future::ready(std::os::windows::fs::symlink_dir(&src, &dst))
}
pub fn symlink_file(
src: impl Into<PathBuf>,
dst: impl Into<PathBuf>,
) -> impl Future<Output = io::Result<()>> {
let src = src.into();
let dst = dst.into();
std::future::ready(std::os::windows::fs::symlink_file(&src, &dst))
}
pub fn symlink_metadata(
path: impl Into<PathBuf>,
) -> impl Future<Output = io::Result<std::fs::Metadata>> {
let path = path.into();
std::future::ready(std::fs::symlink_metadata(&path))
}
pub fn try_exists(path: impl Into<PathBuf>) -> impl Future<Output = io::Result<bool>> {
let path = path.into();
std::future::ready(std::fs::exists(&path))
}
pub fn write(
path: impl Into<PathBuf>,
contents: impl AsRef<[u8]>,
) -> impl Future<Output = io::Result<()>> {
let path = path.into();
std::future::ready(std::fs::write(&path, contents))
}