mod aligned_buf;
mod counting_fs;
#[cfg(feature = "std")]
mod open_flags;
mod mem_fs;
#[cfg(feature = "std")]
mod fault_fs;
#[cfg(feature = "std")]
mod crash_fs;
#[cfg(feature = "std")]
mod std_fs;
pub use aligned_buf::AlignedBuf;
pub use counting_fs::{
CountingFs, DirectoryWrites, FileWrites, WriteIoSnapshot, WriteIoStats, WriteOrigin,
};
#[cfg(all(target_os = "linux", feature = "io-uring"))]
mod io_uring_fs;
#[cfg(all(target_os = "linux", feature = "io-uring-raw"))]
mod io_uring_raw;
#[cfg(all(
test,
feature = "std",
target_os = "linux",
any(feature = "io-uring", feature = "io-uring-raw")
))]
mod batch_conformance;
#[cfg(feature = "std")]
pub use crash_fs::CrashFs;
#[cfg(all(test, feature = "std"))]
pub(crate) use fault_fs::ReadHook;
#[cfg(feature = "std")]
pub use fault_fs::{Fault, FaultFs, FaultInjector, FaultOp, FaultRule, SyncKind};
pub use mem_fs::MemFs;
#[cfg(feature = "std")]
pub use std_fs::StdFs;
#[cfg(feature = "std")]
pub(crate) use std_fs::is_cross_device;
#[cfg(all(target_os = "linux", feature = "io-uring"))]
pub use io_uring_fs::{IoUringFs, is_io_uring_available};
#[cfg(all(target_os = "linux", feature = "io-uring-raw"))]
pub use io_uring_raw::{
IoUringRaw, IoUringRawFile, IoUringRawFs, O_CREAT, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY,
close_raw, open_raw,
};
use crate::io::{self, Read, Seek, Write};
use crate::path::{Path, PathBuf};
#[cfg(not(feature = "std"))]
use alloc::{boxed::Box, string::String, vec::Vec};
#[expect(
clippy::struct_excessive_bools,
reason = "mirrors std::fs::OpenOptions which uses bool flags for each mode"
)]
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct FsOpenOptions {
pub read: bool,
pub write: bool,
pub create: bool,
pub create_new: bool,
pub truncate: bool,
pub append: bool,
pub direct_io: bool,
pub no_follow: bool,
pub no_wait: bool,
pub write_origin: Option<WriteOrigin>,
}
impl Default for FsOpenOptions {
fn default() -> Self {
Self::new()
}
}
impl FsOpenOptions {
#[must_use]
pub const fn new() -> Self {
Self {
read: false,
write: false,
create: false,
create_new: false,
truncate: false,
append: false,
direct_io: false,
no_follow: false,
no_wait: false,
write_origin: None,
}
}
#[must_use]
pub const fn read(mut self, read: bool) -> Self {
self.read = read;
self
}
#[must_use]
pub const fn write(mut self, write: bool) -> Self {
self.write = write;
self
}
#[must_use]
pub const fn create(mut self, create: bool) -> Self {
self.create = create;
self
}
#[must_use]
pub const fn create_new(mut self, create_new: bool) -> Self {
self.create_new = create_new;
self
}
#[must_use]
pub const fn truncate(mut self, truncate: bool) -> Self {
self.truncate = truncate;
self
}
#[must_use]
pub const fn append(mut self, append: bool) -> Self {
self.append = append;
self
}
#[must_use]
pub const fn direct_io(mut self, direct_io: bool) -> Self {
self.direct_io = direct_io;
self
}
#[must_use]
pub const fn no_follow(mut self, no_follow: bool) -> Self {
self.no_follow = no_follow;
self
}
#[must_use]
pub const fn no_wait(mut self, no_wait: bool) -> Self {
self.no_wait = no_wait;
self
}
#[must_use]
pub const fn write_origin(mut self, origin: WriteOrigin) -> Self {
self.write_origin = Some(origin);
self
}
}
#[derive(Clone, Debug)]
pub struct FsMetadata {
pub len: u64,
pub is_dir: bool,
pub is_file: bool,
}
#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
#[expect(
clippy::struct_excessive_bools,
reason = "a capability profile is a flat set of independent yes/no FS guarantees; \
each flag is queried on its own, so distinct bools read clearer than \
bitflags or a state machine (same rationale as FsOpenOptions)"
)]
pub struct FsCapabilities {
pub per_block_integrity_on_read: bool,
pub background_scrub: bool,
pub copy_on_write: bool,
pub reflink: bool,
pub native_snapshot: bool,
pub punch_hole: bool,
pub directory_sync_noop: bool,
}
#[derive(Clone, Debug)]
pub struct FsDirEntry {
pub path: PathBuf,
pub file_name: String,
pub is_dir: bool,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[non_exhaustive]
pub enum FileHint {
#[default]
Default,
Sequential,
Random,
WriteOnce,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum WriteLifetime {
Short,
Medium,
Long,
Extreme,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum SyncMode {
#[default]
Normal,
Full,
Barrier,
}
pub trait FsFile: Read + Write + Seek + Send + Sync {
fn sync_all(&self) -> io::Result<()>;
fn sync_data(&self) -> io::Result<()>;
fn sync_all_with(&self, mode: SyncMode) -> io::Result<()> {
let _ = mode;
self.sync_all()
}
fn sync_data_with(&self, mode: SyncMode) -> io::Result<()> {
let _ = mode;
self.sync_data()
}
fn metadata(&self) -> io::Result<FsMetadata>;
fn hard_link_count(&self) -> io::Result<u64> {
Err(io::Error::new(
io::ErrorKind::Unsupported,
"hard-link count unavailable",
))
}
fn set_len(&self, size: u64) -> io::Result<()>;
fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize>;
fn read_many(&self, regions: &mut [(u64, &mut [u8])]) -> io::Result<()> {
for (offset, buf) in regions.iter_mut() {
let n = self.read_at(buf, *offset)?;
if n != buf.len() {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"read_many: short read on a fixed-size block region",
));
}
}
Ok(())
}
fn backing_fd(&self) -> Option<i32> {
None
}
fn lock_exclusive(&self) -> io::Result<()>;
fn try_lock_exclusive(&self) -> io::Result<bool> {
Err(io::Error::new(
io::ErrorKind::Unsupported,
"FsFile::try_lock_exclusive is not implemented for this backend; \
override it (real lock, or Ok(true) for single-process backends) \
or disable the directory lock via Config::with_directory_lock(false)",
))
}
fn hint(&self, _hint: FileHint) -> io::Result<()> {
Ok(())
}
fn start_writeback(&self, offset: u64, len: u64) -> io::Result<()> {
let _ = (offset, len);
Ok(())
}
fn set_write_lifetime(&self, lifetime: WriteLifetime) -> io::Result<()> {
let _ = lifetime;
Ok(())
}
}
pub(crate) fn hint_writeback(file: &dyn FsFile, offset: u64, len: u64, writeback_bytes: &mut u64) {
if let Err(error) = file.start_writeback(offset, len) {
log::warn!("writeback hint refused, the file is written back by its final sync: {error}");
*writeback_bytes = 0;
}
}
pub struct BlockRead<'a> {
pub file: &'a dyn FsFile,
pub offset: u64,
pub buf: BlockBuf<'a>,
}
pub struct BlockBuf<'a> {
buf: &'a mut [u8],
filled: usize,
}
impl<'a> BlockBuf<'a> {
#[must_use]
pub fn new(buf: &'a mut [u8]) -> Self {
Self { buf, filled: 0 }
}
#[must_use]
pub fn capacity(&self) -> usize {
self.buf.len()
}
#[must_use]
pub fn filled(&self) -> usize {
self.filled
}
#[must_use]
pub fn is_full(&self) -> bool {
self.filled == self.buf.len()
}
#[must_use]
pub fn filled_bytes(&self) -> &[u8] {
#[expect(
clippy::indexing_slicing,
reason = "filled <= buf.len() is the type's invariant"
)]
&self.buf[..self.filled]
}
pub fn append(&mut self, bytes: &[u8]) -> usize {
let room = self.buf.len() - self.filled;
let n = room.min(bytes.len());
for (slot, byte) in self
.buf
.iter_mut()
.skip(self.filled)
.zip(bytes.iter().take(n))
{
*slot = *byte;
}
self.filled += n;
n
}
pub fn unfilled_mut(&mut self) -> &mut [u8] {
let filled = self.filled;
#[expect(
clippy::indexing_slicing,
reason = "filled <= buf.len() is the type's invariant"
)]
&mut self.buf[filled..]
}
pub fn advance(&mut self, n: usize) {
let remaining = self.buf.len() - self.filled;
self.filled += n.min(remaining);
}
}
pub struct QueuedRead {
pub tag: usize,
pub file: alloc::sync::Arc<dyn FsFile>,
pub offset: u64,
pub buf: Vec<u8>,
}
pub struct ReadDone {
pub tag: usize,
pub buf: Vec<u8>,
pub result: io::Result<()>,
}
#[diagnostic::on_unimplemented(
message = "`{Self}` is not a read queue",
label = "this type does not implement `ReadQueue`",
note = "`Fs::read_queue` provides one for every backend; implement `ReadQueue` only for a backend that can take reads while others are in flight"
)]
pub trait ReadQueue {
fn submit(&mut self, read: QueuedRead);
fn outstanding(&self) -> usize;
fn held(&self) -> usize {
self.outstanding()
}
fn issues_at_wait_for_none(&self) -> bool {
false
}
fn take_kept(&mut self) -> Vec<QueuedRead> {
Vec::new()
}
fn wait(&mut self, min: usize, on_done: &mut dyn FnMut(ReadDone));
fn set_wake(&mut self, wake: alloc::sync::Arc<dyn ReadWake>) -> bool {
drop(wake);
false
}
}
#[diagnostic::on_unimplemented(
message = "`{Self}` cannot be woken by a read queue",
label = "this type does not implement `ReadWake`",
note = "a caller waiting on several `ReadQueue`s implements `ReadWake` to learn that one of them has a read back"
)]
pub trait ReadWake: Send + Sync {
fn wake(&self);
}
struct BatchedReadQueue<'a, F: Fs + ?Sized> {
fs: &'a F,
submitted: Vec<QueuedRead>,
}
impl<F: Fs + ?Sized> ReadQueue for BatchedReadQueue<'_, F> {
fn submit(&mut self, read: QueuedRead) {
self.submitted.push(read);
}
fn outstanding(&self) -> usize {
self.submitted.len()
}
fn wait(&mut self, min: usize, on_done: &mut dyn FnMut(ReadDone)) {
if min == 0 || self.submitted.is_empty() {
return;
}
let mut reads = core::mem::take(&mut self.submitted);
let mut filled = alloc::vec![false; reads.len()];
let outcome = {
let mut reqs: Vec<BlockRead<'_>> = reads
.iter_mut()
.map(|read| BlockRead {
file: read.file.as_ref(),
offset: read.offset,
buf: BlockBuf::new(&mut read.buf[..]),
})
.collect();
self.fs
.read_blocks_batched_each(&mut reqs, &mut |index, _| {
if let Some(slot) = filled.get_mut(index) {
*slot = true;
}
})
};
let (mut failure, kind, message) = match outcome {
Ok(()) => (
None,
io::ErrorKind::UnexpectedEof,
"read queue: a read reported done was left unfilled",
),
Err(error) => {
let kind = error.kind();
(
Some(error),
kind,
"read queue: a read of a batch that failed",
)
}
};
for (read, filled) in reads.into_iter().zip(filled) {
let result = if filled {
Ok(())
} else {
Err(failure
.take()
.unwrap_or_else(|| io::Error::new(kind, message)))
};
on_done(ReadDone {
tag: read.tag,
buf: read.buf,
result,
});
}
}
}
pub trait Fs: Send + Sync + 'static {
fn open(&self, path: &Path, opts: &FsOpenOptions) -> io::Result<Box<dyn FsFile>>;
fn read_blocks_batched(&self, reqs: &mut [BlockRead<'_>]) -> io::Result<()> {
for req in reqs.iter_mut() {
let offset = req
.offset
.checked_add(req.buf.filled() as u64)
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"read_blocks_batched: request offset overflows",
)
})?;
let dst = req.buf.unfilled_mut();
let want = dst.len();
let n = req.file.read_at(dst, offset)?;
req.buf.advance(n);
if n != want {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"read_blocks_batched: short read on a fixed-size block",
));
}
}
Ok(())
}
fn read_blocks_batched_each(
&self,
reqs: &mut [BlockRead<'_>],
on_read: &mut dyn FnMut(usize, &BlockRead<'_>),
) -> io::Result<()> {
self.read_blocks_batched(reqs)?;
for (index, req) in reqs.iter().enumerate().filter(|(_, r)| r.buf.is_full()) {
on_read(index, req);
}
Ok(())
}
fn read_queue(&self) -> Box<dyn ReadQueue + '_> {
Box::new(BatchedReadQueue {
fs: self,
submitted: Vec::new(),
})
}
fn reads_by_descriptor(&self) -> bool {
false
}
fn check_descriptor_batch(&self, reqs: &[BlockRead<'_>]) -> io::Result<()> {
let _ = reqs;
Ok(())
}
fn create_dir_all(&self, path: &Path) -> io::Result<()>;
fn create_dir(&self, path: &Path) -> io::Result<()> {
let _ = path;
Err(io::Error::new(
io::ErrorKind::Unsupported,
"Fs::create_dir is not implemented for this backend",
))
}
fn read_dir(&self, path: &Path) -> io::Result<Vec<FsDirEntry>>;
fn remove_file(&self, path: &Path) -> io::Result<()>;
fn remove_dir_all(&self, path: &Path) -> io::Result<()>;
fn remove_dir(&self, path: &Path) -> io::Result<()> {
let _ = path;
Err(io::Error::new(
io::ErrorKind::Unsupported,
"this backend cannot remove a single empty directory",
))
}
fn same_file(&self, a: &Path, b: &Path) -> io::Result<bool> {
Ok(a == b)
}
fn read_link(&self, path: &Path) -> io::Result<Option<PathBuf>> {
let _ = path;
Ok(None)
}
fn symlink(&self, target: &Path, link: &Path, to_dir: bool) -> io::Result<()> {
let _ = (target, link, to_dir);
Err(io::Error::new(
io::ErrorKind::Unsupported,
"this backend has no symbolic links",
))
}
fn symlink_to_dir(&self, path: &Path) -> io::Result<bool> {
let _ = path;
Ok(false)
}
fn rename(&self, from: &Path, to: &Path) -> io::Result<()>;
fn metadata(&self, path: &Path) -> io::Result<FsMetadata>;
fn sync_directory(&self, path: &Path) -> io::Result<()>;
fn sync_directory_with(&self, path: &Path, mode: SyncMode) -> io::Result<()> {
let _ = mode;
self.sync_directory(path)
}
fn sync_device(&self, path: &Path) -> io::Result<()> {
let _ = path;
Ok(())
}
fn exists(&self, path: &Path) -> io::Result<bool>;
fn hard_link(&self, src: &Path, dst: &Path) -> io::Result<()> {
let _ = (src, dst);
Err(io::Error::new(
io::ErrorKind::Unsupported,
"Fs::hard_link is not implemented for this backend",
))
}
fn backend_id(&self) -> Option<u64> {
None
}
fn volume_id(&self, path: &Path) -> Option<u64> {
let _ = path;
None
}
fn capabilities(&self, path: &Path) -> FsCapabilities {
let _ = path;
FsCapabilities::default()
}
fn try_disable_cow(&self, path: &Path) -> io::Result<()> {
let _ = path;
Ok(())
}
fn punch_hole(&self, path: &Path, offset: u64, len: u64) -> io::Result<()> {
let _ = (path, offset, len);
Err(io::Error::new(
io::ErrorKind::Unsupported,
"punch_hole is not supported by this backend",
))
}
fn reflink_file(&self, src: &Path, dst: &Path) -> io::Result<()> {
copy_file_streamed(self, src, dst)
}
fn truncate_file(&self, path: &Path) -> io::Result<()> {
self.open(path, &FsOpenOptions::new().write(true))?
.set_len(0)
}
fn write_io_stats(&self) -> Option<&alloc::sync::Arc<WriteIoStats>> {
None
}
fn write_budget(&self) -> Option<&alloc::sync::Arc<crate::write_budget::WriteBudget>> {
None
}
fn hard_link_count(&self, path: &Path) -> io::Result<u64> {
let _ = path;
Err(io::Error::new(
io::ErrorKind::Unsupported,
"hard_link_count is not supported by this backend",
))
}
fn available_space(&self, path: &Path) -> io::Result<u64> {
let _ = path;
Ok(u64::MAX)
}
fn allocated_size(&self, path: &Path) -> io::Result<Option<u64>> {
let _ = path;
Ok(None)
}
fn extent_is_hole(&self, path: &Path, offset: u64, len: u64) -> io::Result<Option<bool>> {
let _ = (path, offset, len);
Ok(None)
}
fn extent_contains_hole(&self, path: &Path, offset: u64, len: u64) -> io::Result<Option<bool>> {
let _ = (path, offset, len);
Ok(None)
}
}
#[cfg(all(unix, feature = "std"))]
#[allow(
clippy::unnecessary_cast,
clippy::cast_lossless,
reason = "statvfs field widths vary by platform (e.g. fsblkcnt_t is u32 on \
macOS, u64 on Linux); `as u64` is the portable widening that is a \
no-op where the field is already u64"
)]
pub(crate) fn statvfs_available_space(path: &Path) -> std::io::Result<u64> {
use core::mem::MaybeUninit;
use std::ffi::CString;
use std::os::unix::ffi::OsStrExt;
let c = CString::new(path.as_os_str().as_bytes()).map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"path contains an interior NUL byte",
)
})?;
let mut buf = MaybeUninit::<libc::statvfs>::uninit();
#[expect(unsafe_code, reason = "statvfs FFI to read filesystem free space")]
let rc = unsafe { libc::statvfs(c.as_ptr(), buf.as_mut_ptr()) };
if rc != 0 {
return Err(std::io::Error::last_os_error());
}
#[expect(unsafe_code, reason = "read initialized statvfs fields")]
let st = unsafe { buf.assume_init() };
let frsize = st.f_frsize as u64;
let bavail = st.f_bavail as u64;
Ok(bavail.saturating_mul(frsize))
}
#[cfg(any(feature = "std", all(target_os = "linux", feature = "io-uring-raw")))]
pub(crate) const KERNEL_BACKEND_ID: u64 = 0x4b45_524e_454c_5f46;
#[cfg(all(
target_os = "linux",
any(
all(feature = "std", target_pointer_width = "64"),
all(
feature = "io-uring-raw",
any(target_arch = "x86_64", target_arch = "aarch64")
)
)
))]
pub(crate) fn linux_fs_type_capabilities(f_type: i64) -> FsCapabilities {
const BTRFS: i64 = 0x9123_683E;
const ZFS: i64 = 0x2FC1_2FC1;
const XFS: i64 = 0x5846_5342;
const EXT_FAMILY: i64 = 0x0000_EF53; const TMPFS: i64 = 0x0102_1994;
match f_type {
BTRFS | ZFS => FsCapabilities {
per_block_integrity_on_read: true,
background_scrub: true,
copy_on_write: true,
reflink: true,
native_snapshot: true,
punch_hole: true,
..FsCapabilities::default()
},
XFS => FsCapabilities {
copy_on_write: true,
reflink: true,
punch_hole: true,
..FsCapabilities::default()
},
EXT_FAMILY | TMPFS => FsCapabilities {
punch_hole: true,
..FsCapabilities::default()
},
_ => FsCapabilities::default(),
}
}
#[cfg(all(unix, feature = "std"))]
pub(crate) fn unix_volume_id(path: &Path) -> Option<u64> {
use std::os::unix::fs::MetadataExt;
std::fs::metadata(path).ok().map(|m| m.dev())
}
#[cfg(all(unix, feature = "std"))]
pub(crate) fn unix_allocated_size(path: &Path) -> std::io::Result<Option<u64>> {
use std::os::unix::fs::MetadataExt;
Ok(Some(std::fs::metadata(path)?.blocks() * 512))
}
pub(crate) fn copy_file_streamed<F: Fs + ?Sized>(fs: &F, src: &Path, dst: &Path) -> io::Result<()> {
let mut src_file = fs.open(src, &FsOpenOptions::new().read(true))?;
let mut dst_file = fs.open(dst, &FsOpenOptions::new().write(true).create_new(true))?;
let mut buf = vec![0u8; 64 * 1024].into_boxed_slice();
let result: io::Result<()> = (|| {
loop {
let n = match src_file.read(&mut buf) {
Ok(0) => break,
Ok(n) => n,
#[cfg(feature = "std")]
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
#[cfg(not(feature = "std"))]
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e.into()),
};
let chunk = buf.get(..n).ok_or_else(|| {
io::Error::other("read returned more bytes than the buffer holds")
})?;
dst_file.write_all(chunk)?;
}
dst_file.sync_all()?;
Ok(())
})();
if let Err(e) = result {
drop(dst_file);
let _ = fs.remove_file(dst);
return Err(e);
}
Ok(())
}
#[cfg(test)]
mod block_buf_tests;
#[cfg(test)]
mod read_queue_tests;
#[cfg(all(
test,
target_os = "linux",
any(
all(feature = "std", target_pointer_width = "64"),
all(
feature = "io-uring-raw",
any(target_arch = "x86_64", target_arch = "aarch64")
)
)
))]
mod fs_type_tests;