use std::fmt;
use std::fs::{File, OpenOptions};
use std::io;
use std::os::fd::AsRawFd;
use std::os::unix::fs::{FileExt, OpenOptionsExt};
use std::path::Path;
use memmap2::MmapOptions;
pub const DIO_ALIGN: u64 = 4096;
#[cfg(all(
target_os = "linux",
any(
target_arch = "x86",
target_arch = "x86_64",
target_arch = "arm",
target_arch = "aarch64",
target_arch = "riscv32",
target_arch = "riscv64",
target_arch = "s390x",
target_arch = "loongarch64",
)
))]
pub(super) const O_DIRECT: Option<i32> = Some(0o0040000);
#[cfg(not(all(
target_os = "linux",
any(
target_arch = "x86",
target_arch = "x86_64",
target_arch = "arm",
target_arch = "aarch64",
target_arch = "riscv32",
target_arch = "riscv64",
target_arch = "s390x",
target_arch = "loongarch64",
)
)))]
pub(super) const O_DIRECT: Option<i32> = None;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DirectFault {
Unsupported,
OpenRefused,
FileTooSmall,
UnalignedGeometry,
BadProbeBuffer,
ReadFailed,
ShortRead,
DirtyBaseline,
PageCacheGrew,
NoResidencySignal,
ResidencyDenied,
ProbeFailed,
}
impl fmt::Display for DirectFault {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let text = match self {
Self::Unsupported => "O_DIRECT is not implemented for this architecture",
Self::OpenRefused => "the filesystem refused an O_DIRECT open",
Self::FileTooSmall => "the layer file is shorter than one aligned block",
Self::UnalignedGeometry => "the layout's blob stride is not 4096-aligned",
Self::BadProbeBuffer => "the probe buffer is misaligned or too small",
Self::ReadFailed => "an aligned O_DIRECT read failed",
Self::ShortRead => "an aligned O_DIRECT read came back short",
Self::DirtyBaseline => "the page cache could not be cleared for the probe window",
Self::PageCacheGrew => "the read populated the page cache: a silent buffered fallback",
Self::NoResidencySignal => {
"mincore reports nothing resident even after a buffered read"
}
Self::ResidencyDenied => {
"mincore will not report page-cache residency for a file this \
process neither owns nor may write, so O_DIRECT cannot be \
measured on this install"
}
Self::ProbeFailed => "the probe's own mmap/mincore/fadvise failed",
};
f.write_str(text)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DirectSupport {
Verified,
Degraded(DirectFault),
Unusable(DirectFault),
}
impl DirectSupport {
pub fn is_verified(self) -> bool {
self == Self::Verified
}
pub fn is_usable(self) -> bool {
!matches!(self, Self::Unusable(_))
}
pub fn fault(self) -> Option<DirectFault> {
match self {
Self::Verified => None,
Self::Degraded(fault) | Self::Unusable(fault) => Some(fault),
}
}
}
impl fmt::Display for DirectSupport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Verified => f.write_str("verified"),
Self::Degraded(fault) => write!(f, "degraded: {fault}"),
Self::Unusable(fault) => write!(f, "unusable: {fault}"),
}
}
}
pub fn open(path: &Path, direct: bool) -> io::Result<(File, bool)> {
if direct && let Some(flag) = O_DIRECT {
match OpenOptions::new().read(true).custom_flags(flag).open(path) {
Ok(file) => return Ok((file, true)),
Err(error) => {
tracing::debug!(
path = %path.display(),
%error,
"O_DIRECT open refused, retrying buffered"
);
}
}
}
Ok((File::open(path)?, false))
}
pub fn probe(path: &Path, buf: &mut [u8]) -> DirectSupport {
let window = DIO_ALIGN as usize;
if buf.len() < window || !is_aligned(buf.as_ptr() as usize as u64) {
return DirectSupport::Unusable(DirectFault::BadProbeBuffer);
}
let buf = &mut buf[..window];
let Some(_) = O_DIRECT else {
return DirectSupport::Unusable(DirectFault::Unsupported);
};
let (direct, applied) = match open(path, true) {
Ok(opened) => opened,
Err(error) => {
tracing::warn!(path = %path.display(), %error, "cannot open layer file for probing");
return DirectSupport::Unusable(DirectFault::OpenRefused);
}
};
if !applied {
return DirectSupport::Unusable(DirectFault::OpenRefused);
}
let Ok(buffered) = File::open(path) else {
return DirectSupport::Unusable(DirectFault::OpenRefused);
};
match buffered.metadata() {
Ok(meta) if meta.len() >= DIO_ALIGN => {}
Ok(_) => return DirectSupport::Unusable(DirectFault::FileTooSmall),
Err(_) => return DirectSupport::Unusable(DirectFault::ProbeFailed),
}
if !residency_is_visible(path, &buffered) {
tracing::warn!(
path = %path.display(),
"the install is owned by another user, so the kernel will not \
report its page-cache residency; O_DIRECT cannot be verified"
);
return DirectSupport::Degraded(DirectFault::ResidencyDenied);
}
match clear_window(&buffered, window) {
Ok(0) => {}
Ok(_) => return DirectSupport::Degraded(DirectFault::DirtyBaseline),
Err(error) => {
tracing::debug!(path = %path.display(), %error, "residency baseline failed");
return DirectSupport::Degraded(DirectFault::ProbeFailed);
}
}
match direct.read_at(buf, 0) {
Ok(read) if read == window => {}
Ok(read) => {
tracing::debug!(path = %path.display(), read, "short O_DIRECT probe read");
return DirectSupport::Unusable(DirectFault::ShortRead);
}
Err(error) => {
tracing::debug!(path = %path.display(), %error, "O_DIRECT probe read failed");
return DirectSupport::Unusable(DirectFault::ReadFailed);
}
}
let grew = match resident_pages(&buffered, window) {
Ok(pages) => pages,
Err(_) => return DirectSupport::Degraded(DirectFault::ProbeFailed),
};
let verdict = if grew > 0 {
DirectSupport::Degraded(DirectFault::PageCacheGrew)
} else {
let mut control = vec![0u8; window];
match buffered.read_at(&mut control, 0) {
Ok(read) if read == window => match resident_pages(&buffered, window) {
Ok(0) => DirectSupport::Degraded(DirectFault::NoResidencySignal),
Ok(_) => DirectSupport::Verified,
Err(_) => DirectSupport::Degraded(DirectFault::ProbeFailed),
},
_ => DirectSupport::Degraded(DirectFault::ProbeFailed),
}
};
let _ = fadvise_dontneed(&buffered, 0, DIO_ALIGN);
verdict
}
pub fn is_aligned(value: u64) -> bool {
value.is_multiple_of(DIO_ALIGN)
}
#[cfg(target_os = "linux")]
fn residency_is_visible(path: &Path, file: &File) -> bool {
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::MetadataExt;
let euid = unsafe { libc::geteuid() };
if euid == 0 {
return true;
}
if file.metadata().is_ok_and(|meta| meta.uid() == euid) {
return true;
}
let Ok(c_path) = std::ffi::CString::new(path.as_os_str().as_bytes()) else {
return false;
};
let rc = unsafe {
libc::faccessat(
libc::AT_FDCWD,
c_path.as_ptr(),
libc::W_OK,
libc::AT_EACCESS,
)
};
rc == 0
}
#[cfg(not(target_os = "linux"))]
fn residency_is_visible(_path: &Path, _file: &File) -> bool {
false
}
fn clear_window(file: &File, len: usize) -> io::Result<usize> {
let _ = fadvise_dontneed(file, 0, len as u64);
let resident = resident_pages(file, len)?;
if resident == 0 {
return Ok(0);
}
if writeback(file, 0, len as u64).is_err() {
return Ok(resident);
}
let _ = fadvise_dontneed(file, 0, len as u64);
resident_pages(file, len)
}
#[cfg(target_os = "linux")]
fn writeback(file: &File, offset: u64, len: u64) -> io::Result<()> {
let (Ok(offset), Ok(len)) = (i64::try_from(offset), i64::try_from(len)) else {
return Err(io::Error::from(io::ErrorKind::InvalidInput));
};
let rc = unsafe {
libc::sync_file_range(
file.as_raw_fd(),
offset,
len,
libc::SYNC_FILE_RANGE_WAIT_BEFORE
| libc::SYNC_FILE_RANGE_WRITE
| libc::SYNC_FILE_RANGE_WAIT_AFTER,
)
};
if rc != 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
#[cfg(not(target_os = "linux"))]
fn writeback(_file: &File, _offset: u64, _len: u64) -> io::Result<()> {
Err(io::Error::from(io::ErrorKind::Unsupported))
}
#[cfg(target_os = "linux")]
pub fn fadvise_dontneed(file: &File, offset: u64, len: u64) -> io::Result<()> {
let (Ok(offset), Ok(len)) = (i64::try_from(offset), i64::try_from(len)) else {
return Err(io::Error::from(io::ErrorKind::InvalidInput));
};
let rc = unsafe {
libc::posix_fadvise(
file.as_raw_fd(),
offset as libc::off_t,
len as libc::off_t,
libc::POSIX_FADV_DONTNEED,
)
};
if rc != 0 {
return Err(io::Error::from_raw_os_error(rc));
}
Ok(())
}
#[cfg(not(target_os = "linux"))]
pub fn fadvise_dontneed(_file: &File, _offset: u64, _len: u64) -> io::Result<()> {
Err(io::Error::from(io::ErrorKind::Unsupported))
}
#[cfg(target_os = "linux")]
pub fn resident_pages(file: &File, len: usize) -> io::Result<usize> {
if len == 0 {
return Ok(0);
}
let map = unsafe { MmapOptions::new().len(len).map(file)? };
let pages = len.div_ceil(page_size());
let mut resident = vec![0u8; pages];
let rc = unsafe {
libc::mincore(
map.as_ptr() as *mut libc::c_void,
len,
resident.as_mut_ptr(),
)
};
if rc != 0 {
return Err(io::Error::last_os_error());
}
Ok(resident.iter().filter(|byte| *byte & 1 == 1).count())
}
#[cfg(not(target_os = "linux"))]
pub fn resident_pages(_file: &File, _len: usize) -> io::Result<usize> {
Err(io::Error::from(io::ErrorKind::Unsupported))
}
fn page_size() -> usize {
let size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) };
if size > 0 {
size as usize
} else {
DIO_ALIGN as usize
}
}
#[cfg(test)]
mod tests {
use super::super::testutil::build_install;
use super::*;
use crate::io::SlotPool;
use std::path::PathBuf;
struct Scratch {
pool: SlotPool,
}
impl Scratch {
fn new() -> Self {
Self {
pool: SlotPool::new(1, &[2 * DIO_ALIGN]).expect("two-page pool"),
}
}
fn with<R>(&self, f: impl FnOnce(&mut [u8]) -> R) -> R {
let mut slot = self.pool.acquire(0).expect("free slot");
f(slot.as_mut_slice())
}
}
fn shm_dir() -> Option<PathBuf> {
let dir = PathBuf::from("/dev/shm").join(format!("ramvamp-probe-{}", std::process::id()));
std::fs::create_dir_all(&dir).ok().map(|()| dir)
}
fn read_happened(outcome: DirectSupport) -> bool {
matches!(
outcome,
DirectSupport::Verified
| DirectSupport::Degraded(
DirectFault::PageCacheGrew | DirectFault::NoResidencySignal
)
)
}
#[test]
fn probe_reads_the_head_of_the_file_whatever_it_concludes() {
let fx = build_install("direct-probe");
let path = fx.root.join(&fx.layout.layers[0].file);
let expected = std::fs::read(&path).unwrap();
let scratch = Scratch::new();
let (outcome, got) = scratch.with(|buf| (probe(&path, buf), buf.to_vec()));
println!("probe outcome on the test filesystem: {outcome}");
assert!(
!matches!(
outcome.fault(),
Some(DirectFault::ReadFailed | DirectFault::ShortRead)
),
"aligned O_DIRECT read failed: {outcome}"
);
if read_happened(outcome) {
let window = DIO_ALIGN as usize;
assert_eq!(
&got[..window],
&expected[..window],
"probe read wrong bytes"
);
}
assert_eq!(outcome.is_verified(), outcome.fault().is_none());
}
#[test]
fn probe_separates_a_compressed_extent_from_a_plain_one() {
let fx = build_install("direct-compressible");
let path = fx.root.join("incompressible.bin");
let mut state = 0x2545_f491_4f6c_dd1du64;
let mut bytes = Vec::with_capacity(1 << 20);
while bytes.len() < (1 << 20) {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
bytes.extend_from_slice(&state.to_le_bytes());
}
{
use std::io::Write;
let mut file = File::create(&path).unwrap();
file.write_all(&bytes).unwrap();
file.sync_all().unwrap();
}
let scratch = Scratch::new();
let plain = scratch.with(|buf| probe(&path, buf));
let packed = scratch.with(|buf| probe(&fx.root.join(&fx.layout.layers[0].file), buf));
println!("probe: incompressible={plain} compressible={packed}");
for outcome in [plain, packed] {
assert!(
!matches!(
outcome.fault(),
Some(DirectFault::ReadFailed | DirectFault::ShortRead)
),
"aligned O_DIRECT read failed: {outcome}"
);
}
if packed.is_verified() {
assert!(plain.is_verified(), "verified only for compressible data");
}
}
#[test]
fn probe_refuses_to_claim_direct_io_on_tmpfs() {
let Some(dir) = shm_dir() else {
return; };
let path = dir.join("probe.bin");
std::fs::write(&path, vec![7u8; 64 * 1024]).unwrap();
let scratch = Scratch::new();
let outcome = scratch.with(|buf| probe(&path, buf));
println!("probe outcome on tmpfs: {outcome}");
assert!(!outcome.is_verified(), "tmpfs claimed real O_DIRECT");
let _ = std::fs::remove_file(&path);
let _ = std::fs::remove_dir(&dir);
}
#[test]
fn probe_rejects_a_misaligned_or_short_buffer() {
let fx = build_install("direct-badbuf");
let path = fx.root.join(&fx.layout.layers[0].file);
let scratch = Scratch::new();
let short = scratch.with(|buf| probe(&path, &mut buf[..8]));
assert_eq!(
short,
DirectSupport::Unusable(DirectFault::BadProbeBuffer),
"a short buffer must be rejected"
);
let misaligned = scratch.with(|buf| probe(&path, &mut buf[8..]));
assert_eq!(
misaligned,
DirectSupport::Unusable(DirectFault::BadProbeBuffer),
"a misaligned buffer must be rejected"
);
}
#[test]
fn probe_reports_a_missing_file_rather_than_failing() {
let scratch = Scratch::new();
let outcome = scratch.with(|buf| probe(Path::new("/nonexistent/layer_00.bin"), buf));
assert!(!outcome.is_usable());
assert_eq!(outcome.fault(), Some(DirectFault::OpenRefused));
}
#[test]
fn residency_tracks_a_buffered_read() {
let fx = build_install("direct-residency");
let path = fx.root.join(&fx.layout.layers[0].file);
let file = File::open(&path).unwrap();
let window = DIO_ALIGN as usize;
if fadvise_dontneed(&file, 0, DIO_ALIGN).is_err() {
return;
}
let Ok(before) = resident_pages(&file, window) else {
return; };
let mut buf = vec![0u8; window];
file.read_at(&mut buf, 0).unwrap();
let after = resident_pages(&file, window).unwrap();
if before == 0 {
assert!(after > 0, "buffered read left nothing resident");
}
}
#[cfg(target_os = "linux")]
fn foreign_file() -> Option<PathBuf> {
use std::os::unix::fs::MetadataExt;
if unsafe { libc::geteuid() } == 0 {
return None;
}
[
"/usr/lib/libc.so.6",
"/usr/lib/x86_64-linux-gnu/libc.so.6",
"/usr/bin/env",
"/bin/sh",
]
.into_iter()
.map(PathBuf::from)
.find(|path| {
let Ok(meta) = std::fs::metadata(path) else {
return false;
};
meta.is_file()
&& meta.len() >= DIO_ALIGN
&& meta.uid() != unsafe { libc::geteuid() }
&& File::open(path).is_ok_and(|file| !residency_is_visible(path, &file))
})
}
#[test]
#[cfg(target_os = "linux")]
fn residency_is_visible_for_a_file_this_process_owns() {
let fx = build_install("direct-residency-owner");
let path = fx.root.join(&fx.layout.layers[0].file);
let file = File::open(&path).unwrap();
assert!(
residency_is_visible(&path, &file),
"our own install must have a residency signal"
);
let mut mode = std::fs::metadata(&path).unwrap().permissions();
mode.set_readonly(true);
std::fs::set_permissions(&path, mode).unwrap();
assert!(
residency_is_visible(&path, &file),
"chmod 444 on our own file must not hide residency"
);
}
#[test]
#[cfg(target_os = "linux")]
fn probe_names_the_permission_fault_on_a_file_owned_by_someone_else() {
let Some(path) = foreign_file() else {
eprintln!("skipping: no foreign-owned probe target (running as root?)");
return;
};
let scratch = Scratch::new();
let outcome = scratch.with(|buf| probe(&path, buf));
println!("probe outcome on {}: {outcome}", path.display());
assert_eq!(
outcome,
DirectSupport::Degraded(DirectFault::ResidencyDenied),
"a file we do not own must report the permission fault, not a \
filesystem one"
);
let file = File::open(&path).unwrap();
let _ = fadvise_dontneed(&file, 0, DIO_ALIGN);
let pages = resident_pages(&file, DIO_ALIGN as usize).unwrap();
assert_eq!(
pages,
(DIO_ALIGN as usize).div_ceil(page_size()),
"mincore was expected to claim full residency for a foreign file"
);
}
#[test]
fn alignment_helper_matches_the_documented_requirement() {
assert!(is_aligned(0));
assert!(is_aligned(DIO_ALIGN));
assert!(is_aligned(3_059_712)); assert!(is_aligned(2_654_208)); assert!(!is_aligned(512)); assert!(!is_aligned(DIO_ALIGN + 1));
}
#[test]
fn open_applies_o_direct_only_when_asked() {
let fx = build_install("direct-open");
let path = fx.root.join(&fx.layout.layers[0].file);
let (buffered, applied) = open(&path, false).unwrap();
assert!(!applied, "O_DIRECT applied when it was not requested");
let mut small = vec![0u8; 7];
buffered.read_at(&mut small, 3).unwrap();
let (direct, applied) = open(&path, true).unwrap();
assert_eq!(
direct.metadata().unwrap().len(),
buffered.metadata().unwrap().len()
);
if applied {
let scratch = Scratch::new();
let read = scratch.with(|buf| direct.read_at(&mut buf[..DIO_ALIGN as usize], 0));
assert_eq!(read.unwrap(), DIO_ALIGN as usize);
}
}
}