use std::fs::File;
use std::io;
use std::os::unix::io::AsRawFd;
use std::ptr;
use libc::{
c_int, c_void, mmap, munmap, msync, madvise, PROT_NONE, PROT_READ, PROT_WRITE, PROT_EXEC,
MAP_SHARED, MAP_PRIVATE, MAP_ANONYMOUS, MAP_HUGETLB, MAP_HUGE_2MB, MAP_HUGE_1GB,
MAP_STACK, MAP_POPULATE, MAP_FIXED_NOREPLACE, MS_ASYNC, MS_SYNC, MS_INVALIDATE,
MADV_NORMAL, MADV_RANDOM, MADV_SEQUENTIAL, MADV_WILLNEED, MADV_DONTNEED, MADV_FREE,
};
use crate::error::{Error, Result};
use crate::mmap::MmapRaw;
use crate::advanced::{HugePageSize, NumaPolicy};
use crate::platform::Advice;
use crate::utils::alignment;
pub unsafe fn map_file(
file: &File,
offset: u64,
len: usize,
readable: bool,
writable: bool,
executable: bool,
huge_pages: Option<HugePageSize>,
numa_policy: Option<NumaPolicy>,
stack: bool,
copy_on_write: bool,
populate: bool,
alignment: Option<usize>,
) -> Result<MmapRaw> {
let mut prot = PROT_NONE;
if readable {
prot |= PROT_READ;
}
if writable {
prot |= PROT_WRITE;
}
if executable {
prot |= PROT_EXEC;
}
let mut flags = if copy_on_write {
MAP_PRIVATE
} else {
MAP_SHARED
};
if stack {
flags |= MAP_STACK;
}
if let Some(page_size) = huge_pages {
flags |= MAP_HUGETLB;
match page_size {
HugePageSize::TwoMB => flags |= MAP_HUGE_2MB,
HugePageSize::OneGB => flags |= MAP_HUGE_1GB,
}
}
if populate {
flags |= MAP_POPULATE;
}
let page_size = page_size();
let aligned_offset = offset & !(page_size as u64 - 1);
let offset_delta = offset - aligned_offset;
let aligned_len = len + offset_delta as usize;
let mut aligned_addr: *mut c_void = ptr::null_mut();
if let Some(align) = alignment {
if align > page_size {
let extra = align - 1;
let map_len = aligned_len + extra;
aligned_addr = mmap(
ptr::null_mut(),
map_len,
prot,
flags,
file.as_raw_fd(),
aligned_offset as i64,
);
if aligned_addr == libc::MAP_FAILED {
return Err(Error::Io(io::Error::last_os_error()));
}
let addr_value = aligned_addr as usize;
let aligned_value = (addr_value + extra) & !(align - 1);
let prefix_size = aligned_value - addr_value;
if prefix_size > 0 {
munmap(aligned_addr, prefix_size);
}
let suffix_size = extra - prefix_size;
if suffix_size > 0 {
munmap(
(aligned_addr as usize + prefix_size + aligned_len) as *mut c_void,
suffix_size,
);
}
aligned_addr = aligned_value as *mut c_void;
}
}
let addr = if aligned_addr.is_null() {
mmap(
ptr::null_mut(),
aligned_len,
prot,
flags,
file.as_raw_fd(),
aligned_offset as i64,
)
} else {
aligned_addr
};
if addr == libc::MAP_FAILED {
return Err(Error::Io(io::Error::last_os_error()));
}
if let Some(policy) = numa_policy {
apply_numa_policy(addr, aligned_len, policy)?;
}
let ptr = (addr as usize + offset_delta as usize) as *mut u8;
Ok(MmapRaw { ptr, len })
}
pub unsafe fn map_anon(
len: usize,
readable: bool,
writable: bool,
executable: bool,
huge_pages: Option<HugePageSize>,
numa_policy: Option<NumaPolicy>,
stack: bool,
populate: bool,
alignment: Option<usize>,
) -> Result<MmapRaw> {
let mut prot = PROT_NONE;
if readable {
prot |= PROT_READ;
}
if writable {
prot |= PROT_WRITE;
}
if executable {
prot |= PROT_EXEC;
}
let mut flags = MAP_PRIVATE | MAP_ANONYMOUS;
if stack {
flags |= MAP_STACK;
}
if let Some(page_size) = huge_pages {
flags |= MAP_HUGETLB;
match page_size {
HugePageSize::TwoMB => flags |= MAP_HUGE_2MB,
HugePageSize::OneGB => flags |= MAP_HUGE_1GB,
}
}
if populate {
flags |= MAP_POPULATE;
}
let mut aligned_len = len;
let mut aligned_addr: *mut c_void = ptr::null_mut();
if let Some(align) = alignment {
let page_size = page_size();
if align > page_size {
let extra = align - 1;
aligned_len = len + extra;
aligned_addr = mmap(
ptr::null_mut(),
aligned_len,
prot,
flags,
-1,
0,
);
if aligned_addr == libc::MAP_FAILED {
return Err(Error::Io(io::Error::last_os_error()));
}
let addr_value = aligned_addr as usize;
let aligned_value = (addr_value + extra) & !(align - 1);
let prefix_size = aligned_value - addr_value;
if prefix_size > 0 {
munmap(aligned_addr, prefix_size);
}
let suffix_size = extra - prefix_size;
if suffix_size > 0 {
munmap(
(aligned_addr as usize + prefix_size + len) as *mut c_void,
suffix_size,
);
}
aligned_addr = aligned_value as *mut c_void;
aligned_len = len;
}
}
let addr = if aligned_addr.is_null() {
mmap(
ptr::null_mut(),
len,
prot,
flags,
-1,
0,
)
} else {
aligned_addr
};
if addr == libc::MAP_FAILED {
return Err(Error::Io(io::Error::last_os_error()));
}
if let Some(policy) = numa_policy {
apply_numa_policy(addr, aligned_len, policy)?;
}
Ok(MmapRaw { ptr: addr as *mut u8, len: aligned_len })
}
pub unsafe fn flush(addr: *mut u8, len: usize, async_flush: bool) -> Result<()> {
let flags = if async_flush { MS_ASYNC } else { MS_SYNC };
let result = msync(addr as *mut c_void, len, flags);
if result == 0 {
Ok(())
} else {
Err(Error::Io(io::Error::last_os_error()))
}
}
pub unsafe fn unmap(addr: *mut u8, len: usize) -> Result<()> {
let result = munmap(addr as *mut c_void, len);
if result == 0 {
Ok(())
} else {
Err(Error::Io(io::Error::last_os_error()))
}
}
pub unsafe fn advise(addr: *mut u8, len: usize, advice: Advice) -> Result<()> {
let advice_flag = match advice {
Advice::Normal => MADV_NORMAL,
Advice::Random => MADV_RANDOM,
Advice::Sequential => MADV_SEQUENTIAL,
Advice::WillNeed => MADV_WILLNEED,
Advice::DontNeed => MADV_DONTNEED,
Advice::SequentialOnce => MADV_SEQUENTIAL, Advice::RandomOnce => MADV_RANDOM, Advice::Free => MADV_FREE,
};
let result = madvise(addr as *mut c_void, len, advice_flag);
if result == 0 {
Ok(())
} else {
Err(Error::Io(io::Error::last_os_error()))
}
}
unsafe fn apply_numa_policy(addr: *mut c_void, len: usize, policy: NumaPolicy) -> Result<()> {
#[cfg(target_os = "linux")]
{
match policy {
NumaPolicy::Interleave(nodes) => {
Ok(())
},
NumaPolicy::Bind(node) => {
Ok(())
},
NumaPolicy::Preferred(node) => {
Ok(())
},
}
}
#[cfg(not(target_os = "linux"))]
{
Err(Error::PlatformError(libc::ENOSYS))
}
}
#[inline]
fn page_size() -> usize {
unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize }
}