use std::fs::File;
use std::ops::{Deref, DerefMut};
use std::slice;
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::error::{Error, Result};
use crate::platform;
use crate::advanced::{HugePageSize, NumaPolicy, PrefetchStrategy};
static TOTAL_MAPPED_MEMORY: AtomicUsize = AtomicUsize::new(0);
static ACTIVE_MAPPINGS: AtomicUsize = AtomicUsize::new(0);
#[derive(Debug, Clone)]
pub struct MmapOptions {
offset: u64,
len: Option<usize>,
pub readable: bool,
pub writable: bool,
pub executable: bool,
pub huge_pages: Option<HugePageSize>,
pub numa_policy: Option<NumaPolicy>,
pub prefetch: Option<PrefetchStrategy>,
pub stack: bool,
pub copy_on_write: bool,
pub populate: bool,
pub alignment: Option<usize>,
}
impl Default for MmapOptions {
fn default() -> MmapOptions {
MmapOptions {
offset: 0,
len: None,
readable: true,
writable: false,
executable: false,
huge_pages: None,
numa_policy: None,
prefetch: None,
stack: false,
copy_on_write: false,
populate: false,
alignment: None,
}
}
}
impl MmapOptions {
#[inline]
pub fn new() -> MmapOptions {
MmapOptions::default()
}
#[inline]
pub fn offset(mut self, offset: u64) -> MmapOptions {
self.offset = offset;
self
}
#[inline]
pub fn len(mut self, len: usize) -> MmapOptions {
self.len = Some(len);
self
}
#[inline]
pub fn read(mut self, readable: bool) -> MmapOptions {
self.readable = readable;
self
}
#[inline]
pub fn write(mut self, writable: bool) -> MmapOptions {
self.writable = writable;
self
}
#[inline]
pub fn exec(mut self, executable: bool) -> MmapOptions {
self.executable = executable;
self
}
#[inline]
pub fn huge_pages(mut self, size: HugePageSize) -> MmapOptions {
self.huge_pages = Some(size);
self
}
#[inline]
pub fn numa_policy(mut self, policy: NumaPolicy) -> MmapOptions {
self.numa_policy = Some(policy);
self
}
#[inline]
pub fn prefetch(mut self, strategy: PrefetchStrategy) -> MmapOptions {
self.prefetch = Some(strategy);
self
}
#[inline]
pub fn stack(mut self, stack: bool) -> MmapOptions {
self.stack = stack;
self
}
#[inline]
pub fn copy_on_write(mut self, cow: bool) -> MmapOptions {
self.copy_on_write = cow;
self
}
#[inline]
pub fn populate(mut self, populate: bool) -> MmapOptions {
self.populate = populate;
self
}
#[inline]
pub fn alignment(mut self, alignment: usize) -> MmapOptions {
self.alignment = Some(alignment);
self
}
#[inline]
pub unsafe fn map(&self, file: &File) -> Result<Mmap> {
self.map_impl(file).map(|raw| Mmap { inner: raw })
}
#[inline]
pub unsafe fn map_mut(&self, file: &File) -> Result<MmapMut> {
let mut options = self.clone();
options.writable = true;
options.map_impl(file).map(|raw| MmapMut { inner: raw })
}
#[inline]
pub unsafe fn map_anon(&self, len: usize) -> Result<MmapMut> {
let mut options = self.clone();
options.len = Some(len);
options.map_anon_impl().map(|raw| MmapMut { inner: raw })
}
unsafe fn map_impl(&self, file: &File) -> Result<MmapRaw> {
if let Some(len) = self.len {
if len == 0 {
return Err(Error::ZeroSizedMapping);
}
}
let len = match self.len {
Some(len) => len,
None => {
let metadata = file.metadata()?;
metadata.len().try_into().map_err(|_| Error::SizeExceedsSystemLimit)?
}
};
let raw = platform::map_file(
file,
self.offset,
len,
self.readable,
self.writable,
self.executable,
self.huge_pages,
self.numa_policy.clone(), self.stack,
self.copy_on_write,
self.populate,
self.alignment,
)?;
TOTAL_MAPPED_MEMORY.fetch_add(len, Ordering::Relaxed);
ACTIVE_MAPPINGS.fetch_add(1, Ordering::Relaxed);
if let Some(strategy) = self.prefetch {
crate::advanced::prefetch::apply_strategy(raw.ptr, len, strategy);
}
Ok(raw)
}
unsafe fn map_anon_impl(&self) -> Result<MmapRaw> {
let len = self.len.ok_or(Error::InvalidArgument("Length must be specified for anonymous mapping".into()))?;
if len == 0 {
return Err(Error::ZeroSizedMapping);
}
let raw = platform::map_anon(
len,
self.readable,
self.writable,
self.executable,
self.huge_pages,
self.numa_policy,
self.stack,
self.populate,
self.alignment,
)?;
TOTAL_MAPPED_MEMORY.fetch_add(len, Ordering::Relaxed);
ACTIVE_MAPPINGS.fetch_add(1, Ordering::Relaxed);
if let Some(strategy) = self.prefetch {
crate::advanced::prefetch::apply_strategy(raw.ptr, len, strategy);
}
Ok(raw)
}
}
#[derive(Debug)]
pub struct MmapRaw {
pub(crate) ptr: *mut u8,
pub(crate) len: usize,
}
impl MmapRaw {
#[inline]
pub fn flush(&self) -> Result<()> {
unsafe { platform::flush(self.ptr, self.len, false) }
}
#[inline]
pub fn flush_async(&self) -> Result<()> {
unsafe { platform::flush(self.ptr, self.len, true) }
}
#[inline]
pub fn advise(&self, advice: platform::Advice) -> Result<()> {
unsafe { platform::advise(self.ptr, self.len, advice) }
}
}
impl Drop for MmapRaw {
fn drop(&mut self) {
unsafe {
if !self.ptr.is_null() {
ACTIVE_MAPPINGS.fetch_sub(1, Ordering::Relaxed);
TOTAL_MAPPED_MEMORY.fetch_sub(self.len, Ordering::Relaxed);
let _ = platform::unmap(self.ptr, self.len);
}
}
}
}
#[derive(Debug)]
pub struct Mmap {
inner: MmapRaw,
}
impl Mmap {
#[inline]
pub unsafe fn map(file: &File) -> Result<Mmap> {
MmapOptions::new().map(file)
}
#[inline]
pub fn len(&self) -> usize {
self.inner.len
}
#[inline]
pub fn is_empty(&self) -> bool {
self.inner.len == 0
}
#[inline]
pub fn as_ptr(&self) -> *const u8 {
self.inner.ptr
}
#[inline]
pub fn flush(&self) -> Result<()> {
self.inner.flush()
}
#[inline]
pub fn flush_async(&self) -> Result<()> {
self.inner.flush_async()
}
#[inline]
pub fn advise(&self, advice: platform::Advice) -> Result<()> {
self.inner.advise(advice)
}
}
impl Deref for Mmap {
type Target = [u8];
#[inline]
fn deref(&self) -> &[u8] {
unsafe { slice::from_raw_parts(self.inner.ptr, self.inner.len) }
}
}
impl AsRef<[u8]> for Mmap {
#[inline]
fn as_ref(&self) -> &[u8] {
self
}
}
#[derive(Debug)]
pub struct MmapMut {
inner: MmapRaw,
}
impl MmapMut {
#[inline]
pub unsafe fn map(file: &File) -> Result<MmapMut> {
MmapOptions::new().write(true).map_mut(file)
}
#[inline]
pub unsafe fn map_anon(len: usize) -> Result<MmapMut> {
MmapOptions::new().write(true).map_anon(len)
}
#[inline]
pub fn len(&self) -> usize {
self.inner.len
}
#[inline]
pub fn is_empty(&self) -> bool {
self.inner.len == 0
}
#[inline]
pub fn as_ptr(&self) -> *const u8 {
self.inner.ptr
}
#[inline]
pub fn as_mut_ptr(&mut self) -> *mut u8 {
self.inner.ptr
}
#[inline]
pub fn flush(&self) -> Result<()> {
self.inner.flush()
}
#[inline]
pub fn flush_async(&self) -> Result<()> {
self.inner.flush_async()
}
#[inline]
pub fn advise(&self, advice: platform::Advice) -> Result<()> {
self.inner.advise(advice)
}
}
impl Deref for MmapMut {
type Target = [u8];
#[inline]
fn deref(&self) -> &[u8] {
unsafe { slice::from_raw_parts(self.inner.ptr, self.inner.len) }
}
}
impl DerefMut for MmapMut {
#[inline]
fn deref_mut(&mut self) -> &mut [u8] {
unsafe { slice::from_raw_parts_mut(self.inner.ptr, self.inner.len) }
}
}
impl AsRef<[u8]> for MmapMut {
#[inline]
fn as_ref(&self) -> &[u8] {
self
}
}
impl AsMut<[u8]> for MmapMut {
#[inline]
fn as_mut(&mut self) -> &mut [u8] {
self
}
}
#[inline]
pub fn total_mapped_memory() -> usize {
TOTAL_MAPPED_MEMORY.load(Ordering::Relaxed)
}
#[inline]
pub fn active_mappings() -> usize {
ACTIVE_MAPPINGS.load(Ordering::Relaxed)
}