use crate::trap::Trap;
use crate::{mmap::Mmap, store::MaybeInstanceOwned, vmcontext::VMMemoryDefinition};
use more_asserts::assert_ge;
use std::cell::UnsafeCell;
use std::convert::TryInto;
use std::ptr::NonNull;
use std::slice;
use std::sync::{Arc, RwLock};
use wasmer_types::{Bytes, MemoryError, MemoryStyle, MemoryType, Pages};
#[derive(Debug)]
struct WasmMmap {
alloc: Mmap,
size: Pages,
vm_memory_definition: MaybeInstanceOwned<VMMemoryDefinition>,
}
impl WasmMmap {
fn get_vm_memory_definition(&self) -> NonNull<VMMemoryDefinition> {
self.vm_memory_definition.as_ptr()
}
fn size(&self) -> Pages {
unsafe {
let md_ptr = self.get_vm_memory_definition();
let md = md_ptr.as_ref();
Bytes::from(md.current_length).try_into().unwrap()
}
}
fn grow(&mut self, delta: Pages, conf: VMMemoryConfig) -> Result<Pages, MemoryError> {
if delta.0 == 0 {
return Ok(self.size);
}
let new_pages = self
.size
.checked_add(delta)
.ok_or(MemoryError::CouldNotGrow {
current: self.size,
attempted_delta: delta,
})?;
let prev_pages = self.size;
if let Some(maximum) = conf.maximum {
if new_pages > maximum {
return Err(MemoryError::CouldNotGrow {
current: self.size,
attempted_delta: delta,
});
}
}
if new_pages >= Pages::max_value() {
return Err(MemoryError::CouldNotGrow {
current: self.size,
attempted_delta: delta,
});
}
let delta_bytes = delta.bytes().0;
let prev_bytes = prev_pages.bytes().0;
let new_bytes = new_pages.bytes().0;
if new_bytes > self.alloc.len() - conf.offset_guard_size {
let guard_bytes = conf.offset_guard_size;
let request_bytes =
new_bytes
.checked_add(guard_bytes)
.ok_or_else(|| MemoryError::CouldNotGrow {
current: new_pages,
attempted_delta: Bytes(guard_bytes).try_into().unwrap(),
})?;
let mut new_mmap =
Mmap::accessible_reserved(new_bytes, request_bytes).map_err(MemoryError::Region)?;
let copy_len = self.alloc.len() - conf.offset_guard_size;
new_mmap.as_mut_slice()[..copy_len].copy_from_slice(&self.alloc.as_slice()[..copy_len]);
self.alloc = new_mmap;
} else if delta_bytes > 0 {
self.alloc
.make_accessible(prev_bytes, delta_bytes)
.map_err(MemoryError::Region)?;
}
self.size = new_pages;
unsafe {
let mut md_ptr = self.vm_memory_definition.as_ptr();
let md = md_ptr.as_mut();
md.current_length = new_pages.bytes().0;
md.base = self.alloc.as_mut_ptr() as _;
}
Ok(prev_pages)
}
pub fn duplicate(&mut self) -> Result<Self, MemoryError> {
let mem_length = self.size.bytes().0;
let mut alloc = self
.alloc
.duplicate(Some(mem_length))
.map_err(MemoryError::Generic)?;
let base_ptr = alloc.as_mut_ptr();
Ok(Self {
vm_memory_definition: MaybeInstanceOwned::Host(Box::new(UnsafeCell::new(
VMMemoryDefinition {
base: base_ptr,
current_length: mem_length,
},
))),
alloc,
size: self.size,
})
}
}
#[derive(Debug, Clone)]
struct VMMemoryConfig {
maximum: Option<Pages>,
memory: MemoryType,
style: MemoryStyle,
offset_guard_size: usize,
}
impl VMMemoryConfig {
fn ty(&self, minimum: Pages) -> MemoryType {
let mut out = self.memory;
out.minimum = minimum;
out
}
fn style(&self) -> MemoryStyle {
self.style
}
}
#[derive(Debug)]
pub struct VMOwnedMemory {
mmap: WasmMmap,
config: VMMemoryConfig,
}
unsafe impl Send for VMOwnedMemory {}
unsafe impl Sync for VMOwnedMemory {}
impl VMOwnedMemory {
pub fn new(memory: &MemoryType, style: &MemoryStyle) -> Result<Self, MemoryError> {
unsafe { Self::new_internal(memory, style, None) }
}
pub unsafe fn from_definition(
memory: &MemoryType,
style: &MemoryStyle,
vm_memory_location: NonNull<VMMemoryDefinition>,
) -> Result<Self, MemoryError> {
Self::new_internal(memory, style, Some(vm_memory_location))
}
unsafe fn new_internal(
memory: &MemoryType,
style: &MemoryStyle,
vm_memory_location: Option<NonNull<VMMemoryDefinition>>,
) -> Result<Self, MemoryError> {
if memory.minimum > Pages::max_value() {
return Err(MemoryError::MinimumMemoryTooLarge {
min_requested: memory.minimum,
max_allowed: Pages::max_value(),
});
}
if let Some(max) = memory.maximum {
if max > Pages::max_value() {
return Err(MemoryError::MaximumMemoryTooLarge {
max_requested: max,
max_allowed: Pages::max_value(),
});
}
if max < memory.minimum {
return Err(MemoryError::InvalidMemory {
reason: format!(
"the maximum ({} pages) is less than the minimum ({} pages)",
max.0, memory.minimum.0
),
});
}
}
let offset_guard_bytes = style.offset_guard_size() as usize;
let minimum_pages = match style {
MemoryStyle::Dynamic { .. } => memory.minimum,
MemoryStyle::Static { bound, .. } => {
assert_ge!(*bound, memory.minimum);
*bound
}
};
let minimum_bytes = minimum_pages.bytes().0;
let request_bytes = minimum_bytes.checked_add(offset_guard_bytes).unwrap();
let mapped_pages = memory.minimum;
let mapped_bytes = mapped_pages.bytes();
let mut alloc = Mmap::accessible_reserved(mapped_bytes.0, request_bytes)
.map_err(MemoryError::Region)?;
let base_ptr = alloc.as_mut_ptr();
let mem_length = memory.minimum.bytes().0;
let mmap = WasmMmap {
vm_memory_definition: if let Some(mem_loc) = vm_memory_location {
{
let mut ptr = mem_loc;
let md = ptr.as_mut();
md.base = base_ptr;
md.current_length = mem_length;
}
MaybeInstanceOwned::Instance(mem_loc)
} else {
MaybeInstanceOwned::Host(Box::new(UnsafeCell::new(VMMemoryDefinition {
base: base_ptr,
current_length: mem_length,
})))
},
alloc,
size: memory.minimum,
};
Ok(Self {
mmap,
config: VMMemoryConfig {
maximum: memory.maximum,
offset_guard_size: offset_guard_bytes,
memory: *memory,
style: *style,
},
})
}
pub fn to_shared(self) -> VMSharedMemory {
VMSharedMemory {
mmap: Arc::new(RwLock::new(self.mmap)),
config: self.config,
}
}
pub fn duplicate(&mut self) -> Result<Self, MemoryError> {
Ok(Self {
mmap: self.mmap.duplicate()?,
config: self.config.clone(),
})
}
}
impl LinearMemory for VMOwnedMemory {
fn ty(&self) -> MemoryType {
let minimum = self.mmap.size();
self.config.ty(minimum)
}
fn size(&self) -> Pages {
self.mmap.size()
}
fn style(&self) -> MemoryStyle {
self.config.style()
}
fn grow(&mut self, delta: Pages) -> Result<Pages, MemoryError> {
self.mmap.grow(delta, self.config.clone())
}
fn vmmemory(&self) -> NonNull<VMMemoryDefinition> {
self.mmap.vm_memory_definition.as_ptr()
}
fn try_clone(&self) -> Option<Box<dyn LinearMemory + 'static>> {
None
}
fn duplicate(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
let forked = Self::duplicate(self)?;
Ok(Box::new(forked))
}
}
#[derive(Debug, Clone)]
pub struct VMSharedMemory {
mmap: Arc<RwLock<WasmMmap>>,
config: VMMemoryConfig,
}
unsafe impl Send for VMSharedMemory {}
unsafe impl Sync for VMSharedMemory {}
impl VMSharedMemory {
pub fn new(memory: &MemoryType, style: &MemoryStyle) -> Result<Self, MemoryError> {
Ok(VMOwnedMemory::new(memory, style)?.to_shared())
}
pub unsafe fn from_definition(
memory: &MemoryType,
style: &MemoryStyle,
vm_memory_location: NonNull<VMMemoryDefinition>,
) -> Result<Self, MemoryError> {
Ok(VMOwnedMemory::from_definition(memory, style, vm_memory_location)?.to_shared())
}
pub fn duplicate(&mut self) -> Result<Self, MemoryError> {
let mut guard = self.mmap.write().unwrap();
Ok(Self {
mmap: Arc::new(RwLock::new(guard.duplicate()?)),
config: self.config.clone(),
})
}
}
impl LinearMemory for VMSharedMemory {
fn ty(&self) -> MemoryType {
let minimum = {
let guard = self.mmap.read().unwrap();
guard.size()
};
self.config.ty(minimum)
}
fn size(&self) -> Pages {
let guard = self.mmap.read().unwrap();
guard.size()
}
fn style(&self) -> MemoryStyle {
self.config.style()
}
fn grow(&mut self, delta: Pages) -> Result<Pages, MemoryError> {
let mut guard = self.mmap.write().unwrap();
guard.grow(delta, self.config.clone())
}
fn vmmemory(&self) -> NonNull<VMMemoryDefinition> {
let guard = self.mmap.read().unwrap();
guard.vm_memory_definition.as_ptr()
}
fn try_clone(&self) -> Option<Box<dyn LinearMemory + 'static>> {
Some(Box::new(self.clone()))
}
fn duplicate(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
let forked = Self::duplicate(self)?;
Ok(Box::new(forked))
}
}
impl From<VMOwnedMemory> for VMMemory {
fn from(mem: VMOwnedMemory) -> Self {
Self(Box::new(mem))
}
}
impl From<VMSharedMemory> for VMMemory {
fn from(mem: VMSharedMemory) -> Self {
Self(Box::new(mem))
}
}
#[derive(Debug)]
pub struct VMMemory(pub Box<dyn LinearMemory + 'static>);
impl From<Box<dyn LinearMemory + 'static>> for VMMemory {
fn from(mem: Box<dyn LinearMemory + 'static>) -> Self {
Self(mem)
}
}
impl LinearMemory for VMMemory {
fn ty(&self) -> MemoryType {
self.0.ty()
}
fn size(&self) -> Pages {
self.0.size()
}
fn grow(&mut self, delta: Pages) -> Result<Pages, MemoryError> {
self.0.grow(delta)
}
fn style(&self) -> MemoryStyle {
self.0.style()
}
fn vmmemory(&self) -> NonNull<VMMemoryDefinition> {
self.0.vmmemory()
}
fn try_clone(&self) -> Option<Box<dyn LinearMemory + 'static>> {
self.0.try_clone()
}
unsafe fn initialize_with_data(&self, start: usize, data: &[u8]) -> Result<(), Trap> {
self.0.initialize_with_data(start, data)
}
fn duplicate(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
self.0.duplicate()
}
}
impl VMMemory {
pub fn new(memory: &MemoryType, style: &MemoryStyle) -> Result<Self, MemoryError> {
Ok(if memory.shared {
Self(Box::new(VMSharedMemory::new(memory, style)?))
} else {
Self(Box::new(VMOwnedMemory::new(memory, style)?))
})
}
pub fn get_runtime_size(&self) -> u32 {
self.0.size().0
}
pub unsafe fn from_definition(
memory: &MemoryType,
style: &MemoryStyle,
vm_memory_location: NonNull<VMMemoryDefinition>,
) -> Result<Self, MemoryError> {
Ok(if memory.shared {
Self(Box::new(VMSharedMemory::from_definition(
memory,
style,
vm_memory_location,
)?))
} else {
Self(Box::new(VMOwnedMemory::from_definition(
memory,
style,
vm_memory_location,
)?))
})
}
pub fn from_custom<IntoVMMemory>(memory: IntoVMMemory) -> Self
where
IntoVMMemory: Into<Self>,
{
memory.into()
}
pub fn duplicate(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
LinearMemory::duplicate(self)
}
}
#[doc(hidden)]
pub unsafe fn initialize_memory_with_data(
memory: &VMMemoryDefinition,
start: usize,
data: &[u8],
) -> Result<(), Trap> {
let mem_slice = slice::from_raw_parts_mut(memory.base, memory.current_length);
let end = start + data.len();
let to_init = &mut mem_slice[start..end];
to_init.copy_from_slice(data);
Ok(())
}
pub trait LinearMemory
where
Self: std::fmt::Debug + Send,
{
fn ty(&self) -> MemoryType;
fn size(&self) -> Pages;
fn style(&self) -> MemoryStyle;
fn grow(&mut self, delta: Pages) -> Result<Pages, MemoryError>;
fn vmmemory(&self) -> NonNull<VMMemoryDefinition>;
fn try_clone(&self) -> Option<Box<dyn LinearMemory + 'static>>;
#[doc(hidden)]
unsafe fn initialize_with_data(&self, start: usize, data: &[u8]) -> Result<(), Trap> {
let memory = self.vmmemory().as_ref();
initialize_memory_with_data(memory, start, data)
}
fn duplicate(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError>;
}