use crate::export::Export;
use crate::externref::{StackMapRegistry, VMExternRefActivationsTable};
use crate::memory::{Memory, RuntimeMemoryCreator};
use crate::table::{Table, TableElement};
use crate::traphandlers::Trap;
use crate::vmcontext::{
VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionImport,
VMGlobalDefinition, VMGlobalImport, VMInterrupts, VMMemoryDefinition, VMMemoryImport,
VMSharedSignatureIndex, VMTableDefinition, VMTableImport,
};
use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable};
use indexmap::IndexMap;
use memoffset::offset_of;
use more_asserts::assert_lt;
use std::alloc::Layout;
use std::any::Any;
use std::cell::RefCell;
use std::collections::HashMap;
use std::convert::TryFrom;
use std::hash::Hash;
use std::ptr::NonNull;
use std::rc::Rc;
use std::sync::Arc;
use std::{mem, ptr, slice};
use wasmtime_environ::entity::{packed_option::ReservedValue, EntityRef, EntitySet, PrimaryMap};
use wasmtime_environ::wasm::{
DataIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, ElemIndex, EntityIndex,
FuncIndex, GlobalIndex, MemoryIndex, TableElementType, TableIndex,
};
use wasmtime_environ::{ir, Module, VMOffsets};
mod allocator;
pub use allocator::*;
pub type RuntimeInstance = Rc<IndexMap<String, Export>>;
#[repr(C)]
pub(crate) struct Instance {
module: Arc<Module>,
offsets: VMOffsets,
memories: PrimaryMap<DefinedMemoryIndex, Memory>,
tables: PrimaryMap<DefinedTableIndex, Table>,
dropped_elements: RefCell<EntitySet<ElemIndex>>,
dropped_data: RefCell<EntitySet<DataIndex>>,
host_state: Box<dyn Any>,
vmctx: VMContext,
}
#[allow(clippy::cast_ptr_alignment)]
impl Instance {
unsafe fn vmctx_plus_offset<T>(&self, offset: u32) -> *mut T {
(self.vmctx_ptr() as *mut u8)
.add(usize::try_from(offset).unwrap())
.cast()
}
pub(crate) fn module(&self) -> &Module {
&self.module
}
fn signature_ids_ptr(&self) -> *mut VMSharedSignatureIndex {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_signature_ids_begin()) }
}
fn imported_function(&self, index: FuncIndex) -> &VMFunctionImport {
let index = usize::try_from(index.as_u32()).unwrap();
unsafe { &*self.imported_functions_ptr().add(index) }
}
fn imported_functions_ptr(&self) -> *mut VMFunctionImport {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_imported_functions_begin()) }
}
fn imported_table(&self, index: TableIndex) -> &VMTableImport {
let index = usize::try_from(index.as_u32()).unwrap();
unsafe { &*self.imported_tables_ptr().add(index) }
}
fn imported_tables_ptr(&self) -> *mut VMTableImport {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_imported_tables_begin()) }
}
fn imported_memory(&self, index: MemoryIndex) -> &VMMemoryImport {
let index = usize::try_from(index.as_u32()).unwrap();
unsafe { &*self.imported_memories_ptr().add(index) }
}
fn imported_memories_ptr(&self) -> *mut VMMemoryImport {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_imported_memories_begin()) }
}
fn imported_global(&self, index: GlobalIndex) -> &VMGlobalImport {
let index = usize::try_from(index.as_u32()).unwrap();
unsafe { &*self.imported_globals_ptr().add(index) }
}
fn imported_globals_ptr(&self) -> *mut VMGlobalImport {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_imported_globals_begin()) }
}
#[allow(dead_code)]
fn table(&self, index: DefinedTableIndex) -> VMTableDefinition {
unsafe { *self.table_ptr(index) }
}
fn set_table(&self, index: DefinedTableIndex, table: VMTableDefinition) {
unsafe {
*self.table_ptr(index) = table;
}
}
fn table_ptr(&self, index: DefinedTableIndex) -> *mut VMTableDefinition {
let index = usize::try_from(index.as_u32()).unwrap();
unsafe { self.tables_ptr().add(index) }
}
fn tables_ptr(&self) -> *mut VMTableDefinition {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_tables_begin()) }
}
pub(crate) fn get_memory(&self, index: MemoryIndex) -> VMMemoryDefinition {
if let Some(defined_index) = self.module.defined_memory_index(index) {
self.memory(defined_index)
} else {
let import = self.imported_memory(index);
*unsafe { import.from.as_ref().unwrap() }
}
}
fn memory(&self, index: DefinedMemoryIndex) -> VMMemoryDefinition {
unsafe { *self.memory_ptr(index) }
}
fn set_memory(&self, index: DefinedMemoryIndex, mem: VMMemoryDefinition) {
unsafe {
*self.memory_ptr(index) = mem;
}
}
fn memory_ptr(&self, index: DefinedMemoryIndex) -> *mut VMMemoryDefinition {
let index = usize::try_from(index.as_u32()).unwrap();
unsafe { self.memories_ptr().add(index) }
}
fn memories_ptr(&self) -> *mut VMMemoryDefinition {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_memories_begin()) }
}
fn global(&self, index: DefinedGlobalIndex) -> VMGlobalDefinition {
unsafe { *self.global_ptr(index) }
}
#[allow(dead_code)]
fn set_global(&self, index: DefinedGlobalIndex, global: VMGlobalDefinition) {
unsafe {
*self.global_ptr(index) = global;
}
}
fn global_ptr(&self, index: DefinedGlobalIndex) -> *mut VMGlobalDefinition {
let index = usize::try_from(index.as_u32()).unwrap();
unsafe { self.globals_ptr().add(index) }
}
pub(crate) fn defined_or_imported_global_ptr(
&self,
index: GlobalIndex,
) -> *mut VMGlobalDefinition {
if let Some(index) = self.module.defined_global_index(index) {
self.global_ptr(index)
} else {
self.imported_global(index).from
}
}
fn globals_ptr(&self) -> *mut VMGlobalDefinition {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_globals_begin()) }
}
fn builtin_functions_ptr(&self) -> *mut VMBuiltinFunctionsArray {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_builtin_functions_begin()) }
}
pub fn interrupts(&self) -> *mut *const VMInterrupts {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_interrupts()) }
}
pub fn externref_activations_table(&self) -> *mut *mut VMExternRefActivationsTable {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_externref_activations_table()) }
}
pub fn stack_map_registry(&self) -> *mut *mut StackMapRegistry {
unsafe { self.vmctx_plus_offset(self.offsets.vmctx_stack_map_registry()) }
}
pub fn vmctx(&self) -> &VMContext {
&self.vmctx
}
pub fn vmctx_ptr(&self) -> *mut VMContext {
self.vmctx() as *const VMContext as *mut VMContext
}
pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export {
match export {
EntityIndex::Function(index) => {
let anyfunc = self.get_caller_checked_anyfunc(*index).unwrap();
let anyfunc =
NonNull::new(anyfunc as *const VMCallerCheckedAnyfunc as *mut _).unwrap();
ExportFunction { anyfunc }.into()
}
EntityIndex::Table(index) => {
let (definition, vmctx) =
if let Some(def_index) = self.module.defined_table_index(*index) {
(self.table_ptr(def_index), self.vmctx_ptr())
} else {
let import = self.imported_table(*index);
(import.from, import.vmctx)
};
ExportTable {
definition,
vmctx,
table: self.module.table_plans[*index].clone(),
}
.into()
}
EntityIndex::Memory(index) => {
let (definition, vmctx) =
if let Some(def_index) = self.module.defined_memory_index(*index) {
(self.memory_ptr(def_index), self.vmctx_ptr())
} else {
let import = self.imported_memory(*index);
(import.from, import.vmctx)
};
ExportMemory {
definition,
vmctx,
memory: self.module.memory_plans[*index].clone(),
}
.into()
}
EntityIndex::Global(index) => ExportGlobal {
definition: if let Some(def_index) = self.module.defined_global_index(*index) {
self.global_ptr(def_index)
} else {
self.imported_global(*index).from
},
vmctx: self.vmctx_ptr(),
global: self.module.globals[*index],
}
.into(),
EntityIndex::Instance(_) | EntityIndex::Module(_) => {
panic!("can't use this api for modules/instances")
}
}
}
pub fn exports(&self) -> indexmap::map::Iter<String, EntityIndex> {
self.module.exports.iter()
}
#[inline]
pub fn host_state(&self) -> &dyn Any {
&*self.host_state
}
#[inline]
pub(crate) fn vmctx_offset() -> isize {
offset_of!(Self, vmctx) as isize
}
pub(crate) fn table_index(&self, table: &VMTableDefinition) -> DefinedTableIndex {
let offsets = &self.offsets;
let begin = unsafe {
(&self.vmctx as *const VMContext as *const u8)
.add(usize::try_from(offsets.vmctx_tables_begin()).unwrap())
} as *const VMTableDefinition;
let end: *const VMTableDefinition = table;
let index = DefinedTableIndex::new(
(end as usize - begin as usize) / mem::size_of::<VMTableDefinition>(),
);
assert_lt!(index.index(), self.tables.len());
index
}
pub(crate) fn memory_index(&self, memory: &VMMemoryDefinition) -> DefinedMemoryIndex {
let offsets = &self.offsets;
let begin = unsafe {
(&self.vmctx as *const VMContext as *const u8)
.add(usize::try_from(offsets.vmctx_memories_begin()).unwrap())
} as *const VMMemoryDefinition;
let end: *const VMMemoryDefinition = memory;
let index = DefinedMemoryIndex::new(
(end as usize - begin as usize) / mem::size_of::<VMMemoryDefinition>(),
);
assert_lt!(index.index(), self.memories.len());
index
}
pub(crate) fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option<u32> {
let result = self
.memories
.get(memory_index)
.unwrap_or_else(|| panic!("no memory for index {}", memory_index.index()))
.grow(delta);
self.set_memory(memory_index, self.memories[memory_index].vmmemory());
result
}
pub(crate) unsafe fn imported_memory_grow(
&self,
memory_index: MemoryIndex,
delta: u32,
) -> Option<u32> {
let import = self.imported_memory(memory_index);
let foreign_instance = (&*import.vmctx).instance();
let foreign_memory = &*import.from;
let foreign_index = foreign_instance.memory_index(foreign_memory);
foreign_instance.memory_grow(foreign_index, delta)
}
pub(crate) fn memory_size(&self, memory_index: DefinedMemoryIndex) -> u32 {
self.memories
.get(memory_index)
.unwrap_or_else(|| panic!("no memory for index {}", memory_index.index()))
.size()
}
pub(crate) unsafe fn imported_memory_size(&self, memory_index: MemoryIndex) -> u32 {
let import = self.imported_memory(memory_index);
let foreign_instance = (&mut *import.vmctx).instance();
let foreign_memory = &mut *import.from;
let foreign_index = foreign_instance.memory_index(foreign_memory);
foreign_instance.memory_size(foreign_index)
}
pub(crate) fn table_element_type(&self, table_index: TableIndex) -> TableElementType {
let table = self.get_table(table_index);
table.element_type()
}
pub(crate) fn table_grow(
&self,
table_index: TableIndex,
delta: u32,
init_value: TableElement,
) -> Option<u32> {
let (defined_table_index, instance) =
self.get_defined_table_index_and_instance(table_index);
instance.defined_table_grow(defined_table_index, delta, init_value)
}
fn defined_table_grow(
&self,
table_index: DefinedTableIndex,
delta: u32,
init_value: TableElement,
) -> Option<u32> {
unsafe {
let orig_size = self
.tables
.get(table_index)
.unwrap_or_else(|| panic!("no table for index {}", table_index.index()))
.grow(delta, init_value)?;
self.set_table(table_index, self.tables[table_index].vmtable());
Some(orig_size)
}
}
pub(crate) fn defined_table_fill(
&self,
table_index: DefinedTableIndex,
dst: u32,
val: TableElement,
len: u32,
) -> Result<(), Trap> {
self.tables.get(table_index).unwrap().fill(dst, val, len)
}
fn table_get(&self, table_index: DefinedTableIndex, index: u32) -> Option<TableElement> {
self.tables
.get(table_index)
.unwrap_or_else(|| panic!("no table for index {}", table_index.index()))
.get(index)
}
fn table_set(
&self,
table_index: DefinedTableIndex,
index: u32,
val: TableElement,
) -> Result<(), ()> {
self.tables
.get(table_index)
.unwrap_or_else(|| panic!("no table for index {}", table_index.index()))
.set(index, val)
}
fn alloc_layout(&self) -> Layout {
let size = mem::size_of_val(self)
.checked_add(usize::try_from(self.offsets.size_of_vmctx()).unwrap())
.unwrap();
let align = mem::align_of_val(self);
Layout::from_size_align(size, align).unwrap()
}
pub(crate) fn get_caller_checked_anyfunc(
&self,
index: FuncIndex,
) -> Option<&VMCallerCheckedAnyfunc> {
if index == FuncIndex::reserved_value() {
return None;
}
Some(unsafe { &*self.anyfunc_ptr(index) })
}
unsafe fn anyfunc_ptr(&self, index: FuncIndex) -> *mut VMCallerCheckedAnyfunc {
self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index))
}
fn find_passive_segment<'a, I, D, T>(
index: I,
index_map: &HashMap<I, usize>,
data: &'a Vec<D>,
dropped: &RefCell<EntitySet<I>>,
) -> &'a [T]
where
D: AsRef<[T]>,
I: EntityRef + Hash,
{
match index_map.get(&index) {
Some(index) if !dropped.borrow().contains(I::new(*index)) => data[*index].as_ref(),
_ => &[],
}
}
pub(crate) fn table_init(
&self,
table_index: TableIndex,
elem_index: ElemIndex,
dst: u32,
src: u32,
len: u32,
) -> Result<(), Trap> {
let table = self.get_table(table_index);
let elements = Self::find_passive_segment(
elem_index,
&self.module.passive_elements_map,
&self.module.passive_elements,
&self.dropped_elements,
);
if src
.checked_add(len)
.map_or(true, |n| n as usize > elements.len())
|| dst.checked_add(len).map_or(true, |m| m > table.size())
{
return Err(Trap::wasm(ir::TrapCode::TableOutOfBounds));
}
for (dst, src) in (dst..dst + len).zip(src..src + len) {
let elem = self
.get_caller_checked_anyfunc(elements[src as usize])
.map_or(ptr::null_mut(), |f: &VMCallerCheckedAnyfunc| {
f as *const VMCallerCheckedAnyfunc as *mut _
});
table
.set(dst, TableElement::FuncRef(elem))
.expect("should never panic because we already did the bounds check above");
}
Ok(())
}
pub(crate) fn elem_drop(&self, elem_index: ElemIndex) {
if let Some(index) = self.module.passive_elements_map.get(&elem_index) {
self.dropped_elements
.borrow_mut()
.insert(ElemIndex::new(*index));
}
}
pub(crate) fn memory_copy(
&self,
dst_index: MemoryIndex,
dst: u32,
src_index: MemoryIndex,
src: u32,
len: u32,
) -> Result<(), Trap> {
let src_mem = self.get_memory(src_index);
let dst_mem = self.get_memory(dst_index);
if src
.checked_add(len)
.map_or(true, |n| n as usize > src_mem.current_length)
|| dst
.checked_add(len)
.map_or(true, |m| m as usize > dst_mem.current_length)
{
return Err(Trap::wasm(ir::TrapCode::HeapOutOfBounds));
}
let dst = usize::try_from(dst).unwrap();
let src = usize::try_from(src).unwrap();
unsafe {
let dst = dst_mem.base.add(dst);
let src = src_mem.base.add(src);
ptr::copy(src, dst, len as usize);
}
Ok(())
}
pub(crate) fn defined_memory_fill(
&self,
memory_index: DefinedMemoryIndex,
dst: u32,
val: u32,
len: u32,
) -> Result<(), Trap> {
let memory = self.memory(memory_index);
if dst
.checked_add(len)
.map_or(true, |m| m as usize > memory.current_length)
{
return Err(Trap::wasm(ir::TrapCode::HeapOutOfBounds));
}
let dst = isize::try_from(dst).unwrap();
let val = val as u8;
unsafe {
let dst = memory.base.offset(dst);
ptr::write_bytes(dst, val, len as usize);
}
Ok(())
}
pub(crate) fn imported_memory_fill(
&self,
memory_index: MemoryIndex,
dst: u32,
val: u32,
len: u32,
) -> Result<(), Trap> {
let import = self.imported_memory(memory_index);
unsafe {
let foreign_instance = (&*import.vmctx).instance();
let foreign_memory = &*import.from;
let foreign_index = foreign_instance.memory_index(foreign_memory);
foreign_instance.defined_memory_fill(foreign_index, dst, val, len)
}
}
pub(crate) fn memory_init(
&self,
memory_index: MemoryIndex,
data_index: DataIndex,
dst: u32,
src: u32,
len: u32,
) -> Result<(), Trap> {
let memory = self.get_memory(memory_index);
let data = Self::find_passive_segment(
data_index,
&self.module.passive_data_map,
&self.module.passive_data,
&self.dropped_data,
);
if src
.checked_add(len)
.map_or(true, |n| n as usize > data.len())
|| dst
.checked_add(len)
.map_or(true, |m| m as usize > memory.current_length)
{
return Err(Trap::wasm(ir::TrapCode::HeapOutOfBounds));
}
let src_slice = &data[src as usize..(src + len) as usize];
unsafe {
let dst_start = memory.base.add(dst as usize);
let dst_slice = slice::from_raw_parts_mut(dst_start, len as usize);
dst_slice.copy_from_slice(src_slice);
}
Ok(())
}
pub(crate) fn data_drop(&self, data_index: DataIndex) {
if let Some(index) = self.module.passive_data_map.get(&data_index) {
self.dropped_data
.borrow_mut()
.insert(DataIndex::new(*index));
}
}
pub(crate) fn get_table(&self, table_index: TableIndex) -> &Table {
if let Some(defined_table_index) = self.module.defined_table_index(table_index) {
self.get_defined_table(defined_table_index)
} else {
self.get_foreign_table(table_index)
}
}
pub(crate) fn get_defined_table(&self, index: DefinedTableIndex) -> &Table {
&self.tables[index]
}
pub(crate) fn get_foreign_table(&self, index: TableIndex) -> &Table {
let import = self.imported_table(index);
let foreign_instance = unsafe { (&mut *(import).vmctx).instance() };
let foreign_table = unsafe { &mut *(import).from };
let foreign_index = foreign_instance.table_index(foreign_table);
&foreign_instance.tables[foreign_index]
}
pub(crate) fn get_defined_table_index_and_instance(
&self,
index: TableIndex,
) -> (DefinedTableIndex, &Instance) {
if let Some(defined_table_index) = self.module.defined_table_index(index) {
(defined_table_index, self)
} else {
let import = self.imported_table(index);
let foreign_instance = unsafe { (&mut *(import).vmctx).instance() };
let foreign_table_def = unsafe { &mut *(import).from };
let foreign_table_index = foreign_instance.table_index(foreign_table_def);
(foreign_table_index, foreign_instance)
}
}
}
#[derive(Hash, PartialEq, Eq)]
pub struct InstanceHandle {
instance: *mut Instance,
}
impl InstanceHandle {
pub(crate) unsafe fn new(instance: *mut Instance) -> Self {
Self { instance }
}
pub unsafe fn from_vmctx(vmctx: *mut VMContext) -> Self {
let instance = (&mut *vmctx).instance();
Self {
instance: instance as *const Instance as *mut Instance,
}
}
pub fn vmctx(&self) -> &VMContext {
self.instance().vmctx()
}
pub fn vmctx_ptr(&self) -> *mut VMContext {
self.instance().vmctx_ptr()
}
pub fn module(&self) -> &Module {
self.instance().module()
}
pub fn lookup_by_declaration(&self, export: &EntityIndex) -> Export {
self.instance().lookup_by_declaration(export)
}
pub fn exports(&self) -> indexmap::map::Iter<String, EntityIndex> {
self.instance().exports()
}
pub fn host_state(&self) -> &dyn Any {
self.instance().host_state()
}
pub fn memory_index(&self, memory: &VMMemoryDefinition) -> DefinedMemoryIndex {
self.instance().memory_index(memory)
}
pub fn memory_grow(&self, memory_index: DefinedMemoryIndex, delta: u32) -> Option<u32> {
self.instance().memory_grow(memory_index, delta)
}
pub fn table_index(&self, table: &VMTableDefinition) -> DefinedTableIndex {
self.instance().table_index(table)
}
pub fn table_grow(
&self,
table_index: TableIndex,
delta: u32,
init_value: TableElement,
) -> Option<u32> {
self.instance().table_grow(table_index, delta, init_value)
}
pub fn defined_table_grow(
&self,
table_index: DefinedTableIndex,
delta: u32,
init_value: TableElement,
) -> Option<u32> {
self.instance()
.defined_table_grow(table_index, delta, init_value)
}
pub fn table_get(&self, table_index: DefinedTableIndex, index: u32) -> Option<TableElement> {
self.instance().table_get(table_index, index)
}
pub fn table_set(
&self,
table_index: DefinedTableIndex,
index: u32,
val: TableElement,
) -> Result<(), ()> {
self.instance().table_set(table_index, index, val)
}
pub fn defined_table_fill(
&self,
table_index: DefinedTableIndex,
dst: u32,
val: TableElement,
len: u32,
) -> Result<(), Trap> {
self.instance()
.defined_table_fill(table_index, dst, val, len)
}
pub fn get_defined_table(&self, index: DefinedTableIndex) -> &Table {
self.instance().get_defined_table(index)
}
pub(crate) fn instance(&self) -> &Instance {
unsafe { &*(self.instance as *const Instance) }
}
pub unsafe fn clone(&self) -> InstanceHandle {
InstanceHandle {
instance: self.instance,
}
}
}