use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition};
use crate::{Store, VMExternRef};
use anyhow::{bail, format_err, Error, Result};
use std::convert::{TryFrom, TryInto};
use std::ops::Range;
use std::ptr;
use wasmtime_environ::{TablePlan, Trap, WasmType, FUNCREF_INIT_BIT, FUNCREF_MASK};
#[derive(Clone)]
pub enum TableElement {
FuncRef(*mut VMCallerCheckedAnyfunc),
ExternRef(Option<VMExternRef>),
UninitFunc,
}
#[derive(Copy, Clone, PartialEq, Eq)]
pub enum TableElementType {
Func,
Extern,
}
unsafe impl Send for TableElement where VMExternRef: Send {}
unsafe impl Sync for TableElement where VMExternRef: Sync {}
impl TableElement {
unsafe fn from_table_value(ty: TableElementType, ptr: usize) -> Self {
match (ty, ptr) {
(TableElementType::Func, 0) => Self::UninitFunc,
(TableElementType::Func, ptr) => Self::FuncRef((ptr & FUNCREF_MASK) as _),
(TableElementType::Extern, 0) => Self::ExternRef(None),
(TableElementType::Extern, ptr) => {
Self::ExternRef(Some(VMExternRef::from_raw(ptr as *mut u8)))
}
}
}
unsafe fn clone_from_table_value(ty: TableElementType, ptr: usize) -> Self {
match (ty, ptr) {
(TableElementType::Func, 0) => Self::UninitFunc,
(TableElementType::Func, ptr) => Self::FuncRef((ptr & FUNCREF_MASK) as _),
(TableElementType::Extern, 0) => Self::ExternRef(None),
(TableElementType::Extern, ptr) => {
Self::ExternRef(Some(VMExternRef::clone_from_raw(ptr as *mut u8)))
}
}
}
unsafe fn into_table_value(self) -> usize {
match self {
Self::UninitFunc => 0,
Self::FuncRef(e) => (e as usize) | FUNCREF_INIT_BIT,
Self::ExternRef(e) => e.map_or(0, |e| e.into_raw() as usize),
}
}
pub(crate) unsafe fn into_ref_asserting_initialized(self) -> usize {
match self {
Self::FuncRef(e) => e as usize,
Self::ExternRef(e) => e.map_or(0, |e| e.into_raw() as usize),
Self::UninitFunc => panic!("Uninitialized table element value outside of table slot"),
}
}
pub(crate) fn is_uninit(&self) -> bool {
match self {
Self::UninitFunc => true,
_ => false,
}
}
}
impl From<*mut VMCallerCheckedAnyfunc> for TableElement {
fn from(f: *mut VMCallerCheckedAnyfunc) -> TableElement {
TableElement::FuncRef(f)
}
}
impl From<Option<VMExternRef>> for TableElement {
fn from(x: Option<VMExternRef>) -> TableElement {
TableElement::ExternRef(x)
}
}
impl From<VMExternRef> for TableElement {
fn from(x: VMExternRef) -> TableElement {
TableElement::ExternRef(Some(x))
}
}
pub enum Table {
Static {
data: &'static mut [usize],
size: u32,
ty: TableElementType,
},
Dynamic {
elements: Vec<usize>,
ty: TableElementType,
maximum: Option<u32>,
},
}
fn wasm_to_table_type(ty: WasmType) -> Result<TableElementType> {
match ty {
WasmType::FuncRef => Ok(TableElementType::Func),
WasmType::ExternRef => Ok(TableElementType::Extern),
ty => bail!("invalid table element type {:?}", ty),
}
}
impl Table {
pub fn new_dynamic(plan: &TablePlan, store: &mut dyn Store) -> Result<Self> {
Self::limit_new(plan, store)?;
let elements = vec![0; plan.table.minimum as usize];
let ty = wasm_to_table_type(plan.table.wasm_ty)?;
let maximum = plan.table.maximum;
Ok(Table::Dynamic {
elements,
ty,
maximum,
})
}
pub fn new_static(
plan: &TablePlan,
data: &'static mut [usize],
store: &mut dyn Store,
) -> Result<Self> {
Self::limit_new(plan, store)?;
let size = plan.table.minimum;
let ty = wasm_to_table_type(plan.table.wasm_ty)?;
if data.len() < (plan.table.minimum as usize) {
bail!(
"initial table size of {} exceeds the pooling allocator's \
configured maximum table size of {} elements",
plan.table.minimum,
data.len(),
);
}
let data = match plan.table.maximum {
Some(max) if (max as usize) < data.len() => &mut data[..max as usize],
_ => data,
};
Ok(Table::Static { data, size, ty })
}
fn limit_new(plan: &TablePlan, store: &mut dyn Store) -> Result<()> {
if !store.table_growing(0, plan.table.minimum, plan.table.maximum)? {
bail!(
"table minimum size of {} elements exceeds table limits",
plan.table.minimum
);
}
Ok(())
}
pub fn element_type(&self) -> TableElementType {
match self {
Table::Static { ty, .. } => *ty,
Table::Dynamic { ty, .. } => *ty,
}
}
#[cfg(feature = "pooling-allocator")]
pub(crate) fn is_static(&self) -> bool {
if let Table::Static { .. } = self {
true
} else {
false
}
}
pub fn size(&self) -> u32 {
match self {
Table::Static { size, .. } => *size,
Table::Dynamic { elements, .. } => elements.len().try_into().unwrap(),
}
}
pub fn maximum(&self) -> Option<u32> {
match self {
Table::Static { data, .. } => Some(data.len() as u32),
Table::Dynamic { maximum, .. } => maximum.clone(),
}
}
pub fn init_funcs(
&mut self,
dst: u32,
items: impl ExactSizeIterator<Item = *mut VMCallerCheckedAnyfunc>,
) -> Result<(), Trap> {
assert!(self.element_type() == TableElementType::Func);
let elements = match self
.elements_mut()
.get_mut(usize::try_from(dst).unwrap()..)
.and_then(|s| s.get_mut(..items.len()))
{
Some(elements) => elements,
None => return Err(Trap::TableOutOfBounds),
};
for (item, slot) in items.zip(elements) {
unsafe {
*slot = TableElement::FuncRef(item).into_table_value();
}
}
Ok(())
}
pub fn fill(&mut self, dst: u32, val: TableElement, len: u32) -> Result<(), Trap> {
let start = dst as usize;
let end = start
.checked_add(len as usize)
.ok_or_else(|| Trap::TableOutOfBounds)?;
if end > self.size() as usize {
return Err(Trap::TableOutOfBounds);
}
debug_assert!(self.type_matches(&val));
let ty = self.element_type();
if let Some((last, elements)) = self.elements_mut()[start..end].split_last_mut() {
for e in elements {
Self::set_raw(ty, e, val.clone());
}
Self::set_raw(ty, last, val);
}
Ok(())
}
pub unsafe fn grow(
&mut self,
delta: u32,
init_value: TableElement,
store: &mut dyn Store,
) -> Result<Option<u32>, Error> {
let old_size = self.size();
let new_size = match old_size.checked_add(delta) {
Some(s) => s,
None => return Ok(None),
};
if !store.table_growing(old_size, new_size, self.maximum())? {
return Ok(None);
}
if let Some(max) = self.maximum() {
if new_size > max {
store.table_grow_failed(&format_err!("Table maximum size exceeded"));
return Ok(None);
}
}
debug_assert!(self.type_matches(&init_value));
match self {
Table::Static { size, data, .. } => {
debug_assert!(data[*size as usize..new_size as usize]
.iter()
.all(|x| *x == 0));
*size = new_size;
}
Table::Dynamic { elements, .. } => {
elements.resize(new_size as usize, 0);
}
}
self.fill(old_size, init_value, delta)
.expect("table should not be out of bounds");
Ok(Some(old_size))
}
pub fn get(&self, index: u32) -> Option<TableElement> {
self.elements()
.get(index as usize)
.map(|p| unsafe { TableElement::clone_from_table_value(self.element_type(), *p) })
}
pub fn set(&mut self, index: u32, elem: TableElement) -> Result<(), ()> {
if !self.type_matches(&elem) {
return Err(());
}
let ty = self.element_type();
let e = self.elements_mut().get_mut(index as usize).ok_or(())?;
Self::set_raw(ty, e, elem);
Ok(())
}
pub unsafe fn copy(
dst_table: *mut Self,
src_table: *mut Self,
dst_index: u32,
src_index: u32,
len: u32,
) -> Result<(), Trap> {
if src_index
.checked_add(len)
.map_or(true, |n| n > (*src_table).size())
|| dst_index
.checked_add(len)
.map_or(true, |m| m > (*dst_table).size())
{
return Err(Trap::TableOutOfBounds);
}
debug_assert!(
(*dst_table).element_type() == (*src_table).element_type(),
"table element type mismatch"
);
let src_range = src_index as usize..src_index as usize + len as usize;
let dst_range = dst_index as usize..dst_index as usize + len as usize;
if ptr::eq(dst_table, src_table) {
(*dst_table).copy_elements_within(dst_range, src_range);
} else {
Self::copy_elements(&mut *dst_table, &*src_table, dst_range, src_range);
}
Ok(())
}
pub fn vmtable(&mut self) -> VMTableDefinition {
match self {
Table::Static { data, size, .. } => VMTableDefinition {
base: data.as_mut_ptr().cast(),
current_elements: *size,
},
Table::Dynamic { elements, .. } => VMTableDefinition {
base: elements.as_mut_ptr().cast(),
current_elements: elements.len().try_into().unwrap(),
},
}
}
fn type_matches(&self, val: &TableElement) -> bool {
match (&val, self.element_type()) {
(TableElement::FuncRef(_), TableElementType::Func) => true,
(TableElement::ExternRef(_), TableElementType::Extern) => true,
_ => false,
}
}
fn elements(&self) -> &[usize] {
match self {
Table::Static { data, size, .. } => &data[..*size as usize],
Table::Dynamic { elements, .. } => &elements[..],
}
}
fn elements_mut(&mut self) -> &mut [usize] {
match self {
Table::Static { data, size, .. } => &mut data[..*size as usize],
Table::Dynamic { elements, .. } => &mut elements[..],
}
}
fn set_raw(ty: TableElementType, elem: &mut usize, val: TableElement) {
unsafe {
let old = *elem;
*elem = val.into_table_value();
let _ = TableElement::from_table_value(ty, old);
}
}
fn copy_elements(
dst_table: &mut Self,
src_table: &Self,
dst_range: Range<usize>,
src_range: Range<usize>,
) {
debug_assert!(!ptr::eq(dst_table, src_table));
let ty = dst_table.element_type();
match ty {
TableElementType::Func => {
dst_table.elements_mut()[dst_range]
.copy_from_slice(&src_table.elements()[src_range]);
}
TableElementType::Extern => {
let dst = dst_table.elements_mut();
let src = src_table.elements();
for (s, d) in src_range.zip(dst_range) {
let elem = unsafe { TableElement::clone_from_table_value(ty, src[s]) };
Self::set_raw(ty, &mut dst[d], elem);
}
}
}
}
fn copy_elements_within(&mut self, dst_range: Range<usize>, src_range: Range<usize>) {
let ty = self.element_type();
let dst = self.elements_mut();
match ty {
TableElementType::Func => {
dst.copy_within(src_range, dst_range.start);
}
TableElementType::Extern => {
if dst_range.start <= src_range.start {
for (s, d) in src_range.zip(dst_range) {
let elem = unsafe { TableElement::clone_from_table_value(ty, dst[s]) };
Self::set_raw(ty, &mut dst[d], elem);
}
} else {
for (s, d) in src_range.rev().zip(dst_range.rev()) {
let elem = unsafe { TableElement::clone_from_table_value(ty, dst[s]) };
Self::set_raw(ty, &mut dst[d], elem);
}
}
}
}
}
}
impl Drop for Table {
fn drop(&mut self) {
let ty = self.element_type();
if let TableElementType::Func = ty {
return;
}
for element in self.elements() {
drop(unsafe { TableElement::from_table_value(ty, *element) });
}
}
}
impl Default for Table {
fn default() -> Self {
Table::Static {
data: &mut [],
size: 0,
ty: TableElementType::Func,
}
}
}