mod block_type;
mod code_map;
mod config;
mod executor;
mod func_types;
mod limits;
mod resumable;
mod translator;
mod utils;
pub(crate) use self::{
block_type::BlockType,
executor::{
Cell,
InOutParams,
InOutResults,
Inst,
LiftFromCells,
LiftFromCellsByValue,
LoadByVal,
LowerToCells,
Stack,
},
func_types::DedupFuncType,
translator::{
FuncTranslationDriver,
FuncTranslator,
FuncTranslatorAllocations,
LazyFuncTranslator,
ValidatingFuncTranslator,
WasmTranslator,
required_cells_for_tys,
},
};
use self::{
code_map::{CodeMap, CompiledFuncEntity},
func_types::FuncTypeRegistry,
resumable::ResumableCallBase,
};
pub use self::{
code_map::{EngineFunc, EngineFuncSpan, EngineFuncSpanIter},
config::{CompilationMode, Config},
limits::{EnforcedLimits, EnforcedLimitsError, StackConfig},
resumable::{
ResumableCall,
ResumableCallHostTrap,
ResumableCallOutOfFuel,
ResumableHostTrapError,
ResumableOutOfFuelError,
TypedResumableCall,
TypedResumableCallHostTrap,
TypedResumableCallOutOfFuel,
},
translator::TranslationError,
};
use crate::{
Error,
Func,
FuncType,
StoreContextMut,
module::{FuncIdx, ModuleHeader},
};
use alloc::{
sync::{Arc, Weak},
vec::Vec,
};
use core::sync::atomic::{AtomicU32, Ordering};
use spin::{Mutex, RwLock};
use wasmparser::{FuncToValidate, FuncValidatorAllocations, ValidatorResources};
#[cfg(doc)]
use crate::Store;
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct EngineId(u32);
impl EngineId {
fn new() -> Self {
static CURRENT_STORE_IDX: AtomicU32 = AtomicU32::new(0);
let next_idx = CURRENT_STORE_IDX.fetch_add(1, Ordering::AcqRel);
Self(next_idx)
}
pub fn wrap<T>(self, value: T) -> EngineOwned<T> {
EngineOwned {
engine: self,
value,
}
}
pub fn unwrap<T>(self, owned: EngineOwned<T>) -> Option<T> {
if self != owned.engine {
return None;
}
Some(owned.value)
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct EngineOwned<T> {
engine: EngineId,
value: T,
}
#[derive(Debug, Clone)]
pub struct Engine {
inner: Arc<EngineInner>,
}
#[derive(Debug, Clone)]
pub struct EngineWeak {
inner: Weak<EngineInner>,
}
impl EngineWeak {
pub fn upgrade(&self) -> Option<Engine> {
let inner = self.inner.upgrade()?;
Some(Engine { inner })
}
}
impl Default for Engine {
fn default() -> Self {
Self::new(&Config::default())
}
}
impl Engine {
pub fn new(config: &Config) -> Self {
Self {
inner: Arc::new(EngineInner::new(config)),
}
}
pub fn weak(&self) -> EngineWeak {
EngineWeak {
inner: Arc::downgrade(&self.inner),
}
}
pub fn config(&self) -> &Config {
self.inner.config()
}
pub fn same(a: &Engine, b: &Engine) -> bool {
Arc::ptr_eq(&a.inner, &b.inner)
}
pub(super) fn alloc_func_type(&self, func_type: FuncType) -> DedupFuncType {
self.inner.alloc_func_type(func_type)
}
pub(super) fn resolve_func_type<F, R>(&self, func_type: &DedupFuncType, f: F) -> R
where
F: FnOnce(&FuncType) -> R,
{
self.inner.resolve_func_type(func_type, f)
}
pub(super) fn alloc_funcs(&self, amount: usize) -> EngineFuncSpan {
self.inner.alloc_funcs(amount)
}
pub(crate) fn translate_func(
&self,
func_index: FuncIdx,
engine_func: EngineFunc,
offset: usize,
bytes: &[u8],
module: ModuleHeader,
func_to_validate: Option<FuncToValidate<ValidatorResources>>,
) -> Result<(), Error> {
self.inner.translate_func(
func_index,
engine_func,
offset,
bytes,
module,
func_to_validate,
)
}
pub(crate) fn get_translation_allocs(&self) -> FuncTranslatorAllocations {
self.inner.get_translation_allocs()
}
pub(crate) fn get_allocs(&self) -> (FuncTranslatorAllocations, FuncValidatorAllocations) {
self.inner.get_allocs()
}
pub(crate) fn recycle_translation_allocs(&self, allocs: FuncTranslatorAllocations) {
self.inner.recycle_translation_allocs(allocs)
}
pub(crate) fn recycle_allocs(
&self,
translation: FuncTranslatorAllocations,
validation: FuncValidatorAllocations,
) {
self.inner.recycle_allocs(translation, validation)
}
fn init_lazy_func(
&self,
func_idx: FuncIdx,
func: EngineFunc,
bytes: &[u8],
module: &ModuleHeader,
func_to_validate: Option<FuncToValidate<ValidatorResources>>,
) {
self.inner
.init_lazy_func(func_idx, func, bytes, module, func_to_validate)
}
#[inline]
pub(crate) fn execute_func<T, Params, Results>(
&self,
ctx: StoreContextMut<T>,
func: &Func,
params: Params,
results: Results,
) -> Result<Results::Value, Error>
where
Params: LowerToCells,
Results: LiftFromCells,
{
self.inner.execute_func(ctx, func, params, results)
}
#[inline]
pub(crate) fn execute_func_resumable<T, Params, Results>(
&self,
ctx: StoreContextMut<T>,
func: &Func,
params: Params,
results: Results,
) -> Result<ResumableCallBase<Results::Value>, Error>
where
Params: LowerToCells,
Results: LiftFromCells,
{
self.inner
.execute_func_resumable(ctx, func, params, results)
}
#[inline]
pub(crate) fn resume_func_host_trap<T, Params, Results>(
&self,
ctx: StoreContextMut<T>,
invocation: ResumableCallHostTrap,
params: Params,
results: Results,
) -> Result<ResumableCallBase<Results::Value>, Error>
where
Params: LowerToCells,
Results: LiftFromCells,
{
self.inner
.resume_func_host_trap(ctx, invocation, params, results)
}
#[inline]
pub(crate) fn resume_func_out_of_fuel<T, Results>(
&self,
ctx: StoreContextMut<T>,
invocation: ResumableCallOutOfFuel,
results: Results,
) -> Result<ResumableCallBase<Results::Value>, Error>
where
Results: LiftFromCells,
{
self.inner.resume_func_out_of_fuel(ctx, invocation, results)
}
pub(crate) fn recycle_stack(&self, stack: Stack) {
self.inner.recycle_stack(stack)
}
}
#[derive(Debug)]
pub struct EngineInner {
config: Config,
code_map: CodeMap,
func_types: RwLock<FuncTypeRegistry>,
allocs: Mutex<ReusableAllocationStack>,
stacks: Mutex<EngineStacks>,
}
pub struct ReusableAllocationStack {
max_height: usize,
translation: Vec<FuncTranslatorAllocations>,
validation: Vec<FuncValidatorAllocations>,
}
impl Default for ReusableAllocationStack {
fn default() -> Self {
Self {
max_height: 1,
translation: Vec::new(),
validation: Vec::new(),
}
}
}
impl core::fmt::Debug for ReusableAllocationStack {
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
f.debug_struct("ReusableAllocationStack")
.field("translation", &self.translation)
.field("validation", &self.validation.len())
.finish()
}
}
impl ReusableAllocationStack {
pub fn get_translation_allocs(&mut self) -> FuncTranslatorAllocations {
self.translation.pop().unwrap_or_default()
}
pub fn get_validation_allocs(&mut self) -> FuncValidatorAllocations {
self.validation.pop().unwrap_or_default()
}
pub fn recycle_translation_allocs(&mut self, recycled: FuncTranslatorAllocations) {
debug_assert!(self.translation.len() <= self.max_height);
if self.translation.len() >= self.max_height {
return;
}
self.translation.push(recycled);
}
pub fn recycle_validation_allocs(&mut self, recycled: FuncValidatorAllocations) {
debug_assert!(self.validation.len() <= self.max_height);
if self.validation.len() >= self.max_height {
return;
}
self.validation.push(recycled);
}
}
#[derive(Debug)]
pub struct EngineStacks {
stacks: Vec<Stack>,
config: StackConfig,
}
unsafe impl Send for EngineStacks {}
unsafe impl Sync for EngineStacks {}
impl EngineStacks {
pub fn new(config: &StackConfig) -> Self {
Self {
stacks: Vec::new(),
config: *config,
}
}
pub fn reuse_or_new(&mut self) -> Stack {
match self.stacks.pop() {
Some(stack) => stack,
None => Stack::new(&self.config),
}
}
pub fn recycle(&mut self, stack: Stack) {
if stack.bytes_allocated() > 0 && self.stacks.len() < self.config.max_cached_stacks() {
self.stacks.push(stack);
}
}
}
impl EngineInner {
fn new(config: &Config) -> Self {
let engine_idx = EngineId::new();
Self {
config: config.clone(),
code_map: CodeMap::new(config),
func_types: RwLock::new(FuncTypeRegistry::new(engine_idx)),
allocs: Mutex::new(ReusableAllocationStack::default()),
stacks: Mutex::new(EngineStacks::new(&config.stack)),
}
}
fn config(&self) -> &Config {
&self.config
}
fn alloc_func_type(&self, func_type: FuncType) -> DedupFuncType {
self.func_types.write().alloc_func_type(func_type)
}
fn resolve_func_type<F, R>(&self, func_type: &DedupFuncType, f: F) -> R
where
F: FnOnce(&FuncType) -> R,
{
f(self.func_types.read().resolve_func_type(func_type))
}
fn alloc_funcs(&self, amount: usize) -> EngineFuncSpan {
self.code_map.alloc_funcs(amount)
}
fn translate_func(
&self,
func_index: FuncIdx,
engine_func: EngineFunc,
offset: usize,
bytes: &[u8],
module: ModuleHeader,
func_to_validate: Option<FuncToValidate<ValidatorResources>>,
) -> Result<(), Error> {
let features = self.config().wasm_features();
match (self.config.get_compilation_mode(), func_to_validate) {
(CompilationMode::Eager, Some(func_to_validate)) => {
let (translation_allocs, validation_allocs) = self.get_allocs();
let validator = func_to_validate.into_validator(validation_allocs);
let translator = FuncTranslator::new(func_index, module, translation_allocs)?;
let translator = ValidatingFuncTranslator::new(validator, translator)?;
let allocs = FuncTranslationDriver::new(offset, bytes, translator)?
.translate(|func_entity| self.init_func(engine_func, func_entity))?;
self.recycle_allocs(allocs.translation, allocs.validation);
}
(CompilationMode::Eager, None) => {
let allocs = self.get_translation_allocs();
let translator = FuncTranslator::new(func_index, module, allocs)?;
let allocs = FuncTranslationDriver::new(offset, bytes, translator)?
.translate(|func_entity| self.init_func(engine_func, func_entity))?;
self.recycle_translation_allocs(allocs);
}
(CompilationMode::LazyTranslation, Some(func_to_validate)) => {
let allocs = self.get_validation_allocs();
let translator =
LazyFuncTranslator::new_unchecked(func_index, engine_func, module, features);
let validator = func_to_validate.into_validator(allocs);
let translator = ValidatingFuncTranslator::new(validator, translator)?;
let allocs = FuncTranslationDriver::new(offset, bytes, translator)?
.translate(|func_entity| self.init_func(engine_func, func_entity))?;
self.recycle_validation_allocs(allocs.validation);
}
(CompilationMode::Lazy | CompilationMode::LazyTranslation, func_to_validate) => {
let translator = match func_to_validate {
Some(func_to_validate) => {
LazyFuncTranslator::new(func_index, engine_func, module, func_to_validate)
}
None => {
LazyFuncTranslator::new_unchecked(func_index, engine_func, module, features)
}
};
FuncTranslationDriver::new(offset, bytes, translator)?
.translate(|func_entity| self.init_func(engine_func, func_entity))?;
}
}
Ok(())
}
fn get_translation_allocs(&self) -> FuncTranslatorAllocations {
self.allocs.lock().get_translation_allocs()
}
fn get_validation_allocs(&self) -> FuncValidatorAllocations {
self.allocs.lock().get_validation_allocs()
}
fn get_allocs(&self) -> (FuncTranslatorAllocations, FuncValidatorAllocations) {
let mut allocs = self.allocs.lock();
let translation = allocs.get_translation_allocs();
let validation = allocs.get_validation_allocs();
(translation, validation)
}
fn recycle_translation_allocs(&self, allocs: FuncTranslatorAllocations) {
self.allocs.lock().recycle_translation_allocs(allocs)
}
fn recycle_validation_allocs(&self, allocs: FuncValidatorAllocations) {
self.allocs.lock().recycle_validation_allocs(allocs)
}
fn recycle_allocs(
&self,
translation: FuncTranslatorAllocations,
validation: FuncValidatorAllocations,
) {
let mut allocs = self.allocs.lock();
allocs.recycle_translation_allocs(translation);
allocs.recycle_validation_allocs(validation);
}
fn init_func(&self, engine_func: EngineFunc, func_entity: CompiledFuncEntity) {
self.code_map
.init_func_as_compiled(engine_func, func_entity)
}
fn init_lazy_func(
&self,
func_idx: FuncIdx,
func: EngineFunc,
bytes: &[u8],
module: &ModuleHeader,
func_to_validate: Option<FuncToValidate<ValidatorResources>>,
) {
self.code_map
.init_func_as_uncompiled(func, func_idx, bytes, module, func_to_validate)
}
fn recycle_stack(&self, stack: Stack) {
self.stacks.lock().recycle(stack)
}
}