use crate::{
host_state::HostState,
instance_wrapper::{EntryPoint, InstanceWrapper},
memory_wrapper::MemoryWrapper,
store_data::StoreData,
util as memory_util,
util::replace_strategy_if_broken,
};
use parking_lot::Mutex;
use sc_executor_common::{
error::{Result, WasmError},
runtime_blob::RuntimeBlob,
wasm_runtime::{HeapAllocStrategy, WasmInstance, WasmModule},
};
use sp_allocator::{AllocationStats, FreeingBumpHeapAllocator};
use sp_runtime_interface::unpack_ptr_and_len;
use sp_wasm_interface::{HostFunctions, Pointer, Value, WordSize};
use std::{
path::{Path, PathBuf},
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
};
use wasmtime::{AsContext, Cache, Engine};
const MAX_INSTANCE_COUNT: u32 = 64;
pub(crate) type Store = wasmtime::Store<StoreData>;
enum Strategy {
RecreateInstance(InstanceCreator),
}
struct InstanceCreator {
engine: Engine,
instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
instance_counter: Arc<InstanceCounter>,
}
impl InstanceCreator {
fn instantiate(&mut self) -> Result<InstanceWrapper> {
InstanceWrapper::new(
&self.engine,
&self.instance_pre,
self.instance_counter.clone(),
)
}
}
pub(crate) struct ReleaseInstanceHandle {
counter: Arc<InstanceCounter>,
}
impl Drop for ReleaseInstanceHandle {
fn drop(&mut self) {
{
let mut counter = self.counter.counter.lock();
*counter = counter.saturating_sub(1);
}
self.counter.wait_for_instance.notify_one();
}
}
#[derive(Default)]
pub(crate) struct InstanceCounter {
counter: Mutex<u32>,
wait_for_instance: parking_lot::Condvar,
}
impl InstanceCounter {
pub fn acquire_instance(self: Arc<Self>) -> ReleaseInstanceHandle {
let mut counter = self.counter.lock();
while *counter >= MAX_INSTANCE_COUNT {
self.wait_for_instance.wait(&mut counter);
}
*counter += 1;
ReleaseInstanceHandle {
counter: self.clone(),
}
}
}
pub struct WasmtimeRuntime {
engine: Engine,
instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
instantiation_strategy: InternalInstantiationStrategy,
instance_counter: Arc<InstanceCounter>,
}
impl WasmModule for WasmtimeRuntime {
fn new_instance(&self) -> Result<Box<dyn WasmInstance>> {
let strategy = match self.instantiation_strategy {
InternalInstantiationStrategy::Builtin => Strategy::RecreateInstance(InstanceCreator {
engine: self.engine.clone(),
instance_pre: self.instance_pre.clone(),
instance_counter: self.instance_counter.clone(),
}),
};
Ok(Box::new(WasmtimeInstance { strategy }))
}
}
pub struct WasmtimeInstance {
strategy: Strategy,
}
impl WasmtimeInstance {
fn call_impl(
&mut self,
method: &str,
data: &[u8],
allocation_stats: &mut Option<AllocationStats>,
) -> Result<Vec<u8>> {
match &mut self.strategy {
Strategy::RecreateInstance(instance_creator) => {
let mut instance_wrapper = instance_creator.instantiate()?;
let heap_base = instance_wrapper.extract_heap_base()?;
let entrypoint = instance_wrapper.resolve_entrypoint(method)?;
let allocator = FreeingBumpHeapAllocator::new(heap_base);
perform_call(
data,
&mut instance_wrapper,
entrypoint,
allocator,
allocation_stats,
)
}
}
}
}
impl WasmInstance for WasmtimeInstance {
fn call_with_allocation_stats(
&mut self,
method: &str,
data: &[u8],
) -> (Result<Vec<u8>>, Option<AllocationStats>) {
let mut allocation_stats = None;
let result = self.call_impl(method, data, &mut allocation_stats);
(result, allocation_stats)
}
fn get_global_const(&mut self, name: &str) -> Result<Option<Value>> {
match &mut self.strategy {
Strategy::RecreateInstance(instance_creator) => {
instance_creator.instantiate()?.get_global_val(name)
}
}
}
}
fn setup_wasmtime_caching(
cache_path: &Path,
config: &mut wasmtime::Config,
) -> std::result::Result<(), String> {
use std::fs;
let wasmtime_cache_root = cache_path.join("wasmtime");
fs::create_dir_all(&wasmtime_cache_root)
.map_err(|err| format!("cannot create the dirs to cache: {}", err))?;
let wasmtime_cache_root = wasmtime_cache_root
.canonicalize()
.map_err(|err| format!("failed to canonicalize the path: {}", err))?;
let cache_config_path = wasmtime_cache_root.join("cache-config.toml");
let config_content = format!(
"\
[cache]
directory = {cache_dir}
",
cache_dir = toml_basic_string(&wasmtime_cache_root.to_string_lossy())
);
fs::write(&cache_config_path, config_content)
.map_err(|err| format!("cannot write the cache config: {}", err))?;
let cache = Cache::from_file(Some(&cache_config_path))
.map_err(|err| format!("failed to parse the config: {:#}", err))?;
config.cache(Some(cache));
Ok(())
}
fn toml_basic_string(value: &str) -> String {
use std::fmt::Write as _;
let mut escaped = String::with_capacity(value.len() + 2);
escaped.push('"');
for ch in value.chars() {
match ch {
'\\' => escaped.push_str("\\\\"),
'"' => escaped.push_str("\\\""),
'\n' => escaped.push_str("\\n"),
'\r' => escaped.push_str("\\r"),
'\t' => escaped.push_str("\\t"),
'\u{08}' => escaped.push_str("\\b"),
'\u{0c}' => escaped.push_str("\\f"),
ch if ch.is_control() => {
write!(escaped, "\\u{:04X}", ch as u32).expect("writing to a string cannot fail")
}
ch => escaped.push(ch),
}
}
escaped.push('"');
escaped
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn toml_basic_string_escapes_cache_paths() {
assert_eq!(
toml_basic_string(r#"C:\Users\gear"cache"\wasmtime"#),
r#""C:\\Users\\gear\"cache\"\\wasmtime""#
);
}
}
fn common_config(semantics: &Semantics) -> std::result::Result<wasmtime::Config, WasmError> {
let mut config = wasmtime::Config::new();
config.macos_use_mach_ports(false);
config.cranelift_opt_level(wasmtime::OptLevel::SpeedAndSize);
config.cranelift_nan_canonicalization(semantics.canonicalize_nans);
let profiler = match std::env::var_os("WASMTIME_PROFILING_STRATEGY") {
Some(os_string) if os_string == "jitdump" => wasmtime::ProfilingStrategy::JitDump,
None => wasmtime::ProfilingStrategy::None,
Some(_) => {
static UNKNOWN_PROFILING_STRATEGY: AtomicBool = AtomicBool::new(false);
if !UNKNOWN_PROFILING_STRATEGY.swap(true, Ordering::Relaxed) {
log::warn!("WASMTIME_PROFILING_STRATEGY is set to unknown value, ignored.");
}
wasmtime::ProfilingStrategy::None
}
};
config.profiler(profiler);
let native_stack_max = match semantics.deterministic_stack_limit {
Some(DeterministicStackLimit {
native_stack_max, ..
}) => native_stack_max,
None => 1024 * 1024,
};
config.max_wasm_stack(native_stack_max as usize);
config.parallel_compilation(semantics.parallel_compilation);
config.wasm_simd(semantics.wasm_simd);
config.wasm_relaxed_simd(false);
config.wasm_bulk_memory(semantics.wasm_bulk_memory);
config.wasm_multi_value(semantics.wasm_multi_value);
config.wasm_multi_memory(false);
config.wasm_memory64(false);
let (use_pooling, use_cow) = match semantics.instantiation_strategy {
InstantiationStrategy::PoolingCopyOnWrite => (true, true),
InstantiationStrategy::Pooling => (true, false),
InstantiationStrategy::RecreateInstanceCopyOnWrite => (false, true),
InstantiationStrategy::RecreateInstance => (false, false),
};
const WASM_PAGE_SIZE: u64 = 65536;
config.memory_init_cow(use_cow);
config.memory_guaranteed_dense_image_size(match semantics.heap_alloc_strategy {
HeapAllocStrategy::Dynamic { maximum_pages } => maximum_pages
.map(|p| p as u64 * WASM_PAGE_SIZE)
.unwrap_or(u64::MAX),
HeapAllocStrategy::Static { .. } => u64::MAX,
});
if use_pooling {
const MAX_WASM_PAGES: u64 = 0x10000;
let memory_pages = match semantics.heap_alloc_strategy {
HeapAllocStrategy::Dynamic { maximum_pages } => {
maximum_pages.map(|p| p as u64).unwrap_or(MAX_WASM_PAGES)
}
HeapAllocStrategy::Static { .. } => MAX_WASM_PAGES,
};
let mut pooling_config = wasmtime::PoolingAllocationConfig::default();
pooling_config
.max_unused_warm_slots(4)
.max_core_instance_size(128 * 1024)
.table_elements(8192)
.max_memory_size((memory_pages * WASM_PAGE_SIZE) as usize)
.max_tables_per_module(1)
.max_memories_per_module(1)
.total_core_instances(MAX_INSTANCE_COUNT);
config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(
pooling_config,
));
}
Ok(config)
}
#[derive(Clone)]
pub struct DeterministicStackLimit {
pub logical_max: u32,
pub native_stack_max: u32,
}
#[non_exhaustive]
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
pub enum InstantiationStrategy {
PoolingCopyOnWrite,
RecreateInstanceCopyOnWrite,
Pooling,
RecreateInstance,
}
enum InternalInstantiationStrategy {
Builtin,
}
#[derive(Clone)]
pub struct Semantics {
pub instantiation_strategy: InstantiationStrategy,
pub deterministic_stack_limit: Option<DeterministicStackLimit>,
pub canonicalize_nans: bool,
pub parallel_compilation: bool,
pub heap_alloc_strategy: HeapAllocStrategy,
pub wasm_multi_value: bool,
pub wasm_bulk_memory: bool,
pub wasm_reference_types: bool,
pub wasm_simd: bool,
}
#[derive(Clone)]
pub struct Config {
pub allow_missing_func_imports: bool,
pub cache_path: Option<PathBuf>,
pub semantics: Semantics,
}
enum CodeSupplyMode<'a> {
Fresh(RuntimeBlob),
Precompiled(&'a Path),
PrecompiledBytes(&'a [u8]),
}
pub fn create_runtime<H>(
blob: RuntimeBlob,
config: Config,
) -> std::result::Result<WasmtimeRuntime, WasmError>
where
H: HostFunctions,
{
unsafe { do_create_runtime::<H>(CodeSupplyMode::Fresh(blob), config) }
}
pub unsafe fn create_runtime_from_artifact<H>(
compiled_artifact_path: &Path,
config: Config,
) -> std::result::Result<WasmtimeRuntime, WasmError>
where
H: HostFunctions,
{
unsafe { do_create_runtime::<H>(CodeSupplyMode::Precompiled(compiled_artifact_path), config) }
}
pub unsafe fn create_runtime_from_artifact_bytes<H>(
compiled_artifact_bytes: &[u8],
config: Config,
) -> std::result::Result<WasmtimeRuntime, WasmError>
where
H: HostFunctions,
{
unsafe {
do_create_runtime::<H>(
CodeSupplyMode::PrecompiledBytes(compiled_artifact_bytes),
config,
)
}
}
unsafe fn do_create_runtime<H>(
code_supply_mode: CodeSupplyMode<'_>,
mut config: Config,
) -> std::result::Result<WasmtimeRuntime, WasmError>
where
H: HostFunctions,
{
replace_strategy_if_broken(&mut config.semantics.instantiation_strategy);
let mut wasmtime_config = common_config(&config.semantics)?;
if let Some(ref cache_path) = config.cache_path
&& let Err(reason) = setup_wasmtime_caching(cache_path, &mut wasmtime_config)
{
log::warn!(
"failed to setup wasmtime cache. Performance may degrade significantly: {}.",
reason,
);
}
let engine = Engine::new(&wasmtime_config)
.map_err(|e| WasmError::Other(format!("cannot create the wasmtime engine: {:#}", e)))?;
let (module, instantiation_strategy) = match code_supply_mode {
CodeSupplyMode::Fresh(blob) => {
let blob = prepare_blob_for_compilation(blob, &config.semantics)?;
let serialized_blob = blob.clone().serialize();
let module = wasmtime::Module::new(&engine, &serialized_blob)
.map_err(|e| WasmError::Other(format!("cannot create module: {:#}", e)))?;
match config.semantics.instantiation_strategy {
InstantiationStrategy::Pooling
| InstantiationStrategy::PoolingCopyOnWrite
| InstantiationStrategy::RecreateInstance
| InstantiationStrategy::RecreateInstanceCopyOnWrite => {
(module, InternalInstantiationStrategy::Builtin)
}
}
}
CodeSupplyMode::Precompiled(compiled_artifact_path) => {
let module =
unsafe { wasmtime::Module::deserialize_file(&engine, compiled_artifact_path) }
.map_err(|e| WasmError::Other(format!("cannot deserialize module: {:#}", e)))?;
(module, InternalInstantiationStrategy::Builtin)
}
CodeSupplyMode::PrecompiledBytes(compiled_artifact_bytes) => {
let module = unsafe { wasmtime::Module::deserialize(&engine, compiled_artifact_bytes) }
.map_err(|e| WasmError::Other(format!("cannot deserialize module: {:#}", e)))?;
(module, InternalInstantiationStrategy::Builtin)
}
};
let mut linker = wasmtime::Linker::new(&engine);
crate::imports::prepare_imports::<H>(&mut linker, &module, config.allow_missing_func_imports)?;
let instance_pre = linker
.instantiate_pre(&module)
.map_err(|e| WasmError::Other(format!("cannot preinstantiate module: {:#}", e)))?;
Ok(WasmtimeRuntime {
engine,
instance_pre: Arc::new(instance_pre),
instantiation_strategy,
instance_counter: Default::default(),
})
}
fn prepare_blob_for_compilation(
mut blob: RuntimeBlob,
semantics: &Semantics,
) -> std::result::Result<RuntimeBlob, WasmError> {
if let Some(DeterministicStackLimit { logical_max, .. }) = semantics.deterministic_stack_limit {
blob = blob.inject_stack_depth_metering(logical_max)?;
}
blob.convert_memory_import_into_export()?;
blob.setup_memory_according_to_heap_alloc_strategy(semantics.heap_alloc_strategy)?;
Ok(blob)
}
pub fn prepare_runtime_artifact(
blob: RuntimeBlob,
semantics: &Semantics,
) -> std::result::Result<Vec<u8>, WasmError> {
let mut semantics = semantics.clone();
replace_strategy_if_broken(&mut semantics.instantiation_strategy);
let blob = prepare_blob_for_compilation(blob, &semantics)?;
let engine = Engine::new(&common_config(&semantics)?)
.map_err(|e| WasmError::Other(format!("cannot create the engine: {:#}", e)))?;
engine
.precompile_module(&blob.serialize())
.map_err(|e| WasmError::Other(format!("cannot precompile module: {:#}", e)))
}
fn perform_call(
data: &[u8],
instance_wrapper: &mut InstanceWrapper,
entrypoint: EntryPoint,
mut allocator: FreeingBumpHeapAllocator,
allocation_stats: &mut Option<AllocationStats>,
) -> Result<Vec<u8>> {
let (data_ptr, data_len) = inject_input_data(instance_wrapper, &mut allocator, data)?;
let host_state = HostState::new(allocator);
instance_wrapper.store_mut().data_mut().host_state = Some(host_state);
let ret = entrypoint
.call(instance_wrapper.store_mut(), data_ptr, data_len)
.map(unpack_ptr_and_len);
let host_state = instance_wrapper
.store_mut()
.data_mut()
.host_state
.take()
.expect(
"the host state is always set before calling into WASM so it can't be None here; qed",
);
*allocation_stats = Some(host_state.allocation_stats());
let (output_ptr, output_len) = ret?;
let output = extract_output_data(instance_wrapper, output_ptr, output_len)?;
Ok(output)
}
fn inject_input_data(
instance: &mut InstanceWrapper,
allocator: &mut FreeingBumpHeapAllocator,
data: &[u8],
) -> Result<(Pointer<u8>, WordSize)> {
let mut ctx = instance.store_mut();
let memory = ctx.data().memory();
let data_len = data.len() as WordSize;
let data_ptr = allocator.allocate(&mut MemoryWrapper::from((&memory, &mut ctx)), data_len)?;
memory_util::write_memory_from(instance.store_mut(), data_ptr, data)?;
Ok((data_ptr, data_len))
}
fn extract_output_data(
instance: &InstanceWrapper,
output_ptr: u32,
output_len: u32,
) -> Result<Vec<u8>> {
let ctx = instance.store();
let memory_size = ctx.as_context().data().memory().data_size(ctx);
if memory_util::checked_range(output_ptr as usize, output_len as usize, memory_size).is_none() {
Err(WasmError::Other(
"output exceeds bounds of wasm memory".into(),
))?
}
let mut output = vec![0; output_len as usize];
memory_util::read_memory_into(ctx, Pointer::new(output_ptr), &mut output)?;
Ok(output)
}