1use crate::{
22 host::HostState,
23 instance_wrapper::{EntryPoint, InstanceWrapper, MemoryWrapper},
24 util::{self, replace_strategy_if_broken},
25};
26
27use parking_lot::Mutex;
28use sc_allocator::{AllocationStats, FreeingBumpHeapAllocator};
29use sc_executor_common::{
30 error::{Error, Result, WasmError},
31 runtime_blob::RuntimeBlob,
32 util::checked_range,
33 wasm_runtime::{HeapAllocStrategy, WasmInstance, WasmModule},
34};
35use sp_runtime_interface::unpack_ptr_and_len;
36use sp_wasm_interface::{HostFunctions, Pointer, WordSize};
37use std::{
38 path::{Path, PathBuf},
39 sync::{
40 atomic::{AtomicBool, Ordering},
41 Arc,
42 },
43};
44use wasmtime::{AsContext, Cache, CacheConfig, Engine, Memory};
45
46const MAX_INSTANCE_COUNT: u32 = 64;
47
48#[derive(Default)]
49pub(crate) struct StoreData {
50 pub(crate) host_state: Option<HostState>,
52 pub(crate) memory: Option<Memory>,
54 pub(crate) limits: wasmtime::StoreLimits,
56 pub(crate) memory_max_pages: Option<u32>,
58}
59
60impl StoreData {
61 pub fn host_state_mut(&mut self) -> Option<&mut HostState> {
63 self.host_state.as_mut()
64 }
65
66 pub fn memory(&self) -> Memory {
68 self.memory.expect("memory is always set; qed")
69 }
70}
71
72pub(crate) type Store = wasmtime::Store<StoreData>;
73
74enum Strategy {
75 RecreateInstance(InstanceCreator),
76}
77
78struct InstanceCreator {
79 engine: Engine,
80 instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
81 instance_counter: Arc<InstanceCounter>,
82 heap_alloc_strategy: HeapAllocStrategy,
83}
84
85impl InstanceCreator {
86 fn instantiate(&mut self) -> Result<InstanceWrapper> {
87 InstanceWrapper::new(
88 &self.engine,
89 &self.instance_pre,
90 self.instance_counter.clone(),
91 self.heap_alloc_strategy,
92 )
93 }
94}
95
96pub(crate) struct ReleaseInstanceHandle {
98 counter: Arc<InstanceCounter>,
99}
100
101impl Drop for ReleaseInstanceHandle {
102 fn drop(&mut self) {
103 {
104 let mut counter = self.counter.counter.lock();
105 *counter = counter.saturating_sub(1);
106 }
107
108 self.counter.wait_for_instance.notify_one();
109 }
110}
111
112#[derive(Default)]
119pub(crate) struct InstanceCounter {
120 counter: Mutex<u32>,
121 wait_for_instance: parking_lot::Condvar,
122}
123
124impl InstanceCounter {
125 pub fn acquire_instance(self: Arc<Self>) -> ReleaseInstanceHandle {
132 let mut counter = self.counter.lock();
133
134 while *counter >= MAX_INSTANCE_COUNT {
135 self.wait_for_instance.wait(&mut counter);
136 }
137 *counter += 1;
138
139 ReleaseInstanceHandle { counter: self.clone() }
140 }
141}
142
143pub struct WasmtimeRuntime {
146 engine: Engine,
147 instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
148 instantiation_strategy: InternalInstantiationStrategy,
149 instance_counter: Arc<InstanceCounter>,
150}
151
152impl WasmModule for WasmtimeRuntime {
153 fn new_instance(
154 &self,
155 heap_alloc_strategy: HeapAllocStrategy,
156 ) -> Result<Box<dyn WasmInstance>> {
157 let strategy = match self.instantiation_strategy {
158 InternalInstantiationStrategy::Builtin => Strategy::RecreateInstance(InstanceCreator {
159 engine: self.engine.clone(),
160 instance_pre: self.instance_pre.clone(),
161 instance_counter: self.instance_counter.clone(),
162 heap_alloc_strategy,
163 }),
164 };
165
166 Ok(Box::new(WasmtimeInstance { strategy }))
167 }
168}
169
170pub struct WasmtimeInstance {
173 strategy: Strategy,
174}
175
176impl WasmtimeInstance {
177 fn call_impl(
178 &mut self,
179 method: &str,
180 data: &[u8],
181 allocation_stats: &mut Option<AllocationStats>,
182 ) -> Result<Vec<u8>> {
183 match &mut self.strategy {
184 Strategy::RecreateInstance(ref mut instance_creator) => {
185 let mut instance_wrapper = instance_creator.instantiate()?;
186 let heap_base = instance_wrapper.extract_heap_base()?;
187 let entrypoint = instance_wrapper.resolve_entrypoint(method)?;
188 let allocator = FreeingBumpHeapAllocator::new(heap_base);
189
190 perform_call(data, &mut instance_wrapper, entrypoint, allocator, allocation_stats)
191 },
192 }
193 }
194}
195
196impl WasmInstance for WasmtimeInstance {
197 fn call_with_allocation_stats(
198 &mut self,
199 method: &str,
200 data: &[u8],
201 ) -> (Result<Vec<u8>>, Option<AllocationStats>) {
202 let mut allocation_stats = None;
203 let result = self.call_impl(method, data, &mut allocation_stats);
204 (result, allocation_stats)
205 }
206
207 fn set_heap_alloc_strategy(&mut self, heap_alloc_strategy: HeapAllocStrategy) {
208 match &mut self.strategy {
209 Strategy::RecreateInstance(ref mut creator) => {
210 creator.heap_alloc_strategy = heap_alloc_strategy;
211 },
212 }
213 }
214}
215
216fn setup_wasmtime_caching(
220 cache_path: &Path,
221 config: &mut wasmtime::Config,
222) -> std::result::Result<(), String> {
223 use std::fs;
224
225 let wasmtime_cache_root = cache_path.join("wasmtime");
226 fs::create_dir_all(&wasmtime_cache_root)
227 .map_err(|err| format!("cannot create the dirs to cache: {}", err))?;
228
229 let mut cache_config = CacheConfig::new();
230 cache_config.with_directory(cache_path);
231
232 let cache =
233 Cache::new(cache_config).map_err(|err| format!("failed to initiate Cache: {err:?}"))?;
234
235 config.cache(Some(cache));
236
237 Ok(())
238}
239
240fn common_config(semantics: &Semantics) -> std::result::Result<wasmtime::Config, WasmError> {
241 let mut config = wasmtime::Config::new();
242 config.cranelift_opt_level(wasmtime::OptLevel::SpeedAndSize);
243 config.cranelift_nan_canonicalization(semantics.canonicalize_nans);
244
245 let profiler = match std::env::var_os("WASMTIME_PROFILING_STRATEGY") {
246 Some(os_string) if os_string == "jitdump" => wasmtime::ProfilingStrategy::JitDump,
247 Some(os_string) if os_string == "perfmap" => wasmtime::ProfilingStrategy::PerfMap,
248 None => wasmtime::ProfilingStrategy::None,
249 Some(_) => {
250 static UNKNOWN_PROFILING_STRATEGY: AtomicBool = AtomicBool::new(false);
252 if !UNKNOWN_PROFILING_STRATEGY.swap(true, Ordering::Relaxed) {
254 log::warn!("WASMTIME_PROFILING_STRATEGY is set to unknown value, ignored.");
255 }
256 wasmtime::ProfilingStrategy::None
257 },
258 };
259 config.profiler(profiler);
260
261 let native_stack_max = match semantics.deterministic_stack_limit {
262 Some(DeterministicStackLimit { native_stack_max, .. }) => native_stack_max,
263
264 None => 1024 * 1024,
269 };
270
271 config.max_wasm_stack(native_stack_max as usize);
272
273 config.parallel_compilation(semantics.parallel_compilation);
274
275 config.wasm_reference_types(semantics.wasm_reference_types);
278 config.wasm_simd(semantics.wasm_simd);
279 config.wasm_relaxed_simd(semantics.wasm_simd);
280 config.wasm_bulk_memory(semantics.wasm_bulk_memory);
281 config.wasm_multi_value(semantics.wasm_multi_value);
282 config.wasm_multi_memory(false);
283 config.wasm_threads(false);
284 config.wasm_memory64(false);
285 config.wasm_tail_call(false);
286 config.wasm_extended_const(false);
287
288 let (use_pooling, use_cow) = match semantics.instantiation_strategy {
289 InstantiationStrategy::PoolingCopyOnWrite => (true, true),
290 InstantiationStrategy::Pooling => (true, false),
291 InstantiationStrategy::RecreateInstanceCopyOnWrite => (false, true),
292 InstantiationStrategy::RecreateInstance => (false, false),
293 };
294
295 const WASM_PAGE_SIZE: u64 = 65536;
296
297 config.memory_init_cow(use_cow);
298 config.memory_guaranteed_dense_image_size(u64::MAX);
302
303 if use_pooling {
304 const MAX_WASM_PAGES: u64 = 0x10000;
305
306 let mut pooling_config = wasmtime::PoolingAllocationConfig::default();
307 pooling_config
308 .max_unused_warm_slots(4)
309 .max_core_instance_size(512 * 1024)
317 .table_elements(8192)
318 .max_memory_size(MAX_WASM_PAGES as usize * WASM_PAGE_SIZE as usize)
324 .total_tables(MAX_INSTANCE_COUNT)
325 .total_memories(MAX_INSTANCE_COUNT)
326 .total_core_instances(MAX_INSTANCE_COUNT);
329
330 config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(pooling_config));
331 }
332
333 Ok(config)
334}
335
336#[derive(Clone)]
362pub struct DeterministicStackLimit {
363 pub logical_max: u32,
368 pub native_stack_max: u32,
379}
380
381#[non_exhaustive]
391#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
392pub enum InstantiationStrategy {
393 PoolingCopyOnWrite,
398
399 RecreateInstanceCopyOnWrite,
402
403 Pooling,
406
407 RecreateInstance,
409}
410
411enum InternalInstantiationStrategy {
412 Builtin,
413}
414
415#[derive(Clone)]
416pub struct Semantics {
417 pub instantiation_strategy: InstantiationStrategy,
419
420 pub deterministic_stack_limit: Option<DeterministicStackLimit>,
431
432 pub canonicalize_nans: bool,
445
446 pub parallel_compilation: bool,
448
449 pub heap_alloc_strategy: HeapAllocStrategy,
451
452 pub wasm_multi_value: bool,
454
455 pub wasm_bulk_memory: bool,
457
458 pub wasm_reference_types: bool,
460
461 pub wasm_simd: bool,
463}
464
465#[derive(Clone)]
466pub struct Config {
467 pub allow_missing_func_imports: bool,
472
473 pub cache_path: Option<PathBuf>,
475
476 pub semantics: Semantics,
478}
479
480enum CodeSupplyMode<'a> {
481 Fresh(RuntimeBlob),
483
484 Precompiled(&'a Path),
492
493 PrecompiledBytes(&'a [u8]),
498}
499
500pub fn create_runtime<H>(
506 blob: RuntimeBlob,
507 config: Config,
508) -> std::result::Result<WasmtimeRuntime, WasmError>
509where
510 H: HostFunctions,
511{
512 unsafe { do_create_runtime::<H>(CodeSupplyMode::Fresh(blob), config) }
514}
515
516pub unsafe fn create_runtime_from_artifact<H>(
533 compiled_artifact_path: &Path,
534 config: Config,
535) -> std::result::Result<WasmtimeRuntime, WasmError>
536where
537 H: HostFunctions,
538{
539 do_create_runtime::<H>(CodeSupplyMode::Precompiled(compiled_artifact_path), config)
540}
541
542pub unsafe fn create_runtime_from_artifact_bytes<H>(
558 compiled_artifact_bytes: &[u8],
559 config: Config,
560) -> std::result::Result<WasmtimeRuntime, WasmError>
561where
562 H: HostFunctions,
563{
564 do_create_runtime::<H>(CodeSupplyMode::PrecompiledBytes(compiled_artifact_bytes), config)
565}
566
567unsafe fn do_create_runtime<H>(
572 code_supply_mode: CodeSupplyMode<'_>,
573 mut config: Config,
574) -> std::result::Result<WasmtimeRuntime, WasmError>
575where
576 H: HostFunctions,
577{
578 replace_strategy_if_broken(&mut config.semantics.instantiation_strategy);
579
580 let mut wasmtime_config = common_config(&config.semantics)?;
581 if let Some(ref cache_path) = config.cache_path {
582 if let Err(reason) = setup_wasmtime_caching(cache_path, &mut wasmtime_config) {
583 log::warn!(
584 "failed to setup wasmtime cache. Performance may degrade significantly: {}.",
585 reason,
586 );
587 }
588 }
589
590 let engine = Engine::new(&wasmtime_config)
591 .map_err(|e| WasmError::Other(format!("cannot create the wasmtime engine: {:#}", e)))?;
592
593 let (module, instantiation_strategy) = match code_supply_mode {
594 CodeSupplyMode::Fresh(blob) => {
595 let blob = prepare_blob_for_compilation(blob, &config.semantics)?;
596 let serialized_blob = blob.clone().serialize();
597
598 let module = wasmtime::Module::new(&engine, &serialized_blob)
599 .map_err(|e| WasmError::Other(format!("cannot create module: {:#}", e)))?;
600
601 match config.semantics.instantiation_strategy {
602 InstantiationStrategy::Pooling |
603 InstantiationStrategy::PoolingCopyOnWrite |
604 InstantiationStrategy::RecreateInstance |
605 InstantiationStrategy::RecreateInstanceCopyOnWrite => {
606 (module, InternalInstantiationStrategy::Builtin)
607 },
608 }
609 },
610 CodeSupplyMode::Precompiled(compiled_artifact_path) => {
611 let module = wasmtime::Module::deserialize_file(&engine, compiled_artifact_path)
616 .map_err(|e| WasmError::Other(format!("cannot deserialize module: {:#}", e)))?;
617
618 (module, InternalInstantiationStrategy::Builtin)
619 },
620 CodeSupplyMode::PrecompiledBytes(compiled_artifact_bytes) => {
621 let module = wasmtime::Module::deserialize(&engine, compiled_artifact_bytes)
626 .map_err(|e| WasmError::Other(format!("cannot deserialize module: {:#}", e)))?;
627
628 (module, InternalInstantiationStrategy::Builtin)
629 },
630 };
631
632 let mut linker = wasmtime::Linker::new(&engine);
633 crate::imports::prepare_imports::<H>(&mut linker, &module, config.allow_missing_func_imports)?;
634
635 let instance_pre = linker
636 .instantiate_pre(&module)
637 .map_err(|e| WasmError::Other(format!("cannot preinstantiate module: {:#}", e)))?;
638
639 Ok(WasmtimeRuntime {
640 engine,
641 instance_pre: Arc::new(instance_pre),
642 instantiation_strategy,
643 instance_counter: Default::default(),
644 })
645}
646
647fn prepare_blob_for_compilation(
648 mut blob: RuntimeBlob,
649 semantics: &Semantics,
650) -> std::result::Result<RuntimeBlob, WasmError> {
651 if let Some(DeterministicStackLimit { logical_max, .. }) = semantics.deterministic_stack_limit {
652 blob = blob.inject_stack_depth_metering(logical_max)?;
653 }
654
655 blob.convert_memory_import_into_export()?;
660 blob.clear_memory_max_limit()?;
661
662 Ok(blob)
663}
664
665pub fn prepare_runtime_artifact(
668 blob: RuntimeBlob,
669 semantics: &Semantics,
670) -> std::result::Result<Vec<u8>, WasmError> {
671 let mut semantics = semantics.clone();
672 replace_strategy_if_broken(&mut semantics.instantiation_strategy);
673
674 let blob = prepare_blob_for_compilation(blob, &semantics)?;
675
676 let engine = Engine::new(&common_config(&semantics)?)
677 .map_err(|e| WasmError::Other(format!("cannot create the engine: {:#}", e)))?;
678
679 engine
680 .precompile_module(&blob.serialize())
681 .map_err(|e| WasmError::Other(format!("cannot precompile module: {:#}", e)))
682}
683
684fn perform_call(
685 data: &[u8],
686 instance_wrapper: &mut InstanceWrapper,
687 entrypoint: EntryPoint,
688 mut allocator: FreeingBumpHeapAllocator,
689 allocation_stats: &mut Option<AllocationStats>,
690) -> Result<Vec<u8>> {
691 let (data_ptr, data_len) = inject_input_data(instance_wrapper, &mut allocator, data)?;
692
693 let host_state = HostState::new(allocator);
694
695 instance_wrapper.store_mut().data_mut().host_state = Some(host_state);
697
698 let ret = entrypoint
699 .call(instance_wrapper.store_mut(), data_ptr, data_len)
700 .map(unpack_ptr_and_len);
701
702 let host_state = instance_wrapper.store_mut().data_mut().host_state.take().expect(
704 "the host state is always set before calling into WASM so it can't be None here; qed",
705 );
706 *allocation_stats = Some(host_state.allocation_stats());
707
708 let (output_ptr, output_len) = ret?;
709 let output = extract_output_data(instance_wrapper, output_ptr, output_len)?;
710
711 Ok(output)
712}
713
714fn inject_input_data(
715 instance: &mut InstanceWrapper,
716 allocator: &mut FreeingBumpHeapAllocator,
717 data: &[u8],
718) -> Result<(Pointer<u8>, WordSize)> {
719 let mut ctx = instance.store_mut();
720 let memory = ctx.data().memory();
721 let data_len = data.len() as WordSize;
722 let data_ptr = allocator.allocate(&mut MemoryWrapper(&memory, &mut ctx), data_len)?;
723 util::write_memory_from(instance.store_mut(), data_ptr, data)?;
724 Ok((data_ptr, data_len))
725}
726
727fn extract_output_data(
728 instance: &InstanceWrapper,
729 output_ptr: u32,
730 output_len: u32,
731) -> Result<Vec<u8>> {
732 let ctx = instance.store();
733
734 let memory_size = ctx.as_context().data().memory().data_size(ctx);
740 if checked_range(output_ptr as usize, output_len as usize, memory_size).is_none() {
741 Err(Error::OutputExceedsBounds)?
742 }
743 let mut output = vec![0; output_len as usize];
744
745 util::read_memory_into(ctx, Pointer::new(output_ptr), &mut output)?;
746 Ok(output)
747}