use crate::code_memory::CodeMemory;
use crate::instantiate::SetupError;
use crate::target_tunables::target_tunables;
use cranelift_codegen::ir::ExternalName;
use cranelift_codegen::ir::InstBuilder;
use cranelift_codegen::print_errors::pretty_error;
use cranelift_codegen::Context;
use cranelift_codegen::{binemit, ir};
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_wasm::ModuleTranslationState;
use std::collections::HashMap;
use std::convert::TryFrom;
use wasmtime_debug::{emit_debugsections_image, DebugInfoData};
use wasmtime_environ::entity::{EntityRef, PrimaryMap};
use wasmtime_environ::isa::{TargetFrontendConfig, TargetIsa};
use wasmtime_environ::wasm::{DefinedFuncIndex, DefinedMemoryIndex, MemoryIndex};
use wasmtime_environ::RelocationTarget;
use wasmtime_environ::{
CacheConfig, CompileError, CompiledFunction, CompiledFunctionUnwindInfo, Compiler as _C,
FunctionBodyData, Module, ModuleMemoryOffset, ModuleVmctxInfo, Relocation, Relocations, Traps,
Tunables, VMOffsets,
};
use wasmtime_runtime::{
InstantiationError, SignatureRegistry, TrapRegistration, TrapRegistry, VMFunctionBody,
VMSharedSignatureIndex, VMTrampoline,
};
#[derive(Copy, Clone, Debug)]
pub enum CompilationStrategy {
Auto,
Cranelift,
#[cfg(feature = "lightbeam")]
Lightbeam,
}
pub struct Compiler {
isa: Box<dyn TargetIsa>,
code_memory: CodeMemory,
trap_registry: TrapRegistry,
signatures: SignatureRegistry,
strategy: CompilationStrategy,
cache_config: CacheConfig,
}
impl Compiler {
pub fn new(
isa: Box<dyn TargetIsa>,
strategy: CompilationStrategy,
cache_config: CacheConfig,
) -> Self {
Self {
isa,
code_memory: CodeMemory::new(),
signatures: SignatureRegistry::new(),
strategy,
trap_registry: TrapRegistry::default(),
cache_config,
}
}
}
#[allow(missing_docs)]
pub struct Compilation {
pub finished_functions: PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
pub relocations: Relocations,
pub trampolines: HashMap<VMSharedSignatureIndex, VMTrampoline>,
pub trampoline_relocations: HashMap<VMSharedSignatureIndex, Vec<Relocation>>,
pub jt_offsets: PrimaryMap<DefinedFuncIndex, ir::JumpTableOffsets>,
pub dbg_image: Option<Vec<u8>>,
pub trap_registration: TrapRegistration,
}
impl Compiler {
pub fn frontend_config(&self) -> TargetFrontendConfig {
self.isa.frontend_config()
}
pub fn tunables(&self) -> Tunables {
target_tunables(self.isa.triple())
}
pub(crate) fn compile<'data>(
&mut self,
module: &Module,
module_translation: &ModuleTranslationState,
function_body_inputs: PrimaryMap<DefinedFuncIndex, FunctionBodyData<'data>>,
debug_data: Option<DebugInfoData>,
) -> Result<Compilation, SetupError> {
let (
compilation,
relocations,
address_transform,
value_ranges,
stack_slots,
traps,
frame_layouts,
) = match self.strategy {
CompilationStrategy::Auto | CompilationStrategy::Cranelift => {
wasmtime_environ::cranelift::Cranelift::compile_module(
module,
module_translation,
function_body_inputs,
&*self.isa,
debug_data.is_some(),
&self.cache_config,
)
}
#[cfg(feature = "lightbeam")]
CompilationStrategy::Lightbeam => {
wasmtime_environ::lightbeam::Lightbeam::compile_module(
module,
module_translation,
function_body_inputs,
&*self.isa,
debug_data.is_some(),
&self.cache_config,
)
}
}
.map_err(SetupError::Compile)?;
let finished_functions =
allocate_functions(&mut self.code_memory, &compilation).map_err(|message| {
SetupError::Instantiate(InstantiationError::Resource(format!(
"failed to allocate memory for functions: {}",
message
)))
})?;
let trap_registration = register_traps(&finished_functions, &traps, &self.trap_registry);
let mut cx = FunctionBuilderContext::new();
let mut trampolines = HashMap::new();
let mut trampoline_relocations = HashMap::new();
for sig in module.local.signatures.values() {
let index = self.signatures.register(sig);
if trampolines.contains_key(&index) {
continue;
}
let (trampoline, relocations) = make_trampoline(
&*self.isa,
&mut self.code_memory,
&mut cx,
sig,
std::mem::size_of::<u128>(),
)?;
trampolines.insert(index, trampoline);
if relocations.len() > 0 {
log::info!("relocations found in trampoline for {:?}", sig);
trampoline_relocations.insert(index, relocations);
}
}
let dbg_image = if debug_data.is_some() && !finished_functions.is_empty() {
let target_config = self.isa.frontend_config();
let ofs = VMOffsets::new(target_config.pointer_bytes(), &module.local);
let mut funcs = Vec::new();
for (i, allocated) in finished_functions.into_iter() {
let ptr = (*allocated) as *const u8;
let body_len = compilation.get(i).body.len();
funcs.push((ptr, body_len));
}
let module_vmctx_info = {
ModuleVmctxInfo {
memory_offset: if ofs.num_imported_memories > 0 {
ModuleMemoryOffset::Imported(ofs.vmctx_vmmemory_import(MemoryIndex::new(0)))
} else if ofs.num_defined_memories > 0 {
ModuleMemoryOffset::Defined(
ofs.vmctx_vmmemory_definition_base(DefinedMemoryIndex::new(0)),
)
} else {
ModuleMemoryOffset::None
},
stack_slots,
}
};
let bytes = emit_debugsections_image(
&*self.isa,
debug_data.as_ref().unwrap(),
&module_vmctx_info,
&address_transform,
&value_ranges,
&frame_layouts,
&funcs,
)
.map_err(SetupError::DebugInfo)?;
Some(bytes)
} else {
None
};
let jt_offsets = compilation.get_jt_offsets();
Ok(Compilation {
finished_functions,
relocations,
trampolines,
trampoline_relocations,
jt_offsets,
dbg_image,
trap_registration,
})
}
pub(crate) fn publish_compiled_code(&mut self) {
self.code_memory.publish();
}
pub fn signatures(&self) -> &SignatureRegistry {
&self.signatures
}
pub fn trap_registry(&self) -> &TrapRegistry {
&self.trap_registry
}
}
pub fn make_trampoline(
isa: &dyn TargetIsa,
code_memory: &mut CodeMemory,
fn_builder_ctx: &mut FunctionBuilderContext,
signature: &ir::Signature,
value_size: usize,
) -> Result<(VMTrampoline, Vec<Relocation>), SetupError> {
let pointer_type = isa.pointer_type();
let mut wrapper_sig = ir::Signature::new(isa.frontend_config().default_call_conv);
wrapper_sig.params.push(ir::AbiParam::special(
pointer_type,
ir::ArgumentPurpose::VMContext,
));
wrapper_sig.params.push(ir::AbiParam::new(pointer_type));
wrapper_sig.params.push(ir::AbiParam::new(pointer_type));
wrapper_sig.params.push(ir::AbiParam::new(pointer_type));
let mut context = Context::new();
context.func = ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wrapper_sig);
context.func.collect_frame_layout_info();
{
let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx);
let block0 = builder.create_block();
builder.append_block_params_for_function_params(block0);
builder.switch_to_block(block0);
builder.seal_block(block0);
let (vmctx_ptr_val, caller_vmctx_ptr_val, callee_value, values_vec_ptr_val) = {
let params = builder.func.dfg.block_params(block0);
(params[0], params[1], params[2], params[3])
};
let mflags = ir::MemFlags::trusted();
let callee_args = signature
.params
.iter()
.enumerate()
.map(|(i, r)| {
match i {
0 => vmctx_ptr_val,
1 => caller_vmctx_ptr_val,
_ =>
{
builder.ins().load(
r.value_type,
mflags,
values_vec_ptr_val,
((i - 2) * value_size) as i32,
)
}
}
})
.collect::<Vec<_>>();
let new_sig = builder.import_signature(signature.clone());
let call = builder
.ins()
.call_indirect(new_sig, callee_value, &callee_args);
let results = builder.func.dfg.inst_results(call).to_vec();
let mflags = ir::MemFlags::trusted();
for (i, r) in results.iter().enumerate() {
builder
.ins()
.store(mflags, *r, values_vec_ptr_val, (i * value_size) as i32);
}
builder.ins().return_(&[]);
builder.finalize()
}
let mut code_buf = Vec::new();
let mut reloc_sink = RelocSink::default();
let mut trap_sink = binemit::NullTrapSink {};
let mut stackmap_sink = binemit::NullStackmapSink {};
context
.compile_and_emit(
isa,
&mut code_buf,
&mut reloc_sink,
&mut trap_sink,
&mut stackmap_sink,
)
.map_err(|error| {
SetupError::Compile(CompileError::Codegen(pretty_error(
&context.func,
Some(isa),
error,
)))
})?;
let unwind_info = CompiledFunctionUnwindInfo::new(isa, &context);
let ptr = code_memory
.allocate_for_function(&CompiledFunction {
body: code_buf,
jt_offsets: context.func.jt_offsets,
unwind_info,
})
.map_err(|message| SetupError::Instantiate(InstantiationError::Resource(message)))?
.as_ptr();
Ok((
unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) },
reloc_sink.relocs,
))
}
fn allocate_functions(
code_memory: &mut CodeMemory,
compilation: &wasmtime_environ::Compilation,
) -> Result<PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>, String> {
let fat_ptrs = code_memory.allocate_for_compilation(compilation)?;
let mut result = PrimaryMap::with_capacity(compilation.len());
for i in 0..fat_ptrs.len() {
let fat_ptr: *mut [VMFunctionBody] = fat_ptrs[i];
result.push(fat_ptr);
}
Ok(result)
}
fn register_traps(
allocated_functions: &PrimaryMap<DefinedFuncIndex, *mut [VMFunctionBody]>,
traps: &Traps,
registry: &TrapRegistry,
) -> TrapRegistration {
let traps =
allocated_functions
.values()
.zip(traps.values())
.flat_map(|(func_addr, func_traps)| {
func_traps.iter().map(move |trap_desc| {
let func_addr = *func_addr as *const u8 as usize;
let offset = usize::try_from(trap_desc.code_offset).unwrap();
let trap_addr = func_addr + offset;
(trap_addr, trap_desc.source_loc, trap_desc.trap_code)
})
});
registry.register_traps(traps)
}
#[derive(Default)]
struct RelocSink {
relocs: Vec<Relocation>,
}
impl binemit::RelocSink for RelocSink {
fn reloc_block(
&mut self,
_offset: binemit::CodeOffset,
_reloc: binemit::Reloc,
_block_offset: binemit::CodeOffset,
) {
panic!("trampoline compilation should not produce block relocs");
}
fn reloc_external(
&mut self,
offset: binemit::CodeOffset,
reloc: binemit::Reloc,
name: &ir::ExternalName,
addend: binemit::Addend,
) {
let reloc_target = if let ExternalName::LibCall(libcall) = *name {
RelocationTarget::LibCall(libcall)
} else {
panic!("unrecognized external name")
};
self.relocs.push(Relocation {
reloc,
reloc_target,
offset,
addend,
});
}
fn reloc_constant(
&mut self,
_code_offset: binemit::CodeOffset,
_reloc: binemit::Reloc,
_constant_offset: ir::ConstantOffset,
) {
panic!("trampoline compilation should not produce constant relocs");
}
fn reloc_jt(
&mut self,
_offset: binemit::CodeOffset,
_reloc: binemit::Reloc,
_jt: ir::JumpTable,
) {
panic!("trampoline compilation should not produce jump table relocs");
}
}