mod code;
pub mod func;
pub mod external_func;
pub(crate) mod integer_math;
mod float_math;
mod bool_math;
mod calling_convention;
mod unwind_info;
use std::cell::{Ref, RefCell, RefMut};
use std::collections::HashMap;
use std::marker::PhantomData;
use std::{mem, ptr, slice};
use std::any::TypeId;
use std::fmt::{Debug, Formatter};
use std::mem::MaybeUninit;
use std::ptr::NonNull;
use std::rc::Rc;
use std::sync::{Arc, Mutex, RwLock};
use edlc_core::prelude::mir_backend::{Backend, IntrinsicExecutionError, StaticData};
use edlc_core::prelude::{AmorphusData, AmorphusDataCopy, AmorphusDataMut, EdlVarId, FunctionBinding, HirPhase, MirError, MirPhase};
use edlc_core::prelude::index_map::IndexMap;
use edlc_core::prelude::mir_funcs::{MirFuncId, MirFuncRegistry};
use edlc_core::prelude::mir_type::{MirTypeId, MirTypeRegistry};
use edlc_core::prelude::mir_type::abi::AbiConfig;
use cranelift::prelude::*;
use cranelift_jit::{ArenaMemoryProvider, JITBuilder, JITModule};
use cranelift_module::{DataDescription, DataId, FuncId, Linkage, Module};
use log::debug;
use crate::codegen::{FunctionTranslator, CodeCtx};
use crate::compiler::code::{JITCode};
use crate::error::{JITError, JITErrorType};
pub use unwind_info::eh_frames;
pub use unwind_info::host_eh_frames;
pub use unwind_info::unwind_ctx;
pub(crate) use crate::compiler::unwind_info::{UnwindInfo, HostUnwindInfo};
#[derive(Default)]
struct NativeFunctionLookup {
funcs: HashMap<String, MirFuncId>,
function_bindings: IndexMap<FunctionBinding>,
}
impl NativeFunctionLookup {
fn get_symbol(&self, name: &str) -> Option<&FunctionBinding> {
self.funcs.get(name)
.and_then(|idx| self.function_bindings.get(idx.ordinal()))
}
fn for_id(&self, id: &MirFuncId) -> Option<&FunctionBinding> {
self.function_bindings.get(id.ordinal())
}
fn insert(&mut self, symbol: String, id: &MirFuncId, binding: FunctionBinding) {
assert!(self.funcs.get(&symbol).is_none(), "symbol already registered");
assert!(self.function_bindings.get(id.ordinal()).is_none(), "function id already registered");
self.function_bindings.view_mut(id.ordinal()).set(binding);
self.funcs.insert(symbol, *id);
}
fn insert_anonymous(&mut self, id: &MirFuncId, binding: FunctionBinding) {
assert!(self.function_bindings.get(id.ordinal()).is_none(), "function id already registered");
self.function_bindings.view_mut(id.ordinal()).set(binding);
}
}
#[derive(Clone, Debug)]
pub struct GlobalVar {
pub(crate) data_id: DataId,
pub ty: MirTypeId,
}
#[derive(Debug, Clone, Copy, PartialOrd, PartialEq, Ord, Eq, Hash)]
pub struct RuntimeId(u16);
impl Default for RuntimeId {
fn default() -> Self {
RuntimeId(0)
}
}
impl From<u16> for RuntimeId {
fn from(value: u16) -> Self {
RuntimeId(value)
}
}
impl RuntimeId {
pub fn oridnal(self) -> u16 {
self.0
}
}
pub struct JIT<Runtime: 'static> {
pub builder_context: FunctionBuilderContext,
pub ctx: codegen::Context,
pub data_description: DataDescription,
pub module: JITModule,
native_functions: Arc<Mutex<NativeFunctionLookup>>,
pub func_reg: Rc<RefCell<MirFuncRegistry<Self>>>,
pub(crate) global_vars: IndexMap<GlobalVar>,
pub(crate) runtime_data: IndexMap<DataId>,
static_data: Mutex<Vec<StaticData>>,
code: Rc<RefCell<JITCode>>,
_rt: PhantomData<Runtime>,
pub abi: Arc<AbiConfig>,
pub(crate) unwind_info: UnwindInfo,
}
impl<Runtime: 'static> Default for JIT<Runtime> {
fn default() -> Self {
let mut flag_builder = settings::builder();
flag_builder.set("use_colocated_libcalls", "false").unwrap();
flag_builder.set("is_pic", "false").unwrap();
flag_builder.set("enable_llvm_abi_extensions", "true").unwrap();
flag_builder.set("unwind_info", "true").unwrap();
let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {
panic!("host machine is not supported: {}", msg);
});
let isa = isa_builder
.finish(settings::Flags::new(flag_builder))
.unwrap();
let lookup = Arc::new(Mutex::new(NativeFunctionLookup::default()));
let mut builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
let lookup_clone = lookup.clone();
builder.symbol_lookup_fn(Box::new(move |name| {
debug!("Requesting external function with symbol `{name}`");
let tmp = lookup_clone.lock().unwrap();
tmp.get_symbol(name).map(|binding| unsafe { binding.as_raw_ptr() })
}));
let mem_provider = ArenaMemoryProvider::new_with_size(1 << 40)
.unwrap();
builder.memory_provider(Box::new(mem_provider));
let mut module = JITModule::new(builder);
let mut data_description = DataDescription::new();
Self {
builder_context: FunctionBuilderContext::new(),
ctx: module.make_context(),
data_description,
module,
native_functions: lookup,
func_reg: Rc::new(RefCell::new(MirFuncRegistry::default())),
global_vars: IndexMap::default(),
code: Rc::new(RefCell::new(JITCode::default())),
runtime_data: IndexMap::default(),
static_data: Mutex::new(Vec::default()),
_rt: PhantomData,
abi: Arc::new(AbiConfig::local_system_v()),
unwind_info: UnwindInfo::new(),
}
}
}
#[derive(Debug)]
pub struct TypedProgram<R, Runtime: 'static> {
pub(crate) _r: PhantomData<R>,
pub(crate) _rt: PhantomData<Runtime>,
pub(crate) func_id: FuncId,
}
impl<R, Runtime: 'static> Clone for TypedProgram<R, Runtime> {
fn clone(&self) -> Self {
TypedProgram {
_r: self._r.clone(),
_rt: self._rt.clone(),
func_id: self.func_id,
}
}
}
impl<R, Runtime: 'static> TypedProgram<R, Runtime> {
pub fn exec(&self, jit: &mut JIT<Runtime>) -> Result<R, JITError> {
let mut return_buffer = MaybeUninit::<R>::uninit();
let code = jit.module.get_finalized_function(self.func_id);
let program = unsafe { mem::transmute::<*const u8, fn(*mut R)>(code) };
program(return_buffer.as_mut_ptr());
Ok(unsafe { return_buffer.assume_init() })
}
pub fn exec_with_buffer(&self, ret_buf: &mut R, jit: &mut JIT<Runtime>) -> Result<(), JITError> {
let code = jit.module.get_finalized_function(self.func_id);
let program = unsafe { mem::transmute::<*const u8, fn(*mut R)>(code) };
program(ret_buf as *mut R);
Ok(())
}
pub fn to_unchecked(
&self,
jit: &mut JIT<Runtime>
) -> Result<UncheckedProgram<R, Runtime>, JITError> {
let code = jit.module.get_finalized_function(self.func_id);
Ok(UncheckedProgram {
_r: self._r,
_rt: self._rt,
function: code,
})
}
}
#[inline]
fn check_type<R: 'static>(lhs: TypeId) -> Result<(), JITError> {
if lhs != TypeId::of::<R>() {
Err(JITError { ty: JITErrorType::RustTypeMismatch(lhs, TypeId::of::<R>()) })
} else {
Ok(())
}
}
#[derive(Debug)]
pub struct Program<Runtime: 'static> {
_rt: PhantomData<Runtime>,
func_id: FuncId,
guard: TypeId,
}
impl<Runtime: 'static> Clone for Program<Runtime> {
fn clone(&self) -> Self {
Program {
_rt: self._rt.clone(),
func_id: self.func_id,
guard: self.guard.clone(),
}
}
}
impl<Runtime: 'static> Program<Runtime> {
pub fn to_typed<R: 'static>(&self) -> Result<TypedProgram<R, Runtime>, JITError> {
check_type::<R>(self.guard)?;
Ok(TypedProgram {
_r: PhantomData::<R>::default(),
_rt: self._rt,
func_id: self.func_id,
})
}
pub unsafe fn to_unchecked<R: 'static>(&self, jit: &mut JIT<Runtime>) -> Result<UncheckedProgram<R, Runtime>, JITError> {
check_type::<R>(self.guard)?;
let code = jit.module.get_finalized_function(self.func_id);
Ok(UncheckedProgram {
_r: PhantomData::<R>::default(),
_rt: self._rt,
function: code,
})
}
pub fn to_maybe(&self, jit: &mut JIT<Runtime>) -> Result<MaybeProgram<Runtime>, JITError> {
let code = jit.module.get_finalized_function(self.func_id);
Ok(MaybeProgram {
_rt: PhantomData,
guard: self.guard,
func: code,
})
}
}
pub struct UncheckedProgram<R, Runtime: 'static> {
_r: PhantomData<R>,
_rt: PhantomData<Runtime>,
function: *const u8,
}
impl<R, Runtime: 'static> Debug for UncheckedProgram<R, Runtime> {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "UncheckedProgram {{ {:?}, {:?}, {:?} }}", self._r, self._rt, self.function)
}
}
impl<R, Runtime: 'static> Clone for UncheckedProgram<R, Runtime> {
fn clone(&self) -> Self {
UncheckedProgram {
_r: self._r.clone(),
_rt: self._rt.clone(),
function: self.function,
}
}
}
impl<R, Runtime: 'static> UncheckedProgram<R, Runtime> {
pub unsafe fn exec(&self) -> Result<R, JITError> {
let mut return_buffer = MaybeUninit::<R>::uninit();
let program = mem::transmute::<*const u8, fn(*mut R)>(self.function);
program(return_buffer.as_mut_ptr());
Ok(return_buffer.assume_init())
}
pub unsafe fn exec_no_unwind(&self) -> Result<R, JITError> {
let mut return_buffer = MaybeUninit::<R>::uninit();
let program = mem::transmute::<*const u8, fn(*mut R)>(self.function);
program(return_buffer.as_mut_ptr());
Ok(return_buffer.assume_init())
}
}
#[derive(Debug)]
pub struct MaybeProgram<Runtime: 'static> {
_rt: PhantomData<Runtime>,
func: *const u8,
guard: TypeId,
}
impl<Runtime: 'static> Clone for MaybeProgram<Runtime> {
fn clone(&self) -> Self {
MaybeProgram {
_rt: self._rt.clone(),
func: self.func,
guard: self.guard.clone(),
}
}
}
impl<Runtime: 'static> MaybeProgram<Runtime> {
pub fn to_unchecked<R: 'static>(&self) -> Result<UncheckedProgram<R, Runtime>, JITError> {
check_type::<R>(self.guard)?;
Ok(UncheckedProgram {
_r: PhantomData::<R>::default(),
_rt: self._rt,
function: self.func,
})
}
}
impl<Runtime: 'static> Drop for JIT<Runtime> {
fn drop(&mut self) {
let ids: Vec<_> = self.runtime_data.iter()
.map(|(idx, _)| RuntimeId(idx as u16))
.collect();
for id in ids.into_iter() {
let _rt = unsafe { self.remove_runtime(id).unwrap() };
}
}
}
impl<Runtime: 'static> JIT<Runtime> {
pub fn finalize_definitions(&mut self) -> Result<(), MirError<JIT<Runtime>>> {
self.module.finalize_definitions()
.map_err(|err| MirError::BackendError(JITError {
ty: JITErrorType::ModuleErr(err),
}))?;
self.unwind_info.rebuild(&self.module)
.map_err(|err| MirError::BackendError(JITError {
ty: JITErrorType::Gimli(err),
}))?;
Ok(())
}
pub fn insert_runtime<Id: Into<RuntimeId>>(
&mut self,
id: Id,
name: &str,
rt: Runtime
) -> Result<(), MirError<JIT<Runtime>>> {
let id: RuntimeId = id.into();
let loc: Option<RwLock<Runtime>> = Some(RwLock::new(rt));
let bytes = unsafe {
slice::from_raw_parts(&loc as *const Option<RwLock<Runtime>> as *const u8, mem::size_of::<Option<RwLock<Runtime>>>())
};
self.data_description.define(bytes.to_vec().into_boxed_slice());
mem::forget(loc);
let data_id = self.module
.declare_data(name, Linkage::Export, true, false)
.map_err(|err| MirError::BackendError(JITError {
ty: JITErrorType::ModuleErr(err)
}))?;
self.module
.define_data(data_id, &self.data_description)
.map_err(|err| MirError::BackendError(JITError {
ty: JITErrorType::ModuleErr(err)
}))?;
self.module.finalize_definitions()
.map_err(|err| MirError::BackendError(JITError {
ty: JITErrorType::ModuleErr(err)
}))?;
self.data_description.clear();
self.runtime_data.view_mut(id.0 as usize).set(data_id);
Ok(())
}
pub unsafe fn remove_runtime(&mut self, id: RuntimeId) -> Result<Runtime, MirError<JIT<Runtime>>> {
let data = self.get_runtime_data(id)?;
let (ptr, _) = self.module.get_finalized_data(data);
let ptr = ptr as *mut Option<RwLock<Runtime>>;
let mut data: Option<RwLock<Runtime>> = None;
ptr::swap(&mut data, ptr); let _rt = self.runtime_data.view_mut(id.0 as usize).remove();
let data = data.ok_or(MirError::BackendError(JITError {
ty: JITErrorType::InvalidRuntimeData(id)
}))?.into_inner()
.map_err(|err| MirError::BackendError(JITError {
ty: JITErrorType::RuntimeDataState(id, err.to_string())
}))?;
Ok(data)
}
pub fn insert_function(&mut self, symbol: String, id: &MirFuncId, binding: FunctionBinding) {
let mut native_functions = self.native_functions.lock().unwrap();
native_functions.insert(symbol, id, binding);
}
pub fn insert_anonymous_function(&mut self, id: &MirFuncId, binding: FunctionBinding) {
let mut native_functions = self.native_functions.lock().unwrap();
native_functions.insert_anonymous(id, binding);
}
pub fn get_runtime_data(&self, RuntimeId(id): RuntimeId) -> Result<DataId, MirError<JIT<Runtime>>> {
self.runtime_data.get(id as usize)
.ok_or(MirError::BackendError(JITError {
ty: JITErrorType::InvalidRuntimeData(RuntimeId(id))
}))
.cloned()
}
pub fn get_runtime(&self, id: RuntimeId) -> Result<&RwLock<Runtime>, MirError<JIT<Runtime>>> {
let data_id = self.get_runtime_data(id)?;
let (ptr, _) = self.module.get_finalized_data(data_id);
let rt: &Option<RwLock<Runtime>> = unsafe { &*(ptr as *const Option<RwLock<Runtime>>) };
rt.as_ref().ok_or(MirError::BackendError(JITError {
ty: JITErrorType::InvalidRuntimeData(id)
}))
}
pub fn get_runtime_mut(&mut self, id: RuntimeId) -> Result<&mut RwLock<Runtime>, MirError<JIT<Runtime>>> {
let data_id = self.get_runtime_data(id)?;
let (ptr, _) = self.module.get_finalized_data(data_id);
let rt: &mut Option<RwLock<Runtime>> = unsafe { &mut*(ptr as *mut Option<RwLock<Runtime>>) };
rt.as_mut().ok_or(MirError::BackendError(JITError {
ty: JITErrorType::InvalidRuntimeData(id)
}))
}
pub fn get_global_var<R: 'static>(
&self,
var_id: EdlVarId,
phase: &MirPhase
) -> Result<R, MirError<JIT<Runtime>>> {
if let Some(global) = &self.global_vars.get(var_id.0) {
let (ptr, len) = self.module.get_finalized_data(global.data_id);
let slice = unsafe { slice::from_raw_parts(ptr, len) };
phase.types.check_type::<R>(global.ty)
.ok_or(MirError::UnknownType(global.ty))
.and_then(|d| if d {
Ok(())
} else {
Err(MirError::UnknownType(global.ty))
})?;
Ok(unsafe { ptr::read(slice.as_ptr() as *const _) })
} else {
panic!("failed to find global variable");
}
}
pub fn register_global_var(&mut self, id: EdlVarId, data: AmorphusDataCopy) -> Result<(), MirError<JIT<Runtime>>> {
let ty = data.mir_type();
let (align, raw) = data.deconstruct();
let data = self.module.declare_anonymous_data(false, false)
.map_err(|err| MirError::BackendError(JITError { ty: JITErrorType::ModuleErr(err) }))?;
let mut description = DataDescription::new();
description.align = Some(align as u64);
description.define(raw.into_boxed_slice());
self.module.define_data(data, &description)
.map_err(|err| MirError::BackendError(JITError { ty: JITErrorType::ModuleErr(err) }))?;
self.module.finalize_definitions()
.map_err(|err| MirError::BackendError(JITError { ty: JITErrorType::ModuleErr(err) }))?;
self.global_vars.view_mut(id.0).set(GlobalVar {
ty,
data_id: data,
});
Ok(())
}
pub fn get_global_var_data(&mut self, id: EdlVarId) -> Option<DataId> {
self.global_vars.get(id.0)
.map(|var| var.data_id.clone())
}
pub fn get_global_var_type(&mut self, id: EdlVarId) -> Option<MirTypeId> {
self.global_vars.get(id.0)
.map(|var| var.ty)
}
pub(crate) fn compile_associated_functions(
&mut self,
phase: &mut MirPhase,
hir_phase: &mut HirPhase,
) -> Result<(), MirError<Self>> {
let code = self.code.clone();
code.borrow_mut().add_from_executor(self, phase, hir_phase)?;
Ok(())
}
pub(crate) unsafe fn get_func_ptr(&self, mir_id: MirFuncId) -> Option<*const u8> {
{
let native = self.native_functions.lock().unwrap();
if let Some(func) = native.for_id(&mir_id) {
return Some(func.as_raw_ptr());
}
}
let code = self.code.borrow();
if let Some(func_id) = code.get_func_id(mir_id) {
let ptr = self.module.get_finalized_function(func_id);
Some(ptr)
} else {
None
}
}
#[allow(dead_code)]
pub(crate) fn get_func_id(&self, mir_id: MirFuncId) -> Option<FuncId> {
self.code.borrow().get_func_id(mir_id)
}
}
impl<Runtime: 'static> Backend for JIT<Runtime> {
type Error = JITError;
type FuncGen<'a> = FunctionTranslator<'a, Runtime>;
fn func_reg(
&self
) -> Ref<'_, MirFuncRegistry<Self>> {
self.func_reg.borrow()
}
fn func_reg_mut(&mut self) -> RefMut<'_, MirFuncRegistry<Self>> {
self.func_reg.borrow_mut()
}
fn intrinsic_runtime(
&self,
func: &MirFuncId,
) -> Option<u16> {
let lock = self.native_functions.lock().unwrap();
let rt = lock.for_id(func).and_then(|func| func.runtime_ordinal);
rt
}
fn call_intrinsic(
&self,
func: &MirFuncId,
params: &[AmorphusData<'_>],
ret_buffer: AmorphusDataMut<'_>,
reg: &MirTypeRegistry,
) -> Result<(), IntrinsicExecutionError> {
let lock = self.native_functions.lock().unwrap();
let func = lock
.for_id(func)
.expect("intrinsic function does not exist");
let res = func.run(params, ret_buffer, reg);
res.map_err(|err| IntrinsicExecutionError::TypeError(err))
}
fn is_call_intrinsic(&self, func: &MirFuncId) -> bool {
let lock = self.native_functions.lock().unwrap();
let out = lock
.for_id(func)
.is_some();
out
}
fn global_var_mut(&mut self, var: EdlVarId) -> Option<NonNull<()>> {
self.global_var(var)
}
fn global_var(&self, var: EdlVarId) -> Option<NonNull<()>> {
self.global_vars.get(var.0)
.and_then(|global| {
let data = global.data_id;
let (ptr, _size) = self.module.get_finalized_data(data);
NonNull::new(ptr as *mut _)
})
}
fn alloc_static(&self, data: StaticData) -> NonNull<()> {
let mut lock = self.static_data.lock().unwrap();
let idx = if let Some(idx) = lock
.iter()
.position(|d| d == &data) {
idx
} else {
let i = lock.len();
lock.push(data);
i
};
NonNull::new(lock[idx].as_ptr() as *mut ()).unwrap()
}
fn runtime(&self, ordinal: u16) -> Option<NonNull<()>> {
self.runtime_data.get(ordinal as usize)
.and_then(|rt| {
let (ptr, _size) = self.module.get_finalized_data(*rt);
NonNull::new(ptr as *mut _)
})
}
}