use std::{
collections::BTreeMap,
error::Error,
fmt,
marker::PhantomData,
ops::Deref,
ptr::NonNull,
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
};
use cranelift::codegen::ir::{
AbiParam, InstBuilder, MemFlagsData, Signature, StackSlotData, StackSlotKind, Value, types,
};
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_jit::{JITBuilder, JITModule};
use cranelift_module::{Linkage, Module};
use crate::{
runtime::codegen::{
AbiValueClass, CodegenError, classify_abi_value, clif_signature_with_execution_mode,
clif_signature_with_metadata, lower_typed_body_with_module,
},
runtime::{
ExecutionMode, NativePassMode, NativeRegisterClass, RuntimeAllocator, RuntimeErrorSink,
TypeMetadataRegistry,
},
rust_lib::{NativeEntrypointAbi, NativeFunction},
semantic::{
CanonicalSymbolPath, FunctionRegistry, GlobalFunctionId, SemanticFunctionType, TypedBody,
},
};
static NEXT_COMPILATION_IDENTITY: AtomicU64 = AtomicU64::new(1);
fn next_compilation_identity() -> u64 {
NEXT_COMPILATION_IDENTITY.fetch_add(1, Ordering::Relaxed)
}
#[derive(Debug)]
pub enum JitError {
Builder(String),
Codegen(CodegenError),
Module(String),
NullFunctionPointer,
MissingFunction(GlobalFunctionId),
NativeAggregateRequiresIndirectAbi(CanonicalSymbolPath),
}
impl fmt::Display for JitError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Builder(error) => write!(f, "cannot create JIT builder: {error}"),
Self::Codegen(error) => error.fmt(f),
Self::Module(error) => write!(f, "JIT module error: {error}"),
Self::NullFunctionPointer => f.write_str("JIT returned a null function pointer"),
Self::MissingFunction(function) => {
write!(f, "function {:?} is absent from the registry", function)
}
Self::NativeAggregateRequiresIndirectAbi(path) => write!(
f,
"native aggregate function `{}::{}` must use an explicit zero-copy indirect ABI entrypoint",
path.module, path.local
),
}
}
}
impl Error for JitError {}
impl From<CodegenError> for JitError {
fn from(value: CodegenError) -> Self {
Self::Codegen(value)
}
}
#[derive(Debug, Clone)]
pub struct LinkedFunction {
pub path: CanonicalSymbolPath,
pub symbol: String,
pub address: *const u8,
pub signature: SemanticFunctionType,
pub execution_mode: ExecutionMode,
}
pub struct JitCompilationUnit {
module: JITModule,
next_function: u64,
return_drop_adapters: BTreeMap<CanonicalSymbolPath, NonNull<u8>>,
lehit_abi_adapters: BTreeMap<GlobalFunctionId, NonNull<u8>>,
}
impl JitCompilationUnit {
pub fn new() -> Result<Self, JitError> {
Self::with_linked_functions(&BTreeMap::new(), &BTreeMap::new())
}
pub fn with_native_functions(
functions: &BTreeMap<CanonicalSymbolPath, NativeFunction>,
) -> Result<Self, JitError> {
Self::with_linked_functions(functions, &BTreeMap::new())
}
pub fn with_linked_functions(
native_functions: &BTreeMap<CanonicalSymbolPath, NativeFunction>,
linked_functions: &BTreeMap<CanonicalSymbolPath, LinkedFunction>,
) -> Result<Self, JitError> {
Self::with_linked_functions_and_allocator(
native_functions,
linked_functions,
RuntimeAllocator::default(),
)
}
pub fn with_linked_functions_and_allocator(
native_functions: &BTreeMap<CanonicalSymbolPath, NativeFunction>,
linked_functions: &BTreeMap<CanonicalSymbolPath, LinkedFunction>,
allocator: RuntimeAllocator,
) -> Result<Self, JitError> {
let mut builder = JITBuilder::new(cranelift_module::default_libcall_names())
.map_err(|error| JitError::Builder(error.to_string()))?;
builder.symbol(
crate::runtime::allocator::ALLOCATE_SYMBOL,
allocator.allocate as *const u8,
);
builder.symbol(
crate::runtime::allocator::DEALLOCATE_SYMBOL,
allocator.deallocate as *const u8,
);
builder.symbol(
crate::runtime::execution::RECORD_BOUNDS_VIOLATION_SYMBOL,
crate::runtime::execution::record_bounds_violation as *const u8,
);
for function in native_functions.values() {
builder.symbol(function.symbol(), function.address());
}
for function in linked_functions.values() {
builder.symbol(&function.symbol, function.address);
}
Ok(Self {
module: JITModule::new(builder),
next_function: 0,
return_drop_adapters: BTreeMap::new(),
lehit_abi_adapters: BTreeMap::new(),
})
}
pub fn compile<'unit>(
&'unit mut self,
name: &str,
body: &TypedBody,
signature: &SemanticFunctionType,
functions: &FunctionRegistry,
) -> Result<JitFunctionHandle<'unit>, JitError> {
let pointer =
self.compile_pointer(name, body, signature, functions, &BTreeMap::new(), None)?;
Ok(JitFunctionHandle {
pointer,
_unit: PhantomData,
})
}
pub(crate) fn compile_group(
&mut self,
bodies: &BTreeMap<GlobalFunctionId, TypedBody>,
functions: &FunctionRegistry,
native_functions: &BTreeMap<CanonicalSymbolPath, NativeFunction>,
linked_functions: &BTreeMap<CanonicalSymbolPath, LinkedFunction>,
type_metadata: Option<&TypeMetadataRegistry>,
execution_mode: ExecutionMode,
error_sink: Option<&RuntimeErrorSink>,
) -> Result<BTreeMap<GlobalFunctionId, NonNull<u8>>, JitError> {
let target_config = self.module.target_config();
let execution_error_reporter = if execution_mode == ExecutionMode::Checked {
let pointer = target_config.pointer_type();
let mut signature = Signature::new(target_config.default_call_conv);
signature.params.push(AbiParam::new(pointer));
signature.params.push(AbiParam::new(pointer));
signature.params.push(AbiParam::new(pointer));
Some(
self.module
.declare_function(
crate::runtime::execution::RECORD_BOUNDS_VIOLATION_SYMBOL,
Linkage::Import,
&signature,
)
.map_err(|error| JitError::Module(error.to_string()))?,
)
} else {
None
};
let mut callable = BTreeMap::new();
let mut checked_script_functions = std::collections::BTreeSet::new();
let mut native_adapters = BTreeMap::new();
for function in functions.iter().filter(|function| {
function.path.module == crate::core_lib::BOX_MODULE_FILE_PATH
&& function.path.local == "Box.new"
}) {
let pointer = target_config.pointer_type();
let mut signature = Signature::new(target_config.default_call_conv);
signature.params.push(AbiParam::new(pointer));
signature.params.push(AbiParam::new(pointer));
signature.returns.push(AbiParam::new(pointer));
let allocator = self
.module
.declare_function(
crate::runtime::allocator::ALLOCATE_SYMBOL,
Linkage::Import,
&signature,
)
.map_err(|error| JitError::Module(error.to_string()))?;
callable.insert(function.id, allocator);
}
for function in functions.iter().filter(|function| {
function.path.module == crate::core_lib::BOX_MODULE_FILE_PATH
&& function.path.local.ends_with(".Drop.drop")
}) {
let pointer = target_config.pointer_type();
let mut signature = Signature::new(target_config.default_call_conv);
signature.params.push(AbiParam::new(pointer));
signature.params.push(AbiParam::new(pointer));
signature.params.push(AbiParam::new(pointer));
let deallocator = self
.module
.declare_function(
crate::runtime::allocator::DEALLOCATE_SYMBOL,
Linkage::Import,
&signature,
)
.map_err(|error| JitError::Module(error.to_string()))?;
callable.insert(function.id, deallocator);
}
for (path, native) in native_functions {
let Some(global) = functions.function_by_path(path) else {
continue;
};
let has_aggregate = global.signature.arguments.iter().any(|argument| {
matches!(
classify_abi_value(
&argument.type_,
target_config.pointer_type(),
type_metadata
),
Ok(AbiValueClass::Indirect { .. })
)
}) || (!matches!(
global.signature.return_type.as_ref(),
crate::semantic::SemanticType::Void
) && matches!(
classify_abi_value(
global.signature.return_type.as_ref(),
target_config.pointer_type(),
type_metadata
),
Ok(AbiValueClass::Indirect { .. })
));
let signature =
clif_signature_with_metadata(&global.signature, target_config, type_metadata)?;
let id = if has_aggregate && native.abi() == NativeEntrypointAbi::Rust {
self.declare_register_chunk_native_adapter(
path,
native,
&global.signature,
&signature,
type_metadata,
)?
} else {
self.module
.declare_function(native.symbol(), Linkage::Import, &signature)
.map_err(|error| JitError::Module(error.to_string()))?
};
callable.insert(global.id, id);
if has_aggregate && native.abi() == NativeEntrypointAbi::Rust {
native_adapters.insert(global.id, id);
}
}
for (path, linked) in linked_functions {
let Some(global) = functions.function_by_path(path) else {
continue;
};
let signature = clif_signature_with_execution_mode(
&linked.signature,
target_config,
type_metadata,
execution_mode,
)?;
let id = self
.module
.declare_function(&linked.symbol, Linkage::Import, &signature)
.map_err(|error| JitError::Module(error.to_string()))?;
callable.insert(global.id, id);
if execution_mode == ExecutionMode::Checked {
checked_script_functions.insert(global.id);
}
}
let mut definitions = native_adapters;
for function_id in bodies.keys() {
let function = functions
.function(*function_id)
.ok_or(JitError::MissingFunction(*function_id))?;
let signature = clif_signature_with_execution_mode(
&function.signature,
target_config,
type_metadata,
execution_mode,
)?;
let name = format!(
"lehit_{}_{}",
self.next_function,
sanitize(&format!(
"{}__{}",
function.path.module, function.path.local
))
);
self.next_function += 1;
let id = self
.module
.declare_function(&name, Linkage::Local, &signature)
.map_err(|error| JitError::Module(error.to_string()))?;
callable.insert(*function_id, id);
definitions.insert(*function_id, id);
if execution_mode == ExecutionMode::Checked {
checked_script_functions.insert(*function_id);
}
}
for (function_id, body) in bodies {
let function = functions
.function(*function_id)
.ok_or(JitError::MissingFunction(*function_id))?;
let lowered = lower_typed_body_with_module(
body,
&function.signature,
functions,
&mut self.module,
&callable,
type_metadata,
execution_mode,
&checked_script_functions,
execution_error_reporter,
)?;
let id = definitions[function_id];
let mut context = self.module.make_context();
context.func = lowered;
let rendered = context.func.display().to_string();
self.module
.define_function(id, &mut context)
.map_err(|error| JitError::Module(format!("{error:?}\n{rendered}")))?;
self.module.clear_context(&mut context);
}
let mut lehit_abi_adapters = Vec::new();
for function_id in bodies.keys() {
let function = functions
.function(*function_id)
.ok_or(JitError::MissingFunction(*function_id))?;
let adapter = self.declare_lehit_abi_adapter(
&function.path,
&function.signature,
definitions[function_id],
type_metadata,
execution_mode,
error_sink,
)?;
lehit_abi_adapters.push((*function_id, adapter));
}
let mut return_drop_adapters = Vec::new();
for (function_id, body) in bodies {
if body.return_cleanup.drops.is_empty() {
continue;
}
let function = functions
.function(*function_id)
.ok_or(JitError::MissingFunction(*function_id))?;
let registry = type_metadata.ok_or_else(|| {
JitError::Module("return cleanup adapter requires type metadata".into())
})?;
let adapter = self.declare_return_cleanup_adapter(
&function.path,
body,
functions,
&callable,
registry,
execution_mode,
&checked_script_functions,
)?;
return_drop_adapters.push((function.path.clone(), adapter));
}
self.module
.finalize_definitions()
.map_err(|error| JitError::Module(error.to_string()))?;
for (path, id) in return_drop_adapters {
let pointer = NonNull::new(self.module.get_finalized_function(id) as *mut u8)
.ok_or(JitError::NullFunctionPointer)?;
self.return_drop_adapters.insert(path, pointer);
}
for (function, id) in lehit_abi_adapters {
let pointer = NonNull::new(self.module.get_finalized_function(id) as *mut u8)
.ok_or(JitError::NullFunctionPointer)?;
self.lehit_abi_adapters.insert(function, pointer);
}
definitions
.into_iter()
.map(|(global, id)| {
NonNull::new(self.module.get_finalized_function(id) as *mut u8)
.map(|pointer| (global, pointer))
.ok_or(JitError::NullFunctionPointer)
})
.collect()
}
fn declare_return_cleanup_adapter(
&mut self,
function_path: &CanonicalSymbolPath,
body: &TypedBody,
functions: &FunctionRegistry,
callable: &BTreeMap<GlobalFunctionId, cranelift_module::FuncId>,
registry: &TypeMetadataRegistry,
execution_mode: ExecutionMode,
checked_script_functions: &std::collections::BTreeSet<GlobalFunctionId>,
) -> Result<cranelift_module::FuncId, JitError> {
let target_config = self.module.target_config();
let pointer_type = target_config.pointer_type();
let mut signature = Signature::new(target_config.default_call_conv);
signature.params.push(AbiParam::new(pointer_type));
let name = format!(
"lehit_return_cleanup_{}_{}",
self.next_function,
sanitize(&format!(
"{}__{}",
function_path.module, function_path.local
))
);
self.next_function += 1;
let adapter = self
.module
.declare_function(&name, Linkage::Local, &signature)
.map_err(|error| JitError::Module(error.to_string()))?;
let mut context = self.module.make_context();
context.func.signature = signature;
let mut function_builder_context = FunctionBuilderContext::new();
{
let mut builder =
FunctionBuilder::new(&mut context.func, &mut function_builder_context);
let entry = builder.create_block();
builder.append_block_params_for_function_params(entry);
builder.switch_to_block(entry);
builder.seal_block(entry);
let root = builder.block_params(entry)[0];
let execution_context = if execution_mode == ExecutionMode::Checked {
let slot = builder.create_sized_stack_slot(StackSlotData::new(
StackSlotKind::ExplicitSlot,
std::mem::size_of::<crate::runtime::ExecutionContext>() as u32,
std::mem::align_of::<crate::runtime::ExecutionContext>().trailing_zeros() as u8,
));
let address = builder.ins().stack_addr(pointer_type, slot, 0);
let zero_i32 = builder.ins().iconst(types::I32, 0);
builder
.ins()
.store(MemFlagsData::new(), zero_i32, address, 0);
let zero = builder.ins().iconst(pointer_type, 0);
for offset in [
std::mem::offset_of!(crate::runtime::ExecutionContext, first),
std::mem::offset_of!(crate::runtime::ExecutionContext, second),
] {
builder
.ins()
.store(MemFlagsData::new(), zero, address, offset as i32);
}
Some(address)
} else {
None
};
for action in &body.return_cleanup.drops {
let mut selected = None;
for condition in &action.conditions {
let (enum_address, enum_type) = project_return_address(
&mut builder,
root,
&body.return_type,
&condition.enum_path,
registry,
pointer_type,
)?;
if enum_type != condition.enum_type {
return Err(JitError::Module(
"return enum cleanup path type mismatch".into(),
));
}
let crate::semantic::SemanticType::Named(named) = &enum_type else {
return Err(JitError::Module(
"return cleanup condition is not an enum".into(),
));
};
let metadata = registry.get_named(named).ok_or_else(|| {
JitError::Module("return enum cleanup metadata is missing".into())
})?;
let crate::runtime::TypeMetadataKind::Enum { tag_offset, .. } = &metadata.kind
else {
return Err(JitError::Module(
"return cleanup condition metadata is not enum".into(),
));
};
let tag_address = if *tag_offset == 0 {
enum_address
} else {
builder.ins().iadd_imm_u(enum_address, *tag_offset as i64)
};
let tag_type = match condition.tag_bits {
0..=8 => types::I8,
9..=16 => types::I16,
17..=32 => types::I32,
33..=64 => types::I64,
_ => {
return Err(JitError::Module(
"return cleanup tag exceeds 64 bits".into(),
));
}
};
let tag = builder
.ins()
.load(tag_type, MemFlagsData::new(), tag_address, 0);
let candidates = if condition.fallback {
&condition.explicit_tags
} else {
&condition.matching_tags
};
let mut matches_any = builder.ins().iconst(types::I8, 0);
for candidate in candidates {
let candidate = i64::try_from(*candidate).map_err(|_| {
JitError::Module("return cleanup tag exceeds i64".into())
})?;
let matches = builder.ins().icmp_imm_u(
cranelift::codegen::ir::condcodes::IntCC::Equal,
tag,
candidate,
);
matches_any = builder.ins().bor(matches_any, matches);
}
let condition_selected = if condition.fallback {
builder.ins().bnot(matches_any)
} else {
matches_any
};
selected = Some(match selected {
Some(current) => builder.ins().band(current, condition_selected),
None => condition_selected,
});
}
let continuation = if let Some(selected) = selected {
let execute = builder.create_block();
let continuation = builder.create_block();
builder
.ins()
.brif(selected, execute, &[], continuation, &[]);
builder.switch_to_block(execute);
Some((execute, continuation))
} else {
None
};
let (address, projected_type) = project_return_address(
&mut builder,
root,
&body.return_type,
&action.path,
registry,
pointer_type,
)?;
if projected_type != action.type_ {
return Err(JitError::Module(
"return cleanup action path type mismatch".into(),
));
}
let drop_data = functions
.function(action.function)
.ok_or(JitError::MissingFunction(action.function))?;
let drop_id = *callable
.get(&action.function)
.ok_or(JitError::MissingFunction(action.function))?;
let drop_ref = self.module.declare_func_in_func(drop_id, builder.func);
if drop_data.path.module == crate::core_lib::BOX_MODULE_FILE_PATH
&& drop_data.path.local.ends_with(".Drop.drop")
{
let crate::semantic::SemanticType::Named(box_type) = &action.type_ else {
return Err(JitError::Module(
"Box cleanup action has non-Box type".into(),
));
};
let [crate::semantic::SemanticGenericArgument::Type(payload)] =
box_type.generic_arguments.as_slice()
else {
return Err(JitError::Module(
"Box cleanup action has invalid arguments".into(),
));
};
let (size, align) = registry.layout_of(payload).ok_or_else(|| {
JitError::Module("Box cleanup payload layout is missing".into())
})?;
let allocation =
builder
.ins()
.load(pointer_type, MemFlagsData::new(), address, 0);
let size = builder.ins().iconst(pointer_type, size as i64);
let align = builder.ins().iconst(pointer_type, align as i64);
builder.ins().call(drop_ref, &[allocation, size, align]);
} else {
let mut arguments = vec![address];
if checked_script_functions.contains(&action.function) {
arguments.push(execution_context.ok_or_else(|| {
JitError::Module("checked Drop has no execution context".into())
})?);
}
builder.ins().call(drop_ref, &arguments);
}
if let Some((execute, continuation)) = continuation {
builder.ins().jump(continuation, &[]);
builder.switch_to_block(continuation);
builder.seal_block(execute);
builder.seal_block(continuation);
}
}
builder.ins().return_(&[]);
builder.finalize(target_config);
}
let rendered = context.func.display().to_string();
self.module
.define_function(adapter, &mut context)
.map_err(|error| JitError::Module(format!("{error:?}\n{rendered}")))?;
self.module.clear_context(&mut context);
Ok(adapter)
}
fn declare_lehit_abi_adapter(
&mut self,
function_path: &CanonicalSymbolPath,
semantic_signature: &SemanticFunctionType,
body: cranelift_module::FuncId,
type_metadata: Option<&TypeMetadataRegistry>,
execution_mode: ExecutionMode,
error_sink: Option<&RuntimeErrorSink>,
) -> Result<cranelift_module::FuncId, JitError> {
let target_config = self.module.target_config();
let pointer = target_config.pointer_type();
let pointer_size = u64::from(pointer.bytes());
let returns_void = matches!(
semantic_signature.return_type.as_ref(),
crate::semantic::SemanticType::Void
);
let return_class = (!returns_void)
.then(|| {
classify_abi_value(
semantic_signature.return_type.as_ref(),
pointer,
type_metadata,
)
})
.transpose()?;
let mut argument_layout = Vec::new();
if semantic_signature.receiver.is_some() {
argument_layout.push((pointer, pointer_size));
}
for argument in &semantic_signature.arguments {
match classify_abi_value(&argument.type_, pointer, type_metadata)? {
AbiValueClass::Direct(type_) => {
argument_layout.push((type_, u64::from(type_.bytes())));
}
AbiValueClass::Indirect { .. } => argument_layout.push((pointer, pointer_size)),
}
}
let mut signature = Signature::new(target_config.default_call_conv);
signature.params.push(AbiParam::new(pointer));
signature.params.push(AbiParam::new(pointer));
signature.returns.push(AbiParam::new(types::I32));
let name = format!(
"lehit_abi_adapter_{}_{}",
self.next_function,
sanitize(&format!(
"{}__{}",
function_path.module, function_path.local
))
);
self.next_function += 1;
let adapter = self
.module
.declare_function(&name, Linkage::Local, &signature)
.map_err(|error| JitError::Module(error.to_string()))?;
let mut context = self.module.make_context();
context.func.signature = signature;
let mut function_builder_context = FunctionBuilderContext::new();
{
let mut builder =
FunctionBuilder::new(&mut context.func, &mut function_builder_context);
let entry = builder.create_block();
let success = builder.create_block();
let failure =
(execution_mode == ExecutionMode::Checked).then(|| builder.create_block());
builder.append_block_params_for_function_params(entry);
builder.switch_to_block(entry);
builder.seal_block(entry);
let result = builder.block_params(entry)[0];
let arguments = builder.block_params(entry)[1];
let no_error = builder.ins().iconst(types::I32, 0);
let execution_context = if execution_mode == ExecutionMode::Checked {
let context_slot = builder.create_sized_stack_slot(StackSlotData::new(
StackSlotKind::ExplicitSlot,
std::mem::size_of::<crate::runtime::ExecutionContext>() as u32,
std::mem::align_of::<crate::runtime::ExecutionContext>().trailing_zeros() as u8,
));
let execution_context = builder.ins().stack_addr(pointer, context_slot, 0);
builder
.ins()
.store(MemFlagsData::new(), no_error, execution_context, 0);
let zero = builder.ins().iconst(pointer, 0);
for offset in [
std::mem::offset_of!(crate::runtime::ExecutionContext, first),
std::mem::offset_of!(crate::runtime::ExecutionContext, second),
] {
builder.ins().store(
MemFlagsData::new(),
zero,
execution_context,
offset as i32,
);
}
Some(execution_context)
} else {
None
};
let mut body_arguments = Vec::new();
let mut aggregate_temporary = None;
if let Some(AbiValueClass::Indirect { size, align }) = return_class {
let slot = builder.create_sized_stack_slot(StackSlotData::new(
StackSlotKind::ExplicitSlot,
u32::try_from(size).map_err(|_| {
JitError::Module(
"Lehit ABI aggregate return exceeds stack-slot size".into(),
)
})?,
u8::try_from(align.trailing_zeros()).map_err(|_| {
JitError::Module("Lehit ABI aggregate return alignment is invalid".into())
})?,
));
let temporary = builder.ins().stack_addr(pointer, slot, 0);
body_arguments.push(temporary);
aggregate_temporary = Some(temporary);
}
let mut offset = 0_u64;
for (type_, align) in argument_layout {
offset = offset
.checked_add(align - 1)
.map(|value| value & !(align - 1))
.ok_or_else(|| JitError::Module("Lehit ABI argument offset overflow".into()))?;
let offset_i32 = i32::try_from(offset).map_err(|_| {
JitError::Module("Lehit ABI arguments exceed i32 offset".into())
})?;
body_arguments.push(builder.ins().load(
type_,
MemFlagsData::new(),
arguments,
offset_i32,
));
offset = offset
.checked_add(u64::from(type_.bytes()))
.ok_or_else(|| JitError::Module("Lehit ABI argument offset overflow".into()))?;
}
if let Some(execution_context) = execution_context {
body_arguments.push(execution_context);
}
let body_ref = self.module.declare_func_in_func(body, builder.func);
let call = builder.ins().call(body_ref, &body_arguments);
let scalar_result = builder.inst_results(call).first().copied();
let error_code = execution_context.map(|execution_context| {
builder
.ins()
.load(types::I32, MemFlagsData::new(), execution_context, 0)
});
if let (Some(error_code), Some(failure)) = (error_code, failure) {
builder.ins().brif(error_code, failure, &[], success, &[]);
} else {
builder.ins().jump(success, &[]);
}
builder.switch_to_block(success);
match return_class {
Some(AbiValueClass::Direct(_)) => {
let value = scalar_result.ok_or_else(|| {
JitError::Module("Lehit ABI scalar body returned no value".into())
})?;
builder.ins().store(MemFlagsData::new(), value, result, 0);
}
Some(AbiValueClass::Indirect { size, align }) => {
let temporary = aggregate_temporary.ok_or_else(|| {
JitError::Module("Lehit ABI aggregate temporary is missing".into())
})?;
builder.emit_small_memory_copy(
target_config,
result,
temporary,
size as u64,
align.trailing_zeros() as u8,
align.trailing_zeros() as u8,
true,
MemFlagsData::new(),
);
}
None => {}
}
builder.ins().return_(&[no_error]);
if let (Some(failure), Some(execution_context), Some(error_code)) =
(failure, execution_context, error_code)
{
builder.switch_to_block(failure);
if let Some(error_sink) = error_sink {
let first = builder.ins().load(
pointer,
MemFlagsData::new(),
execution_context,
std::mem::offset_of!(crate::runtime::ExecutionContext, first) as i32,
);
let second = builder.ins().load(
pointer,
MemFlagsData::new(),
execution_context,
std::mem::offset_of!(crate::runtime::ExecutionContext, second) as i32,
);
let mut callback_signature = Signature::new(target_config.default_call_conv);
callback_signature.params.push(AbiParam::new(pointer));
callback_signature.params.push(AbiParam::new(types::I32));
callback_signature.params.push(AbiParam::new(pointer));
callback_signature.params.push(AbiParam::new(pointer));
let callback_signature = builder.import_signature(callback_signature);
let callback = builder
.ins()
.iconst(pointer, error_sink.callback() as usize as i64);
let callback_context = builder
.ins()
.iconst(pointer, error_sink.context() as usize as i64);
builder.ins().call_indirect(
callback_signature,
callback,
&[callback_context, error_code, first, second],
);
}
builder.ins().return_(&[error_code]);
builder.seal_block(failure);
}
builder.seal_block(success);
builder.finalize(target_config);
}
let rendered = context.func.display().to_string();
self.module
.define_function(adapter, &mut context)
.map_err(|error| JitError::Module(format!("{error:?}\n{rendered}")))?;
self.module.clear_context(&mut context);
Ok(adapter)
}
fn declare_register_chunk_native_adapter(
&mut self,
path: &CanonicalSymbolPath,
native: &NativeFunction,
semantic_signature: &SemanticFunctionType,
lehit_signature: &Signature,
type_metadata: Option<&TypeMetadataRegistry>,
) -> Result<cranelift_module::FuncId, JitError> {
if semantic_signature.receiver.is_some() || semantic_signature.arguments.is_empty() {
return Err(JitError::NativeAggregateRequiresIndirectAbi(path.clone()));
}
let registry = type_metadata
.ok_or_else(|| JitError::NativeAggregateRequiresIndirectAbi(path.clone()))?;
let target_config = self.module.target_config();
let native_signature = native.signature();
let argument_modes = semantic_signature
.arguments
.iter()
.enumerate()
.map(|(index, argument)| {
adapter_value_mode(
registry,
&argument.type_,
target_config.pointer_type(),
native_signature
.and_then(|signature| signature.argument_pass_modes().get(index)),
)
.ok_or_else(|| JitError::NativeAggregateRequiresIndirectAbi(path.clone()))
})
.collect::<Result<Vec<_>, _>>()?;
let return_mode = if matches!(
semantic_signature.return_type.as_ref(),
crate::semantic::SemanticType::Void
) {
AdapterValueMode::Void
} else {
adapter_value_mode(
registry,
semantic_signature.return_type.as_ref(),
target_config.pointer_type(),
native_signature.and_then(|signature| signature.return_pass_mode()),
)
.ok_or_else(|| JitError::NativeAggregateRequiresIndirectAbi(path.clone()))?
};
let mut native_signature = Signature::new(target_config.default_call_conv);
for mode in &argument_modes {
match mode {
AdapterValueMode::Direct(type_) => {
native_signature.params.push(AbiParam::new(*type_));
}
AdapterValueMode::Registers(chunks) => native_signature
.params
.extend(chunks.iter().map(|(_, type_)| AbiParam::new(*type_))),
AdapterValueMode::Void => unreachable!("arguments cannot have void type"),
}
}
match &return_mode {
AdapterValueMode::Direct(type_) => native_signature.returns.push(AbiParam::new(*type_)),
AdapterValueMode::Registers(chunks) => native_signature
.returns
.extend(chunks.iter().map(|(_, type_)| AbiParam::new(*type_))),
AdapterValueMode::Void => {}
}
let native_id = self
.module
.declare_function(native.symbol(), Linkage::Import, &native_signature)
.map_err(|error| JitError::Module(error.to_string()))?;
let name = format!(
"lehit_native_adapter_{}_{}",
self.next_function,
sanitize(&path.local)
);
self.next_function += 1;
let adapter_id = self
.module
.declare_function(&name, Linkage::Local, lehit_signature)
.map_err(|error| JitError::Module(error.to_string()))?;
let mut context = self.module.make_context();
context.func.signature = lehit_signature.clone();
let mut function_builder_context = FunctionBuilderContext::new();
{
let mut builder =
FunctionBuilder::new(&mut context.func, &mut function_builder_context);
let entry = builder.create_block();
builder.append_block_params_for_function_params(entry);
builder.switch_to_block(entry);
builder.seal_block(entry);
let parameters = builder.block_params(entry).to_vec();
let (output, inputs) = match &return_mode {
AdapterValueMode::Registers(_) => {
let Some((output, inputs)) = parameters.split_first() else {
return Err(JitError::NativeAggregateRequiresIndirectAbi(path.clone()));
};
(Some(*output), inputs)
}
AdapterValueMode::Direct(_) | AdapterValueMode::Void => {
(None, parameters.as_slice())
}
};
if inputs.len() != argument_modes.len() {
return Err(JitError::NativeAggregateRequiresIndirectAbi(path.clone()));
}
let mut arguments = Vec::new();
for (input, mode) in inputs.iter().zip(&argument_modes) {
match mode {
AdapterValueMode::Direct(_) => arguments.push(*input),
AdapterValueMode::Registers(chunks) => {
for (offset, type_) in chunks {
let offset = i32::try_from(*offset).map_err(|_| {
JitError::NativeAggregateRequiresIndirectAbi(path.clone())
})?;
arguments.push(builder.ins().load(
*type_,
MemFlagsData::new(),
*input,
offset,
));
}
}
AdapterValueMode::Void => unreachable!("arguments cannot have void type"),
}
}
let native_ref = self.module.declare_func_in_func(native_id, builder.func);
let call = builder.ins().call(native_ref, &arguments);
let results = builder.inst_results(call).to_vec();
match &return_mode {
AdapterValueMode::Direct(_) => {
let [result] = results.as_slice() else {
return Err(JitError::NativeAggregateRequiresIndirectAbi(path.clone()));
};
builder.ins().return_(&[*result]);
}
AdapterValueMode::Registers(chunks) => {
if results.len() != chunks.len() {
return Err(JitError::NativeAggregateRequiresIndirectAbi(path.clone()));
}
let output = output.expect("register return has an output pointer");
for ((offset, _), result) in chunks.iter().zip(results) {
builder.ins().store(
MemFlagsData::new(),
result,
output,
i32::try_from(*offset).map_err(|_| {
JitError::NativeAggregateRequiresIndirectAbi(path.clone())
})?,
);
}
builder.ins().return_(&[]);
}
AdapterValueMode::Void => {
if !results.is_empty() {
return Err(JitError::NativeAggregateRequiresIndirectAbi(path.clone()));
}
builder.ins().return_(&[]);
}
}
builder.finalize(target_config);
}
let rendered = context.func.display().to_string();
self.module
.define_function(adapter_id, &mut context)
.map_err(|error| JitError::Module(format!("{error:?}\n{rendered}")))?;
self.module.clear_context(&mut context);
Ok(adapter_id)
}
fn compile_pointer(
&mut self,
name: &str,
body: &TypedBody,
signature: &SemanticFunctionType,
functions: &FunctionRegistry,
native_functions: &BTreeMap<CanonicalSymbolPath, NativeFunction>,
type_metadata: Option<&TypeMetadataRegistry>,
) -> Result<NonNull<u8>, JitError> {
let target_config = self.module.target_config();
let mut imports = BTreeMap::new();
for (path, native) in native_functions {
let Some(global) = functions.function_by_path(path) else {
continue;
};
let signature =
clif_signature_with_metadata(&global.signature, target_config, type_metadata)?;
let module_function = self
.module
.declare_function(native.symbol(), Linkage::Import, &signature)
.map_err(|error| JitError::Module(error.to_string()))?;
imports.insert(global.id, module_function);
}
let function = lower_typed_body_with_module(
body,
signature,
functions,
&mut self.module,
&imports,
type_metadata,
ExecutionMode::Direct,
&std::collections::BTreeSet::new(),
None,
)?;
let declaration_name = format!("lehit_{}_{}", self.next_function, sanitize(name));
self.next_function += 1;
let id = self
.module
.declare_function(&declaration_name, Linkage::Local, &function.signature)
.map_err(|error| JitError::Module(error.to_string()))?;
let mut context = self.module.make_context();
context.func = function;
let rendered = context.func.display().to_string();
self.module
.define_function(id, &mut context)
.map_err(|error| JitError::Module(format!("{error:?}\n{rendered}")))?;
self.module.clear_context(&mut context);
self.module
.finalize_definitions()
.map_err(|error| JitError::Module(error.to_string()))?;
NonNull::new(self.module.get_finalized_function(id) as *mut u8)
.ok_or(JitError::NullFunctionPointer)
}
}
pub struct LeasedJitCompilationUnit {
executable: OwnedJitCompilationUnit,
dependencies: Vec<Arc<LeasedJitCompilationUnit>>,
}
impl LeasedJitCompilationUnit {
pub(crate) fn new(
executable: OwnedJitCompilationUnit,
dependencies: Vec<Arc<LeasedJitCompilationUnit>>,
) -> Self {
Self {
executable,
dependencies,
}
}
pub fn dependencies(&self) -> &[Arc<LeasedJitCompilationUnit>] {
&self.dependencies
}
}
impl Deref for LeasedJitCompilationUnit {
type Target = OwnedJitCompilationUnit;
fn deref(&self) -> &Self::Target {
&self.executable
}
}
pub struct OwnedJitCompilationUnit {
identity: u64,
execution_mode: ExecutionMode,
public_pointers: BTreeMap<CanonicalSymbolPath, NonNull<u8>>,
internal_body_pointers: BTreeMap<CanonicalSymbolPath, NonNull<u8>>,
signatures: BTreeMap<CanonicalSymbolPath, SemanticFunctionType>,
public_functions: std::collections::BTreeSet<CanonicalSymbolPath>,
return_drop_pointers: BTreeMap<CanonicalSymbolPath, NonNull<u8>>,
lehit_abi_pointers: BTreeMap<CanonicalSymbolPath, NonNull<u8>>,
type_metadata: TypeMetadataRegistry,
_error_sink: Option<RuntimeErrorSink>,
_unit: JitCompilationUnit,
}
impl OwnedJitCompilationUnit {
pub(crate) fn compile_with_allocator_and_mode(
bodies: &BTreeMap<GlobalFunctionId, TypedBody>,
functions: &FunctionRegistry,
native_functions: &BTreeMap<CanonicalSymbolPath, NativeFunction>,
type_metadata: &TypeMetadataRegistry,
allocator: RuntimeAllocator,
execution_mode: ExecutionMode,
error_sink: Option<RuntimeErrorSink>,
) -> Result<Self, JitError> {
Self::compile_with_linked_and_allocator_and_mode(
bodies,
functions,
native_functions,
&BTreeMap::new(),
type_metadata,
allocator,
execution_mode,
error_sink,
)
}
pub(crate) fn compile_with_linked_and_allocator_and_mode(
bodies: &BTreeMap<GlobalFunctionId, TypedBody>,
functions: &FunctionRegistry,
native_functions: &BTreeMap<CanonicalSymbolPath, NativeFunction>,
linked_functions: &BTreeMap<CanonicalSymbolPath, LinkedFunction>,
type_metadata: &TypeMetadataRegistry,
allocator: RuntimeAllocator,
execution_mode: ExecutionMode,
error_sink: Option<RuntimeErrorSink>,
) -> Result<Self, JitError> {
if let Some(linked) = linked_functions
.values()
.find(|linked| linked.execution_mode != execution_mode)
{
return Err(JitError::Module(format!(
"cannot link {:?} function `{}::{}` into a {:?} compilation unit",
linked.execution_mode, linked.path.module, linked.path.local, execution_mode,
)));
}
let mut unit = JitCompilationUnit::with_linked_functions_and_allocator(
native_functions,
linked_functions,
allocator,
)?;
let by_id = unit.compile_group(
bodies,
functions,
native_functions,
linked_functions,
Some(type_metadata),
execution_mode,
error_sink.as_ref(),
)?;
let mut pointers = BTreeMap::new();
let mut internal_body_pointers = BTreeMap::new();
let mut signatures = BTreeMap::new();
let mut public_functions = std::collections::BTreeSet::new();
for (id, pointer) in by_id {
let function = functions
.function(id)
.ok_or(JitError::MissingFunction(id))?;
internal_body_pointers.insert(function.path.clone(), pointer);
pointers.insert(function.path.clone(), pointer);
signatures.insert(function.path.clone(), function.signature.clone());
if function.public {
public_functions.insert(function.path.clone());
}
}
for (path, native) in native_functions {
let Some(function) = functions.function_by_path(path) else {
continue;
};
let pointer =
NonNull::new(native.address().cast_mut()).ok_or(JitError::NullFunctionPointer)?;
pointers.entry(path.clone()).or_insert(pointer);
internal_body_pointers
.entry(path.clone())
.or_insert(pointer);
signatures
.entry(path.clone())
.or_insert_with(|| function.signature.clone());
if function.public {
public_functions.insert(path.clone());
}
}
let mut lehit_abi_pointers = BTreeMap::new();
for function_id in bodies.keys() {
let function = functions
.function(*function_id)
.ok_or(JitError::MissingFunction(*function_id))?;
let pointer = unit
.lehit_abi_adapters
.get(function_id)
.copied()
.ok_or(JitError::MissingFunction(*function_id))?;
lehit_abi_pointers.insert(function.path.clone(), pointer);
}
let mut return_drop_pointers = BTreeMap::new();
for (function_id, body) in bodies {
if body.return_cleanup.drops.is_empty() {
continue;
}
let function = functions
.function(*function_id)
.ok_or(JitError::MissingFunction(*function_id))?;
let pointer = unit
.return_drop_adapters
.get(&function.path)
.copied()
.ok_or_else(|| {
JitError::Module(format!(
"return `{}::{}` has no Rust-callable cleanup adapter",
function.path.module, function.path.local
))
})?;
return_drop_pointers.insert(function.path.clone(), pointer);
}
Ok(Self {
identity: next_compilation_identity(),
execution_mode,
public_pointers: pointers,
internal_body_pointers,
signatures,
public_functions,
return_drop_pointers,
lehit_abi_pointers,
type_metadata: type_metadata.clone(),
_error_sink: error_sink,
_unit: unit,
})
}
pub(crate) fn identity(&self) -> u64 {
self.identity
}
pub fn execution_mode(&self) -> ExecutionMode {
self.execution_mode
}
pub fn function_ptr(&self, path: &CanonicalSymbolPath) -> Option<*const u8> {
if self.execution_mode != ExecutionMode::Direct {
return None;
}
self.public_pointers
.get(path)
.map(|pointer| pointer.as_ptr().cast_const())
}
pub fn lehit_abi_function_ptr(&self, path: &CanonicalSymbolPath) -> Option<*const u8> {
self.lehit_abi_pointers
.get(path)
.map(|pointer| pointer.as_ptr().cast_const())
}
pub fn signature(&self, path: &CanonicalSymbolPath) -> Option<&SemanticFunctionType> {
self.signatures.get(path)
}
pub fn type_metadata(&self) -> &TypeMetadataRegistry {
&self.type_metadata
}
pub fn return_drop_ptr(&self, function: &CanonicalSymbolPath) -> Option<*const u8> {
self.return_drop_pointers
.get(function)
.map(|pointer| pointer.as_ptr().cast_const())
}
pub fn exports(&self) -> BTreeMap<CanonicalSymbolPath, LinkedFunction> {
self.public_functions
.iter()
.filter_map(|path| {
Some((
path.clone(),
LinkedFunction {
path: path.clone(),
symbol: linked_symbol(path),
address: self.internal_body_pointers.get(path)?.as_ptr().cast_const(),
signature: self.signature(path)?.clone(),
execution_mode: self.execution_mode,
},
))
})
.collect()
}
}
pub struct OwnedJitFunction {
pointer: NonNull<u8>,
signature: SemanticFunctionType,
_unit: JitCompilationUnit,
}
impl OwnedJitFunction {
pub fn compile(
name: &str,
body: &TypedBody,
signature: &SemanticFunctionType,
functions: &FunctionRegistry,
) -> Result<Self, JitError> {
let mut unit = JitCompilationUnit::new()?;
let pointer =
unit.compile_pointer(name, body, signature, functions, &BTreeMap::new(), None)?;
Ok(Self {
pointer,
signature: signature.clone(),
_unit: unit,
})
}
pub fn compile_with_native_functions(
name: &str,
body: &TypedBody,
signature: &SemanticFunctionType,
functions: &FunctionRegistry,
native_functions: &BTreeMap<CanonicalSymbolPath, NativeFunction>,
type_metadata: &TypeMetadataRegistry,
) -> Result<Self, JitError> {
let mut unit = JitCompilationUnit::with_native_functions(native_functions)?;
let pointer = unit.compile_pointer(
name,
body,
signature,
functions,
native_functions,
Some(type_metadata),
)?;
Ok(Self {
pointer,
signature: signature.clone(),
_unit: unit,
})
}
pub fn signature(&self) -> &SemanticFunctionType {
&self.signature
}
pub fn as_ptr(&self) -> *const u8 {
self.pointer.as_ptr()
}
}
pub struct JitFunctionHandle<'unit> {
pointer: NonNull<u8>,
_unit: PhantomData<&'unit JitCompilationUnit>,
}
impl JitFunctionHandle<'_> {
pub fn as_ptr(&self) -> *const u8 {
self.pointer.as_ptr()
}
}
fn linked_symbol(path: &CanonicalSymbolPath) -> String {
format!(
"lehit_export_{}",
sanitize(&format!("{}__{}", path.module, path.local))
)
}
enum AdapterValueMode {
Direct(cranelift::codegen::ir::Type),
Registers(Vec<(usize, cranelift::codegen::ir::Type)>),
Void,
}
fn adapter_value_mode(
registry: &TypeMetadataRegistry,
type_: &crate::semantic::SemanticType,
pointer_type: cranelift::codegen::ir::Type,
native_pass_mode: Option<&NativePassMode>,
) -> Option<AdapterValueMode> {
match classify_abi_value(type_, pointer_type, Some(registry)).ok()? {
AbiValueClass::Direct(type_) => Some(AdapterValueMode::Direct(type_)),
AbiValueClass::Indirect { .. } => native_pass_mode
.and_then(pass_mode_register_chunks)
.or_else(|| register_chunks(registry, type_))
.map(AdapterValueMode::Registers),
}
}
fn pass_mode_register_chunks(
pass_mode: &NativePassMode,
) -> Option<Vec<(usize, cranelift::codegen::ir::Type)>> {
let NativePassMode::Registers(chunks) = pass_mode else {
return None;
};
chunks
.iter()
.map(|chunk| native_register_chunk_type(chunk.offset, chunk.class))
.collect()
}
fn register_chunks(
registry: &TypeMetadataRegistry,
type_: &crate::semantic::SemanticType,
) -> Option<Vec<(usize, cranelift::codegen::ir::Type)>> {
let crate::semantic::SemanticType::Named(named) = type_ else {
return None;
};
let metadata = registry.get_named(named)?;
let NativePassMode::Registers(chunks) = metadata.native_pass_mode.as_ref()? else {
return None;
};
chunks
.iter()
.map(|chunk| native_register_chunk_type(chunk.offset, chunk.class))
.collect()
}
fn native_register_chunk_type(
offset: usize,
class: NativeRegisterClass,
) -> Option<(usize, cranelift::codegen::ir::Type)> {
let type_ = match class {
NativeRegisterClass::Integer { bits: 8 } => types::I8,
NativeRegisterClass::Integer { bits: 16 } => types::I16,
NativeRegisterClass::Integer { bits: 32 } => types::I32,
NativeRegisterClass::Integer { bits: 64 } => types::I64,
NativeRegisterClass::Float { bits: 32 } => types::F32,
NativeRegisterClass::Float { bits: 64 } => types::F64,
_ => return None,
};
Some((offset, type_))
}
fn project_return_address(
builder: &mut FunctionBuilder<'_>,
mut address: Value,
root_type: &crate::semantic::SemanticType,
projections: &[crate::semantic::ReturnPlaceProjection],
registry: &TypeMetadataRegistry,
pointer_type: cranelift::codegen::ir::Type,
) -> Result<(Value, crate::semantic::SemanticType), JitError> {
let mut type_ = root_type.clone();
for projection in projections {
match projection {
crate::semantic::ReturnPlaceProjection::Dereference => {
let (crate::semantic::SemanticType::Pointer { target, .. }
| crate::semantic::SemanticType::Reference { target, .. }) = &type_
else {
return Err(JitError::Module(
"return cleanup dereferences a non-pointer".into(),
));
};
address = builder
.ins()
.load(pointer_type, MemFlagsData::new(), address, 0);
type_ = target.as_ref().clone();
}
crate::semantic::ReturnPlaceProjection::Field(name) => {
let crate::semantic::SemanticType::Named(named) = &type_ else {
return Err(JitError::Module(
"return cleanup field base is not named".into(),
));
};
let metadata = registry.get_named(named).ok_or_else(|| {
JitError::Module("return cleanup field metadata is missing".into())
})?;
let crate::runtime::TypeMetadataKind::Structure { fields } = &metadata.kind else {
return Err(JitError::Module(
"return cleanup field base is not a structure".into(),
));
};
let field = fields
.iter()
.find(|field| field.name == *name)
.ok_or_else(|| {
JitError::Module(format!("return cleanup field `{name}` is missing"))
})?;
if field.offset != 0 {
address = builder.ins().iadd_imm_u(address, field.offset as i64);
}
type_ = field.type_.clone();
}
crate::semantic::ReturnPlaceProjection::EnumPayload(index) => {
let crate::semantic::SemanticType::Named(named) = &type_ else {
return Err(JitError::Module(
"return cleanup enum base is not named".into(),
));
};
let metadata = registry.get_named(named).ok_or_else(|| {
JitError::Module("return cleanup enum metadata is missing".into())
})?;
let crate::runtime::TypeMetadataKind::Enum { variants, .. } = &metadata.kind else {
return Err(JitError::Module(
"return cleanup payload base is not enum".into(),
));
};
let variant = variants
.iter()
.find(|variant| variant.index == *index)
.ok_or_else(|| {
JitError::Module("return cleanup enum variant is missing".into())
})?;
let offset = variant.payload_offset.ok_or_else(|| {
JitError::Module("return cleanup unit variant has no payload".into())
})?;
if offset != 0 {
address = builder.ins().iadd_imm_u(address, offset as i64);
}
type_ = variant.payload_type.clone().ok_or_else(|| {
JitError::Module("return cleanup enum payload type is missing".into())
})?;
}
crate::semantic::ReturnPlaceProjection::TupleField(index) => {
let crate::semantic::SemanticType::Tuple(items) = &type_ else {
return Err(JitError::Module(
"return cleanup tuple base is not tuple".into(),
));
};
let item = items.get(*index).cloned().ok_or_else(|| {
JitError::Module("return cleanup tuple index is out of bounds".into())
})?;
let mut offset = 0usize;
for preceding in &items[..*index] {
let (size, align) = registry.layout_of(preceding).ok_or_else(|| {
JitError::Module("return cleanup tuple layout is missing".into())
})?;
offset = align_up_jit(offset, align)
.checked_add(size)
.ok_or_else(|| {
JitError::Module("return cleanup tuple layout overflow".into())
})?;
}
let (_, align) = registry.layout_of(&item).ok_or_else(|| {
JitError::Module("return cleanup tuple item layout is missing".into())
})?;
offset = align_up_jit(offset, align);
if offset != 0 {
address = builder.ins().iadd_imm_u(address, offset as i64);
}
type_ = item;
}
crate::semantic::ReturnPlaceProjection::ConstantIndex(index) => {
let crate::semantic::SemanticType::Array {
item,
size: Some(length),
} = &type_
else {
return Err(JitError::Module(
"return cleanup index base is not fixed array".into(),
));
};
if *index >= *length {
return Err(JitError::Module(
"return cleanup array index is out of bounds".into(),
));
}
let item_type = item.as_ref().clone();
let (size, align) = registry.layout_of(&item_type).ok_or_else(|| {
JitError::Module("return cleanup array item layout is missing".into())
})?;
let offset = index
.checked_mul(align_up_jit(size, align))
.ok_or_else(|| {
JitError::Module("return cleanup array layout overflow".into())
})?;
if offset != 0 {
address = builder.ins().iadd_imm_u(address, offset as i64);
}
type_ = item_type;
}
}
}
Ok((address, type_))
}
fn align_up_jit(value: usize, align: usize) -> usize {
debug_assert!(align.is_power_of_two());
(value + align - 1) & !(align - 1)
}
fn sanitize(name: &str) -> String {
name.chars()
.map(|character| {
if character.is_ascii_alphanumeric() || character == '_' {
character
} else {
'_'
}
})
.collect()
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use super::*;
use crate::{
core_lib,
parsing::{ParsedFile, SourceFile, Span, parse_file},
semantic::{
BasicBlockId, BodyAnalyzer, CanonicalSymbolPath, DeclarationIndex, IrValueId,
ModuleGraph, SemanticNamedType, SemanticType, TypedBasicBlock, TypedConstant,
TypedInstruction, TypedInstructionKind, TypedTerminator, TypedValue,
},
};
fn i32_type() -> SemanticType {
SemanticType::Named(SemanticNamedType {
path: CanonicalSymbolPath {
module: "core://primitive.lt".into(),
local: "i32".into(),
},
generic_arguments: Vec::new(),
})
}
fn insert(
graph: &mut ModuleGraph,
files: &mut BTreeMap<String, ParsedFile>,
path: &str,
source: &str,
) {
let file = parse_file(SourceFile { path, text: source }).unwrap();
graph
.insert(path, DeclarationIndex::collect(&file).unwrap())
.unwrap();
files.insert(path.to_owned(), file);
}
#[test]
fn compiles_source_through_the_full_frontend_and_executes_it() {
let mut graph = ModuleGraph::default();
let mut files = BTreeMap::new();
for (path, source) in [
("core://ops.lt", core_lib::OPS_MODULE_FILE_CONTENT),
("core://cmp.lt", core_lib::CMP_MODULE_FILE_CONTENT),
(
"core://primitive.lt",
core_lib::PRIMITIVE_MODULE_FILE_CONTENT,
),
("core://prelude.lt", core_lib::PRELUDE_MODULE_FILE_CONTENT),
("app://main.lt", "return 20 + 22"),
] {
insert(&mut graph, &mut files, path, source);
}
graph
.set_prelude(core_lib::PRELUDE_MODULE_FILE_PATH)
.unwrap();
let registry = FunctionRegistry::build(&graph, &files).unwrap();
let file = files.get("app://main.lt").unwrap();
let signature = SemanticFunctionType {
receiver: None,
arguments: Vec::new(),
return_type: Box::new(i32_type()),
};
let body = BodyAnalyzer::new(&graph, ®istry, file, "app://main.lt", &signature)
.unwrap()
.analyze(&file.ast.contract_definition_content.statements)
.unwrap();
let function_owner =
OwnedJitFunction::compile("source_answer", &body, &signature, ®istry).unwrap();
let function: unsafe extern "C" fn() -> i32 =
unsafe { std::mem::transmute(function_owner.as_ptr()) };
assert_eq!(unsafe { function() }, 42);
}
#[test]
fn compiles_script_calls_in_one_finalized_unit() {
let mut graph = ModuleGraph::default();
let mut files = BTreeMap::new();
for (path, source) in [
("core://ops.lt", core_lib::OPS_MODULE_FILE_CONTENT),
("core://cmp.lt", core_lib::CMP_MODULE_FILE_CONTENT),
("core://clone.lt", core_lib::CLONE_MODULE_FILE_CONTENT),
("core://drop.lt", core_lib::DROP_MODULE_FILE_CONTENT),
(
"core://primitive.lt",
core_lib::PRIMITIVE_MODULE_FILE_CONTENT,
),
("core://prelude.lt", core_lib::PRELUDE_MODULE_FILE_CONTENT),
(
"app://unit.lt",
"acs callee: fn() -> i32 { return 42 } acs caller: fn() -> i32 { return callee() }",
),
] {
insert(&mut graph, &mut files, path, source);
}
graph
.set_prelude(core_lib::PRELUDE_MODULE_FILE_PATH)
.unwrap();
let registry = FunctionRegistry::build(&graph, &files).unwrap();
let file = files.get("app://unit.lt").unwrap();
let mut bodies = BTreeMap::new();
for name in ["callee", "caller"] {
let path = CanonicalSymbolPath {
module: "app://unit.lt".into(),
local: name.into(),
};
let function = registry.function_by_path(&path).unwrap();
let access = file
.ast
.contract_definition_content
.contract_accesses
.iter()
.find(|access| file.data.strings[access.ident.string.0 as usize].as_str() == name)
.unwrap();
let crate::parsing::ast::ContractSource::Declaration(declaration) = &access.source
else {
panic!("function must be a declaration")
};
let definition = declaration.definition.as_ref().unwrap();
let body = BodyAnalyzer::new(
&graph,
®istry,
file,
"app://unit.lt",
&function.signature,
)
.unwrap()
.analyze(&definition.content.statements)
.unwrap();
bodies.insert(function.id, body);
}
let caller = registry
.function_id(&CanonicalSymbolPath {
module: "app://unit.lt".into(),
local: "caller".into(),
})
.unwrap();
let mut unit = JitCompilationUnit::new().unwrap();
let pointers = unit
.compile_group(
&bodies,
®istry,
&BTreeMap::new(),
&BTreeMap::new(),
None,
ExecutionMode::Direct,
None,
)
.unwrap();
let function: unsafe extern "C" fn() -> i32 =
unsafe { std::mem::transmute(pointers[&caller].as_ptr()) };
assert_eq!(unsafe { function() }, 42);
let mut checked_unit = JitCompilationUnit::new().unwrap();
let checked_pointers = checked_unit
.compile_group(
&bodies,
®istry,
&BTreeMap::new(),
&BTreeMap::new(),
None,
ExecutionMode::Checked,
None,
)
.unwrap();
let checked_function: unsafe extern "C" fn(*mut crate::runtime::ExecutionContext) -> i32 =
unsafe { std::mem::transmute(checked_pointers[&caller].as_ptr()) };
let mut execution_context = crate::runtime::ExecutionContext::new();
assert_eq!(unsafe { checked_function(&mut execution_context) }, 42);
assert_eq!(execution_context.error(), None);
}
#[test]
fn compiles_and_executes_a_constant_function() {
let type_ = SemanticType::Named(SemanticNamedType {
path: CanonicalSymbolPath {
module: "core://primitive.lt".into(),
local: "i32".into(),
},
generic_arguments: Vec::new(),
});
let value = TypedValue {
id: IrValueId(0),
type_: type_.clone(),
storage_place: None,
borrow_origins: Vec::new(),
};
let body = TypedBody {
locals: Vec::new(),
places: Vec::new(),
entry: BasicBlockId(0),
blocks: vec![TypedBasicBlock {
id: BasicBlockId(0),
parameters: Vec::new(),
instructions: vec![TypedInstruction {
span: Span::new(0, 2),
result: Some(value.clone()),
kind: TypedInstructionKind::Constant(TypedConstant::Signed(42)),
}],
terminator: TypedTerminator::Return(Some(value.id)),
}],
return_type: type_.clone(),
return_cleanup: crate::semantic::ReturnCleanupPlan::default(),
};
let signature = SemanticFunctionType {
receiver: None,
arguments: Vec::new(),
return_type: Box::new(type_),
};
let mut unit = JitCompilationUnit::new().unwrap();
let handle = unit
.compile("answer", &body, &signature, &FunctionRegistry::default())
.unwrap();
let function: unsafe extern "C" fn() -> i32 =
unsafe { std::mem::transmute(handle.as_ptr()) };
assert_eq!(unsafe { function() }, 42);
}
}