use std::{
collections::{BTreeMap, BTreeSet},
error::Error,
fmt,
};
use cranelift::{
codegen::ir::{
AbiParam, Block, BlockArg, FuncRef, Function, InstBuilder, MemFlagsData, Signature,
StackSlotData, StackSlotKind, Type, UserFuncName, Value, types,
},
prelude::{FunctionBuilder, FunctionBuilderContext},
};
use cranelift_module::{FuncId, Module, ModuleError};
use crate::{
runtime::{ExecutionMode, TypeMetadataKind, TypeMetadataRegistry},
semantic::{
BasicBlockId, CheckedCleanupPlan, FunctionRegistry, GlobalFunctionId, IrValueId, LocalId,
PlaceBase, PlaceId, PlaceProjection, SemanticFunctionType, SemanticType, TypedBody,
TypedConstant, TypedInstructionKind, TypedTerminator,
},
};
#[derive(Debug)]
pub enum CodegenError {
UnsupportedType(SemanticType),
MissingValue(IrValueId),
MissingBlock(BasicBlockId),
MissingFunction(GlobalFunctionId),
UnsupportedCall(GlobalFunctionId),
UnsupportedInstruction(&'static str),
InvalidIntegerConstant { value: i128, type_: Type },
MissingTypeMetadata(SemanticType),
InvalidStackLayout(SemanticType),
Module(ModuleError),
}
impl fmt::Display for CodegenError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnsupportedType(type_) => {
write!(f, "type `{type_:?}` has no Cranelift ABI mapping")
}
Self::MissingValue(value) => write!(f, "IR value {:?} was not lowered", value),
Self::MissingBlock(block) => write!(f, "IR block {:?} does not exist", block),
Self::MissingFunction(function) => {
write!(f, "function {:?} is not registered", function)
}
Self::UnsupportedCall(function) => {
write!(f, "function {:?} has no codegen implementation", function)
}
Self::UnsupportedInstruction(feature) => {
write!(f, "unsupported codegen instruction: {feature}")
}
Self::InvalidIntegerConstant { value, type_ } => {
write!(f, "integer `{value}` does not fit Cranelift type `{type_}`")
}
Self::MissingTypeMetadata(type_) => {
write!(f, "type `{type_:?}` has no layout metadata")
}
Self::InvalidStackLayout(type_) => write!(
f,
"type `{type_:?}` cannot be represented in a Cranelift stack slot"
),
Self::Module(error) => error.fmt(f),
}
}
}
impl Error for CodegenError {}
impl From<ModuleError> for CodegenError {
fn from(value: ModuleError) -> Self {
Self::Module(value)
}
}
pub fn lower_typed_body(
body: &TypedBody,
signature: &SemanticFunctionType,
functions: &FunctionRegistry,
) -> Result<Function, CodegenError> {
lower_typed_body_with_config(
body,
signature,
functions,
cranelift::codegen::isa::TargetFrontendConfig {
default_call_conv: cranelift::codegen::isa::CallConv::SystemV,
pointer_width: target_lexicon::PointerWidth::U64,
page_size_align_log2: 12,
},
)
}
pub fn lower_typed_body_with_config(
body: &TypedBody,
signature: &SemanticFunctionType,
functions: &FunctionRegistry,
target_config: cranelift::codegen::isa::TargetFrontendConfig,
) -> Result<Function, CodegenError> {
lower_typed_body_impl(
body,
signature,
functions,
None,
target_config,
ExecutionMode::Direct,
&BTreeSet::new(),
|_| (BTreeMap::new(), None),
)
}
pub(crate) fn lower_typed_body_with_module<M: Module>(
body: &TypedBody,
signature: &SemanticFunctionType,
functions: &FunctionRegistry,
module: &mut M,
imports: &BTreeMap<GlobalFunctionId, FuncId>,
type_metadata: Option<&TypeMetadataRegistry>,
execution_mode: ExecutionMode,
checked_script_functions: &BTreeSet<GlobalFunctionId>,
execution_error_reporter: Option<FuncId>,
) -> Result<Function, CodegenError> {
let target_config = module.target_config();
lower_typed_body_impl(
body,
signature,
functions,
type_metadata,
target_config,
execution_mode,
checked_script_functions,
|function| {
let imports = imports
.iter()
.map(|(global, module_function)| {
(
*global,
module.declare_func_in_func(*module_function, function),
)
})
.collect();
let execution_error_reporter = execution_error_reporter
.map(|reporter| module.declare_func_in_func(reporter, function));
(imports, execution_error_reporter)
},
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum AbiValueClass {
Direct(Type),
Indirect { size: usize, align: usize },
}
pub(crate) fn classify_abi_value(
type_: &SemanticType,
pointer_type: Type,
type_metadata: Option<&TypeMetadataRegistry>,
) -> Result<AbiValueClass, CodegenError> {
if let Ok(type_) = clif_type(type_, pointer_type) {
return Ok(AbiValueClass::Direct(type_));
}
let (size, align) = type_metadata
.and_then(|registry| registry.layout_of(type_))
.ok_or_else(|| CodegenError::UnsupportedType(type_.clone()))?;
Ok(AbiValueClass::Indirect { size, align })
}
#[allow(dead_code)]
pub(crate) fn clif_signature(
signature: &SemanticFunctionType,
target_config: cranelift::codegen::isa::TargetFrontendConfig,
) -> Result<Signature, CodegenError> {
clif_signature_with_metadata(signature, target_config, None)
}
pub(crate) fn clif_signature_with_metadata(
signature: &SemanticFunctionType,
target_config: cranelift::codegen::isa::TargetFrontendConfig,
type_metadata: Option<&TypeMetadataRegistry>,
) -> Result<Signature, CodegenError> {
let mut lowered = Signature::new(target_config.default_call_conv);
if !matches!(signature.return_type.as_ref(), SemanticType::Void)
&& matches!(
classify_abi_value(
signature.return_type.as_ref(),
target_config.pointer_type(),
type_metadata
)?,
AbiValueClass::Indirect { .. }
)
{
lowered
.params
.push(AbiParam::new(target_config.pointer_type()));
}
if let Some(receiver) = &signature.receiver {
if !receiver.borrow {
return Err(CodegenError::UnsupportedType(SemanticType::SelfType));
}
lowered
.params
.push(AbiParam::new(target_config.pointer_type()));
}
for argument in &signature.arguments {
let type_ =
match classify_abi_value(&argument.type_, target_config.pointer_type(), type_metadata)?
{
AbiValueClass::Direct(type_) => type_,
AbiValueClass::Indirect { .. } => target_config.pointer_type(),
};
lowered.params.push(AbiParam::new(type_));
}
if !matches!(signature.return_type.as_ref(), SemanticType::Void)
&& let AbiValueClass::Direct(type_) = classify_abi_value(
signature.return_type.as_ref(),
target_config.pointer_type(),
type_metadata,
)?
{
lowered.returns.push(AbiParam::new(type_));
}
Ok(lowered)
}
pub(crate) fn clif_signature_with_execution_mode(
signature: &SemanticFunctionType,
target_config: cranelift::codegen::isa::TargetFrontendConfig,
type_metadata: Option<&TypeMetadataRegistry>,
execution_mode: ExecutionMode,
) -> Result<Signature, CodegenError> {
let mut lowered = clif_signature_with_metadata(signature, target_config, type_metadata)?;
if execution_mode == ExecutionMode::Checked {
lowered
.params
.push(AbiParam::new(target_config.pointer_type()));
}
Ok(lowered)
}
fn lower_typed_body_impl(
body: &TypedBody,
signature: &SemanticFunctionType,
functions: &FunctionRegistry,
type_metadata: Option<&TypeMetadataRegistry>,
target_config: cranelift::codegen::isa::TargetFrontendConfig,
execution_mode: ExecutionMode,
checked_script_functions: &BTreeSet<GlobalFunctionId>,
declare_imports: impl FnOnce(
&mut Function,
) -> (BTreeMap<GlobalFunctionId, FuncRef>, Option<FuncRef>),
) -> Result<Function, CodegenError> {
let clif_signature = clif_signature_with_execution_mode(
signature,
target_config,
type_metadata,
execution_mode,
)?;
let mut function = Function::with_name_signature(UserFuncName::user(0, 0), clif_signature);
let (imported_functions, execution_error_reporter) = declare_imports(&mut function);
let mut context = FunctionBuilderContext::new();
let mut builder = FunctionBuilder::new(&mut function, &mut context);
let mut blocks = BTreeMap::new();
for block in &body.blocks {
blocks.insert(block.id, builder.create_block());
}
let checked_failure =
(execution_mode == ExecutionMode::Checked).then(|| builder.create_block());
let entry = *blocks
.get(&body.entry)
.ok_or(CodegenError::MissingBlock(body.entry))?;
builder.append_block_params_for_function_params(entry);
for block in &body.blocks {
let clif_block = blocks[&block.id];
for parameter in &block.parameters {
builder.append_block_param(
clif_block,
clif_type(¶meter.type_, target_config.pointer_type())?,
);
}
}
let mut stack_locals = addressable_locals(body);
for local in &body.locals {
if matches!(
classify_abi_value(&local.type_, target_config.pointer_type(), type_metadata)?,
AbiValueClass::Indirect { .. }
) {
stack_locals.insert(local.id);
}
}
let mut variables = BTreeMap::new();
let mut stack_slots = BTreeMap::new();
for local in &body.locals {
if stack_locals.contains(&local.id) {
let registry = type_metadata
.ok_or_else(|| CodegenError::MissingTypeMetadata(local.type_.clone()))?;
let (size, align) = registry
.layout_of(&local.type_)
.ok_or_else(|| CodegenError::MissingTypeMetadata(local.type_.clone()))?;
let size = u32::try_from(size)
.map_err(|_| CodegenError::InvalidStackLayout(local.type_.clone()))?;
let align_shift = u8::try_from(align.trailing_zeros())
.map_err(|_| CodegenError::InvalidStackLayout(local.type_.clone()))?;
let slot = builder.create_sized_stack_slot(StackSlotData::new(
StackSlotKind::ExplicitSlot,
size,
align_shift,
));
stack_slots.insert(local.id, slot);
} else {
let variable =
builder.declare_var(clif_type(&local.type_, target_config.pointer_type())?);
variables.insert(local.id, variable);
}
}
builder.switch_to_block(entry);
let entry_parameters = builder.block_params(entry).to_vec();
let indirect_return = !matches!(signature.return_type.as_ref(), SemanticType::Void)
&& matches!(
classify_abi_value(
signature.return_type.as_ref(),
target_config.pointer_type(),
type_metadata,
)?,
AbiValueClass::Indirect { .. }
);
let return_pointer = indirect_return.then(|| entry_parameters[0]);
let execution_context = if execution_mode == ExecutionMode::Checked {
Some(
*entry_parameters
.last()
.ok_or(CodegenError::UnsupportedInstruction(
"checked function has no execution context parameter",
))?,
)
} else {
None
};
let receiver_offset = usize::from(signature.receiver.is_some()) + usize::from(indirect_return);
for (index, local) in body
.locals
.iter()
.filter(|local| local.parameter)
.enumerate()
{
let value = *entry_parameters.get(index + receiver_offset).ok_or(
CodegenError::UnsupportedInstruction("missing function parameter"),
)?;
if let Some(variable) = variables.get(&local.id) {
builder.def_var(*variable, value);
} else {
let slot = stack_slots[&local.id];
match classify_abi_value(&local.type_, target_config.pointer_type(), type_metadata)? {
AbiValueClass::Direct(_) => {
builder
.ins()
.stack_store(target_config.pointer_type(), value, slot, 0);
}
AbiValueClass::Indirect { size, .. } => {
let destination =
builder
.ins()
.stack_addr(target_config.pointer_type(), slot, 0);
copy_memory(
&mut builder,
destination,
value,
size,
target_config.pointer_type(),
)?;
}
}
}
}
let mut values = BTreeMap::<IrValueId, Value>::new();
for block in &body.blocks {
let clif_block = blocks[&block.id];
if block.id != body.entry {
builder.switch_to_block(clif_block);
}
let function_parameter_count = signature.arguments.len()
+ usize::from(signature.receiver.is_some())
+ usize::from(indirect_return)
+ usize::from(execution_mode == ExecutionMode::Checked);
let parameter_offset = usize::from(block.id == body.entry) * function_parameter_count;
for (parameter, value) in block.parameters.iter().zip(
builder.block_params(clif_block)[parameter_offset..]
.iter()
.copied(),
) {
values.insert(parameter.id, value);
}
for instruction in &block.instructions {
let produced =
match &instruction.kind {
TypedInstructionKind::Constant(constant) => {
let result = instruction.result.as_ref().ok_or(
CodegenError::UnsupportedInstruction("constant without result"),
)?;
lower_constant(
&mut builder,
constant,
&result.type_,
target_config.pointer_type(),
)?
}
TypedInstructionKind::Load(place) => {
let place_data = body
.places
.get(place.0 as usize)
.ok_or(CodegenError::UnsupportedInstruction("unknown place"))?;
let local = place_local(place_data);
if place_data.projections.is_empty() && variables.contains_key(&local) {
Some(builder.use_var(variables[&local]))
} else {
let address = place_address(
&mut builder,
body,
*place,
&values,
&variables,
&stack_slots,
type_metadata,
target_config.pointer_type(),
)?;
match classify_abi_value(
&place_data.type_,
target_config.pointer_type(),
type_metadata,
)? {
AbiValueClass::Direct(type_) => {
Some(builder.ins().load(type_, MemFlagsData::new(), address, 0))
}
AbiValueClass::Indirect { .. } => Some(address),
}
}
}
TypedInstructionKind::Store { place, value } => {
let place_data = body
.places
.get(place.0 as usize)
.ok_or(CodegenError::UnsupportedInstruction("unknown place"))?;
let local = place_local(place_data);
let value = value_of(&values, *value)?;
if place_data.projections.is_empty() && variables.contains_key(&local) {
builder.def_var(variables[&local], value);
} else {
let address = place_address(
&mut builder,
body,
*place,
&values,
&variables,
&stack_slots,
type_metadata,
target_config.pointer_type(),
)?;
match classify_abi_value(
&place_data.type_,
target_config.pointer_type(),
type_metadata,
)? {
AbiValueClass::Direct(_) => {
builder.ins().store(MemFlagsData::new(), value, address, 0);
}
AbiValueClass::Indirect { size, .. } => copy_memory(
&mut builder,
address,
value,
size,
target_config.pointer_type(),
)?,
}
}
None
}
TypedInstructionKind::Borrow { place, .. } => Some(place_address(
&mut builder,
body,
*place,
&values,
&variables,
&stack_slots,
type_metadata,
target_config.pointer_type(),
)?),
TypedInstructionKind::Drop {
place,
function,
conditions,
} => {
let action = crate::semantic::CheckedDropAction {
place: *place,
function: *function,
conditions: conditions.clone(),
};
let cleanup_context = match execution_context {
Some(context) => context,
None => builder.ins().iconst(target_config.pointer_type(), 0),
};
lower_conditional_cleanup_drop(
&mut builder,
body,
functions,
&imported_functions,
&action,
&values,
&variables,
&stack_slots,
type_metadata,
target_config.pointer_type(),
cleanup_context,
checked_script_functions,
)?;
if checked_script_functions.contains(function) {
branch_to_checked_failure_on_error(
&mut builder,
execution_context,
checked_failure,
)?;
}
None
}
TypedInstructionKind::BoundsCheck {
index,
length,
cleanup,
} => {
let index = value_of(&values, *index)?;
let length_imm = i64::try_from(*length).map_err(|_| {
CodegenError::UnsupportedInstruction("array length exceeds i64")
})?;
let out_of_bounds = builder.ins().icmp_imm_u(
cranelift::codegen::ir::condcodes::IntCC::UnsignedGreaterThanOrEqual,
index,
length_imm,
);
if execution_mode == ExecutionMode::Direct {
builder.ins().trapnz(
out_of_bounds,
cranelift::codegen::ir::TrapCode::unwrap_user(1),
);
} else {
let reporter = execution_error_reporter.ok_or(
CodegenError::UnsupportedInstruction(
"checked bounds reporter is not imported",
),
)?;
let execution_context =
execution_context.ok_or(CodegenError::UnsupportedInstruction(
"checked bounds has no execution context",
))?;
let length_value = builder
.ins()
.iconst(target_config.pointer_type(), length_imm);
let continuation = builder.create_block();
let failure =
checked_failure.ok_or(CodegenError::UnsupportedInstruction(
"checked bounds has no failure block",
))?;
let report_block = builder.create_block();
builder
.ins()
.brif(out_of_bounds, report_block, &[], continuation, &[]);
builder.switch_to_block(report_block);
builder
.ins()
.call(reporter, &[execution_context, index, length_value]);
for action in &cleanup.drops {
lower_conditional_cleanup_drop(
&mut builder,
body,
functions,
&imported_functions,
action,
&values,
&variables,
&stack_slots,
type_metadata,
target_config.pointer_type(),
execution_context,
checked_script_functions,
)?;
}
builder.ins().jump(failure, &[]);
builder.switch_to_block(continuation);
builder.seal_block(report_block);
builder.seal_block(continuation);
}
None
}
TypedInstructionKind::IntegerEqual { left, right } => Some(builder.ins().icmp(
cranelift::codegen::ir::condcodes::IntCC::Equal,
value_of(&values, *left)?,
value_of(&values, *right)?,
)),
TypedInstructionKind::Call {
function,
arguments,
cleanup,
} => lower_call(
&mut builder,
functions,
&imported_functions,
*function,
arguments,
&values,
instruction.result.as_ref().map(|result| &result.type_),
type_metadata,
target_config,
target_config.pointer_type(),
execution_context,
checked_script_functions,
checked_failure,
cleanup,
body,
&variables,
&stack_slots,
)?,
};
if let (Some(result), Some(value)) = (&instruction.result, produced) {
values.insert(result.id, value);
}
}
match &block.terminator {
TypedTerminator::Return(value) => {
if let Some(destination) = return_pointer {
let source = value
.map(|value| value_of(&values, value))
.transpose()?
.ok_or(CodegenError::UnsupportedInstruction(
"indirect return has no value",
))?;
let (size, _) = type_metadata
.and_then(|registry| registry.layout_of(signature.return_type.as_ref()))
.ok_or_else(|| {
CodegenError::MissingTypeMetadata(
signature.return_type.as_ref().clone(),
)
})?;
copy_memory(
&mut builder,
destination,
source,
size,
target_config.pointer_type(),
)?;
builder.ins().return_(&[]);
} else {
let returned = value
.map(|value| value_of(&values, value))
.transpose()?
.into_iter()
.collect::<Vec<_>>();
builder.ins().return_(&returned);
}
}
TypedTerminator::Jump { target, arguments } => {
let target = *blocks
.get(target)
.ok_or(CodegenError::MissingBlock(*target))?;
let arguments = arguments
.iter()
.map(|value| value_of(&values, *value).map(BlockArg::from))
.collect::<Result<Vec<_>, _>>()?;
builder.ins().jump(target, &arguments);
}
TypedTerminator::Branch {
condition,
then_block,
else_block,
} => {
let condition = value_of(&values, *condition)?;
let then_block = *blocks
.get(then_block)
.ok_or(CodegenError::MissingBlock(*then_block))?;
let else_block = *blocks
.get(else_block)
.ok_or(CodegenError::MissingBlock(*else_block))?;
builder
.ins()
.brif(condition, then_block, &[], else_block, &[]);
}
TypedTerminator::Unreachable => {
builder
.ins()
.trap(cranelift::codegen::ir::TrapCode::unwrap_user(0));
}
}
}
if let Some(failure) = checked_failure {
builder.switch_to_block(failure);
if indirect_return || matches!(signature.return_type.as_ref(), SemanticType::Void) {
builder.ins().return_(&[]);
} else {
let return_type = classify_abi_value(
signature.return_type.as_ref(),
target_config.pointer_type(),
type_metadata,
)?;
let AbiValueClass::Direct(return_type) = return_type else {
return Err(CodegenError::UnsupportedInstruction(
"checked aggregate failure requires an indirect return",
));
};
let zero = if return_type == types::F32 {
builder.ins().f32const(0.0)
} else if return_type == types::F64 {
builder.ins().f64const(0.0)
} else {
builder.ins().iconst(return_type, 0)
};
builder.ins().return_(&[zero]);
}
}
builder.seal_all_blocks();
builder.finalize(target_config);
Ok(function)
}
fn lower_conditional_cleanup_drop(
builder: &mut FunctionBuilder<'_>,
body: &TypedBody,
functions: &FunctionRegistry,
imported_functions: &BTreeMap<GlobalFunctionId, FuncRef>,
action: &crate::semantic::CheckedDropAction,
values: &BTreeMap<IrValueId, Value>,
variables: &BTreeMap<LocalId, cranelift_frontend::Variable>,
stack_slots: &BTreeMap<LocalId, cranelift::codegen::ir::StackSlot>,
type_metadata: Option<&TypeMetadataRegistry>,
pointer_type: Type,
execution_context: Value,
checked_script_functions: &BTreeSet<GlobalFunctionId>,
) -> Result<(), CodegenError> {
if action.conditions.is_empty() {
return lower_cleanup_drop(
builder,
body,
functions,
imported_functions,
action.place,
action.function,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
execution_context,
checked_script_functions,
);
}
let registry = type_metadata.ok_or(CodegenError::UnsupportedInstruction(
"conditional enum cleanup requires type metadata",
))?;
let mut selected = None;
for condition in &action.conditions {
let enum_type = &body.places[condition.enum_place.0 as usize].type_;
let SemanticType::Named(named) = enum_type else {
return Err(CodegenError::MissingTypeMetadata(enum_type.clone()));
};
let metadata = registry
.get_named(named)
.ok_or_else(|| CodegenError::MissingTypeMetadata(enum_type.clone()))?;
let TypeMetadataKind::Enum { tag_offset, .. } = &metadata.kind else {
return Err(CodegenError::MissingTypeMetadata(enum_type.clone()));
};
let mut tag_address = place_address(
builder,
body,
condition.enum_place,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
)?;
if *tag_offset != 0 {
let tag_offset = i64::try_from(*tag_offset)
.map_err(|_| CodegenError::UnsupportedInstruction("enum tag offset exceeds i64"))?;
tag_address = builder.ins().iadd_imm_u(tag_address, tag_offset);
}
let tag_type = match condition.tag_bits {
0..=8 => types::I8,
9..=16 => types::I16,
17..=32 => types::I32,
33..=64 => types::I64,
_ => {
return Err(CodegenError::UnsupportedInstruction(
"enum cleanup tags wider than 64 bits",
));
}
};
let tag = builder
.ins()
.load(tag_type, MemFlagsData::new(), tag_address, 0);
let compared_tags = if condition.fallback {
&condition.explicit_tags
} else {
&condition.matching_tags
};
let mut matches_any = builder.ins().iconst(types::I8, 0);
for candidate in compared_tags {
let candidate = i64::try_from(*candidate).map_err(|_| {
CodegenError::UnsupportedInstruction("enum cleanup tag exceeds i64")
})?;
let matches = builder.ins().icmp_imm_u(
cranelift::codegen::ir::condcodes::IntCC::Equal,
tag,
candidate,
);
matches_any = builder.ins().bor(matches_any, matches);
}
let condition_selected = if condition.fallback {
builder.ins().bnot(matches_any)
} else {
matches_any
};
selected = Some(match selected {
Some(current) => builder.ins().band(current, condition_selected),
None => condition_selected,
});
}
let execute = builder.create_block();
let continuation = builder.create_block();
builder.ins().brif(
selected.ok_or(CodegenError::UnsupportedInstruction(
"conditional cleanup has no enum condition",
))?,
execute,
&[],
continuation,
&[],
);
builder.switch_to_block(execute);
lower_cleanup_drop(
builder,
body,
functions,
imported_functions,
action.place,
action.function,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
execution_context,
checked_script_functions,
)?;
builder.ins().jump(continuation, &[]);
builder.switch_to_block(continuation);
builder.seal_block(execute);
builder.seal_block(continuation);
Ok(())
}
fn lower_cleanup_drop(
builder: &mut FunctionBuilder<'_>,
body: &TypedBody,
functions: &FunctionRegistry,
imported_functions: &BTreeMap<GlobalFunctionId, FuncRef>,
place: PlaceId,
function: GlobalFunctionId,
values: &BTreeMap<IrValueId, Value>,
variables: &BTreeMap<LocalId, cranelift_frontend::Variable>,
stack_slots: &BTreeMap<LocalId, cranelift::codegen::ir::StackSlot>,
type_metadata: Option<&TypeMetadataRegistry>,
pointer_type: Type,
execution_context: Value,
checked_script_functions: &BTreeSet<GlobalFunctionId>,
) -> Result<(), CodegenError> {
let imported = imported_functions
.get(&function)
.copied()
.ok_or(CodegenError::UnsupportedCall(function))?;
let address = place_address(
builder,
body,
place,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
)?;
let function_data = functions
.function(function)
.ok_or(CodegenError::MissingFunction(function))?;
if function_data.path.module == crate::core_lib::BOX_MODULE_FILE_PATH
&& function_data.path.local.ends_with(".Drop.drop")
{
let place_type = &body.places[place.0 as usize].type_;
let SemanticType::Named(box_type) = place_type else {
return Err(CodegenError::UnsupportedCall(function));
};
let [crate::semantic::SemanticGenericArgument::Type(payload)] =
box_type.generic_arguments.as_slice()
else {
return Err(CodegenError::UnsupportedCall(function));
};
let registry =
type_metadata.ok_or_else(|| CodegenError::UnsupportedType(payload.clone()))?;
let (size, align) = registry
.layout_of(payload)
.ok_or_else(|| CodegenError::UnsupportedType(payload.clone()))?;
let allocation = builder
.ins()
.load(pointer_type, MemFlagsData::new(), address, 0);
let size = builder.ins().iconst(pointer_type, size as i64);
let align = builder.ins().iconst(pointer_type, align as i64);
builder.ins().call(imported, &[allocation, size, align]);
} else {
let mut arguments = vec![address];
if checked_script_functions.contains(&function) {
arguments.push(execution_context);
}
builder.ins().call(imported, &arguments);
}
Ok(())
}
fn copy_memory(
builder: &mut FunctionBuilder<'_>,
destination: Value,
source: Value,
size: usize,
_pointer_type: Type,
) -> Result<(), CodegenError> {
let mut offset = 0usize;
while offset < size {
let remaining = size - offset;
let (type_, width) = if remaining >= 8 {
(types::I64, 8)
} else if remaining >= 4 {
(types::I32, 4)
} else if remaining >= 2 {
(types::I16, 2)
} else {
(types::I8, 1)
};
let offset_i32 = i32::try_from(offset).map_err(|_| {
CodegenError::UnsupportedInstruction("aggregate copy exceeds i32 offset")
})?;
let value = builder
.ins()
.load(type_, MemFlagsData::new(), source, offset_i32);
builder
.ins()
.store(MemFlagsData::new(), value, destination, offset_i32);
offset += width;
}
Ok(())
}
fn lower_constant(
builder: &mut FunctionBuilder<'_>,
constant: &TypedConstant,
semantic_type: &SemanticType,
pointer_type: Type,
) -> Result<Option<Value>, CodegenError> {
if matches!(constant, TypedConstant::Void) {
return Ok(None);
}
let type_ = clif_type(semantic_type, pointer_type)?;
let value = match constant {
TypedConstant::Void => unreachable!("void constants return before type lowering"),
TypedConstant::Bool(value) => builder.ins().iconst(type_, i64::from(*value)),
TypedConstant::Char(value) => builder.ins().iconst(type_, i64::from(u32::from(*value))),
TypedConstant::Signed(value) => builder.ins().iconst(
type_,
i64::try_from(*value).map_err(|_| CodegenError::InvalidIntegerConstant {
value: *value,
type_,
})?,
),
TypedConstant::Unsigned(value) => builder.ins().iconst(
type_,
i64::try_from(*value).map_err(|_| CodegenError::InvalidIntegerConstant {
value: *value as i128,
type_,
})?,
),
TypedConstant::Float(value) if type_ == types::F32 => builder.ins().f32const(*value as f32),
TypedConstant::Float(value) if type_ == types::F64 => builder.ins().f64const(*value),
TypedConstant::Float(_) => {
return Err(CodegenError::UnsupportedType(semantic_type.clone()));
}
TypedConstant::String(_) => {
return Err(CodegenError::UnsupportedInstruction("string constants"));
}
};
Ok(Some(value))
}
fn lower_call(
builder: &mut FunctionBuilder<'_>,
functions: &FunctionRegistry,
imports: &BTreeMap<GlobalFunctionId, FuncRef>,
function: GlobalFunctionId,
arguments: &[IrValueId],
values: &BTreeMap<IrValueId, Value>,
result_type: Option<&SemanticType>,
type_metadata: Option<&TypeMetadataRegistry>,
target_config: cranelift::codegen::isa::TargetFrontendConfig,
pointer_type: Type,
execution_context: Option<Value>,
checked_script_functions: &BTreeSet<GlobalFunctionId>,
checked_failure: Option<Block>,
cleanup: &CheckedCleanupPlan,
body: &TypedBody,
variables: &BTreeMap<LocalId, cranelift_frontend::Variable>,
stack_slots: &BTreeMap<LocalId, cranelift::codegen::ir::StackSlot>,
) -> Result<Option<Value>, CodegenError> {
let function_data = functions
.function(function)
.ok_or(CodegenError::MissingFunction(function))?;
let arguments = arguments
.iter()
.map(|value| value_of(values, *value))
.collect::<Result<Vec<_>, _>>()?;
let path = function_data.path.local.as_str();
if function_data.path.module == crate::core_lib::BOX_MODULE_FILE_PATH && path == "Box.new" {
let [value] = arguments.as_slice() else {
return Err(CodegenError::UnsupportedCall(function));
};
let result_type = result_type.ok_or(CodegenError::UnsupportedInstruction(
"Box.new without concrete result type",
))?;
let SemanticType::Named(box_type) = result_type else {
return Err(CodegenError::UnsupportedCall(function));
};
let [crate::semantic::SemanticGenericArgument::Type(payload_type)] =
box_type.generic_arguments.as_slice()
else {
return Err(CodegenError::UnsupportedCall(function));
};
let registry =
type_metadata.ok_or_else(|| CodegenError::UnsupportedType(payload_type.clone()))?;
let (payload_size, payload_align) = registry
.layout_of(payload_type)
.ok_or_else(|| CodegenError::UnsupportedType(payload_type.clone()))?;
let allocator = imports
.get(&function)
.ok_or(CodegenError::UnsupportedCall(function))?;
let size = builder.ins().iconst(pointer_type, payload_size as i64);
let align = builder.ins().iconst(pointer_type, payload_align as i64);
let allocation_call = builder.ins().call(*allocator, &[size, align]);
let allocation = *builder
.inst_results(allocation_call)
.first()
.ok_or(CodegenError::UnsupportedCall(function))?;
match classify_abi_value(payload_type, pointer_type, type_metadata)? {
AbiValueClass::Direct(_) => {
builder
.ins()
.store(MemFlagsData::new(), *value, allocation, 0);
}
AbiValueClass::Indirect { .. } => builder.emit_small_memory_copy(
target_config,
allocation,
*value,
payload_size as u64,
payload_align.trailing_zeros() as u8,
payload_align.trailing_zeros() as u8,
true,
MemFlagsData::new(),
),
}
let (box_size, box_align) = registry
.layout_of(result_type)
.ok_or_else(|| CodegenError::UnsupportedType(result_type.clone()))?;
let slot = builder.create_sized_stack_slot(StackSlotData::new(
StackSlotKind::ExplicitSlot,
u32::try_from(box_size)
.map_err(|_| CodegenError::InvalidStackLayout(result_type.clone()))?,
u8::try_from(box_align.trailing_zeros())
.map_err(|_| CodegenError::InvalidStackLayout(result_type.clone()))?,
));
let output = builder.ins().stack_addr(pointer_type, slot, 0);
builder
.ins()
.store(MemFlagsData::new(), allocation, output, 0);
return Ok(Some(output));
}
if function_data.path.module == crate::core_lib::NON_ZERO_MODULE_FILE_PATH
&& path == "NonZero.new"
{
let [value] = arguments.as_slice() else {
return Err(CodegenError::UnsupportedCall(function));
};
let result_type = result_type.ok_or(CodegenError::UnsupportedInstruction(
"NonZero.new without concrete result type",
))?;
let AbiValueClass::Indirect { size, align } =
classify_abi_value(result_type, pointer_type, type_metadata)?
else {
return Err(CodegenError::UnsupportedCall(function));
};
let size = u32::try_from(size)
.map_err(|_| CodegenError::InvalidStackLayout(result_type.clone()))?;
let align_shift = u8::try_from(align.trailing_zeros())
.map_err(|_| CodegenError::InvalidStackLayout(result_type.clone()))?;
let slot = builder.create_sized_stack_slot(StackSlotData::new(
StackSlotKind::ExplicitSlot,
size,
align_shift,
));
let output = builder.ins().stack_addr(pointer_type, slot, 0);
builder.ins().store(MemFlagsData::new(), *value, output, 0);
return Ok(Some(output));
}
if path.ends_with(".Clone.clone") {
let [receiver] = arguments.as_slice() else {
return Err(CodegenError::UnsupportedCall(function));
};
let pointer_type = builder.func.dfg.value_type(*receiver);
let return_type = clif_type(
result_type.ok_or(CodegenError::UnsupportedInstruction(
"Clone call without concrete result type",
))?,
pointer_type,
)?;
return Ok(Some(builder.ins().load(
return_type,
MemFlagsData::new(),
*receiver,
0,
)));
}
if let Some(import) = imports.get(&function) {
if let Some(result_type) = result_type
&& let AbiValueClass::Indirect { size, align } =
classify_abi_value(result_type, pointer_type, type_metadata)?
{
let size = u32::try_from(size)
.map_err(|_| CodegenError::InvalidStackLayout(result_type.clone()))?;
let align_shift = u8::try_from(align.trailing_zeros())
.map_err(|_| CodegenError::InvalidStackLayout(result_type.clone()))?;
let slot = builder.create_sized_stack_slot(StackSlotData::new(
StackSlotKind::ExplicitSlot,
size,
align_shift,
));
let output = builder.ins().stack_addr(pointer_type, slot, 0);
let mut indirect_arguments = Vec::with_capacity(arguments.len() + 2);
indirect_arguments.push(output);
indirect_arguments.extend(arguments);
if checked_script_functions.contains(&function) {
indirect_arguments.push(execution_context.ok_or(
CodegenError::UnsupportedInstruction(
"checked script call has no execution context",
),
)?);
}
builder.ins().call(*import, &indirect_arguments);
if checked_script_functions.contains(&function) {
branch_to_checked_failure_with_cleanup(
builder,
execution_context,
checked_failure,
cleanup,
body,
functions,
imports,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
checked_script_functions,
)?;
}
return Ok(Some(output));
}
let mut call_arguments = arguments;
if checked_script_functions.contains(&function) {
call_arguments.push(
execution_context.ok_or(CodegenError::UnsupportedInstruction(
"checked script call has no execution context",
))?,
);
}
let call = builder.ins().call(*import, &call_arguments);
let result = builder.inst_results(call).first().copied();
if checked_script_functions.contains(&function) {
branch_to_checked_failure_with_cleanup(
builder,
execution_context,
checked_failure,
cleanup,
body,
functions,
imports,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
checked_script_functions,
)?;
}
return Ok(result);
}
let [left, right] = arguments.as_slice() else {
return Err(CodegenError::UnsupportedCall(function));
};
let value = if path.ends_with(".PartialEq.eq") || path.ends_with(".PartialEq.ne") {
let owner = path
.split(".PartialEq.")
.next()
.ok_or(CodegenError::UnsupportedCall(function))?;
let owner_type = SemanticType::Named(crate::semantic::SemanticNamedType {
path: crate::semantic::CanonicalSymbolPath {
module: function_data.path.module.clone(),
local: owner.to_owned(),
},
generic_arguments: Vec::new(),
});
let pointer_type = builder.func.dfg.value_type(*left);
let operand_type = clif_type(&owner_type, pointer_type)?;
let left = builder
.ins()
.load(operand_type, MemFlagsData::new(), *left, 0);
let right = builder
.ins()
.load(operand_type, MemFlagsData::new(), *right, 0);
let condition = if path.ends_with(".PartialEq.eq") {
cranelift::codegen::ir::condcodes::IntCC::Equal
} else {
cranelift::codegen::ir::condcodes::IntCC::NotEqual
};
builder.ins().icmp(condition, left, right)
} else if path.ends_with(".Add.add") {
let operand_type = builder.func.dfg.value_type(*left);
if operand_type == types::F32 || operand_type == types::F64 {
builder.ins().fadd(*left, *right)
} else if execution_context.is_some() {
let owner = path.split(".Add.").next().unwrap_or_default();
let (result, overflow) = if owner.starts_with('i') {
builder.ins().sadd_overflow(*left, *right)
} else {
builder.ins().uadd_overflow(*left, *right)
};
branch_to_arithmetic_failure_with_cleanup(
builder,
overflow,
crate::runtime::ExecutionErrorCode::IntegerOverflow as u32,
execution_context,
checked_failure,
cleanup,
body,
functions,
imports,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
checked_script_functions,
)?;
result
} else {
builder.ins().iadd(*left, *right)
}
} else if path.ends_with(".Sub.sub") {
let operand_type = builder.func.dfg.value_type(*left);
if operand_type == types::F32 || operand_type == types::F64 {
builder.ins().fsub(*left, *right)
} else if execution_context.is_some() {
let owner = path.split(".Sub.").next().unwrap_or_default();
let (result, overflow) = if owner.starts_with('i') {
builder.ins().ssub_overflow(*left, *right)
} else {
builder.ins().usub_overflow(*left, *right)
};
branch_to_arithmetic_failure_with_cleanup(
builder,
overflow,
crate::runtime::ExecutionErrorCode::IntegerOverflow as u32,
execution_context,
checked_failure,
cleanup,
body,
functions,
imports,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
checked_script_functions,
)?;
result
} else {
builder.ins().isub(*left, *right)
}
} else if path.ends_with(".Mul.mul") {
let operand_type = builder.func.dfg.value_type(*left);
if operand_type == types::F32 || operand_type == types::F64 {
builder.ins().fmul(*left, *right)
} else if execution_context.is_some() {
let owner = path.split(".Mul.").next().unwrap_or_default();
let (result, overflow) = if owner.starts_with('i') {
builder.ins().smul_overflow(*left, *right)
} else {
builder.ins().umul_overflow(*left, *right)
};
branch_to_arithmetic_failure_with_cleanup(
builder,
overflow,
crate::runtime::ExecutionErrorCode::IntegerOverflow as u32,
execution_context,
checked_failure,
cleanup,
body,
functions,
imports,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
checked_script_functions,
)?;
result
} else {
builder.ins().imul(*left, *right)
}
} else if path.ends_with(".Shl.shl") || path.ends_with(".Shr.shr") {
let operand_type = builder.func.dfg.value_type(*left);
if execution_context.is_some() {
let bit_width = i64::from(operand_type.bits());
let invalid = builder.ins().icmp_imm_u(
cranelift::codegen::ir::condcodes::IntCC::UnsignedGreaterThanOrEqual,
*right,
bit_width,
);
let context = execution_context.expect("checked shift context was tested");
let amount_offset = i32::try_from(std::mem::offset_of!(
crate::runtime::ExecutionContext,
first
))
.map_err(|_| CodegenError::UnsupportedInstruction("execution context offset"))?;
let width_offset = i32::try_from(std::mem::offset_of!(
crate::runtime::ExecutionContext,
second
))
.map_err(|_| CodegenError::UnsupportedInstruction("execution context offset"))?;
let amount = if builder.func.dfg.value_type(*right) == pointer_type {
*right
} else {
builder.ins().uextend(pointer_type, *right)
};
builder
.ins()
.store(MemFlagsData::new(), amount, context, amount_offset);
let width = builder.ins().iconst(pointer_type, bit_width);
builder
.ins()
.store(MemFlagsData::new(), width, context, width_offset);
branch_to_arithmetic_failure_with_cleanup(
builder,
invalid,
crate::runtime::ExecutionErrorCode::InvalidShift as u32,
execution_context,
checked_failure,
cleanup,
body,
functions,
imports,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
checked_script_functions,
)?;
}
if path.ends_with(".Shl.shl") {
builder.ins().ishl(*left, *right)
} else {
let owner = path.split(".Shr.").next().unwrap_or_default();
if owner.starts_with('i') {
builder.ins().sshr(*left, *right)
} else {
builder.ins().ushr(*left, *right)
}
}
} else if path.ends_with(".Div.div") {
let owner = path
.split(".Div.")
.next()
.ok_or(CodegenError::UnsupportedCall(function))?;
let operand_type = builder.func.dfg.value_type(*left);
if operand_type == types::F32 {
builder.ins().fdiv(*left, *right)
} else if operand_type == types::F64 {
builder.ins().fdiv(*left, *right)
} else {
if execution_context.is_some() {
let division_by_zero = builder.ins().icmp_imm_u(
cranelift::codegen::ir::condcodes::IntCC::Equal,
*right,
0,
);
branch_to_arithmetic_failure_with_cleanup(
builder,
division_by_zero,
crate::runtime::ExecutionErrorCode::DivisionByZero as u32,
execution_context,
checked_failure,
cleanup,
body,
functions,
imports,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
checked_script_functions,
)?;
}
let signed = owner.starts_with('i') && owner != "isize" || owner == "isize";
if signed {
if execution_context.is_some() {
let minimum = match operand_type {
types::I8 => i8::MIN as i64,
types::I16 => i16::MIN as i64,
types::I32 => i32::MIN as i64,
types::I64 => i64::MIN,
_ => return Err(CodegenError::UnsupportedCall(function)),
};
let is_minimum = builder.ins().icmp_imm_s(
cranelift::codegen::ir::condcodes::IntCC::Equal,
*left,
minimum,
);
let is_negative_one = builder.ins().icmp_imm_s(
cranelift::codegen::ir::condcodes::IntCC::Equal,
*right,
-1,
);
let overflow = builder.ins().band(is_minimum, is_negative_one);
branch_to_arithmetic_failure_with_cleanup(
builder,
overflow,
crate::runtime::ExecutionErrorCode::IntegerOverflow as u32,
execution_context,
checked_failure,
cleanup,
body,
functions,
imports,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
checked_script_functions,
)?;
}
builder.ins().sdiv(*left, *right)
} else {
builder.ins().udiv(*left, *right)
}
}
} else {
return Err(CodegenError::UnsupportedCall(function));
};
Ok(Some(value))
}
fn branch_to_arithmetic_failure_with_cleanup(
builder: &mut FunctionBuilder<'_>,
condition: Value,
error_code: u32,
execution_context: Option<Value>,
checked_failure: Option<Block>,
cleanup: &CheckedCleanupPlan,
body: &TypedBody,
functions: &FunctionRegistry,
imports: &BTreeMap<GlobalFunctionId, FuncRef>,
values: &BTreeMap<IrValueId, Value>,
variables: &BTreeMap<LocalId, cranelift_frontend::Variable>,
stack_slots: &BTreeMap<LocalId, cranelift::codegen::ir::StackSlot>,
type_metadata: Option<&TypeMetadataRegistry>,
pointer_type: Type,
checked_script_functions: &BTreeSet<GlobalFunctionId>,
) -> Result<(), CodegenError> {
let context = execution_context.ok_or(CodegenError::UnsupportedInstruction(
"checked arithmetic has no execution context",
))?;
let failure = checked_failure.ok_or(CodegenError::UnsupportedInstruction(
"checked arithmetic has no failure block",
))?;
let error_block = builder.create_block();
let continuation = builder.create_block();
builder
.ins()
.brif(condition, error_block, &[], continuation, &[]);
builder.switch_to_block(error_block);
let error_code = builder.ins().iconst(types::I32, i64::from(error_code));
builder
.ins()
.store(MemFlagsData::new(), error_code, context, 0);
for action in &cleanup.drops {
lower_conditional_cleanup_drop(
builder,
body,
functions,
imports,
action,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
context,
checked_script_functions,
)?;
}
builder.ins().jump(failure, &[]);
builder.switch_to_block(continuation);
builder.seal_block(error_block);
builder.seal_block(continuation);
Ok(())
}
fn branch_to_checked_failure_on_error(
builder: &mut FunctionBuilder<'_>,
execution_context: Option<Value>,
checked_failure: Option<Block>,
) -> Result<(), CodegenError> {
let context = execution_context.ok_or(CodegenError::UnsupportedInstruction(
"checked call has no execution context",
))?;
let failure = checked_failure.ok_or(CodegenError::UnsupportedInstruction(
"checked call has no failure block",
))?;
let error_code = builder
.ins()
.load(types::I32, MemFlagsData::new(), context, 0);
let continuation = builder.create_block();
builder
.ins()
.brif(error_code, failure, &[], continuation, &[]);
builder.switch_to_block(continuation);
builder.seal_block(continuation);
Ok(())
}
fn branch_to_checked_failure_with_cleanup(
builder: &mut FunctionBuilder<'_>,
execution_context: Option<Value>,
checked_failure: Option<Block>,
cleanup: &CheckedCleanupPlan,
body: &TypedBody,
functions: &FunctionRegistry,
imports: &BTreeMap<GlobalFunctionId, FuncRef>,
values: &BTreeMap<IrValueId, Value>,
variables: &BTreeMap<LocalId, cranelift_frontend::Variable>,
stack_slots: &BTreeMap<LocalId, cranelift::codegen::ir::StackSlot>,
type_metadata: Option<&TypeMetadataRegistry>,
pointer_type: Type,
checked_script_functions: &BTreeSet<GlobalFunctionId>,
) -> Result<(), CodegenError> {
let context = execution_context.ok_or(CodegenError::UnsupportedInstruction(
"checked call has no execution context",
))?;
let failure = checked_failure.ok_or(CodegenError::UnsupportedInstruction(
"checked call has no failure block",
))?;
let error_code = builder
.ins()
.load(types::I32, MemFlagsData::new(), context, 0);
let cleanup_block = builder.create_block();
let continuation = builder.create_block();
builder
.ins()
.brif(error_code, cleanup_block, &[], continuation, &[]);
builder.switch_to_block(cleanup_block);
for action in &cleanup.drops {
lower_conditional_cleanup_drop(
builder,
body,
functions,
imports,
action,
values,
variables,
stack_slots,
type_metadata,
pointer_type,
context,
checked_script_functions,
)?;
}
builder.ins().jump(failure, &[]);
builder.switch_to_block(continuation);
builder.seal_block(cleanup_block);
builder.seal_block(continuation);
Ok(())
}
fn value_of(values: &BTreeMap<IrValueId, Value>, id: IrValueId) -> Result<Value, CodegenError> {
values
.get(&id)
.copied()
.ok_or(CodegenError::MissingValue(id))
}
fn addressable_locals(body: &TypedBody) -> BTreeSet<LocalId> {
let mut result = BTreeSet::new();
for place in &body.places {
if !place.projections.is_empty()
&& !matches!(
place.projections.first(),
Some(PlaceProjection::Dereference)
)
{
result.insert(place_local(place));
}
}
for block in &body.blocks {
for instruction in &block.instructions {
let address_place = match &instruction.kind {
TypedInstructionKind::Borrow { place, .. }
| TypedInstructionKind::Drop { place, .. } => Some(*place),
_ => None,
};
if let Some(place) = address_place
&& let Some(place) = body.places.get(place.0 as usize)
&& !matches!(
place.projections.first(),
Some(PlaceProjection::Dereference)
)
{
result.insert(place_local(place));
}
}
}
result
}
fn place_local(place: &crate::semantic::TypedPlace) -> LocalId {
match place.base {
PlaceBase::Local(local) => local,
}
}
fn place_address(
builder: &mut FunctionBuilder<'_>,
body: &TypedBody,
place: PlaceId,
values: &BTreeMap<IrValueId, Value>,
variables: &BTreeMap<LocalId, cranelift::prelude::Variable>,
stack_slots: &BTreeMap<LocalId, cranelift::codegen::ir::StackSlot>,
type_metadata: Option<&TypeMetadataRegistry>,
pointer_type: Type,
) -> Result<Value, CodegenError> {
let place = body
.places
.get(place.0 as usize)
.ok_or(CodegenError::UnsupportedInstruction("unknown place"))?;
let local = place_local(place);
let local_type = &body
.locals
.get(local.0 as usize)
.ok_or(CodegenError::UnsupportedInstruction("unknown local"))?
.type_;
let mut type_ = local_type.clone();
let mut projection_index = 0usize;
let mut address = if matches!(
place.projections.first(),
Some(PlaceProjection::Dereference)
) {
let variable =
variables
.get(&local)
.copied()
.ok_or(CodegenError::UnsupportedInstruction(
"reference place has no pointer value",
))?;
let (SemanticType::Reference { target, .. } | SemanticType::Pointer { target, .. }) =
&type_
else {
return Err(CodegenError::UnsupportedInstruction(
"dereference of non-pointer place",
));
};
type_ = target.as_ref().clone();
projection_index = 1;
builder.use_var(variable)
} else {
let slot = stack_slots
.get(&local)
.copied()
.ok_or(CodegenError::UnsupportedInstruction(
"place has no stack slot",
))?;
builder.ins().stack_addr(pointer_type, slot, 0)
};
let registry = type_metadata.ok_or_else(|| CodegenError::MissingTypeMetadata(type_.clone()))?;
for projection in &place.projections[projection_index..] {
match projection {
PlaceProjection::Dereference => {
let (SemanticType::Reference { target, .. } | SemanticType::Pointer { target, .. }) =
&type_
else {
return Err(CodegenError::UnsupportedInstruction(
"dereference of non-pointer place",
));
};
address = builder
.ins()
.load(pointer_type, MemFlagsData::new(), address, 0);
type_ = target.as_ref().clone();
}
PlaceProjection::Field(name) => {
let SemanticType::Named(named) = &type_ else {
return Err(CodegenError::MissingTypeMetadata(type_));
};
let metadata = registry
.get_named(named)
.ok_or_else(|| CodegenError::MissingTypeMetadata(type_.clone()))?;
let TypeMetadataKind::Structure { fields } = &metadata.kind else {
return Err(CodegenError::MissingTypeMetadata(type_));
};
let field = fields.iter().find(|field| field.name == *name).ok_or(
CodegenError::UnsupportedInstruction("unknown metadata field"),
)?;
if field.offset != 0 {
let offset = i64::try_from(field.offset)
.map_err(|_| CodegenError::InvalidStackLayout(local_type.clone()))?;
address = builder.ins().iadd_imm_u(address, offset);
}
type_ = field.type_.clone();
}
PlaceProjection::EnumTag => {
let SemanticType::Named(named) = &type_ else {
return Err(CodegenError::MissingTypeMetadata(type_));
};
let metadata = registry
.get_named(named)
.ok_or_else(|| CodegenError::MissingTypeMetadata(type_.clone()))?;
let TypeMetadataKind::Enum { tag_offset, .. } = &metadata.kind else {
return Err(CodegenError::MissingTypeMetadata(type_));
};
if *tag_offset != 0 {
let offset = i64::try_from(*tag_offset)
.map_err(|_| CodegenError::InvalidStackLayout(local_type.clone()))?;
address = builder.ins().iadd_imm_u(address, offset);
}
type_ = place.type_.clone();
}
PlaceProjection::EnumPayload(index) => {
let SemanticType::Named(named) = &type_ else {
return Err(CodegenError::MissingTypeMetadata(type_));
};
let metadata = registry
.get_named(named)
.ok_or_else(|| CodegenError::MissingTypeMetadata(type_.clone()))?;
let TypeMetadataKind::Enum { variants, .. } = &metadata.kind else {
return Err(CodegenError::MissingTypeMetadata(type_));
};
let variant = variants
.iter()
.find(|variant| variant.index == *index)
.ok_or(CodegenError::UnsupportedInstruction("unknown enum variant"))?;
let offset = variant
.payload_offset
.ok_or(CodegenError::UnsupportedInstruction(
"unit enum variant has no payload",
))?;
if offset != 0 {
let offset = i64::try_from(offset)
.map_err(|_| CodegenError::InvalidStackLayout(local_type.clone()))?;
address = builder.ins().iadd_imm_u(address, offset);
}
type_ =
variant
.payload_type
.clone()
.ok_or(CodegenError::UnsupportedInstruction(
"enum variant has no payload type",
))?;
}
PlaceProjection::TupleField(index) => {
let SemanticType::Tuple(items) = &type_ else {
return Err(CodegenError::MissingTypeMetadata(type_));
};
let item_type =
items
.get(*index)
.cloned()
.ok_or(CodegenError::UnsupportedInstruction(
"tuple index out of bounds",
))?;
let mut offset = 0usize;
for preceding in &items[..*index] {
let (size, align) = registry
.layout_of(preceding)
.ok_or_else(|| CodegenError::MissingTypeMetadata(preceding.clone()))?;
offset = align_offset(offset, align)
.checked_add(size)
.ok_or_else(|| CodegenError::InvalidStackLayout(local_type.clone()))?;
}
let (_, item_align) = registry
.layout_of(&item_type)
.ok_or_else(|| CodegenError::MissingTypeMetadata(item_type.clone()))?;
offset = align_offset(offset, item_align);
if offset != 0 {
let offset = i64::try_from(offset)
.map_err(|_| CodegenError::InvalidStackLayout(local_type.clone()))?;
address = builder.ins().iadd_imm_u(address, offset);
}
type_ = item_type;
}
PlaceProjection::ConstantIndex(index) => {
let SemanticType::Array {
item,
size: Some(length),
} = &type_
else {
return Err(CodegenError::UnsupportedInstruction(
"indexing non-fixed array",
));
};
if *index >= *length {
return Err(CodegenError::UnsupportedInstruction(
"constant array index out of bounds",
));
}
let item_type = item.as_ref().clone();
let (item_size, item_align) = registry
.layout_of(&item_type)
.ok_or_else(|| CodegenError::MissingTypeMetadata(item_type.clone()))?;
let stride = align_offset(item_size, item_align);
let offset = index
.checked_mul(stride)
.ok_or_else(|| CodegenError::InvalidStackLayout(local_type.clone()))?;
if offset != 0 {
let offset = i64::try_from(offset)
.map_err(|_| CodegenError::InvalidStackLayout(local_type.clone()))?;
address = builder.ins().iadd_imm_u(address, offset);
}
type_ = item_type;
}
PlaceProjection::Index(index) => {
let SemanticType::Array {
item,
size: Some(_),
} = &type_
else {
return Err(CodegenError::UnsupportedInstruction(
"indexing non-fixed array",
));
};
let item_type = item.as_ref().clone();
let (item_size, item_align) = registry
.layout_of(&item_type)
.ok_or_else(|| CodegenError::MissingTypeMetadata(item_type.clone()))?;
let stride = align_offset(item_size, item_align);
let stride = i64::try_from(stride)
.map_err(|_| CodegenError::InvalidStackLayout(local_type.clone()))?;
let index = value_of(values, *index)?;
let stride = builder.ins().iconst(pointer_type, stride);
let byte_offset = builder.ins().imul(index, stride);
address = builder.ins().iadd(address, byte_offset);
type_ = item_type;
}
}
}
Ok(address)
}
fn align_offset(value: usize, align: usize) -> usize {
debug_assert!(align.is_power_of_two());
(value + align - 1) & !(align - 1)
}
fn clif_type(type_: &SemanticType, pointer_type: Type) -> Result<Type, CodegenError> {
match type_ {
SemanticType::Named(named) => match named.path.local.as_str() {
"bool" | "i8" | "u8" => Ok(types::I8),
"i16" | "u16" => Ok(types::I16),
"i32" | "u32" | "char" => Ok(types::I32),
"i64" | "u64" => Ok(types::I64),
"isize" | "usize" => Ok(pointer_type),
"f32" => Ok(types::F32),
"f64" => Ok(types::F64),
_ => Err(CodegenError::UnsupportedType(type_.clone())),
},
SemanticType::Reference { .. } | SemanticType::Pointer { .. } => Ok(pointer_type),
_ => Err(CodegenError::UnsupportedType(type_.clone())),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::semantic::{
CanonicalSymbolPath, PlaceBase, PlaceId, TypedBasicBlock, TypedInstruction, TypedLocal,
TypedPlace, TypedValue,
};
fn i32_type() -> SemanticType {
SemanticType::Named(crate::semantic::SemanticNamedType {
path: CanonicalSymbolPath {
module: "core://primitive.lt".into(),
local: "i32".into(),
},
generic_arguments: Vec::new(),
})
}
#[test]
fn lowers_pointer_sized_types_for_the_target() {
let reference = SemanticType::Reference {
lifetime: None,
mutable: false,
target: Box::new(i32_type()),
};
let usize_type = SemanticType::Named(crate::semantic::SemanticNamedType {
path: CanonicalSymbolPath {
module: "core://primitive.lt".into(),
local: "usize".into(),
},
generic_arguments: Vec::new(),
});
assert_eq!(clif_type(&reference, types::I32).unwrap(), types::I32);
assert_eq!(clif_type(&usize_type, types::I32).unwrap(), types::I32);
assert_eq!(clif_type(&reference, types::I64).unwrap(), types::I64);
assert_eq!(clif_type(&usize_type, types::I64).unwrap(), types::I64);
}
#[test]
fn checked_signature_appends_one_hidden_execution_context_pointer() {
let target = cranelift::codegen::isa::TargetFrontendConfig {
default_call_conv: cranelift::codegen::isa::CallConv::SystemV,
pointer_width: target_lexicon::PointerWidth::U64,
page_size_align_log2: 12,
};
let signature = SemanticFunctionType {
receiver: None,
arguments: vec![crate::semantic::SemanticFunctionArgument {
name: "value".into(),
mutable: false,
type_: i32_type(),
}],
return_type: Box::new(i32_type()),
};
let direct =
clif_signature_with_execution_mode(&signature, target, None, ExecutionMode::Direct)
.unwrap();
let checked =
clif_signature_with_execution_mode(&signature, target, None, ExecutionMode::Checked)
.unwrap();
assert_eq!(direct.params.len(), 1);
assert_eq!(checked.params.len(), 2);
assert_eq!(checked.params[0], direct.params[0]);
assert_eq!(checked.params[1].value_type, types::I64);
assert_eq!(checked.returns, direct.returns);
}
#[test]
fn lowers_constant_local_and_return_to_cranelift_ir() {
let type_ = i32_type();
let value = TypedValue {
id: IrValueId(0),
type_: type_.clone(),
storage_place: None,
borrow_origins: Vec::new(),
};
let body = TypedBody {
locals: vec![TypedLocal {
id: LocalId(0),
name: "value".into(),
type_: type_.clone(),
mutable: true,
parameter: false,
}],
places: vec![TypedPlace {
id: PlaceId(0),
base: PlaceBase::Local(LocalId(0)),
projections: Vec::new(),
type_: type_.clone(),
mutable: true,
}],
entry: BasicBlockId(0),
blocks: vec![TypedBasicBlock {
id: BasicBlockId(0),
parameters: Vec::new(),
instructions: vec![
TypedInstruction {
span: crate::parsing::Span::new(0, 1),
result: Some(value.clone()),
kind: TypedInstructionKind::Constant(TypedConstant::Signed(42)),
},
TypedInstruction {
span: crate::parsing::Span::new(0, 1),
result: None,
kind: TypedInstructionKind::Store {
place: PlaceId(0),
value: value.id,
},
},
],
terminator: TypedTerminator::Return(Some(value.id)),
}],
return_type: type_.clone(),
return_cleanup: crate::semantic::ReturnCleanupPlan::default(),
};
let signature = SemanticFunctionType {
receiver: None,
arguments: Vec::new(),
return_type: Box::new(type_),
};
let function = lower_typed_body(&body, &signature, &FunctionRegistry::default()).unwrap();
let flags =
cranelift::codegen::settings::Flags::new(cranelift::codegen::settings::builder());
cranelift::codegen::verify_function(&function, &flags).unwrap();
let text = function.display().to_string();
assert!(text.contains("iconst.i32 42"));
assert!(text.contains("return"));
}
}