use super::*;
use cranelift_codegen::ir::{AbiParam, InstBuilder};
use cranelift_codegen::ir::types::{I64, F64};
use cranelift_codegen::settings::{self, Configurable};
use cranelift_codegen::Context;
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
use cranelift_jit::{JITBuilder, JITModule};
use cranelift_module::{default_libcall_names, Module, Linkage, FuncId};
use std::collections::HashMap;
pub(crate) struct JitFunction {
_module: JITModule,
func_ptr: *const u8,
param_count: usize,
}
unsafe impl Send for JitFunction {}
#[allow(dead_code)]
struct HelperFuncs {
add: FuncId,
sub: FuncId,
mul: FuncId,
div: FuncId,
eq: FuncId,
ne: FuncId,
gt: FuncId,
lt: FuncId,
ge: FuncId,
le: FuncId,
not: FuncId,
neg: FuncId,
truthy: FuncId,
wrapok: FuncId,
wraperr: FuncId,
isok: FuncId,
iserr: FuncId,
unwrap: FuncId,
jit_move: FuncId,
drop_rc: FuncId,
len: FuncId,
str_fn: FuncId,
num: FuncId,
abs: FuncId,
min: FuncId,
max: FuncId,
flr: FuncId,
cel: FuncId,
rou: FuncId,
rnd0: FuncId,
rnd2: FuncId,
now: FuncId,
env: FuncId,
get: FuncId,
spl: FuncId,
cat: FuncId,
has: FuncId,
hd: FuncId,
tl: FuncId,
rev: FuncId,
srt: FuncId,
slc: FuncId,
listappend: FuncId,
index: FuncId,
recfld: FuncId,
recnew: FuncId,
recwith: FuncId,
listnew: FuncId,
listget: FuncId,
jpth: FuncId,
jdmp: FuncId,
jpar: FuncId,
call: FuncId,
}
fn declare_helper(module: &mut JITModule, name: &str, n_params: usize, n_returns: usize) -> FuncId {
let mut sig = module.make_signature();
for _ in 0..n_params {
sig.params.push(AbiParam::new(I64));
}
for _ in 0..n_returns {
sig.returns.push(AbiParam::new(I64));
}
module.declare_function(name, Linkage::Import, &sig).unwrap()
}
fn register_helpers(builder: &mut JITBuilder) {
let helpers: &[(&str, *const u8)] = &[
("jit_add", jit_add as *const u8),
("jit_sub", jit_sub as *const u8),
("jit_mul", jit_mul as *const u8),
("jit_div", jit_div as *const u8),
("jit_eq", jit_eq as *const u8),
("jit_ne", jit_ne as *const u8),
("jit_gt", jit_gt as *const u8),
("jit_lt", jit_lt as *const u8),
("jit_ge", jit_ge as *const u8),
("jit_le", jit_le as *const u8),
("jit_not", jit_not as *const u8),
("jit_neg", jit_neg as *const u8),
("jit_truthy", jit_truthy as *const u8),
("jit_wrapok", jit_wrapok as *const u8),
("jit_wraperr", jit_wraperr as *const u8),
("jit_isok", jit_isok as *const u8),
("jit_iserr", jit_iserr as *const u8),
("jit_unwrap", jit_unwrap as *const u8),
("jit_move", jit_move as *const u8),
("jit_drop_rc", jit_drop_rc as *const u8),
("jit_len", jit_len as *const u8),
("jit_str", jit_str as *const u8),
("jit_num", jit_num as *const u8),
("jit_abs", jit_abs as *const u8),
("jit_min", jit_min as *const u8),
("jit_max", jit_max as *const u8),
("jit_flr", jit_flr as *const u8),
("jit_cel", jit_cel as *const u8),
("jit_rou", jit_rou as *const u8),
("jit_rnd0", jit_rnd0 as *const u8),
("jit_rnd2", jit_rnd2 as *const u8),
("jit_now", jit_now as *const u8),
("jit_env", jit_env as *const u8),
("jit_get", jit_get as *const u8),
("jit_spl", jit_spl as *const u8),
("jit_cat", jit_cat as *const u8),
("jit_has", jit_has as *const u8),
("jit_hd", jit_hd as *const u8),
("jit_tl", jit_tl as *const u8),
("jit_rev", jit_rev as *const u8),
("jit_srt", jit_srt as *const u8),
("jit_slc", jit_slc as *const u8),
("jit_listappend", jit_listappend as *const u8),
("jit_index", jit_index as *const u8),
("jit_recfld", jit_recfld as *const u8),
("jit_recnew", jit_recnew as *const u8),
("jit_recwith", jit_recwith as *const u8),
("jit_listnew", jit_listnew as *const u8),
("jit_listget", jit_listget as *const u8),
("jit_jpth", jit_jpth as *const u8),
("jit_jdmp", jit_jdmp as *const u8),
("jit_jpar", jit_jpar as *const u8),
("jit_call", jit_call as *const u8),
];
for &(name, ptr) in helpers {
builder.symbol(name, ptr);
}
}
fn declare_all_helpers(module: &mut JITModule) -> HelperFuncs {
HelperFuncs {
add: declare_helper(module, "jit_add", 2, 1),
sub: declare_helper(module, "jit_sub", 2, 1),
mul: declare_helper(module, "jit_mul", 2, 1),
div: declare_helper(module, "jit_div", 2, 1),
eq: declare_helper(module, "jit_eq", 2, 1),
ne: declare_helper(module, "jit_ne", 2, 1),
gt: declare_helper(module, "jit_gt", 2, 1),
lt: declare_helper(module, "jit_lt", 2, 1),
ge: declare_helper(module, "jit_ge", 2, 1),
le: declare_helper(module, "jit_le", 2, 1),
not: declare_helper(module, "jit_not", 1, 1),
neg: declare_helper(module, "jit_neg", 1, 1),
truthy: declare_helper(module, "jit_truthy", 1, 1),
wrapok: declare_helper(module, "jit_wrapok", 1, 1),
wraperr: declare_helper(module, "jit_wraperr", 1, 1),
isok: declare_helper(module, "jit_isok", 1, 1),
iserr: declare_helper(module, "jit_iserr", 1, 1),
unwrap: declare_helper(module, "jit_unwrap", 1, 1),
jit_move: declare_helper(module, "jit_move", 1, 1),
drop_rc: declare_helper(module, "jit_drop_rc", 1, 0),
len: declare_helper(module, "jit_len", 1, 1),
str_fn: declare_helper(module, "jit_str", 1, 1),
num: declare_helper(module, "jit_num", 1, 1),
abs: declare_helper(module, "jit_abs", 1, 1),
min: declare_helper(module, "jit_min", 2, 1),
max: declare_helper(module, "jit_max", 2, 1),
flr: declare_helper(module, "jit_flr", 1, 1),
cel: declare_helper(module, "jit_cel", 1, 1),
rou: declare_helper(module, "jit_rou", 1, 1),
rnd0: declare_helper(module, "jit_rnd0", 0, 1),
rnd2: declare_helper(module, "jit_rnd2", 2, 1),
now: declare_helper(module, "jit_now", 0, 1),
env: declare_helper(module, "jit_env", 1, 1),
get: declare_helper(module, "jit_get", 1, 1),
spl: declare_helper(module, "jit_spl", 2, 1),
cat: declare_helper(module, "jit_cat", 2, 1),
has: declare_helper(module, "jit_has", 2, 1),
hd: declare_helper(module, "jit_hd", 1, 1),
tl: declare_helper(module, "jit_tl", 1, 1),
rev: declare_helper(module, "jit_rev", 1, 1),
srt: declare_helper(module, "jit_srt", 1, 1),
slc: declare_helper(module, "jit_slc", 3, 1),
listappend: declare_helper(module, "jit_listappend", 2, 1),
index: declare_helper(module, "jit_index", 2, 1),
recfld: declare_helper(module, "jit_recfld", 2, 1),
recnew: declare_helper(module, "jit_recnew", 4, 1),
recwith: declare_helper(module, "jit_recwith", 4, 1),
listnew: declare_helper(module, "jit_listnew", 2, 1),
listget: declare_helper(module, "jit_listget", 2, 1),
jpth: declare_helper(module, "jit_jpth", 2, 1),
jdmp: declare_helper(module, "jit_jdmp", 1, 1),
jpar: declare_helper(module, "jit_jpar", 1, 1),
call: declare_helper(module, "jit_call", 4, 1),
}
}
pub(crate) fn compile(chunk: &Chunk, nan_consts: &[NanVal], program: &CompiledProgram) -> Option<JitFunction> {
let mut flag_builder = settings::builder();
flag_builder.set("opt_level", "speed").ok()?;
let isa_builder = cranelift_native::builder().ok()?;
let isa = isa_builder.finish(settings::Flags::new(flag_builder)).ok()?;
let mut jit_builder = JITBuilder::with_isa(isa, default_libcall_names());
register_helpers(&mut jit_builder);
let mut module = JITModule::new(jit_builder);
let helpers = declare_all_helpers(&mut module);
let mut sig = module.make_signature();
for _ in 0..chunk.param_count {
sig.params.push(AbiParam::new(I64));
}
sig.returns.push(AbiParam::new(I64));
let func_id = module.declare_function("jit_func", Linkage::Local, &sig).ok()?;
let mut ctx = Context::new();
ctx.func.signature = sig;
let mut fn_builder_ctx = FunctionBuilderContext::new();
let mut builder = FunctionBuilder::new(&mut ctx.func, &mut fn_builder_ctx);
let reg_count = chunk.reg_count.max(chunk.param_count) as usize;
let mut vars: Vec<Variable> = Vec::with_capacity(reg_count);
for i in 0..reg_count {
let var = Variable::from_u32(i as u32);
builder.declare_var(var, I64);
vars.push(var);
}
let leaders = find_block_leaders(&chunk.code);
let mut block_map: HashMap<usize, cranelift_codegen::ir::Block> = HashMap::new();
for &leader in &leaders {
let block = builder.create_block();
block_map.insert(leader, block);
}
let entry_block = block_map[&0];
builder.append_block_params_for_function_params(entry_block);
builder.switch_to_block(entry_block);
for (i, var) in vars.iter().enumerate().take(chunk.param_count as usize) {
let val = builder.block_params(entry_block)[i];
builder.def_var(*var, val);
}
let nil_bits = TAG_NIL;
for var in vars.iter().take(reg_count).skip(chunk.param_count as usize) {
let zero = builder.ins().iconst(I64, nil_bits as i64);
builder.def_var(*var, zero);
}
let mut func_refs: HashMap<FuncId, cranelift_codegen::ir::FuncRef> = HashMap::new();
let mut get_func_ref = |builder: &mut FunctionBuilder, module: &mut JITModule, id: FuncId| -> cranelift_codegen::ir::FuncRef {
*func_refs.entry(id).or_insert_with(|| module.declare_func_in_func(id, builder.func))
};
let program_ptr_val = program as *const CompiledProgram as u64;
let mut block_terminated = false;
for (ip, &inst) in chunk.code.iter().enumerate() {
if ip > 0 && block_map.contains_key(&ip) {
let block = block_map[&ip];
if !block_terminated {
builder.ins().jump(block, &[]);
}
builder.switch_to_block(block);
block_terminated = false;
}
if block_terminated {
continue;
}
let op = (inst >> 24) as u8;
let a_idx = ((inst >> 16) & 0xFF) as usize;
let b_idx = ((inst >> 8) & 0xFF) as usize;
let c_idx = (inst & 0xFF) as usize;
match op {
OP_ADD_NN => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let bf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), bv);
let cf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), cv);
let result_f = builder.ins().fadd(bf, cf);
let result = builder.ins().bitcast(I64, cranelift_codegen::ir::MemFlags::new(), result_f);
builder.def_var(vars[a_idx], result);
}
OP_SUB_NN => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let bf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), bv);
let cf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), cv);
let result_f = builder.ins().fsub(bf, cf);
let result = builder.ins().bitcast(I64, cranelift_codegen::ir::MemFlags::new(), result_f);
builder.def_var(vars[a_idx], result);
}
OP_MUL_NN => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let bf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), bv);
let cf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), cv);
let result_f = builder.ins().fmul(bf, cf);
let result = builder.ins().bitcast(I64, cranelift_codegen::ir::MemFlags::new(), result_f);
builder.def_var(vars[a_idx], result);
}
OP_DIV_NN => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let bf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), bv);
let cf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), cv);
let result_f = builder.ins().fdiv(bf, cf);
let result = builder.ins().bitcast(I64, cranelift_codegen::ir::MemFlags::new(), result_f);
builder.def_var(vars[a_idx], result);
}
OP_ADDK_N => {
let bv = builder.use_var(vars[b_idx]);
let kv = nan_consts.get(c_idx)?.as_number();
let bf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), bv);
let kval = builder.ins().f64const(kv);
let result_f = builder.ins().fadd(bf, kval);
let result = builder.ins().bitcast(I64, cranelift_codegen::ir::MemFlags::new(), result_f);
builder.def_var(vars[a_idx], result);
}
OP_SUBK_N => {
let bv = builder.use_var(vars[b_idx]);
let kv = nan_consts.get(c_idx)?.as_number();
let bf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), bv);
let kval = builder.ins().f64const(kv);
let result_f = builder.ins().fsub(bf, kval);
let result = builder.ins().bitcast(I64, cranelift_codegen::ir::MemFlags::new(), result_f);
builder.def_var(vars[a_idx], result);
}
OP_MULK_N => {
let bv = builder.use_var(vars[b_idx]);
let kv = nan_consts.get(c_idx)?.as_number();
let bf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), bv);
let kval = builder.ins().f64const(kv);
let result_f = builder.ins().fmul(bf, kval);
let result = builder.ins().bitcast(I64, cranelift_codegen::ir::MemFlags::new(), result_f);
builder.def_var(vars[a_idx], result);
}
OP_DIVK_N => {
let bv = builder.use_var(vars[b_idx]);
let kv = nan_consts.get(c_idx)?.as_number();
let bf = builder.ins().bitcast(F64, cranelift_codegen::ir::MemFlags::new(), bv);
let kval = builder.ins().f64const(kv);
let result_f = builder.ins().fdiv(bf, kval);
let result = builder.ins().bitcast(I64, cranelift_codegen::ir::MemFlags::new(), result_f);
builder.def_var(vars[a_idx], result);
}
OP_ADD | OP_SUB | OP_MUL | OP_DIV => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let qnan_val = builder.ins().iconst(I64, QNAN as i64);
let b_masked = builder.ins().band(bv, qnan_val);
let c_masked = builder.ins().band(cv, qnan_val);
let b_or_c = builder.ins().bor(b_masked, c_masked);
let both_num = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::NotEqual, b_or_c, qnan_val);
let num_block = builder.create_block();
let slow_block = builder.create_block();
let merge_block = builder.create_block();
builder.append_block_param(merge_block, I64);
builder.ins().brif(both_num, num_block, &[], slow_block, &[]);
builder.switch_to_block(num_block);
let mf = cranelift_codegen::ir::MemFlags::new();
let bf = builder.ins().bitcast(F64, mf, bv);
let cf = builder.ins().bitcast(F64, mf, cv);
let result_f = match op {
OP_ADD => builder.ins().fadd(bf, cf),
OP_SUB => builder.ins().fsub(bf, cf),
OP_MUL => builder.ins().fmul(bf, cf),
OP_DIV => builder.ins().fdiv(bf, cf),
_ => unreachable!(),
};
let fast_result = builder.ins().bitcast(I64, mf, result_f);
builder.ins().jump(merge_block, &[fast_result]);
builder.switch_to_block(slow_block);
let helper = match op {
OP_ADD => helpers.add,
OP_SUB => helpers.sub,
OP_MUL => helpers.mul,
OP_DIV => helpers.div,
_ => unreachable!(),
};
let fref = get_func_ref(&mut builder, &mut module, helper);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let slow_result = builder.inst_results(call_inst)[0];
builder.ins().jump(merge_block, &[slow_result]);
builder.switch_to_block(merge_block);
let result = builder.block_params(merge_block)[0];
builder.def_var(vars[a_idx], result);
}
OP_LT | OP_GT | OP_LE | OP_GE | OP_EQ | OP_NE => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let qnan_val = builder.ins().iconst(I64, QNAN as i64);
let b_masked = builder.ins().band(bv, qnan_val);
let c_masked = builder.ins().band(cv, qnan_val);
let b_or_c = builder.ins().bor(b_masked, c_masked);
let both_num = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::NotEqual, b_or_c, qnan_val);
let num_block = builder.create_block();
let slow_block = builder.create_block();
let merge_block = builder.create_block();
builder.append_block_param(merge_block, I64);
builder.ins().brif(both_num, num_block, &[], slow_block, &[]);
builder.switch_to_block(num_block);
let mf = cranelift_codegen::ir::MemFlags::new();
let bf = builder.ins().bitcast(F64, mf, bv);
let cf = builder.ins().bitcast(F64, mf, cv);
use cranelift_codegen::ir::condcodes::FloatCC;
let cc = match op {
OP_LT => FloatCC::LessThan,
OP_GT => FloatCC::GreaterThan,
OP_LE => FloatCC::LessThanOrEqual,
OP_GE => FloatCC::GreaterThanOrEqual,
OP_EQ => FloatCC::Equal,
OP_NE => FloatCC::NotEqual,
_ => unreachable!(),
};
let cmp = builder.ins().fcmp(cc, bf, cf);
let true_val = builder.ins().iconst(I64, TAG_TRUE as i64);
let false_val = builder.ins().iconst(I64, TAG_FALSE as i64);
let fast_result = builder.ins().select(cmp, true_val, false_val);
builder.ins().jump(merge_block, &[fast_result]);
builder.switch_to_block(slow_block);
let helper = match op {
OP_LT => helpers.lt,
OP_GT => helpers.gt,
OP_LE => helpers.le,
OP_GE => helpers.ge,
OP_EQ => helpers.eq,
OP_NE => helpers.ne,
_ => unreachable!(),
};
let fref = get_func_ref(&mut builder, &mut module, helper);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let slow_result = builder.inst_results(call_inst)[0];
builder.ins().jump(merge_block, &[slow_result]);
builder.switch_to_block(merge_block);
let result = builder.block_params(merge_block)[0];
builder.def_var(vars[a_idx], result);
}
OP_MOVE => {
if a_idx != b_idx {
let bv = builder.use_var(vars[b_idx]);
let qnan_val = builder.ins().iconst(I64, QNAN as i64);
let masked = builder.ins().band(bv, qnan_val);
let is_heap = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::Equal, masked, qnan_val);
let clone_block = builder.create_block();
let after_block = builder.create_block();
builder.ins().brif(is_heap, clone_block, &[], after_block, &[]);
builder.switch_to_block(clone_block);
let fref = get_func_ref(&mut builder, &mut module, helpers.jit_move);
builder.ins().call(fref, &[bv]);
builder.ins().jump(after_block, &[]);
builder.switch_to_block(after_block);
builder.def_var(vars[a_idx], bv);
}
}
OP_NOT => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.not);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_NEG => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.neg);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_WRAPOK => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.wrapok);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_WRAPERR => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.wraperr);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_ISOK => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.isok);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_ISERR => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.iserr);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_UNWRAP => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.unwrap);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_LOADK => {
let bx = (inst & 0xFFFF) as usize;
let bits = nan_consts.get(bx)?.0;
let kval = builder.ins().iconst(I64, bits as i64);
let nv = NanVal(bits);
if nv.is_heap() {
let fref = get_func_ref(&mut builder, &mut module, helpers.jit_move);
let call_inst = builder.ins().call(fref, &[kval]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
} else {
builder.def_var(vars[a_idx], kval);
}
}
OP_JMP => {
let sbx = (inst & 0xFFFF) as i16;
let target = (ip as isize + 1 + sbx as isize) as usize;
if let Some(&target_block) = block_map.get(&target) {
builder.ins().jump(target_block, &[]);
block_terminated = true;
}
}
OP_JMPF | OP_JMPT => {
let sbx = (inst & 0xFFFF) as i16;
let target = (ip as isize + 1 + sbx as isize) as usize;
let fallthrough = ip + 1;
let av = builder.use_var(vars[a_idx]);
let qnan_val = builder.ins().iconst(I64, QNAN as i64);
let masked = builder.ins().band(av, qnan_val);
let is_num = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::NotEqual, masked, qnan_val);
let num_truthy_block = builder.create_block();
let tag_truthy_block = builder.create_block();
let merge_truthy = builder.create_block();
builder.append_block_param(merge_truthy, I64);
builder.ins().brif(is_num, num_truthy_block, &[], tag_truthy_block, &[]);
builder.switch_to_block(num_truthy_block);
let mf = cranelift_codegen::ir::MemFlags::new();
let af = builder.ins().bitcast(F64, mf, av);
let zero = builder.ins().f64const(0.0);
let cmp = builder.ins().fcmp(cranelift_codegen::ir::condcodes::FloatCC::NotEqual, af, zero);
let num_result = builder.ins().uextend(I64, cmp);
builder.ins().jump(merge_truthy, &[num_result]);
builder.switch_to_block(tag_truthy_block);
let nil_val = builder.ins().iconst(I64, TAG_NIL as i64);
let false_val = builder.ins().iconst(I64, TAG_FALSE as i64);
let not_nil = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::NotEqual, av, nil_val);
let not_false = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::NotEqual, av, false_val);
let tag_truthy = builder.ins().band(not_nil, not_false);
let tag_result = builder.ins().uextend(I64, tag_truthy);
builder.ins().jump(merge_truthy, &[tag_result]);
builder.switch_to_block(merge_truthy);
let truthy_val = builder.block_params(merge_truthy)[0];
if let (Some(&target_block), Some(&fall_block)) = (block_map.get(&target), block_map.get(&fallthrough)) {
if op == OP_JMPF {
builder.ins().brif(truthy_val, fall_block, &[], target_block, &[]);
} else {
builder.ins().brif(truthy_val, target_block, &[], fall_block, &[]);
}
block_terminated = true;
}
}
OP_JMPNN => {
let sbx = (inst & 0xFFFF) as i16;
let target = (ip as isize + 1 + sbx as isize) as usize;
let fallthrough = ip + 1;
let av = builder.use_var(vars[a_idx]);
let nil_const = builder.ins().iconst(I64, TAG_NIL as i64);
let is_nil = builder.ins().icmp(cranelift_codegen::ir::condcodes::IntCC::Equal, av, nil_const);
if let (Some(&target_block), Some(&fall_block)) = (block_map.get(&target), block_map.get(&fallthrough)) {
builder.ins().brif(is_nil, fall_block, &[], target_block, &[]);
block_terminated = true;
}
}
OP_RET => {
let av = builder.use_var(vars[a_idx]);
builder.ins().return_(&[av]);
block_terminated = true;
}
OP_LEN => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.len);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_STR => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.str_fn);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_NUM => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.num);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_ABS => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.abs);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_MIN => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.min);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_MAX => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.max);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_FLR => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.flr);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_CEL => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.cel);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_ROU => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.rou);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_RND0 => {
let fref = get_func_ref(&mut builder, &mut module, helpers.rnd0);
let call_inst = builder.ins().call(fref, &[]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_RND2 => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.rnd2);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_NOW => {
let fref = get_func_ref(&mut builder, &mut module, helpers.now);
let call_inst = builder.ins().call(fref, &[]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_ENV => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.env);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_GET => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.get);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_SPL => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.spl);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_CAT => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.cat);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_HAS => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.has);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_HD => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.hd);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_TL => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.tl);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_REV => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.rev);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_SRT => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.srt);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_SLC => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let dv = builder.use_var(vars[c_idx + 1]);
let fref = get_func_ref(&mut builder, &mut module, helpers.slc);
let call_inst = builder.ins().call(fref, &[bv, cv, dv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_LISTAPPEND => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.listappend);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_INDEX => {
let bv = builder.use_var(vars[b_idx]);
let idx_val = builder.ins().iconst(I64, c_idx as i64);
let fref = get_func_ref(&mut builder, &mut module, helpers.index);
let call_inst = builder.ins().call(fref, &[bv, idx_val]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_RECFLD => {
let bv = builder.use_var(vars[b_idx]);
let tag_mask_val = builder.ins().iconst(I64, TAG_MASK as i64);
let tag = builder.ins().band(bv, tag_mask_val);
let arena_tag_val = builder.ins().iconst(I64, TAG_ARENA_REC as i64);
let is_arena = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::Equal, tag, arena_tag_val);
let arena_block = builder.create_block();
let heap_block = builder.create_block();
let merge_block = builder.create_block();
builder.append_block_param(merge_block, I64);
builder.ins().brif(is_arena, arena_block, &[], heap_block, &[]);
builder.switch_to_block(arena_block);
let ptr_mask_val = builder.ins().iconst(I64, PTR_MASK as i64);
let ptr = builder.ins().band(bv, ptr_mask_val);
let field_offset = builder.ins().iconst(I64, (8 + c_idx * 8) as i64);
let field_addr = builder.ins().iadd(ptr, field_offset);
let field_val = builder.ins().load(I64, cranelift_codegen::ir::MemFlags::trusted(), field_addr, 0);
let qnan_val = builder.ins().iconst(I64, QNAN as i64);
let masked = builder.ins().band(field_val, qnan_val);
let is_nan_tagged = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::Equal, masked, qnan_val);
let clone_block = builder.create_block();
let skip_clone_block = builder.create_block();
builder.ins().brif(is_nan_tagged, clone_block, &[], skip_clone_block, &[]);
builder.switch_to_block(clone_block);
let fref_move = get_func_ref(&mut builder, &mut module, helpers.jit_move);
let move_inst = builder.ins().call(fref_move, &[field_val]);
let _cloned = builder.inst_results(move_inst)[0];
builder.ins().jump(skip_clone_block, &[]);
builder.switch_to_block(skip_clone_block);
builder.ins().jump(merge_block, &[field_val]);
builder.switch_to_block(heap_block);
let field_idx_val = builder.ins().iconst(I64, c_idx as i64);
let fref = get_func_ref(&mut builder, &mut module, helpers.recfld);
let call_inst = builder.ins().call(fref, &[bv, field_idx_val]);
let heap_result = builder.inst_results(call_inst)[0];
builder.ins().jump(merge_block, &[heap_result]);
builder.switch_to_block(merge_block);
let result = builder.block_params(merge_block)[0];
builder.def_var(vars[a_idx], result);
}
OP_RECFLD_NAME => {
return None;
}
OP_RECNEW => {
let bx = (inst & 0xFFFF) as usize;
let type_id = (bx >> 8) as u16;
let n_fields = bx & 0xFF;
let record_size = 8 + n_fields * 8;
let arena_ptr = jit_arena_ptr();
let arena_ptr_val = builder.ins().iconst(I64, arena_ptr as i64);
let cur_offset = builder.ins().load(I64,
cranelift_codegen::ir::MemFlags::trusted(), arena_ptr_val, 16);
let seven = builder.ins().iconst(I64, 7);
let off_plus_7 = builder.ins().iadd(cur_offset, seven);
let neg8 = builder.ins().iconst(I64, !7i64);
let aligned = builder.ins().band(off_plus_7, neg8);
let size_val = builder.ins().iconst(I64, record_size as i64);
let new_offset = builder.ins().iadd(aligned, size_val);
let buf_cap = builder.ins().load(I64,
cranelift_codegen::ir::MemFlags::trusted(), arena_ptr_val, 8);
let has_space = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::UnsignedLessThanOrEqual,
new_offset, buf_cap);
let alloc_block = builder.create_block();
let fallback_block = builder.create_block();
let merge_block = builder.create_block();
builder.append_block_param(merge_block, I64);
builder.ins().brif(has_space, alloc_block, &[], fallback_block, &[]);
builder.switch_to_block(alloc_block);
let buf_ptr = builder.ins().load(I64,
cranelift_codegen::ir::MemFlags::trusted(), arena_ptr_val, 0);
let rec_ptr = builder.ins().iadd(buf_ptr, aligned);
let header = ((n_fields as u64) << 16) | (type_id as u64);
let header_val = builder.ins().iconst(I64, header as i64);
builder.ins().store(cranelift_codegen::ir::MemFlags::trusted(),
header_val, rec_ptr, 0);
for i in 0..n_fields {
let field_v = builder.use_var(vars[a_idx + 1 + i]);
let field_off = (8 + i * 8) as i32;
builder.ins().store(cranelift_codegen::ir::MemFlags::trusted(),
field_v, rec_ptr, field_off);
let qnan_val = builder.ins().iconst(I64, QNAN as i64);
let masked = builder.ins().band(field_v, qnan_val);
let is_heap = builder.ins().icmp(
cranelift_codegen::ir::condcodes::IntCC::Equal, masked, qnan_val);
let do_clone = builder.create_block();
let after_clone = builder.create_block();
builder.ins().brif(is_heap, do_clone, &[], after_clone, &[]);
builder.switch_to_block(do_clone);
let fref_move = get_func_ref(&mut builder, &mut module, helpers.jit_move);
builder.ins().call(fref_move, &[field_v]);
builder.ins().jump(after_clone, &[]);
builder.switch_to_block(after_clone);
}
builder.ins().store(cranelift_codegen::ir::MemFlags::trusted(),
new_offset, arena_ptr_val, 16);
let tag_val = builder.ins().iconst(I64, TAG_ARENA_REC as i64);
let result_val = builder.ins().bor(rec_ptr, tag_val);
builder.ins().jump(merge_block, &[result_val]);
builder.switch_to_block(fallback_block);
let slot = builder.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
(n_fields * 8) as u32,
0,
));
for i in 0..n_fields {
let v = builder.use_var(vars[a_idx + 1 + i]);
builder.ins().stack_store(v, slot, (i * 8) as i32);
}
let regs_ptr = builder.ins().stack_addr(I64, slot, 0);
let type_id_and_nfields = ((type_id as u64) << 16) | (n_fields as u64);
let type_id_nfields_val = builder.ins().iconst(I64, type_id_and_nfields as i64);
let registry_ptr_val = builder.ins().iconst(I64, &program.type_registry as *const TypeRegistry as i64);
let fref = get_func_ref(&mut builder, &mut module, helpers.recnew);
let call_inst = builder.ins().call(fref, &[arena_ptr_val, type_id_nfields_val, regs_ptr, registry_ptr_val]);
let fb_result = builder.inst_results(call_inst)[0];
builder.ins().jump(merge_block, &[fb_result]);
builder.switch_to_block(merge_block);
let result = builder.block_params(merge_block)[0];
builder.def_var(vars[a_idx], result);
}
OP_RECWITH => {
let bx = (inst & 0xFFFF) as usize;
let indices_idx = bx >> 8;
let n_updates = bx & 0xFF;
let update_indices: Vec<u8> = match &chunk.constants[indices_idx] {
Value::List(items) => items.iter().map(|v| match v {
Value::Number(n) => *n as u8,
_ => 0,
}).collect(),
_ => return None,
};
let indices_bytes: &'static [u8] = Box::leak(update_indices.into_boxed_slice());
let old_rec = builder.use_var(vars[a_idx]);
let slot = builder.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
(n_updates * 8) as u32,
0,
));
for i in 0..n_updates {
let v = builder.use_var(vars[a_idx + 1 + i]);
builder.ins().stack_store(v, slot, (i * 8) as i32);
}
let regs_ptr = builder.ins().stack_addr(I64, slot, 0);
let indices_ptr_val = builder.ins().iconst(I64, indices_bytes.as_ptr() as i64);
let n_updates_val = builder.ins().iconst(I64, n_updates as i64);
let fref = get_func_ref(&mut builder, &mut module, helpers.recwith);
let call_inst = builder.ins().call(fref, &[old_rec, indices_ptr_val, n_updates_val, regs_ptr]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_LISTNEW => {
let n = (inst & 0xFFFF) as usize;
if n == 0 {
let null_ptr = builder.ins().iconst(I64, 0i64);
let n_val = builder.ins().iconst(I64, 0i64);
let fref = get_func_ref(&mut builder, &mut module, helpers.listnew);
let call_inst = builder.ins().call(fref, &[null_ptr, n_val]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
} else {
let slot = builder.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
(n * 8) as u32,
0,
));
for i in 0..n {
let v = builder.use_var(vars[a_idx + 1 + i]);
builder.ins().stack_store(v, slot, (i * 8) as i32);
}
let regs_ptr = builder.ins().stack_addr(I64, slot, 0);
let n_val = builder.ins().iconst(I64, n as i64);
let fref = get_func_ref(&mut builder, &mut module, helpers.listnew);
let call_inst = builder.ins().call(fref, &[regs_ptr, n_val]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
}
OP_LISTGET => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.listget);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let result = builder.inst_results(call_inst)[0];
let nil_const = builder.ins().iconst(I64, TAG_NIL as i64);
let is_nil = builder.ins().icmp(cranelift_codegen::ir::condcodes::IntCC::Equal, result, nil_const);
let jmp_block = block_map.get(&(ip + 1)).copied();
let body_block = block_map.get(&(ip + 2)).copied();
if let (Some(jb), Some(bb)) = (jmp_block, body_block) {
let unwrap_block = builder.create_block();
builder.ins().brif(is_nil, jb, &[], unwrap_block, &[]);
builder.switch_to_block(unwrap_block);
builder.seal_block(unwrap_block);
let fref2 = get_func_ref(&mut builder, &mut module, helpers.unwrap);
let call_inst2 = builder.ins().call(fref2, &[result]);
let item = builder.inst_results(call_inst2)[0];
let fref_drop = get_func_ref(&mut builder, &mut module, helpers.drop_rc);
builder.ins().call(fref_drop, &[result]);
builder.def_var(vars[a_idx], item);
builder.ins().jump(bb, &[]);
block_terminated = true;
} else {
builder.def_var(vars[a_idx], result);
}
}
OP_CALL => {
let a = ((inst >> 16) & 0xFF) as u8;
let bx = (inst & 0xFFFF) as usize;
let func_idx = (bx >> 8) as u16;
let n_args = bx & 0xFF;
if n_args > 0 {
let slot = builder.create_sized_stack_slot(cranelift_codegen::ir::StackSlotData::new(
cranelift_codegen::ir::StackSlotKind::ExplicitSlot,
(n_args * 8) as u32,
0,
));
for i in 0..n_args {
let v = builder.use_var(vars[a as usize + 1 + i]);
builder.ins().stack_store(v, slot, (i * 8) as i32);
}
let args_ptr = builder.ins().stack_addr(I64, slot, 0);
let prog_ptr = builder.ins().iconst(I64, program_ptr_val as i64);
let func_idx_val = builder.ins().iconst(I64, func_idx as i64);
let n_args_val = builder.ins().iconst(I64, n_args as i64);
let fref = get_func_ref(&mut builder, &mut module, helpers.call);
let call_inst = builder.ins().call(fref, &[prog_ptr, func_idx_val, args_ptr, n_args_val]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a as usize], result);
} else {
let null_ptr = builder.ins().iconst(I64, 0i64);
let prog_ptr = builder.ins().iconst(I64, program_ptr_val as i64);
let func_idx_val = builder.ins().iconst(I64, func_idx as i64);
let n_args_val = builder.ins().iconst(I64, 0i64);
let fref = get_func_ref(&mut builder, &mut module, helpers.call);
let call_inst = builder.ins().call(fref, &[prog_ptr, func_idx_val, null_ptr, n_args_val]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a as usize], result);
}
}
OP_JPTH => {
let bv = builder.use_var(vars[b_idx]);
let cv = builder.use_var(vars[c_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.jpth);
let call_inst = builder.ins().call(fref, &[bv, cv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_JDMP => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.jdmp);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
OP_JPAR => {
let bv = builder.use_var(vars[b_idx]);
let fref = get_func_ref(&mut builder, &mut module, helpers.jpar);
let call_inst = builder.ins().call(fref, &[bv]);
let result = builder.inst_results(call_inst)[0];
builder.def_var(vars[a_idx], result);
}
_ => {
return None;
}
}
}
if !block_terminated {
let nil = builder.ins().iconst(I64, TAG_NIL as i64);
builder.ins().return_(&[nil]);
}
builder.seal_all_blocks();
builder.finalize();
module.define_function(func_id, &mut ctx).ok()?;
module.finalize_definitions().ok()?;
let func_ptr = module.get_finalized_function(func_id);
Some(JitFunction {
_module: module,
func_ptr,
param_count: chunk.param_count as usize,
})
}
fn call_raw(func: &JitFunction, args: &[u64]) -> Option<u64> {
if args.len() != func.param_count { return None; }
Some(match args.len() {
0 => {
let f: extern "C" fn() -> u64 = unsafe { std::mem::transmute(func.func_ptr) };
f()
}
1 => {
let f: extern "C" fn(u64) -> u64 = unsafe { std::mem::transmute(func.func_ptr) };
f(args[0])
}
2 => {
let f: extern "C" fn(u64, u64) -> u64 = unsafe { std::mem::transmute(func.func_ptr) };
f(args[0], args[1])
}
3 => {
let f: extern "C" fn(u64, u64, u64) -> u64 = unsafe { std::mem::transmute(func.func_ptr) };
f(args[0], args[1], args[2])
}
4 => {
let f: extern "C" fn(u64, u64, u64, u64) -> u64 = unsafe { std::mem::transmute(func.func_ptr) };
f(args[0], args[1], args[2], args[3])
}
5 => {
let f: extern "C" fn(u64, u64, u64, u64, u64) -> u64 = unsafe { std::mem::transmute(func.func_ptr) };
f(args[0], args[1], args[2], args[3], args[4])
}
6 => {
let f: extern "C" fn(u64, u64, u64, u64, u64, u64) -> u64 = unsafe { std::mem::transmute(func.func_ptr) };
f(args[0], args[1], args[2], args[3], args[4], args[5])
}
7 => {
let f: extern "C" fn(u64, u64, u64, u64, u64, u64, u64) -> u64 = unsafe { std::mem::transmute(func.func_ptr) };
f(args[0], args[1], args[2], args[3], args[4], args[5], args[6])
}
8 => {
let f: extern "C" fn(u64, u64, u64, u64, u64, u64, u64, u64) -> u64 = unsafe { std::mem::transmute(func.func_ptr) };
f(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7])
}
_ => return None,
})
}
pub(crate) fn call(func: &JitFunction, args: &[u64]) -> Option<u64> {
let mut result = call_raw(func, args)?;
let rv = NanVal(result);
if rv.is_arena_record() {
let registry_ptr = ACTIVE_REGISTRY.with(|r| r.get());
if !registry_ptr.is_null() {
let promoted = rv.promote_arena_to_heap(unsafe { &*registry_ptr });
result = promoted.0;
}
}
jit_arena_reset();
Some(result)
}
pub(crate) fn compile_and_call(chunk: &Chunk, nan_consts: &[NanVal], args: &[u64], program: &CompiledProgram) -> Option<u64> {
with_active_registry(program, || {
let func = compile(chunk, nan_consts, program)?;
call(&func, args)
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::lexer;
use crate::parser;
fn jit_run(source: &str, func_name: &str, args: &[Value]) -> Option<Value> {
let tokens: Vec<crate::lexer::Token> = lexer::lex(source)
.unwrap()
.into_iter()
.map(|(t, _)| t)
.collect();
let prog = parser::parse_tokens(tokens).unwrap();
let compiled = crate::vm::compile(&prog).unwrap();
let idx = compiled.func_names.iter().position(|n| n == func_name)?;
let chunk = &compiled.chunks[idx];
let nan_consts = &compiled.nan_constants[idx];
let nan_args: Vec<u64> = args.iter().map(|v| NanVal::from_value(v).0).collect();
let result = compile_and_call(chunk, nan_consts, &nan_args, &compiled)?;
Some(NanVal(result).to_value())
}
fn jit_run_numeric(source: &str, func_name: &str, args: &[f64]) -> Option<f64> {
let val_args: Vec<Value> = args.iter().map(|n| Value::Number(*n)).collect();
match jit_run(source, func_name, &val_args)? {
Value::Number(n) => Some(n),
_ => None,
}
}
#[test]
fn cranelift_sub_nn() {
let result = jit_run_numeric("f a:n b:n>n;-a b", "f", &[10.0, 3.0]);
assert_eq!(result, Some(7.0));
}
#[test]
fn cranelift_div_nn() {
let result = jit_run_numeric("f a:n b:n>n;/a b", "f", &[10.0, 2.0]);
assert_eq!(result, Some(5.0));
}
#[test]
fn cranelift_subk_n() {
let result = jit_run_numeric("f x:n>n;-x 3", "f", &[10.0]);
assert_eq!(result, Some(7.0));
}
#[test]
fn cranelift_divk_n() {
let result = jit_run_numeric("f x:n>n;/x 4", "f", &[20.0]);
assert_eq!(result, Some(5.0));
}
#[test]
fn cranelift_neg() {
let result = jit_run_numeric("f x:n>n;-x", "f", &[5.0]);
assert_eq!(result, Some(-5.0));
}
#[test]
fn cranelift_zero_arg_function() {
let result = jit_run_numeric("f>n;42", "f", &[]);
assert_eq!(result, Some(42.0));
}
#[test]
fn cranelift_add_k_n() {
let result = jit_run_numeric("f x:n>n;+x 10", "f", &[5.0]);
assert_eq!(result, Some(15.0));
}
#[test]
fn cranelift_move_op() {
let result = jit_run_numeric("f x:n>n;x", "f", &[7.0]);
assert_eq!(result, Some(7.0));
}
#[test]
fn cranelift_arg_count_mismatch() {
let result = jit_run_numeric("f x:n y:n>n;+x y", "f", &[1.0]);
assert_eq!(result, None);
}
#[test]
fn cranelift_move_a_ne_b() {
let result = jit_run_numeric("f x:n>n;y=x;y", "f", &[7.0]);
assert_eq!(result, Some(7.0));
}
#[test]
fn cranelift_4_args() {
let result = jit_run_numeric("f a:n b:n c:n d:n>n;+a +b +c d", "f", &[1.0, 2.0, 3.0, 4.0]);
assert_eq!(result, Some(10.0));
}
#[test]
fn cranelift_5_args() {
let result = jit_run_numeric("f a:n b:n c:n d:n e:n>n;+a +b +c +d e", "f", &[1.0, 2.0, 3.0, 4.0, 5.0]);
assert_eq!(result, Some(15.0));
}
#[test]
fn cranelift_6_args() {
let result = jit_run_numeric("f a:n b:n c:n d:n e:n f0:n>n;+a +b +c +d +e f0", "f", &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
assert_eq!(result, Some(21.0));
}
#[test]
fn cranelift_7_args() {
let result = jit_run_numeric("f a:n b:n c:n d:n e:n f0:n g0:n>n;+a +b +c +d +e +f0 g0", "f", &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]);
assert_eq!(result, Some(28.0));
}
#[test]
fn cranelift_8_args() {
let result = jit_run_numeric("f a:n b:n c:n d:n e:n f0:n g0:n h:n>n;+a +b +c +d +e +f0 +g0 h", "f", &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]);
assert_eq!(result, Some(36.0));
}
#[test]
fn cranelift_9_args_hits_fallback() {
let tokens: Vec<crate::lexer::Token> = crate::lexer::lex(
"f a:n b:n c:n d:n e:n f0:n g0:n h:n i:n>n;+a +b +c +d +e +f0 +g0 +h i"
).unwrap().into_iter().map(|(t, _)| t).collect();
let prog = crate::parser::parse_tokens(tokens).unwrap();
let compiled = crate::vm::compile(&prog).unwrap();
let idx = compiled.func_names.iter().position(|n| n == "f").unwrap();
let chunk = &compiled.chunks[idx];
let nan_consts = &compiled.nan_constants[idx];
if let Some(func) = compile(chunk, nan_consts, &compiled) {
let args: Vec<u64> = (1..=9).map(|i| NanVal::number(i as f64).0).collect();
let result = call(&func, &args);
assert_eq!(result, None);
}
}
#[test]
fn cranelift_string_concat() {
let result = jit_run(r#"f a:t b:t>t;+ a b"#, "f", &[Value::Text("hello".into()), Value::Text(" world".into())]);
assert_eq!(result, Some(Value::Text("hello world".into())));
}
#[test]
fn cranelift_string_constant() {
let result = jit_run(r#"f>t;"hello""#, "f", &[]);
assert_eq!(result, Some(Value::Text("hello".into())));
}
#[test]
fn cranelift_bool_true() {
let result = jit_run("f>b;true", "f", &[]);
assert_eq!(result, Some(Value::Bool(true)));
}
#[test]
fn cranelift_bool_false() {
let result = jit_run("f>b;false", "f", &[]);
assert_eq!(result, Some(Value::Bool(false)));
}
#[test]
fn cranelift_equality() {
let result = jit_run("f a:n b:n>b;= a b", "f", &[Value::Number(5.0), Value::Number(5.0)]);
assert_eq!(result, Some(Value::Bool(true)));
}
#[test]
fn cranelift_inequality() {
let result = jit_run("f a:n b:n>b;!= a b", "f", &[Value::Number(5.0), Value::Number(3.0)]);
assert_eq!(result, Some(Value::Bool(true)));
}
#[test]
fn cranelift_guard_ternary() {
let result = jit_run("f x:n>n;>x 0{x}{0}", "f", &[Value::Number(5.0)]);
assert_eq!(result, Some(Value::Number(5.0)));
let result2 = jit_run("f x:n>n;>x 0{x}{0}", "f", &[Value::Number(-1.0)]);
assert_eq!(result2, Some(Value::Number(0.0)));
}
#[test]
fn cranelift_wrapok() {
let result = jit_run("f x:n>R n t;~x", "f", &[Value::Number(42.0)]);
assert_eq!(result, Some(Value::Ok(Box::new(Value::Number(42.0)))));
}
#[test]
fn cranelift_wraperr() {
let result = jit_run(r#"f x:t>R n t;^"bad""#, "f", &[Value::Text("bad".into())]);
assert_eq!(result, Some(Value::Err(Box::new(Value::Text("bad".into())))));
}
#[test]
fn cranelift_len_string() {
let result = jit_run(r#"f s:t>n;len s"#, "f", &[Value::Text("hello".into())]);
assert_eq!(result, Some(Value::Number(5.0)));
}
#[test]
fn cranelift_len_list() {
let result = jit_run("f xs:L n>n;len xs", "f", &[Value::List(vec![Value::Number(1.0), Value::Number(2.0), Value::Number(3.0)])]);
assert_eq!(result, Some(Value::Number(3.0)));
}
#[test]
fn cranelift_not() {
let result = jit_run("f x:b>b;! x", "f", &[Value::Bool(true)]);
assert_eq!(result, Some(Value::Bool(false)));
}
#[test]
fn cranelift_comparison_gt() {
let result = jit_run("f a:n b:n>b;> a b", "f", &[Value::Number(5.0), Value::Number(3.0)]);
assert_eq!(result, Some(Value::Bool(true)));
}
#[test]
fn cranelift_comparison_lt() {
let result = jit_run("f a:n b:n>b;< a b", "f", &[Value::Number(3.0), Value::Number(5.0)]);
assert_eq!(result, Some(Value::Bool(true)));
}
#[test]
fn cranelift_while_loop() {
let result = jit_run("f n:n>n;s=0;i=1;wh <= i n{s=+s i;i=+i 1};s", "f", &[Value::Number(10.0)]);
assert_eq!(result, Some(Value::Number(55.0)));
}
#[test]
fn cranelift_str_builtin() {
let result = jit_run("f x:n>t;str x", "f", &[Value::Number(42.0)]);
assert_eq!(result, Some(Value::Text("42".into())));
}
#[test]
fn cranelift_abs_builtin() {
let result = jit_run("f x:n>n;abs x", "f", &[Value::Number(-5.0)]);
assert_eq!(result, Some(Value::Number(5.0)));
}
#[test]
fn cranelift_function_call() {
let result = jit_run("double x:n>n;* x 2\nf x:n>n;double x", "f", &[Value::Number(5.0)]);
assert_eq!(result, Some(Value::Number(10.0)));
}
#[test]
fn cranelift_num_builtin() {
let result = jit_run(r#"f s:t>n;r=num s;?r{~v:v;^_:0}"#, "f", &[Value::Text("3.14".into())]);
assert_eq!(result, Some(Value::Number(3.14)));
}
#[test]
fn cranelift_flr_builtin() {
let result = jit_run("f x:n>n;flr x", "f", &[Value::Number(4.7)]);
assert_eq!(result, Some(Value::Number(4.0)));
}
#[test]
fn cranelift_cel_builtin() {
let result = jit_run("f x:n>n;cel x", "f", &[Value::Number(4.1)]);
assert_eq!(result, Some(Value::Number(5.0)));
}
#[test]
fn cranelift_min_builtin() {
let result = jit_run("f a:n b:n>n;min a b", "f", &[Value::Number(3.0), Value::Number(7.0)]);
assert_eq!(result, Some(Value::Number(3.0)));
}
#[test]
fn cranelift_max_builtin() {
let result = jit_run("f a:n b:n>n;max a b", "f", &[Value::Number(3.0), Value::Number(7.0)]);
assert_eq!(result, Some(Value::Number(7.0)));
}
#[test]
fn cranelift_rnd0_returns_number() {
let result = jit_run("f>n;rnd", "f", &[]);
assert!(matches!(result, Some(Value::Number(_))));
}
#[test]
fn cranelift_rnd2_range_returns_number() {
let result = jit_run("f>n;rnd 1 10", "f", &[]);
assert!(matches!(result, Some(Value::Number(_))));
}
#[test]
fn cranelift_env_builtin() {
unsafe { std::env::set_var("ILO_JIT_TEST_VAR", "hello"); }
let result = jit_run(r#"f k:t>R t t;env k"#, "f", &[Value::Text("ILO_JIT_TEST_VAR".into())]);
assert_eq!(result, Some(Value::Ok(Box::new(Value::Text("hello".into())))));
}
#[test]
fn cranelift_spl_builtin() {
let result = jit_run(r#"f s:t sep:t>L t;spl s sep"#, "f", &[
Value::Text("a,b,c".into()),
Value::Text(",".into()),
]);
assert_eq!(result, Some(Value::List(vec![
Value::Text("a".into()),
Value::Text("b".into()),
Value::Text("c".into()),
])));
}
#[test]
fn cranelift_cat_builtin() {
let result = jit_run(r#"f xs:L t sep:t>t;cat xs sep"#, "f", &[
Value::List(vec![Value::Text("x".into()), Value::Text("y".into())]),
Value::Text("-".into()),
]);
assert_eq!(result, Some(Value::Text("x-y".into())));
}
#[test]
fn cranelift_has_list() {
let result = jit_run("f xs:L n v:n>b;has xs v", "f", &[
Value::List(vec![Value::Number(1.0), Value::Number(2.0), Value::Number(3.0)]),
Value::Number(2.0),
]);
assert_eq!(result, Some(Value::Bool(true)));
}
#[test]
fn cranelift_hd_builtin() {
let result = jit_run("f xs:L n>n;hd xs", "f", &[
Value::List(vec![Value::Number(10.0), Value::Number(20.0)]),
]);
assert_eq!(result, Some(Value::Number(10.0)));
}
#[test]
fn cranelift_tl_builtin() {
let result = jit_run("f xs:L n>L n;tl xs", "f", &[
Value::List(vec![Value::Number(1.0), Value::Number(2.0), Value::Number(3.0)]),
]);
assert_eq!(result, Some(Value::List(vec![Value::Number(2.0), Value::Number(3.0)])));
}
#[test]
fn cranelift_rev_builtin() {
let result = jit_run("f xs:L n>L n;rev xs", "f", &[
Value::List(vec![Value::Number(1.0), Value::Number(2.0), Value::Number(3.0)]),
]);
assert_eq!(result, Some(Value::List(vec![Value::Number(3.0), Value::Number(2.0), Value::Number(1.0)])));
}
#[test]
fn cranelift_srt_builtin() {
let result = jit_run("f xs:L n>L n;srt xs", "f", &[
Value::List(vec![Value::Number(3.0), Value::Number(1.0), Value::Number(2.0)]),
]);
assert_eq!(result, Some(Value::List(vec![Value::Number(1.0), Value::Number(2.0), Value::Number(3.0)])));
}
#[test]
fn cranelift_slc_builtin() {
let result = jit_run("f xs:L n a:n b:n>L n;slc xs a b", "f", &[
Value::List(vec![Value::Number(10.0), Value::Number(20.0), Value::Number(30.0), Value::Number(40.0)]),
Value::Number(1.0),
Value::Number(3.0),
]);
assert_eq!(result, Some(Value::List(vec![Value::Number(20.0), Value::Number(30.0)])));
}
#[test]
fn cranelift_listappend() {
let result = jit_run("f xs:L n v:n>L n;r=+=xs v;r", "f", &[
Value::List(vec![Value::Number(1.0), Value::Number(2.0)]),
Value::Number(3.0),
]);
assert_eq!(result, Some(Value::List(vec![Value::Number(1.0), Value::Number(2.0), Value::Number(3.0)])));
}
#[test]
fn cranelift_listnew() {
let result = jit_run("f a:n b:n>L n;[a, b]", "f", &[Value::Number(5.0), Value::Number(6.0)]);
assert_eq!(result, Some(Value::List(vec![Value::Number(5.0), Value::Number(6.0)])));
}
#[test]
fn cranelift_index_literal() {
let result = jit_run("f xs:L n>n;xs.0", "f", &[
Value::List(vec![Value::Number(10.0), Value::Number(20.0), Value::Number(30.0)]),
]);
assert_eq!(result, Some(Value::Number(10.0)));
}
#[test]
fn cranelift_recnew_and_field() {
let src = "type pt{x:n;y:n} f a:n b:n>n;p=pt x:a y:b;p.x";
let result = jit_run(src, "f", &[Value::Number(3.0), Value::Number(4.0)]);
assert_eq!(result, Some(Value::Number(3.0)));
}
#[test]
fn cranelift_recwith() {
let src = "type pt{x:n;y:n} f a:n b:n>n;p=pt x:a y:b;q=p with x:99;q.x";
let result = jit_run(src, "f", &[Value::Number(3.0), Value::Number(4.0)]);
assert_eq!(result, Some(Value::Number(99.0)));
}
#[test]
fn cranelift_jdmp_number() {
let result = jit_run("f x:n>t;jdmp x", "f", &[Value::Number(42.0)]);
assert_eq!(result, Some(Value::Text("42".into())));
}
#[test]
fn cranelift_jpar_ok() {
let result = jit_run(r#"f s:t>R t t;jpar s"#, "f", &[Value::Text(r#"{"k":"v"}"#.into())]);
assert!(matches!(result, Some(Value::Ok(_))));
}
#[test]
fn cranelift_jpth_ok() {
let result = jit_run(r#"f j:t p:t>R t t;jpth j p"#, "f", &[
Value::Text(r#"{"name":"alice"}"#.into()),
Value::Text("name".into()),
]);
assert_eq!(result, Some(Value::Ok(Box::new(Value::Text("alice".into())))));
}
#[test]
fn cranelift_isok_via_match() {
let result = jit_run("f x:R n t>n;?x{~v:v;^_:0}", "f", &[Value::Ok(Box::new(Value::Number(7.0)))]);
assert_eq!(result, Some(Value::Number(7.0)));
}
#[test]
fn cranelift_iserr_via_match() {
let result = jit_run(r#"f x:R n t>n;?x{~_:1;^_:99}"#, "f", &[Value::Err(Box::new(Value::Text("bad".into())))]);
assert_eq!(result, Some(Value::Number(99.0)));
}
#[test]
fn cranelift_unwrap_via_match() {
let src = "f x:R n t>n;?x{~v:v;^_:0}";
let result = jit_run(src, "f", &[Value::Ok(Box::new(Value::Number(42.0)))]);
assert_eq!(result, Some(Value::Number(42.0)));
}
#[test]
fn cranelift_now_returns_number() {
let result = jit_run("f>n;now", "f", &[]);
assert!(matches!(result, Some(Value::Number(_))));
}
#[test]
fn cranelift_nil_coalesce_with_value() {
let result = jit_run("f x:O n>n;x??42", "f", &[Value::Number(7.0)]);
assert_eq!(result, Some(Value::Number(7.0)));
}
#[test]
fn cranelift_nil_coalesce_with_nil() {
let result = jit_run("f x:O n>n;x??42", "f", &[Value::Nil]);
assert_eq!(result, Some(Value::Number(42.0)));
}
#[test]
fn cranelift_empty_list_literal() {
let result = jit_run("f>L n;[]", "f", &[]);
assert_eq!(result, Some(Value::List(vec![])));
}
#[test]
fn jit_run_numeric_non_number_returns_none() {
let result = jit_run_numeric("f>b;true", "f", &[]);
assert_eq!(result, None);
}
#[test]
fn cranelift_recfld_name_bails_out() {
use crate::vm::{compile as vm_compile, OP_RECFLD_NAME};
let tokens: Vec<crate::lexer::Token> = crate::lexer::lex(
"f x:n>n;x"
).unwrap().into_iter().map(|(t, _)| t).collect();
let prog = crate::parser::parse_tokens(tokens).unwrap();
let mut compiled = vm_compile(&prog).unwrap();
let idx = compiled.func_names.iter().position(|n| n == "f").unwrap();
compiled.chunks[idx].code.insert(0, (OP_RECFLD_NAME as u32) << 24);
let chunk = &compiled.chunks[idx];
let nan_consts = &compiled.nan_constants[idx];
let result = compile(chunk, nan_consts, &compiled);
assert!(result.is_none(), "JIT should bail on OP_RECFLD_NAME");
}
#[test]
fn cranelift_function_ends_without_explicit_terminator() {
let result = jit_run("f x:n>n;wh > x 0{x=-x};x", "f", &[Value::Number(5.0)]);
assert_eq!(result, Some(Value::Number(-5.0)));
}
#[test]
fn cranelift_rou_builtin() {
let result = jit_run("f x:n>n;rou x", "f", &[Value::Number(4.5)]);
assert_eq!(result, Some(Value::Number(5.0)));
}
#[test]
fn cranelift_rou_down() {
let result = jit_run("f x:n>n;rou x", "f", &[Value::Number(4.4)]);
assert_eq!(result, Some(Value::Number(4.0)));
}
#[test]
fn cranelift_call_zero_args_injected() {
use crate::vm::{compile as vm_compile, OP_CALL, OP_RET};
let tokens: Vec<crate::lexer::Token> = crate::lexer::lex(
"g>n;42\nf>n;42"
).unwrap().into_iter().map(|(t, _)| t).collect();
let prog = crate::parser::parse_tokens(tokens).unwrap();
let mut compiled = vm_compile(&prog).unwrap();
let f_idx = compiled.func_names.iter().position(|n| n == "f").unwrap();
let g_idx = compiled.func_names.iter().position(|n| n == "g").unwrap();
let call_inst = (OP_CALL as u32) << 24 | ((g_idx as u32) << 8);
let ret_inst = (OP_RET as u32) << 24;
compiled.chunks[f_idx].code = vec![call_inst, ret_inst];
let chunk = &compiled.chunks[f_idx];
let nan_consts = &compiled.nan_constants[f_idx];
let func = compile(chunk, nan_consts, &compiled);
if let Some(f) = func {
let result = call(&f, &[]);
assert_eq!(result, Some(NanVal::number(42.0).0));
}
}
#[test]
fn cranelift_record_return_promotes_arena() {
let src = "type pt{x:n;y:n} f a:n b:n>pt;pt x:a y:b";
let result = jit_run(src, "f", &[Value::Number(10.0), Value::Number(20.0)]);
match result {
Some(Value::Record { type_name, fields }) => {
assert_eq!(type_name, "pt");
assert_eq!(fields.get("x"), Some(&Value::Number(10.0)));
assert_eq!(fields.get("y"), Some(&Value::Number(20.0)));
}
other => panic!("expected Record, got {:?}", other),
}
}
#[test]
fn cranelift_recwith_update() {
let src = "type pt{x:n;y:n} f>pt;p=pt x:1 y:2;p with x:99";
let result = jit_run(src, "f", &[]);
match result {
Some(Value::Record { type_name, fields }) => {
assert_eq!(type_name, "pt");
assert_eq!(fields.get("x"), Some(&Value::Number(99.0)));
assert_eq!(fields.get("y"), Some(&Value::Number(2.0)));
}
other => panic!("expected Record, got {:?}", other),
}
}
#[test]
fn cranelift_foreach_loop() {
let result = jit_run("f xs:L n>n;s=0;@x xs{s=+s x};s", "f", &[
Value::List(vec![Value::Number(1.0), Value::Number(2.0), Value::Number(3.0)]),
]);
assert_eq!(result, Some(Value::Number(6.0)));
}
}