use super::*;
impl<'a> Emitter<'a> {
pub(super) fn load_reg(&mut self, idx: usize) -> Value {
if idx == reg::SR {
self.flush_pending_flags();
}
if let Some(var) = self.promoted.vars[idx] {
if !self.promoted.valid[idx] {
let scope = self.scope_stack.last_mut().unwrap();
if !scope.entry_valid[idx] {
scope.deferred[idx] = true;
} else {
let val = self.load_u32(Self::reg_offset(idx));
self.builder.def_var(var, val);
}
self.promoted.valid[idx] = true;
}
self.builder.use_var(var)
} else {
match idx {
reg::A0 | reg::A1 | reg::A2 | reg::B0 | reg::B1 | reg::B2 => {
self.load_acc_subreg(idx)
}
_ => self.load_u32(Self::reg_offset(idx)),
}
}
}
fn load_acc_subreg(&mut self, idx: usize) -> Value {
let (acc, extract): (Accumulator, fn(&mut Self, Value) -> Value) = match idx {
reg::A0 => (Accumulator::A, Self::extract_acc_lo),
reg::A1 => (Accumulator::A, Self::extract_acc_mid),
reg::A2 => (Accumulator::A, Self::extract_acc_hi),
reg::B0 => (Accumulator::B, Self::extract_acc_lo),
reg::B1 => (Accumulator::B, Self::extract_acc_mid),
reg::B2 => (Accumulator::B, Self::extract_acc_hi),
_ => unreachable!(),
};
let acc_val = self.load_acc(acc);
extract(self, acc_val)
}
pub(super) fn store_reg(&mut self, idx: usize, val: Value) {
if idx == reg::SR {
self.pending_flags = None;
}
let mask = REG_MASKS[idx];
let v = if mask != 0xFFFFFFFF {
let m = self.builder.ins().iconst(types::I32, mask as i64);
self.builder.ins().band(val, m)
} else {
val
};
if let Some(var) = self.promoted.vars[idx] {
self.builder.def_var(var, v);
self.promoted.valid[idx] = true;
self.promoted.dirty[idx] = true;
} else {
match idx {
reg::A0 | reg::A1 | reg::A2 | reg::B0 | reg::B1 | reg::B2 => {
self.store_acc_subreg(idx, v);
}
_ => self.store_u32(Self::reg_offset(idx), v),
}
}
}
fn store_acc_subreg(&mut self, idx: usize, val: Value) {
let (acc, shift, field_mask): (Accumulator, u32, u64) = match idx {
reg::A0 => (Accumulator::A, 0, 0x00FFFFFF),
reg::A1 => (Accumulator::A, 24, 0x00FFFFFF),
reg::A2 => (Accumulator::A, 48, 0xFF),
reg::B0 => (Accumulator::B, 0, 0x00FFFFFF),
reg::B1 => (Accumulator::B, 24, 0x00FFFFFF),
reg::B2 => (Accumulator::B, 48, 0xFF),
_ => unreachable!(),
};
let cur = self.load_acc(acc);
let clear_mask = !(field_mask << shift);
let clear = self.builder.ins().iconst(types::I64, clear_mask as i64);
let cleared = self.builder.ins().band(cur, clear);
let val64 = self.builder.ins().uextend(types::I64, val);
let positioned = if shift > 0 {
let fm = self.builder.ins().iconst(types::I64, field_mask as i64);
let masked = self.builder.ins().band(val64, fm);
let c = self.builder.ins().iconst(types::I32, shift as i64);
self.builder.ins().ishl(masked, c)
} else {
let fm = self.builder.ins().iconst(types::I64, field_mask as i64);
self.builder.ins().band(val64, fm)
};
let result = self.builder.ins().bor(cleared, positioned);
self.store_acc(acc, result);
}
pub(super) fn store_pc(&mut self, v: Value) {
let masked = self.mask24(v);
self.store_u32(OFF_PC, masked);
}
pub(super) fn flush_reg(&mut self, idx: usize) {
if self.promoted.dirty[idx] {
let val = self.builder.use_var(self.promoted.vars[idx].unwrap());
self.store_u32(Self::reg_offset(idx), val);
}
}
pub(super) fn flush_promoted(&mut self) {
for &idx in &PROMOTED_REGS {
self.flush_reg(idx);
self.promoted.dirty[idx] = false;
}
for acc in [Accumulator::A, Accumulator::B] {
let i = Self::acc_idx(acc);
if self.promoted.acc_dirty[i] {
self.flush_acc_to_memory(acc);
self.promoted.acc_dirty[i] = false;
}
}
}
pub(super) fn invalidate_promoted(&mut self) {
for &idx in &PROMOTED_REGS {
self.promoted.valid[idx] = false;
}
self.promoted.acc_valid = [false; 2];
self.promoted.dirty = [false; 64];
self.promoted.acc_dirty = [false; 2];
if let Some(scope) = self.scope_stack.last_mut() {
for &idx in &PROMOTED_REGS {
scope.entry_valid[idx] = true;
}
scope.entry_acc_valid = [true; 2];
}
}
pub(super) fn begin_conditional(&mut self) -> ConditionalState {
ConditionalState {
saved_dirty: self.promoted.dirty,
saved_acc_dirty: self.promoted.acc_dirty,
saved_valid: self.promoted.valid,
saved_acc_valid: self.promoted.acc_valid,
modified: [false; 64],
modified_acc: [false; 2],
}
}
pub(super) fn end_conditional_arm(&mut self, state: &mut ConditionalState) {
for &idx in &PROMOTED_REGS {
if self.promoted.dirty[idx] && !state.saved_dirty[idx] {
self.flush_reg(idx);
state.modified[idx] = true;
}
}
for i in 0..2 {
if self.promoted.acc_dirty[i] && !state.saved_acc_dirty[i] {
let acc = if i == 0 {
Accumulator::A
} else {
Accumulator::B
};
self.flush_acc_to_memory(acc);
state.modified_acc[i] = true;
}
}
self.promoted.dirty = state.saved_dirty;
self.promoted.acc_dirty = state.saved_acc_dirty;
self.promoted.valid = state.saved_valid;
self.promoted.acc_valid = state.saved_acc_valid;
}
pub(super) fn merge_conditional(&mut self, state: &ConditionalState) {
let scope = self.scope_stack.last_mut().unwrap();
for &idx in &PROMOTED_REGS {
if state.modified[idx] {
self.promoted.valid[idx] = false;
scope.entry_valid[idx] = true;
}
}
for i in 0..2 {
if state.modified_acc[i] {
self.promoted.acc_valid[i] = false;
scope.entry_acc_valid[i] = true;
}
}
}
pub(super) fn flush_sr(&mut self) {
self.flush_reg(reg::SR);
self.promoted.dirty[reg::SR] = false;
}
pub(super) fn flush_all_to_memory(&mut self) {
for &idx in &PROMOTED_REGS {
self.flush_reg(idx);
}
for acc in [Accumulator::A, Accumulator::B] {
if self.promoted.acc_dirty[Self::acc_idx(acc)] {
self.flush_acc_to_memory(acc);
}
}
let il = self.builder.use_var(self.inst_len);
self.store_u32(OFF_PC_ADVANCE, il);
}
pub(super) fn acc_regs(acc: Accumulator) -> (usize, usize, usize) {
match acc {
Accumulator::A => (reg::A2, reg::A1, reg::A0),
Accumulator::B => (reg::B2, reg::B1, reg::B0),
}
}
pub(super) fn acc_idx(acc: Accumulator) -> usize {
match acc {
Accumulator::A => 0,
Accumulator::B => 1,
}
}
pub(super) fn load_acc(&mut self, acc: Accumulator) -> Value {
let i = Self::acc_idx(acc);
if !self.promoted.acc_valid[i] {
let scope = self.scope_stack.last_mut().unwrap();
if !scope.entry_acc_valid[i] {
scope.acc_deferred[i] = true;
} else {
self.reload_acc(acc);
}
self.promoted.acc_valid[i] = true;
}
self.builder.use_var(self.promoted.acc[i])
}
pub(super) fn store_acc(&mut self, acc: Accumulator, val: Value) {
let i = Self::acc_idx(acc);
self.builder.def_var(self.promoted.acc[i], val);
self.promoted.acc_valid[i] = true;
self.promoted.acc_dirty[i] = true;
}
pub(super) fn pack_acc(&mut self, a2: Value, a1: Value, a0: Value) -> Value {
let v2 = self.builder.ins().uextend(types::I64, a2);
let v1 = self.builder.ins().uextend(types::I64, a1);
let v0 = self.builder.ins().uextend(types::I64, a0);
let c48 = self.builder.ins().iconst(types::I32, 48);
let c24 = self.builder.ins().iconst(types::I32, 24);
let hi = self.builder.ins().ishl(v2, c48);
let mid = self.builder.ins().ishl(v1, c24);
let tmp = self.builder.ins().bor(hi, mid);
self.builder.ins().bor(tmp, v0)
}
pub(super) fn extract_acc_lo(&mut self, acc_val: Value) -> Value {
let mask = self.builder.ins().iconst(types::I64, 0x00FFFFFF);
let lo = self.builder.ins().band(acc_val, mask);
self.builder.ins().ireduce(types::I32, lo)
}
pub(super) fn extract_acc_mid(&mut self, acc_val: Value) -> Value {
let c24 = self.builder.ins().iconst(types::I32, 24);
let shifted = self.builder.ins().ushr(acc_val, c24);
let mask = self.builder.ins().iconst(types::I64, 0x00FFFFFF);
let mid = self.builder.ins().band(shifted, mask);
self.builder.ins().ireduce(types::I32, mid)
}
pub(super) fn extract_acc_hi(&mut self, acc_val: Value) -> Value {
let c48 = self.builder.ins().iconst(types::I32, 48);
let shifted = self.builder.ins().ushr(acc_val, c48);
let mask = self.builder.ins().iconst(types::I64, 0xFF);
let hi = self.builder.ins().band(shifted, mask);
self.builder.ins().ireduce(types::I32, hi)
}
pub(super) fn flush_acc_to_memory(&mut self, acc: Accumulator) {
let (r2, r1, r0) = Self::acc_regs(acc);
let val = self.load_acc(acc);
let lo = self.extract_acc_lo(val);
let mid = self.extract_acc_mid(val);
let hi = self.extract_acc_hi(val);
self.store_u32(Self::reg_offset(r0), lo);
self.store_u32(Self::reg_offset(r1), mid);
self.store_u32(Self::reg_offset(r2), hi);
}
pub(super) fn reload_acc(&mut self, acc: Accumulator) {
let (r2, r1, r0) = Self::acc_regs(acc);
let v0 = self.load_u32(Self::reg_offset(r0));
let v1 = self.load_u32(Self::reg_offset(r1));
let v2 = self.load_u32(Self::reg_offset(r2));
let packed = self.pack_acc(v2, v1, v0);
let i = Self::acc_idx(acc);
self.builder.def_var(self.promoted.acc[i], packed);
}
pub(super) fn val24_to_acc56(&mut self, val32: Value) -> Value {
let val64 = self.builder.ins().uextend(types::I64, val32);
let c24 = self.builder.ins().iconst(types::I32, 24);
let shifted = self.builder.ins().ishl(val64, c24);
let c16 = self.builder.ins().iconst(types::I32, 16);
let shl = self.builder.ins().ishl(shifted, c16);
let sext = self.builder.ins().sshr(shl, c16);
self.mask56(sext)
}
pub(super) fn val24_to_acc56_unsigned(&mut self, val32: Value) -> Value {
let val64 = self.builder.ins().uextend(types::I64, val32);
let c24 = self.builder.ins().iconst(types::I32, 24);
let shifted = self.builder.ins().ishl(val64, c24);
self.mask56(shifted)
}
pub(super) fn mask56(&mut self, val: Value) -> Value {
let m = self
.builder
.ins()
.iconst(types::I64, 0x00FFFFFFFFFFFFFF_u64 as i64);
self.builder.ins().band(val, m)
}
pub(super) fn write_acc_lo_mid(&mut self, acc: Accumulator, mid_val: Value, lo_val: Value) {
let cur = self.load_acc(acc);
let a2_mask = self
.builder
.ins()
.iconst(types::I64, 0x00FF_0000_0000_0000u64 as i64);
let a2_preserved = self.builder.ins().band(cur, a2_mask);
let lx64 = self.builder.ins().uextend(types::I64, mid_val);
let ly64 = self.builder.ins().uextend(types::I64, lo_val);
let mask24 = self.builder.ins().iconst(types::I64, 0x00FFFFFF);
let lx_m = self.builder.ins().band(lx64, mask24);
let ly_m = self.builder.ins().band(ly64, mask24);
let c24 = self.builder.ins().iconst(types::I32, 24);
let mid = self.builder.ins().ishl(lx_m, c24);
let lo_mid = self.builder.ins().bor(mid, ly_m);
let result = self.builder.ins().bor(a2_preserved, lo_mid);
self.store_acc(acc, result);
}
pub(super) fn read_reg_for_move(&mut self, idx: usize) -> Value {
if idx == reg::A || idx == reg::B {
self.read_accu24(idx)
} else if idx == reg::SSH {
self.emit_call_extern_ret(jit_read_ssh as *const () as usize)
} else {
self.load_reg(idx)
}
}
pub(super) fn read_accu24(&mut self, idx: usize) -> Value {
let (val, _no_limit) = self.read_accu24_inner(idx);
val
}
pub(super) fn read_accu24_inner(&mut self, idx: usize) -> (Value, Value) {
let acc_idx = if idx == reg::A { 0u32 } else { 1u32 };
let raw = self.emit_call_read_accu24(acc_idx);
let result = self.mask24(raw);
let no_limit = self.builder.ins().ushr_imm(raw, 24);
(result, no_limit)
}
pub(super) fn write_reg_for_move(&mut self, idx: usize, val: Value) {
if idx == reg::A || idx == reg::B {
let acc = if idx == reg::A {
Accumulator::A
} else {
Accumulator::B
};
let packed = self.val24_to_acc56(val);
self.store_acc(acc, packed);
} else if idx == reg::SSH {
self.emit_call_extern_val(jit_write_ssh as *const () as usize, val);
} else if idx == reg::SSL {
self.emit_call_extern_val(jit_write_ssl as *const () as usize, val);
} else if idx == reg::SP {
self.emit_call_extern_val(jit_write_sp as *const () as usize, val);
} else {
self.store_reg(idx, val);
}
}
pub(super) fn load_reg24_as_acc56(&mut self, reg_idx: usize) -> Value {
let val32 = self.load_reg(reg_idx);
self.val24_to_acc56(val32)
}
pub(super) fn read_l_reg(&mut self, numreg: u32) -> (Value, Value) {
match numreg {
0 => {
(self.load_reg(reg::A1), self.load_reg(reg::A0))
}
1 => {
(self.load_reg(reg::B1), self.load_reg(reg::B0))
}
2 => {
(self.load_reg(reg::X1), self.load_reg(reg::X0))
}
3 => {
(self.load_reg(reg::Y1), self.load_reg(reg::Y0))
}
4 => self.read_l_reg_limited(reg::A, reg::A0),
5 => self.read_l_reg_limited(reg::B, reg::B0),
6 => {
let lx = self.read_accu24(reg::A);
let ly = self.read_accu24(reg::B);
(lx, ly)
}
7 => {
let lx = self.read_accu24(reg::B);
let ly = self.read_accu24(reg::A);
(lx, ly)
}
_ => unreachable!(),
}
}
fn read_l_reg_limited(&mut self, acc: usize, lo_reg: usize) -> (Value, Value) {
let (save_lx, no_limit) = self.read_accu24_inner(acc);
let c23 = self.builder.ins().iconst(types::I32, 23);
let bit23 = self.builder.ins().ushr(save_lx, c23);
let one = self.builder.ins().iconst(types::I32, 1);
let bit23 = self.builder.ins().band(bit23, one);
let is_neg = self.builder.ins().icmp_imm(IntCC::NotEqual, bit23, 0);
let zero = self.builder.ins().iconst(types::I32, 0);
let all_ones = self.builder.ins().iconst(types::I32, 0x00FF_FFFF);
let limited_ly = self.builder.ins().select(is_neg, zero, all_ones);
let lo = self.load_reg(lo_reg);
let save_ly = self.builder.ins().select(no_limit, lo, limited_ly);
(save_lx, save_ly)
}
pub(super) fn write_l_reg(&mut self, numreg: u32, lx: Value, ly: Value) {
match numreg {
0 => {
self.write_acc_lo_mid(Accumulator::A, lx, ly);
}
1 => {
self.write_acc_lo_mid(Accumulator::B, lx, ly);
}
2 => {
self.store_reg(reg::X1, lx);
self.store_reg(reg::X0, ly);
}
3 => {
self.store_reg(reg::Y1, lx);
self.store_reg(reg::Y0, ly);
}
4 | 5 => {
let acc = if numreg == 4 {
Accumulator::A
} else {
Accumulator::B
};
let ext = self.sign_ext_bit23(lx);
let packed = self.pack_acc(ext, lx, ly);
self.store_acc(acc, packed);
}
6 | 7 => {
let (first, second) = if numreg == 6 {
(Accumulator::A, Accumulator::B)
} else {
(Accumulator::B, Accumulator::A)
};
let first_packed = self.val24_to_acc56(lx);
self.store_acc(first, first_packed);
let second_packed = self.val24_to_acc56(ly);
self.store_acc(second, second_packed);
}
_ => unreachable!(),
}
}
pub(super) fn sign_ext_bit23(&mut self, val: Value) -> Value {
let c23 = self.builder.ins().iconst(types::I32, 23);
let sign = self.builder.ins().ushr(val, c23);
let one = self.builder.ins().iconst(types::I32, 1);
let sign = self.builder.ins().band(sign, one);
let zero = self.builder.ins().iconst(types::I32, 0);
let neg = self.builder.ins().isub(zero, sign);
let mask = self.builder.ins().iconst(types::I32, 0xFF);
self.builder.ins().band(neg, mask)
}
pub(super) fn sext24_to_i64(&mut self, val: Value) -> Value {
let c8 = self.builder.ins().iconst(types::I32, 8);
let shifted = self.builder.ins().ishl(val, c8);
let sext32 = self.builder.ins().sshr(shifted, c8);
self.builder.ins().sextend(types::I64, sext32)
}
pub(super) fn zext24_to_i64(&mut self, val: Value) -> Value {
let mask = self.builder.ins().iconst(types::I32, 0x00FF_FFFF);
let masked = self.builder.ins().band(val, mask);
self.builder.ins().uextend(types::I64, masked)
}
pub(super) fn load_xy_as_acc56(&mut self, hi_reg: usize, lo_reg: usize) -> Value {
let hi32 = self.load_reg(hi_reg);
let lo32 = self.load_reg(lo_reg);
let hi64 = self.builder.ins().uextend(types::I64, hi32);
let lo64 = self.builder.ins().uextend(types::I64, lo32);
let c24 = self.builder.ins().iconst(types::I32, 24);
let mid = self.builder.ins().ishl(hi64, c24);
let lo_mid = self.builder.ins().bor(mid, lo64);
let c16 = self.builder.ins().iconst(types::I32, 16);
let shl = self.builder.ins().ishl(lo_mid, c16);
let sext = self.builder.ins().sshr(shl, c16);
self.mask56(sext)
}
}