use crate::abi::Arch;
use super::{Backend, Exit, GuestMemory, Vcpu, VcpuError};
const MAX_STEPS: u64 = 50_000_000;
#[derive(Debug)]
pub struct InterpBackend {
guest: Arch,
}
impl InterpBackend {
pub fn new(guest: Arch) -> Result<Self, VcpuError> {
Ok(Self { guest })
}
}
impl Backend for InterpBackend {
fn name(&self) -> &'static str {
"interp"
}
fn guest_arch(&self) -> Arch {
self.guest
}
fn new_vcpu(&self, entry: u64, stack: u64) -> Result<Box<dyn Vcpu>, VcpuError> {
match self.guest {
Arch::Aarch64 => Ok(Box::new(Aarch64Interp::new(entry, stack))),
Arch::X86_64 => Err(VcpuError::Backend(
"interp x86-64 not implemented yet (ROADMAP Phase 10)".into(),
)),
}
}
}
enum Step {
Next,
Branched,
Syscall,
Illegal,
Fault {
addr: u64,
write: bool,
},
}
#[derive(Default, Clone, Copy)]
#[allow(clippy::struct_excessive_bools)]
struct Flags {
n: bool,
z: bool,
c: bool,
v: bool,
}
#[derive(Clone)]
struct Aarch64Interp {
x: [u64; 31],
sp: u64,
pc: u64,
tpidr: u64,
cntvct: u64,
fpcr: u64,
fpsr: u64,
flags: Flags,
v: [u128; 32],
watch: Option<u64>,
excl_monitor: bool,
}
impl Aarch64Interp {
fn new(entry: u64, stack: u64) -> Self {
Self {
x: [0; 31],
sp: stack,
pc: entry,
tpidr: 0,
cntvct: 0,
fpcr: 0,
fpsr: 0,
flags: Flags::default(),
v: [0; 32],
watch: std::env::var("NIXVM_WATCH")
.ok()
.and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()),
excl_monitor: true,
}
}
fn ldst_vec(
&mut self,
addr: u64,
scale: u32,
is_load: bool,
rt: usize,
mem: &mut GuestMemory,
) -> Step {
let nbytes = 1usize << scale; if is_load {
let mut buf = [0u8; 16];
if mem.read(addr, &mut buf[..nbytes]).is_err() {
return Step::Fault { addr, write: false };
}
self.v[rt] = u128::from_le_bytes(buf); } else {
self.note_store(addr, self.v[rt], nbytes);
let bytes = self.v[rt].to_le_bytes();
if mem.write(addr, &bytes[..nbytes]).is_err() {
return Step::Fault { addr, write: true };
}
}
Step::Next
}
fn read_x(&self, i: usize) -> u64 {
if i == 31 { 0 } else { self.x[i] }
}
fn write_x(&mut self, i: usize, v: u64) {
if i != 31 {
self.x[i] = v;
}
}
fn read_sp(&self, i: usize) -> u64 {
if i == 31 { self.sp } else { self.x[i] }
}
fn write_sp(&mut self, i: usize, v: u64) {
if i == 31 {
self.sp = v;
} else {
self.x[i] = v;
}
}
fn branch(&mut self, offset: i64) -> Step {
self.pc = (self.pc as i64).wrapping_add(offset) as u64;
Step::Branched
}
#[allow(clippy::match_same_arms)]
fn read_sysreg(&mut self, instr: u32) -> u64 {
match (instr >> 5) & 0x7fff {
TPIDR_EL0 => self.tpidr,
TPIDRRO_EL0 => 0,
FPCR => self.fpcr,
FPSR => self.fpsr,
MIDR_EL1 => MIDR_EL1_VAL,
MPIDR_EL1 => 0,
REVIDR_EL1 => 0,
CTR_EL0 => CTR_EL0_VAL,
DCZID_EL0 => DCZID_EL0_VAL,
CNTFRQ_EL0 => CNTFRQ_EL0_VAL,
CNTVCT_EL0 | CNTVCTSS_EL0 => {
self.cntvct = self.cntvct.wrapping_add(1);
self.cntvct
}
ID_AA64ISAR0_EL1 => ID_AA64ISAR0_EL1_VAL,
ID_AA64PFR0_EL1 => ID_AA64PFR0_EL1_VAL,
_ => 0,
}
}
fn write_sysreg(&mut self, instr: u32, value: u64) {
match (instr >> 5) & 0x7fff {
TPIDR_EL0 => self.tpidr = value,
FPCR => self.fpcr = value,
FPSR => self.fpsr = value,
_ => {}
}
}
fn fp32(&self, n: usize) -> f32 {
f32::from_bits(self.v[n] as u32)
}
fn fp64(&self, n: usize) -> f64 {
f64::from_bits(self.v[n] as u64)
}
fn set_fp32(&mut self, d: usize, x: f32) {
self.v[d] = u128::from(x.to_bits());
}
fn set_fp64(&mut self, d: usize, x: f64) {
self.v[d] = u128::from(x.to_bits());
}
fn fp16(&self, n: usize) -> u16 {
self.v[n] as u16
}
fn set_fp16(&mut self, d: usize, bits: u16) {
self.v[d] = u128::from(bits);
}
#[allow(clippy::float_cmp)]
fn set_fcmp_flags(&mut self, a: f64, b: f64) {
self.flags = if a.is_nan() || b.is_nan() {
Flags {
n: false,
z: false,
c: true,
v: true,
}
} else if a == b {
Flags {
n: false,
z: true,
c: true,
v: false,
}
} else if a < b {
Flags {
n: true,
z: false,
c: false,
v: false,
}
} else {
Flags {
n: false,
z: false,
c: true,
v: false,
}
};
}
fn note_store(&mut self, addr: u64, value: u128, nbytes: usize) {
self.excl_monitor = false;
if let Some(w) = self.watch
&& w >= addr
&& w < addr + nbytes as u64
{
eprintln!(
"[watch] pc={:#x} store {value:#x} ({nbytes}B) -> {addr:#x}",
self.pc
);
}
}
fn ldst(&mut self, addr: u64, size: u32, opc: u32, rt: usize, mem: &mut GuestMemory) -> Step {
let nbytes = 1usize << size;
if opc == 0b00 {
let value = self.read_x(rt);
self.note_store(addr, u128::from(value), nbytes);
let val = value.to_le_bytes();
if mem.write(addr, &val[..nbytes]).is_err() {
return Step::Fault { addr, write: true };
}
return Step::Next;
}
let mut buf = [0u8; 8];
if mem.read(addr, &mut buf[..nbytes]).is_err() {
return Step::Fault { addr, write: false };
}
let raw = u64::from_le_bytes(buf);
let val = match opc {
0b01 => raw, 0b10 => sign_extend(raw, (nbytes * 8) as u32) as u64, _ => sign_extend(raw, (nbytes * 8) as u32) as u64 & 0xffff_ffff, };
self.write_x(rt, val);
Step::Next
}
fn mem_write_sized(
&mut self,
mem: &mut GuestMemory,
addr: u64,
nbytes: usize,
val: u64,
) -> bool {
self.note_store(addr, u128::from(val), nbytes);
mem.write(addr, &val.to_le_bytes()[..nbytes]).is_ok()
}
fn cas_single(
&mut self,
addr: u64,
nbytes: usize,
rs: usize,
rt: usize,
mem: &mut GuestMemory,
) -> Step {
let Some(old) = mem_read_sized(mem, addr, nbytes) else {
return Step::Fault { addr, write: false };
};
if old == self.read_x(rs) & ones((nbytes * 8) as u32) {
let new = self.read_x(rt);
if !self.mem_write_sized(mem, addr, nbytes, new) {
return Step::Fault { addr, write: true };
}
}
self.write_x(rs, old);
Step::Next
}
fn cas_pair(
&mut self,
addr: u64,
nbytes: usize,
rs: usize,
rt: usize,
mem: &mut GuestMemory,
) -> Step {
let rs2 = (rs + 1) & 0x1f;
let rt2 = (rt + 1) & 0x1f;
let addr2 = addr.wrapping_add(nbytes as u64);
let Some(old0) = mem_read_sized(mem, addr, nbytes) else {
return Step::Fault { addr, write: false };
};
let Some(old1) = mem_read_sized(mem, addr2, nbytes) else {
return Step::Fault {
addr: addr2,
write: false,
};
};
let mask = ones((nbytes * 8) as u32);
if old0 == self.read_x(rs) & mask && old1 == self.read_x(rs2) & mask {
let (new0, new1) = (self.read_x(rt), self.read_x(rt2));
if !self.mem_write_sized(mem, addr, nbytes, new0) {
return Step::Fault { addr, write: true };
}
if !self.mem_write_sized(mem, addr2, nbytes, new1) {
return Step::Fault {
addr: addr2,
write: true,
};
}
}
self.write_x(rs, old0);
self.write_x(rs2, old1);
Step::Next
}
fn swp(
&mut self,
addr: u64,
nbytes: usize,
rs: usize,
rt: usize,
mem: &mut GuestMemory,
) -> Step {
let Some(old) = mem_read_sized(mem, addr, nbytes) else {
return Step::Fault { addr, write: false };
};
let new = self.read_x(rs);
if !self.mem_write_sized(mem, addr, nbytes, new) {
return Step::Fault { addr, write: true };
}
self.write_x(rt, old);
Step::Next
}
fn ld_op(
&mut self,
addr: u64,
nbytes: usize,
rs: usize,
rt: usize,
op: u32,
mem: &mut GuestMemory,
) -> Step {
let Some(old) = mem_read_sized(mem, addr, nbytes) else {
return Step::Fault { addr, write: false };
};
let bits = (nbytes * 8) as u32;
let mask = ones(bits);
let s = self.read_x(rs) & mask;
let new = match op {
0b000 => old.wrapping_add(s) & mask, 0b001 => old & !s & mask, 0b010 => (old ^ s) & mask, 0b011 => (old | s) & mask, 0b100 => {
if sign_extend(old, bits) >= sign_extend(s, bits) {
old
} else {
s
}
}
0b101 => {
if sign_extend(old, bits) <= sign_extend(s, bits) {
old
} else {
s
}
}
0b110 => {
if old >= s { old } else { s } }
_ => {
if old <= s { old } else { s } }
};
if !self.mem_write_sized(mem, addr, nbytes, new) {
return Step::Fault { addr, write: true };
}
self.write_x(rt, old);
Step::Next
}
fn addsub_flags(&mut self, a: u64, b: u64, sub: bool, sf: bool) -> u64 {
let (operand, carry_in) = if sub { (!b, 1u128) } else { (b, 0u128) };
if sf {
let sum = u128::from(a) + u128::from(operand) + carry_in;
let r = sum as u64;
self.flags = Flags {
n: (r >> 63) & 1 == 1,
z: r == 0,
c: (sum >> 64) & 1 == 1,
v: (((a ^ r) & (operand ^ r)) >> 63) & 1 == 1,
};
r
} else {
let (a, operand) = (a as u32, operand as u32);
let sum = u64::from(a) + u64::from(operand) + carry_in as u64;
let r = sum as u32;
self.flags = Flags {
n: (r >> 31) & 1 == 1,
z: r == 0,
c: (sum >> 32) & 1 == 1,
v: (((a ^ r) & (operand ^ r)) >> 31) & 1 == 1,
};
u64::from(r)
}
}
fn cond_holds(&self, cond: u32) -> bool {
let f = &self.flags;
match cond {
0b0000 => f.z,
0b0001 => !f.z,
0b0010 => f.c,
0b0011 => !f.c,
0b0100 => f.n,
0b0101 => !f.n,
0b0110 => f.v,
0b0111 => !f.v,
0b1000 => f.c && !f.z, 0b1001 => !f.c || f.z, 0b1010 => f.n == f.v, 0b1011 => f.n != f.v, 0b1100 => !f.z && (f.n == f.v), 0b1101 => f.z || (f.n != f.v), _ => true, }
}
#[allow(clippy::too_many_lines)]
#[allow(clippy::many_single_char_names)]
fn exec(&mut self, instr: u32, mem: &mut GuestMemory) -> Step {
if instr & 0xFFE0_001F == 0xD400_0001 {
return Step::Syscall; }
if instr & 0xFFFF_F000 == 0xD503_2000 || instr & 0xFFFF_F000 == 0xD503_3000 {
return Step::Next;
}
if (instr >> 20) & 0xfff == 0xD53 {
let rt = reg_field(instr, 0);
let val = self.read_sysreg(instr);
self.write_x(rt, val);
return Step::Next;
}
if (instr >> 20) & 0xfff == 0xD51 {
let rt = reg_field(instr, 0);
self.write_sysreg(instr, self.read_x(rt));
return Step::Next;
}
if (instr >> 19) & 0x1fff == 0x1AA1 {
let key = (instr >> 5) & 0x3fff; let rt = reg_field(instr, 0);
return match key {
SYS_DC_ZVA => {
let addr = self.read_x(rt) & !(DC_ZVA_BLOCK_BYTES - 1);
self.note_store(addr, 0u128, DC_ZVA_BLOCK_BYTES as usize);
if mem
.write(addr, &[0u8; DC_ZVA_BLOCK_BYTES as usize])
.is_err()
{
Step::Fault { addr, write: true }
} else {
Step::Next
}
}
SYS_DC_CVAC | SYS_DC_CVAU | SYS_DC_CIVAC | SYS_DC_IVAC | SYS_IC_IALLU
| SYS_IC_IALLUIS | SYS_IC_IVAU => Step::Next,
_ => Step::Illegal,
};
}
if instr & 0xEFFE_0C00 == 0x4E28_0800 {
let u = (instr >> 28) & 1;
let opcode = (instr >> 12) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
self.v[rd] = match (u, opcode) {
(0, 0b00100) => aes_round(self.v[rd], self.v[rn], true), (0, 0b00101) => aes_round(self.v[rd], self.v[rn], false), (0, 0b00110) => aes_mix_columns(self.v[rn], true), (0, 0b00111) => aes_mix_columns(self.v[rn], false), (1, 0b00000) => u128::from((self.v[rn] as u32).rotate_left(30)),
(1, 0b00001) => sha1_su1(self.v[rd], self.v[rn]), (1, 0b00010) => sha256_su0(self.v[rd], self.v[rn]), _ => return Step::Illegal,
};
return Step::Next;
}
if instr & 0xFFE0_0C00 == 0x5E00_0000 {
let opcode = (instr >> 12) & 7;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let e = self.v[rn] as u32;
self.v[rd] = match opcode {
0b000 => sha1_quad_round(self.v[rd], e, self.v[rm], Sha1Op::Choose), 0b001 => sha1_quad_round(self.v[rd], e, self.v[rm], Sha1Op::Parity), 0b010 => sha1_quad_round(self.v[rd], e, self.v[rm], Sha1Op::Majority), 0b011 => sha1_su0(self.v[rd], self.v[rn], self.v[rm]), 0b100 => sha256_hash(self.v[rd], self.v[rn], self.v[rm], false), 0b101 => sha256_hash(self.v[rn], self.v[rd], self.v[rm], true), 0b110 => sha256_su1(self.v[rd], self.v[rn], self.v[rm]), _ => return Step::Illegal,
};
return Step::Next;
}
if instr & 0xFFFF_FC1F == 0xD65F_0000 {
self.pc = self.read_x(reg_field(instr, 5)); return Step::Branched;
}
if instr & 0xFFFF_FC1F == 0xD61F_0000 {
self.pc = self.read_x(reg_field(instr, 5)); return Step::Branched;
}
if instr & 0xFFFF_FC1F == 0xD63F_0000 {
let target = self.read_x(reg_field(instr, 5)); self.x[30] = self.pc.wrapping_add(4);
self.pc = target;
return Step::Branched;
}
if (instr >> 26) & 0x3f == 0b00_0101 {
let off = sign_extend(u64::from(instr & 0x03ff_ffff), 26) << 2; return self.branch(off);
}
if (instr >> 26) & 0x3f == 0b10_0101 {
self.x[30] = self.pc.wrapping_add(4); let off = sign_extend(u64::from(instr & 0x03ff_ffff), 26) << 2;
return self.branch(off);
}
if instr & 0xFF00_0010 == 0x5400_0000 {
let cond = instr & 0xf; if self.cond_holds(cond) {
let off = sign_extend(u64::from((instr >> 5) & 0x7ffff), 19) << 2;
return self.branch(off);
}
return Step::Next;
}
if (instr >> 25) & 0x3f == 0b01_1010 {
let sf = (instr >> 31) & 1; let op = (instr >> 24) & 1;
let rt = reg_field(instr, 0);
let mut val = self.read_x(rt);
if sf == 0 {
val &= 0xffff_ffff;
}
let take = if op == 0 { val == 0 } else { val != 0 };
if take {
let off = sign_extend(u64::from((instr >> 5) & 0x7ffff), 19) << 2;
return self.branch(off);
}
return Step::Next;
}
if (instr >> 23) & 0x3f == 0b1_00101 {
let sf = (instr >> 31) & 1;
let opc = (instr >> 29) & 3;
let hw = (instr >> 21) & 3;
let imm16 = u64::from((instr >> 5) & 0xffff);
let rd = reg_field(instr, 0);
if sf == 0 && hw > 1 {
return Step::Illegal;
}
let shift = hw * 16;
let val = imm16 << shift;
let result = match opc {
0b10 => val,
0b00 => !val,
0b11 => (self.read_x(rd) & !(0xffff_u64 << shift)) | val,
_ => return Step::Illegal,
};
self.write_x(rd, mask_sf(result, sf));
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b1_0000 {
let op = (instr >> 31) & 1;
let immlo = u64::from((instr >> 29) & 3);
let immhi = u64::from((instr >> 5) & 0x7ffff);
let rd = reg_field(instr, 0);
let imm = sign_extend((immhi << 2) | immlo, 21);
let result = if op == 0 {
(self.pc as i64).wrapping_add(imm) as u64
} else {
((self.pc & !0xfff) as i64).wrapping_add(imm << 12) as u64
};
self.write_x(rd, result);
return Step::Next;
}
if (instr >> 23) & 0x3f == 0b1_00100 {
let sf = (instr >> 31) & 1;
let opc = (instr >> 29) & 3;
let n = (instr >> 22) & 1;
let immr = (instr >> 16) & 0x3f;
let imms = (instr >> 10) & 0x3f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let width = if sf == 1 { 64 } else { 32 };
if sf == 0 && n == 1 {
return Step::Illegal;
}
let Some((imm, _)) = decode_bit_masks(n, imms, immr, width) else {
return Step::Illegal;
};
let a = self.read_x(rn);
let r = mask_sf(
match opc {
0b00 | 0b11 => a & imm, 0b01 => a | imm, _ => a ^ imm, },
sf,
);
if opc == 0b11 {
self.flags = Flags {
n: (r >> if sf == 1 { 63 } else { 31 }) & 1 == 1,
z: r == 0,
c: false,
v: false,
};
self.write_x(rd, r);
} else {
self.write_sp(rd, r);
}
return Step::Next;
}
if (instr >> 23) & 0xff == 0b00100111 {
let sf = (instr >> 31) & 1;
let rm = reg_field(instr, 16);
let imms = (instr >> 10) & 0x3f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let (n, m) = (self.read_x(rn), self.read_x(rm));
let r = if sf == 1 {
if imms == 0 {
m
} else {
(n << (64 - imms)) | (m >> imms)
}
} else {
let (n, m) = (n & 0xffff_ffff, m & 0xffff_ffff);
if imms == 0 {
m
} else {
(n << (32 - imms)) | (m >> imms)
}
};
self.write_x(rd, mask_sf(r, sf));
return Step::Next;
}
if (instr >> 23) & 0x3f == 0b1_00110 {
let sf = (instr >> 31) & 1;
let opc = (instr >> 29) & 3;
let n = (instr >> 22) & 1;
let immr = (instr >> 16) & 0x3f;
let imms = (instr >> 10) & 0x3f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let width = if sf == 1 { 64u32 } else { 32 };
if (sf == 1) != (n == 1) {
return Step::Illegal; }
let Some((wmask, tmask)) = decode_bit_masks(n, imms, immr, width) else {
return Step::Illegal;
};
let src = self.read_x(rn);
let rotated = ror_val(src, immr, width) & wmask;
let (bot, top) = match opc {
0b10 => (rotated, 0u64), 0b00 => {
let top = if (src >> imms) & 1 == 1 {
ones(width)
} else {
0
};
(rotated, top)
}
0b01 => {
let dst = self.read_x(rd);
((dst & !wmask) | rotated, dst)
}
_ => return Step::Illegal,
};
let result = mask_sf((top & !tmask) | (bot & tmask), sf);
self.write_x(rd, result);
return Step::Next;
}
if (instr >> 23) & 0x3f == 0b1_00010 {
let sf = (instr >> 31) & 1;
let op = (instr >> 30) & 1;
let s = (instr >> 29) & 1;
let sh = (instr >> 22) & 1;
let imm12 = u64::from((instr >> 10) & 0xfff);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let imm = if sh == 1 { imm12 << 12 } else { imm12 };
let a = self.read_sp(rn);
if s == 1 {
let r = self.addsub_flags(a, imm, op == 1, sf == 1);
self.write_x(rd, r); } else {
let r = if op == 0 {
a.wrapping_add(imm)
} else {
a.wrapping_sub(imm)
};
self.write_sp(rd, mask_sf(r, sf));
}
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1011 && (instr >> 21) & 1 == 0 {
let sf = (instr >> 31) & 1;
let op = (instr >> 30) & 1;
let s = (instr >> 29) & 1;
let shift_type = (instr >> 22) & 3;
let rm = reg_field(instr, 16);
let imm6 = (instr >> 10) & 0x3f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let a = self.read_x(rn);
let b = shift_reg(self.read_x(rm), shift_type, imm6, sf == 1);
let r = if s == 1 {
self.addsub_flags(a, b, op == 1, sf == 1)
} else {
let r = if op == 0 {
a.wrapping_add(b)
} else {
a.wrapping_sub(b)
};
mask_sf(r, sf)
};
self.write_x(rd, r);
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1011 && (instr >> 21) & 0x7 == 0b001 {
let sf = (instr >> 31) & 1;
let op = (instr >> 30) & 1;
let s = (instr >> 29) & 1;
let rm = reg_field(instr, 16);
let option = (instr >> 13) & 7;
let imm3 = (instr >> 10) & 7;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let a = self.read_sp(rn);
let b = extend_reg(self.read_x(rm), option, imm3);
if s == 1 {
let r = self.addsub_flags(a, b, op == 1, sf == 1);
self.write_x(rd, r);
} else {
let r = if op == 0 {
a.wrapping_add(b)
} else {
a.wrapping_sub(b)
};
self.write_sp(rd, mask_sf(r, sf));
}
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1010 {
let sf = (instr >> 31) & 1;
let opc = (instr >> 29) & 3;
let shift_type = (instr >> 22) & 3;
let n_bit = (instr >> 21) & 1;
let rm = reg_field(instr, 16);
let imm6 = (instr >> 10) & 0x3f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let a = self.read_x(rn);
let mut b = shift_reg(self.read_x(rm), shift_type, imm6, sf == 1);
if n_bit == 1 {
b = mask_sf(!b, sf);
}
let r = match opc {
0b00 | 0b11 => a & b, 0b01 => a | b, 0b10 => a ^ b, _ => return Step::Illegal,
};
let r = mask_sf(r, sf);
if opc == 0b11 {
self.flags = Flags {
n: (r >> if sf == 1 { 63 } else { 31 }) & 1 == 1,
z: r == 0,
c: false,
v: false,
};
}
self.write_x(rd, r);
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b1_1011 {
let sf = (instr >> 31) & 1;
let op31 = (instr >> 21) & 0x7;
let o0 = (instr >> 15) & 1;
let rm = reg_field(instr, 16);
let ra = reg_field(instr, 10);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let (n, m, a) = (self.read_x(rn), self.read_x(rm), self.read_x(ra));
let r = match op31 {
0b000 => {
let prod = n.wrapping_mul(m);
let r = if o0 == 0 {
a.wrapping_add(prod)
} else {
a.wrapping_sub(prod)
};
mask_sf(r, sf)
}
0b001 => {
let prod = i64::from(n as i32).wrapping_mul(i64::from(m as i32)) as u64;
if o0 == 0 {
a.wrapping_add(prod)
} else {
a.wrapping_sub(prod)
}
}
0b101 => {
let prod = u64::from(n as u32).wrapping_mul(u64::from(m as u32));
if o0 == 0 {
a.wrapping_add(prod)
} else {
a.wrapping_sub(prod)
}
}
0b010 => ((i128::from(n as i64).wrapping_mul(i128::from(m as i64))) >> 64) as u64, 0b110 => ((u128::from(n).wrapping_mul(u128::from(m))) >> 64) as u64, _ => return Step::Illegal,
};
self.write_x(rd, r);
return Step::Next;
}
if (instr >> 21) & 0x3ff == 0b10_1101_0110 {
let sf = (instr >> 31) & 1;
let opcode = (instr >> 10) & 0x3f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let width = if sf == 1 { 64u32 } else { 32 };
let x = if sf == 1 {
self.read_x(rn)
} else {
self.read_x(rn) & 0xffff_ffff
};
let r = match opcode {
0b000000 => rbit(x, width),
0b000001 => rev16(x, width),
0b000010 => {
if sf == 1 {
rev32(x)
} else {
u64::from((x as u32).swap_bytes())
}
}
0b000011 => x.swap_bytes(), 0b000100 => u64::from(if sf == 1 {
x.leading_zeros()
} else {
(x as u32).leading_zeros()
}),
0b000101 => u64::from(cls(x, width)),
_ => return Step::Illegal,
};
self.write_x(rd, mask_sf(r, sf));
return Step::Next;
}
if (instr >> 21) & 0x3ff == 0b00_1101_0110 {
let sf = (instr >> 31) & 1;
let opcode = (instr >> 10) & 0x3f;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let a = self.read_x(rn);
let b = self.read_x(rm);
let width = if sf == 1 { 64u32 } else { 32 };
let amount = (b % u64::from(width)) as u32;
let r = match opcode {
0b00_0010 => udiv(a, b, sf == 1),
0b00_0011 => sdiv(a, b, sf == 1),
0b00_1000 => shift_reg(a, 0, amount, sf == 1),
0b00_1001 => shift_reg(a, 1, amount, sf == 1),
0b00_1010 => shift_reg(a, 2, amount, sf == 1),
0b00_1011 => shift_reg(a, 3, amount, sf == 1),
0b01_0000..=0b01_0111 => {
let sz = opcode & 3;
if (sz == 3) != (sf == 1) {
return Step::Illegal; }
let is_c = (opcode >> 2) & 1 == 1;
let nbytes = 1usize << sz;
let val = b & ones((nbytes * 8) as u32);
let poly = if is_c { 0x82F6_3B78u32 } else { 0xEDB8_8320u32 };
let mut crc = a as u32;
for i in 0..nbytes {
crc = crc32_step(crc, (val >> (i * 8)) as u8, poly);
}
u64::from(crc)
}
_ => return Step::Illegal,
};
self.write_x(rd, mask_sf(r, sf));
return Step::Next;
}
if (instr >> 21) & 0xff == 0b1101_0000 {
let sf = (instr >> 31) & 1;
let op = (instr >> 30) & 1; let s = (instr >> 29) & 1;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let a = self.read_x(rn);
let b = if op == 1 {
!self.read_x(rm)
} else {
self.read_x(rm)
};
let carry = u128::from(self.flags.c);
let r = if sf == 1 {
let sum = u128::from(a) + u128::from(b) + carry;
let r = sum as u64;
if s == 1 {
self.flags = Flags {
n: (r >> 63) & 1 == 1,
z: r == 0,
c: (sum >> 64) & 1 == 1,
v: (((a ^ r) & (b ^ r)) >> 63) & 1 == 1,
};
}
r
} else {
let (a32, b32) = (a as u32, b as u32);
let sum = u64::from(a32) + u64::from(b32) + carry as u64;
let r = sum as u32;
if s == 1 {
self.flags = Flags {
n: (r >> 31) & 1 == 1,
z: r == 0,
c: (sum >> 32) & 1 == 1,
v: (((a32 ^ r) & (b32 ^ r)) >> 31) & 1 == 1,
};
}
u64::from(r)
};
self.write_x(rd, r);
return Step::Next;
}
if (instr >> 21) & 0xff == 0b1101_0010 {
let sf = (instr >> 31) & 1;
let op = (instr >> 30) & 1; let cond = (instr >> 12) & 0xf;
let rn = reg_field(instr, 5);
let nzcv = instr & 0xf;
let operand = if (instr >> 11) & 1 == 1 {
u64::from((instr >> 16) & 0x1f) } else {
self.read_x(reg_field(instr, 16)) };
if self.cond_holds(cond) {
self.addsub_flags(self.read_x(rn), operand, op == 1, sf == 1);
} else {
self.flags = Flags {
n: nzcv & 8 != 0,
z: nzcv & 4 != 0,
c: nzcv & 2 != 0,
v: nzcv & 1 != 0,
};
}
return Step::Next;
}
if (instr >> 21) & 0xff == 0b1101_0100 && (instr >> 29) & 1 == 0 {
let sf = (instr >> 31) & 1;
let op = (instr >> 30) & 1;
let op2 = (instr >> 10) & 3;
if op2 > 1 {
return Step::Illegal;
}
let cond = (instr >> 12) & 0xf;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let r = if self.cond_holds(cond) {
self.read_x(rn)
} else {
let m = self.read_x(rm);
match (op, op2) {
(0, 0) => m, (0, 1) => m.wrapping_add(1), (1, 0) => !m, _ => m.wrapping_neg(), }
};
self.write_x(rd, mask_sf(r, sf));
return Step::Next;
}
if (instr >> 24) & 0x3f == 0b01_1000 {
let opc = (instr >> 30) & 3;
let v = (instr >> 26) & 1;
let imm19 = sign_extend(u64::from((instr >> 5) & 0x7ffff), 19) << 2;
let rt = reg_field(instr, 0);
let addr = (self.pc as i64).wrapping_add(imm19) as u64;
if v == 1 {
if opc == 3 {
return Step::Illegal;
}
return self.ldst_vec(addr, opc + 2, true, rt, mem);
}
return match opc {
0 => self.ldst(addr, 2, 0b01, rt, mem), 1 => self.ldst(addr, 3, 0b01, rt, mem), 2 => self.ldst(addr, 2, 0b10, rt, mem), _ => Step::Next, };
}
if (instr >> 27) & 0x7 == 0b101 && (instr >> 26) & 1 == 1 && (instr >> 25) & 1 == 0 {
let opc = (instr >> 30) & 3;
if opc == 3 {
return Step::Illegal;
}
let nbytes = 4usize << opc; let class = (instr >> 23) & 3; let is_load = (instr >> 22) & 1 == 1;
let imm7 = sign_extend(u64::from((instr >> 15) & 0x7f), 7);
let rt2 = reg_field(instr, 10);
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let offset = imm7 * nbytes as i64;
let base = self.read_sp(rn);
let addr = if class == 1 {
base
} else {
(base as i64).wrapping_add(offset) as u64
};
for (i, r) in [rt, rt2].into_iter().enumerate() {
let a = addr.wrapping_add((i * nbytes) as u64);
if is_load {
let mut buf = [0u8; 16];
if mem.read(a, &mut buf[..nbytes]).is_err() {
return Step::Fault {
addr: a,
write: false,
};
}
self.v[r] = u128::from_le_bytes(buf);
} else {
self.note_store(a, self.v[r], nbytes);
let bytes = self.v[r].to_le_bytes();
if mem.write(a, &bytes[..nbytes]).is_err() {
return Step::Fault {
addr: a,
write: true,
};
}
}
}
if class == 1 || class == 3 {
self.write_sp(rn, (base as i64).wrapping_add(offset) as u64);
}
return Step::Next;
}
if (instr >> 24) & 0x3f == 0b00_1000
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 0x1f == 0b1_1111
{
let rs = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let addr = self.read_sp(rn);
if (instr >> 23) & 1 == 1 {
let size = (instr >> 30) & 3;
return self.cas_single(addr, 1usize << size, rs, rt, mem);
}
let nbytes = if (instr >> 30) & 1 == 1 { 8 } else { 4 };
return self.cas_pair(addr, nbytes, rs, rt, mem);
}
if (instr >> 24) & 0x3f == 0b11_1000 && (instr >> 21) & 1 == 1 && (instr >> 10) & 3 == 0 {
let size = (instr >> 30) & 3;
let nbytes = 1usize << size;
let rs = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let addr = self.read_sp(rn);
let o3 = (instr >> 15) & 1; let opc = (instr >> 12) & 7;
if o3 == 1 {
return if opc == 0 {
self.swp(addr, nbytes, rs, rt, mem)
} else {
Step::Illegal };
}
return self.ld_op(addr, nbytes, rs, rt, opc, mem);
}
if (instr >> 24) & 0x3f == 0b00_1000 {
let size = (instr >> 30) & 3;
let o2 = (instr >> 23) & 1; let l = (instr >> 22) & 1; let o1 = (instr >> 21) & 1; if o1 == 1 {
return Step::Illegal; }
let rs = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let addr = self.read_sp(rn);
if l == 1 {
let step = self.ldst(addr, size, 0b01, rt, mem);
if o2 == 0 && matches!(step, Step::Next) {
self.excl_monitor = true;
}
return step;
}
if o2 == 0 {
if !self.excl_monitor {
self.write_x(rs, 1); return Step::Next;
}
let step = self.ldst(addr, size, 0b00, rt, mem);
self.excl_monitor = false; if matches!(step, Step::Next) {
self.write_x(rs, 0); }
return step;
}
return self.ldst(addr, size, 0b00, rt, mem);
}
if (instr >> 27) & 0x7 == 0b101 && (instr >> 26) & 1 == 0 && (instr >> 25) & 1 == 0 {
let opc = (instr >> 30) & 3;
if opc == 0b11 {
return Step::Illegal; }
let is64 = opc == 0b10;
let is_load = (instr >> 22) & 1 == 1;
if opc == 0b01 && !is_load {
return Step::Illegal; }
let class = (instr >> 23) & 3; let imm7 = sign_extend(u64::from((instr >> 15) & 0x7f), 7);
let rt2 = reg_field(instr, 10);
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let nbytes = if is64 { 8usize } else { 4 };
let offset = imm7 * nbytes as i64;
let base = self.read_sp(rn);
let addr = if class == 1 {
base
} else {
(base as i64).wrapping_add(offset) as u64
};
for (i, r) in [rt, rt2].into_iter().enumerate() {
let a = addr.wrapping_add((i * nbytes) as u64);
if is_load {
let mut buf = [0u8; 8];
if mem.read(a, &mut buf[..nbytes]).is_err() {
return Step::Fault {
addr: a,
write: false,
};
}
let raw = u64::from_le_bytes(buf);
let val = if opc == 0b01 {
sign_extend(raw, 32) as u64 } else {
raw
};
self.write_x(r, val);
} else {
self.note_store(a, u128::from(self.read_x(r)), nbytes);
let val = self.read_x(r).to_le_bytes();
if mem.write(a, &val[..nbytes]).is_err() {
return Step::Fault {
addr: a,
write: true,
};
}
}
}
if class == 1 || class == 3 {
self.write_sp(rn, (base as i64).wrapping_add(offset) as u64);
}
return Step::Next;
}
if (instr >> 27) & 0x7 == 0b111
&& (instr >> 26) & 1 == 0
&& (instr >> 30) & 3 == 0b11
&& (instr >> 22) & 3 == 0b10
&& matches!((instr >> 24) & 0x3, 0b00 | 0b01)
{
return Step::Next;
}
if (instr >> 27) & 0x7 == 0b111 && (instr >> 24) & 0x3 == 0b01 && (instr >> 26) & 1 == 0 {
let size = (instr >> 30) & 3;
let opc = (instr >> 22) & 3;
let imm12 = u64::from((instr >> 10) & 0xfff);
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let addr = self.read_sp(rn).wrapping_add(imm12 << size);
return self.ldst(addr, size, opc, rt, mem);
}
if (instr >> 27) & 0x7 == 0b111
&& (instr >> 24) & 0x3 == 0b00
&& (instr >> 26) & 1 == 0
&& (instr >> 21) & 1 == 0
{
let size = (instr >> 30) & 3;
let opc = (instr >> 22) & 3;
let imm9 = sign_extend(u64::from((instr >> 12) & 0x1ff), 9);
let idx = (instr >> 10) & 3; let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let base = self.read_sp(rn);
let addr = if idx == 0b01 {
base
} else {
(base as i64).wrapping_add(imm9) as u64
};
let step = self.ldst(addr, size, opc, rt, mem);
if matches!(step, Step::Next) && (idx == 0b01 || idx == 0b11) {
self.write_sp(rn, (base as i64).wrapping_add(imm9) as u64);
}
return step;
}
if (instr >> 27) & 0x7 == 0b111
&& (instr >> 24) & 0x3 == 0b00
&& (instr >> 26) & 1 == 0
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 3 == 0b10
{
let size = (instr >> 30) & 3;
let opc = (instr >> 22) & 3;
let rm = reg_field(instr, 16);
let option = (instr >> 13) & 7;
let s = (instr >> 12) & 1;
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let shift = if s == 1 { size } else { 0 };
let addr = self
.read_sp(rn)
.wrapping_add(extend_reg(self.read_x(rm), option, shift));
return self.ldst(addr, size, opc, rt, mem);
}
if (instr >> 27) & 0x7 == 0b111 && (instr >> 24) & 0x3 == 0b01 && (instr >> 26) & 1 == 1 {
let size = (instr >> 30) & 3;
let opc = (instr >> 22) & 3;
let scale = if opc & 2 != 0 { 4 } else { size }; let imm12 = u64::from((instr >> 10) & 0xfff);
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let addr = self.read_sp(rn).wrapping_add(imm12 << scale);
return self.ldst_vec(addr, scale, opc & 1 == 1, rt, mem);
}
if (instr >> 27) & 0x7 == 0b111
&& (instr >> 24) & 0x3 == 0b00
&& (instr >> 26) & 1 == 1
&& (instr >> 21) & 1 == 0
{
let size = (instr >> 30) & 3;
let opc = (instr >> 22) & 3;
let scale = if opc & 2 != 0 { 4 } else { size };
let imm9 = sign_extend(u64::from((instr >> 12) & 0x1ff), 9);
let idx = (instr >> 10) & 3;
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let base = self.read_sp(rn);
let addr = if idx == 0b01 {
base
} else {
(base as i64).wrapping_add(imm9) as u64
};
let step = self.ldst_vec(addr, scale, opc & 1 == 1, rt, mem);
if matches!(step, Step::Next) && (idx == 0b01 || idx == 0b11) {
self.write_sp(rn, (base as i64).wrapping_add(imm9) as u64);
}
return step;
}
if (instr >> 31) & 1 == 0
&& (instr >> 29) & 1 == 0
&& (instr >> 24) & 0x1f == 0b0_1100
&& (instr >> 21) & 1 == 0
&& (instr >> 12) & 0xf == 0b0111
{
let q = (instr >> 30) & 1;
let post = (instr >> 23) & 1 == 1;
let l = (instr >> 22) & 1;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rt = reg_field(instr, 0);
let (nbytes, scale) = if q == 1 { (16u64, 4) } else { (8u64, 3) };
let addr = self.read_sp(rn);
let step = self.ldst_vec(addr, scale, l == 1, rt, mem);
if matches!(step, Step::Next) && post {
let inc = if rm == 31 { nbytes } else { self.read_x(rm) };
self.write_sp(rn, addr.wrapping_add(inc));
}
return step;
}
if (instr >> 19) & 0x3ff == 0b0111100000 && (instr >> 10) & 1 == 1 {
let q = (instr >> 30) & 1;
let op = (instr >> 29) & 1;
let cmode = (instr >> 12) & 0xf;
let imm8 = u64::from((((instr >> 16) & 0x7) << 5) | ((instr >> 5) & 0x1f));
let rd = reg_field(instr, 0);
let imm64 = adv_simd_expand_imm(cmode, op, imm8);
let to_q = |x: u64| {
if q == 1 {
(u128::from(x) << 64) | u128::from(x)
} else {
u128::from(x)
}
};
self.v[rd] = if cmode == 0b1110 || cmode == 0b1111 {
to_q(imm64) } else if cmode & 1 == 0 {
to_q(if op == 0 { imm64 } else { !imm64 }) } else {
let m = to_q(imm64);
if op == 0 {
self.v[rd] | m
} else {
self.v[rd] & !m
}
};
return Step::Next;
}
if (instr >> 21) & 0x1ff == 0b0_0111_0000 && (instr >> 10) & 0x3f == 0b00_0001 {
let q = (instr >> 30) & 1;
let imm5 = (instr >> 16) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = elem_bits(imm5);
let index = imm5 >> (esize / 8).trailing_zeros().wrapping_add(1);
let elem = (self.v[rn] >> (u128::from(index) * u128::from(esize))) & ones_u128(esize);
let width = if q == 1 { 128 } else { 64 };
let mut val = 0u128;
let mut shift = 0;
while shift < width {
val |= elem << shift;
shift += esize;
}
self.v[rd] = val;
return Step::Next;
}
if (instr >> 21) & 0x1ff == 0b0_0111_0000 && (instr >> 10) & 0x3f == 0b00_0011 {
let q = (instr >> 30) & 1;
let imm5 = (instr >> 16) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = elem_bits(imm5);
let elem = u128::from(self.read_x(rn)) & ones_u128(esize);
let width = if q == 1 { 128 } else { 64 };
let mut val = 0u128;
let mut shift = 0;
while shift < width {
val |= elem << shift;
shift += esize;
}
self.v[rd] = val;
return Step::Next;
}
if (instr >> 21) & 0x1ff == 0b0_0111_0000 && (instr >> 10) & 0x3f == 0b00_1111 {
let imm5 = (instr >> 16) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = elem_bits(imm5);
let index = imm5 >> (esize / 8).trailing_zeros().wrapping_add(1);
let elem = (self.v[rn] >> (u128::from(index) * u128::from(esize))) & ones_u128(esize);
self.write_x(rd, elem as u64);
return Step::Next;
}
if (instr >> 21) & 0x1ff == 0b0_0111_0000 && (instr >> 10) & 0x3f == 0b00_1011 {
let q = (instr >> 30) & 1;
let imm5 = (instr >> 16) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = elem_bits(imm5);
let index = imm5 >> (esize / 8).trailing_zeros().wrapping_add(1);
let elem = (self.v[rn] >> (u128::from(index) * u128::from(esize))) & ones_u128(esize);
let se = sign_extend(elem as u64, esize);
let result = if q == 1 {
se as u64
} else {
u64::from(se as u32)
};
self.write_x(rd, result);
return Step::Next;
}
if (instr >> 21) & 0x1ff == 0b0_0111_0000 && (instr >> 10) & 0x3f == 0b00_0111 {
let imm5 = (instr >> 16) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = elem_bits(imm5);
let index = imm5 >> (esize / 8).trailing_zeros().wrapping_add(1);
let shift = u128::from(index) * u128::from(esize);
let mask = ones_u128(esize) << shift;
let val = (u128::from(self.read_x(rn)) & ones_u128(esize)) << shift;
self.v[rd] = (self.v[rd] & !mask) | val;
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 29) & 1 == 0
&& (instr >> 21) & 0x7 == 0
&& (instr >> 15) & 1 == 0
&& (instr >> 10) & 0x3 == 0
{
let q = (instr >> 30) & 1;
let len = ((instr >> 13) & 3) + 1; let is_tbx = (instr >> 12) & 1 == 1;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let table_bytes = (len as usize) * 16;
let idx_bytes = self.v[rm].to_le_bytes();
let nbytes = if q == 1 { 16usize } else { 8 };
let mut out = self.v[rd].to_le_bytes();
for i in 0..nbytes {
let idx = idx_bytes[i] as usize;
if idx < table_bytes {
let reg = (rn + idx / 16) & 0x1f;
out[i] = self.v[reg].to_le_bytes()[idx % 16];
} else if !is_tbx {
out[i] = 0;
} }
for b in out.iter_mut().skip(nbytes) {
*b = 0;
}
self.v[rd] = u128::from_le_bytes(out);
return Step::Next;
}
if (instr >> 24) & 0x3f == 0b10_1110
&& (instr >> 22) & 3 == 0
&& (instr >> 21) & 1 == 0
&& (instr >> 15) & 1 == 0
&& (instr >> 10) & 1 == 0
{
let q = (instr >> 30) & 1;
let imm4 = (instr >> 11) & 0xf;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let size = if q == 1 { 16usize } else { 8 };
let mut concat = [0u8; 32];
concat[..16].copy_from_slice(&self.v[rn].to_le_bytes());
concat[16..].copy_from_slice(&self.v[rm].to_le_bytes());
let off = imm4 as usize;
let mut out = [0u8; 16];
out[..size].copy_from_slice(&concat[off..off + size]);
self.v[rd] = u128::from_le_bytes(out);
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 29) & 1 == 0
&& (instr >> 21) & 1 == 0
&& (instr >> 15) & 1 == 0
&& (instr >> 11) & 1 == 1
&& (instr >> 10) & 1 == 0
{
let q = (instr >> 30) & 1;
let size = (instr >> 22) & 3;
let opcode = (instr >> 12) & 0x7;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = 8u32 << size;
let width = if q == 1 { 128 } else { 64 };
let lanes = width / esize;
let half = lanes / 2;
let mask = ones_u128(esize);
let lane = |v: u128, i: u32| (v >> (i * esize)) & mask;
let second = opcode & 0b100 != 0; let mut result = 0u128;
match opcode & 0b011 {
0b11 => {
let base = if second { half } else { 0 };
for i in 0..half {
result |= lane(self.v[rn], base + i) << (2 * i * esize);
result |= lane(self.v[rm], base + i) << ((2 * i + 1) * esize);
}
}
0b01 => {
let start = u32::from(second);
for i in 0..half {
result |= lane(self.v[rn], 2 * i + start) << (i * esize);
result |= lane(self.v[rm], 2 * i + start) << ((half + i) * esize);
}
}
_ => {
let start = u32::from(second);
for i in 0..half {
result |= lane(self.v[rn], 2 * i + start) << (2 * i * esize);
result |= lane(self.v[rm], 2 * i + start) << ((2 * i + 1) * esize);
}
}
}
self.v[rd] = result & (if q == 1 { u128::MAX } else { ones_u128(64) });
return Step::Next;
}
if (instr >> 24) & 0x7f == 0b0011110 && (instr >> 21) & 1 == 1 && (instr >> 10) & 0x3f == 0
{
let ftype = (instr >> 22) & 3; let sf = (instr >> 31) & 1; let rmode = (instr >> 19) & 3;
let opcode = (instr >> 16) & 7;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
match (rmode, opcode) {
(0, 0b010 | 0b011) => {
let signed = opcode == 0b010;
let val = self.read_x(rn);
match ftype {
0 => self.set_fp32(rd, int_to_f32(val, signed, sf == 1)),
1 => self.set_fp64(rd, int_to_f64(val, signed, sf == 1)),
_ => return Step::Illegal,
}
}
(3, 0b000 | 0b001) => {
let signed = opcode == 0b000;
let x = match ftype {
0 => f64::from(self.fp32(rn)),
1 => self.fp64(rn),
_ => return Step::Illegal,
};
self.write_x(rd, fp_to_int(x, signed, sf == 1));
}
(0, 0b111) => {
let val = self.read_x(rn);
self.v[rd] = if ftype == 0 {
u128::from(val & 0xffff_ffff)
} else {
u128::from(val)
};
}
(0, 0b110) => {
let val = if ftype == 0 {
(self.v[rn] as u64) & 0xffff_ffff
} else {
self.v[rn] as u64
};
self.write_x(rd, val);
}
(1, 0b111) => {
let val = u128::from(self.read_x(rn));
self.v[rd] = (self.v[rd] & u128::from(u64::MAX)) | (val << 64);
}
(1, 0b110) => self.write_x(rd, (self.v[rn] >> 64) as u64), _ => return Step::Illegal, }
return Step::Next;
}
if (instr >> 24) & 0x7f == 0b0011110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 0x1f == 0b10000
{
let ftype = (instr >> 22) & 3;
let opcode = (instr >> 15) & 0x3f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
if opcode & 0b11_1100 == 0b00_0100 {
match (ftype, opcode & 3) {
(0, 1) => self.set_fp64(rd, f64::from(self.fp32(rn))), (1, 0) => self.set_fp32(rd, self.fp64(rn) as f32), (0, 3) => self.set_fp16(rd, f32_to_f16(self.fp32(rn))), (1, 3) => self.set_fp16(rd, f32_to_f16(self.fp64(rn) as f32)), (3, 0) => self.set_fp32(rd, f16_to_f32(self.fp16(rn))), (3, 1) => self.set_fp64(rd, f64::from(f16_to_f32(self.fp16(rn)))), _ => return Step::Illegal,
}
return Step::Next;
}
match ftype {
0 => {
let a = self.fp32(rn);
let r = match opcode {
0b000000 => a, 0b000001 => a.abs(), 0b000010 => -a, 0b000011 => a.sqrt(), 0b001000 => a.round_ties_even(), 0b001001 => a.ceil(), 0b001010 => a.floor(), 0b001011 => a.trunc(), 0b001100 => a.round(), _ => return Step::Illegal,
};
self.set_fp32(rd, r);
}
1 => {
let a = self.fp64(rn);
let r = match opcode {
0b000000 => a,
0b000001 => a.abs(),
0b000010 => -a,
0b000011 => a.sqrt(),
0b001000 => a.round_ties_even(),
0b001001 => a.ceil(),
0b001010 => a.floor(),
0b001011 => a.trunc(),
0b001100 => a.round(),
_ => return Step::Illegal,
};
self.set_fp64(rd, r);
}
_ => return Step::Illegal,
}
return Step::Next;
}
if (instr >> 24) & 0x7f == 0b0011110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 0x3 == 0b10
{
let ftype = (instr >> 22) & 3;
let opcode = (instr >> 12) & 0xf;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
match ftype {
0 => {
let (a, b) = (self.fp32(rn), self.fp32(rm));
let r = match opcode {
0b0000 => a * b, 0b0001 => a / b, 0b0010 => a + b, 0b0011 => a - b, 0b0100 => fmax32(a, b), 0b0101 => fmin32(a, b), 0b0110 => a.max(b), 0b0111 => a.min(b), 0b1000 => -(a * b), _ => return Step::Illegal,
};
self.set_fp32(rd, r);
}
1 => {
let (a, b) = (self.fp64(rn), self.fp64(rm));
let r = match opcode {
0b0000 => a * b,
0b0001 => a / b,
0b0010 => a + b,
0b0011 => a - b,
0b0100 => fmax64(a, b),
0b0101 => fmin64(a, b),
0b0110 => a.max(b),
0b0111 => a.min(b),
0b1000 => -(a * b),
_ => return Step::Illegal,
};
self.set_fp64(rd, r);
}
_ => return Step::Illegal,
}
return Step::Next;
}
if (instr >> 24) & 0x7f == 0b0011111 {
let ftype = (instr >> 22) & 3;
let o1 = (instr >> 21) & 1;
let o0 = (instr >> 15) & 1;
let rm = reg_field(instr, 16);
let ra = reg_field(instr, 10);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
match ftype {
0 => {
let (n, m, a) = (self.fp32(rn), self.fp32(rm), self.fp32(ra));
let r = match (o1, o0) {
(0, 0) => n.mul_add(m, a), (0, 1) => (-n).mul_add(m, a), (1, 0) => (-n).mul_add(m, -a), _ => n.mul_add(m, -a), };
self.set_fp32(rd, r);
}
1 => {
let (n, m, a) = (self.fp64(rn), self.fp64(rm), self.fp64(ra));
let r = match (o1, o0) {
(0, 0) => n.mul_add(m, a),
(0, 1) => (-n).mul_add(m, a),
(1, 0) => (-n).mul_add(m, -a),
_ => n.mul_add(m, -a),
};
self.set_fp64(rd, r);
}
_ => return Step::Illegal,
}
return Step::Next;
}
if (instr >> 24) & 0x7f == 0b0011110
&& (instr >> 21) & 1 == 1
&& (instr >> 14) & 0x3 == 0b00
&& (instr >> 10) & 0xf == 0b1000
{
let ftype = (instr >> 22) & 3;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let cmp_zero = (instr >> 3) & 1 == 1; let (a, b) = match ftype {
0 => (
f64::from(self.fp32(rn)),
if cmp_zero {
0.0
} else {
f64::from(self.fp32(rm))
},
),
1 => (self.fp64(rn), if cmp_zero { 0.0 } else { self.fp64(rm) }),
_ => return Step::Illegal,
};
self.set_fcmp_flags(a, b);
return Step::Next;
}
if (instr >> 24) & 0x7f == 0b0011110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 0x3 == 0b01
{
let ftype = (instr >> 22) & 3;
let rm = reg_field(instr, 16);
let cond = (instr >> 12) & 0xf;
let rn = reg_field(instr, 5);
let nzcv = instr & 0xf;
if self.cond_holds(cond) {
let (a, b) = match ftype {
0 => (f64::from(self.fp32(rn)), f64::from(self.fp32(rm))),
1 => (self.fp64(rn), self.fp64(rm)),
_ => return Step::Illegal,
};
self.set_fcmp_flags(a, b);
} else {
self.flags = Flags {
n: nzcv & 8 != 0,
z: nzcv & 4 != 0,
c: nzcv & 2 != 0,
v: nzcv & 1 != 0,
};
}
return Step::Next;
}
if (instr >> 24) & 0x7f == 0b0011110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 0x3 == 0b11
{
let ftype = (instr >> 22) & 3;
let rm = reg_field(instr, 16);
let cond = (instr >> 12) & 0xf;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let src = if self.cond_holds(cond) { rn } else { rm };
match ftype {
0 => self.set_fp32(rd, self.fp32(src)),
1 => self.set_fp64(rd, self.fp64(src)),
_ => return Step::Illegal,
}
return Step::Next;
}
if (instr >> 24) & 0x7f == 0b0011110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 0x7 == 0b100
&& (instr >> 5) & 0x1f == 0
{
let ftype = (instr >> 22) & 3;
let imm8 = (instr >> 13) & 0xff;
let rd = reg_field(instr, 0);
match ftype {
0 => self.v[rd] = u128::from(vfp_expand_imm32(imm8)),
1 => self.v[rd] = u128::from(vfp_expand_imm64(imm8)),
_ => return Step::Illegal,
}
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 0x3f == 0b000111
{
let q = (instr >> 30) & 1;
let u = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let (vn, vm, vd) = (self.v[rn], self.v[rm], self.v[rd]);
let out = match (u, size) {
(0, 0b00) => vn & vm, (0, 0b01) => vn & !vm, (0, 0b10) => vn | vm, (0, 0b11) => vn | !vm, (1, 0b00) => vn ^ vm, (1, 0b01) => (vd & vn) | (!vd & vm), (1, 0b10) => (vm & vn) | (!vm & vd), _ => (!vm & vn) | (vm & vd), };
let mask = if q == 1 { u128::MAX } else { ones_u128(64) };
self.v[rd] = out & mask;
return Step::Next;
}
if (instr >> 24) & 0x9f == 0b0_0001110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 1 == 1
&& matches!(
((instr >> 29) & 1, (instr >> 23) & 1, (instr >> 11) & 0x1f),
(0, 0 | 1, 0b11010 | 0b11001 | 0b11111) | (1, 0, 0b11011 | 0b11111) | (1, 1, 0b11010) | (0, 0, 0b11100) | (1, 0 | 1, 0b11100) )
{
let q = (instr >> 30) & 1;
let uns = (instr >> 29) & 1;
let asub = (instr >> 23) & 1; let dbl = (instr >> 22) & 1;
let opcode = (instr >> 11) & 0x1f;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
if dbl == 1 && q == 0 {
return Step::Illegal; }
self.v[rd] = if dbl == 0 {
let lanes = if q == 1 { 4 } else { 2 };
let mut result = 0u128;
for i in 0..lanes {
let sh = i * 32;
let lhs = f32::from_bits(((self.v[rn] >> sh) & 0xffff_ffff) as u32);
let rhs = f32::from_bits(((self.v[rm] >> sh) & 0xffff_ffff) as u32);
let acc = || f32::from_bits(((self.v[rd] >> sh) & 0xffff_ffff) as u32);
let bits = match (uns, asub, opcode) {
(0, 0, 0b11010) => (lhs + rhs).to_bits(),
(0, 1, 0b11010) => (lhs - rhs).to_bits(),
(1, 0, 0b11011) => (lhs * rhs).to_bits(),
(1, 0, 0b11111) => (lhs / rhs).to_bits(),
(0, 0, 0b11001) => lhs.mul_add(rhs, acc()).to_bits(),
(0, 1, 0b11001) => (-lhs).mul_add(rhs, acc()).to_bits(),
(1, 1, 0b11010) => (lhs - rhs).abs().to_bits(), (0, 0, 0b11111) => (2.0 - lhs * rhs).to_bits(), (0, 1, 0b11111) => ((3.0 - lhs * rhs) / 2.0).to_bits(), (0, 0, 0b11100) => fbits_bool32(fp_eq(f64::from(lhs), f64::from(rhs))),
(1, 0, 0b11100) => fbits_bool32(f64::from(lhs) >= f64::from(rhs)),
_ => fbits_bool32(f64::from(lhs) > f64::from(rhs)), };
result |= u128::from(bits) << sh;
}
result
} else {
let mut result = 0u128;
for i in 0..2u32 {
let sh = i * 64;
let lhs = f64::from_bits(((self.v[rn] >> sh) & u128::from(u64::MAX)) as u64);
let rhs = f64::from_bits(((self.v[rm] >> sh) & u128::from(u64::MAX)) as u64);
let acc = || f64::from_bits(((self.v[rd] >> sh) & u128::from(u64::MAX)) as u64);
let bits = match (uns, asub, opcode) {
(0, 0, 0b11010) => (lhs + rhs).to_bits(),
(0, 1, 0b11010) => (lhs - rhs).to_bits(),
(1, 0, 0b11011) => (lhs * rhs).to_bits(),
(1, 0, 0b11111) => (lhs / rhs).to_bits(),
(0, 0, 0b11001) => lhs.mul_add(rhs, acc()).to_bits(),
(0, 1, 0b11001) => (-lhs).mul_add(rhs, acc()).to_bits(),
(1, 1, 0b11010) => (lhs - rhs).abs().to_bits(), (0, 0, 0b11111) => (2.0 - lhs * rhs).to_bits(), (0, 1, 0b11111) => ((3.0 - lhs * rhs) / 2.0).to_bits(), (0, 0, 0b11100) => fbits_bool64(fp_eq(lhs, rhs)),
(1, 0, 0b11100) => fbits_bool64(lhs >= rhs),
_ => fbits_bool64(lhs > rhs), };
result |= u128::from(bits) << sh;
}
result
};
return Step::Next;
}
if (instr >> 24) & 0x9f == 0b0_0011110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 1 == 1
&& matches!(
((instr >> 29) & 1, (instr >> 23) & 1, (instr >> 11) & 0x1f),
(1, 1, 0b11010) | (0, 0 | 1, 0b11111) )
{
let uns = (instr >> 29) & 1;
let asub = (instr >> 23) & 1;
let dbl = (instr >> 22) & 1;
let opcode = (instr >> 11) & 0x1f;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
if dbl == 0 {
let (a, b) = (self.fp32(rn), self.fp32(rm));
let r = match (uns, asub, opcode) {
(1, 1, 0b11010) => (a - b).abs(), (0, 0, 0b11111) => 2.0 - a * b, _ => (3.0 - a * b) / 2.0, };
self.set_fp32(rd, r);
} else {
let (a, b) = (self.fp64(rn), self.fp64(rm));
let r = match (uns, asub, opcode) {
(1, 1, 0b11010) => (a - b).abs(),
(0, 0, 0b11111) => 2.0 - a * b,
_ => (3.0 - a * b) / 2.0,
};
self.set_fp64(rd, r);
}
return Step::Next;
}
if (instr >> 24) & 0x9f == 0b0_0001110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 1 == 1
&& matches!((instr >> 11) & 0x1f, 0b10111 | 0b10100 | 0b10101)
{
let q = (instr >> 30) & 1;
let uns = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let opcode = (instr >> 11) & 0x1f;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
if opcode == 0b10111 && uns == 1 {
return Step::Illegal; }
let esize = 8u32 << size;
let lanes = (if q == 1 { 128 } else { 64 }) / esize;
let mask = ones_u128(esize);
let lane = |v: u128, i: u32| (v >> (i * esize)) & mask;
let concat = |v0: u128, v1: u128, j: u32| {
if j < lanes {
lane(v0, j)
} else {
lane(v1, j - lanes)
}
};
let mut result = 0u128;
for i in 0..lanes {
let (lhs, rhs) = (
concat(self.v[rn], self.v[rm], 2 * i),
concat(self.v[rn], self.v[rm], 2 * i + 1),
);
let pair = match (opcode, uns) {
(0b10111, _) => lhs.wrapping_add(rhs) & mask, (0b10100, 0) => {
if sign_extend(lhs as u64, esize) >= sign_extend(rhs as u64, esize) {
lhs
} else {
rhs
}
}
(0b10100, _) => {
if lhs >= rhs { lhs } else { rhs } }
(0b10101, 0) => {
if sign_extend(lhs as u64, esize) <= sign_extend(rhs as u64, esize) {
lhs
} else {
rhs
}
}
_ => {
if lhs <= rhs { lhs } else { rhs } }
};
result |= pair << (i * esize);
}
self.v[rd] = result;
return Step::Next;
}
if (instr >> 24) & 0x9f == 0b0_0001110
&& (instr >> 21) & 1 == 1
&& (instr >> 10) & 1 == 1
&& (instr >> 29) & 1 == 1
&& (instr >> 23) & 1 == 0
&& matches!((instr >> 11) & 0x1f, 0b11010 | 0b11110)
{
let q = (instr >> 30) & 1;
let dbl = (instr >> 22) & 1;
let is_max = (instr >> 11) & 0x1f == 0b11110;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
if dbl == 1 && q == 0 {
return Step::Illegal;
}
self.v[rd] = if dbl == 0 {
let lanes = if q == 1 { 4 } else { 2 };
let lane = |v: u128, i: u32| f32::from_bits(((v >> (i * 32)) & 0xffff_ffff) as u32);
let concat = |v0: u128, v1: u128, j: u32| {
if j < lanes {
lane(v0, j)
} else {
lane(v1, j - lanes)
}
};
let mut result = 0u128;
for i in 0..lanes {
let (a, b) = (
concat(self.v[rn], self.v[rm], 2 * i),
concat(self.v[rn], self.v[rm], 2 * i + 1),
);
let r = if is_max { fmax32(a, b) } else { a + b };
result |= u128::from(r.to_bits()) << (i * 32);
}
result
} else {
let lane = |v: u128, i: u32| {
f64::from_bits(((v >> (i * 64)) & u128::from(u64::MAX)) as u64)
};
let concat = |v0: u128, v1: u128, j: u32| {
if j < 2 { lane(v0, j) } else { lane(v1, j - 2) }
};
let mut result = 0u128;
for i in 0..2u32 {
let (a, b) = (
concat(self.v[rn], self.v[rm], 2 * i),
concat(self.v[rn], self.v[rm], 2 * i + 1),
);
let r = if is_max { fmax64(a, b) } else { a + b };
result |= u128::from(r.to_bits()) << (i * 64);
}
result
};
return Step::Next;
}
if (instr >> 24) & 0x9f == 0b0_0001110 && (instr >> 21) & 1 == 1 && (instr >> 10) & 1 == 1 {
let q = (instr >> 30) & 1;
let u = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let opcode = (instr >> 11) & 0x1f;
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let op = match (u, opcode) {
(0, 0b10000) => 0u8,
(1, 0b10000) => 1,
(1, 0b10001) => 2,
(0, 0b00110) => 3,
(0, 0b00111) => 4,
(1, 0b00110) => 5,
(1, 0b00111) => 6,
(0, 0b01000) => 7,
(1, 0b01000) => 8,
(0, 0b10011) => 9,
(0, 0b10010) => 10,
(1, 0b10010) => 11,
(0, 0b01100) => 12,
(0, 0b01101) => 13,
(1, 0b01100) => 14,
(1, 0b01101) => 15,
(0, 0b00001) => 16,
(1, 0b00001) => 17,
(0, 0b00101) => 18,
(1, 0b00101) => 19,
(0, 0b01001) => 20,
(1, 0b01001) => 21,
(0, 0b01011) => 22,
(1, 0b01011) => 23,
_ => return Step::Illegal,
};
let esize = 8u32 << size;
let lanes = (if q == 1 { 128 } else { 64 }) / esize;
let mask = ones_u128(esize);
let mut result = 0u128;
for i in 0..lanes {
let sh = i * esize;
let x = (self.v[rn] >> sh) & mask;
let y = (self.v[rm] >> sh) & mask;
let lane = match op {
0 => x.wrapping_add(y) & mask,
1 => x.wrapping_sub(y) & mask,
2 => {
if x == y {
mask
} else {
0
}
}
3 => {
if sign_extend(x as u64, esize) > sign_extend(y as u64, esize) {
mask
} else {
0
}
}
4 => {
if sign_extend(x as u64, esize) >= sign_extend(y as u64, esize) {
mask
} else {
0
}
}
5 => {
if x > y {
mask
} else {
0
}
}
6 => {
if x >= y {
mask
} else {
0
}
}
7 | 8 => {
let amt = sign_extend(y as u64 & 0xff, 8);
simd_shl(x, amt, esize, op == 7)
}
9 => x.wrapping_mul(y) & mask, 10 => {
let acc = (self.v[rd] >> sh) & mask;
acc.wrapping_add(x.wrapping_mul(y)) & mask
}
11 => {
let acc = (self.v[rd] >> sh) & mask;
acc.wrapping_sub(x.wrapping_mul(y)) & mask
}
12 => {
if sign_extend(x as u64, esize) > sign_extend(y as u64, esize) {
x
} else {
y
}
}
13 => {
if sign_extend(x as u64, esize) < sign_extend(y as u64, esize) {
x
} else {
y
}
}
14 => {
if x > y { x } else { y } }
15 => {
if x < y { x } else { y } }
16 => signed_sat(
i128::from(sign_extend(x as u64, esize))
+ i128::from(sign_extend(y as u64, esize)),
esize,
), 17 => unsigned_sat(x as i128 + y as i128, esize), 18 => signed_sat(
i128::from(sign_extend(x as u64, esize))
- i128::from(sign_extend(y as u64, esize)),
esize,
), 19 => unsigned_sat(x as i128 - y as i128, esize), 20 => sat_shl(x, sign_extend(y as u64 & 0xff, 8), esize, true, false), 21 => sat_shl(x, sign_extend(y as u64 & 0xff, 8), esize, false, false), 22 => sat_shl(x, sign_extend(y as u64 & 0xff, 8), esize, true, true), _ => sat_shl(x, sign_extend(y as u64 & 0xff, 8), esize, false, true), };
result |= lane << sh;
}
self.v[rd] = result;
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 29) & 1 == 1
&& (instr >> 22) & 3 == 0
&& (instr >> 17) & 0x1f == 0b1_0000
&& (instr >> 12) & 0x1f == 0b0_0101
&& (instr >> 10) & 3 == 0b10
{
let q = (instr >> 30) & 1;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let mask = if q == 1 { u128::MAX } else { ones_u128(64) };
self.v[rd] = !self.v[rn] & mask;
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 17) & 0x1f == 0b1_0000
&& (instr >> 10) & 3 == 0b10
&& (instr >> 12) & 0x1f <= 1
{
let u = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let opcode = (instr >> 12) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let container = match (u, opcode) {
(0, 0b00000) => 64u32, (1, 0b00000) => 32, (0, 0b00001) => 16, _ => return Step::Illegal,
};
let esize = 8u32 << size;
if esize >= container {
return Step::Illegal;
}
let q = (instr >> 30) & 1;
let width = if q == 1 { 128 } else { 64 };
let per = container / esize; let mask = ones_u128(esize);
let mut result = 0u128;
let mut i = 0u32;
while i < width / esize {
let c = i / per; let within = i % per;
let src = c * per + (per - 1 - within);
let lane = (self.v[rn] >> (src * esize)) & mask;
result |= lane << (i * esize);
i += 1;
}
self.v[rd] = result;
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 17) & 0x1f == 0b1_0000
&& (instr >> 10) & 3 == 0b10
&& matches!((instr >> 12) & 0x1f, 0b10010 | 0b10100)
{
let q = (instr >> 30) & 1;
let u = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let opcode = (instr >> 12) & 0x1f;
if size == 3 {
return Step::Illegal;
}
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = 8u32 << size; let lanes = 64 / esize;
let mask = ones_u128(esize);
let src_mask = ones_u128(esize * 2);
let mut narrow = 0u128;
for i in 0..lanes {
let wide = (self.v[rn] >> (i * esize * 2)) & src_mask;
let lane = match (u, opcode) {
(0, 0b10010) => wide & mask, (0, 0b10100) => {
signed_sat(i128::from(sign_extend(wide as u64, esize * 2)), esize)
} (1, 0b10100) => unsigned_sat(wide as i128, esize), _ => unsigned_sat(i128::from(sign_extend(wide as u64, esize * 2)), esize), };
narrow |= lane << (i * esize);
}
self.v[rd] = if q == 1 {
(self.v[rd] & ones_u128(64)) | (narrow << 64)
} else {
narrow
};
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 17) & 0x1f == 0b1_0000
&& (instr >> 10) & 3 == 0b10
&& (instr >> 12) & 0x1f == 0b0_0011
{
let q = (instr >> 30) & 1;
let u = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = 8u32 << size;
let width = if q == 1 { 128 } else { 64 };
let lanes = width / esize;
let mask = ones_u128(esize);
let mut result = 0u128;
for i in 0..lanes {
let sh = i * esize;
let vd = (self.v[rd] >> sh) & mask;
let vn = (self.v[rn] >> sh) & mask;
let lane = if u == 0 {
let sum = i128::from(sign_extend(vd as u64, esize)) + vn as i128;
signed_sat(sum, esize)
} else {
let sum = vd as i128 + i128::from(sign_extend(vn as u64, esize));
unsigned_sat(sum, esize)
};
result |= lane << sh;
}
self.v[rd] = result;
return Step::Next;
}
if ((instr >> 24) & 0x1f == 0b0_1110 || (instr >> 24) & 0x1f == 0b1_1110)
&& (instr >> 17) & 0x1f == 0b1_0000
&& (instr >> 10) & 3 == 0b10
&& matches!(
(instr >> 12) & 0x1f,
0b0_1101 | 0b0_1100 | 0b0_1110 | 0b1_1101 | 0b1_1100
)
{
let scalar = (instr >> 24) & 0x1f == 0b1_1110;
let q = (instr >> 30) & 1;
let uns = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let opcode = (instr >> 12) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
if size >> 1 == 0 {
return Step::Illegal; }
let dbl = size & 1 == 1;
if opcode == 0b1_1100 {
if scalar || dbl {
return Step::Illegal;
}
let lanes = if q == 1 { 4 } else { 2 };
let mut result = 0u128;
for i in 0..lanes {
let sh = i * 32;
let x = ((self.v[rn] >> sh) & 0xffff_ffff) as u32;
let r = if uns == 0 {
u32_recip_estimate(x)
} else {
u32_rsqrt_estimate(x)
};
result |= u128::from(r) << sh;
}
self.v[rd] = result;
return Step::Next;
}
if dbl && !scalar && q == 0 {
return Step::Illegal; }
let lanes: u32 = if scalar {
1
} else if dbl {
2
} else if q == 1 {
4
} else {
2
};
let esize = if dbl { 64u32 } else { 32 };
let mask = ones_u128(esize);
let mut result = self.v[rd];
for i in 0..lanes {
let sh = i * esize;
let bits: u128 = if dbl {
let x = f64::from_bits(((self.v[rn] >> sh) & mask) as u64);
let r = match (opcode, uns) {
(0b0_1101, 0) => fbits_bool64(fp_eq(x, 0.0)), (0b0_1101, 1) => fbits_bool64(x <= 0.0), (0b0_1100, 0) => fbits_bool64(x > 0.0), (0b0_1100, 1) => fbits_bool64(x >= 0.0), (0b0_1110, 0) => fbits_bool64(x < 0.0), (0b1_1101, 0) => (1.0f64 / x).to_bits(), (0b1_1101, 1) => (1.0f64 / x.sqrt()).to_bits(), _ => return Step::Illegal,
};
u128::from(r)
} else {
let x = f32::from_bits(((self.v[rn] >> sh) & mask) as u32);
let r = match (opcode, uns) {
(0b0_1101, 0) => fbits_bool32(fp_eq(f64::from(x), 0.0)),
(0b0_1101, 1) => fbits_bool32(x <= 0.0),
(0b0_1100, 0) => fbits_bool32(x > 0.0),
(0b0_1100, 1) => fbits_bool32(x >= 0.0),
(0b0_1110, 0) => fbits_bool32(x < 0.0),
(0b1_1101, 0) => (1.0f32 / x).to_bits(),
(0b1_1101, 1) => (1.0f32 / x.sqrt()).to_bits(),
_ => return Step::Illegal,
};
u128::from(r)
};
result = (result & !(mask << sh)) | (bits << sh);
}
self.v[rd] = if scalar {
result & mask
} else if q == 1 {
result
} else {
result & ones_u128(64)
};
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 17) & 0x1f == 0b1_0000
&& (instr >> 10) & 3 == 0b10
{
let q = (instr >> 30) & 1;
let uns = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let hi = size >> 1;
let dbl = size & 1;
let opcode = (instr >> 12) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
match (uns, hi, opcode) {
(0 | 1, 1, 0b01111) | (1, 1, 0b11111) => {
if dbl == 1 && q == 0 {
return Step::Illegal;
}
self.v[rd] = if dbl == 0 {
let lanes = if q == 1 { 4 } else { 2 };
let mut result = 0u128;
for i in 0..lanes {
let sh = i * 32;
let val = f32::from_bits(((self.v[rn] >> sh) & 0xffff_ffff) as u32);
let res = match (uns, opcode) {
(0, 0b01111) => val.abs(),
(1, 0b01111) => -val,
_ => val.sqrt(),
};
result |= u128::from(res.to_bits()) << sh;
}
result
} else {
let mut result = 0u128;
for i in 0..2u32 {
let sh = i * 64;
let val =
f64::from_bits(((self.v[rn] >> sh) & u128::from(u64::MAX)) as u64);
let res = match (uns, opcode) {
(0, 0b01111) => val.abs(),
(1, 0b01111) => -val,
_ => val.sqrt(),
};
result |= u128::from(res.to_bits()) << sh;
}
result
};
}
(0 | 1, _, 0b01011) => {
let esize = 8u32 << size;
let lanes = (if q == 1 { 128 } else { 64 }) / esize;
let mask = ones_u128(esize);
let mut result = 0u128;
for i in 0..lanes {
let sh = i * esize;
let val = (self.v[rn] >> sh) & mask;
let signed = sign_extend(val as u64, esize);
let res = if uns == 0 {
signed.wrapping_abs()
} else {
0i64.wrapping_sub(signed)
};
result |= (u128::from(res as u64) & mask) << sh;
}
self.v[rd] = result;
}
_ => return Step::Illegal,
}
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 21) & 1 == 1
&& (instr >> 12) & 0xf == 0b0000
&& (instr >> 10) & 3 == 0b00
{
let q = (instr >> 30) & 1;
let unsigned = (instr >> 29) & 1 == 1;
let size = (instr >> 22) & 3;
if size == 3 {
return Step::Illegal; }
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = 8u32 << size;
let mask = ones_u128(esize);
let src_shift = if q == 1 { 64 } else { 0 };
let lanes = 64 / esize;
let mut result = 0u128;
for i in 0..lanes {
let sh = i * esize;
let xn = (self.v[rn] >> (src_shift + sh)) & mask;
let xm = (self.v[rm] >> (src_shift + sh)) & mask;
let (wn, wm) = if unsigned {
(xn, xm)
} else {
(
(sign_extend(xn as u64, esize) as u128) & ones_u128(2 * esize),
(sign_extend(xm as u64, esize) as u128) & ones_u128(2 * esize),
)
};
let sum = (wn.wrapping_add(wm)) & ones_u128(2 * esize);
result |= sum << (i * 2 * esize);
}
self.v[rd] = result;
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 21) & 1 == 1
&& (instr >> 12) & 0xf == 0b0001
&& (instr >> 10) & 3 == 0b00
{
let q = (instr >> 30) & 1;
let unsigned = (instr >> 29) & 1 == 1;
let size = (instr >> 22) & 3;
if size == 3 {
return Step::Illegal;
}
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = 8u32 << size;
let mask = ones_u128(esize);
let src_shift = if q == 1 { 64 } else { 0 };
let lanes = 64 / esize;
let wide_mask = ones_u128(2 * esize);
let mut result = 0u128;
for i in 0..lanes {
let sh = i * 2 * esize;
let xn = (self.v[rn] >> sh) & wide_mask;
let xm = (self.v[rm] >> (src_shift + i * esize)) & mask;
let wm = if unsigned {
xm
} else {
(sign_extend(xm as u64, esize) as u128) & wide_mask
};
result |= (xn.wrapping_add(wm) & wide_mask) << sh;
}
self.v[rd] = result;
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 29) & 1 == 0
&& (instr >> 21) & 1 == 1
&& (instr >> 12) & 0xf == 0b0100
&& (instr >> 10) & 3 == 0b00
{
let q = (instr >> 30) & 1;
let size = (instr >> 22) & 3;
if size == 3 {
return Step::Illegal;
}
let rm = reg_field(instr, 16);
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = 8u32 << size; let src_mask = ones_u128(esize * 2);
let lanes = 64 / esize;
let mut narrow = 0u128;
for i in 0..lanes {
let sh = i * esize * 2;
let xn = (self.v[rn] >> sh) & src_mask;
let xm = (self.v[rm] >> sh) & src_mask;
let sum = xn.wrapping_add(xm) & src_mask;
narrow |= (sum >> esize) << (i * esize);
}
self.v[rd] = if q == 1 {
(self.v[rd] & ones_u128(64)) | (narrow << 64)
} else {
narrow
};
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 17) & 0x1f == 0b1_1000
&& (instr >> 10) & 3 == 0b10
&& matches!((instr >> 12) & 0x1f, 0b0_1100 | 0b0_1111)
{
let q = (instr >> 30) & 1;
let is_min = (instr >> 22) & 1 == 1;
let is_nm = (instr >> 12) & 0x1f == 0b0_1100;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
if q == 0 {
return Step::Illegal; }
let lane = |i: u32| f32::from_bits(((self.v[rn] >> (i * 32)) & 0xffff_ffff) as u32);
let mut acc = lane(0);
for i in 1..4 {
let v = lane(i);
acc = match (is_nm, is_min) {
(true, true) => acc.min(v), (true, false) => acc.max(v), (false, true) => fmin32(acc, v), (false, false) => fmax32(acc, v), };
}
self.set_fp32(rd, acc);
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b0_1110
&& (instr >> 17) & 0x1f == 0b1_1000
&& (instr >> 10) & 3 == 0b10
{
let q = (instr >> 30) & 1;
let u = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let opcode = (instr >> 12) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
if size == 3 {
return Step::Illegal;
}
let esize = 8u32 << size;
let width = if q == 1 { 128 } else { 64 };
let lanes = width / esize;
let mask = ones_u128(esize);
match (u, opcode) {
(0, 0b1_1011) => {
let mut sum = 0u128;
for i in 0..lanes {
sum = sum.wrapping_add((self.v[rn] >> (i * esize)) & mask);
}
self.v[rd] = sum & mask;
}
(1, 0b0_0011) => {
let mut sum = 0u128;
for i in 0..lanes {
sum = sum.wrapping_add((self.v[rn] >> (i * esize)) & mask);
}
self.v[rd] = sum & ones_u128(esize * 2);
}
(0 | 1, 0b0_1010 | 0b1_1010) => {
let is_min = opcode == 0b1_1010;
let lane_at = |i: u32| (self.v[rn] >> (i * esize)) & mask;
let mut acc = lane_at(0);
for i in 1..lanes {
let v = lane_at(i);
let better = if u == 0 {
let (sv, sacc) =
(sign_extend(v as u64, esize), sign_extend(acc as u64, esize));
if is_min { sv < sacc } else { sv > sacc }
} else if is_min {
v < acc
} else {
v > acc
};
if better {
acc = v;
}
}
self.v[rd] = acc;
}
_ => return Step::Illegal,
}
return Step::Next;
}
if (instr >> 24) & 0x1f == 0b1_1110
&& (instr >> 30) & 1 == 1
&& (instr >> 17) & 0x1f == 0b1_1000
&& (instr >> 10) & 3 == 0b10
{
let u = (instr >> 29) & 1;
let sz = (instr >> 22) & 1;
let opcode = (instr >> 12) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
match (u, opcode) {
(0, 0b1_1011) => {
let lo = self.v[rn] as u64;
let hi = (self.v[rn] >> 64) as u64;
self.v[rd] = u128::from(lo.wrapping_add(hi));
}
(1, 0b0_1101) => {
if sz == 0 {
let a = f32::from_bits(self.v[rn] as u32);
let b = f32::from_bits((self.v[rn] >> 32) as u32);
self.set_fp32(rd, a + b);
} else {
let a = f64::from_bits(self.v[rn] as u64);
let b = f64::from_bits((self.v[rn] >> 64) as u64);
self.set_fp64(rd, a + b);
}
}
_ => return Step::Illegal,
}
return Step::Next;
}
if (instr >> 23) & 0x3f == 0b0_11110 && (instr >> 10) & 0x3f == 0b10_1001 {
let q = (instr >> 30) & 1;
let unsigned = (instr >> 29) & 1 == 1;
let immh = (instr >> 19) & 0xf;
let immb = (instr >> 16) & 7;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize = if immh & 0b1000 != 0 {
return Step::Illegal;
} else if immh & 0b0100 != 0 {
32
} else if immh & 0b0010 != 0 {
16
} else if immh & 0b0001 != 0 {
8
} else {
return Step::Illegal;
};
let shift = ((immh << 3) | immb) - esize;
let src = if q == 0 {
self.v[rn] as u64
} else {
(self.v[rn] >> 64) as u64
};
let lanes = 64 / esize;
let mut result = 0u128;
for i in 0..lanes {
let e = (src >> (i * esize)) & ones(esize);
let ext = if unsigned {
u128::from(e)
} else {
(sign_extend(e, esize) as u128) & ones_u128(2 * esize)
};
let widened = (ext << shift) & ones_u128(2 * esize);
result |= widened << (i * 2 * esize);
}
self.v[rd] = result;
return Step::Next;
}
if ((instr >> 23) & 0x3f == 0b0_11110 || (instr >> 23) & 0x3f == 0b1_11110)
&& (instr >> 10) & 1 == 1
{
let scalar = (instr >> 23) & 0x3f == 0b1_11110;
let q = (instr >> 30) & 1;
let u = (instr >> 29) & 1;
let immh = (instr >> 19) & 0xf;
let immb = (instr >> 16) & 7;
let opcode = (instr >> 11) & 0x1f;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let esize: u32 = if immh & 0b1000 != 0 {
64
} else if immh & 0b0100 != 0 {
32
} else if immh & 0b0010 != 0 {
16
} else if immh & 0b0001 != 0 {
8
} else {
return Step::Illegal;
};
let immhb = (immh << 3) | immb;
let lanes_bits = if scalar {
esize
} else if q == 1 {
128
} else {
64
};
let mut result = 0u128;
let mut i = 0;
while i < lanes_bits / esize {
let e = ((self.v[rn] >> (i * esize)) & ones_u128(esize)) as u64;
let lane = match opcode {
0b01010 => e << (immhb - esize), 0b00000 => {
let sh = (2 * esize - immhb).min(63);
if u == 1 {
e >> sh
} else {
(sign_extend(e, esize) >> sh) as u64
}
}
_ => return Step::Illegal,
};
result |= u128::from(lane & ones(esize)) << (i * esize);
i += 1;
}
self.v[rd] = result;
return Step::Next;
}
if ((instr >> 24) & 0x1f == 0b0_1111 || (instr >> 24) & 0x1f == 0b1_1111)
&& (instr >> 10) & 1 == 0
{
let scalar = (instr >> 24) & 0x1f == 0b1_1111;
let q = (instr >> 30) & 1;
let u = (instr >> 29) & 1;
let size = (instr >> 22) & 3;
let l = (instr >> 21) & 1;
let m = (instr >> 20) & 1;
let rm_lo4 = (instr >> 16) & 0xf; let opcode = (instr >> 12) & 0xf;
let h = (instr >> 11) & 1;
let rn = reg_field(instr, 5);
let rd = reg_field(instr, 0);
let (esize, rm, index): (u32, usize, u32) = match size {
0b01 => (16, rm_lo4 as usize, (h << 2) | (l << 1) | m),
0b10 => (32, (((m << 4) | rm_lo4) & 0x1f) as usize, (h << 1) | l),
0b11 => (64, (((m << 4) | rm_lo4) & 0x1f) as usize, h),
_ => return Step::Illegal,
};
let mask = ones_u128(esize);
let m_elem = (self.v[rm] >> (index * esize)) & mask;
if matches!(opcode, 0b0001 | 0b0101 | 0b1001) {
if esize != 32 && esize != 64 {
return Step::Illegal;
}
if esize == 64 && !scalar && q == 0 {
return Step::Illegal; }
let lanes: u32 = if scalar {
1
} else if esize == 32 {
if q == 1 { 4 } else { 2 }
} else {
2
};
let mut result = self.v[rd];
for i in 0..lanes {
let sh = i * esize;
let bits: u128 = if esize == 32 {
let mv = f32::from_bits(m_elem as u32);
let nv = f32::from_bits(((self.v[rn] >> sh) & mask) as u32);
let acc = f32::from_bits(((self.v[rd] >> sh) & mask) as u32);
let r = match (opcode, u) {
(0b0001, 0) => nv.mul_add(mv, acc), (0b0101, 0) => (-nv).mul_add(mv, acc), (0b1001, 0) => nv * mv, (0b1001, 1) => fmulx32(nv, mv), _ => return Step::Illegal,
};
u128::from(r.to_bits())
} else {
let mv = f64::from_bits(m_elem as u64);
let nv = f64::from_bits(((self.v[rn] >> sh) & mask) as u64);
let acc = f64::from_bits(((self.v[rd] >> sh) & mask) as u64);
let r = match (opcode, u) {
(0b0001, 0) => nv.mul_add(mv, acc),
(0b0101, 0) => (-nv).mul_add(mv, acc),
(0b1001, 0) => nv * mv,
(0b1001, 1) => fmulx64(nv, mv),
_ => return Step::Illegal,
};
u128::from(r.to_bits())
};
result = (result & !(mask << sh)) | (bits << sh);
}
self.v[rd] = if scalar {
result & mask
} else if esize == 32 && q == 0 {
result & ones_u128(64)
} else {
result
};
return Step::Next;
}
if scalar || esize == 64 {
return Step::Illegal;
}
match (opcode, u) {
(0b1000, 0) | (0b0000 | 0b0100, 1) => {
let lanes = (if q == 1 { 128 } else { 64 }) / esize;
let mut result = 0u128;
for i in 0..lanes {
let sh = i * esize;
let n_elem = (self.v[rn] >> sh) & mask;
let prod = n_elem.wrapping_mul(m_elem) & mask;
let lane = match (opcode, u) {
(0b1000, 0) => prod, (0b0000, 1) => ((self.v[rd] >> sh) & mask).wrapping_add(prod) & mask, _ => ((self.v[rd] >> sh) & mask).wrapping_sub(prod) & mask, };
result |= lane << sh;
}
self.v[rd] = result;
}
(0b1010, _) | (0b1011, 0) => {
let src_shift = if q == 1 { 64 } else { 0 };
let lanes = 64 / esize;
let wide_mask = ones_u128(esize * 2);
let mut result = 0u128;
for i in 0..lanes {
let n_elem = (self.v[rn] >> (src_shift + i * esize)) & mask;
let lane = if opcode == 0b1011 {
signed_sat(
2 * i128::from(sign_extend(n_elem as u64, esize))
* i128::from(sign_extend(m_elem as u64, esize)),
esize * 2,
)
} else if u == 0 {
((i128::from(sign_extend(n_elem as u64, esize))
* i128::from(sign_extend(m_elem as u64, esize)))
as u128)
& wide_mask
} else {
n_elem.wrapping_mul(m_elem) & wide_mask
};
result |= lane << (i * esize * 2);
}
self.v[rd] = result;
}
_ => return Step::Illegal,
}
return Step::Next;
}
if (instr >> 25) & 0x3f == 0b01_1011 {
let op = (instr >> 24) & 1;
let bitpos = (((instr >> 31) & 1) << 5) | ((instr >> 19) & 0x1f);
let rt = reg_field(instr, 0);
let bit = (self.read_x(rt) >> bitpos) & 1;
let take = if op == 0 { bit == 0 } else { bit == 1 };
if take {
let off = sign_extend(u64::from((instr >> 5) & 0x3fff), 14) << 2;
return self.branch(off);
}
return Step::Next;
}
Step::Illegal
}
}
impl Vcpu for Aarch64Interp {
fn run(&mut self, mem: &mut GuestMemory) -> Result<Exit, VcpuError> {
for _ in 0..MAX_STEPS {
let Ok(instr) = mem.read_u32(self.pc) else {
return Ok(Exit::MemFault {
addr: self.pc,
write: false,
});
};
match self.exec(instr, mem) {
Step::Next => self.pc = self.pc.wrapping_add(4),
Step::Branched => {}
Step::Syscall => return Ok(Exit::Syscall),
Step::Illegal => return Ok(Exit::IllegalInstruction { pc: self.pc }),
Step::Fault { addr, write } => return Ok(Exit::MemFault { addr, write }),
}
}
Ok(Exit::Interrupted)
}
fn syscall_nr(&self) -> u64 {
self.x[8]
}
fn syscall_args(&self) -> [u64; 6] {
[
self.x[0], self.x[1], self.x[2], self.x[3], self.x[4], self.x[5],
]
}
fn set_syscall_ret(&mut self, value: u64) {
self.x[0] = value;
self.pc = self.pc.wrapping_add(4);
}
fn reg(&self, idx: usize) -> u64 {
if idx < 31 { self.x[idx] } else { self.sp }
}
fn set_reg(&mut self, idx: usize, value: u64) {
if idx < 31 {
self.x[idx] = value;
} else {
self.sp = value;
}
}
fn pc(&self) -> u64 {
self.pc
}
fn set_pc(&mut self, pc: u64) {
self.pc = pc;
}
fn sp(&self) -> u64 {
self.sp
}
fn set_sp(&mut self, sp: u64) {
self.sp = sp;
}
fn set_tls(&mut self, value: u64) {
self.tpidr = value;
}
fn fork(&self) -> Box<dyn Vcpu> {
Box::new(self.clone())
}
fn reset(&mut self, entry: u64, sp: u64) {
self.x = [0; 31];
self.v = [0; 32];
self.sp = sp;
self.pc = entry;
self.tpidr = 0;
self.fpcr = 0;
self.fpsr = 0;
self.flags = Flags::default();
self.excl_monitor = true;
}
}
const TPIDR_EL0: u32 = 0x5E82;
const TPIDRRO_EL0: u32 = 0x5E83;
const FPCR: u32 = 0x5A20;
const FPSR: u32 = 0x5A21;
const MIDR_EL1: u32 = 0x4000;
const MPIDR_EL1: u32 = 0x4005;
const REVIDR_EL1: u32 = 0x4006;
const CTR_EL0: u32 = 0x5801;
const DCZID_EL0: u32 = 0x5807;
const CNTFRQ_EL0: u32 = 0x5F00;
const CNTVCT_EL0: u32 = 0x5F02;
const CNTVCTSS_EL0: u32 = 0x5F06;
const ID_AA64ISAR0_EL1: u32 = 0x4030;
const ID_AA64PFR0_EL1: u32 = 0x4020;
const MIDR_EL1_VAL: u64 = 0x410F_D0C0;
const CTR_EL0_VAL: u64 = 0x8444_c004;
const DCZID_EL0_VAL: u64 = 0x4;
const DC_ZVA_BLOCK_BYTES: u64 = 64;
const CNTFRQ_EL0_VAL: u64 = 1_000_000_000;
const ID_AA64ISAR0_EL1_VAL: u64 = 0x0021_1110;
const ID_AA64PFR0_EL1_VAL: u64 = 0x0000_0011;
const SYS_DC_ZVA: u32 = 0x1BA1;
const SYS_DC_CVAC: u32 = 0x1BD1;
const SYS_DC_CVAU: u32 = 0x1BD9;
const SYS_DC_CIVAC: u32 = 0x1BF1;
const SYS_DC_IVAC: u32 = 0x03B1;
const SYS_IC_IALLU: u32 = 0x03A8;
const SYS_IC_IALLUIS: u32 = 0x0388;
const SYS_IC_IVAU: u32 = 0x1BA9;
fn reg_field(instr: u32, lsb: u32) -> usize {
((instr >> lsb) & 0x1f) as usize
}
const fn mask_sf(v: u64, sf: u32) -> u64 {
if sf == 0 { v & 0xffff_ffff } else { v }
}
fn shift_reg(v: u64, shift_type: u32, amount: u32, sf: bool) -> u64 {
let width = if sf { 64 } else { 32 };
let amt = amount % width;
let v = if sf { v } else { v & 0xffff_ffff };
let r = match shift_type {
0 => v << amt,
1 => v >> amt,
2 => {
if sf {
((v as i64) >> amt) as u64
} else {
u64::from(((v as u32 as i32) >> amt) as u32)
}
}
_ => {
if sf {
v.rotate_right(amt)
} else {
u64::from((v as u32).rotate_right(amt))
}
}
};
if sf { r } else { r & 0xffff_ffff }
}
fn rbit(v: u64, width: u32) -> u64 {
if width == 64 {
v.reverse_bits()
} else {
u64::from((v as u32).reverse_bits())
}
}
fn rev16(v: u64, width: u32) -> u64 {
let mut r = 0u64;
let mut i = 0;
while i < width {
let h = ((v >> i) & 0xffff) as u16;
r |= u64::from(h.swap_bytes()) << i;
i += 16;
}
r
}
fn rev32(v: u64) -> u64 {
u64::from((v as u32).swap_bytes()) | (u64::from(((v >> 32) as u32).swap_bytes()) << 32)
}
fn cls(v: u64, width: u32) -> u32 {
let sign = (v >> (width - 1)) & 1;
let mut count = 0;
let mut i = width - 1;
while i > 0 {
i -= 1;
if (v >> i) & 1 == sign {
count += 1;
} else {
break;
}
}
count
}
fn udiv(a: u64, b: u64, sf: bool) -> u64 {
if sf {
a.checked_div(b).unwrap_or(0)
} else {
(a as u32).checked_div(b as u32).map_or(0, u64::from)
}
}
fn sdiv(a: u64, b: u64, sf: bool) -> u64 {
if sf {
let (a, b) = (a as i64, b as i64);
if b == 0 { 0 } else { a.wrapping_div(b) as u64 }
} else {
let (a, b) = (a as i32, b as i32);
if b == 0 {
0
} else {
u64::from(a.wrapping_div(b) as u32)
}
}
}
fn extend_reg(val: u64, option: u32, shift: u32) -> u64 {
let extended = match option {
0b000 => val & 0xff, 0b001 => val & 0xffff, 0b010 => val & 0xffff_ffff, 0b100 => sign_extend(val & 0xff, 8) as u64, 0b101 => sign_extend(val & 0xffff, 16) as u64, 0b110 => sign_extend(val & 0xffff_ffff, 32) as u64, _ => val, };
extended << shift
}
fn ones(n: u32) -> u64 {
if n >= 64 { u64::MAX } else { (1u64 << n) - 1 }
}
fn mem_read_sized(mem: &mut GuestMemory, addr: u64, nbytes: usize) -> Option<u64> {
let mut buf = [0u8; 8];
mem.read(addr, &mut buf[..nbytes]).ok()?;
Some(u64::from_le_bytes(buf))
}
fn adv_simd_expand_imm(cmode: u32, op: u32, imm8: u64) -> u64 {
let rep32 = |x: u64| (x & 0xffff_ffff) | ((x & 0xffff_ffff) << 32);
let rep16 = |x: u64| {
let x = x & 0xffff;
x | (x << 16) | (x << 32) | (x << 48)
};
match cmode >> 1 {
0b000 => rep32(imm8),
0b001 => rep32(imm8 << 8),
0b010 => rep32(imm8 << 16),
0b011 => rep32(imm8 << 24),
0b100 => rep16(imm8),
0b101 => rep16(imm8 << 8),
0b110 => {
if cmode & 1 == 0 {
rep32((imm8 << 8) | 0xff)
} else {
rep32((imm8 << 16) | 0xffff)
}
}
_ => {
if cmode == 0b1110 {
if op == 0 {
(imm8 & 0xff) * 0x0101_0101_0101_0101 } else {
let mut r = 0u64;
for i in 0..8 {
if (imm8 >> i) & 1 == 1 {
r |= 0xffu64 << (i * 8);
}
}
r
}
} else if op == 0 {
rep32(u64::from(vfp_expand_imm32(imm8 as u32)))
} else {
vfp_expand_imm64(imm8 as u32)
}
}
}
}
fn vfp_expand_imm32(imm8: u32) -> u32 {
let sign = (imm8 >> 7) & 1;
let b6 = (imm8 >> 6) & 1;
let exp = ((1 - b6) << 7) | (if b6 == 1 { 0x1f } else { 0 } << 2) | ((imm8 >> 4) & 3);
let frac = (imm8 & 0xf) << 19;
(sign << 31) | (exp << 23) | frac
}
fn vfp_expand_imm64(imm8: u32) -> u64 {
let sign = u64::from((imm8 >> 7) & 1);
let b6 = u64::from((imm8 >> 6) & 1);
let exp = ((1 - b6) << 10) | (if b6 == 1 { 0xff } else { 0 } << 2) | u64::from((imm8 >> 4) & 3);
let frac = u64::from(imm8 & 0xf) << 48;
(sign << 63) | (exp << 52) | frac
}
fn simd_shl(x: u128, amt: i64, esize: u32, signed: bool) -> u128 {
let mask = ones_u128(esize);
if amt >= 0 {
let sh = amt.min(i64::from(esize)) as u32;
if sh >= esize { 0 } else { (x << sh) & mask }
} else {
let sh = (-amt).min(i64::from(esize)) as u32;
if signed {
let se = sign_extend(x as u64, esize);
let shifted = if sh >= 64 {
if se < 0 { -1i64 } else { 0 }
} else {
se >> sh.min(63)
};
u128::from(shifted as u64) & mask
} else if sh >= esize {
0
} else {
(x >> sh) & mask
}
}
}
fn signed_sat(v: i128, esize: u32) -> u128 {
let max = (1i128 << (esize - 1)) - 1;
let min = -(1i128 << (esize - 1));
(v.clamp(min, max) as u128) & ones_u128(esize)
}
fn unsigned_sat(v: i128, esize: u32) -> u128 {
let max = (1i128 << esize) - 1;
v.clamp(0, max) as u128
}
fn sat_shl(x: u128, amt: i64, esize: u32, signed: bool, rounding: bool) -> u128 {
if amt >= 0 {
let sh = amt.min(i64::from(esize)) as u32;
if signed {
let se = i128::from(sign_extend(x as u64, esize));
signed_sat(se << sh, esize)
} else {
let shifted = x << sh;
let max = ones_u128(esize);
if shifted > max { max } else { shifted }
}
} else {
let sh = (-amt) as u32; if signed {
let se = i128::from(sign_extend(x as u64, esize));
let r = if sh >= 127 {
if se < 0 { -1 } else { 0 }
} else if rounding {
(se + (1i128 << (sh - 1))) >> sh
} else {
se >> sh
};
(r as u128) & ones_u128(esize)
} else {
let r = if sh >= 128 {
0
} else if rounding {
(x + (1u128 << (sh - 1))) >> sh
} else {
x >> sh
};
r & ones_u128(esize)
}
}
}
fn crc32_step(crc: u32, byte: u8, poly: u32) -> u32 {
let mut crc = crc ^ u32::from(byte);
for _ in 0..8 {
crc = if crc & 1 != 0 {
(crc >> 1) ^ poly
} else {
crc >> 1
};
}
crc
}
fn ones_u128(n: u32) -> u128 {
if n >= 128 {
u128::MAX
} else {
(1u128 << n) - 1
}
}
fn elem_bits(imm5: u32) -> u32 {
if imm5 & 1 != 0 {
8
} else if imm5 & 2 != 0 {
16
} else if imm5 & 4 != 0 {
32
} else {
64
}
}
fn ror_val(v: u64, r: u32, size: u32) -> u64 {
if size == 0 {
return v;
}
let r = r % size;
let v = v & ones(size);
if r == 0 {
return v;
}
((v >> r) | (v << (size - r))) & ones(size)
}
fn replicate(pattern: u64, esize: u32, width: u32) -> u64 {
let pat = pattern & ones(esize);
let mut result = 0u64;
let mut i = 0u32;
while i < width {
result |= pat << i;
i += esize;
}
result & ones(width)
}
fn decode_bit_masks(n: u32, imms: u32, immr: u32, width: u32) -> Option<(u64, u64)> {
let x = (n << 6) | ((!imms) & 0x3f);
if x == 0 {
return None;
}
let len = x.ilog2();
if len < 1 {
return None;
}
let levels = (1u32 << len) - 1;
let s = imms & levels;
let r = immr & levels;
let diff = s.wrapping_sub(r) & levels;
let esize = 1u32 << len;
let wmask = replicate(ror_val(ones(s + 1), r, esize), esize, width);
let tmask = replicate(ones(diff + 1), esize, width);
Some((wmask, tmask))
}
#[allow(clippy::cast_precision_loss)]
fn int_to_f32(v: u64, signed: bool, sf: bool) -> f32 {
match (signed, sf) {
(true, true) => v as i64 as f32,
(true, false) => v as i32 as f32,
(false, true) => v as f32,
(false, false) => v as u32 as f32,
}
}
#[allow(clippy::cast_precision_loss)]
fn int_to_f64(v: u64, signed: bool, sf: bool) -> f64 {
match (signed, sf) {
(true, true) => v as i64 as f64,
(true, false) => f64::from(v as i32),
(false, true) => v as f64,
(false, false) => f64::from(v as u32),
}
}
fn fp_to_int(x: f64, signed: bool, sf: bool) -> u64 {
match (signed, sf) {
(true, true) => x as i64 as u64,
(true, false) => u64::from(x as i32 as u32),
(false, true) => x as u64,
(false, false) => u64::from(x as u32),
}
}
fn fmax32(a: f32, b: f32) -> f32 {
if a.is_nan() || b.is_nan() {
f32::NAN
} else {
a.max(b)
}
}
fn fmin32(a: f32, b: f32) -> f32 {
if a.is_nan() || b.is_nan() {
f32::NAN
} else {
a.min(b)
}
}
fn fmax64(a: f64, b: f64) -> f64 {
if a.is_nan() || b.is_nan() {
f64::NAN
} else {
a.max(b)
}
}
fn fp_eq(a: f64, b: f64) -> bool {
a == b
}
fn fbits_bool32(cond: bool) -> u32 {
if cond { u32::MAX } else { 0 }
}
fn fbits_bool64(cond: bool) -> u64 {
if cond { u64::MAX } else { 0 }
}
fn fmin64(a: f64, b: f64) -> f64 {
if a.is_nan() || b.is_nan() {
f64::NAN
} else {
a.min(b)
}
}
fn fmulx32(a: f32, b: f32) -> f32 {
let a_zero = a.to_bits() & 0x7fff_ffff == 0;
let b_zero = b.to_bits() & 0x7fff_ffff == 0;
if (a_zero && b.is_infinite()) || (a.is_infinite() && b_zero) {
if a.is_sign_negative() ^ b.is_sign_negative() {
-2.0
} else {
2.0
}
} else {
a * b
}
}
fn fmulx64(a: f64, b: f64) -> f64 {
let a_zero = a.to_bits() & 0x7fff_ffff_ffff_ffff == 0;
let b_zero = b.to_bits() & 0x7fff_ffff_ffff_ffff == 0;
if (a_zero && b.is_infinite()) || (a.is_infinite() && b_zero) {
if a.is_sign_negative() ^ b.is_sign_negative() {
-2.0
} else {
2.0
}
} else {
a * b
}
}
fn u32_recip_estimate(operand: u32) -> u32 {
let x = operand.max(0x8000_0000); ((1u64 << 63) / u64::from(x)).min(u64::from(u32::MAX)) as u32
}
fn u32_rsqrt_estimate(operand: u32) -> u32 {
let x = operand.max(0x4000_0000); let r = 2f64.powi(47) / f64::from(x).sqrt();
r.min(f64::from(u32::MAX)) as u32
}
fn f16_to_f32(h: u16) -> f32 {
let sign = u32::from(h & 0x8000) << 16;
let exp = u32::from((h >> 10) & 0x1f);
let frac = u32::from(h & 0x3ff);
if exp == 0 {
if frac == 0 {
return f32::from_bits(sign);
}
let mut e = 0i32;
let mut f = frac;
while f & 0x400 == 0 {
f <<= 1;
e -= 1;
}
f &= 0x3ff;
let exp32 = (127 - 15 + 1 + e) as u32;
return f32::from_bits(sign | (exp32 << 23) | (f << 13));
}
if exp == 0x1f {
return f32::from_bits(sign | 0xff80_0000 | (frac << 13)); }
let exp32 = exp + (127 - 15);
f32::from_bits(sign | (exp32 << 23) | (frac << 13))
}
#[allow(clippy::cast_sign_loss)]
fn f32_to_f16(v: f32) -> u16 {
let bits = v.to_bits();
let sign = ((bits >> 16) & 0x8000) as u16;
let exp = i32::try_from((bits >> 23) & 0xff).unwrap_or(0);
let frac = bits & 0x007f_ffff;
if exp == 0xff {
let payload = if frac != 0 {
(frac >> 13) as u16 | 0x200
} else {
0
};
return sign | 0x7c00 | payload;
}
let hexp = exp - 127 + 15;
if hexp >= 0x1f {
return sign | 0x7c00; }
if hexp <= 0 {
if hexp < -10 {
return sign; }
let frac_full = frac | 0x0080_0000; let shift = (14 - hexp) as u32;
let mut hfrac = (frac_full >> shift) as u16;
let round_bit = 1u32 << (shift - 1);
let rem = frac_full & ((round_bit << 1) - 1);
if rem > round_bit || (rem == round_bit && hfrac & 1 == 1) {
hfrac += 1;
}
return sign | hfrac;
}
let mut hexp16 = hexp as u16;
let mut hfrac = (frac >> 13) as u16;
let round_bit = frac & 0x1000;
let sticky = frac & 0x0fff;
if round_bit != 0 && (sticky != 0 || hfrac & 1 == 1) {
hfrac += 1;
if hfrac == 0x400 {
hfrac = 0;
hexp16 += 1;
if hexp16 >= 0x1f {
return sign | 0x7c00;
}
}
}
sign | (hexp16 << 10) | hfrac
}
const fn sign_extend(v: u64, bits: u32) -> i64 {
let shift = 64 - bits;
((v << shift) as i64) >> shift
}
const AES_SBOX: [u8; 256] = [
0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16,
];
const AES_INV_SBOX: [u8; 256] = [
0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d,
];
fn aes_round(vd: u128, vn: u128, encrypt: bool) -> u128 {
let state = (vd ^ vn).to_le_bytes();
let shifted = if encrypt {
aes_shift_rows(state)
} else {
aes_inv_shift_rows(state)
};
let sbox = if encrypt { &AES_SBOX } else { &AES_INV_SBOX };
u128::from_le_bytes(shifted.map(|b| sbox[b as usize]))
}
fn aes_mix_columns(vn: u128, forward: bool) -> u128 {
let state = vn.to_le_bytes();
let mut out = [0u8; 16];
for (out_col, in_col) in out.chunks_exact_mut(4).zip(state.chunks_exact(4)) {
let a = [in_col[0], in_col[1], in_col[2], in_col[3]];
let r = if forward {
aes_mix_column(a)
} else {
aes_inv_mix_column(a)
};
out_col.copy_from_slice(&r);
}
u128::from_le_bytes(out)
}
fn aes_shift_rows(state: [u8; 16]) -> [u8; 16] {
let mut out = [0u8; 16];
for r in 0..4usize {
for c in 0..4usize {
out[r + 4 * c] = state[r + 4 * ((c + r) % 4)];
}
}
out
}
fn aes_inv_shift_rows(state: [u8; 16]) -> [u8; 16] {
let mut out = [0u8; 16];
for r in 0..4usize {
for c in 0..4usize {
out[r + 4 * ((c + r) % 4)] = state[r + 4 * c];
}
}
out
}
fn gf_mul(mut a: u8, mut b: u8) -> u8 {
let mut p = 0u8;
for _ in 0..8 {
if b & 1 != 0 {
p ^= a;
}
let hi = a & 0x80;
a <<= 1;
if hi != 0 {
a ^= 0x1b;
}
b >>= 1;
}
p
}
fn aes_mix_column(a: [u8; 4]) -> [u8; 4] {
[
gf_mul(2, a[0]) ^ gf_mul(3, a[1]) ^ a[2] ^ a[3],
a[0] ^ gf_mul(2, a[1]) ^ gf_mul(3, a[2]) ^ a[3],
a[0] ^ a[1] ^ gf_mul(2, a[2]) ^ gf_mul(3, a[3]),
gf_mul(3, a[0]) ^ a[1] ^ a[2] ^ gf_mul(2, a[3]),
]
}
fn aes_inv_mix_column(a: [u8; 4]) -> [u8; 4] {
[
gf_mul(14, a[0]) ^ gf_mul(11, a[1]) ^ gf_mul(13, a[2]) ^ gf_mul(9, a[3]),
gf_mul(9, a[0]) ^ gf_mul(14, a[1]) ^ gf_mul(11, a[2]) ^ gf_mul(13, a[3]),
gf_mul(13, a[0]) ^ gf_mul(9, a[1]) ^ gf_mul(14, a[2]) ^ gf_mul(11, a[3]),
gf_mul(11, a[0]) ^ gf_mul(13, a[1]) ^ gf_mul(9, a[2]) ^ gf_mul(14, a[3]),
]
}
fn u32_lanes(v: u128) -> [u32; 4] {
[
v as u32,
(v >> 32) as u32,
(v >> 64) as u32,
(v >> 96) as u32,
]
}
fn lane32(v: u128, i: u32) -> u32 {
(v >> (i * 32)) as u32
}
fn pack_u32_lanes(l: [u32; 4]) -> u128 {
u128::from(l[0])
| (u128::from(l[1]) << 32)
| (u128::from(l[2]) << 64)
| (u128::from(l[3]) << 96)
}
#[derive(Clone, Copy)]
enum Sha1Op {
Choose,
Parity,
Majority,
}
fn sha1_f(op: Sha1Op, b: u32, c: u32, d: u32) -> u32 {
match op {
Sha1Op::Choose => (b & c) ^ (!b & d),
Sha1Op::Parity => b ^ c ^ d,
Sha1Op::Majority => (b & c) ^ (b & d) ^ (c & d),
}
}
#[allow(clippy::many_single_char_names)]
fn sha1_quad_round(abcd: u128, mut e: u32, wk: u128, op: Sha1Op) -> u128 {
let [mut a, mut b, mut c, mut d] = u32_lanes(abcd);
for i in 0..4u32 {
let w = lane32(wk, i);
let t = a
.rotate_left(5)
.wrapping_add(sha1_f(op, b, c, d))
.wrapping_add(e)
.wrapping_add(w);
e = d;
d = c;
c = b.rotate_left(30);
b = a;
a = t;
}
pack_u32_lanes([a, b, c, d])
}
fn sha1_su0(vd: u128, vn: u128, vm: u128) -> u128 {
let d = u32_lanes(vd);
let n = u32_lanes(vn);
let m = u32_lanes(vm);
pack_u32_lanes([
d[0] ^ d[2] ^ m[0],
d[1] ^ d[3] ^ m[1],
d[2] ^ n[0] ^ m[2],
d[3] ^ n[1] ^ m[3],
])
}
fn sha1_su1(vd: u128, vn: u128) -> u128 {
let d = u32_lanes(vd);
let n = u32_lanes(vn);
let w0 = (d[0] ^ n[1]).rotate_left(1);
let w1 = (d[1] ^ n[2]).rotate_left(1);
let w2 = (d[2] ^ n[3]).rotate_left(1);
let w3 = (d[3] ^ w0).rotate_left(1);
pack_u32_lanes([w0, w1, w2, w3])
}
fn sha256_ch(e: u32, f: u32, g: u32) -> u32 {
(e & f) ^ (!e & g)
}
fn sha256_maj(a: u32, b: u32, c: u32) -> u32 {
(a & b) ^ (a & c) ^ (b & c)
}
fn sha256_bsig0(a: u32) -> u32 {
a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22)
}
fn sha256_bsig1(e: u32) -> u32 {
e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25)
}
fn sha256_ssig0(x: u32) -> u32 {
x.rotate_right(7) ^ x.rotate_right(18) ^ (x >> 3)
}
fn sha256_ssig1(x: u32) -> u32 {
x.rotate_right(17) ^ x.rotate_right(19) ^ (x >> 10)
}
#[allow(clippy::many_single_char_names)]
fn sha256_hash(abcd: u128, efgh: u128, wk: u128, want_efgh: bool) -> u128 {
let [mut a, mut b, mut c, mut d] = u32_lanes(abcd);
let [mut e, mut f, mut g, mut h] = u32_lanes(efgh);
for i in 0..4u32 {
let w = lane32(wk, i);
let t1 = h
.wrapping_add(sha256_bsig1(e))
.wrapping_add(sha256_ch(e, f, g))
.wrapping_add(w);
let t2 = sha256_bsig0(a).wrapping_add(sha256_maj(a, b, c));
h = g;
g = f;
f = e;
e = d.wrapping_add(t1);
d = c;
c = b;
b = a;
a = t1.wrapping_add(t2);
}
if want_efgh {
pack_u32_lanes([e, f, g, h])
} else {
pack_u32_lanes([a, b, c, d])
}
}
fn sha256_su0(vd: u128, vn: u128) -> u128 {
let d = u32_lanes(vd);
let n = u32_lanes(vn);
pack_u32_lanes([
d[0].wrapping_add(sha256_ssig0(d[1])),
d[1].wrapping_add(sha256_ssig0(d[2])),
d[2].wrapping_add(sha256_ssig0(d[3])),
d[3].wrapping_add(sha256_ssig0(n[0])),
])
}
fn sha256_su1(vd: u128, vn: u128, vm: u128) -> u128 {
let d = u32_lanes(vd);
let n = u32_lanes(vn);
let m = u32_lanes(vm);
let w0 = d[0].wrapping_add(sha256_ssig1(m[2])).wrapping_add(n[1]);
let w1 = d[1].wrapping_add(sha256_ssig1(m[3])).wrapping_add(n[2]);
let w2 = d[2].wrapping_add(sha256_ssig1(w0)).wrapping_add(n[3]);
let w3 = d[3].wrapping_add(sha256_ssig1(w1)).wrapping_add(m[0]);
pack_u32_lanes([w0, w1, w2, w3])
}
#[cfg(test)]
mod tests {
#![allow(clippy::float_cmp)]
use super::*;
use crate::vcpu::mem::{PAGE_SIZE, Prot};
fn cpu() -> Aarch64Interp {
Aarch64Interp::new(0x1_0000, 0x2_0000)
}
fn scratch() -> GuestMemory {
GuestMemory::new(0x1_0000, PAGE_SIZE)
}
#[test]
fn fmov_ins_sshll() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 0x1234_5678;
c.exec(0x1E27_0020, &mut m);
assert_eq!(c.v[0], 0x1234_5678);
c.exec(0x1E26_0003, &mut m);
assert_eq!(c.x[3], 0x1234_5678);
c.v[0] = 0;
c.x[2] = 0xAABB;
c.exec(0x4E0C_1C40, &mut m);
assert_eq!(c.v[0], 0xAABB_u128 << 32);
c.v[0] = (2u128 << 32) | 0xFFFF_FFFF;
c.exec(0x0F20_A400, &mut m);
assert_eq!(c.v[0], (2u128 << 64) | u128::from(u64::MAX));
}
#[test]
fn simd_modified_immediate_movi_mvni() {
let (mut c, mut m) = (cpu(), scratch());
c.exec(0x4F00_0400, &mut m); assert_eq!(c.v[0], 0);
c.v[3] = 0xdead; c.exec(0x2F00_0403, &mut m); assert_eq!(c.v[3], 0xFFFF_FFFF_FFFF_FFFF);
}
#[test]
fn movz_movk_build_64bit_immediate() {
let (mut c, mut m) = (cpu(), scratch());
assert!(matches!(c.exec(0xD282_0001, &mut m), Step::Next)); assert_eq!(c.x[1], 0x1000);
assert!(matches!(c.exec(0xF2A0_0021, &mut m), Step::Next)); assert_eq!(c.x[1], 0x1_1000);
}
#[test]
fn add_sub_immediate() {
let (mut c, mut m) = (cpu(), scratch());
c.x[0] = 100;
c.exec(0x9100_1401, &mut m); assert_eq!(c.x[1], 105);
c.exec(0xD100_2802, &mut m); assert_eq!(c.x[2], 90);
}
#[test]
fn add_extended_register() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 0x1000;
c.x[2] = 0x1FF;
c.exec(0x8B22_0020, &mut m); assert_eq!(c.x[0], 0x10FF);
}
#[test]
fn add_shifted_register() {
let (mut c, mut m) = (cpu(), scratch());
c.x[0] = 10;
c.x[1] = 20;
c.exec(0x8B01_0002, &mut m); assert_eq!(c.x[2], 30);
}
#[test]
fn cmp_sets_flags_for_branch() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 6;
c.exec(0xF100_183F, &mut m); assert!(c.flags.z, "6 == 6 sets Z");
assert!(c.cond_holds(0b0000), "EQ holds");
assert!(!c.cond_holds(0b0001), "NE does not hold");
}
#[test]
fn bitfield_shifts_and_extends() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 0x1234;
c.exec(0xD37C_EC20, &mut m); assert_eq!(c.x[0], 0x1234 << 4);
c.exec(0xD344_FC20, &mut m); assert_eq!(c.x[0], 0x1234 >> 4);
c.x[1] = (-16i64) as u64;
c.exec(0x9344_FC20, &mut m); assert_eq!(c.x[0] as i64, -1);
c.x[1] = 0x1234_5678_9abc_def0;
c.exec(0x5300_1C20, &mut m); assert_eq!(c.x[0], 0xf0);
c.x[1] = 0x80; c.exec(0x9340_1C20, &mut m); assert_eq!(c.x[0] as i64, -128);
}
#[test]
fn logical_immediate() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 0x1_2345;
c.exec(0x9240_1C20, &mut m); assert_eq!(c.x[0], 0x45);
}
#[test]
fn mul_and_madd() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 6;
c.x[2] = 7;
c.exec(0x9B02_7C20, &mut m); assert_eq!(c.x[0], 42);
c.x[3] = 1;
c.exec(0x9B02_0C20, &mut m); assert_eq!(c.x[0], 43);
}
#[test]
fn udiv_sdiv_and_div_by_zero() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 100;
c.x[2] = 7;
c.exec(0x9AC2_0820, &mut m); assert_eq!(c.x[0], 14);
c.x[1] = (-100i64) as u64;
c.exec(0x9AC2_0C20, &mut m); assert_eq!(c.x[0] as i64, -14);
c.x[2] = 0;
c.exec(0x9AC2_0820, &mut m); assert_eq!(c.x[0], 0);
}
#[test]
fn lslv_variable_shift() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 1;
c.x[2] = 4;
c.exec(0x9AC2_2020, &mut m); assert_eq!(c.x[0], 16);
}
#[test]
fn csel_and_csinc_use_flags() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 111;
c.x[2] = 222;
c.flags.z = true; c.exec(0x9A82_0020, &mut m); assert_eq!(c.x[0], 111);
c.flags.z = false; c.exec(0x9A82_0020, &mut m); assert_eq!(c.x[0], 222);
c.exec(0x9A82_0420, &mut m);
assert_eq!(c.x[0], 223);
}
#[test]
fn mov_via_orr() {
let (mut c, mut m) = (cpu(), scratch());
c.x[5] = 0xabcd;
c.exec(0xAA05_03E0, &mut m); assert_eq!(c.x[0], 0xabcd);
}
#[test]
fn ldr_str_roundtrip() {
let mut c = cpu();
let mut m = GuestMemory::new(0x1_0000, 4 * PAGE_SIZE);
m.map(0x1_0000, PAGE_SIZE, Prot::rw()).unwrap();
c.x[1] = 0x1_0040; c.x[0] = 0x1122_3344_5566_7788;
assert!(matches!(c.exec(0xF900_0020, &mut m), Step::Next)); c.x[0] = 0;
assert!(matches!(c.exec(0xF940_0022, &mut m), Step::Next)); assert_eq!(c.x[2], 0x1122_3344_5566_7788);
}
#[test]
fn store_to_unmapped_faults() {
let mut c = cpu();
let mut m = GuestMemory::new(0x1_0000, PAGE_SIZE);
c.x[1] = 0x1_0000; assert!(matches!(
c.exec(0xF900_0020, &mut m),
Step::Fault { write: true, .. }
));
}
#[test]
fn sum_loop_runs_control_flow() {
let base = 0x1_0000u64;
let program: [u32; 8] = [
0xD280_0000, 0xD280_0021, 0x8B01_0000, 0x9100_0421, 0xF100_183F, 0x54FF_FFA1, 0xD280_0BA8, 0xD400_0001, ];
let mut mem = GuestMemory::new(base, 4 * PAGE_SIZE);
mem.map(base, PAGE_SIZE, Prot::rx()).unwrap();
let mut bytes = Vec::new();
for w in program {
bytes.extend_from_slice(&w.to_le_bytes());
}
mem.write_init(base, &bytes).unwrap();
let mut c = Aarch64Interp::new(base, base + 3 * PAGE_SIZE);
assert_eq!(c.run(&mut mem).unwrap(), Exit::Syscall);
assert_eq!(c.x[8], 93, "exit syscall");
assert_eq!(c.x[0], 15, "sum of 1..=5");
}
#[test]
fn bl_ret_calls_subroutine() {
let base = 0x1_0000u64;
let program: [u32; 5] = [
0x9400_0003, 0xD280_0BA8, 0xD400_0001, 0xD280_00E0, 0xD65F_03C0, ];
let mut mem = GuestMemory::new(base, 4 * PAGE_SIZE);
mem.map(base, PAGE_SIZE, Prot::rx()).unwrap();
let mut bytes = Vec::new();
for w in program {
bytes.extend_from_slice(&w.to_le_bytes());
}
mem.write_init(base, &bytes).unwrap();
let mut c = Aarch64Interp::new(base, base + 3 * PAGE_SIZE);
assert_eq!(c.run(&mut mem).unwrap(), Exit::Syscall);
assert_eq!(c.x[0], 7, "subroutine set x0");
assert_eq!(c.x[8], 93);
}
#[test]
fn stp_ldp_push_pop_roundtrip() {
let base = 0x1_0000u64;
let mut mem = GuestMemory::new(base, 8 * PAGE_SIZE);
mem.map(base, PAGE_SIZE, Prot::rx()).unwrap();
mem.map(base + 4 * PAGE_SIZE, PAGE_SIZE, Prot::rw())
.unwrap();
let sp = base + 5 * PAGE_SIZE;
let program: [u32; 7] = [
0xD282_4680, 0xD28A_CF01, 0xA9BF_07E0, 0xD280_0000, 0xD280_0001, 0xA8C1_07E0, 0xD400_0001, ];
let mut bytes = Vec::new();
for w in program {
bytes.extend_from_slice(&w.to_le_bytes());
}
mem.write_init(base, &bytes).unwrap();
let mut c = Aarch64Interp::new(base, sp);
assert_eq!(c.run(&mut mem).unwrap(), Exit::Syscall);
assert_eq!(c.x[0], 0x1234, "x0 restored from stack");
assert_eq!(c.x[1], 0x5678, "x1 restored from stack");
assert_eq!(c.sp, sp, "sp restored to its original value");
}
#[test]
fn indexed_and_register_offset_load_store() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
c.x[1] = base + 0x100;
c.x[0] = 0xAABB_CCDD;
assert!(matches!(c.exec(0xF81F_8C20, &mut m), Step::Next));
assert_eq!(c.x[1], base + 0xF8, "pre-index writeback");
assert!(matches!(c.exec(0xF840_0022, &mut m), Step::Next));
assert_eq!(c.x[2], 0xAABB_CCDD);
c.x[5] = base;
c.x[6] = 0xF8;
assert!(matches!(c.exec(0xF866_68A3, &mut m), Step::Next));
assert_eq!(c.x[3], 0xAABB_CCDD);
}
#[test]
fn ldrsb_sign_extends() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
c.x[1] = base + 0x200;
c.x[0] = 0x80;
c.exec(0x3900_0020, &mut m); c.exec(0x3880_0022, &mut m); assert_eq!(c.x[2] as i64, -128, "signed byte load sign-extends");
}
#[test]
fn exclusive_store_load_roundtrip() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
c.x[1] = base + 0x40;
c.x[3] = 0x42;
assert!(matches!(c.exec(0xC802_7C23, &mut m), Step::Next));
assert_eq!(c.x[2], 0, "store-exclusive reports success");
assert!(matches!(c.exec(0xC85F_7C20, &mut m), Step::Next));
assert_eq!(c.x[0], 0x42);
}
#[test]
fn ldxr_stxr_sequence_reports_success() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
let addr = base + 0x40;
m.write(addr, &0x42u64.to_le_bytes()).unwrap();
c.x[1] = addr;
assert!(matches!(c.exec(0xC85F_7C20, &mut m), Step::Next));
assert_eq!(c.x[0], 0x42);
c.x[3] = 0x99;
assert!(matches!(c.exec(0xC802_7C23, &mut m), Step::Next));
assert_eq!(c.x[2], 0, "STXR reports success while the monitor is open");
let mut buf = [0u8; 8];
m.read(addr, &mut buf).unwrap();
assert_eq!(u64::from_le_bytes(buf), 0x99, "STXR's store took effect");
}
#[test]
fn intervening_store_clears_exclusive_monitor() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
let addr = base + 0x40;
m.write(addr, &0x42u64.to_le_bytes()).unwrap();
c.x[1] = addr;
assert!(matches!(c.exec(0xC85F_7C20, &mut m), Step::Next)); c.x[4] = 0x1234;
assert!(matches!(c.exec(0xB900_0024, &mut m), Step::Next)); c.x[3] = 0x99;
assert!(matches!(c.exec(0xC802_7C23, &mut m), Step::Next));
assert_eq!(
c.x[2], 1,
"STXR reports failure once the monitor is cleared"
);
let mut buf = [0u8; 8];
m.read(addr, &mut buf).unwrap();
assert_eq!(
u64::from_le_bytes(buf) & 0xffff_ffff,
0x1234,
"failed STXR must not have written memory"
);
}
#[test]
fn cas_success_and_failure_paths() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
let addr = base + 0x300;
m.write(addr, &0x1111_1111u32.to_le_bytes()).unwrap();
c.x[0] = addr;
c.x[1] = 0xdead_beef;
c.x[2] = 0x2222_2222;
assert!(matches!(c.exec(0x88a1_7c02, &mut m), Step::Next));
assert_eq!(
c.x[1], 0x1111_1111,
"CAS returns the original value even on mismatch"
);
let mut buf = [0u8; 4];
m.read(addr, &mut buf).unwrap();
assert_eq!(
u32::from_le_bytes(buf),
0x1111_1111,
"no swap on a failed compare"
);
c.x[1] = 0x1111_1111;
c.x[2] = 0x2222_2222;
assert!(matches!(c.exec(0x88a1_7c02, &mut m), Step::Next));
assert_eq!(c.x[1], 0x1111_1111, "CAS still returns the pre-swap value");
m.read(addr, &mut buf).unwrap();
assert_eq!(
u32::from_le_bytes(buf),
0x2222_2222,
"swap happens on a matching compare"
);
}
#[test]
fn swp_round_trip() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
let addr = base + 0x300;
m.write(addr, &0x1234_5678u32.to_le_bytes()).unwrap();
c.x[0] = addr;
c.x[1] = 0xAAAA_BBBB; assert!(matches!(c.exec(0xb821_8002, &mut m), Step::Next));
assert_eq!(c.x[2], 0x1234_5678, "SWP returns the original value");
let mut buf = [0u8; 4];
m.read(addr, &mut buf).unwrap();
assert_eq!(
u32::from_le_bytes(buf),
0xAAAA_BBBB,
"SWP stores the new value"
);
}
#[test]
fn ldadd_ldset_ldclr_return_old_value_and_update_memory() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
let addr = base + 0x300;
m.write(addr, &0x0000_000fu32.to_le_bytes()).unwrap();
c.x[0] = addr;
let mut buf = [0u8; 4];
c.x[1] = 0x10;
assert!(matches!(c.exec(0xb821_0002, &mut m), Step::Next));
assert_eq!(c.x[2], 0x0f, "LDADD returns the original value");
m.read(addr, &mut buf).unwrap();
assert_eq!(
u32::from_le_bytes(buf),
0x1f,
"LDADD writes old+rs back to memory"
);
c.x[1] = 0xf0;
assert!(matches!(c.exec(0xb821_3002, &mut m), Step::Next));
assert_eq!(c.x[2], 0x1f, "LDSET returns the original value");
m.read(addr, &mut buf).unwrap();
assert_eq!(
u32::from_le_bytes(buf),
0xff,
"LDSET writes old|rs back to memory"
);
c.x[1] = 0x0f;
assert!(matches!(c.exec(0xb821_1002, &mut m), Step::Next));
assert_eq!(c.x[2], 0xff, "LDCLR returns the original value");
m.read(addr, &mut buf).unwrap();
assert_eq!(
u32::from_le_bytes(buf),
0xf0,
"LDCLR writes old&!rs back to memory"
);
}
#[test]
fn stadd_updates_memory_without_register_writeback() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
let addr = base + 0x300;
m.write(addr, &0x5u32.to_le_bytes()).unwrap();
c.x[0] = addr;
c.x[1] = 0x3;
assert!(matches!(c.exec(0xb821_001f, &mut m), Step::Next));
let mut buf = [0u8; 4];
m.read(addr, &mut buf).unwrap();
assert_eq!(u32::from_le_bytes(buf), 0x8, "STADD still updates memory");
}
#[test]
fn msr_mrs_tpidr_roundtrip() {
let (mut c, mut m) = (cpu(), scratch());
c.x[0] = 0x1234_5678;
c.exec(0xD51B_D040, &mut m); assert_eq!(c.tpidr, 0x1234_5678);
c.exec(0xD53B_D041, &mut m); assert_eq!(c.x[1], 0x1234_5678);
}
#[test]
fn tbz_tests_a_bit() {
let (mut c, mut m) = (cpu(), scratch());
c.pc = 0x1000;
c.x[0] = 0; assert!(matches!(c.exec(0x3618_0040, &mut m), Step::Branched));
assert_eq!(c.pc, 0x1008);
c.pc = 0x1000;
c.x[0] = 8; assert!(matches!(c.exec(0x3618_0040, &mut m), Step::Next));
}
#[test]
fn adc_sbc_use_carry() {
let (mut c, mut m) = (cpu(), scratch());
c.x[0] = 10;
c.x[1] = 3;
c.flags.c = true;
c.exec(0x9A01_0002, &mut m); assert_eq!(c.x[2], 14);
c.flags.c = false;
c.exec(0xDA01_0002, &mut m); assert_eq!(c.x[2], 6);
}
#[test]
fn shl_scalar_and_vector_mov() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = 0xff;
c.exec(0x5F74_5421, &mut m); assert_eq!(c.v[1], 0xff << 52);
c.v[0] = 0x1234_5678_9abc_def0;
c.exec(0x4EA0_1C02, &mut m); assert_eq!(c.v[2], c.v[0]);
}
#[test]
fn svc_traps_without_advancing() {
let (mut c, mut m) = (cpu(), scratch());
c.pc = 0x1_0004;
assert!(matches!(c.exec(0xD400_0001, &mut m), Step::Syscall));
assert_eq!(c.pc, 0x1_0004);
c.set_syscall_ret(0);
assert_eq!(c.pc, 0x1_0008);
}
fn setd(c: &mut Aarch64Interp, n: usize, x: f64) {
c.v[n] = u128::from(x.to_bits());
}
fn sets(c: &mut Aarch64Interp, n: usize, x: f32) {
c.v[n] = u128::from(x.to_bits());
}
fn getd(c: &Aarch64Interp, n: usize) -> f64 {
f64::from_bits(c.v[n] as u64)
}
fn gets(c: &Aarch64Interp, n: usize) -> f32 {
f32::from_bits(c.v[n] as u32)
}
#[test]
fn fp_arithmetic_double() {
let (mut c, mut m) = (cpu(), scratch());
setd(&mut c, 1, 3.5);
setd(&mut c, 2, 2.0);
c.v[0] = u128::MAX; c.exec(0x1E62_2820, &mut m); assert_eq!(getd(&c, 0), 5.5);
assert_eq!(c.v[0] >> 64, 0, "upper bits cleared");
c.exec(0x1E62_3820, &mut m); assert_eq!(getd(&c, 0), 1.5);
c.exec(0x1E62_0820, &mut m); assert_eq!(getd(&c, 0), 7.0);
c.exec(0x1E62_1820, &mut m); assert_eq!(getd(&c, 0), 1.75);
c.exec(0x1E62_4820, &mut m); assert_eq!(getd(&c, 0), 3.5);
c.exec(0x1E62_5820, &mut m); assert_eq!(getd(&c, 0), 2.0);
c.exec(0x1E62_8820, &mut m); assert_eq!(getd(&c, 0), -7.0);
}
#[test]
fn fp_arithmetic_single() {
let (mut c, mut m) = (cpu(), scratch());
sets(&mut c, 1, 1.5);
sets(&mut c, 2, 4.0);
c.exec(0x1E22_2820, &mut m); assert_eq!(gets(&c, 0), 5.5);
c.exec(0x1E22_0820, &mut m); assert_eq!(gets(&c, 0), 6.0);
}
#[test]
fn fp_one_source_ops() {
let (mut c, mut m) = (cpu(), scratch());
setd(&mut c, 1, -3.25);
c.exec(0x1E60_C020, &mut m); assert_eq!(getd(&c, 0), 3.25);
c.exec(0x1E61_4020, &mut m); assert_eq!(getd(&c, 0), 3.25);
setd(&mut c, 1, 9.0);
c.exec(0x1E61_C020, &mut m); assert_eq!(getd(&c, 0), 3.0);
setd(&mut c, 1, 2.7);
c.exec(0x1E65_C020, &mut m); assert_eq!(getd(&c, 0), 2.0);
c.exec(0x1E65_4020, &mut m); assert_eq!(getd(&c, 0), 2.0);
c.exec(0x1E64_C020, &mut m); assert_eq!(getd(&c, 0), 3.0);
setd(&mut c, 1, 2.5);
c.exec(0x1E64_4020, &mut m); assert_eq!(getd(&c, 0), 2.0);
c.exec(0x1E66_4020, &mut m); assert_eq!(getd(&c, 0), 3.0);
}
#[test]
fn fcvt_single_double_roundtrip() {
let (mut c, mut m) = (cpu(), scratch());
sets(&mut c, 1, 1.5);
c.exec(0x1E22_C020, &mut m); assert_eq!(getd(&c, 0), 1.5);
setd(&mut c, 1, 1.5);
c.exec(0x1E62_4020, &mut m); assert_eq!(gets(&c, 0), 1.5);
}
#[test]
fn fmov_immediate() {
let (mut c, mut m) = (cpu(), scratch());
c.exec(0x1E6E_1000, &mut m); assert_eq!(getd(&c, 0), 1.0);
c.exec(0x1E60_1000, &mut m); assert_eq!(getd(&c, 0), 2.0);
c.exec(0x1E2E_1000, &mut m); assert_eq!(gets(&c, 0), 1.0);
}
#[test]
fn scvtf_fcvtzs_roundtrip_integer() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = (-42i64) as u64;
c.exec(0x9E62_0020, &mut m); assert_eq!(getd(&c, 0), -42.0);
c.exec(0x9E78_0000, &mut m); assert_eq!(c.x[0] as i64, -42);
c.x[3] = 300;
c.exec(0x9E63_0060, &mut m); assert_eq!(getd(&c, 0), 300.0);
setd(&mut c, 0, 300.0);
c.exec(0x9E79_0000, &mut m); assert_eq!(c.x[0], 300);
setd(&mut c, 0, f64::NAN);
c.exec(0x9E78_0000, &mut m); assert_eq!(c.x[0], 0, "NaN converts to 0");
setd(&mut c, 0, f64::INFINITY);
c.exec(0x9E78_0000, &mut m); assert_eq!(c.x[0] as i64, i64::MAX, "+inf saturates");
sets(&mut c, 1, -2.9);
c.exec(0x1E38_0020, &mut m); assert_eq!(c.x[0] as i32, -2);
}
#[test]
fn fcmp_sets_flags() {
let (mut c, mut m) = (cpu(), scratch());
setd(&mut c, 1, 1.0);
setd(&mut c, 2, 2.0);
c.exec(0x1E62_2020, &mut m); assert!(
c.flags.n && !c.flags.z && !c.flags.c && !c.flags.v,
"less-than"
);
setd(&mut c, 2, 1.0);
c.exec(0x1E62_2020, &mut m); assert!(!c.flags.n && c.flags.z && c.flags.c && !c.flags.v, "equal");
setd(&mut c, 2, 0.5);
c.exec(0x1E62_2020, &mut m); assert!(
!c.flags.n && !c.flags.z && c.flags.c && !c.flags.v,
"greater-than"
);
setd(&mut c, 1, f64::NAN);
c.exec(0x1E62_2020, &mut m); assert!(
!c.flags.n && !c.flags.z && c.flags.c && c.flags.v,
"unordered"
);
setd(&mut c, 1, 0.0);
c.exec(0x1E60_2028, &mut m); assert!(c.flags.z && c.flags.c, "d1 == 0.0");
}
#[test]
fn fcsel_picks_by_condition() {
let (mut c, mut m) = (cpu(), scratch());
setd(&mut c, 1, 11.0);
setd(&mut c, 2, 22.0);
c.flags.z = true; c.exec(0x1E62_0C20, &mut m); assert_eq!(getd(&c, 0), 11.0);
c.flags.z = false; c.exec(0x1E62_0C20, &mut m);
assert_eq!(getd(&c, 0), 22.0);
}
#[test]
fn fccmp_conditional() {
let (mut c, mut m) = (cpu(), scratch());
setd(&mut c, 1, 1.0);
setd(&mut c, 2, 1.0);
c.flags.z = true; c.exec(0x1E62_0420, &mut m); assert!(c.flags.z && c.flags.c, "compare taken: equal");
c.flags = Flags::default();
c.flags.z = false; c.exec(0x1E62_042F, &mut m); assert!(
c.flags.n && c.flags.z && c.flags.c && c.flags.v,
"nzcv loaded"
);
}
#[test]
fn vector_add_sub_cmeq() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = (5u128 << 64) | 3;
c.v[2] = (7u128 << 64) | 4;
c.exec(0x4EE2_8420, &mut m); assert_eq!(c.v[0], (12u128 << 64) | 7);
c.exec(0x6EA2_8420, &mut m); assert_eq!(c.v[0] & 0xffff_ffff, 0xffff_ffff);
c.v[1] = 0x1111_2222;
c.v[2] = 0x1111_9999;
c.exec(0x6EA2_8C20, &mut m); assert_eq!(c.v[0] & 0xffff_ffff, 0, "low lane differs -> 0");
assert_eq!(
(c.v[0] >> 32) & 0xffff_ffff,
0xffff_ffff,
"high lane equal -> all ones"
);
}
#[test]
fn sbfx_ubfx_extract_bitfield() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 0x1234_5678_9abc_def0;
c.exec(0x9344_2C20, &mut m);
assert_eq!(c.x[0] as i64, -17);
c.exec(0xD344_2C20, &mut m);
assert_eq!(c.x[0], 0xef);
}
#[test]
fn ccmp_feeds_csel() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 5;
c.x[3] = 111;
c.x[4] = 222;
c.flags.z = true;
assert!(matches!(c.exec(0xFA45_0820, &mut m), Step::Next));
assert!(c.flags.z && c.flags.c, "5 == 5 sets Z and C");
assert!(matches!(c.exec(0x9A84_0060, &mut m), Step::Next));
assert_eq!(c.x[0], 111);
c.flags = Flags::default();
c.flags.z = true; assert!(matches!(c.exec(0xFA45_182F, &mut m), Step::Next));
assert!(
c.flags.n && c.flags.z && c.flags.c && c.flags.v,
"nzcv literal loaded when outer cond fails"
);
assert!(matches!(c.exec(0x9A84_0060, &mut m), Step::Next));
assert_eq!(c.x[0], 111);
c.flags.z = false;
assert!(matches!(c.exec(0x9A84_0060, &mut m), Step::Next));
assert_eq!(c.x[0], 222);
}
#[test]
fn rev_rbit_clz_ops() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 0x1122_3344_5566_7788;
c.exec(0xDAC0_0C20, &mut m); assert_eq!(c.x[0], 0x8877_6655_4433_2211);
c.exec(0xDAC0_0020, &mut m); assert_eq!(c.x[0], 0x11ee_66aa_22cc_4488);
c.exec(0xDAC0_1020, &mut m); assert_eq!(c.x[0], 3);
}
#[test]
fn ldr_literal_and_ldpsw() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
c.pc = base;
m.write(base + 16, &0x1122_3344_5566_7788u64.to_le_bytes())
.unwrap();
assert!(matches!(c.exec(0x5800_0082, &mut m), Step::Next));
assert_eq!(c.x[2], 0x1122_3344_5566_7788);
c.x[2] = base + 0x100;
m.write(base + 0x100, &(-1i32).to_le_bytes()).unwrap();
m.write(base + 0x104, &5i32.to_le_bytes()).unwrap();
assert!(matches!(c.exec(0x6940_0440, &mut m), Step::Next));
assert_eq!(c.x[0] as i64, -1, "LDPSW sign-extends the first word");
assert_eq!(c.x[1], 5);
}
#[test]
fn neon_dup_umov_roundtrip() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 0xDEAD_BEEF;
c.exec(0x4E04_0C20, &mut m); let expect = (0xDEAD_BEEFu128 << 96)
| (0xDEAD_BEEFu128 << 64)
| (0xDEAD_BEEFu128 << 32)
| 0xDEAD_BEEFu128;
assert_eq!(c.v[0], expect);
c.exec(0x0E0C_3C00, &mut m); assert_eq!(c.x[0], 0xDEAD_BEEF);
}
#[test]
fn neon_dup_element_and_smov() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = 0x8442_0201;
c.exec(0x0E07_0420, &mut m); assert_eq!(c.v[0], 0x8484_8484_8484_8484);
c.exec(0x4E07_2C20, &mut m); assert_eq!(c.x[0] as i64, -124);
}
#[test]
fn neon_vector_compares() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = 0x0000_0000_0000_000A_FFFF_FFFD_0000_0005u128;
c.v[2] = 0x0000_0001_0000_000A_FFFF_FFFD_0000_0003u128;
c.exec(0x4EA2_3420, &mut m); assert_eq!(c.v[0], 0xffff_ffffu128);
c.exec(0x4EA2_3C20, &mut m); assert_eq!(c.v[0], 0xffff_ffff_ffff_ffff_ffff_ffffu128);
c.exec(0x6EA2_3420, &mut m); assert_eq!(c.v[0], 0xffff_ffffu128);
c.exec(0x6EA2_3C20, &mut m); assert_eq!(c.v[0], 0xffff_ffff_ffff_ffff_ffff_ffffu128);
}
#[test]
fn neon_sshl_ushl_signed_shift_amount() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = 0xFFFF_FFFF_1000_0000_8000_0000_0000_0001u128;
c.v[2] = 0xFFFF_FFFF_0000_001F_FFFF_FFFC_0000_0004u128;
c.exec(0x4EA2_4420, &mut m); assert_eq!(c.v[0], 0xFFFF_FFFF_0000_0000_F800_0000_0000_0010u128);
c.exec(0x6EA2_4420, &mut m); assert_eq!(c.v[0], 0x7FFF_FFFF_0000_0000_0800_0000_0000_0010u128);
}
#[test]
fn neon_not_addv_uaddlv() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = 0x1234_5678_9ABC_DEF0_1122_3344_5566_7788u128;
c.exec(0x6E20_5820, &mut m); assert_eq!(c.v[0], !c.v[1]);
c.v[1] = 0x100F_0E0D_0C0B_0A09_0807_0605_0403_0201u128;
c.exec(0x4E31_B820, &mut m); assert_eq!(c.v[0], 136);
c.exec(0x6E30_3820, &mut m); assert_eq!(c.v[0], 136);
}
#[test]
fn neon_ld1_st1_multiple_structures() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let mut c = cpu();
c.x[1] = base + 0x40;
c.v[0] = 0x1122_3344_5566_7788_99AA_BBCC_DDEE_FF00u128;
assert!(matches!(c.exec(0x4C00_7020, &mut m), Step::Next));
c.v[0] = 0;
assert!(matches!(c.exec(0x4CDF_7020, &mut m), Step::Next));
assert_eq!(c.v[0], 0x1122_3344_5566_7788_99AA_BBCC_DDEE_FF00u128);
assert_eq!(c.x[1], base + 0x50, "post-index advanced by 16 bytes");
}
fn quad_f32(a: f32, b: f32, c: f32, d: f32) -> u128 {
(u128::from(d.to_bits()) << 96)
| (u128::from(c.to_bits()) << 64)
| (u128::from(b.to_bits()) << 32)
| u128::from(a.to_bits())
}
fn quad_u32(a: u32, b: u32, c: u32, d: u32) -> u128 {
(u128::from(d) << 96) | (u128::from(c) << 64) | (u128::from(b) << 32) | u128::from(a)
}
fn quad_u16(a: u16, b: u16, c: u16, d: u16) -> u128 {
(u128::from(d) << 48) | (u128::from(c) << 32) | (u128::from(b) << 16) | u128::from(a)
}
#[test]
fn fmadd_fmsub_fnmadd_fnmsub() {
let (mut c, mut m) = (cpu(), scratch());
sets(&mut c, 1, 2.0);
sets(&mut c, 2, 3.0);
sets(&mut c, 3, 1.0);
c.exec(0x1F02_0C20, &mut m); assert_eq!(gets(&c, 0), 7.0);
setd(&mut c, 1, 2.0);
setd(&mut c, 2, 3.0);
setd(&mut c, 3, 1.0);
c.exec(0x1F42_8C20, &mut m); assert_eq!(getd(&c, 0), -5.0);
c.exec(0x1F62_0C20, &mut m); assert_eq!(getd(&c, 0), -7.0);
c.exec(0x1F62_8C20, &mut m); assert_eq!(getd(&c, 0), 5.0);
}
#[test]
fn fcvt_half_precision_roundtrip() {
let (mut c, mut m) = (cpu(), scratch());
sets(&mut c, 1, 1.5);
c.exec(0x1E23_C020, &mut m); assert_eq!(c.v[0] as u16, 0x3E00, "1.5 as f16");
c.v[1] = c.v[0]; c.exec(0x1EE2_4020, &mut m); assert_eq!(gets(&c, 0), 1.5);
setd(&mut c, 1, 0.5);
c.exec(0x1E63_C020, &mut m); assert_eq!(c.v[0] as u16, 0x3800, "0.5 as f16");
c.v[1] = c.v[0];
c.exec(0x1EE2_C020, &mut m); assert_eq!(getd(&c, 0), 0.5);
}
#[test]
fn neon_fp_vector_arithmetic_4s() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_f32(1.0, 2.0, 3.0, 4.0);
c.v[2] = quad_f32(1.0, 1.0, 1.0, 2.0);
c.exec(0x4E22_D420, &mut m); assert_eq!(c.v[0], quad_f32(2.0, 3.0, 4.0, 6.0));
c.exec(0x6E22_DC20, &mut m); assert_eq!(c.v[0], quad_f32(1.0, 2.0, 3.0, 8.0));
c.v[0] = quad_f32(10.0, 10.0, 10.0, 10.0);
c.exec(0x4E22_CC20, &mut m); assert_eq!(c.v[0], quad_f32(11.0, 12.0, 13.0, 18.0));
c.v[1] = quad_f32(-1.0, -2.0, 3.0, -4.0);
c.exec(0x4EA0_F820, &mut m); assert_eq!(c.v[0], quad_f32(1.0, 2.0, 3.0, 4.0));
c.v[1] = quad_f32(4.0, 9.0, 16.0, 25.0);
c.exec(0x6EA1_F820, &mut m); assert_eq!(c.v[0], quad_f32(2.0, 3.0, 4.0, 5.0));
}
#[test]
fn neon_integer_mul_mla_abs_neg_minmax() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_u32(2, 3, 4, 5);
c.v[2] = quad_u32(10, 10, 10, 10);
c.exec(0x4EA2_9C20, &mut m); assert_eq!(c.v[0], quad_u32(20, 30, 40, 50));
c.v[0] = quad_u32(1, 1, 1, 1);
c.exec(0x4EA2_9420, &mut m); assert_eq!(c.v[0], quad_u32(21, 31, 41, 51));
c.v[1] = quad_u32((-1i32) as u32, (-2i32) as u32, 3, (-4i32) as u32);
c.exec(0x4EA0_B820, &mut m); assert_eq!(c.v[0], quad_u32(1, 2, 3, 4));
c.exec(0x6EA0_B820, &mut m); assert_eq!(c.v[0], quad_u32(1, 2, (-3i32) as u32, 4));
c.v[1] = quad_u32(5, 5, (-1i32) as u32, 100);
c.v[2] = quad_u32(3, 3, 1, 200);
c.exec(0x4EA2_6420, &mut m); assert_eq!(c.v[0], quad_u32(5, 5, 1, 200));
c.exec(0x6EA2_6C20, &mut m); assert_eq!(c.v[0], quad_u32(3, 3, 1, 100));
}
#[test]
fn neon_saddl_uaddl_widening_add() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_u16(1, 2, 3, 0xFFFF); c.v[2] = quad_u16(10, 20, 30, 1);
c.exec(0x0E62_0020, &mut m); assert_eq!(c.v[0], quad_u32(11, 22, 33, 0), "signed: -1 + 1 == 0");
c.exec(0x2E62_0020, &mut m); assert_eq!(
c.v[0],
quad_u32(11, 22, 33, 0x1_0000),
"unsigned: 0xFFFF + 1"
);
}
#[test]
fn fpcr_fpsr_read_zero() {
let (mut c, mut m) = (cpu(), scratch());
c.x[0] = 0xdead;
c.exec(0xD53B_4400, &mut m); assert_eq!(c.x[0], 0, "FPCR reads 0");
c.exec(0xD51B_4400, &mut m); }
#[test]
fn tbl_tbx_and_ext() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = u128::from_le_bytes([
100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115,
]);
c.v[2] = u128::from_le_bytes([0, 5, 15, 16, 255, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
c.exec(0x0E02_0020, &mut m);
assert_eq!(
c.v[0].to_le_bytes(),
[100, 105, 115, 0, 0, 103, 100, 100, 0, 0, 0, 0, 0, 0, 0, 0]
);
c.v[0] = u128::from_le_bytes([9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0]);
c.exec(0x0E02_1020, &mut m);
assert_eq!(
c.v[0].to_le_bytes(),
[100, 105, 115, 9, 9, 103, 100, 100, 0, 0, 0, 0, 0, 0, 0, 0]
);
c.v[1] = u128::from_le_bytes([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
c.v[2] = u128::from_le_bytes([
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
]);
c.exec(0x6E02_2020, &mut m);
assert_eq!(
c.v[0].to_le_bytes(),
[4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
);
}
#[test]
fn zip_uzp_trn_permute() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_u32(0x10, 0x11, 0x12, 0x13);
c.v[2] = quad_u32(0x20, 0x21, 0x22, 0x23);
c.exec(0x4E82_3820, &mut m); assert_eq!(c.v[0], quad_u32(0x10, 0x20, 0x11, 0x21));
c.exec(0x4E82_7820, &mut m); assert_eq!(c.v[0], quad_u32(0x12, 0x22, 0x13, 0x23));
c.exec(0x4E82_1820, &mut m); assert_eq!(c.v[0], quad_u32(0x10, 0x12, 0x20, 0x22));
c.exec(0x4E82_5820, &mut m); assert_eq!(c.v[0], quad_u32(0x11, 0x13, 0x21, 0x23));
c.exec(0x4E82_2820, &mut m); assert_eq!(c.v[0], quad_u32(0x10, 0x20, 0x12, 0x22));
c.exec(0x4E82_6820, &mut m); assert_eq!(c.v[0], quad_u32(0x11, 0x21, 0x13, 0x23));
}
#[test]
fn rev64_rev32_rev16_vector() {
let (mut c, mut m) = (cpu(), scratch());
let bytes: [u8; 16] = core::array::from_fn(|i| i as u8);
c.v[1] = u128::from_le_bytes(bytes);
c.exec(0x4EA0_0820, &mut m); assert_eq!(
c.v[0].to_le_bytes(),
[4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11]
);
c.exec(0x6E60_0820, &mut m); assert_eq!(
c.v[0].to_le_bytes(),
[2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13]
);
c.exec(0x4E20_1820, &mut m); assert_eq!(
c.v[0].to_le_bytes(),
[1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14]
);
}
#[test]
fn xtn_sqxtn_uqxtn_sqxtun_narrow() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_u32(0x0001_FFFF, 0x7FFF_FFFF, 0x8000_0000, 0xFFFF_FFFF);
c.exec(0x0E61_2820, &mut m); assert_eq!(c.v[0], quad_u16(0xFFFF, 0xFFFF, 0x0000, 0xFFFF));
c.exec(0x0E61_4820, &mut m); assert_eq!(c.v[0], quad_u16(0x7FFF, 0x7FFF, 0x8000, 0xFFFF));
c.exec(0x2E61_4820, &mut m); assert_eq!(c.v[0], quad_u16(0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF));
c.exec(0x2E61_2820, &mut m); assert_eq!(c.v[0], quad_u16(0xFFFF, 0xFFFF, 0x0000, 0x0000));
}
#[test]
fn addhn_uaddw_saddw_wide() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_u32(0xFFFF_FFFF, 0x0001_0000, 0x8000_0000, 1);
c.v[2] = quad_u32(1, 0x0000_FFFF, 0x8000_0000, 1);
c.exec(0x0E62_4020, &mut m); assert_eq!(c.v[0], quad_u16(0, 1, 0, 0));
c.v[1] = quad_u32(1, 2, 3, 4);
c.v[2] = quad_u16(0xFFFF, 1, 2, 3);
c.exec(0x2E62_1020, &mut m); assert_eq!(c.v[0], quad_u32(0x1_0000, 3, 5, 7));
c.exec(0x0E62_1020, &mut m); assert_eq!(c.v[0], quad_u32(0, 3, 5, 7));
}
#[test]
fn sqadd_uqadd_sqsub_uqsub_saturate() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_u32(0x7FFF_FFFF, 0x8000_0000, 0xFFFF_FFFF, 5);
c.v[2] = quad_u32(1, 0xFFFF_FFFF, 1, 3);
c.exec(0x4EA2_0C20, &mut m); assert_eq!(
c.v[0],
quad_u32(0x7FFF_FFFF, 0x8000_0000, 0, 8),
"SQADD clamps at INT32_MAX/INT32_MIN instead of wrapping"
);
c.exec(0x6EA2_0C20, &mut m); assert_eq!(c.v[0], quad_u32(0x8000_0000, 0xFFFF_FFFF, 0xFFFF_FFFF, 8));
c.exec(0x4EA2_2C20, &mut m); assert_eq!(c.v[0], quad_u32(0x7FFF_FFFE, 0x8000_0001, 0xFFFF_FFFE, 2));
c.exec(0x6EA2_2C20, &mut m); assert_eq!(c.v[0], quad_u32(0x7FFF_FFFE, 0, 0xFFFF_FFFE, 2));
}
#[test]
fn sqshl_uqshl_sqrshl_uqrshl_register() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_u32(1, 0x4000_0000, 0xFFFF_FFFF, 0x8000_0000);
c.v[2] = quad_u32(4, 2, 0xFFFF_FFFF, 0xFFFF_FFFC);
c.exec(0x4EA2_4C20, &mut m); assert_eq!(
c.v[0],
quad_u32(0x10, 0x7FFF_FFFF, 0xFFFF_FFFF, 0xF800_0000)
);
c.exec(0x6EA2_4C20, &mut m); assert_eq!(
c.v[0],
quad_u32(0x10, 0xFFFF_FFFF, 0x7FFF_FFFF, 0x0800_0000)
);
c.exec(0x4EA2_5C20, &mut m); assert_eq!(c.v[0], quad_u32(0x10, 0x7FFF_FFFF, 0, 0xF800_0000));
c.exec(0x6EA2_5C20, &mut m); assert_eq!(
c.v[0],
quad_u32(0x10, 0xFFFF_FFFF, 0x8000_0000, 0x0800_0000)
);
}
#[test]
fn suqadd_usqadd_saturating_accumulate() {
let (mut c, mut m) = (cpu(), scratch());
c.v[0] = quad_u32(5, (-5i32) as u32, 0x7FFF_FFFF, (-2_000_000_000i32) as u32);
c.v[1] = quad_u32(10, 3, 10, 0xFFFF_FFFF);
c.exec(0x4EA0_3820, &mut m); assert_eq!(
c.v[0],
quad_u32(15, (-2i32) as u32, 0x7FFF_FFFF, 0x7FFF_FFFF)
);
c.v[0] = quad_u32(5, 3, 0xFFFF_FFF0, 0);
c.v[1] = quad_u32(10, (-5i32) as u32, 10, (-1i32) as u32);
c.exec(0x6EA0_3820, &mut m); assert_eq!(c.v[0], quad_u32(15, 0, 0xFFFF_FFFA, 0));
}
#[test]
fn addp_smaxp_sminp_umaxp_uminp_pairwise() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_u32(1, 2, 3, 4);
c.v[2] = quad_u32(10, 20, 30, 40);
c.exec(0x4EA2_BC20, &mut m); assert_eq!(c.v[0], quad_u32(3, 7, 30, 70));
c.exec(0x4EA2_A420, &mut m); assert_eq!(c.v[0], quad_u32(2, 4, 20, 40));
c.exec(0x4EA2_AC20, &mut m); assert_eq!(c.v[0], quad_u32(1, 3, 10, 30));
c.exec(0x6EA2_A420, &mut m); assert_eq!(c.v[0], quad_u32(2, 4, 20, 40));
c.exec(0x6EA2_AC20, &mut m); assert_eq!(c.v[0], quad_u32(1, 3, 10, 30));
c.v[1] = (200u128 << 64) | 0x64;
c.exec(0x5EF1_B820, &mut m);
assert_eq!(c.v[0], 300);
}
#[test]
fn faddp_fmaxp_vector_and_scalar() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_f32(1.0, 2.0, 3.0, 4.0);
c.v[2] = quad_f32(10.0, 20.0, 30.0, 40.0);
c.exec(0x6E22_D420, &mut m); assert_eq!(c.v[0], quad_f32(3.0, 7.0, 30.0, 70.0));
c.exec(0x6E22_F420, &mut m); assert_eq!(c.v[0], quad_f32(2.0, 4.0, 20.0, 40.0));
c.v[1] = quad_f32(3.5, 4.5, 0.0, 0.0);
c.exec(0x7E30_D820, &mut m);
assert_eq!(f32::from_bits(c.v[0] as u32), 8.0);
}
#[test]
fn smaxv_sminv_umaxv_uminv_across_lanes() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_u32((-5i32) as u32, 10, (-20i32) as u32, 3);
c.exec(0x4EB0_A820, &mut m); assert_eq!(c.v[0] as u32, 10);
c.exec(0x4EB1_A820, &mut m); assert_eq!(c.v[0] as u32 as i32, -20);
c.exec(0x6EB0_A820, &mut m); assert_eq!(c.v[0] as u32, (-5i32) as u32);
c.exec(0x6EB1_A820, &mut m); assert_eq!(c.v[0] as u32, 3);
}
#[test]
fn crc32x_and_crc32cx_known_vectors() {
let (mut c, mut m) = (cpu(), scratch());
c.x[1] = 0xFFFF_FFFF;
c.x[2] = 0x3736_3534_3332_3130;
c.exec(0x9AC2_4C20, &mut m); assert_eq!(c.x[0] as u32, 0xd27f_c50a);
c.exec(0x9AC2_5C20, &mut m); assert_eq!(c.x[0] as u32, 0x53dd_dcdf);
c.x[1] = 0xFFFF_FFFF;
for &byte in b"123456789" {
c.x[2] = u64::from(byte);
c.exec(0x1AC2_4020, &mut m); c.x[1] = c.x[0];
}
assert_eq!(!(c.x[1] as u32), 0xcbf4_3926);
}
#[test]
fn unknown_instruction_is_illegal() {
let (mut c, mut m) = (cpu(), scratch());
assert!(matches!(c.exec(0x0000_0000, &mut m), Step::Illegal));
}
#[test]
fn run_faults_on_unmapped_pc() {
let mut mem = GuestMemory::new(0x1_0000, PAGE_SIZE);
let mut c = Aarch64Interp::new(0x1_0000, 0x1_0000);
assert_eq!(
c.run(&mut mem).unwrap(),
Exit::MemFault {
addr: 0x1_0000,
write: false
}
);
}
#[test]
fn fmul_by_element() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = u128::from(3.0f32.to_bits());
c.v[2] = quad_f32(2.0, 100.0, 100.0, 100.0); c.exec(0x5F82_9020, &mut m);
assert_eq!(f32::from_bits(c.v[0] as u32), 6.0);
c.v[1] = quad_f32(1.0, 2.0, 3.0, 4.0);
c.v[2] = quad_f32(10.0, 5.0, 10.0, 10.0);
c.exec(0x4FA2_9020, &mut m);
assert_eq!(
c.v[0],
quad_f32(5.0, 10.0, 15.0, 20.0),
"each lane of v1 * v2.s[1] (5.0)"
);
}
#[test]
fn fmla_by_element_accumulates() {
let (mut c, mut m) = (cpu(), scratch());
c.v[0] = quad_f32(1.0, 1.0, 1.0, 1.0); c.v[1] = quad_f32(1.0, 2.0, 3.0, 4.0);
c.v[2] = quad_f32(10.0, 5.0, 10.0, 10.0); c.exec(0x4FA2_1020, &mut m);
assert_eq!(c.v[0], quad_f32(6.0, 11.0, 16.0, 21.0));
}
#[test]
fn frecpe_then_frecps_converges_toward_reciprocal() {
let (mut c, mut m) = (cpu(), scratch());
let x = 4.0f32;
c.v[1] = u128::from(x.to_bits());
c.exec(0x5EA1_D820, &mut m);
let estimate = f32::from_bits(c.v[0] as u32);
assert_eq!(
estimate, 0.25,
"this interpreter's FRECPE is the exact reciprocal"
);
c.v[1] = u128::from(x.to_bits()); c.v[2] = u128::from(estimate.to_bits()); c.exec(0x5E22_FC20, &mut m); let step = f32::from_bits(c.v[0] as u32);
let refined = estimate * step;
assert!(
(refined - 1.0 / x).abs() < 1e-6,
"refined={refined} should converge to 1/x={}",
1.0 / x
);
}
#[test]
fn fcmeq_vector_vs_zero_mask() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = quad_f32(0.0, -0.0, 1.0, f32::NAN);
c.exec(0x4EA0_D820, &mut m);
assert_eq!(
c.v[0],
quad_u32(u32::MAX, u32::MAX, 0, 0),
"lanes 0/1 (+0.0/-0.0) compare equal to zero, lane 2 (1.0) and \
lane 3 (NaN, unordered) do not"
);
}
#[test]
fn fabd_scalar_and_vector() {
let (mut c, mut m) = (cpu(), scratch());
c.v[1] = u128::from(5.0f32.to_bits());
c.v[2] = u128::from(8.0f32.to_bits());
c.exec(0x7EA2_D420, &mut m);
assert_eq!(f32::from_bits(c.v[0] as u32), 3.0);
c.v[1] = quad_f32(1.0, -2.0, 10.0, 0.0);
c.v[2] = quad_f32(4.0, 2.0, 3.0, -5.0);
c.exec(0x6EA2_D420, &mut m);
assert_eq!(c.v[0], quad_f32(3.0, 4.0, 7.0, 5.0));
}
#[test]
fn ldnp_stnp_pair_roundtrip() {
let mut c = cpu();
let mut m = GuestMemory::new(0x1_0000, 4 * PAGE_SIZE);
m.map(0x1_0000, PAGE_SIZE, Prot::rw()).unwrap();
c.x[2] = 0x1_0040; c.x[0] = 0x1111_1111_1111_1111;
c.x[1] = 0x2222_2222_2222_2222;
assert!(matches!(
c.exec(0xA800_0440, &mut m), Step::Next
));
c.x[0] = 0;
c.x[1] = 0;
assert!(matches!(
c.exec(0xA840_0440, &mut m), Step::Next
));
assert_eq!(c.x[0], 0x1111_1111_1111_1111);
assert_eq!(c.x[1], 0x2222_2222_2222_2222);
}
#[test]
fn prfm_decodes_as_noop() {
let (mut c, mut m) = (cpu(), scratch());
c.x[0] = 0xDEAD_0000;
let pc_before = c.pc;
assert!(matches!(c.exec(0xF980_0000, &mut m), Step::Next));
assert_eq!(c.x[0], 0xDEAD_0000, "PRFM must not touch any register");
assert_eq!(
c.pc, pc_before,
"exec() itself doesn't advance pc; run() does"
);
c.x[1] = 8;
assert!(matches!(c.exec(0xF8A1_6800, &mut m), Step::Next));
}
#[test]
fn mrs_ctr_el0_returns_constant() {
let (mut c, mut m) = (cpu(), scratch());
assert!(matches!(c.exec(0xD53B_0020, &mut m), Step::Next));
assert_eq!(c.x[0], CTR_EL0_VAL);
}
#[test]
fn mrs_cntvct_el0_increases_across_reads() {
let (mut c, mut m) = (cpu(), scratch());
assert!(matches!(c.exec(0xD53B_E040, &mut m), Step::Next));
let first = c.x[0];
assert!(matches!(c.exec(0xD53B_E040, &mut m), Step::Next));
let second = c.x[0];
assert!(
second > first,
"a spin on CNTVCT_EL0 must observe it advance"
);
}
#[test]
fn dc_zva_zeroes_a_64_byte_block() {
let base = 0x1_0000u64;
let mut m = GuestMemory::new(base, 4 * PAGE_SIZE);
m.map(base, PAGE_SIZE, Prot::rw()).unwrap();
let region = base + 0x100;
m.write(region, &[0xAAu8; 256]).unwrap();
let mut c = cpu();
let unaligned_off = 0x72usize;
c.x[0] = region + unaligned_off as u64;
assert!(matches!(c.exec(0xD50B_7420, &mut m), Step::Next));
let mut buf = [0u8; 256];
m.read(region, &mut buf).unwrap();
let block = DC_ZVA_BLOCK_BYTES as usize;
let aligned_off = unaligned_off & !(block - 1);
assert!(
buf[..aligned_off].iter().all(|&b| b == 0xAA),
"before the block"
);
assert!(
buf[aligned_off..aligned_off + block]
.iter()
.all(|&b| b == 0),
"the DC ZVA block itself"
);
assert!(
buf[aligned_off + block..].iter().all(|&b| b == 0xAA),
"after the block"
);
}
#[test]
fn dmb_isb_decode_as_noops() {
let (mut c, mut m) = (cpu(), scratch());
c.pc = 0x1000;
assert!(matches!(c.exec(0xD503_3FBF, &mut m), Step::Next));
assert_eq!(c.pc, 0x1000);
assert!(matches!(c.exec(0xD503_3FDF, &mut m), Step::Next));
assert_eq!(c.pc, 0x1000);
}
#[test]
fn aese_matches_hardware_vector() {
let (mut c, mut m) = (cpu(), scratch());
c.v[0] = 0; c.v[1] = 0x0f0e_0d0c_0b0a_0908_0706_0504_0302_0100; assert!(matches!(c.exec(0x4E28_4820, &mut m), Step::Next));
assert_eq!(c.v[0], 0x2b6f_7cfe_c577_d730_7bab_01f2_7667_6b63);
}
#[test]
fn aes_sbox_matches_generated_table() {
fn gf_mul(mut a: u8, mut b: u8) -> u8 {
let mut p = 0u8;
for _ in 0..8 {
if b & 1 != 0 {
p ^= a;
}
let hi = a & 0x80;
a <<= 1;
if hi != 0 {
a ^= 0x1b;
}
b >>= 1;
}
p
}
let mut inv = [0u8; 256];
for a in 1..=255u16 {
for b in 1..=255u16 {
if gf_mul(a as u8, b as u8) == 1 {
inv[a as usize] = b as u8;
break;
}
}
}
let rol = |x: u8, n: u32| x.rotate_left(n);
for i in 0..256usize {
let b = inv[i];
let generated = b ^ rol(b, 1) ^ rol(b, 2) ^ rol(b, 3) ^ rol(b, 4) ^ 0x63;
assert_eq!(AES_SBOX[i], generated, "AES_SBOX[{i:#04x}]");
assert_eq!(AES_INV_SBOX[generated as usize], i as u8, "AES_INV_SBOX");
}
}
#[test]
fn sha256h_matches_hardware_vector() {
let (mut c, mut m) = (cpu(), scratch());
c.v[0] = 0xfeed_face_0def_aced_8bad_f00d_dead_beef; c.v[1] = 0xba5e_ba11_5ca1_ab1e_1337_c0de_cafe_babe; c.v[2] = 0xc0ff_ee11_ba1d_2323_0fac_ade1_000f_f1ce; assert!(matches!(c.exec(0x5E02_4020, &mut m), Step::Next));
assert_eq!(c.v[0], 0xea32_0a6e_642e_88ee_9b73_03d0_dd3d_d598);
}
#[test]
fn sha256_block_compression_matches_known_digest() {
const K: [u32; 64] = [
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4,
0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe,
0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f,
0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc,
0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116,
0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
0xc67178f2,
];
const H0: [u32; 8] = [
0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab,
0x5be0cd19,
];
const EXPECT: [u32; 8] = [
0xba7816bf, 0x8f01cfea, 0x414140de, 0x5dae2223, 0xb00361a3, 0x96177a9c, 0xb410ff61,
0xf20015ad,
];
let mut block = [0u8; 64];
block[0..3].copy_from_slice(b"abc");
block[3] = 0x80;
block[63] = 0x18; let mut w = [0u32; 64];
for i in 0..16 {
w[i] = u32::from_be_bytes(block[4 * i..4 * i + 4].try_into().unwrap());
}
let (mut c, mut m) = (cpu(), scratch());
for i in (16..64).step_by(4) {
let wd = pack4(&w, i - 16);
let wn = pack4(&w, i - 12);
let wm8 = pack4(&w, i - 8);
let wm4 = pack4(&w, i - 4);
c.v[0] = wd; c.v[1] = wn;
assert!(matches!(c.exec(0x5E28_2820, &mut m), Step::Next));
c.v[2] = wm8;
c.v[3] = wm4;
assert!(matches!(c.exec(0x5E03_6040, &mut m), Step::Next));
for (j, word) in u32_lanes(c.v[0]).into_iter().enumerate() {
w[i + j] = word;
}
}
let abcd0 = pack_u32_lanes([H0[0], H0[1], H0[2], H0[3]]);
let efgh0 = pack_u32_lanes([H0[4], H0[5], H0[6], H0[7]]);
c.v[10] = abcd0; c.v[11] = efgh0; c.v[0] = abcd0;
c.v[1] = efgh0;
for i in (0..64).step_by(4) {
let wk = pack_u32_lanes([
w[i].wrapping_add(K[i]),
w[i + 1].wrapping_add(K[i + 1]),
w[i + 2].wrapping_add(K[i + 2]),
w[i + 3].wrapping_add(K[i + 3]),
]);
c.v[2] = wk;
c.v[10] = c.v[0]; assert!(matches!(c.exec(0x5E02_4020, &mut m), Step::Next));
assert!(matches!(c.exec(0x5E02_5141, &mut m), Step::Next));
}
let final_a = u32_lanes(c.v[0]);
let final_e = u32_lanes(c.v[1]);
let digest = [
H0[0].wrapping_add(final_a[0]),
H0[1].wrapping_add(final_a[1]),
H0[2].wrapping_add(final_a[2]),
H0[3].wrapping_add(final_a[3]),
H0[4].wrapping_add(final_e[0]),
H0[5].wrapping_add(final_e[1]),
H0[6].wrapping_add(final_e[2]),
H0[7].wrapping_add(final_e[3]),
];
assert_eq!(digest, EXPECT);
}
fn pack4(w: &[u32; 64], i: usize) -> u128 {
pack_u32_lanes([w[i], w[i + 1], w[i + 2], w[i + 3]])
}
}