rigless 0.2.1

// Copyright (c) 2026 Rigless Labs.
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Multiplies, divides, the parallel arithmetic and the miscellaneous operations, DDI 0403E.b A5.3.13 to A5.3.17.
//!
//! Every instruction here makes SP and the PC UNPREDICTABLE as a register, and a should-be-one bit
//! that isn't one UNPREDICTABLE too, A5.1. The glossary lets an UNPREDICTABLE instruction be
//! UNDEFINED, so that's what you get for each, and no multiply ever has to write the PC.

use std::mem::offset_of;

use cranelift_codegen::ir::{InstBuilder, Value, condcodes::IntCC, types};

use super::super::family::Family;
use super::super::registers::{ccr, is_set};
use super::super::state::State;
use super::alu::{add_with_carry, signed_sat_q, unsigned_sat_q};
use super::{Emitter, Fault, Translated, Undefined, bits};

/// Which halfword of a register an operand comes from, the encodings' `N` and `M` bits.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Half {
    Bottom,
    Top,
}

impl Half {
    fn of(bit: u32) -> Self {
        if bit == 0 { Self::Bottom } else { Self::Top }
    }
}

/// Whether the second term is added or subtracted.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Combine {
    Add,
    Subtract,
}

impl Combine {
    fn apply(self, e: &mut Emitter<'_, '_>, x: Value, y: Value) -> Value {
        match self {
            Self::Add => e.b.ins().iadd(x, y),
            Self::Subtract => e.b.ins().isub(x, y),
        }
    }
}

/// `R[n]`, `R[m]`, and `R[a]` when the instruction accumulates.
#[derive(Clone, Copy)]
struct Operands {
    rn: Value,
    rm: Value,
    ra: Option<Value>,
}

/// Whether none of `registers` is SP or the PC.
fn usable(registers: &[u32]) -> bool {
    registers.iter().all(|&r| r != 13 && r != 15)
}

/// A5.3.16, multiply, multiply accumulate, and absolute difference.
pub(super) fn multiply_accumulate(e: &mut Emitter<'_, '_>, hw1: u16, hw2: u16) -> Translated {
    let (hw1, hw2) = (u32::from(hw1), u32::from(hw2));
    let (op1, op2) = (bits(hw1, 6, 4), bits(hw2, 5, 4));
    let (n, a, d, m) = (
        bits(hw1, 3, 0),
        bits(hw2, 15, 12),
        bits(hw2, 11, 8),
        bits(hw2, 3, 0),
    );
    // Ra of 15 means no accumulator, except for MLS and SMMLS, which always have one and make
    // it UNPREDICTABLE.
    let accumulate = a != 15;
    let exchange_or_round = op2 & 1 != 0;
    let operation = match (op1, op2) {
        (0b000, 0b00) => Accumulating::Multiply,
        (0b000, 0b01) if accumulate => Accumulating::MultiplySubtract,
        (0b001, _) => Accumulating::Halfwords(Half::of(op2 >> 1), Half::of(op2 & 1)),
        (0b010, 0b00 | 0b01) => Accumulating::Dual(Combine::Add, exchange_or_round),
        (0b011, 0b00 | 0b01) => Accumulating::WordByHalfword(Half::of(op2)),
        (0b100, 0b00 | 0b01) => Accumulating::Dual(Combine::Subtract, exchange_or_round),
        (0b101, 0b00 | 0b01) => Accumulating::MostSignificant(Combine::Add, exchange_or_round),
        (0b110, 0b00 | 0b01) if accumulate => {
            Accumulating::MostSignificant(Combine::Subtract, exchange_or_round)
        }
        (0b111, 0b00) => Accumulating::AbsoluteDifferences,
        _ => return Err(Undefined),
    };
    if bits(hw2, 7, 6) != 0 || !usable(&[d, n, m]) || a == 13 {
        return Err(Undefined);
    }
    let (rn, rm) = (e.reg(n), e.reg(m));
    let ra = accumulate.then(|| e.reg(a));
    let operands = Operands { rn, rm, ra };
    let result = match operation {
        Accumulating::Multiply => {
            let product = e.b.ins().imul(rn, rm);
            ra.map_or(product, |ra| e.b.ins().iadd(product, ra))
        }
        Accumulating::MultiplySubtract => {
            let product = e.b.ins().imul(rn, rm);
            e.b.ins().isub(ra.expect("MLS accumulates"), product)
        }
        Accumulating::Halfwords(of_n, of_m) => halfwords(e, operands, of_n, of_m),
        Accumulating::Dual(combine, swap) => dual(e, operands, combine, swap),
        Accumulating::WordByHalfword(of_m) => word_by_halfword(e, operands, of_m),
        Accumulating::MostSignificant(combine, round) => {
            most_significant(e, operands, combine, round)
        }
        Accumulating::AbsoluteDifferences => absolute_differences(e, operands),
    };
    e.set_reg(d, result);
    Ok(())
}

/// What Table A5-28 allocates, with what each encoding's bits choose.
#[derive(Clone, Copy)]
enum Accumulating {
    /// MUL, A7.7.83 T2, and MLA, A7.7.73. The low 32 bits are the same signed or not.
    Multiply,
    /// MLS, A7.7.74.
    MultiplySubtract,
    /// SMUL and SMLA by halfwords, with `N` and `M`.
    Halfwords(Half, Half),
    /// SMUAD and SMLAD, or SMUSD and SMLSD, with `M`, the exchange.
    Dual(Combine, bool),
    /// SMULW and SMLAW, with `M`.
    WordByHalfword(Half),
    /// SMMUL and SMMLA, or SMMLS, with `R`, the rounding.
    MostSignificant(Combine, bool),
    /// USAD8 and USADA8.
    AbsoluteDifferences,
}

/// SMULBB to SMULTT, A7.7.146, and SMLABB to SMLATT, A7.7.134, which sets Q when adding the
/// accumulator overflows. The product itself never does.
fn halfwords(e: &mut Emitter<'_, '_>, operands: Operands, x: Half, y: Half) -> Value {
    let (operand1, operand2) = (half(e, operands.rn, x), half(e, operands.rm, y));
    let product = e.b.ins().imul(operand1, operand2);
    let Some(ra) = operands.ra else {
        return product;
    };
    let carry_in = e.constant(0);
    let (result, _, overflow) = add_with_carry(e, product, ra, carry_in);
    set_q_when(e, overflow);
    result
}

/// SMUAD, A7.7.145, SMLAD, A7.7.135, SMUSD, A7.7.149, and SMLSD, A7.7.140. SMUSD can't
/// overflow, so the Q check the others need never fires for it.
fn dual(e: &mut Emitter<'_, '_>, operands: Operands, combine: Combine, swap: bool) -> Value {
    let mut result = dual_sum(e, operands.rn, operands.rm, combine, swap);
    if let Some(ra) = operands.ra {
        let ra = e.b.ins().sextend(types::I64, ra);
        result = e.b.ins().iadd(result, ra);
    }
    let overflow = overflows(e, result);
    set_q_when(e, overflow);
    e.b.ins().ireduce(types::I32, result)
}

/// SMULWB and SMULWT, A7.7.148, and SMLAWB and SMLAWT, A7.7.139: bits 47 to 16 of a 48-bit
/// product, which only the accumulating forms can overflow.
fn word_by_halfword(e: &mut Emitter<'_, '_>, operands: Operands, y: Half) -> Value {
    let operand2 = half(e, operands.rm, y);
    let operand1 = e.b.ins().sextend(types::I64, operands.rn);
    let operand2 = e.b.ins().sextend(types::I64, operand2);
    let mut product = e.b.ins().imul(operand1, operand2);
    if let Some(ra) = operands.ra {
        let ra = e.b.ins().sextend(types::I64, ra);
        let ra = e.b.ins().ishl_imm_u(ra, 16);
        product = e.b.ins().iadd(product, ra);
    }
    let shifted = e.b.ins().sshr_imm_u(product, 16);
    if operands.ra.is_some() {
        let overflow = overflows(e, shifted);
        set_q_when(e, overflow);
    }
    e.b.ins().ireduce(types::I32, shifted)
}

/// SMMUL, A7.7.144, SMMLA, A7.7.142, and SMMLS, A7.7.143, rounding with `R`. Only bits 63 to 32
/// are kept, and arithmetic modulo 2^64 gives those exactly.
fn most_significant(
    e: &mut Emitter<'_, '_>,
    operands: Operands,
    combine: Combine,
    round: bool,
) -> Value {
    let product = long_product(e, operands.rn, operands.rm, true);
    let mut result = match operands.ra {
        Some(ra) => {
            let ra = e.b.ins().uextend(types::I64, ra);
            let ra = e.b.ins().ishl_imm_u(ra, 32);
            combine.apply(e, ra, product)
        }
        None => product,
    };
    if round {
        result = e.b.ins().iadd_imm_u(result, 0x8000_0000_i64);
    }
    high_word(e, result)
}

/// USAD8, A7.7.208, and USADA8, A7.7.209.
fn absolute_differences(e: &mut Emitter<'_, '_>, operands: Operands) -> Value {
    let mut sum = operands.ra.unwrap_or_else(|| e.constant(0));
    for byte in 0..4 {
        let x = lane(e, operands.rn, byte, 8, false);
        let y = lane(e, operands.rm, byte, 8, false);
        let difference = e.b.ins().isub(x, y);
        let difference = e.b.ins().iabs(difference);
        sum = e.b.ins().iadd(sum, difference);
    }
    sum
}

/// A5.3.17, long multiply, long multiply accumulate, and divide.
pub(super) fn long_multiply_divide(e: &mut Emitter<'_, '_>, hw1: u16, hw2: u16) -> Translated {
    let (hw1, hw2) = (u32::from(hw1), u32::from(hw2));
    let (op1, op2) = (bits(hw1, 6, 4), bits(hw2, 7, 4));
    let swap = op2 & 1 != 0;
    let operation = match (op1, op2) {
        (0b001, 0b1111) => return divide(e, hw1, hw2, true),
        (0b011, 0b1111) => return divide(e, hw1, hw2, false),
        (0b000, 0b0000) => Long::Multiply(true),
        (0b010, 0b0000) => Long::Multiply(false),
        (0b100, 0b0000) => Long::Accumulate(true),
        (0b110, 0b0000) => Long::Accumulate(false),
        (0b100, 0b1000..=0b1011) => Long::Halfwords(Half::of(op2 >> 1 & 1), Half::of(op2 & 1)),
        (0b100, 0b1100 | 0b1101) => Long::Dual(Combine::Add, swap),
        (0b101, 0b1100 | 0b1101) => Long::Dual(Combine::Subtract, swap),
        (0b110, 0b0110) => Long::AccumulateAccumulate,
        _ => return Err(Undefined),
    };
    let (n, lo, hi, m) = (
        bits(hw1, 3, 0),
        bits(hw2, 15, 12),
        bits(hw2, 11, 8),
        bits(hw2, 3, 0),
    );
    if !usable(&[lo, hi, n, m]) || lo == hi {
        return Err(Undefined);
    }
    let (rn, rm) = (e.reg(n), e.reg(m));
    let (rd_lo, rd_hi) = (e.reg(lo), e.reg(hi));
    let rd_lo = e.b.ins().uextend(types::I64, rd_lo);
    let rd_hi = e.b.ins().uextend(types::I64, rd_hi);
    let result = match operation {
        Long::Multiply(signed) => long_product(e, rn, rm, signed),
        Long::Accumulate(signed) => {
            let product = long_product(e, rn, rm, signed);
            let accumulator = pair(e, rd_lo, rd_hi);
            e.b.ins().iadd(product, accumulator)
        }
        Long::Halfwords(x, y) => {
            let (operand1, operand2) = (half(e, rn, x), half(e, rm, y));
            let product = e.b.ins().imul(operand1, operand2);
            let product = e.b.ins().sextend(types::I64, product);
            let accumulator = pair(e, rd_lo, rd_hi);
            e.b.ins().iadd(product, accumulator)
        }
        Long::Dual(combine, swap) => {
            let sum = dual_sum(e, rn, rm, combine, swap);
            let accumulator = pair(e, rd_lo, rd_hi);
            e.b.ins().iadd(sum, accumulator)
        }
        Long::AccumulateAccumulate => {
            let product = long_product(e, rn, rm, false);
            let sum = e.b.ins().iadd(product, rd_lo);
            e.b.ins().iadd(sum, rd_hi)
        }
    };
    let low = e.b.ins().ireduce(types::I32, result);
    let high = high_word(e, result);
    e.set_reg(hi, high);
    e.set_reg(lo, low);
    Ok(())
}

/// What Table A5-29 allocates beside the divides.
#[derive(Clone, Copy)]
enum Long {
    /// SMULL, A7.7.147, or UMULL, A7.7.201, by whether it's signed.
    Multiply(bool),
    /// SMLAL, A7.7.136, or UMLAL, A7.7.200, by whether it's signed.
    Accumulate(bool),
    /// SMLALBB to SMLALTT, A7.7.137, with `N` and `M`. The 64-bit sum wraps unseen.
    Halfwords(Half, Half),
    /// SMLALD, A7.7.138, or SMLSLD, A7.7.141, with `M`, the exchange.
    Dual(Combine, bool),
    /// UMAAL, A7.7.199, whose sum can't pass 64 bits.
    AccumulateAccumulate,
}

/// The 64-bit product of two registers, signed or not.
fn long_product(e: &mut Emitter<'_, '_>, rn: Value, rm: Value, signed: bool) -> Value {
    let (x, y) = if signed {
        (
            e.b.ins().sextend(types::I64, rn),
            e.b.ins().sextend(types::I64, rm),
        )
    } else {
        (
            e.b.ins().uextend(types::I64, rn),
            e.b.ins().uextend(types::I64, rm),
        )
    };
    e.b.ins().imul(x, y)
}

/// `R[dHi]:R[dLo]` from both zero-extended. Signed or not is the same modulo 2^64, which is all
/// the long multiplies keep.
fn pair(e: &mut Emitter<'_, '_>, rd_lo: Value, rd_hi: Value) -> Value {
    let rd_hi = e.b.ins().ishl_imm_u(rd_hi, 32);
    e.b.ins().bor(rd_hi, rd_lo)
}

/// SDIV, A7.7.125, and UDIV, A7.7.192. A zero divisor gives 0, or a UsageFault when
/// `CCR.DIV_0_TRP` says to trap, `GenerateIntegerZeroDivide()`, D6.7.20.
fn divide(e: &mut Emitter<'_, '_>, hw1: u32, hw2: u32, signed: bool) -> Translated {
    let (n, d, m) = (bits(hw1, 3, 0), bits(hw2, 11, 8), bits(hw2, 3, 0));
    if bits(hw2, 15, 12) != 0b1111 || !usable(&[d, n, m]) {
        return Err(Undefined);
    }
    let (rn, rm) = (e.reg(n), e.reg(m));
    let zero = e.b.ins().icmp_imm_s(IntCC::Equal, rm, 0);
    let check = e.b.create_block();
    let divide = e.b.create_block();
    // Only a zero divisor pays for the call that reads CCR.
    e.b.ins().brif(zero, check, &[], divide, &[]);
    e.b.seal_block(check);
    e.b.switch_to_block(check);
    let traps = e.call_plain(zero_divide_traps as *const () as usize, &[]);
    e.raise_if(traps, Fault::DivideByZero);
    e.b.ins().jump(divide, &[]);
    e.b.seal_block(divide);
    e.b.switch_to_block(divide);
    // The host traps on a zero divisor too, so it never sees one. The result is thrown away then.
    let one = e.constant(1);
    let divisor = e.b.ins().select(zero, one, rm);
    let quotient = if signed {
        // 0x80000000 by -1 is 2^31, which an i64 holds, and its low 32 bits are what the
        // pseudocode writes.
        let dividend = e.b.ins().sextend(types::I64, rn);
        let divisor = e.b.ins().sextend(types::I64, divisor);
        let quotient = e.b.ins().sdiv(dividend, divisor);
        e.b.ins().ireduce(types::I32, quotient)
    } else {
        e.b.ins().udiv(rn, divisor)
    };
    let nothing = e.constant(0);
    let result = e.b.ins().select(zero, nothing, quotient);
    e.set_reg(d, result);
    Ok(())
}

/// `IntegerZeroDivideTrappingEnabled()`, D6.7.29: `CCR.DIV_0_TRP`, as 0 or 1.
extern "C" fn zero_divide_traps(core: &mut Family) -> u32 {
    u32::from(is_set(ccr::DIV_0_TRP, core.system.ccr))
}

/// A5.3.13, parallel addition and subtraction, signed.
pub(super) fn parallel_signed(e: &mut Emitter<'_, '_>, hw1: u16, hw2: u16) -> Translated {
    parallel(e, u32::from(hw1), u32::from(hw2), true)
}

/// A5.3.14, parallel addition and subtraction, unsigned.
pub(super) fn parallel_unsigned(e: &mut Emitter<'_, '_>, hw1: u16, hw2: u16) -> Translated {
    parallel(e, u32::from(hw1), u32::from(hw2), false)
}

/// Tables A5-25 and A5-26, which share their encodings but for bit 6 of `hw2`. `op1` picks the
/// lanes and the operation in each, `op2` what's done with each lane's result: GE set from it,
/// A7.7.119 and its kind, saturated without touching Q, A7.7.101 and its kind, or halved, A7.7.128
/// and its kind.
fn parallel(e: &mut Emitter<'_, '_>, hw1: u32, hw2: u32, signed: bool) -> Translated {
    use Combine::{Add, Subtract};
    let (op1, op2) = (bits(hw1, 6, 4), bits(hw2, 5, 4));
    let (n, d, m) = (bits(hw1, 3, 0), bits(hw2, 11, 8), bits(hw2, 3, 0));
    // Each entry is the lane written, the lane of Rm it takes, and how. The exchanging forms,
    // ASX and SAX, cross Rm's halfwords, so there the two lanes differ.
    let (width, lanes): (u32, &[(u32, u32, Combine)]) = match op1 {
        0b000 => (8, &[(0, 0, Add), (1, 1, Add), (2, 2, Add), (3, 3, Add)]),
        0b100 => (
            8,
            &[
                (0, 0, Subtract),
                (1, 1, Subtract),
                (2, 2, Subtract),
                (3, 3, Subtract),
            ],
        ),
        0b001 => (16, &[(0, 0, Add), (1, 1, Add)]),
        0b101 => (16, &[(0, 0, Subtract), (1, 1, Subtract)]),
        0b010 => (16, &[(0, 1, Subtract), (1, 0, Add)]),
        0b110 => (16, &[(0, 1, Add), (1, 0, Subtract)]),
        _ => return Err(Undefined),
    };
    if op2 == 0b11 || bits(hw2, 15, 12) != 0b1111 || !usable(&[d, n, m]) {
        return Err(Undefined);
    }
    let (rn, rm) = (e.reg(n), e.reg(m));
    let mut result = e.constant(0);
    let mut ge = e.constant(0);
    for &(at, from, combine) in lanes {
        let x = lane(e, rn, at, width, signed);
        let y = lane(e, rm, from, width, signed);
        let exact = combine.apply(e, x, y);
        let kept = match op2 {
            0b00 => {
                // An unsigned sum sets GE when it carries out of the lane, everything else when
                // it isn't negative.
                let (condition, bound) = if signed || combine == Subtract {
                    (IntCC::SignedGreaterThanOrEqual, 0)
                } else {
                    (IntCC::UnsignedGreaterThanOrEqual, 1 << width)
                };
                let set = e.b.ins().icmp_imm_s(condition, exact, bound);
                // A halfword lane sets two GE bits, a byte lane one.
                let lane_bits = if width == 16 {
                    0b11 << (2 * at)
                } else {
                    1 << at
                };
                let (lane_bits, clear) = (e.constant(lane_bits), e.constant(0));
                let lane_bits = e.b.ins().select(set, lane_bits, clear);
                ge = e.b.ins().bor(ge, lane_bits);
                exact
            }
            0b01 => {
                let exact = e.b.ins().sextend(types::I64, exact);
                if signed {
                    signed_sat_q(e, exact, width).0
                } else {
                    unsigned_sat_q(e, exact, width).0
                }
            }
            _ => e.b.ins().sshr_imm_u(exact, 1),
        };
        let kept = e.b.ins().band_imm_u(kept, (1_i64 << width) - 1);
        let kept = e.b.ins().ishl_imm_u(kept, i64::from(at * width));
        result = e.b.ins().bor(result, kept);
    }
    e.set_reg(d, result);
    if op2 == 0b00 {
        e.store_field(offset_of!(State, ge), ge);
    }
    Ok(())
}

/// A5.3.15, miscellaneous operations.
pub(super) fn miscellaneous(e: &mut Emitter<'_, '_>, hw1: u16, hw2: u16) -> Translated {
    let (hw1, hw2) = (u32::from(hw1), u32::from(hw2));
    let (op1, op2) = (bits(hw1, 5, 4), bits(hw2, 5, 4));
    let (n, d, m) = (bits(hw1, 3, 0), bits(hw2, 11, 8), bits(hw2, 3, 0));
    if (op1 >= 0b10 && op2 != 0b00) || bits(hw2, 15, 12) != 0b1111 || !usable(&[d, n, m]) {
        return Err(Undefined);
    }
    // The one-operand forms encode Rm twice, and two that differ fail `Consistent(Rm)`, which is
    // UNPREDICTABLE.
    if op1 & 1 == 1 && n != m {
        return Err(Undefined);
    }
    let result = match op1 {
        0b00 => {
            let combine = if op2 & 0b10 == 0 {
                Combine::Add
            } else {
                Combine::Subtract
            };
            saturating(e, n, m, combine, op2 & 1 != 0)
        }
        0b10 => select_bytes(e, n, m),
        _ => {
            let rm = e.reg(m);
            reverse_count(e, rm, op1, op2)
        }
    };
    e.set_reg(d, result);
    Ok(())
}

/// QADD, A7.7.100, QSUB, A7.7.107, QDADD, A7.7.104, and QDSUB, A7.7.105: `Rm` plus or minus `Rn`,
/// which the doubling forms double first. Each saturation sets Q.
fn saturating(e: &mut Emitter<'_, '_>, n: u32, m: u32, combine: Combine, double: bool) -> Value {
    let (rn, rm) = (e.reg(n), e.reg(m));
    let mut operand = e.b.ins().sextend(types::I64, rn);
    let mut saturated = e.constant(0);
    if double {
        let twice = e.b.ins().ishl_imm_u(operand, 1);
        let (doubled, doubling_saturated) = signed_sat_q(e, twice, 32);
        operand = e.b.ins().sextend(types::I64, doubled);
        saturated = doubling_saturated;
    }
    let rm = e.b.ins().sextend(types::I64, rm);
    let exact = combine.apply(e, rm, operand);
    let (result, result_saturated) = signed_sat_q(e, exact, 32);
    let saturated = e.b.ins().bor(saturated, result_saturated);
    set_q_when(e, saturated);
    result
}

/// SEL, A7.7.126: each byte from `Rn` where its GE bit is set, and from `Rm` where it isn't.
fn select_bytes(e: &mut Emitter<'_, '_>, n: u32, m: u32) -> Value {
    let (rn, rm) = (e.reg(n), e.reg(m));
    let ge = e.load_field(offset_of!(State, ge));
    let mut mask = e.constant(0);
    for byte in 0..4 {
        let bit = e.b.ins().ushr_imm_u(ge, byte);
        let bit = e.b.ins().band_imm_u(bit, 1);
        let lanes = e.b.ins().imul_imm_u(bit, 0xFF_i64 << (8 * byte));
        mask = e.b.ins().bor(mask, lanes);
    }
    let from_n = e.b.ins().band(rn, mask);
    let from_m = e.b.ins().band_not(rm, mask);
    e.b.ins().bor(from_n, from_m)
}

/// REV, A7.7.111 T2, REV16, A7.7.112 T2, RBIT, A7.7.110, REVSH, A7.7.113 T2, and CLZ, A7.7.24,
/// by Table A5-27's `op1` and `op2`.
fn reverse_count(e: &mut Emitter<'_, '_>, rm: Value, op1: u32, op2: u32) -> Value {
    match (op1, op2) {
        (0b01, 0b00) => e.b.ins().bswap(rm),
        (0b01, 0b01) => {
            // Swapping all four bytes and then the halfwords swaps the bytes in each halfword.
            let swapped = e.b.ins().bswap(rm);
            e.b.ins().rotr_imm_u(swapped, 16)
        }
        (0b01, 0b10) => e.b.ins().bitrev(rm),
        (0b01, _) => {
            let top = e.b.ins().ishl_imm_u(rm, 24);
            let top = e.b.ins().sshr_imm_u(top, 16);
            let bottom = e.b.ins().ushr_imm_u(rm, 8);
            let bottom = e.b.ins().band_imm_u(bottom, 0xFF);
            e.b.ins().bor(top, bottom)
        }
        // CountLeadingZeroBits gives 32 for zero, as the host's count does.
        _ => e.b.ins().clz(rm),
    }
}

/// `product1` plus or minus `product2` of the dual multiplies, A7.7.135, as an `i64`: the bottom
/// halfwords' product and the top halfwords', after `ROR(R[m], 16)` when `swap`.
fn dual_sum(e: &mut Emitter<'_, '_>, rn: Value, rm: Value, combine: Combine, swap: bool) -> Value {
    let operand2 = if swap {
        e.b.ins().rotr_imm_u(rm, 16)
    } else {
        rm
    };
    let [product1, product2] = [Half::Bottom, Half::Top].map(|which| {
        let (x, y) = (half(e, rn, which), half(e, operand2, which));
        // Two 16-bit signed values multiply to at most 2^30, which an `i32` holds.
        let product = e.b.ins().imul(x, y);
        e.b.ins().sextend(types::I64, product)
    });
    combine.apply(e, product1, product2)
}

/// `SInt` of the halfword of `value` that `which` names.
fn half(e: &mut Emitter<'_, '_>, value: Value, which: Half) -> Value {
    lane(e, value, u32::from(which == Half::Top), 16, true)
}

/// Lane `index` of `value`, `width` bits wide, sign- or zero-extended to an `i32`.
fn lane(e: &mut Emitter<'_, '_>, value: Value, index: u32, width: u32, signed: bool) -> Value {
    let low = index * width;
    if signed {
        let top = e.b.ins().ishl_imm_u(value, i64::from(32 - width - low));
        e.b.ins().sshr_imm_u(top, i64::from(32 - width))
    } else {
        let shifted = e.b.ins().ushr_imm_u(value, i64::from(low));
        e.b.ins().band_imm_u(shifted, (1_i64 << width) - 1)
    }
}

/// Bits 63 to 32 of an `i64`.
fn high_word(e: &mut Emitter<'_, '_>, value: Value) -> Value {
    let shifted = e.b.ins().ushr_imm_u(value, 32);
    e.b.ins().ireduce(types::I32, shifted)
}

/// Whether an exact `i64` result isn't `SInt(result<31:0>)`, the manual's signed overflow, as 0
/// or 1.
fn overflows(e: &mut Emitter<'_, '_>, value: Value) -> Value {
    let low = e.b.ins().ireduce(types::I32, value);
    let back = e.b.ins().sextend(types::I64, low);
    let differs = e.b.ins().icmp(IntCC::NotEqual, back, value);
    e.b.ins().uextend(types::I32, differs)
}

/// Sets `APSR.Q` when `happened` is 1. Nothing here ever clears it, A2.3.2.
fn set_q_when(e: &mut Emitter<'_, '_>, happened: Value) {
    let q = e.load_field(offset_of!(State, q));
    let q = e.b.ins().bor(q, happened);
    e.store_field(offset_of!(State, q), q);
}

#[cfg(test)]
mod tests {
    use cranelift_codegen::ir;
    use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};

    use super::super::super::family::helpers;
    use super::super::super::registers::mask;
    use super::super::status;
    use super::super::testing::Bench;
    use super::*;

    /// What A5.3's decode answers for a 32-bit instruction, written `hw1` then `hw2` as a
    /// disassembler prints it, translated into a function of its own that never runs.
    fn decoded(instruction: u32) -> Translated {
        let mut function = ir::Function::new();
        function
            .signature
            .params
            .push(ir::AbiParam::new(types::I64));
        let mut context = FunctionBuilderContext::new();
        let mut builder = FunctionBuilder::new(&mut function, &mut context);
        let entry = builder.create_block();
        builder.append_block_params_for_function_params(entry);
        builder.switch_to_block(entry);
        let mut e = Emitter::new(&mut builder, types::I64);
        let (hw1, hw2) = halfwords_of(instruction);
        super::super::decode32(&mut e, hw1, hw2)
    }

    fn halfwords_of(instruction: u32) -> (u16, u16) {
        let [hw1_high, hw1_low, hw2_high, hw2_low] = instruction.to_be_bytes();
        (
            u16::from_be_bytes([hw1_high, hw1_low]),
            u16::from_be_bytes([hw2_high, hw2_low]),
        )
    }

    /// Runs `instruction` with `r` in R0 to R3, Q and GE as given, and N, Z, C, V as 1, 0, 1, 1,
    /// and answers the processor after. None of these instructions touches the flags, so every
    /// run checks they're kept.
    fn run(instruction: u32, r: [u32; 4], q: u32, ge: u32) -> Bench {
        let (hw1, hw2) = halfwords_of(instruction);
        let mut bench = Bench::new(&[hw1, hw2]);
        let state = &mut bench.core.state;
        state.r[..4].copy_from_slice(&r);
        (state.q, state.ge) = (q, ge);
        (state.n, state.z, state.c, state.v) = (1, 0, 1, 1);
        let answer = run_block(&mut bench, &[instruction]);
        let state = &bench.core.state;
        assert_eq!(
            (answer, state.r[15]),
            (1 << 8 | status::NEXT, 4),
            "{instruction:#010x} retired"
        );
        assert_eq!((state.n, state.z, state.c, state.v), (1, 0, 1, 1));
        bench
    }

    /// Lowers `instructions` from address 0 into one block as the translator does, runs it once
    /// on `bench`'s processor, and answers what it answered. The Bench's own run loop would spin
    /// on a fault until EX's exception model takes it, so a faulting block is run here instead.
    fn run_block(bench: &mut Bench, instructions: &[u32]) -> u32 {
        helpers::run_built(&mut bench.core, |builder, core| {
            let pointer = builder.func.dfg.value_type(core);
            let mut e = Emitter::new(builder, pointer);
            for (index, &instruction) in (0..).zip(instructions) {
                e.begin(index, index * 4, 4, 0);
                let (hw1, hw2) = halfwords_of(instruction);
                assert_eq!(
                    super::super::decode32(&mut e, hw1, hw2),
                    Ok(()),
                    "{instruction:#010x} translates"
                );
                if e.ended() {
                    e.finish();
                    return;
                }
            }
            e.end_block();
            e.finish();
        })
    }

    /// R0 to R3 after `instruction` runs from `r`, with Q and GE clear and both checked kept.
    fn registers(instruction: u32, r: [u32; 4]) -> [u32; 4] {
        let bench = run(instruction, r, 0, 0);
        assert_eq!(
            (bench.core.state.q, bench.core.state.ge),
            (0, 0),
            "{instruction:#010x} keeps Q and GE"
        );
        bench.core.state.r[..4].try_into().unwrap()
    }

    /// R0 and Q after `instruction` runs from `r` with Q clear.
    fn saturated(instruction: u32, r: [u32; 4]) -> (u32, u32) {
        let bench = run(instruction, r, 0, 0);
        (bench.core.state.r[0], bench.core.state.q)
    }

    /// MUL T2, MLA and MLS, A7.7.83, A7.7.73 and A7.7.74: the low 32 bits, signed or not.
    #[test]
    fn mul_mla_and_mls_keep_the_low_word() {
        const MUL: u32 = 0xFB01_F002; // MUL r0, r1, r2
        const MLA: u32 = 0xFB01_3002; // MLA r0, r1, r2, r3
        const MLS: u32 = 0xFB01_3012; // MLS r0, r1, r2, r3
        assert_eq!(registers(MUL, [9, 7, 6, 0])[0], 42);
        assert_eq!(registers(MUL, [9, 0xFFFF_FFFF, 5, 0])[0], 0xFFFF_FFFB);
        assert_eq!(registers(MUL, [9, 0x1_0000, 0x1_0000, 0])[0], 0);
        assert_eq!(registers(MLA, [9, 3, 4, 5]), [17, 3, 4, 5]);
        assert_eq!(registers(MLA, [9, 0xFFFF_FFFF, 2, 1])[0], 0xFFFF_FFFF);
        assert_eq!(registers(MLS, [9, 3, 4, 100])[0], 88);
        assert_eq!(registers(MLS, [9, 1, 1, 0])[0], 0xFFFF_FFFF);
    }

    /// SMULBB to SMULTT, A7.7.146: `N` and `M` pick the halfwords, each signed.
    #[test]
    fn smul_by_halfwords_multiplies_the_halves_it_names() {
        let r = [9, 0x0003_FFFE, 0xFFFF_0005, 0];
        assert_eq!(registers(0xFB11_F002, r)[0], 0xFFFF_FFF6, "SMULBB -2 * 5");
        assert_eq!(registers(0xFB11_F012, r)[0], 2, "SMULBT -2 * -1");
        assert_eq!(registers(0xFB11_F022, r)[0], 15, "SMULTB 3 * 5");
        assert_eq!(registers(0xFB11_F032, r)[0], 0xFFFF_FFFD, "SMULTT 3 * -1");
    }

    /// SMLABB to SMLATT, A7.7.134: Q is set when adding the accumulator overflows, and kept
    /// when it doesn't.
    #[test]
    fn smla_by_halfwords_sets_q_on_an_overflowing_accumulate() {
        const SMLABB: u32 = 0xFB11_3002; // SMLABB r0, r1, r2, r3
        const SMLATT: u32 = 0xFB11_3032;
        assert_eq!(registers(SMLABB, [9, 0x0003_FFFE, 0xFFFF_0005, 100])[0], 90);
        assert_eq!(
            registers(SMLATT, [9, 0x0003_FFFE, 0xFFFF_0005, 0])[0],
            0xFFFF_FFFD
        );
        assert_eq!(
            saturated(SMLABB, [9, 0x8000, 0x8000, 0x4000_0000]),
            (0x8000_0000, 1)
        );
        let bench = run(SMLABB, [9, 2, 3, 4], 1, 0);
        assert_eq!((bench.core.state.r[0], bench.core.state.q), (10, 1));
    }

    /// SMUAD, SMUADX, SMLAD and SMLADX, A7.7.145 and A7.7.135: `X` exchanges Rm's halfwords, and
    /// Q is set when the sum overflows.
    #[test]
    fn smuad_and_smlad_add_both_products() {
        let r = [9, 0x0002_0003, 0x0004_0005, 100];
        assert_eq!(registers(0xFB21_F002, r)[0], 23, "SMUAD 3*5 + 2*4");
        assert_eq!(registers(0xFB21_F012, r)[0], 22, "SMUADX 3*4 + 2*5");
        assert_eq!(registers(0xFB21_3002, r)[0], 123, "SMLAD");
        assert_eq!(registers(0xFB21_3012, r)[0], 122, "SMLADX");
        assert_eq!(
            saturated(0xFB21_F002, [9, 0x8000_8000, 0x8000_8000, 0]),
            (0x8000_0000, 1),
            "SMUAD of four -32768s is 2^31"
        );
        assert_eq!(
            saturated(0xFB21_3002, [9, 0x0001_0001, 0x0001_0001, 0x7FFF_FFFF]),
            (0x8000_0001, 1)
        );
    }

    /// SMUSD, SMUSDX, SMLSD and SMLSDX, A7.7.149 and A7.7.140: the bottom product less the top,
    /// and only the accumulating form can overflow.
    #[test]
    fn smusd_and_smlsd_subtract_the_top_product() {
        let r = [9, 0x0002_0003, 0x0004_0005, 100];
        assert_eq!(registers(0xFB41_F002, r)[0], 7, "SMUSD 3*5 - 2*4");
        assert_eq!(registers(0xFB41_F012, r)[0], 2, "SMUSDX 3*4 - 2*5");
        assert_eq!(registers(0xFB41_3002, r)[0], 107, "SMLSD");
        assert_eq!(registers(0xFB41_3012, r)[0], 102, "SMLSDX");
        assert_eq!(
            registers(0xFB41_F002, [9, 0x0003_0002, 0x0005_0004, 0])[0],
            0xFFFF_FFF9
        );
        assert_eq!(
            saturated(0xFB41_3002, [9, 0x8000_0000, 0x7FFF_0000, 0x7FFF_FFFF]),
            (0xBFFF_7FFF, 1)
        );
    }

    /// SMULWB, SMULWT, SMLAWB and SMLAWT, A7.7.148 and A7.7.139: bits 47 to 16 of the product,
    /// rounded toward minus infinity as an arithmetic shift is.
    #[test]
    fn smulw_and_smlaw_keep_the_top_of_a_48_bit_product() {
        const SMULWB: u32 = 0xFB31_F002;
        const SMULWT: u32 = 0xFB31_F012;
        const SMLAWB: u32 = 0xFB31_3002;
        const SMLAWT: u32 = 0xFB31_3012;
        assert_eq!(registers(SMULWB, [9, 0x1_0000, 3, 0])[0], 3);
        assert_eq!(registers(SMULWT, [9, 0x1_0000, 0x0002_0000, 0])[0], 2);
        assert_eq!(registers(SMULWB, [9, 0xFFFF_FFFF, 1, 0])[0], 0xFFFF_FFFF);
        assert_eq!(
            registers(SMULWB, [9, 0x7FFF_FFFF, 0x8000, 0])[0],
            0xC000_0000
        );
        assert_eq!(registers(SMLAWB, [9, 0x1_0000, 3, 10])[0], 13);
        assert_eq!(registers(SMLAWT, [9, 0x1_0000, 0x0002_0000, 10])[0], 12);
        assert_eq!(
            saturated(SMLAWB, [9, 0x1_0000, 1, 0x7FFF_FFFF]),
            (0x8000_0000, 1)
        );
    }

    /// SMMUL, SMMLA and SMMLS and their `R` forms, A7.7.144, A7.7.142 and A7.7.143: the top word,
    /// with 0x80000000 added first when rounding.
    #[test]
    fn smmul_smmla_and_smmls_keep_the_top_word() {
        assert_eq!(registers(0xFB51_F002, [9, 0x4000_0000, 4, 0])[0], 1);
        assert_eq!(
            registers(0xFB51_F002, [9, 0x8000_0000, 1, 0])[0],
            0xFFFF_FFFF
        );
        assert_eq!(
            registers(0xFB51_F012, [9, 0x8000_0000, 1, 0])[0],
            0,
            "SMMULR"
        );
        assert_eq!(registers(0xFB51_3002, [9, 0x4000_0000, 4, 5])[0], 6);
        assert_eq!(registers(0xFB51_3002, [9, 0x8000_0000, 1, 5])[0], 4);
        assert_eq!(
            registers(0xFB51_3012, [9, 0x8000_0000, 1, 5])[0],
            5,
            "SMMLAR"
        );
        assert_eq!(registers(0xFB61_3002, [9, 0x4000_0000, 4, 5])[0], 4);
        assert_eq!(registers(0xFB61_3002, [9, 1, 1, 0])[0], 0xFFFF_FFFF);
        assert_eq!(registers(0xFB61_3012, [9, 1, 1, 0])[0], 0, "SMMLSR");
    }

    /// USAD8 and USADA8, A7.7.208 and A7.7.209: the bytes' absolute differences, summed.
    #[test]
    fn usad8_and_usada8_sum_absolute_differences() {
        let r = [9, 0x01FF_1005, 0x0300_2001, 1000];
        assert_eq!(registers(0xFB71_F002, r)[0], 2 + 255 + 16 + 4);
        assert_eq!(registers(0xFB71_3002, r)[0], 1000 + 277);
    }

    /// SMULL and UMULL, A7.7.147 and A7.7.201, with R0 as `RdLo` and R1 as `RdHi`.
    #[test]
    fn smull_and_umull_write_both_halves() {
        const SMULL: u32 = 0xFB82_0103; // SMULL r0, r1, r2, r3
        const UMULL: u32 = 0xFBA2_0103;
        assert_eq!(
            registers(SMULL, [9, 9, 0xFFFF_FFFE, 3]),
            [0xFFFF_FFFA, 0xFFFF_FFFF, 0xFFFF_FFFE, 3]
        );
        assert_eq!(
            registers(SMULL, [9, 9, 0x8000_0000, 0x8000_0000])[..2],
            [0, 0x4000_0000]
        );
        assert_eq!(
            registers(UMULL, [9, 9, 0xFFFF_FFFF, 0xFFFF_FFFF])[..2],
            [1, 0xFFFF_FFFE]
        );
    }

    /// SMLAL, UMLAL and UMAAL, A7.7.136, A7.7.200 and A7.7.199, accumulating into R1:R0.
    #[test]
    fn smlal_umlal_and_umaal_accumulate_64_bits() {
        const SMLAL: u32 = 0xFBC2_0103;
        const UMLAL: u32 = 0xFBE2_0103;
        const UMAAL: u32 = 0xFBE2_0163;
        assert_eq!(
            registers(SMLAL, [0, 1, 0xFFFF_FFFF, 1])[..2],
            [0xFFFF_FFFF, 0]
        );
        assert_eq!(
            registers(SMLAL, [0, 0, 0xFFFF_FFFE, 3])[..2],
            [0xFFFF_FFFA, 0xFFFF_FFFF]
        );
        assert_eq!(registers(UMLAL, [0xFFFF_FFFF, 0, 1, 1])[..2], [0, 1]);
        assert_eq!(
            registers(UMLAL, [0, 0, 0xFFFF_FFFF, 0xFFFF_FFFF])[..2],
            [1, 0xFFFF_FFFE]
        );
        assert_eq!(
            registers(UMAAL, [u32::MAX; 4])[..2],
            [0xFFFF_FFFF, 0xFFFF_FFFF],
            "the largest sum UMAAL can make just fits"
        );
        assert_eq!(registers(UMAAL, [5, 7, 2, 3])[..2], [18, 0]);
    }

    /// SMLALBB to SMLALTT, A7.7.137: a halfword product sign-extended into a 64-bit sum that
    /// wraps unseen.
    #[test]
    fn smlal_by_halfwords_accumulates_a_sign_extended_product() {
        let (n, m) = (0x0003_FFFE, 0xFFFF_0005);
        assert_eq!(
            registers(0xFBC2_0183, [0, 0, n, m])[..2],
            [0xFFFF_FFF6, 0xFFFF_FFFF],
            "SMLALBB"
        );
        assert_eq!(registers(0xFBC2_0193, [0, 0, n, m])[..2], [2, 0], "SMLALBT");
        assert_eq!(
            registers(0xFBC2_01A3, [0, 0, n, m])[..2],
            [15, 0],
            "SMLALTB"
        );
        assert_eq!(
            registers(0xFBC2_01B3, [0, 0, n, m])[..2],
            [0xFFFF_FFFD, 0xFFFF_FFFF],
            "SMLALTT"
        );
        assert_eq!(
            registers(0xFBC2_01A3, [u32::MAX, u32::MAX, n, m])[..2],
            [14, 0]
        );
    }

    /// SMLALD, SMLALDX, SMLSLD and SMLSLDX, A7.7.138 and A7.7.141.
    #[test]
    fn smlald_and_smlsld_accumulate_both_products() {
        let r = [0, 1, 0x0002_0003, 0x0004_0005];
        assert_eq!(registers(0xFBC2_01C3, r)[..2], [23, 1], "SMLALD");
        assert_eq!(registers(0xFBC2_01D3, r)[..2], [22, 1], "SMLALDX");
        assert_eq!(registers(0xFBD2_01C3, r)[..2], [7, 1], "SMLSLD");
        assert_eq!(registers(0xFBD2_01D3, r)[..2], [2, 1], "SMLSLDX");
        assert_eq!(
            registers(0xFBD2_01C3, [0, 1, 0x0003_0002, 0x0005_0004])[..2],
            [0xFFFF_FFF9, 0],
            "a negative difference borrows from RdHi"
        );
    }

    /// SDIV and UDIV, A7.7.125 and A7.7.192: rounded toward zero, 0x80000000 by -1 wraps, and a
    /// zero divisor gives 0 while `CCR.DIV_0_TRP` is clear.
    #[test]
    fn sdiv_and_udiv_round_toward_zero_and_give_zero_for_zero() {
        const SDIV: u32 = 0xFB91_F0F2; // SDIV r0, r1, r2
        const UDIV: u32 = 0xFBB1_F0F2;
        assert_eq!(registers(SDIV, [9, 7, 0xFFFF_FFFE, 0])[0], 0xFFFF_FFFD);
        assert_eq!(registers(SDIV, [9, 0xFFFF_FFF9, 2, 0])[0], 0xFFFF_FFFD);
        assert_eq!(
            registers(SDIV, [9, 0x8000_0000, 0xFFFF_FFFF, 0])[0],
            0x8000_0000
        );
        assert_eq!(registers(SDIV, [9, 5, 0, 0])[0], 0);
        assert_eq!(registers(UDIV, [9, 0xFFFF_FFFF, 2, 0])[0], 0x7FFF_FFFF);
        assert_eq!(registers(UDIV, [9, 7, 0xFFFF_FFFF, 0])[0], 0);
        assert_eq!(registers(UDIV, [9, 7, 0, 0])[0], 0);
    }

    /// SDIV and UDIV by zero with `CCR.DIV_0_TRP` set, `GenerateIntegerZeroDivide()`, D6.7.20:
    /// the block leaves through its fault exit at the divide, the MUL before it retired and the
    /// divide wrote nothing. With the trap clear the same block retires both and divides to 0,
    /// and a divisor that isn't zero divides as ever with the trap set.
    #[test]
    fn a_zero_divisor_faults_only_while_ccr_traps_it() {
        const MUL: u32 = 0xFB01_F302; // MUL r3, r1, r2
        for divide in [0xFB91_F0F4, 0xFBB1_F0F4] {
            let mut bench = Bench::new(&[]);
            bench.core.system.ccr |= mask(ccr::DIV_0_TRP);
            bench.core.state.r[..5].copy_from_slice(&[9, 42, 6, 8, 0]);
            let answer = run_block(&mut bench, &[MUL, divide]);
            let state = &bench.core.state;
            assert_eq!(answer, 1 << 8 | status::EXCEPTION, "{divide:#010x}");
            assert_eq!((&state.r[..5], state.r[15]), (&[9, 42, 6, 252, 0][..], 4));
            bench.core.state.r[4] = 5;
            assert_eq!(run_block(&mut bench, &[divide]), 1 << 8 | status::NEXT);
            assert_eq!(bench.core.state.r[0], 8, "{divide:#010x} 42 by 5");
            bench.core.system.ccr &= !mask(ccr::DIV_0_TRP);
            bench.core.state.r[4] = 0;
            let answer = run_block(&mut bench, &[MUL, divide]);
            let state = &bench.core.state;
            assert_eq!(answer, 2 << 8 | status::NEXT, "{divide:#010x}");
            assert_eq!((state.r[0], state.r[15]), (0, 8));
        }
    }

    /// A5.3.16's UNDEFINED encodings, and its UNPREDICTABLE ones, which are UNDEFINED here.
    #[test]
    fn multiply_accumulate_refuses_what_table_a5_28_leaves_out() {
        for instruction in [
            0xFB01_F042, // bits 7 and 6 aren't zero
            0xFB01_F082,
            0xFB01_F022, // op1 000, op2 1x
            0xFB01_F012, // MLS with Ra 15
            0xFB21_F022, // op1 010 to 110, op2 1x
            0xFB31_3022,
            0xFB41_F032,
            0xFB51_3022,
            0xFB61_3022,
            0xFB61_F002, // SMMLS with Ra 15
            0xFB71_F012, // op1 111, op2 not 00
            0xFB71_3022,
            0xFB01_FD02, // Rd 13
            0xFB01_FF02, // Rd 15
            0xFB0D_F002, // Rn 13
            0xFB0F_3002, // Rn 15
            0xFB01_F00D, // Rm 13
            0xFB01_F00F, // Rm 15
            0xFB01_D002, // Ra 13
            0xFB11_D002,
        ] {
            assert_eq!(decoded(instruction), Err(Undefined), "{instruction:#010x}");
        }
    }

    /// A5.3.17's UNDEFINED encodings, and its UNPREDICTABLE ones, which are UNDEFINED here.
    #[test]
    fn long_multiply_divide_refuses_what_table_a5_29_leaves_out() {
        for instruction in [
            0xFB82_0113, // op1 000, op2 0001
            0xFB91_F002, // op1 001, op2 0000
            0xFBA1_F0F2, // op1 010, op2 1111
            0xFBB1_F002, // op1 011, op2 0000
            0xFBC2_0113, // op1 100, op2 0001
            0xFBC2_01E3, // op1 100, op2 1110
            0xFBD2_0103, // op1 101, op2 0000
            0xFBD2_0183, // op1 101, op2 1000
            0xFBE2_0113, // op1 110, op2 0001
            0xFBE2_0173, // op1 110, op2 0111
            0xFBF2_0103, // op1 111
            0xFB82_0003, // RdHi is RdLo
            0xFBE2_1163,
            0xFB82_D103, // RdLo 13
            0xFB82_0F03, // RdHi 15
            0xFB8D_0103, // Rn 13
            0xFB82_010F, // Rm 15
            0xFB91_70F2, // SDIV's should-be-ones aren't
            0xFBB1_E0F2, // UDIV's
            0xFB91_FDF2, // SDIV Rd 13
            0xFB9F_F0F2, // SDIV Rn 15
            0xFBB1_F0FD, // UDIV Rm 13
        ] {
            assert_eq!(decoded(instruction), Err(Undefined), "{instruction:#010x}");
        }
    }

    /// SADD16, SASX, SSAX, SSUB16, SADD8 and SSUB8, A7.7.119 to A7.7.121 and A7.7.152 to
    /// A7.7.154: GE is set for each lane that isn't negative.
    #[test]
    fn signed_parallel_arithmetic_sets_ge_per_lane() {
        for (instruction, r, result, ge) in [
            (0xFA91_F002, [0x7FFF_0001, 0x0001_FFFE], 0x8000_FFFF, 0b1100), // SADD16
            (0xFAA1_F002, [0x0005_0003, 0x0004_0002], 0x0007_FFFF, 0b1100), // SASX
            (0xFAE1_F002, [0x0005_0003, 0x0004_0006], 0xFFFF_0007, 0b0011), // SSAX
            (0xFAD1_F002, [0x0005_0003, 0x0004_0006], 0x0001_FFFD, 0b1100), // SSUB16
            (0xFA81_F002, [0x7F80_01FF, 0x01FF_0101], 0x807F_0200, 0b1011), // SADD8
            (0xFAC1_F002, [0x0580_0010, 0x0601_0101], 0xFF7F_FF0F, 0b0001), // SSUB8
        ] {
            check_parallel(instruction, r, result, Some(ge));
        }
    }

    /// QADD16 to QSUB8, A7.7.101 to A7.7.109: each lane saturates, and neither Q nor GE moves.
    #[test]
    fn signed_saturating_parallel_arithmetic_saturates_each_lane() {
        for (instruction, r, result) in [
            (0xFA91_F012, [0x7FFF_8000, 0x0001_FFFF], 0x7FFF_8000), // QADD16
            (0xFAA1_F012, [0x0010_0020, 0x0005_0003], 0x0013_001B), // QASX
            (0xFAA1_F012, [0x7FFF_8000, 0x0001_0001], 0x7FFF_8000),
            (0xFAE1_F012, [0x0010_0020, 0x0005_0003], 0x000D_0025), // QSAX
            (0xFAD1_F012, [0x8000_0005, 0x0001_0007], 0x8000_FFFE), // QSUB16
            (0xFA81_F012, [0x7F80_1001, 0x01FF_1002], 0x7F80_2003), // QADD8
            (0xFAC1_F012, [0x807F_0500, 0x01FF_0700], 0x807F_FE00), // QSUB8
        ] {
            check_parallel(instruction, r, result, None);
        }
    }

    /// SHADD16 to SHSUB8, A7.7.128 to A7.7.133: each lane's exact result, halved toward minus
    /// infinity.
    #[test]
    fn signed_halving_parallel_arithmetic_halves_each_lane() {
        for (instruction, r, result) in [
            (0xFA91_F022, [0x7FFF_FFFF, 0x7FFF_FFFE], 0x7FFF_FFFE), // SHADD16
            (0xFAA1_F022, [0x0010_0000, 0x0005_0003], 0x0009_FFFD), // SHASX
            (0xFAE1_F022, [0x0010_0000, 0x0005_0003], 0x0006_0002), // SHSAX
            (0xFAD1_F022, [0x8000_0000, 0x7FFF_0001], 0x8000_FFFF), // SHSUB16
            (0xFA81_F022, [0x7F80_FF03, 0x7F80_FE04], 0x7F80_FE03), // SHADD8
            (0xFAC1_F022, [0x807F_0005, 0x7F80_0102], 0x807F_FF01), // SHSUB8
        ] {
            check_parallel(instruction, r, result, None);
        }
    }

    /// UADD16, UASX, USAX, USUB16, UADD8 and USUB8, A7.7.187 to A7.7.189 and A7.7.212 to
    /// A7.7.214: GE is set for a sum that carries out of its lane and a difference that doesn't
    /// borrow.
    #[test]
    fn unsigned_parallel_arithmetic_sets_ge_per_lane() {
        for (instruction, r, result, ge) in [
            (0xFA91_F042, [0xFFFF_0001, 0x0001_0002], 0x0000_0003, 0b1100), // UADD16
            (0xFAA1_F042, [0xFFFF_0003, 0x0004_0001], 0x0000_FFFF, 0b1100), // UASX
            (0xFAE1_F042, [0x0003_FFFF, 0x0001_0004], 0xFFFF_0000, 0b0011), // USAX
            (0xFAD1_F042, [0x0005_0003, 0x0004_0006], 0x0001_FFFD, 0b1100), // USUB16
            (0xFA81_F042, [0xFF80_0110, 0x0180_0110], 0x0000_0220, 0b1100), // UADD8
            (0xFAC1_F042, [0x0005_8010, 0x0105_7F20], 0xFF00_01F0, 0b0110), // USUB8
        ] {
            check_parallel(instruction, r, result, Some(ge));
        }
    }

    /// UQADD16 to UQSUB8, A7.7.202 to A7.7.207: each lane held to 0 to its largest value.
    #[test]
    fn unsigned_saturating_parallel_arithmetic_saturates_each_lane() {
        for (instruction, r, result) in [
            (0xFA91_F052, [0xFFFF_0001, 0x0001_0002], 0xFFFF_0003), // UQADD16
            (0xFAA1_F052, [0xFFFF_0003, 0x0004_0001], 0xFFFF_0000), // UQASX
            (0xFAE1_F052, [0x0003_FFFF, 0x0001_0004], 0x0000_FFFF), // UQSAX
            (0xFAD1_F052, [0x0005_0003, 0x0004_0006], 0x0001_0000), // UQSUB16
            (0xFA81_F052, [0xFF80_0110, 0x0180_0110], 0xFFFF_0220), // UQADD8
            (0xFAC1_F052, [0x0005_8010, 0x0105_7F20], 0x0000_0100), // UQSUB8
        ] {
            check_parallel(instruction, r, result, None);
        }
    }

    /// UHADD16 to UHSUB8, A7.7.193 to A7.7.198: a difference that borrows halves to a negative
    /// lane.
    #[test]
    fn unsigned_halving_parallel_arithmetic_halves_each_lane() {
        for (instruction, r, result) in [
            (0xFA91_F062, [0xFFFF_0001, 0xFFFF_0002], 0xFFFF_0001), // UHADD16
            (0xFAA1_F062, [0xFFFF_0003, 0x0004_0001], 0x8000_FFFF), // UHASX
            (0xFAE1_F062, [0x0003_FFFF, 0x0001_0004], 0xFFFF_8000), // UHSAX
            (0xFAD1_F062, [0x0000_FFFF, 0xFFFF_0001], 0x8000_7FFF), // UHSUB16
            (0xFA81_F062, [0xFF80_0110, 0xFF80_0230], 0xFF80_0120), // UHADD8
            (0xFAC1_F062, [0x0005_8010, 0xFF05_7F20], 0x8000_00F8), // UHSUB8
        ] {
            check_parallel(instruction, r, result, None);
        }
    }

    /// Runs a parallel instruction from Rn and Rm into R0, with GE 0b1010 and Q clear and then
    /// set, and checks the result and GE, which only `ge` changes. Q never moves, since A7.7.101
    /// and its kind write only `R[d]`.
    fn check_parallel(instruction: u32, [rn, rm]: [u32; 2], result: u32, ge: Option<u32>) {
        for q in [0, 1] {
            let bench = run(instruction, [9, rn, rm, 0], q, 0b1010);
            let state = &bench.core.state;
            assert_eq!(
                (state.r[0], state.ge, state.q),
                (result, ge.unwrap_or(0b1010), q),
                "{instruction:#010x} from Q {q}"
            );
        }
    }

    /// A5.3.13's and A5.3.14's UNDEFINED encodings, and their UNPREDICTABLE ones.
    #[test]
    fn parallel_arithmetic_refuses_what_tables_a5_25_and_a5_26_leave_out() {
        for instruction in [
            0xFAB1_F002, // op1 011
            0xFAF1_F002, // op1 111
            0xFA91_F032, // op2 11
            0xFA91_E002, // bits 15 to 12 of hw2 aren't ones
            0xFA91_FF02, // Rd 15
            0xFA9D_F002, // Rn 13
            0xFA91_F00F, // Rm 15
        ] {
            assert_eq!(decoded(instruction), Err(Undefined), "{instruction:#010x}");
            assert_eq!(
                decoded(instruction | 0x40),
                Err(Undefined),
                "{instruction:#010x} unsigned"
            );
        }
    }

    /// QADD, QSUB, QDADD and QDSUB, A7.7.100, A7.7.107, A7.7.104 and A7.7.105: `Rm` first, and
    /// a saturation in the doubling or the sum sets Q.
    #[test]
    fn saturating_add_and_subtract_set_q_when_they_saturate() {
        const QADD: u32 = 0xFA81_F082; // QADD r0, r2, r1
        const QDADD: u32 = 0xFA81_F092;
        const QSUB: u32 = 0xFA81_F0A2;
        const QDSUB: u32 = 0xFA81_F0B2;
        assert_eq!(saturated(QADD, [9, 6, 5, 0]), (11, 0));
        assert_eq!(saturated(QADD, [9, 0x20, 0x7FFF_FFF0, 0]), (0x7FFF_FFFF, 1));
        assert_eq!(saturated(QSUB, [9, 3, 10, 0]), (7, 0));
        assert_eq!(saturated(QSUB, [9, 1, 0x8000_0000, 0]), (0x8000_0000, 1));
        assert_eq!(saturated(QDADD, [9, 3, 10, 0]), (16, 0));
        assert_eq!(
            saturated(QDADD, [9, 0x4000_0000, 0xFFFF_FFFF, 0]),
            (0x7FFF_FFFE, 1),
            "the doubling saturates though the sum doesn't"
        );
        assert_eq!(saturated(QDSUB, [9, 3, 10, 0]), (4, 0));
        assert_eq!(
            saturated(QDSUB, [9, 0xC000_0000, 0, 0]),
            (0x7FFF_FFFF, 1),
            "the doubling is exact and the difference saturates"
        );
        let bench = run(QADD, [9, 1, 2, 0], 1, 0);
        assert_eq!(bench.core.state.q, 1, "nothing clears Q");
    }

    /// REV, REV16, RBIT and REVSH T2, A7.7.111, A7.7.112, A7.7.110 and A7.7.113, and CLZ,
    /// A7.7.24, each with Rm in both its fields.
    #[test]
    fn reverses_and_clz_rearrange_one_register() {
        const REV: u32 = 0xFA92_F082; // REV r0, r2
        const REV16: u32 = 0xFA92_F092;
        const RBIT: u32 = 0xFA92_F0A2;
        const REVSH: u32 = 0xFA92_F0B2;
        const CLZ: u32 = 0xFAB2_F082;
        let r = [9, 0, 0x1234_5678, 0];
        assert_eq!(registers(REV, r)[0], 0x7856_3412);
        assert_eq!(registers(REV16, r)[0], 0x3412_7856);
        assert_eq!(registers(RBIT, r)[0], 0x1E6A_2C48);
        assert_eq!(registers(RBIT, [9, 0, 1, 0])[0], 0x8000_0000);
        assert_eq!(registers(REVSH, [9, 0, 0x1234_5680, 0])[0], 0xFFFF_8056);
        assert_eq!(registers(REVSH, [9, 0, 0x0000_1234, 0])[0], 0x0000_3412);
        for (value, zeros) in [(0, 32), (1, 31), (0x8000_0000, 0), (0x1_0000, 15)] {
            assert_eq!(registers(CLZ, [9, 0, value, 0])[0], zeros, "CLZ {value:#x}");
        }
    }

    /// SEL, A7.7.126: GE picks each byte, from Rn where it's set.
    #[test]
    fn sel_picks_each_byte_by_its_ge_bit() {
        const SEL: u32 = 0xFAA1_F082; // SEL r0, r1, r2
        let bench = run(SEL, [9, 0x1122_3344, 0xAABB_CCDD, 0], 0, 0b0101);
        assert_eq!(bench.core.state.r[0], 0xAA22_CC44);
        assert_eq!(bench.core.state.ge, 0b0101);
        let bench = run(SEL, [9, 0x1122_3344, 0xAABB_CCDD, 0], 0, 0b1111);
        assert_eq!(bench.core.state.r[0], 0x1122_3344);
    }

    /// A5.3.15's UNDEFINED encodings, and its UNPREDICTABLE ones.
    #[test]
    fn miscellaneous_refuses_what_table_a5_27_leaves_out() {
        for instruction in [
            0xFAA1_F092, // op1 10, op2 not 00
            0xFAA1_F0B2,
            0xFAB2_F092, // op1 11, op2 not 00
            0xFAB2_F0A2,
            0xFA93_F082, // REV with Rm fields that differ
            0xFAB1_F082, // CLZ with Rm fields that differ
            0xFA81_E082, // bits 15 to 12 of hw2 aren't ones
            0xFA81_FD82, // Rd 13
            0xFA8F_F082, // Rn 15
            0xFA81_F08F, // Rm 15
            0xFA9D_F08D, // REV of SP
        ] {
            assert_eq!(decoded(instruction), Err(Undefined), "{instruction:#010x}");
        }
    }
}