rigless 0.2.1

// Copyright (c) 2026 Rigless Labs.
// SPDX-License-Identifier: MIT OR Apache-2.0
//! Translates a block of Thumb instructions to one host function, DDI 0403E.b chapters A5 to A7.
//!
//! [`translate`] decodes from an address until an instruction ends the block, the next one has a
//! breakpoint, can't be fetched, or the block is as long as it may be, and lowers each instruction
//! through an [`Emitter`]. The encoding groups are split by DDI 0403E.b's own tables, A5.2 for
//! 16-bit instructions and A5.3 for 32-bit ones, and each group lives in a module of its own.
//!
//! The block's function takes the core, `&mut Family`, and answers why it ended in its low eight
//! bits, one of [`status`], with the instructions it retired above them.

mod alu;
mod control;
mod data16;
mod data32;
mod float;
mod memory;
mod multiply;
#[cfg(any(test, fuzzing))]
pub(crate) mod testing;

use cranelift_codegen::ir::{self, InstBuilder, MemFlagsData, Value, condcodes::IntCC, types};
use cranelift_frontend::{FunctionBuilder, Variable};

use super::family::{Features, offsets};
use super::registers::{exc_return, mask};
use crate::emulator::Error;
use crate::emulator::core::{
    Block, Fma, Jit, Lookup, Tier, emit_lookup, host_convention, load_guest, store_guest,
};

/// Why a block ended, in the low eight bits of what it answers.
pub(crate) mod status {
    /// It ran to its end, or took a branch, and the next instruction is at `r[15]`.
    pub(crate) const NEXT: u32 = 0;
    /// The budget was spent before the block, so it ran nothing.
    pub(crate) const SHORT: u32 = 1;
    /// An instruction raised an exception, which the core takes before the next one.
    pub(crate) const EXCEPTION: u32 = 2;
    /// `WFI` retired, and the core waits for an interrupt.
    pub(crate) const WAIT_FOR_INTERRUPT: u32 = 3;
    /// `WFE` retired, and the core waits for an event.
    pub(crate) const WAIT_FOR_EVENT: u32 = 4;
    /// A branch wrote an `EXC_RETURN` value to the PC in Handler mode, B1.5.8, which is in `r[15]`.
    pub(crate) const EXCEPTION_RETURN: u32 = 5;
    /// A branch to itself retired, which runs nothing but itself until an interrupt.
    pub(crate) const SPIN: u32 = 6;
    /// A quick translation has run often enough to translate again with better code, and ran
    /// nothing this time.
    pub(crate) const HOT: u32 = 7;
}

/// A helper's answer with this set raised a fault, and its instruction leaves as if it hadn't run.
pub(crate) const FAULTED: u64 = 1 << 32;
/// With [`FAULTED`]: the helper locked the processor up and set `r[15]` to the lockup address,
/// which the fault exit keeps, B1.5.15.
pub(crate) const KEEP_PC: u64 = 1 << 33;

/// How many runs make a quickly translated block worth translating again.
pub(crate) const HOT_RUNS: i64 = 256;

/// The most instructions a block holds.
pub(crate) const LONGEST_BLOCK: u32 = 64;

/// What an instruction's encoding doesn't define, A5.1: the core raises a UsageFault with
/// `UFSR.UNDEFINSTR` for it.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct Undefined;

/// What a group's function answers for one encoding.
pub(crate) type Translated = Result<(), Undefined>;

/// A fault an instruction raises, which the exception model turns into its exception and its
/// status bits, B1.5.14.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(u32)]
pub(crate) enum Fault {
    /// `UFSR.UNDEFINSTR`.
    Undefined = 1,
    /// `UFSR.INVSTATE`: an instruction with `EPSR.T` 0 or a bad `ITSTATE`.
    InvalidState = 2,
    /// `UFSR.NOCP`: a coprocessor instruction with the coprocessor off or absent.
    NoCoprocessor = 3,
    /// `UFSR.UNALIGNED`.
    Unaligned = 4,
    /// `UFSR.DIVBYZERO`.
    DivideByZero = 5,
    /// `BFSR.IBUSERR`, or `MMFSR.IACCVIOL` when the MPU refuses the fetch.
    InstructionFetch = 6,
    /// `SVC`, which is SVCall rather than a fault.
    SupervisorCall = 8,
    /// `BKPT`, DebugMonitor or HardFault, C1.7.
    Breakpoint = 9,
    /// `BKPT #0xAB`, the M profile's trap for a semihosting call, which the run may serve before
    /// it's the breakpoint it otherwise is.
    HostCall = 10,
}

impl Fault {
    pub(crate) fn from_code(code: u32) -> Option<Self> {
        Some(match code {
            1 => Self::Undefined,
            2 => Self::InvalidState,
            3 => Self::NoCoprocessor,
            4 => Self::Unaligned,
            5 => Self::DivideByZero,
            6 => Self::InstructionFetch,
            8 => Self::SupervisorCall,
            9 => Self::Breakpoint,
            10 => Self::HostCall,
            _ => return None,
        })
    }
}

/// How wide a load or store is.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Size {
    Byte = 1,
    Halfword = 2,
    Word = 4,
}

impl Size {
    pub(crate) fn bytes(self) -> u32 {
        self as u32
    }

    fn host(self) -> ir::Type {
        match self {
            Self::Byte => types::I8,
            Self::Halfword => types::I16,
            Self::Word => types::I32,
        }
    }
}

/// What a load or store asks of the memory system beyond its address and size.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) struct Attributes {
    /// The address must be aligned to the size, whatever `CCR.UNALIGN_TRP` says, A3.2.1: `LDM`,
    /// `STM`, `LDRD`, `STRD`, the exclusives and the floating-point loads and stores.
    pub(crate) aligned: bool,
    /// The access is checked as an unprivileged one, `LDRT` and its kind, A5.3.
    pub(crate) unprivileged: bool,
}

impl Attributes {
    pub(crate) const ALIGNED: Self = Self {
        aligned: true,
        unprivileged: false,
    };

    pub(crate) fn bits(self) -> u32 {
        u32::from(self.aligned) | u32::from(self.unprivileged) << 1
    }

    pub(crate) fn from_bits(bits: u32) -> Self {
        Self {
            aligned: bits & 1 != 0,
            unprivileged: bits & 2 != 0,
        }
    }
}

/// One of the condition flags, A2.3.2.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Flag {
    N = 0,
    Z = 1,
    C = 2,
    V = 3,
}

/// Bits `high` down to `low` of `value`, as the manual writes `value<high:low>`.
pub(crate) fn bits(value: u32, high: u32, low: u32) -> u32 {
    (value >> low) & (u32::MAX >> (31 - (high - low)))
}

/// The offset from the core of the state field at `offset` into [`super::state::State`].
fn field_offset(offset: usize) -> i32 {
    i32::try_from(offsets::STATE + offset).expect("inside the core")
}

/// `ITSTATE` after one more instruction of its IT block, A7.3.3's `ITAdvance()`.
pub(crate) fn it_advance(it: u32) -> u32 {
    if it.trailing_zeros() >= 3 {
        0
    } else {
        (it & 0xE0) | ((it << 1) & 0x1F)
    }
}

/// The written registers and flags, a bit each: `R0` to `R14`, then N, Z, C, V from bit 16.
type Written = u32;

const FLAG_BIT: u32 = 16;

/// Lowers one block's instructions to Cranelift IR.
///
/// Registers `R0` to `R14` and the N, Z, C and V flags live in variables while the block runs, and
/// each instruction writes the ones it changed to the core's state once it's done, so an exit after
/// an instruction finds the state current and one that leaves as if it hadn't run finds the state
/// as the instruction found it. Everything else,
/// `Q`, `GE`, the floating-point registers and the special registers, stays in the state, which
/// [`Emitter::load_field`] and [`Emitter::store_field`] reach.
///
/// An instruction's function reads its operands with [`Emitter::reg`] and [`Emitter::flag`],
/// computes with the builder in [`Emitter::b`], and writes with [`Emitter::set_reg`] and
/// [`Emitter::set_flag`]. It accesses memory with [`Emitter::load`] and [`Emitter::store`], which
/// leave the block through a fault on their own. It changes the flow with the `branch` methods,
/// [`Emitter::raise`] and [`Emitter::wait`], each of which ends the block, and it ends the block
/// with [`Emitter::end_block`] after changing anything the next instruction's translation or the
/// interrupt it might take depends on.
pub(crate) struct Emitter<'a, 'b> {
    /// The builder, positioned where the instruction's code goes.
    pub(crate) b: &'a mut FunctionBuilder<'b>,
    regs: [Variable; 15],
    flags: [Variable; 4],
    /// The registers and flags the instruction has written, which reach the state once it's done.
    written: Written,
    /// The address of the instruction being translated.
    pc: u32,
    /// Its length in bytes, 2 or 4.
    length: u32,
    /// `ITSTATE` as the instruction sees it.
    it: u32,
    /// `ITSTATE` for the next instruction when this one sets it, as `IT` does.
    next_it: Option<u32>,
    /// The instructions of the block before this one.
    index: u32,
    /// The instruction ended the block.
    ended: bool,
    /// The instruction made an access that may have gone to the bus.
    accessed: bool,
    pointer: ir::Type,
    /// The budget when this pass through the block started, which every exit leaves less what
    /// it retired.
    budget: Variable,
    /// The block's first instruction, for a branch back to it to loop inside the block.
    head: Option<Head>,
    /// Where an exit to a known next block can chain to it, when the block may.
    chain: Option<Chain<'a>>,
    /// Where every exit goes, to write the block's registers back once for all of them.
    leave: Option<ir::Block>,
    /// What the core implements, which decides what an encoding is.
    features: Features,
    /// How the host makes a fused multiply-add.
    fma: Fma,
    /// An instruction of the block has passed B1.6.3's `ExecuteFPCheck()` on every way to this
    /// one, so the next has nothing left to check.
    pub(crate) fp_checked: bool,
}

/// A block's first instruction: its address, its `ITSTATE`, where its code starts, and whether a
/// branch back to it loops inside the block.
#[derive(Clone, Copy, Debug)]
struct Head {
    pc: u32,
    it: u32,
    block: ir::Block,
    looped: bool,
}

/// Where an exit leaves the block for: the next instruction's `ITSTATE`, the instructions the
/// pass retired, and the status the block answers.
#[derive(Clone, Copy, Debug)]
struct Exit {
    it: u32,
    retired: u32,
    status: u32,
}

/// A new link cell for an exit to the block at an address with an `ITSTATE`: its number and the
/// address the exit reads it at.
pub(crate) type Cells<'a> = &'a mut dyn FnMut(u32, u32) -> (u32, usize);

/// What a block needs to chain an exit to the block after it.
pub(crate) struct Chain<'a> {
    pub(crate) cells: Cells<'a>,
    /// The signature every block has, for the tail call.
    pub(crate) signature: ir::SigRef,
    /// The core's cache of recent lookups, where an exit to a computed address finds the next
    /// block.
    pub(crate) recent: usize,
}

impl<'a, 'b> Emitter<'a, 'b> {
    fn new(b: &'a mut FunctionBuilder<'b>, pointer: ir::Type) -> Self {
        let regs = std::array::from_fn(|_| b.declare_var(types::I32));
        let flags = std::array::from_fn(|_| b.declare_var(types::I32));
        let budget = b.declare_var(types::I32);
        let core = b.ins().get_pinned_reg(pointer);
        let left = b
            .ins()
            .load(types::I32, MemFlagsData::trusted(), core, offsets::BUDGET);
        b.def_var(budget, left);
        let mut emitter = Self {
            b,
            regs,
            flags,
            written: 0,
            pc: 0,
            length: 2,
            it: 0,
            next_it: None,
            index: 0,
            ended: false,
            accessed: false,
            pointer,
            budget,
            head: None,
            chain: None,
            leave: None,
            features: Features::default(),
            fma: Fma::Library,
            fp_checked: false,
        };
        emitter.reload();
        emitter
    }

    /// The core's state, from the register the core is pinned to while blocks run, so no block
    /// keeps it anywhere else.
    fn core(&mut self) -> Value {
        self.b.ins().get_pinned_reg(self.pointer)
    }

    /// The page table, read again from the core wherever it's needed rather than kept.
    fn table(&mut self) -> Value {
        let core = self.core();
        self.b
            .ins()
            .load(self.pointer, MemFlagsData::trusted(), core, offsets::TABLE)
    }

    /// The address of this instruction.
    pub(crate) fn pc(&self) -> u32 {
        self.pc
    }

    /// The address of the next instruction.
    pub(crate) fn next_pc(&self) -> u32 {
        self.pc.wrapping_add(self.length)
    }

    /// Whether this instruction is inside an IT block, A7.3.3's `InITBlock()`.
    pub(crate) fn in_it(&self) -> bool {
        self.it & 0xF != 0
    }

    /// Sets `ITSTATE` for the instructions after this one, as `IT` does, A7.7.37. The block goes
    /// on, with each instruction after it under its condition.
    pub(crate) fn set_next_it(&mut self, it: u32) {
        self.next_it = Some(it);
    }

    /// `R[n]`, A2.3.1: `R15` reads as this instruction's address plus 4.
    pub(crate) fn reg(&mut self, n: u32) -> Value {
        if n == 15 {
            return self.constant(self.pc.wrapping_add(4));
        }
        self.b.use_var(self.regs[n as usize])
    }

    /// Writes `R[n]` for `n` below 15. The PC is written only through the branch methods.
    pub(crate) fn set_reg(&mut self, n: u32, value: Value) {
        assert!(n < 15, "the PC is written by a branch");
        self.b.def_var(self.regs[n as usize], value);
        self.written |= 1 << n;
    }

    /// A flag, as 0 or 1 in an `i32`.
    pub(crate) fn flag(&mut self, flag: Flag) -> Value {
        self.b.use_var(self.flags[flag as usize])
    }

    /// Sets a flag from `value`, which is 0 or 1 in an `i32`.
    pub(crate) fn set_flag(&mut self, flag: Flag, value: Value) {
        self.b.def_var(self.flags[flag as usize], value);
        self.written |= 1 << (FLAG_BIT + flag as u32);
    }

    /// Sets N and Z from a 32-bit result.
    pub(crate) fn set_nz(&mut self, result: Value) {
        let n = self.b.ins().ushr_imm_u(result, 31);
        let z = self.b.ins().icmp_imm_s(IntCC::Equal, result, 0);
        let z = self.b.ins().uextend(types::I32, z);
        self.set_flag(Flag::N, n);
        self.set_flag(Flag::Z, z);
    }

    /// An `i32` constant.
    pub(crate) fn constant(&mut self, value: u32) -> Value {
        self.b
            .ins()
            .iconst(types::I32, i64::from(value.cast_signed()))
    }

    /// Whether condition `cond` holds, A7.3.1's `ConditionPassed()`, as an `i8` that's 0 or 1.
    pub(crate) fn condition(&mut self, cond: u32) -> Value {
        let n = self.flag(Flag::N);
        let z = self.flag(Flag::Z);
        let c = self.flag(Flag::C);
        let v = self.flag(Flag::V);
        let holds = match cond >> 1 {
            0b000 => z,
            0b001 => c,
            0b010 => n,
            0b011 => v,
            0b100 => {
                let not_z = self.b.ins().bxor_imm_s(z, 1);
                self.b.ins().band(c, not_z)
            }
            0b101 => self.b.ins().icmp(IntCC::Equal, n, v),
            0b110 => {
                let ge = self.b.ins().icmp(IntCC::Equal, n, v);
                let ge = self.b.ins().uextend(types::I32, ge);
                let not_z = self.b.ins().bxor_imm_s(z, 1);
                self.b.ins().band(ge, not_z)
            }
            _ => self.constant(1),
        };
        let holds = self.b.ins().icmp_imm_s(IntCC::NotEqual, holds, 0);
        if cond & 1 == 1 && cond != 0b1111 {
            self.b.ins().bxor_imm_s(holds, 1)
        } else {
            holds
        }
    }

    /// Loads the `i32` at `offset` into the core, one of [`offsets`].
    pub(crate) fn load_core(&mut self, offset: i32) -> Value {
        let core = self.core();
        self.b
            .ins()
            .load(types::I32, MemFlagsData::trusted(), core, offset)
    }

    /// Loads the `i32` state field at `offset` into [`super::state::State`].
    pub(crate) fn load_field(&mut self, offset: usize) -> Value {
        let offset = field_offset(offset);
        let core = self.core();
        self.b
            .ins()
            .load(types::I32, MemFlagsData::trusted(), core, offset)
    }

    /// Stores an `i32` to the state field at `offset`.
    pub(crate) fn store_field(&mut self, offset: usize, value: Value) {
        let offset = field_offset(offset);
        let core = self.core();
        self.b
            .ins()
            .store(MemFlagsData::trusted(), value, core, offset);
    }

    /// Loads `size` bytes at `address`, zero-extended to an `i32`, A3.2. A fault leaves the block
    /// as if the instruction hadn't run, so an instruction loads before it writes a register.
    pub(crate) fn load(&mut self, address: Value, size: Size, attributes: Attributes) -> Value {
        self.accessed = true;
        let slow = self.b.create_block();
        self.b.set_cold_block(slow);
        let join = self.b.create_block();
        self.b.append_block_param(join, types::I32);
        let table = self.table();
        let host = emit_lookup(self.b, table, address, Lookup::Read(size.bytes()), slow);
        // A3.3.1: the core's data order.
        let value = load_guest(self.b, size.host(), host, self.features.endian);
        let value = if size == Size::Word {
            value
        } else {
            self.b.ins().uextend(types::I32, value)
        };
        self.b.ins().jump(join, &[value.into()]);
        self.b.switch_to_block(slow);
        let size = self.constant(size.bytes());
        let attributes = self.constant(attributes.bits());
        let pc = self.constant(self.pc);
        let index = self.constant(self.index);
        let answer = self.call_helper(
            super::family::helpers::read as *const () as usize,
            &[address, size, attributes, pc, index],
            types::I64,
        );
        self.fault_unless(answer);
        let value = self.b.ins().ireduce(types::I32, answer);
        self.b.ins().jump(join, &[value.into()]);
        self.b.seal_block(join);
        self.b.switch_to_block(join);
        self.b.block_params(join)[0]
    }

    /// Stores the low `size` bytes of `value` at `address`, A3.2. A fault leaves the block as
    /// [`Emitter::load`]'s does.
    pub(crate) fn store(
        &mut self,
        address: Value,
        value: Value,
        size: Size,
        attributes: Attributes,
    ) {
        self.accessed = true;
        let slow = self.b.create_block();
        self.b.set_cold_block(slow);
        let join = self.b.create_block();
        let table = self.table();
        let host = emit_lookup(self.b, table, address, Lookup::Write(size.bytes()), slow);
        let narrow = if size == Size::Word {
            value
        } else {
            self.b.ins().ireduce(size.host(), value)
        };
        store_guest(self.b, narrow, host, self.features.endian);
        self.b.ins().jump(join, &[]);
        self.b.switch_to_block(slow);
        let size = self.constant(size.bytes());
        let attributes = self.constant(attributes.bits());
        let pc = self.constant(self.pc);
        let index = self.constant(self.index);
        let answer = self.call_helper(
            super::family::helpers::write as *const () as usize,
            &[address, value, size, attributes, pc, index],
            types::I64,
        );
        self.fault_unless(answer);
        self.b.ins().jump(join, &[]);
        self.b.seal_block(join);
        self.b.switch_to_block(join);
    }

    /// Leaves the block through a fault exit when a helper's answer has [`FAULTED`] set, and goes
    /// on with the instruction where it hasn't. The exit is at the instruction, or where the
    /// helper left `r[15]` with [`KEEP_PC`].
    fn fault_unless(&mut self, answer: Value) {
        let failed = self
            .b
            .ins()
            .ushr_imm_u(answer, i64::from(FAULTED.trailing_zeros()));
        let fault = self.b.create_block();
        self.b.set_cold_block(fault);
        let fine = self.b.create_block();
        self.b.ins().brif(failed, fault, &[], fine, &[]);
        self.b.seal_block(fault);
        self.b.seal_block(fine);
        self.b.switch_to_block(fault);
        let keep = self.b.ins().band_imm_s(answer, KEEP_PC.cast_signed());
        let here = self.constant(self.pc);
        let kept = self.load_field(std::mem::offset_of!(super::state::State, r) + 15 * 4);
        let pc = self.b.ins().select(keep, kept, here);
        self.exit_before_at(pc, status::EXCEPTION);
        self.ended = false;
        self.b.switch_to_block(fine);
    }

    /// Calls `function`, an `extern "C" fn(&mut Family, u32...) -> u32`, with `args`, and answers
    /// what it returns. The registers and flags reach the state before the call and are read back
    /// after it, so the helper may read or write any of them. Its answer is the instruction's to
    /// use, and the call doesn't end the block.
    pub(crate) fn call(&mut self, function: usize, args: &[Value]) -> Value {
        self.commit();
        let answer = self.call_helper(function, args, types::I32);
        self.reload();
        self.written = 0;
        answer
    }

    /// Calls `function` as [`Emitter::call`] does, for a helper that reads and writes none of
    /// `R0` to `R14` or the N, Z, C and V flags, so nothing is written back or read again.
    pub(crate) fn call_plain(&mut self, function: usize, args: &[Value]) -> Value {
        self.call_helper(function, args, types::I32)
    }

    fn call_helper(&mut self, function: usize, args: &[Value], returns: ir::Type) -> Value {
        let mut params = vec![self.pointer];
        params.extend(args.iter().map(|&arg| self.b.func.dfg.value_type(arg)));
        let mut signature = ir::Signature::new(host_convention());
        signature.params = params.into_iter().map(ir::AbiParam::new).collect();
        signature.returns = vec![ir::AbiParam::new(returns)];
        let signature = self.b.import_signature(signature);
        let callee = self
            .b
            .ins()
            .iconst(self.pointer, i64::try_from(function).expect("an address"));
        let core = self.core();
        let mut all = vec![core];
        all.extend_from_slice(args);
        let call = self.b.ins().call_indirect(signature, callee, &all);
        self.b.inst_results(call)[0]
    }

    /// `BranchWritePC(address)`, A2.3.1: the next instruction is at `address` with bit 0 cleared.
    /// Ends the block.
    pub(crate) fn branch(&mut self, address: Value) {
        let target = self.b.ins().band_imm_s(address, !1);
        let thumb = self.constant(1);
        self.exit_to_computed(target, thumb);
    }

    /// A branch to the constant `address`, which ends the block. A branch to itself outside an IT
    /// block spins, and the core treats it as waiting for an interrupt.
    pub(crate) fn branch_to(&mut self, address: u32) {
        let target = address & !1;
        if target == self.pc && !self.in_it() {
            let target = self.constant(target);
            self.exit_to(target, status::SPIN);
        } else {
            self.exit_to_known(target);
        }
    }

    /// Branches to `address` when `holds`, an `i8`, is set, and goes on to the next instruction
    /// otherwise. Ends the block.
    pub(crate) fn branch_if(&mut self, holds: Value, address: u32) {
        let taken = self.b.create_block();
        let not_taken = self.b.create_block();
        self.b.ins().brif(holds, taken, &[], not_taken, &[]);
        self.b.seal_block(taken);
        self.b.seal_block(not_taken);
        self.b.switch_to_block(not_taken);
        self.exit_to_known(self.next_pc());
        self.ended = false;
        self.b.switch_to_block(taken);
        let target = address & !1;
        if target == self.pc && !self.in_it() {
            let target = self.constant(target);
            self.exit_to(target, status::SPIN);
        } else {
            self.exit_to_known(target);
        }
    }

    /// `BXWritePC(address)`, A2.3.1, which `LoadWritePC` is too: in Handler mode an address
    /// from `0xF000_0000` up is an exception return, and otherwise bit 0 is `EPSR.T` and the next
    /// instruction is at the rest. Ends the block.
    pub(crate) fn branch_exchange(&mut self, address: Value) {
        let returning = self.b.ins().icmp_imm_s(
            IntCC::UnsignedGreaterThanOrEqual,
            address,
            i64::from(mask(exc_return::PREFIX).cast_signed()),
        );
        let ipsr = self.load_field(std::mem::offset_of!(super::state::State, ipsr));
        let handler = self.b.ins().icmp_imm_s(IntCC::NotEqual, ipsr, 0);
        let returning = self.b.ins().band(returning, handler);
        let exception_return = self.b.create_block();
        let exchange = self.b.create_block();
        self.b
            .ins()
            .brif(returning, exception_return, &[], exchange, &[]);
        self.b.seal_block(exception_return);
        self.b.seal_block(exchange);
        self.b.switch_to_block(exception_return);
        self.exit_to(address, status::EXCEPTION_RETURN);
        self.b.switch_to_block(exchange);
        self.blx_write_pc(address);
    }

    /// `BLXWritePC(address)`, A2.3.1: bit 0 is `EPSR.T` and the next instruction is at the rest,
    /// with no exception return. A target with bit 0 clear never chains, so the core takes the
    /// INVSTATE UsageFault on it. Ends the block.
    pub(crate) fn blx_write_pc(&mut self, address: Value) {
        let thumb = self.b.ins().band_imm_s(address, 1);
        self.store_field(std::mem::offset_of!(super::state::State, t), thumb);
        let target = self.b.ins().band_imm_s(address, !1);
        self.exit_to_computed(target, thumb);
    }

    /// Raises `fault` for this instruction: the processor takes its exception before the next
    /// instruction. An `SVC` retires first and returns past itself, unless it locked up, which
    /// makes it UNDEFINED, B1.5.15. Every other fault leaves as if the instruction hadn't run.
    /// Ends the block.
    pub(crate) fn raise(&mut self, fault: Fault) {
        let code = self.constant(fault as u32);
        let pc = self.constant(self.pc);
        let locked_up = self.call_plain(
            super::family::helpers::raise as *const () as usize,
            &[code, pc],
        );
        if fault != Fault::SupervisorCall {
            self.exit_before(status::EXCEPTION);
            return;
        }
        let undefined = self.b.create_block();
        let retired = self.b.create_block();
        self.b.ins().brif(locked_up, undefined, &[], retired, &[]);
        self.b.seal_block(undefined);
        self.b.seal_block(retired);
        self.b.switch_to_block(undefined);
        self.exit_before(status::EXCEPTION);
        self.b.switch_to_block(retired);
        let next = self.constant(self.next_pc());
        self.exit_to(next, status::EXCEPTION);
    }

    /// Raises `fault` as [`Emitter::raise`] does where `holds`, an `i8`, is set, and goes on with
    /// the instruction where it isn't.
    pub(crate) fn raise_if(&mut self, holds: Value, fault: Fault) {
        let raise = self.b.create_block();
        self.b.set_cold_block(raise);
        let go_on = self.b.create_block();
        self.b.ins().brif(holds, raise, &[], go_on, &[]);
        self.b.seal_block(raise);
        self.b.seal_block(go_on);
        self.b.switch_to_block(raise);
        self.raise(fault);
        self.ended = false;
        self.b.switch_to_block(go_on);
    }

    /// Retires a `WFI`, or a `WFE` when `event`, and leaves the core waiting. Ends the block.
    pub(crate) fn wait(&mut self, event: bool) {
        let next = self.constant(self.next_pc());
        self.exit_to(
            next,
            if event {
                status::WAIT_FOR_EVENT
            } else {
                status::WAIT_FOR_INTERRUPT
            },
        );
    }

    /// Ends the block after this instruction, for one that changed what the next instruction's
    /// translation depends on, or the masks an interrupt is taken by.
    pub(crate) fn end_block(&mut self) {
        let next = self.constant(self.next_pc());
        self.exit_to(next, status::NEXT);
    }

    /// Leaves after the `retired`-th instruction of the block, which has just run, when that
    /// spends the budget. The block runs as far as its budget and no further, with nothing
    /// translated for a shorter one.
    fn stop_if_spent(&mut self, retired: u32) {
        let budget = self.b.use_var(self.budget);
        let spent = self
            .b
            .ins()
            .icmp_imm_s(IntCC::Equal, budget, i64::from(retired));
        let next = self.next_pc();
        let exit = Exit {
            it: self.it_after(),
            retired: self.index + 1,
            status: status::NEXT,
        };
        self.leave_if(spent, next, exit);
    }

    /// Leaves for `pc` with `exit` where `condition` is set, and goes on where it isn't. The way
    /// out is laid out away from the way on, which is the one usually taken.
    fn leave_if(&mut self, condition: Value, pc: u32, exit: Exit) {
        let out = self.b.create_block();
        let stay = self.b.create_block();
        self.b.set_cold_block(out);
        self.b.ins().brif(condition, out, &[], stay, &[]);
        self.b.seal_block(out);
        self.b.seal_block(stay);
        self.b.switch_to_block(out);
        let pc = self.constant(pc);
        let arguments = self.leaving(pc, exit, None);
        let leave = self.leave_block();
        self.b.ins().jump(leave, &arguments);
        self.b.switch_to_block(stay);
    }

    /// Whether the instruction ended the block.
    pub(crate) fn ended(&self) -> bool {
        self.ended
    }

    /// Goes round the block again from `head`, its first instruction, with the registers and
    /// flags still in hand. The pass that ends here spends its instructions, and when that's the
    /// whole budget, or a helper asked the core to return, the block leaves instead, at its
    /// first instruction.
    fn loop_to(&mut self, head: ir::Block) {
        self.commit();
        self.written = 0;
        let retired = self.index + 1;
        let budget = self.b.use_var(self.budget);
        let left = self.b.ins().iadd_imm_s(budget, -i64::from(retired));
        let core = self.core();
        let returning = self.b.ins().load(
            types::I32,
            MemFlagsData::trusted(),
            core,
            offsets::RETURNING,
        );
        let spent = self.b.ins().icmp_imm_s(IntCC::Equal, left, 0);
        let asked = self.b.ins().icmp_imm_s(IntCC::NotEqual, returning, 0);
        let leave = self.b.ins().bor(spent, asked);
        let first = self.head.as_mut().expect("looping to the head");
        first.looped = true;
        let (pc, it) = (first.pc, first.it);
        let pc = self.constant(pc);
        let arguments = self.leaving(
            pc,
            Exit {
                it,
                retired,
                status: status::NEXT,
            },
            None,
        );
        let again = self.b.create_block();
        let out = self.leave_block();
        self.b.ins().brif(leave, out, &arguments, again, &[]);
        self.b.seal_block(again);
        self.b.switch_to_block(again);
        self.b.def_var(self.budget, left);
        let core = self.core();
        // A helper counts the instructions before its access from the budget in the state.
        self.b
            .ins()
            .store(MemFlagsData::trusted(), left, core, offsets::BUDGET);
        self.b.ins().jump(head, &[]);
        self.ended = true;
    }

    /// Leaves after this instruction, which retires, to `pc`, with `status`.
    fn exit_to(&mut self, pc: Value, status: u32) {
        let arguments = self.leaving(
            pc,
            Exit {
                it: self.it_after(),
                retired: self.index + 1,
                status,
            },
            None,
        );
        let leave = self.leave_block();
        self.b.ins().jump(leave, &arguments);
        self.ended = true;
    }

    /// Leaves after this instruction for `target`, which it computed. Where the block may chain,
    /// the exit hands over to the block there when the core has it at hand, as an exit to a known
    /// address does through its link cell, unless `thumb` is 0, since the core faults on that.
    fn exit_to_computed(&mut self, target: Value, thumb: Value) {
        self.commit();
        let address = if self.chain.is_some() {
            let find = self.b.ins().iconst(self.pointer, FIND);
            let none = self.b.ins().iconst(self.pointer, 0);
            self.b.ins().select(thumb, find, none)
        } else {
            self.b.ins().iconst(self.pointer, 0)
        };
        let exit = Exit {
            it: self.it_after(),
            retired: self.index + 1,
            status: status::NEXT,
        };
        let packed = self.constant(exit.it | exit.retired << 8 | exit.status << 16);
        let leave = self.leave_block();
        self.b
            .ins()
            .jump(leave, &[target.into(), packed.into(), address.into()]);
        self.ended = true;
    }

    /// `ITSTATE` for the instruction after this one.
    fn it_after(&self) -> u32 {
        self.next_it.unwrap_or_else(|| it_advance(self.it))
    }

    /// The block every exit goes to, made the first time one does. Its parameters are the next
    /// instruction's address, `ITSTATE`, the instructions the pass retired and the status to
    /// answer packed in a byte each, and a link cell's address or 0.
    fn leave_block(&mut self) -> ir::Block {
        if let Some(leave) = self.leave {
            return leave;
        }
        let leave = self.b.create_block();
        self.b.append_block_param(leave, types::I32);
        self.b.append_block_param(leave, types::I32);
        self.b.append_block_param(leave, self.pointer);
        self.leave = Some(leave);
        leave
    }

    /// The arguments of a jump to the leave block.
    fn leaving(&mut self, pc: Value, exit: Exit, cell: Option<(u32, usize)>) -> Vec<ir::BlockArg> {
        if exit.retired > self.index {
            self.commit();
        }
        let address = cell.map_or(0, |(_, address)| address);
        let packed = self.constant(exit.it | exit.retired << 8 | exit.status << 16);
        let address = self
            .b
            .ins()
            .iconst(self.pointer, i64::try_from(address).expect("an address"));
        [pc, packed, address]
            .into_iter()
            .map(ir::BlockArg::from)
            .collect()
    }

    /// Fills the leave block, once every exit has gone to it: every register and flag the block
    /// wrote goes back to the state, as each exit had it, with the next instruction's address,
    /// `ITSTATE` and the budget left. An exit with a link cell hands over to the next block when
    /// it's linked and nothing asked the core to return, and notes the cell for the core
    /// otherwise.
    pub(crate) fn finish(&mut self) {
        let Some(leave) = self.leave.take() else {
            return;
        };
        self.b.switch_to_block(leave);
        self.b.seal_block(leave);
        let parameters = self.b.block_params(leave).to_vec();
        let [pc, packed, address] = parameters[..] else {
            unreachable!("the leave block's three parameters");
        };
        let it = self.b.ins().band_imm_s(packed, 0xFF);
        let retired = self.b.ins().ushr_imm_u(packed, 8);
        let retired = self.b.ins().band_imm_s(retired, 0xFF);
        let status = self.b.ins().ushr_imm_u(packed, 16);
        self.store_field(std::mem::offset_of!(super::state::State, r) + 15 * 4, pc);
        self.store_field(std::mem::offset_of!(super::state::State, it), it);
        let budget = self.b.use_var(self.budget);
        let left = self.b.ins().isub(budget, retired);
        let core = self.core();
        self.b
            .ins()
            .store(MemFlagsData::trusted(), left, core, offsets::BUDGET);
        let Some(chain) = self.chain.as_ref() else {
            self.b.ins().return_(&[status]);
            return;
        };
        let signature = chain.signature;
        let recent = chain.recent;
        let chained = self.b.create_block();
        let plain = self.b.create_block();
        self.b.ins().brif(address, chained, &[], plain, &[]);
        self.b.seal_block(chained);
        self.b.seal_block(plain);
        self.b.switch_to_block(plain);
        self.b.ins().return_(&[status]);
        self.b.switch_to_block(chained);
        // An exit to a computed address finds the next block by its key, and one to a known
        // address reads its link cell.
        let computed = self.b.ins().icmp_imm_s(IntCC::Equal, address, FIND);
        let by_key = self.b.create_block();
        let by_cell = self.b.create_block();
        let found = self.b.create_block();
        self.b.append_block_param(found, self.pointer);
        self.b.ins().brif(computed, by_key, &[], by_cell, &[]);
        self.b.seal_block(by_key);
        self.b.seal_block(by_cell);
        self.b.switch_to_block(by_key);
        let wide_pc = self.b.ins().uextend(types::I64, pc);
        let wide_it = self.b.ins().uextend(types::I64, it);
        let wide_it = self.b.ins().ishl_imm_s(wide_it, 32);
        let key = self.b.ins().bor(wide_pc, wide_it);
        let key = self.b.ins().bor_imm_s(key, i64::from(LONGEST_BLOCK) << 40);
        let next = crate::emulator::core::emit_find(self.b, self.pointer, recent, key);
        self.b.ins().jump(found, &[next.into()]);
        self.b.switch_to_block(by_cell);
        let next = self
            .b
            .ins()
            .load(self.pointer, MemFlagsData::trusted(), address, 0);
        self.b.ins().jump(found, &[next.into()]);
        self.b.seal_block(found);
        self.b.switch_to_block(found);
        let next = self.b.block_params(found)[0];
        // Only a cell is noted for the core to link.
        let none = self.b.ins().iconst(self.pointer, 0);
        let address = self.b.ins().select(computed, none, address);
        let core = self.core();
        // A helper that reached the bus asked the core to return, which only the run loop does.
        let returning = self.b.ins().load(
            types::I32,
            MemFlagsData::trusted(),
            core,
            offsets::RETURNING,
        );
        let returning = self.b.ins().icmp_imm_s(IntCC::NotEqual, returning, 0);
        let returning = self.b.ins().uextend(self.pointer, returning);
        let stay = self.b.ins().band_imm_s(returning, 0);
        let next = self.b.ins().select(returning, stay, next);
        let linked = self.b.create_block();
        let unlinked = self.b.create_block();
        self.b.ins().brif(next, linked, &[], unlinked, &[]);
        self.b.seal_block(linked);
        self.b.seal_block(unlinked);
        self.b.switch_to_block(linked);
        let core = self.core();
        self.b.ins().return_call_indirect(signature, next, &[core]);
        self.b.switch_to_block(unlinked);
        let core = self.core();
        self.b
            .ins()
            .store(MemFlagsData::trusted(), address, core, offsets::LINK);
        self.b.ins().return_(&[status]);
    }

    /// Leaves after this instruction for the next one at `target`, which the core runs next as
    /// it would after any block. Where the block may chain, the exit hands over to the block
    /// at `target` itself once the core has linked it, and the core comes back only when the
    /// budget is too short for that block or nothing is linked yet.
    fn exit_to_known(&mut self, target: u32) {
        let it = self.next_it.unwrap_or_else(|| it_advance(self.it));
        if let Some(head) = self.head
            && head.pc == target
            && head.it == it
        {
            self.loop_to(head.block);
            return;
        }
        let pc = self.constant(target);
        let cell = self.chain.as_mut().map(|chain| (chain.cells)(target, it));
        let arguments = self.leaving(
            pc,
            Exit {
                it,
                retired: self.index + 1,
                status: status::NEXT,
            },
            cell,
        );
        let leave = self.leave_block();
        self.b.ins().jump(leave, &arguments);
        self.ended = true;
    }

    /// Leaves as if this instruction hadn't run, with `status`.
    fn exit_before(&mut self, status: u32) {
        let pc = self.constant(self.pc);
        self.exit_before_at(pc, status);
    }

    /// Leaves for `pc` as if this instruction hadn't run, with `status`.
    fn exit_before_at(&mut self, pc: Value, status: u32) {
        let arguments = self.leaving(
            pc,
            Exit {
                it: self.it,
                retired: self.index,
                status,
            },
            None,
        );
        let leave = self.leave_block();
        self.b.ins().jump(leave, &arguments);
        self.ended = true;
    }

    /// Writes the registers and flags the instruction has written to the state, on the way the
    /// builder is on.
    fn commit(&mut self) {
        for bit in 0..32 {
            if self.written & (1 << bit) != 0 {
                let (variable, offset) = self.slot(bit);
                let value = self.b.use_var(variable);
                self.store_field(offset, value);
            }
        }
    }

    /// Finishes the instruction where it didn't end the block: what it wrote reaches the state.
    fn done(&mut self) {
        if !self.ended {
            self.commit();
            self.written = 0;
        }
    }

    /// Reads every register and flag from the state into its variable.
    fn reload(&mut self) {
        for bit in (0..15).chain(FLAG_BIT..FLAG_BIT + 4) {
            let (variable, offset) = self.slot(bit);
            let value = self.load_field(offset);
            self.b.def_var(variable, value);
        }
    }

    /// The variable and the state's offset of written-bit `bit`.
    fn slot(&self, bit: u32) -> (Variable, usize) {
        use super::state::State;
        if bit < FLAG_BIT {
            let n = bit as usize;
            (self.regs[n], std::mem::offset_of!(State, r) + 4 * n)
        } else {
            let flag = (bit - FLAG_BIT) as usize;
            let offset = [
                std::mem::offset_of!(State, n),
                std::mem::offset_of!(State, z),
                std::mem::offset_of!(State, c),
                std::mem::offset_of!(State, v),
            ][flag];
            (self.flags[flag], offset)
        }
    }

    /// Starts instruction `index` at `pc`, `length` bytes long, with `ITSTATE` `it`, once the one
    /// before is done.
    fn begin(&mut self, index: u32, pc: u32, length: u32, it: u32) {
        self.done();
        self.index = index;
        self.pc = pc;
        self.length = length;
        self.it = it;
        self.next_it = None;
        self.ended = false;
        self.accessed = false;
        self.written = 0;
    }

    /// Ends the block after this instruction when an access of it went to the bus, which asks
    /// the core to return to the runner before the next instruction.
    fn return_if_asked(&mut self) {
        if !self.accessed || self.ended {
            return;
        }
        let core = self.core();
        let returning = self.b.ins().load(
            types::I32,
            MemFlagsData::trusted(),
            core,
            offsets::RETURNING,
        );
        let exit = Exit {
            it: self.it_after(),
            retired: self.index + 1,
            status: status::NEXT,
        };
        self.leave_if(returning, self.next_pc(), exit);
    }
}

/// How many ways a lowered function has of leaving: every branch to its leave block, which is
/// the only block with three parameters, before the leave block is filled.
#[cfg(test)]
pub(crate) fn exits_of(function: &ir::Function) -> usize {
    let dfg = &function.dfg;
    function
        .layout
        .blocks()
        .flat_map(|block| function.layout.block_insts(block))
        .flat_map(|inst| {
            dfg.insts[inst]
                .branch_destination(&dfg.jump_tables, &dfg.exception_tables)
                .iter()
                .map(|call| call.block(&dfg.value_lists))
        })
        .filter(|&block| dfg.num_block_params(block) == 3)
        .count()
}

/// One instruction fetched: its first halfword, and its second when it's 32 bits, A5.1.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Instruction {
    Narrow(u16),
    Wide(u16, u16),
}

impl Instruction {
    /// Whether `first` starts a 32-bit instruction, A5.1.
    pub(crate) fn is_wide(first: u16) -> bool {
        matches!(first >> 11, 0b11101..=0b11111)
    }

    fn length(self) -> u32 {
        match self {
            Self::Narrow(_) => 2,
            Self::Wide(..) => 4,
        }
    }

    /// Whether it's one of B1.5.10's exception-continuable instructions on a core with the
    /// floating-point extension when `fp`: `STM` and `LDM`, Table A5-1, `PUSH` and `POP`, A5.2.5,
    /// the wide `STM`, `LDM`, `STMDB`, `LDMDB`, `PUSH` and `POP`, Table A5-16, and only with the
    /// extension, `VSTM`, `VLDM`, `VPUSH` and `VPOP`, Table A6-7. `VSTR` and `VLDR` are the rows
    /// of that table with `P` set and `W` clear, and aren't.
    fn continuable(self, fp: bool) -> bool {
        match self {
            Self::Narrow(hw) => matches!(hw >> 11, 0b11000 | 0b11001) || hw & 0xF600 == 0xB400,
            Self::Wide(hw1, hw2) => {
                let multiple = hw1 & 0xFE40 == 0xE800 && matches!((hw1 >> 7) & 0b11, 0b01 | 0b10);
                let (p, u, w) = ((hw1 >> 8) & 1, (hw1 >> 7) & 1, (hw1 >> 5) & 1);
                let extension = fp
                    && hw1 & 0xFE00 == 0xEC00
                    && hw2 & 0x0E00 == 0x0A00
                    && matches!((p, u, w), (0, 1, _) | (1, 0, 1));
                multiple || extension
            }
        }
    }
}

/// What a block is translated from: where it starts, the `ITSTATE` there, and the most
/// instructions it may hold.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct Start {
    pub(crate) pc: u32,
    pub(crate) it: u32,
    pub(crate) longest: u32,
}

/// The key of the block at `start`, with the MPU off. [`Emitter::finish`] works it out the same
/// way in code.
pub(crate) fn block_key(start: Start) -> u64 {
    u64::from(start.pc) | u64::from(start.it) << 32 | u64::from(start.longest) << 40
}

/// The leave block's cell address for an exit that finds the next block by its key, which no
/// cell's address is, since cells are words.
const FIND: i64 = 1;

/// Translates one block from `start` for a core with `features`, fetching halfwords with `fetch`,
/// and ending before any address `stop_before` names but the first. Answers the block, the first
/// and last byte it came from, and whether it loops back to its first instruction inside itself,
/// which only one that doesn't start at a breakpoint does.
#[expect(
    clippy::too_many_arguments,
    reason = "a block's start, its bytes, where it stops, how it chains and how hard to compile it are each the caller's"
)]
pub(crate) fn translate(
    jit: &mut Jit,
    features: Features,
    start: Start,
    fetch: &mut dyn FnMut(u32) -> Option<u16>,
    stop_before: &dyn Fn(u32) -> bool,
    cells: Option<(Cells<'_>, usize)>,
    tier: Tier,
    counter: Option<usize>,
) -> Result<(Block, (u32, u32), bool), Error> {
    let pointer = jit.pointer();
    let fma = jit.fma();
    // The first and last byte the block came from.
    let mut range = (start.pc, start.pc);
    let mut looped = false;
    let block = jit.compile(tier, |builder, core, signature| {
        let entry = builder.current_block().expect("in the entry block");
        // The builder lays a block out when it first holds an instruction, and the body fills
        // before the entry does, so the entry goes first now.
        builder.func.layout.append_block(entry);
        let body = builder.create_block();
        builder.switch_to_block(body);
        let signature = builder.import_signature(signature.clone());
        let mut e = Emitter::new(builder, pointer);
        e.features = features;
        e.fma = fma;
        e.chain = cells.map(|(cells, recent)| Chain {
            cells,
            signature,
            recent,
        });
        // The registers, the flags and the budget are read in `body`, and each pass round the
        // block starts after them, at `head`.
        let head = e.b.create_block();
        e.b.ins().jump(head, &[]);
        e.b.switch_to_block(head);
        // A block that starts at a breakpoint, run once to pass it, leaves at its back edge, so
        // the next lap stops there.
        e.head = (!stop_before(start.pc)).then_some(Head {
            pc: start.pc,
            it: start.it,
            block: head,
            looped: false,
        });
        let mut pc = start.pc;
        let mut it = start.it;
        let mut count = 0;
        loop {
            if count > 0 && (count == start.longest || stop_before(pc)) {
                // The block ends where the next instruction starts, which it hasn't run: the
                // last one it ran stands as the one that retired.
                e.index = count - 1;
                e.next_it = Some(it);
                if stop_before(pc) {
                    let next = e.constant(pc);
                    e.exit_to(next, status::NEXT);
                } else {
                    e.exit_to_known(pc);
                }
                break;
            }
            let Some(instruction) = fetch_one(fetch, pc) else {
                // A wide instruction's first halfword was fetched, and a change to it drops this
                // block too.
                if fetch(pc).is_some() {
                    range.1 = pc + 1;
                }
                e.begin(count, pc, 2, it);
                e.raise(Fault::InstructionFetch);
                break;
            };
            if count == 0 && it != 0 && it.trailing_zeros() >= 4 {
                // B1.5.10: ICI bits, which only an exception return brings back, belong to a Load
                // Multiple or Store Multiple, which this core restarts from its first register.
                // On anything else they're an INVSTATE UsageFault.
                if !instruction.continuable(features.fp()) {
                    range.1 = pc + instruction.length() - 1;
                    e.begin(count, pc, instruction.length(), it);
                    e.raise(Fault::InvalidState);
                    break;
                }
                it = 0;
            }
            e.begin(count, pc, instruction.length(), it);
            range.1 = pc + instruction.length() - 1;
            emit_one(&mut e, instruction);
            count += 1;
            if e.ended() {
                break;
            }
            e.stop_if_spent(count);
            it = e.next_it.take().unwrap_or_else(|| it_advance(it));
            pc = e.next_pc();
        }
        looped = e.head.is_some_and(|head| head.looped);
        e.finish();
        e.b.switch_to_block(entry);
        let budget =
            e.b.ins()
                .load(types::I32, MemFlagsData::trusted(), core, offsets::BUDGET);
        let short = e.b.ins().icmp_imm_s(IntCC::Equal, budget, 0);
        let refuse = e.b.create_block();
        if let Some(address) = counter {
            // A quick translation counts its runs, and hands back untouched once they make it
            // worth translating again.
            let count = e.b.create_block();
            e.b.ins().brif(short, refuse, &[], count, &[]);
            e.b.seal_block(count);
            e.b.switch_to_block(count);
            let counter =
                e.b.ins()
                    .iconst(pointer, i64::try_from(address).expect("an address"));
            let runs = e.b.ins().load(pointer, MemFlagsData::trusted(), counter, 0);
            let runs = e.b.ins().iadd_imm_s(runs, 1);
            e.b.ins().store(MemFlagsData::trusted(), runs, counter, 0);
            let hot = e.b.ins().icmp_imm_s(IntCC::Equal, runs, HOT_RUNS);
            let promote = e.b.create_block();
            e.b.ins().brif(hot, promote, &[], body, &[]);
            e.b.seal_block(promote);
            e.b.switch_to_block(promote);
            let answer = e.constant(status::HOT);
            e.b.ins().return_(&[answer]);
        } else {
            e.b.ins().brif(short, refuse, &[], body, &[]);
        }
        e.b.seal_block(body);
        e.b.seal_block(head);
        e.b.seal_block(refuse);
        e.b.switch_to_block(refuse);
        let answer = e.constant(status::SHORT);
        e.b.ins().return_(&[answer]);
    })?;
    Ok((block, range, looped))
}

fn fetch_one(fetch: &mut dyn FnMut(u32) -> Option<u16>, pc: u32) -> Option<Instruction> {
    let first = fetch(pc)?;
    if !Instruction::is_wide(first) {
        return Some(Instruction::Narrow(first));
    }
    Some(Instruction::Wide(first, fetch(pc.wrapping_add(2))?))
}

/// Lowers one instruction, inside its IT block's condition when it has one and the instruction
/// tests it, A7.3.1.
fn emit_one(e: &mut Emitter<'_, '_>, instruction: Instruction) {
    let conditional =
        e.in_it() && (e.it >> 4) != 0b1110 && !control::ignores_it_condition(instruction);
    if !conditional {
        decode(e, instruction);
        e.done();
        e.return_if_asked();
        return;
    }
    // A check the condition may skip leaves the next instruction its own.
    let checked = e.fp_checked;
    let holds = e.condition(e.it >> 4);
    let run = e.b.create_block();
    let skip = e.b.create_block();
    e.b.ins().brif(holds, run, &[], skip, &[]);
    e.b.seal_block(run);
    e.b.switch_to_block(run);
    decode(e, instruction);
    e.fp_checked = checked;
    e.done();
    e.return_if_asked();
    if e.ended() {
        // It ended the block where it ran, so where it doesn't, the block ends after it too,
        // having written nothing.
        e.b.seal_block(skip);
        e.b.switch_to_block(skip);
        e.written = 0;
        e.end_block();
        return;
    }
    e.b.ins().jump(skip, &[]);
    e.b.seal_block(skip);
    e.b.switch_to_block(skip);
}

/// Decodes one instruction to its group, A5.1, and raises UNDEFINED for one no group defines.
fn decode(e: &mut Emitter<'_, '_>, instruction: Instruction) {
    let translated = match instruction {
        Instruction::Narrow(hw) => decode16(e, hw),
        Instruction::Wide(hw1, hw2) => decode32(e, hw1, hw2),
    };
    if translated.is_err() {
        e.raise(Fault::Undefined);
    }
}

/// A5.2, the 16-bit encodings, by `opcode`, bits 15 to 10.
fn decode16(e: &mut Emitter<'_, '_>, hw: u16) -> Translated {
    let opcode = hw >> 10;
    match opcode {
        0b00_0000..=0b00_1111 => data16::shift_add_subtract_move_compare(e, hw),
        0b01_0000 => data16::data_processing(e, hw),
        0b01_0001 => data16::special_data_branch_exchange(e, hw),
        0b01_0010 | 0b01_0011 => memory::load_literal16(e, hw),
        0b01_0100..=0b10_0111 => memory::load_store_single16(e, hw)?,
        0b10_1000 | 0b10_1001 => data16::adr(e, hw),
        0b10_1010 | 0b10_1011 => data16::add_sp_immediate(e, hw),
        0b10_1100..=0b10_1111 => miscellaneous16(e, hw)?,
        0b11_0000 | 0b11_0001 => memory::store_multiple16(e, hw)?,
        0b11_0010 | 0b11_0011 => memory::load_multiple16(e, hw)?,
        0b11_0100..=0b11_0111 => control::conditional_branch_supervisor_call(e, hw)?,
        0b11_1000 | 0b11_1001 => control::branch16(e, hw),
        _ => return Err(Undefined),
    }
    Ok(())
}

/// A5.2.5, the miscellaneous 16-bit instructions, by `opcode`, bits 11 to 5, Table A5-6.
fn miscellaneous16(e: &mut Emitter<'_, '_>, hw: u16) -> Translated {
    let opcode = (hw >> 5) & 0x7F;
    match opcode {
        0b011_0011 => control::change_processor_state(e, hw),
        0b000_0000..=0b000_0111 => data16::adjust_sp(e, hw),
        0b000_1000..=0b000_1111
        | 0b001_1000..=0b001_1111
        | 0b100_1000..=0b100_1111
        | 0b101_1000..=0b101_1111 => control::compare_branch(e, hw),
        0b001_0000..=0b001_0111 => data16::extend(e, hw),
        0b010_0000..=0b010_1111 => memory::push16(e, hw)?,
        0b101_0000..=0b101_0011 | 0b101_0110 | 0b101_0111 => data16::reverse(e, hw)?,
        0b110_0000..=0b110_1111 => memory::pop16(e, hw)?,
        0b111_0000..=0b111_0111 => control::breakpoint(e, hw),
        0b111_1000..=0b111_1111 => control::if_then_hints(e, hw),
        _ => return Err(Undefined),
    }
    Ok(())
}

/// A5.3, the 32-bit encodings, by `op1`, `op2` and `op`.
fn decode32(e: &mut Emitter<'_, '_>, hw1: u16, hw2: u16) -> Translated {
    if !e.features.dsp && dsp_only(u32::from(hw1), u32::from(hw2)) {
        return Err(Undefined);
    }
    let op1 = (hw1 >> 11) & 0b11;
    let op2 = (hw1 >> 4) & 0x7F;
    let op = hw2 >> 15;
    match op1 {
        0b01 => match op2 >> 5 {
            0b00 if op2 & 0b100 == 0 => memory::load_store_multiple(e, hw1, hw2),
            0b00 => memory::load_store_dual_exclusive_table_branch(e, hw1, hw2),
            0b01 => data32::shifted_register(e, hw1, hw2),
            _ => float::coprocessor(e, hw1, hw2),
        },
        0b10 if op == 0 && op2 & 0b10_0000 == 0 => data32::modified_immediate(e, hw1, hw2),
        0b10 if op == 0 => data32::plain_binary_immediate(e, hw1, hw2),
        0b10 => control::branches_miscellaneous_control(e, hw1, hw2),
        0b11 => match op2 {
            _ if op2 & 0b111_0001 == 0 => memory::store_single(e, hw1, hw2),
            _ if op2 & 0b110_0111 == 0b000_0001 => memory::load_byte_hints(e, hw1, hw2),
            _ if op2 & 0b110_0111 == 0b000_0011 => memory::load_halfword_hints(e, hw1, hw2),
            _ if op2 & 0b110_0111 == 0b000_0101 => memory::load_word(e, hw1, hw2),
            _ if op2 & 0b110_0111 == 0b000_0111 => Err(Undefined),
            _ if op2 & 0b111_0000 == 0b010_0000 => data32::register(e, hw1, hw2),
            _ if op2 & 0b111_1000 == 0b011_0000 => multiply::multiply_accumulate(e, hw1, hw2),
            _ if op2 & 0b111_1000 == 0b011_1000 => multiply::long_multiply_divide(e, hw1, hw2),
            _ if op2 & 0b100_0000 != 0 => float::coprocessor(e, hw1, hw2),
            _ => Err(Undefined),
        },
        _ => Err(Undefined),
    }
}

/// Whether a 32-bit encoding is one the DSP extension adds: the rows A5.3's tables mark v7E-M, in
/// Tables A5-12, A5-22, A5-24, A5-27, A5-28 and A5-29, and all of A5-25 and A5-26.
fn dsp_only(hw1: u32, hw2: u32) -> bool {
    let op2 = bits(hw1, 10, 4);
    match bits(hw1, 12, 11) {
        // Table A5-12: SSAT16 and USAT16 are SSAT and USAT with no shift.
        0b10 if bits(hw2, 15, 15) == 0 && bits(hw1, 9, 9) == 1 => {
            matches!(bits(hw1, 8, 4), 0b1_0010 | 0b1_1010)
                && bits(hw2, 14, 12) == 0
                && bits(hw2, 7, 6) == 0
        }
        // Table A5-22: PKHBT and PKHTB.
        0b01 if op2 >> 5 == 0b01 => bits(hw1, 8, 5) == 0b0110,
        0b11 => match op2 >> 3 {
            0b0100 | 0b0101 => register_dsp_only(hw1, hw2),
            // Table A5-28: all but MLA, MUL and MLS.
            0b0110 => bits(hw1, 6, 4) != 0,
            // Table A5-29: the halfword and dual accumulates, and UMAAL.
            0b0111 => match (bits(hw1, 6, 4), bits(hw2, 7, 4)) {
                (0b100, op2) => op2 != 0,
                (0b101, _) | (0b110, 0b0110) => true,
                _ => false,
            },
            _ => false,
        },
        _ => false,
    }
}

/// [`dsp_only`] for Table A5-24, by its `op1` and `op2`: the extends that add or work on byte
/// pairs, the parallel arithmetic of Tables A5-25 and A5-26, and Table A5-27's saturating
/// arithmetic and SEL.
fn register_dsp_only(hw1: u32, hw2: u32) -> bool {
    let (op1, op2) = (bits(hw1, 7, 4), bits(hw2, 7, 4));
    match (op1 >> 3, op2 >> 2) {
        (0, 0b10 | 0b11) => bits(hw1, 3, 0) != 0b1111 || op1 >> 1 == 0b001,
        (1, 0b00 | 0b01) => true,
        (1, 0b10) => matches!(bits(hw1, 5, 4), 0b00 | 0b10),
        _ => false,
    }
}

#[cfg(test)]
mod tests {
    use cranelift_codegen::ir::Function;
    use cranelift_frontend::FunctionBuilderContext;

    use super::*;

    /// One instruction lowered by `lower` into a function that never runs: whether it ended the
    /// block, and how many ways out of the block the function has.
    fn lowered(lower: impl FnOnce(&mut Emitter<'_, '_>)) -> (bool, usize) {
        let mut function = Function::new();
        function
            .signature
            .params
            .push(ir::AbiParam::new(types::I64));
        let mut context = FunctionBuilderContext::new();
        let mut builder = FunctionBuilder::new(&mut function, &mut context);
        let entry = builder.create_block();
        builder.append_block_params_for_function_params(entry);
        builder.switch_to_block(entry);
        let mut e = Emitter::new(&mut builder, types::I64);
        e.begin(0, 0, 4, 0);
        lower(&mut e);
        let ended = e.ended();
        let exits = exits_of(&function);
        (ended, exits)
    }

    /// Every row the tables mark v7E-M is the DSP extension's, and its siblings marked All aren't.
    #[test]
    fn the_dsp_extension_is_the_rows_marked_v7e_m() {
        let dsp = [
            (0xF321_0003, "SSAT16"),
            (0xF3A1_0003, "USAT16"),
            (0xEAC1_0002, "PKHBT"),
            (0xFA01_F082, "SXTAH"),
            (0xFA2F_F082, "SXTB16"),
            (0xFA51_F082, "UXTAB"),
            (0xFA91_F002, "SADD16"),
            (0xFA81_F042, "UADD8"),
            (0xFA81_F082, "QADD"),
            (0xFAA1_F082, "SEL"),
            (0xFB11_F002, "SMULBB"),
            (0xFB71_F002, "USAD8"),
            (0xFBC2_0183, "SMLALBB"),
            (0xFBD2_01C3, "SMLSLD"),
            (0xFBE2_0163, "UMAAL"),
        ];
        let base = [
            (0xF321_0043, "SSAT with a shift"),
            (0xF201_0000, "ADDW"),
            (0xEA01_0002, "AND"),
            (0xFA0F_F082, "SXTH"),
            (0xFA5F_F082, "UXTB"),
            (0xFA01_F002, "LSL"),
            (0xFA91_F081, "REV"),
            (0xFAB1_F081, "CLZ"),
            (0xFB01_F002, "MUL"),
            (0xFB01_3012, "MLS"),
            (0xFB82_0103, "SMULL"),
            (0xFB91_F0F2, "SDIV"),
            (0xFBE2_0103, "UMLAL"),
            (0xF000_B800, "B"),
        ];
        let split = |instruction: u32| (instruction >> 16, instruction & 0xFFFF);
        for (instruction, name) in dsp {
            let (hw1, hw2) = split(instruction);
            assert!(dsp_only(hw1, hw2), "{name}");
        }
        for (instruction, name) in base {
            let (hw1, hw2) = split(instruction);
            assert!(!dsp_only(hw1, hw2), "{name}");
        }
    }

    /// B1.5.10's exception-continuable instructions, from every row of Table A5-16, the two
    /// multiples of Table A5-1, `PUSH` and `POP` from A5.2.5, and the multiples of Table A6-7 only
    /// with the floating-point extension, against their neighbors in the same tables.
    #[test]
    fn the_exception_continuable_instructions_are_b1_5_10s() {
        use Instruction::{Narrow, Wide};
        let always = [
            (Narrow(0xC002), "STM r0!, {r1}"),
            (Narrow(0xC806), "LDM r0!, {r1, r2}"),
            (Narrow(0xB510), "PUSH {r4, lr}"),
            (Narrow(0xBD10), "POP {r4, pc}"),
            (Wide(0xE8A0, 0x0006), "STM.W r0!, {r1, r2}"),
            (Wide(0xE890, 0x0006), "LDM.W r0, {r1, r2}"),
            (Wide(0xE8BD, 0x8010), "POP.W {r4, pc}"),
            (Wide(0xE900, 0x0006), "STMDB r0, {r1, r2}"),
            (Wide(0xE92D, 0x4010), "PUSH.W {r4, lr}"),
            (Wide(0xE910, 0x0006), "LDMDB r0, {r1, r2}"),
        ];
        let with_the_extension = [
            (Wide(0xECA0, 0x0A02), "VSTMIA r0!, {s0, s1}"),
            (Wide(0xEC90, 0x0A02), "VLDMIA r0, {s0, s1}"),
            (Wide(0xED20, 0x0A02), "VSTMDB r0!, {s0, s1}"),
            (Wide(0xED30, 0x0A02), "VLDMDB r0!, {s0, s1}"),
            (Wide(0xED2D, 0x8B02), "VPUSH {d8}"),
            (Wide(0xECBD, 0x8A02), "VPOP {s16, s17}"),
        ];
        let never = [
            (Narrow(0x2001), "MOVS r0, #1"),
            (Narrow(0x6808), "LDR r0, [r1]"),
            (Narrow(0x9000), "STR r0, [sp]"),
            (Narrow(0xB001), "ADD sp, #4"),
            (Narrow(0xB100), "CBZ r0"),
            (Narrow(0xB672), "CPSID i"),
            (Wide(0xE9D2, 0x0100), "LDRD r0, r1, [r2]"),
            (Wide(0xE841, 0x0200), "STREX r2, r0, [r1]"),
            (Wide(0xE8D0, 0xF000), "TBB [r0, r0]"),
            (Wide(0xED90, 0x0A00), "VLDR s0, [r0]"),
            (Wide(0xED80, 0x0A00), "VSTR s0, [r0]"),
            (Wide(0xEC51, 0x0B10), "VMOV r0, r1, d0"),
        ];
        for (instruction, name) in always {
            assert!(instruction.continuable(false), "{name}");
            assert!(instruction.continuable(true), "{name}");
        }
        for (instruction, name) in with_the_extension {
            assert!(
                !instruction.continuable(false),
                "{name} without the extension"
            );
            assert!(instruction.continuable(true), "{name}");
        }
        for (instruction, name) in never {
            assert!(!instruction.continuable(true), "{name}");
        }
    }

    /// A core without the DSP extension takes its instructions as UNDEFINED, A5.3.
    #[test]
    fn a_core_without_the_dsp_extension_refuses_its_instructions() {
        use super::super::registers::cfsr;
        use super::testing::Bench;
        use super::testing::spec_of;
        use crate::machine::armv7m::{Fpu, MpuRegions, Variant};
        let undefined = mask(cfsr::UNDEFINSTR);
        for (model, status, sum) in [(Variant::M3, undefined, 0), (Variant::M4, 0, 0x0003_0005)] {
            let sadd16 = [0xFA91, 0xF002];
            let mut bench = Bench::of(&spec_of(model, Fpu::None, MpuRegions::Eight), &sadd16);
            bench.core.state.r[1] = 0x0001_0002;
            bench.core.state.r[2] = 0x0002_0003;
            bench.run(1);
            let core = &bench.core;
            assert_eq!(
                (core.system.cfsr & undefined, core.state.r[0]),
                (status, sum),
                "{}",
                model.name()
            );
        }
    }

    /// A load or store leaves through its fault exit only where it faults, so the instruction and
    /// the block go on after it.
    #[test]
    fn an_access_goes_on_after_its_fault_exit() {
        let load = lowered(|e| {
            let address = e.constant(0x2000_0000);
            e.load(address, Size::Word, Attributes::default());
        });
        let store = lowered(|e| {
            let address = e.constant(0x2000_0000);
            e.store(address, address, Size::Word, Attributes::default());
        });
        assert_eq!((load, store), ((false, 1), (false, 1)));
    }

    #[test]
    fn raise_if_goes_on_where_it_does_not_raise() {
        let raised = lowered(|e| {
            let holds = e.b.ins().iconst(types::I8, 1);
            e.raise_if(holds, Fault::Unaligned);
        });
        assert_eq!(raised, (false, 1));
    }

    /// B1.5.15: an SVC that locks up is UNDEFINED and leaves unretired, which is a way out of its
    /// own beside the one past it.
    #[test]
    fn svc_has_a_way_out_for_locking_up() {
        assert_eq!(lowered(|e| e.raise(Fault::SupervisorCall)), (true, 2));
        assert_eq!(lowered(|e| e.raise(Fault::Undefined)), (true, 1));
    }

    /// A7.7.192: UDIV raises DIVBYZERO only where `CCR.DIV_0_TRP` traps a zero divisor, so the
    /// block goes on after it.
    #[test]
    fn a_divide_goes_on_in_its_block() {
        let divided = lowered(|e| assert_eq!(decode32(e, 0xFBB1, 0xF0F2), Ok(())));
        assert_eq!(divided, (false, 1));
    }
}