rigless 0.2.0

// Copyright (c) 2026 Rigless Labs.
// SPDX-License-Identifier: MIT OR Apache-2.0
//! The parts of a translating core that know no instruction set: the page table its code reads
//! memory through, the cache of blocks it translated, and the code generator that makes them.
//!
//! A family's core translates a block of guest instructions to one host function with
//! [`Jit::compile`], and runs blocks until its budget is spent. A load or store looks its page up
//! in [`Pages`] inline, and reaches the host's copy of the guest's memory directly when the page is
//! plain memory. Anything else goes through a helper to the family, which answers its own space and
//! hands the rest to the [`CoreBus`](super::CoreBus), and the block ends after that instruction so the
//! runner sees every such access between two instructions.
#![expect(
    unsafe_code,
    reason = "generated code reads the page table and calls back into Rust through raw pointers"
)]

use std::cell::Cell;
use std::collections::{BTreeMap, BTreeSet};
use std::ops::RangeInclusive;

use cranelift_codegen::ir::{
    self, AbiParam, Endianness, InstBuilder, MemFlagsData, Signature, Value, types,
};
use cranelift_codegen::isa::CallConv;
use cranelift_codegen::settings::{self, Configurable};
use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_jit::{BranchProtection, JITBuilder, JITMemoryKind, JITMemoryProvider, JITModule};
use cranelift_module::{Linkage, Module, ModuleError};

use super::{DataModel, Error};
use crate::soc::Endian;

/// The bytes of the guest addresses the page table covers, every one a 32-bit address.
// ceiling: one flat table over a 32-bit address space, which `emit_lookup` indexes with the address
// zero-extended; upgrade to a two-level table sized by the family's address width when a 64-bit
// family runs on the core.
pub(crate) const ADDRESSES: u64 = 1 << 32;

/// The bits of an address inside one of the code generator's pages.
pub(crate) const PAGE_BITS: u32 = 10;
/// The bytes in one of the code generator's pages, the least a family on it maps or routes.
pub(crate) const PAGE_SIZE: u64 = 1 << PAGE_BITS;

/// A page table entry's bit for a page the code reads directly.
const READ: usize = 1;
/// A page table entry's bit for a page the code writes directly.
const WRITE: usize = 2;
/// A page table entry's bit for a page whose bytes, through it or another view of them, hold
/// translated code. Generated code reads none of the bits from here on.
const CODE: usize = 4;
/// A page table entry's bit for a page that's plain memory, whose entry holds its host address
/// whether or not anything of it is inline.
const MAPPED: usize = 8;
/// A page table entry's bit for plain memory the guest may write.
const WRITABLE: usize = 16;
/// A page table entry's bit for a page routed to the bus.
const ROUTED: usize = 32;

/// Where the guest's memory is in the host's, a page at a time, for the code to read inline.
///
/// An entry is the host address of the page's first byte with [`READ`] and [`WRITE`] in its low
/// bits. A page that's routed, or that isn't plain memory, has neither. A page whose bytes hold
/// translated code, through it or through any other guest page mapping the same host page, has no
/// [`WRITE`], so a store to those bytes through any view reaches the family, which drops the code
/// before it writes. While every access is checked, a page is inline only as far as the family
/// allowed it. Translated code reads the entries while this writes them, so each is a cell.
pub(crate) struct Pages {
    entries: Box<[Cell<usize>]>,
    /// Every page mapped: its host address and whether the guest may write it.
    mapped: BTreeMap<u64, (usize, bool)>,
    /// Every guest page mapping each host page, by the host page's address.
    views: BTreeMap<usize, Vec<u64>>,
    routed: BTreeSet<u64>,
    code: BTreeSet<u64>,
    /// Every access is checked by the family, so none is inline unless `allowed` says so.
    checked: bool,
    /// What the family let be inline of each page while every access is checked.
    allowed: BTreeMap<u64, Inline>,
}

/// The accesses of a page that may be inline.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) enum Inline {
    #[default]
    Neither,
    Reads,
    Both,
}

impl std::fmt::Debug for Pages {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter
            .debug_struct("Pages")
            .field("mapped", &self.mapped.len())
            .field("routed", &self.routed)
            .field("code", &self.code.len())
            .finish_non_exhaustive()
    }
}

impl Pages {
    /// A table for a 32-bit address space, the one [`emit_lookup`] indexes, with nothing mapped,
    /// for a family whose data is laid out as `model` says. One whose pointers are wider than the
    /// table's addresses is refused.
    pub(crate) fn new(model: DataModel) -> Result<Self, Error> {
        if u64::from(model.pointer().bits()) > u64::from(ADDRESSES.trailing_zeros()) {
            return Err(Error::Machine(format!(
                "the core's page table holds {}-bit addresses, and a family with {}-bit pointers needs a wider one",
                ADDRESSES.trailing_zeros(),
                model.pointer().bits()
            )));
        }
        let pages = usize::try_from(ADDRESSES >> PAGE_BITS).expect("a table the host can hold");
        let zeros = vec![0_usize; pages].into_boxed_slice();
        Ok(Self {
            // SAFETY: `Cell<usize>` is `repr(transparent)` over `usize`. The zeroed allocation
            // stays untouched until a page is mapped, where a table of cells built one by one
            // would write all of it.
            entries: unsafe { Box::from_raw(Box::into_raw(zeros) as *mut [Cell<usize>]) },
            mapped: BTreeMap::new(),
            views: BTreeMap::new(),
            routed: BTreeSet::new(),
            code: BTreeSet::new(),
            checked: false,
            allowed: BTreeMap::new(),
        })
    }

    /// Whether the `length` bytes from `start` are all addresses the table covers, so a family
    /// refuses memory anywhere else before it maps it.
    pub(crate) fn holds(start: u64, length: u64) -> bool {
        start
            .checked_add(length)
            .is_some_and(|end| end <= ADDRESSES)
    }

    /// The table, for generated code to index by page number.
    pub(crate) fn table(&self) -> *const usize {
        self.entries.as_ptr().cast()
    }

    /// Maps `length` bytes at `start` to the host's bytes at `host`. Both are whole pages.
    ///
    /// # Safety
    ///
    /// As [`Processor::map`](super::Processor::map).
    pub(crate) unsafe fn map(&mut self, start: u64, length: u64, host: *mut u8, writable: bool) {
        assert!(
            start.is_multiple_of(PAGE_SIZE) && length.is_multiple_of(PAGE_SIZE),
            "whole pages"
        );
        assert!(Self::holds(start, length), "addresses the table covers");
        assert!(
            (host as usize).is_multiple_of(usize::try_from(PAGE_SIZE).expect("a page")),
            "a page-aligned host page"
        );
        for offset in (0..length).step_by(usize::try_from(PAGE_SIZE).expect("a page")) {
            let page = (start + offset) >> PAGE_BITS;
            let at = host as usize + usize::try_from(offset).expect("inside a region in memory");
            if let Some((before, _)) = self.mapped.insert(page, (at, writable))
                && let Some(views) = self.views.get_mut(&before)
            {
                views.retain(|&view| view != page);
            }
            self.views.entry(at).or_default().push(page);
            self.refresh_views(page);
        }
    }

    /// Every guest range the bytes in `start..end` answer at: each mapped page's part of them at
    /// every guest page mapping the same host page, and the rest where they are.
    pub(crate) fn views(&self, start: u64, end: u64) -> Vec<(u64, u64)> {
        let mut ranges = Vec::new();
        let mut from = start;
        while from < end {
            let page = from >> PAGE_BITS;
            let to = end.min((page + 1) << PAGE_BITS);
            match self.mapped.get(&page) {
                Some((at, _)) => {
                    for &view in &self.views[at] {
                        let moved = view << PAGE_BITS | from & (PAGE_SIZE - 1);
                        ranges.push((moved, moved + (to - from)));
                    }
                }
                None => ranges.push((from, to)),
            }
            from = to;
        }
        ranges
    }

    /// Whether code was translated from any view of the bytes in `start..end`.
    pub(crate) fn touches_code(&self, start: u64, end: u64) -> bool {
        if self.code.is_empty() || end <= start {
            return false;
        }
        (start >> PAGE_BITS..=(end - 1) >> PAGE_BITS).any(|page| {
            match usize::try_from(page)
                .ok()
                .and_then(|page| self.entries.get(page))
                .map(Cell::get)
            {
                // A page that isn't mapped has code only the set knows of.
                Some(entry) if entry & MAPPED != 0 => entry & CODE != 0,
                _ => self.code.contains(&page),
            }
        })
    }

    /// The host address of the guest's byte at `address`, and whether the guest may write it,
    /// when it's mapped.
    pub(crate) fn host(&self, address: u64) -> Option<(*mut u8, bool)> {
        let entry = self.entry(address);
        if entry & MAPPED == 0 {
            return None;
        }
        let at = entry & !usize::try_from(PAGE_SIZE - 1).expect("a page");
        let offset = usize::try_from(address & (PAGE_SIZE - 1)).expect("inside a page");
        Some(((at + offset) as *mut u8, entry & WRITABLE != 0))
    }

    /// The entry of the page holding `address`, or 0 past the table.
    fn entry(&self, address: u64) -> usize {
        usize::try_from(address >> PAGE_BITS)
            .ok()
            .and_then(|page| self.entries.get(page))
            .map_or(0, Cell::get)
    }

    /// Whether the page holding `address` is routed to the bus.
    pub(crate) fn routed(&self, address: u64) -> bool {
        let entry = self.entry(address);
        if entry & MAPPED != 0 {
            return entry & ROUTED != 0;
        }
        self.routed.contains(&(address >> PAGE_BITS))
    }

    /// Sends every data access to the page holding `address` to the bus, or stops.
    pub(crate) fn route(&mut self, address: u64, routed: bool) {
        let page = address >> PAGE_BITS;
        if routed {
            self.routed.insert(page);
        } else {
            self.routed.remove(&page);
        }
        self.refresh(page);
    }

    /// Notes that the page holding `address` has translated code, so a store to its bytes isn't
    /// inline through any view of them.
    pub(crate) fn hold_code(&mut self, address: u64) {
        let page = address >> PAGE_BITS;
        if self.code.insert(page) {
            self.refresh_views(page);
        }
    }

    /// Notes that the page holding `address` has no translated code left.
    pub(crate) fn release_code(&mut self, address: u64) {
        let page = address >> PAGE_BITS;
        if self.code.remove(&page) {
            self.refresh_views(page);
        }
    }

    /// Notes that no page has translated code left.
    pub(crate) fn release_all_code(&mut self) {
        for page in std::mem::take(&mut self.code) {
            self.refresh_views(page);
        }
    }

    /// Sends every access to the family, which checks it, for as long as `checked`, or lets plain
    /// memory be inline again. Either way, what the family let be inline is forgotten.
    pub(crate) fn check_all(&mut self, checked: bool) {
        if self.checked != checked || !self.allowed.is_empty() {
            self.checked = checked;
            self.allowed.clear();
            let pages: Vec<u64> = self.mapped.keys().copied().collect();
            for page in pages {
                self.refresh(page);
            }
        }
    }

    /// Whether the family has said what's inline of the page holding `address` since every access
    /// was last checked.
    pub(crate) fn decided(&self, address: u64) -> bool {
        self.allowed.contains_key(&(address >> PAGE_BITS))
    }

    /// Lets `inline` of the page holding `address` be inline while every access is checked,
    /// until [`Pages::check_all`] forgets it.
    pub(crate) fn allow(&mut self, address: u64, inline: Inline) {
        let page = address >> PAGE_BITS;
        self.allowed.insert(page, inline);
        self.refresh(page);
    }

    /// The accesses of the page holding `address` that are inline.
    #[cfg(test)]
    pub(crate) fn inline(&self, address: u64) -> Inline {
        let entry = self.entries[usize::try_from(address >> PAGE_BITS).expect("a page")].get();
        match (entry & READ != 0, entry & WRITE != 0) {
            (false, _) => Inline::Neither,
            (true, false) => Inline::Reads,
            (true, true) => Inline::Both,
        }
    }

    /// Refreshes every page mapping the same host page as `page`, since its code decides theirs.
    fn refresh_views(&mut self, page: u64) {
        let Some(&(at, _)) = self.mapped.get(&page) else {
            return;
        };
        for view in self.views[&at].clone() {
            self.refresh(view);
        }
    }

    fn refresh(&mut self, page: u64) {
        let entry = match self.mapped.get(&page) {
            Some(&(at, writable)) => {
                let inline = if self.routed.contains(&page) {
                    Inline::Neither
                } else if self.checked {
                    self.allowed.get(&page).copied().unwrap_or_default()
                } else {
                    Inline::Both
                };
                let code = self.views[&at].iter().any(|view| self.code.contains(view));
                let mut entry = match inline {
                    Inline::Neither => at,
                    Inline::Both if writable && !code => at | READ | WRITE,
                    Inline::Reads | Inline::Both => at | READ,
                } | MAPPED;
                for (holds, bit) in [
                    (code, CODE),
                    (writable, WRITABLE),
                    (self.routed.contains(&page), ROUTED),
                ] {
                    if holds {
                        entry |= bit;
                    }
                }
                entry
            }
            None => 0,
        };
        if let Some(slot) = usize::try_from(page)
            .ok()
            .and_then(|page| self.entries.get(page))
        {
            slot.set(entry);
        }
    }
}

/// The flags of every load and store of the guest's memory in the host's: never trapping, since
/// the page table has checked it, and little-endian, however the host keeps its own. The code
/// generator has no big-endian access on the hosts it runs on, so [`load_guest`] and
/// [`store_guest`] swap for a family that keeps its data big-endian.
pub(crate) fn guest_memory() -> MemFlagsData {
    MemFlagsData::new()
        .with_notrap()
        .with_endianness(Endianness::Little)
}

/// Emits a load of a `kind` of the guest's memory at `host`, in the order `endian` the family
/// keeps its data in.
pub(crate) fn load_guest(
    builder: &mut FunctionBuilder<'_>,
    kind: ir::Type,
    host: Value,
    endian: Endian,
) -> Value {
    let value = builder.ins().load(kind, guest_memory(), host, 0);
    in_order(builder, value, endian)
}

/// Emits a store of `value` to the guest's memory at `host`, in the order `endian` the family
/// keeps its data in.
pub(crate) fn store_guest(
    builder: &mut FunctionBuilder<'_>,
    value: Value,
    host: Value,
    endian: Endian,
) {
    let value = in_order(builder, value, endian);
    builder.ins().store(guest_memory(), value, host, 0);
}

/// `value` as little-endian memory holds it, in `endian`. Swapping is its own inverse, so a store
/// takes the same.
fn in_order(builder: &mut FunctionBuilder<'_>, value: Value, endian: Endian) -> Value {
    if endian == Endian::Big && builder.func.dfg.value_type(value).bytes() > 1 {
        builder.ins().bswap(value)
    } else {
        value
    }
}

/// The access a page table lookup is for, a read or a write of this many bytes.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Lookup {
    Read(u32),
    Write(u32),
}

/// Emits the page table lookup for `lookup` at the 32-bit guest `address`, reading the table from
/// `table`. It branches to `slow` with nothing when the access can't be inline, because its page
/// isn't plain memory or it isn't aligned, and otherwise answers the host address it's at, in a
/// block the builder is left in.
pub(crate) fn emit_lookup(
    builder: &mut FunctionBuilder<'_>,
    table: Value,
    address: Value,
    lookup: Lookup,
    slow: ir::Block,
) -> Value {
    // Every host the code generator runs on is 64-bit, which `Jit::new` checks.
    #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
    const _: () = assert!(size_of::<usize>() == 8, "a table entry is eight bytes");
    let (bytes, need) = match lookup {
        Lookup::Read(bytes) => (bytes, READ),
        Lookup::Write(bytes) => (bytes, WRITE),
    };
    let wide = builder.ins().uextend(types::I64, address);
    let page = builder.ins().ushr_imm_u(wide, i64::from(PAGE_BITS));
    let slot = builder.ins().ishl_imm_s(page, 3);
    let slot = builder.ins().iadd(table, slot);
    let entry = builder
        .ins()
        .load(types::I64, MemFlagsData::trusted(), slot, 0);
    let allowed = builder
        .ins()
        .band_imm_s(entry, i64::try_from(need).expect("a flag"));
    // `WRITE` isn't bit 0, so it's made 0 or 1 to meet `aligned` below.
    let allowed = builder
        .ins()
        .icmp_imm_s(ir::condcodes::IntCC::NotEqual, allowed, 0);
    let allowed = builder.ins().uextend(types::I64, allowed);
    let offset = builder
        .ins()
        .band_imm_s(wide, i64::try_from(PAGE_SIZE - 1).expect("a page"));
    // An aligned access never crosses a page, and an unaligned one takes the slow path, which
    // checks whether it may be unaligned at all.
    let misaligned = builder.ins().band_imm_s(wide, i64::from(bytes) - 1);
    let aligned = builder
        .ins()
        .icmp_imm_s(ir::condcodes::IntCC::Equal, misaligned, 0);
    let aligned = builder.ins().uextend(types::I64, aligned);
    let fast = builder.ins().band(allowed, aligned);
    let inline = builder.create_block();
    builder.ins().brif(fast, inline, &[], slow, &[]);
    builder.seal_block(inline);
    builder.switch_to_block(inline);
    let base = builder
        .ins()
        .band_imm_s(entry, !i64::try_from(PAGE_SIZE - 1).expect("a page"));
    builder.ins().iadd(base, offset)
}

/// A translated block's code. It runs through [`Jit::enter`], with the family's core, and answers
/// why it ended, in the family's own numbers. Blocks use the code generator's tail-call
/// convention, so one block can hand over to the next with no return in between.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(transparent)]
pub(crate) struct Block(*const u8);

impl Block {
    /// The code's address, for another block to tail-call.
    pub(crate) fn address(self) -> usize {
        self.0 as usize
    }
}

/// The host's own calling convention, the one Rust's `extern "C"` functions use, which a block
/// calls a helper in.
pub(crate) fn host_convention() -> CallConv {
    if cfg!(all(windows, target_arch = "x86_64")) {
        CallConv::WindowsFastcall
    } else if cfg!(all(target_vendor = "apple", target_arch = "aarch64")) {
        CallConv::AppleAarch64
    } else {
        CallConv::SystemV
    }
}

/// The host function that runs a block: the core and the block's code in, its answer out.
type Enter = unsafe extern "C" fn(*mut u8, *const u8) -> u32;

/// The address space a code generator reserves for its code. Only what's used is ever backed.
const CODE_SPACE: usize = 1 << 30;

/// Code memory packed tight in one reserved region, each block straight after the last, so the
/// blocks a firmware runs share pages rather than taking one each. The pages a block is emitted
/// into are writable while it is, and executable once it's done, never both, and since a core
/// translates only between blocks, nothing runs from a page while it's writable.
struct Packed {
    region: region::Allocation,
    /// The bytes handed out so far.
    used: usize,
    /// The bytes handed out since the last block was finished.
    pending: Option<std::ops::Range<usize>>,
}

// SAFETY: the region is this provider's own, and the code generator that owns the provider
// moves with the core it belongs to.
unsafe impl Send for Packed {}

impl Packed {
    fn new() -> Result<Self, region::Error> {
        Ok(Self {
            region: region::alloc(CODE_SPACE, region::Protection::NONE)?,
            used: 0,
            pending: None,
        })
    }

    /// The pages holding the bytes in `range`, as a start and a length.
    fn pages(&self, range: &std::ops::Range<usize>) -> (*const u8, usize) {
        let page = region::page::size();
        let start = range.start / page * page;
        let end = range.end.div_ceil(page) * page;
        (self.region.as_ptr::<u8>().wrapping_add(start), end - start)
    }
}

impl JITMemoryProvider for Packed {
    fn allocate(
        &mut self,
        size: usize,
        align: u64,
        kind: JITMemoryKind,
    ) -> std::io::Result<*mut u8> {
        // Every page here ends up executable and never writable, which fits code and nothing else.
        if !matches!(kind, JITMemoryKind::Executable) {
            return Err(std::io::Error::other(
                "the core's code space holds only code",
            ));
        }
        let align = usize::try_from(align).map_err(std::io::Error::other)?;
        let start = self.used.next_multiple_of(align.max(1));
        let end = start + size;
        if end > CODE_SPACE {
            return Err(std::io::Error::new(
                std::io::ErrorKind::OutOfMemory,
                "the core's code space is full",
            ));
        }
        let pending = self.pending.take().unwrap_or(start..start);
        let pending = pending.start.min(start)..end;
        let (pages, length) = self.pages(&pending);
        // SAFETY: inside the region. A core translates only from its own loop between blocks,
        // never from code a block calls, so nothing runs from these pages until `finalize` makes
        // them executable again. A translation that fails is followed by clearing every block, or
        // by the end of the run.
        unsafe { region::protect(pages, length, region::Protection::READ_WRITE) }
            .map_err(std::io::Error::other)?;
        self.pending = Some(pending);
        self.used = end;
        Ok(self.region.as_mut_ptr::<u8>().wrapping_add(start))
    }

    fn finalize(&mut self, _protection: BranchProtection) -> cranelift_module::ModuleResult<()> {
        let Some(pending) = self.pending.take() else {
            return Ok(());
        };
        let (pages, length) = self.pages(&pending);
        let backend = |error: &dyn std::fmt::Display| ModuleError::Allocation {
            err: std::io::Error::other(error.to_string()),
        };
        // SAFETY: the bytes just written, inside the region.
        unsafe { wasmtime_internal_jit_icache_coherence::clear_cache(pages.cast(), length) }
            .map_err(|error| backend(&error))?;
        // SAFETY: as above.
        unsafe { region::protect(pages, length, region::Protection::READ_EXECUTE) }
            .map_err(|error| backend(&error))
    }

    unsafe fn free_memory(&mut self) {
        self.used = 0;
        self.pending = None;
        // SAFETY: the caller's promise that no code from here runs again.
        unsafe {
            region::protect(
                self.region.as_ptr::<u8>(),
                CODE_SPACE,
                region::Protection::NONE,
            )
        }
        .ok();
    }
}

/// How hard the code generator works on a block.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Tier {
    /// Quick to make, for a block's first translation, since most blocks run only a few times.
    Quick,
    /// Better code, for a block that turned out to run often.
    Best,
}

/// How the code generator makes Cranelift's `fma` on this host, from [`Jit::fma`].
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Fma {
    /// One instruction of the host's.
    Instruction,
    /// A call to the C library's `fmaf`, which isn't the same on every host.
    Library,
}

/// What runs a block, from [`Jit::entry`].
#[derive(Clone, Copy)]
pub(crate) struct Entry(Enter);

impl Entry {
    /// Runs `block` on `core` and answers what it does.
    ///
    /// # Safety
    ///
    /// `block` was made by the code generator this came from, and this was taken from it, both
    /// since it was last cleared. `block` was made for the family whose core `core` is, and
    /// nothing else touches the core until it returns.
    pub(crate) unsafe fn run(self, core: *mut u8, block: Block) -> u32 {
        // SAFETY: the caller's promise, and the entry calls the block with exactly that.
        unsafe { (self.0)(core, block.0) }
    }
}

/// The code generator, and the host memory every block it made lives in.
pub(crate) struct Jit {
    /// The code generator for hot blocks, which takes longer and makes better code.
    module: JITModule,
    /// The code generator for a block's first translation, quick to make and slower to run.
    quick: JITModule,
    context: cranelift_codegen::Context,
    builder: FunctionBuilderContext,
    /// Every block's signature: the core in, the answer out, in the tail-call convention.
    signature: Signature,
    enter: Enter,
    made: u64,
    /// The last [`Jit::compile`] failed for want of code space.
    out_of_room: bool,
}

impl std::fmt::Debug for Jit {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter
            .debug_struct("Jit")
            .field("made", &self.made)
            .finish_non_exhaustive()
    }
}

fn failed(error: &dyn std::fmt::Display) -> Error {
    Error::Machine(format!("the core couldn't translate a block: {error}"))
}

impl Jit {
    /// A code generator for this host.
    pub(crate) fn new() -> Result<Self, Error> {
        // ceiling: x86-64 and AArch64 hosts, whose conventions leave the bits above a 32-bit
        // argument unread, so no helper's signature widens one; upgrade when another host is tried.
        if !cfg!(any(target_arch = "x86_64", target_arch = "aarch64")) {
            return Err(Error::Machine(
                "the core generates code only on an x86-64 or AArch64 host".to_owned(),
            ));
        }
        let mut module = Self::module(Tier::Best)?;
        let quick = Self::module(Tier::Quick)?;
        let pointer = module.isa().pointer_type();
        let mut signature = Signature::new(CallConv::Tail);
        signature.params.push(AbiParam::new(pointer));
        signature.returns.push(AbiParam::new(types::I32));
        let mut context = module.make_context();
        let mut builder = FunctionBuilderContext::new();
        let enter = Self::make_enter(&mut module, &mut context, &mut builder, &signature)?;
        Ok(Self {
            context,
            module,
            quick,
            builder,
            signature,
            enter,
            made: 0,
            out_of_room: false,
        })
    }

    fn module(tier: Tier) -> Result<JITModule, Error> {
        let unsupported = |error: &dyn std::fmt::Display| {
            Error::Machine(format!(
                "the core can't generate code for this host: {error}"
            ))
        };
        // A hot block gains from a better register allocator and not from the optimizer, whose
        // passes cost a third of compiling one and find little in what a family builds.
        let (level, allocator) = match tier {
            Tier::Quick => ("none", "single_pass"),
            Tier::Best => ("none", "backtracking"),
        };
        let mut flags = settings::builder();
        flags
            .set("opt_level", level)
            .map_err(|error| unsupported(&error))?;
        flags
            .set("regalloc_algorithm", allocator)
            .map_err(|error| unsupported(&error))?;
        // The verifier checks every block's IR against Cranelift's own rules and costs as much as
        // compiling. A debug build pays for it, this crate's tests and a dependent's debug build
        // alike, and a release doesn't.
        flags
            .set(
                "enable_verifier",
                if cfg!(debug_assertions) {
                    "true"
                } else {
                    "false"
                },
            )
            .map_err(|error| unsupported(&error))?;
        // A block hands over to the next by a tail call, which the code generator makes only with
        // frame pointers kept.
        flags
            .set("preserve_frame_pointers", "true")
            .map_err(|error| unsupported(&error))?;
        // The core's state stays in one register while blocks run, which helpers in the host's
        // convention keep as they found it.
        flags
            .set("enable_pinned_reg", "true")
            .map_err(|error| unsupported(&error))?;
        let isa = cranelift_native::builder()
            .map_err(|error| unsupported(&error))?
            .finish(settings::Flags::new(flags))
            .map_err(|error| unsupported(&error))?;
        let mut builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
        builder.memory_provider(Box::new(
            Packed::new().map_err(|error| unsupported(&error))?,
        ));
        Ok(JITModule::new(builder))
    }

    /// The host function that calls a block in the tail-call convention, which Rust can't.
    #[expect(
        clippy::disallowed_methods,
        reason = "a function pointer isn't plain old data, and only transmute makes one from code"
    )]
    fn make_enter(
        module: &mut JITModule,
        context: &mut cranelift_codegen::Context,
        builder: &mut FunctionBuilderContext,
        block: &Signature,
    ) -> Result<Enter, Error> {
        let pointer = module.isa().pointer_type();
        let mut signature = module.make_signature();
        signature.params.push(AbiParam::new(pointer));
        signature.params.push(AbiParam::new(pointer));
        signature.returns.push(AbiParam::new(types::I32));
        module.clear_context(context);
        context.func.signature = signature.clone();
        let config = module.target_config();
        {
            let mut builder = FunctionBuilder::new(&mut context.func, builder);
            let entry = builder.create_block();
            builder.append_block_params_for_function_params(entry);
            builder.switch_to_block(entry);
            builder.seal_block(entry);
            let (core, code) = (
                builder.block_params(entry)[0],
                builder.block_params(entry)[1],
            );
            let block = builder.import_signature(block.clone());
            // The pinned register is the host caller's to keep, so it goes back as it was.
            let kept = builder.ins().get_pinned_reg(pointer);
            builder.ins().set_pinned_reg(core);
            let call = builder.ins().call_indirect(block, code, &[core]);
            let answer = builder.inst_results(call)[0];
            builder.ins().set_pinned_reg(kept);
            builder.ins().return_(&[answer]);
            builder.finalize(config);
        }
        let id = module
            .declare_function("enter", Linkage::Local, &signature)
            .map_err(|error| failed(&error))?;
        module
            .define_function(id, context)
            .map_err(|error| failed(&error))?;
        module
            .finalize_definitions()
            .map_err(|error| failed(&error))?;
        let code = module.get_finalized_function(id);
        // SAFETY: the function just made, with `Enter`'s signature in the host's convention.
        Ok(unsafe { std::mem::transmute::<*const u8, Enter>(code) })
    }

    /// The host's pointer type, which every address in the host is.
    pub(crate) fn pointer(&self) -> ir::Type {
        self.module.isa().pointer_type()
    }

    /// How both tiers make Cranelift's `fma`.
    pub(crate) fn fma(&self) -> Fma {
        if self.module.isa().has_native_fma() {
            Fma::Instruction
        } else {
            Fma::Library
        }
    }

    /// Makes one block. `build` gets a builder in the entry block, already holding the core's
    /// pointer as its parameter, and the signature every block has, for a tail call to the next.
    /// It ends every path with a return or a tail call.
    pub(crate) fn compile(
        &mut self,
        tier: Tier,
        build: impl FnOnce(&mut FunctionBuilder<'_>, Value, &Signature),
    ) -> Result<Block, Error> {
        self.out_of_room = false;
        let module = match tier {
            Tier::Quick => &mut self.quick,
            Tier::Best => &mut self.module,
        };
        module.clear_context(&mut self.context);
        self.context.func.signature = self.signature.clone();
        let config = module.target_config();
        {
            let mut builder = FunctionBuilder::new(&mut self.context.func, &mut self.builder);
            let entry = builder.create_block();
            builder.append_block_params_for_function_params(entry);
            builder.switch_to_block(entry);
            builder.seal_block(entry);
            let core = builder.block_params(entry)[0];
            build(&mut builder, core, &self.signature);
            builder.seal_all_blocks();
            builder.finalize(config);
        }
        self.made += 1;
        let id = module
            .declare_function(&format!("b{}", self.made), Linkage::Local, &self.signature)
            .map_err(|error| failed(&error))?;
        module
            .define_function(id, &mut self.context)
            .map_err(|error| {
                self.out_of_room = matches!(&error, ModuleError::Allocation { err }
                    if err.kind() == std::io::ErrorKind::OutOfMemory);
                failed(&error)
            })?;
        module
            .finalize_definitions()
            .map_err(|error| failed(&error))?;
        Ok(Block(module.get_finalized_function(id)))
    }

    /// What runs a block this made, for as long as it isn't cleared.
    pub(crate) fn entry(&self) -> Entry {
        Entry(self.enter)
    }

    /// How many blocks this has made since it was last cleared.
    pub(crate) fn made(&self) -> u64 {
        self.made
    }

    /// Whether the last [`Jit::compile`] failed for want of code space, which [`Jit::clear`]
    /// makes.
    pub(crate) fn out_of_room(&self) -> bool {
        self.out_of_room
    }

    /// Frees every block this made.
    ///
    /// # Safety
    ///
    /// None of them is running, and none is called again.
    pub(crate) unsafe fn clear(&mut self) -> Result<(), Error> {
        let mut module = Self::module(Tier::Best)?;
        let quick = std::mem::replace(&mut self.quick, Self::module(Tier::Quick)?);
        // A panic in the middle of a translation leaves the builder holding half a function.
        self.builder = FunctionBuilderContext::new();
        // SAFETY: the caller's promise.
        unsafe { quick.free_memory() };
        self.enter = Self::make_enter(
            &mut module,
            &mut self.context,
            &mut self.builder,
            &self.signature,
        )?;
        let old = std::mem::replace(&mut self.module, module);
        // SAFETY: the caller's promise.
        unsafe { old.free_memory() };
        self.made = 0;
        Ok(())
    }
}

/// The slots of [`Blocks`]'s cache of recent lookups, a power of two.
const RECENT: usize = 1 << 12;

/// One slot of [`Blocks`]'s cache of recent lookups, laid out for translated code to read: a
/// key, and the code of its block, which is null where the slot is empty.
#[derive(Clone, Copy, Debug)]
#[repr(C)]
struct Recent {
    key: u64,
    block: Block,
}

const EMPTY: Recent = Recent {
    key: 0,
    block: Block(std::ptr::null()),
};

/// The link cells in each of [`Blocks`]'s chunks of them.
const CELLS: usize = 1 << 12;

/// Every block a core translated, by the key its family gives them, and the pages each came from.
///
/// A block's exit to a next block it knows can tail-call that block, through a link cell: a word
/// the exit reads, holding the next block's code or 0. The family links a cell once the block it
/// leads to is translated, and a cell goes back to 0 when that block is dropped.
#[derive(Debug)]
pub(crate) struct Blocks {
    /// Every block, with the first and last byte it was translated from.
    by_key: BTreeMap<u64, (Block, u64, u64)>,
    /// Every block on each page, with the first and last byte it was translated from.
    by_page: BTreeMap<u64, Vec<(u64, u64, u64)>>,
    /// A direct-mapped cache in front of `by_key`, since a core looks a block up before every one
    /// it runs.
    /// Translated code reads it too, so it never moves, and each slot is a cell.
    recent: Box<[Cell<Recent>]>,
    /// The link cells, in chunks that never move, since translated code holds their addresses
    /// and counts its runs in some.
    cells: Vec<Box<[Cell<usize>]>>,
    /// The key of the block each cell may lead to, by cell.
    leads_to: Vec<u64>,
    /// The cells linked to each block, by its key.
    linked: BTreeMap<u64, Vec<u32>>,
}

impl Default for Blocks {
    fn default() -> Self {
        Self {
            by_key: BTreeMap::new(),
            by_page: BTreeMap::new(),
            recent: vec![Cell::new(EMPTY); RECENT].into_boxed_slice(),
            cells: Vec::new(),
            leads_to: Vec::new(),
            linked: BTreeMap::new(),
        }
    }
}

/// Where `key` goes in the cache of recent lookups. Keys differ mostly in their low bits, the
/// address, whose bit 0 is always clear.
/// [`emit_find`] works it out the same way.
fn slot(key: u64) -> usize {
    let mixed = (key >> 1) ^ (key >> 32);
    usize::try_from(mixed & (RECENT as u64 - 1)).expect("below the cache's size")
}

/// Emits a lookup of the block keyed `key`, an `i64`, in the cache of recent lookups at
/// `recent`, from [`Blocks::recent`]: its code, or 0 where the cache doesn't hold it.
pub(crate) fn emit_find(
    builder: &mut FunctionBuilder<'_>,
    pointer: ir::Type,
    recent: usize,
    key: Value,
) -> Value {
    let low = builder.ins().ushr_imm_u(key, 1);
    let high = builder.ins().ushr_imm_u(key, 32);
    let mixed = builder.ins().bxor(low, high);
    let slot = builder
        .ins()
        .band_imm_s(mixed, i64::try_from(RECENT - 1).expect("a mask"));
    let offset = builder
        .ins()
        .imul_imm_u(slot, i64::try_from(size_of::<Recent>()).expect("a size"));
    let base = builder
        .ins()
        .iconst(pointer, i64::try_from(recent).expect("an address"));
    let entry = builder.ins().iadd(base, offset);
    let flags = MemFlagsData::trusted();
    let cached = builder.ins().load(
        types::I64,
        flags,
        entry,
        i32::try_from(std::mem::offset_of!(Recent, key)).expect("an offset"),
    );
    let code = builder.ins().load(
        pointer,
        flags,
        entry,
        i32::try_from(std::mem::offset_of!(Recent, block)).expect("an offset"),
    );
    let hit = builder.ins().icmp(ir::condcodes::IntCC::Equal, cached, key);
    let none = builder.ins().iconst(pointer, 0);
    builder.ins().select(hit, code, none)
}

impl Blocks {
    pub(crate) fn get(&mut self, key: u64) -> Option<Block> {
        let slot = slot(key);
        let cached = self.recent[slot].get();
        if cached.key == key && !cached.block.0.is_null() {
            return Some(cached.block);
        }
        let (block, _, _) = *self.by_key.get(&key)?;
        self.recent[slot].set(Recent { key, block });
        Some(block)
    }

    /// Where the cache of recent lookups is, for [`emit_find`].
    pub(crate) fn recent(&self) -> usize {
        self.recent.as_ptr() as usize
    }

    /// Whether a block was translated from the byte at `address`.
    pub(crate) fn covers(&self, address: u64) -> bool {
        self.by_page
            .get(&(address >> PAGE_BITS))
            .is_some_and(|held| {
                held.iter()
                    .any(|&(_, first, last)| (first..=last).contains(&address))
            })
    }

    /// The key of every block translated from bytes starting at `address`, with the last byte it
    /// came from.
    pub(crate) fn starting_at(&self, address: u64) -> Vec<(u64, u64)> {
        self.by_page
            .get(&(address >> PAGE_BITS))
            .into_iter()
            .flatten()
            .filter(|&&(_, first, _)| first == address)
            .map(|&(key, _, last)| (key, last))
            .collect()
    }

    /// Keeps `block` under `key`, translated from the guest's bytes in `first..=last`, in place of
    /// any block kept under it before.
    pub(crate) fn insert(&mut self, key: u64, first: u64, last: u64, block: Block) {
        self.unlist(key);
        self.by_key.insert(key, (block, first, last));
        self.recent[slot(key)].set(Recent { key, block });
        for page in first >> PAGE_BITS..=last >> PAGE_BITS {
            self.by_page
                .entry(page)
                .or_default()
                .push((key, first, last));
        }
    }

    /// Drops every block translated from any of the bytes in `start..end`, and answers whether
    /// each page they were on still holds a block, as the page and that answer.
    pub(crate) fn drop_range(&mut self, start: u64, end: u64) -> Vec<(u64, bool)> {
        self.drop_range_but(start, end, &BTreeSet::new())
    }

    /// Moves each block in `moving` to the key `to` gives it, translated from the same bytes, and
    /// drops every other block translated from the byte at `address`, as [`Blocks::drop_range`]
    /// does. The blocks moved are kept here the whole time, so nothing that clears the code
    /// generator can free them while they're out.
    pub(crate) fn rekey_at(
        &mut self,
        address: u64,
        moving: &[u64],
        to: impl Fn(u64) -> u64,
    ) -> Vec<(u64, bool)> {
        let mut moved = BTreeSet::new();
        for &from in moving {
            let Some(&(block, first, last)) = self.by_key.get(&from) else {
                continue;
            };
            self.forget(from);
            self.insert(to(from), first, last, block);
            moved.insert(to(from));
        }
        self.drop_range_but(address, address.saturating_add(1), &moved)
    }

    /// [`Blocks::drop_range`], keeping the blocks keyed in `kept`.
    fn drop_range_but(&mut self, start: u64, end: u64, kept: &BTreeSet<u64>) -> Vec<(u64, bool)> {
        if end <= start {
            return Vec::new();
        }
        let mut dropped = BTreeSet::new();
        for page in start >> PAGE_BITS..=(end - 1) >> PAGE_BITS {
            for &(key, first, last) in self.by_page.get(&page).into_iter().flatten() {
                if first < end && start <= last && !kept.contains(&key) {
                    dropped.insert(key);
                }
            }
        }
        let mut pages = BTreeSet::new();
        for key in dropped {
            pages.extend(self.forget(key).into_iter().flatten());
        }
        pages
            .into_iter()
            .map(|page| (page << PAGE_BITS, self.by_page.contains_key(&page)))
            .collect()
    }

    /// Forgets the block keyed `key`, on every page it came from, and unlinks every exit linked
    /// to it. Answers those pages.
    fn forget(&mut self, key: u64) -> Option<RangeInclusive<u64>> {
        let pages = self.unlist(key);
        self.by_key.remove(&key);
        let slot = slot(key);
        if self.recent[slot].get().key == key {
            self.recent[slot].set(EMPTY);
        }
        for cell in self.linked.remove(&key).unwrap_or_default() {
            self.cell(cell).set(0);
        }
        pages
    }

    /// Takes the block keyed `key` off every page it came from, and answers those pages, or `None`
    /// when there's no such block.
    fn unlist(&mut self, key: u64) -> Option<RangeInclusive<u64>> {
        let &(_, first, last) = self.by_key.get(&key)?;
        let pages = first >> PAGE_BITS..=last >> PAGE_BITS;
        for page in pages.clone() {
            if let Some(held) = self.by_page.get_mut(&page) {
                held.retain(|&(each, _, _)| each != key);
                if held.is_empty() {
                    self.by_page.remove(&page);
                }
            }
        }
        Some(pages)
    }

    /// How many blocks the page holding `address` lists.
    #[cfg(test)]
    pub(crate) fn listed_on(&self, address: u64) -> usize {
        self.by_page
            .get(&(address >> PAGE_BITS))
            .map_or(0, Vec::len)
    }

    /// Drops every block. The cells stay where they are, since a block that's running when its
    /// family drops everything still reads its own, and they're all unlinked.
    pub(crate) fn clear(&mut self) {
        self.by_key.clear();
        self.by_page.clear();
        for slot in &self.recent {
            slot.set(EMPTY);
        }
        self.linked.clear();
        for cell in self.cells.iter().flatten() {
            cell.set(0);
        }
        self.leads_to.fill(u64::MAX);
    }

    /// Frees every cell, once the code that read them is gone too, so dropping every block
    /// again and again doesn't keep their cells for good.
    pub(crate) fn free_cells(&mut self) {
        self.cells.clear();
        self.leads_to.clear();
    }

    /// A new link cell for an exit to the block keyed `key`, unlinked, as its number and the
    /// address translated code reads it at.
    pub(crate) fn new_cell(&mut self, key: u64) -> (u32, usize) {
        let number = self.leads_to.len();
        if number.is_multiple_of(CELLS) {
            self.cells
                .push(vec![Cell::new(0); CELLS].into_boxed_slice());
        }
        self.leads_to.push(key);
        let number = u32::try_from(number).expect("a cell number");
        (number, self.cell(number).as_ptr() as usize)
    }

    /// A counter a block counts its runs in, as the address it reads it at.
    pub(crate) fn new_counter(&mut self) -> usize {
        self.new_cell(u64::MAX).1
    }

    /// Puts `block` in place of the block keyed `key`, translated again from the same bytes, and
    /// links every exit that was linked to the old one to it.
    pub(crate) fn replace(&mut self, key: u64, block: Block) {
        if let Some(kept) = self.by_key.get_mut(&key) {
            kept.0 = block;
        }
        self.recent[slot(key)].set(Recent { key, block });
        for &cell in self.linked.get(&key).into_iter().flatten() {
            self.cell(cell).set(block.address());
        }
    }

    /// Links cell `number` to `block`, keyed `key`, when that's the block its exit leads to.
    pub(crate) fn link(&mut self, number: u32, key: u64, block: Block) {
        if self.leads_to.get(number as usize) == Some(&key) && self.by_key.contains_key(&key) {
            self.cell(number).set(block.address());
            self.linked.entry(key).or_default().push(number);
        }
    }

    /// The number of the link cell at `address`, if it's one of these.
    pub(crate) fn cell_number(&self, address: usize) -> Option<u32> {
        self.cells.iter().enumerate().find_map(|(chunk, cells)| {
            let first = cells.as_ptr() as usize;
            let offset = address.checked_sub(first)? / size_of::<usize>();
            (offset < CELLS).then(|| u32::try_from(chunk * CELLS + offset).expect("a cell number"))
        })
    }

    fn cell(&self, number: u32) -> &Cell<usize> {
        let number = number as usize;
        &self.cells[number / CELLS][number % CELLS]
    }

    pub(crate) fn len(&self) -> usize {
        self.by_key.len()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A block translated from bytes on two pages is forgotten on both when a store to either
    /// drops it, so neither page lists it again, and kept again under its key it's listed once.
    #[test]
    fn a_dropped_block_leaves_every_page_it_came_from() {
        let mut blocks = Blocks::default();
        let block = Block(std::ptr::null());
        let (first, last) = (PAGE_SIZE - 4, PAGE_SIZE + 3);
        blocks.insert(7, first, last, block);
        blocks.insert(7, first, last, block);
        assert_eq!((blocks.listed_on(first), blocks.listed_on(last)), (1, 1));
        assert_eq!(
            blocks.drop_range(first, first + 1),
            [(0, false), (PAGE_SIZE, false)]
        );
        assert_eq!((blocks.listed_on(first), blocks.listed_on(last)), (0, 0));
        assert!(!blocks.covers(last));
    }

    /// Runs `block` with `core` as its core, whose first word is the page table.
    fn run(jit: &Jit, core: &mut [*const usize; 1], block: Block) -> u32 {
        let entry = jit.entry();
        // SAFETY: the block reads only the table through the core's first word, which `core`
        // holds, and was made by a code generator that's still alive.
        unsafe { entry.run(core.as_mut_ptr().cast(), block) }
    }

    /// A block that reads a word through the page table, from a mapped page, and takes the slow
    /// branch from one that isn't, unaligned, or routed.
    #[test]
    fn a_block_reads_a_mapped_page_inline_and_takes_the_slow_branch_elsewhere() {
        #[repr(C, align(4096))]
        struct Page([u8; 4096]);
        let mut page = Box::new(Page([0; 4096]));
        page.0[8..12].copy_from_slice(&0xDEAD_BEEF_u32.to_le_bytes());
        let mut pages = Pages::new(DataModel::Ilp32).unwrap();
        // SAFETY: the page is the test's own and outlives the table.
        unsafe { pages.map(0x2000_0000, PAGE_SIZE, page.0.as_mut_ptr(), true) };
        let mut jit = Jit::new().unwrap();
        let block = |jit: &mut Jit, address: u32| {
            jit.compile(Tier::Best, |builder, core, _| {
                let table = builder
                    .ins()
                    .load(types::I64, MemFlagsData::trusted(), core, 0);
                let address = builder.ins().iconst(types::I32, i64::from(address));
                let slow = builder.create_block();
                let host = emit_lookup(builder, table, address, Lookup::Read(4), slow);
                let value = builder.ins().load(types::I32, guest_memory(), host, 0);
                builder.ins().return_(&[value]);
                builder.switch_to_block(slow);
                let missed = builder.ins().iconst(types::I32, 7);
                builder.ins().return_(&[missed]);
            })
            .unwrap()
        };
        let mut core = [pages.table()];
        let read = block(&mut jit, 0x2000_0008);
        assert_eq!(run(&jit, &mut core, read), 0xDEAD_BEEF);
        let unmapped = block(&mut jit, 0x3000_0000);
        assert_eq!(run(&jit, &mut core, unmapped), 7);
        let across = block(
            &mut jit,
            u32::try_from(0x2000_0000 + PAGE_SIZE - 2).unwrap(),
        );
        let answer = run(&jit, &mut core, across);
        assert_eq!(answer, 7, "across a page");
        pages.route(0x2000_0000, true);
        assert_eq!(run(&jit, &mut core, read), 7, "routed");
    }

    /// The table covers the 32-bit address space and nothing above it, and is no table for a
    /// family whose pointers are wider.
    #[test]
    fn the_table_holds_the_32_bit_space_and_nothing_above_it() {
        assert!(matches!(
            Pages::new(DataModel::Lp64),
            Err(Error::Machine(_))
        ));
        assert!(Pages::holds(0, ADDRESSES));
        assert!(Pages::holds(ADDRESSES - PAGE_SIZE, PAGE_SIZE));
        assert!(!Pages::holds(ADDRESSES - PAGE_SIZE, 2 * PAGE_SIZE));
        assert!(!Pages::holds(ADDRESSES, PAGE_SIZE));
        assert!(!Pages::holds(u64::MAX, 2));
    }

    /// A load and a store inline keep the byte order they're given: a big-endian word reads its
    /// most significant byte from the lowest address, and stores it back there.
    #[test]
    fn a_big_endian_load_and_store_keep_the_most_significant_byte_first() {
        #[repr(C, align(4096))]
        struct Page([u8; 4096]);
        let mut page = Box::new(Page([0; 4096]));
        page.0[8..12].copy_from_slice(&0xDEAD_BEEF_u32.to_be_bytes());
        let mut pages = Pages::new(DataModel::Ilp32).unwrap();
        // SAFETY: the page is the test's own and outlives the table.
        unsafe { pages.map(0x2000_0000, PAGE_SIZE, page.0.as_mut_ptr(), true) };
        let mut jit = Jit::new().unwrap();
        let block = jit
            .compile(Tier::Best, |builder, core, _| {
                let table = builder
                    .ins()
                    .load(types::I64, MemFlagsData::trusted(), core, 0);
                let slow = builder.create_block();
                let from = builder.ins().iconst(types::I32, 0x2000_0008);
                let host = emit_lookup(builder, table, from, Lookup::Read(4), slow);
                let value = load_guest(builder, types::I32, host, Endian::Big);
                let to = builder.ins().iconst(types::I32, 0x2000_0010);
                let host = emit_lookup(builder, table, to, Lookup::Write(4), slow);
                store_guest(builder, value, host, Endian::Big);
                builder.ins().return_(&[value]);
                builder.switch_to_block(slow);
                let missed = builder.ins().iconst(types::I32, 7);
                builder.ins().return_(&[missed]);
            })
            .unwrap();
        let mut core = [pages.table()];
        assert_eq!(run(&jit, &mut core, block), 0xDEAD_BEEF);
        assert_eq!(page.0[16..20], [0xDE, 0xAD, 0xBE, 0xEF]);
    }

    /// A block stores to a writable page inline, and takes the slow branch from one it can't write.
    #[test]
    fn a_block_stores_to_a_writable_page_inline() {
        #[repr(C, align(4096))]
        struct Page([u8; 4096]);
        let mut page = Box::new(Page([0; 4096]));
        let mut pages = Pages::new(DataModel::Ilp32).unwrap();
        // SAFETY: the page is the test's own and outlives the table.
        unsafe { pages.map(0x2000_0000, PAGE_SIZE, page.0.as_mut_ptr(), true) };
        let mut jit = Jit::new().unwrap();
        let store = jit
            .compile(Tier::Best, |builder, core, _| {
                let table = builder
                    .ins()
                    .load(types::I64, MemFlagsData::trusted(), core, 0);
                let address = builder.ins().iconst(types::I32, 0x2000_0008);
                let slow = builder.create_block();
                let host = emit_lookup(builder, table, address, Lookup::Write(4), slow);
                let value = builder.ins().iconst(types::I32, 0x1234);
                builder.ins().store(guest_memory(), value, host, 0);
                let stored = builder.ins().iconst(types::I32, 0);
                builder.ins().return_(&[stored]);
                builder.switch_to_block(slow);
                let missed = builder.ins().iconst(types::I32, 7);
                builder.ins().return_(&[missed]);
            })
            .unwrap();
        let mut core = [pages.table()];
        assert_eq!(run(&jit, &mut core, store), 0);
        assert_eq!(page.0[8..12], 0x1234_u32.to_le_bytes());
        pages.hold_code(0x2000_0000);
        assert_eq!(run(&jit, &mut core, store), 7, "code");
    }

    /// A page of code takes no inline store, and one routed takes no inline access at all, until
    /// each is let go.
    #[test]
    fn a_page_of_code_takes_no_inline_store_until_its_code_is_dropped() {
        #[repr(C, align(4096))]
        struct Page([u8; 4096]);
        let mut page = Box::new(Page([0; 4096]));
        let mut pages = Pages::new(DataModel::Ilp32).unwrap();
        // SAFETY: the page is the test's own and outlives the table.
        unsafe { pages.map(0, PAGE_SIZE, page.0.as_mut_ptr(), true) };
        let entry = |pages: &Pages| pages.entries[0].get() & (READ | WRITE);
        assert_eq!(entry(&pages), READ | WRITE);
        pages.hold_code(0x10);
        assert_eq!(entry(&pages), READ);
        pages.route(0, true);
        assert_eq!(entry(&pages), 0);
        pages.release_all_code();
        assert_eq!(entry(&pages), 0, "still routed");
        pages.route(0, false);
        assert_eq!(entry(&pages), READ | WRITE);
    }

    /// Code translated through one view of a host page takes the inline store from every view of
    /// it, and a range of those bytes answers at every view.
    #[test]
    fn code_through_one_view_takes_the_inline_store_from_every_view() {
        #[repr(C, align(4096))]
        struct Page([u8; 4096]);
        const VIEW: u64 = 0x0800_0000;
        let mut page = Box::new(Page([0; 4096]));
        let mut pages = Pages::new(DataModel::Ilp32).unwrap();
        // SAFETY: the page is the test's own and outlives the table.
        unsafe { pages.map(0, 2 * PAGE_SIZE, page.0.as_mut_ptr(), true) };
        // SAFETY: the page is the test's own and outlives the table.
        unsafe { pages.map(VIEW, 2 * PAGE_SIZE, page.0.as_mut_ptr(), true) };
        pages.hold_code(VIEW + 0x10);
        assert_eq!(pages.inline(0), Inline::Reads);
        assert_eq!(pages.inline(VIEW), Inline::Reads);
        assert_eq!(pages.inline(PAGE_SIZE), Inline::Both, "the next page");
        assert_eq!(
            pages.views(PAGE_SIZE - 2, PAGE_SIZE + 2),
            [
                (PAGE_SIZE - 2, PAGE_SIZE),
                (VIEW + PAGE_SIZE - 2, VIEW + PAGE_SIZE),
                (PAGE_SIZE, PAGE_SIZE + 2),
                (VIEW + PAGE_SIZE, VIEW + PAGE_SIZE + 2),
            ]
        );
        assert_eq!(
            pages.views(0x3000_0000, 0x3000_0004),
            [(0x3000_0000, 0x3000_0004)]
        );
        pages.release_code(VIEW);
        assert_eq!(pages.inline(0), Inline::Both);
    }

    /// While every access is checked, a page is inline only as far as it was let be, and checking
    /// again forgets that.
    #[test]
    fn a_checked_page_is_inline_only_as_far_as_it_was_let_be() {
        #[repr(C, align(4096))]
        struct Page([u8; 4096]);
        let mut page = Box::new(Page([0; 4096]));
        let mut pages = Pages::new(DataModel::Ilp32).unwrap();
        // SAFETY: the page is the test's own and outlives the table.
        unsafe { pages.map(0, PAGE_SIZE, page.0.as_mut_ptr(), true) };
        pages.check_all(true);
        assert_eq!(pages.inline(0), Inline::Neither);
        assert!(!pages.decided(0));
        pages.allow(0, Inline::Reads);
        assert!(pages.decided(0));
        assert_eq!(pages.inline(0), Inline::Reads);
        pages.allow(0, Inline::Both);
        assert_eq!(pages.inline(0), Inline::Both);
        pages.check_all(true);
        assert_eq!(pages.inline(0), Inline::Neither, "forgotten");
        assert!(!pages.decided(0));
        pages.check_all(false);
        assert_eq!(pages.inline(0), Inline::Both);
    }
}