rpi-hal 0.10.0

//! MMU bring-up: enables address translation so RAM can be marked
//! Normal memory, needed for `core::sync::atomic`'s exclusive-access
//! instructions (`ldrex`/`strex` on AArch32, `ldxr`/`stxr` on AArch64)
//! to behave correctly -- those are architecturally UNPREDICTABLE on the
//! Strongly-Ordered/Device memory every address is with the MMU off.
//! Every peripheral MMIO region keeps behaving exactly as it does today
//! (Device memory, same ordering guarantees raw register access already
//! relies on everywhere else in this crate).
//!
//! Identity-mapped only: every virtual address equals its physical
//! address. This exists purely to change memory *attributes*, not
//! layout -- no relocation, no higher-half kernel.
//!
//! The data cache is enabled and RAM is mapped Cacheable -- confirmed
//! necessary on real hardware (AArch32), not a nicety: with RAM mapped
//! Normal-but-Non-cacheable, the exclusive monitor never let `strex`
//! succeed, because this core ties the monitor to cache-line state.
//!
//! The implementation differs enough between execution states to live in
//! two files (as `boot.s`/`boot64.s` do), selected by target
//! architecture:
//!
//! - [`mmu32`](mod@self) -> `mmu32.rs`: VMSA short-descriptor
//!   first-level table (1MB sections), CP15-programmed, run at PL1.
//!   Called from `rt`'s `boot.s` after its HYP->SVC drop, or from
//!   `boot6.s` on ARMv6, where there is no Hyp mode to drop out of.
//! - `mmu64.rs`: VMSAv8-64 long-descriptor tables (2MB blocks),
//!   programmed via `MAIR_EL1`/`TCR_EL1`/`TTBR0_EL1`, run at EL1. The
//!   caller must already be at EL1 (a consumer's boot stub drops
//!   EL2->EL1 first).
//!
//! Both define the same `#[no_mangle] extern "C" fn rpi_hal_mmu_init`
//! entry point. It is a *strong* symbol; `rt`'s own boot sequence
//! (`boot.s`) calls it unconditionally, falling back to a weak no-op
//! (`mmu_fallback.s`, only included when the `mmu` feature is off -- see
//! `boot.rs`) when this feature is off. A consumer wanting a different
//! memory map turns this feature off and defines their own
//! `#[no_mangle] extern "C" fn rpi_hal_mmu_init()` instead.

// SoC memory map, shared by both implementations. The BCM2836/2837
// values are confirmed against this crate's own peripheral access types
// (e.g. the System Timer at `0x3f00_3000`, the legacy interrupt
// controller at `0x3f00_b000`, out to `0x3f98_0e00` for USB), not assumed
// from a datasheet. The BCM2711 values are the "low peripheral mode"
// addresses from Linux's `bcm2711.dtsi` (the peripheral block's `ranges`
// entry and `local_intc`/GIC-400 nodes), not yet cross-checked against
// this crate's own drivers the way the BCM2836/2837 base was -- there's
// no BCM2711 target to build until these consts exist, which is what
// they're for. `PERIPHERAL_BASE` itself lives in `crate::soc`, not here
// -- `crate::dma`/`crate::watchdog` need it regardless of whether this
// (`mmu`-gated) module is even compiled.

//
// The BCM2712 is the odd one out: every peripheral it has sits above 4 GiB,
// and the whole low 4 GiB is RAM. So it has none of the low-memory
// peripheral consts below, and instead names its MMIO as whole 1 GiB
// regions in `HIGH_DEVICE` (from `bcm2712.dtsi`'s `soc` and `pcie2`
// `ranges`).

#[cfg(not(feature = "bcm2712"))]
use crate::soc::PERIPHERAL_BASE;

/// Exclusive top of the RAM the identity map marks Normal: everything
/// below the peripheral block on chips that keep it in the low 4 GiB, and
/// the whole low 4 GiB on the BCM2712. Also the bound [`set_uncached`]
/// checks a region against.
///
/// On the BCM2712 that is 4 GiB of address space whatever the board's RAM:
/// RAM above 4 GiB (8 GB and 16 GB boards) is not mapped, and on a 2 GB
/// board the top half of what is mapped has nothing behind it.
#[cfg(not(feature = "bcm2712"))]
const RAM_END: u64 = PERIPHERAL_BASE as u64;
#[cfg(feature = "bcm2712")]
const RAM_END: u64 = 1 << 32;

/// 1 GiB regions above 4 GiB to map as Device, by base address.
///
/// On the BCM2712: the two that hold the SoC's own peripheral bus
/// (`0x10_0000_0000`, 2 GiB, which includes the System Timer, the always-on
/// GPIO and the GIC), and the one RP1's peripheral window opens into
/// (`0x1f_0000_0000`). Empty on every other chip.
#[cfg(feature = "bcm2712")]
const HIGH_DEVICE: &[u64] = &[0x10_0000_0000, 0x10_4000_0000, 0x1f_0000_0000];
#[cfg(all(target_arch = "aarch64", not(feature = "bcm2712")))]
const HIGH_DEVICE: &[u64] = &[];

/// End of the mapped peripheral block (inclusive) -- a round,
/// 1MB-aligned number comfortably covering every peripheral address this
/// crate can reach.
#[cfg(all(feature = "bcm2711", not(feature = "bcm2712")))]
const PERIPHERAL_END: u32 = 0xFEFF_FFFF;
#[cfg(all(
    not(feature = "bcm2712"),
    not(feature = "bcm2711"),
    feature = "bcm2837"
))]
const PERIPHERAL_END: u32 = 0x3FFF_FFFF;
/// BCM2835's peripheral block is 16MB, from `PERIPHERAL_BASE` out past
/// USB at `0x2098_0e00`. Same "no chip selected" arm as
/// [`PERIPHERAL_BASE`] -- see `soc.rs`.
#[cfg(all(
    not(feature = "bcm2712"),
    not(feature = "bcm2711"),
    not(feature = "bcm2837")
))]
const PERIPHERAL_END: u32 = 0x20FF_FFFF;

/// The ARM-local peripheral block (per-core timers, IRQ routing, and the
/// inter-core mailbox registers on BCM2836/2837; the same block plus
/// GIC-400 on BCM2711) as an inclusive `(base, end)` range -- a
/// physically separate MMIO region from [`PERIPHERAL_BASE`]. Needed
/// device-mapped so [`crate::multicore`] can reach the mailbox registers
/// with the MMU on.
///
/// `None` on a chip that has no such block at all, in which case the
/// identity map has no device region above the peripheral block.
#[cfg(all(
    not(feature = "bcm2712"),
    not(feature = "bcm2711"),
    feature = "bcm2837"
))]
const LOCAL_PERIPHERAL: Option<(u32, u32)> = Some((0x4000_0000, 0x400F_FFFF));
/// BCM2711 low-peripheral-mode base (`bcm2711.dtsi`'s `ranges` entry
/// mapping bus address `0x4000_0000` to this physical address; GIC-400's
/// distributor/CPU-interface registers live at `0xFF84_1000`/`0xFF84_2000`
/// within this block), for the 8MB that entry's size covers -- which runs
/// to the top of the 32-bit address space.
#[cfg(all(feature = "bcm2711", not(feature = "bcm2712")))]
const LOCAL_PERIPHERAL: Option<(u32, u32)> = Some((0xFF80_0000, 0xFFFF_FFFF));
/// The BCM2835 has no ARM-local peripheral block. It is a uniprocessor
/// part: the block first appears on the BCM2836 to carry what a second
/// core needs -- per-core timers and interrupt routing, and the mailboxes
/// cores signal each other through -- none of which exists here. Nothing
/// above the peripheral block is mapped at all.
#[cfg(all(
    not(feature = "bcm2712"),
    not(feature = "bcm2711"),
    not(feature = "bcm2837")
))]
const LOCAL_PERIPHERAL: Option<(u32, u32)> = None;

/// Whether the 2MB block at `base`, somewhere at or above [`RAM_END`] in
/// the low 4 GiB, is MMIO to map as Device rather than leave invalid.
#[cfg(all(target_arch = "aarch64", not(feature = "bcm2712")))]
fn is_low_device(base: u64) -> bool {
    let in_local_block = match LOCAL_PERIPHERAL {
        Some((local_base, local_end)) => {
            base >= u64::from(local_base) && base <= u64::from(local_end)
        }
        None => false,
    };
    base <= u64::from(PERIPHERAL_END) || in_local_block
}
/// The BCM2712 has no MMIO in the low 4 GiB, all of which is below
/// [`RAM_END`] anyway.
#[cfg(feature = "bcm2712")]
fn is_low_device(_base: u64) -> bool {
    false
}

#[cfg(target_arch = "arm")]
#[path = "mmu32.rs"]
mod imp;

#[cfg(target_arch = "aarch64")]
#[path = "mmu64.rs"]
mod imp;

use crate::cache::clean_invalidate_range;

/// Granularity of [`set_uncached`]: 1MB (AArch32's section) or 2MB
/// (AArch64's level-2 block), whichever this build's table uses. A region
/// handed to that function must be aligned to this and a whole multiple of
/// it, since a translation table entry is the smallest thing whose memory
/// type can be changed.
///
/// Sized for whichever implementation is compiled, so a `static` intended to
/// be remapped has to be aligned and padded to the larger of the two to
/// build for both targets — 2MB, the value below on AArch64.
pub const UNCACHED_GRANULE: usize = imp::UNCACHED_GRANULE;

/// Bytes one translation-table descriptor covers.
///
/// The same number [`UNCACHED_GRANULE`] publishes, named again because
/// [`install_stack_guard`] cares about the shape of the table rather than
/// about `set_uncached`'s contract — and the day those stop being the same
/// number, each wants to move on its own.
#[cfg(feature = "rt")]
const DESCRIPTOR_BYTES: usize = imp::UNCACHED_GRANULE;

#[cfg(feature = "rt")]
extern "C" {
    /// Lowest address of the reserved margin below the main stack.
    static __stack_slack_bottom: u8;
    /// Lowest address of the main stack, and one past the top of the
    /// margin below it.
    static __stack_bottom: u8;
}

/// Leaves the reserved margin below the main stack with no translation, so
/// that an overflow faults on the instruction that causes it.
///
/// Without this a stack overflow is not an error at all. The map is an
/// identity map of every address below the peripheral base, so `sp`
/// descending past `__stack_bottom` crosses nothing the hardware objects
/// to: it walks the margin, reaches `.data` and `.text` below it, and
/// overwrites the running program. What that looks like is not a fault but
/// a board that behaves strangely and later reboots, arbitrarily far from
/// the call that actually went too deep.
///
/// The margin is reserved by this crate's linker scripts (`__stack_slack`,
/// 2 MiB) and is 2 MiB-aligned precisely so it can be given up here: 2 MiB
/// is the coarser of the two granules in play, so the region is a whole
/// number of descriptors on either architecture and no real memory shares
/// one with it. It holds nothing — `.stack` is a `NOLOAD` region between
/// `.data` and `.bss` — so nothing is lost by making it unreachable, and
/// the 2 MiB of address space is noise on a board with at least a
/// gigabyte.
///
/// Rounded inward, so a margin that has been overridden to something not a
/// multiple of a descriptor gives up only the descriptors lying entirely
/// inside it. A margin smaller than one descriptor yields no guard at all
/// rather than an approximate one.
///
/// # What this does not cover
///
/// Only the main stack of the core that boots. The AArch32 exception-mode
/// stacks sit *above* `__stack_top`, and a secondary core runs on a
/// [`Stack`](crate::multicore::Stack) the application placed in `.bss`;
/// both are ordinary mapped memory with their neighbours right below them,
/// exactly as this one used to be.
///
/// # Safety
///
/// Called from `rpi_hal_mmu_init` on each core, after the table is built
/// and before the MMU is enabled. That ordering is what makes it free of
/// cache and TLB maintenance: the descriptor write lands in RAM with the
/// caches off, and there is no stale translation to invalidate because
/// there are no translations yet. Writing the same zeroes on every core is
/// why calling it more than once is harmless.
#[cfg(feature = "rt")]
pub(crate) unsafe fn install_stack_guard() {
    let margin_bottom = &raw const __stack_slack_bottom as usize;
    let margin_top = &raw const __stack_bottom as usize;

    let first = margin_bottom.next_multiple_of(DESCRIPTOR_BYTES);
    let last = margin_top - margin_top % DESCRIPTOR_BYTES;

    let mut block = first;
    while block < last {
        // SAFETY: `block` is descriptor-aligned and lies inside the region
        // the linker script reserved, which holds nothing. The caller's
        // contract puts this before the MMU is enabled.
        unsafe { imp::invalidate_block(block as u32) };
        block += DESCRIPTOR_BYTES;
    }
}

/// Why a [`set_uncached`] call was rejected.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Error {
    /// The base address or the length isn't a multiple of
    /// [`UNCACHED_GRANULE`]. A translation table entry covers exactly that
    /// much, so a partial one can't be given a different memory type
    /// without splitting the entry — which this map, deliberately flat,
    /// doesn't do.
    Misaligned,
    /// The region isn't entirely inside the RAM this table maps as Normal
    /// memory (below the peripheral block). Remapping MMIO space, or an
    /// address with no descriptor at all, is a caller bug rather than
    /// something to silently do.
    NotRam,
}

/// Remaps `len` bytes at `base` as Normal **Non-cacheable** memory, and
/// drops whatever this core had cached of it.
///
/// This is what makes a shared-memory protocol with the VideoCore possible.
/// The rest of this crate's bus-master traffic gets by with explicit cache
/// maintenance (`cache.rs`) because each buffer has one owner at a time: the
/// ARM writes it, cleans it, and hands it over. VCHIQ's shared state
/// (`crate::vchiq`) is not like that — both sides write *different fields of
/// the same cache line* concurrently, so there is no correct maintenance
/// sequence at all. Cleaning the line to publish this core's field writes
/// the stale copy of the peer's field back over it; invalidating it to read
/// the peer's field discards this core's not-yet-written-back one. Only
/// taking the line out of the picture fixes that, which is why Linux
/// allocates the same region with `dma_alloc_coherent`.
///
/// Both the descriptor write and the TLB invalidation are broadcast to the
/// inner-shareable domain, so secondary cores (which share this one table)
/// see the new memory type too rather than keeping a stale cached
/// translation.
///
/// # Safety
///
/// The region must be memory this caller owns outright — the entire
/// [`UNCACHED_GRANULE`]-sized block gets a new memory type, so anything else
/// that happens to live in it silently becomes non-cacheable and much
/// slower. Nothing may be concurrently accessing the region on any core
/// during the call: its cached copies are discarded as part of it.
pub unsafe fn set_uncached(base: usize, len: usize) -> Result<(), Error> {
    if !base.is_multiple_of(UNCACHED_GRANULE) || len == 0 || !len.is_multiple_of(UNCACHED_GRANULE) {
        return Err(Error::Misaligned);
    }
    let end = base
        .checked_add(len)
        .filter(|end| *end as u64 <= RAM_END)
        .ok_or(Error::NotRam)?;

    // Write back and drop every cached copy *before* the memory type
    // changes: afterwards these lines are no longer reachable by
    // maintenance-by-address through this (now non-cacheable) mapping, and a
    // dirty one left behind could be evicted over data written through the
    // new mapping.
    clean_invalidate_range(base as u32, len);

    for block in (base..end).step_by(UNCACHED_GRANULE) {
        // SAFETY: `block` is granule-aligned and inside the RAM range the
        // table maps as Normal memory, both checked above.
        unsafe { imp::set_uncached_block(block as u32) };
    }
    Ok(())
}