rpi-hal 0.1.0

//! AArch32 (ARMv7-A short-descriptor) MMU implementation -- see the
//! parent [`mmu`](super) module's doc comment for the overall design.
//!
//! A single flat first-level table of 1MB sections covering all 4GB,
//! CP15-programmed, run at PL1.

use super::{LOCAL_PERIPHERAL_BASE, LOCAL_PERIPHERAL_END, PERIPHERAL_BASE, PERIPHERAL_END};
use core::arch::asm;

/// Entries in a full first-level short-descriptor translation table:
/// one 1MB section per entry, covering the whole 4GB address space.
const SECTION_COUNT: usize = 4096;

/// Shift from a section index to its base physical address (each
/// section covers `1 << 20` = 1MB).
const SECTION_SHIFT: u32 = 20;

// Section descriptor fields (ARMv7-A VMSA short-descriptor format --
// see the ARM Architecture Reference Manual's translation table
// descriptor and memory region attribute tables).
/// Bits[1:0]: marks this entry as a Section descriptor with PXN
/// (bit 0) clear, i.e. privileged execution is allowed.
const DESCRIPTOR_SECTION: u32 = 0b10;
/// Bits[11:10] (AP\[1:0\]) with APX (bit 15) left clear: full
/// read/write access at any privilege level.
const AP_FULL_ACCESS: u32 = 0b11 << 10;
/// Bit[4]: execute-never.
const XN: u32 = 1 << 4;
/// Bit[2] (B) -- meaning depends on which `TEX`/`C` it's paired with;
/// see `SECTION_RAM`/`SECTION_DEVICE`'s comments.
const B_BIT: u32 = 1 << 2;
/// Bits[14:12] (TEX) = `0b001`, paired with C=1/B=1: Normal memory,
/// Outer and Inner Write-Back, Write-Allocate -- see the parent module's
/// doc comment on why RAM needs to be Cacheable, not just Shareable, for
/// `ldrex`/`strex` to actually succeed on this core.
const TEX_NORMAL_CACHEABLE: u32 = 0b001 << 12;
/// Bit[3] (C), paired with `TEX_NORMAL_CACHEABLE`/`B_BIT` above.
const C_BIT: u32 = 1 << 3;
/// Bit[16] (S): Shareable. Necessary but not sufficient on its own on
/// this core (Cortex-A7) -- see the parent module's doc comment.
const S_BIT: u32 = 1 << 16;

/// RAM: Normal, Write-Back Write-Allocate Cacheable, Shareable, full
/// access, executable -- covers every address this crate's own code,
/// data, and stacks can occupy.
const SECTION_RAM: u32 =
    DESCRIPTOR_SECTION | AP_FULL_ACCESS | TEX_NORMAL_CACHEABLE | C_BIT | B_BIT | S_BIT;

/// Peripherals: TEX=000/C=0/B=1 is "Shareable Device" memory -- the
/// same ordering/no-caching guarantees every raw register access in
/// this crate already depends on with the MMU off, now made explicit.
/// Execute-never, since nothing should ever jump into MMIO space.
const SECTION_DEVICE: u32 = DESCRIPTOR_SECTION | AP_FULL_ACCESS | B_BIT | XN;

/// Builds the identity map: RAM below the peripheral base as
/// [`SECTION_RAM`], the peripheral block and the ARM-local peripheral
/// block as [`SECTION_DEVICE`], and everything else left as an invalid
/// descriptor (bits\[1:0\] = `00`) -- touching genuinely unbacked address
/// space still faults instead of being silently redefined as valid.
const fn build_page_table() -> [u32; SECTION_COUNT] {
    let mut table = [0u32; SECTION_COUNT];
    let mut i = 0;
    while i < SECTION_COUNT {
        let base = (i as u32) << SECTION_SHIFT;
        if base < PERIPHERAL_BASE {
            table[i] = base | SECTION_RAM;
        } else if base <= PERIPHERAL_END || {
            // Under `bcm2711`, `LOCAL_PERIPHERAL_END` is `u32::MAX` (the
            // ARM-local block runs to the top of the address space), which
            // makes the upper bound below trivially true -- still the
            // right check for the BCM2836/2837 case, where it isn't.
            #[allow(clippy::absurd_extreme_comparisons)]
            let in_local_block = base >= LOCAL_PERIPHERAL_BASE && base <= LOCAL_PERIPHERAL_END;
            in_local_block
        } {
            table[i] = base | SECTION_DEVICE;
        }
        i += 1;
    }
    table
}

/// The first-level translation table itself, built entirely at compile
/// time (a plain `static`, not populated by any runtime loop). `TTBR0`
/// with `N=0` (a single top-level table, no split) requires 16KB
/// alignment -- `#[repr(align(16384))]` on a wrapper struct, since
/// `align` can't be attached directly to a `static`'s type when that type
/// is a plain array.
#[repr(align(16384))]
struct PageTable(
    // Never read through Rust field access -- only its address (via
    // `&PAGE_TABLE`) matters; the actual contents are read by the MMU's
    // hardware page-table walker, invisible to the compiler.
    #[allow(dead_code)] [u32; SECTION_COUNT],
);

static PAGE_TABLE: PageTable = PageTable(build_page_table());

/// Builds the identity-mapped page table (above) and enables the MMU.
/// Called from `boot.s`, after `VBAR`/`SCTLR.V` setup and before `.bss`
/// zeroing/`kmain`: a fault during this sequence is at least catchable
/// once `VBAR` is live, and nothing before `kmain` needs the MMU already
/// on.
///
/// Safe to call once per core, not just once overall: with the
/// `multicore` feature on, every secondary core calls this again as part
/// of its own bring-up (see `boot.s`'s `__secondary_core_entry`).
/// TTBR0/DACR/SCTLR and the TLB are all per-core banked state, so each
/// call only ever reprograms the calling core's own copy against the
/// single, already-built, read-only [`PAGE_TABLE`] -- never a second
/// build of the table itself.
///
/// # Safety
///
/// Must only be called early in boot, by `rt`'s own boot sequence, on
/// each core before any code on that core relies on today's MMU-off
/// memory ordering guarantees changing underneath it.
#[no_mangle]
pub unsafe extern "C" fn rpi_hal_mmu_init() {
    let ttbr0 = &PAGE_TABLE as *const PageTable as u32;

    unsafe {
        // ACTLR.SMP (bit 6): per the Cortex-A7 TRM, a core must set this
        // before enabling its caches for cache coherency (via the Snoop
        // Control Unit) to actually apply to it in a multiprocessor
        // system. Without it, this core's cacheable writes are not
        // guaranteed to ever become visible to another core no matter how
        // many dsb/dmb barriers follow. Harmless to set unconditionally
        // even when only core 0 ever runs.
        let mut actlr: u32;
        asm!("mrc p15, 0, {0}, c1, c0, 1", out(reg) actlr);
        actlr |= 1 << 6;
        asm!("mcr p15, 0, {0}, c1, c0, 1", in(reg) actlr);

        // TTBR0: point at the page table. Low attribute bits (RGN/S/IRGN,
        // meaningful for cached/shared page-table walks) left 0 --
        // irrelevant with caches off and a single core.
        asm!("mcr p15, 0, {0}, c2, c0, 0", in(reg) ttbr0);

        // DACR: domain 0 set to "client" (0b01) -- respects the page
        // table's own AP bits rather than bypassing them. Every other
        // domain left at 0 = "no access".
        asm!("mcr p15, 0, {0}, c3, c0, 0", in(reg) 0b01u32);

        // Invalidate the entire TLB -- this core's TLB state coming out of
        // whatever GPU firmware ran before us isn't something to assume is
        // clean.
        asm!("mcr p15, 0, {0}, c8, c7, 0", in(reg) 0u32);
        asm!("dsb");
        asm!("isb");

        // SCTLR: set M (bit 0) and C (bit 2, data cache) -- see the parent
        // module's doc comment on why C is needed for `ldrex`/`strex` to
        // work on this core. I (bit 12, instruction cache) stays clear --
        // unrelated to this.
        let mut sctlr: u32;
        asm!("mrc p15, 0, {0}, c1, c0, 0", out(reg) sctlr);
        sctlr |= 1 | (1 << 2);
        asm!("mcr p15, 0, {0}, c1, c0, 0", in(reg) sctlr);

        // Architecturally required right after enabling the MMU: the
        // pipeline may have already fetched ahead using the old (MMU-off)
        // address translation behavior.
        asm!("isb");
    }
}