rpi-hal 0.8.0

//! Internal D-cache maintenance for the DMA-driven drivers.
//!
//! The mailbox, SD, touch, and USB paths hand buffers to bus masters
//! (the VideoCore, the EMMC controller, the DWC2 USB core) that read and
//! write RAM directly, bypassing this core's data cache. Before telling
//! one of them to read an ARM-written buffer, [`clean_range`] writes the
//! covering lines back to RAM; before reading a buffer one of them wrote,
//! [`invalidate_range`] drops the stale cached copies so the read comes
//! from RAM. Each ends with a barrier so the maintenance has completed
//! before the DMA is kicked off or the data is read.
//!
//! Consolidated here (rather than a private copy per driver) so the one
//! architecture-specific piece — the cache-maintenance instructions —
//! lives in a single place.

use crate::barrier::dsb;
use core::arch::asm;

/// Conservative minimum D-cache line size, in bytes.
///
/// The true line size is discoverable at runtime (the `CTR` register's
/// `DminLine` field), but walking a buffer in strides this small is
/// always safe: on hardware with a bigger line (this SoC's Cortex-A7 and
/// Cortex-A53 both use 64 bytes) a clean/invalidate-by-address op affects
/// the whole line containing the address, so a smaller stride just
/// touches each line more than once — harmless — where a stride larger
/// than the true line size could skip one.
pub(crate) const MIN_CACHE_LINE: u32 = 32;

/// Cleans (writes back, without invalidating) every cache line covering
/// `len` bytes from `address`, then barriers — makes an ARM-side write
/// actually visible in RAM before a bus master is told to read it.
pub(crate) fn clean_range(address: u32, len: usize) {
    let end = address + len as u32;
    let mut line = address & !(MIN_CACHE_LINE - 1);
    while line < end {
        unsafe { clean_line(line) };
        line += MIN_CACHE_LINE;
    }
    dsb();
}

/// Invalidates every cache line covering `len` bytes from `address`, then
/// barriers — forces the next ARM-side read to come from RAM instead of
/// whatever this core cached before a bus master wrote its response
/// there.
pub(crate) fn invalidate_range(address: u32, len: usize) {
    let end = address + len as u32;
    let mut line = address & !(MIN_CACHE_LINE - 1);
    while line < end {
        unsafe { invalidate_line(line) };
        line += MIN_CACHE_LINE;
    }
    dsb();
}

/// Cleans *and* invalidates every cache line covering `len` bytes from
/// `address`, then barriers — writes back anything this core had dirty and
/// leaves nothing cached behind.
///
/// The one place this is needed rather than one of the two halves above is
/// when memory stops being cacheable at all: a region about to be remapped
/// Normal Non-cacheable (`crate::mmu::set_uncached`) must have no cached
/// copy left, dirty or clean. A plain invalidate would discard writes that
/// never reached RAM; a plain clean would leave clean lines that the new,
/// non-cacheable accesses bypass, so a later eviction could still hand back
/// pre-remap data.
#[cfg(feature = "mmu")]
pub(crate) fn clean_invalidate_range(address: u32, len: usize) {
    let end = address + len as u32;
    let mut line = address & !(MIN_CACHE_LINE - 1);
    while line < end {
        unsafe { clean_invalidate_line(line) };
        line += MIN_CACHE_LINE;
    }
    dsb();
}

// The per-line ops are the only architecture-specific part left here;
// the trailing barrier comes from `crate::barrier`, which has its own
// three-way split. AArch32 uses CP15 cache-maintenance coprocessor
// writes (DCCMVAC to clean, DCIMVAC to invalidate, DCCIMVAC for both),
// with the same encodings on ARMv6 and ARMv7 — maintenance *by MVA* did
// not change, unlike the barriers; AArch64 uses the `dc cvac` /
// `dc ivac` / `dc civac` cache-maintenance instructions. All operate to
// the point of coherency — the level shared with the bus masters. On
// AArch64 the `{0:x}` operand selects the 64-bit view of the
// (zero-extended, always < 4 GB on this SoC) address, since `dc` takes an
// X register.

/// Cleans the single cache line containing `line` to the point of
/// coherency.
#[cfg(target_arch = "arm")]
#[inline(always)]
unsafe fn clean_line(line: u32) {
    asm!("mcr p15, 0, {0}, c7, c10, 1", in(reg) line);
}

/// Cleans the single cache line containing `line` to the point of
/// coherency.
#[cfg(target_arch = "aarch64")]
#[inline(always)]
unsafe fn clean_line(line: u32) {
    asm!("dc cvac, {0:x}", in(reg) line);
}

/// Invalidates the single cache line containing `line` to the point of
/// coherency.
#[cfg(target_arch = "arm")]
#[inline(always)]
unsafe fn invalidate_line(line: u32) {
    asm!("mcr p15, 0, {0}, c7, c6, 1", in(reg) line);
}

/// Invalidates the single cache line containing `line` to the point of
/// coherency.
#[cfg(target_arch = "aarch64")]
#[inline(always)]
unsafe fn invalidate_line(line: u32) {
    asm!("dc ivac, {0:x}", in(reg) line);
}

/// Cleans and invalidates the single cache line containing `line` to the
/// point of coherency.
#[cfg(all(feature = "mmu", target_arch = "arm"))]
#[inline(always)]
unsafe fn clean_invalidate_line(line: u32) {
    asm!("mcr p15, 0, {0}, c7, c14, 1", in(reg) line);
}

/// Cleans and invalidates the single cache line containing `line` to the
/// point of coherency.
#[cfg(all(feature = "mmu", target_arch = "aarch64"))]
#[inline(always)]
unsafe fn clean_invalidate_line(line: u32) {
    asm!("dc civac, {0:x}", in(reg) line);
}

#[cfg(not(any(target_arch = "arm", target_arch = "aarch64")))]
compile_error!("rpi-hal supports only ARM (AArch32) and AArch64 targets");