moa_cache 0.1.4

CPU 缓存维护操作（D-cache / I-cache）
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

use core::sync::atomic::{AtomicUsize, Ordering};

// 初值 64 仅为 `arch_cache_init` 调用前的占位（init 早期即填入真实值，运行期按行维护的路径都在其后）。
// 注意：CTR_EL0 读的是当前核的值，异构核（big.LITTLE）须取所有核的最小 line size，否则在 line
// 更小的核上按行循环会跳行漏维护——同构平台（如 QEMU virt）无此问题。
static DCACHE_LINE_SIZE: AtomicUsize = AtomicUsize::new(64);
static ICACHE_LINE_SIZE: AtomicUsize = AtomicUsize::new(64);

#[inline(always)]
pub(super) fn arch_cache_init() {
    let ctr: usize;
    unsafe { core::arch::asm!("mrs {}, ctr_el0", out(reg) ctr) };
    DCACHE_LINE_SIZE.store(4 << ((ctr >> 16) & 0xf), Ordering::Relaxed);
    ICACHE_LINE_SIZE.store(4 << (ctr & 0xf), Ordering::Relaxed);
}

#[inline(always)]
pub(super) fn arch_dcache_line_size() -> usize {
    DCACHE_LINE_SIZE.load(Ordering::Relaxed)
}

#[inline(always)]
fn icache_line_size() -> usize {
    ICACHE_LINE_SIZE.load(Ordering::Relaxed)
}

pub(super) fn arch_clean_dcache_area(addr: usize, size: usize) {
    let line = arch_dcache_line_size();
    let end = addr + size;
    let mut cur = addr & !(line - 1);
    while cur < end {
        unsafe { core::arch::asm!("dc cvac, {}", in(reg) cur) };
        cur += line;
    }
    unsafe { core::arch::aarch64::__dsb(core::arch::aarch64::SY) };
}

// 边界未对齐行用 civac(clean+invalidate)、完整包含行用 ivac(仅 invalidate)的逻辑，
// 须与 boot.rs 的 naked-asm `__inval_dcache_area`/`__flush_dcache_area` 保持一致（两份独立实现）。
pub(super) fn arch_inval_dcache_area(addr: usize, size: usize) {
    let line = arch_dcache_line_size();
    let mask = line - 1;
    let end = addr + size;
    let end_aligned = end & !mask;

    if end & mask != 0 {
        unsafe { core::arch::asm!("dc civac, {}", in(reg) end_aligned) };
    }
    let mut cur = addr & !mask;
    if addr & mask != 0 {
        unsafe { core::arch::asm!("dc civac, {}", in(reg) cur) };
        cur += line;
    }
    while cur < end_aligned {
        unsafe { core::arch::asm!("dc ivac, {}", in(reg) cur) };
        cur += line;
    }
    unsafe { core::arch::aarch64::__dsb(core::arch::aarch64::SY) };
}

pub(super) fn arch_flush_dcache_area(addr: usize, size: usize) {
    let line = arch_dcache_line_size();
    let end = addr + size;
    let mut cur = addr & !(line - 1);
    while cur < end {
        unsafe { core::arch::asm!("dc civac, {}", in(reg) cur) };
        cur += line;
    }
    unsafe { core::arch::aarch64::__dsb(core::arch::aarch64::SY) };
}

#[inline(always)]
pub(super) fn arch_flush_icache_all() {
    unsafe {
        core::arch::asm!("ic iallu");
        core::arch::aarch64::__dsb(core::arch::aarch64::NSH);
        core::arch::aarch64::__isb(core::arch::aarch64::SY);
    }
}

pub(super) fn arch_flush_icache_range(start: usize, end: usize) {
    let dline = arch_dcache_line_size();
    let mut cur = start & !(dline - 1);
    while cur < end {
        unsafe { core::arch::asm!("dc cvau, {}", in(reg) cur) };
        cur += dline;
    }
    unsafe { core::arch::aarch64::__dsb(core::arch::aarch64::ISH) };

    let iline = icache_line_size();
    cur = start & !(iline - 1);
    while cur < end {
        unsafe { core::arch::asm!("ic ivau, {}", in(reg) cur) };
        cur += iline;
    }
    unsafe {
        core::arch::aarch64::__dsb(core::arch::aarch64::ISH);
        core::arch::aarch64::__isb(core::arch::aarch64::SY);
    }
}