aarch64_cpu_ext/
cache.rs

1use core::arch::asm;
2
3use aarch64_cpu::{
4    asm::barrier::{NSH, SY, dsb, isb},
5    registers::*,
6};
7
8use crate::asm::cache::{CISW, CIVAC, CSW, CVAC, IALLU, ISW, IVAC, dc, ic};
9
10pub fn icache_flush_all() {
11    ic(IALLU);
12    dsb(NSH);
13    isb(SY);
14}
15
16#[repr(C)]
17#[derive(Debug, Clone, Copy)]
18pub enum CacheOp {
19    /// Write back to memory
20    Clean,
21    /// Invalidate cache
22    Invalidate,
23    /// Clean and invalidate
24    CleanAndInvalidate,
25}
26
27#[inline(always)]
28pub fn cache_line_size() -> usize {
29    unsafe {
30        let mut ctr_el0: u64;
31        asm!("mrs {}, ctr_el0", out(reg) ctr_el0);
32        // CTR_EL0.DminLine (bits 19:16) - log2 of the number of words in the smallest cache line
33        let log2_cache_line_size = ((ctr_el0 >> 16) & 0xF) as usize;
34        // Calculate the cache line size: 4 * (2^log2_cache_line_size) bytes
35        4 << log2_cache_line_size
36    }
37}
38
39/// Performs a cache operation on a single cache line.
40#[inline]
41fn _dcache_line(op: CacheOp, addr: usize) {
42    let addr = addr as u64;
43    match op {
44        CacheOp::Clean => dc(CVAC, addr),
45        CacheOp::Invalidate => dc(IVAC, addr),
46        CacheOp::CleanAndInvalidate => dc(CIVAC, addr),
47    }
48}
49
50/// Performs a cache operation on a range of memory.
51#[inline]
52pub fn dcache_range(op: CacheOp, addr: usize, size: usize) {
53    let start = addr;
54    let end = start + size;
55    let cache_line_size = cache_line_size();
56
57    let mut aligned_addr = addr & !(cache_line_size - 1);
58
59    while aligned_addr < end {
60        _dcache_line(op, aligned_addr);
61        aligned_addr += cache_line_size;
62    }
63
64    dsb(SY);
65    isb(SY);
66}
67
68/// Performs a cache operation on a value.
69pub fn dcache_value<T>(op: CacheOp, v: &T) {
70    // Get the pointer to the value
71    let ptr = v as *const T as usize;
72    // Calculate the size of the value in bytes
73    let size = core::mem::size_of_val(v);
74    // Perform cache operation on the value
75    dcache_range(op, ptr, size);
76}
77
78/// Performs a cache operation on a cache level.
79/// https://developer.arm.com/documentation/ddi0601/2024-09/AArch64-Instructions/DC-CISW--Data-or-unified-Cache-line-Clean-and-Invalidate-by-Set-Way
80/// https://developer.arm.com/documentation/ddi0601/2024-09/AArch64-Registers/CTR-EL0--Cache-Type-Register?lang=en
81/// https://developer.arm.com/documentation/ddi0601/2024-09/AArch64-Registers/CCSIDR-EL1--Current-Cache-Size-ID-Register?lang=en
82/// https://github.com/u-boot/u-boot/blob/master/arch/arm/cpu/armv8/cache.S
83///
84/// DC instruction set/way format:
85/// - Bits [63:32]: Reserved, RES0
86/// - Bits [31:4]: SetWay field containing:
87///   - Way field: bits[31:32-A] where A = Log2(ASSOCIATIVITY)  
88///   - Set field: bits[B-1:L] where B = L + S, L = Log2(LINELEN), S = Log2(NSETS)
89///   - Bits[L-1:4]: RES0
90/// - Bits [3:1]: Level (cache level minus 1)
91/// - Bit [0]: Reserved, RES0
92#[inline]
93fn dcache_level(op: CacheOp, level: u64) {
94    assert!(level < 8, "armv8 level range is 0-7");
95
96    isb(SY);
97    CSSELR_EL1.write(CSSELR_EL1::InD::Data + CSSELR_EL1::Level.val(level));
98    isb(SY);
99
100    // Read cache parameters from CCSIDR_EL1
101    // Note: All values from CCSIDR_EL1 need to be adjusted according to ARM spec:
102    // - LineSize: (Log2(bytes in cache line)) - 4
103    // - Associativity: (Associativity of cache) - 1
104    // - NumSets: (Number of sets in cache) - 1
105    let line_size_raw = CCSIDR_EL1.read(CCSIDR_EL1::LineSize) as u32;
106    let associativity_raw = CCSIDR_EL1.read(CCSIDR_EL1::AssociativityWithCCIDX) as u32;
107    let num_sets_raw = CCSIDR_EL1.read(CCSIDR_EL1::NumSetsWithCCIDX) as u32;
108
109    // Convert raw values to actual values
110    let line_size_log2_bytes = line_size_raw + 4; // Actual log2 of line size in bytes
111    let associativity = associativity_raw + 1; // Actual associativity
112    let num_sets = num_sets_raw + 1; // Actual number of sets
113
114    // Calculate bit positions for set/way encoding according to ARM spec:
115    // L = Log2(LINELEN) where LINELEN is line length in bytes
116    // S = Log2(NSETS)
117    // A = Log2(ASSOCIATIVITY)
118    // Way field: bits[31:32-A]
119    // Set field: bits[B-1:L] where B = L + S
120
121    let l = line_size_log2_bytes; // Log2 of line length in bytes
122
123    // Calculate the number of bits needed to represent the way index
124    // leading_zeros on (associativity-1) gives us the position of the MSB needed
125    let way_shift = associativity_raw.leading_zeros(); // Way field starts at bit (32-A)
126    let set_shift = l; // Set field starts at bit L (line size offset)
127
128    // Loop over all sets and ways (0-based indexing for hardware)
129    for set in 0..num_sets {
130        for way in 0..associativity {
131            // Construct the set/way value according to ARM DC instruction format:
132            // Way field: bits[31:32-A] - way value shifted to proper bit position
133            // Set field: bits[B-1:L] - set value shifted to proper bit position
134            //
135            // Example: If associativity=4, way indices are 0,1,2,3
136            // We need A=2 bits (Log2(4)=2), so way field is at bits[31:30]
137            // way_shift = 32 - 2 = 30, so way values are shifted left by 30 bits
138            let set_way = (way << way_shift) | (set << set_shift);
139
140            // Complete operand: set_way in bits [31:4], level in bits [3:1], bit [0] is RES0
141            let cisw = (set_way as u64) | (level << 1);
142            match op {
143                CacheOp::Invalidate => dc(ISW, cisw),
144                CacheOp::Clean => dc(CSW, cisw),
145                CacheOp::CleanAndInvalidate => dc(CISW, cisw),
146            }
147        }
148    }
149}
150
151/// Performs a cache operation on all memory.
152pub fn dcache_all(op: CacheOp) {
153    let clidr = CLIDR_EL1.get();
154
155    for level in 0..8 {
156        let ty = (clidr >> (level * 3)) & 0b111;
157
158        // Cache type values:
159        // 0b000 = No cache
160        // 0b001 = Instruction cache only
161        // 0b010 = Data cache only
162        // 0b011 = Separate instruction and data caches
163        // 0b100 = Unified cache
164        // Only process data caches (0b010) and unified caches (0b100)
165        // or separate I+D caches (0b011) - for 0b011, we process the data cache
166        match ty {
167            0b000 => return,   // No cache at this level, we're done
168            0b001 => continue, // Instruction cache only, skip
169            0b010..=0b100 => {
170                // Data cache (0b010), separate I+D caches (0b011), or unified cache (0b100) - process it
171                dcache_level(op, level);
172            }
173            _ => continue, // Reserved values, skip
174        }
175    }
176    dsb(SY);
177    isb(SY);
178}
aarch64_cpu_ext/cache.rs

aarch64_cpu_ext/
cache.rs
