et_kernel/pmu.rs
1//! Performance Monitoring Unit (PMU) counter API for the ET-SoC-1 Minion core.
2//!
3//! The ET-SoC-1 implements a subset of the RISC-V Zihpm extension (PRM
4//! section 1.3.2). The PMU is shared across 8 Minions in a neighbourhood and
5//! provides **six** counters per hart, not the full 3-31 range that the
6//! RISC-V spec permits:
7//!
8//! - `hpmcounter3`-`hpmcounter6` (`mhpmevent3`-`mhpmevent6`): Minion-level
9//! events from [`PmuEvent`] -- one event per counter, configured by firmware.
10//! - `hpmcounter7`-`hpmcounter8` (`mhpmevent7`-`mhpmevent8`): neighbourhood-
11//! level events from [`NeighborhoodEvent`] -- shared across the 8-Minion
12//! neighbourhood; when different harts program different events the lower
13//! `mhartid` wins.
14//! - `hpmcounter9`-`hpmcounter31`: tied to 0 on this implementation.
15//!
16//! # Note on `mcycle` and `minstret`
17//!
18//! The standard `mcycle` (CSR `0xC00`) and `minstret` (CSR `0xC02`) counters
19//! are **permanently zero** on the ET-SoC-1 (PRM section 1.3.2). Use
20//! `hpmcounter3` (or any of 3-6) configured with [`PmuEvent::Cycles`] to
21//! count clock cycles, and [`PmuEvent::RetiredInst0`] / [`PmuEvent::RetiredInst1`]
22//! to count retired instructions. The firmware on aifoundry3 assigns
23//! `PmuEvent::Cycles` to `hpmcounter3` by default, which is why
24//! [`crate::timestamp`] reads CSR `0xC03`.
25//!
26//! # Enabling the PMU
27//!
28//! The PMU must be enabled by firmware (an M-mode ESR write) before any
29//! counter increments. U-mode code can read counts but typically cannot
30//! reconfigure `mhpmeventN` without M-mode delegation.
31//!
32//! # Usage pattern
33//!
34//! ```no_run
35//! use et_kernel::pmu::{PmuEvent, pmu_read};
36//!
37//! // Read counter 4 before and after a tensor operation.
38//! // (Assumes firmware has assigned PmuEvent::TfmaWaitTenb to mhpmevent4.)
39//! let before = pmu_read(4);
40//! // ... tensor operations ...
41//! let after = pmu_read(4);
42//! let delta = after.wrapping_sub(before);
43//! ```
44
45use core::arch::asm;
46
47// ---------------------------------------------------------------------------
48// PMU event codes
49// ---------------------------------------------------------------------------
50
51/// Minion-level PMU event codes (PRM section 1.3.2, Table 1-3).
52///
53/// Written to `mhpmeventN` (CSR `0x320 + N`, for N in `3..=6`) to select what
54/// `hpmcounterN` accumulates. Firmware or a privileged shim configures the
55/// mapping; U-mode code reads counts via [`pmu_read`] and typically cannot
56/// write `mhpmeventN` without M-mode delegation.
57///
58/// These events apply only to `hpmcounter3`-`hpmcounter6`. For
59/// neighbourhood-level events on `hpmcounter7`-`hpmcounter8`, use
60/// [`NeighborhoodEvent`].
61#[repr(u64)]
62#[derive(Clone, Copy, Debug, PartialEq, Eq)]
63pub enum PmuEvent {
64 /// No event; counter does not increment.
65 NoEvent = 0,
66 /// Clock cycles executed by the core.
67 ///
68 /// Use this event in `mhpmevent3`-`mhpmevent6` to count cycles;
69 /// `mcycle` (CSR `0xC00`) is permanently zero on this implementation.
70 Cycles = 1,
71 /// An instruction retired by thread 0 of the core.
72 ///
73 /// `minstret` (CSR `0xC02`) is permanently zero; use this event instead.
74 RetiredInst0 = 2,
75 /// An instruction retired by thread 1 of the core.
76 ///
77 /// `minstret` (CSR `0xC02`) is permanently zero; use this event instead.
78 RetiredInst1 = 3,
79 /// A branch taken by thread 0 of the core.
80 Branches0 = 4,
81 /// A branch taken by thread 1 of the core.
82 Branches1 = 5,
83 /// A load/store by thread 0 accessed the data cache (hit or miss).
84 ///
85 /// Excludes tensor and cache-management operations.
86 DcacheAccess0 = 6,
87 /// A load/store by thread 1 accessed the data cache (hit or miss).
88 ///
89 /// Excludes tensor and cache-management operations.
90 DcacheAccess1 = 7,
91 /// A load/store by thread 0 missed in the data cache.
92 ///
93 /// Excludes tensor and cache-management operations.
94 DcacheMisses0 = 8,
95 /// A load/store by thread 1 missed in the data cache.
96 ///
97 /// Excludes tensor and cache-management operations.
98 DcacheMisses1 = 9,
99 /// The data cache sent a miss request to the L2 cache.
100 L2MissReq = 10,
101 /// The L2 cache rejected a miss request from the data cache.
102 L2MissReqRej = 11,
103 /// The data cache sent an evict request to the L2 cache.
104 L2EvictReq = 12,
105 /// The L2 cache rejected an evict request from the data cache.
106 L2EvictReqRej = 13,
107 /// Started execution of a TensorLoad instruction.
108 TlInst = 14,
109 /// A TensorLoad sent a request to the L2 cache.
110 TlOps = 15,
111 /// Started execution of a TensorStore instruction.
112 TsInst = 16,
113 /// A TensorStore sent a request to the L2 cache.
114 TsOps = 17,
115 /// Cycles a TensorFMA paired with TensorLoadB was blocked waiting for data
116 /// from L2. Measures the B-load serialisation cost; high values indicate
117 /// that the crossbar or DRAM is the bottleneck for B tiles.
118 TfmaWaitTenb = 18,
119 /// Started execution of a micro-op generated by a TensorIMA8A32 instruction.
120 TimaOps = 19,
121 /// Retired a micro-op generated by a TensorFMA16A32 instruction.
122 TxFma3216Ops = 20,
123 /// Retired an FP instruction (packed or scalar), integer multiplication,
124 /// or a micro-op from TensorFMA32.
125 TxFma32Ops = 21,
126 /// Retired a packed integer instruction, int-to-FP conversion, or micro-op
127 /// from integer TensorQuant.
128 TxFmaIntOps = 22,
129 /// Retired a micro-op generated by a transcendental instruction.
130 TransOps = 23,
131 /// Retired a packed integer instruction or a micro-op from TensorFMA32.
132 ShortOps = 24,
133 /// Retired a mask instruction.
134 MaskOps = 25,
135 /// Started execution of a TensorFMA instruction.
136 TfmaInst = 26,
137 /// Started execution of a tensor reduction instruction.
138 TreduceInst = 27,
139 /// Started execution of a TensorQuant instruction.
140 TquantInst = 28,
141}
142
143/// Neighbourhood-level PMU event codes (PRM section 1.3.2, Table 1-4).
144///
145/// Written to `mhpmevent7` or `mhpmevent8` to select what `hpmcounter7` or
146/// `hpmcounter8` accumulates. The counter is shared across the 8-Minion
147/// neighbourhood; when harts program different events the lower `mhartid`
148/// hart's choice takes precedence.
149#[repr(u64)]
150#[derive(Clone, Copy, Debug, PartialEq, Eq)]
151pub enum NeighborhoodEvent {
152 /// No event; counter does not increment.
153 NoEvent = 0,
154 /// Any Minion sent an ET Link request.
155 EtLinkSend = 1,
156 /// Any Minion received an ET Link response.
157 EtLinkRecv = 2,
158 /// A cooperative load request was sent.
159 CoopLoadSend = 3,
160 /// An inter-neighbourhood cooperative load request was sent.
161 CoopLoadInterNeighSend = 4,
162 /// A cooperative load response was received.
163 CoopLoadRecv = 5,
164 /// A cooperative store request was sent.
165 CoopStoreSend = 6,
166 /// A cooperative store response was received.
167 CoopStoreRecv = 7,
168 /// Any Minion sent a request to the I-cache.
169 IcacheReqSend = 8,
170 /// Any Minion received a response from the I-cache.
171 IcacheRespRecv = 9,
172 /// Any Minion sent a request to the page table walker.
173 PtwReqSend = 10,
174 /// Any Minion received a response from the page table walker.
175 PtwRespRecv = 11,
176 /// A message was sent between Minions through the FLN.
177 FlnMsg = 12,
178 /// The I-cache sent an ET Link request.
179 IcacheEtLinkSend = 13,
180 /// The I-cache received an ET Link response.
181 IcacheEtLinkRecv = 14,
182 /// The I-cache sent a request to the L1 data SRAM.
183 IcacheL1Req = 15,
184 /// The I-cache received a response from the L1 data SRAM.
185 IcacheL1Resp = 16,
186 /// Any PTW sent an ET Link request.
187 PtwEtLinkSend = 17,
188 /// Any PTW received an ET Link response.
189 PtwEtLinkRecv = 18,
190 // Codes 19-20 are reserved.
191 /// An ET Link request was pushed into the intermediate FIFO.
192 EtLinkFifoIn = 21,
193 /// An ET Link request was pushed into any BANK/UC FIFO.
194 EtLinkBankFifoIn = 22,
195 /// An ET Link response was received from the SC/UC input.
196 EtLinkScUcIn = 23,
197}
198
199// ---------------------------------------------------------------------------
200// CSR read helper macro
201// ---------------------------------------------------------------------------
202
203// Reads an hpmcounterN CSR where N is a compile-time literal, with the
204// RTLMIN-6496 workaround: four back-to-back reads of the same CSR in a
205// 16-byte-aligned block. The first three reads are discarded; the fourth
206// is the architecturally correct value. `.align 4` aligns the block to
207// 2^4 = 16 bytes. `nomem` is omitted so the compiler treats the block as
208// a potential memory barrier, preventing it from reordering other
209// loads/stores across the four reads.
210macro_rules! csr_read {
211 ($csr:literal) => {{
212 let v: u64;
213 // SAFETY: csrrs with rs1 = x0 reads without side effect.
214 // The four reads must be consecutive in the instruction stream;
215 // placing them in one asm block prevents the compiler inserting
216 // any intervening instructions.
217 unsafe {
218 asm!(
219 ".align 4",
220 concat!("csrrs {v0}, ", stringify!($csr), ", x0"),
221 concat!("csrrs {v1}, ", stringify!($csr), ", x0"),
222 concat!("csrrs {v2}, ", stringify!($csr), ", x0"),
223 concat!("csrrs {v}, ", stringify!($csr), ", x0"),
224 v0 = out(reg) _,
225 v1 = out(reg) _,
226 v2 = out(reg) _,
227 v = out(reg) v,
228 options(nostack, preserves_flags),
229 );
230 }
231 v
232 }};
233}
234
235// ---------------------------------------------------------------------------
236// Public API
237// ---------------------------------------------------------------------------
238
239/// Read the `mcycle` counter (CSR `0xC00`).
240///
241/// **Always returns 0 on the ET-SoC-1.** The standard `mcycle` CSR is
242/// permanently tied to zero on this implementation (PRM section 1.3.2).
243/// To count clock cycles, read `hpmcounter3` via [`pmu_read`]`(3)` after
244/// firmware has assigned [`PmuEvent::Cycles`] to `mhpmevent3`. On aifoundry3
245/// the firmware assigns this event by default; [`crate::timestamp`] relies on
246/// it.
247#[inline(always)]
248pub fn pmu_read_cycle() -> u64 {
249 csr_read!(0xC00)
250}
251
252/// Read the `minstret` counter (CSR `0xC02`).
253///
254/// **Always returns 0 on the ET-SoC-1.** The standard `minstret` CSR is
255/// permanently tied to zero on this implementation (PRM section 1.3.2).
256/// To count retired instructions, read an `hpmcounter3`-`hpmcounter6` via
257/// [`pmu_read`] after firmware has assigned [`PmuEvent::RetiredInst0`] or
258/// [`PmuEvent::RetiredInst1`] to the corresponding `mhpmeventN`.
259#[inline(always)]
260pub fn pmu_read_instret() -> u64 {
261 csr_read!(0xC02)
262}
263
264/// Read hardware performance counter `N` (`hpmcounterN`, CSR `0xC03 + (N-3)`).
265///
266/// `counter` must be in `3..=8` on the ET-SoC-1; counters 9-31 are tied to 0
267/// by the hardware. Values outside `3..=31` also return 0.
268///
269/// The semantics of the count depend on the event assigned to counter N by
270/// firmware via `mhpmeventN`:
271/// - Counters 3-6: Minion-level events from [`PmuEvent`].
272/// - Counters 7-8: neighbourhood-level events from [`NeighborhoodEvent`].
273///
274/// Counter 3 (`hpmcounter3`, CSR `0xC03`) is also used by [`crate::timestamp`].
275#[inline(always)]
276#[rustfmt::skip] // tabular CSR-to-counter dispatch; keep aligned
277pub fn pmu_read(counter: u8) -> u64 {
278 match counter {
279 3 => csr_read!(0xC03),
280 4 => csr_read!(0xC04),
281 5 => csr_read!(0xC05),
282 6 => csr_read!(0xC06),
283 7 => csr_read!(0xC07),
284 8 => csr_read!(0xC08),
285 9 => csr_read!(0xC09),
286 10 => csr_read!(0xC0A),
287 11 => csr_read!(0xC0B),
288 12 => csr_read!(0xC0C),
289 13 => csr_read!(0xC0D),
290 14 => csr_read!(0xC0E),
291 15 => csr_read!(0xC0F),
292 16 => csr_read!(0xC10),
293 17 => csr_read!(0xC11),
294 18 => csr_read!(0xC12),
295 19 => csr_read!(0xC13),
296 20 => csr_read!(0xC14),
297 21 => csr_read!(0xC15),
298 22 => csr_read!(0xC16),
299 23 => csr_read!(0xC17),
300 24 => csr_read!(0xC18),
301 25 => csr_read!(0xC19),
302 26 => csr_read!(0xC1A),
303 27 => csr_read!(0xC1B),
304 28 => csr_read!(0xC1C),
305 29 => csr_read!(0xC1D),
306 30 => csr_read!(0xC1E),
307 31 => csr_read!(0xC1F),
308 _ => 0,
309 }
310}
311
312// ---------------------------------------------------------------------------
313// Compile-time discriminant checks (PRM section 1.3.2, Tables 1-3 and 1-4).
314// Placed outside #[cfg(test)] so they are verified on every build, including
315// cross-compilation for the RISC-V target.
316// ---------------------------------------------------------------------------
317
318#[rustfmt::skip] // tabular discriminant checks; keep the columns aligned
319const _: () = {
320 assert!(PmuEvent::NoEvent as u64 == 0);
321 assert!(PmuEvent::Cycles as u64 == 1);
322 assert!(PmuEvent::RetiredInst0 as u64 == 2);
323 assert!(PmuEvent::RetiredInst1 as u64 == 3);
324 assert!(PmuEvent::Branches0 as u64 == 4);
325 assert!(PmuEvent::Branches1 as u64 == 5);
326 assert!(PmuEvent::DcacheAccess0 as u64 == 6);
327 assert!(PmuEvent::DcacheAccess1 as u64 == 7);
328 assert!(PmuEvent::DcacheMisses0 as u64 == 8);
329 assert!(PmuEvent::DcacheMisses1 as u64 == 9);
330 assert!(PmuEvent::L2MissReq as u64 == 10);
331 assert!(PmuEvent::L2MissReqRej as u64 == 11);
332 assert!(PmuEvent::L2EvictReq as u64 == 12);
333 assert!(PmuEvent::L2EvictReqRej as u64 == 13);
334 assert!(PmuEvent::TlInst as u64 == 14);
335 assert!(PmuEvent::TlOps as u64 == 15);
336 assert!(PmuEvent::TsInst as u64 == 16);
337 assert!(PmuEvent::TsOps as u64 == 17);
338 assert!(PmuEvent::TfmaWaitTenb as u64 == 18);
339 assert!(PmuEvent::TimaOps as u64 == 19);
340 assert!(PmuEvent::TxFma3216Ops as u64 == 20);
341 assert!(PmuEvent::TxFma32Ops as u64 == 21);
342 assert!(PmuEvent::TxFmaIntOps as u64 == 22);
343 assert!(PmuEvent::TransOps as u64 == 23);
344 assert!(PmuEvent::ShortOps as u64 == 24);
345 assert!(PmuEvent::MaskOps as u64 == 25);
346 assert!(PmuEvent::TfmaInst as u64 == 26);
347 assert!(PmuEvent::TreduceInst as u64 == 27);
348 assert!(PmuEvent::TquantInst as u64 == 28);
349
350 assert!(NeighborhoodEvent::NoEvent as u64 == 0);
351 assert!(NeighborhoodEvent::EtLinkSend as u64 == 1);
352 assert!(NeighborhoodEvent::EtLinkRecv as u64 == 2);
353 assert!(NeighborhoodEvent::CoopLoadSend as u64 == 3);
354 assert!(NeighborhoodEvent::CoopLoadInterNeighSend as u64 == 4);
355 assert!(NeighborhoodEvent::CoopLoadRecv as u64 == 5);
356 assert!(NeighborhoodEvent::CoopStoreSend as u64 == 6);
357 assert!(NeighborhoodEvent::CoopStoreRecv as u64 == 7);
358 assert!(NeighborhoodEvent::IcacheReqSend as u64 == 8);
359 assert!(NeighborhoodEvent::IcacheRespRecv as u64 == 9);
360 assert!(NeighborhoodEvent::PtwReqSend as u64 == 10);
361 assert!(NeighborhoodEvent::PtwRespRecv as u64 == 11);
362 assert!(NeighborhoodEvent::FlnMsg as u64 == 12);
363 assert!(NeighborhoodEvent::IcacheEtLinkSend as u64 == 13);
364 assert!(NeighborhoodEvent::IcacheEtLinkRecv as u64 == 14);
365 assert!(NeighborhoodEvent::IcacheL1Req as u64 == 15);
366 assert!(NeighborhoodEvent::IcacheL1Resp as u64 == 16);
367 assert!(NeighborhoodEvent::PtwEtLinkSend as u64 == 17);
368 assert!(NeighborhoodEvent::PtwEtLinkRecv as u64 == 18);
369 assert!(NeighborhoodEvent::EtLinkFifoIn as u64 == 21);
370 assert!(NeighborhoodEvent::EtLinkBankFifoIn as u64 == 22);
371 assert!(NeighborhoodEvent::EtLinkScUcIn as u64 == 23);
372};