ax_cpu/arch/aarch64/pmu/
mod.rs

1//! PMUv3 hardware access. Linux reference: v7.1 arm_pmuv3.c at
2//! 8cd9520d35a6c38db6567e97dd93b1f11f185dc6.
3
4use core::{arch::asm, marker::PhantomData};
5
6mod capability;
7mod registers;
8pub use capability::{EventSupport, PmuInfo};
9
10macro_rules! read_reg {
11    ($register:literal) => {{
12        let value: u64;
13        // SAFETY: the enclosing PMU session owns this CPU's register access.
14        unsafe { asm!(concat!("mrs {}, ", $register), out(reg) value, options(nomem, nostack)); }
15        value
16    }};
17}
18
19macro_rules! write_reg {
20    ($register:literal, $value:expr) => {{
21        // SAFETY: the session has exclusive access and supplies defined bits.
22        unsafe { asm!(concat!("msr ", $register, ", {}"), in(reg) $value as u64, options(nostack)); }
23    }};
24}
25
26mod access;
27mod counter;
28mod overflow;
29
30fn isb() {
31    // SAFETY: instruction synchronization does not access memory.
32    unsafe {
33        asm!("isb", options(nostack));
34    }
35}
36
37/// A requested PMU operation could not be performed.
38#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
39pub enum PmuError {
40    /// This CPU does not implement architectural PMUv3.
41    #[error("PMUv3 is unavailable on this CPU")]
42    Unavailable,
43    /// The counter index is not implemented on this CPU.
44    #[error("counter is not implemented on this CPU")]
45    InvalidCounter,
46    /// The event is architecturally reported as unsupported.
47    #[error("event is reported as unsupported")]
48    UnsupportedEvent,
49    /// The configuration does not apply to the selected counter.
50    #[error("configuration does not apply to this counter")]
51    InvalidConfiguration,
52    /// The requested period is zero or exceeds the active counter width.
53    #[error("period is zero or exceeds the counter width")]
54    InvalidPeriod,
55}
56
57/// Validated hardware counter selector.
58#[derive(Clone, Copy, Debug, Eq, PartialEq)]
59pub struct CounterId(u8);
60
61impl CounterId {
62    /// The dedicated cycle counter at architectural index 31.
63    pub const CYCLE: Self = Self(31);
64
65    /// Dedicated PMICNTR at architectural index 32, when independently present.
66    pub const INSTRUCTIONS: Self = Self(32);
67
68    /// Returns the architectural index, also used in overflow masks.
69    pub const fn index(self) -> usize {
70        self.0 as usize
71    }
72}
73
74/// PMUv3 event code and privilege filters.
75#[derive(Clone, Copy, Debug, Eq, PartialEq)]
76pub struct EventConfig {
77    /// Raw architectural or implementation-defined event code.
78    pub event: u16,
79    /// Excludes execution at EL0.
80    pub exclude_user: bool,
81    /// Excludes execution at EL1.
82    pub exclude_kernel: bool,
83    /// Includes execution at EL2.
84    pub include_hypervisor: bool,
85}
86
87/// Exclusive access to the current CPU's PMU for one non-migrating scope.
88/// This object does not allocate counters or implicitly reset them on drop.
89pub struct Pmu {
90    info: PmuInfo,
91    _not_send_sync: PhantomData<*mut ()>,
92}
93
94impl Pmu {
95    /// Probes PMUv3 without changing hardware state.
96    ///
97    /// # Safety
98    /// The caller must run at a privileged level with PMU access permitted by
99    /// higher exception levels. It must keep this CPU pinned and serialize all
100    /// PMU access, including local IRQs, for the complete returned session.
101    pub unsafe fn current() -> Result<Self, PmuError> {
102        let version = ((read_reg!("ID_AA64DFR0_EL1") >> 8) & 15) as u8;
103        if version == 0 || version == 15 {
104            return Err(PmuError::Unavailable);
105        }
106        let pmcr = read_reg!("PMCR_EL0");
107        Ok(Self {
108            info: PmuInfo {
109                version,
110                num_counters: ((pmcr >> 11) & 31) as usize,
111                has_instruction_counter: (read_reg!("ID_AA64DFR1_EL1") >> 36) & 15 != 0,
112                counter_width: if version >= 6 && pmcr & (1 << 7) != 0 {
113                    64
114                } else {
115                    32
116                },
117                cycle_counter_width: if pmcr & (1 << 6) != 0 { 64 } else { 32 },
118                pmceid0: read_reg!("PMCEID0_EL0"),
119                pmceid1: read_reg!("PMCEID1_EL0"),
120            },
121            _not_send_sync: PhantomData,
122        })
123    }
124
125    /// Returns the capability snapshot for this session's CPU.
126    pub const fn info(&self) -> PmuInfo {
127        self.info
128    }
129
130    /// Validates a programmable counter index on this CPU.
131    pub fn counter(&self, index: usize) -> Result<CounterId, PmuError> {
132        if index < self.info.num_counters {
133            Ok(CounterId(index as u8))
134        } else {
135            Err(PmuError::InvalidCounter)
136        }
137    }
138
139    fn validate(&self, id: CounterId) -> Result<(), PmuError> {
140        if id == CounterId::CYCLE
141            || (id == CounterId::INSTRUCTIONS && self.info.has_instruction_counter)
142            || id.index() < self.info.num_counters
143        {
144            Ok(())
145        } else {
146            Err(PmuError::InvalidCounter)
147        }
148    }
149
150    /// Returns the active overflow width of this counter.
151    pub fn width(&self, id: CounterId) -> Result<u8, PmuError> {
152        self.validate(id)?;
153        Ok(if id == CounterId::INSTRUCTIONS {
154            64
155        } else if id == CounterId::CYCLE {
156            self.info.cycle_counter_width
157        } else {
158            self.info.counter_width
159        })
160    }
161
162    fn mask(&self, id: CounterId) -> Result<u64, PmuError> {
163        Ok(u64::MAX >> (64 - self.width(id)?))
164    }
165
166    fn implemented_mask(&self) -> u64 {
167        ((1u64 << self.info.num_counters) - 1)
168            | (1u64 << 31)
169            | (u64::from(self.info.has_instruction_counter) << 32)
170    }
171
172    /// Returns whether global PMU counting is enabled on this CPU.
173    pub fn is_running(&self) -> bool {
174        read_reg!("PMCR_EL0") & 1 != 0
175    }
176
177    /// Pauses global counting for a bounded snapshot and restores its prior state.
178    /// Counter enables, values, overflow state and user permissions are retained.
179    ///
180    /// # Safety
181    /// The caller must own the complete local PMU scheduling domain: pausing
182    /// must be permitted for every configured event. The callback must not
183    /// block, enable IRQs, migrate, or create another PMU access session.
184    pub unsafe fn with_counting_paused<R>(&mut self, operation: impl FnOnce(&mut Self) -> R) -> R {
185        struct Restore<'a> {
186            pmu: &'a mut Pmu,
187            running: bool,
188        }
189        impl Drop for Restore<'_> {
190            fn drop(&mut self) {
191                if self.running {
192                    self.pmu.start();
193                } else {
194                    self.pmu.stop();
195                }
196            }
197        }
198        let running = self.is_running();
199        self.stop();
200        let restore = Restore { pmu: self, running };
201        operation(restore.pmu)
202    }
203
204    /// Enables global counting without resetting or reallocating any counter.
205    pub fn start(&mut self) {
206        let value = (read_reg!("PMCR_EL0") & 0xf9) | 1;
207        isb();
208        write_reg!("PMCR_EL0", value);
209    }
210
211    /// Stops global counting without changing counter values.
212    pub fn stop(&mut self) {
213        let value = read_reg!("PMCR_EL0") & 0xf8;
214        isb();
215        write_reg!("PMCR_EL0", value);
216        isb();
217    }
218
219    /// Resets all counters while globally stopped and disables user access.
220    /// Long programmable counters are enabled when PMUv3p5 supports them.
221    ///
222    /// # Safety
223    /// The caller must own every counter on this CPU and have withdrawn all
224    /// event users. No active perf or guest owner may retain counter state.
225    pub unsafe fn reset(&mut self) {
226        let mask = self.implemented_mask();
227        write_reg!("PMCNTENCLR_EL0", mask);
228        isb();
229        write_reg!("PMINTENCLR_EL1", mask);
230        isb();
231        write_reg!("PMOVSCLR_EL0", mask);
232        isb();
233        write_reg!("PMUSERENR_EL0", 0);
234        let value = 2
235            | 4
236            | 64
237            | if self.info.has_long_counters() {
238                128
239            } else {
240                0
241            };
242        isb();
243        write_reg!("PMCR_EL0", value);
244        isb();
245        if self.info.has_instruction_counter {
246            write_reg!("S3_3_C9_C4_0", 0); // PMICNTR_EL0
247        }
248        self.info.counter_width = if self.info.has_long_counters() {
249            64
250        } else {
251            32
252        };
253        self.info.cycle_counter_width = 64;
254    }
255}
ax_cpu/arch/aarch64/pmu/mod.rs

ax_cpu/arch/aarch64/pmu/
mod.rs
