ax_cpu/arch/aarch64/pmu/
mod.rs

1//! PMUv3 hardware access. Linux reference: v7.1 arm_pmuv3.c at
2//! 8cd9520d35a6c38db6567e97dd93b1f11f185dc6.
3
4use core::{arch::asm, marker::PhantomData};
5
6mod capability;
7mod registers;
8pub use capability::{EventSupport, PmuInfo};
9
10macro_rules! read_reg {
11    ($register:literal) => {{
12        let value: u64;
13        // SAFETY: the enclosing PMU session owns this CPU's register access.
14        unsafe { asm!(concat!("mrs {}, ", $register), out(reg) value, options(nomem, nostack)); }
15        value
16    }};
17}
18
19macro_rules! write_reg {
20    ($register:literal, $value:expr) => {{
21        // SAFETY: the session has exclusive access and supplies defined bits.
22        unsafe { asm!(concat!("msr ", $register, ", {}"), in(reg) $value as u64, options(nostack)); }
23    }};
24}
25
26mod access;
27mod counter;
28mod overflow;
29
30fn isb() {
31    // SAFETY: instruction synchronization does not access memory.
32    unsafe {
33        asm!("isb", options(nostack));
34    }
35}
36
37/// A requested PMU operation could not be performed.
38#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
39pub enum PmuError {
40    /// This CPU does not implement architectural PMUv3.
41    #[error("PMUv3 is unavailable on this CPU")]
42    Unavailable,
43    /// The counter index is not implemented on this CPU.
44    #[error("counter is not implemented on this CPU")]
45    InvalidCounter,
46    /// The event is architecturally reported as unsupported.
47    #[error("event is reported as unsupported")]
48    UnsupportedEvent,
49    /// The configuration does not apply to the selected counter.
50    #[error("configuration does not apply to this counter")]
51    InvalidConfiguration,
52    /// The requested period is zero or exceeds the active counter width.
53    #[error("period is zero or exceeds the counter width")]
54    InvalidPeriod,
55}
56
57/// Validated hardware counter selector.
58#[derive(Clone, Copy, Debug, Eq, PartialEq)]
59pub struct CounterId(u8);
60
61impl CounterId {
62    /// The dedicated cycle counter at architectural index 31.
63    pub const CYCLE: Self = Self(31);
64
65    /// Dedicated PMICNTR at architectural index 32, when independently present.
66    pub const INSTRUCTIONS: Self = Self(32);
67
68    /// Returns the architectural index, also used in overflow masks.
69    pub const fn index(self) -> usize {
70        self.0 as usize
71    }
72}
73
74/// PMUv3 event code and privilege filters.
75#[derive(Clone, Copy, Debug, Eq, PartialEq)]
76pub struct EventConfig {
77    /// Raw architectural or implementation-defined event code.
78    pub event: u16,
79    /// Excludes execution at EL0.
80    pub exclude_user: bool,
81    /// Excludes execution at EL1.
82    pub exclude_kernel: bool,
83    /// Includes execution at EL2.
84    pub include_hypervisor: bool,
85}
86
87/// Exclusive access to the current CPU's PMU for one non-migrating scope.
88/// This object does not allocate counters or implicitly reset them on drop.
89pub struct Pmu {
90    info: PmuInfo,
91    _not_send_sync: PhantomData<*mut ()>,
92}
93
94impl Pmu {
95    /// Probes PMUv3 without changing hardware state.
96    ///
97    /// # Safety
98    /// The caller must run at a privileged level with PMU access permitted by
99    /// higher exception levels. It must keep this CPU pinned and serialize all
100    /// PMU access, including local IRQs, for the complete returned session.
101    pub unsafe fn current() -> Result<Self, PmuError> {
102        let version = ((read_reg!("ID_AA64DFR0_EL1") >> 8) & 15) as u8;
103        if version == 0 || version == 15 {
104            return Err(PmuError::Unavailable);
105        }
106        let pmcr = read_reg!("PMCR_EL0");
107        Ok(Self {
108            info: PmuInfo {
109                midr: crate::capability::read_midr_el1(),
110                version,
111                num_counters: ((pmcr >> 11) & 31) as usize,
112                has_instruction_counter: (read_reg!("ID_AA64DFR1_EL1") >> 36) & 15 != 0,
113                counter_width: if version >= 6 && pmcr & (1 << 7) != 0 {
114                    64
115                } else {
116                    32
117                },
118                cycle_counter_width: if pmcr & (1 << 6) != 0 { 64 } else { 32 },
119                pmceid0: read_reg!("PMCEID0_EL0"),
120                pmceid1: read_reg!("PMCEID1_EL0"),
121            },
122            _not_send_sync: PhantomData,
123        })
124    }
125
126    /// Returns the capability snapshot for this session's CPU.
127    pub const fn info(&self) -> PmuInfo {
128        self.info
129    }
130
131    /// Validates a programmable counter index on this CPU.
132    pub fn counter(&self, index: usize) -> Result<CounterId, PmuError> {
133        if index < self.info.num_counters {
134            Ok(CounterId(index as u8))
135        } else {
136            Err(PmuError::InvalidCounter)
137        }
138    }
139
140    fn validate(&self, id: CounterId) -> Result<(), PmuError> {
141        if id == CounterId::CYCLE
142            || (id == CounterId::INSTRUCTIONS && self.info.has_instruction_counter)
143            || id.index() < self.info.num_counters
144        {
145            Ok(())
146        } else {
147            Err(PmuError::InvalidCounter)
148        }
149    }
150
151    /// Returns the active overflow width of this counter.
152    pub fn width(&self, id: CounterId) -> Result<u8, PmuError> {
153        self.validate(id)?;
154        Ok(if id == CounterId::INSTRUCTIONS {
155            64
156        } else if id == CounterId::CYCLE {
157            self.info.cycle_counter_width
158        } else {
159            self.info.counter_width
160        })
161    }
162
163    fn mask(&self, id: CounterId) -> Result<u64, PmuError> {
164        Ok(u64::MAX >> (64 - self.width(id)?))
165    }
166
167    fn implemented_mask(&self) -> u64 {
168        ((1u64 << self.info.num_counters) - 1)
169            | (1u64 << 31)
170            | (u64::from(self.info.has_instruction_counter) << 32)
171    }
172
173    /// Returns whether global PMU counting is enabled on this CPU.
174    pub fn is_running(&self) -> bool {
175        read_reg!("PMCR_EL0") & 1 != 0
176    }
177
178    /// Selects programmable overflow width while the PMU domain is stopped.
179    ///
180    /// # Safety
181    /// The caller must own every counter and have withdrawn all event users.
182    /// Changing width invalidates their preload and software extension state.
183    pub unsafe fn set_long_counters(&mut self, enabled: bool) -> Result<(), PmuError> {
184        if self.is_running() || (enabled && !self.info.has_long_counters()) {
185            return Err(PmuError::InvalidConfiguration);
186        }
187        if self.info.has_long_counters() {
188            // Preserve D/X/DP/LC; leave E/P/C clear and select LP below.
189            let value = (read_reg!("PMCR_EL0") & 0x78) | (u64::from(enabled) << 7);
190            write_reg!("PMCR_EL0", value);
191            isb();
192        }
193        self.info.counter_width = if enabled { 64 } else { 32 };
194        Ok(())
195    }
196
197    /// Pauses global counting for a bounded snapshot and restores its prior state.
198    /// Counter enables, values, overflow state and user permissions are retained.
199    ///
200    /// # Safety
201    /// The caller must own the complete local PMU scheduling domain: pausing
202    /// must be permitted for every configured event. The callback must not
203    /// block, enable IRQs, migrate, or create another PMU access session.
204    pub unsafe fn with_counting_paused<R>(&mut self, operation: impl FnOnce(&mut Self) -> R) -> R {
205        struct Restore<'a> {
206            pmu: &'a mut Pmu,
207            running: bool,
208        }
209        impl Drop for Restore<'_> {
210            fn drop(&mut self) {
211                if self.running {
212                    self.pmu.start();
213                } else {
214                    self.pmu.stop();
215                }
216            }
217        }
218        let running = self.is_running();
219        self.stop();
220        let restore = Restore { pmu: self, running };
221        operation(restore.pmu)
222    }
223
224    /// Enables global counting without resetting or reallocating any counter.
225    pub fn start(&mut self) {
226        let value = (read_reg!("PMCR_EL0") & 0xf9) | 1;
227        isb();
228        write_reg!("PMCR_EL0", value);
229    }
230
231    /// Stops global counting without changing counter values.
232    pub fn stop(&mut self) {
233        let value = read_reg!("PMCR_EL0") & 0xf8;
234        isb();
235        write_reg!("PMCR_EL0", value);
236        isb();
237    }
238
239    /// Resets all counters while globally stopped and disables user access.
240    /// Long programmable counters are enabled when PMUv3p5 supports them.
241    ///
242    /// # Safety
243    /// The caller must own every counter on this CPU and have withdrawn all
244    /// event users. No active perf or guest owner may retain counter state.
245    pub unsafe fn reset(&mut self) {
246        let mask = self.implemented_mask();
247        write_reg!("PMCNTENCLR_EL0", mask);
248        isb();
249        write_reg!("PMINTENCLR_EL1", mask);
250        isb();
251        write_reg!("PMOVSCLR_EL0", mask);
252        isb();
253        write_reg!("PMUSERENR_EL0", 0);
254        let value = 2
255            | 4
256            | 64
257            | if self.info.has_long_counters() {
258                128
259            } else {
260                0
261            };
262        isb();
263        write_reg!("PMCR_EL0", value);
264        isb();
265        if self.info.has_instruction_counter {
266            write_reg!("S3_3_C9_C4_0", 0); // PMICNTR_EL0
267        }
268        self.info.counter_width = if self.info.has_long_counters() {
269            64
270        } else {
271            32
272        };
273        self.info.cycle_counter_width = 64;
274    }
275}
ax_cpu/arch/aarch64/pmu/mod.rs

ax_cpu/arch/aarch64/pmu/
mod.rs
