gpu_async_reactor/backend/wordwait_arm64.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! aarch64 Linux: `ldxr` arms the core's exclusive monitor on the line; a store to any word of
4//! it clears the monitor and raises the event `wfe`/`wfet` waits for. The monitor is left armed
5//! on return (as Linux's `__cmpwait`): a `clrex` would itself raise the event and make the next
6//! park return at once.
7
8use core::arch::asm;
9use core::time::Duration;
10
11use crate::core::line::CompletionLine;
12
13/// `HWCAP2_WFXT` (arm64 `asm/hwcap.h`): FEAT_WFxT, the timed wait-for-event.
14const HWCAP2_WFXT: u64 = 1 << 31;
15
16/// This core's word-wait capabilities, read once. A value is proof that the core has the
17/// instructions [`WordWait::park`](super::WordWait::park) executes, so the only way to get one is
18/// [`WordWait::probe`](super::WordWait::probe); it cannot be written by hand (in 0.1.0 the fields
19/// were public, and a forged `timed: true` ran `wfet` on a core without FEAT_WFxT: SIGILL).
20///
21/// ```compile_fail,E0451
22/// use gpu_async_reactor::backend::wordwait::WordWait;
23/// let forged = WordWait { timed: true, counter_hz: 24_000_000 };
24/// ```
25///
26/// ```
27/// use gpu_async_reactor::backend::wordwait::WordWait;
28/// let w = WordWait::probe().expect("every aarch64 Linux core has ldxr + wfe");
29/// assert!(w.counter_hz() > 0);
30/// let _: bool = w.timed();
31/// ```
32#[derive(Clone, Copy, Debug, PartialEq, Eq)]
33pub struct WordWait {
34 timed: bool,
35 counter_hz: u64,
36}
37
38fn counter() -> u64 {
39 let v: u64;
40 // WHY: the virtual counter is a system register only `mrs` reads.
41 // SAFETY: `mrs cntvct_el0` is readable at EL0 on Linux (CNTKCTL_EL1.EL0VCTEN is set by the
42 // kernel for the vDSO); it touches no memory and no flags.
43 // NO SAFE ALTERNATIVE: std's clock is a syscall or vDSO call 20-50x slower than the register,
44 // and the park deadline must be in counter ticks.
45 unsafe {
46 asm!("mrs {v}, cntvct_el0", v = out(reg) v, options(nomem, nostack, preserves_flags))
47 };
48 v
49}
50
51impl WordWait {
52 /// FEAT_WFxT present: a park is bounded by its own deadline (else by the kernel's event
53 /// stream).
54 #[must_use]
55 pub fn timed(&self) -> bool {
56 self.timed
57 }
58
59 /// The generic timer's frequency (`cntfrq_el0`), Hz.
60 #[must_use]
61 pub fn counter_hz(&self) -> u64 {
62 self.counter_hz
63 }
64
65 /// Read the counter frequency and whether FEAT_WFxT is present (every aarch64 Linux core
66 /// has `ldxr` + `wfe`).
67 pub(super) fn probe_arch() -> Result<Self, &'static str> {
68 let hz: u64;
69 // WHY: the counter frequency is a system register only `mrs` reads.
70 // SAFETY: `cntfrq_el0` is readable at EL0 on Linux; no memory, no flags.
71 // NO SAFE ALTERNATIVE: no std or libc call returns the generic timer's frequency.
72 unsafe {
73 asm!("mrs {v}, cntfrq_el0", v = out(reg) hz, options(nomem, nostack, preserves_flags))
74 };
75 // WHY: `getauxval` is a foreign call.
76 // SAFETY: it only reads the process's auxiliary vector and takes a plain integer.
77 // NO SAFE ALTERNATIVE: std exposes no hardware-capability query.
78 let hwcap2 = unsafe { libc::getauxval(libc::AT_HWCAP2) };
79 Ok(Self {
80 timed: hwcap2 & HWCAP2_WFXT != 0,
81 counter_hz: hz,
82 })
83 }
84
85 /// See [`WordWait::park`](super::WordWait::park). `ldxr` loads the line's first word and
86 /// arms the exclusive monitor on its granule; then `ready` is checked (so a store before the
87 /// arm is seen, and one after it raises the event); `wfet`/`wfe` waits for that event.
88 pub(super) fn park_arch(
89 &self,
90 line: &CompletionLine,
91 ready: impl Fn(&CompletionLine) -> bool,
92 bound: Duration,
93 ) -> bool {
94 let ticks = u64::try_from(
95 bound.as_nanos().saturating_mul(u128::from(self.counter_hz)) / 1_000_000_000,
96 )
97 .unwrap_or(u64::MAX);
98 let deadline = counter().saturating_add(ticks);
99 let _armed: u64;
100 // WHY: an exclusive load is the only way to arm the monitor the store's event clears.
101 // SAFETY: `line.words[0].as_ptr()` is valid, 64-aligned and live for the borrow; `ldxr`
102 // only reads.
103 // NO SAFE ALTERNATIVE: `AtomicU64::load` compiles to `ldar`/`ldr`, which arm no monitor.
104 unsafe {
105 asm!("ldxr {v}, [{p}]", p = in(reg) line.words[0].as_ptr(), v = out(reg) _armed,
106 options(nostack, preserves_flags, readonly));
107 }
108 if !ready(line) {
109 if self.timed {
110 // WHY: `wfet` (FEAT_WFxT) parks the core until an event or the counter deadline.
111 // SAFETY: it reads only the register operand and waits; no memory access. Encoded
112 // as `.inst` (`wfet x0`) because the assembler may predate the mnemonic.
113 // NO SAFE ALTERNATIVE: no std or libc call waits on the exclusive monitor.
114 unsafe {
115 asm!(".inst 0xd5031000", in("x0") deadline, options(nomem, nostack, preserves_flags))
116 };
117 } else {
118 // WHY: `wfe` parks the core until an event (the store, an interrupt, the
119 // kernel's event stream).
120 // SAFETY: it only waits; no memory access.
121 // NO SAFE ALTERNATIVE: as above.
122 unsafe { asm!("wfe", options(nomem, nostack, preserves_flags)) };
123 }
124 }
125 // No `clrex` here. Clearing the monitor moves it from exclusive to open, which
126 // itself raises the event, so the next park's `wfe` returned at once and every park spun
127 // (~60 ns per call, ~1e5 per 10 ms). As Linux's `__cmpwait`, the monitor is left armed:
128 // every `stxr` the compiler emits follows its own `ldxr`, which re-arms it.
129 ready(line)
130 }
131}
132
133#[cfg(test)]
134mod tests {
135 use super::*;
136 extern crate std;
137 use core::sync::atomic::Ordering;
138
139 fn slot_is(slot: usize, v: u64) -> impl Fn(&CompletionLine) -> bool {
140 move |l| l.words[slot].load(Ordering::Acquire) == v
141 }
142
143 #[test]
144 fn a_slot_already_at_expect_returns_true_at_once() {
145 let w = WordWait::probe().expect("aarch64 Linux has a word wait");
146 assert!(w.counter_hz > 0);
147 let line = CompletionLine::default();
148 line.words[5].store(7, Ordering::Release);
149 let t = std::time::Instant::now();
150 assert!(w.park(&line, slot_is(5, 7), Duration::from_millis(50)));
151 assert!(
152 t.elapsed() < Duration::from_millis(5),
153 "must not park when the slot is set"
154 );
155 }
156
157 #[test]
158 fn a_slot_never_written_returns_false_within_the_bound() {
159 let w = WordWait::probe().expect("aarch64 Linux has a word wait");
160 let line = CompletionLine::default();
161 let t = std::time::Instant::now();
162 let mut fired = false;
163 while t.elapsed() < Duration::from_millis(2) {
164 fired |= w.park(&line, slot_is(0, 1), Duration::from_micros(100));
165 }
166 assert!(!fired, "an unwritten slot must never read as written");
167 }
168
169 #[test]
170 fn a_store_to_another_slot_of_the_line_wakes_the_park() {
171 // The monitor is armed on word 0; the writer stores word 6: the line-wide granule must
172 // still wake the park (the multiplexing case).
173 let w = WordWait::probe().expect("aarch64 Linux has a word wait");
174 let line = std::sync::Arc::new(CompletionLine::default());
175 let writer = {
176 let line = std::sync::Arc::clone(&line);
177 std::thread::spawn(move || {
178 std::thread::sleep(Duration::from_millis(2));
179 line.words[6].store(5, Ordering::Release);
180 })
181 };
182 let t = std::time::Instant::now();
183 while !w.park(&line, slot_is(6, 5), Duration::from_millis(1)) {
184 assert!(
185 t.elapsed() < Duration::from_secs(2),
186 "the store never woke the park"
187 );
188 }
189 writer.join().expect("writer");
190 }
191 /// Park calls until a store 10 ms out, on `w`.
192 fn parks_until_a_store_in_10_ms(w: WordWait) -> u64 {
193 let line = std::sync::Arc::new(CompletionLine::default());
194 let writer = {
195 let line = std::sync::Arc::clone(&line);
196 std::thread::spawn(move || {
197 std::thread::sleep(Duration::from_millis(10));
198 line.words[0].store(1, Ordering::Release);
199 })
200 };
201 let mut parks = 0u64;
202 while !w.park(&line, slot_is(0, 1), Duration::from_micros(100)) {
203 parks += 1;
204 }
205 writer.join().expect("writer");
206 parks
207 }
208
209 #[test]
210 fn a_park_sleeps_until_the_store_instead_of_spinning() {
211 // Each park returns on the store, its 100 us bound or the kernel's event stream: ~100
212 // calls in 10 ms on GB10 (10 kHz stream), ~450 on a GitHub Neoverse runner (faster
213 // stream). A park that spins returns in ~60 ns: ~1e5 calls (the `clrex` bug,
214 // 2026-11-14, measured 160,636). The bound sits 20x under a spin and >10x over the
215 // fastest event stream seen, so it separates the two on any host. Both the timed and
216 // the plain arm, back to back (the monitor state of one park must not wake the next).
217 let probed = WordWait::probe().expect("aarch64 Linux has a word wait");
218 let plain = WordWait {
219 timed: false,
220 ..probed
221 };
222 for w in [probed, plain, probed] {
223 let parks = parks_until_a_store_in_10_ms(w);
224 assert!(parks <= 5_000, "{parks} parks in ~10 ms (timed {}): the park spins", w.timed);
225 }
226 }
227}