gpu_async_reactor/backend/wordwait.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! The user-mode wait on a [`CompletionLine`] that another agent (a GPU's store over a coherent
4//! fabric, or another thread) writes. The CPU arms a monitor on the line, re-checks the
5//! condition, and sleeps in hardware until the line is written or a bound passes:
6//!
7//! | target | arm | wait | bound |
8//! |---|---|---|---|
9//! | aarch64 Linux | `ldxr` (exclusive monitor; granule = the 64-byte line) | `wfet` (FEAT_WFxT) or `wfe` | counter deadline, or the kernel's ~100 us event stream |
10//! | x86-64 Linux, Intel WAITPKG | `umonitor` | `umwait` (C0.2) | TSC deadline |
11//! | x86-64 Linux, AMD MONITORX | `monitorx` | `mwaitx` with the timer enabled | TSC-rate timeout |
12//!
13//! Measured on GB10 (aarch64): the wake follows the store by 0.3-0.4 us; a 10 ms wait takes
14//! ~95 parks at a 100 us bound (README, benchmarks). The x86 paths are compile-checked only.
15//!
16//! Invariants: every public item is safe; a park only reads the line through its atomics; a park
17//! is bounded; [`WordWait::probe`] refuses a CPU that lacks the instructions.
18
19use core::time::Duration;
20
21use crate::core::line::CompletionLine;
22
23#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
24#[path = "wordwait_arm64.rs"]
25mod arch;
26
27#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
28#[path = "wordwait_x86.rs"]
29mod arch;
30
31#[cfg(not(any(
32 all(target_arch = "aarch64", target_os = "linux"),
33 all(target_arch = "x86_64", target_os = "linux")
34)))]
35mod arch {
36 use super::{CompletionLine, Duration};
37
38 /// No user-mode word wait on this target.
39 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
40 pub struct WordWait {
41 _none: (),
42 }
43
44 impl WordWait {
45 pub(super) fn probe_arch() -> Result<Self, &'static str> {
46 Err("no user-mode word wait on this target (aarch64 or x86-64 Linux only)")
47 }
48
49 pub(super) fn park_arch(
50 &self,
51 line: &CompletionLine,
52 ready: impl Fn(&CompletionLine) -> bool,
53 _bound: Duration,
54 ) -> bool {
55 ready(line)
56 }
57 }
58}
59
60pub use arch::WordWait;
61#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
62pub use arch::Flavor;
63
64impl WordWait {
65 /// This CPU's word wait, or why there is none. Probe once and keep the value.
66 ///
67 /// # Errors
68 /// The CPU or target lacks a user-mode monitor/wait.
69 pub fn probe() -> Result<Self, &'static str> {
70 Self::probe_arch()
71 }
72
73 /// Park on `line` until `ready(line)` holds, a store to the line wakes the core, or `bound`
74 /// passes; returns `ready(line)`. Any store to the line wakes the park, so one waiter can
75 /// watch every slot; a single-word wait is
76 /// `ready = |l| l.words[slot].load(Acquire) == expect`. May return `false` early.
77 #[must_use]
78 pub fn park(
79 &self,
80 line: &CompletionLine,
81 ready: impl Fn(&CompletionLine) -> bool,
82 bound: Duration,
83 ) -> bool {
84 self.park_arch(line, ready, bound)
85 }
86}