Skip to main content

gpu_async_reactor/backend/
wordwait.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! The user-mode wait on a [`CompletionLine`] that another agent (a GPU's store over a coherent
4//! fabric, or another thread) writes. The CPU arms a monitor on the line, re-checks the
5//! condition, and sleeps in hardware until the line is written or a bound passes:
6//!
7//! | target | arm | wait | bound |
8//! |---|---|---|---|
9//! | aarch64 Linux | `ldxr` (exclusive monitor; granule = the 64-byte line) | `wfet` (FEAT_WFxT) or `wfe` | counter deadline, or the kernel's ~100 us event stream |
10//! | x86-64 Linux, Intel WAITPKG | `umonitor` | `umwait` (C0.2) | TSC deadline |
11//! | x86-64 Linux, AMD MONITORX | `monitorx` | `mwaitx` with the timer enabled | TSC-rate timeout |
12//!
13//! Measured on GB10 (aarch64): the wake follows the store by 0.3-0.4 us; a 10 ms wait takes
14//! ~95 parks at a 100 us bound (README, benchmarks). The x86 paths are compile-checked only.
15//!
16//! Invariants: every public item is safe; a park only reads the line through its atomics; a park
17//! is bounded; [`WordWait::probe`] refuses a CPU that lacks the instructions.
18
19use core::time::Duration;
20
21use crate::core::line::CompletionLine;
22
23#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
24#[path = "wordwait_arm64.rs"]
25mod arch;
26
27#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
28#[path = "wordwait_x86.rs"]
29mod arch;
30
31#[cfg(not(any(
32    all(target_arch = "aarch64", target_os = "linux"),
33    all(target_arch = "x86_64", target_os = "linux")
34)))]
35mod arch {
36    use super::{CompletionLine, Duration};
37
38    /// No user-mode word wait on this target.
39    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
40    pub struct WordWait {
41        _none: (),
42    }
43
44    impl WordWait {
45        pub(super) fn probe_arch() -> Result<Self, &'static str> {
46            Err("no user-mode word wait on this target (aarch64 or x86-64 Linux only)")
47        }
48
49        pub(super) fn park_arch(
50            &self,
51            line: &CompletionLine,
52            ready: impl Fn(&CompletionLine) -> bool,
53            _bound: Duration,
54        ) -> bool {
55            ready(line)
56        }
57    }
58}
59
60pub use arch::WordWait;
61#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
62pub use arch::Flavor;
63
64impl WordWait {
65    /// This CPU's word wait, or why there is none. Probe once and keep the value.
66    ///
67    /// # Errors
68    /// The CPU or target lacks a user-mode monitor/wait.
69    pub fn probe() -> Result<Self, &'static str> {
70        Self::probe_arch()
71    }
72
73    /// Park on `line` until `ready(line)` holds, a store to the line wakes the core, or `bound`
74    /// passes; returns `ready(line)`. Any store to the line wakes the park, so one waiter can
75    /// watch every slot; a single-word wait is
76    /// `ready = |l| l.words[slot].load(Acquire) == expect`. May return `false` early.
77    #[must_use]
78    pub fn park(
79        &self,
80        line: &CompletionLine,
81        ready: impl Fn(&CompletionLine) -> bool,
82        bound: Duration,
83    ) -> bool {
84        self.park_arch(line, ready, bound)
85    }
86}