gpu_async_reactor/backend/mapped.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! The parent of the CUDA and HIP lines: one page-locked, device-mapped host page holding one
4//! [`CompletionLine`], which a stream stores sequence numbers into behind its work. Each vendor
5//! supplies only its driver table ([`LineDriver`]); the ownership rules live here once:
6//! - a store is accepted only from a stream of the domain (CUDA context, HIP device) the line was
7//! made in, so `Drop` can wait for it;
8//! - `Drop` frees the page only after that domain settled; if it cannot, the page is leaked,
9//! never freed under a pending store (CWE-416);
10//! - the host reads the line only through atomics; the device writes only the words named.
11
12#![allow(unsafe_code)]
13
14use core::cell::Cell;
15use core::ffi::c_void;
16use core::ptr::NonNull;
17use core::sync::atomic::Ordering;
18
19use crate::core::line::{CompletionLine, Slot};
20
21/// The page the line lives in: a whole page, so no other allocation shares the line with a
22/// device writer.
23pub(crate) const PAGE: usize = 4096;
24
25/// Why a line could not be made or a store not queued.
26#[derive(Clone, Copy, Debug, PartialEq, Eq)]
27pub enum LineError {
28 /// The driver returned this nonzero status.
29 Status(i32),
30 /// The driver library or one of its entry points is missing.
31 Unavailable,
32 /// No current context/device on the calling thread.
33 NoDomain,
34 /// The stream belongs to another context/device than the line's.
35 ForeignStream,
36}
37
38impl LineError {
39 /// One integer per error, as the line's first users logged them: the driver's status, `-1`
40 /// unavailable, `-3` no domain, `-4` foreign stream.
41 #[must_use]
42 pub fn status(self) -> i32 {
43 match self {
44 Self::Status(s) => s,
45 Self::Unavailable => -1,
46 Self::NoDomain => -3,
47 Self::ForeignStream => -4,
48 }
49 }
50}
51
52impl core::fmt::Display for LineError {
53 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
54 match self {
55 Self::Status(s) => write!(f, "driver status {s}"),
56 Self::Unavailable => f.write_str("the driver or one of its entry points is missing"),
57 Self::NoDomain => f.write_str("no current context on this thread"),
58 Self::ForeignStream => f.write_str("the stream belongs to another context"),
59 }
60 }
61}
62
63impl core::error::Error for LineError {}
64
65impl From<LineError> for crate::Error {
66 fn from(e: LineError) -> Self {
67 crate::error!(Backend, "completion line", e.status())
68 }
69}
70
71/// What a line needs of a vendor driver. Crate-private: the line is its only caller and passes
72/// only values the driver handed out.
73pub(crate) trait LineDriver {
74 /// The vendor's stream handle.
75 type Stream: Copy;
76 /// The calling thread's current domain (context or device).
77 fn domain_current(&self) -> Result<u64, LineError>;
78 /// A page-locked, device-mapped host page of `bytes`.
79 fn alloc_host(&self, bytes: usize) -> Result<NonNull<c_void>, LineError>;
80 /// The device address of `host`, a page from [`Self::alloc_host`] not yet freed.
81 fn device_address(&self, host: NonNull<c_void>) -> Result<u64, LineError>;
82 /// Free a page from [`Self::alloc_host`], once.
83 fn free_host(&self, host: NonNull<c_void>);
84 /// The domain `stream` belongs to.
85 fn stream_domain(&self, stream: Self::Stream) -> Result<u64, LineError>;
86 /// Queue a store of `value` at device address `addr` on `stream`.
87 fn write_value(&self, stream: Self::Stream, addr: u64, value: u64) -> Result<(), LineError>;
88 /// Wait for every queued work of `domain`; `false` if the driver could not.
89 fn settle(&self, domain: u64) -> bool;
90}
91
92/// A device-mapped host line over driver `D`.
93pub(crate) struct Line<D: LineDriver> {
94 host: NonNull<CompletionLine>,
95 device: u64,
96 driver: D,
97 domain: u64,
98 queued: Cell<bool>,
99}
100
101// WHY: a line owns a raw mapped page, which withholds the auto `Send`; the executor moves the owner between threads.
102// SAFETY: the page is owned by this value alone and the host touches it only through atomics
103// (which are `Sync`); the driver table is `Send` by the bound. `queued` is a `Cell`, so the value
104// is `Send` but not `Sync`, which is all a line needs.
105// NO SAFE ALTERNATIVE: a raw-pointer owner is never `Send` by derivation, and wrapping the page would add an indirection on every host read.
106unsafe impl<D: LineDriver + Send> Send for Line<D> {}
107
108impl<D: LineDriver> Line<D> {
109 /// A fresh line reading all 0, mapped for the calling thread's current domain.
110 pub(crate) fn with_driver(driver: D) -> Result<Self, LineError> {
111 let domain = driver.domain_current();
112 let page = driver.alloc_host(PAGE)?;
113 let device = driver.device_address(page);
114 // Built before the checks, so every error below frees the page through `Drop`.
115 let line = Self {
116 host: page.cast::<CompletionLine>(),
117 device: device.unwrap_or(0),
118 driver,
119 domain: domain.unwrap_or(0),
120 queued: Cell::new(false),
121 };
122 device?;
123 if domain? == 0 {
124 return Err(LineError::NoDomain);
125 }
126 for w in &line.line().words {
127 w.store(0, Ordering::Release);
128 }
129 Ok(line)
130 }
131
132 /// The driver table.
133 #[cfg(feature = "cuda")]
134 pub(crate) fn driver(&self) -> &D {
135 &self.driver
136 }
137
138 /// The device address of word 0 (for a stream-side wait on the line).
139 pub(crate) fn device_address(&self) -> u64 {
140 self.device
141 }
142
143 /// The line, as the host reads it.
144 pub(crate) fn line(&self) -> &CompletionLine {
145 // WHY: reads the owned mapped page as a `CompletionLine` without copying it.
146 // SAFETY: `host` is the live, page-aligned (so 64-aligned, as `CompletionLine` needs) page
147 // this value owns until `Drop`; the device writes it only with aligned 8-byte stores to
148 // its words and every host access goes through the atomics, so nothing aliases it
149 // non-atomically.
150 // NO SAFE ALTERNATIVE: a `NonNull<T>` deref is unsafe; the page must stay in place because the device writes it.
151 unsafe { self.host.as_ref() }
152 }
153
154 /// Queue a store of `value` into word `slot` on `stream`, behind the work queued there.
155 pub(crate) fn enqueue_store(
156 &self,
157 stream: D::Stream,
158 slot: Slot,
159 value: u64,
160 ) -> Result<(), LineError> {
161 let offset = 8 * u64::try_from(slot.index()).map_err(|_| LineError::Unavailable)?;
162 if self.driver.stream_domain(stream)? != self.domain {
163 return Err(LineError::ForeignStream);
164 }
165 // Set before the call: a store the driver queued despite a failing status still makes
166 // `Drop` settle first.
167 self.queued.set(true);
168 self.driver.write_value(stream, self.device + offset, value)
169 }
170
171 /// Every store queued into the line has landed (`true` at once when none was queued).
172 fn settled(&self) -> bool {
173 !self.queued.get() || self.driver.settle(self.domain)
174 }
175}
176
177impl<D: LineDriver> Drop for Line<D> {
178 fn drop(&mut self) {
179 if !self.settled() {
180 // A store may still be pending into the page: leak 4 KiB rather than free memory the
181 // device may yet write.
182 return;
183 }
184 self.driver.free_host(self.host.cast::<c_void>());
185 }
186}
187
188#[cfg(test)]
189#[path = "mapped_tests.rs"]
190mod tests;