Skip to main content

gpu_async_reactor/backend/
mapped.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! The parent of the CUDA and HIP lines: one page-locked, device-mapped host page holding one
4//! [`CompletionLine`], which a stream stores sequence numbers into behind its work. Each vendor
5//! supplies only its driver table ([`LineDriver`]); the ownership rules live here once:
6//! - a store is accepted only from a stream of the domain (CUDA context, HIP device) the line was
7//!   made in, so `Drop` can wait for it;
8//! - `Drop` frees the page only after that domain settled; if it cannot, the page is leaked,
9//!   never freed under a pending store (CWE-416);
10//! - the host reads the line only through atomics; the device writes only the words named.
11
12#![allow(unsafe_code)]
13
14use core::cell::Cell;
15use core::ffi::c_void;
16use core::ptr::NonNull;
17use core::sync::atomic::Ordering;
18
19use crate::core::line::{CompletionLine, Slot};
20
21/// The page the line lives in: a whole page, so no other allocation shares the line with a
22/// device writer.
23pub(crate) const PAGE: usize = 4096;
24
25/// Why a line could not be made or a store not queued.
26#[derive(Clone, Copy, Debug, PartialEq, Eq)]
27pub enum LineError {
28    /// The driver returned this nonzero status.
29    Status(i32),
30    /// The driver library or one of its entry points is missing.
31    Unavailable,
32    /// No current context/device on the calling thread.
33    NoDomain,
34    /// The stream belongs to another context/device than the line's.
35    ForeignStream,
36}
37
38impl LineError {
39    /// One integer per error, as the line's first users logged them: the driver's status, `-1`
40    /// unavailable, `-3` no domain, `-4` foreign stream.
41    #[must_use]
42    pub fn status(self) -> i32 {
43        match self {
44            Self::Status(s) => s,
45            Self::Unavailable => -1,
46            Self::NoDomain => -3,
47            Self::ForeignStream => -4,
48        }
49    }
50}
51
52impl core::fmt::Display for LineError {
53    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
54        match self {
55            Self::Status(s) => write!(f, "driver status {s}"),
56            Self::Unavailable => f.write_str("the driver or one of its entry points is missing"),
57            Self::NoDomain => f.write_str("no current context on this thread"),
58            Self::ForeignStream => f.write_str("the stream belongs to another context"),
59        }
60    }
61}
62
63impl core::error::Error for LineError {}
64
65impl From<LineError> for crate::Error {
66    fn from(e: LineError) -> Self {
67        crate::error!(Backend, "completion line", e.status())
68    }
69}
70
71/// What a line needs of a vendor driver. Crate-private: the line is its only caller and passes
72/// only values the driver handed out.
73pub(crate) trait LineDriver {
74    /// The vendor's stream handle.
75    type Stream: Copy;
76    /// The calling thread's current domain (context or device).
77    fn domain_current(&self) -> Result<u64, LineError>;
78    /// A page-locked, device-mapped host page of `bytes`.
79    fn alloc_host(&self, bytes: usize) -> Result<NonNull<c_void>, LineError>;
80    /// The device address of `host`, a page from [`Self::alloc_host`] not yet freed.
81    fn device_address(&self, host: NonNull<c_void>) -> Result<u64, LineError>;
82    /// Free a page from [`Self::alloc_host`], once.
83    fn free_host(&self, host: NonNull<c_void>);
84    /// The domain `stream` belongs to.
85    fn stream_domain(&self, stream: Self::Stream) -> Result<u64, LineError>;
86    /// Queue a store of `value` at device address `addr` on `stream`.
87    fn write_value(&self, stream: Self::Stream, addr: u64, value: u64) -> Result<(), LineError>;
88    /// Wait for every queued work of `domain`; `false` if the driver could not.
89    fn settle(&self, domain: u64) -> bool;
90}
91
92/// A device-mapped host line over driver `D`.
93pub(crate) struct Line<D: LineDriver> {
94    host: NonNull<CompletionLine>,
95    device: u64,
96    driver: D,
97    domain: u64,
98    queued: Cell<bool>,
99}
100
101// WHY: a line owns a raw mapped page, which withholds the auto `Send`; the executor moves the owner between threads.
102// SAFETY: the page is owned by this value alone and the host touches it only through atomics
103// (which are `Sync`); the driver table is `Send` by the bound. `queued` is a `Cell`, so the value
104// is `Send` but not `Sync`, which is all a line needs.
105// NO SAFE ALTERNATIVE: a raw-pointer owner is never `Send` by derivation, and wrapping the page would add an indirection on every host read.
106unsafe impl<D: LineDriver + Send> Send for Line<D> {}
107
108impl<D: LineDriver> Line<D> {
109    /// A fresh line reading all 0, mapped for the calling thread's current domain.
110    pub(crate) fn with_driver(driver: D) -> Result<Self, LineError> {
111        let domain = driver.domain_current();
112        let page = driver.alloc_host(PAGE)?;
113        let device = driver.device_address(page);
114        // Built before the checks, so every error below frees the page through `Drop`.
115        let line = Self {
116            host: page.cast::<CompletionLine>(),
117            device: device.unwrap_or(0),
118            driver,
119            domain: domain.unwrap_or(0),
120            queued: Cell::new(false),
121        };
122        device?;
123        if domain? == 0 {
124            return Err(LineError::NoDomain);
125        }
126        for w in &line.line().words {
127            w.store(0, Ordering::Release);
128        }
129        Ok(line)
130    }
131
132    /// The driver table.
133    #[cfg(feature = "cuda")]
134    pub(crate) fn driver(&self) -> &D {
135        &self.driver
136    }
137
138    /// The device address of word 0 (for a stream-side wait on the line).
139    pub(crate) fn device_address(&self) -> u64 {
140        self.device
141    }
142
143    /// The line, as the host reads it.
144    pub(crate) fn line(&self) -> &CompletionLine {
145        // WHY: reads the owned mapped page as a `CompletionLine` without copying it.
146        // SAFETY: `host` is the live, page-aligned (so 64-aligned, as `CompletionLine` needs) page
147        // this value owns until `Drop`; the device writes it only with aligned 8-byte stores to
148        // its words and every host access goes through the atomics, so nothing aliases it
149        // non-atomically.
150        // NO SAFE ALTERNATIVE: a `NonNull<T>` deref is unsafe; the page must stay in place because the device writes it.
151        unsafe { self.host.as_ref() }
152    }
153
154    /// Queue a store of `value` into word `slot` on `stream`, behind the work queued there.
155    pub(crate) fn enqueue_store(
156        &self,
157        stream: D::Stream,
158        slot: Slot,
159        value: u64,
160    ) -> Result<(), LineError> {
161        let offset = 8 * u64::try_from(slot.index()).map_err(|_| LineError::Unavailable)?;
162        if self.driver.stream_domain(stream)? != self.domain {
163            return Err(LineError::ForeignStream);
164        }
165        // Set before the call: a store the driver queued despite a failing status still makes
166        // `Drop` settle first.
167        self.queued.set(true);
168        self.driver.write_value(stream, self.device + offset, value)
169    }
170
171    /// Every store queued into the line has landed (`true` at once when none was queued).
172    fn settled(&self) -> bool {
173        !self.queued.get() || self.driver.settle(self.domain)
174    }
175}
176
177impl<D: LineDriver> Drop for Line<D> {
178    fn drop(&mut self) {
179        if !self.settled() {
180            // A store may still be pending into the page: leak 4 KiB rather than free memory the
181            // device may yet write.
182            return;
183        }
184        self.driver.free_host(self.host.cast::<c_void>());
185    }
186}
187
188#[cfg(test)]
189#[path = "mapped_tests.rs"]
190mod tests;