Skip to main content

gpu_async_reactor/backend/
hip.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! HIP (AMD): a [`MappedLine`] that HIP streams store into with `hipStreamWriteValue64` into
4//! host-mapped, coherent memory (`hipHostMalloc(Mapped | Coherent)`), so a host thread waits on
5//! it with the x86 `mwaitx`/`umwait` (or aarch64 `wfe`) park of the `wordwait` backend. The
6//! domain is the HIP device: the line holds a reference to the device's primary context
7//! (`hipDevicePrimaryCtxRetain`) from before the page is allocated until after it is freed, a
8//! store is accepted only from a stream of that device, and `Drop` synchronizes the device before
9//! freeing the page.
10//!
11//! Entry points are resolved by name at run time from `libamdhip64.so`; compile-checked, not yet
12//! run on AMD hardware.
13
14use core::ffi::{c_int, c_void};
15use core::ptr::NonNull;
16
17use libloading::Library;
18
19use super::mapped::{Line, LineDriver, LineError};
20use crate::core::line::{CompletionLine, Slot};
21
22/// A `hipStream_t` handle.
23#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub struct HipStream(usize);
25
26impl HipStream {
27    /// Wrap a raw `hipStream_t`.
28    ///
29    /// # Why
30    /// the typed stream is the only handle the reactor waits on, and the driver hands streams out as bare integers.
31    ///
32    /// # Safety
33    /// `raw` must be a live `hipStream_t` (or null, the default stream) for every call made with
34    /// the value: the runtime dereferences it.
35    ///
36    /// # No safe alternative
37    /// a handle is an integer the crate cannot validate; the typed stream is built only from it, so the constructor is unsafe.
38    #[must_use]
39    pub const unsafe fn from_raw(raw: usize) -> Self {
40        Self(raw)
41    }
42
43    /// The raw handle.
44    #[must_use]
45    pub const fn raw(self) -> usize {
46        self.0
47    }
48}
49
50/// `hipHostMallocMapped`.
51const MAPPED: u32 = 0x2;
52/// `hipHostMallocCoherent`: CPU and device see each other's writes without a flush.
53const COHERENT: u32 = 0x4000_0000;
54
55type HostMalloc = unsafe extern "C" fn(*mut *mut c_void, usize, u32) -> i32;
56type HostDevPtr = unsafe extern "C" fn(*mut *mut c_void, *mut c_void, u32) -> i32;
57type HostFree = unsafe extern "C" fn(*mut c_void) -> i32;
58type GetDevice = unsafe extern "C" fn(*mut c_int) -> i32;
59type SetDevice = unsafe extern "C" fn(c_int) -> i32;
60type DevSync = unsafe extern "C" fn() -> i32;
61type StreamDevice = unsafe extern "C" fn(*mut c_void, *mut c_int) -> i32;
62type PrimaryRetain = unsafe extern "C" fn(*mut *mut c_void, c_int) -> i32;
63type PrimaryRelease = unsafe extern "C" fn(c_int) -> i32;
64type WriteValue64 = unsafe extern "C" fn(*mut c_void, *mut c_void, u64, u32) -> i32;
65
66/// The HIP runtime's table and the library handle that keeps it valid.
67pub(crate) struct HipDriver {
68    malloc: HostMalloc,
69    dev_ptr: HostDevPtr,
70    free: HostFree,
71    get_device: GetDevice,
72    set_device: SetDevice,
73    dev_sync: DevSync,
74    stream_device: StreamDevice,
75    retain: PrimaryRetain,
76    release: PrimaryRelease,
77    write: WriteValue64,
78    _lib: Library,
79}
80
81macro_rules! sym {
82    ($lib:ident, $ty:ty, $name:literal) => {
83        // WHY: resolves one driver entry point by name, to call it through a typed fn pointer.
84        // SAFETY: the type is the signature hip_runtime_api.h (ROCm 6) declares for this name:
85        // `hipError_t` is a C enum read as `i32`, handles and device pointers are pointers,
86        // `hipDevice_t` is `int`; the name is NUL-free ASCII.
87        // NO SAFE ALTERNATIVE: a symbol lookup asserts a foreign signature the compiler cannot check; `libloading::Library::get` is unsafe.
88        *unsafe { $lib.get::<$ty>($name) }.ok()?
89    };
90}
91
92impl HipDriver {
93    /// Every entry point, or `None` if the runtime or any of them is missing.
94    pub(crate) fn resolve() -> Option<Self> {
95        // WHY: loads the vendor library at run time, so the crate links no vendor SDK and builds on a host without one.
96        // SAFETY: loading the HIP runtime runs its initialisers, which have no preconditions on
97        // this process; the handle is kept for as long as the table.
98        // NO SAFE ALTERNATIVE: dlopen runs foreign initialisers; `libloading::Library::new` is unsafe and there is no checked loader.
99        let lib = unsafe { Library::new("libamdhip64.so") }.ok()?;
100        Some(Self {
101            malloc: sym!(lib, HostMalloc, b"hipHostMalloc"),
102            dev_ptr: sym!(lib, HostDevPtr, b"hipHostGetDevicePointer"),
103            free: sym!(lib, HostFree, b"hipHostFree"),
104            get_device: sym!(lib, GetDevice, b"hipGetDevice"),
105            set_device: sym!(lib, SetDevice, b"hipSetDevice"),
106            dev_sync: sym!(lib, DevSync, b"hipDeviceSynchronize"),
107            stream_device: sym!(lib, StreamDevice, b"hipStreamGetDevice"),
108            retain: sym!(lib, PrimaryRetain, b"hipDevicePrimaryCtxRetain"),
109            release: sym!(lib, PrimaryRelease, b"hipDevicePrimaryCtxRelease"),
110            write: sym!(lib, WriteValue64, b"hipStreamWriteValue64"),
111            _lib: lib,
112        })
113    }
114}
115
116fn ok(status: i32) -> Result<(), LineError> {
117    if status == 0 { Ok(()) } else { Err(LineError::Status(status)) }
118}
119
120/// A device whose primary context the line holds a reference to.
121#[derive(Clone, Copy, Debug, PartialEq, Eq)]
122pub(crate) struct HipDevice(c_int);
123
124impl HipDriver {
125    /// Run `f` with `device` current on the calling thread, then restore the thread's device.
126    fn on_device<T>(&self, device: HipDevice, f: impl FnOnce() -> T) -> Result<T, LineError> {
127        let mut prev: c_int = 0;
128        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
129        // SAFETY: plain runtime calls on integers; `prev` is a live out-pointer.
130        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
131        unsafe {
132            ok((self.get_device)(&raw mut prev))?;
133            ok((self.set_device)(device.0))?;
134        }
135        let r = f();
136        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
137        // SAFETY: restores the device read above.
138        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
139        ok(unsafe { (self.set_device)(prev) })?;
140        Ok(r)
141    }
142}
143
144impl LineDriver for HipDriver {
145    type Stream = HipStream;
146    type Domain = HipDevice;
147
148    fn retain_current(&self) -> Result<HipDevice, LineError> {
149        let mut d: c_int = -1;
150        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
151        // SAFETY: `d` is a live out-pointer, the only memory written.
152        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
153        ok(unsafe { (self.get_device)(&raw mut d) })?;
154        if d < 0 {
155            return Err(LineError::NoDomain);
156        }
157        let mut ctx: *mut c_void = core::ptr::null_mut();
158        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
159        // SAFETY: `d` is the current device; `ctx` is a live out-pointer. The reference taken here
160        // is dropped once, by `release` (or leaked with the page).
161        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
162        ok(unsafe { (self.retain)(&raw mut ctx, d) })?;
163        Ok(HipDevice(d))
164    }
165
166    fn release(&self, device: HipDevice) {
167        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
168        // SAFETY: drops the one reference `retain_current` took on this device's primary context.
169        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
170        let _ = unsafe { (self.release)(device.0) };
171    }
172
173    fn alloc_host(&self, device: HipDevice, bytes: usize) -> Result<NonNull<c_void>, LineError> {
174        let mut page: *mut c_void = core::ptr::null_mut();
175        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
176        // SAFETY: `page` is a live out-pointer, the only memory written; the line's device is
177        // current for the call.
178        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
179        ok(self.on_device(device, || unsafe {
180            (self.malloc)(&raw mut page, bytes, MAPPED | COHERENT)
181        })?)?;
182        NonNull::new(page).ok_or(LineError::Unavailable)
183    }
184
185    fn device_address(&self, device: HipDevice, host: NonNull<c_void>) -> Result<u64, LineError> {
186        let mut addr: *mut c_void = core::ptr::null_mut();
187        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
188        // SAFETY: `host` is a live mapped page from `alloc_host`; `addr` is a live out-pointer.
189        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
190        ok(self.on_device(device, || unsafe {
191            (self.dev_ptr)(&raw mut addr, host.as_ptr(), 0)
192        })?)?;
193        Ok(addr as u64)
194    }
195
196    fn free_host(&self, _device: HipDevice, host: NonNull<c_void>) {
197        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
198        // SAFETY: `host` came from `alloc_host` and is freed once, after every store landed, while
199        // the line still holds its device's primary context.
200        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
201        let _ = unsafe { (self.free)(host.as_ptr()) };
202    }
203
204    fn stream_in(&self, stream: HipStream, device: HipDevice) -> Result<bool, LineError> {
205        let mut d: c_int = -1;
206        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
207        // SAFETY: `stream` is live by `HipStream::from_raw`'s contract; `d` is a live out-pointer.
208        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
209        ok(unsafe { (self.stream_device)(stream.0 as *mut c_void, &raw mut d) })?;
210        Ok(d == device.0)
211    }
212
213    fn write_value(&self, stream: HipStream, addr: u64, value: u64) -> Result<(), LineError> {
214        let ptr = usize::try_from(addr).map_err(|_| LineError::Unavailable)? as *mut c_void;
215        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
216        // SAFETY: `stream` is live and of the line's device (checked by the line); `ptr` is a word
217        // of the line's own live mapped page, which `Drop` frees only after the device settled.
218        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
219        ok(unsafe { (self.write)(stream.0 as *mut c_void, ptr, value, 0) })
220    }
221
222    fn settle(&self, device: HipDevice) -> bool {
223        // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
224        // SAFETY: a plain runtime call, with the line's device current for it.
225        // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
226        matches!(self.on_device(device, || unsafe { (self.dev_sync)() }), Ok(0))
227    }
228}
229
230/// A page-locked, host-mapped, coherent [`CompletionLine`] that HIP streams store into.
231pub struct MappedLine(Line<HipDriver>);
232
233impl MappedLine {
234    /// A fresh line reading all 0, mapped for the calling thread's current device. The line holds
235    /// a reference to that device's primary context until after its page is freed.
236    ///
237    /// # Errors
238    /// The runtime's status, [`LineError::Unavailable`] when the runtime or an entry point is
239    /// missing, [`LineError::NoDomain`] when no device is current.
240    pub fn new() -> Result<Self, LineError> {
241        Line::with_driver(HipDriver::resolve().ok_or(LineError::Unavailable)?).map(Self)
242    }
243
244    /// The line, as the host reads it.
245    #[must_use]
246    pub fn line(&self) -> &CompletionLine {
247        self.0.line()
248    }
249
250    /// The device address of word 0; word `i` is at `+ 8 * i` (for a stream-side wait on it).
251    #[must_use]
252    pub fn device_address(&self) -> u64 {
253        self.0.device_address()
254    }
255
256    /// Queue a store of `value` into word `slot` on `stream`, behind the work queued there.
257    ///
258    /// # Errors
259    /// The runtime's status, or [`LineError::ForeignStream`] for a stream of another device.
260    pub fn enqueue_store(&self, stream: HipStream, slot: Slot, value: u64) -> Result<(), LineError> {
261        self.0.enqueue_store(stream, slot, value)
262    }
263}