gpu_async_reactor/backend/hip.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! HIP (AMD): a [`MappedLine`] that HIP streams store into with `hipStreamWriteValue64` into
4//! host-mapped, coherent memory (`hipHostMalloc(Mapped | Coherent)`), so a host thread waits on
5//! it with the x86 `mwaitx`/`umwait` (or aarch64 `wfe`) park of the `wordwait` backend. The
6//! domain is the HIP device: the line holds a reference to the device's primary context
7//! (`hipDevicePrimaryCtxRetain`) from before the page is allocated until after it is freed, a
8//! store is accepted only from a stream of that device, and `Drop` synchronizes the device before
9//! freeing the page.
10//!
11//! Entry points are resolved by name at run time from `libamdhip64.so`; compile-checked, not yet
12//! run on AMD hardware.
13
14use core::ffi::{c_int, c_void};
15use core::ptr::NonNull;
16
17use libloading::Library;
18
19use super::mapped::{Line, LineDriver, LineError};
20use crate::core::line::{CompletionLine, Slot};
21
22/// A `hipStream_t` handle.
23#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub struct HipStream(usize);
25
26impl HipStream {
27 /// Wrap a raw `hipStream_t`.
28 ///
29 /// # Why
30 /// the typed stream is the only handle the reactor waits on, and the driver hands streams out as bare integers.
31 ///
32 /// # Safety
33 /// `raw` must be a live `hipStream_t` (or null, the default stream) for every call made with
34 /// the value: the runtime dereferences it.
35 ///
36 /// # No safe alternative
37 /// a handle is an integer the crate cannot validate; the typed stream is built only from it, so the constructor is unsafe.
38 #[must_use]
39 pub const unsafe fn from_raw(raw: usize) -> Self {
40 Self(raw)
41 }
42
43 /// The raw handle.
44 #[must_use]
45 pub const fn raw(self) -> usize {
46 self.0
47 }
48}
49
50/// `hipHostMallocMapped`.
51const MAPPED: u32 = 0x2;
52/// `hipHostMallocCoherent`: CPU and device see each other's writes without a flush.
53const COHERENT: u32 = 0x4000_0000;
54
55type HostMalloc = unsafe extern "C" fn(*mut *mut c_void, usize, u32) -> i32;
56type HostDevPtr = unsafe extern "C" fn(*mut *mut c_void, *mut c_void, u32) -> i32;
57type HostFree = unsafe extern "C" fn(*mut c_void) -> i32;
58type GetDevice = unsafe extern "C" fn(*mut c_int) -> i32;
59type SetDevice = unsafe extern "C" fn(c_int) -> i32;
60type DevSync = unsafe extern "C" fn() -> i32;
61type StreamDevice = unsafe extern "C" fn(*mut c_void, *mut c_int) -> i32;
62type PrimaryRetain = unsafe extern "C" fn(*mut *mut c_void, c_int) -> i32;
63type PrimaryRelease = unsafe extern "C" fn(c_int) -> i32;
64type WriteValue64 = unsafe extern "C" fn(*mut c_void, *mut c_void, u64, u32) -> i32;
65
66/// The HIP runtime's table and the library handle that keeps it valid.
67pub(crate) struct HipDriver {
68 malloc: HostMalloc,
69 dev_ptr: HostDevPtr,
70 free: HostFree,
71 get_device: GetDevice,
72 set_device: SetDevice,
73 dev_sync: DevSync,
74 stream_device: StreamDevice,
75 retain: PrimaryRetain,
76 release: PrimaryRelease,
77 write: WriteValue64,
78 _lib: Library,
79}
80
81macro_rules! sym {
82 ($lib:ident, $ty:ty, $name:literal) => {
83 // WHY: resolves one driver entry point by name, to call it through a typed fn pointer.
84 // SAFETY: the type is the signature hip_runtime_api.h (ROCm 6) declares for this name:
85 // `hipError_t` is a C enum read as `i32`, handles and device pointers are pointers,
86 // `hipDevice_t` is `int`; the name is NUL-free ASCII.
87 // NO SAFE ALTERNATIVE: a symbol lookup asserts a foreign signature the compiler cannot check; `libloading::Library::get` is unsafe.
88 *unsafe { $lib.get::<$ty>($name) }.ok()?
89 };
90}
91
92impl HipDriver {
93 /// Every entry point, or `None` if the runtime or any of them is missing.
94 pub(crate) fn resolve() -> Option<Self> {
95 // WHY: loads the vendor library at run time, so the crate links no vendor SDK and builds on a host without one.
96 // SAFETY: loading the HIP runtime runs its initialisers, which have no preconditions on
97 // this process; the handle is kept for as long as the table.
98 // NO SAFE ALTERNATIVE: dlopen runs foreign initialisers; `libloading::Library::new` is unsafe and there is no checked loader.
99 let lib = unsafe { Library::new("libamdhip64.so") }.ok()?;
100 Some(Self {
101 malloc: sym!(lib, HostMalloc, b"hipHostMalloc"),
102 dev_ptr: sym!(lib, HostDevPtr, b"hipHostGetDevicePointer"),
103 free: sym!(lib, HostFree, b"hipHostFree"),
104 get_device: sym!(lib, GetDevice, b"hipGetDevice"),
105 set_device: sym!(lib, SetDevice, b"hipSetDevice"),
106 dev_sync: sym!(lib, DevSync, b"hipDeviceSynchronize"),
107 stream_device: sym!(lib, StreamDevice, b"hipStreamGetDevice"),
108 retain: sym!(lib, PrimaryRetain, b"hipDevicePrimaryCtxRetain"),
109 release: sym!(lib, PrimaryRelease, b"hipDevicePrimaryCtxRelease"),
110 write: sym!(lib, WriteValue64, b"hipStreamWriteValue64"),
111 _lib: lib,
112 })
113 }
114}
115
116fn ok(status: i32) -> Result<(), LineError> {
117 if status == 0 { Ok(()) } else { Err(LineError::Status(status)) }
118}
119
120/// A device whose primary context the line holds a reference to.
121#[derive(Clone, Copy, Debug, PartialEq, Eq)]
122pub(crate) struct HipDevice(c_int);
123
124impl HipDriver {
125 /// Run `f` with `device` current on the calling thread, then restore the thread's device.
126 fn on_device<T>(&self, device: HipDevice, f: impl FnOnce() -> T) -> Result<T, LineError> {
127 let mut prev: c_int = 0;
128 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
129 // SAFETY: plain runtime calls on integers; `prev` is a live out-pointer.
130 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
131 unsafe {
132 ok((self.get_device)(&raw mut prev))?;
133 ok((self.set_device)(device.0))?;
134 }
135 let r = f();
136 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
137 // SAFETY: restores the device read above.
138 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
139 ok(unsafe { (self.set_device)(prev) })?;
140 Ok(r)
141 }
142}
143
144impl LineDriver for HipDriver {
145 type Stream = HipStream;
146 type Domain = HipDevice;
147
148 fn retain_current(&self) -> Result<HipDevice, LineError> {
149 let mut d: c_int = -1;
150 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
151 // SAFETY: `d` is a live out-pointer, the only memory written.
152 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
153 ok(unsafe { (self.get_device)(&raw mut d) })?;
154 if d < 0 {
155 return Err(LineError::NoDomain);
156 }
157 let mut ctx: *mut c_void = core::ptr::null_mut();
158 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
159 // SAFETY: `d` is the current device; `ctx` is a live out-pointer. The reference taken here
160 // is dropped once, by `release` (or leaked with the page).
161 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
162 ok(unsafe { (self.retain)(&raw mut ctx, d) })?;
163 Ok(HipDevice(d))
164 }
165
166 fn release(&self, device: HipDevice) {
167 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
168 // SAFETY: drops the one reference `retain_current` took on this device's primary context.
169 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
170 let _ = unsafe { (self.release)(device.0) };
171 }
172
173 fn alloc_host(&self, device: HipDevice, bytes: usize) -> Result<NonNull<c_void>, LineError> {
174 let mut page: *mut c_void = core::ptr::null_mut();
175 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
176 // SAFETY: `page` is a live out-pointer, the only memory written; the line's device is
177 // current for the call.
178 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
179 ok(self.on_device(device, || unsafe {
180 (self.malloc)(&raw mut page, bytes, MAPPED | COHERENT)
181 })?)?;
182 NonNull::new(page).ok_or(LineError::Unavailable)
183 }
184
185 fn device_address(&self, device: HipDevice, host: NonNull<c_void>) -> Result<u64, LineError> {
186 let mut addr: *mut c_void = core::ptr::null_mut();
187 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
188 // SAFETY: `host` is a live mapped page from `alloc_host`; `addr` is a live out-pointer.
189 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
190 ok(self.on_device(device, || unsafe {
191 (self.dev_ptr)(&raw mut addr, host.as_ptr(), 0)
192 })?)?;
193 Ok(addr as u64)
194 }
195
196 fn free_host(&self, _device: HipDevice, host: NonNull<c_void>) {
197 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
198 // SAFETY: `host` came from `alloc_host` and is freed once, after every store landed, while
199 // the line still holds its device's primary context.
200 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
201 let _ = unsafe { (self.free)(host.as_ptr()) };
202 }
203
204 fn stream_in(&self, stream: HipStream, device: HipDevice) -> Result<bool, LineError> {
205 let mut d: c_int = -1;
206 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
207 // SAFETY: `stream` is live by `HipStream::from_raw`'s contract; `d` is a live out-pointer.
208 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
209 ok(unsafe { (self.stream_device)(stream.0 as *mut c_void, &raw mut d) })?;
210 Ok(d == device.0)
211 }
212
213 fn write_value(&self, stream: HipStream, addr: u64, value: u64) -> Result<(), LineError> {
214 let ptr = usize::try_from(addr).map_err(|_| LineError::Unavailable)? as *mut c_void;
215 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
216 // SAFETY: `stream` is live and of the line's device (checked by the line); `ptr` is a word
217 // of the line's own live mapped page, which `Drop` frees only after the device settled.
218 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
219 ok(unsafe { (self.write)(stream.0 as *mut c_void, ptr, value, 0) })
220 }
221
222 fn settle(&self, device: HipDevice) -> bool {
223 // WHY: calls a HIP runtime entry point resolved at run time (the line, stream or context operation).
224 // SAFETY: a plain runtime call, with the line's device current for it.
225 // NO SAFE ALTERNATIVE: the driver API is C: every entry point is an unsafe fn pointer, and the vendor crates' wrappers are unsafe too.
226 matches!(self.on_device(device, || unsafe { (self.dev_sync)() }), Ok(0))
227 }
228}
229
230/// A page-locked, host-mapped, coherent [`CompletionLine`] that HIP streams store into.
231pub struct MappedLine(Line<HipDriver>);
232
233impl MappedLine {
234 /// A fresh line reading all 0, mapped for the calling thread's current device. The line holds
235 /// a reference to that device's primary context until after its page is freed.
236 ///
237 /// # Errors
238 /// The runtime's status, [`LineError::Unavailable`] when the runtime or an entry point is
239 /// missing, [`LineError::NoDomain`] when no device is current.
240 pub fn new() -> Result<Self, LineError> {
241 Line::with_driver(HipDriver::resolve().ok_or(LineError::Unavailable)?).map(Self)
242 }
243
244 /// The line, as the host reads it.
245 #[must_use]
246 pub fn line(&self) -> &CompletionLine {
247 self.0.line()
248 }
249
250 /// The device address of word 0; word `i` is at `+ 8 * i` (for a stream-side wait on it).
251 #[must_use]
252 pub fn device_address(&self) -> u64 {
253 self.0.device_address()
254 }
255
256 /// Queue a store of `value` into word `slot` on `stream`, behind the work queued there.
257 ///
258 /// # Errors
259 /// The runtime's status, or [`LineError::ForeignStream`] for a stream of another device.
260 pub fn enqueue_store(&self, stream: HipStream, slot: Slot, value: u64) -> Result<(), LineError> {
261 self.0.enqueue_store(stream, slot, value)
262 }
263}