use core::{
ffi::{c_uint, c_ulonglong},
mem::{ManuallyDrop, transmute},
ptr::from_ref,
};
use rocm_sys::hip::{
hipChildGraphNodeParams, hipEventRecordNodeParams, hipEventWaitNodeParams,
hipExternalSemaphoreSignalNodeParams, hipExternalSemaphoreWaitNodeParams, hipGraph_t,
hipGraphAddKernelNode, hipGraphClone, hipGraphCreate, hipGraphDestroy, hipGraphExec_t,
hipGraphExecDestroy, hipGraphInstantiateWithFlags, hipGraphKernelNodeGetParams, hipGraphLaunch,
hipGraphNode_t, hipGraphNodeParams, hipGraphNodeParams__bindgen_ty_1, hipGraphNodeType,
hipGraphUpload, hipHostNodeParams, hipKernelNodeParams, hipMemAllocNodeParams,
hipMemFreeNodeParams, hipMemcpyNodeParams, hipMemsetParams, hipStreamBeginCapture,
hipStreamCaptureMode, hipStreamCaptureStatus, hipStreamEndCapture, hipStreamIsCapturing,
};
use crate::hip::{HipError, stream::HipStream};
#[repr(transparent)]
pub struct Graph {
raw: hipGraph_t,
}
impl Graph {
pub fn new(flags: GraphInstantiateFlags) -> Result<Self, HipError> {
let raw: Result<hipGraph_t, HipError> = unsafe {
wrap_sys_res!(|graph| hipGraphCreate((&raw mut graph).cast(), flags.bits() as c_uint))
};
let raw = raw?;
Ok(Self { raw })
}
pub fn destroy(self) -> Result<(), HipError> {
let handle = ManuallyDrop::new(self);
unsafe {
handle.destroy_unchecked()?;
}
Ok(())
}
pub unsafe fn destroy_unchecked(&self) -> Result<(), HipError> {
unsafe { try_err!(hipGraphDestroy(self.raw), Ok(())) }
}
pub fn as_raw(&self) -> hipGraph_t {
self.raw
}
pub fn from_raw(raw: hipGraph_t) -> Self {
Self { raw }
}
pub fn try_clone(&self) -> Result<Self, HipError> {
unsafe { wrap_sys_res!(|clone| hipGraphClone((&raw mut clone).cast(), self.raw)) }
}
pub fn add_kernel_node(
&mut self,
dep: &[Node],
params: &KernelParams,
) -> Result<Node, HipError> {
unsafe {
wrap_sys_res!(|node| hipGraphAddKernelNode(
(&raw mut node).cast(),
self.raw,
dep.as_ptr().cast(),
dep.len(),
from_ref(params).cast()
))
}
}
pub fn init(self, flags: GraphInstantiateFlags) -> Result<ExecGraph, HipError> {
let exec: Result<hipGraphExec_t, HipError> = unsafe {
wrap_sys_res!(|graph_exec| hipGraphInstantiateWithFlags(
(&raw mut graph_exec).cast(),
self.raw,
flags.bits() as c_ulonglong
))
};
let raw = exec?;
Ok(ExecGraph { raw })
}
}
impl Drop for Graph {
fn drop(&mut self) {
unsafe {
let _ = self.destroy_unchecked();
}
}
}
#[repr(transparent)]
pub struct ExecGraph {
raw: hipGraphExec_t,
}
impl ExecGraph {
pub fn destroy(self) -> Result<(), HipError> {
let handle = ManuallyDrop::new(self);
unsafe {
handle.destroy_unchecked()?;
}
Ok(())
}
pub unsafe fn destroy_unchecked(&self) -> Result<(), HipError> {
unsafe { try_err!(hipGraphExecDestroy(self.raw), Ok(())) }
}
#[inline(always)]
pub fn as_raw(&self) -> hipGraphExec_t {
self.raw
}
#[inline(always)]
pub unsafe fn from_raw(raw: hipGraphExec_t) -> Self {
Self { raw }
}
pub fn upload(&self, stream: &HipStream) -> Result<(), HipError> {
unsafe { try_err!(hipGraphUpload(self.raw, stream.raw.raw), Ok(())) }
}
}
impl Drop for ExecGraph {
fn drop(&mut self) {
unsafe {
let _ = self.destroy_unchecked();
}
}
}
impl HipStream {
pub fn capture<F: FnOnce(&Self) -> Result<(), HipError>>(
&self,
mode: StreamCaptureMode,
f: F,
) -> Result<Graph, HipError> {
self.start_capture(mode)?;
match f(self) {
Ok(()) => self.end_capture(),
Err(err) => {
let _ = self.end_capture();
Err(err)
}
}
}
pub fn start_capture(&self, mode: StreamCaptureMode) -> Result<(), HipError> {
unsafe {
try_err!(
hipStreamBeginCapture(
self.raw.raw,
transmute::<StreamCaptureMode, hipStreamCaptureMode>(mode)
),
Ok(())
)
}
}
pub fn end_capture(&self) -> Result<Graph, HipError> {
let raw: Result<hipGraph_t, HipError> = unsafe {
wrap_sys_res!(|graph| hipStreamEndCapture(self.raw.raw, (&raw mut graph).cast()))
};
let raw = raw?;
Ok(Graph { raw })
}
pub fn is_capturing(&self) -> Result<bool, HipError> {
let status: Result<hipStreamCaptureStatus, HipError> = unsafe {
wrap_sys_res!(|status| hipStreamIsCapturing(self.raw.raw, (&raw mut status).cast()))
};
match status? {
hipStreamCaptureStatus::hipStreamCaptureStatusActive => Ok(true),
hipStreamCaptureStatus::hipStreamCaptureStatusNone => Ok(false),
hipStreamCaptureStatus::hipStreamCaptureStatusInvalidated => {
Err(HipError::StreamCaptureInvalidated)
}
}
}
pub unsafe fn launch_graph(&self, graph: &ExecGraph) -> Result<(), HipError> {
unsafe { try_err!(hipGraphLaunch(graph.raw, self.raw.raw), Ok(())) }
}
}
#[repr(u32)]
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub enum StreamCaptureMode {
Global = 0,
ThreadLocal = 1,
Relaxed = 2,
}
#[repr(transparent)]
#[derive(Debug, Clone)]
pub struct Node {
raw: hipGraphNode_t,
}
impl Node {
pub unsafe fn get_kernel_params(&self) -> Result<KernelParams, HipError> {
unsafe {
wrap_sys_res!(|params| hipGraphKernelNodeGetParams(self.raw, (&raw mut params).cast()))
}
}
}
#[repr(transparent)]
#[derive(Clone)]
pub struct KernelParams {
raw: hipKernelNodeParams,
}
#[cfg(all(feature = "hiprtc", any(feature = "dynamic-loading", hiprtc)))]
impl KernelParams {
pub const fn new(
func: &crate::hip::module::Func,
args: &mut [*mut u8],
conf: crate::hip::module::LaunchConfig,
) -> Self {
use rocm_sys::hip::dim3;
Self {
raw: hipKernelNodeParams {
blockDim: unsafe { transmute::<[u32; 3], dim3>(conf.block) },
gridDim: unsafe { transmute::<[u32; 3], dim3>(conf.grid) },
func: func.raw.cast(),
kernelParams: args.as_mut_ptr().cast(),
sharedMemBytes: 0,
extra: core::ptr::null_mut(),
},
}
}
}
#[repr(transparent)]
#[derive(Clone)]
pub struct GraphParams {
raw: hipGraphNodeParams,
}
impl GraphParams {
pub const fn new(ty: NodeType, desc: ParamDescriptor) -> Self {
Self {
raw: hipGraphNodeParams {
type_: unsafe { transmute::<NodeType, hipGraphNodeType>(ty) },
reserved0: [0; 3],
__bindgen_anon_1: unsafe {
transmute::<ParamDescriptor, hipGraphNodeParams__bindgen_ty_1>(desc)
},
reserved2: 0,
},
}
}
}
#[repr(C)]
#[derive(Copy, Clone)]
pub union ParamDescriptor {
pub reserved1: [::core::ffi::c_longlong; 29usize],
pub kernel: ParamKernel,
pub memcpy: ParamMemcpy,
pub memset: ParamMemset,
pub host: ParamHost,
pub graph: ParamChildGraph,
pub event_wait: ParamEventWait,
pub event_record: ParamEventRecord,
pub ext_sem_signal: ParamExternalSemaphoreSignal,
pub ext_sem_wait: ParamExternalSemaphoreWait,
pub alloc: ParamMemAlloc,
pub free: ParamMemFree,
}
pub type ParamMemFree = hipMemFreeNodeParams;
pub type ParamMemAlloc = hipMemAllocNodeParams;
pub type ParamExternalSemaphoreWait = hipExternalSemaphoreWaitNodeParams;
pub type ParamExternalSemaphoreSignal = hipExternalSemaphoreSignalNodeParams;
pub type ParamEventRecord = hipEventRecordNodeParams;
pub type ParamEventWait = hipEventWaitNodeParams;
pub type ParamChildGraph = hipChildGraphNodeParams;
pub type ParamHost = hipHostNodeParams;
pub type ParamMemset = hipMemsetParams;
pub type ParamMemcpy = hipMemcpyNodeParams;
pub type ParamKernel = hipKernelNodeParams;
#[repr(u32)]
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub enum NodeType {
Kernel = 0,
Memcpy = 1,
Memset = 2,
Host = 3,
Graph = 4,
Empty = 5,
WaitEvent = 6,
EventRecord = 7,
ExtSemaphoreSignal = 8,
ExtSemaphoreWait = 9,
MemAlloc = 10,
MemFree = 11,
MemcpyFromSymbol = 12,
MemcpyToSymbol = 13,
BatchMemOp = 14,
Count = 15,
}
bitflags::bitflags! {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct GraphInstantiateFlags: u32 {
const AUTO_FREE_ON_LAUNCH = 1 << 0;
const UPLOAD = 1 << 1;
const DEV_LAUNCH = 1 << 2;
const USE_NODE_PRIO = 1 << 3;
}
}