use crate::error::{ErrorCode, TptrError, TptrResult};
#[cfg(feature = "cuda")]
mod imp {
use super::*;
use std::sync::Arc;
use tpt_gpu_primitives::sym;
use tpt_gpu_primitives::vendor::dynlink::Library;
type CUresult = i32;
type CUdevice = i32;
type CUcontext = *mut std::ffi::c_void;
type CUdeviceptr = u64;
const CUDA_SUCCESS: CUresult = 0;
fn map(e: impl std::fmt::Display) -> TptrError {
TptrError::new(ErrorCode::DeviceNotFound, e.to_string())
}
pub struct CudaContext {
#[allow(dead_code)]
lib_cuda: Arc<Library>,
#[allow(dead_code)]
lib_cublas: Arc<Library>,
#[allow(dead_code)]
ctx: CUcontext,
cu_mem_alloc: unsafe extern "C" fn(*mut CUdeviceptr, usize) -> CUresult,
cu_mem_free: unsafe extern "C" fn(CUdeviceptr) -> CUresult,
cu_memcpy_htod:
unsafe extern "C" fn(CUdeviceptr, *const std::ffi::c_void, usize) -> CUresult,
cu_memcpy_dtoh: unsafe extern "C" fn(*mut std::ffi::c_void, CUdeviceptr, usize) -> CUresult,
cu_ctx_sync: unsafe extern "C" fn() -> CUresult,
}
unsafe impl Send for CudaContext {}
unsafe impl Sync for CudaContext {}
impl CudaContext {
pub(super) fn new() -> TptrResult<Self> {
let lib_cublas = Arc::new(
Library::open("cublas64_12.dll")
.or_else(|_| Library::open("cublas64_11.dll"))
.or_else(|_| Library::open("libcublas.so.12"))
.or_else(|_| Library::open("libcublas.so.11"))
.or_else(|_| Library::open("libcublas.so"))
.map_err(map)?,
);
let lib_cuda = Arc::new(
Library::open("nvcuda.dll")
.or_else(|_| Library::open("libcuda.so"))
.map_err(map)?,
);
let cu_init: unsafe extern "C" fn(u32) -> CUresult =
unsafe { *lib_cuda.get(sym!("cuInit")).map_err(map)? };
let cu_device_get: unsafe extern "C" fn(*mut CUdevice, i32) -> CUresult =
unsafe { *lib_cuda.get(sym!("cuDeviceGet")).map_err(map)? };
let cu_ctx_create: unsafe extern "C" fn(*mut CUcontext, u32, CUdevice) -> CUresult =
unsafe { *lib_cuda.get(sym!("cuCtxCreate")).map_err(map)? };
let cu_mem_alloc: unsafe extern "C" fn(*mut CUdeviceptr, usize) -> CUresult =
unsafe { *lib_cuda.get(sym!("cuMemAlloc_v2")).map_err(map)? };
let cu_mem_free: unsafe extern "C" fn(CUdeviceptr) -> CUresult =
unsafe { *lib_cuda.get(sym!("cuMemFree_v2")).map_err(map)? };
let cu_memcpy_htod: unsafe extern "C" fn(
CUdeviceptr,
*const std::ffi::c_void,
usize,
) -> CUresult = unsafe { *lib_cuda.get(sym!("cuMemcpyHtoD_v2")).map_err(map)? };
let cu_memcpy_dtoh: unsafe extern "C" fn(
*mut std::ffi::c_void,
CUdeviceptr,
usize,
) -> CUresult = unsafe { *lib_cuda.get(sym!("cuMemcpyDtoH_v2")).map_err(map)? };
let cu_ctx_sync: unsafe extern "C" fn() -> CUresult =
unsafe { *lib_cuda.get(sym!("cuCtxSynchronize")).map_err(map)? };
let cu_ctx_set_current: unsafe extern "C" fn(CUcontext) -> CUresult =
unsafe { *lib_cuda.get(sym!("cuCtxSetCurrent")).map_err(map)? };
let cu_device_primary_ctx_retain: unsafe extern "C" fn(
*mut CUcontext,
CUdevice,
) -> CUresult = unsafe {
*lib_cuda
.get(sym!("cuDevicePrimaryCtxRetain"))
.map_err(map)?
};
if unsafe { cu_init(0) } != CUDA_SUCCESS {
return Err(TptrError::new(
ErrorCode::DeviceNotFound,
"CUDA: cuInit failed",
));
}
let mut device: CUdevice = 0;
if unsafe { cu_device_get(&mut device, 0) } != CUDA_SUCCESS {
return Err(TptrError::new(
ErrorCode::DeviceNotFound,
"CUDA: cuDeviceGet failed",
));
}
let mut ctx: CUcontext = std::ptr::null_mut();
let retain_res = unsafe { cu_device_primary_ctx_retain(&mut ctx, device) };
if retain_res != CUDA_SUCCESS || ctx.is_null() {
return Err(TptrError::new(
ErrorCode::DeviceNotFound,
format!("CUDA: cuDevicePrimaryCtxRetain failed (res={})", retain_res),
));
}
if unsafe { cu_ctx_set_current(ctx) } != CUDA_SUCCESS {
return Err(TptrError::new(
ErrorCode::DeviceNotFound,
"CUDA: cuCtxSetCurrent failed",
));
}
Ok(Self {
lib_cuda,
lib_cublas,
ctx,
cu_mem_alloc,
cu_mem_free,
cu_memcpy_htod,
cu_memcpy_dtoh,
cu_ctx_sync,
})
}
pub(super) fn alloc(&self, size: u64) -> TptrResult<u64> {
let mut ptr: CUdeviceptr = 0;
if unsafe { (self.cu_mem_alloc)(&mut ptr, size as usize) } != CUDA_SUCCESS || ptr == 0 {
return Err(TptrError::new(
ErrorCode::OutOfMemory,
"CUDA: cuMemAlloc failed",
));
}
Ok(ptr)
}
pub(super) fn free(&self, ptr: u64) -> TptrResult<()> {
if ptr == 0 {
return Ok(());
}
if unsafe { (self.cu_mem_free)(ptr) } != CUDA_SUCCESS {
return Err(TptrError::new(
ErrorCode::InvalidAddress,
"CUDA: cuMemFree failed",
));
}
Ok(())
}
pub(super) fn upload(&self, dst: u64, src: &[u8]) -> TptrResult<()> {
if unsafe {
(self.cu_memcpy_htod)(dst, src.as_ptr() as *const std::ffi::c_void, src.len())
} != CUDA_SUCCESS
{
return Err(TptrError::new(
ErrorCode::DeviceLost,
"CUDA: cuMemcpyHtoD failed",
));
}
Ok(())
}
pub(super) fn download(&self, src: u64, dst: &mut [u8]) -> TptrResult<()> {
if unsafe {
(self.cu_memcpy_dtoh)(dst.as_mut_ptr() as *mut std::ffi::c_void, src, dst.len())
} != CUDA_SUCCESS
{
return Err(TptrError::new(
ErrorCode::DeviceLost,
"CUDA: cuMemcpyDtoH failed",
));
}
Ok(())
}
pub(super) fn synchronize(&self) -> TptrResult<()> {
if unsafe { (self.cu_ctx_sync)() } != CUDA_SUCCESS {
return Err(TptrError::new(
ErrorCode::SynchronizationError,
"CUDA: cuCtxSynchronize failed",
));
}
Ok(())
}
}
impl Drop for CudaContext {
fn drop(&mut self) {
unsafe {
(self.cu_ctx_sync)();
}
}
}
}
pub enum DeviceBackend {
#[cfg(feature = "cuda")]
Cuda(imp::CudaContext),
#[allow(dead_code)]
None,
}
impl DeviceBackend {
#[allow(dead_code)]
pub(super) fn try_cuda() -> Option<Self> {
#[cfg(feature = "cuda")]
{
imp::CudaContext::new().ok().map(DeviceBackend::Cuda)
}
#[cfg(not(feature = "cuda"))]
{
None
}
}
#[allow(unused_variables)]
pub(super) fn alloc(&self, size: u64) -> TptrResult<u64> {
match self {
#[cfg(feature = "cuda")]
DeviceBackend::Cuda(ctx) => ctx.alloc(size),
DeviceBackend::None => Err(TptrError::new(
ErrorCode::DeviceNotFound,
"no real device backend",
)),
}
}
#[allow(unused_variables)]
pub(super) fn free(&self, ptr: u64) -> TptrResult<()> {
match self {
#[cfg(feature = "cuda")]
DeviceBackend::Cuda(ctx) => ctx.free(ptr),
DeviceBackend::None => Ok(()),
}
}
#[allow(unused_variables)]
pub(super) fn upload(&self, dst: u64, src: &[u8]) -> TptrResult<()> {
match self {
#[cfg(feature = "cuda")]
DeviceBackend::Cuda(ctx) => ctx.upload(dst, src),
DeviceBackend::None => Err(TptrError::new(
ErrorCode::DeviceNotFound,
"no real device backend",
)),
}
}
#[allow(unused_variables)]
pub(super) fn download(&self, src: u64, dst: &mut [u8]) -> TptrResult<()> {
match self {
#[cfg(feature = "cuda")]
DeviceBackend::Cuda(ctx) => ctx.download(src, dst),
DeviceBackend::None => Err(TptrError::new(
ErrorCode::DeviceNotFound,
"no real device backend",
)),
}
}
pub(super) fn synchronize(&self) -> TptrResult<()> {
match self {
#[cfg(feature = "cuda")]
DeviceBackend::Cuda(ctx) => ctx.synchronize(),
DeviceBackend::None => Ok(()),
}
}
pub(super) fn is_real(&self) -> bool {
#[cfg(feature = "cuda")]
{
matches!(self, DeviceBackend::Cuda(_))
}
#[cfg(not(feature = "cuda"))]
{
false
}
}
}