1use crate::error::{ErrorCode, TptrError, TptrResult};
12
13#[cfg(feature = "cuda")]
14mod imp {
15 use super::*;
16 use std::sync::Arc;
17 use tpt_gpu_primitives::sym;
18 use tpt_gpu_primitives::vendor::dynlink::Library;
19
20 type CUresult = i32;
21 type CUdevice = i32;
22 type CUcontext = *mut std::ffi::c_void;
23 type CUdeviceptr = u64;
24
25 const CUDA_SUCCESS: CUresult = 0;
26
27 fn map(e: impl std::fmt::Display) -> TptrError {
28 TptrError::new(ErrorCode::DeviceNotFound, e.to_string())
29 }
30
31 pub struct CudaContext {
33 #[allow(dead_code)]
34 lib_cuda: Arc<Library>,
35 #[allow(dead_code)]
36 lib_cublas: Arc<Library>,
37 #[allow(dead_code)]
38 ctx: CUcontext,
39 cu_mem_alloc: unsafe extern "C" fn(*mut CUdeviceptr, usize) -> CUresult,
40 cu_mem_free: unsafe extern "C" fn(CUdeviceptr) -> CUresult,
41 cu_memcpy_htod:
42 unsafe extern "C" fn(CUdeviceptr, *const std::ffi::c_void, usize) -> CUresult,
43 cu_memcpy_dtoh: unsafe extern "C" fn(*mut std::ffi::c_void, CUdeviceptr, usize) -> CUresult,
44 cu_ctx_sync: unsafe extern "C" fn() -> CUresult,
45 }
46
47 unsafe impl Send for CudaContext {}
48 unsafe impl Sync for CudaContext {}
49
50 impl CudaContext {
51 pub(super) fn new() -> TptrResult<Self> {
53 let lib_cublas = Arc::new(
54 Library::open("cublas64_12.dll")
55 .or_else(|_| Library::open("cublas64_11.dll"))
56 .or_else(|_| Library::open("libcublas.so.12"))
57 .or_else(|_| Library::open("libcublas.so.11"))
58 .or_else(|_| Library::open("libcublas.so"))
59 .map_err(map)?,
60 );
61 let lib_cuda = Arc::new(
62 Library::open("nvcuda.dll")
63 .or_else(|_| Library::open("libcuda.so"))
64 .map_err(map)?,
65 );
66
67 let cu_init: unsafe extern "C" fn(u32) -> CUresult =
68 unsafe { *lib_cuda.get(sym!("cuInit")).map_err(map)? };
69 let cu_device_get: unsafe extern "C" fn(*mut CUdevice, i32) -> CUresult =
70 unsafe { *lib_cuda.get(sym!("cuDeviceGet")).map_err(map)? };
71 let cu_ctx_create: unsafe extern "C" fn(*mut CUcontext, u32, CUdevice) -> CUresult =
72 unsafe { *lib_cuda.get(sym!("cuCtxCreate")).map_err(map)? };
73 let cu_mem_alloc: unsafe extern "C" fn(*mut CUdeviceptr, usize) -> CUresult =
74 unsafe { *lib_cuda.get(sym!("cuMemAlloc_v2")).map_err(map)? };
75 let cu_mem_free: unsafe extern "C" fn(CUdeviceptr) -> CUresult =
76 unsafe { *lib_cuda.get(sym!("cuMemFree_v2")).map_err(map)? };
77 let cu_memcpy_htod: unsafe extern "C" fn(
78 CUdeviceptr,
79 *const std::ffi::c_void,
80 usize,
81 ) -> CUresult = unsafe { *lib_cuda.get(sym!("cuMemcpyHtoD_v2")).map_err(map)? };
82 let cu_memcpy_dtoh: unsafe extern "C" fn(
83 *mut std::ffi::c_void,
84 CUdeviceptr,
85 usize,
86 ) -> CUresult = unsafe { *lib_cuda.get(sym!("cuMemcpyDtoH_v2")).map_err(map)? };
87 let cu_ctx_sync: unsafe extern "C" fn() -> CUresult =
88 unsafe { *lib_cuda.get(sym!("cuCtxSynchronize")).map_err(map)? };
89 let cu_ctx_set_current: unsafe extern "C" fn(CUcontext) -> CUresult =
90 unsafe { *lib_cuda.get(sym!("cuCtxSetCurrent")).map_err(map)? };
91 let cu_device_primary_ctx_retain: unsafe extern "C" fn(
92 *mut CUcontext,
93 CUdevice,
94 ) -> CUresult = unsafe {
95 *lib_cuda
96 .get(sym!("cuDevicePrimaryCtxRetain"))
97 .map_err(map)?
98 };
99
100 if unsafe { cu_init(0) } != CUDA_SUCCESS {
101 return Err(TptrError::new(
102 ErrorCode::DeviceNotFound,
103 "CUDA: cuInit failed",
104 ));
105 }
106 let mut device: CUdevice = 0;
107 if unsafe { cu_device_get(&mut device, 0) } != CUDA_SUCCESS {
108 return Err(TptrError::new(
109 ErrorCode::DeviceNotFound,
110 "CUDA: cuDeviceGet failed",
111 ));
112 }
113 let mut ctx: CUcontext = std::ptr::null_mut();
119 let retain_res = unsafe { cu_device_primary_ctx_retain(&mut ctx, device) };
120 if retain_res != CUDA_SUCCESS || ctx.is_null() {
121 return Err(TptrError::new(
122 ErrorCode::DeviceNotFound,
123 format!("CUDA: cuDevicePrimaryCtxRetain failed (res={})", retain_res),
124 ));
125 }
126 if unsafe { cu_ctx_set_current(ctx) } != CUDA_SUCCESS {
127 return Err(TptrError::new(
128 ErrorCode::DeviceNotFound,
129 "CUDA: cuCtxSetCurrent failed",
130 ));
131 }
132
133 Ok(Self {
134 lib_cuda,
135 lib_cublas,
136 ctx,
137 cu_mem_alloc,
138 cu_mem_free,
139 cu_memcpy_htod,
140 cu_memcpy_dtoh,
141 cu_ctx_sync,
142 })
143 }
144
145 pub(super) fn alloc(&self, size: u64) -> TptrResult<u64> {
147 let mut ptr: CUdeviceptr = 0;
148 if unsafe { (self.cu_mem_alloc)(&mut ptr, size as usize) } != CUDA_SUCCESS || ptr == 0 {
149 return Err(TptrError::new(
150 ErrorCode::OutOfMemory,
151 "CUDA: cuMemAlloc failed",
152 ));
153 }
154 Ok(ptr)
155 }
156
157 pub(super) fn free(&self, ptr: u64) -> TptrResult<()> {
158 if ptr == 0 {
159 return Ok(());
160 }
161 if unsafe { (self.cu_mem_free)(ptr) } != CUDA_SUCCESS {
162 return Err(TptrError::new(
163 ErrorCode::InvalidAddress,
164 "CUDA: cuMemFree failed",
165 ));
166 }
167 Ok(())
168 }
169
170 pub(super) fn upload(&self, dst: u64, src: &[u8]) -> TptrResult<()> {
171 if unsafe {
172 (self.cu_memcpy_htod)(dst, src.as_ptr() as *const std::ffi::c_void, src.len())
173 } != CUDA_SUCCESS
174 {
175 return Err(TptrError::new(
176 ErrorCode::DeviceLost,
177 "CUDA: cuMemcpyHtoD failed",
178 ));
179 }
180 Ok(())
181 }
182
183 pub(super) fn download(&self, src: u64, dst: &mut [u8]) -> TptrResult<()> {
184 if unsafe {
185 (self.cu_memcpy_dtoh)(dst.as_mut_ptr() as *mut std::ffi::c_void, src, dst.len())
186 } != CUDA_SUCCESS
187 {
188 return Err(TptrError::new(
189 ErrorCode::DeviceLost,
190 "CUDA: cuMemcpyDtoH failed",
191 ));
192 }
193 Ok(())
194 }
195
196 pub(super) fn synchronize(&self) -> TptrResult<()> {
197 if unsafe { (self.cu_ctx_sync)() } != CUDA_SUCCESS {
198 return Err(TptrError::new(
199 ErrorCode::SynchronizationError,
200 "CUDA: cuCtxSynchronize failed",
201 ));
202 }
203 Ok(())
204 }
205 }
206
207 impl Drop for CudaContext {
208 fn drop(&mut self) {
209 unsafe {
212 (self.cu_ctx_sync)();
213 }
214 }
215 }
216}
217
218pub enum DeviceBackend {
222 #[cfg(feature = "cuda")]
223 Cuda(imp::CudaContext),
224 #[allow(dead_code)]
225 None,
226}
227
228impl DeviceBackend {
229 #[allow(dead_code)]
233 pub(super) fn try_cuda() -> Option<Self> {
234 #[cfg(feature = "cuda")]
235 {
236 imp::CudaContext::new().ok().map(DeviceBackend::Cuda)
237 }
238 #[cfg(not(feature = "cuda"))]
239 {
240 None
241 }
242 }
243
244 #[allow(unused_variables)]
245 pub(super) fn alloc(&self, size: u64) -> TptrResult<u64> {
246 match self {
247 #[cfg(feature = "cuda")]
248 DeviceBackend::Cuda(ctx) => ctx.alloc(size),
249 DeviceBackend::None => Err(TptrError::new(
250 ErrorCode::DeviceNotFound,
251 "no real device backend",
252 )),
253 }
254 }
255
256 #[allow(unused_variables)]
257 pub(super) fn free(&self, ptr: u64) -> TptrResult<()> {
258 match self {
259 #[cfg(feature = "cuda")]
260 DeviceBackend::Cuda(ctx) => ctx.free(ptr),
261 DeviceBackend::None => Ok(()),
262 }
263 }
264
265 #[allow(unused_variables)]
266 pub(super) fn upload(&self, dst: u64, src: &[u8]) -> TptrResult<()> {
267 match self {
268 #[cfg(feature = "cuda")]
269 DeviceBackend::Cuda(ctx) => ctx.upload(dst, src),
270 DeviceBackend::None => Err(TptrError::new(
271 ErrorCode::DeviceNotFound,
272 "no real device backend",
273 )),
274 }
275 }
276
277 #[allow(unused_variables)]
278 pub(super) fn download(&self, src: u64, dst: &mut [u8]) -> TptrResult<()> {
279 match self {
280 #[cfg(feature = "cuda")]
281 DeviceBackend::Cuda(ctx) => ctx.download(src, dst),
282 DeviceBackend::None => Err(TptrError::new(
283 ErrorCode::DeviceNotFound,
284 "no real device backend",
285 )),
286 }
287 }
288
289 pub(super) fn synchronize(&self) -> TptrResult<()> {
290 match self {
291 #[cfg(feature = "cuda")]
292 DeviceBackend::Cuda(ctx) => ctx.synchronize(),
293 DeviceBackend::None => Ok(()),
294 }
295 }
296
297 pub(super) fn is_real(&self) -> bool {
298 #[cfg(feature = "cuda")]
299 {
300 matches!(self, DeviceBackend::Cuda(_))
301 }
302 #[cfg(not(feature = "cuda"))]
303 {
304 false
305 }
306 }
307}