1use std::ffi::CStr;
4use std::os::raw::c_void;
5use std::path::Path;
6
7use crate::cuda::DeviceBuffer;
8use crate::cuda::Stream;
9use crate::error::{last_message, trt_check, Error};
10use crate::ffi;
11use crate::Result;
12
13pub const MAX_DIMS: usize = 8;
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum DataType {
19 Float32,
21 Float16,
23 Int8,
25 Int32,
27 Bool,
29 UInt8,
31 Fp8,
33 BFloat16,
35 Int64,
37 Int4,
39}
40
41impl DataType {
42 fn from_code(code: i32) -> Option<Self> {
43 Some(match code {
44 0 => Self::Float32,
45 1 => Self::Float16,
46 2 => Self::Int8,
47 3 => Self::Int32,
48 4 => Self::Bool,
49 5 => Self::UInt8,
50 6 => Self::Fp8,
51 7 => Self::BFloat16,
52 8 => Self::Int64,
53 9 => Self::Int4,
54 _ => return None,
55 })
56 }
57
58 pub fn element_bytes(self) -> Option<usize> {
60 match self {
61 Self::Float32 | Self::Int32 | Self::Int64 => Some(4),
62 Self::Float16 | Self::BFloat16 => Some(2),
63 Self::Int8 | Self::Bool | Self::UInt8 | Self::Fp8 => Some(1),
64 Self::Int4 => None,
65 }
66 }
67}
68
69#[derive(Debug, Clone, Copy, PartialEq, Eq)]
71pub enum IoMode {
72 Input,
74 Output,
76 Internal,
78}
79
80#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct TensorInfo {
83 pub name: String,
85 pub mode: IoMode,
87 pub dtype: DataType,
89 pub shape: Vec<i64>,
91}
92
93impl TensorInfo {
94 pub fn elements(&self) -> Option<usize> {
96 if self.shape.iter().any(|&dim| dim < 0) {
97 return None;
98 }
99 Some(self.shape.iter().map(|&dim| dim as usize).product())
100 }
101
102 pub fn size_bytes(&self) -> Option<usize> {
104 Some(self.elements()? * self.dtype.element_bytes()?)
105 }
106
107 pub fn is_static(&self) -> bool {
109 self.elements().is_some()
110 }
111}
112
113pub fn header_version() -> (u32, u32, u32) {
115 unsafe {
116 (
117 ffi::trt_version_major() as u32,
118 ffi::trt_version_minor() as u32,
119 ffi::trt_version_patch() as u32,
120 )
121 }
122}
123
124#[derive(Debug)]
130pub struct Engine {
131 io: Vec<TensorInfo>,
133 engine: *mut c_void,
134 runtime: *mut c_void,
135}
136
137impl Engine {
138 pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
140 let bytes = std::fs::read(path.as_ref())?;
141 Self::from_bytes(&bytes)
142 }
143
144 pub fn from_bytes_with_verbosity(bytes: &[u8], verbosity: i32) -> Result<Self> {
148 let runtime = unsafe { ffi::trt_runtime_create(verbosity) };
149 if runtime.is_null() {
150 return Err(Error::TensorRt {
151 call: "createInferRuntime",
152 message: last_message(),
153 });
154 }
155 let engine = unsafe {
156 ffi::trt_runtime_deserialize(runtime, bytes.as_ptr() as *const c_void, bytes.len())
157 };
158 if engine.is_null() {
159 let message = last_message();
160 unsafe { ffi::trt_runtime_destroy(runtime) };
161 return Err(Error::TensorRt {
162 call: "deserializeCudaEngine",
163 message,
164 });
165 }
166 let io = match collect_io(engine) {
167 Ok(io) => io,
168 Err(error) => {
169 unsafe {
170 ffi::trt_engine_destroy(engine);
171 ffi::trt_runtime_destroy(runtime);
172 }
173 return Err(error);
174 }
175 };
176 Ok(Self {
177 io,
178 engine,
179 runtime,
180 })
181 }
182
183 pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
185 Self::from_bytes_with_verbosity(bytes, 2)
186 }
187
188 pub fn io(&self) -> &[TensorInfo] {
190 &self.io
191 }
192
193 pub fn tensor(&self, name: &str) -> Option<&TensorInfo> {
195 self.io.iter().find(|info| info.name == name)
196 }
197
198 pub fn inputs(&self) -> impl Iterator<Item = &TensorInfo> {
200 self.io.iter().filter(|info| info.mode == IoMode::Input)
201 }
202
203 pub fn outputs(&self) -> impl Iterator<Item = &TensorInfo> {
205 self.io.iter().filter(|info| info.mode == IoMode::Output)
206 }
207
208 pub fn tensor_names(&self) -> Vec<String> {
210 self.io.iter().map(|info| info.name.clone()).collect()
211 }
212
213 pub fn context(&self) -> Result<Context<'_>> {
216 let context = create_context(self.engine)?;
217 Ok(Context {
218 context,
219 engine: self,
220 })
221 }
222
223 pub fn into_context(self) -> Result<OwnedContext> {
227 let context = create_context(self.engine)?;
228 let owned = OwnedContext {
229 io: self.io.clone(),
230 context,
231 engine: self.engine,
232 runtime: self.runtime,
233 };
234 std::mem::forget(self);
236 Ok(owned)
237 }
238}
239
240impl Drop for Engine {
241 fn drop(&mut self) {
242 unsafe {
243 ffi::trt_engine_destroy(self.engine);
244 ffi::trt_runtime_destroy(self.runtime);
245 }
246 }
247}
248
249fn create_context(engine: *mut c_void) -> Result<*mut c_void> {
250 let context = unsafe { ffi::trt_context_create(engine) };
251 if context.is_null() {
252 return Err(Error::TensorRt {
253 call: "createExecutionContext",
254 message: last_message(),
255 });
256 }
257 Ok(context)
258}
259
260#[derive(Debug)]
262pub struct Context<'engine> {
263 context: *mut c_void,
264 engine: &'engine Engine,
265}
266
267impl<'engine> Context<'engine> {
268 pub fn engine(&self) -> &'engine Engine {
270 self.engine
271 }
272
273 pub fn set_profile(&mut self, index: i32, stream: &Stream) -> Result<()> {
276 set_profile(self.context, index, stream)
277 }
278
279 pub fn set_shape(&mut self, name: &str, shape: &[i64]) -> Result<()> {
281 set_shape(self.context, name, shape)
282 }
283
284 pub unsafe fn bind_raw(
291 &mut self,
292 name: &str,
293 address: *mut c_void,
294 bytes: usize,
295 ) -> Result<()> {
296 bind_raw(self.context, self.engine.io(), name, address, bytes)
297 }
298
299 pub unsafe fn bind_buffer(&mut self, name: &str, buffer: &DeviceBuffer) -> Result<()> {
304 unsafe { self.bind_raw(name, buffer.as_ptr() as *mut c_void, buffer.size_bytes()) }
305 }
306
307 pub fn enqueue(&mut self, stream: &Stream) -> Result<()> {
310 enqueue(self.context, stream)
311 }
312
313 pub fn run(&mut self, stream: &Stream) -> Result<()> {
315 run(self.context, stream)
316 }
317}
318
319impl Drop for Context<'_> {
320 fn drop(&mut self) {
321 unsafe { ffi::trt_context_destroy(self.context) };
322 }
323}
324
325#[derive(Debug)]
328pub struct OwnedContext {
329 io: Vec<TensorInfo>,
330 context: *mut c_void,
331 engine: *mut c_void,
333 runtime: *mut c_void,
334}
335
336impl OwnedContext {
337 pub fn io(&self) -> &[TensorInfo] {
339 &self.io
340 }
341
342 pub fn tensor(&self, name: &str) -> Option<&TensorInfo> {
344 self.io.iter().find(|info| info.name == name)
345 }
346
347 pub fn inputs(&self) -> impl Iterator<Item = &TensorInfo> {
349 self.io.iter().filter(|info| info.mode == IoMode::Input)
350 }
351
352 pub fn outputs(&self) -> impl Iterator<Item = &TensorInfo> {
354 self.io.iter().filter(|info| info.mode == IoMode::Output)
355 }
356
357 pub fn set_profile(&mut self, index: i32, stream: &Stream) -> Result<()> {
359 set_profile(self.context, index, stream)
360 }
361
362 pub fn set_shape(&mut self, name: &str, shape: &[i64]) -> Result<()> {
364 set_shape(self.context, name, shape)
365 }
366
367 pub unsafe fn bind_raw(
372 &mut self,
373 name: &str,
374 address: *mut c_void,
375 bytes: usize,
376 ) -> Result<()> {
377 bind_raw(self.context, &self.io, name, address, bytes)
378 }
379
380 pub unsafe fn bind_buffer(&mut self, name: &str, buffer: &DeviceBuffer) -> Result<()> {
385 unsafe { self.bind_raw(name, buffer.as_ptr() as *mut c_void, buffer.size_bytes()) }
386 }
387
388 pub fn enqueue(&mut self, stream: &Stream) -> Result<()> {
390 enqueue(self.context, stream)
391 }
392
393 pub fn run(&mut self, stream: &Stream) -> Result<()> {
395 run(self.context, stream)
396 }
397}
398
399unsafe impl Send for OwnedContext {}
402
403impl Drop for OwnedContext {
404 fn drop(&mut self) {
405 unsafe {
406 ffi::trt_context_destroy(self.context);
407 ffi::trt_engine_destroy(self.engine);
408 ffi::trt_runtime_destroy(self.runtime);
409 }
410 }
411}
412
413fn set_profile(context: *mut c_void, index: i32, stream: &Stream) -> Result<()> {
414 trt_check(
415 unsafe { ffi::trt_context_set_profile(context, index, stream.as_raw()) },
416 "setOptimizationProfileAsync",
417 )
418}
419
420fn set_shape(context: *mut c_void, name: &str, shape: &[i64]) -> Result<()> {
421 if shape.len() > MAX_DIMS {
422 return Err(Error::TooManyDimensions {
423 found: shape.len(),
424 max: MAX_DIMS,
425 });
426 }
427 let cname = crate::util::to_cstring(name)?;
428 trt_check(
429 unsafe {
430 ffi::trt_context_set_shape(context, cname.as_ptr(), shape.as_ptr(), shape.len() as i32)
431 },
432 "setInputShape",
433 )
434}
435
436unsafe fn bind_raw(
437 context: *mut c_void,
438 io: &[TensorInfo],
439 name: &str,
440 address: *mut c_void,
441 bytes: usize,
442) -> Result<()> {
443 let info = io
444 .iter()
445 .find(|info| info.name == name)
446 .ok_or_else(|| Error::UnknownTensor {
447 name: name.to_string(),
448 available: io.iter().map(|info| info.name.clone()).collect(),
449 })?;
450 if address.is_null() {
451 return Err(Error::SizeMismatch {
452 tensor: name.to_string(),
453 expected: info.size_bytes().unwrap_or(bytes),
454 found: 0,
455 });
456 }
457 if let Some(expected) = info.size_bytes() {
458 if expected != bytes {
459 return Err(Error::SizeMismatch {
460 tensor: name.to_string(),
461 expected,
462 found: bytes,
463 });
464 }
465 }
466 let cname = crate::util::to_cstring(name)?;
467 trt_check(
468 unsafe { ffi::trt_context_bind(context, cname.as_ptr(), address as *const c_void) },
469 "setTensorAddress",
470 )
471}
472
473fn enqueue(context: *mut c_void, stream: &Stream) -> Result<()> {
474 trt_check(
475 unsafe { ffi::trt_context_enqueue(context, stream.as_raw()) },
476 "enqueueV3",
477 )
478}
479
480fn run(context: *mut c_void, stream: &Stream) -> Result<()> {
481 enqueue(context, stream)?;
482 stream.synchronize()
483}
484
485fn collect_io(engine: *mut c_void) -> Result<Vec<TensorInfo>> {
486 let count = unsafe { ffi::trt_engine_num_io(engine) };
487 let mut out = Vec::with_capacity(count.max(0) as usize);
488 let mut dims = [0i64; MAX_DIMS];
489 for index in 0..count {
490 let raw_name = unsafe { ffi::trt_engine_io_name(engine, index) };
491 if raw_name.is_null() {
492 return Err(Error::TensorRt {
493 call: "getIOTensorName",
494 message: last_message(),
495 });
496 }
497 let name = unsafe { CStr::from_ptr(raw_name) }
498 .to_string_lossy()
499 .into_owned();
500 let mode_code = unsafe { ffi::trt_engine_io_mode(engine, raw_name) };
501 let mode = match mode_code {
502 0 => IoMode::Internal,
503 1 => IoMode::Input,
504 2 => IoMode::Output,
505 _ => {
506 return Err(Error::TensorRt {
507 call: "getTensorIOMode",
508 message: format!("{name}: unknown io mode {mode_code}"),
509 })
510 }
511 };
512 let dtype_code = unsafe { ffi::trt_engine_io_dtype(engine, raw_name) };
513 let dtype = DataType::from_code(dtype_code).ok_or(Error::UnsupportedDataType {
514 tensor: name.clone(),
515 code: dtype_code,
516 })?;
517 let count_dims = unsafe {
518 ffi::trt_engine_io_dims(engine, raw_name, dims.as_mut_ptr(), MAX_DIMS as i32)
519 };
520 if count_dims < 0 {
521 return Err(Error::TensorRt {
522 call: "getTensorShape",
523 message: last_message(),
524 });
525 }
526 out.push(TensorInfo {
527 name,
528 mode,
529 dtype,
530 shape: dims[..count_dims as usize].to_vec(),
531 });
532 }
533 Ok(out)
534}