Skip to main content

trt_runner/
engine.rs

1//! Loading an engine, reading its I/O tensors, and running one execution context.
2
3use std::ffi::CStr;
4use std::os::raw::c_void;
5use std::path::Path;
6
7use crate::cuda::DeviceBuffer;
8use crate::cuda::Stream;
9use crate::error::{last_message, trt_check, Error};
10use crate::ffi;
11use crate::Result;
12
13/// Maximum dimensions TensorRT allows for one tensor.
14pub const MAX_DIMS: usize = 8;
15
16/// Element type of a tensor, as reported by the engine.
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum DataType {
19    /// `f32`.
20    Float32,
21    /// IEEE half.
22    Float16,
23    /// Signed 8 bit integer.
24    Int8,
25    /// `i32`.
26    Int32,
27    /// One byte per value.
28    Bool,
29    /// Unsigned 8 bit integer.
30    UInt8,
31    /// 8 bit floating point.
32    Fp8,
33    /// Brain float.
34    BFloat16,
35    /// `i64`.
36    Int64,
37    /// Two values per byte, so a whole-tensor byte count is not `elements * size`.
38    Int4,
39}
40
41impl DataType {
42    fn from_code(code: i32) -> Option<Self> {
43        Some(match code {
44            0 => Self::Float32,
45            1 => Self::Float16,
46            2 => Self::Int8,
47            3 => Self::Int32,
48            4 => Self::Bool,
49            5 => Self::UInt8,
50            6 => Self::Fp8,
51            7 => Self::BFloat16,
52            8 => Self::Int64,
53            9 => Self::Int4,
54            _ => return None,
55        })
56    }
57
58    /// Bytes per element, or `None` for the sub-byte types.
59    pub fn element_bytes(self) -> Option<usize> {
60        match self {
61            Self::Float32 | Self::Int32 | Self::Int64 => Some(4),
62            Self::Float16 | Self::BFloat16 => Some(2),
63            Self::Int8 | Self::Bool | Self::UInt8 | Self::Fp8 => Some(1),
64            Self::Int4 => None,
65        }
66    }
67}
68
69/// Whether the engine reads or writes a tensor.
70#[derive(Debug, Clone, Copy, PartialEq, Eq)]
71pub enum IoMode {
72    /// The caller writes it.
73    Input,
74    /// The caller reads it back.
75    Output,
76    /// Present in the graph but neither bound by the caller nor read back.
77    Internal,
78}
79
80/// One I/O tensor of an engine: name, direction, element type and declared shape.
81#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct TensorInfo {
83    /// Engine side name, e.g. `images` or `output0`.
84    pub name: String,
85    /// Direction of the tensor.
86    pub mode: IoMode,
87    /// Element type.
88    pub dtype: DataType,
89    /// Dimensions as declared in the engine. `-1` marks a dynamic dimension.
90    pub shape: Vec<i64>,
91}
92
93impl TensorInfo {
94    /// Number of elements, or `None` when any dimension is dynamic.
95    pub fn elements(&self) -> Option<usize> {
96        if self.shape.iter().any(|&dim| dim < 0) {
97            return None;
98        }
99        Some(self.shape.iter().map(|&dim| dim as usize).product())
100    }
101
102    /// Size in bytes, or `None` for dynamic shapes and sub-byte element types.
103    pub fn size_bytes(&self) -> Option<usize> {
104        Some(self.elements()? * self.dtype.element_bytes()?)
105    }
106
107    /// True when the shape is fixed, which is the common case for an exported detection model.
108    pub fn is_static(&self) -> bool {
109        self.elements().is_some()
110    }
111}
112
113/// The TensorRT header version this crate was compiled against.
114pub fn header_version() -> (u32, u32, u32) {
115    unsafe {
116        (
117            ffi::trt_version_major() as u32,
118            ffi::trt_version_minor() as u32,
119            ffi::trt_version_patch() as u32,
120        )
121    }
122}
123
124/// A deserialized engine plus the runtime that produced it.
125///
126/// Engines are built for one GPU, one driver and one exact TensorRT build: an engine built elsewhere
127/// will usually fail to load here, and that failure is reported as
128/// [`Error::TensorRt`][Error::TensorRt] rather than a crash.
129#[derive(Debug)]
130pub struct Engine {
131    // declaration order matters on drop: tensors, then engine, then the runtime that owns it
132    io: Vec<TensorInfo>,
133    engine: *mut c_void,
134    runtime: *mut c_void,
135}
136
137impl Engine {
138    /// Reads and deserializes a serialized engine.
139    pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
140        let bytes = std::fs::read(path.as_ref())?;
141        Self::from_bytes(&bytes)
142    }
143
144    /// Deserializes engine bytes already in memory.
145    ///
146    /// `verbosity` follows TensorRT's logger levels: 2 keeps warnings and errors, 4 is very chatty.
147    pub fn from_bytes_with_verbosity(bytes: &[u8], verbosity: i32) -> Result<Self> {
148        let runtime = unsafe { ffi::trt_runtime_create(verbosity) };
149        if runtime.is_null() {
150            return Err(Error::TensorRt {
151                call: "createInferRuntime",
152                message: last_message(),
153            });
154        }
155        let engine = unsafe {
156            ffi::trt_runtime_deserialize(runtime, bytes.as_ptr() as *const c_void, bytes.len())
157        };
158        if engine.is_null() {
159            let message = last_message();
160            unsafe { ffi::trt_runtime_destroy(runtime) };
161            return Err(Error::TensorRt {
162                call: "deserializeCudaEngine",
163                message,
164            });
165        }
166        let io = match collect_io(engine) {
167            Ok(io) => io,
168            Err(error) => {
169                unsafe {
170                    ffi::trt_engine_destroy(engine);
171                    ffi::trt_runtime_destroy(runtime);
172                }
173                return Err(error);
174            }
175        };
176        Ok(Self {
177            io,
178            engine,
179            runtime,
180        })
181    }
182
183    /// Deserializes with warnings and errors only.
184    pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
185        Self::from_bytes_with_verbosity(bytes, 2)
186    }
187
188    /// Every tensor the caller can bind, in engine order.
189    pub fn io(&self) -> &[TensorInfo] {
190        &self.io
191    }
192
193    /// Looks a tensor up by name.
194    pub fn tensor(&self, name: &str) -> Option<&TensorInfo> {
195        self.io.iter().find(|info| info.name == name)
196    }
197
198    /// Tensors the caller must fill.
199    pub fn inputs(&self) -> impl Iterator<Item = &TensorInfo> {
200        self.io.iter().filter(|info| info.mode == IoMode::Input)
201    }
202
203    /// Tensors the caller reads back.
204    pub fn outputs(&self) -> impl Iterator<Item = &TensorInfo> {
205        self.io.iter().filter(|info| info.mode == IoMode::Output)
206    }
207
208    /// Names of all I/O tensors, for error messages.
209    pub fn tensor_names(&self) -> Vec<String> {
210        self.io.iter().map(|info| info.name.clone()).collect()
211    }
212
213    /// Creates an execution context. One context handles one shape combination at a time and is not
214    /// `Sync`; create one per thread instead.
215    pub fn context(&self) -> Result<Context<'_>> {
216        let context = create_context(self.engine)?;
217        Ok(Context {
218            context,
219            engine: self,
220        })
221    }
222
223    /// Consumes the engine into an [`OwnedContext`], which keeps the engine and the runtime alive
224    /// beside the execution context. This is the shape a long lived runner usually wants: one value
225    /// to store in a struct, without a lifetime parameter chasing it.
226    pub fn into_context(self) -> Result<OwnedContext> {
227        let context = create_context(self.engine)?;
228        let owned = OwnedContext {
229            io: self.io.clone(),
230            context,
231            engine: self.engine,
232            runtime: self.runtime,
233        };
234        // the three raw handles now belong to the OwnedContext, which drops them in the same order
235        std::mem::forget(self);
236        Ok(owned)
237    }
238}
239
240impl Drop for Engine {
241    fn drop(&mut self) {
242        unsafe {
243            ffi::trt_engine_destroy(self.engine);
244            ffi::trt_runtime_destroy(self.runtime);
245        }
246    }
247}
248
249fn create_context(engine: *mut c_void) -> Result<*mut c_void> {
250    let context = unsafe { ffi::trt_context_create(engine) };
251    if context.is_null() {
252        return Err(Error::TensorRt {
253            call: "createExecutionContext",
254            message: last_message(),
255        });
256    }
257    Ok(context)
258}
259
260/// A bound execution context. Borrows the engine so it can never outlive it.
261#[derive(Debug)]
262pub struct Context<'engine> {
263    context: *mut c_void,
264    engine: &'engine Engine,
265}
266
267impl<'engine> Context<'engine> {
268    /// The engine this context came from.
269    pub fn engine(&self) -> &'engine Engine {
270        self.engine
271    }
272
273    /// Selects an optimization profile. Only needed for engines with more than one dynamic shape
274    /// profile, and only before [`Context::set_shape`].
275    pub fn set_profile(&mut self, index: i32, stream: &Stream) -> Result<()> {
276        set_profile(self.context, index, stream)
277    }
278
279    /// Declares the shape of a dynamic input.
280    pub fn set_shape(&mut self, name: &str, shape: &[i64]) -> Result<()> {
281        set_shape(self.context, name, shape)
282    }
283
284    /// Points a tensor at device memory. The buffer has to stay alive, and stay put, until the last
285    /// `enqueue` using this binding has been synchronised.
286    ///
287    /// # Safety
288    /// `address` must be a device pointer with at least `bytes` readable or writable, depending on
289    /// the direction of the tensor.
290    pub unsafe fn bind_raw(
291        &mut self,
292        name: &str,
293        address: *mut c_void,
294        bytes: usize,
295    ) -> Result<()> {
296        bind_raw(self.context, self.engine.io(), name, address, bytes)
297    }
298
299    /// Binds a [`DeviceBuffer`] the caller allocated, checking its size against the engine.
300    ///
301    /// # Safety
302    /// Same contract as [`Context::bind_raw`]: `buffer` must outlive every enqueue using the binding.
303    pub unsafe fn bind_buffer(&mut self, name: &str, buffer: &DeviceBuffer) -> Result<()> {
304        unsafe { self.bind_raw(name, buffer.as_ptr() as *mut c_void, buffer.size_bytes()) }
305    }
306
307    /// Queues the network on `stream` and returns immediately. The output buffers are not readable
308    /// until `stream` reports the work finished.
309    pub fn enqueue(&mut self, stream: &Stream) -> Result<()> {
310        enqueue(self.context, stream)
311    }
312
313    /// Queues the network and blocks until it is done.
314    pub fn run(&mut self, stream: &Stream) -> Result<()> {
315        run(self.context, stream)
316    }
317}
318
319impl Drop for Context<'_> {
320    fn drop(&mut self) {
321        unsafe { ffi::trt_context_destroy(self.context) };
322    }
323}
324
325/// An execution context that owns the engine and runtime behind it, so it can be stored in a struct
326/// without a lifetime parameter.
327#[derive(Debug)]
328pub struct OwnedContext {
329    io: Vec<TensorInfo>,
330    context: *mut c_void,
331    // kept alive, and destroyed after, the context
332    engine: *mut c_void,
333    runtime: *mut c_void,
334}
335
336impl OwnedContext {
337    /// Every bindable tensor.
338    pub fn io(&self) -> &[TensorInfo] {
339        &self.io
340    }
341
342    /// Looks a tensor up by name.
343    pub fn tensor(&self, name: &str) -> Option<&TensorInfo> {
344        self.io.iter().find(|info| info.name == name)
345    }
346
347    /// Tensors the caller must fill.
348    pub fn inputs(&self) -> impl Iterator<Item = &TensorInfo> {
349        self.io.iter().filter(|info| info.mode == IoMode::Input)
350    }
351
352    /// Tensors the caller reads back.
353    pub fn outputs(&self) -> impl Iterator<Item = &TensorInfo> {
354        self.io.iter().filter(|info| info.mode == IoMode::Output)
355    }
356
357    /// See [`Context::set_profile`].
358    pub fn set_profile(&mut self, index: i32, stream: &Stream) -> Result<()> {
359        set_profile(self.context, index, stream)
360    }
361
362    /// See [`Context::set_shape`].
363    pub fn set_shape(&mut self, name: &str, shape: &[i64]) -> Result<()> {
364        set_shape(self.context, name, shape)
365    }
366
367    /// See [`Context::bind_raw`].
368    ///
369    /// # Safety
370    /// `address` must be a device pointer that stays valid for the lifetime of the binding.
371    pub unsafe fn bind_raw(
372        &mut self,
373        name: &str,
374        address: *mut c_void,
375        bytes: usize,
376    ) -> Result<()> {
377        bind_raw(self.context, &self.io, name, address, bytes)
378    }
379
380    /// See [`Context::bind_buffer`].
381    ///
382    /// # Safety
383    /// `buffer` must outlive every enqueue that uses the binding.
384    pub unsafe fn bind_buffer(&mut self, name: &str, buffer: &DeviceBuffer) -> Result<()> {
385        unsafe { self.bind_raw(name, buffer.as_ptr() as *mut c_void, buffer.size_bytes()) }
386    }
387
388    /// See [`Context::enqueue`].
389    pub fn enqueue(&mut self, stream: &Stream) -> Result<()> {
390        enqueue(self.context, stream)
391    }
392
393    /// See [`Context::run`].
394    pub fn run(&mut self, stream: &Stream) -> Result<()> {
395        run(self.context, stream)
396    }
397}
398
399// A context belongs to one thread at a time and is not `Sync`; handing ownership to another thread
400// is fine as long as that thread has the CUDA context current, which is the caller's job.
401unsafe impl Send for OwnedContext {}
402
403impl Drop for OwnedContext {
404    fn drop(&mut self) {
405        unsafe {
406            ffi::trt_context_destroy(self.context);
407            ffi::trt_engine_destroy(self.engine);
408            ffi::trt_runtime_destroy(self.runtime);
409        }
410    }
411}
412
413fn set_profile(context: *mut c_void, index: i32, stream: &Stream) -> Result<()> {
414    trt_check(
415        unsafe { ffi::trt_context_set_profile(context, index, stream.as_raw()) },
416        "setOptimizationProfileAsync",
417    )
418}
419
420fn set_shape(context: *mut c_void, name: &str, shape: &[i64]) -> Result<()> {
421    if shape.len() > MAX_DIMS {
422        return Err(Error::TooManyDimensions {
423            found: shape.len(),
424            max: MAX_DIMS,
425        });
426    }
427    let cname = crate::util::to_cstring(name)?;
428    trt_check(
429        unsafe {
430            ffi::trt_context_set_shape(context, cname.as_ptr(), shape.as_ptr(), shape.len() as i32)
431        },
432        "setInputShape",
433    )
434}
435
436unsafe fn bind_raw(
437    context: *mut c_void,
438    io: &[TensorInfo],
439    name: &str,
440    address: *mut c_void,
441    bytes: usize,
442) -> Result<()> {
443    let info = io
444        .iter()
445        .find(|info| info.name == name)
446        .ok_or_else(|| Error::UnknownTensor {
447            name: name.to_string(),
448            available: io.iter().map(|info| info.name.clone()).collect(),
449        })?;
450    if address.is_null() {
451        return Err(Error::SizeMismatch {
452            tensor: name.to_string(),
453            expected: info.size_bytes().unwrap_or(bytes),
454            found: 0,
455        });
456    }
457    if let Some(expected) = info.size_bytes() {
458        if expected != bytes {
459            return Err(Error::SizeMismatch {
460                tensor: name.to_string(),
461                expected,
462                found: bytes,
463            });
464        }
465    }
466    let cname = crate::util::to_cstring(name)?;
467    trt_check(
468        unsafe { ffi::trt_context_bind(context, cname.as_ptr(), address as *const c_void) },
469        "setTensorAddress",
470    )
471}
472
473fn enqueue(context: *mut c_void, stream: &Stream) -> Result<()> {
474    trt_check(
475        unsafe { ffi::trt_context_enqueue(context, stream.as_raw()) },
476        "enqueueV3",
477    )
478}
479
480fn run(context: *mut c_void, stream: &Stream) -> Result<()> {
481    enqueue(context, stream)?;
482    stream.synchronize()
483}
484
485fn collect_io(engine: *mut c_void) -> Result<Vec<TensorInfo>> {
486    let count = unsafe { ffi::trt_engine_num_io(engine) };
487    let mut out = Vec::with_capacity(count.max(0) as usize);
488    let mut dims = [0i64; MAX_DIMS];
489    for index in 0..count {
490        let raw_name = unsafe { ffi::trt_engine_io_name(engine, index) };
491        if raw_name.is_null() {
492            return Err(Error::TensorRt {
493                call: "getIOTensorName",
494                message: last_message(),
495            });
496        }
497        let name = unsafe { CStr::from_ptr(raw_name) }
498            .to_string_lossy()
499            .into_owned();
500        let mode_code = unsafe { ffi::trt_engine_io_mode(engine, raw_name) };
501        let mode = match mode_code {
502            0 => IoMode::Internal,
503            1 => IoMode::Input,
504            2 => IoMode::Output,
505            _ => {
506                return Err(Error::TensorRt {
507                    call: "getTensorIOMode",
508                    message: format!("{name}: unknown io mode {mode_code}"),
509                })
510            }
511        };
512        let dtype_code = unsafe { ffi::trt_engine_io_dtype(engine, raw_name) };
513        let dtype = DataType::from_code(dtype_code).ok_or(Error::UnsupportedDataType {
514            tensor: name.clone(),
515            code: dtype_code,
516        })?;
517        let count_dims = unsafe {
518            ffi::trt_engine_io_dims(engine, raw_name, dims.as_mut_ptr(), MAX_DIMS as i32)
519        };
520        if count_dims < 0 {
521            return Err(Error::TensorRt {
522                call: "getTensorShape",
523                message: last_message(),
524            });
525        }
526        out.push(TensorInfo {
527            name,
528            mode,
529            dtype,
530            shape: dims[..count_dims as usize].to_vec(),
531        });
532    }
533    Ok(out)
534}