Skip to main content

trt_runner/
lib.rs

1//! Inference against a serialized [TensorRT] engine, in as few layers as possible.
2//!
3//! [`Engine`] deserializes bytes, [`Context`] binds device buffers to the engine's I/O tensors and
4//! runs [`enqueueV3`][Context::enqueue]. Nothing else: no preprocessing, no model-specific decoding,
5//! no bindings generator, and no third-party runtime dependencies.
6//!
7//! # Targets TensorRT 10
8//!
9//! Only the inference runtime (`libnvinfer`) and the CUDA runtime (`libcudart`) are linked. In
10//! particular `libnvparsers` is not, because NVIDIA removed the NvParser API in TensorRT 10; bindings
11//! that still link it cannot be built on JetPack 6. Building against TensorRT 8 headers fails at
12//! compile time with a message rather than at run time with a missing symbol.
13//!
14//! Set `TENSORRT_INCLUDE_PATH`, `TENSORRT_LIB_PATH`, `CUDA_INCLUDE_PATH` and `CUDA_LIB_PATH` when the
15//! toolchain is in an unusual place. On Jetson the headers live in `/usr/include/aarch64-linux-gnu`.
16//!
17//! # Two rules this crate is designed around
18//!
19//! **A context has to be current on the calling thread.** Use [`cuda::set_device`] rather than
20//! driver-API context retention: `cudaSetDevice` activates the primary context for the thread, while
21//! `cuDevicePrimaryCtxRetain` only takes a reference to it and leaves later calls failing with
22//! `CUDA_ERROR_INVALID_CONTEXT`.
23//!
24//! **Asynchronous copies are not finished when they return.** [`DeviceBuffer::copy_to_host`] waits
25//! for the stream and is the safe default; [`DeviceBuffer::copy_to_host_async`] exists for pipelines
26//! that want to overlap several transfers, and reading the destination before synchronising the
27//! stream that carried them yields zeroes. That failure mode looks exactly like a preprocessing bug,
28//! so it is worth the extra method name.
29//!
30//! # Example
31//!
32//! ```no_run
33//! use trt_runner::{cuda, Context, DeviceBuffer, Engine, IoMode, Stream};
34//!
35//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
36//! cuda::set_device(0)?;
37//! let engine = Engine::from_file("model_fp16.engine")?;
38//! let stream = Stream::new()?;
39//! let mut context = engine.context()?;
40//!
41//! // one device buffer per I/O tensor, sized from the engine's own metadata
42//! let mut buffers = Vec::new();
43//! for info in engine.io() {
44//!     let bytes = info.size_bytes().expect("a static shape");
45//!     let mut buffer = DeviceBuffer::new(bytes)?;
46//!     // SAFETY: the buffers live until after the last synchronised enqueue.
47//!     unsafe { context.bind_buffer(&info.name, &buffer)? };
48//!     buffers.push((info.name.clone(), info.mode, buffer));
49//! }
50//!
51//! context.run(&stream)?; // enqueue plus synchronize
52//!
53//! for (name, mode, buffer) in &buffers {
54//!     if *mode == IoMode::Output {
55//!         let mut head = vec![0u8; 16.min(buffer.size_bytes())];
56//!         buffer.copy_to_host(&mut head, &stream)?;
57//!         println!("{name}: {} bytes", buffer.size_bytes());
58//!     }
59//! }
60//! # Ok(())
61//! # }
62//! ```
63//!
64//! [TensorRT]: https://docs.nvidia.com/deeplearning/tensorrt/
65
66#![deny(missing_docs)]
67
68pub mod cuda;
69mod engine;
70mod error;
71mod ffi;
72mod util;
73
74pub use cuda::{DeviceBuffer, PinnedBuffer, Stream};
75pub use engine::{
76    header_version, Context, DataType, Engine, IoMode, OwnedContext, TensorInfo, MAX_DIMS,
77};
78pub use error::Error;
79
80/// Result alias with this crate's error type.
81pub type Result<T> = std::result::Result<T, Error>;