use std::ffi::CStr;
use std::os::raw::c_void;
use std::path::Path;
use crate::cuda::DeviceBuffer;
use crate::cuda::Stream;
use crate::error::{last_message, trt_check, Error};
use crate::ffi;
use crate::Result;
pub const MAX_DIMS: usize = 8;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DataType {
Float32,
Float16,
Int8,
Int32,
Bool,
UInt8,
Fp8,
BFloat16,
Int64,
Int4,
}
impl DataType {
fn from_code(code: i32) -> Option<Self> {
Some(match code {
0 => Self::Float32,
1 => Self::Float16,
2 => Self::Int8,
3 => Self::Int32,
4 => Self::Bool,
5 => Self::UInt8,
6 => Self::Fp8,
7 => Self::BFloat16,
8 => Self::Int64,
9 => Self::Int4,
_ => return None,
})
}
pub fn element_bytes(self) -> Option<usize> {
match self {
Self::Float32 | Self::Int32 | Self::Int64 => Some(4),
Self::Float16 | Self::BFloat16 => Some(2),
Self::Int8 | Self::Bool | Self::UInt8 | Self::Fp8 => Some(1),
Self::Int4 => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IoMode {
Input,
Output,
Internal,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TensorInfo {
pub name: String,
pub mode: IoMode,
pub dtype: DataType,
pub shape: Vec<i64>,
}
impl TensorInfo {
pub fn elements(&self) -> Option<usize> {
if self.shape.iter().any(|&dim| dim < 0) {
return None;
}
Some(self.shape.iter().map(|&dim| dim as usize).product())
}
pub fn size_bytes(&self) -> Option<usize> {
Some(self.elements()? * self.dtype.element_bytes()?)
}
pub fn is_static(&self) -> bool {
self.elements().is_some()
}
}
pub fn header_version() -> (u32, u32, u32) {
unsafe {
(
ffi::trt_version_major() as u32,
ffi::trt_version_minor() as u32,
ffi::trt_version_patch() as u32,
)
}
}
#[derive(Debug)]
pub struct Engine {
io: Vec<TensorInfo>,
engine: *mut c_void,
runtime: *mut c_void,
}
impl Engine {
pub fn from_file(path: impl AsRef<Path>) -> Result<Self> {
let bytes = std::fs::read(path.as_ref())?;
Self::from_bytes(&bytes)
}
pub fn from_bytes_with_verbosity(bytes: &[u8], verbosity: i32) -> Result<Self> {
let runtime = unsafe { ffi::trt_runtime_create(verbosity) };
if runtime.is_null() {
return Err(Error::TensorRt {
call: "createInferRuntime",
message: last_message(),
});
}
let engine = unsafe {
ffi::trt_runtime_deserialize(runtime, bytes.as_ptr() as *const c_void, bytes.len())
};
if engine.is_null() {
let message = last_message();
unsafe { ffi::trt_runtime_destroy(runtime) };
return Err(Error::TensorRt {
call: "deserializeCudaEngine",
message,
});
}
let io = match collect_io(engine) {
Ok(io) => io,
Err(error) => {
unsafe {
ffi::trt_engine_destroy(engine);
ffi::trt_runtime_destroy(runtime);
}
return Err(error);
}
};
Ok(Self {
io,
engine,
runtime,
})
}
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
Self::from_bytes_with_verbosity(bytes, 2)
}
pub fn io(&self) -> &[TensorInfo] {
&self.io
}
pub fn tensor(&self, name: &str) -> Option<&TensorInfo> {
self.io.iter().find(|info| info.name == name)
}
pub fn inputs(&self) -> impl Iterator<Item = &TensorInfo> {
self.io.iter().filter(|info| info.mode == IoMode::Input)
}
pub fn outputs(&self) -> impl Iterator<Item = &TensorInfo> {
self.io.iter().filter(|info| info.mode == IoMode::Output)
}
pub fn tensor_names(&self) -> Vec<String> {
self.io.iter().map(|info| info.name.clone()).collect()
}
pub fn context(&self) -> Result<Context<'_>> {
let context = create_context(self.engine)?;
Ok(Context {
context,
engine: self,
})
}
pub fn into_context(self) -> Result<OwnedContext> {
let context = create_context(self.engine)?;
let owned = OwnedContext {
io: self.io.clone(),
context,
engine: self.engine,
runtime: self.runtime,
};
std::mem::forget(self);
Ok(owned)
}
}
impl Drop for Engine {
fn drop(&mut self) {
unsafe {
ffi::trt_engine_destroy(self.engine);
ffi::trt_runtime_destroy(self.runtime);
}
}
}
fn create_context(engine: *mut c_void) -> Result<*mut c_void> {
let context = unsafe { ffi::trt_context_create(engine) };
if context.is_null() {
return Err(Error::TensorRt {
call: "createExecutionContext",
message: last_message(),
});
}
Ok(context)
}
#[derive(Debug)]
pub struct Context<'engine> {
context: *mut c_void,
engine: &'engine Engine,
}
impl<'engine> Context<'engine> {
pub fn engine(&self) -> &'engine Engine {
self.engine
}
pub fn set_profile(&mut self, index: i32, stream: &Stream) -> Result<()> {
set_profile(self.context, index, stream)
}
pub fn set_shape(&mut self, name: &str, shape: &[i64]) -> Result<()> {
set_shape(self.context, name, shape)
}
pub unsafe fn bind_raw(
&mut self,
name: &str,
address: *mut c_void,
bytes: usize,
) -> Result<()> {
bind_raw(self.context, self.engine.io(), name, address, bytes)
}
pub unsafe fn bind_buffer(&mut self, name: &str, buffer: &DeviceBuffer) -> Result<()> {
unsafe { self.bind_raw(name, buffer.as_ptr() as *mut c_void, buffer.size_bytes()) }
}
pub fn enqueue(&mut self, stream: &Stream) -> Result<()> {
enqueue(self.context, stream)
}
pub fn run(&mut self, stream: &Stream) -> Result<()> {
run(self.context, stream)
}
}
impl Drop for Context<'_> {
fn drop(&mut self) {
unsafe { ffi::trt_context_destroy(self.context) };
}
}
#[derive(Debug)]
pub struct OwnedContext {
io: Vec<TensorInfo>,
context: *mut c_void,
engine: *mut c_void,
runtime: *mut c_void,
}
impl OwnedContext {
pub fn io(&self) -> &[TensorInfo] {
&self.io
}
pub fn tensor(&self, name: &str) -> Option<&TensorInfo> {
self.io.iter().find(|info| info.name == name)
}
pub fn inputs(&self) -> impl Iterator<Item = &TensorInfo> {
self.io.iter().filter(|info| info.mode == IoMode::Input)
}
pub fn outputs(&self) -> impl Iterator<Item = &TensorInfo> {
self.io.iter().filter(|info| info.mode == IoMode::Output)
}
pub fn set_profile(&mut self, index: i32, stream: &Stream) -> Result<()> {
set_profile(self.context, index, stream)
}
pub fn set_shape(&mut self, name: &str, shape: &[i64]) -> Result<()> {
set_shape(self.context, name, shape)
}
pub unsafe fn bind_raw(
&mut self,
name: &str,
address: *mut c_void,
bytes: usize,
) -> Result<()> {
bind_raw(self.context, &self.io, name, address, bytes)
}
pub unsafe fn bind_buffer(&mut self, name: &str, buffer: &DeviceBuffer) -> Result<()> {
unsafe { self.bind_raw(name, buffer.as_ptr() as *mut c_void, buffer.size_bytes()) }
}
pub fn enqueue(&mut self, stream: &Stream) -> Result<()> {
enqueue(self.context, stream)
}
pub fn run(&mut self, stream: &Stream) -> Result<()> {
run(self.context, stream)
}
}
unsafe impl Send for OwnedContext {}
impl Drop for OwnedContext {
fn drop(&mut self) {
unsafe {
ffi::trt_context_destroy(self.context);
ffi::trt_engine_destroy(self.engine);
ffi::trt_runtime_destroy(self.runtime);
}
}
}
fn set_profile(context: *mut c_void, index: i32, stream: &Stream) -> Result<()> {
trt_check(
unsafe { ffi::trt_context_set_profile(context, index, stream.as_raw()) },
"setOptimizationProfileAsync",
)
}
fn set_shape(context: *mut c_void, name: &str, shape: &[i64]) -> Result<()> {
if shape.len() > MAX_DIMS {
return Err(Error::TooManyDimensions {
found: shape.len(),
max: MAX_DIMS,
});
}
let cname = crate::util::to_cstring(name)?;
trt_check(
unsafe {
ffi::trt_context_set_shape(context, cname.as_ptr(), shape.as_ptr(), shape.len() as i32)
},
"setInputShape",
)
}
unsafe fn bind_raw(
context: *mut c_void,
io: &[TensorInfo],
name: &str,
address: *mut c_void,
bytes: usize,
) -> Result<()> {
let info = io
.iter()
.find(|info| info.name == name)
.ok_or_else(|| Error::UnknownTensor {
name: name.to_string(),
available: io.iter().map(|info| info.name.clone()).collect(),
})?;
if address.is_null() {
return Err(Error::SizeMismatch {
tensor: name.to_string(),
expected: info.size_bytes().unwrap_or(bytes),
found: 0,
});
}
if let Some(expected) = info.size_bytes() {
if expected != bytes {
return Err(Error::SizeMismatch {
tensor: name.to_string(),
expected,
found: bytes,
});
}
}
let cname = crate::util::to_cstring(name)?;
trt_check(
unsafe { ffi::trt_context_bind(context, cname.as_ptr(), address as *const c_void) },
"setTensorAddress",
)
}
fn enqueue(context: *mut c_void, stream: &Stream) -> Result<()> {
trt_check(
unsafe { ffi::trt_context_enqueue(context, stream.as_raw()) },
"enqueueV3",
)
}
fn run(context: *mut c_void, stream: &Stream) -> Result<()> {
enqueue(context, stream)?;
stream.synchronize()
}
fn collect_io(engine: *mut c_void) -> Result<Vec<TensorInfo>> {
let count = unsafe { ffi::trt_engine_num_io(engine) };
let mut out = Vec::with_capacity(count.max(0) as usize);
let mut dims = [0i64; MAX_DIMS];
for index in 0..count {
let raw_name = unsafe { ffi::trt_engine_io_name(engine, index) };
if raw_name.is_null() {
return Err(Error::TensorRt {
call: "getIOTensorName",
message: last_message(),
});
}
let name = unsafe { CStr::from_ptr(raw_name) }
.to_string_lossy()
.into_owned();
let mode_code = unsafe { ffi::trt_engine_io_mode(engine, raw_name) };
let mode = match mode_code {
0 => IoMode::Internal,
1 => IoMode::Input,
2 => IoMode::Output,
_ => {
return Err(Error::TensorRt {
call: "getTensorIOMode",
message: format!("{name}: unknown io mode {mode_code}"),
})
}
};
let dtype_code = unsafe { ffi::trt_engine_io_dtype(engine, raw_name) };
let dtype = DataType::from_code(dtype_code).ok_or(Error::UnsupportedDataType {
tensor: name.clone(),
code: dtype_code,
})?;
let count_dims = unsafe {
ffi::trt_engine_io_dims(engine, raw_name, dims.as_mut_ptr(), MAX_DIMS as i32)
};
if count_dims < 0 {
return Err(Error::TensorRt {
call: "getTensorShape",
message: last_message(),
});
}
out.push(TensorInfo {
name,
mode,
dtype,
shape: dims[..count_dims as usize].to_vec(),
});
}
Ok(out)
}