use std::time::Instant;
use trt_runner::{cuda, DataType, DeviceBuffer, Engine, IoMode, PinnedBuffer, Stream};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut engine_path: Option<String> = None;
let mut blob_path: Option<String> = None;
let mut repeat: usize = 1;
let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
match arg.as_str() {
"--blob" => blob_path = args.next(),
"--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
other if engine_path.is_none() => engine_path = Some(other.to_string()),
other => return Err(format!("unexpected argument {other:?}").into()),
}
}
let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
let (major, minor, patch) = trt_runner::header_version();
println!("compiled against TensorRT {major}.{minor}.{patch}");
cuda::set_device(0)?;
let engine = Engine::from_file(&engine_path)?;
println!("{} has {} I/O tensors", engine_path, engine.io().len());
for info in engine.io() {
println!(
" {:10} {:7} {:?} {:?} {}",
info.name,
match info.mode {
IoMode::Input => "input",
IoMode::Output => "output",
IoMode::Internal => "internal",
},
info.dtype,
info.shape,
info.size_bytes()
.map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
);
}
let stream = Stream::new()?;
let mut context = engine.context()?;
let mut buffers = Vec::new();
for info in engine.io() {
let bytes = info
.size_bytes()
.ok_or_else(|| trt_runner::Error::DynamicShape {
tensor: info.name.clone(),
shape: info.shape.clone(),
})?;
let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
buffers.push((info.name.clone(), buffer));
}
for (name, buffer) in &mut buffers {
unsafe { context.bind_buffer(name, buffer)? };
}
let input = engine
.inputs()
.next()
.ok_or("the engine has no input tensor to feed")?
.clone();
let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
match blob_path {
Some(path) => {
let raw = std::fs::read(&path)?;
if raw.len() != host.size_bytes() {
return Err(format!(
"{} holds {} bytes, input wants {}",
path,
raw.len(),
host.size_bytes()
)
.into());
}
host.bytes_mut().copy_from_slice(&raw);
println!("fed {} bytes from {path}", raw.len());
}
None => host.f32s_mut().fill(0.5),
}
let input_buffer = &mut buffers
.iter_mut()
.find(|(name, _)| *name == input.name)
.unwrap()
.1;
input_buffer.copy_from_host(host.bytes(), &stream)?;
let mut timings = Vec::with_capacity(repeat);
for _ in 0..repeat {
let started = Instant::now();
context.run(&stream)?;
timings.push(started.elapsed().as_secs_f64() * 1000.0);
}
timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
let mean = timings.iter().sum::<f64>() / timings.len() as f64;
println!(
"{} runs: mean {mean:.2} ms p50 {:.2} p95 {:.2} -> {:.1} fps",
timings.len(),
timings[timings.len() / 2],
timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
1000.0 / mean
);
for info in engine.outputs() {
let buffer = &buffers
.iter()
.find(|(name, _)| *name == info.name)
.unwrap()
.1;
let mut scratch = vec![0u8; buffer.size_bytes()];
buffer.copy_to_host(&mut scratch, &stream)?;
let stats = describe(&scratch, info.dtype);
println!(" {} : {}", info.name, stats);
}
Ok(())
}
fn describe(bytes: &[u8], dtype: DataType) -> String {
if dtype != DataType::Float32 || bytes.len() % 4 != 0 {
return format!("{} bytes of {:?}", bytes.len(), dtype);
}
let values: Vec<f32> = bytes
.chunks_exact(4)
.map(|chunk| f32::from_le_bytes(chunk.try_into().expect("four bytes")))
.collect();
let sum: f64 = values.iter().map(|&v| v as f64).sum();
let head: Vec<String> = values.iter().take(6).map(|v| format!("{v:.4}")).collect();
format!(
"{} values, mean {:.6}, min {:.4}, max {:.4}, first [{}]",
values.len(),
sum / values.len() as f64,
values.iter().cloned().fold(f32::INFINITY, f32::min),
values.iter().cloned().fold(f32::NEG_INFINITY, f32::max),
head.join(" ")
)
}