trt-runner 0.1.0

Minimal TensorRT 10 inference runner: deserialize an engine, bind I/O tensors, enqueue V3. No bindings generator, no runtime dependencies.
Documentation
//! Runs one serialized engine end to end: load, bind, upload, infer, read back.
//!
//! ```text
//! cargo run --release --example run_engine -- model.engine [--blob input.bin] [--repeat 200]
//! ```
//!
//! `--blob` takes raw little-endian float32 bytes matching the size of the engine's input tensor,
//! which makes it easy to feed the same preprocessing result into two different runtimes and compare.

use std::time::Instant;

use trt_runner::{cuda, DataType, DeviceBuffer, Engine, IoMode, PinnedBuffer, Stream};

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut engine_path: Option<String> = None;
    let mut blob_path: Option<String> = None;
    let mut repeat: usize = 1;
    let mut args = std::env::args().skip(1);
    while let Some(arg) = args.next() {
        match arg.as_str() {
            "--blob" => blob_path = args.next(),
            "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
            other if engine_path.is_none() => engine_path = Some(other.to_string()),
            other => return Err(format!("unexpected argument {other:?}").into()),
        }
    }
    let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;

    let (major, minor, patch) = trt_runner::header_version();
    println!("compiled against TensorRT {major}.{minor}.{patch}");
    cuda::set_device(0)?;

    let engine = Engine::from_file(&engine_path)?;
    println!("{} has {} I/O tensors", engine_path, engine.io().len());
    for info in engine.io() {
        println!(
            "  {:10} {:7} {:?} {:?} {}",
            info.name,
            match info.mode {
                IoMode::Input => "input",
                IoMode::Output => "output",
                IoMode::Internal => "internal",
            },
            info.dtype,
            info.shape,
            info.size_bytes()
                .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
        );
    }

    let stream = Stream::new()?;
    let mut context = engine.context()?;

    // one device buffer per bindable tensor, sized from the engine's own metadata
    let mut buffers = Vec::new();
    for info in engine.io() {
        let bytes = info
            .size_bytes()
            .ok_or_else(|| trt_runner::Error::DynamicShape {
                tensor: info.name.clone(),
                shape: info.shape.clone(),
            })?;
        let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
        buffers.push((info.name.clone(), buffer));
    }
    for (name, buffer) in &mut buffers {
        // SAFETY: the buffers live until after the final synchronised run below.
        unsafe { context.bind_buffer(name, buffer)? };
    }

    let input = engine
        .inputs()
        .next()
        .ok_or("the engine has no input tensor to feed")?
        .clone();
    let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
    match blob_path {
        Some(path) => {
            let raw = std::fs::read(&path)?;
            if raw.len() != host.size_bytes() {
                return Err(format!(
                    "{} holds {} bytes, input wants {}",
                    path,
                    raw.len(),
                    host.size_bytes()
                )
                .into());
            }
            host.bytes_mut().copy_from_slice(&raw);
            println!("fed {} bytes from {path}", raw.len());
        }
        None => host.f32s_mut().fill(0.5),
    }
    let input_buffer = &mut buffers
        .iter_mut()
        .find(|(name, _)| *name == input.name)
        .unwrap()
        .1;
    input_buffer.copy_from_host(host.bytes(), &stream)?;

    let mut timings = Vec::with_capacity(repeat);
    for _ in 0..repeat {
        let started = Instant::now();
        context.run(&stream)?;
        timings.push(started.elapsed().as_secs_f64() * 1000.0);
    }
    timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
    let mean = timings.iter().sum::<f64>() / timings.len() as f64;
    println!(
        "{} runs: mean {mean:.2} ms  p50 {:.2}  p95 {:.2}  -> {:.1} fps",
        timings.len(),
        timings[timings.len() / 2],
        timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
        1000.0 / mean
    );

    for info in engine.outputs() {
        let buffer = &buffers
            .iter()
            .find(|(name, _)| *name == info.name)
            .unwrap()
            .1;
        let mut scratch = vec![0u8; buffer.size_bytes()];
        // copy_to_host waits for the stream, which is the point: the values below are real.
        buffer.copy_to_host(&mut scratch, &stream)?;
        let stats = describe(&scratch, info.dtype);
        println!("  {} : {}", info.name, stats);
    }
    Ok(())
}

/// Mean, min, max and the first few values, so a wrong result is visible at a glance.
fn describe(bytes: &[u8], dtype: DataType) -> String {
    if dtype != DataType::Float32 || bytes.len() % 4 != 0 {
        return format!("{} bytes of {:?}", bytes.len(), dtype);
    }
    let values: Vec<f32> = bytes
        .chunks_exact(4)
        .map(|chunk| f32::from_le_bytes(chunk.try_into().expect("four bytes")))
        .collect();
    let sum: f64 = values.iter().map(|&v| v as f64).sum();
    let head: Vec<String> = values.iter().take(6).map(|v| format!("{v:.4}")).collect();
    format!(
        "{} values, mean {:.6}, min {:.4}, max {:.4}, first [{}]",
        values.len(),
        sum / values.len() as f64,
        values.iter().cloned().fold(f32::INFINITY, f32::min),
        values.iter().cloned().fold(f32::NEG_INFINITY, f32::max),
        head.join(" ")
    )
}