Skip to main content

Engine

Struct Engine 

Source
pub struct Engine { /* private fields */ }
Expand description

A deserialized engine plus the runtime that produced it.

Engines are built for one GPU, one driver and one exact TensorRT build: an engine built elsewhere will usually fail to load here, and that failure is reported as Error::TensorRt rather than a crash.

Implementations§

Source§

impl Engine

Source

pub fn from_file(path: impl AsRef<Path>) -> Result<Self>

Reads and deserializes a serialized engine.

Examples found in repository?
examples/quickstart.rs (line 16)
10fn main() -> trt_runner::Result<()> {
11    let path = std::env::args()
12        .nth(1)
13        .unwrap_or_else(|| "model_fp16.engine".to_string());
14
15    cuda::set_device(0)?;
16    let engine = Engine::from_file(&path)?;
17    let stream = Stream::new()?;
18    let mut context = engine.context()?;
19
20    // One device buffer per I/O tensor, sized from the engine's own metadata.
21    let mut buffers = Vec::new();
22    for info in engine.io() {
23        let bytes = info.size_bytes().expect("a static shape");
24        let mut buffer = DeviceBuffer::zeroed(bytes, &stream)?;
25        if info.mode == IoMode::Input {
26            let mut host = PinnedBuffer::new(bytes)?;
27            host.f32s_mut().fill(0.5);
28            buffer.copy_from_host(host.bytes(), &stream)?;
29        }
30        // SAFETY: `buffers` is alive until after the synchronised run below.
31        unsafe { context.bind_buffer(&info.name, &buffer)? };
32        buffers.push((info.name.clone(), info.mode, buffer));
33    }
34
35    context.run(&stream)?; // enqueue plus synchronize
36
37    for (name, mode, buffer) in &buffers {
38        if *mode == IoMode::Output {
39            let mut out = vec![0u8; buffer.size_bytes()];
40            // copy_to_host waits for the stream, so these bytes are real.
41            buffer.copy_to_host(&mut out, &stream)?;
42            println!("{name}: read back {} bytes", out.len());
43        }
44    }
45    Ok(())
46}
More examples
Hide additional examples
examples/run_engine.rs (line 33)
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15    let mut engine_path: Option<String> = None;
16    let mut blob_path: Option<String> = None;
17    let mut repeat: usize = 1;
18    let mut args = std::env::args().skip(1);
19    while let Some(arg) = args.next() {
20        match arg.as_str() {
21            "--blob" => blob_path = args.next(),
22            "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23            other if engine_path.is_none() => engine_path = Some(other.to_string()),
24            other => return Err(format!("unexpected argument {other:?}").into()),
25        }
26    }
27    let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29    let (major, minor, patch) = trt_runner::header_version();
30    println!("compiled against TensorRT {major}.{minor}.{patch}");
31    cuda::set_device(0)?;
32
33    let engine = Engine::from_file(&engine_path)?;
34    println!("{} has {} I/O tensors", engine_path, engine.io().len());
35    for info in engine.io() {
36        println!(
37            "  {:10} {:7} {:?} {:?} {}",
38            info.name,
39            match info.mode {
40                IoMode::Input => "input",
41                IoMode::Output => "output",
42                IoMode::Internal => "internal",
43            },
44            info.dtype,
45            info.shape,
46            info.size_bytes()
47                .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48        );
49    }
50
51    let stream = Stream::new()?;
52    let mut context = engine.context()?;
53
54    // one device buffer per bindable tensor, sized from the engine's own metadata
55    let mut buffers = Vec::new();
56    for info in engine.io() {
57        let bytes = info
58            .size_bytes()
59            .ok_or_else(|| trt_runner::Error::DynamicShape {
60                tensor: info.name.clone(),
61                shape: info.shape.clone(),
62            })?;
63        let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64        buffers.push((info.name.clone(), buffer));
65    }
66    for (name, buffer) in &mut buffers {
67        // SAFETY: the buffers live until after the final synchronised run below.
68        unsafe { context.bind_buffer(name, buffer)? };
69    }
70
71    let input = engine
72        .inputs()
73        .next()
74        .ok_or("the engine has no input tensor to feed")?
75        .clone();
76    let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77    match blob_path {
78        Some(path) => {
79            let raw = std::fs::read(&path)?;
80            if raw.len() != host.size_bytes() {
81                return Err(format!(
82                    "{} holds {} bytes, input wants {}",
83                    path,
84                    raw.len(),
85                    host.size_bytes()
86                )
87                .into());
88            }
89            host.bytes_mut().copy_from_slice(&raw);
90            println!("fed {} bytes from {path}", raw.len());
91        }
92        None => host.f32s_mut().fill(0.5),
93    }
94    let input_buffer = &mut buffers
95        .iter_mut()
96        .find(|(name, _)| *name == input.name)
97        .unwrap()
98        .1;
99    input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101    let mut timings = Vec::with_capacity(repeat);
102    for _ in 0..repeat {
103        let started = Instant::now();
104        context.run(&stream)?;
105        timings.push(started.elapsed().as_secs_f64() * 1000.0);
106    }
107    timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108    let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109    println!(
110        "{} runs: mean {mean:.2} ms  p50 {:.2}  p95 {:.2}  -> {:.1} fps",
111        timings.len(),
112        timings[timings.len() / 2],
113        timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114        1000.0 / mean
115    );
116
117    for info in engine.outputs() {
118        let buffer = &buffers
119            .iter()
120            .find(|(name, _)| *name == info.name)
121            .unwrap()
122            .1;
123        let mut scratch = vec![0u8; buffer.size_bytes()];
124        // copy_to_host waits for the stream, which is the point: the values below are real.
125        buffer.copy_to_host(&mut scratch, &stream)?;
126        let stats = describe(&scratch, info.dtype);
127        println!("  {} : {}", info.name, stats);
128    }
129    Ok(())
130}
Source

pub fn from_bytes_with_verbosity(bytes: &[u8], verbosity: i32) -> Result<Self>

Deserializes engine bytes already in memory.

verbosity follows TensorRT’s logger levels: 2 keeps warnings and errors, 4 is very chatty.

Source

pub fn from_bytes(bytes: &[u8]) -> Result<Self>

Deserializes with warnings and errors only.

Source

pub fn io(&self) -> &[TensorInfo]

Every tensor the caller can bind, in engine order.

Examples found in repository?
examples/quickstart.rs (line 22)
10fn main() -> trt_runner::Result<()> {
11    let path = std::env::args()
12        .nth(1)
13        .unwrap_or_else(|| "model_fp16.engine".to_string());
14
15    cuda::set_device(0)?;
16    let engine = Engine::from_file(&path)?;
17    let stream = Stream::new()?;
18    let mut context = engine.context()?;
19
20    // One device buffer per I/O tensor, sized from the engine's own metadata.
21    let mut buffers = Vec::new();
22    for info in engine.io() {
23        let bytes = info.size_bytes().expect("a static shape");
24        let mut buffer = DeviceBuffer::zeroed(bytes, &stream)?;
25        if info.mode == IoMode::Input {
26            let mut host = PinnedBuffer::new(bytes)?;
27            host.f32s_mut().fill(0.5);
28            buffer.copy_from_host(host.bytes(), &stream)?;
29        }
30        // SAFETY: `buffers` is alive until after the synchronised run below.
31        unsafe { context.bind_buffer(&info.name, &buffer)? };
32        buffers.push((info.name.clone(), info.mode, buffer));
33    }
34
35    context.run(&stream)?; // enqueue plus synchronize
36
37    for (name, mode, buffer) in &buffers {
38        if *mode == IoMode::Output {
39            let mut out = vec![0u8; buffer.size_bytes()];
40            // copy_to_host waits for the stream, so these bytes are real.
41            buffer.copy_to_host(&mut out, &stream)?;
42            println!("{name}: read back {} bytes", out.len());
43        }
44    }
45    Ok(())
46}
More examples
Hide additional examples
examples/run_engine.rs (line 34)
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15    let mut engine_path: Option<String> = None;
16    let mut blob_path: Option<String> = None;
17    let mut repeat: usize = 1;
18    let mut args = std::env::args().skip(1);
19    while let Some(arg) = args.next() {
20        match arg.as_str() {
21            "--blob" => blob_path = args.next(),
22            "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23            other if engine_path.is_none() => engine_path = Some(other.to_string()),
24            other => return Err(format!("unexpected argument {other:?}").into()),
25        }
26    }
27    let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29    let (major, minor, patch) = trt_runner::header_version();
30    println!("compiled against TensorRT {major}.{minor}.{patch}");
31    cuda::set_device(0)?;
32
33    let engine = Engine::from_file(&engine_path)?;
34    println!("{} has {} I/O tensors", engine_path, engine.io().len());
35    for info in engine.io() {
36        println!(
37            "  {:10} {:7} {:?} {:?} {}",
38            info.name,
39            match info.mode {
40                IoMode::Input => "input",
41                IoMode::Output => "output",
42                IoMode::Internal => "internal",
43            },
44            info.dtype,
45            info.shape,
46            info.size_bytes()
47                .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48        );
49    }
50
51    let stream = Stream::new()?;
52    let mut context = engine.context()?;
53
54    // one device buffer per bindable tensor, sized from the engine's own metadata
55    let mut buffers = Vec::new();
56    for info in engine.io() {
57        let bytes = info
58            .size_bytes()
59            .ok_or_else(|| trt_runner::Error::DynamicShape {
60                tensor: info.name.clone(),
61                shape: info.shape.clone(),
62            })?;
63        let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64        buffers.push((info.name.clone(), buffer));
65    }
66    for (name, buffer) in &mut buffers {
67        // SAFETY: the buffers live until after the final synchronised run below.
68        unsafe { context.bind_buffer(name, buffer)? };
69    }
70
71    let input = engine
72        .inputs()
73        .next()
74        .ok_or("the engine has no input tensor to feed")?
75        .clone();
76    let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77    match blob_path {
78        Some(path) => {
79            let raw = std::fs::read(&path)?;
80            if raw.len() != host.size_bytes() {
81                return Err(format!(
82                    "{} holds {} bytes, input wants {}",
83                    path,
84                    raw.len(),
85                    host.size_bytes()
86                )
87                .into());
88            }
89            host.bytes_mut().copy_from_slice(&raw);
90            println!("fed {} bytes from {path}", raw.len());
91        }
92        None => host.f32s_mut().fill(0.5),
93    }
94    let input_buffer = &mut buffers
95        .iter_mut()
96        .find(|(name, _)| *name == input.name)
97        .unwrap()
98        .1;
99    input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101    let mut timings = Vec::with_capacity(repeat);
102    for _ in 0..repeat {
103        let started = Instant::now();
104        context.run(&stream)?;
105        timings.push(started.elapsed().as_secs_f64() * 1000.0);
106    }
107    timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108    let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109    println!(
110        "{} runs: mean {mean:.2} ms  p50 {:.2}  p95 {:.2}  -> {:.1} fps",
111        timings.len(),
112        timings[timings.len() / 2],
113        timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114        1000.0 / mean
115    );
116
117    for info in engine.outputs() {
118        let buffer = &buffers
119            .iter()
120            .find(|(name, _)| *name == info.name)
121            .unwrap()
122            .1;
123        let mut scratch = vec![0u8; buffer.size_bytes()];
124        // copy_to_host waits for the stream, which is the point: the values below are real.
125        buffer.copy_to_host(&mut scratch, &stream)?;
126        let stats = describe(&scratch, info.dtype);
127        println!("  {} : {}", info.name, stats);
128    }
129    Ok(())
130}
Source

pub fn tensor(&self, name: &str) -> Option<&TensorInfo>

Looks a tensor up by name.

Source

pub fn inputs(&self) -> impl Iterator<Item = &TensorInfo>

Tensors the caller must fill.

Examples found in repository?
examples/run_engine.rs (line 72)
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15    let mut engine_path: Option<String> = None;
16    let mut blob_path: Option<String> = None;
17    let mut repeat: usize = 1;
18    let mut args = std::env::args().skip(1);
19    while let Some(arg) = args.next() {
20        match arg.as_str() {
21            "--blob" => blob_path = args.next(),
22            "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23            other if engine_path.is_none() => engine_path = Some(other.to_string()),
24            other => return Err(format!("unexpected argument {other:?}").into()),
25        }
26    }
27    let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29    let (major, minor, patch) = trt_runner::header_version();
30    println!("compiled against TensorRT {major}.{minor}.{patch}");
31    cuda::set_device(0)?;
32
33    let engine = Engine::from_file(&engine_path)?;
34    println!("{} has {} I/O tensors", engine_path, engine.io().len());
35    for info in engine.io() {
36        println!(
37            "  {:10} {:7} {:?} {:?} {}",
38            info.name,
39            match info.mode {
40                IoMode::Input => "input",
41                IoMode::Output => "output",
42                IoMode::Internal => "internal",
43            },
44            info.dtype,
45            info.shape,
46            info.size_bytes()
47                .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48        );
49    }
50
51    let stream = Stream::new()?;
52    let mut context = engine.context()?;
53
54    // one device buffer per bindable tensor, sized from the engine's own metadata
55    let mut buffers = Vec::new();
56    for info in engine.io() {
57        let bytes = info
58            .size_bytes()
59            .ok_or_else(|| trt_runner::Error::DynamicShape {
60                tensor: info.name.clone(),
61                shape: info.shape.clone(),
62            })?;
63        let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64        buffers.push((info.name.clone(), buffer));
65    }
66    for (name, buffer) in &mut buffers {
67        // SAFETY: the buffers live until after the final synchronised run below.
68        unsafe { context.bind_buffer(name, buffer)? };
69    }
70
71    let input = engine
72        .inputs()
73        .next()
74        .ok_or("the engine has no input tensor to feed")?
75        .clone();
76    let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77    match blob_path {
78        Some(path) => {
79            let raw = std::fs::read(&path)?;
80            if raw.len() != host.size_bytes() {
81                return Err(format!(
82                    "{} holds {} bytes, input wants {}",
83                    path,
84                    raw.len(),
85                    host.size_bytes()
86                )
87                .into());
88            }
89            host.bytes_mut().copy_from_slice(&raw);
90            println!("fed {} bytes from {path}", raw.len());
91        }
92        None => host.f32s_mut().fill(0.5),
93    }
94    let input_buffer = &mut buffers
95        .iter_mut()
96        .find(|(name, _)| *name == input.name)
97        .unwrap()
98        .1;
99    input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101    let mut timings = Vec::with_capacity(repeat);
102    for _ in 0..repeat {
103        let started = Instant::now();
104        context.run(&stream)?;
105        timings.push(started.elapsed().as_secs_f64() * 1000.0);
106    }
107    timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108    let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109    println!(
110        "{} runs: mean {mean:.2} ms  p50 {:.2}  p95 {:.2}  -> {:.1} fps",
111        timings.len(),
112        timings[timings.len() / 2],
113        timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114        1000.0 / mean
115    );
116
117    for info in engine.outputs() {
118        let buffer = &buffers
119            .iter()
120            .find(|(name, _)| *name == info.name)
121            .unwrap()
122            .1;
123        let mut scratch = vec![0u8; buffer.size_bytes()];
124        // copy_to_host waits for the stream, which is the point: the values below are real.
125        buffer.copy_to_host(&mut scratch, &stream)?;
126        let stats = describe(&scratch, info.dtype);
127        println!("  {} : {}", info.name, stats);
128    }
129    Ok(())
130}
Source

pub fn outputs(&self) -> impl Iterator<Item = &TensorInfo>

Tensors the caller reads back.

Examples found in repository?
examples/run_engine.rs (line 117)
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15    let mut engine_path: Option<String> = None;
16    let mut blob_path: Option<String> = None;
17    let mut repeat: usize = 1;
18    let mut args = std::env::args().skip(1);
19    while let Some(arg) = args.next() {
20        match arg.as_str() {
21            "--blob" => blob_path = args.next(),
22            "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23            other if engine_path.is_none() => engine_path = Some(other.to_string()),
24            other => return Err(format!("unexpected argument {other:?}").into()),
25        }
26    }
27    let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29    let (major, minor, patch) = trt_runner::header_version();
30    println!("compiled against TensorRT {major}.{minor}.{patch}");
31    cuda::set_device(0)?;
32
33    let engine = Engine::from_file(&engine_path)?;
34    println!("{} has {} I/O tensors", engine_path, engine.io().len());
35    for info in engine.io() {
36        println!(
37            "  {:10} {:7} {:?} {:?} {}",
38            info.name,
39            match info.mode {
40                IoMode::Input => "input",
41                IoMode::Output => "output",
42                IoMode::Internal => "internal",
43            },
44            info.dtype,
45            info.shape,
46            info.size_bytes()
47                .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48        );
49    }
50
51    let stream = Stream::new()?;
52    let mut context = engine.context()?;
53
54    // one device buffer per bindable tensor, sized from the engine's own metadata
55    let mut buffers = Vec::new();
56    for info in engine.io() {
57        let bytes = info
58            .size_bytes()
59            .ok_or_else(|| trt_runner::Error::DynamicShape {
60                tensor: info.name.clone(),
61                shape: info.shape.clone(),
62            })?;
63        let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64        buffers.push((info.name.clone(), buffer));
65    }
66    for (name, buffer) in &mut buffers {
67        // SAFETY: the buffers live until after the final synchronised run below.
68        unsafe { context.bind_buffer(name, buffer)? };
69    }
70
71    let input = engine
72        .inputs()
73        .next()
74        .ok_or("the engine has no input tensor to feed")?
75        .clone();
76    let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77    match blob_path {
78        Some(path) => {
79            let raw = std::fs::read(&path)?;
80            if raw.len() != host.size_bytes() {
81                return Err(format!(
82                    "{} holds {} bytes, input wants {}",
83                    path,
84                    raw.len(),
85                    host.size_bytes()
86                )
87                .into());
88            }
89            host.bytes_mut().copy_from_slice(&raw);
90            println!("fed {} bytes from {path}", raw.len());
91        }
92        None => host.f32s_mut().fill(0.5),
93    }
94    let input_buffer = &mut buffers
95        .iter_mut()
96        .find(|(name, _)| *name == input.name)
97        .unwrap()
98        .1;
99    input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101    let mut timings = Vec::with_capacity(repeat);
102    for _ in 0..repeat {
103        let started = Instant::now();
104        context.run(&stream)?;
105        timings.push(started.elapsed().as_secs_f64() * 1000.0);
106    }
107    timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108    let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109    println!(
110        "{} runs: mean {mean:.2} ms  p50 {:.2}  p95 {:.2}  -> {:.1} fps",
111        timings.len(),
112        timings[timings.len() / 2],
113        timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114        1000.0 / mean
115    );
116
117    for info in engine.outputs() {
118        let buffer = &buffers
119            .iter()
120            .find(|(name, _)| *name == info.name)
121            .unwrap()
122            .1;
123        let mut scratch = vec![0u8; buffer.size_bytes()];
124        // copy_to_host waits for the stream, which is the point: the values below are real.
125        buffer.copy_to_host(&mut scratch, &stream)?;
126        let stats = describe(&scratch, info.dtype);
127        println!("  {} : {}", info.name, stats);
128    }
129    Ok(())
130}
Source

pub fn tensor_names(&self) -> Vec<String>

Names of all I/O tensors, for error messages.

Source

pub fn context(&self) -> Result<Context<'_>>

Creates an execution context. One context handles one shape combination at a time and is not Sync; create one per thread instead.

Examples found in repository?
examples/quickstart.rs (line 18)
10fn main() -> trt_runner::Result<()> {
11    let path = std::env::args()
12        .nth(1)
13        .unwrap_or_else(|| "model_fp16.engine".to_string());
14
15    cuda::set_device(0)?;
16    let engine = Engine::from_file(&path)?;
17    let stream = Stream::new()?;
18    let mut context = engine.context()?;
19
20    // One device buffer per I/O tensor, sized from the engine's own metadata.
21    let mut buffers = Vec::new();
22    for info in engine.io() {
23        let bytes = info.size_bytes().expect("a static shape");
24        let mut buffer = DeviceBuffer::zeroed(bytes, &stream)?;
25        if info.mode == IoMode::Input {
26            let mut host = PinnedBuffer::new(bytes)?;
27            host.f32s_mut().fill(0.5);
28            buffer.copy_from_host(host.bytes(), &stream)?;
29        }
30        // SAFETY: `buffers` is alive until after the synchronised run below.
31        unsafe { context.bind_buffer(&info.name, &buffer)? };
32        buffers.push((info.name.clone(), info.mode, buffer));
33    }
34
35    context.run(&stream)?; // enqueue plus synchronize
36
37    for (name, mode, buffer) in &buffers {
38        if *mode == IoMode::Output {
39            let mut out = vec![0u8; buffer.size_bytes()];
40            // copy_to_host waits for the stream, so these bytes are real.
41            buffer.copy_to_host(&mut out, &stream)?;
42            println!("{name}: read back {} bytes", out.len());
43        }
44    }
45    Ok(())
46}
More examples
Hide additional examples
examples/run_engine.rs (line 52)
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15    let mut engine_path: Option<String> = None;
16    let mut blob_path: Option<String> = None;
17    let mut repeat: usize = 1;
18    let mut args = std::env::args().skip(1);
19    while let Some(arg) = args.next() {
20        match arg.as_str() {
21            "--blob" => blob_path = args.next(),
22            "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23            other if engine_path.is_none() => engine_path = Some(other.to_string()),
24            other => return Err(format!("unexpected argument {other:?}").into()),
25        }
26    }
27    let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29    let (major, minor, patch) = trt_runner::header_version();
30    println!("compiled against TensorRT {major}.{minor}.{patch}");
31    cuda::set_device(0)?;
32
33    let engine = Engine::from_file(&engine_path)?;
34    println!("{} has {} I/O tensors", engine_path, engine.io().len());
35    for info in engine.io() {
36        println!(
37            "  {:10} {:7} {:?} {:?} {}",
38            info.name,
39            match info.mode {
40                IoMode::Input => "input",
41                IoMode::Output => "output",
42                IoMode::Internal => "internal",
43            },
44            info.dtype,
45            info.shape,
46            info.size_bytes()
47                .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48        );
49    }
50
51    let stream = Stream::new()?;
52    let mut context = engine.context()?;
53
54    // one device buffer per bindable tensor, sized from the engine's own metadata
55    let mut buffers = Vec::new();
56    for info in engine.io() {
57        let bytes = info
58            .size_bytes()
59            .ok_or_else(|| trt_runner::Error::DynamicShape {
60                tensor: info.name.clone(),
61                shape: info.shape.clone(),
62            })?;
63        let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64        buffers.push((info.name.clone(), buffer));
65    }
66    for (name, buffer) in &mut buffers {
67        // SAFETY: the buffers live until after the final synchronised run below.
68        unsafe { context.bind_buffer(name, buffer)? };
69    }
70
71    let input = engine
72        .inputs()
73        .next()
74        .ok_or("the engine has no input tensor to feed")?
75        .clone();
76    let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77    match blob_path {
78        Some(path) => {
79            let raw = std::fs::read(&path)?;
80            if raw.len() != host.size_bytes() {
81                return Err(format!(
82                    "{} holds {} bytes, input wants {}",
83                    path,
84                    raw.len(),
85                    host.size_bytes()
86                )
87                .into());
88            }
89            host.bytes_mut().copy_from_slice(&raw);
90            println!("fed {} bytes from {path}", raw.len());
91        }
92        None => host.f32s_mut().fill(0.5),
93    }
94    let input_buffer = &mut buffers
95        .iter_mut()
96        .find(|(name, _)| *name == input.name)
97        .unwrap()
98        .1;
99    input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101    let mut timings = Vec::with_capacity(repeat);
102    for _ in 0..repeat {
103        let started = Instant::now();
104        context.run(&stream)?;
105        timings.push(started.elapsed().as_secs_f64() * 1000.0);
106    }
107    timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108    let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109    println!(
110        "{} runs: mean {mean:.2} ms  p50 {:.2}  p95 {:.2}  -> {:.1} fps",
111        timings.len(),
112        timings[timings.len() / 2],
113        timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114        1000.0 / mean
115    );
116
117    for info in engine.outputs() {
118        let buffer = &buffers
119            .iter()
120            .find(|(name, _)| *name == info.name)
121            .unwrap()
122            .1;
123        let mut scratch = vec![0u8; buffer.size_bytes()];
124        // copy_to_host waits for the stream, which is the point: the values below are real.
125        buffer.copy_to_host(&mut scratch, &stream)?;
126        let stats = describe(&scratch, info.dtype);
127        println!("  {} : {}", info.name, stats);
128    }
129    Ok(())
130}
Source

pub fn into_context(self) -> Result<OwnedContext>

Consumes the engine into an OwnedContext, which keeps the engine and the runtime alive beside the execution context. This is the shape a long lived runner usually wants: one value to store in a struct, without a lifetime parameter chasing it.

Trait Implementations§

Source§

impl Debug for Engine

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more
Source§

impl Drop for Engine

Source§

fn drop(&mut self)

Executes the destructor for this type. Read more
Source§

fn pin_drop(self: Pin<&mut Self>)

🔬This is a nightly-only experimental API. (pin_ergonomics)
Execute the destructor for this type, but different to Drop::drop, it requires self to be pinned. Read more

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.