pub struct Engine { /* private fields */ }Expand description
A deserialized engine plus the runtime that produced it.
Engines are built for one GPU, one driver and one exact TensorRT build: an engine built elsewhere
will usually fail to load here, and that failure is reported as
Error::TensorRt rather than a crash.
Implementations§
Source§impl Engine
impl Engine
Sourcepub fn from_file(path: impl AsRef<Path>) -> Result<Self>
pub fn from_file(path: impl AsRef<Path>) -> Result<Self>
Reads and deserializes a serialized engine.
Examples found in repository?
10fn main() -> trt_runner::Result<()> {
11 let path = std::env::args()
12 .nth(1)
13 .unwrap_or_else(|| "model_fp16.engine".to_string());
14
15 cuda::set_device(0)?;
16 let engine = Engine::from_file(&path)?;
17 let stream = Stream::new()?;
18 let mut context = engine.context()?;
19
20 // One device buffer per I/O tensor, sized from the engine's own metadata.
21 let mut buffers = Vec::new();
22 for info in engine.io() {
23 let bytes = info.size_bytes().expect("a static shape");
24 let mut buffer = DeviceBuffer::zeroed(bytes, &stream)?;
25 if info.mode == IoMode::Input {
26 let mut host = PinnedBuffer::new(bytes)?;
27 host.f32s_mut().fill(0.5);
28 buffer.copy_from_host(host.bytes(), &stream)?;
29 }
30 // SAFETY: `buffers` is alive until after the synchronised run below.
31 unsafe { context.bind_buffer(&info.name, &buffer)? };
32 buffers.push((info.name.clone(), info.mode, buffer));
33 }
34
35 context.run(&stream)?; // enqueue plus synchronize
36
37 for (name, mode, buffer) in &buffers {
38 if *mode == IoMode::Output {
39 let mut out = vec![0u8; buffer.size_bytes()];
40 // copy_to_host waits for the stream, so these bytes are real.
41 buffer.copy_to_host(&mut out, &stream)?;
42 println!("{name}: read back {} bytes", out.len());
43 }
44 }
45 Ok(())
46}More examples
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15 let mut engine_path: Option<String> = None;
16 let mut blob_path: Option<String> = None;
17 let mut repeat: usize = 1;
18 let mut args = std::env::args().skip(1);
19 while let Some(arg) = args.next() {
20 match arg.as_str() {
21 "--blob" => blob_path = args.next(),
22 "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23 other if engine_path.is_none() => engine_path = Some(other.to_string()),
24 other => return Err(format!("unexpected argument {other:?}").into()),
25 }
26 }
27 let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29 let (major, minor, patch) = trt_runner::header_version();
30 println!("compiled against TensorRT {major}.{minor}.{patch}");
31 cuda::set_device(0)?;
32
33 let engine = Engine::from_file(&engine_path)?;
34 println!("{} has {} I/O tensors", engine_path, engine.io().len());
35 for info in engine.io() {
36 println!(
37 " {:10} {:7} {:?} {:?} {}",
38 info.name,
39 match info.mode {
40 IoMode::Input => "input",
41 IoMode::Output => "output",
42 IoMode::Internal => "internal",
43 },
44 info.dtype,
45 info.shape,
46 info.size_bytes()
47 .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48 );
49 }
50
51 let stream = Stream::new()?;
52 let mut context = engine.context()?;
53
54 // one device buffer per bindable tensor, sized from the engine's own metadata
55 let mut buffers = Vec::new();
56 for info in engine.io() {
57 let bytes = info
58 .size_bytes()
59 .ok_or_else(|| trt_runner::Error::DynamicShape {
60 tensor: info.name.clone(),
61 shape: info.shape.clone(),
62 })?;
63 let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64 buffers.push((info.name.clone(), buffer));
65 }
66 for (name, buffer) in &mut buffers {
67 // SAFETY: the buffers live until after the final synchronised run below.
68 unsafe { context.bind_buffer(name, buffer)? };
69 }
70
71 let input = engine
72 .inputs()
73 .next()
74 .ok_or("the engine has no input tensor to feed")?
75 .clone();
76 let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77 match blob_path {
78 Some(path) => {
79 let raw = std::fs::read(&path)?;
80 if raw.len() != host.size_bytes() {
81 return Err(format!(
82 "{} holds {} bytes, input wants {}",
83 path,
84 raw.len(),
85 host.size_bytes()
86 )
87 .into());
88 }
89 host.bytes_mut().copy_from_slice(&raw);
90 println!("fed {} bytes from {path}", raw.len());
91 }
92 None => host.f32s_mut().fill(0.5),
93 }
94 let input_buffer = &mut buffers
95 .iter_mut()
96 .find(|(name, _)| *name == input.name)
97 .unwrap()
98 .1;
99 input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101 let mut timings = Vec::with_capacity(repeat);
102 for _ in 0..repeat {
103 let started = Instant::now();
104 context.run(&stream)?;
105 timings.push(started.elapsed().as_secs_f64() * 1000.0);
106 }
107 timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108 let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109 println!(
110 "{} runs: mean {mean:.2} ms p50 {:.2} p95 {:.2} -> {:.1} fps",
111 timings.len(),
112 timings[timings.len() / 2],
113 timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114 1000.0 / mean
115 );
116
117 for info in engine.outputs() {
118 let buffer = &buffers
119 .iter()
120 .find(|(name, _)| *name == info.name)
121 .unwrap()
122 .1;
123 let mut scratch = vec![0u8; buffer.size_bytes()];
124 // copy_to_host waits for the stream, which is the point: the values below are real.
125 buffer.copy_to_host(&mut scratch, &stream)?;
126 let stats = describe(&scratch, info.dtype);
127 println!(" {} : {}", info.name, stats);
128 }
129 Ok(())
130}Sourcepub fn from_bytes_with_verbosity(bytes: &[u8], verbosity: i32) -> Result<Self>
pub fn from_bytes_with_verbosity(bytes: &[u8], verbosity: i32) -> Result<Self>
Deserializes engine bytes already in memory.
verbosity follows TensorRT’s logger levels: 2 keeps warnings and errors, 4 is very chatty.
Sourcepub fn from_bytes(bytes: &[u8]) -> Result<Self>
pub fn from_bytes(bytes: &[u8]) -> Result<Self>
Deserializes with warnings and errors only.
Sourcepub fn io(&self) -> &[TensorInfo]
pub fn io(&self) -> &[TensorInfo]
Every tensor the caller can bind, in engine order.
Examples found in repository?
10fn main() -> trt_runner::Result<()> {
11 let path = std::env::args()
12 .nth(1)
13 .unwrap_or_else(|| "model_fp16.engine".to_string());
14
15 cuda::set_device(0)?;
16 let engine = Engine::from_file(&path)?;
17 let stream = Stream::new()?;
18 let mut context = engine.context()?;
19
20 // One device buffer per I/O tensor, sized from the engine's own metadata.
21 let mut buffers = Vec::new();
22 for info in engine.io() {
23 let bytes = info.size_bytes().expect("a static shape");
24 let mut buffer = DeviceBuffer::zeroed(bytes, &stream)?;
25 if info.mode == IoMode::Input {
26 let mut host = PinnedBuffer::new(bytes)?;
27 host.f32s_mut().fill(0.5);
28 buffer.copy_from_host(host.bytes(), &stream)?;
29 }
30 // SAFETY: `buffers` is alive until after the synchronised run below.
31 unsafe { context.bind_buffer(&info.name, &buffer)? };
32 buffers.push((info.name.clone(), info.mode, buffer));
33 }
34
35 context.run(&stream)?; // enqueue plus synchronize
36
37 for (name, mode, buffer) in &buffers {
38 if *mode == IoMode::Output {
39 let mut out = vec![0u8; buffer.size_bytes()];
40 // copy_to_host waits for the stream, so these bytes are real.
41 buffer.copy_to_host(&mut out, &stream)?;
42 println!("{name}: read back {} bytes", out.len());
43 }
44 }
45 Ok(())
46}More examples
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15 let mut engine_path: Option<String> = None;
16 let mut blob_path: Option<String> = None;
17 let mut repeat: usize = 1;
18 let mut args = std::env::args().skip(1);
19 while let Some(arg) = args.next() {
20 match arg.as_str() {
21 "--blob" => blob_path = args.next(),
22 "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23 other if engine_path.is_none() => engine_path = Some(other.to_string()),
24 other => return Err(format!("unexpected argument {other:?}").into()),
25 }
26 }
27 let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29 let (major, minor, patch) = trt_runner::header_version();
30 println!("compiled against TensorRT {major}.{minor}.{patch}");
31 cuda::set_device(0)?;
32
33 let engine = Engine::from_file(&engine_path)?;
34 println!("{} has {} I/O tensors", engine_path, engine.io().len());
35 for info in engine.io() {
36 println!(
37 " {:10} {:7} {:?} {:?} {}",
38 info.name,
39 match info.mode {
40 IoMode::Input => "input",
41 IoMode::Output => "output",
42 IoMode::Internal => "internal",
43 },
44 info.dtype,
45 info.shape,
46 info.size_bytes()
47 .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48 );
49 }
50
51 let stream = Stream::new()?;
52 let mut context = engine.context()?;
53
54 // one device buffer per bindable tensor, sized from the engine's own metadata
55 let mut buffers = Vec::new();
56 for info in engine.io() {
57 let bytes = info
58 .size_bytes()
59 .ok_or_else(|| trt_runner::Error::DynamicShape {
60 tensor: info.name.clone(),
61 shape: info.shape.clone(),
62 })?;
63 let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64 buffers.push((info.name.clone(), buffer));
65 }
66 for (name, buffer) in &mut buffers {
67 // SAFETY: the buffers live until after the final synchronised run below.
68 unsafe { context.bind_buffer(name, buffer)? };
69 }
70
71 let input = engine
72 .inputs()
73 .next()
74 .ok_or("the engine has no input tensor to feed")?
75 .clone();
76 let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77 match blob_path {
78 Some(path) => {
79 let raw = std::fs::read(&path)?;
80 if raw.len() != host.size_bytes() {
81 return Err(format!(
82 "{} holds {} bytes, input wants {}",
83 path,
84 raw.len(),
85 host.size_bytes()
86 )
87 .into());
88 }
89 host.bytes_mut().copy_from_slice(&raw);
90 println!("fed {} bytes from {path}", raw.len());
91 }
92 None => host.f32s_mut().fill(0.5),
93 }
94 let input_buffer = &mut buffers
95 .iter_mut()
96 .find(|(name, _)| *name == input.name)
97 .unwrap()
98 .1;
99 input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101 let mut timings = Vec::with_capacity(repeat);
102 for _ in 0..repeat {
103 let started = Instant::now();
104 context.run(&stream)?;
105 timings.push(started.elapsed().as_secs_f64() * 1000.0);
106 }
107 timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108 let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109 println!(
110 "{} runs: mean {mean:.2} ms p50 {:.2} p95 {:.2} -> {:.1} fps",
111 timings.len(),
112 timings[timings.len() / 2],
113 timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114 1000.0 / mean
115 );
116
117 for info in engine.outputs() {
118 let buffer = &buffers
119 .iter()
120 .find(|(name, _)| *name == info.name)
121 .unwrap()
122 .1;
123 let mut scratch = vec![0u8; buffer.size_bytes()];
124 // copy_to_host waits for the stream, which is the point: the values below are real.
125 buffer.copy_to_host(&mut scratch, &stream)?;
126 let stats = describe(&scratch, info.dtype);
127 println!(" {} : {}", info.name, stats);
128 }
129 Ok(())
130}Sourcepub fn tensor(&self, name: &str) -> Option<&TensorInfo>
pub fn tensor(&self, name: &str) -> Option<&TensorInfo>
Looks a tensor up by name.
Sourcepub fn inputs(&self) -> impl Iterator<Item = &TensorInfo>
pub fn inputs(&self) -> impl Iterator<Item = &TensorInfo>
Tensors the caller must fill.
Examples found in repository?
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15 let mut engine_path: Option<String> = None;
16 let mut blob_path: Option<String> = None;
17 let mut repeat: usize = 1;
18 let mut args = std::env::args().skip(1);
19 while let Some(arg) = args.next() {
20 match arg.as_str() {
21 "--blob" => blob_path = args.next(),
22 "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23 other if engine_path.is_none() => engine_path = Some(other.to_string()),
24 other => return Err(format!("unexpected argument {other:?}").into()),
25 }
26 }
27 let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29 let (major, minor, patch) = trt_runner::header_version();
30 println!("compiled against TensorRT {major}.{minor}.{patch}");
31 cuda::set_device(0)?;
32
33 let engine = Engine::from_file(&engine_path)?;
34 println!("{} has {} I/O tensors", engine_path, engine.io().len());
35 for info in engine.io() {
36 println!(
37 " {:10} {:7} {:?} {:?} {}",
38 info.name,
39 match info.mode {
40 IoMode::Input => "input",
41 IoMode::Output => "output",
42 IoMode::Internal => "internal",
43 },
44 info.dtype,
45 info.shape,
46 info.size_bytes()
47 .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48 );
49 }
50
51 let stream = Stream::new()?;
52 let mut context = engine.context()?;
53
54 // one device buffer per bindable tensor, sized from the engine's own metadata
55 let mut buffers = Vec::new();
56 for info in engine.io() {
57 let bytes = info
58 .size_bytes()
59 .ok_or_else(|| trt_runner::Error::DynamicShape {
60 tensor: info.name.clone(),
61 shape: info.shape.clone(),
62 })?;
63 let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64 buffers.push((info.name.clone(), buffer));
65 }
66 for (name, buffer) in &mut buffers {
67 // SAFETY: the buffers live until after the final synchronised run below.
68 unsafe { context.bind_buffer(name, buffer)? };
69 }
70
71 let input = engine
72 .inputs()
73 .next()
74 .ok_or("the engine has no input tensor to feed")?
75 .clone();
76 let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77 match blob_path {
78 Some(path) => {
79 let raw = std::fs::read(&path)?;
80 if raw.len() != host.size_bytes() {
81 return Err(format!(
82 "{} holds {} bytes, input wants {}",
83 path,
84 raw.len(),
85 host.size_bytes()
86 )
87 .into());
88 }
89 host.bytes_mut().copy_from_slice(&raw);
90 println!("fed {} bytes from {path}", raw.len());
91 }
92 None => host.f32s_mut().fill(0.5),
93 }
94 let input_buffer = &mut buffers
95 .iter_mut()
96 .find(|(name, _)| *name == input.name)
97 .unwrap()
98 .1;
99 input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101 let mut timings = Vec::with_capacity(repeat);
102 for _ in 0..repeat {
103 let started = Instant::now();
104 context.run(&stream)?;
105 timings.push(started.elapsed().as_secs_f64() * 1000.0);
106 }
107 timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108 let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109 println!(
110 "{} runs: mean {mean:.2} ms p50 {:.2} p95 {:.2} -> {:.1} fps",
111 timings.len(),
112 timings[timings.len() / 2],
113 timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114 1000.0 / mean
115 );
116
117 for info in engine.outputs() {
118 let buffer = &buffers
119 .iter()
120 .find(|(name, _)| *name == info.name)
121 .unwrap()
122 .1;
123 let mut scratch = vec![0u8; buffer.size_bytes()];
124 // copy_to_host waits for the stream, which is the point: the values below are real.
125 buffer.copy_to_host(&mut scratch, &stream)?;
126 let stats = describe(&scratch, info.dtype);
127 println!(" {} : {}", info.name, stats);
128 }
129 Ok(())
130}Sourcepub fn outputs(&self) -> impl Iterator<Item = &TensorInfo>
pub fn outputs(&self) -> impl Iterator<Item = &TensorInfo>
Tensors the caller reads back.
Examples found in repository?
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15 let mut engine_path: Option<String> = None;
16 let mut blob_path: Option<String> = None;
17 let mut repeat: usize = 1;
18 let mut args = std::env::args().skip(1);
19 while let Some(arg) = args.next() {
20 match arg.as_str() {
21 "--blob" => blob_path = args.next(),
22 "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23 other if engine_path.is_none() => engine_path = Some(other.to_string()),
24 other => return Err(format!("unexpected argument {other:?}").into()),
25 }
26 }
27 let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29 let (major, minor, patch) = trt_runner::header_version();
30 println!("compiled against TensorRT {major}.{minor}.{patch}");
31 cuda::set_device(0)?;
32
33 let engine = Engine::from_file(&engine_path)?;
34 println!("{} has {} I/O tensors", engine_path, engine.io().len());
35 for info in engine.io() {
36 println!(
37 " {:10} {:7} {:?} {:?} {}",
38 info.name,
39 match info.mode {
40 IoMode::Input => "input",
41 IoMode::Output => "output",
42 IoMode::Internal => "internal",
43 },
44 info.dtype,
45 info.shape,
46 info.size_bytes()
47 .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48 );
49 }
50
51 let stream = Stream::new()?;
52 let mut context = engine.context()?;
53
54 // one device buffer per bindable tensor, sized from the engine's own metadata
55 let mut buffers = Vec::new();
56 for info in engine.io() {
57 let bytes = info
58 .size_bytes()
59 .ok_or_else(|| trt_runner::Error::DynamicShape {
60 tensor: info.name.clone(),
61 shape: info.shape.clone(),
62 })?;
63 let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64 buffers.push((info.name.clone(), buffer));
65 }
66 for (name, buffer) in &mut buffers {
67 // SAFETY: the buffers live until after the final synchronised run below.
68 unsafe { context.bind_buffer(name, buffer)? };
69 }
70
71 let input = engine
72 .inputs()
73 .next()
74 .ok_or("the engine has no input tensor to feed")?
75 .clone();
76 let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77 match blob_path {
78 Some(path) => {
79 let raw = std::fs::read(&path)?;
80 if raw.len() != host.size_bytes() {
81 return Err(format!(
82 "{} holds {} bytes, input wants {}",
83 path,
84 raw.len(),
85 host.size_bytes()
86 )
87 .into());
88 }
89 host.bytes_mut().copy_from_slice(&raw);
90 println!("fed {} bytes from {path}", raw.len());
91 }
92 None => host.f32s_mut().fill(0.5),
93 }
94 let input_buffer = &mut buffers
95 .iter_mut()
96 .find(|(name, _)| *name == input.name)
97 .unwrap()
98 .1;
99 input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101 let mut timings = Vec::with_capacity(repeat);
102 for _ in 0..repeat {
103 let started = Instant::now();
104 context.run(&stream)?;
105 timings.push(started.elapsed().as_secs_f64() * 1000.0);
106 }
107 timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108 let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109 println!(
110 "{} runs: mean {mean:.2} ms p50 {:.2} p95 {:.2} -> {:.1} fps",
111 timings.len(),
112 timings[timings.len() / 2],
113 timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114 1000.0 / mean
115 );
116
117 for info in engine.outputs() {
118 let buffer = &buffers
119 .iter()
120 .find(|(name, _)| *name == info.name)
121 .unwrap()
122 .1;
123 let mut scratch = vec![0u8; buffer.size_bytes()];
124 // copy_to_host waits for the stream, which is the point: the values below are real.
125 buffer.copy_to_host(&mut scratch, &stream)?;
126 let stats = describe(&scratch, info.dtype);
127 println!(" {} : {}", info.name, stats);
128 }
129 Ok(())
130}Sourcepub fn tensor_names(&self) -> Vec<String>
pub fn tensor_names(&self) -> Vec<String>
Names of all I/O tensors, for error messages.
Sourcepub fn context(&self) -> Result<Context<'_>>
pub fn context(&self) -> Result<Context<'_>>
Creates an execution context. One context handles one shape combination at a time and is not
Sync; create one per thread instead.
Examples found in repository?
10fn main() -> trt_runner::Result<()> {
11 let path = std::env::args()
12 .nth(1)
13 .unwrap_or_else(|| "model_fp16.engine".to_string());
14
15 cuda::set_device(0)?;
16 let engine = Engine::from_file(&path)?;
17 let stream = Stream::new()?;
18 let mut context = engine.context()?;
19
20 // One device buffer per I/O tensor, sized from the engine's own metadata.
21 let mut buffers = Vec::new();
22 for info in engine.io() {
23 let bytes = info.size_bytes().expect("a static shape");
24 let mut buffer = DeviceBuffer::zeroed(bytes, &stream)?;
25 if info.mode == IoMode::Input {
26 let mut host = PinnedBuffer::new(bytes)?;
27 host.f32s_mut().fill(0.5);
28 buffer.copy_from_host(host.bytes(), &stream)?;
29 }
30 // SAFETY: `buffers` is alive until after the synchronised run below.
31 unsafe { context.bind_buffer(&info.name, &buffer)? };
32 buffers.push((info.name.clone(), info.mode, buffer));
33 }
34
35 context.run(&stream)?; // enqueue plus synchronize
36
37 for (name, mode, buffer) in &buffers {
38 if *mode == IoMode::Output {
39 let mut out = vec![0u8; buffer.size_bytes()];
40 // copy_to_host waits for the stream, so these bytes are real.
41 buffer.copy_to_host(&mut out, &stream)?;
42 println!("{name}: read back {} bytes", out.len());
43 }
44 }
45 Ok(())
46}More examples
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15 let mut engine_path: Option<String> = None;
16 let mut blob_path: Option<String> = None;
17 let mut repeat: usize = 1;
18 let mut args = std::env::args().skip(1);
19 while let Some(arg) = args.next() {
20 match arg.as_str() {
21 "--blob" => blob_path = args.next(),
22 "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23 other if engine_path.is_none() => engine_path = Some(other.to_string()),
24 other => return Err(format!("unexpected argument {other:?}").into()),
25 }
26 }
27 let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29 let (major, minor, patch) = trt_runner::header_version();
30 println!("compiled against TensorRT {major}.{minor}.{patch}");
31 cuda::set_device(0)?;
32
33 let engine = Engine::from_file(&engine_path)?;
34 println!("{} has {} I/O tensors", engine_path, engine.io().len());
35 for info in engine.io() {
36 println!(
37 " {:10} {:7} {:?} {:?} {}",
38 info.name,
39 match info.mode {
40 IoMode::Input => "input",
41 IoMode::Output => "output",
42 IoMode::Internal => "internal",
43 },
44 info.dtype,
45 info.shape,
46 info.size_bytes()
47 .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48 );
49 }
50
51 let stream = Stream::new()?;
52 let mut context = engine.context()?;
53
54 // one device buffer per bindable tensor, sized from the engine's own metadata
55 let mut buffers = Vec::new();
56 for info in engine.io() {
57 let bytes = info
58 .size_bytes()
59 .ok_or_else(|| trt_runner::Error::DynamicShape {
60 tensor: info.name.clone(),
61 shape: info.shape.clone(),
62 })?;
63 let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64 buffers.push((info.name.clone(), buffer));
65 }
66 for (name, buffer) in &mut buffers {
67 // SAFETY: the buffers live until after the final synchronised run below.
68 unsafe { context.bind_buffer(name, buffer)? };
69 }
70
71 let input = engine
72 .inputs()
73 .next()
74 .ok_or("the engine has no input tensor to feed")?
75 .clone();
76 let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77 match blob_path {
78 Some(path) => {
79 let raw = std::fs::read(&path)?;
80 if raw.len() != host.size_bytes() {
81 return Err(format!(
82 "{} holds {} bytes, input wants {}",
83 path,
84 raw.len(),
85 host.size_bytes()
86 )
87 .into());
88 }
89 host.bytes_mut().copy_from_slice(&raw);
90 println!("fed {} bytes from {path}", raw.len());
91 }
92 None => host.f32s_mut().fill(0.5),
93 }
94 let input_buffer = &mut buffers
95 .iter_mut()
96 .find(|(name, _)| *name == input.name)
97 .unwrap()
98 .1;
99 input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101 let mut timings = Vec::with_capacity(repeat);
102 for _ in 0..repeat {
103 let started = Instant::now();
104 context.run(&stream)?;
105 timings.push(started.elapsed().as_secs_f64() * 1000.0);
106 }
107 timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108 let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109 println!(
110 "{} runs: mean {mean:.2} ms p50 {:.2} p95 {:.2} -> {:.1} fps",
111 timings.len(),
112 timings[timings.len() / 2],
113 timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114 1000.0 / mean
115 );
116
117 for info in engine.outputs() {
118 let buffer = &buffers
119 .iter()
120 .find(|(name, _)| *name == info.name)
121 .unwrap()
122 .1;
123 let mut scratch = vec![0u8; buffer.size_bytes()];
124 // copy_to_host waits for the stream, which is the point: the values below are real.
125 buffer.copy_to_host(&mut scratch, &stream)?;
126 let stats = describe(&scratch, info.dtype);
127 println!(" {} : {}", info.name, stats);
128 }
129 Ok(())
130}Sourcepub fn into_context(self) -> Result<OwnedContext>
pub fn into_context(self) -> Result<OwnedContext>
Consumes the engine into an OwnedContext, which keeps the engine and the runtime alive
beside the execution context. This is the shape a long lived runner usually wants: one value
to store in a struct, without a lifetime parameter chasing it.