pub struct TensorInfo {
pub name: String,
pub mode: IoMode,
pub dtype: DataType,
pub shape: Vec<i64>,
}Expand description
One I/O tensor of an engine: name, direction, element type and declared shape.
Fields§
§name: StringEngine side name, e.g. images or output0.
mode: IoModeDirection of the tensor.
dtype: DataTypeElement type.
shape: Vec<i64>Dimensions as declared in the engine. -1 marks a dynamic dimension.
Implementations§
Source§impl TensorInfo
impl TensorInfo
Sourcepub fn elements(&self) -> Option<usize>
pub fn elements(&self) -> Option<usize>
Number of elements, or None when any dimension is dynamic.
Sourcepub fn size_bytes(&self) -> Option<usize>
pub fn size_bytes(&self) -> Option<usize>
Size in bytes, or None for dynamic shapes and sub-byte element types.
Examples found in repository?
examples/quickstart.rs (line 23)
10fn main() -> trt_runner::Result<()> {
11 let path = std::env::args()
12 .nth(1)
13 .unwrap_or_else(|| "model_fp16.engine".to_string());
14
15 cuda::set_device(0)?;
16 let engine = Engine::from_file(&path)?;
17 let stream = Stream::new()?;
18 let mut context = engine.context()?;
19
20 // One device buffer per I/O tensor, sized from the engine's own metadata.
21 let mut buffers = Vec::new();
22 for info in engine.io() {
23 let bytes = info.size_bytes().expect("a static shape");
24 let mut buffer = DeviceBuffer::zeroed(bytes, &stream)?;
25 if info.mode == IoMode::Input {
26 let mut host = PinnedBuffer::new(bytes)?;
27 host.f32s_mut().fill(0.5);
28 buffer.copy_from_host(host.bytes(), &stream)?;
29 }
30 // SAFETY: `buffers` is alive until after the synchronised run below.
31 unsafe { context.bind_buffer(&info.name, &buffer)? };
32 buffers.push((info.name.clone(), info.mode, buffer));
33 }
34
35 context.run(&stream)?; // enqueue plus synchronize
36
37 for (name, mode, buffer) in &buffers {
38 if *mode == IoMode::Output {
39 let mut out = vec![0u8; buffer.size_bytes()];
40 // copy_to_host waits for the stream, so these bytes are real.
41 buffer.copy_to_host(&mut out, &stream)?;
42 println!("{name}: read back {} bytes", out.len());
43 }
44 }
45 Ok(())
46}More examples
examples/run_engine.rs (line 46)
14fn main() -> Result<(), Box<dyn std::error::Error>> {
15 let mut engine_path: Option<String> = None;
16 let mut blob_path: Option<String> = None;
17 let mut repeat: usize = 1;
18 let mut args = std::env::args().skip(1);
19 while let Some(arg) = args.next() {
20 match arg.as_str() {
21 "--blob" => blob_path = args.next(),
22 "--repeat" => repeat = args.next().and_then(|v| v.parse().ok()).unwrap_or(1).max(1),
23 other if engine_path.is_none() => engine_path = Some(other.to_string()),
24 other => return Err(format!("unexpected argument {other:?}").into()),
25 }
26 }
27 let engine_path = engine_path.ok_or("usage: run_engine <engine> [--blob file] [--repeat n]")?;
28
29 let (major, minor, patch) = trt_runner::header_version();
30 println!("compiled against TensorRT {major}.{minor}.{patch}");
31 cuda::set_device(0)?;
32
33 let engine = Engine::from_file(&engine_path)?;
34 println!("{} has {} I/O tensors", engine_path, engine.io().len());
35 for info in engine.io() {
36 println!(
37 " {:10} {:7} {:?} {:?} {}",
38 info.name,
39 match info.mode {
40 IoMode::Input => "input",
41 IoMode::Output => "output",
42 IoMode::Internal => "internal",
43 },
44 info.dtype,
45 info.shape,
46 info.size_bytes()
47 .map_or_else(|| "dynamic".to_string(), |bytes| format!("{bytes} B"))
48 );
49 }
50
51 let stream = Stream::new()?;
52 let mut context = engine.context()?;
53
54 // one device buffer per bindable tensor, sized from the engine's own metadata
55 let mut buffers = Vec::new();
56 for info in engine.io() {
57 let bytes = info
58 .size_bytes()
59 .ok_or_else(|| trt_runner::Error::DynamicShape {
60 tensor: info.name.clone(),
61 shape: info.shape.clone(),
62 })?;
63 let buffer = DeviceBuffer::zeroed(bytes, &stream)?;
64 buffers.push((info.name.clone(), buffer));
65 }
66 for (name, buffer) in &mut buffers {
67 // SAFETY: the buffers live until after the final synchronised run below.
68 unsafe { context.bind_buffer(name, buffer)? };
69 }
70
71 let input = engine
72 .inputs()
73 .next()
74 .ok_or("the engine has no input tensor to feed")?
75 .clone();
76 let mut host = PinnedBuffer::new(input.size_bytes().unwrap())?;
77 match blob_path {
78 Some(path) => {
79 let raw = std::fs::read(&path)?;
80 if raw.len() != host.size_bytes() {
81 return Err(format!(
82 "{} holds {} bytes, input wants {}",
83 path,
84 raw.len(),
85 host.size_bytes()
86 )
87 .into());
88 }
89 host.bytes_mut().copy_from_slice(&raw);
90 println!("fed {} bytes from {path}", raw.len());
91 }
92 None => host.f32s_mut().fill(0.5),
93 }
94 let input_buffer = &mut buffers
95 .iter_mut()
96 .find(|(name, _)| *name == input.name)
97 .unwrap()
98 .1;
99 input_buffer.copy_from_host(host.bytes(), &stream)?;
100
101 let mut timings = Vec::with_capacity(repeat);
102 for _ in 0..repeat {
103 let started = Instant::now();
104 context.run(&stream)?;
105 timings.push(started.elapsed().as_secs_f64() * 1000.0);
106 }
107 timings.sort_by(|a, b| a.partial_cmp(b).unwrap());
108 let mean = timings.iter().sum::<f64>() / timings.len() as f64;
109 println!(
110 "{} runs: mean {mean:.2} ms p50 {:.2} p95 {:.2} -> {:.1} fps",
111 timings.len(),
112 timings[timings.len() / 2],
113 timings[((timings.len() as f64 * 0.95) as usize).min(timings.len() - 1)],
114 1000.0 / mean
115 );
116
117 for info in engine.outputs() {
118 let buffer = &buffers
119 .iter()
120 .find(|(name, _)| *name == info.name)
121 .unwrap()
122 .1;
123 let mut scratch = vec![0u8; buffer.size_bytes()];
124 // copy_to_host waits for the stream, which is the point: the values below are real.
125 buffer.copy_to_host(&mut scratch, &stream)?;
126 let stats = describe(&scratch, info.dtype);
127 println!(" {} : {}", info.name, stats);
128 }
129 Ok(())
130}Trait Implementations§
Source§impl Clone for TensorInfo
impl Clone for TensorInfo
Source§impl Debug for TensorInfo
impl Debug for TensorInfo
impl Eq for TensorInfo
Source§impl PartialEq for TensorInfo
impl PartialEq for TensorInfo
impl StructuralPartialEq for TensorInfo
Auto Trait Implementations§
impl Freeze for TensorInfo
impl RefUnwindSafe for TensorInfo
impl Send for TensorInfo
impl Sync for TensorInfo
impl Unpin for TensorInfo
impl UnsafeUnpin for TensorInfo
impl UnwindSafe for TensorInfo
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more