pub struct Model { /* private fields */ }Expand description
A loaded, shareable MNN model.
Clones address the same model worker. Calls are executed in submission order.
Implementations§
Source§impl Model
impl Model
Sourcepub fn info(&self) -> &ModelInfo
pub fn info(&self) -> &ModelInfo
Model input/output metadata.
Examples found in repository?
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example inspect -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = Runtime::cpu().and_then(|runtime| runtime.load_file(path));
14 match result {
15 Ok(model) => {
16 println!("MNN {}", Runtime::native_version());
17 println!("inputs:");
18 for tensor in model.info().inputs() {
19 println!(" {} {:?}", tensor.name(), tensor.shape());
20 }
21 println!("outputs:");
22 for tensor in model.info().outputs() {
23 println!(" {} {:?}", tensor.name(), tensor.shape());
24 }
25 ExitCode::SUCCESS
26 }
27 Err(error) => {
28 eprintln!("error: {error}");
29 ExitCode::FAILURE
30 }
31 }
32}More examples
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = (|| {
14 let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15 Ok("metal") => Backend::Metal,
16 Ok("coreml") => Backend::CoreMl,
17 Ok("opencl") => Backend::OpenCl,
18 Ok("opengl") => Backend::OpenGl,
19 Ok("vulkan") => Backend::Vulkan,
20 Ok("cuda") => Backend::Cuda,
21 Ok("auto") => Backend::Auto,
22 Ok(_) | Err(_) => Backend::Cpu,
23 };
24 let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25 let model = runtime.load_file(path)?;
26 let inputs = model
27 .info()
28 .inputs()
29 .iter()
30 .map(|info| {
31 Tensor::new(
32 info.name(),
33 info.concrete_shape()?,
34 vec![
35 0.0;
36 info.element_count().ok_or_else(|| {
37 mnn_runtime::Error::UnresolvedShape {
38 name: info.name().to_owned(),
39 }
40 })?
41 ],
42 )
43 })
44 .collect::<mnn_runtime::Result<Vec<_>>>()?;
45 model.run(&inputs)
46 })();
47
48 match result {
49 Ok(outputs) => {
50 println!("MNN {}", Runtime::native_version());
51 for tensor in outputs {
52 let first = tensor.data().first().copied().unwrap_or_default();
53 println!(
54 "{} {:?}: {} values, first={first}",
55 tensor.name(),
56 tensor.shape(),
57 tensor.data().len()
58 );
59 }
60 ExitCode::SUCCESS
61 }
62 Err(error) => {
63 eprintln!("error: {error}");
64 ExitCode::FAILURE
65 }
66 }
67}Sourcepub fn cache_file(&self) -> Option<&Path>
pub fn cache_file(&self) -> Option<&Path>
Persistent cache path, isolated by model and native configuration.
Sourcepub fn save_cache(&self) -> Result<()>
pub fn save_cache(&self) -> Result<()>
Save kernel tuning data after earlier accepted requests have completed. Without a cache directory this is a no-op. Shutdown also attempts a save.
§Errors
Returns native cache I/O or worker errors.
Sourcepub fn run(&self, inputs: &[Tensor]) -> Result<Vec<Tensor>>
pub fn run(&self, inputs: &[Tensor]) -> Result<Vec<Tensor>>
Run inference, waiting for queue capacity, and return all graph outputs. Concrete graph dimensions must match; unresolved dimensions use the input.
§Errors
Returns validation, native inference, or worker errors.
Examples found in repository?
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24 let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25 let model =
26 runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27 let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28 let borrowed = measure(
29 || {
30 std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31 },
32 1000,
33 );
34 let mut outputs = model.run(std::slice::from_ref(&input))?;
35 let reused = measure(
36 || {
37 let result = model
38 .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39 .unwrap();
40 outputs = std::hint::black_box(result.tensors);
41 },
42 1000,
43 );
44 println!("tiny model, current compilation profile; P50/P95/P99");
45 println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46 println!(
47 "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48 );
49 Ok(())
50}More examples
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = (|| {
14 let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15 Ok("metal") => Backend::Metal,
16 Ok("coreml") => Backend::CoreMl,
17 Ok("opencl") => Backend::OpenCl,
18 Ok("opengl") => Backend::OpenGl,
19 Ok("vulkan") => Backend::Vulkan,
20 Ok("cuda") => Backend::Cuda,
21 Ok("auto") => Backend::Auto,
22 Ok(_) | Err(_) => Backend::Cpu,
23 };
24 let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25 let model = runtime.load_file(path)?;
26 let inputs = model
27 .info()
28 .inputs()
29 .iter()
30 .map(|info| {
31 Tensor::new(
32 info.name(),
33 info.concrete_shape()?,
34 vec![
35 0.0;
36 info.element_count().ok_or_else(|| {
37 mnn_runtime::Error::UnresolvedShape {
38 name: info.name().to_owned(),
39 }
40 })?
41 ],
42 )
43 })
44 .collect::<mnn_runtime::Result<Vec<_>>>()?;
45 model.run(&inputs)
46 })();
47
48 match result {
49 Ok(outputs) => {
50 println!("MNN {}", Runtime::native_version());
51 for tensor in outputs {
52 let first = tensor.data().first().copied().unwrap_or_default();
53 println!(
54 "{} {:?}: {} values, first={first}",
55 tensor.name(),
56 tensor.shape(),
57 tensor.data().len()
58 );
59 }
60 ExitCode::SUCCESS
61 }
62 Err(error) => {
63 eprintln!("error: {error}");
64 ExitCode::FAILURE
65 }
66 }
67}Sourcepub fn run_dynamic(&self, inputs: &[Tensor]) -> Result<Vec<Tensor>>
pub fn run_dynamic(&self, inputs: &[Tensor]) -> Result<Vec<Tensor>>
Run with concrete input shapes, permitting changes to graph dimensions including a fixed batch or width. Rank must match; MNN validates the graph. Calls wait for queue capacity. Actual output shapes belong to the returned tensors.
§Errors
Returns validation, native resize/inference, or worker errors.
Sourcepub fn run_owned(&self, inputs: Vec<Tensor>) -> Result<Vec<Tensor>>
pub fn run_owned(&self, inputs: Vec<Tensor>) -> Result<Vec<Tensor>>
Move inputs into a blocking inference request.
§Errors
Returns validation, native inference, or worker errors.
Sourcepub fn run_dynamic_owned(&self, inputs: Vec<Tensor>) -> Result<Vec<Tensor>>
pub fn run_dynamic_owned(&self, inputs: Vec<Tensor>) -> Result<Vec<Tensor>>
Move inputs into a blocking request that can resize concrete graph axes.
§Errors
Returns validation, native resize/inference, or worker errors.
Sourcepub fn run_for_outputs(
&self,
inputs: &[Tensor],
outputs: &[&str],
) -> Result<Vec<Tensor>>
pub fn run_for_outputs( &self, inputs: &[Tensor], outputs: &[&str], ) -> Result<Vec<Tensor>>
Run inference and copy only the named graph outputs in the given order.
§Errors
Returns validation, native inference, or worker errors.
Sourcepub fn run_dynamic_for_outputs(
&self,
inputs: &[Tensor],
outputs: &[&str],
) -> Result<Vec<Tensor>>
pub fn run_dynamic_for_outputs( &self, inputs: &[Tensor], outputs: &[&str], ) -> Result<Vec<Tensor>>
Resize inputs and retrieve only the named outputs in the given order.
§Errors
Returns validation, native resize/inference, or worker errors.
Sourcepub fn run_reusing(
&self,
inputs: Vec<Tensor>,
outputs: Vec<Tensor>,
) -> Result<InferenceOutput>
pub fn run_reusing( &self, inputs: Vec<Tensor>, outputs: Vec<Tensor>, ) -> Result<InferenceOutput>
Reuse previous output allocations, selected by name, resizing them after
inference if necessary. Inputs may resize concrete graph axes, as with
run_dynamic. Buffers are consumed even on error; queue admission blocks.
§Errors
Returns validation, native resize/inference, or worker errors.
Examples found in repository?
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24 let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25 let model =
26 runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27 let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28 let borrowed = measure(
29 || {
30 std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31 },
32 1000,
33 );
34 let mut outputs = model.run(std::slice::from_ref(&input))?;
35 let reused = measure(
36 || {
37 let result = model
38 .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39 .unwrap();
40 outputs = std::hint::black_box(result.tensors);
41 },
42 1000,
43 );
44 println!("tiny model, current compilation profile; P50/P95/P99");
45 println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46 println!(
47 "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48 );
49 Ok(())
50}Sourcepub fn try_submit_owned(
&self,
inputs: Vec<Tensor>,
deadline: Option<Instant>,
) -> Result<PendingRun>
pub fn try_submit_owned( &self, inputs: Vec<Tensor>, deadline: Option<Instant>, ) -> Result<PendingRun>
Submit without waiting for queue capacity. Dropping the returned handle cancels queued work. The deadline is checked before entering MNN.
§Errors
Returns QueueFull if not accepted, or validation/deadline errors.
Sourcepub fn try_submit_dynamic_owned(
&self,
inputs: Vec<Tensor>,
deadline: Option<Instant>,
) -> Result<PendingRun>
pub fn try_submit_dynamic_owned( &self, inputs: Vec<Tensor>, deadline: Option<Instant>, ) -> Result<PendingRun>
Nonblocking submission permitting concrete graph axes to resize.
§Errors
Returns QueueFull if not accepted, or validation/deadline errors.