pub struct Runtime { /* private fields */ }Expand description
Shareable model factory and process-level MNN execution policy.
Implementations§
Source§impl Runtime
impl Runtime
Sourcepub fn native_version() -> String
pub fn native_version() -> String
Version reported by the linked native MNN library.
Examples found in repository?
examples/inspect.rs (line 16)
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example inspect -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = Runtime::cpu().and_then(|runtime| runtime.load_file(path));
14 match result {
15 Ok(model) => {
16 println!("MNN {}", Runtime::native_version());
17 println!("inputs:");
18 for tensor in model.info().inputs() {
19 println!(" {} {:?}", tensor.name(), tensor.shape());
20 }
21 println!("outputs:");
22 for tensor in model.info().outputs() {
23 println!(" {} {:?}", tensor.name(), tensor.shape());
24 }
25 ExitCode::SUCCESS
26 }
27 Err(error) => {
28 eprintln!("error: {error}");
29 ExitCode::FAILURE
30 }
31 }
32}More examples
examples/smoke.rs (line 50)
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = (|| {
14 let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15 Ok("metal") => Backend::Metal,
16 Ok("coreml") => Backend::CoreMl,
17 Ok("opencl") => Backend::OpenCl,
18 Ok("opengl") => Backend::OpenGl,
19 Ok("vulkan") => Backend::Vulkan,
20 Ok("cuda") => Backend::Cuda,
21 Ok("auto") => Backend::Auto,
22 Ok(_) | Err(_) => Backend::Cpu,
23 };
24 let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25 let model = runtime.load_file(path)?;
26 let inputs = model
27 .info()
28 .inputs()
29 .iter()
30 .map(|info| {
31 Tensor::new(
32 info.name(),
33 info.concrete_shape()?,
34 vec![
35 0.0;
36 info.element_count().ok_or_else(|| {
37 mnn_runtime::Error::UnresolvedShape {
38 name: info.name().to_owned(),
39 }
40 })?
41 ],
42 )
43 })
44 .collect::<mnn_runtime::Result<Vec<_>>>()?;
45 model.run(&inputs)
46 })();
47
48 match result {
49 Ok(outputs) => {
50 println!("MNN {}", Runtime::native_version());
51 for tensor in outputs {
52 let first = tensor.data().first().copied().unwrap_or_default();
53 println!(
54 "{} {:?}: {} values, first={first}",
55 tensor.name(),
56 tensor.shape(),
57 tensor.data().len()
58 );
59 }
60 ExitCode::SUCCESS
61 }
62 Err(error) => {
63 eprintln!("error: {error}");
64 ExitCode::FAILURE
65 }
66 }
67}Sourcepub fn new(config: RuntimeConfig) -> Result<Self>
pub fn new(config: RuntimeConfig) -> Result<Self>
Create a runtime after validating its configuration and compiled backend.
§Errors
Returns an error if the thread count is invalid or the selected backend was not compiled or registered by the linked MNN library.
Examples found in repository?
examples/overhead.rs (line 24)
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24 let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25 let model =
26 runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27 let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28 let borrowed = measure(
29 || {
30 std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31 },
32 1000,
33 );
34 let mut outputs = model.run(std::slice::from_ref(&input))?;
35 let reused = measure(
36 || {
37 let result = model
38 .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39 .unwrap();
40 outputs = std::hint::black_box(result.tensors);
41 },
42 1000,
43 );
44 println!("tiny model, current compilation profile; P50/P95/P99");
45 println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46 println!(
47 "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48 );
49 Ok(())
50}More examples
examples/smoke.rs (line 24)
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = (|| {
14 let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15 Ok("metal") => Backend::Metal,
16 Ok("coreml") => Backend::CoreMl,
17 Ok("opencl") => Backend::OpenCl,
18 Ok("opengl") => Backend::OpenGl,
19 Ok("vulkan") => Backend::Vulkan,
20 Ok("cuda") => Backend::Cuda,
21 Ok("auto") => Backend::Auto,
22 Ok(_) | Err(_) => Backend::Cpu,
23 };
24 let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25 let model = runtime.load_file(path)?;
26 let inputs = model
27 .info()
28 .inputs()
29 .iter()
30 .map(|info| {
31 Tensor::new(
32 info.name(),
33 info.concrete_shape()?,
34 vec![
35 0.0;
36 info.element_count().ok_or_else(|| {
37 mnn_runtime::Error::UnresolvedShape {
38 name: info.name().to_owned(),
39 }
40 })?
41 ],
42 )
43 })
44 .collect::<mnn_runtime::Result<Vec<_>>>()?;
45 model.run(&inputs)
46 })();
47
48 match result {
49 Ok(outputs) => {
50 println!("MNN {}", Runtime::native_version());
51 for tensor in outputs {
52 let first = tensor.data().first().copied().unwrap_or_default();
53 println!(
54 "{} {:?}: {} values, first={first}",
55 tensor.name(),
56 tensor.shape(),
57 tensor.data().len()
58 );
59 }
60 ExitCode::SUCCESS
61 }
62 Err(error) => {
63 eprintln!("error: {error}");
64 ExitCode::FAILURE
65 }
66 }
67}Sourcepub fn cpu() -> Result<Self>
pub fn cpu() -> Result<Self>
Create the default CPU runtime.
§Errors
Returns an error when the linked MNN library has no CPU backend.
Examples found in repository?
examples/inspect.rs (line 13)
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example inspect -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = Runtime::cpu().and_then(|runtime| runtime.load_file(path));
14 match result {
15 Ok(model) => {
16 println!("MNN {}", Runtime::native_version());
17 println!("inputs:");
18 for tensor in model.info().inputs() {
19 println!(" {} {:?}", tensor.name(), tensor.shape());
20 }
21 println!("outputs:");
22 for tensor in model.info().outputs() {
23 println!(" {} {:?}", tensor.name(), tensor.shape());
24 }
25 ExitCode::SUCCESS
26 }
27 Err(error) => {
28 eprintln!("error: {error}");
29 ExitCode::FAILURE
30 }
31 }
32}Sourcepub fn config(&self) -> &RuntimeConfig
pub fn config(&self) -> &RuntimeConfig
Runtime configuration used for newly loaded models.
Sourcepub fn load_file(&self, path: impl AsRef<Path>) -> Result<Model>
pub fn load_file(&self, path: impl AsRef<Path>) -> Result<Model>
Load an MNN model file and start its dedicated worker.
§Errors
Returns an error if the file cannot be read or MNN cannot load it.
Examples found in repository?
examples/inspect.rs (line 13)
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example inspect -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = Runtime::cpu().and_then(|runtime| runtime.load_file(path));
14 match result {
15 Ok(model) => {
16 println!("MNN {}", Runtime::native_version());
17 println!("inputs:");
18 for tensor in model.info().inputs() {
19 println!(" {} {:?}", tensor.name(), tensor.shape());
20 }
21 println!("outputs:");
22 for tensor in model.info().outputs() {
23 println!(" {} {:?}", tensor.name(), tensor.shape());
24 }
25 ExitCode::SUCCESS
26 }
27 Err(error) => {
28 eprintln!("error: {error}");
29 ExitCode::FAILURE
30 }
31 }
32}More examples
examples/smoke.rs (line 25)
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = (|| {
14 let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15 Ok("metal") => Backend::Metal,
16 Ok("coreml") => Backend::CoreMl,
17 Ok("opencl") => Backend::OpenCl,
18 Ok("opengl") => Backend::OpenGl,
19 Ok("vulkan") => Backend::Vulkan,
20 Ok("cuda") => Backend::Cuda,
21 Ok("auto") => Backend::Auto,
22 Ok(_) | Err(_) => Backend::Cpu,
23 };
24 let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25 let model = runtime.load_file(path)?;
26 let inputs = model
27 .info()
28 .inputs()
29 .iter()
30 .map(|info| {
31 Tensor::new(
32 info.name(),
33 info.concrete_shape()?,
34 vec![
35 0.0;
36 info.element_count().ok_or_else(|| {
37 mnn_runtime::Error::UnresolvedShape {
38 name: info.name().to_owned(),
39 }
40 })?
41 ],
42 )
43 })
44 .collect::<mnn_runtime::Result<Vec<_>>>()?;
45 model.run(&inputs)
46 })();
47
48 match result {
49 Ok(outputs) => {
50 println!("MNN {}", Runtime::native_version());
51 for tensor in outputs {
52 let first = tensor.data().first().copied().unwrap_or_default();
53 println!(
54 "{} {:?}: {} values, first={first}",
55 tensor.name(),
56 tensor.shape(),
57 tensor.data().len()
58 );
59 }
60 ExitCode::SUCCESS
61 }
62 Err(error) => {
63 eprintln!("error: {error}");
64 ExitCode::FAILURE
65 }
66 }
67}Sourcepub fn load_bytes(&self, bytes: impl Into<Vec<u8>>) -> Result<Model>
pub fn load_bytes(&self, bytes: impl Into<Vec<u8>>) -> Result<Model>
Load an MNN model from owned bytes and start its dedicated worker.
§Errors
Returns an error if the model is empty, invalid, unsupported, or its worker cannot be started.
Examples found in repository?
examples/overhead.rs (line 26)
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24 let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25 let model =
26 runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27 let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28 let borrowed = measure(
29 || {
30 std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31 },
32 1000,
33 );
34 let mut outputs = model.run(std::slice::from_ref(&input))?;
35 let reused = measure(
36 || {
37 let result = model
38 .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39 .unwrap();
40 outputs = std::hint::black_box(result.tensors);
41 },
42 1000,
43 );
44 println!("tiny model, current compilation profile; P50/P95/P99");
45 println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46 println!(
47 "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48 );
49 Ok(())
50}Trait Implementations§
Auto Trait Implementations§
impl Freeze for Runtime
impl RefUnwindSafe for Runtime
impl Send for Runtime
impl Sync for Runtime
impl Unpin for Runtime
impl UnsafeUnpin for Runtime
impl UnwindSafe for Runtime
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more