Skip to main content

Model

Struct Model 

Source
pub struct Model { /* private fields */ }
Expand description

A loaded, shareable MNN model.

Clones address the same model worker. Calls are executed in submission order.

Implementations§

Source§

impl Model

Source

pub fn info(&self) -> &ModelInfo

Model input/output metadata.

Examples found in repository?
examples/inspect.rs (line 18)
7fn main() -> ExitCode {
8    let Some(path) = std::env::args_os().nth(1) else {
9        eprintln!("usage: cargo run -p mnn-runtime --example inspect -- <model.mnn>");
10        return ExitCode::FAILURE;
11    };
12
13    let result = Runtime::cpu().and_then(|runtime| runtime.load_file(path));
14    match result {
15        Ok(model) => {
16            println!("MNN {}", Runtime::native_version());
17            println!("inputs:");
18            for tensor in model.info().inputs() {
19                println!("  {} {:?}", tensor.name(), tensor.shape());
20            }
21            println!("outputs:");
22            for tensor in model.info().outputs() {
23                println!("  {} {:?}", tensor.name(), tensor.shape());
24            }
25            ExitCode::SUCCESS
26        }
27        Err(error) => {
28            eprintln!("error: {error}");
29            ExitCode::FAILURE
30        }
31    }
32}
More examples
Hide additional examples
examples/smoke.rs (line 27)
7fn main() -> ExitCode {
8    let Some(path) = std::env::args_os().nth(1) else {
9        eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10        return ExitCode::FAILURE;
11    };
12
13    let result = (|| {
14        let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15            Ok("metal") => Backend::Metal,
16            Ok("coreml") => Backend::CoreMl,
17            Ok("opencl") => Backend::OpenCl,
18            Ok("opengl") => Backend::OpenGl,
19            Ok("vulkan") => Backend::Vulkan,
20            Ok("cuda") => Backend::Cuda,
21            Ok("auto") => Backend::Auto,
22            Ok(_) | Err(_) => Backend::Cpu,
23        };
24        let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25        let model = runtime.load_file(path)?;
26        let inputs = model
27            .info()
28            .inputs()
29            .iter()
30            .map(|info| {
31                Tensor::new(
32                    info.name(),
33                    info.concrete_shape()?,
34                    vec![
35                        0.0;
36                        info.element_count().ok_or_else(|| {
37                            mnn_runtime::Error::UnresolvedShape {
38                                name: info.name().to_owned(),
39                            }
40                        })?
41                    ],
42                )
43            })
44            .collect::<mnn_runtime::Result<Vec<_>>>()?;
45        model.run(&inputs)
46    })();
47
48    match result {
49        Ok(outputs) => {
50            println!("MNN {}", Runtime::native_version());
51            for tensor in outputs {
52                let first = tensor.data().first().copied().unwrap_or_default();
53                println!(
54                    "{} {:?}: {} values, first={first}",
55                    tensor.name(),
56                    tensor.shape(),
57                    tensor.data().len()
58                );
59            }
60            ExitCode::SUCCESS
61        }
62        Err(error) => {
63            eprintln!("error: {error}");
64            ExitCode::FAILURE
65        }
66    }
67}
Source

pub fn cache_file(&self) -> Option<&Path>

Persistent cache path, isolated by model and native configuration.

Source

pub fn save_cache(&self) -> Result<()>

Save kernel tuning data after earlier accepted requests have completed. Without a cache directory this is a no-op. Shutdown also attempts a save.

§Errors

Returns native cache I/O or worker errors.

Source

pub fn run(&self, inputs: &[Tensor]) -> Result<Vec<Tensor>>

Run inference, waiting for queue capacity, and return all graph outputs. Concrete graph dimensions must match; unresolved dimensions use the input.

§Errors

Returns validation, native inference, or worker errors.

Examples found in repository?
examples/overhead.rs (line 30)
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24    let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25    let model =
26        runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27    let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28    let borrowed = measure(
29        || {
30            std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31        },
32        1000,
33    );
34    let mut outputs = model.run(std::slice::from_ref(&input))?;
35    let reused = measure(
36        || {
37            let result = model
38                .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39                .unwrap();
40            outputs = std::hint::black_box(result.tensors);
41        },
42        1000,
43    );
44    println!("tiny model, current compilation profile; P50/P95/P99");
45    println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46    println!(
47        "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48    );
49    Ok(())
50}
More examples
Hide additional examples
examples/smoke.rs (line 45)
7fn main() -> ExitCode {
8    let Some(path) = std::env::args_os().nth(1) else {
9        eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10        return ExitCode::FAILURE;
11    };
12
13    let result = (|| {
14        let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15            Ok("metal") => Backend::Metal,
16            Ok("coreml") => Backend::CoreMl,
17            Ok("opencl") => Backend::OpenCl,
18            Ok("opengl") => Backend::OpenGl,
19            Ok("vulkan") => Backend::Vulkan,
20            Ok("cuda") => Backend::Cuda,
21            Ok("auto") => Backend::Auto,
22            Ok(_) | Err(_) => Backend::Cpu,
23        };
24        let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25        let model = runtime.load_file(path)?;
26        let inputs = model
27            .info()
28            .inputs()
29            .iter()
30            .map(|info| {
31                Tensor::new(
32                    info.name(),
33                    info.concrete_shape()?,
34                    vec![
35                        0.0;
36                        info.element_count().ok_or_else(|| {
37                            mnn_runtime::Error::UnresolvedShape {
38                                name: info.name().to_owned(),
39                            }
40                        })?
41                    ],
42                )
43            })
44            .collect::<mnn_runtime::Result<Vec<_>>>()?;
45        model.run(&inputs)
46    })();
47
48    match result {
49        Ok(outputs) => {
50            println!("MNN {}", Runtime::native_version());
51            for tensor in outputs {
52                let first = tensor.data().first().copied().unwrap_or_default();
53                println!(
54                    "{} {:?}: {} values, first={first}",
55                    tensor.name(),
56                    tensor.shape(),
57                    tensor.data().len()
58                );
59            }
60            ExitCode::SUCCESS
61        }
62        Err(error) => {
63            eprintln!("error: {error}");
64            ExitCode::FAILURE
65        }
66    }
67}
Source

pub fn run_dynamic(&self, inputs: &[Tensor]) -> Result<Vec<Tensor>>

Run with concrete input shapes, permitting changes to graph dimensions including a fixed batch or width. Rank must match; MNN validates the graph. Calls wait for queue capacity. Actual output shapes belong to the returned tensors.

§Errors

Returns validation, native resize/inference, or worker errors.

Source

pub fn run_owned(&self, inputs: Vec<Tensor>) -> Result<Vec<Tensor>>

Move inputs into a blocking inference request.

§Errors

Returns validation, native inference, or worker errors.

Source

pub fn run_dynamic_owned(&self, inputs: Vec<Tensor>) -> Result<Vec<Tensor>>

Move inputs into a blocking request that can resize concrete graph axes.

§Errors

Returns validation, native resize/inference, or worker errors.

Source

pub fn run_for_outputs( &self, inputs: &[Tensor], outputs: &[&str], ) -> Result<Vec<Tensor>>

Run inference and copy only the named graph outputs in the given order.

§Errors

Returns validation, native inference, or worker errors.

Source

pub fn run_dynamic_for_outputs( &self, inputs: &[Tensor], outputs: &[&str], ) -> Result<Vec<Tensor>>

Resize inputs and retrieve only the named outputs in the given order.

§Errors

Returns validation, native resize/inference, or worker errors.

Source

pub fn run_reusing( &self, inputs: Vec<Tensor>, outputs: Vec<Tensor>, ) -> Result<InferenceOutput>

Reuse previous output allocations, selected by name, resizing them after inference if necessary. Inputs may resize concrete graph axes, as with run_dynamic. Buffers are consumed even on error; queue admission blocks.

§Errors

Returns validation, native resize/inference, or worker errors.

Examples found in repository?
examples/overhead.rs (line 38)
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24    let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25    let model =
26        runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27    let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28    let borrowed = measure(
29        || {
30            std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31        },
32        1000,
33    );
34    let mut outputs = model.run(std::slice::from_ref(&input))?;
35    let reused = measure(
36        || {
37            let result = model
38                .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39                .unwrap();
40            outputs = std::hint::black_box(result.tensors);
41        },
42        1000,
43    );
44    println!("tiny model, current compilation profile; P50/P95/P99");
45    println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46    println!(
47        "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48    );
49    Ok(())
50}
Source

pub fn try_submit_owned( &self, inputs: Vec<Tensor>, deadline: Option<Instant>, ) -> Result<PendingRun>

Submit without waiting for queue capacity. Dropping the returned handle cancels queued work. The deadline is checked before entering MNN.

§Errors

Returns QueueFull if not accepted, or validation/deadline errors.

Source

pub fn try_submit_dynamic_owned( &self, inputs: Vec<Tensor>, deadline: Option<Instant>, ) -> Result<PendingRun>

Nonblocking submission permitting concrete graph axes to resize.

§Errors

Returns QueueFull if not accepted, or validation/deadline errors.

Trait Implementations§

Source§

impl Clone for Model

Source§

fn clone(&self) -> Self

Returns a duplicate of the value. Read more
1.0.0 (const: unstable) · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl Debug for Model

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more

Auto Trait Implementations§

§

impl Freeze for Model

§

impl RefUnwindSafe for Model

§

impl Send for Model

§

impl Sync for Model

§

impl Unpin for Model

§

impl UnsafeUnpin for Model

§

impl UnwindSafe for Model

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dest. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.