Skip to main content

RuntimeConfig

Struct RuntimeConfig 

Source
pub struct RuntimeConfig {
    pub backend: Backend,
    pub threads: usize,
    pub precision: PrecisionMode,
    pub power: PowerMode,
    pub memory: MemoryMode,
    pub execution: ExecutionMode,
    pub queue_capacity: usize,
    pub gpu_tuning: GpuTuning,
    pub gpu_memory: GpuMemoryMode,
    pub gpu_cache_dir: Option<PathBuf>,
}
Expand description

Configuration used for subsequently loaded models.

Fields§

§backend: Backend

Execution backend.

§threads: usize

Number of MNN worker threads. Must be greater than zero.

§precision: PrecisionMode

Precision preference.

§power: PowerMode

Power preference.

§memory: MemoryMode

Memory preference.

§execution: ExecutionMode

Cross-model execution policy.

§queue_capacity: usize

Maximum queued requests per model, excluding the active request.

§gpu_tuning: GpuTuning

GPU tuning policy.

§gpu_memory: GpuMemoryMode

OpenCL storage policy.

§gpu_cache_dir: Option<PathBuf>

Directory for caches isolated by model, native version and configuration. Cache writes occur on Model::save_cache and worker shutdown.

Implementations§

Source§

impl RuntimeConfig

Source

pub fn new() -> Self

Start with the default CPU configuration.

Examples found in repository?
examples/overhead.rs (line 24)
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24    let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25    let model =
26        runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27    let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28    let borrowed = measure(
29        || {
30            std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31        },
32        1000,
33    );
34    let mut outputs = model.run(std::slice::from_ref(&input))?;
35    let reused = measure(
36        || {
37            let result = model
38                .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39                .unwrap();
40            outputs = std::hint::black_box(result.tensors);
41        },
42        1000,
43    );
44    println!("tiny model, current compilation profile; P50/P95/P99");
45    println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46    println!(
47        "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48    );
49    Ok(())
50}
More examples
Hide additional examples
examples/smoke.rs (line 24)
7fn main() -> ExitCode {
8    let Some(path) = std::env::args_os().nth(1) else {
9        eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10        return ExitCode::FAILURE;
11    };
12
13    let result = (|| {
14        let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15            Ok("metal") => Backend::Metal,
16            Ok("coreml") => Backend::CoreMl,
17            Ok("opencl") => Backend::OpenCl,
18            Ok("opengl") => Backend::OpenGl,
19            Ok("vulkan") => Backend::Vulkan,
20            Ok("cuda") => Backend::Cuda,
21            Ok("auto") => Backend::Auto,
22            Ok(_) | Err(_) => Backend::Cpu,
23        };
24        let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25        let model = runtime.load_file(path)?;
26        let inputs = model
27            .info()
28            .inputs()
29            .iter()
30            .map(|info| {
31                Tensor::new(
32                    info.name(),
33                    info.concrete_shape()?,
34                    vec![
35                        0.0;
36                        info.element_count().ok_or_else(|| {
37                            mnn_runtime::Error::UnresolvedShape {
38                                name: info.name().to_owned(),
39                            }
40                        })?
41                    ],
42                )
43            })
44            .collect::<mnn_runtime::Result<Vec<_>>>()?;
45        model.run(&inputs)
46    })();
47
48    match result {
49        Ok(outputs) => {
50            println!("MNN {}", Runtime::native_version());
51            for tensor in outputs {
52                let first = tensor.data().first().copied().unwrap_or_default();
53                println!(
54                    "{} {:?}: {} values, first={first}",
55                    tensor.name(),
56                    tensor.shape(),
57                    tensor.data().len()
58                );
59            }
60            ExitCode::SUCCESS
61        }
62        Err(error) => {
63            eprintln!("error: {error}");
64            ExitCode::FAILURE
65        }
66    }
67}
Source

pub const fn with_backend(self, backend: Backend) -> Self

Select a backend.

Examples found in repository?
examples/smoke.rs (line 24)
7fn main() -> ExitCode {
8    let Some(path) = std::env::args_os().nth(1) else {
9        eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10        return ExitCode::FAILURE;
11    };
12
13    let result = (|| {
14        let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15            Ok("metal") => Backend::Metal,
16            Ok("coreml") => Backend::CoreMl,
17            Ok("opencl") => Backend::OpenCl,
18            Ok("opengl") => Backend::OpenGl,
19            Ok("vulkan") => Backend::Vulkan,
20            Ok("cuda") => Backend::Cuda,
21            Ok("auto") => Backend::Auto,
22            Ok(_) | Err(_) => Backend::Cpu,
23        };
24        let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25        let model = runtime.load_file(path)?;
26        let inputs = model
27            .info()
28            .inputs()
29            .iter()
30            .map(|info| {
31                Tensor::new(
32                    info.name(),
33                    info.concrete_shape()?,
34                    vec![
35                        0.0;
36                        info.element_count().ok_or_else(|| {
37                            mnn_runtime::Error::UnresolvedShape {
38                                name: info.name().to_owned(),
39                            }
40                        })?
41                    ],
42                )
43            })
44            .collect::<mnn_runtime::Result<Vec<_>>>()?;
45        model.run(&inputs)
46    })();
47
48    match result {
49        Ok(outputs) => {
50            println!("MNN {}", Runtime::native_version());
51            for tensor in outputs {
52                let first = tensor.data().first().copied().unwrap_or_default();
53                println!(
54                    "{} {:?}: {} values, first={first}",
55                    tensor.name(),
56                    tensor.shape(),
57                    tensor.data().len()
58                );
59            }
60            ExitCode::SUCCESS
61        }
62        Err(error) => {
63            eprintln!("error: {error}");
64            ExitCode::FAILURE
65        }
66    }
67}
Source

pub const fn with_threads(self, threads: usize) -> Self

Select the MNN worker-thread count.

Examples found in repository?
examples/overhead.rs (line 24)
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24    let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25    let model =
26        runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27    let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28    let borrowed = measure(
29        || {
30            std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31        },
32        1000,
33    );
34    let mut outputs = model.run(std::slice::from_ref(&input))?;
35    let reused = measure(
36        || {
37            let result = model
38                .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39                .unwrap();
40            outputs = std::hint::black_box(result.tensors);
41        },
42        1000,
43    );
44    println!("tiny model, current compilation profile; P50/P95/P99");
45    println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46    println!(
47        "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48    );
49    Ok(())
50}
Source

pub const fn with_precision(self, precision: PrecisionMode) -> Self

Select the precision preference.

Source

pub const fn with_queue_capacity(self, capacity: usize) -> Self

Set the maximum number of queued requests (must be positive).

Source

pub const fn with_gpu_tuning(self, tuning: GpuTuning) -> Self

Select GPU tuning.

Source

pub const fn with_gpu_memory(self, memory: GpuMemoryMode) -> Self

Select OpenCL tensor storage.

Source

pub fn with_gpu_cache_dir(self, directory: impl Into<PathBuf>) -> Self

Enable persistent kernel caches for subsequently loaded models.

Source

pub const fn with_execution(self, execution: ExecutionMode) -> Self

Select the cross-model execution policy.

Trait Implementations§

Source§

impl Clone for RuntimeConfig

Source§

fn clone(&self) -> Self

Returns a duplicate of the value. Read more
1.0.0 (const: unstable) · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl Debug for RuntimeConfig

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more
Source§

impl Default for RuntimeConfig

Source§

fn default() -> Self

Returns the “default value” for a type. Read more
Source§

impl Eq for RuntimeConfig

Source§

impl PartialEq for RuntimeConfig

Source§

fn eq(&self, other: &Self) -> bool

Equality operator ==. Read more
1.0.0 (const: unstable) · Source§

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=. Read more
Source§

impl StructuralPartialEq for RuntimeConfig

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dest. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.