pub struct RuntimeConfig {
pub backend: Backend,
pub threads: usize,
pub precision: PrecisionMode,
pub power: PowerMode,
pub memory: MemoryMode,
pub execution: ExecutionMode,
pub queue_capacity: usize,
pub gpu_tuning: GpuTuning,
pub gpu_memory: GpuMemoryMode,
pub gpu_cache_dir: Option<PathBuf>,
}Expand description
Configuration used for subsequently loaded models.
Fields§
§backend: BackendExecution backend.
threads: usizeNumber of MNN worker threads. Must be greater than zero.
precision: PrecisionModePrecision preference.
power: PowerModePower preference.
memory: MemoryModeMemory preference.
execution: ExecutionModeCross-model execution policy.
queue_capacity: usizeMaximum queued requests per model, excluding the active request.
gpu_tuning: GpuTuningGPU tuning policy.
gpu_memory: GpuMemoryModeOpenCL storage policy.
gpu_cache_dir: Option<PathBuf>Directory for caches isolated by model, native version and configuration.
Cache writes occur on Model::save_cache and worker shutdown.
Implementations§
Source§impl RuntimeConfig
impl RuntimeConfig
Sourcepub fn new() -> Self
pub fn new() -> Self
Start with the default CPU configuration.
Examples found in repository?
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24 let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25 let model =
26 runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27 let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28 let borrowed = measure(
29 || {
30 std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31 },
32 1000,
33 );
34 let mut outputs = model.run(std::slice::from_ref(&input))?;
35 let reused = measure(
36 || {
37 let result = model
38 .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39 .unwrap();
40 outputs = std::hint::black_box(result.tensors);
41 },
42 1000,
43 );
44 println!("tiny model, current compilation profile; P50/P95/P99");
45 println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46 println!(
47 "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48 );
49 Ok(())
50}More examples
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = (|| {
14 let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15 Ok("metal") => Backend::Metal,
16 Ok("coreml") => Backend::CoreMl,
17 Ok("opencl") => Backend::OpenCl,
18 Ok("opengl") => Backend::OpenGl,
19 Ok("vulkan") => Backend::Vulkan,
20 Ok("cuda") => Backend::Cuda,
21 Ok("auto") => Backend::Auto,
22 Ok(_) | Err(_) => Backend::Cpu,
23 };
24 let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25 let model = runtime.load_file(path)?;
26 let inputs = model
27 .info()
28 .inputs()
29 .iter()
30 .map(|info| {
31 Tensor::new(
32 info.name(),
33 info.concrete_shape()?,
34 vec![
35 0.0;
36 info.element_count().ok_or_else(|| {
37 mnn_runtime::Error::UnresolvedShape {
38 name: info.name().to_owned(),
39 }
40 })?
41 ],
42 )
43 })
44 .collect::<mnn_runtime::Result<Vec<_>>>()?;
45 model.run(&inputs)
46 })();
47
48 match result {
49 Ok(outputs) => {
50 println!("MNN {}", Runtime::native_version());
51 for tensor in outputs {
52 let first = tensor.data().first().copied().unwrap_or_default();
53 println!(
54 "{} {:?}: {} values, first={first}",
55 tensor.name(),
56 tensor.shape(),
57 tensor.data().len()
58 );
59 }
60 ExitCode::SUCCESS
61 }
62 Err(error) => {
63 eprintln!("error: {error}");
64 ExitCode::FAILURE
65 }
66 }
67}Sourcepub const fn with_backend(self, backend: Backend) -> Self
pub const fn with_backend(self, backend: Backend) -> Self
Select a backend.
Examples found in repository?
7fn main() -> ExitCode {
8 let Some(path) = std::env::args_os().nth(1) else {
9 eprintln!("usage: cargo run -p mnn-runtime --example smoke -- <model.mnn>");
10 return ExitCode::FAILURE;
11 };
12
13 let result = (|| {
14 let backend = match std::env::var("MNN_SMOKE_BACKEND").as_deref() {
15 Ok("metal") => Backend::Metal,
16 Ok("coreml") => Backend::CoreMl,
17 Ok("opencl") => Backend::OpenCl,
18 Ok("opengl") => Backend::OpenGl,
19 Ok("vulkan") => Backend::Vulkan,
20 Ok("cuda") => Backend::Cuda,
21 Ok("auto") => Backend::Auto,
22 Ok(_) | Err(_) => Backend::Cpu,
23 };
24 let runtime = Runtime::new(RuntimeConfig::new().with_backend(backend))?;
25 let model = runtime.load_file(path)?;
26 let inputs = model
27 .info()
28 .inputs()
29 .iter()
30 .map(|info| {
31 Tensor::new(
32 info.name(),
33 info.concrete_shape()?,
34 vec![
35 0.0;
36 info.element_count().ok_or_else(|| {
37 mnn_runtime::Error::UnresolvedShape {
38 name: info.name().to_owned(),
39 }
40 })?
41 ],
42 )
43 })
44 .collect::<mnn_runtime::Result<Vec<_>>>()?;
45 model.run(&inputs)
46 })();
47
48 match result {
49 Ok(outputs) => {
50 println!("MNN {}", Runtime::native_version());
51 for tensor in outputs {
52 let first = tensor.data().first().copied().unwrap_or_default();
53 println!(
54 "{} {:?}: {} values, first={first}",
55 tensor.name(),
56 tensor.shape(),
57 tensor.data().len()
58 );
59 }
60 ExitCode::SUCCESS
61 }
62 Err(error) => {
63 eprintln!("error: {error}");
64 ExitCode::FAILURE
65 }
66 }
67}Sourcepub const fn with_threads(self, threads: usize) -> Self
pub const fn with_threads(self, threads: usize) -> Self
Select the MNN worker-thread count.
Examples found in repository?
23fn main() -> Result<(), Box<dyn std::error::Error>> {
24 let runtime = Runtime::new(RuntimeConfig::new().with_threads(1))?;
25 let model =
26 runtime.load_bytes(include_bytes!("../tests/fixtures/packed_output3.mnn").as_slice())?;
27 let input = Tensor::new("input", vec![1, 3, 1, 2], vec![1.0; 6])?;
28 let borrowed = measure(
29 || {
30 std::hint::black_box(model.run(std::slice::from_ref(&input)).unwrap());
31 },
32 1000,
33 );
34 let mut outputs = model.run(std::slice::from_ref(&input))?;
35 let reused = measure(
36 || {
37 let result = model
38 .run_reusing(vec![input.clone()], std::mem::take(&mut outputs))
39 .unwrap();
40 outputs = std::hint::black_box(result.tensors);
41 },
42 1000,
43 );
44 println!("tiny model, current compilation profile; P50/P95/P99");
45 println!("borrowed: {borrowed:?}; reused outputs: {reused:?}");
46 println!(
47 "Input cloning is included in both loops; run_owned transfers an existing Vec without that clone."
48 );
49 Ok(())
50}Sourcepub const fn with_precision(self, precision: PrecisionMode) -> Self
pub const fn with_precision(self, precision: PrecisionMode) -> Self
Select the precision preference.
Sourcepub const fn with_queue_capacity(self, capacity: usize) -> Self
pub const fn with_queue_capacity(self, capacity: usize) -> Self
Set the maximum number of queued requests (must be positive).
Sourcepub const fn with_gpu_tuning(self, tuning: GpuTuning) -> Self
pub const fn with_gpu_tuning(self, tuning: GpuTuning) -> Self
Select GPU tuning.
Sourcepub const fn with_gpu_memory(self, memory: GpuMemoryMode) -> Self
pub const fn with_gpu_memory(self, memory: GpuMemoryMode) -> Self
Select OpenCL tensor storage.
Sourcepub fn with_gpu_cache_dir(self, directory: impl Into<PathBuf>) -> Self
pub fn with_gpu_cache_dir(self, directory: impl Into<PathBuf>) -> Self
Enable persistent kernel caches for subsequently loaded models.
Sourcepub const fn with_execution(self, execution: ExecutionMode) -> Self
pub const fn with_execution(self, execution: ExecutionMode) -> Self
Select the cross-model execution policy.