Skip to main content

mnn_runtime/
config.rs

1//! Runtime configuration.
2
3/// MNN execution backend.
4#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
5#[non_exhaustive]
6pub enum Backend {
7    /// Let MNN select a backend.
8    Auto,
9    /// CPU inference.
10    #[default]
11    Cpu,
12    /// Apple Metal.
13    Metal,
14    /// Apple Core ML.
15    CoreMl,
16    /// `OpenCL`.
17    OpenCl,
18    /// OpenGL.
19    OpenGl,
20    /// Vulkan.
21    Vulkan,
22    /// NVIDIA CUDA.
23    Cuda,
24}
25
26impl Backend {
27    /// Stable lowercase backend name.
28    #[must_use]
29    pub const fn as_str(self) -> &'static str {
30        match self {
31            Self::Auto => "auto",
32            Self::Cpu => "cpu",
33            Self::Metal => "metal",
34            Self::CoreMl => "coreml",
35            Self::OpenCl => "opencl",
36            Self::OpenGl => "opengl",
37            Self::Vulkan => "vulkan",
38            Self::Cuda => "cuda",
39        }
40    }
41}
42
43/// Precision preference passed to MNN.
44#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
45pub enum PrecisionMode {
46    /// Normal precision.
47    #[default]
48    Normal,
49    /// Prefer accuracy.
50    High,
51    /// Prefer throughput and memory reduction.
52    Low,
53    /// Prefer BF16 where supported.
54    LowBf16,
55}
56
57/// Power preference passed to MNN.
58#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
59pub enum PowerMode {
60    /// Prefer lower power usage.
61    Low,
62    /// Balanced power behavior.
63    #[default]
64    Normal,
65    /// Prefer performance.
66    High,
67}
68
69/// Backend memory preference passed to MNN.
70#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
71pub enum MemoryMode {
72    /// Prefer lower memory usage.
73    Low,
74    /// Balanced memory behavior.
75    #[default]
76    Normal,
77    /// Prefer performance even when it uses more memory.
78    High,
79}
80
81/// Coordination policy for multiple models in one process.
82#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
83pub enum ExecutionMode {
84    /// Serialize inference through one process-wide gate.
85    ///
86    /// This is the conservative default for applications combining OCR, gaze,
87    /// and other models backed by the same MNN thread pool.
88    #[default]
89    Serialized,
90    /// Allow model worker threads to enter MNN concurrently.
91    Parallel,
92}
93
94/// GPU kernel search policy. Fast/Normal are `OpenCL`-only.
95#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
96pub enum GpuTuning {
97    /// Fast for `OpenCL`, no tuning for Vulkan.
98    #[default]
99    Auto,
100    /// Disable tuning.
101    None,
102    /// Search a small `OpenCL` candidate set.
103    Fast,
104    /// Search a medium `OpenCL` candidate set.
105    Normal,
106    /// Search a wide candidate set (slower first inference).
107    Wide,
108    /// Exhaustive tuning (slowest initialization).
109    Heavy,
110}
111
112/// `OpenCL` tensor storage; ignored by other backends.
113#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
114pub enum GpuMemoryMode {
115    /// Avoid device image-size limits for wide tensors.
116    #[default]
117    Buffer,
118    /// Use GPU images.
119    Image,
120    /// Let MNN choose.
121    Auto,
122}
123
124/// Configuration used for subsequently loaded models.
125#[derive(Debug, Clone, PartialEq, Eq)]
126pub struct RuntimeConfig {
127    /// Execution backend.
128    pub backend: Backend,
129    /// Number of MNN worker threads. Must be greater than zero.
130    pub threads: usize,
131    /// Precision preference.
132    pub precision: PrecisionMode,
133    /// Power preference.
134    pub power: PowerMode,
135    /// Memory preference.
136    pub memory: MemoryMode,
137    /// Cross-model execution policy.
138    pub execution: ExecutionMode,
139    /// Maximum queued requests per model, excluding the active request.
140    pub queue_capacity: usize,
141    /// GPU tuning policy.
142    pub gpu_tuning: GpuTuning,
143    /// `OpenCL` storage policy.
144    pub gpu_memory: GpuMemoryMode,
145    /// Directory for caches isolated by model, native version and configuration.
146    /// Cache writes occur on `Model::save_cache` and worker shutdown.
147    pub gpu_cache_dir: Option<std::path::PathBuf>,
148}
149
150impl Default for RuntimeConfig {
151    fn default() -> Self {
152        Self {
153            backend: Backend::Cpu,
154            threads: 4,
155            precision: PrecisionMode::Normal,
156            power: PowerMode::Normal,
157            memory: MemoryMode::Normal,
158            execution: ExecutionMode::Serialized,
159            queue_capacity: 2,
160            gpu_tuning: GpuTuning::Auto,
161            gpu_memory: GpuMemoryMode::Buffer,
162            gpu_cache_dir: None,
163        }
164    }
165}
166
167impl RuntimeConfig {
168    /// Start with the default CPU configuration.
169    #[must_use]
170    pub fn new() -> Self {
171        Self::default()
172    }
173
174    /// Select a backend.
175    #[must_use]
176    pub const fn with_backend(mut self, backend: Backend) -> Self {
177        self.backend = backend;
178        self
179    }
180
181    /// Select the MNN worker-thread count.
182    #[must_use]
183    pub const fn with_threads(mut self, threads: usize) -> Self {
184        self.threads = threads;
185        self
186    }
187
188    /// Select the precision preference.
189    #[must_use]
190    pub const fn with_precision(mut self, precision: PrecisionMode) -> Self {
191        self.precision = precision;
192        self
193    }
194
195    /// Set the maximum number of queued requests (must be positive).
196    #[must_use]
197    pub const fn with_queue_capacity(mut self, capacity: usize) -> Self {
198        self.queue_capacity = capacity;
199        self
200    }
201
202    /// Select GPU tuning.
203    #[must_use]
204    pub const fn with_gpu_tuning(mut self, tuning: GpuTuning) -> Self {
205        self.gpu_tuning = tuning;
206        self
207    }
208
209    /// Select `OpenCL` tensor storage.
210    #[must_use]
211    pub const fn with_gpu_memory(mut self, memory: GpuMemoryMode) -> Self {
212        self.gpu_memory = memory;
213        self
214    }
215
216    /// Enable persistent kernel caches for subsequently loaded models.
217    #[must_use]
218    pub fn with_gpu_cache_dir(mut self, directory: impl Into<std::path::PathBuf>) -> Self {
219        self.gpu_cache_dir = Some(directory.into());
220        self
221    }
222
223    /// Select the cross-model execution policy.
224    #[must_use]
225    pub const fn with_execution(mut self, execution: ExecutionMode) -> Self {
226        self.execution = execution;
227        self
228    }
229}