1#![allow(unsafe_code)]
7
8use std::{
9 ffi::{CStr, CString},
10 path::Path,
11 ptr::NonNull,
12};
13
14pub const MNN_SYS_CRATE_VERSION: &str = env!("CARGO_PKG_VERSION");
16
17pub fn execution_gate() -> std::sync::Arc<std::sync::Mutex<()>> {
20 static GATE: std::sync::OnceLock<std::sync::Arc<std::sync::Mutex<()>>> =
21 std::sync::OnceLock::new();
22 GATE.get_or_init(|| std::sync::Arc::new(std::sync::Mutex::new(())))
23 .clone()
24}
25
26#[derive(Debug, Clone, Copy)]
28#[repr(i32)]
29pub enum Backend {
30 Cpu = 0,
32 Metal = 1,
34 Cuda = 2,
36 OpenCl = 3,
38 CoreMl = 5,
40 OpenGl = 6,
42 Vulkan = 7,
44 Auto = 4,
46}
47
48#[derive(Debug, Clone, Copy)]
50pub struct Config {
51 pub backend: Backend,
53 pub threads: i32,
55 pub precision: i32,
57 pub power: i32,
59 pub memory: i32,
61 pub gpu_mode: i32,
63}
64
65#[derive(Debug, Clone, PartialEq, Eq)]
67pub struct TensorInfo {
68 pub name: String,
70 pub shape: Vec<i32>,
72 pub channel_last: bool,
74}
75
76#[derive(Debug, Clone, PartialEq, Eq)]
78pub struct NativeError {
79 pub status: Option<i32>,
81 pub message: String,
83}
84
85impl std::fmt::Display for NativeError {
86 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87 match self.status {
88 Some(status) => write!(formatter, "{} (status {status})", self.message),
89 None => formatter.write_str(&self.message),
90 }
91 }
92}
93
94impl std::error::Error for NativeError {}
95
96pub struct Engine {
101 pointer: NonNull<ffi::MnnRuntimeEngine>,
102}
103
104impl Engine {
105 pub fn new(model: &[u8], config: Config) -> Result<Self, NativeError> {
107 Self::new_with_cache_file(model, config, None)
108 }
109
110 pub fn new_with_cache_file(
113 model: &[u8],
114 config: Config,
115 cache_file: Option<&Path>,
116 ) -> Result<Self, NativeError> {
117 let cache_file = cache_file
118 .map(|path| {
119 let path = path.to_str().ok_or_else(|| NativeError {
120 status: Some(1),
121 message: "cache path must be valid UTF-8".to_owned(),
122 })?;
123 CString::new(path).map_err(|_| NativeError {
124 status: Some(1),
125 message: "cache path contains a NUL byte".to_owned(),
126 })
127 })
128 .transpose()?;
129 let native_config = ffi::MnnRuntimeConfig {
130 backend: config.backend as i32,
131 threads: config.threads,
132 precision: config.precision,
133 power: config.power,
134 memory: config.memory,
135 gpu_mode: config.gpu_mode,
136 cache_file: cache_file.as_ref().map_or(std::ptr::null(), |p| p.as_ptr()),
137 };
138 let pointer = unsafe {
139 ffi::mnn_runtime_engine_create(model.as_ptr().cast(), model.len(), &native_config)
140 };
141 NonNull::new(pointer)
142 .map(|pointer| Self { pointer })
143 .ok_or_else(|| last_error(None, None))
144 }
145
146 pub fn effective_threads(&self) -> Option<usize> {
148 let value = unsafe { ffi::mnn_runtime_effective_threads(self.pointer.as_ptr()) };
149 (value > 0).then_some(value as usize)
150 }
151
152 pub fn inputs(&self) -> Result<Vec<TensorInfo>, NativeError> {
154 self.tensor_infos(true)
155 }
156
157 pub fn outputs(&self) -> Result<Vec<TensorInfo>, NativeError> {
159 self.tensor_infos(false)
160 }
161
162 pub fn write_input(&mut self, name: &str, data: &[f32]) -> Result<(), NativeError> {
164 let index = self
165 .inputs()?
166 .iter()
167 .position(|t| t.name == name)
168 .ok_or_else(|| NativeError {
169 status: Some(4),
170 message: format!("unknown input {name}"),
171 })?;
172 self.write_input_index(index, data)
173 }
174
175 pub fn write_input_index(&mut self, index: usize, data: &[f32]) -> Result<(), NativeError> {
177 let status = unsafe {
178 ffi::mnn_runtime_write_input_index_f32(
179 self.pointer.as_ptr(),
180 index,
181 data.as_ptr(),
182 data.len(),
183 )
184 };
185 self.check(status)
186 }
187
188 pub fn resize_inputs(&mut self, shapes: &[(usize, &[i32])]) -> Result<(), NativeError> {
192 let shapes: Vec<_> = shapes
193 .iter()
194 .map(|(index, dimensions)| ffi::MnnRuntimeInputShape {
195 index: *index,
196 dimensions: dimensions.as_ptr(),
197 rank: dimensions.len(),
198 })
199 .collect();
200 let status = unsafe {
201 ffi::mnn_runtime_resize_inputs(self.pointer.as_ptr(), shapes.as_ptr(), shapes.len())
202 };
203 self.check(status)
204 }
205
206 pub fn save_cache(&mut self) -> Result<(), NativeError> {
208 let status = unsafe { ffi::mnn_runtime_save_cache(self.pointer.as_ptr()) };
209 self.check(status)
210 }
211
212 pub fn run(&mut self) -> Result<(), NativeError> {
214 let status = unsafe { ffi::mnn_runtime_run(self.pointer.as_ptr()) };
215 self.check(status)
216 }
217
218 pub fn read_output(&mut self, name: &str, data: &mut [f32]) -> Result<(), NativeError> {
220 let index = self
221 .outputs()?
222 .iter()
223 .position(|t| t.name == name)
224 .ok_or_else(|| NativeError {
225 status: Some(4),
226 message: format!("unknown output {name}"),
227 })?;
228 self.read_output_index(index, data)
229 }
230
231 pub fn read_output_index(&mut self, index: usize, data: &mut [f32]) -> Result<(), NativeError> {
233 let status = unsafe {
234 ffi::mnn_runtime_read_output_index_f32(
235 self.pointer.as_ptr(),
236 index,
237 data.as_mut_ptr(),
238 data.len(),
239 )
240 };
241 self.check(status)
242 }
243
244 fn tensor_infos(&self, input: bool) -> Result<Vec<TensorInfo>, NativeError> {
245 let input = i32::from(input);
246 let count = unsafe { ffi::mnn_runtime_tensor_count(self.pointer.as_ptr(), input) };
247 let mut tensors = Vec::with_capacity(count);
248 for index in 0..count {
249 let name = unsafe { ffi::mnn_runtime_tensor_name(self.pointer.as_ptr(), input, index) };
250 if name.is_null() {
251 return Err(last_error(Some(self.pointer), Some(1)));
252 }
253 let name = unsafe { CStr::from_ptr(name) }
254 .to_string_lossy()
255 .into_owned();
256 let rank = unsafe { ffi::mnn_runtime_tensor_rank(self.pointer.as_ptr(), input, index) };
257 let mut shape = vec![0; rank];
258 let status = unsafe {
259 ffi::mnn_runtime_tensor_shape(
260 self.pointer.as_ptr(),
261 input,
262 index,
263 shape.as_mut_ptr(),
264 shape.len(),
265 )
266 };
267 self.check(status)?;
268 let channel_last =
269 unsafe { ffi::mnn_runtime_tensor_layout(self.pointer.as_ptr(), input, index) == 1 };
270 tensors.push(TensorInfo {
271 name,
272 shape,
273 channel_last,
274 });
275 }
276 Ok(tensors)
277 }
278
279 fn check(&self, status: i32) -> Result<(), NativeError> {
280 if status == ffi::MNN_RUNTIME_OK {
281 Ok(())
282 } else {
283 Err(last_error(Some(self.pointer), Some(status)))
284 }
285 }
286}
287
288impl Drop for Engine {
289 fn drop(&mut self) {
290 unsafe { ffi::mnn_runtime_engine_destroy(self.pointer.as_ptr()) };
291 }
292}
293
294pub fn version() -> String {
296 let version = unsafe { ffi::mnn_runtime_version() };
297 if version.is_null() {
298 return "unknown".to_owned();
299 }
300 unsafe { CStr::from_ptr(version) }
301 .to_string_lossy()
302 .into_owned()
303}
304
305pub fn backend_available(backend: Backend) -> bool {
307 unsafe { ffi::mnn_runtime_backend_available(backend as i32) != 0 }
308}
309
310fn last_error(engine: Option<NonNull<ffi::MnnRuntimeEngine>>, status: Option<i32>) -> NativeError {
311 let message = unsafe {
312 let pointer =
313 ffi::mnn_runtime_last_error(engine.map_or(std::ptr::null(), |engine| engine.as_ptr()));
314 if pointer.is_null() {
315 "unknown native error".to_owned()
316 } else {
317 CStr::from_ptr(pointer).to_string_lossy().into_owned()
318 }
319 };
320 NativeError { status, message }
321}
322
323#[allow(non_camel_case_types, dead_code)]
324mod ffi {
325 use std::ffi::{c_char, c_void};
326
327 pub const MNN_RUNTIME_OK: i32 = 0;
328
329 #[repr(C)]
330 pub struct MnnRuntimeEngine {
331 _private: [u8; 0],
332 }
333
334 #[repr(C)]
335 pub struct MnnRuntimeConfig {
336 pub backend: i32,
337 pub threads: i32,
338 pub precision: i32,
339 pub power: i32,
340 pub memory: i32,
341 pub gpu_mode: i32,
342 pub cache_file: *const c_char,
343 }
344
345 #[repr(C)]
346 pub struct MnnRuntimeInputShape {
347 pub index: usize,
348 pub dimensions: *const i32,
349 pub rank: usize,
350 }
351
352 extern "C" {
353 pub fn mnn_runtime_resize_inputs(
354 engine: *mut MnnRuntimeEngine,
355 shapes: *const MnnRuntimeInputShape,
356 count: usize,
357 ) -> i32;
358 pub fn mnn_runtime_save_cache(engine: *mut MnnRuntimeEngine) -> i32;
359 pub fn mnn_runtime_version() -> *const c_char;
360 pub fn mnn_runtime_backend_available(backend: i32) -> i32;
361 pub fn mnn_runtime_engine_create(
362 model: *const c_void,
363 model_size: usize,
364 config: *const MnnRuntimeConfig,
365 ) -> *mut MnnRuntimeEngine;
366 pub fn mnn_runtime_effective_threads(engine: *const MnnRuntimeEngine) -> i32;
367 pub fn mnn_runtime_tensor_layout(
368 engine: *const MnnRuntimeEngine,
369 input: i32,
370 index: usize,
371 ) -> i32;
372 pub fn mnn_runtime_engine_destroy(engine: *mut MnnRuntimeEngine);
373 pub fn mnn_runtime_last_error(engine: *const MnnRuntimeEngine) -> *const c_char;
374 pub fn mnn_runtime_tensor_count(engine: *const MnnRuntimeEngine, input: i32) -> usize;
375 pub fn mnn_runtime_tensor_name(
376 engine: *const MnnRuntimeEngine,
377 input: i32,
378 index: usize,
379 ) -> *const c_char;
380 pub fn mnn_runtime_tensor_rank(
381 engine: *const MnnRuntimeEngine,
382 input: i32,
383 index: usize,
384 ) -> usize;
385 pub fn mnn_runtime_tensor_shape(
386 engine: *const MnnRuntimeEngine,
387 input: i32,
388 index: usize,
389 dimensions: *mut i32,
390 capacity: usize,
391 ) -> i32;
392 pub fn mnn_runtime_write_input_index_f32(
393 engine: *mut MnnRuntimeEngine,
394 index: usize,
395 data: *const f32,
396 element_count: usize,
397 ) -> i32;
398 pub fn mnn_runtime_run(engine: *mut MnnRuntimeEngine) -> i32;
399 pub fn mnn_runtime_read_output_index_f32(
400 engine: *mut MnnRuntimeEngine,
401 index: usize,
402 data: *mut f32,
403 element_count: usize,
404 ) -> i32;
405 }
406}