mnn-runtime-sys 0.1.0

A publishable native MNN bridge with verified prebuilt binaries
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
//! Narrow Rust boundary for the project-owned MNN C++ bridge.
//!
//! All raw pointers and FFI calls remain in this crate, the sole owner of
//! `links = "mnn"` and process coordination shared by higher-level runtimes.

#![allow(unsafe_code)]

use std::{
    ffi::{CStr, CString},
    path::Path,
    ptr::NonNull,
};

/// The pinned Rust ABI crate version.
pub const MNN_SYS_CRATE_VERSION: &str = env!("CARGO_PKG_VERSION");

/// Shared gate for complete native transactions, including creation and destruction.
/// Advanced users opting out must coordinate their own MNN access.
pub fn execution_gate() -> std::sync::Arc<std::sync::Mutex<()>> {
    static GATE: std::sync::OnceLock<std::sync::Arc<std::sync::Mutex<()>>> =
        std::sync::OnceLock::new();
    GATE.get_or_init(|| std::sync::Arc::new(std::sync::Mutex::new(())))
        .clone()
}

/// Native backend identifier understood by MNN.
#[derive(Debug, Clone, Copy)]
#[repr(i32)]
pub enum Backend {
    /// CPU.
    Cpu = 0,
    /// Metal.
    Metal = 1,
    /// CUDA.
    Cuda = 2,
    /// OpenCL.
    OpenCl = 3,
    /// Core ML / neural-network backend.
    CoreMl = 5,
    /// OpenGL.
    OpenGl = 6,
    /// Vulkan.
    Vulkan = 7,
    /// MNN automatic selection.
    Auto = 4,
}

/// Native engine configuration.
#[derive(Debug, Clone, Copy)]
pub struct Config {
    /// Backend.
    pub backend: Backend,
    /// MNN thread count.
    pub threads: i32,
    /// MNN precision enum value.
    pub precision: i32,
    /// MNN power enum value.
    pub power: i32,
    /// MNN memory enum value.
    pub memory: i32,
    /// GPU tuning/memory bitmask (not a CPU thread count).
    pub gpu_mode: i32,
}

/// Native tensor metadata.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TensorInfo {
    /// Graph tensor name.
    pub name: String,
    /// Signed native shape. Negative dimensions represent dynamic axes.
    pub shape: Vec<i32>,
    /// Whether the public contiguous tensor is channel-last (NHWC).
    pub channel_last: bool,
}

/// Error returned by the native bridge.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NativeError {
    /// Bridge status code, or `None` when engine creation returned null.
    pub status: Option<i32>,
    /// Native diagnostic message.
    pub message: String,
}

impl std::fmt::Display for NativeError {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self.status {
            Some(status) => write!(formatter, "{} (status {status})", self.message),
            None => formatter.write_str(&self.message),
        }
    }
}

impl std::error::Error for NativeError {}

/// RAII wrapper around one interpreter and session.
///
/// This type deliberately does not implement `Send` or `Sync`. The safe runtime
/// keeps it on the model worker that created it.
pub struct Engine {
    pointer: NonNull<ffi::MnnRuntimeEngine>,
}

impl Engine {
    /// Create an engine from a model buffer.
    pub fn new(model: &[u8], config: Config) -> Result<Self, NativeError> {
        Self::new_with_cache_file(model, config, None)
    }

    /// Create an engine with a persistent kernel cache loaded before session creation.
    /// The parent directory must already exist; no cache is used when `None`.
    pub fn new_with_cache_file(
        model: &[u8],
        config: Config,
        cache_file: Option<&Path>,
    ) -> Result<Self, NativeError> {
        let cache_file = cache_file
            .map(|path| {
                let path = path.to_str().ok_or_else(|| NativeError {
                    status: Some(1),
                    message: "cache path must be valid UTF-8".to_owned(),
                })?;
                CString::new(path).map_err(|_| NativeError {
                    status: Some(1),
                    message: "cache path contains a NUL byte".to_owned(),
                })
            })
            .transpose()?;
        let native_config = ffi::MnnRuntimeConfig {
            backend: config.backend as i32,
            threads: config.threads,
            precision: config.precision,
            power: config.power,
            memory: config.memory,
            gpu_mode: config.gpu_mode,
            cache_file: cache_file.as_ref().map_or(std::ptr::null(), |p| p.as_ptr()),
        };
        let pointer = unsafe {
            ffi::mnn_runtime_engine_create(model.as_ptr().cast(), model.len(), &native_config)
        };
        NonNull::new(pointer)
            .map(|pointer| Self { pointer })
            .ok_or_else(|| last_error(None, None))
    }

    /// Actual CPU thread count, or `None` for GPU sessions.
    pub fn effective_threads(&self) -> Option<usize> {
        let value = unsafe { ffi::mnn_runtime_effective_threads(self.pointer.as_ptr()) };
        (value > 0).then_some(value as usize)
    }

    /// Inspect all input tensors.
    pub fn inputs(&self) -> Result<Vec<TensorInfo>, NativeError> {
        self.tensor_infos(true)
    }

    /// Inspect all output tensors.
    pub fn outputs(&self) -> Result<Vec<TensorInfo>, NativeError> {
        self.tensor_infos(false)
    }

    /// Copy a named contiguous `f32` input into MNN.
    pub fn write_input(&mut self, name: &str, data: &[f32]) -> Result<(), NativeError> {
        let index = self
            .inputs()?
            .iter()
            .position(|t| t.name == name)
            .ok_or_else(|| NativeError {
                status: Some(4),
                message: format!("unknown input {name}"),
            })?;
        self.write_input_index(index, data)
    }

    /// Copy into a pre-resolved input slot; an invalid index is rejected by the bridge.
    pub fn write_input_index(&mut self, index: usize, data: &[f32]) -> Result<(), NativeError> {
        let status = unsafe {
            ffi::mnn_runtime_write_input_index_f32(
                self.pointer.as_ptr(),
                index,
                data.as_ptr(),
                data.len(),
            )
        };
        self.check(status)
    }

    /// Resize every input in one transaction. Entries are `(input_index, shape)`.
    /// All entries are checked before changing native tensors; native resize
    /// failures leave the session requiring another successful resize.
    pub fn resize_inputs(&mut self, shapes: &[(usize, &[i32])]) -> Result<(), NativeError> {
        let shapes: Vec<_> = shapes
            .iter()
            .map(|(index, dimensions)| ffi::MnnRuntimeInputShape {
                index: *index,
                dimensions: dimensions.as_ptr(),
                rank: dimensions.len(),
            })
            .collect();
        let status = unsafe {
            ffi::mnn_runtime_resize_inputs(self.pointer.as_ptr(), shapes.as_ptr(), shapes.len())
        };
        self.check(status)
    }

    /// Persist kernel tuning data. A model without a configured cache is a no-op.
    pub fn save_cache(&mut self) -> Result<(), NativeError> {
        let status = unsafe { ffi::mnn_runtime_save_cache(self.pointer.as_ptr()) };
        self.check(status)
    }

    /// Run one inference.
    pub fn run(&mut self) -> Result<(), NativeError> {
        let status = unsafe { ffi::mnn_runtime_run(self.pointer.as_ptr()) };
        self.check(status)
    }

    /// Copy a named contiguous `f32` output from MNN.
    pub fn read_output(&mut self, name: &str, data: &mut [f32]) -> Result<(), NativeError> {
        let index = self
            .outputs()?
            .iter()
            .position(|t| t.name == name)
            .ok_or_else(|| NativeError {
                status: Some(4),
                message: format!("unknown output {name}"),
            })?;
        self.read_output_index(index, data)
    }

    /// Copy from a pre-resolved output slot; an invalid index is rejected by the bridge.
    pub fn read_output_index(&mut self, index: usize, data: &mut [f32]) -> Result<(), NativeError> {
        let status = unsafe {
            ffi::mnn_runtime_read_output_index_f32(
                self.pointer.as_ptr(),
                index,
                data.as_mut_ptr(),
                data.len(),
            )
        };
        self.check(status)
    }

    fn tensor_infos(&self, input: bool) -> Result<Vec<TensorInfo>, NativeError> {
        let input = i32::from(input);
        let count = unsafe { ffi::mnn_runtime_tensor_count(self.pointer.as_ptr(), input) };
        let mut tensors = Vec::with_capacity(count);
        for index in 0..count {
            let name = unsafe { ffi::mnn_runtime_tensor_name(self.pointer.as_ptr(), input, index) };
            if name.is_null() {
                return Err(last_error(Some(self.pointer), Some(1)));
            }
            let name = unsafe { CStr::from_ptr(name) }
                .to_string_lossy()
                .into_owned();
            let rank = unsafe { ffi::mnn_runtime_tensor_rank(self.pointer.as_ptr(), input, index) };
            let mut shape = vec![0; rank];
            let status = unsafe {
                ffi::mnn_runtime_tensor_shape(
                    self.pointer.as_ptr(),
                    input,
                    index,
                    shape.as_mut_ptr(),
                    shape.len(),
                )
            };
            self.check(status)?;
            let channel_last =
                unsafe { ffi::mnn_runtime_tensor_layout(self.pointer.as_ptr(), input, index) == 1 };
            tensors.push(TensorInfo {
                name,
                shape,
                channel_last,
            });
        }
        Ok(tensors)
    }

    fn check(&self, status: i32) -> Result<(), NativeError> {
        if status == ffi::MNN_RUNTIME_OK {
            Ok(())
        } else {
            Err(last_error(Some(self.pointer), Some(status)))
        }
    }
}

impl Drop for Engine {
    fn drop(&mut self) {
        unsafe { ffi::mnn_runtime_engine_destroy(self.pointer.as_ptr()) };
    }
}

/// Linked MNN native version.
pub fn version() -> String {
    let version = unsafe { ffi::mnn_runtime_version() };
    if version.is_null() {
        return "unknown".to_owned();
    }
    unsafe { CStr::from_ptr(version) }
        .to_string_lossy()
        .into_owned()
}

/// Whether the linked MNN library registered a backend.
pub fn backend_available(backend: Backend) -> bool {
    unsafe { ffi::mnn_runtime_backend_available(backend as i32) != 0 }
}

fn last_error(engine: Option<NonNull<ffi::MnnRuntimeEngine>>, status: Option<i32>) -> NativeError {
    let message = unsafe {
        let pointer =
            ffi::mnn_runtime_last_error(engine.map_or(std::ptr::null(), |engine| engine.as_ptr()));
        if pointer.is_null() {
            "unknown native error".to_owned()
        } else {
            CStr::from_ptr(pointer).to_string_lossy().into_owned()
        }
    };
    NativeError { status, message }
}

#[allow(non_camel_case_types, dead_code)]
mod ffi {
    use std::ffi::{c_char, c_void};

    pub const MNN_RUNTIME_OK: i32 = 0;

    #[repr(C)]
    pub struct MnnRuntimeEngine {
        _private: [u8; 0],
    }

    #[repr(C)]
    pub struct MnnRuntimeConfig {
        pub backend: i32,
        pub threads: i32,
        pub precision: i32,
        pub power: i32,
        pub memory: i32,
        pub gpu_mode: i32,
        pub cache_file: *const c_char,
    }

    #[repr(C)]
    pub struct MnnRuntimeInputShape {
        pub index: usize,
        pub dimensions: *const i32,
        pub rank: usize,
    }

    extern "C" {
        pub fn mnn_runtime_resize_inputs(
            engine: *mut MnnRuntimeEngine,
            shapes: *const MnnRuntimeInputShape,
            count: usize,
        ) -> i32;
        pub fn mnn_runtime_save_cache(engine: *mut MnnRuntimeEngine) -> i32;
        pub fn mnn_runtime_version() -> *const c_char;
        pub fn mnn_runtime_backend_available(backend: i32) -> i32;
        pub fn mnn_runtime_engine_create(
            model: *const c_void,
            model_size: usize,
            config: *const MnnRuntimeConfig,
        ) -> *mut MnnRuntimeEngine;
        pub fn mnn_runtime_effective_threads(engine: *const MnnRuntimeEngine) -> i32;
        pub fn mnn_runtime_tensor_layout(
            engine: *const MnnRuntimeEngine,
            input: i32,
            index: usize,
        ) -> i32;
        pub fn mnn_runtime_engine_destroy(engine: *mut MnnRuntimeEngine);
        pub fn mnn_runtime_last_error(engine: *const MnnRuntimeEngine) -> *const c_char;
        pub fn mnn_runtime_tensor_count(engine: *const MnnRuntimeEngine, input: i32) -> usize;
        pub fn mnn_runtime_tensor_name(
            engine: *const MnnRuntimeEngine,
            input: i32,
            index: usize,
        ) -> *const c_char;
        pub fn mnn_runtime_tensor_rank(
            engine: *const MnnRuntimeEngine,
            input: i32,
            index: usize,
        ) -> usize;
        pub fn mnn_runtime_tensor_shape(
            engine: *const MnnRuntimeEngine,
            input: i32,
            index: usize,
            dimensions: *mut i32,
            capacity: usize,
        ) -> i32;
        pub fn mnn_runtime_write_input_index_f32(
            engine: *mut MnnRuntimeEngine,
            index: usize,
            data: *const f32,
            element_count: usize,
        ) -> i32;
        pub fn mnn_runtime_run(engine: *mut MnnRuntimeEngine) -> i32;
        pub fn mnn_runtime_read_output_index_f32(
            engine: *mut MnnRuntimeEngine,
            index: usize,
            data: *mut f32,
            element_count: usize,
        ) -> i32;
    }
}