Skip to main content

mnn_runtime/
model.rs

1//! Thread-confined MNN model implementation.
2
3use std::{
4    collections::HashSet,
5    path::{Path, PathBuf},
6    sync::{
7        atomic::{AtomicBool, Ordering},
8        mpsc, Arc, Mutex,
9    },
10    thread::{self, JoinHandle},
11    time::{Duration, Instant},
12};
13
14use crate::{
15    runtime::RuntimeInner, Backend, Error, MemoryMode, PowerMode, PrecisionMode, Result, Tensor,
16    TensorInfo,
17};
18
19macro_rules! feature_backend {
20    ($feature:literal, $backend:path) => {{
21        #[cfg(feature = $feature)]
22        {
23            Ok($backend)
24        }
25        #[cfg(not(feature = $feature))]
26        {
27            Err(Error::BackendNotCompiled {
28                backend: $feature,
29                feature: $feature,
30            })
31        }
32    }};
33}
34
35/// Immutable model input/output metadata.
36#[derive(Debug, Clone, PartialEq, Eq)]
37pub struct ModelInfo {
38    inputs: Vec<TensorInfo>,
39    outputs: Vec<TensorInfo>,
40    effective_threads: Option<usize>,
41    cache_file: Option<PathBuf>,
42}
43
44impl ModelInfo {
45    /// Actual CPU worker count; GPU sessions return `None`.
46    #[must_use]
47    pub fn effective_threads(&self) -> Option<usize> {
48        self.effective_threads
49    }
50
51    /// Model inputs sorted by graph name.
52    #[must_use]
53    pub fn inputs(&self) -> &[TensorInfo] {
54        &self.inputs
55    }
56
57    /// Model outputs sorted by graph name.
58    #[must_use]
59    pub fn outputs(&self) -> &[TensorInfo] {
60        &self.outputs
61    }
62
63    /// Find input metadata by name.
64    #[must_use]
65    pub fn input(&self, name: &str) -> Option<&TensorInfo> {
66        self.inputs.iter().find(|tensor| tensor.name() == name)
67    }
68
69    /// Find output metadata by name.
70    #[must_use]
71    pub fn output(&self, name: &str) -> Option<&TensorInfo> {
72        self.outputs.iter().find(|tensor| tensor.name() == name)
73    }
74}
75
76/// A loaded, shareable MNN model.
77///
78/// Clones address the same model worker. Calls are executed in submission order.
79#[derive(Debug, Clone)]
80pub struct Model {
81    inner: Arc<ModelInner>,
82}
83
84#[derive(Debug)]
85struct ModelInner {
86    info: ModelInfo,
87    commands: mpsc::SyncSender<Command>,
88    worker: Mutex<Option<JoinHandle<()>>>,
89}
90
91/// Measured wall times for one accepted request.
92#[derive(Debug, Clone, Copy, Default)]
93pub struct RunTimings {
94    /// Queue and cross-model gate wait, excluding caller-side validation/allocation.
95    pub queue_wait: Duration,
96    /// Resizing native tensors and preparing the session for the request.
97    pub resize: Duration,
98    /// Copying inputs into MNN (including layout conversion/upload).
99    pub input_copy: Duration,
100    /// Native session execution (some GPU work can finish during output copy).
101    pub inference: Duration,
102    /// Downloading/converting output values.
103    pub output_copy: Duration,
104}
105
106/// Inference tensors and timing diagnostics.
107#[derive(Debug)]
108pub struct InferenceOutput {
109    /// Outputs in the requested order.
110    pub tensors: Vec<Tensor>,
111    /// Stage timings.
112    pub timings: RunTimings,
113}
114
115/// An accepted request. Dropping it cancels work that has not entered MNN.
116/// Cancellation does not interrupt an active native call.
117#[derive(Debug)]
118pub struct PendingRun {
119    result: mpsc::Receiver<Result<InferenceOutput>>,
120    cancelled: Arc<AtomicBool>,
121}
122
123impl PendingRun {
124    /// Wait for completion.
125    ///
126    /// # Errors
127    /// Returns inference, cancellation, deadline, or worker errors.
128    pub fn wait(self) -> Result<InferenceOutput> {
129        self.result.recv().map_err(|_| Error::WorkerStopped)?
130    }
131
132    /// Wait at most `timeout`; expiry also cancels work that has not started.
133    ///
134    /// # Errors
135    /// Returns `DeadlineExceeded` on timeout, or an inference/worker error.
136    pub fn wait_timeout(self, timeout: Duration) -> Result<InferenceOutput> {
137        self.result
138            .recv_timeout(timeout)
139            .map_err(|error| match error {
140                mpsc::RecvTimeoutError::Timeout => Error::DeadlineExceeded,
141                mpsc::RecvTimeoutError::Disconnected => Error::WorkerStopped,
142            })?
143    }
144
145    /// Request cancellation before native execution starts.
146    pub fn cancel(&self) {
147        self.cancelled.store(true, Ordering::Release);
148    }
149}
150
151impl Drop for PendingRun {
152    fn drop(&mut self) {
153        self.cancel();
154    }
155}
156
157#[derive(Debug)]
158struct RunCommand {
159    inputs: Vec<(usize, Tensor)>,
160    outputs: Vec<(usize, Option<Tensor>)>,
161    reply: mpsc::Sender<Result<InferenceOutput>>,
162    cancelled: Arc<AtomicBool>,
163    deadline: Option<Instant>,
164    submitted: Instant,
165}
166
167impl RunCommand {
168    fn check_live(&self) -> Result<()> {
169        if self.cancelled.load(Ordering::Acquire) {
170            return Err(Error::Cancelled);
171        }
172        if self
173            .deadline
174            .is_some_and(|deadline| Instant::now() >= deadline)
175        {
176            return Err(Error::DeadlineExceeded);
177        }
178        Ok(())
179    }
180}
181
182#[derive(Debug)]
183enum Command {
184    Run(RunCommand),
185    SaveCache(mpsc::Sender<Result<()>>),
186    Shutdown,
187}
188
189impl Model {
190    pub(crate) fn spawn(model: Vec<u8>, runtime: Arc<RuntimeInner>) -> Result<Self> {
191        if model.is_empty() {
192            return Err(Error::InvalidConfig("model bytes are empty".to_owned()));
193        }
194
195        let (commands, receiver) = mpsc::sync_channel(runtime.config.queue_capacity);
196        let (initialized, startup) = mpsc::sync_channel(1);
197        let worker = thread::Builder::new()
198            .name("mnn-model".to_owned())
199            .spawn(move || worker_main(model, &runtime, &receiver, &initialized))
200            .map_err(|error| {
201                Error::InvalidConfig(format!("failed to spawn model worker: {error}"))
202            })?;
203
204        let info = match startup.recv() {
205            Ok(Ok(info)) => info,
206            Ok(Err(error)) => {
207                let _ = worker.join();
208                return Err(error);
209            }
210            Err(_) => {
211                return match worker.join() {
212                    Ok(()) => Err(Error::WorkerStopped),
213                    Err(_) => Err(Error::WorkerPanicked),
214                };
215            }
216        };
217
218        Ok(Self {
219            inner: Arc::new(ModelInner {
220                info,
221                commands,
222                worker: Mutex::new(Some(worker)),
223            }),
224        })
225    }
226
227    /// Model input/output metadata.
228    #[must_use]
229    pub fn info(&self) -> &ModelInfo {
230        &self.inner.info
231    }
232
233    /// Persistent cache path, isolated by model and native configuration.
234    #[must_use]
235    pub fn cache_file(&self) -> Option<&Path> {
236        self.info().cache_file.as_deref()
237    }
238
239    /// Save kernel tuning data after earlier accepted requests have completed.
240    /// Without a cache directory this is a no-op. Shutdown also attempts a save.
241    ///
242    /// # Errors
243    /// Returns native cache I/O or worker errors.
244    pub fn save_cache(&self) -> Result<()> {
245        let (reply, result) = mpsc::channel();
246        self.inner
247            .commands
248            .send(Command::SaveCache(reply))
249            .map_err(|_| Error::WorkerStopped)?;
250        result.recv().map_err(|_| Error::WorkerStopped)?
251    }
252
253    /// Run inference, waiting for queue capacity, and return all graph outputs.
254    /// Concrete graph dimensions must match; unresolved dimensions use the input.
255    ///
256    /// # Errors
257    /// Returns validation, native inference, or worker errors.
258    pub fn run(&self, inputs: &[Tensor]) -> Result<Vec<Tensor>> {
259        self.run_owned(inputs.to_vec())
260    }
261
262    /// Run with concrete input shapes, permitting changes to graph dimensions
263    /// including a fixed batch or width. Rank must match; MNN validates the graph.
264    /// Calls wait for queue capacity. Actual output shapes belong to the returned tensors.
265    ///
266    /// # Errors
267    /// Returns validation, native resize/inference, or worker errors.
268    pub fn run_dynamic(&self, inputs: &[Tensor]) -> Result<Vec<Tensor>> {
269        self.run_dynamic_owned(inputs.to_vec())
270    }
271
272    /// Move inputs into a blocking inference request.
273    ///
274    /// # Errors
275    /// Returns validation, native inference, or worker errors.
276    pub fn run_owned(&self, inputs: Vec<Tensor>) -> Result<Vec<Tensor>> {
277        self.enqueue(inputs, self.all_outputs(), None, false, true)?
278            .wait()
279            .map(|r| r.tensors)
280    }
281
282    /// Move inputs into a blocking request that can resize concrete graph axes.
283    ///
284    /// # Errors
285    /// Returns validation, native resize/inference, or worker errors.
286    pub fn run_dynamic_owned(&self, inputs: Vec<Tensor>) -> Result<Vec<Tensor>> {
287        self.enqueue(inputs, self.all_outputs(), None, true, true)?
288            .wait()
289            .map(|r| r.tensors)
290    }
291
292    /// Run inference and copy only the named graph outputs in the given order.
293    ///
294    /// # Errors
295    /// Returns validation, native inference, or worker errors.
296    pub fn run_for_outputs(&self, inputs: &[Tensor], outputs: &[&str]) -> Result<Vec<Tensor>> {
297        let outputs = outputs
298            .iter()
299            .map(|&name| (name.to_owned(), None))
300            .collect();
301        self.enqueue(inputs.to_vec(), outputs, None, false, true)?
302            .wait()
303            .map(|r| r.tensors)
304    }
305
306    /// Resize inputs and retrieve only the named outputs in the given order.
307    ///
308    /// # Errors
309    /// Returns validation, native resize/inference, or worker errors.
310    pub fn run_dynamic_for_outputs(
311        &self,
312        inputs: &[Tensor],
313        outputs: &[&str],
314    ) -> Result<Vec<Tensor>> {
315        let outputs = outputs
316            .iter()
317            .map(|&name| (name.to_owned(), None))
318            .collect();
319        self.enqueue(inputs.to_vec(), outputs, None, true, true)?
320            .wait()
321            .map(|r| r.tensors)
322    }
323
324    /// Reuse previous output allocations, selected by name, resizing them after
325    /// inference if necessary. Inputs may resize concrete graph axes, as with
326    /// `run_dynamic`. Buffers are consumed even on error; queue admission blocks.
327    ///
328    /// # Errors
329    /// Returns validation, native resize/inference, or worker errors.
330    pub fn run_reusing(
331        &self,
332        inputs: Vec<Tensor>,
333        outputs: Vec<Tensor>,
334    ) -> Result<InferenceOutput> {
335        let outputs = outputs
336            .into_iter()
337            .map(|t| (t.name().to_owned(), Some(t)))
338            .collect();
339        self.enqueue(inputs, outputs, None, true, true)?.wait()
340    }
341
342    /// Submit without waiting for queue capacity. Dropping the returned handle
343    /// cancels queued work. The deadline is checked before entering MNN.
344    ///
345    /// # Errors
346    /// Returns `QueueFull` if not accepted, or validation/deadline errors.
347    pub fn try_submit_owned(
348        &self,
349        inputs: Vec<Tensor>,
350        deadline: Option<Instant>,
351    ) -> Result<PendingRun> {
352        self.enqueue(inputs, self.all_outputs(), deadline, false, false)
353    }
354
355    /// Nonblocking submission permitting concrete graph axes to resize.
356    ///
357    /// # Errors
358    /// Returns `QueueFull` if not accepted, or validation/deadline errors.
359    pub fn try_submit_dynamic_owned(
360        &self,
361        inputs: Vec<Tensor>,
362        deadline: Option<Instant>,
363    ) -> Result<PendingRun> {
364        self.enqueue(inputs, self.all_outputs(), deadline, true, false)
365    }
366
367    fn all_outputs(&self) -> Vec<(String, Option<Tensor>)> {
368        self.info()
369            .outputs()
370            .iter()
371            .map(|t| (t.name().to_owned(), None))
372            .collect()
373    }
374
375    fn enqueue(
376        &self,
377        inputs: Vec<Tensor>,
378        outputs: Vec<(String, Option<Tensor>)>,
379        deadline: Option<Instant>,
380        dynamic: bool,
381        blocking: bool,
382    ) -> Result<PendingRun> {
383        validate_inputs(self.info(), &inputs, dynamic)?;
384        let inputs = inputs
385            .into_iter()
386            .map(|tensor| {
387                let index = self
388                    .info()
389                    .inputs()
390                    .iter()
391                    .position(|t| t.name() == tensor.name())
392                    .expect("validated input");
393                (index, tensor)
394            })
395            .collect();
396        let mut seen = HashSet::new();
397        let outputs = outputs
398            .into_iter()
399            .map(|(name, tensor)| {
400                if !seen.insert(name.clone()) {
401                    return Err(Error::DuplicateTensor {
402                        kind: "output",
403                        name,
404                    });
405                }
406                let index = self
407                    .info()
408                    .outputs()
409                    .iter()
410                    .position(|t| t.name() == name)
411                    .ok_or(Error::UnknownTensor {
412                        kind: "output",
413                        name,
414                    })?;
415                Ok((index, tensor))
416            })
417            .collect::<Result<Vec<_>>>()?;
418        let (reply, result) = mpsc::channel();
419        let cancelled = Arc::new(AtomicBool::new(false));
420        let command = RunCommand {
421            inputs,
422            outputs,
423            reply,
424            cancelled: cancelled.clone(),
425            deadline,
426            submitted: Instant::now(),
427        };
428        command.check_live()?;
429        if blocking {
430            self.inner
431                .commands
432                .send(Command::Run(command))
433                .map_err(|_| Error::WorkerStopped)?;
434        } else {
435            self.inner
436                .commands
437                .try_send(Command::Run(command))
438                .map_err(|error| match error {
439                    mpsc::TrySendError::Full(_) => Error::QueueFull,
440                    mpsc::TrySendError::Disconnected(_) => Error::WorkerStopped,
441                })?;
442        }
443        Ok(PendingRun { result, cancelled })
444    }
445}
446
447impl Drop for ModelInner {
448    fn drop(&mut self) {
449        let _ = self.commands.send(Command::Shutdown);
450        let worker = self.worker.get_mut().ok().and_then(Option::take);
451        if let Some(worker) = worker {
452            let _ = worker.join();
453        }
454    }
455}
456
457fn validate_inputs(info: &ModelInfo, inputs: &[Tensor], dynamic: bool) -> Result<()> {
458    let mut seen = HashSet::with_capacity(inputs.len());
459    for input in inputs {
460        if !seen.insert(input.name()) {
461            return Err(Error::DuplicateTensor {
462                kind: "input",
463                name: input.name().to_owned(),
464            });
465        }
466        let expected = info
467            .input(input.name())
468            .ok_or_else(|| Error::UnknownTensor {
469                kind: "input",
470                name: input.name().to_owned(),
471            })?;
472        if expected.shape().len() != input.shape().len()
473            || (!dynamic
474                && expected
475                    .shape()
476                    .iter()
477                    .zip(input.shape())
478                    .any(|(&e, &a)| e > 0 && usize::try_from(e).ok() != Some(a)))
479        {
480            return Err(Error::ShapeMismatch {
481                name: input.name().to_owned(),
482                expected: expected.shape().to_vec(),
483                actual: input.shape().to_vec(),
484            });
485        }
486    }
487    for expected in info.inputs() {
488        if !seen.contains(expected.name()) {
489            return Err(Error::MissingInput(expected.name().to_owned()));
490        }
491    }
492    Ok(())
493}
494
495fn worker_main(
496    model: Vec<u8>,
497    runtime: &Arc<RuntimeInner>,
498    commands: &mpsc::Receiver<Command>,
499    initialized: &mpsc::SyncSender<Result<ModelInfo>>,
500) {
501    let state = with_gate(runtime, || initialize_model(&model, runtime));
502    // createFromBuffer copied these bytes; keep no model-sized Rust allocation alive.
503    drop(model);
504    let (mut engine, info) = match state {
505        Ok(state) => state,
506        Err(error) => {
507            let _ = initialized.send(Err(error));
508            return;
509        }
510    };
511    if initialized.send(Ok(info.clone())).is_err() {
512        let _ = with_gate(runtime, || {
513            drop(engine);
514            Ok(())
515        });
516        return;
517    }
518
519    while let Ok(command) = commands.recv() {
520        match command {
521            Command::Run(mut request) => {
522                let result = request.check_live().and_then(|()| {
523                    with_gate(runtime, || {
524                        request.check_live()?;
525                        run_model(&mut engine, &mut request)
526                    })
527                });
528                let _ = request.reply.send(result);
529            }
530            Command::SaveCache(reply) => {
531                let result = with_gate(runtime, || {
532                    engine
533                        .save_cache()
534                        .map_err(|e| native_error("save cache", &e))
535                });
536                let _ = reply.send(result);
537            }
538            Command::Shutdown => break,
539        }
540    }
541    // Destruction participates in the same process policy as creation and inference.
542    // Poison recovery is only for cleanup; further inference still returns an error.
543    let _guard = runtime.execution_gate.as_ref().map(|gate| {
544        gate.lock()
545            .unwrap_or_else(std::sync::PoisonError::into_inner)
546    });
547    let _ = engine.save_cache();
548    drop(engine);
549}
550
551fn with_gate<T>(runtime: &RuntimeInner, operation: impl FnOnce() -> Result<T>) -> Result<T> {
552    let _guard = runtime
553        .execution_gate
554        .as_ref()
555        .map(|gate| gate.lock().map_err(|_| Error::ExecutionGatePoisoned))
556        .transpose()?;
557    operation()
558}
559
560fn initialize_model(
561    model: &[u8],
562    runtime: &RuntimeInner,
563) -> Result<(mnn_runtime_sys::Engine, ModelInfo)> {
564    let config = mnn_runtime_sys::Config {
565        backend: mnn_backend(runtime.config.backend)?,
566        threads: i32::try_from(runtime.config.threads)
567            .map_err(|_| Error::InvalidConfig("thread count does not fit in an i32".to_owned()))?,
568        precision: mnn_precision(runtime.config.precision),
569        power: mnn_power(runtime.config.power),
570        memory: mnn_memory(runtime.config.memory),
571        gpu_mode: gpu_mode(&runtime.config)?,
572    };
573    let cache_file = cache_file(model, runtime)?;
574    let engine = mnn_runtime_sys::Engine::new_with_cache_file(model, config, cache_file.as_deref())
575        .map_err(|error| native_error("create engine", &error))?;
576    let effective_threads = engine.effective_threads();
577    if let Some(effective) = effective_threads {
578        if effective != runtime.config.threads {
579            return Err(Error::ThreadCountLimited {
580                requested: runtime.config.threads,
581                effective,
582            });
583        }
584    }
585    let inputs = collect_tensors(
586        engine
587            .inputs()
588            .map_err(|error| native_error("inspect model inputs", &error))?,
589    )?;
590    let outputs = collect_tensors(
591        engine
592            .outputs()
593            .map_err(|error| native_error("inspect model outputs", &error))?,
594    )?;
595    if inputs.is_empty() {
596        return Err(Error::Native {
597            operation: "inspect model",
598            message: "model has no inputs".to_owned(),
599        });
600    }
601    if outputs.is_empty() {
602        return Err(Error::Native {
603            operation: "inspect model",
604            message: "model has no outputs".to_owned(),
605        });
606    }
607
608    Ok((
609        engine,
610        ModelInfo {
611            inputs,
612            outputs,
613            effective_threads,
614            cache_file,
615        },
616    ))
617}
618
619fn collect_tensors(tensors: Vec<mnn_runtime_sys::TensorInfo>) -> Result<Vec<TensorInfo>> {
620    tensors
621        .into_iter()
622        .map(|tensor| tensor_info(tensor.name, &tensor.shape, tensor.channel_last))
623        .collect()
624}
625
626fn tensor_info(name: String, shape: &[i32], channel_last: bool) -> Result<TensorInfo> {
627    TensorInfo::checked(name, shape.to_vec(), channel_last)
628}
629
630fn run_model(
631    engine: &mut mnn_runtime_sys::Engine,
632    request: &mut RunCommand,
633) -> Result<InferenceOutput> {
634    let mut timings = RunTimings {
635        queue_wait: request.submitted.elapsed(),
636        ..RunTimings::default()
637    };
638    let start = Instant::now();
639    let shapes = request
640        .inputs
641        .iter()
642        .map(|(index, tensor)| {
643            let shape = tensor
644                .shape()
645                .iter()
646                .map(|&d| {
647                    i32::try_from(d).map_err(|_| Error::ShapeOverflow {
648                        name: tensor.name().to_owned(),
649                    })
650                })
651                .collect::<Result<Vec<_>>>()?;
652            Ok((*index, shape))
653        })
654        .collect::<Result<Vec<_>>>()?;
655    let shapes = shapes
656        .iter()
657        .map(|(index, shape)| (*index, shape.as_slice()))
658        .collect::<Vec<_>>();
659    engine
660        .resize_inputs(&shapes)
661        .map_err(|e| native_error("resize inputs", &e))?;
662    timings.resize = start.elapsed();
663    let start = Instant::now();
664    for (index, input) in &request.inputs {
665        engine
666            .write_input_index(*index, input.data())
667            .map_err(|e| native_error("copy input tensor", &e))?;
668    }
669    timings.input_copy = start.elapsed();
670    let start = Instant::now();
671    engine.run().map_err(|e| native_error("run session", &e))?;
672    timings.inference = start.elapsed();
673    let start = Instant::now();
674    let metadata = collect_tensors(
675        engine
676            .outputs()
677            .map_err(|e| native_error("inspect outputs after inference", &e))?,
678    )?;
679    let mut tensors = Vec::with_capacity(request.outputs.len());
680    for (index, buffer) in &mut request.outputs {
681        let info = &metadata[*index];
682        let shape = info.concrete_shape()?;
683        let mut output = if let Some(mut output) = buffer.take() {
684            output.resize_output(shape)?;
685            output
686        } else {
687            let count = info.element_count().ok_or_else(|| Error::UnresolvedShape {
688                name: info.name().to_owned(),
689            })?;
690            Tensor::new(info.name(), shape, vec![0.0; count])?
691        };
692        engine
693            .read_output_index(*index, output.data_mut())
694            .map_err(|e| native_error("copy output tensor", &e))?;
695        tensors.push(output);
696    }
697    timings.output_copy = start.elapsed();
698    Ok(InferenceOutput { tensors, timings })
699}
700
701fn cache_file(model: &[u8], runtime: &RuntimeInner) -> Result<Option<PathBuf>> {
702    use sha2::{Digest, Sha256};
703    let Some(directory) = &runtime.config.gpu_cache_dir else {
704        return Ok(None);
705    };
706    std::fs::create_dir_all(directory).map_err(|source| Error::CacheDirectory {
707        path: directory.clone(),
708        source,
709    })?;
710    let mut hash = Sha256::new();
711    hash.update(model);
712    // Exclude the directory/queue policy, which cannot affect compiled kernels.
713    let config = &runtime.config;
714    hash.update(format!(
715        "mnn-runtime-cache-v1:{}:{}:{}:{:?}:{:?}:{:?}:{}",
716        mnn_runtime_sys::version(),
717        config.backend.as_str(),
718        config.threads,
719        config.precision,
720        config.power,
721        config.memory,
722        gpu_mode(config)?
723    ));
724    Ok(Some(
725        directory.join(format!("{:x}.mnncache", hash.finalize())),
726    ))
727}
728
729pub(crate) fn gpu_mode(config: &crate::RuntimeConfig) -> Result<i32> {
730    use crate::{GpuMemoryMode, GpuTuning};
731    if !matches!(config.backend, Backend::OpenCl | Backend::Vulkan) {
732        return Ok(0);
733    }
734    let tuning = match (config.backend, config.gpu_tuning) {
735        (Backend::OpenCl, GpuTuning::Auto | GpuTuning::Fast) => 1 << 4,
736        (Backend::OpenCl, GpuTuning::Normal) => 1 << 3,
737        (_, GpuTuning::None | GpuTuning::Auto) => 1,
738        (_, GpuTuning::Wide) => 1 << 2,
739        (_, GpuTuning::Heavy) => 1 << 1,
740        _ => {
741            return Err(Error::InvalidConfig(
742                "Fast/Normal tuning are supported only by OpenCL".to_owned(),
743            ));
744        }
745    };
746    let memory = if config.backend == Backend::OpenCl {
747        match config.gpu_memory {
748            GpuMemoryMode::Buffer => 1 << 6,
749            GpuMemoryMode::Image => 1 << 7,
750            GpuMemoryMode::Auto => 0,
751        }
752    } else {
753        0
754    };
755    Ok(tuning | memory)
756}
757
758pub(crate) fn validate_backend(backend: Backend) -> Result<()> {
759    let native = mnn_backend(backend)?;
760    if backend != Backend::Auto && !mnn_runtime_sys::backend_available(native) {
761        return Err(Error::BackendUnavailable(backend.as_str()));
762    }
763    Ok(())
764}
765
766fn mnn_backend(backend: Backend) -> Result<mnn_runtime_sys::Backend> {
767    match backend {
768        Backend::Auto => Ok(mnn_runtime_sys::Backend::Auto),
769        Backend::Cpu => Ok(mnn_runtime_sys::Backend::Cpu),
770        Backend::Metal => feature_backend!("metal", mnn_runtime_sys::Backend::Metal),
771        Backend::CoreMl => feature_backend!("coreml", mnn_runtime_sys::Backend::CoreMl),
772        Backend::OpenCl => feature_backend!("opencl", mnn_runtime_sys::Backend::OpenCl),
773        Backend::OpenGl => feature_backend!("opengl", mnn_runtime_sys::Backend::OpenGl),
774        Backend::Vulkan => feature_backend!("vulkan", mnn_runtime_sys::Backend::Vulkan),
775        Backend::Cuda => feature_backend!("cuda", mnn_runtime_sys::Backend::Cuda),
776    }
777}
778
779const fn mnn_precision(mode: PrecisionMode) -> i32 {
780    match mode {
781        PrecisionMode::Normal => 0,
782        PrecisionMode::High => 1,
783        PrecisionMode::Low => 2,
784        PrecisionMode::LowBf16 => 3,
785    }
786}
787
788const fn mnn_power(mode: PowerMode) -> i32 {
789    match mode {
790        PowerMode::Normal => 0,
791        PowerMode::High => 1,
792        PowerMode::Low => 2,
793    }
794}
795
796const fn mnn_memory(mode: MemoryMode) -> i32 {
797    match mode {
798        MemoryMode::Normal => 0,
799        MemoryMode::High => 1,
800        MemoryMode::Low => 2,
801    }
802}
803
804fn native_error(operation: &'static str, error: &mnn_runtime_sys::NativeError) -> Error {
805    Error::Native {
806        operation,
807        message: error.to_string(),
808    }
809}
810
811#[cfg(test)]
812mod tests {
813    use super::*;
814
815    fn two_input_model() -> ModelInfo {
816        ModelInfo {
817            inputs: vec![
818                TensorInfo::new("left_eye".to_owned(), vec![1, 3, 60, 60]),
819                TensorInfo::new("right_eye".to_owned(), vec![1, 3, 60, 60]),
820            ],
821            outputs: vec![TensorInfo::new("gaze_vector".to_owned(), vec![1, 3])],
822            effective_threads: Some(4),
823            cache_file: None,
824        }
825    }
826
827    fn input(name: &str) -> Tensor {
828        Tensor::new(name, vec![1, 3, 60, 60], vec![0.0; 10_800]).expect("valid tensor")
829    }
830
831    #[test]
832    fn named_inputs_may_arrive_in_any_order() {
833        let inputs = [input("right_eye"), input("left_eye")];
834        validate_inputs(&two_input_model(), &inputs, false).expect("both named inputs are present");
835    }
836
837    #[test]
838    fn missing_named_input_is_rejected() {
839        let error = validate_inputs(&two_input_model(), &[input("left_eye")], false)
840            .expect_err("right eye is required");
841        assert!(matches!(error, Error::MissingInput(name) if name == "right_eye"));
842    }
843
844    #[test]
845    fn duplicate_named_input_is_rejected() {
846        let inputs = [input("left_eye"), input("left_eye"), input("right_eye")];
847        let error = validate_inputs(&two_input_model(), &inputs, false)
848            .expect_err("duplicate left eye must be rejected");
849        assert!(matches!(
850            error,
851            Error::DuplicateTensor { kind: "input", .. }
852        ));
853    }
854}