gigastt-core 2.12.0

Core inference engine for gigastt — GigaAM v3 ONNX Runtime, model management, quantization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

#![allow(dead_code)]

use crate::runtime::{
    error::RuntimeError,
    factory::{Runtime, RuntimeFactory},
};

use super::runtime::CandleRuntime;

/// Which hardware device the Candle backend runs on.
#[derive(Clone, Copy)]
pub enum CandleDevice {
    Cpu,
    Metal,
}

/// Factory that creates a `CandleRuntime` for the selected device.
pub struct CandleFactory {
    device: CandleDevice,
}

impl CandleFactory {
    /// Prefer Metal (Apple Silicon Neural Engine / GPU); fall back to CPU.
    pub fn new() -> Self {
        let device = if candle_core::Device::new_metal(0).is_ok() {
            CandleDevice::Metal
        } else {
            CandleDevice::Cpu
        };
        Self { device }
    }

    /// CPU-only factory.
    pub fn cpu() -> Self {
        Self {
            device: CandleDevice::Cpu,
        }
    }
}

impl RuntimeFactory for CandleFactory {
    fn create(&self, _intra_threads: usize) -> Result<Box<dyn Runtime>, RuntimeError> {
        let runtime = CandleRuntime::new(self.device)?;
        Ok(Box::new(runtime))
    }

    fn cpu_fallback(&self) -> Box<dyn RuntimeFactory> {
        Box::new(CandleFactory::cpu())
    }
}