Skip to main content

ml_runtime_onnx_backend/
lib.rs

1use async_trait::async_trait;
2use futures_util::stream;
3mod laya;
4use laya::LayaModel;
5
6use ml_runtime_backend::{
7    Backend, BackendCapabilities, BackendCapability, DecisionModel, DecisionModelProvider,
8};
9use ml_runtime_common::{BoxStream, CancellationToken, RuntimeError, RuntimeResult};
10use ml_runtime_inference::{
11    DecisionModelCapabilities, DeviceKind, ExecutionMetadata, InferenceChunk, InferenceRequest,
12    InferenceResult, Input, ModelArchitecture, Output, Tensor,
13};
14use ml_runtime_model::{ModelFormat, ModelHandle, ModelLocation, ModelSpec};
15use ort::{session::Session, value::Tensor as OrtTensor};
16use std::{
17    path::Path,
18    sync::{Arc, Mutex},
19};
20
21#[derive(Clone, Debug, Default)]
22pub struct OnnxBackend;
23
24struct OnnxLoadedModel {
25    spec: ModelSpec,
26    session: Mutex<Session>,
27}
28
29impl OnnxBackend {
30    pub fn descriptor() -> BackendCapability {
31        BackendCapability::from_parts("onnx", Self.capabilities())
32    }
33
34    fn load_session(model: &ModelSpec) -> RuntimeResult<Session> {
35        let ModelLocation::Path(path) = &model.location else {
36            return Err(RuntimeError::InvalidRequest {
37                reason: "ONNX models require a filesystem artifact".to_owned(),
38            });
39        };
40        Session::builder()
41            .and_then(|builder| builder.commit_from_file(path))
42            .map_err(|error| RuntimeError::execution("load", format!("load ONNX model: {error}")))
43    }
44
45    pub fn validate_laya(artifact: &Path) -> RuntimeResult<String> {
46        laya::LayaModel::validate(artifact)
47    }
48
49    /// Execution facts for Laya's ONNX representation. This is separate from
50    /// the generic ONNX tensor backend because architecture is a model fact.
51    pub fn laya_decision_capabilities() -> DecisionModelCapabilities {
52        DecisionModelCapabilities {
53            backend: "onnx".to_owned(),
54            device: DeviceKind::Cpu,
55            model_architecture: ModelArchitecture::StructuredDecision,
56            supports_batching: true,
57            max_batch_size: Some(16),
58            batch_size: None,
59            supports_async: false,
60            supports_cancellation: false,
61            supports_parallel_execution: false,
62            recommended_parallelism: Some(1),
63            supports_structured_decisions: true,
64        }
65    }
66}
67
68impl DecisionModelProvider for OnnxBackend {
69    fn name(&self) -> &str {
70        "onnx"
71    }
72
73    fn supports_artifact(&self, artifact: &Path) -> bool {
74        artifact.join("laya.onnx").is_file()
75            && artifact.join("laya_config.json").is_file()
76            && artifact.join("tokenizer/tokenizer.json").is_file()
77    }
78
79    fn load_decision_model(&self, artifact: &Path) -> RuntimeResult<Box<dyn DecisionModel>> {
80        LayaModel::load(artifact).map(|model| Box::new(model) as Box<dyn DecisionModel>)
81    }
82}
83
84#[async_trait]
85impl Backend for OnnxBackend {
86    fn name(&self) -> &str {
87        "onnx"
88    }
89
90    fn capabilities(&self) -> BackendCapabilities {
91        BackendCapabilities {
92            available: true,
93            local: true,
94            streaming: true,
95            cancellation: true,
96            structured_output: false,
97            batching: true,
98            max_batch_size: Some(16),
99            supported_batch_shapes: Vec::new(),
100            supported_formats: vec![ModelFormat::Onnx],
101            accelerators: Vec::new(),
102            hardware: Some("cpu".to_owned()),
103            notes: vec!["ONNX Runtime CPU execution".to_owned()],
104        }
105    }
106
107    fn supports(&self, model: &ModelSpec) -> bool {
108        matches!(model.format, ModelFormat::Onnx)
109    }
110
111    async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle> {
112        let session = Self::load_session(model)?;
113        Ok(ModelHandle::new(
114            model.id.clone(),
115            Backend::name(self),
116            model.format.clone(),
117            Arc::new(OnnxLoadedModel {
118                spec: model.clone(),
119                session: Mutex::new(session),
120            }),
121        ))
122    }
123
124    async fn infer(
125        &self,
126        model: &ModelHandle,
127        request: &InferenceRequest,
128        cancellation: Option<CancellationToken>,
129    ) -> RuntimeResult<InferenceResult> {
130        if cancellation
131            .as_ref()
132            .is_some_and(CancellationToken::is_cancelled)
133        {
134            return Err(RuntimeError::Cancelled);
135        }
136        let loaded = model.state::<OnnxLoadedModel>().ok_or_else(|| {
137            RuntimeError::execution("infer", "model handle does not belong to the ONNX backend")
138        })?;
139        let Input::Tensor(input) = &request.input else {
140            return Err(RuntimeError::UnsupportedInput {
141                reason: "ONNX backend currently accepts tensor input".to_owned(),
142            });
143        };
144        if input.shape.iter().product::<usize>() != input.values.len() {
145            return Err(RuntimeError::UnsupportedInput {
146                reason: "tensor shape does not match tensor values".to_owned(),
147            });
148        }
149        let mut session = loaded
150            .session
151            .lock()
152            .map_err(|_| RuntimeError::execution("infer", "ONNX session lock poisoned"))?;
153        let input_name = session
154            .inputs
155            .first()
156            .map(|input| input.name.to_owned())
157            .ok_or_else(|| RuntimeError::execution("infer", "ONNX model has no inputs"))?;
158        let tensor = OrtTensor::from_array((input.shape.clone(), input.values.clone())).map_err(
159            |error| RuntimeError::execution("infer", format!("create ONNX input: {error}")),
160        )?;
161        let outputs = session
162            .run(ort::inputs![input_name => tensor])
163            .map_err(|error| {
164                RuntimeError::execution("infer", format!("execute ONNX model: {error}"))
165            })?;
166        if outputs.len() == 0 {
167            return Err(RuntimeError::execution(
168                "infer",
169                "ONNX model produced no outputs",
170            ));
171        }
172        let (_, values) = outputs[0].try_extract_tensor::<f32>().map_err(|error| {
173            RuntimeError::execution("infer", format!("read ONNX output: {error}"))
174        })?;
175        let shape = outputs[0]
176            .shape()
177            .iter()
178            .map(|dimension| usize::try_from(*dimension).unwrap_or(0))
179            .collect();
180        Ok(InferenceResult {
181            output: Output::Tensor(Tensor {
182                shape,
183                values: values.to_vec(),
184            }),
185            metadata: ExecutionMetadata {
186                model: loaded.spec.id.clone(),
187                provider: "local".to_owned(),
188                backend: Backend::name(self).to_owned(),
189                hardware: Some("cpu".to_owned()),
190                ..ExecutionMetadata::default()
191            },
192        })
193    }
194
195    async fn infer_stream(
196        &self,
197        model: &ModelHandle,
198        request: &InferenceRequest,
199        cancellation: Option<CancellationToken>,
200    ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
201        let result = self.infer(model, request, cancellation).await?;
202        Ok(Box::pin(stream::iter(vec![Ok(InferenceChunk {
203            output: result.output,
204            done: true,
205            metadata: Some(result.metadata),
206        })])))
207    }
208}
209
210#[cfg(test)]
211mod tests {
212    use super::*;
213    #[test]
214    fn descriptor_matches_boundary() {
215        assert_eq!(OnnxBackend::descriptor().name, "onnx");
216    }
217
218    #[test]
219    fn advertises_onnx_format() {
220        let capabilities = OnnxBackend.capabilities();
221        assert!(capabilities.available);
222        assert_eq!(capabilities.supported_formats, vec![ModelFormat::Onnx]);
223        assert!(OnnxBackend.supports(&ModelSpec::new(
224            "model",
225            ModelFormat::Onnx,
226            ModelLocation::Memory,
227        )));
228    }
229
230    #[test]
231    fn reports_factual_laya_execution_capabilities() {
232        let capabilities = OnnxBackend::laya_decision_capabilities();
233        assert_eq!(capabilities.backend, "onnx");
234        assert_eq!(capabilities.device, DeviceKind::Cpu);
235        assert!(capabilities.supports_batching);
236        assert_eq!(capabilities.max_batch_size, Some(16));
237        assert!(!capabilities.supports_cancellation);
238    }
239}