1use async_trait::async_trait;
2use futures_util::stream;
3mod laya;
4use laya::LayaModel;
5
6use ml_runtime_backend::{
7 Backend, BackendCapabilities, BackendCapability, DecisionModel, DecisionModelProvider,
8};
9use ml_runtime_common::{BoxStream, CancellationToken, RuntimeError, RuntimeResult};
10use ml_runtime_inference::{
11 DecisionModelCapabilities, DeviceKind, ExecutionMetadata, InferenceChunk, InferenceRequest,
12 InferenceResult, Input, ModelArchitecture, Output, Tensor,
13};
14use ml_runtime_model::{ModelFormat, ModelHandle, ModelLocation, ModelSpec};
15use ort::{session::Session, value::Tensor as OrtTensor};
16use std::{
17 path::Path,
18 sync::{Arc, Mutex},
19};
20
21#[derive(Clone, Debug, Default)]
22pub struct OnnxBackend;
23
24struct OnnxLoadedModel {
25 spec: ModelSpec,
26 session: Mutex<Session>,
27}
28
29impl OnnxBackend {
30 pub fn descriptor() -> BackendCapability {
31 BackendCapability::from_parts("onnx", Self.capabilities())
32 }
33
34 fn load_session(model: &ModelSpec) -> RuntimeResult<Session> {
35 let ModelLocation::Path(path) = &model.location else {
36 return Err(RuntimeError::InvalidRequest {
37 reason: "ONNX models require a filesystem artifact".to_owned(),
38 });
39 };
40 Session::builder()
41 .and_then(|builder| builder.commit_from_file(path))
42 .map_err(|error| RuntimeError::execution("load", format!("load ONNX model: {error}")))
43 }
44
45 pub fn validate_laya(artifact: &Path) -> RuntimeResult<String> {
46 laya::LayaModel::validate(artifact)
47 }
48
49 pub fn laya_decision_capabilities() -> DecisionModelCapabilities {
52 DecisionModelCapabilities {
53 backend: "onnx".to_owned(),
54 device: DeviceKind::Cpu,
55 model_architecture: ModelArchitecture::StructuredDecision,
56 supports_batching: true,
57 max_batch_size: Some(16),
58 batch_size: None,
59 supports_async: false,
60 supports_cancellation: false,
61 supports_parallel_execution: false,
62 recommended_parallelism: Some(1),
63 supports_structured_decisions: true,
64 }
65 }
66}
67
68impl DecisionModelProvider for OnnxBackend {
69 fn name(&self) -> &str {
70 "onnx"
71 }
72
73 fn supports_artifact(&self, artifact: &Path) -> bool {
74 artifact.join("laya.onnx").is_file()
75 && artifact.join("laya_config.json").is_file()
76 && artifact.join("tokenizer/tokenizer.json").is_file()
77 }
78
79 fn load_decision_model(&self, artifact: &Path) -> RuntimeResult<Box<dyn DecisionModel>> {
80 LayaModel::load(artifact).map(|model| Box::new(model) as Box<dyn DecisionModel>)
81 }
82}
83
84#[async_trait]
85impl Backend for OnnxBackend {
86 fn name(&self) -> &str {
87 "onnx"
88 }
89
90 fn capabilities(&self) -> BackendCapabilities {
91 BackendCapabilities {
92 available: true,
93 local: true,
94 streaming: true,
95 cancellation: true,
96 structured_output: false,
97 batching: true,
98 max_batch_size: Some(16),
99 supported_batch_shapes: Vec::new(),
100 supported_formats: vec![ModelFormat::Onnx],
101 accelerators: Vec::new(),
102 hardware: Some("cpu".to_owned()),
103 notes: vec!["ONNX Runtime CPU execution".to_owned()],
104 }
105 }
106
107 fn supports(&self, model: &ModelSpec) -> bool {
108 matches!(model.format, ModelFormat::Onnx)
109 }
110
111 async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle> {
112 let session = Self::load_session(model)?;
113 Ok(ModelHandle::new(
114 model.id.clone(),
115 Backend::name(self),
116 model.format.clone(),
117 Arc::new(OnnxLoadedModel {
118 spec: model.clone(),
119 session: Mutex::new(session),
120 }),
121 ))
122 }
123
124 async fn infer(
125 &self,
126 model: &ModelHandle,
127 request: &InferenceRequest,
128 cancellation: Option<CancellationToken>,
129 ) -> RuntimeResult<InferenceResult> {
130 if cancellation
131 .as_ref()
132 .is_some_and(CancellationToken::is_cancelled)
133 {
134 return Err(RuntimeError::Cancelled);
135 }
136 let loaded = model.state::<OnnxLoadedModel>().ok_or_else(|| {
137 RuntimeError::execution("infer", "model handle does not belong to the ONNX backend")
138 })?;
139 let Input::Tensor(input) = &request.input else {
140 return Err(RuntimeError::UnsupportedInput {
141 reason: "ONNX backend currently accepts tensor input".to_owned(),
142 });
143 };
144 if input.shape.iter().product::<usize>() != input.values.len() {
145 return Err(RuntimeError::UnsupportedInput {
146 reason: "tensor shape does not match tensor values".to_owned(),
147 });
148 }
149 let mut session = loaded
150 .session
151 .lock()
152 .map_err(|_| RuntimeError::execution("infer", "ONNX session lock poisoned"))?;
153 let input_name = session
154 .inputs
155 .first()
156 .map(|input| input.name.to_owned())
157 .ok_or_else(|| RuntimeError::execution("infer", "ONNX model has no inputs"))?;
158 let tensor = OrtTensor::from_array((input.shape.clone(), input.values.clone())).map_err(
159 |error| RuntimeError::execution("infer", format!("create ONNX input: {error}")),
160 )?;
161 let outputs = session
162 .run(ort::inputs![input_name => tensor])
163 .map_err(|error| {
164 RuntimeError::execution("infer", format!("execute ONNX model: {error}"))
165 })?;
166 if outputs.len() == 0 {
167 return Err(RuntimeError::execution(
168 "infer",
169 "ONNX model produced no outputs",
170 ));
171 }
172 let (_, values) = outputs[0].try_extract_tensor::<f32>().map_err(|error| {
173 RuntimeError::execution("infer", format!("read ONNX output: {error}"))
174 })?;
175 let shape = outputs[0]
176 .shape()
177 .iter()
178 .map(|dimension| usize::try_from(*dimension).unwrap_or(0))
179 .collect();
180 Ok(InferenceResult {
181 output: Output::Tensor(Tensor {
182 shape,
183 values: values.to_vec(),
184 }),
185 metadata: ExecutionMetadata {
186 model: loaded.spec.id.clone(),
187 provider: "local".to_owned(),
188 backend: Backend::name(self).to_owned(),
189 hardware: Some("cpu".to_owned()),
190 ..ExecutionMetadata::default()
191 },
192 })
193 }
194
195 async fn infer_stream(
196 &self,
197 model: &ModelHandle,
198 request: &InferenceRequest,
199 cancellation: Option<CancellationToken>,
200 ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
201 let result = self.infer(model, request, cancellation).await?;
202 Ok(Box::pin(stream::iter(vec![Ok(InferenceChunk {
203 output: result.output,
204 done: true,
205 metadata: Some(result.metadata),
206 })])))
207 }
208}
209
210#[cfg(test)]
211mod tests {
212 use super::*;
213 #[test]
214 fn descriptor_matches_boundary() {
215 assert_eq!(OnnxBackend::descriptor().name, "onnx");
216 }
217
218 #[test]
219 fn advertises_onnx_format() {
220 let capabilities = OnnxBackend.capabilities();
221 assert!(capabilities.available);
222 assert_eq!(capabilities.supported_formats, vec![ModelFormat::Onnx]);
223 assert!(OnnxBackend.supports(&ModelSpec::new(
224 "model",
225 ModelFormat::Onnx,
226 ModelLocation::Memory,
227 )));
228 }
229
230 #[test]
231 fn reports_factual_laya_execution_capabilities() {
232 let capabilities = OnnxBackend::laya_decision_capabilities();
233 assert_eq!(capabilities.backend, "onnx");
234 assert_eq!(capabilities.device, DeviceKind::Cpu);
235 assert!(capabilities.supports_batching);
236 assert_eq!(capabilities.max_batch_size, Some(16));
237 assert!(!capabilities.supports_cancellation);
238 }
239}