ml_runtime_backend/
lib.rs1use async_trait::async_trait;
2use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
3use ml_runtime_inference::{DecisionRequest, DecisionResult, ModelDescription};
4use ml_runtime_inference::{InferenceChunk, InferenceRequest, InferenceResult};
5use ml_runtime_model::{ModelFormat, ModelHandle, ModelSpec};
6use serde::{Deserialize, Serialize};
7use std::path::Path;
8
9#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
10pub struct BackendCapabilities {
11 pub available: bool,
12 pub local: bool,
13 pub streaming: bool,
14 pub cancellation: bool,
15 pub structured_output: bool,
16 pub batching: bool,
17 pub max_batch_size: Option<usize>,
18 pub supported_batch_shapes: Vec<Vec<usize>>,
19 pub supported_formats: Vec<ModelFormat>,
20 pub accelerators: Vec<String>,
21 pub hardware: Option<String>,
22 pub notes: Vec<String>,
23}
24
25#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
26pub struct BackendCapability {
27 pub name: String,
28 pub available: bool,
29 pub local: bool,
30 pub streaming: bool,
31 pub cancellation: bool,
32 pub structured_output: bool,
33 pub batching: bool,
34 pub max_batch_size: Option<usize>,
35 pub supported_batch_shapes: Vec<Vec<usize>>,
36 pub supported_formats: Vec<ModelFormat>,
37 pub accelerators: Vec<String>,
38 pub hardware: Option<String>,
39 pub notes: Vec<String>,
40}
41
42impl BackendCapability {
43 pub fn from_parts(name: impl Into<String>, capabilities: BackendCapabilities) -> Self {
44 Self {
45 name: name.into(),
46 available: capabilities.available,
47 local: capabilities.local,
48 streaming: capabilities.streaming,
49 cancellation: capabilities.cancellation,
50 structured_output: capabilities.structured_output,
51 batching: capabilities.batching,
52 max_batch_size: capabilities.max_batch_size,
53 supported_batch_shapes: capabilities.supported_batch_shapes,
54 supported_formats: capabilities.supported_formats,
55 accelerators: capabilities.accelerators,
56 hardware: capabilities.hardware,
57 notes: capabilities.notes,
58 }
59 }
60}
61
62#[async_trait]
63pub trait Backend: Send + Sync {
64 fn name(&self) -> &str;
65 fn capabilities(&self) -> BackendCapabilities;
66 fn supports(&self, model: &ModelSpec) -> bool;
67 async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle>;
68 async fn infer(
69 &self,
70 model: &ModelHandle,
71 request: &InferenceRequest,
72 cancellation: Option<CancellationToken>,
73 ) -> RuntimeResult<InferenceResult>;
74 async fn infer_batch(
75 &self,
76 model: &ModelHandle,
77 requests: &[InferenceRequest],
78 cancellation: Option<CancellationToken>,
79 ) -> RuntimeResult<Vec<InferenceResult>> {
80 let mut results = Vec::with_capacity(requests.len());
81 for request in requests {
82 results.push(self.infer(model, request, cancellation.clone()).await?);
83 }
84 Ok(results)
85 }
86 async fn infer_stream(
87 &self,
88 model: &ModelHandle,
89 request: &InferenceRequest,
90 cancellation: Option<CancellationToken>,
91 ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>>;
92}
93
94pub trait DecisionModel: Send + Sync {
96 fn describe(&self) -> ModelDescription;
97 fn decide(&self, request: &DecisionRequest) -> RuntimeResult<DecisionResult>;
98}
99
100pub trait DecisionModelProvider: Send + Sync {
102 fn name(&self) -> &str;
103 fn supports_artifact(&self, artifact: &Path) -> bool;
104 fn load_decision_model(&self, artifact: &Path) -> RuntimeResult<Box<dyn DecisionModel>>;
105}