ml_runtime_backend/
lib.rs1use async_trait::async_trait;
2use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
3use ml_runtime_inference::{
4 DecisionModelCapabilities, DecisionRequest, DecisionResult, ModelDescription,
5};
6use ml_runtime_inference::{InferenceChunk, InferenceRequest, InferenceResult};
7use ml_runtime_model::{ModelFormat, ModelHandle, ModelSpec};
8use serde::{Deserialize, Serialize};
9use std::path::Path;
10
11#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
12pub struct BackendCapabilities {
13 pub available: bool,
14 pub local: bool,
15 pub streaming: bool,
16 pub cancellation: bool,
17 pub structured_output: bool,
18 pub batching: bool,
19 pub max_batch_size: Option<usize>,
20 pub supported_batch_shapes: Vec<Vec<usize>>,
21 pub supported_formats: Vec<ModelFormat>,
22 pub accelerators: Vec<String>,
23 pub hardware: Option<String>,
24 pub notes: Vec<String>,
25}
26
27#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
28pub struct BackendCapability {
29 pub name: String,
30 pub available: bool,
31 pub local: bool,
32 pub streaming: bool,
33 pub cancellation: bool,
34 pub structured_output: bool,
35 pub batching: bool,
36 pub max_batch_size: Option<usize>,
37 pub supported_batch_shapes: Vec<Vec<usize>>,
38 pub supported_formats: Vec<ModelFormat>,
39 pub accelerators: Vec<String>,
40 pub hardware: Option<String>,
41 pub notes: Vec<String>,
42}
43
44impl BackendCapability {
45 pub fn from_parts(name: impl Into<String>, capabilities: BackendCapabilities) -> Self {
46 Self {
47 name: name.into(),
48 available: capabilities.available,
49 local: capabilities.local,
50 streaming: capabilities.streaming,
51 cancellation: capabilities.cancellation,
52 structured_output: capabilities.structured_output,
53 batching: capabilities.batching,
54 max_batch_size: capabilities.max_batch_size,
55 supported_batch_shapes: capabilities.supported_batch_shapes,
56 supported_formats: capabilities.supported_formats,
57 accelerators: capabilities.accelerators,
58 hardware: capabilities.hardware,
59 notes: capabilities.notes,
60 }
61 }
62}
63
64#[async_trait]
65pub trait Backend: Send + Sync {
66 fn name(&self) -> &str;
67 fn capabilities(&self) -> BackendCapabilities;
68 fn supports(&self, model: &ModelSpec) -> bool;
69 async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle>;
70 async fn infer(
71 &self,
72 model: &ModelHandle,
73 request: &InferenceRequest,
74 cancellation: Option<CancellationToken>,
75 ) -> RuntimeResult<InferenceResult>;
76 async fn infer_batch(
77 &self,
78 model: &ModelHandle,
79 requests: &[InferenceRequest],
80 cancellation: Option<CancellationToken>,
81 ) -> RuntimeResult<Vec<InferenceResult>> {
82 let mut results = Vec::with_capacity(requests.len());
83 for request in requests {
84 results.push(self.infer(model, request, cancellation.clone()).await?);
85 }
86 Ok(results)
87 }
88 async fn infer_stream(
89 &self,
90 model: &ModelHandle,
91 request: &InferenceRequest,
92 cancellation: Option<CancellationToken>,
93 ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>>;
94}
95
96pub trait DecisionModel: Send + Sync {
98 fn describe(&self) -> ModelDescription;
99 fn capabilities(&self) -> DecisionModelCapabilities {
100 DecisionModelCapabilities::conservative(self.describe().backend)
101 }
102 fn decide(&self, request: &DecisionRequest) -> RuntimeResult<DecisionResult>;
103}
104
105pub trait DecisionModelProvider: Send + Sync {
107 fn name(&self) -> &str;
108 fn supports_artifact(&self, artifact: &Path) -> bool;
109 fn load_decision_model(&self, artifact: &Path) -> RuntimeResult<Box<dyn DecisionModel>>;
110}