Skip to main content

ml_runtime_backend/
lib.rs

1use async_trait::async_trait;
2use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
3use ml_runtime_inference::{
4    DecisionModelCapabilities, DecisionRequest, DecisionResult, ModelDescription,
5};
6use ml_runtime_inference::{InferenceChunk, InferenceRequest, InferenceResult};
7use ml_runtime_model::{ModelFormat, ModelHandle, ModelSpec};
8use serde::{Deserialize, Serialize};
9use std::path::Path;
10
11#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
12pub struct BackendCapabilities {
13    pub available: bool,
14    pub local: bool,
15    pub streaming: bool,
16    pub cancellation: bool,
17    pub structured_output: bool,
18    pub batching: bool,
19    pub max_batch_size: Option<usize>,
20    pub supported_batch_shapes: Vec<Vec<usize>>,
21    pub supported_formats: Vec<ModelFormat>,
22    pub accelerators: Vec<String>,
23    pub hardware: Option<String>,
24    pub notes: Vec<String>,
25}
26
27#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
28pub struct BackendCapability {
29    pub name: String,
30    pub available: bool,
31    pub local: bool,
32    pub streaming: bool,
33    pub cancellation: bool,
34    pub structured_output: bool,
35    pub batching: bool,
36    pub max_batch_size: Option<usize>,
37    pub supported_batch_shapes: Vec<Vec<usize>>,
38    pub supported_formats: Vec<ModelFormat>,
39    pub accelerators: Vec<String>,
40    pub hardware: Option<String>,
41    pub notes: Vec<String>,
42}
43
44impl BackendCapability {
45    pub fn from_parts(name: impl Into<String>, capabilities: BackendCapabilities) -> Self {
46        Self {
47            name: name.into(),
48            available: capabilities.available,
49            local: capabilities.local,
50            streaming: capabilities.streaming,
51            cancellation: capabilities.cancellation,
52            structured_output: capabilities.structured_output,
53            batching: capabilities.batching,
54            max_batch_size: capabilities.max_batch_size,
55            supported_batch_shapes: capabilities.supported_batch_shapes,
56            supported_formats: capabilities.supported_formats,
57            accelerators: capabilities.accelerators,
58            hardware: capabilities.hardware,
59            notes: capabilities.notes,
60        }
61    }
62}
63
64#[async_trait]
65pub trait Backend: Send + Sync {
66    fn name(&self) -> &str;
67    fn capabilities(&self) -> BackendCapabilities;
68    fn supports(&self, model: &ModelSpec) -> bool;
69    async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle>;
70    async fn infer(
71        &self,
72        model: &ModelHandle,
73        request: &InferenceRequest,
74        cancellation: Option<CancellationToken>,
75    ) -> RuntimeResult<InferenceResult>;
76    async fn infer_batch(
77        &self,
78        model: &ModelHandle,
79        requests: &[InferenceRequest],
80        cancellation: Option<CancellationToken>,
81    ) -> RuntimeResult<Vec<InferenceResult>> {
82        let mut results = Vec::with_capacity(requests.len());
83        for request in requests {
84            results.push(self.infer(model, request, cancellation.clone()).await?);
85        }
86        Ok(results)
87    }
88    async fn infer_stream(
89        &self,
90        model: &ModelHandle,
91        request: &InferenceRequest,
92        cancellation: Option<CancellationToken>,
93    ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>>;
94}
95
96/// Loaded model-neutral typed-decision model.
97pub trait DecisionModel: Send + Sync {
98    fn describe(&self) -> ModelDescription;
99    fn capabilities(&self) -> DecisionModelCapabilities {
100        DecisionModelCapabilities::conservative(self.describe().backend)
101    }
102    fn decide(&self, request: &DecisionRequest) -> RuntimeResult<DecisionResult>;
103}
104
105/// Backend extension capable of loading a local typed-decision artifact.
106pub trait DecisionModelProvider: Send + Sync {
107    fn name(&self) -> &str;
108    fn supports_artifact(&self, artifact: &Path) -> bool;
109    fn load_decision_model(&self, artifact: &Path) -> RuntimeResult<Box<dyn DecisionModel>>;
110}