Skip to main content

ml_runtime_backend/
lib.rs

1use async_trait::async_trait;
2use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
3use ml_runtime_inference::{DecisionRequest, DecisionResult, ModelDescription};
4use ml_runtime_inference::{InferenceChunk, InferenceRequest, InferenceResult};
5use ml_runtime_model::{ModelFormat, ModelHandle, ModelSpec};
6use serde::{Deserialize, Serialize};
7use std::path::Path;
8
9#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
10pub struct BackendCapabilities {
11    pub available: bool,
12    pub local: bool,
13    pub streaming: bool,
14    pub cancellation: bool,
15    pub structured_output: bool,
16    pub batching: bool,
17    pub max_batch_size: Option<usize>,
18    pub supported_batch_shapes: Vec<Vec<usize>>,
19    pub supported_formats: Vec<ModelFormat>,
20    pub accelerators: Vec<String>,
21    pub hardware: Option<String>,
22    pub notes: Vec<String>,
23}
24
25#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
26pub struct BackendCapability {
27    pub name: String,
28    pub available: bool,
29    pub local: bool,
30    pub streaming: bool,
31    pub cancellation: bool,
32    pub structured_output: bool,
33    pub batching: bool,
34    pub max_batch_size: Option<usize>,
35    pub supported_batch_shapes: Vec<Vec<usize>>,
36    pub supported_formats: Vec<ModelFormat>,
37    pub accelerators: Vec<String>,
38    pub hardware: Option<String>,
39    pub notes: Vec<String>,
40}
41
42impl BackendCapability {
43    pub fn from_parts(name: impl Into<String>, capabilities: BackendCapabilities) -> Self {
44        Self {
45            name: name.into(),
46            available: capabilities.available,
47            local: capabilities.local,
48            streaming: capabilities.streaming,
49            cancellation: capabilities.cancellation,
50            structured_output: capabilities.structured_output,
51            batching: capabilities.batching,
52            max_batch_size: capabilities.max_batch_size,
53            supported_batch_shapes: capabilities.supported_batch_shapes,
54            supported_formats: capabilities.supported_formats,
55            accelerators: capabilities.accelerators,
56            hardware: capabilities.hardware,
57            notes: capabilities.notes,
58        }
59    }
60}
61
62#[async_trait]
63pub trait Backend: Send + Sync {
64    fn name(&self) -> &str;
65    fn capabilities(&self) -> BackendCapabilities;
66    fn supports(&self, model: &ModelSpec) -> bool;
67    async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle>;
68    async fn infer(
69        &self,
70        model: &ModelHandle,
71        request: &InferenceRequest,
72        cancellation: Option<CancellationToken>,
73    ) -> RuntimeResult<InferenceResult>;
74    async fn infer_batch(
75        &self,
76        model: &ModelHandle,
77        requests: &[InferenceRequest],
78        cancellation: Option<CancellationToken>,
79    ) -> RuntimeResult<Vec<InferenceResult>> {
80        let mut results = Vec::with_capacity(requests.len());
81        for request in requests {
82            results.push(self.infer(model, request, cancellation.clone()).await?);
83        }
84        Ok(results)
85    }
86    async fn infer_stream(
87        &self,
88        model: &ModelHandle,
89        request: &InferenceRequest,
90        cancellation: Option<CancellationToken>,
91    ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>>;
92}
93
94/// Loaded model-neutral typed-decision model.
95pub trait DecisionModel: Send + Sync {
96    fn describe(&self) -> ModelDescription;
97    fn decide(&self, request: &DecisionRequest) -> RuntimeResult<DecisionResult>;
98}
99
100/// Backend extension capable of loading a local typed-decision artifact.
101pub trait DecisionModelProvider: Send + Sync {
102    fn name(&self) -> &str;
103    fn supports_artifact(&self, artifact: &Path) -> bool;
104    fn load_decision_model(&self, artifact: &Path) -> RuntimeResult<Box<dyn DecisionModel>>;
105}