Skip to main content

ml_runtime_provider/
lib.rs

1use async_trait::async_trait;
2use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
3use ml_runtime_inference::{InferenceChunk, InferenceRequest, InferenceResult};
4use serde::{Deserialize, Serialize};
5
6#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
7pub struct ProviderCapabilities {
8    pub available: bool,
9    pub local: bool,
10    pub remote: bool,
11    pub streaming: bool,
12    pub cancellation: bool,
13    pub structured_output: bool,
14    pub batching: bool,
15    pub endpoint: Option<String>,
16    pub notes: Vec<String>,
17}
18
19#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
20pub struct ProviderCapability {
21    pub name: String,
22    pub available: bool,
23    pub local: bool,
24    pub remote: bool,
25    pub streaming: bool,
26    pub cancellation: bool,
27    pub structured_output: bool,
28    pub batching: bool,
29    pub endpoint: Option<String>,
30    pub notes: Vec<String>,
31}
32
33impl ProviderCapability {
34    pub fn from_parts(name: impl Into<String>, capabilities: ProviderCapabilities) -> Self {
35        Self {
36            name: name.into(),
37            available: capabilities.available,
38            local: capabilities.local,
39            remote: capabilities.remote,
40            streaming: capabilities.streaming,
41            cancellation: capabilities.cancellation,
42            structured_output: capabilities.structured_output,
43            batching: capabilities.batching,
44            endpoint: capabilities.endpoint,
45            notes: capabilities.notes,
46        }
47    }
48}
49
50#[async_trait]
51pub trait Provider: Send + Sync {
52    fn name(&self) -> &str;
53    fn capabilities(&self) -> ProviderCapabilities;
54    async fn infer(
55        &self,
56        request: InferenceRequest,
57        cancellation: Option<CancellationToken>,
58    ) -> RuntimeResult<InferenceResult>;
59    async fn infer_stream(
60        &self,
61        request: InferenceRequest,
62        cancellation: Option<CancellationToken>,
63    ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>>;
64}