Skip to main content

ml_runtime_provider/
lib.rs

1mod generation;
2pub use generation::*;
3
4use async_trait::async_trait;
5use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
6use ml_runtime_inference::{InferenceChunk, InferenceRequest, InferenceResult};
7use serde::{Deserialize, Serialize};
8
9#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
10pub struct ProviderCapabilities {
11    pub available: bool,
12    pub local: bool,
13    pub remote: bool,
14    pub streaming: bool,
15    pub cancellation: bool,
16    pub structured_output: bool,
17    pub batching: bool,
18    pub endpoint: Option<String>,
19    pub notes: Vec<String>,
20}
21
22#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
23pub struct ProviderCapability {
24    pub name: String,
25    pub available: bool,
26    pub local: bool,
27    pub remote: bool,
28    pub streaming: bool,
29    pub cancellation: bool,
30    pub structured_output: bool,
31    pub batching: bool,
32    pub endpoint: Option<String>,
33    pub notes: Vec<String>,
34}
35
36impl ProviderCapability {
37    pub fn from_parts(name: impl Into<String>, capabilities: ProviderCapabilities) -> Self {
38        Self {
39            name: name.into(),
40            available: capabilities.available,
41            local: capabilities.local,
42            remote: capabilities.remote,
43            streaming: capabilities.streaming,
44            cancellation: capabilities.cancellation,
45            structured_output: capabilities.structured_output,
46            batching: capabilities.batching,
47            endpoint: capabilities.endpoint,
48            notes: capabilities.notes,
49        }
50    }
51}
52
53#[async_trait]
54pub trait Provider: Send + Sync {
55    fn name(&self) -> &str;
56    fn capabilities(&self) -> ProviderCapabilities;
57    async fn infer(
58        &self,
59        request: InferenceRequest,
60        cancellation: Option<CancellationToken>,
61    ) -> RuntimeResult<InferenceResult>;
62    async fn infer_stream(
63        &self,
64        request: InferenceRequest,
65        cancellation: Option<CancellationToken>,
66    ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>>;
67}