ml_runtime_provider/
lib.rs1use async_trait::async_trait;
2use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
3use ml_runtime_inference::{InferenceChunk, InferenceRequest, InferenceResult};
4use serde::{Deserialize, Serialize};
5
6#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
7pub struct ProviderCapabilities {
8 pub available: bool,
9 pub local: bool,
10 pub remote: bool,
11 pub streaming: bool,
12 pub cancellation: bool,
13 pub structured_output: bool,
14 pub batching: bool,
15 pub endpoint: Option<String>,
16 pub notes: Vec<String>,
17}
18
19#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
20pub struct ProviderCapability {
21 pub name: String,
22 pub available: bool,
23 pub local: bool,
24 pub remote: bool,
25 pub streaming: bool,
26 pub cancellation: bool,
27 pub structured_output: bool,
28 pub batching: bool,
29 pub endpoint: Option<String>,
30 pub notes: Vec<String>,
31}
32
33impl ProviderCapability {
34 pub fn from_parts(name: impl Into<String>, capabilities: ProviderCapabilities) -> Self {
35 Self {
36 name: name.into(),
37 available: capabilities.available,
38 local: capabilities.local,
39 remote: capabilities.remote,
40 streaming: capabilities.streaming,
41 cancellation: capabilities.cancellation,
42 structured_output: capabilities.structured_output,
43 batching: capabilities.batching,
44 endpoint: capabilities.endpoint,
45 notes: capabilities.notes,
46 }
47 }
48}
49
50#[async_trait]
51pub trait Provider: Send + Sync {
52 fn name(&self) -> &str;
53 fn capabilities(&self) -> ProviderCapabilities;
54 async fn infer(
55 &self,
56 request: InferenceRequest,
57 cancellation: Option<CancellationToken>,
58 ) -> RuntimeResult<InferenceResult>;
59 async fn infer_stream(
60 &self,
61 request: InferenceRequest,
62 cancellation: Option<CancellationToken>,
63 ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>>;
64}