ml_runtime_provider/
lib.rs1mod generation;
2pub use generation::*;
3
4use async_trait::async_trait;
5use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
6use ml_runtime_inference::{InferenceChunk, InferenceRequest, InferenceResult};
7use serde::{Deserialize, Serialize};
8
9#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
10pub struct ProviderCapabilities {
11 pub available: bool,
12 pub local: bool,
13 pub remote: bool,
14 pub streaming: bool,
15 pub cancellation: bool,
16 pub structured_output: bool,
17 pub batching: bool,
18 pub endpoint: Option<String>,
19 pub notes: Vec<String>,
20}
21
22#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
23pub struct ProviderCapability {
24 pub name: String,
25 pub available: bool,
26 pub local: bool,
27 pub remote: bool,
28 pub streaming: bool,
29 pub cancellation: bool,
30 pub structured_output: bool,
31 pub batching: bool,
32 pub endpoint: Option<String>,
33 pub notes: Vec<String>,
34}
35
36impl ProviderCapability {
37 pub fn from_parts(name: impl Into<String>, capabilities: ProviderCapabilities) -> Self {
38 Self {
39 name: name.into(),
40 available: capabilities.available,
41 local: capabilities.local,
42 remote: capabilities.remote,
43 streaming: capabilities.streaming,
44 cancellation: capabilities.cancellation,
45 structured_output: capabilities.structured_output,
46 batching: capabilities.batching,
47 endpoint: capabilities.endpoint,
48 notes: capabilities.notes,
49 }
50 }
51}
52
53#[async_trait]
54pub trait Provider: Send + Sync {
55 fn name(&self) -> &str;
56 fn capabilities(&self) -> ProviderCapabilities;
57 async fn infer(
58 &self,
59 request: InferenceRequest,
60 cancellation: Option<CancellationToken>,
61 ) -> RuntimeResult<InferenceResult>;
62 async fn infer_stream(
63 &self,
64 request: InferenceRequest,
65 cancellation: Option<CancellationToken>,
66 ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>>;
67}