hyperinfer_providers/
provider_trait.rs

1use async_trait::async_trait;
2use futures::Stream;
3use hyperinfer_core::{ChatChunk, ChatRequest, ChatResponse};
4use std::pin::Pin;
5use std::sync::Arc;
6
7#[async_trait]
8pub trait LlmProvider: dyn_clone::DynClone + Send + Sync {
9    fn name(&self) -> &str;
10
11    fn base_url(&self) -> &str {
12        ""
13    }
14
15    fn supports_streaming(&self) -> bool {
16        true
17    }
18
19    async fn chat(
20        &self,
21        request: &ChatRequest,
22        api_key: &str,
23    ) -> Result<ChatResponse, hyperinfer_core::HyperInferError>;
24
25    fn stream(
26        &self,
27        request: &ChatRequest,
28        api_key: &str,
29    ) -> Pin<
30        Box<
31            dyn Stream<Item = Result<ChatChunk, hyperinfer_core::HyperInferError>> + Send + 'static,
32        >,
33    >;
34
35    async fn health_check(&self, api_key: &str) -> Result<(), hyperinfer_core::HyperInferError> {
36        let request = ChatRequest {
37            model: "health-check-probe".to_string(),
38            messages: vec![hyperinfer_core::ChatMessage {
39                role: hyperinfer_core::MessageRole::User,
40                content: "ping".to_string(),
41            }],
42            temperature: None,
43            max_tokens: Some(1),
44            stream: None,
45            stop: None,
46        };
47        self.chat(&request, api_key).await?;
48        Ok(())
49    }
50}
51
52dyn_clone::clone_trait_object!(LlmProvider);
53
54/// Holds a reference-counted LlmProvider and produces a 'static stream.
55/// Cloning the Arc is O(1) — no deep-clone of the provider's HTTP client.
56pub struct StreamingProvider {
57    inner: Arc<dyn LlmProvider>,
58}
59
60impl StreamingProvider {
61    pub fn new(provider: Arc<dyn LlmProvider>) -> Self {
62        Self { inner: provider }
63    }
64
65    pub fn into_stream(
66        self,
67        request: &ChatRequest,
68        api_key: &str,
69    ) -> Pin<
70        Box<
71            dyn Stream<Item = Result<ChatChunk, hyperinfer_core::HyperInferError>> + Send + 'static,
72        >,
73    > {
74        self.inner.stream(request, api_key)
75    }
76}
hyperinfer_providers/provider_trait.rs

hyperinfer_providers/
provider_trait.rs
