synaptic-models 0.4.0

ProviderBackend abstraction and ChatModel wrappers (Retry, RateLimit, StructuredOutput, BoundTools)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

use std::sync::Arc;

use async_trait::async_trait;
use synaptic_core::{ChatModel, ChatRequest, ChatResponse, ChatStream, SynapticError};
use tokio::sync::Semaphore;

pub struct RateLimitedChatModel {
    inner: Arc<dyn ChatModel>,
    semaphore: Arc<Semaphore>,
}

impl RateLimitedChatModel {
    pub fn new(inner: Arc<dyn ChatModel>, max_concurrent: usize) -> Self {
        Self {
            inner,
            semaphore: Arc::new(Semaphore::new(max_concurrent)),
        }
    }
}

#[async_trait]
impl ChatModel for RateLimitedChatModel {
    async fn chat(&self, request: ChatRequest) -> Result<ChatResponse, SynapticError> {
        let _permit = self
            .semaphore
            .acquire()
            .await
            .map_err(|e| SynapticError::Model(format!("semaphore error: {e}")))?;
        self.inner.chat(request).await
    }

    fn stream_chat(&self, request: ChatRequest) -> ChatStream<'_> {
        let inner = self.inner.clone();
        let semaphore = self.semaphore.clone();

        Box::pin(async_stream::stream! {
            let _permit = match semaphore.acquire_owned().await {
                Ok(p) => p,
                Err(e) => {
                    yield Err(SynapticError::Model(format!("semaphore error: {e}")));
                    return;
                }
            };

            use futures::StreamExt;
            let mut stream = inner.stream_chat(request);
            while let Some(result) = stream.next().await {
                yield result;
            }
        })
    }
}