llmkit-tower 0.1.0

Tower middleware (retry, rate limit, cost tracking, tracing) for llmkit-rs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

//! Exponential-backoff retry layer.

use std::sync::Arc;
use std::time::Duration;

use async_trait::async_trait;
use llmkit_core::{
    ChatRequest, ChatResponse, ChatStream, CostEstimate, EmbedRequest, EmbedResponse, LlmError,
    LlmProvider, LlmResult,
};

use crate::layer::LlmLayer;

/// Configures exponential-backoff retries for retryable errors.
#[derive(Debug, Clone, Copy)]
pub struct RetryLayer {
    attempts: u32,
    base: Duration,
    max_backoff: Duration,
}

impl RetryLayer {
    /// `attempts` total tries with exponential backoff starting at `base`.
    pub fn exponential(attempts: u32, base: Duration) -> Self {
        Self { attempts: attempts.max(1), base, max_backoff: Duration::from_secs(30) }
    }

    /// Cap the backoff delay.
    pub fn max_backoff(mut self, max: Duration) -> Self {
        self.max_backoff = max;
        self
    }
}

impl LlmLayer for RetryLayer {
    type Provider = Retry;
    fn layer(self, inner: Arc<dyn LlmProvider>) -> Retry {
        Retry { inner, cfg: self }
    }
}

/// Provider produced by [`RetryLayer`].
pub struct Retry {
    inner: Arc<dyn LlmProvider>,
    cfg: RetryLayer,
}

impl Retry {
    fn backoff(&self, attempt: u32, err: &LlmError) -> Duration {
        // Honour a server-supplied retry-after when present.
        if let LlmError::RateLimited { retry_after: Some(d), .. } = err {
            return (*d).min(self.cfg.max_backoff);
        }
        let mult = 2u32.saturating_pow(attempt);
        self.cfg.base.saturating_mul(mult).min(self.cfg.max_backoff)
    }
}

#[async_trait]
impl LlmProvider for Retry {
    async fn chat(&self, req: ChatRequest) -> LlmResult<ChatResponse> {
        let mut last = None;
        for attempt in 0..self.cfg.attempts {
            match self.inner.chat(req.clone()).await {
                Ok(resp) => return Ok(resp),
                Err(e) if e.is_retryable() && attempt + 1 < self.cfg.attempts => {
                    tracing::debug!(provider = self.inner.name(), attempt, error = %e, "retrying");
                    tokio::time::sleep(self.backoff(attempt, &e)).await;
                    last = Some(e);
                }
                Err(e) => return Err(e),
            }
        }
        Err(last.unwrap_or_else(|| LlmError::Other("retry exhausted".into())))
    }

    async fn chat_stream(&self, req: ChatRequest) -> LlmResult<ChatStream> {
        // Retry only the connection establishment, not mid-stream failures.
        let mut last = None;
        for attempt in 0..self.cfg.attempts {
            match self.inner.chat_stream(req.clone()).await {
                Ok(s) => return Ok(s),
                Err(e) if e.is_retryable() && attempt + 1 < self.cfg.attempts => {
                    tokio::time::sleep(self.backoff(attempt, &e)).await;
                    last = Some(e);
                }
                Err(e) => return Err(e),
            }
        }
        Err(last.unwrap_or_else(|| LlmError::Other("retry exhausted".into())))
    }

    async fn embed(&self, req: EmbedRequest) -> LlmResult<EmbedResponse> {
        let mut last = None;
        for attempt in 0..self.cfg.attempts {
            match self.inner.embed(req.clone()).await {
                Ok(r) => return Ok(r),
                Err(e) if e.is_retryable() && attempt + 1 < self.cfg.attempts => {
                    tokio::time::sleep(self.backoff(attempt, &e)).await;
                    last = Some(e);
                }
                Err(e) => return Err(e),
            }
        }
        Err(last.unwrap_or_else(|| LlmError::Other("retry exhausted".into())))
    }

    fn name(&self) -> &'static str {
        self.inner.name()
    }

    fn model(&self) -> &str {
        self.inner.model()
    }

    fn estimate_cost(&self, req: &ChatRequest) -> Option<CostEstimate> {
        self.inner.estimate_cost(req)
    }
}