lash_core/provider/
traits.rs

1use super::support::*;
2use crate::LlmTerminalReason;
3
4/// A configured LLM backend: its identity, host-config serialization, its
5/// generation options, and the request transport.
6#[async_trait]
7pub trait Provider: Send + Sync + std::fmt::Debug {
8    fn kind(&self) -> &'static str;
9
10    fn options(&self) -> ProviderOptions;
11    fn set_options(&mut self, options: ProviderOptions);
12
13    /// Emit the provider-specific JSON body used by [`ProviderSpec`]. The
14    /// object must NOT contain a `type` field — [`ProviderSpec::Serialize`]
15    /// layers that on top.
16    fn serialize_config(&self) -> serde_json::Value;
17
18    async fn complete(&mut self, request: LlmRequest) -> Result<LlmResponse, LlmTransportError>;
19
20    fn requires_streaming(&self) -> bool {
21        false
22    }
23
24    /// Release any host-visible transport resources this provider holds —
25    /// cached connections, pooled sockets, background tasks — sending whatever
26    /// graceful close a clean shutdown requires.
27    ///
28    /// Hosts call this before process exit so protocol niceties (e.g. WebSocket
29    /// Close frames) are sent rather than skipped by an abrupt drop. It takes
30    /// `&self` because a provider's reusable transport state lives behind its
31    /// own synchronization; a shared clone can therefore be closed from the
32    /// shutdown path. The default is a no-op: providers that hold no reusable
33    /// transport state have nothing to release.
34    async fn close(&self) -> Result<(), LlmTransportError> {
35        Ok(())
36    }
37
38    fn clone_boxed(&self) -> Box<dyn Provider>;
39}
40
41pub trait ProviderFailureClassifier: Send + Sync + std::fmt::Debug {
42    fn classify(&self, failure: ProviderFailure) -> ProviderFailure;
43}
44
45#[derive(Clone, Debug, Default)]
46pub struct DefaultProviderFailureClassifier;
47
48impl ProviderFailureClassifier for DefaultProviderFailureClassifier {
49    fn classify(&self, mut failure: ProviderFailure) -> ProviderFailure {
50        if let Some(status) = failure.status.or_else(|| {
51            failure
52                .code
53                .as_deref()
54                .and_then(|code| code.parse::<u16>().ok())
55        }) {
56            failure.status = Some(status);
57            if failure.kind == ProviderFailureKind::Unknown {
58                failure.kind = ProviderFailureKind::Http;
59            }
60            failure.retryable = matches!(status, 408 | 409 | 425 | 429 | 500 | 502 | 503 | 504);
61            if status == 429 {
62                // Provider-side throttling. `Quota` + `retryable: true` is the
63                // combination `ProviderHandle`'s retry ladder defers to as a
64                // throttle; hard quota exhaustion (the text markers below)
65                // downgrades to `retryable: false`.
66                failure.kind = ProviderFailureKind::Quota;
67            } else if matches!(status, 401 | 403) {
68                failure.kind = ProviderFailureKind::Auth;
69            } else if matches!(status, 400 | 413 | 422) {
70                failure.kind = ProviderFailureKind::Validation;
71            }
72        } else if matches!(
73            failure.kind,
74            ProviderFailureKind::Transport | ProviderFailureKind::Timeout
75        ) {
76            failure.retryable = true;
77        }
78
79        let haystack = format!(
80            "{}\n{}\n{}",
81            failure.code.as_deref().unwrap_or_default(),
82            failure.message,
83            failure
84                .raw
85                .as_deref()
86                .map(String::as_str)
87                .unwrap_or_default()
88        )
89        .to_ascii_lowercase();
90        if is_context_overflow_text(&haystack) {
91            failure.kind = ProviderFailureKind::Validation;
92            failure.retryable = false;
93            failure.terminal_reason = LlmTerminalReason::ContextOverflow;
94        }
95        // Hard exhaustion only. A bare "quota" match is too broad: Google
96        // phrases ordinary per-minute throttling as "Quota exceeded for quota
97        // metric ... per minute", which arrives as a retryable 429 and must
98        // stay retryable — downgrading it also loses the throttle-deference
99        // path, which requires `Quota` + `retryable: true`.
100        // A transient Google 429 with none of the RetryInfo, per-metric, or
101        // textual retry markers is indistinguishable here and will therefore
102        // be classified as terminal by this heuristic.
103        let google_hard_quota = haystack.contains("you exceeded your current quota")
104            && !haystack.contains("quota exceeded for metric")
105            && !haystack.contains("retrydelay")
106            && !haystack.contains("please retry");
107        if haystack.contains("insufficient_quota")
108            || haystack.contains("usage_limit_reached")
109            || haystack.contains("usage_not_included")
110            || haystack.contains("credit balance is too low")
111            || google_hard_quota
112        {
113            failure.kind = ProviderFailureKind::Quota;
114            failure.retryable = false;
115        }
116        if haystack.contains("content_filter")
117            || haystack.contains("prohibited_content")
118            || haystack.contains("safety")
119            || haystack.contains("sensitive")
120        {
121            failure.terminal_reason = LlmTerminalReason::ContentFilter;
122        }
123        if haystack.contains("model_not_found")
124            || haystack.contains("unsupported model")
125            || haystack.contains("does not exist")
126        {
127            failure.kind = ProviderFailureKind::Unsupported;
128            failure.retryable = false;
129        }
130        failure
131    }
132}
133
134pub fn is_context_overflow_text(haystack: &str) -> bool {
135    let lower = haystack.to_ascii_lowercase();
136    if lower.contains("rate limit")
137        || lower.contains("rate_limit")
138        || lower.contains("ratelimit")
139        || lower.contains("throttle")
140        || lower.contains("throttling")
141        || lower.contains("too many requests")
142        || lower.contains("tokens per minute")
143        || lower.contains("tpm")
144        || lower.contains("quota")
145    {
146        return false;
147    }
148
149    lower.contains("context_length_exceeded")
150        || lower.contains("context_length")
151        || lower.contains("context length")
152        || lower.contains("maximum context")
153        || lower.contains("max context")
154        || lower.contains("context window")
155        || lower.contains("context window exceeds limit")
156        || lower.contains("exceeds the context window")
157        || lower.contains("prompt is too long")
158        || lower.contains("prompt too long")
159        || lower.contains("request_too_large")
160        || lower.contains("input token count") && lower.contains("exceeds the maximum")
161        || lower.contains("maximum prompt length is")
162        || lower.contains("reduce the length of the messages")
163        || lower.contains("maximum context length is")
164        || lower.contains("model's context length")
165        || lower.contains("models context length")
166        || lower.contains("exceeds the available context size")
167        || lower.contains("greater than the context length")
168        || lower.contains("exceeded model token limit")
169        || lower.contains("too large for model with")
170        || lower.contains("model_context_window_exceeded")
171        || lower.contains("too many tokens")
172        || lower.contains("exceeds the maximum number of tokens")
173        || lower.contains("exceeds maximum number of tokens")
174        || lower.contains("request too large")
175        || lower.contains("input is too long")
176        || lower.contains("token limit exceeded")
177        || lower.contains("reduce the length of the messages")
178        || lower.contains("reduce the length of your prompt")
179}
lash_core/provider/traits.rs

lash_core/provider/
traits.rs
