lash_core/provider/
options.rs

1use super::support::*;
2
3pub const DEFAULT_REQUEST_TIMEOUT_MS: u64 = 300_000;
4pub const DEFAULT_CHUNK_TIMEOUT_MS: u64 = 120_000;
5pub const DEFAULT_THROTTLE_WAIT_BUDGET_MS: u64 = 90_000;
6
7/// Minimum amount a single deferred throttle wait charges against
8/// [`ProviderRetryPolicy::throttle_wait_budget_ms`]. Charging at least this
9/// much per deference bounds the courtesy in count as well as time: a zero or
10/// near-zero `Retry-After` (or a zero [`retry_after_cap_ms`]) cannot spin the
11/// attempt-free retry loop more than `budget / charge` times.
12///
13/// [`retry_after_cap_ms`]: ProviderRetryPolicy::retry_after_cap_ms
14pub(crate) const MIN_THROTTLE_BUDGET_CHARGE: Duration = Duration::from_secs(1);
15
16#[derive(Clone, Copy, Debug, PartialEq, Eq)]
17pub struct LlmTimeouts {
18    pub request_timeout: Option<Duration>,
19    pub chunk_timeout: Duration,
20}
21
22impl Default for LlmTimeouts {
23    fn default() -> Self {
24        Self {
25            request_timeout: Some(Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MS)),
26            chunk_timeout: Duration::from_millis(DEFAULT_CHUNK_TIMEOUT_MS),
27        }
28    }
29}
30
31#[derive(Clone, Copy, Debug, PartialEq, Eq)]
32pub enum RequestTimeout {
33    Disabled,
34    Millis(u64),
35}
36
37impl Serialize for RequestTimeout {
38    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
39    where
40        S: Serializer,
41    {
42        match self {
43            Self::Disabled => serializer.serialize_bool(false),
44            Self::Millis(value) => serializer.serialize_u64(*value),
45        }
46    }
47}
48
49impl<'de> Deserialize<'de> for RequestTimeout {
50    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
51    where
52        D: Deserializer<'de>,
53    {
54        struct RequestTimeoutVisitor;
55
56        impl Visitor<'_> for RequestTimeoutVisitor {
57            type Value = RequestTimeout;
58
59            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
60                formatter.write_str("a positive timeout in milliseconds or false")
61            }
62
63            fn visit_bool<E>(self, value: bool) -> Result<Self::Value, E>
64            where
65                E: de::Error,
66            {
67                if value {
68                    return Err(E::custom("timeout must be a positive integer or false"));
69                }
70                Ok(RequestTimeout::Disabled)
71            }
72
73            fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
74            where
75                E: de::Error,
76            {
77                if value == 0 {
78                    return Err(E::custom("timeout must be greater than 0"));
79                }
80                Ok(RequestTimeout::Millis(value))
81            }
82
83            fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>
84            where
85                E: de::Error,
86            {
87                if value <= 0 {
88                    return Err(E::custom("timeout must be greater than 0"));
89                }
90                Ok(RequestTimeout::Millis(value as u64))
91            }
92        }
93
94        deserializer.deserialize_any(RequestTimeoutVisitor)
95    }
96}
97
98/// Prompt-cache lifetime hint. Providers translate this into their own
99/// wire dialect (Anthropic and OpenRouter Claude/Gemini `cache_control`,
100/// OpenAI Responses and Codex `prompt_cache_key`, and OpenAI
101/// `prompt_cache_retention`). Providers without a cache-control concept,
102/// such as direct Google, read the value but emit nothing for it.
103#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
104#[serde(rename_all = "kebab-case")]
105pub enum CacheRetention {
106    /// Do not emit any prompt-cache hints.
107    None,
108    /// Default Anthropic ephemeral window (5 minutes).
109    #[default]
110    Short,
111    /// Extend to a 1-hour TTL where the API supports it.
112    Long,
113}
114
115impl CacheRetention {
116    pub fn is_default(&self) -> bool {
117        matches!(self, CacheRetention::Short)
118    }
119}
120
121#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
122#[serde(deny_unknown_fields)]
123pub struct ProviderOptions {
124    #[serde(default)]
125    pub reliability: ProviderReliability,
126    /// Surface provider reasoning/thinking output in responses.
127    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
128    pub expose_thinking: bool,
129    /// Per-request output-token cap. `None` lets each provider apply its
130    /// own default. Providers translate to their wire-specific field
131    /// (`max_tokens`, `max_output_tokens`, `maxOutputTokens`, …).
132    #[serde(default, skip_serializing_if = "Option::is_none")]
133    pub max_output_tokens: Option<u64>,
134    /// Prompt-cache lifetime hint; see [`CacheRetention`].
135    #[serde(default, skip_serializing_if = "CacheRetention::is_default")]
136    pub cache_retention: CacheRetention,
137}
138
139impl ProviderOptions {
140    pub fn is_default(&self) -> bool {
141        self.reliability == ProviderReliability::default()
142            && !self.expose_thinking
143            && self.max_output_tokens.is_none()
144            && self.cache_retention.is_default()
145    }
146
147    pub fn llm_timeouts(&self) -> LlmTimeouts {
148        self.reliability.llm_timeouts()
149    }
150}
151
152#[derive(Clone, Debug, PartialEq, Eq)]
153pub struct ResolvedGenerationPolicy<TThinking> {
154    pub max_output_tokens: u64,
155    pub cache_retention: CacheRetention,
156    pub expose_thinking: bool,
157    pub thinking: TThinking,
158}
159
160pub fn resolve_generation_policy<TThinking>(
161    generation: &crate::GenerationOptions,
162    options: &ProviderOptions,
163    provider_default_max_output_tokens: u64,
164    thinking: TThinking,
165) -> ResolvedGenerationPolicy<TThinking> {
166    let max_output_tokens = generation
167        .output_token_cap_u64()
168        .or(options.max_output_tokens)
169        .unwrap_or(provider_default_max_output_tokens);
170    ResolvedGenerationPolicy {
171        max_output_tokens,
172        cache_retention: options.cache_retention,
173        expose_thinking: options.expose_thinking,
174        thinking,
175    }
176}
177
178#[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq, Eq)]
179pub struct ProviderReliability {
180    /// Whole-request timeout. `None` applies [`DEFAULT_REQUEST_TIMEOUT_MS`];
181    /// use [`RequestTimeout::Disabled`] to wait indefinitely.
182    #[serde(default, skip_serializing_if = "Option::is_none")]
183    pub request_timeout: Option<RequestTimeout>,
184    /// Inter-chunk stream timeout in milliseconds. `None` (or `0`) applies
185    /// [`DEFAULT_CHUNK_TIMEOUT_MS`].
186    #[serde(default, skip_serializing_if = "Option::is_none")]
187    pub chunk_timeout: Option<u64>,
188    #[serde(default)]
189    pub retry: ProviderRetryPolicy,
190    #[serde(default)]
191    pub rate_limits: ProviderRateLimitPolicy,
192}
193
194impl ProviderReliability {
195    pub fn codex() -> Self {
196        Self {
197            retry: ProviderRetryPolicy {
198                max_attempts: 4,
199                base_delay_ms: 1_000,
200                max_delay_ms: 4_000,
201                jitter_ms: 0,
202                retry_after_cap_ms: Some(60_000),
203                throttle_wait_budget_ms: DEFAULT_THROTTLE_WAIT_BUDGET_MS,
204                enabled: true,
205            },
206            ..Self::default()
207        }
208    }
209
210    pub fn disabled() -> Self {
211        Self {
212            retry: ProviderRetryPolicy::disabled(),
213            ..Self::default()
214        }
215    }
216
217    pub fn llm_timeouts(&self) -> LlmTimeouts {
218        let request_timeout = match self.request_timeout {
219            Some(RequestTimeout::Disabled) => None,
220            Some(RequestTimeout::Millis(ms)) => Some(Duration::from_millis(ms)),
221            None => Some(Duration::from_millis(DEFAULT_REQUEST_TIMEOUT_MS)),
222        };
223        let chunk_timeout_ms = self
224            .chunk_timeout
225            .filter(|value| *value > 0)
226            .unwrap_or(DEFAULT_CHUNK_TIMEOUT_MS);
227        LlmTimeouts {
228            request_timeout,
229            chunk_timeout: Duration::from_millis(chunk_timeout_ms),
230        }
231    }
232
233    pub fn request_timeout(mut self, timeout: Option<RequestTimeout>) -> Self {
234        self.request_timeout = timeout;
235        self
236    }
237
238    pub fn stream_chunk_timeout_ms(mut self, timeout_ms: Option<u64>) -> Self {
239        self.chunk_timeout = timeout_ms;
240        self
241    }
242
243    pub fn max_attempts(mut self, attempts: u32) -> Self {
244        self.retry.max_attempts = attempts.max(1);
245        self
246    }
247
248    pub fn base_delay_ms(mut self, delay_ms: u64) -> Self {
249        self.retry.base_delay_ms = delay_ms;
250        self
251    }
252
253    pub fn max_delay_ms(mut self, delay_ms: u64) -> Self {
254        self.retry.max_delay_ms = delay_ms;
255        self
256    }
257
258    pub fn retry_after_cap_ms(mut self, cap_ms: Option<u64>) -> Self {
259        self.retry.retry_after_cap_ms = cap_ms;
260        self
261    }
262
263    pub fn throttle_wait_budget_ms(mut self, budget_ms: u64) -> Self {
264        self.retry.throttle_wait_budget_ms = budget_ms;
265        self
266    }
267
268    pub fn max_concurrency(mut self, value: Option<usize>) -> Self {
269        self.rate_limits.max_concurrency = value;
270        self
271    }
272
273    pub fn requests_per_window(mut self, requests: Option<u32>, window_ms: Option<u64>) -> Self {
274        self.rate_limits.requests_per_window = requests;
275        self.rate_limits.request_window_ms = window_ms;
276        self
277    }
278
279    pub fn tokens_per_window(mut self, tokens: Option<u32>, window_ms: Option<u64>) -> Self {
280        self.rate_limits.tokens_per_window = tokens;
281        self.rate_limits.token_window_ms = window_ms;
282        self
283    }
284}
285
286#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
287pub struct ProviderRetryPolicy {
288    pub enabled: bool,
289    pub max_attempts: u32,
290    pub base_delay_ms: u64,
291    pub max_delay_ms: u64,
292    pub jitter_ms: u64,
293    #[serde(default, skip_serializing_if = "Option::is_none")]
294    pub retry_after_cap_ms: Option<u64>,
295    /// Cumulative time [`ProviderHandle::complete`] may spend honoring
296    /// provider throttle waits — a retryable [`ProviderFailureKind::Quota`]
297    /// failure carrying `Retry-After` — without consuming retry attempts.
298    /// Each deferred wait charges what it actually waits, with a one-second
299    /// floor per deference, so the courtesy is bounded in count as well as
300    /// time. Once the budget is spent, throttled failures consume attempts
301    /// like any other retryable failure. `0` disables the deference entirely.
302    #[serde(
303        default = "default_throttle_wait_budget_ms",
304        skip_serializing_if = "is_default_throttle_wait_budget_ms"
305    )]
306    pub throttle_wait_budget_ms: u64,
307}
308
309fn default_throttle_wait_budget_ms() -> u64 {
310    DEFAULT_THROTTLE_WAIT_BUDGET_MS
311}
312
313fn is_default_throttle_wait_budget_ms(budget_ms: &u64) -> bool {
314    *budget_ms == DEFAULT_THROTTLE_WAIT_BUDGET_MS
315}
316
317impl Default for ProviderRetryPolicy {
318    fn default() -> Self {
319        Self {
320            enabled: true,
321            max_attempts: 4,
322            base_delay_ms: 2_000,
323            max_delay_ms: 10_000,
324            jitter_ms: 0,
325            retry_after_cap_ms: Some(60_000),
326            throttle_wait_budget_ms: DEFAULT_THROTTLE_WAIT_BUDGET_MS,
327        }
328    }
329}
330
331impl ProviderRetryPolicy {
332    pub fn disabled() -> Self {
333        Self {
334            enabled: false,
335            max_attempts: 1,
336            base_delay_ms: 0,
337            max_delay_ms: 0,
338            jitter_ms: 0,
339            retry_after_cap_ms: None,
340            throttle_wait_budget_ms: 0,
341        }
342    }
343
344    pub(crate) fn attempts(&self) -> u32 {
345        if self.enabled {
346            self.max_attempts.max(1)
347        } else {
348            1
349        }
350    }
351
352    /// A provider-stated `Retry-After`, bounded by
353    /// [`retry_after_cap_ms`](Self::retry_after_cap_ms) when one is set.
354    pub(crate) fn cap_retry_after(&self, retry_after: Duration) -> Duration {
355        self.retry_after_cap_ms
356            .map(Duration::from_millis)
357            .map(|cap| retry_after.min(cap))
358            .unwrap_or(retry_after)
359    }
360
361    pub(crate) fn delay_for_attempt(
362        &self,
363        retry_index: u32,
364        retry_after: Option<Duration>,
365    ) -> Duration {
366        if let Some(retry_after) = retry_after {
367            return self.cap_retry_after(retry_after);
368        }
369        let multiplier = 1u64.checked_shl(retry_index).unwrap_or(u64::MAX);
370        let delay_ms = self
371            .base_delay_ms
372            .saturating_mul(multiplier)
373            .min(self.max_delay_ms);
374        Duration::from_millis(delay_ms.saturating_add(self.jitter_ms))
375    }
376}
377
378#[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq, Eq)]
379pub struct ProviderRateLimitPolicy {
380    #[serde(default, skip_serializing_if = "Option::is_none")]
381    pub max_concurrency: Option<usize>,
382    #[serde(default, skip_serializing_if = "Option::is_none")]
383    pub requests_per_window: Option<u32>,
384    #[serde(default, skip_serializing_if = "Option::is_none")]
385    pub request_window_ms: Option<u64>,
386    #[serde(default, skip_serializing_if = "Option::is_none")]
387    pub tokens_per_window: Option<u32>,
388    #[serde(default, skip_serializing_if = "Option::is_none")]
389    pub token_window_ms: Option<u64>,
390}
lash_core/provider/options.rs

lash_core/provider/
options.rs
