Skip to main content

rusty_agent_runtime/
llm.rs

1//! LLM abstraction and an OpenAI-compatible chat client.
2//!
3//! [`ChatModel`] is the minimal async chat-completion interface used by
4//! agent nodes (the prebuilt ReAct agent only needs `chat`). Messages use
5//! the OpenAI wire conventions: roles, assistant `tool_calls`, and tool
6//! results carried by `role: "tool"` messages with `tool_call_id`.
7
8use std::time::Duration;
9
10use async_trait::async_trait;
11use serde::{Deserialize, Deserializer, Serialize, Serializer};
12use serde_json::{json, Value};
13
14use crate::error::{Result, RustyError};
15
16/// Maximum characters of an HTTP error body embedded in an error message:
17/// enough for diagnosis, bounded so a verbose server cannot bloat logs.
18/// Shared with `crate::remote`.
19pub(crate) const ERROR_BODY_MAX_CHARS: usize = 512;
20
21/// Truncate a response body for inclusion in an error message.
22pub(crate) fn truncate_body(body: &str) -> String {
23    body.chars().take(ERROR_BODY_MAX_CHARS).collect()
24}
25
26/// Exponential backoff `base * 2^attempt` with the exponent capped at 6 and
27/// cheap time-based jitter (×[0.5, 1.5)) so concurrent retry loops
28/// decorrelate without pulling in a `rand` dependency. Shared with
29/// `crate::remote`.
30pub(crate) fn backoff_delay(base: Duration, attempt: u32) -> Duration {
31    let base = base.saturating_mul(2u32.saturating_pow(attempt.min(6)));
32    let nanos = std::time::SystemTime::now()
33        .duration_since(std::time::UNIX_EPOCH)
34        .map(|d| d.subsec_nanos())
35        .unwrap_or(0);
36    let jitter = 0.5 + f64::from(nanos % 1000) / 1000.0; // [0.5, 1.5)
37    let jittered = (base.as_nanos() as f64 * jitter) as u128;
38    Duration::from_nanos(jittered.min(u64::MAX as u128) as u64)
39}
40
41/// Chat message role.
42#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
43#[serde(rename_all = "lowercase")]
44pub enum Role {
45    /// System prompt / instructions.
46    System,
47    /// End-user input.
48    User,
49    /// Model output (may carry tool calls).
50    Assistant,
51    /// Tool execution result (must carry `tool_call_id`).
52    Tool,
53}
54
55/// A single chat message.
56///
57/// Serialization follows the OpenAI chat-completions schema:
58/// `content` may be null on assistant tool-call messages; `tool_calls` and
59/// `tool_call_id` are omitted when absent.
60#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
61pub struct ChatMessage {
62    /// Who produced this message.
63    pub role: Role,
64
65    /// Text content. `None` is legal for assistant messages that only carry
66    /// tool calls.
67    #[serde(default, skip_serializing_if = "Option::is_none")]
68    pub content: Option<String>,
69
70    /// Tool calls requested by the assistant.
71    #[serde(default, skip_serializing_if = "Vec::is_empty")]
72    pub tool_calls: Vec<ToolCall>,
73
74    /// Required on `role: tool` messages: the tool call this answers.
75    #[serde(default, skip_serializing_if = "Option::is_none")]
76    pub tool_call_id: Option<String>,
77
78    /// Optional participant name (multi-agent disambiguation).
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub name: Option<String>,
81}
82
83impl ChatMessage {
84    /// A system message.
85    pub fn system(content: impl Into<String>) -> Self {
86        Self {
87            role: Role::System,
88            content: Some(content.into()),
89            tool_calls: Vec::new(),
90            tool_call_id: None,
91            name: None,
92        }
93    }
94
95    /// A user message.
96    pub fn user(content: impl Into<String>) -> Self {
97        Self {
98            role: Role::User,
99            content: Some(content.into()),
100            tool_calls: Vec::new(),
101            tool_call_id: None,
102            name: None,
103        }
104    }
105
106    /// An assistant message with text content.
107    pub fn assistant(content: impl Into<String>) -> Self {
108        Self {
109            role: Role::Assistant,
110            content: Some(content.into()),
111            tool_calls: Vec::new(),
112            tool_call_id: None,
113            name: None,
114        }
115    }
116
117    /// An assistant message requesting tool calls (content may be empty).
118    pub fn assistant_tool_calls(tool_calls: Vec<ToolCall>) -> Self {
119        Self {
120            role: Role::Assistant,
121            content: None,
122            tool_calls,
123            tool_call_id: None,
124            name: None,
125        }
126    }
127
128    /// A tool-result message answering `tool_call_id`.
129    pub fn tool_result(tool_call_id: impl Into<String>, content: impl Into<String>) -> Self {
130        Self {
131            role: Role::Tool,
132            content: Some(content.into()),
133            tool_calls: Vec::new(),
134            tool_call_id: Some(tool_call_id.into()),
135            name: None,
136        }
137    }
138
139    /// `true` if this is an assistant message requesting tool calls.
140    pub fn has_tool_calls(&self) -> bool {
141        !self.tool_calls.is_empty()
142    }
143}
144
145/// A tool call requested by the model.
146///
147/// Wire format is the OpenAI function-calling shape:
148/// `{"id": "...", "type": "function", "function": {"name": "...", "arguments": "<json string>"}}`.
149/// The `arguments` field is exposed as a parsed [`Value`]; serialization
150/// re-encodes it to the string form the API expects.
151#[derive(Debug, Clone, PartialEq)]
152pub struct ToolCall {
153    /// Provider-assigned call id (echoed back in the tool-result message).
154    pub id: String,
155    /// Tool name (must match a registered [`crate::tool::Tool::name`]).
156    pub name: String,
157    /// Parsed arguments.
158    pub arguments: Value,
159}
160
161impl ToolCall {
162    /// Convenience constructor.
163    pub fn new(id: impl Into<String>, name: impl Into<String>, arguments: Value) -> Self {
164        Self {
165            id: id.into(),
166            name: name.into(),
167            arguments,
168        }
169    }
170}
171
172impl Serialize for ToolCall {
173    fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
174        json!({
175            "id": self.id,
176            "type": "function",
177            "function": {
178                "name": self.name,
179                "arguments": self.arguments.to_string(),
180            }
181        })
182        .serialize(serializer)
183    }
184}
185
186impl<'de> Deserialize<'de> for ToolCall {
187    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
188        #[derive(Deserialize)]
189        struct FnArgs {
190            name: String,
191            arguments: Value,
192        }
193        #[derive(Deserialize)]
194        struct Wire {
195            id: String,
196            function: FnArgs,
197        }
198        let wire = Wire::deserialize(deserializer)?;
199        let arguments = match wire.function.arguments {
200            // Standard: arguments arrive as a JSON-encoded string.
201            Value::String(s) => serde_json::from_str(&s).map_err(serde::de::Error::custom)?,
202            // Lenient: some providers send a raw object.
203            other => other,
204        };
205        Ok(ToolCall {
206            id: wire.id,
207            name: wire.function.name,
208            arguments,
209        })
210    }
211}
212
213/// Token usage accounting from the provider.
214#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
215pub struct Usage {
216    /// Tokens in the prompt.
217    #[serde(default)]
218    pub prompt_tokens: u64,
219    /// Tokens in the completion.
220    #[serde(default)]
221    pub completion_tokens: u64,
222    /// Total tokens billed.
223    #[serde(default)]
224    pub total_tokens: u64,
225}
226
227/// One chat-completion response (single choice; multi-choice responses are
228/// not modeled).
229#[derive(Debug, Clone)]
230pub struct ChatResponse {
231    /// The assistant message (text and/or tool calls).
232    pub message: ChatMessage,
233    /// The model that produced the response, when reported.
234    pub model: Option<String>,
235    /// Token usage, when reported.
236    pub usage: Option<Usage>,
237}
238
239/// One incremental piece of a streamed completion (the LangGraph `messages`
240/// stream-mode analog at the model level).
241///
242/// `on_token` callbacks of [`ChatModel::chat_stream`] receive a sequence of
243/// `TokenChunk`s: zero or more with `finish: false` carrying text deltas,
244/// terminated by exactly one with `finish: true` (whose `delta` is empty for
245/// truly-streaming implementations).
246#[derive(Debug, Clone)]
247pub struct TokenChunk {
248    /// The incremental text produced since the previous chunk. May be empty
249    /// (e.g. on the terminal chunk, or on chunks that only carry tool-call
250    /// deltas).
251    pub delta: String,
252
253    /// `true` on the final chunk of the stream.
254    pub finish: bool,
255
256    /// The raw provider chunk (the decoded SSE `data:` JSON), when the
257    /// implementation streams from a wire protocol. `None` for synthetic
258    /// chunks (default fallback, mocks).
259    pub raw: Option<Value>,
260}
261
262/// The chat-model interface used by agent nodes.
263///
264/// `tools` are OpenAI-format tool schemas (`{"type": "function", "function":
265/// {...}}`); pass an empty slice for a plain completion. See
266/// [`crate::tool::ToolRegistry::schemas`].
267///
268/// # Streaming tokens into the executor's event channel
269///
270/// [`ChatModel::chat_stream`] is pull-based: the `on_token` callback fires
271/// once per token delta. To surface those deltas as
272/// [`crate::executor::GraphEvent::Token`]s (the LangGraph `messages` stream
273/// mode), clone the run's event sender into the node closure and forward
274/// each chunk — the executor's `event_tx` channel is the shared sink:
275///
276/// ```ignore
277/// use rusty_agent_runtime::executor::{GraphEvent, RunConfig};
278/// use rusty_agent_runtime::llm::{ChatModel, ChatMessage, TokenChunk};
279///
280/// let (tx, mut rx) = tokio::sync::mpsc::channel::<GraphEvent>(64);
281/// let node_tx = tx.clone();                  // captured by the node closure
282/// let config = RunConfig::new("t-1").with_event_tx(tx);
283/// // Convenience handles for wiring the clone into nodes:
284/// //   RunConfig::token_tx()  -> Option<mpsc::Sender<GraphEvent>>
285/// //   Executor::with_token_tx(tx) / Executor::token_tx()
286///
287/// // ...inside the node:
288/// // let response = model
289/// //     .chat_stream(&messages, &tools, &mut |chunk: TokenChunk| {
290/// //         if !chunk.delta.is_empty() {
291/// //             let _ = node_tx.try_send(GraphEvent::Token {
292/// //                 node: "agent".into(),
293/// //                 delta: chunk.delta,
294/// //             });
295/// //         }
296/// //     })
297/// //     .await?;
298/// ```
299///
300/// Forwarding uses `try_send` (best-effort), matching the executor's own
301/// emission policy: a full or closed channel drops tokens but never aborts
302/// the run.
303///
304/// The prebuilt ReAct agent comes in two flavors for exactly this wiring:
305/// [`crate::react::create_react_agent`] never streams (it calls
306/// [`ChatModel::chat`], so no `Token` events can fire), while
307/// [`crate::react::create_react_agent_streaming`] performs the forwarding
308/// above internally.
309#[async_trait]
310pub trait ChatModel: Send + Sync {
311    /// Produce the next assistant message for the conversation.
312    async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse>;
313
314    /// The declared effect classification of calling this model (Flight
315    /// Recorder, R0.5): recorded on model-call journal events and used by
316    /// retry/replay policy.
317    ///
318    /// The default is [`crate::record::Effect::NonIdempotent`]: a provider
319    /// call is billable and unverifiable, so the restrictive class applies.
320    /// Override only with justification (e.g. a local deterministic model
321    /// could argue for `ReadOnly`; cached completions for `Idempotent`).
322    fn effect(&self) -> crate::record::Effect {
323        crate::record::Effect::NonIdempotent
324    }
325
326    /// Produce the next assistant message, streaming token deltas through
327    /// `on_token` as they arrive.
328    ///
329    /// The default implementation falls back to [`ChatModel::chat`] and
330    /// delivers the whole assistant text as a single [`TokenChunk`] with
331    /// `finish: true`, so existing implementors remain source-compatible.
332    /// Implementations with a streaming wire protocol (e.g.
333    /// [`OpenAiCompatibleClient`]) override this to deliver real deltas.
334    async fn chat_stream(
335        &self,
336        messages: &[ChatMessage],
337        tools: &[Value],
338        on_token: &mut (dyn FnMut(TokenChunk) + Send),
339    ) -> Result<ChatResponse> {
340        let response = self.chat(messages, tools).await?;
341        on_token(TokenChunk {
342            delta: response.message.content.clone().unwrap_or_default(),
343            finish: true,
344            raw: None,
345        });
346        Ok(response)
347    }
348}
349
350/// Default number of retries *after* the initial attempt for transient
351/// failures (connect errors, timeouts, HTTP 5xx / 408 / 429).
352pub const DEFAULT_MAX_RETRIES: u32 = 2;
353
354/// Default base delay for exponential backoff between retries (exponent
355/// capped, jittered; see [`OpenAiCompatibleClient::with_backoff`]).
356pub const DEFAULT_BASE_BACKOFF: Duration = Duration::from_millis(100);
357
358/// A client for any OpenAI-compatible `/chat/completions` endpoint (OpenAI,
359/// Azure-OpenAI-compatible gateways, vLLM, Ollama, LM Studio, ...).
360///
361/// Uses `reqwest` with rustls; no default TLS features.
362///
363/// Transient failures are retried with capped, jittered exponential backoff
364/// (same classification policy as `crate::remote::RemoteNode`): connect
365/// errors, timeouts, and HTTP 5xx / 408 / 429 are retryable; a `Retry-After`
366/// header (integer seconds) floors the delay. Other 4xx statuses and
367/// request/response decode errors are fatal. Configure with
368/// [`OpenAiCompatibleClient::with_retries`] /
369/// [`OpenAiCompatibleClient::with_backoff`].
370#[derive(Clone)]
371pub struct OpenAiCompatibleClient {
372    base_url: String,
373    api_key: Option<String>,
374    model: String,
375    client: reqwest::Client,
376    /// Retries after the initial attempt.
377    max_retries: u32,
378    /// Base delay for exponential backoff.
379    base_backoff: Duration,
380}
381
382// Hand-written because the derived impl would print `api_key` in cleartext
383// on any `{:?}` (logging middleware, panic messages, `dbg!`).
384impl std::fmt::Debug for OpenAiCompatibleClient {
385    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
386        f.debug_struct("OpenAiCompatibleClient")
387            .field("base_url", &self.base_url)
388            .field("api_key", &self.api_key.as_ref().map(|_| "***"))
389            .field("model", &self.model)
390            .field("client", &self.client)
391            .field("max_retries", &self.max_retries)
392            .field("base_backoff", &self.base_backoff)
393            .finish()
394    }
395}
396
397impl OpenAiCompatibleClient {
398    /// A client for `base_url` (e.g. `https://api.openai.com/v1`) serving
399    /// `model`. Trailing slashes on `base_url` are trimmed.
400    pub fn new(
401        base_url: impl Into<String>,
402        api_key: Option<String>,
403        model: impl Into<String>,
404    ) -> Self {
405        Self {
406            base_url: base_url.into().trim_end_matches('/').to_string(),
407            api_key,
408            model: model.into(),
409            client: reqwest::Client::new(),
410            max_retries: DEFAULT_MAX_RETRIES,
411            base_backoff: DEFAULT_BASE_BACKOFF,
412        }
413    }
414
415    /// Read the API key from an environment variable.
416    ///
417    /// A missing variable maps to "no key" (requests go out unauthenticated
418    /// and typically fail 401 far from the cause), so a warning is logged at
419    /// construction time to keep the cause close to the effect.
420    pub fn from_env(
421        base_url: impl Into<String>,
422        api_key_env: &str,
423        model: impl Into<String>,
424    ) -> Self {
425        let api_key = std::env::var(api_key_env).ok();
426        if api_key.is_none() {
427            tracing::warn!(
428                env_var = api_key_env,
429                "API key environment variable is not set; sending requests unauthenticated"
430            );
431        }
432        Self::new(base_url, api_key, model)
433    }
434
435    /// Override the underlying `reqwest::Client` (timeouts, proxies, ...).
436    pub fn with_http_client(mut self, client: reqwest::Client) -> Self {
437        self.client = client;
438        self
439    }
440
441    /// Override the number of retries after the initial attempt
442    /// (`0` = single attempt, no retries).
443    pub fn with_retries(mut self, max_retries: u32) -> Self {
444        self.max_retries = max_retries;
445        self
446    }
447
448    /// Override the base backoff delay (attempt *n* waits roughly
449    /// `base * 2^n`, exponent capped and jittered).
450    pub fn with_backoff(mut self, base_backoff: Duration) -> Self {
451        self.base_backoff = base_backoff;
452        self
453    }
454
455    /// The model this client requests.
456    pub fn model(&self) -> &str {
457        &self.model
458    }
459
460    fn endpoint(&self) -> String {
461        format!("{}/chat/completions", self.base_url)
462    }
463
464    /// Send the request produced by `build()` with the configured retry
465    /// policy. Only the initial request/response exchange is covered; once a
466    /// 2xx response is returned, streaming-read failures are not retried.
467    async fn send_with_retries(
468        &self,
469        build: impl Fn() -> reqwest::RequestBuilder,
470    ) -> Result<reqwest::Response> {
471        let mut attempt: u32 = 0;
472        loop {
473            match self.try_send(build()).await {
474                Ok(response) => return Ok(response),
475                Err(AttemptError::Fatal(e)) => return Err(e),
476                Err(AttemptError::Retryable { error, retry_after })
477                    if attempt < self.max_retries =>
478                {
479                    let mut delay = backoff_delay(self.base_backoff, attempt);
480                    if let Some(floor) = retry_after {
481                        // A server that says Retry-After knows better than
482                        // our backoff guess.
483                        delay = delay.max(floor);
484                    }
485                    tracing::warn!(
486                        url = %self.base_url,
487                        attempt = attempt + 1,
488                        max_retries = self.max_retries,
489                        backoff_ms = delay.as_millis() as u64,
490                        error = %error,
491                        "chat completions attempt failed; retrying"
492                    );
493                    tokio::time::sleep(delay).await;
494                    attempt += 1;
495                }
496                Err(AttemptError::Retryable { error, .. }) => return Err(error),
497            }
498        }
499    }
500
501    /// One HTTP attempt. `Ok` means a 2xx response (the body may still be
502    /// streamed); `Err` is classified for retry.
503    async fn try_send(
504        &self,
505        request: reqwest::RequestBuilder,
506    ) -> std::result::Result<reqwest::Response, AttemptError> {
507        let response = request.send().await.map_err(|e| {
508            let err = RustyError::Llm(format!("request to {} failed: {e}", self.base_url));
509            if e.is_timeout() || e.is_connect() {
510                AttemptError::Retryable {
511                    error: err,
512                    retry_after: None,
513                }
514            } else {
515                AttemptError::Fatal(err)
516            }
517        })?;
518
519        let status = response.status();
520        if status.is_success() {
521            return Ok(response);
522        }
523        let retry_after = response
524            .headers()
525            .get(reqwest::header::RETRY_AFTER)
526            .and_then(|v| v.to_str().ok())
527            .and_then(|s| s.trim().parse::<u64>().ok())
528            .map(Duration::from_secs);
529        let body = response.text().await.unwrap_or_default();
530        let err = RustyError::Llm(format!(
531            "chat completions returned {status}: {}",
532            truncate_body(&body)
533        ));
534        // 5xx and 408/429 are transient by convention; other 4xx are
535        // definitive (bad request, auth failure, ...).
536        let retryable = status.is_server_error()
537            || status == reqwest::StatusCode::REQUEST_TIMEOUT
538            || status == reqwest::StatusCode::TOO_MANY_REQUESTS;
539        Err(if retryable {
540            AttemptError::Retryable {
541                error: err,
542                retry_after,
543            }
544        } else {
545            AttemptError::Fatal(err)
546        })
547    }
548}
549
550/// Internal classification of a failed HTTP attempt (mirrors
551/// `crate::remote::AttemptError`).
552#[derive(Debug)]
553enum AttemptError {
554    /// Transient failure eligible for retry (connect, timeout, 5xx/408/429),
555    /// with an optional server-provided `Retry-After` floor.
556    Retryable {
557        error: RustyError,
558        retry_after: Option<Duration>,
559    },
560    /// Definitive failure; never retried.
561    Fatal(RustyError),
562}
563
564/// Wire shape of one completion choice.
565#[derive(Deserialize)]
566struct WireChoice {
567    message: ChatMessage,
568}
569
570/// Wire shape of the completion response body.
571#[derive(Deserialize)]
572struct WireResponse {
573    choices: Vec<WireChoice>,
574    #[serde(default)]
575    model: Option<String>,
576    #[serde(default)]
577    usage: Option<Usage>,
578}
579
580/// Wire shape of one streaming chunk (`stream: true`).
581#[derive(Deserialize)]
582struct WireStreamChunk {
583    #[serde(default)]
584    choices: Vec<WireStreamChoice>,
585    #[serde(default)]
586    model: Option<String>,
587    #[serde(default)]
588    usage: Option<Usage>,
589}
590
591/// Wire shape of one streaming choice.
592///
593/// Note: the wire's `finish_reason` field is deliberately not modeled here.
594/// With `stream_options.include_usage`, OpenAI-compatible servers send the
595/// terminal usage chunk *after* the chunk whose choice carries
596/// `finish_reason: "stop"`, so terminating on `finish_reason` would drop
597/// usage accounting. Stream termination is instead driven by the `[DONE]`
598/// sentinel (with end-of-body as the fallback for providers that omit it).
599#[derive(Deserialize)]
600struct WireStreamChoice {
601    delta: WireStreamDelta,
602}
603
604/// Wire shape of the incremental delta inside a streaming chunk.
605#[derive(Deserialize)]
606struct WireStreamDelta {
607    #[serde(default)]
608    content: Option<String>,
609    #[serde(default)]
610    tool_calls: Option<Vec<WireToolCallDelta>>,
611}
612
613/// Wire shape of an incremental tool-call delta (indexed slots; `id`,
614/// `name`, and `arguments` arrive piecewise and are concatenated per index).
615#[derive(Deserialize)]
616struct WireToolCallDelta {
617    #[serde(default)]
618    index: usize,
619    #[serde(default)]
620    id: Option<String>,
621    #[serde(default)]
622    function: Option<WireFunctionDelta>,
623}
624
625/// Wire shape of the function fragment inside a tool-call delta.
626#[derive(Deserialize)]
627struct WireFunctionDelta {
628    #[serde(default)]
629    name: Option<String>,
630    #[serde(default)]
631    arguments: Option<String>,
632}
633
634/// Accumulates streaming deltas into a final [`ChatResponse`].
635#[derive(Default)]
636struct StreamAccumulator {
637    content: String,
638    tool_calls: Vec<ToolCallAccumulator>,
639    model: Option<String>,
640    usage: Option<Usage>,
641}
642
643/// Per-index accumulation of one streamed tool call.
644#[derive(Default)]
645struct ToolCallAccumulator {
646    id: String,
647    name: String,
648    arguments: String,
649}
650
651impl StreamAccumulator {
652    fn into_response(self) -> Result<ChatResponse> {
653        let mut tool_calls = Vec::with_capacity(self.tool_calls.len());
654        for (index, acc) in self.tool_calls.into_iter().enumerate() {
655            let arguments = if acc.arguments.trim().is_empty() {
656                // Note: a stream truncated mid-arguments also lands here —
657                // the accumulator cannot distinguish "the model sent no
658                // arguments" from "bytes were lost".
659                Value::Object(serde_json::Map::new())
660            } else {
661                serde_json::from_str(&acc.arguments).map_err(|e| {
662                    RustyError::Llm(format!(
663                        "malformed tool-call arguments in stream (index {index}): {e}"
664                    ))
665                })?
666            };
667            tool_calls.push(ToolCall::new(acc.id, acc.name, arguments));
668        }
669        let content = if self.content.is_empty() {
670            None
671        } else {
672            Some(self.content)
673        };
674        Ok(ChatResponse {
675            message: ChatMessage {
676                role: Role::Assistant,
677                content,
678                tool_calls,
679                tool_call_id: None,
680                name: None,
681            },
682            model: self.model,
683            usage: self.usage,
684        })
685    }
686}
687
688/// A minimal hand-rolled Server-Sent-Events decoder.
689///
690/// SSE is a line protocol over an arbitrarily chunked byte stream: events
691/// are separated by blank lines, each event's `data:` lines (possibly
692/// several) join with `\n` into one payload, `:`-prefixed lines are
693/// comments/heartbeats, and other fields (`event:`, `id:`, `retry:`) are
694/// ignored. [`SseDecoder::feed_bytes`] buffers partial lines *and* partial
695/// UTF-8 sequences across calls, so a `data:` line — or a single multi-byte
696/// character — split across TCP chunks still decodes correctly. One leading
697/// BOM is stripped, and a bare `data` line (no colon) is an empty-string
698/// field, both per the SSE spec.
699#[derive(Default)]
700struct SseDecoder {
701    /// Raw bytes held back because they end mid-UTF-8-sequence.
702    pending: Vec<u8>,
703    /// Decoded text received but not yet terminated by `\n`.
704    buf: String,
705    /// `data:` lines of the event currently being assembled.
706    data_lines: Vec<String>,
707    /// Whether any bytes have been consumed yet (for one-time BOM stripping).
708    started: bool,
709}
710
711impl SseDecoder {
712    fn new() -> Self {
713        Self::default()
714    }
715
716    /// Feed raw bytes as they arrive from the transport; returns the payloads
717    /// of all events completed by them (usually zero or one).
718    ///
719    /// Only complete UTF-8 is decoded: a trailing partial sequence is held in
720    /// `pending` for the next feed, so a chunk boundary can never corrupt
721    /// text into U+FFFD. Genuinely invalid bytes (not fixable by more data)
722    /// are replaced with U+FFFD.
723    fn feed_bytes(&mut self, bytes: &[u8]) -> Vec<String> {
724        self.pending.extend_from_slice(bytes);
725        if !self.started {
726            self.started = true;
727            // Spec: a single leading U+FEFF is ignored. (If the BOM itself is
728            // split across feeds, the tail decodes to U+FEFF in the first
729            // line, which the JSON parse of real providers never produces.)
730            if self.pending.starts_with(&[0xEF, 0xBB, 0xBF]) {
731                self.pending.drain(..3);
732            }
733        }
734        let mut text = String::new();
735        let mut consumed = 0;
736        while consumed < self.pending.len() {
737            match std::str::from_utf8(&self.pending[consumed..]) {
738                Ok(s) => {
739                    text.push_str(s);
740                    consumed = self.pending.len();
741                }
742                Err(e) => {
743                    let end = consumed + e.valid_up_to();
744                    text.push_str(
745                        std::str::from_utf8(&self.pending[consumed..end])
746                            .expect("bytes up to valid_up_to are valid UTF-8"),
747                    );
748                    match e.error_len() {
749                        Some(bad) => {
750                            text.push('\u{FFFD}');
751                            consumed = end + bad;
752                        }
753                        // Incomplete trailing sequence: wait for more bytes.
754                        None => {
755                            consumed = end;
756                            break;
757                        }
758                    }
759                }
760            }
761        }
762        self.pending.drain(..consumed);
763        self.feed(&text)
764    }
765
766    /// Feed a text fragment; returns the payloads of all events completed by
767    /// it (usually zero or one).
768    fn feed(&mut self, chunk: &str) -> Vec<String> {
769        self.buf.push_str(chunk);
770        let mut events = Vec::new();
771        while let Some(pos) = self.buf.find('\n') {
772            let line: String = self.buf.drain(..=pos).collect();
773            if let Some(payload) = self.process_line(line.trim_end_matches(['\n', '\r'])) {
774                events.push(payload);
775            }
776        }
777        events
778    }
779
780    /// Flush any undecoded trailing bytes (lossily), any unterminated
781    /// trailing line, and any event that ended without its blank-line
782    /// terminator (end of stream).
783    fn finish(&mut self) -> Vec<String> {
784        let mut events = Vec::new();
785        if !self.pending.is_empty() {
786            // Bytes that never completed a UTF-8 sequence before EOF.
787            let tail = String::from_utf8_lossy(&self.pending).into_owned();
788            self.pending.clear();
789            events.extend(self.feed(&tail));
790        }
791        if !self.buf.is_empty() {
792            let line = std::mem::take(&mut self.buf);
793            if let Some(payload) = self.process_line(line.trim_end_matches('\r')) {
794                events.push(payload);
795            }
796        }
797        if !self.data_lines.is_empty() {
798            events.push(self.data_lines.join("\n"));
799            self.data_lines.clear();
800        }
801        events
802    }
803
804    /// Process one complete line; returns the event payload when a blank
805    /// line terminates an event that carried `data:`.
806    fn process_line(&mut self, line: &str) -> Option<String> {
807        if line.is_empty() {
808            if self.data_lines.is_empty() {
809                return None;
810            }
811            let payload = self.data_lines.join("\n");
812            self.data_lines.clear();
813            return Some(payload);
814        }
815        if line.starts_with(':') {
816            return None; // comment / heartbeat
817        }
818        if let Some(data) = line.strip_prefix("data:") {
819            // Per spec, a single leading space after the colon is stripped.
820            self.data_lines
821                .push(data.strip_prefix(' ').unwrap_or(data).to_owned());
822        } else if line == "data" {
823            // Per spec, a field with no colon has the empty string as value.
824            self.data_lines.push(String::new());
825        }
826        None
827    }
828}
829
830/// Apply one decoded SSE `data:` payload to the accumulator, invoking
831/// `on_token` for text deltas. Returns `Ok(true)` on the terminal `[DONE]`
832/// sentinel.
833///
834/// Fail-fast policy: one malformed payload aborts the whole stream
835/// (discarding accumulated deltas) rather than guessing which bytes to skip
836/// — after a desynchronized JSON parse, event boundaries can no longer be
837/// trusted. Empty payloads (bare `data:` keep-alives) are skipped.
838fn handle_sse_payload(
839    payload: &str,
840    acc: &mut StreamAccumulator,
841    on_token: &mut (dyn FnMut(TokenChunk) + Send),
842) -> Result<bool> {
843    let trimmed = payload.trim();
844    if trimmed == "[DONE]" {
845        return Ok(true);
846    }
847    if trimmed.is_empty() {
848        return Ok(false);
849    }
850
851    let value: Value = serde_json::from_str(trimmed)
852        .map_err(|e| RustyError::Llm(format!("malformed stream chunk: {e}")))?;
853    let chunk: WireStreamChunk = serde_json::from_value(value.clone())
854        .map_err(|e| RustyError::Llm(format!("malformed stream chunk: {e}")))?;
855
856    if chunk.model.is_some() {
857        acc.model = chunk.model;
858    }
859    if chunk.usage.is_some() {
860        acc.usage = chunk.usage;
861    }
862
863    if let Some(choice) = chunk.choices.into_iter().next() {
864        let delta = choice.delta;
865        if let Some(content) = delta.content {
866            if !content.is_empty() {
867                acc.content.push_str(&content);
868                on_token(TokenChunk {
869                    delta: content,
870                    finish: false,
871                    raw: Some(value),
872                });
873            }
874        }
875        if let Some(calls) = delta.tool_calls {
876            for call in calls {
877                if acc.tool_calls.len() <= call.index {
878                    acc.tool_calls
879                        .resize_with(call.index + 1, ToolCallAccumulator::default);
880                }
881                let slot = &mut acc.tool_calls[call.index];
882                if let Some(id) = call.id {
883                    slot.id.push_str(&id);
884                }
885                if let Some(function) = call.function {
886                    if let Some(name) = function.name {
887                        slot.name.push_str(&name);
888                    }
889                    if let Some(arguments) = function.arguments {
890                        slot.arguments.push_str(&arguments);
891                    }
892                }
893            }
894        }
895    }
896    Ok(false)
897}
898
899#[async_trait]
900impl ChatModel for OpenAiCompatibleClient {
901    async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
902        let mut body = json!({
903            "model": self.model,
904            "messages": messages,
905        });
906        if !tools.is_empty() {
907            body["tools"] = Value::Array(tools.to_vec());
908        }
909
910        let response = self
911            .send_with_retries(|| {
912                let mut request = self.client.post(self.endpoint()).json(&body);
913                if let Some(key) = &self.api_key {
914                    request = request.bearer_auth(key);
915                }
916                request
917            })
918            .await?;
919
920        let wire: WireResponse = response
921            .json()
922            .await
923            .map_err(|e| RustyError::Llm(format!("malformed chat completions response: {e}")))?;
924
925        let choice = wire
926            .choices
927            .into_iter()
928            .next()
929            .ok_or_else(|| RustyError::Llm("chat completions returned zero choices".into()))?;
930
931        Ok(ChatResponse {
932            message: choice.message,
933            model: wire.model,
934            usage: wire.usage,
935        })
936    }
937
938    async fn chat_stream(
939        &self,
940        messages: &[ChatMessage],
941        tools: &[Value],
942        on_token: &mut (dyn FnMut(TokenChunk) + Send),
943    ) -> Result<ChatResponse> {
944        let mut body = json!({
945            "model": self.model,
946            "messages": messages,
947            "stream": true,
948            // Ask for a final usage chunk (supported by OpenAI, vLLM, ...);
949            // providers that ignore it simply omit `usage`.
950            "stream_options": {"include_usage": true},
951        });
952        if !tools.is_empty() {
953            body["tools"] = Value::Array(tools.to_vec());
954        }
955
956        let mut response = self
957            .send_with_retries(|| {
958                let mut request = self.client.post(self.endpoint()).json(&body);
959                if let Some(key) = &self.api_key {
960                    request = request.bearer_auth(key);
961                }
962                request
963            })
964            .await?;
965
966        // Read the body as raw bytes and decode SSE manually (`chunk()` is
967        // used because the `stream` feature of reqwest is not enabled; the
968        // SseDecoder is byte-chunk agnostic either way).
969        let mut decoder = SseDecoder::new();
970        let mut acc = StreamAccumulator::default();
971        let mut done = false;
972
973        while !done {
974            let bytes = match response.chunk().await {
975                Ok(Some(bytes)) => bytes,
976                Ok(None) => break, // end of body
977                Err(e) => {
978                    return Err(RustyError::Llm(format!(
979                        "stream read from {} failed: {e}",
980                        self.base_url
981                    )))
982                }
983            };
984            for payload in decoder.feed_bytes(&bytes) {
985                if handle_sse_payload(&payload, &mut acc, on_token)? {
986                    done = true;
987                    break;
988                }
989            }
990        }
991        if !done {
992            // Stream ended without `[DONE]`: flush whatever the decoder holds.
993            for payload in decoder.finish() {
994                if handle_sse_payload(&payload, &mut acc, on_token)? {
995                    break;
996                }
997            }
998        }
999
1000        on_token(TokenChunk {
1001            delta: String::new(),
1002            finish: true,
1003            raw: None,
1004        });
1005        acc.into_response()
1006    }
1007}
1008
1009#[cfg(test)]
1010mod tests {
1011    use super::*;
1012
1013    #[test]
1014    fn tool_call_wire_roundtrip() {
1015        let call = ToolCall::new("call_1", "search", json!({"q": "rust"}));
1016        let serialized = serde_json::to_value(&call).unwrap();
1017        assert_eq!(
1018            serialized,
1019            json!({
1020                "id": "call_1",
1021                "type": "function",
1022                "function": {"name": "search", "arguments": "{\"q\":\"rust\"}"}
1023            })
1024        );
1025        let back: ToolCall = serde_json::from_value(serialized).unwrap();
1026        assert_eq!(back, call);
1027    }
1028
1029    #[test]
1030    fn message_builders() {
1031        let m = ChatMessage::tool_result("call_1", "42");
1032        assert_eq!(m.role, Role::Tool);
1033        assert_eq!(m.tool_call_id.as_deref(), Some("call_1"));
1034
1035        let m = ChatMessage::assistant_tool_calls(vec![ToolCall::new("c", "t", json!({}))]);
1036        assert!(m.has_tool_calls());
1037        assert_eq!(m.content, None);
1038
1039        // Roles serialize to OpenAI lowercase strings.
1040        assert_eq!(
1041            serde_json::to_value(Role::Assistant).unwrap(),
1042            json!("assistant")
1043        );
1044    }
1045
1046    #[test]
1047    fn sse_decoder_handles_multi_chunk_delivery() {
1048        let mut decoder = SseDecoder::new();
1049        // One event split across two arbitrary byte chunks (split inside a
1050        // `data:` line), followed by a comment and a blank-line terminator;
1051        // then a CRLF-framed event.
1052        assert!(decoder.feed("data: {\"hel").is_empty());
1053        assert_eq!(
1054            decoder.feed("lo\"}\n: heartbeat\n\nda"),
1055            vec!["{\"hello\"}".to_string()]
1056        );
1057        assert_eq!(decoder.feed("ta: world\r\n\r\n"), vec!["world".to_string()]);
1058    }
1059
1060    #[test]
1061    fn sse_decoder_joins_multi_line_data_and_flushes_trailing_event() {
1062        let mut decoder = SseDecoder::new();
1063        // Multiple `data:` lines in one event join with `\n` per the SSE spec.
1064        assert_eq!(
1065            decoder.feed("data: a\ndata: b\n\n"),
1066            vec!["a\nb".to_string()]
1067        );
1068        // A final event with no blank-line terminator is flushed by finish().
1069        assert!(decoder.feed("data: tail").is_empty());
1070        assert_eq!(decoder.finish(), vec!["tail".to_string()]);
1071        // A blank line with no pending `data:` is not an event.
1072        assert!(SseDecoder::new().process_line("").is_none());
1073    }
1074
1075    #[test]
1076    fn sse_done_sentinel_terminates_and_content_deltas_accumulate() {
1077        let mut acc = StreamAccumulator::default();
1078        let mut deltas: Vec<String> = Vec::new();
1079        let mut on_token = |chunk: TokenChunk| {
1080            assert!(!chunk.finish);
1081            assert!(chunk.raw.is_some(), "wire chunks carry the raw JSON");
1082            deltas.push(chunk.delta);
1083        };
1084
1085        let done = handle_sse_payload(
1086            r#"{"choices":[{"delta":{"content":"Hel"},"finish_reason":null}]}"#,
1087            &mut acc,
1088            &mut on_token,
1089        )
1090        .unwrap();
1091        assert!(!done);
1092        let done = handle_sse_payload(
1093            r#"{"choices":[{"delta":{"content":"lo"},"finish_reason":"stop"}],
1094                "model":"gpt-x","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3}}"#,
1095            &mut acc,
1096            &mut on_token,
1097        )
1098        .unwrap();
1099        assert!(!done);
1100        // The [DONE] sentinel terminates the stream and is not parsed as JSON.
1101        assert!(handle_sse_payload("[DONE]", &mut acc, &mut on_token).unwrap());
1102
1103        assert_eq!(deltas, ["Hel", "lo"]);
1104        let response = acc.into_response().unwrap();
1105        assert_eq!(response.message.content.as_deref(), Some("Hello"));
1106        assert_eq!(response.model.as_deref(), Some("gpt-x"));
1107        assert_eq!(response.usage.unwrap().total_tokens, 3);
1108    }
1109
1110    #[test]
1111    fn sse_stream_accumulates_tool_call_deltas() {
1112        let mut acc = StreamAccumulator::default();
1113        let mut on_token = |_chunk: TokenChunk| {};
1114        // id/name/arguments arrive piecewise across chunks at the same index.
1115        for payload in [
1116            r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"search","arguments":"{\"q\":"}}]}}]}"#,
1117            r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"rust\"}"}}]}}]}"#,
1118        ] {
1119            assert!(!handle_sse_payload(payload, &mut acc, &mut on_token).unwrap());
1120        }
1121        let response = acc.into_response().unwrap();
1122        assert_eq!(
1123            response.message.tool_calls,
1124            vec![ToolCall::new("call_1", "search", json!({"q": "rust"}))]
1125        );
1126        assert_eq!(response.message.content, None);
1127    }
1128
1129    /// A model that only implements `chat` (the pre-streaming API surface).
1130    struct NonStreamingMock;
1131
1132    #[async_trait]
1133    impl ChatModel for NonStreamingMock {
1134        async fn chat(&self, _messages: &[ChatMessage], _tools: &[Value]) -> Result<ChatResponse> {
1135            Ok(ChatResponse {
1136                message: ChatMessage::assistant("full answer"),
1137                model: Some("mock".into()),
1138                usage: None,
1139            })
1140        }
1141    }
1142
1143    #[tokio::test]
1144    async fn chat_stream_default_falls_back_to_single_chunk() {
1145        let model = NonStreamingMock;
1146        let mut chunks: Vec<TokenChunk> = Vec::new();
1147        let response = model
1148            .chat_stream(&[], &[], &mut |chunk| chunks.push(chunk))
1149            .await
1150            .unwrap();
1151
1152        // Exactly one terminal chunk carrying the whole message.
1153        assert_eq!(chunks.len(), 1);
1154        assert_eq!(chunks[0].delta, "full answer");
1155        assert!(chunks[0].finish);
1156        assert!(chunks[0].raw.is_none());
1157        assert_eq!(
1158            response.message.content.as_deref(),
1159            Some("full answer"),
1160            "the fallback returns the chat() response unchanged"
1161        );
1162    }
1163
1164    #[test]
1165    fn feed_bytes_preserves_multibyte_chars_split_across_chunks() {
1166        let mut decoder = SseDecoder::new();
1167        // "你" is E4 BD A0 in UTF-8; split the sequence across two feeds.
1168        let mut first = b"data: {\"c\":\"".to_vec();
1169        first.push(0xE4);
1170        assert!(decoder.feed_bytes(&first).is_empty());
1171        assert!(decoder.feed_bytes(&[0xBD, 0xA0]).is_empty());
1172        let events = decoder.feed_bytes(b"\"}\n\n");
1173        assert_eq!(events, vec!["{\"c\":\"你\"}".to_string()]);
1174    }
1175
1176    #[test]
1177    fn sse_strips_leading_bom_and_handles_bare_data_line() {
1178        let mut decoder = SseDecoder::new();
1179        assert!(decoder.feed_bytes(&[0xEF, 0xBB, 0xBF]).is_empty());
1180        // A bare `data` line (no colon) is an empty-string field per spec.
1181        assert_eq!(decoder.feed("data\n\n"), vec![String::new()]);
1182        assert_eq!(decoder.feed("data: x\n\n"), vec!["x".to_string()]);
1183    }
1184
1185    #[test]
1186    fn empty_payload_is_skipped_not_parsed() {
1187        let mut acc = StreamAccumulator::default();
1188        let mut on_token = |_chunk: TokenChunk| {};
1189        // Bare `data:` keep-alive events decode to empty payloads and must
1190        // not abort the stream as "malformed JSON".
1191        assert!(!handle_sse_payload("", &mut acc, &mut on_token).unwrap());
1192    }
1193
1194    #[test]
1195    fn client_debug_redacts_the_api_key() {
1196        let client =
1197            OpenAiCompatibleClient::new("http://localhost", Some("sk-secret-123".into()), "m");
1198        let debug = format!("{client:?}");
1199        assert!(!debug.contains("sk-secret-123"), "got: {debug}");
1200        assert!(debug.contains("***"), "got: {debug}");
1201
1202        let no_key = OpenAiCompatibleClient::new("http://localhost", None, "m");
1203        assert!(format!("{no_key:?}").contains("api_key: None"));
1204    }
1205
1206    // ---------- retry policy over a hand-rolled mock HTTP server ----------
1207
1208    /// Start a minimal HTTP/1.1 server; `handler` maps the 1-based attempt
1209    /// number to a (status, body) response.
1210    fn start_http_mock(
1211        handler: impl Fn(usize) -> (u16, String) + Send + Sync + 'static,
1212    ) -> (
1213        std::net::SocketAddr,
1214        std::sync::Arc<std::sync::atomic::AtomicUsize>,
1215    ) {
1216        use std::sync::atomic::{AtomicUsize, Ordering};
1217        use std::sync::Arc as StdArc;
1218        use tokio::io::{AsyncReadExt, AsyncWriteExt};
1219
1220        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
1221        listener.set_nonblocking(true).unwrap();
1222        let addr = listener.local_addr().unwrap();
1223        let attempts = StdArc::new(AtomicUsize::new(0));
1224        let attempts2 = attempts.clone();
1225        let handler = StdArc::new(handler);
1226        tokio::spawn(async move {
1227            let listener = tokio::net::TcpListener::from_std(listener).unwrap();
1228            loop {
1229                let Ok((mut stream, _)) = listener.accept().await else {
1230                    continue;
1231                };
1232                let attempts = attempts2.clone();
1233                let handler = handler.clone();
1234                tokio::spawn(async move {
1235                    // Read the request: headers up to \r\n\r\n, then the
1236                    // content-length bytes of body (contents irrelevant).
1237                    let mut buf: Vec<u8> = Vec::new();
1238                    let mut chunk = [0u8; 4096];
1239                    loop {
1240                        if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
1241                            let headers = String::from_utf8_lossy(&buf[..pos]).to_lowercase();
1242                            let len: usize = headers
1243                                .lines()
1244                                .find_map(|l| l.strip_prefix("content-length:"))
1245                                .and_then(|v| v.trim().parse().ok())
1246                                .unwrap_or(0);
1247                            if buf.len() >= pos + 4 + len {
1248                                break;
1249                            }
1250                        }
1251                        let Ok(n) = stream.read(&mut chunk).await else {
1252                            return;
1253                        };
1254                        if n == 0 {
1255                            return;
1256                        }
1257                        buf.extend_from_slice(&chunk[..n]);
1258                    }
1259                    let n = attempts.fetch_add(1, Ordering::SeqCst) + 1;
1260                    let (status, body) = handler(n);
1261                    let reason = match status {
1262                        200 => "OK",
1263                        400 => "Bad Request",
1264                        429 => "Too Many Requests",
1265                        500 => "Internal Server Error",
1266                        _ => "Status",
1267                    };
1268                    let response = format!(
1269                        "HTTP/1.1 {status} {reason}\r\n\
1270                         content-type: application/json\r\n\
1271                         content-length: {}\r\n\
1272                         connection: close\r\n\
1273                         \r\n\
1274                         {body}",
1275                        body.len()
1276                    );
1277                    let _ = stream.write_all(response.as_bytes()).await;
1278                });
1279            }
1280        });
1281        (addr, attempts)
1282    }
1283
1284    #[tokio::test]
1285    async fn chat_retries_transient_5xx_then_succeeds() {
1286        let (addr, attempts) = start_http_mock(|n| {
1287            if n < 3 {
1288                (500, r#"{"error":"overloaded"}"#.to_string())
1289            } else {
1290                (
1291                    200,
1292                    r#"{"choices":[{"message":{"role":"assistant","content":"hi after retry"}}]}"#
1293                        .to_string(),
1294                )
1295            }
1296        });
1297        let client = OpenAiCompatibleClient::new(format!("http://{addr}"), None, "m")
1298            .with_backoff(Duration::from_millis(1));
1299        let response = client.chat(&[ChatMessage::user("hi")], &[]).await.unwrap();
1300        assert_eq!(response.message.content.as_deref(), Some("hi after retry"));
1301        // 1 initial + 2 retries = 3 attempts.
1302        assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 3);
1303    }
1304
1305    #[tokio::test]
1306    async fn chat_does_not_retry_fatal_4xx() {
1307        let (addr, attempts) =
1308            start_http_mock(|_n| (400, r#"{"error":"bad request"}"#.to_string()));
1309        let client = OpenAiCompatibleClient::new(format!("http://{addr}"), None, "m")
1310            .with_backoff(Duration::from_millis(1));
1311        let err = client
1312            .chat(&[ChatMessage::user("hi")], &[])
1313            .await
1314            .unwrap_err();
1315        assert!(err.to_string().contains("400"), "got: {err}");
1316        assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);
1317    }
1318
1319    #[tokio::test]
1320    async fn chat_gives_up_after_retries_exhausted() {
1321        let (addr, attempts) =
1322            start_http_mock(|_n| (429, r#"{"error":"rate limited"}"#.to_string()));
1323        let client = OpenAiCompatibleClient::new(format!("http://{addr}"), None, "m")
1324            .with_retries(1)
1325            .with_backoff(Duration::from_millis(1));
1326        let err = client
1327            .chat(&[ChatMessage::user("hi")], &[])
1328            .await
1329            .unwrap_err();
1330        assert!(err.to_string().contains("429"), "got: {err}");
1331        // 1 initial + 1 retry = 2 attempts.
1332        assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 2);
1333    }
1334}