Skip to main content

tinyharness_lib/provider/
openai_compat.rs

1use std::collections::HashMap;
2use std::time::Duration;
3
4use reqwest::Client;
5use serde::{Deserialize, Serialize};
6use tokio_stream::StreamExt;
7
8use crate::{
9    SecretString,
10    provider::{
11        ChatMessage, ChatMessageResponse, Message, Role, ToolCall, ToolCallFunction, ToolDefinition,
12    },
13};
14
15/// Default per-request timeout for OpenAI-compatible providers when no
16/// explicit value is configured (via `/timeout` or settings).
17pub const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 30;
18
19/// Shared inner state for OpenAI-compatible providers (llama.cpp, vLLM, etc.).
20///
21/// Encapsulates the common `{client, base_url, model, api_key}` fields and all
22/// shared logic so that provider implementations only need to differ in
23/// `list_models()`.
24pub struct OpenAiCompatInner {
25    client: Client,
26    base_url: String,
27    model: Option<String>,
28    /// Optional bearer token sent as `Authorization: Bearer <key>` on every
29    /// request. Used by hosted OpenAI-compatible APIs (e.g. OpenRouter,
30    /// Together, self-hosted gateways) that require authentication.
31    api_key: Option<SecretString>,
32    /// Per-request timeout in seconds, applied to connecting and reading the
33    /// HTTP response. The default (30s) covers a slow backend; increase via
34    /// `/timeout <secs>` for large-context requests.
35    timeout_secs: u64,
36    /// Maximum number of attempts for transient failures (connection errors,
37    /// timeouts, 5xx responses). 1 = no retries. 4xx responses are never
38    /// retried. Backoff between attempts: 1s, 2s, 4s, …
39    max_retries: u32,
40}
41
42impl OpenAiCompatInner {
43    pub fn new(base_url: String) -> Self {
44        Self::with_api_key(base_url, None)
45    }
46
47    /// Create a new inner state with an optional bearer token.
48    pub fn with_api_key(base_url: String, api_key: Option<SecretString>) -> Self {
49        Self::with_options(base_url, api_key, DEFAULT_REQUEST_TIMEOUT_SECS, 0)
50    }
51
52    /// Create a new inner state with explicit timeout/retry configuration.
53    pub fn with_options(
54        base_url: String,
55        api_key: Option<SecretString>,
56        timeout_secs: u64,
57        max_retries: u32,
58    ) -> Self {
59        // Read timeout bounds how long we wait between bytes on an active
60        // stream; add slack over the request timeout so long generations
61        // that keep producing tokens are never cut off mid-stream.
62        let client = Client::builder()
63            .connect_timeout(Duration::from_secs(15))
64            .read_timeout(Duration::from_secs(timeout_secs + 60))
65            .build()
66            .unwrap_or_else(|_| Client::new());
67        OpenAiCompatInner {
68            client,
69            base_url,
70            model: None,
71            api_key,
72            timeout_secs,
73            max_retries,
74        }
75    }
76
77    /// Current request timeout in seconds.
78    pub fn timeout_secs(&self) -> u64 {
79        self.timeout_secs
80    }
81
82    /// Current maximum retry count.
83    pub fn max_retries(&self) -> u32 {
84        self.max_retries
85    }
86
87    /// Update the request timeout. Applies to subsequent requests (the HTTP
88    /// client's read timeout is rebuilt lazily on the next `chat()`).
89    pub fn set_timeout(&mut self, timeout_secs: u64) {
90        self.timeout_secs = timeout_secs;
91        self.client = Client::builder()
92            .connect_timeout(Duration::from_secs(15))
93            .read_timeout(Duration::from_secs(timeout_secs + 60))
94            .build()
95            .unwrap_or_else(|_| Client::new());
96    }
97
98    /// Update the maximum retry count. Applies to subsequent requests.
99    pub fn set_retries(&mut self, max_retries: u32) {
100        self.max_retries = max_retries;
101    }
102
103    /// Perform a health check against the server's `/health` endpoint.
104    ///
105    /// Servers that don't expose `/health` answer with HTTP 404 (llama.cpp,
106    /// many gateways and proxies). In that case we fall back to
107    /// `/v1/models`: a reachable endpoint returning a non-empty model list
108    /// proves the backend is up and usable, so it counts as a successful
109    /// health check. An empty list — or a failing `/v1/models` request — is
110    /// reported as a failure, since it tells us nothing about the server.
111    ///
112    /// Any other `/health` status (5xx, 401, …) is reported directly and is
113    /// never masked by the fallback: the endpoint exists, it is unhappy.
114    pub async fn health_check(&self) -> Result<(), String> {
115        let url = format!("{}/health", self.base_url.trim_end_matches('/'));
116        let mut req = self.client.get(&url);
117        if let Some(key) = &self.api_key {
118            req = req.bearer_auth(key.expose_secret());
119        }
120        match req.send().await {
121            Ok(resp) if resp.status().is_success() => Ok(()),
122            // A 404 usually means the server simply has no /health
123            // endpoint. Don't dump its response body (often a large HTML
124            // or JSON error page) into the warning — probe /v1/models
125            // instead.
126            Ok(resp) if resp.status() == reqwest::StatusCode::NOT_FOUND => {
127                match self.fetch_models_raw().await {
128                    Ok(models) if !models.is_empty() => Ok(()),
129                    Ok(_) => Err(
130                        "Server has no /health endpoint and /v1/models returned an empty list"
131                            .to_string(),
132                    ),
133                    Err(e) => Err(format!(
134                        "Server has no /health endpoint and /v1/models failed: {e}"
135                    )),
136                }
137            }
138            Ok(resp) => Err(format!(
139                "Server returned {}: {}",
140                resp.status().as_u16(),
141                resp.text().await.unwrap_or_default()
142            )),
143            Err(e) => Err(format!("Cannot reach {}: {}", url, e)),
144        }
145    }
146
147    pub fn select_model(&mut self, name: String) {
148        self.model = Some(name);
149    }
150
151    pub fn current_model(&self) -> Option<String> {
152        self.model.clone()
153    }
154
155    /// Return the `/v1/chat/completions` URL for this server.
156    pub fn chat_url(&self) -> String {
157        format!(
158            "{}/v1/chat/completions",
159            self.base_url.trim_end_matches('/').trim_end_matches("/v1")
160        )
161    }
162
163    /// Return the `/v1/models` URL for this server.
164    pub fn models_url(&self) -> String {
165        format!(
166            "{}/v1/models",
167            self.base_url.trim_end_matches('/').trim_end_matches("/v1")
168        )
169    }
170
171    /// Fetch the model list from `/v1/models`, propagating failures.
172    ///
173    /// Unlike [`Self::fetch_model_list`], this distinguishes "the server
174    /// answered with an empty list" (`Ok(vec![])`) from "the request could
175    /// not be completed" (`Err`), which callers such as the health check
176    /// need in order to interpret the result.
177    pub async fn fetch_models_raw(&self) -> Result<Vec<String>, String> {
178        let url = self.models_url();
179        let mut req = self.client.get(&url);
180        if let Some(key) = &self.api_key {
181            req = req.bearer_auth(key.expose_secret());
182        }
183        let resp = match req.send().await {
184            Ok(resp) => resp,
185            Err(e) => return Err(format!("cannot reach {url}: {e}")),
186        };
187        if !resp.status().is_success() {
188            return Err(format!("server returned {}", resp.status().as_u16()));
189        }
190        match resp.json::<ModelListResponse>().await {
191            Ok(list) => Ok(list.data.into_iter().map(|m| m.id).collect()),
192            Err(e) => Err(format!("invalid model list response: {e}")),
193        }
194    }
195
196    /// Fetch the model list from the server's `/v1/models` endpoint.
197    /// Returns the list of model IDs, or an empty vec on failure.
198    pub async fn fetch_model_list(&self) -> Vec<String> {
199        match self.fetch_models_raw().await {
200            Ok(models) => models,
201            Err(_) => self.model.clone().into_iter().collect(),
202        }
203    }
204
205    /// Stream chat completions using the OpenAI-compatible API.
206    /// Returns a receiver for streaming response chunks, or an error string
207    /// if the request cannot be started.
208    pub async fn chat(
209        &self,
210        messages: Vec<Message>,
211        tools: Vec<ToolDefinition>,
212    ) -> Result<tokio::sync::mpsc::Receiver<ChatMessageResponse>, String> {
213        let (send, recv) =
214            tokio::sync::mpsc::channel::<ChatMessageResponse>(super::STREAM_CHANNEL_CAPACITY);
215
216        let model = self.model.clone().unwrap_or_default();
217        let openai_messages = messages.into_iter().map(to_openai_message).collect();
218        let openai_tools = tools.into_iter().map(to_openai_tool).collect();
219        let client = self.client.clone();
220        let chat_url = self.chat_url();
221        let api_key = self.api_key.clone();
222        let timeout_secs = self.timeout_secs;
223        let max_retries = self.max_retries;
224
225        let body = ChatRequest {
226            model,
227            messages: openai_messages,
228            stream: true,
229            stream_options: Some(StreamOptions {
230                include_usage: true,
231            }),
232            tools: openai_tools,
233        };
234
235        // Spawn the streaming work on a background task
236        tokio::spawn(async move {
237            let _usage = chat_with_retries(
238                &client,
239                &chat_url,
240                &body,
241                api_key.as_ref(),
242                timeout_secs,
243                max_retries,
244                &send,
245            )
246            .await;
247        });
248
249        Ok(recv)
250    }
251}
252
253/// Send the chat request with retry + exponential backoff for transient
254/// failures, then stream the response chunks through `send`.
255///
256/// Retryable failures: connection errors, request timeouts (when the whole
257/// attempt is cut off before any output), and 5xx / 429 responses. Once a
258/// terminal chunk has reached the receiver (success, mid-stream error, or a
259/// 4xx error), the outcome is final — retrying would duplicate output.
260///
261/// Backoff between attempts: 1s, 2s, 4s, … Returns the token usage when a
262/// final attempt succeeds.
263async fn chat_with_retries(
264    client: &reqwest::Client,
265    url: &str,
266    body: &ChatRequest,
267    api_key: Option<&SecretString>,
268    timeout_secs: u64,
269    max_retries: u32,
270    send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
271) -> Option<crate::provider::TokenUsage> {
272    let max_attempts = max_retries.max(1);
273
274    for attempt in 1..=max_attempts {
275        let result = tokio::time::timeout(
276            Duration::from_secs(timeout_secs),
277            stream_chat_completions(client, url, body, api_key, send),
278        )
279        .await;
280
281        match result {
282            // The attempt produced a terminal chunk (success or surfaced
283            // error) — the receiver already has the outcome, don't retry.
284            Ok(AttemptOutcome::Final(usage)) => return usage,
285            // The attempt failed before delivering anything.
286            Ok(AttemptOutcome::Retryable(e)) => {
287                if attempt >= max_attempts {
288                    send_error_chunk(send, &format!("Error after {attempt} attempt(s): {e}")).await;
289                    return None;
290                }
291            }
292            // The whole attempt timed out before completing.
293            Err(_) => {
294                if attempt >= max_attempts {
295                    send_error_chunk(
296                        send,
297                        &format!(
298                            "Error: Request timed out after {timeout_secs}s ({attempt} attempt(s))"
299                        ),
300                    )
301                    .await;
302                    return None;
303                }
304            }
305        }
306
307        // Exponential backoff: 1s, 2s, 4s, ...
308        let backoff = Duration::from_secs(1 << (attempt - 1));
309        tokio::time::sleep(backoff).await;
310    }
311
312    None
313}
314
315// ── OpenAI-compatible request/response types ──
316
317#[derive(Serialize)]
318pub struct ChatRequest {
319    pub model: String,
320    pub messages: Vec<OpenAIMessage>,
321    pub stream: bool,
322    #[serde(skip_serializing_if = "Option::is_none")]
323    pub stream_options: Option<StreamOptions>,
324    #[serde(skip_serializing_if = "Vec::is_empty")]
325    pub tools: Vec<OpenAITool>,
326}
327
328#[derive(Serialize)]
329pub struct StreamOptions {
330    pub include_usage: bool,
331}
332
333#[derive(Serialize)]
334pub struct OpenAIMessage {
335    pub role: String,
336    pub content: serde_json::Value,
337    #[serde(skip_serializing_if = "Option::is_none")]
338    pub tool_calls: Option<Vec<OpenAIToolCall>>,
339    #[serde(skip_serializing_if = "Option::is_none")]
340    pub tool_call_id: Option<String>,
341}
342
343#[derive(Serialize)]
344pub struct OpenAITool {
345    #[serde(rename = "type")]
346    pub tool_type: String,
347    pub function: OpenAIToolFunction,
348}
349
350#[derive(Serialize)]
351pub struct OpenAIToolFunction {
352    pub name: String,
353    pub description: String,
354    pub parameters: serde_json::Value,
355}
356
357#[derive(Serialize, Deserialize, Clone)]
358pub struct OpenAIToolCall {
359    #[serde(default)]
360    pub index: usize,
361    #[serde(default)]
362    pub id: String,
363    #[serde(rename = "type", default)]
364    pub call_type: String,
365    #[serde(default)]
366    pub function: OpenAIToolCallFunction,
367}
368
369#[derive(Serialize, Deserialize, Clone, Default)]
370pub struct OpenAIToolCallFunction {
371    #[serde(default)]
372    pub name: String,
373    #[serde(default)]
374    pub arguments: String,
375}
376
377#[derive(Deserialize)]
378pub struct ChunkChoice {
379    pub delta: Delta,
380    #[serde(default, rename = "finish_reason")]
381    pub _finish_reason: Option<String>,
382}
383
384#[derive(Deserialize, Default)]
385pub struct Delta {
386    #[serde(default)]
387    pub content: Option<String>,
388    #[serde(default)]
389    pub tool_calls: Option<Vec<OpenAIToolCall>>,
390    /// Reasoning text — OpenRouter / DeepSeek spelling.
391    #[serde(default)]
392    pub reasoning: Option<String>,
393    /// Reasoning text — vLLM / Qwen `--reasoning-parser` spelling.
394    #[serde(default)]
395    pub reasoning_content: Option<String>,
396    /// Structured reasoning parts (OpenRouter): the portable representation
397    /// used for providers whose reasoning is signed or encrypted.
398    #[serde(default)]
399    pub reasoning_details: Option<Vec<ReasoningDetail>>,
400}
401
402/// One item of an OpenAI-compatible `delta.reasoning_details` array.
403///
404/// Only the displayable shapes are modelled: `text` / `reasoning_text` items
405/// carry their content in `text`, Anthropic-style `summary` items carry an
406/// array of `{ "text": … }` parts. `encrypted` / `redacted_reasoning` items
407/// have no readable payload, so they contribute nothing to the display.
408#[derive(Deserialize, Clone, Debug, Default)]
409pub struct ReasoningDetail {
410    #[serde(default)]
411    pub text: Option<String>,
412    #[serde(default)]
413    pub summary: Option<serde_json::Value>,
414}
415
416impl ReasoningDetail {
417    /// The human-readable text carried by this reasoning part (may be empty).
418    fn display_text(&self) -> String {
419        let mut out = String::new();
420        if let Some(text) = &self.text {
421            out.push_str(text);
422        }
423        if let Some(summary) = &self.summary {
424            match summary {
425                serde_json::Value::String(s) => out.push_str(s),
426                serde_json::Value::Array(parts) => {
427                    for part in parts {
428                        if let Some(s) = part.get("text").and_then(|t| t.as_str()) {
429                            out.push_str(s);
430                        }
431                    }
432                }
433                _ => {}
434            }
435        }
436        out
437    }
438}
439
440/// Extract the displayable reasoning chain from a streaming delta.
441///
442/// Servers disagree on how reasoning is spelled (`reasoning`,
443/// `reasoning_content`, or the structured `reasoning_details` array), so all
444/// shapes collapse into a single string. The plain fields win when present —
445/// OpenRouter emits both the text and its structured form, and concatenating
446/// them would duplicate the chain. Returns `None` when the delta carries no
447/// reasoning at all.
448fn delta_reasoning(delta: &Delta) -> Option<String> {
449    let mut out = String::new();
450    for text in [&delta.reasoning, &delta.reasoning_content]
451        .into_iter()
452        .flatten()
453    {
454        out.push_str(text);
455    }
456    if out.is_empty()
457        && let Some(details) = &delta.reasoning_details
458    {
459        for detail in details {
460            out.push_str(&detail.display_text());
461        }
462    }
463    if out.is_empty() { None } else { Some(out) }
464}
465
466#[derive(Deserialize)]
467pub struct StreamChunk {
468    #[serde(default)]
469    pub choices: Vec<ChunkChoice>,
470    #[serde(default)]
471    pub usage: Option<OpenAIUsage>,
472}
473
474#[derive(Deserialize, Clone)]
475pub struct OpenAIUsage {
476    #[serde(default)]
477    pub prompt_tokens: u32,
478    #[serde(default)]
479    pub completion_tokens: u32,
480    #[serde(default)]
481    pub total_tokens: u32,
482}
483
484// ── Model list response types ──
485
486#[derive(Deserialize)]
487pub struct ModelListResponse {
488    pub data: Vec<ModelEntry>,
489}
490
491#[derive(Deserialize)]
492pub struct ModelEntry {
493    pub id: String,
494}
495
496// ── Conversion helpers ──
497
498pub fn to_openai_message(msg: Message) -> OpenAIMessage {
499    /// Build the content value: if images are present, use multipart array format;
500    /// otherwise use plain string.
501    fn build_content(msg: &Message) -> serde_json::Value {
502        if msg.images.is_empty() {
503            serde_json::Value::String(msg.content.clone())
504        } else {
505            let mut parts: Vec<serde_json::Value> = Vec::new();
506            // Add text part
507            if !msg.content.is_empty() {
508                parts.push(serde_json::json!({
509                    "type": "text",
510                    "text": msg.content
511                }));
512            }
513            // Add image parts
514            for img in &msg.images {
515                parts.push(serde_json::json!({
516                    "type": "image_url",
517                    "image_url": {
518                        "url": img.data_uri()
519                    }
520                }));
521            }
522            serde_json::Value::Array(parts)
523        }
524    }
525
526    match msg.role {
527        Role::System => OpenAIMessage {
528            role: "system".to_string(),
529            content: serde_json::Value::String(msg.content),
530            tool_calls: None,
531            tool_call_id: None,
532        },
533        Role::User => OpenAIMessage {
534            role: "user".to_string(),
535            content: build_content(&msg),
536            tool_calls: None,
537            tool_call_id: None,
538        },
539        Role::Assistant => {
540            if msg.tool_calls.is_empty() {
541                OpenAIMessage {
542                    role: "assistant".to_string(),
543                    content: serde_json::Value::String(msg.content),
544                    tool_calls: None,
545                    tool_call_id: None,
546                }
547            } else {
548                let tool_calls: Vec<OpenAIToolCall> = msg
549                    .tool_calls
550                    .into_iter()
551                    .enumerate()
552                    .map(|(i, tc)| {
553                        // Synthesize a stable id when the upstream provider
554                        // didn't return one (some local / non-OpenAI servers
555                        // omit it). OpenAI requires a non-empty id.
556                        let id = tc
557                            .id
558                            .filter(|s| !s.is_empty())
559                            .unwrap_or_else(|| format!("call_{}", i));
560                        let args_str = if tc.function.arguments.is_null() {
561                            "{}".to_string()
562                        } else {
563                            tc.function.arguments.to_string()
564                        };
565                        OpenAIToolCall {
566                            index: i,
567                            id,
568                            call_type: "function".to_string(),
569                            function: OpenAIToolCallFunction {
570                                name: tc.function.name,
571                                arguments: args_str,
572                            },
573                        }
574                    })
575                    .collect();
576                OpenAIMessage {
577                    role: "assistant".to_string(),
578                    // MiniMax and some other strict providers reject empty
579                    // strings in assistant messages that contain only tool
580                    // calls.  Send `null` when there is no text content.
581                    content: if msg.content.is_empty() {
582                        serde_json::Value::Null
583                    } else {
584                        serde_json::Value::String(msg.content)
585                    },
586                    tool_calls: Some(tool_calls),
587                    tool_call_id: None,
588                }
589            }
590        }
591        Role::Tool => {
592            // tool_call_id should have been set by the agent loop. If somehow
593            // missing, skip the field rather than emitting a bogus id that the
594            // server will reject.
595            let tool_call_id = msg.tool_call_id.filter(|s| !s.is_empty());
596            OpenAIMessage {
597                role: "tool".to_string(),
598                content: serde_json::Value::String(msg.content),
599                tool_calls: None,
600                tool_call_id,
601            }
602        }
603    }
604}
605
606pub fn to_openai_tool(ti: ToolDefinition) -> OpenAITool {
607    OpenAITool {
608        tool_type: "function".to_string(),
609        function: OpenAIToolFunction {
610            name: ti.name,
611            description: ti.description,
612            parameters: serde_json::to_value(ti.parameters).unwrap_or_default(),
613        },
614    }
615}
616
617/// Outcome of a single streaming attempt.
618///
619/// `Retryable` failures happen before anything was delivered to the
620/// receiver, so the attempt can be repeated cleanly. `Final` outcomes
621/// already produced a terminal chunk on the channel (success or an error
622/// the receiver has seen), so retrying would duplicate output.
623enum AttemptOutcome {
624    /// Nothing was sent to the receiver — safe to retry.
625    Retryable(String),
626    /// A final chunk (success or error) was already sent.
627    Final(Option<crate::provider::TokenUsage>),
628}
629
630/// Determine whether an HTTP status code should be retried.
631///
632/// 5xx responses (server-side trouble) and 429 (rate limit) are transient;
633/// other 4xx errors indicate a request problem that retrying won't fix.
634fn is_retryable_status(status: reqwest::StatusCode) -> bool {
635    status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS
636}
637
638/// Stream chat completions from an OpenAI-compatible endpoint.
639///
640/// Returns `AttemptOutcome::Retryable` when the request failed before any
641/// chunk reached the receiver (connection error, non-retryable status
642/// surfaced via `Err`); returns `AttemptOutcome::Final` once a terminal
643/// chunk (success or surfaced error) has been delivered.
644///
645/// If `api_key` is `Some`, the request includes an
646/// `Authorization: Bearer <key>` header.
647async fn stream_chat_completions(
648    client: &reqwest::Client,
649    url: &str,
650    body: &ChatRequest,
651    api_key: Option<&SecretString>,
652    send: &tokio::sync::mpsc::Sender<ChatMessageResponse>,
653) -> AttemptOutcome {
654    let mut request = client.post(url).json(body);
655    if let Some(key) = api_key {
656        request = request.bearer_auth(key.expose_secret());
657    }
658    let response = match request.send().await {
659        Ok(r) => r,
660        Err(e) => return AttemptOutcome::Retryable(format!("Error: {}", e)),
661    };
662
663    // If the server returned a non-success status, surface the body as an
664    // error instead of feeding it into the SSE parser (which would silently
665    // drop it). Server-side (5xx) and rate-limit (429) failures are retried
666    // by the caller; client errors (4xx) are surfaced immediately.
667    let status = response.status();
668    if !status.is_success() {
669        let body = response.text().await.unwrap_or_default();
670        let msg = format!("Error: HTTP {} — {}", status.as_u16(), body);
671        if is_retryable_status(status) {
672            return AttemptOutcome::Retryable(msg);
673        }
674        send_error_chunk(send, &msg).await;
675        return AttemptOutcome::Final(None);
676    }
677
678    let mut stream = response.bytes_stream();
679    let mut buf = String::new();
680
681    let mut acc_tool_calls: HashMap<usize, OpenAIToolCall> = HashMap::new();
682    let mut response_content = String::new();
683    let mut response_thinking = String::new();
684    let mut token_usage: Option<crate::provider::TokenUsage> = None;
685
686    while let Some(chunk_result) = stream.next().await {
687        let chunk = match chunk_result {
688            Ok(c) => c,
689            Err(e) => {
690                // Stream read error — surface it rather than silently breaking.
691                // This is final: partial output may already have been delivered.
692                send_error_chunk(send, &format!("\n\nStream error: {}", e)).await;
693                return AttemptOutcome::Final(token_usage);
694            }
695        };
696
697        buf.push_str(&String::from_utf8_lossy(&chunk));
698
699        loop {
700            match buf.find('\n') {
701                None => break,
702                Some(pos) => {
703                    let line = buf[..pos].trim().to_string();
704                    buf = buf[pos + 1..].to_string();
705
706                    if line.is_empty() || line == "data: [DONE]" {
707                        continue;
708                    }
709
710                    if let Some(data) = line.strip_prefix("data: ")
711                        && let Ok(chunk) = serde_json::from_str::<StreamChunk>(data)
712                    {
713                        // Capture token usage if present (usually in the final chunk)
714                        if let Some(usage) = &chunk.usage {
715                            token_usage = Some(crate::provider::TokenUsage {
716                                prompt_tokens: usage.prompt_tokens,
717                                completion_tokens: usage.completion_tokens,
718                                total_tokens: usage.total_tokens,
719                            });
720                        }
721
722                        for choice in chunk.choices {
723                            if let Some(content) = &choice.delta.content {
724                                response_content.push_str(content);
725                            }
726
727                            // Reasoning deltas arrive before (and sometimes
728                            // interleaved with) the visible content; keep them
729                            // in their own buffer so the UI can render the
730                            // `[thinking]` stream separately.
731                            if let Some(reasoning) = delta_reasoning(&choice.delta) {
732                                response_thinking.push_str(&reasoning);
733                            }
734
735                            if let Some(tool_calls) = &choice.delta.tool_calls {
736                                for tc in tool_calls {
737                                    let entry =
738                                        acc_tool_calls.entry(tc.index).or_insert(OpenAIToolCall {
739                                            index: tc.index,
740                                            id: String::new(),
741                                            call_type: "function".to_string(),
742                                            function: OpenAIToolCallFunction::default(),
743                                        });
744
745                                    if !tc.id.is_empty() {
746                                        entry.id = tc.id.clone();
747                                    }
748                                    if !tc.function.name.is_empty() {
749                                        entry.function.name = tc.function.name.clone();
750                                    }
751                                    entry.function.arguments.push_str(&tc.function.arguments);
752                                }
753                            }
754                        }
755                    }
756                }
757            }
758        }
759
760        // Flush whichever buffer has data — a chunk may carry only reasoning
761        // (models that think before they answer) or only content.
762        if !response_thinking.is_empty() {
763            let thinking = std::mem::take(&mut response_thinking);
764            if send
765                .send(ChatMessageResponse {
766                    message: ChatMessage {
767                        content: String::new(),
768                        tool_calls: vec![],
769                        thinking: Some(thinking),
770                    },
771                    done: false,
772                    is_error: false,
773                    usage: None,
774                })
775                .await
776                .is_err()
777            {
778                return AttemptOutcome::Final(token_usage);
779            }
780        }
781
782        if !response_content.is_empty() {
783            // If the receiver is gone (user interrupt, shutdown), stop
784            // reading the HTTP stream instead of draining the rest of it.
785            if send
786                .send(ChatMessageResponse {
787                    message: ChatMessage {
788                        content: response_content.clone(),
789                        tool_calls: vec![],
790                        thinking: None,
791                    },
792                    done: false,
793                    is_error: false,
794                    usage: None,
795                })
796                .await
797                .is_err()
798            {
799                return AttemptOutcome::Final(token_usage);
800            }
801            response_content.clear();
802        }
803    }
804
805    // Reasoning can still be buffered when the server closes the stream
806    // without another content/usage chunk — flush it so the tail is not lost.
807    if !response_thinking.is_empty() {
808        let thinking = std::mem::take(&mut response_thinking);
809        if send
810            .send(ChatMessageResponse {
811                message: ChatMessage {
812                    content: String::new(),
813                    tool_calls: vec![],
814                    thinking: Some(thinking),
815                },
816                done: false,
817                is_error: false,
818                usage: None,
819            })
820            .await
821            .is_err()
822        {
823            return AttemptOutcome::Final(token_usage);
824        }
825    }
826
827    let tool_calls: Vec<ToolCall> = if !acc_tool_calls.is_empty() {
828        acc_tool_calls
829            .into_values()
830            .map(|tc| {
831                let args: serde_json::Value =
832                    serde_json::from_str(&tc.function.arguments).unwrap_or(serde_json::Value::Null);
833                ToolCall {
834                    id: if tc.id.is_empty() { None } else { Some(tc.id) },
835                    function: ToolCallFunction {
836                        name: tc.function.name,
837                        arguments: args,
838                        thought_signature: None,
839                    },
840                }
841            })
842            .collect()
843    } else {
844        vec![]
845    };
846
847    // Send the final response with tool calls and token usage
848    let _ = send
849        .send(ChatMessageResponse {
850            message: ChatMessage {
851                content: String::new(),
852                tool_calls,
853                thinking: None,
854            },
855            done: true,
856            is_error: false,
857            usage: token_usage.clone(),
858        })
859        .await;
860
861    AttemptOutcome::Final(token_usage)
862}
863
864/// Send a terminal error chunk (done: true, is_error: true) to the receiver.
865async fn send_error_chunk(send: &tokio::sync::mpsc::Sender<ChatMessageResponse>, message: &str) {
866    let _ = send
867        .send(ChatMessageResponse {
868            message: ChatMessage {
869                content: message.to_string(),
870                tool_calls: vec![],
871                thinking: None,
872            },
873            done: true,
874            is_error: true,
875            usage: None,
876        })
877        .await;
878}
879
880#[cfg(test)]
881mod tests {
882    use super::*;
883    use std::sync::{Arc, Mutex};
884
885    /// Reason phrase for the canned status lines used by the mock server.
886    fn reason_phrase(status: u16) -> &'static str {
887        match status {
888            200 => "OK",
889            404 => "Not Found",
890            401 => "Unauthorized",
891            500 => "Internal Server Error",
892            503 => "Service Unavailable",
893            _ => "Status",
894        }
895    }
896
897    /// Spawn a minimal HTTP server answering fixed paths, logging every
898    /// requested path in order. Unknown paths get a 404.
899    ///
900    /// Returns `(base_url, request_log, server_handle)`.
901    async fn spawn_mock_server(
902        routes: Vec<(&'static str, u16, &'static str)>,
903    ) -> (String, Arc<Mutex<Vec<String>>>, tokio::task::JoinHandle<()>) {
904        use tokio::io::{AsyncReadExt, AsyncWriteExt};
905
906        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
907            .await
908            .expect("mock server bind");
909        let addr = listener.local_addr().expect("mock server addr");
910        let log: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(vec![]));
911        let routes = Arc::new(routes);
912        let log_in = Arc::clone(&log);
913
914        let handle = tokio::spawn(async move {
915            while let Ok((mut stream, _)) = listener.accept().await {
916                let routes = Arc::clone(&routes);
917                let log = Arc::clone(&log_in);
918                tokio::spawn(async move {
919                    let mut buf = [0u8; 4096];
920                    let n = match stream.read(&mut buf).await {
921                        Ok(0) | Err(_) => return,
922                        Ok(n) => n,
923                    };
924                    let head = String::from_utf8_lossy(&buf[..n]);
925                    let path = head
926                        .split_whitespace()
927                        .nth(1)
928                        .unwrap_or_default()
929                        .to_string();
930                    log.lock().unwrap().push(path.clone());
931
932                    let (status, body) = routes
933                        .iter()
934                        .find(|(p, _, _)| *p == path)
935                        .map(|(_, s, b)| (*s, *b))
936                        .unwrap_or((404, "{\"error\":\"not found\"}"));
937                    let resp = format!(
938                        "HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\n\
939                         content-length: {len}\r\nconnection: close\r\n\r\n{body}",
940                        reason = reason_phrase(status),
941                        len = body.len(),
942                    );
943                    let _ = stream.write_all(resp.as_bytes()).await;
944                    let _ = stream.shutdown().await;
945                });
946            }
947        });
948
949        (format!("http://{addr}"), log, handle)
950    }
951
952    /// 5xx and 429 are transient and should be retried.
953    #[test]
954    fn retryable_status_codes() {
955        assert!(is_retryable_status(reqwest::StatusCode::BAD_GATEWAY));
956        assert!(is_retryable_status(
957            reqwest::StatusCode::SERVICE_UNAVAILABLE
958        ));
959        assert!(is_retryable_status(
960            reqwest::StatusCode::INTERNAL_SERVER_ERROR
961        ));
962        assert!(is_retryable_status(reqwest::StatusCode::TOO_MANY_REQUESTS));
963    }
964
965    /// 4xx responses (other than 429) mean the request itself is wrong —
966    /// retrying won't help, so they must not be retried.
967    #[test]
968    fn non_retryable_status_codes() {
969        assert!(!is_retryable_status(reqwest::StatusCode::UNAUTHORIZED));
970        assert!(!is_retryable_status(reqwest::StatusCode::NOT_FOUND));
971        assert!(!is_retryable_status(reqwest::StatusCode::BAD_REQUEST));
972        assert!(!is_retryable_status(
973            reqwest::StatusCode::UNPROCESSABLE_ENTITY
974        ));
975    }
976
977    /// Defaults for fresh inner state: 30s timeout, no retries (matches the
978    /// pre-change hardcoded behaviour for retries and read-timeout headroom).
979    #[test]
980    fn defaults_timeout_and_retries() {
981        let inner = OpenAiCompatInner::new("http://localhost:8080".to_string());
982        assert_eq!(inner.timeout_secs(), 30);
983        assert_eq!(inner.max_retries(), 0);
984    }
985
986    /// Explicit options constructor stores timeout and retries.
987    #[test]
988    fn with_options_stores_configuration() {
989        let inner =
990            OpenAiCompatInner::with_options("http://localhost:8080".to_string(), None, 120, 5);
991        assert_eq!(inner.timeout_secs(), 120);
992        assert_eq!(inner.max_retries(), 5);
993    }
994
995    /// set_timeout rebuilds the HTTP client and updates the stored value.
996    #[test]
997    fn set_timeout_updates_value() {
998        let mut inner = OpenAiCompatInner::new("http://localhost:8080".to_string());
999        inner.set_timeout(60);
1000        assert_eq!(inner.timeout_secs(), 60);
1001    }
1002
1003    /// set_retries updates the stored value.
1004    #[test]
1005    fn set_retries_updates_value() {
1006        let mut inner = OpenAiCompatInner::new("http://localhost:8080".to_string());
1007        inner.set_retries(7);
1008        assert_eq!(inner.max_retries(), 7);
1009    }
1010
1011    /// `models_url` strips a trailing `/v1` so both `http://host` and
1012    /// `http://host/v1` resolve to `http://host/v1/models`.
1013    #[test]
1014    fn models_url_normalises_trailing_v1() {
1015        let plain = OpenAiCompatInner::new("http://example.test:1234".to_string());
1016        assert_eq!(plain.models_url(), "http://example.test:1234/v1/models");
1017        let with_v1 = OpenAiCompatInner::new("http://example.test:1234/v1".to_string());
1018        assert_eq!(with_v1.models_url(), "http://example.test:1234/v1/models");
1019    }
1020
1021    /// A working `/health` endpoint short-circuits the check — `/v1/models`
1022    /// is never queried.
1023    #[tokio::test]
1024    async fn health_check_uses_health_endpoint_when_present() {
1025        let (base, log, _server) =
1026            spawn_mock_server(vec![("/health", 200, "{\"status\":\"ok\"}")]).await;
1027        let inner = OpenAiCompatInner::new(base);
1028
1029        assert!(inner.health_check().await.is_ok());
1030        assert_eq!(log.lock().unwrap().as_slice(), ["/health".to_string()]);
1031    }
1032
1033    /// No `/health` endpoint (404) but a non-empty `/v1/models` list counts
1034    /// as a successful health check.
1035    #[tokio::test]
1036    async fn health_check_falls_back_to_models_endpoint() {
1037        let (base, log, _server) = spawn_mock_server(vec![(
1038            "/v1/models",
1039            200,
1040            "{\"data\":[{\"id\":\"gpt-4o-mini\"}]}",
1041        )])
1042        .await;
1043        let inner = OpenAiCompatInner::new(base);
1044
1045        assert!(inner.health_check().await.is_ok());
1046        assert_eq!(
1047            log.lock().unwrap().as_slice(),
1048            ["/health".to_string(), "/v1/models".to_string()]
1049        );
1050    }
1051
1052    /// An empty model list proves nothing, so the fallback fails with a
1053    /// message mentioning both endpoints.
1054    #[tokio::test]
1055    async fn health_check_fallback_rejects_empty_model_list() {
1056        let (base, _log, _server) =
1057            spawn_mock_server(vec![("/v1/models", 200, "{\"data\":[]}")]).await;
1058        let inner = OpenAiCompatInner::new(base);
1059
1060        let err = inner.health_check().await.expect_err("should fail");
1061        assert!(err.contains("/health"), "unexpected error: {err}");
1062        assert!(err.contains("empty list"), "unexpected error: {err}");
1063    }
1064
1065    /// A `/v1/models` endpoint that errors out also fails the health check.
1066    #[tokio::test]
1067    async fn health_check_fallback_reports_models_failure() {
1068        let (base, _log, _server) =
1069            spawn_mock_server(vec![("/v1/models", 500, "{\"error\":\"boom\"}")]).await;
1070        let inner = OpenAiCompatInner::new(base);
1071
1072        let err = inner.health_check().await.expect_err("should fail");
1073        assert!(err.contains("/health"), "unexpected error: {err}");
1074        assert!(err.contains("500"), "unexpected error: {err}");
1075    }
1076
1077    /// Non-404 `/health` responses are reported as-is — the fallback must
1078    /// not mask a health endpoint that exists but is unhappy.
1079    #[tokio::test]
1080    async fn health_check_does_not_mask_non_404_errors() {
1081        let (base, log, _server) =
1082            spawn_mock_server(vec![("/health", 503, "{\"status\":\"unhealthy\"}")]).await;
1083        let inner = OpenAiCompatInner::new(base);
1084
1085        let err = inner.health_check().await.expect_err("should fail");
1086        assert!(err.contains("503"), "unexpected error: {err}");
1087        assert_eq!(log.lock().unwrap().as_slice(), ["/health".to_string()]);
1088    }
1089
1090    /// `fetch_models_raw` propagates errors instead of silently returning
1091    /// the currently selected model.
1092    #[tokio::test]
1093    async fn fetch_models_raw_propagates_errors() {
1094        let (base, _log, _server) = spawn_mock_server(vec![]).await;
1095        let inner = OpenAiCompatInner::new(base);
1096        assert!(inner.fetch_models_raw().await.is_err());
1097
1098        let (base, _log, _server) = spawn_mock_server(vec![(
1099            "/v1/models",
1100            200,
1101            "{\"data\":[{\"id\":\"alpha\"},{\"id\":\"beta\"}]}",
1102        )])
1103        .await;
1104        let inner = OpenAiCompatInner::new(base);
1105        assert_eq!(
1106            inner.fetch_models_raw().await.expect("ok"),
1107            vec!["alpha".to_string(), "beta".to_string()]
1108        );
1109    }
1110
1111    /// `fetch_model_list` keeps its lenient contract: on failure it falls
1112    /// back to the currently selected model.
1113    #[tokio::test]
1114    async fn fetch_model_list_falls_back_to_selected_model() {
1115        let (base, _log, _server) = spawn_mock_server(vec![]).await;
1116        let mut inner = OpenAiCompatInner::new(base);
1117        inner.select_model("chosen-model".to_string());
1118
1119        assert_eq!(
1120            inner.fetch_model_list().await,
1121            vec!["chosen-model".to_string()]
1122        );
1123    }
1124
1125    // ── reasoning / thinking extraction ──
1126
1127    /// Parse a single SSE `data:` payload into a `StreamChunk`.
1128    fn parse_chunk(json: &str) -> StreamChunk {
1129        serde_json::from_str(json).unwrap_or_else(|e| panic!("invalid chunk {json}: {e}"))
1130    }
1131
1132    /// Pull the reasoning out of the first choice of a chunk.
1133    fn reasoning_of(json: &str) -> Option<String> {
1134        let chunk = parse_chunk(json);
1135        chunk
1136            .choices
1137            .first()
1138            .and_then(|c| delta_reasoning(&c.delta))
1139    }
1140
1141    /// OpenRouter / DeepSeek spell the field `reasoning`.
1142    #[test]
1143    fn delta_reasoning_reads_openrouter_field() {
1144        assert_eq!(
1145            reasoning_of(r#"{"choices":[{"delta":{"reasoning":"hmm"}}]}"#).as_deref(),
1146            Some("hmm")
1147        );
1148    }
1149
1150    /// vLLM / Qwen reasoning parsers emit `reasoning_content`.
1151    #[test]
1152    fn delta_reasoning_reads_vllm_field() {
1153        assert_eq!(
1154            reasoning_of(r#"{"choices":[{"delta":{"reasoning_content":"hmm"}}]}"#).as_deref(),
1155            Some("hmm")
1156        );
1157    }
1158
1159    /// OpenRouter's structured `reasoning_details` array is the fallback when
1160    /// no plain-text field is present.
1161    #[test]
1162    fn delta_reasoning_reads_structured_details() {
1163        assert_eq!(
1164            reasoning_of(
1165                r#"{"choices":[{"delta":{"reasoning_details":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}}]}"#
1166            )
1167            .as_deref(),
1168            Some("ab")
1169        );
1170    }
1171
1172    /// Anthropic-style summary parts nest their text in a `summary` array.
1173    #[test]
1174    fn delta_reasoning_reads_summary_parts() {
1175        assert_eq!(
1176            reasoning_of(
1177                r#"{"choices":[{"delta":{"reasoning_details":[{"type":"summary_text","summary":[{"text":"x"}]}]}}]}"#
1178            )
1179            .as_deref(),
1180            Some("x")
1181        );
1182    }
1183
1184    /// Encrypted / redacted reasoning has no readable payload — it must not
1185    /// produce an empty `Some("")` that would trip the UI into printing a
1186    /// bare `[thinking]` header.
1187    #[test]
1188    fn delta_reasoning_ignores_opaque_details() {
1189        assert_eq!(
1190            reasoning_of(
1191                r#"{"choices":[{"delta":{"reasoning_details":[{"type":"reasoning.encrypted","data":"gAAAAA"}]}}]}"#
1192            ),
1193            None
1194        );
1195    }
1196
1197    /// When both spellings are present (OpenRouter does this), the plain
1198    /// field wins so the chain is not duplicated.
1199    #[test]
1200    fn delta_reasoning_does_not_duplicate_plain_and_structured() {
1201        assert_eq!(
1202            reasoning_of(
1203                r#"{"choices":[{"delta":{"reasoning":"same","reasoning_details":[{"type":"text","text":"same"}]}}]}"#
1204            )
1205            .as_deref(),
1206            Some("same")
1207        );
1208    }
1209
1210    /// A content-only delta carries no reasoning.
1211    #[test]
1212    fn delta_reasoning_is_none_without_reasoning_fields() {
1213        assert_eq!(
1214            reasoning_of(r#"{"choices":[{"delta":{"content":"hello"}}]}"#),
1215            None
1216        );
1217    }
1218
1219    /// SSE body serving a reasoning stream: thinking deltas, then content,
1220    /// then a usage-only final chunk.
1221    const REASONING_SSE: &str = concat!(
1222        "data: {\"choices\":[{\"delta\":{\"reasoning\":\"think \"}}]}\n\n",
1223        "data: {\"choices\":[{\"delta\":{\"reasoning\":\"more\"}}]}\n\n",
1224        "data: {\"choices\":[{\"delta\":{\"content\":\"Hel\"}}]}\n\n",
1225        "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\n",
1226        "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":2,\"total_tokens\":3}}\n\n",
1227        "data: [DONE]\n\n",
1228    );
1229
1230    /// End-to-end: reasoning deltas reach the consumer through
1231    /// `ChatMessage::thinking`, and content is not mixed in with them.
1232    #[tokio::test]
1233    async fn stream_forwards_reasoning_as_thinking_chunks() {
1234        let (base, _log, _server) =
1235            spawn_mock_server(vec![("/v1/chat/completions", 200, REASONING_SSE)]).await;
1236        let mut inner = OpenAiCompatInner::new(base);
1237        inner.select_model("test-model".to_string());
1238
1239        let mut recv = inner
1240            .chat(vec![Message::simple(Role::User, "hi")], vec![])
1241            .await
1242            .expect("chat started");
1243
1244        let mut thinking = String::new();
1245        let mut content = String::new();
1246        let mut done = false;
1247        while let Some(resp) = recv.recv().await {
1248            if let Some(t) = &resp.message.thinking {
1249                thinking.push_str(t);
1250            }
1251            content.push_str(&resp.message.content);
1252            if resp.done {
1253                done = true;
1254                break;
1255            }
1256        }
1257        assert!(done, "stream should terminate with a done chunk");
1258        assert_eq!(thinking, "think more");
1259        assert_eq!(content, "Hello");
1260    }
1261
1262    /// A server that never emits reasoning keeps `thinking` unset, so the UI
1263    /// skips the `[thinking]` block entirely.
1264    #[tokio::test]
1265    async fn stream_without_reasoning_leaves_thinking_unset() {
1266        let (base, _log, _server) = spawn_mock_server(vec![(
1267            "/v1/chat/completions",
1268            200,
1269            "data: {\"choices\":[{\"delta\":{\"content\":\"plain\"}}]}\n\ndata: [DONE]\n\n",
1270        )])
1271        .await;
1272        let mut inner = OpenAiCompatInner::new(base);
1273        inner.select_model("test-model".to_string());
1274
1275        let mut recv = inner
1276            .chat(vec![Message::simple(Role::User, "hi")], vec![])
1277            .await
1278            .expect("chat started");
1279
1280        while let Some(resp) = recv.recv().await {
1281            assert!(resp.message.thinking.is_none());
1282            if resp.done {
1283                break;
1284            }
1285        }
1286    }
1287}