Skip to main content

graphrt_llm/
openai.rs

1//! OpenAI-compatible chat completions provider.
2//!
3//! Works with any endpoint implementing `POST {base_url}/chat/completions` with
4//! streaming: OpenAI, Azure OpenAI (set [`AuthStyle::Header`] with `api-key`
5//! and an `api-version` query parameter), Ollama, vLLM, LM Studio, Groq,
6//! OpenRouter, DeepSeek, Together, Mistral and others.
7
8use std::collections::HashMap;
9use std::fmt;
10use std::sync::Arc;
11use std::time::Duration;
12
13use futures::future::BoxFuture;
14use graphrt_core::event::Usage;
15use serde_json::{Map, Value as Json, json};
16
17use crate::credentials::{Credential, CredentialSource};
18use crate::error::LlmError;
19use crate::http::{Decoded, SseDecoder, send_sse, truncated};
20use crate::sse::SseEvent;
21use crate::types::{
22    ChatDelta, ChatModel, ChatRequest, ChatStream, ContentPart, FinishReason, Message, ResponseFormat, Role,
23    ToolChoice, check_extra, tool_images_intro, tool_text_with_image_note,
24};
25
26/// How the API key is sent.
27#[derive(Clone, Debug, PartialEq, Eq)]
28#[non_exhaustive]
29pub enum AuthStyle {
30    /// `Authorization: Bearer <key>`.
31    Bearer,
32    /// A custom header, e.g. `api-key` for Azure OpenAI.
33    Header(String),
34}
35
36/// Configuration of an OpenAI-compatible endpoint.
37#[derive(Clone)]
38pub struct OpenAiConfig {
39    /// Base URL without the `/chat/completions` suffix, e.g. `https://api.openai.com/v1`.
40    pub base_url: String,
41    pub api_key: Option<String>,
42    pub auth: AuthStyle,
43    /// Extra headers sent with every request.
44    pub headers: Vec<(String, String)>,
45    /// Extra query parameters (e.g. `api-version`).
46    pub query: Vec<(String, String)>,
47    /// Ask for a final usage chunk (`stream_options.include_usage`). Disable for
48    /// endpoints that reject the field.
49    pub include_usage: bool,
50    /// Send `max_completion_tokens` instead of `max_tokens` (required by some newer models).
51    pub use_max_completion_tokens: bool,
52    pub connect_timeout: Duration,
53    /// Per-request credential source. When set it replaces `api_key`; its headers are
54    /// appended and its `base_url`, if any, overrides `base_url`. A 401/403 response
55    /// invalidates the credential and the request is retried once.
56    pub credentials: Option<Arc<dyn CredentialSource>>,
57    /// Provider name passed to the credential source (default `openai`).
58    pub credential_name: String,
59}
60
61impl fmt::Debug for OpenAiConfig {
62    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
63        f.debug_struct("OpenAiConfig")
64            .field("base_url", &self.base_url)
65            .field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
66            .field("auth", &self.auth)
67            .field("headers", &self.headers.iter().map(|(k, _)| k).collect::<Vec<_>>())
68            .field("include_usage", &self.include_usage)
69            .field("credentials", &self.credentials.as_ref().map(|_| "<source>"))
70            .field("credential_name", &self.credential_name)
71            .finish_non_exhaustive()
72    }
73}
74
75impl Default for OpenAiConfig {
76    fn default() -> Self {
77        Self {
78            base_url: "https://api.openai.com/v1".to_owned(),
79            api_key: None,
80            auth: AuthStyle::Bearer,
81            headers: Vec::new(),
82            query: Vec::new(),
83            include_usage: true,
84            use_max_completion_tokens: false,
85            connect_timeout: Duration::from_secs(10),
86            credentials: None,
87            credential_name: "openai".to_owned(),
88        }
89    }
90}
91
92impl OpenAiConfig {
93    pub fn new(base_url: impl Into<String>, api_key: Option<String>) -> Self {
94        Self { base_url: base_url.into(), api_key, ..Self::default() }
95    }
96
97    /// Ask `source` for a credential before every request (see [`crate::credentials`]).
98    pub fn with_credentials(mut self, source: Arc<dyn CredentialSource>) -> Self {
99        self.credentials = Some(source);
100        self
101    }
102
103    /// Provider name passed to the credential source.
104    pub fn credential_name(mut self, name: impl Into<String>) -> Self {
105        self.credential_name = name.into();
106        self
107    }
108}
109
110/// An OpenAI-compatible chat model endpoint.
111#[derive(Clone)]
112pub struct OpenAiModel {
113    config: OpenAiConfig,
114    client: crate::http::LazyClient,
115}
116
117impl fmt::Debug for OpenAiModel {
118    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
119        f.debug_struct("OpenAiModel").field("config", &self.config).finish_non_exhaustive()
120    }
121}
122
123impl OpenAiModel {
124    pub fn new(config: OpenAiConfig) -> Result<Self, LlmError> {
125        let client = crate::http::LazyClient::new(config.connect_timeout);
126        Ok(Self { config, client })
127    }
128
129    fn endpoint(&self, base_url: &str, with_query: bool) -> String {
130        let mut url = format!("{}/chat/completions", base_url.trim_end_matches('/'));
131        let query: &[(String, String)] = if with_query { &self.config.query } else { &[] };
132        for (i, (k, v)) in query.iter().enumerate() {
133            url.push(if i == 0 { '?' } else { '&' });
134            url.push_str(&percent_encode(k));
135            url.push('=');
136            url.push_str(&percent_encode(v));
137        }
138        url
139    }
140}
141
142/// Percent-encode a query component (RFC 3986 unreserved characters pass through).
143fn percent_encode(s: &str) -> String {
144    let mut out = String::with_capacity(s.len());
145    for b in s.bytes() {
146        if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~') {
147            out.push(char::from(b));
148        } else {
149            out.push_str(&format!("%{b:02X}"));
150        }
151    }
152    out
153}
154
155impl OpenAiModel {
156    fn request(&self, body: &Json, credential: Option<&Credential>) -> Result<reqwest::RequestBuilder, LlmError> {
157        let endpoint =
158            crate::http::resolve_endpoint(&self.config.base_url, credential.and_then(|c| c.base_url.as_deref()))?;
159        let mut builder = self.client.get()?.post(self.endpoint(&endpoint.base_url, endpoint.send_static)).json(body);
160        let key = credential.map(|c| c.secret.as_str()).or(self.config.api_key.as_deref());
161        if let Some(key) = key {
162            builder = match &self.config.auth {
163                AuthStyle::Bearer => builder.bearer_auth(key),
164                AuthStyle::Header(name) => builder.header(name.as_str(), key),
165            };
166        }
167        // Operator headers (gateway keys, org ids) never follow a credential override
168        // to another origin.
169        if endpoint.send_static {
170            for (k, v) in &self.config.headers {
171                builder = builder.header(k.as_str(), v.as_str());
172            }
173        }
174        if let Some(credential) = credential {
175            for (k, v) in &credential.headers {
176                builder = builder.header(k.as_str(), v.as_str());
177            }
178        }
179        Ok(builder)
180    }
181}
182
183impl ChatModel for OpenAiModel {
184    fn chat(&self, request: ChatRequest) -> BoxFuture<'_, Result<ChatStream, LlmError>> {
185        Box::pin(async move {
186            let body = build_body(&request, &self.config)?;
187            let name = self.config.credential_name.as_str();
188            let mut retried = false;
189            loop {
190                let credential = match &self.config.credentials {
191                    Some(source) => Some(source.credential(name).await?),
192                    None => None,
193                };
194                let result = send_sse(self.request(&body, credential.as_ref())?, OpenAiDecoder::default()).await;
195                match (result, &self.config.credentials) {
196                    // The credential was rejected: drop it and retry once with a fresh one.
197                    (Err(LlmError::Auth(_)), Some(source)) if !retried => {
198                        source.invalidate(name);
199                        retried = true;
200                    }
201                    (result, _) => return result,
202                }
203            }
204        })
205    }
206}
207
208fn content_json(parts: &[ContentPart]) -> Json {
209    if let [ContentPart::Text { text }] = parts {
210        return Json::String(text.clone());
211    }
212    let items: Vec<Json> = parts
213        .iter()
214        .map(|p| match p {
215            ContentPart::Text { text } => json!({"type": "text", "text": text}),
216            ContentPart::Image { url, detail } => {
217                let mut image = Map::new();
218                image.insert("url".into(), Json::String(url.clone()));
219                if let Some(d) = detail {
220                    image.insert("detail".into(), Json::String(d.clone()));
221                }
222                json!({"type": "image_url", "image_url": image})
223            }
224        })
225        .collect();
226    Json::Array(items)
227}
228
229fn message_json(m: &Message) -> Result<Json, LlmError> {
230    Ok(match m.role {
231        Role::System => json!({"role": "system", "content": content_json(&m.content)}),
232        Role::User => json!({"role": "user", "content": content_json(&m.content)}),
233        Role::Tool => {
234            let id = m
235                .tool_call_id
236                .as_deref()
237                .filter(|id| !id.is_empty())
238                .ok_or_else(|| LlmError::Config("tool message without tool_call_id".into()))?;
239            json!({"role": "tool", "tool_call_id": id, "content": tool_text_with_image_note(m)})
240        }
241        Role::Assistant => {
242            let mut obj = Map::new();
243            obj.insert("role".into(), json!("assistant"));
244            let text = m.text();
245            obj.insert("content".into(), if text.is_empty() { Json::Null } else { Json::String(text) });
246            if !m.tool_calls.is_empty() {
247                let calls: Vec<Json> = m
248                    .tool_calls
249                    .iter()
250                    .map(|c| json!({"id": c.id, "type": "function", "function": {"name": c.name, "arguments": c.arguments}}))
251                    .collect();
252                obj.insert("tool_calls".into(), Json::Array(calls));
253            }
254            Json::Object(obj)
255        }
256    })
257}
258
259/// Encode the conversation. Tool messages are text-only in chat completions,
260/// so images from tool results go in one user message after the run of tool
261/// messages they belong to.
262fn messages_json(messages: &[Message]) -> Result<Vec<Json>, LlmError> {
263    let mut out = Vec::with_capacity(messages.len());
264    let mut call_names: HashMap<&str, &str> = HashMap::new();
265    let mut deferred: Vec<Json> = Vec::new();
266    for m in messages {
267        if m.role != Role::Tool && !deferred.is_empty() {
268            out.push(json!({"role": "user", "content": std::mem::take(&mut deferred)}));
269        }
270        for c in &m.tool_calls {
271            call_names.insert(c.id.as_str(), c.name.as_str());
272        }
273        out.push(message_json(m)?);
274        if m.role == Role::Tool && m.images().next().is_some() {
275            let id = m.tool_call_id.as_deref().unwrap_or_default();
276            let intro = tool_images_intro(call_names.get(id).copied(), id);
277            deferred.push(json!({"type": "text", "text": intro}));
278            let images: Vec<ContentPart> = m.images().cloned().collect();
279            if let Json::Array(parts) = content_json(&images) {
280                deferred.extend(parts);
281            }
282        }
283    }
284    if !deferred.is_empty() {
285        out.push(json!({"role": "user", "content": deferred}));
286    }
287    Ok(out)
288}
289
290/// Build the JSON request body.
291pub(crate) fn build_body(req: &ChatRequest, config: &OpenAiConfig) -> Result<Json, LlmError> {
292    check_extra(req)?;
293    // Only the first choice is decoded; asking for more would silently discard them.
294    if let Some(n) = req.extra.get("n") {
295        if n.as_u64() != Some(1) {
296            return Err(LlmError::Config(format!("`n` must be 1 (got {n}); only one choice is supported")));
297        }
298    }
299    let mut body = Map::new();
300    body.insert("model".into(), json!(req.model));
301    body.insert("messages".into(), Json::Array(messages_json(&req.messages)?));
302    body.insert("stream".into(), json!(true));
303    if config.include_usage {
304        body.insert("stream_options".into(), json!({"include_usage": true}));
305    }
306    if !req.tools.is_empty() {
307        let tools: Vec<Json> = req
308            .tools
309            .iter()
310            .map(|t| json!({"type": "function", "function": {"name": t.name, "description": t.description, "parameters": t.parameters}}))
311            .collect();
312        body.insert("tools".into(), Json::Array(tools));
313    }
314    if let Some(choice) = &req.tool_choice {
315        let v = match choice {
316            ToolChoice::Auto => json!("auto"),
317            ToolChoice::None => json!("none"),
318            ToolChoice::Required => json!("required"),
319            ToolChoice::Tool(name) => json!({"type": "function", "function": {"name": name}}),
320        };
321        body.insert("tool_choice".into(), v);
322    }
323    if let Some(t) = req.temperature {
324        body.insert("temperature".into(), json!(t));
325    }
326    if let Some(p) = req.top_p {
327        body.insert("top_p".into(), json!(p));
328    }
329    if let Some(max) = req.max_tokens {
330        let key = if config.use_max_completion_tokens { "max_completion_tokens" } else { "max_tokens" };
331        body.insert(key.into(), json!(max));
332    }
333    if !req.stop.is_empty() {
334        body.insert("stop".into(), json!(req.stop));
335    }
336    match &req.response_format {
337        None | Some(ResponseFormat::Text) => {}
338        Some(ResponseFormat::JsonObject) => {
339            body.insert("response_format".into(), json!({"type": "json_object"}));
340        }
341        Some(ResponseFormat::JsonSchema { name, schema, strict }) => {
342            body.insert(
343                "response_format".into(),
344                json!({"type": "json_schema", "json_schema": {"name": name, "schema": schema, "strict": strict}}),
345            );
346        }
347    }
348    for (k, v) in &req.extra {
349        body.insert(k.clone(), v.clone());
350    }
351    Ok(Json::Object(body))
352}
353
354/// Decodes `chat.completion.chunk` events.
355///
356/// Tool-call fragments are keyed by their `index`. Some compatible servers omit
357/// `index`; then a fragment with a new `id` starts a new call and a fragment
358/// without `id` continues the most recent one.
359#[derive(Debug, Default)]
360pub(crate) struct OpenAiDecoder {
361    /// Call index assigned to each call id seen so far.
362    ids: HashMap<String, usize>,
363    /// Index of the most recent call fragment.
364    last: Option<usize>,
365    /// Smallest index not used yet.
366    next: usize,
367    /// A `finish_reason` was received (some servers end without `[DONE]`).
368    saw_finish: bool,
369}
370
371impl OpenAiDecoder {
372    fn call_index(&mut self, explicit: Option<usize>, id: Option<&str>) -> usize {
373        let index = match (explicit, id) {
374            (Some(i), _) => i,
375            (None, Some(id)) => match self.ids.get(id) {
376                Some(&i) => i,
377                None => self.next,
378            },
379            (None, None) => self.last.unwrap_or(self.next),
380        };
381        if let Some(id) = id {
382            self.ids.entry(id.to_owned()).or_insert(index);
383        }
384        self.last = Some(index);
385        self.next = self.next.max(index.saturating_add(1));
386        index
387    }
388}
389
390/// Classify an error object embedded in the stream.
391fn stream_error(err: &Json) -> LlmError {
392    let message = err.get("message").and_then(Json::as_str).map(str::to_owned).unwrap_or_else(|| err.to_string());
393    let kind = err.get("type").and_then(Json::as_str).unwrap_or_default();
394    let code = match err.get("code") {
395        Some(Json::String(c)) => c.clone(),
396        Some(Json::Number(n)) => n.to_string(),
397        _ => String::new(),
398    };
399    let tag = format!("{kind} {code}").to_ascii_lowercase();
400    if tag.contains("rate_limit") || code == "429" {
401        LlmError::RateLimit { message, retry_after_ms: None }
402    } else if tag.contains("context_length") {
403        LlmError::ContextLength(message)
404    } else if tag.contains("auth") || tag.contains("permission") || code == "401" || code == "403" {
405        LlmError::Auth(message)
406    } else if tag.contains("invalid_request") || code == "400" {
407        LlmError::BadRequest(message)
408    } else {
409        LlmError::Server { status: 0, message }
410    }
411}
412
413fn as_u64(v: Option<&Json>) -> u64 {
414    v.and_then(Json::as_u64).unwrap_or(0)
415}
416
417fn finish_reason(s: &str) -> FinishReason {
418    match s {
419        "stop" => FinishReason::Stop,
420        "length" => FinishReason::Length,
421        "tool_calls" | "function_call" => FinishReason::ToolCalls,
422        "content_filter" => FinishReason::ContentFilter,
423        other => FinishReason::Other(other.to_owned()),
424    }
425}
426
427impl SseDecoder for OpenAiDecoder {
428    fn decode(&mut self, event: SseEvent) -> Result<Decoded, LlmError> {
429        let data = event.data.trim();
430        if data == "[DONE]" {
431            if !self.saw_finish {
432                return Err(truncated());
433            }
434            return Ok(Decoded::done(Vec::new()));
435        }
436        if data.is_empty() {
437            return Ok(Decoded::default());
438        }
439        let chunk: Json = serde_json::from_str(data).map_err(|e| LlmError::Decode(format!("{e}: {data}")))?;
440        if let Some(err) = chunk.get("error").filter(|e| !e.is_null()) {
441            return Err(stream_error(err));
442        }
443        let mut deltas = Vec::new();
444        if let Some(choice) = chunk.get("choices").and_then(Json::as_array).and_then(|c| c.first()) {
445            if let Some(delta) = choice.get("delta") {
446                // Reasoning is exposed as `reasoning_content` (DeepSeek, vLLM) or `reasoning`
447                // (OpenRouter, Ollama). Some servers send both with the same text: use the first.
448                let reasoning = ["reasoning_content", "reasoning"]
449                    .iter()
450                    .find_map(|key| delta.get(*key).and_then(Json::as_str).filter(|s| !s.is_empty()));
451                if let Some(r) = reasoning {
452                    deltas.push(ChatDelta::Reasoning(r.to_owned()));
453                }
454                if let Some(text) = delta.get("content").and_then(Json::as_str).filter(|s| !s.is_empty()) {
455                    deltas.push(ChatDelta::Text(text.to_owned()));
456                }
457                if let Some(calls) = delta.get("tool_calls").and_then(Json::as_array) {
458                    for call in calls {
459                        let explicit = call.get("index").and_then(Json::as_u64).and_then(|i| usize::try_from(i).ok());
460                        let id = call.get("id").and_then(Json::as_str).filter(|s| !s.is_empty());
461                        let index = self.call_index(explicit, id);
462                        let function = call.get("function");
463                        let arguments = match function.and_then(|f| f.get("arguments")) {
464                            None | Some(Json::Null) => String::new(),
465                            Some(Json::String(s)) => s.clone(),
466                            // Some servers send the arguments as a JSON object instead of a string.
467                            Some(other) => other.to_string(),
468                        };
469                        deltas.push(ChatDelta::ToolCall {
470                            index,
471                            id: id.map(str::to_owned),
472                            name: function.and_then(|f| f.get("name")).and_then(Json::as_str).map(str::to_owned),
473                            arguments,
474                        });
475                    }
476                }
477            }
478            if let Some(reason) = choice.get("finish_reason").and_then(Json::as_str) {
479                self.saw_finish = true;
480                deltas.push(ChatDelta::Finish(finish_reason(reason)));
481            }
482        }
483        if let Some(usage) = chunk.get("usage").filter(|u| u.is_object()) {
484            deltas.push(ChatDelta::Usage(Usage {
485                prompt_tokens: as_u64(usage.get("prompt_tokens")),
486                completion_tokens: as_u64(usage.get("completion_tokens")),
487                total_tokens: as_u64(usage.get("total_tokens")),
488            }));
489        }
490        Ok(Decoded::deltas(deltas))
491    }
492
493    /// Servers that omit `[DONE]` are accepted once a `finish_reason` arrived.
494    fn end_of_stream(&mut self) -> Result<Vec<ChatDelta>, LlmError> {
495        if self.saw_finish { Ok(Vec::new()) } else { Err(truncated()) }
496    }
497}
498
499#[cfg(test)]
500mod tests {
501    use super::*;
502    use crate::sse::SseParser;
503    use crate::types::ToolCall;
504    use graphrt_core::tool::ToolSpec;
505
506    #[test]
507    fn body_shape() {
508        let mut assistant = Message::assistant("");
509        assistant.tool_calls = vec![ToolCall { id: "c1".into(), name: "search".into(), arguments: "{}".into() }];
510        let mut req = ChatRequest::new(
511            "gpt-4o-mini",
512            vec![
513                Message::system("sys"),
514                Message {
515                    role: Role::User,
516                    content: vec![
517                        ContentPart::Text { text: "look".into() },
518                        ContentPart::Image { url: "https://x/y.png".into(), detail: Some("high".into()) },
519                    ],
520                    tool_calls: vec![],
521                    tool_call_id: None,
522                    provider_state: None,
523                },
524                assistant,
525                Message::tool("c1", "result"),
526            ],
527        );
528        req.tools =
529            vec![ToolSpec { name: "search".into(), description: "d".into(), parameters: json!({"type": "object"}) }];
530        req.tool_choice = Some(ToolChoice::Tool("search".into()));
531        req.max_tokens = Some(10);
532        req.response_format =
533            Some(ResponseFormat::JsonSchema { name: "out".into(), schema: json!({"type": "object"}), strict: true });
534        req.extra.insert("seed".into(), json!(7));
535        let body = build_body(&req, &OpenAiConfig::default()).unwrap();
536        assert_eq!(body["stream"], true);
537        assert_eq!(body["stream_options"]["include_usage"], true);
538        assert_eq!(body["messages"][0]["content"], "sys");
539        assert_eq!(body["messages"][1]["content"][1]["image_url"]["detail"], "high");
540        assert_eq!(body["messages"][2]["content"], Json::Null);
541        assert_eq!(body["messages"][2]["tool_calls"][0]["function"]["name"], "search");
542        assert_eq!(body["messages"][3]["tool_call_id"], "c1");
543        assert_eq!(body["tools"][0]["function"]["parameters"]["type"], "object");
544        assert_eq!(body["tool_choice"]["function"]["name"], "search");
545        assert_eq!(body["max_tokens"], 10);
546        assert_eq!(body["response_format"]["json_schema"]["strict"], true);
547        assert_eq!(body["seed"], 7);
548
549        let cfg = OpenAiConfig { use_max_completion_tokens: true, include_usage: false, ..OpenAiConfig::default() };
550        let body = build_body(&req, &cfg).unwrap();
551        assert_eq!(body["max_completion_tokens"], 10);
552        assert!(body.get("stream_options").is_none());
553    }
554
555    fn two_tool_calls() -> Message {
556        let mut assistant = Message::assistant("");
557        assistant.tool_calls = vec![
558            ToolCall { id: "c1".into(), name: "screenshot".into(), arguments: "{}".into() },
559            ToolCall { id: "c2".into(), name: "clock".into(), arguments: "{}".into() },
560        ];
561        assistant
562    }
563
564    #[test]
565    fn tool_images_follow_the_tool_messages_in_a_user_message() {
566        let req = ChatRequest::new(
567            "gpt-4o",
568            vec![
569                Message::user("what is on screen?"),
570                two_tool_calls(),
571                Message::tool_with_images("c1", "1280x800", ["data:image/png;base64,AAAA".to_owned()]),
572                Message::tool("c2", "noon"),
573                Message::user("thanks"),
574            ],
575        );
576        let body = build_body(&req, &OpenAiConfig::default()).unwrap();
577        assert_eq!(
578            body["messages"],
579            json!([
580                {"role": "user", "content": "what is on screen?"},
581                {"role": "assistant", "content": null, "tool_calls": [
582                    {"id": "c1", "type": "function", "function": {"name": "screenshot", "arguments": "{}"}},
583                    {"id": "c2", "type": "function", "function": {"name": "clock", "arguments": "{}"}}
584                ]},
585                {"role": "tool", "tool_call_id": "c1", "content": "1280x800\n[1 image attached after the tool results]"},
586                {"role": "tool", "tool_call_id": "c2", "content": "noon"},
587                {"role": "user", "content": [
588                    {"type": "text", "text": "Images returned by tool `screenshot` (call c1):"},
589                    {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
590                ]},
591                {"role": "user", "content": "thanks"}
592            ])
593        );
594    }
595
596    #[test]
597    fn tool_images_at_the_end_of_the_history_are_flushed() {
598        let req = ChatRequest::new(
599            "gpt-4o",
600            vec![
601                Message::user("q"),
602                two_tool_calls(),
603                Message::tool_with_images("c1", "", ["https://x/a.png".to_owned()]),
604                Message::tool_with_images("c2", "b", ["https://x/b.png".to_owned()]),
605            ],
606        );
607        let body = build_body(&req, &OpenAiConfig::default()).unwrap();
608        let messages = body["messages"].as_array().unwrap();
609        assert_eq!(messages.len(), 5);
610        assert_eq!(messages[2]["content"], "[1 image attached after the tool results]");
611        let images = &messages[4];
612        assert_eq!(images["role"], "user");
613        assert_eq!(images["content"][2]["text"], "Images returned by tool `clock` (call c2):");
614        assert_eq!(images["content"][3]["image_url"]["url"], "https://x/b.png");
615    }
616
617    #[test]
618    fn body_rejects_reserved_extra_n_and_missing_tool_id() {
619        let req = ChatRequest::new("m", vec![Message::user("x")]).with_extra("stream", json!(false));
620        assert!(matches!(build_body(&req, &OpenAiConfig::default()), Err(LlmError::Config(_))));
621        let req = ChatRequest::new("m", vec![Message::user("x")]).with_extra("n", json!(2));
622        assert!(matches!(build_body(&req, &OpenAiConfig::default()), Err(LlmError::Config(_))));
623        let mut tool = Message::tool("x", "r");
624        tool.tool_call_id = None;
625        let req = ChatRequest::new("m", vec![tool]);
626        assert!(matches!(build_body(&req, &OpenAiConfig::default()), Err(LlmError::Config(_))));
627    }
628
629    fn decode_all(fixture: &str) -> (Vec<ChatDelta>, Result<bool, LlmError>) {
630        let mut parser = SseParser::new();
631        let mut decoder = OpenAiDecoder::default();
632        let mut deltas = Vec::new();
633        for ev in parser.push(fixture.as_bytes()).unwrap() {
634            match decoder.decode(ev) {
635                Ok(d) => {
636                    deltas.extend(d.deltas);
637                    if d.done {
638                        return (deltas, Ok(true));
639                    }
640                }
641                Err(e) => return (deltas, Err(e)),
642            }
643        }
644        match decoder.end_of_stream() {
645            Ok(d) => {
646                deltas.extend(d);
647                (deltas, Ok(false))
648            }
649            Err(e) => (deltas, Err(e)),
650        }
651    }
652
653    #[test]
654    fn truncated_stream_is_a_retryable_error() {
655        let (_, end) = decode_all("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hel\"}}]}\n\n");
656        let err = end.unwrap_err();
657        assert!(matches!(err, LlmError::Network(_)));
658        assert!(err.is_retryable());
659        // `[DONE]` without any finish_reason is also incomplete.
660        let (_, end) =
661            decode_all("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"x\"}}]}\n\ndata: [DONE]\n\n");
662        assert!(end.is_err());
663        // A finish_reason without `[DONE]` is accepted.
664        let (deltas, end) = decode_all(
665            "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"x\"},\"finish_reason\":\"stop\"}]}\n\n",
666        );
667        assert!(!end.unwrap());
668        assert_eq!(deltas.last(), Some(&ChatDelta::Finish(FinishReason::Stop)));
669    }
670
671    #[tokio::test]
672    async fn tool_calls_without_index_and_object_arguments() {
673        let fixture = concat!(
674            "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"id\":\"a\",\"function\":{\"name\":\"one\",\"arguments\":{\"x\":1}}}]}}]}\n\n",
675            "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"id\":\"b\",\"function\":{\"name\":\"two\",\"arguments\":\"{\\\"y\\\":\"}}]}}]}\n\n",
676            "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"2}\"}}]}}]}\n\n",
677            "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n",
678            "data: [DONE]\n\n",
679        );
680        let (deltas, end) = decode_all(fixture);
681        assert!(end.unwrap());
682        let r =
683            crate::types::collect(Box::pin(futures::stream::iter(deltas.into_iter().map(Ok))), |_| {}).await.unwrap();
684        assert_eq!(r.tool_calls.len(), 2, "{:?}", r.tool_calls);
685        assert_eq!(r.tool_calls[0].arguments_json().unwrap()["x"], 1);
686        assert_eq!(r.tool_calls[1].name, "two");
687        assert_eq!(r.tool_calls[1].arguments_json().unwrap()["y"], 2);
688    }
689
690    #[test]
691    fn duplicate_reasoning_fields_are_not_doubled() {
692        let (deltas, _) = decode_all(
693            "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"r\",\"reasoning\":\"r\"},\"finish_reason\":\"stop\"}]}\n\n",
694        );
695        assert_eq!(deltas.iter().filter(|d| matches!(d, ChatDelta::Reasoning(_))).count(), 1);
696    }
697
698    #[test]
699    fn decode_recorded_stream() {
700        let fixture = concat!(
701            "data: {\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"reasoning_content\":\"think\"}}]}\n\n",
702            "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hel\"}}]}\n\n",
703            "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"lo\"}}]}\n\n",
704            "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get\",\"arguments\":\"\"}}]}}]}\n\n",
705            "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"a\\\":1}\"}}]}}]}\n\n",
706            "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n",
707            "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":2,\"total_tokens\":7}}\n\n",
708            "data: [DONE]\n\n",
709        );
710        let (deltas, done) = decode_all(fixture);
711        assert!(done.unwrap());
712        assert_eq!(deltas[0], ChatDelta::Reasoning("think".into()));
713        assert_eq!(deltas[1], ChatDelta::Text("Hel".into()));
714        assert!(matches!(&deltas[3], ChatDelta::ToolCall { index: 0, name: Some(n), .. } if n == "get"));
715        assert!(matches!(&deltas[4], ChatDelta::ToolCall { arguments, .. } if arguments == "{\"a\":1}"));
716        assert_eq!(deltas[5], ChatDelta::Finish(FinishReason::ToolCalls));
717        assert_eq!(deltas[6], ChatDelta::Usage(Usage { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }));
718    }
719
720    #[test]
721    fn stream_error_object() {
722        let mut d = OpenAiDecoder::default();
723        let ev = SseEvent { data: "{\"error\":{\"message\":\"boom\"}}".into(), ..SseEvent::default() };
724        assert!(matches!(d.decode(ev), Err(LlmError::Server { .. })));
725        let ev = SseEvent {
726            data: "{\"error\":{\"message\":\"slow\",\"type\":\"rate_limit_exceeded\"}}".into(),
727            ..SseEvent::default()
728        };
729        assert!(matches!(d.decode(ev), Err(LlmError::RateLimit { .. })));
730        let ev = SseEvent {
731            data: "{\"error\":{\"message\":\"long\",\"code\":\"context_length_exceeded\"}}".into(),
732            ..SseEvent::default()
733        };
734        assert!(matches!(d.decode(ev), Err(LlmError::ContextLength(_))));
735    }
736
737    #[test]
738    fn debug_redacts_key() {
739        let cfg = OpenAiConfig::new("https://x", Some("sk-secret".into()));
740        assert!(!format!("{cfg:?}").contains("sk-secret"));
741    }
742}