1pub use molo_core::{message, provider, tool};
8
9use crate::message::{ContentBlock, Message, ToolCall};
10use crate::provider::{
11 ChatRequest, ChatResponse, FinishReason, ModelOptions, Provider, ProviderCapabilities,
12 ProviderError, ProviderRequestContext, StreamEvent, TimeoutStage, Usage,
13};
14use crate::tool::ToolSchema;
15use async_trait::async_trait;
16use base64::Engine;
17use futures::stream::{BoxStream, Stream, StreamExt, unfold};
18use serde::Serialize;
19use std::collections::BTreeMap;
20use std::sync::{Arc, Mutex};
21use std::time::{Duration, Instant};
22
23#[derive(Clone)]
56pub struct OpenAiProvider {
57 base_url: String,
58 api_key: String,
59 model: String,
60 client: reqwest::Client,
61 request_timeout: Duration,
66 idle_timeout: Duration,
70 stream_timeout: Duration,
75 structured_mode: StructuredOutputMode,
78}
79
80#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
103pub enum StructuredOutputMode {
104 #[default]
106 Native,
107 JsonObject,
109 Off,
111}
112
113impl std::fmt::Debug for OpenAiProvider {
114 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
115 f.debug_struct("OpenAiProvider")
119 .field("base_url", &self.base_url)
120 .field("api_key", &"***")
121 .field("model", &self.model)
122 .field("request_timeout", &self.request_timeout)
123 .field("idle_timeout", &self.idle_timeout)
124 .field("stream_timeout", &self.stream_timeout)
125 .field("structured_mode", &self.structured_mode)
126 .finish_non_exhaustive()
127 }
128}
129
130impl OpenAiProvider {
131 pub fn new(
138 base_url: impl Into<String>,
139 api_key: impl Into<String>,
140 model: impl Into<String>,
141 ) -> Self {
142 Self {
143 base_url: base_url.into().trim_end_matches('/').to_string(),
144 api_key: api_key.into(),
145 model: model.into(),
146 client: build_client(Duration::from_secs(30)),
147 request_timeout: Duration::from_secs(600),
148 idle_timeout: Duration::from_secs(60),
149 stream_timeout: Duration::from_secs(1800),
150 structured_mode: StructuredOutputMode::default(),
151 }
152 }
153
154 pub fn with_connect_timeout(mut self, timeout: Duration) -> Self {
157 self.client = build_client(timeout);
158 self
159 }
160
161 pub fn with_request_timeout(mut self, timeout: Duration) -> Self {
167 self.request_timeout = timeout;
168 self
169 }
170
171 pub fn with_idle_timeout(mut self, timeout: Duration) -> Self {
175 self.idle_timeout = timeout;
176 self
177 }
178
179 pub fn with_stream_timeout(mut self, timeout: Duration) -> Self {
202 self.stream_timeout = timeout;
203 self
204 }
205
206 pub fn with_structured_output_mode(mut self, mode: StructuredOutputMode) -> Self {
216 self.structured_mode = mode;
217 self
218 }
219
220 fn wire_response_format(
223 &self,
224 schema: Option<&serde_json::Value>,
225 ) -> Option<OpenAiResponseFormat> {
226 match (self.structured_mode, schema) {
227 (StructuredOutputMode::Off, _) | (_, None) => None,
228 (StructuredOutputMode::Native, Some(schema)) => {
229 Some(OpenAiResponseFormat::from_schema(schema))
230 }
231 (StructuredOutputMode::JsonObject, Some(_)) => {
232 Some(OpenAiResponseFormat::json_object())
233 }
234 }
235 }
236
237 fn wire_request(&self, request: &ChatRequest, stream: bool) -> OpenAiChatRequest {
246 let options = &request.options;
247 let extra = wire_extra(options);
248 let temperature = if extra.contains_key("temperature") {
249 None
250 } else {
251 options.temperature
252 };
253 let max_tokens = if extra.contains_key("max_tokens") {
254 None
255 } else {
256 options.max_tokens
257 };
258 let response_format = if extra.contains_key("response_format") {
259 None
260 } else {
261 self.wire_response_format(options.structured.as_ref())
262 };
263
264 OpenAiChatRequest {
265 model: self.model.clone(),
266 messages: request
267 .messages
268 .iter()
269 .map(OpenAiMessage::from_message)
270 .collect(),
271 tools: request
272 .tools
273 .iter()
274 .map(OpenAiToolDef::from_schema)
275 .collect(),
276 stream: stream.then_some(true),
277 stream_options: stream.then_some(StreamOptions {
278 include_usage: true,
279 }),
280 temperature,
281 max_tokens,
282 response_format,
283 extra,
284 }
285 }
286
287 fn build_request(
305 &self,
306 wire: &OpenAiChatRequest,
307 options: &ModelOptions,
308 ) -> Result<reqwest::RequestBuilder, ProviderError> {
309 let url = format!("{}/chat/completions", self.base_url);
310 let headers = wire_headers(options)?;
311 let caller_authorization = headers.contains_key(reqwest::header::AUTHORIZATION);
312 let mut req = self.client.post(url).headers(headers).json(wire);
313 if !self.api_key.is_empty() && !caller_authorization {
314 req = req.bearer_auth(&self.api_key);
315 }
316 Ok(req)
317 }
318}
319
320fn build_client(connect_timeout: Duration) -> reqwest::Client {
323 reqwest::Client::builder()
324 .connect_timeout(connect_timeout)
325 .build()
326 .expect("reqwest Client::builder cannot fail without TLS config")
327}
328
329#[async_trait]
330impl Provider for OpenAiProvider {
331 fn model(&self) -> Option<&str> {
332 Some(&self.model)
333 }
334
335 fn capabilities(&self) -> ProviderCapabilities {
336 ProviderCapabilities {
337 streaming: true,
338 reasoning: true,
339 tool_calls: true,
340 parallel_tool_calls: true,
341 structured_output: !matches!(self.structured_mode, StructuredOutputMode::Off),
342 usage: true,
343 context_cancellation: true,
344 context_deadline: true,
345 }
346 }
347
348 async fn chat(&self, request: ChatRequest) -> Result<ChatResponse, ProviderError> {
349 let wire = self.wire_request(&request, false);
350 let req = self
355 .build_request(&wire, &request.options)?
356 .timeout(self.request_timeout);
357
358 let resp = req.send().await.map_err(|e| {
359 if e.is_timeout() {
360 ProviderError::Timeout(TimeoutStage::Request)
361 } else {
362 map_network_error(e)
363 }
364 })?;
365 let status = resp.status();
366 let headers = resp.headers().clone();
367 let body = read_response_body(resp).await?;
368 if !status.is_success() {
369 return Err(map_status_error(status.as_u16(), &headers, &body));
370 }
371
372 parse_response(&body)
373 }
374
375 async fn chat_with_context(
376 &self,
377 request: ChatRequest,
378 context: &ProviderRequestContext,
379 ) -> Result<ChatResponse, ProviderError> {
380 run_with_context(context, self.chat(request)).await
381 }
382
383 async fn stream_chat(
384 &self,
385 request: ChatRequest,
386 ) -> Result<BoxStream<'static, Result<StreamEvent, ProviderError>>, ProviderError> {
387 let wire = self.wire_request(&request, true);
388 let req = self.build_request(&wire, &request.options)?;
389 let resp = match tokio::time::timeout(self.request_timeout, req.send()).await {
397 Ok(result) => result.map_err(map_network_error)?,
398 Err(_) => return Err(ProviderError::Timeout(TimeoutStage::Request)),
399 };
400 let status = resp.status();
401 let headers = resp.headers().clone();
402 if !status.is_success() {
403 let body = read_response_body(resp).await?;
404 return Err(map_status_error(status.as_u16(), &headers, &body));
405 }
406
407 let aggregator = Arc::new(Mutex::new(ToolCallAggregator::default()));
408 Ok(Box::pin(sse_stream(
409 resp.bytes_stream(),
410 self.idle_timeout,
411 self.stream_timeout,
412 aggregator,
413 )))
414 }
415
416 async fn stream_chat_with_context(
417 &self,
418 request: ChatRequest,
419 context: &ProviderRequestContext,
420 ) -> Result<BoxStream<'static, Result<StreamEvent, ProviderError>>, ProviderError> {
421 let stream = run_with_context(context, self.stream_chat(request)).await?;
422 Ok(Box::pin(stream_with_context(stream, context.clone())))
423 }
424}
425
426async fn run_with_context<T, F>(
427 context: &ProviderRequestContext,
428 future: F,
429) -> Result<T, ProviderError>
430where
431 F: std::future::Future<Output = Result<T, ProviderError>>,
432{
433 check_provider_context(context)?;
434 match context.remaining() {
435 Some(remaining) if remaining.is_zero() => {
436 Err(ProviderError::Timeout(TimeoutStage::Request))
437 }
438 Some(remaining) => {
439 tokio::select! {
440 _ = context.cancellation.cancelled() => Err(ProviderError::Cancelled),
441 _ = tokio::time::sleep(remaining) => Err(ProviderError::Timeout(TimeoutStage::Request)),
442 output = future => output,
443 }
444 }
445 None => {
446 tokio::select! {
447 _ = context.cancellation.cancelled() => Err(ProviderError::Cancelled),
448 output = future => output,
449 }
450 }
451 }
452}
453
454fn stream_with_context(
455 mut stream: BoxStream<'static, Result<StreamEvent, ProviderError>>,
456 context: ProviderRequestContext,
457) -> impl futures::Stream<Item = Result<StreamEvent, ProviderError>> {
458 async_stream::stream! {
459 loop {
460 if let Err(error) = check_provider_context(&context) {
461 yield Err(error);
462 break;
463 }
464 let next = match context.remaining() {
465 Some(remaining) if remaining.is_zero() => {
466 yield Err(ProviderError::Timeout(TimeoutStage::StreamTotal));
467 break;
468 }
469 Some(remaining) => {
470 tokio::select! {
471 _ = context.cancellation.cancelled() => {
472 yield Err(ProviderError::Cancelled);
473 break;
474 }
475 _ = tokio::time::sleep(remaining) => {
476 yield Err(ProviderError::Timeout(TimeoutStage::StreamTotal));
477 break;
478 }
479 next = stream.next() => next,
480 }
481 }
482 None => {
483 tokio::select! {
484 _ = context.cancellation.cancelled() => {
485 yield Err(ProviderError::Cancelled);
486 break;
487 }
488 next = stream.next() => next,
489 }
490 }
491 };
492 match next {
493 Some(event) => yield event,
494 None => break,
495 }
496 }
497 }
498}
499
500fn check_provider_context(context: &ProviderRequestContext) -> Result<(), ProviderError> {
501 if context.is_cancelled() {
502 Err(ProviderError::Cancelled)
503 } else if context.is_expired() {
504 Err(ProviderError::Timeout(TimeoutStage::Request))
505 } else {
506 Ok(())
507 }
508}
509
510async fn read_response_body(resp: reqwest::Response) -> Result<String, ProviderError> {
520 let result = tokio::time::timeout(RESPONSE_BODY_TIMEOUT, async {
521 if let Some(len) = resp.content_length()
522 && len > MAX_RESPONSE_BODY as u64
523 {
524 return Err(limit_error(MAX_RESPONSE_BODY));
525 }
526 let mut stream = resp.bytes_stream();
527 let mut buf = Vec::new();
528 while let Some(chunk) = stream.next().await {
529 let chunk = chunk.map_err(map_network_error)?;
530 if buf.len() + chunk.len() > MAX_RESPONSE_BODY {
531 return Err(limit_error(MAX_RESPONSE_BODY));
532 }
533 buf.extend_from_slice(&chunk);
534 }
535 Ok(String::from_utf8_lossy(&buf).into_owned())
536 })
537 .await;
538 match result {
539 Ok(r) => r,
540 Err(_) => Err(ProviderError::Timeout(TimeoutStage::ResponseBody)),
541 }
542}
543
544const RESPONSE_BODY_TIMEOUT: Duration = Duration::from_secs(60);
547
548fn sse_stream(
555 chunks: impl Stream<Item = Result<impl AsRef<[u8]>, reqwest::Error>> + Unpin,
556 idle_timeout: Duration,
557 stream_timeout: Duration,
558 aggregator: Arc<Mutex<ToolCallAggregator>>,
559) -> impl Stream<Item = Result<StreamEvent, ProviderError>> {
560 let tail = aggregator.clone();
561 sse_lines(chunks, idle_timeout, stream_timeout)
562 .map(move |line| match line {
563 Err(e) => {
564 let mut agg = aggregator.lock().expect("SSE aggregator lock poisoned");
565 if agg.finished {
570 return Vec::new();
571 }
572 agg.set_errored();
577 vec![Err(e)]
578 }
579 Ok(line) => {
580 let mut agg = aggregator.lock().expect("SSE aggregator lock poisoned");
581 if agg.finished {
584 Vec::new()
585 } else {
586 parse_sse_line(&line, &mut agg)
587 }
588 }
589 })
590 .flat_map(futures::stream::iter)
591 .chain(
598 futures::stream::once(async move {
599 tail.lock()
600 .expect("SSE aggregator lock poisoned")
601 .flush_done()
602 })
603 .flat_map(futures::stream::iter),
604 )
605}
606
607fn sse_lines(
633 chunks: impl Stream<Item = Result<impl AsRef<[u8]>, reqwest::Error>> + Unpin,
634 idle_timeout: Duration,
635 stream_timeout: Duration,
636) -> impl Stream<Item = Result<String, ProviderError>> {
637 let deadline = Instant::now() + stream_timeout;
644 unfold(
645 (chunks, Vec::new(), false, deadline),
646 move |(mut chunks, mut buf, terminated, deadline)| async move {
647 if terminated {
648 return None;
649 }
650 loop {
651 if let Some(line) = pop_line(&mut buf) {
654 return Some((Ok(line), (chunks, buf, false, deadline)));
655 }
656 if Instant::now() >= deadline {
660 return Some((
661 Err(ProviderError::Timeout(TimeoutStage::StreamTotal)),
662 (chunks, buf, true, deadline),
663 ));
664 }
665 let next = match tokio::time::timeout(idle_timeout, chunks.next()).await {
666 Ok(next) => next,
667 Err(_) => {
668 return Some((
669 Err(ProviderError::Timeout(TimeoutStage::Idle)),
670 (chunks, buf, true, deadline),
671 ));
672 }
673 };
674 match next {
675 Some(Ok(bytes)) => {
676 if buf.len() + bytes.as_ref().len() > MAX_SSE_LINE {
677 return Some((
678 Err(ProviderError::ResponseTooLarge {
679 limit_bytes: MAX_SSE_LINE,
680 }),
681 (chunks, buf, true, deadline),
682 ));
683 }
684 buf.extend_from_slice(bytes.as_ref());
685 }
686 Some(Err(e)) => {
691 return Some((Err(map_network_error(e)), (chunks, buf, true, deadline)));
692 }
693 None => {
694 if buf.is_empty() {
695 return None;
696 }
697 return Some((
698 Ok(to_line(std::mem::take(&mut buf))),
699 (chunks, buf, false, deadline),
700 ));
701 }
702 }
703 }
704 },
705 )
706}
707
708const MAX_SSE_LINE: usize = 1 << 20;
711const MAX_RESPONSE_BODY: usize = 16 << 20;
716
717fn pop_line(buf: &mut Vec<u8>) -> Option<String> {
720 let pos = buf.iter().position(|&b| b == b'\n')?;
721 let rest = buf.split_off(pos + 1);
722 let line = std::mem::replace(buf, rest);
723 Some(to_line(line))
724}
725
726fn to_line(mut bytes: Vec<u8>) -> String {
728 while matches!(bytes.last(), Some(b'\r' | b'\n')) {
729 bytes.pop();
730 }
731 String::from_utf8_lossy(&bytes).into_owned()
732}
733
734fn parse_sse_line(
754 line: &str,
755 tool_calls: &mut ToolCallAggregator,
756) -> Vec<Result<StreamEvent, ProviderError>> {
757 let mut events = Vec::new();
758 if line.is_empty() || line.starts_with(':') {
759 return events;
760 }
761 let Some(data) = line.strip_prefix("data:") else {
762 return events;
763 };
764 let data = data.trim_start();
765 if data == "[DONE]" {
766 let events = tool_calls.flush_done();
770 tool_calls.finished = true;
771 return events;
772 }
773
774 let chunk = match serde_json::from_str::<OpenAiStreamChunk>(data) {
775 Ok(chunk) => chunk,
776 Err(_) => {
777 return line_error(
778 tool_calls,
779 ProviderError::Decode {
780 message: format!("invalid stream event: {}", extract_error_message(data)),
781 },
782 );
783 }
784 };
785
786 if let Some(error) = chunk.error {
789 return line_error(
790 tool_calls,
791 ProviderError::Protocol {
792 message: format!("provider stream error: {}", error.message),
793 },
794 );
795 }
796
797 if let Some(usage) = chunk.usage {
801 tool_calls.push_usage(usage);
802 }
803 let Some(choice) = chunk.choices.into_iter().next() else {
804 return events;
805 };
806 if let Some(content) = choice.delta.content
810 && !tool_calls.errored
811 {
812 events.push(Ok(StreamEvent::Delta(content)));
813 }
814 if let Some(reasoning) = choice.delta.reasoning_content
815 && !tool_calls.errored
816 {
817 events.push(Ok(StreamEvent::Reasoning(reasoning)));
822 }
823
824 if let Some(calls) = choice.delta.tool_calls {
825 for call in calls {
826 if let Err(e) = tool_calls.push_chunk(call) {
827 return line_error(tool_calls, e);
828 }
829 }
830 }
831 if let Some(finish_reason) = choice.finish_reason {
832 if !tool_calls.errored {
835 for call in tool_calls.take_all() {
838 events.push(Ok(StreamEvent::ToolCall {
839 id: call.id,
840 name: call.name,
841 arguments: call.arguments,
842 }));
843 }
844 }
845 tool_calls.set_done_reason(map_finish_reason(&finish_reason));
846 }
847 events
848}
849
850fn map_finish_reason(reason: &str) -> FinishReason {
857 match reason {
858 "stop" | "tool_calls" => FinishReason::Stop,
859 "length" => FinishReason::Length,
860 other => FinishReason::Other(other.to_string()),
861 }
862}
863
864#[derive(Default)]
879struct ToolCallAggregator {
880 calls: Vec<AccumulatedCall>,
882 done_reason: Option<FinishReason>,
885 usage: Option<Usage>,
888 errored: bool,
892 finished: bool,
896}
897
898const MAX_TOOL_CALLS: usize = 64;
902const MAX_ACCUMULATED_TEXT: usize = 1 << 20;
905
906fn limit_error(limit_bytes: usize) -> ProviderError {
908 ProviderError::ResponseTooLarge { limit_bytes }
909}
910
911fn line_error(
916 tool_calls: &mut ToolCallAggregator,
917 e: ProviderError,
918) -> Vec<Result<StreamEvent, ProviderError>> {
919 tool_calls.set_errored();
920 vec![Err(e)]
921}
922
923struct AccumulatedCall {
925 index: usize,
926 id: String,
927 name: String,
928 arguments: String,
929}
930
931impl ToolCallAggregator {
932 fn push_chunk(&mut self, chunk: OpenAiToolCallChunk) -> Result<(), ProviderError> {
933 let name = chunk
937 .function
938 .as_ref()
939 .and_then(|f| f.name.clone())
940 .unwrap_or_default();
941 let arguments = chunk
942 .function
943 .as_ref()
944 .and_then(|f| f.arguments.clone())
945 .unwrap_or_default();
946 if let Some(call) = self.calls.iter_mut().find(|c| c.index == chunk.index) {
947 if call.id.is_empty() {
948 call.id = chunk.id.unwrap_or_default();
949 }
950 if call.name.is_empty() {
951 call.name = name;
952 }
953 if call.arguments.len() + arguments.len() > MAX_ACCUMULATED_TEXT {
954 return Err(limit_error(MAX_ACCUMULATED_TEXT));
955 }
956 call.arguments.push_str(&arguments);
957 } else {
958 if self.calls.len() >= MAX_TOOL_CALLS {
963 return Err(ProviderError::Protocol {
964 message: "tool call count exceeds limit".to_string(),
965 });
966 }
967 if arguments.len() > MAX_ACCUMULATED_TEXT {
970 return Err(limit_error(MAX_ACCUMULATED_TEXT));
971 }
972 self.calls.push(AccumulatedCall {
973 index: chunk.index,
974 id: chunk.id.unwrap_or_default(),
975 name,
976 arguments,
977 });
978 }
979 Ok(())
980 }
981
982 fn take_all(&mut self) -> Vec<AccumulatedCall> {
984 std::mem::take(&mut self.calls)
985 }
986
987 fn set_done_reason(&mut self, reason: FinishReason) {
990 if !self.errored {
991 self.done_reason = Some(reason);
992 }
993 }
994
995 fn push_usage(&mut self, usage: OpenAiUsage) {
999 if !self.errored {
1000 self.usage = Some(map_usage(usage));
1001 }
1002 }
1003
1004 fn flush_done(&mut self) -> Vec<Result<StreamEvent, ProviderError>> {
1018 if self.errored || self.finished {
1019 return Vec::new();
1020 }
1021 let Some(reason) = self.done_reason.take() else {
1022 return vec![Err(ProviderError::Protocol {
1023 message: "stream ended without finish_reason".into(),
1024 })];
1025 };
1026 let usage = self.usage.take();
1027 vec![Ok(StreamEvent::Done { reason, usage })]
1028 }
1029
1030 fn set_errored(&mut self) {
1034 self.errored = true;
1035 self.done_reason = None;
1036 }
1037}
1038
1039fn map_usage(wire: OpenAiUsage) -> Usage {
1047 let cached_tokens = wire
1048 .prompt_tokens_details
1049 .and_then(|details| details.cached_tokens)
1050 .or(wire.prompt_cache_hit_tokens);
1051 Usage {
1052 prompt_tokens: wire.prompt_tokens,
1053 completion_tokens: wire.completion_tokens,
1054 total_tokens: wire.total_tokens,
1055 cached_tokens,
1056 }
1057}
1058
1059fn map_network_error(e: reqwest::Error) -> ProviderError {
1066 if e.is_timeout() {
1067 ProviderError::Timeout(TimeoutStage::Transport)
1068 } else {
1069 ProviderError::Network(e.to_string())
1070 }
1071}
1072
1073fn map_status_error(
1080 status: u16,
1081 headers: &reqwest::header::HeaderMap,
1082 body: &str,
1083) -> ProviderError {
1084 let message = extract_error_message(body);
1085 if status == 429 && !message.contains("insufficient_quota") {
1086 ProviderError::RateLimited {
1087 retry_after: extract_retry_after(headers),
1088 }
1089 } else {
1090 ProviderError::Api {
1091 status,
1092 code: None,
1093 message,
1094 }
1095 }
1096}
1097
1098fn extract_retry_after(headers: &reqwest::header::HeaderMap) -> Option<Duration> {
1101 headers
1102 .get(reqwest::header::RETRY_AFTER)?
1103 .to_str()
1104 .ok()?
1105 .trim()
1106 .parse::<u64>()
1107 .ok()
1108 .map(Duration::from_secs)
1109}
1110
1111fn extract_error_message(body: &str) -> String {
1114 serde_json::from_str::<OpenAiErrorBody>(body)
1115 .ok()
1116 .and_then(|body| body.error)
1117 .map(|error| error.message)
1118 .unwrap_or_else(|| body.to_string())
1119}
1120
1121fn parse_response(body: &str) -> Result<ChatResponse, ProviderError> {
1128 let parsed: OpenAiChatResponse =
1129 serde_json::from_str(body).map_err(|e| ProviderError::Decode {
1130 message: format!("invalid provider response: {e}"),
1131 })?;
1132 let choice = parsed
1133 .choices
1134 .into_iter()
1135 .next()
1136 .ok_or_else(|| ProviderError::Protocol {
1137 message: "provider returned empty choices".to_string(),
1138 })?;
1139
1140 let wire = choice.message;
1141 let content = match wire.content {
1144 Some(serde_json::Value::String(s)) => s,
1145 Some(other) => other.to_string(),
1146 None => String::new(),
1147 };
1148 let raw_tool_calls = wire.tool_calls.unwrap_or_default();
1149 if raw_tool_calls.len() > MAX_TOOL_CALLS {
1153 return Err(ProviderError::Protocol {
1154 message: format!("too many tool calls ({})", raw_tool_calls.len()),
1155 });
1156 }
1157 let tool_calls: Vec<ToolCall> = raw_tool_calls
1158 .into_iter()
1159 .map(|call| ToolCall {
1160 id: call.id,
1161 name: call.function.name,
1162 arguments: call.function.arguments,
1163 })
1164 .collect();
1165 Ok(ChatResponse {
1171 message: Message::Assistant {
1172 content,
1173 reasoning: wire.reasoning_content,
1174 tool_calls,
1175 },
1176 finish_reason: map_finish_reason(&choice.finish_reason),
1177 usage: parsed.usage.map(map_usage),
1178 })
1179}
1180
1181#[derive(serde::Serialize)]
1184struct OpenAiChatRequest {
1185 model: String,
1186 messages: Vec<OpenAiMessage>,
1187 #[serde(skip_serializing_if = "Vec::is_empty")]
1189 tools: Vec<OpenAiToolDef>,
1190 #[serde(skip_serializing_if = "Option::is_none")]
1193 stream: Option<bool>,
1194 #[serde(skip_serializing_if = "Option::is_none")]
1198 stream_options: Option<StreamOptions>,
1199 #[serde(skip_serializing_if = "Option::is_none")]
1201 temperature: Option<f32>,
1202 #[serde(skip_serializing_if = "Option::is_none")]
1204 max_tokens: Option<u32>,
1205 #[serde(skip_serializing_if = "Option::is_none")]
1209 response_format: Option<OpenAiResponseFormat>,
1210 #[serde(flatten)]
1213 extra: BTreeMap<String, serde_json::Value>,
1214}
1215
1216#[derive(Serialize)]
1220struct OpenAiResponseFormat {
1221 #[serde(rename = "type")]
1222 kind: &'static str,
1223 #[serde(skip_serializing_if = "Option::is_none")]
1226 strict: Option<bool>,
1227 #[serde(skip_serializing_if = "Option::is_none")]
1229 json_schema: Option<OpenAiJsonSchema>,
1230}
1231
1232#[derive(Serialize)]
1233struct OpenAiJsonSchema {
1234 name: &'static str,
1235 schema: serde_json::Value,
1236}
1237
1238impl OpenAiResponseFormat {
1239 fn from_schema(schema: &serde_json::Value) -> Self {
1251 if schema.get("type").and_then(|t| t.as_str()) == Some("object")
1252 && let (Some(properties), Some(required)) = (
1253 schema.get("properties").and_then(|p| p.as_object()),
1254 schema.get("required").and_then(|r| r.as_array()),
1255 )
1256 && required.len() == properties.len()
1257 && required.iter().all(|k| k.is_string())
1258 {
1259 let mut strict_schema = schema.clone();
1260 strict_schema["additionalProperties"] = serde_json::Value::Bool(false);
1261 Self {
1262 kind: "json_schema",
1263 strict: Some(true),
1264 json_schema: Some(OpenAiJsonSchema {
1265 name: "output",
1266 schema: strict_schema,
1267 }),
1268 }
1269 } else {
1270 Self {
1271 kind: "json_schema",
1272 strict: None,
1273 json_schema: Some(OpenAiJsonSchema {
1274 name: "output",
1275 schema: schema.clone(),
1276 }),
1277 }
1278 }
1279 }
1280
1281 fn json_object() -> Self {
1284 Self {
1285 kind: "json_object",
1286 strict: None,
1287 json_schema: None,
1288 }
1289 }
1290}
1291
1292fn wire_extra(options: &ModelOptions) -> BTreeMap<String, serde_json::Value> {
1302 options
1303 .extra
1304 .iter()
1305 .filter(|(key, _)| {
1306 !matches!(
1307 key.as_str(),
1308 "model" | "messages" | "tools" | "stream" | "stream_options"
1309 )
1310 })
1311 .map(|(key, value)| (key.clone(), value.clone()))
1312 .collect()
1313}
1314
1315const FRAMEWORK_HEADERS: [&str; 5] = [
1325 "content-length",
1326 "transfer-encoding",
1327 "connection",
1328 "host",
1329 "upgrade",
1330];
1331
1332fn wire_headers(options: &ModelOptions) -> Result<reqwest::header::HeaderMap, ProviderError> {
1339 let mut headers = reqwest::header::HeaderMap::new();
1340 for (name, value) in &options.headers {
1341 if FRAMEWORK_HEADERS
1342 .iter()
1343 .any(|owned| name.eq_ignore_ascii_case(owned))
1344 {
1345 continue;
1346 }
1347 let header_name =
1348 reqwest::header::HeaderName::from_bytes(name.as_bytes()).map_err(|_| {
1349 ProviderError::Protocol {
1350 message: format!("invalid request header name: {name:?}"),
1351 }
1352 })?;
1353 let header_value =
1354 reqwest::header::HeaderValue::from_str(value).map_err(|_| ProviderError::Protocol {
1355 message: format!("invalid value for request header {name:?}"),
1356 })?;
1357 headers.insert(header_name, header_value);
1358 }
1359 Ok(headers)
1360}
1361
1362#[derive(serde::Serialize)]
1365struct StreamOptions {
1366 include_usage: bool,
1367}
1368
1369#[derive(serde::Serialize, serde::Deserialize)]
1372struct OpenAiMessage {
1373 role: String,
1374 #[serde(default, skip_serializing_if = "Option::is_none")]
1377 content: Option<serde_json::Value>,
1378 #[serde(default, skip_serializing_if = "Option::is_none", alias = "reasoning")]
1385 reasoning_content: Option<String>,
1386 #[serde(default, skip_serializing_if = "Option::is_none")]
1388 tool_calls: Option<Vec<OpenAiToolCall>>,
1389 #[serde(default, skip_serializing_if = "Option::is_none")]
1392 tool_call_id: Option<String>,
1393}
1394
1395impl OpenAiMessage {
1396 fn from_message(message: &Message) -> Self {
1397 match message {
1398 Message::System(content) => Self::text("system", content),
1399 Message::User(blocks) => Self {
1400 role: "user".to_string(),
1401 content: user_content(blocks),
1402 reasoning_content: None,
1403 tool_calls: None,
1404 tool_call_id: None,
1405 },
1406 Message::Assistant {
1407 content,
1408 reasoning,
1409 tool_calls,
1410 } => Self {
1411 role: "assistant".to_string(),
1412 content: (!content.is_empty()).then(|| serde_json::Value::String(content.clone())),
1415 reasoning_content: reasoning.clone(),
1416 tool_calls: (!tool_calls.is_empty()).then(|| {
1417 tool_calls
1418 .iter()
1419 .map(|call| OpenAiToolCall {
1420 id: call.id.clone(),
1421 kind: "function".to_string(),
1422 function: OpenAiFunctionCall {
1423 name: call.name.clone(),
1424 arguments: call.arguments.clone(),
1425 },
1426 })
1427 .collect()
1428 }),
1429 tool_call_id: None,
1430 },
1431 Message::ToolResult { id, content } => Self {
1432 role: "tool".to_string(),
1433 content: Some(serde_json::Value::String(content.clone())),
1434 reasoning_content: None,
1435 tool_calls: None,
1436 tool_call_id: Some(id.clone()),
1437 },
1438 }
1439 }
1440
1441 fn text(role: &str, content: &str) -> Self {
1442 Self {
1443 role: role.to_string(),
1444 content: Some(serde_json::Value::String(content.to_string())),
1445 reasoning_content: None,
1446 tool_calls: None,
1447 tool_call_id: None,
1448 }
1449 }
1450}
1451
1452fn user_content(blocks: &[ContentBlock]) -> Option<serde_json::Value> {
1459 match blocks {
1460 [ContentBlock::Text(text)] => Some(serde_json::Value::String(text.clone())),
1461 blocks => {
1462 let parts: Vec<serde_json::Value> = blocks.iter().map(content_block_wire).collect();
1463 (!parts.is_empty()).then_some(serde_json::Value::Array(parts))
1464 }
1465 }
1466}
1467
1468fn content_block_wire(block: &ContentBlock) -> serde_json::Value {
1475 match block {
1476 ContentBlock::Text(text) => serde_json::json!({"type": "text", "text": text}),
1477 ContentBlock::Image(image) => serde_json::json!({
1478 "type": "image_url",
1479 "image_url": {
1480 "url": format!(
1481 "data:{};base64,{}",
1482 image.mime_type,
1483 base64::engine::general_purpose::STANDARD.encode(&image.data)
1484 )
1485 }
1486 }),
1487 ContentBlock::Wire(value) => value.clone(),
1490 }
1491}
1492
1493#[derive(serde::Serialize, serde::Deserialize)]
1495struct OpenAiToolCall {
1496 id: String,
1497 #[serde(rename = "type")]
1499 kind: String,
1500 function: OpenAiFunctionCall,
1501}
1502
1503#[derive(serde::Serialize, serde::Deserialize)]
1504struct OpenAiFunctionCall {
1505 name: String,
1506 arguments: String,
1508}
1509
1510#[derive(serde::Serialize)]
1512struct OpenAiToolDef {
1513 #[serde(rename = "type")]
1514 kind: &'static str,
1515 function: OpenAiFunctionDef,
1516}
1517
1518#[derive(serde::Serialize)]
1519struct OpenAiFunctionDef {
1520 name: String,
1521 description: String,
1522 parameters: serde_json::Value,
1523}
1524
1525impl OpenAiToolDef {
1526 fn from_schema(schema: &ToolSchema) -> Self {
1527 Self {
1528 kind: "function",
1529 function: OpenAiFunctionDef {
1530 name: schema.name.clone(),
1531 description: schema.description.clone(),
1532 parameters: schema.parameters.clone(),
1533 },
1534 }
1535 }
1536}
1537
1538#[derive(serde::Deserialize)]
1541struct OpenAiUsage {
1542 prompt_tokens: u32,
1543 completion_tokens: u32,
1544 total_tokens: u32,
1545 #[serde(default)]
1549 prompt_tokens_details: Option<OpenAiPromptTokensDetails>,
1550 #[serde(default)]
1553 prompt_cache_hit_tokens: Option<u32>,
1554}
1555
1556#[derive(serde::Deserialize)]
1560struct OpenAiPromptTokensDetails {
1561 #[serde(default)]
1563 cached_tokens: Option<u32>,
1564}
1565
1566#[derive(serde::Deserialize)]
1569struct OpenAiChatResponse {
1570 choices: Vec<OpenAiChoice>,
1571 #[serde(default)]
1575 usage: Option<OpenAiUsage>,
1576}
1577
1578#[derive(serde::Deserialize)]
1579struct OpenAiChoice {
1580 message: OpenAiMessage,
1581 finish_reason: String,
1582}
1583
1584#[derive(serde::Deserialize)]
1587struct OpenAiStreamChunk {
1588 #[serde(default)]
1591 choices: Vec<OpenAiStreamChoice>,
1592 #[serde(default)]
1596 usage: Option<OpenAiUsage>,
1597 #[serde(default)]
1601 error: Option<OpenAiStreamError>,
1602}
1603
1604#[derive(serde::Deserialize)]
1606struct OpenAiStreamError {
1607 message: String,
1608}
1609
1610#[derive(serde::Deserialize)]
1611struct OpenAiStreamChoice {
1612 delta: OpenAiStreamDelta,
1613 #[serde(default)]
1615 finish_reason: Option<String>,
1616}
1617
1618#[derive(serde::Deserialize)]
1619struct OpenAiStreamDelta {
1620 #[serde(default)]
1623 content: Option<String>,
1624 #[serde(default, alias = "reasoning")]
1629 reasoning_content: Option<String>,
1630 #[serde(default)]
1633 tool_calls: Option<Vec<OpenAiToolCallChunk>>,
1634}
1635
1636#[derive(serde::Deserialize)]
1638struct OpenAiToolCallChunk {
1639 index: usize,
1642 #[serde(default)]
1644 id: Option<String>,
1645 #[serde(default)]
1647 function: Option<OpenAiFunctionChunk>,
1648}
1649
1650#[derive(serde::Deserialize)]
1651struct OpenAiFunctionChunk {
1652 #[serde(default)]
1654 name: Option<String>,
1655 #[serde(default)]
1658 arguments: Option<String>,
1659}
1660
1661#[derive(serde::Deserialize)]
1663struct OpenAiErrorBody {
1664 error: Option<OpenAiErrorDetail>,
1665}
1666
1667#[derive(serde::Deserialize)]
1668struct OpenAiErrorDetail {
1669 message: String,
1670}
1671
1672#[cfg(test)]
1673mod tests {
1674 use super::*;
1675 use crate::message::ImageContent;
1676 use reqwest::header::{HeaderMap, RETRY_AFTER};
1677 use serde_json::json;
1678
1679 #[test]
1682 fn status_429_quota_goes_to_api() {
1683 let body = r#"{"error":{"message":"You exceeded your current quota: insufficient_quota"}}"#;
1686 let err = map_status_error(429, &HeaderMap::new(), body);
1687 assert!(matches!(err, ProviderError::Api { status: 429, .. }));
1688 }
1689
1690 #[test]
1691 fn status_429_rate_limited_with_retry_after() {
1692 let mut headers = HeaderMap::new();
1693 headers.insert(RETRY_AFTER, "5".parse().unwrap());
1694 let body = r#"{"error":{"message":"rate limit exceeded"}}"#;
1695 let err = map_status_error(429, &headers, body);
1696 assert!(matches!(
1697 err,
1698 ProviderError::RateLimited { retry_after: Some(d) } if d == Duration::from_secs(5)
1699 ));
1700 }
1701
1702 #[test]
1703 fn status_429_without_retry_after_header() {
1704 let body = r#"{"error":{"message":"rate limit exceeded"}}"#;
1705 let err = map_status_error(429, &HeaderMap::new(), body);
1706 assert!(matches!(
1707 err,
1708 ProviderError::RateLimited { retry_after: None }
1709 ));
1710 }
1711
1712 #[test]
1713 fn status_5xx_goes_to_api_with_status() {
1714 let body = r#"{"error":{"message":"server overloaded"}}"#;
1715 let err = map_status_error(503, &HeaderMap::new(), body);
1716 assert!(matches!(err, ProviderError::Api { status: 503, .. }));
1717 }
1718
1719 #[test]
1720 fn maps_message_to_wire() {
1721 let message = Message::user("hi");
1722 let wire = OpenAiMessage::from_message(&message);
1723 assert_eq!(
1724 serde_json::to_value(&wire).unwrap(),
1725 json!({"role": "user", "content": "hi"})
1726 );
1727 }
1728
1729 #[test]
1730 fn maps_multi_block_user_content_to_wire_parts() {
1731 let message = Message::user_blocks(vec![
1734 ContentBlock::Text("take a look:".into()),
1735 ContentBlock::Text("see the attachment".into()),
1736 ]);
1737 assert_eq!(
1738 serde_json::to_value(OpenAiMessage::from_message(&message)).unwrap(),
1739 json!({
1740 "role": "user",
1741 "content": [
1742 {"type": "text", "text": "take a look:"},
1743 {"type": "text", "text": "see the attachment"}
1744 ]
1745 })
1746 );
1747 }
1748
1749 #[test]
1750 fn maps_image_block_to_wire_data_url() {
1751 let message = Message::user_blocks(vec![ContentBlock::Image(ImageContent::new(
1755 "image/png",
1756 b"fake-png-bytes".to_vec(),
1757 ))]);
1758 let expected_b64 = base64::engine::general_purpose::STANDARD.encode(b"fake-png-bytes");
1759 assert_eq!(
1760 serde_json::to_value(OpenAiMessage::from_message(&message)).unwrap(),
1761 json!({
1762 "role": "user",
1763 "content": [{
1764 "type": "image_url",
1765 "image_url": {"url": format!("data:image/png;base64,{expected_b64}")}
1766 }]
1767 })
1768 );
1769 }
1770
1771 #[test]
1772 fn passes_through_wire_blocks() {
1773 let message = Message::user_blocks(vec![
1777 ContentBlock::Wire(json!({
1778 "type": "input_audio",
1779 "input_audio": {
1780 "data": "UklGRi4A",
1781 "format": "wav"
1782 }
1783 })),
1784 ContentBlock::Text("transcribe this".into()),
1785 ]);
1786 assert_eq!(
1787 serde_json::to_value(OpenAiMessage::from_message(&message)).unwrap(),
1788 json!({
1789 "role": "user",
1790 "content": [
1791 {
1792 "type": "input_audio",
1793 "input_audio": {"data": "UklGRi4A", "format": "wav"}
1794 },
1795 {"type": "text", "text": "transcribe this"}
1796 ]
1797 })
1798 );
1799 }
1800
1801 #[test]
1802 fn maps_mixed_text_and_image_blocks() {
1803 let message = Message::user_blocks(vec![
1807 ContentBlock::Text("what is this?".into()),
1808 ContentBlock::Image(ImageContent::new("image/jpeg", vec![1, 2, 3])),
1809 ]);
1810 assert_eq!(
1811 serde_json::to_value(OpenAiMessage::from_message(&message)).unwrap(),
1812 json!({
1813 "role": "user",
1814 "content": [
1815 {"type": "text", "text": "what is this?"},
1816 {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AQID"}}
1817 ]
1818 })
1819 );
1820 }
1821
1822 #[test]
1823 fn maps_system_and_assistant_roles() {
1824 assert_eq!(
1825 OpenAiMessage::from_message(&Message::system("s")).role,
1826 "system"
1827 );
1828 assert_eq!(
1829 OpenAiMessage::from_message(&Message::assistant("a")).role,
1830 "assistant"
1831 );
1832 }
1833
1834 #[test]
1835 fn maps_tool_call_to_wire() {
1836 let message = Message::Assistant {
1837 content: String::new(),
1838 reasoning: None,
1839 tool_calls: vec![ToolCall {
1840 id: "call_1".into(),
1841 name: "calculator".into(),
1842 arguments: r#"{"expression":"1+1"}"#.into(),
1843 }],
1844 };
1845 assert_eq!(
1846 serde_json::to_value(OpenAiMessage::from_message(&message)).unwrap(),
1847 json!({
1848 "role": "assistant",
1849 "tool_calls": [{
1850 "id": "call_1",
1851 "type": "function",
1852 "function": {"name": "calculator", "arguments": "{\"expression\":\"1+1\"}"}
1853 }]
1854 })
1855 );
1856 }
1857
1858 #[test]
1859 fn maps_tool_result_to_wire() {
1860 let message = Message::ToolResult {
1861 id: "call_1".into(),
1862 content: "2".into(),
1863 };
1864 assert_eq!(
1865 serde_json::to_value(OpenAiMessage::from_message(&message)).unwrap(),
1866 json!({"role": "tool", "tool_call_id": "call_1", "content": "2"})
1867 );
1868 }
1869
1870 #[test]
1871 fn maps_reasoning_to_wire() {
1872 let message = Message::assistant_with_reasoning("2", "thinking process");
1873 assert_eq!(
1874 serde_json::to_value(OpenAiMessage::from_message(&message)).unwrap(),
1875 json!({
1876 "role": "assistant",
1877 "content": "2",
1878 "reasoning_content": "thinking process"
1879 })
1880 );
1881 }
1882
1883 #[test]
1884 fn omits_empty_reasoning_in_wire() {
1885 let message = Message::assistant("2");
1886 assert_eq!(
1887 serde_json::to_value(OpenAiMessage::from_message(&message)).unwrap(),
1888 json!({"role": "assistant", "content": "2"})
1889 );
1890 }
1891
1892 #[test]
1893 fn parses_reasoning_content() {
1894 let body = r#"{"choices":[{"message":{"role":"assistant","content":"2","reasoning_content":"thinking process"},"finish_reason":"stop"}]}"#;
1895 let response = parse_response(body).unwrap();
1896 assert_eq!(
1897 response.message,
1898 Message::assistant_with_reasoning("2", "thinking process")
1899 );
1900 }
1901
1902 #[test]
1903 fn parses_reasoning_with_tool_call() {
1904 let body = r#"{"choices":[{"message":{"role":"assistant","content":null,"reasoning_content":"decided","tool_calls":[{"id":"call_1","type":"function","function":{"name":"calculator","arguments":"{}"}}]},"finish_reason":"tool_calls"}]}"#;
1907 let response = parse_response(body).unwrap();
1908 assert_eq!(
1909 response.message,
1910 Message::Assistant {
1911 content: String::new(),
1912 reasoning: Some("decided".into()),
1913 tool_calls: vec![ToolCall {
1914 id: "call_1".into(),
1915 name: "calculator".into(),
1916 arguments: "{}".into(),
1917 }],
1918 }
1919 );
1920 }
1921
1922 #[test]
1923 fn accepts_ollama_reasoning_field() {
1924 let body = r#"{"choices":[{"message":{"role":"assistant","content":"2","reasoning":"thinking"},"finish_reason":"stop"}]}"#;
1927 let response = parse_response(body).unwrap();
1928 assert_eq!(
1929 response.message,
1930 Message::assistant_with_reasoning("2", "thinking")
1931 );
1932 }
1933
1934 #[test]
1935 fn serializes_tool_defs() {
1936 let schema = ToolSchema::new(
1937 "calculator",
1938 "Evaluate expressions",
1939 json!({"type": "object"}),
1940 );
1941 let def = OpenAiToolDef::from_schema(&schema);
1942 assert_eq!(
1943 serde_json::to_value(&def).unwrap(),
1944 json!({
1945 "type": "function",
1946 "function": {
1947 "name": "calculator",
1948 "description": "Evaluate expressions",
1949 "parameters": {"type": "object"}
1950 }
1951 })
1952 );
1953 }
1954
1955 #[test]
1956 fn omits_empty_tools_in_request() {
1957 let request = OpenAiChatRequest {
1958 model: "m".into(),
1959 messages: vec![OpenAiMessage::from_message(&Message::user("hi"))],
1960 tools: vec![],
1961 stream: None,
1962 stream_options: None,
1963 temperature: None,
1964 max_tokens: None,
1965 response_format: None,
1966 extra: BTreeMap::new(),
1967 };
1968 assert_eq!(
1969 serde_json::to_value(&request).unwrap(),
1970 json!({"model": "m", "messages": [{"role": "user", "content": "hi"}]})
1971 );
1972 }
1973
1974 #[test]
1975 fn serializes_structured_output_as_response_format() {
1976 let schema = json!({
1980 "type": "object",
1981 "properties": { "city": { "type": "string" } },
1982 "required": ["city"],
1983 });
1984 let options = ModelOptions {
1985 structured: Some(schema.clone()),
1986 ..Default::default()
1987 };
1988 let request = OpenAiChatRequest {
1989 model: "m".into(),
1990 messages: vec![OpenAiMessage::from_message(&Message::user("hi"))],
1991 tools: vec![],
1992 stream: None,
1993 stream_options: None,
1994 temperature: None,
1995 max_tokens: None,
1996 response_format: options
1997 .structured
1998 .as_ref()
1999 .map(OpenAiResponseFormat::from_schema),
2000 extra: BTreeMap::new(),
2001 };
2002 let mut expected_schema = schema.clone();
2003 expected_schema["additionalProperties"] = json!(false);
2004 assert_eq!(
2005 serde_json::to_value(&request).unwrap(),
2006 json!({
2007 "model": "m",
2008 "messages": [{"role": "user", "content": "hi"}],
2009 "response_format": {
2010 "type": "json_schema",
2011 "strict": true,
2012 "json_schema": { "name": "output", "schema": expected_schema }
2013 }
2014 })
2015 );
2016 }
2017
2018 #[test]
2019 fn strict_mode_skipped_when_required_incomplete() {
2020 let schema = json!({
2024 "type": "object",
2025 "properties": {
2026 "city": { "type": "string" },
2027 "country": { "type": "string" },
2028 },
2029 "required": ["city"],
2030 });
2031 let request = OpenAiChatRequest {
2032 model: "m".into(),
2033 messages: vec![OpenAiMessage::from_message(&Message::user("hi"))],
2034 tools: vec![],
2035 stream: None,
2036 stream_options: None,
2037 temperature: None,
2038 max_tokens: None,
2039 response_format: Some(OpenAiResponseFormat::from_schema(&schema)),
2040 extra: BTreeMap::new(),
2041 };
2042 assert_eq!(
2043 serde_json::to_value(&request).unwrap(),
2044 json!({
2045 "model": "m",
2046 "messages": [{"role": "user", "content": "hi"}],
2047 "response_format": {
2048 "type": "json_schema",
2049 "json_schema": { "name": "output", "schema": schema }
2050 }
2051 })
2052 );
2053 }
2054
2055 #[test]
2056 fn json_object_mode_omits_schema() {
2057 let wire = OpenAiResponseFormat::json_object();
2060 assert_eq!(
2061 serde_json::to_value(&wire).unwrap(),
2062 json!({ "type": "json_object" })
2063 );
2064 }
2065
2066 #[test]
2067 fn response_format_mode_gating() {
2068 let schema = json!({ "type": "object" });
2072 let provider = OpenAiProvider::new("http://x", "k", "m");
2073 assert!(provider.wire_response_format(None).is_none());
2074 assert!(matches!(
2075 provider.wire_response_format(Some(&schema)),
2076 Some(OpenAiResponseFormat {
2077 kind: "json_schema",
2078 ..
2079 })
2080 ));
2081
2082 let off = OpenAiProvider::new("http://x", "k", "m")
2083 .with_structured_output_mode(StructuredOutputMode::Off);
2084 assert!(off.wire_response_format(Some(&schema)).is_none());
2085
2086 let json_object = OpenAiProvider::new("http://x", "k", "m")
2087 .with_structured_output_mode(StructuredOutputMode::JsonObject);
2088 assert!(matches!(
2089 json_object.wire_response_format(Some(&schema)),
2090 Some(OpenAiResponseFormat {
2091 kind: "json_object",
2092 ..
2093 })
2094 ));
2095 }
2096
2097 #[test]
2098 fn serializes_model_options_in_request() {
2099 let request = OpenAiChatRequest {
2100 model: "m".into(),
2101 messages: vec![OpenAiMessage::from_message(&Message::user("hi"))],
2102 tools: vec![],
2103 stream: None,
2104 stream_options: None,
2105 temperature: Some(0.5), max_tokens: Some(128),
2107 response_format: None,
2108 extra: BTreeMap::new(),
2109 };
2110 assert_eq!(
2111 serde_json::to_value(&request).unwrap(),
2112 json!({
2113 "model": "m",
2114 "messages": [{"role": "user", "content": "hi"}],
2115 "temperature": 0.5,
2116 "max_tokens": 128
2117 })
2118 );
2119 }
2120
2121 fn test_provider() -> OpenAiProvider {
2122 OpenAiProvider::new("https://api.example.com", "sk-test", "m")
2123 }
2124
2125 fn wire_json(options: ModelOptions) -> String {
2126 let request = ChatRequest {
2127 messages: vec![Message::user("hi")],
2128 tools: vec![],
2129 options,
2130 };
2131 serde_json::to_string(&test_provider().wire_request(&request, false)).unwrap()
2132 }
2133
2134 #[test]
2135 fn passes_through_extra_options() {
2136 let mut options = ModelOptions::default();
2139 options.extra.insert("top_p".into(), serde_json::json!(0.9));
2140 options.extra.insert(
2141 "response_format".into(),
2142 serde_json::json!({"type": "json_object"}),
2143 );
2144
2145 assert_eq!(
2146 serde_json::from_str::<serde_json::Value>(&wire_json(options)).unwrap(),
2147 json!({
2148 "model": "m",
2149 "messages": [{"role": "user", "content": "hi"}],
2150 "top_p": 0.9,
2151 "response_format": {"type": "json_object"}
2152 })
2153 );
2154 }
2155
2156 #[test]
2160 fn explicit_extra_overrides_managed_parameters() {
2161 let mut options = ModelOptions {
2162 temperature: Some(0.5),
2163 max_tokens: Some(64),
2164 structured: Some(json!({"type": "object"})),
2165 ..Default::default()
2166 };
2167 options
2168 .extra
2169 .insert("temperature".into(), serde_json::json!(1.0));
2170 options
2171 .extra
2172 .insert("max_tokens".into(), serde_json::json!(256));
2173 options
2174 .extra
2175 .insert("response_format".into(), json!({"type": "json_object"}));
2176
2177 let wire = wire_json(options);
2178 assert_eq!(
2179 serde_json::from_str::<serde_json::Value>(&wire).unwrap(),
2180 json!({
2181 "model": "m",
2182 "messages": [{"role": "user", "content": "hi"}],
2183 "temperature": 1.0,
2184 "max_tokens": 256,
2185 "response_format": {"type": "json_object"}
2186 })
2187 );
2188 for key in ["temperature", "max_tokens", "response_format"] {
2189 assert_eq!(
2190 wire.matches(&format!("\"{key}\"")).count(),
2191 1,
2192 "{key} must appear exactly once: {wire}"
2193 );
2194 }
2195 }
2196
2197 #[test]
2201 fn extra_entries_naming_envelope_fields_are_ignored() {
2202 let mut options = ModelOptions::default();
2203 options
2204 .extra
2205 .insert("model".into(), serde_json::json!("other-model"));
2206 options
2207 .extra
2208 .insert("messages".into(), serde_json::json!([]));
2209 options
2210 .extra
2211 .insert("stream".into(), serde_json::json!(true));
2212 options.extra.insert("tools".into(), serde_json::json!([]));
2213 options
2214 .extra
2215 .insert("stream_options".into(), serde_json::json!({}));
2216
2217 assert_eq!(
2218 serde_json::from_str::<serde_json::Value>(&wire_json(options)).unwrap(),
2219 json!({
2220 "model": "m",
2221 "messages": [{"role": "user", "content": "hi"}]
2222 })
2223 );
2224 }
2225
2226 #[test]
2227 fn parses_chat_response() {
2228 let body = r#"{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"gpt-4o-mini","choices":[{"index":0,"message":{"role":"assistant","content":"hi!"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8}}"#;
2229 let response = parse_response(body).unwrap();
2230 assert_eq!(response.message, Message::assistant("hi!"));
2231 assert_eq!(response.finish_reason, FinishReason::Stop);
2232 assert_eq!(response.usage, Some(Usage::new(5, 3)));
2233 }
2234
2235 #[test]
2236 fn parses_prompt_cache_breakdown() {
2237 let body = r#"{"choices":[{"message":{"role":"assistant","content":"hi!"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8,"prompt_tokens_details":{"cached_tokens":4}}}"#;
2239 let response = parse_response(body).unwrap();
2240 assert_eq!(response.usage, Some(Usage::new(5, 3).with_cached_tokens(4)));
2241 }
2242
2243 #[test]
2244 fn parses_deepseek_flat_cache_hit_tokens() {
2245 let body = r#"{"choices":[{"message":{"role":"assistant","content":"hi!"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8,"prompt_cache_hit_tokens":4,"prompt_cache_miss_tokens":1}}"#;
2248 let response = parse_response(body).unwrap();
2249 assert_eq!(response.usage, Some(Usage::new(5, 3).with_cached_tokens(4)));
2250 }
2251
2252 #[test]
2253 fn reported_cache_breakdown_takes_precedence_over_flat_fallback() {
2254 let body = r#"{"choices":[{"message":{"role":"assistant","content":"hi!"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8,"prompt_tokens_details":{"cached_tokens":4},"prompt_cache_hit_tokens":9}}"#;
2257 let response = parse_response(body).unwrap();
2258 assert_eq!(response.usage, Some(Usage::new(5, 3).with_cached_tokens(4)));
2259 }
2260
2261 #[test]
2262 fn missing_cache_breakdown_stays_unreported() {
2263 let body = r#"{"choices":[{"message":{"role":"assistant","content":"hi!"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8}}"#;
2265 let response = parse_response(body).unwrap();
2266 assert_eq!(response.usage.unwrap().cached_tokens, None);
2267
2268 let body = r#"{"choices":[{"message":{"role":"assistant","content":"hi!"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8,"prompt_tokens_details":{"cached_tokens":0}}}"#;
2270 let response = parse_response(body).unwrap();
2271 assert_eq!(response.usage.unwrap().cached_tokens, Some(0));
2272 }
2273
2274 #[test]
2275 fn stream_parses_cache_breakdown_in_usage_tail_chunk() {
2276 let mut aggregator = ToolCallAggregator::default();
2279 let done_line = r#"data: {"choices":[{"delta":{"content":"ok"},"finish_reason":"stop"}]}"#;
2280 let usage_line = r#"data: {"choices":[],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8,"prompt_tokens_details":{"cached_tokens":4}}}"#;
2281
2282 assert_eq!(
2283 parse_events(done_line, &mut aggregator),
2284 vec![StreamEvent::Delta("ok".to_string())]
2285 );
2286 assert!(parse_events(usage_line, &mut aggregator).is_empty());
2287 assert_eq!(
2288 aggregator.flush_done(),
2289 vec![Ok(StreamEvent::Done {
2290 reason: FinishReason::Stop,
2291 usage: Some(Usage::new(5, 3).with_cached_tokens(4)),
2292 })]
2293 );
2294 }
2295
2296 #[test]
2297 fn parses_tool_call_response() {
2298 let body = r#"{"choices":[{"message":{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"calculator","arguments":"{\"expression\": \"1+1\"}"}}]},"finish_reason":"tool_calls"}]}"#;
2299 let response = parse_response(body).unwrap();
2300 assert_eq!(
2301 response.message,
2302 Message::Assistant {
2303 content: String::new(),
2304 reasoning: None,
2305 tool_calls: vec![ToolCall {
2306 id: "call_1".into(),
2307 name: "calculator".into(),
2308 arguments: r#"{"expression": "1+1"}"#.into(),
2309 }],
2310 }
2311 );
2312 assert_eq!(response.finish_reason, FinishReason::Stop);
2313 }
2314
2315 #[test]
2316 fn parses_text_with_tool_calls() {
2317 let body = r#"{"choices":[{"message":{"role":"assistant","content":"let me compute","tool_calls":[{"id":"call_1","type":"function","function":{"name":"calculator","arguments":"{}"}}]},"finish_reason":"tool_calls"}]}"#;
2320 let response = parse_response(body).unwrap();
2321 assert_eq!(
2322 response.message,
2323 Message::Assistant {
2324 content: "let me compute".into(),
2325 reasoning: None,
2326 tool_calls: vec![ToolCall {
2327 id: "call_1".into(),
2328 name: "calculator".into(),
2329 arguments: "{}".into(),
2330 }],
2331 }
2332 );
2333 }
2334
2335 #[test]
2336 fn parses_multiple_tool_calls_in_one_message() {
2337 let body = r#"{"choices":[{"message":{"role":"assistant","content":null,"tool_calls":[{"id":"a","type":"function","function":{"name":"calculator","arguments":"{}"}},{"id":"b","type":"function","function":{"name":"calculator","arguments":"{}"}}]},"finish_reason":"tool_calls"}]}"#;
2340 let response = parse_response(body).unwrap();
2341 assert_eq!(
2342 response.message,
2343 Message::Assistant {
2344 content: String::new(),
2345 reasoning: None,
2346 tool_calls: vec![
2347 ToolCall {
2348 id: "a".into(),
2349 name: "calculator".into(),
2350 arguments: "{}".into(),
2351 },
2352 ToolCall {
2353 id: "b".into(),
2354 name: "calculator".into(),
2355 arguments: "{}".into(),
2356 },
2357 ],
2358 }
2359 );
2360 }
2361
2362 #[test]
2363 fn maps_length_finish_reason() {
2364 let body = r#"{"choices":[{"message":{"role":"assistant","content":""},"finish_reason":"length"}]}"#;
2365 let response = parse_response(body).unwrap();
2366 assert_eq!(response.message, Message::assistant(""));
2367 assert_eq!(response.finish_reason, FinishReason::Length);
2368 }
2369
2370 #[test]
2371 fn maps_unknown_finish_reason_to_other() {
2372 let body = r#"{"choices":[{"message":{"role":"assistant","content":""},"finish_reason":"content_filter"}]}"#;
2375 let response = parse_response(body).unwrap();
2376 assert_eq!(
2377 response.finish_reason,
2378 FinishReason::Other("content_filter".into())
2379 );
2380 }
2381
2382 #[test]
2383 fn rejects_empty_choices() {
2384 let body = r#"{"choices":[]}"#;
2385 assert!(parse_response(body).is_err());
2386 }
2387
2388 #[test]
2389 fn missing_usage_is_none() {
2390 let body = r#"{"choices":[{"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}"#;
2394 let response = parse_response(body).unwrap();
2395 assert_eq!(response.usage, None);
2396 }
2397
2398 #[test]
2399 fn extracts_error_message() {
2400 let body = r#"{"error":{"message":"Incorrect API key","type":"invalid_request_error"}}"#;
2401 assert_eq!(extract_error_message(body), "Incorrect API key");
2402 }
2403
2404 #[test]
2405 fn falls_back_to_raw_body() {
2406 let body = "not json";
2407 assert_eq!(extract_error_message(body), "not json");
2408 }
2409
2410 fn parse_events(line: &str, aggregator: &mut ToolCallAggregator) -> Vec<StreamEvent> {
2413 parse_sse_line(line, aggregator)
2414 .into_iter()
2415 .map(|event| event.expect("test input must not produce an error"))
2416 .collect()
2417 }
2418
2419 #[test]
2420 fn stream_parses_delta() {
2421 let line = r#"data: {"choices":[{"delta":{"content":"hi"},"finish_reason":null}]}"#;
2422 assert_eq!(
2423 parse_events(line, &mut ToolCallAggregator::default()),
2424 vec![StreamEvent::Delta("hi".to_string())]
2425 );
2426 }
2427
2428 #[test]
2429 fn stream_parses_finish_reason() {
2430 let line = r#"data: {"choices":[{"delta":{},"finish_reason":"stop"}]}"#;
2431 let mut aggregator = ToolCallAggregator::default();
2432 assert!(parse_events(line, &mut aggregator).is_empty());
2435 assert_eq!(
2436 aggregator.flush_done(),
2437 vec![Ok(StreamEvent::Done {
2438 reason: FinishReason::Stop,
2439 usage: None,
2440 })]
2441 );
2442 }
2443
2444 #[test]
2445 fn stream_maps_length_finish_reason() {
2446 let line = r#"data: {"choices":[{"delta":{},"finish_reason":"length"}]}"#;
2447 let mut aggregator = ToolCallAggregator::default();
2448 assert!(parse_events(line, &mut aggregator).is_empty());
2449 assert_eq!(
2450 aggregator.flush_done(),
2451 vec![Ok(StreamEvent::Done {
2452 reason: FinishReason::Length,
2453 usage: None,
2454 })]
2455 );
2456 }
2457
2458 #[test]
2459 fn stream_maps_tool_calls_finish_reason() {
2460 let line = r#"data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}"#;
2461 let mut aggregator = ToolCallAggregator::default();
2462 assert!(parse_events(line, &mut aggregator).is_empty());
2463 assert_eq!(
2464 aggregator.flush_done(),
2465 vec![Ok(StreamEvent::Done {
2466 reason: FinishReason::Stop,
2467 usage: None,
2468 })]
2469 );
2470 }
2471
2472 #[test]
2473 fn stream_parses_usage_only_tail_chunk() {
2474 let mut aggregator = ToolCallAggregator::default();
2478 let done_line = r#"data: {"choices":[{"delta":{"content":"ok"},"finish_reason":"stop"}]}"#;
2479 let usage_line = r#"data: {"choices":[],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8}}"#;
2480
2481 assert_eq!(
2482 parse_events(done_line, &mut aggregator),
2483 vec![StreamEvent::Delta("ok".to_string())]
2484 );
2485 assert!(parse_events(usage_line, &mut aggregator).is_empty());
2486 assert_eq!(
2487 aggregator.flush_done(),
2488 vec![Ok(StreamEvent::Done {
2489 reason: FinishReason::Stop,
2490 usage: Some(Usage::new(5, 3)),
2491 })]
2492 );
2493 }
2494
2495 #[test]
2496 fn stream_parses_usage_in_finish_chunk() {
2497 let line = r#"data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":3,"total_tokens":8}}"#;
2500 let mut aggregator = ToolCallAggregator::default();
2501 assert!(parse_events(line, &mut aggregator).is_empty());
2502 assert_eq!(
2503 aggregator.flush_done(),
2504 vec![Ok(StreamEvent::Done {
2505 reason: FinishReason::Stop,
2506 usage: Some(Usage::new(5, 3)),
2507 })]
2508 );
2509 }
2510
2511 #[test]
2512 fn stream_skips_irrelevant_lines() {
2513 let mut aggregator = ToolCallAggregator::default();
2514 assert!(parse_sse_line("", &mut aggregator).is_empty());
2515 assert!(parse_sse_line(": keep-alive", &mut aggregator).is_empty());
2516 assert!(parse_sse_line("event: message", &mut aggregator).is_empty());
2517 let events = parse_sse_line("data: [DONE]", &mut aggregator);
2519 assert_eq!(events.len(), 1);
2520 assert!(matches!(events[0], Err(ProviderError::Protocol { .. })));
2521 assert!(parse_sse_line(r#"data: {"choices":[]}"#, &mut aggregator).is_empty());
2522 assert!(
2523 parse_sse_line(
2524 r#"data: {"choices":[{"delta":{"role":"assistant"},"finish_reason":null}]}"#,
2525 &mut aggregator
2526 )
2527 .is_empty()
2528 );
2529 }
2530
2531 #[test]
2532 fn stream_rejects_invalid_json() {
2533 let line = "data: not-json";
2534 let events = parse_sse_line(line, &mut ToolCallAggregator::default());
2535 match events.as_slice() {
2536 [Err(ProviderError::Decode { message })] => {
2537 assert_eq!(message, "invalid stream event: not-json")
2538 }
2539 _ => panic!("expected single decode error, got {events:?}"),
2540 }
2541 }
2542
2543 #[test]
2544 fn stream_extracts_provider_error() {
2545 let line = r#"data: {"error":{"message":"rate limited"}}"#;
2546 let events = parse_sse_line(line, &mut ToolCallAggregator::default());
2547 let err = events.into_iter().next().unwrap().unwrap_err();
2548 assert!(err.to_string().contains("rate limited"));
2549 }
2550
2551 #[test]
2552 fn stream_forwards_reasoning_chunks() {
2553 let mut aggregator = ToolCallAggregator::default();
2556 let line1 =
2557 r#"data: {"choices":[{"delta":{"reasoning_content":"first"},"finish_reason":null}]}"#;
2558 let line2 = r#"data: {"choices":[{"delta":{"reasoning_content":" multiply"},"finish_reason":null}]}"#;
2559 let line3 = r#"data: {"choices":[{"delta":{"content":"answer"},"finish_reason":"stop"}]}"#;
2560
2561 assert_eq!(
2562 parse_events(line1, &mut aggregator),
2563 vec![StreamEvent::Reasoning("first".to_string())]
2564 );
2565 assert_eq!(
2566 parse_events(line2, &mut aggregator),
2567 vec![StreamEvent::Reasoning(" multiply".to_string())]
2568 );
2569 assert_eq!(
2570 parse_events(line3, &mut aggregator),
2571 vec![StreamEvent::Delta("answer".to_string())]
2572 );
2573 assert_eq!(
2574 aggregator.flush_done(),
2575 vec![Ok(StreamEvent::Done {
2576 reason: FinishReason::Stop,
2577 usage: None,
2578 })]
2579 );
2580 }
2581
2582 #[test]
2583 fn stream_reasoning_with_tool_call() {
2584 let mut aggregator = ToolCallAggregator::default();
2588 let line1 =
2589 r#"data: {"choices":[{"delta":{"reasoning_content":"all"},"finish_reason":null}]}"#;
2590 let line2 =
2591 r#"data: {"choices":[{"delta":{"reasoning_content":" set"},"finish_reason":null}]}"#;
2592 let line3 = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"calculator","arguments":"{}"}}]},"finish_reason":"tool_calls"}]}"#;
2593
2594 assert_eq!(
2595 parse_events(line1, &mut aggregator),
2596 vec![StreamEvent::Reasoning("all".to_string())]
2597 );
2598 assert_eq!(
2599 parse_events(line2, &mut aggregator),
2600 vec![StreamEvent::Reasoning(" set".to_string())]
2601 );
2602 assert_eq!(
2603 parse_events(line3, &mut aggregator),
2604 vec![StreamEvent::ToolCall {
2605 id: "call_1".to_string(),
2606 name: "calculator".to_string(),
2607 arguments: "{}".to_string(),
2608 },]
2609 );
2610 assert_eq!(
2611 aggregator.flush_done(),
2612 vec![Ok(StreamEvent::Done {
2613 reason: FinishReason::Stop,
2614 usage: None,
2615 })]
2616 );
2617 }
2618
2619 #[test]
2620 fn stream_aggregates_tool_call_chunks() {
2621 let mut aggregator = ToolCallAggregator::default();
2624 let line1 = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"calculator","arguments":"{\"expression\": \""}}]},"finish_reason":null}]}"#;
2625 let line2 = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"1+1"}}]},"finish_reason":null}]}"#;
2626 let line3 = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"finish_reason":"tool_calls"}]}"#;
2627
2628 assert!(parse_events(line1, &mut aggregator).is_empty());
2629 assert!(parse_events(line2, &mut aggregator).is_empty());
2630 assert_eq!(
2631 parse_events(line3, &mut aggregator),
2632 vec![StreamEvent::ToolCall {
2633 id: "call_1".to_string(),
2634 name: "calculator".to_string(),
2635 arguments: "{\"expression\": \"1+1\"}".to_string(),
2636 },]
2637 );
2638 assert_eq!(
2639 aggregator.flush_done(),
2640 vec![Ok(StreamEvent::Done {
2641 reason: FinishReason::Stop,
2642 usage: None,
2643 })]
2644 );
2645 }
2646
2647 #[test]
2648 fn stream_handles_multiple_interleaved_tool_calls() {
2649 let mut aggregator = ToolCallAggregator::default();
2652 let line1 = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"now","arguments":"{}"}},{"index":1,"id":"call_2","function":{"name":"calculator","arguments":"{\"exp"}}]},"finish_reason":null}]}"#;
2653 let line2 = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":1,"function":{"arguments":"ression\":\"1\"}"}}]},"finish_reason":"tool_calls"}]}"#;
2654
2655 assert!(parse_events(line1, &mut aggregator).is_empty());
2656 assert_eq!(
2657 parse_events(line2, &mut aggregator),
2658 vec![
2659 StreamEvent::ToolCall {
2660 id: "call_1".to_string(),
2661 name: "now".to_string(),
2662 arguments: "{}".to_string(),
2663 },
2664 StreamEvent::ToolCall {
2665 id: "call_2".to_string(),
2666 name: "calculator".to_string(),
2667 arguments: "{\"expression\":\"1\"}".to_string(),
2668 },
2669 ]
2670 );
2671 assert_eq!(
2672 aggregator.flush_done(),
2673 vec![Ok(StreamEvent::Done {
2674 reason: FinishReason::Stop,
2675 usage: None,
2676 })]
2677 );
2678 }
2679
2680 #[test]
2681 fn stream_keeps_delta_and_tool_call_order() {
2682 let mut aggregator = ToolCallAggregator::default();
2685 let line1 =
2686 r#"data: {"choices":[{"delta":{"content":"let me compute"},"finish_reason":null}]}"#;
2687 let line2 = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"calculator","arguments":"{}"}}]},"finish_reason":"tool_calls"}]}"#;
2688
2689 assert_eq!(
2690 parse_events(line1, &mut aggregator),
2691 vec![StreamEvent::Delta("let me compute".to_string())]
2692 );
2693 assert_eq!(
2694 parse_events(line2, &mut aggregator),
2695 vec![StreamEvent::ToolCall {
2696 id: "call_1".to_string(),
2697 name: "calculator".to_string(),
2698 arguments: "{}".to_string(),
2699 },]
2700 );
2701 assert_eq!(
2702 aggregator.flush_done(),
2703 vec![Ok(StreamEvent::Done {
2704 reason: FinishReason::Stop,
2705 usage: None,
2706 })]
2707 );
2708 }
2709
2710 #[tokio::test]
2713 async fn sse_lines_joins_chunks_and_strips_terminators() {
2714 let stream = sse_lines(
2715 futures::stream::iter([
2716 Ok(b"data: a\r".to_vec()),
2717 Ok(b"\ndata: b\ndata: c".to_vec()),
2718 ]),
2719 Duration::from_secs(60),
2720 Duration::from_secs(60),
2721 );
2722 let lines: Vec<_> = stream.map(|line| line.unwrap()).collect().await;
2723 assert_eq!(
2724 lines,
2725 vec![
2726 "data: a".to_string(),
2727 "data: b".to_string(),
2728 "data: c".to_string()
2729 ]
2730 );
2731 }
2732
2733 #[tokio::test]
2734 async fn sse_lines_emits_tail_without_newline() {
2735 let stream = sse_lines(
2736 futures::stream::iter([Ok(b"data: x".to_vec())]),
2737 Duration::from_secs(60),
2738 Duration::from_secs(60),
2739 );
2740 let lines: Vec<_> = stream.map(|line| line.unwrap()).collect().await;
2741 assert_eq!(lines, vec!["data: x".to_string()]);
2742 }
2743
2744 #[tokio::test]
2745 async fn sse_lines_idle_timeout_emits_timeout() {
2746 let mut stream = Box::pin(sse_lines(
2750 futures::stream::pending::<Result<Vec<u8>, reqwest::Error>>(),
2751 Duration::from_millis(10),
2752 Duration::from_secs(60),
2753 ));
2754 let first = stream.next().await.unwrap();
2755 assert!(matches!(first, Err(ProviderError::Timeout(_))));
2756 assert!(stream.next().await.is_none());
2757 }
2758
2759 #[tokio::test]
2766 async fn sse_lines_total_deadline_kills_keepalive_stream() {
2767 let mut stream = Box::pin(sse_lines(
2770 futures::stream::iter(std::iter::repeat_with(|| Ok(b": keep-alive\n".to_vec()))),
2771 Duration::from_secs(60),
2772 Duration::from_millis(50),
2773 ));
2774 let mut lines = 0usize;
2777 loop {
2778 match stream.next().await {
2779 Some(Ok(_)) => lines += 1,
2780 Some(Err(ProviderError::Timeout(_))) => break,
2781 other => panic!("expected line or Timeout, got {other:?}"),
2782 }
2783 }
2784 assert!(lines > 0, "lines must be produced before the deadline");
2785 assert!(
2786 stream.next().await.is_none(),
2787 "stream must terminate after an error"
2788 );
2789 }
2790
2791 #[tokio::test]
2795 async fn sse_lines_keepalive_lines_delivered_within_deadline() {
2796 let mut stream = Box::pin(sse_lines(
2797 futures::stream::iter(std::iter::repeat_with(|| Ok(b": keep-alive\n".to_vec()))),
2798 Duration::from_secs(60),
2799 Duration::from_secs(60),
2800 ));
2801 let first = stream.next().await.unwrap().unwrap();
2802 assert_eq!(first, ": keep-alive");
2803 }
2804
2805 #[tokio::test]
2809 async fn sse_lines_over_limit_terminates_stream() {
2810 let mut stream = Box::pin(sse_lines(
2811 futures::stream::iter([Ok(vec![b'x'; MAX_SSE_LINE + 1])]),
2812 Duration::from_secs(60),
2813 Duration::from_secs(60),
2814 ));
2815 let first = stream.next().await.unwrap();
2816 assert!(matches!(
2817 first,
2818 Err(ProviderError::ResponseTooLarge {
2819 limit_bytes: MAX_SSE_LINE
2820 })
2821 ));
2822 assert!(
2823 stream.next().await.is_none(),
2824 "stream must terminate after exceeding the limit"
2825 );
2826 }
2827
2828 #[test]
2831 fn bad_line_discards_pending_done() {
2832 let mut aggregator = ToolCallAggregator::default();
2833 let finish = r#"data: {"choices":[{"delta":{},"finish_reason":"stop"}]}"#;
2834 assert!(parse_sse_line(finish, &mut aggregator).is_empty()); let bad = "data: not-json";
2837 let events = parse_sse_line(bad, &mut aggregator);
2838 assert_eq!(events.len(), 1);
2839 assert!(matches!(events[0], Err(ProviderError::Decode { .. })));
2840
2841 assert!(parse_sse_line("data: [DONE]", &mut aggregator).is_empty());
2844 }
2845
2846 #[test]
2850 fn tool_call_count_limit_only_for_new_calls() {
2851 let mut aggregator = ToolCallAggregator::default();
2852 for i in 0..MAX_TOOL_CALLS {
2853 let line = format!(
2854 r#"data: {{"choices":[{{"delta":{{"tool_calls":[{{"index":{i},"id":"c{i}","function":{{"name":"f","arguments":"{{}}"}}}}]}},"finish_reason":null}}]}}"#
2855 );
2856 assert!(
2857 parse_sse_line(&line, &mut aggregator).is_empty(),
2858 "tool call {i} must be accepted"
2859 );
2860 }
2861 let over = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":64,"id":"c64","function":{"name":"f","arguments":"{}"}}]},"finish_reason":null}]}"#;
2863 let events = parse_sse_line(over, &mut aggregator);
2864 assert_eq!(events.len(), 1);
2865 assert!(matches!(events[0], Err(ProviderError::Protocol { .. })));
2866 let cont = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"more\":true}"}}]},"finish_reason":null}]}"#;
2869 assert!(parse_sse_line(cont, &mut aggregator).is_empty());
2870 }
2871
2872 #[test]
2876 fn tool_arguments_size_limit() {
2877 let mut aggregator = ToolCallAggregator::default();
2878 let huge = "x".repeat(MAX_ACCUMULATED_TEXT + 1);
2879 let line = format!(
2880 r#"data: {{"choices":[{{"delta":{{"tool_calls":[{{"index":0,"id":"c0","function":{{"name":"f","arguments":"{huge}"}}}}]}},"finish_reason":null}}]}}"#
2881 );
2882 let events = parse_sse_line(&line, &mut aggregator);
2883 assert_eq!(events.len(), 1);
2884 assert!(matches!(
2885 events[0],
2886 Err(ProviderError::ResponseTooLarge {
2887 limit_bytes: MAX_ACCUMULATED_TEXT
2888 })
2889 ));
2890 }
2891
2892 #[test]
2895 fn debug_masks_api_key() {
2896 let provider = OpenAiProvider::new("https://api.example.com", "sk-super-secret", "model");
2897 let debug = format!("{provider:?}");
2898 assert!(
2899 !debug.contains("sk-super-secret"),
2900 "Debug leaked api_key: {debug}"
2901 );
2902 assert!(
2903 debug.contains("\"***\""),
2904 "mask placeholder missing: {debug}"
2905 );
2906 }
2907
2908 #[test]
2911 fn tool_arguments_continuation_size_limit() {
2912 let mut aggregator = ToolCallAggregator::default();
2913 let first = "x".repeat(MAX_ACCUMULATED_TEXT - 1);
2916 let line1 = format!(
2917 r#"data: {{"choices":[{{"delta":{{"tool_calls":[{{"index":0,"id":"c0","function":{{"name":"f","arguments":"{first}"}}}}]}},"finish_reason":null}}]}}"#
2918 );
2919 assert!(parse_sse_line(&line1, &mut aggregator).is_empty());
2920 let cont = r#"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"yy"}}]},"finish_reason":null}]}"#;
2921 let events = parse_sse_line(cont, &mut aggregator);
2922 assert_eq!(events.len(), 1);
2923 assert!(matches!(
2924 events[0],
2925 Err(ProviderError::ResponseTooLarge {
2926 limit_bytes: MAX_ACCUMULATED_TEXT
2927 })
2928 ));
2929 }
2930
2931 #[test]
2934 fn errored_latch_blocks_resurrected_done() {
2935 let mut aggregator = ToolCallAggregator::default();
2936 let events = parse_sse_line("data: not-json", &mut aggregator);
2938 assert_eq!(events.len(), 1);
2939 assert!(matches!(events[0], Err(ProviderError::Decode { .. })));
2940 let finish = r#"data: {"choices":[{"delta":{},"finish_reason":"stop"}]}"#;
2943 assert!(parse_sse_line(finish, &mut aggregator).is_empty());
2944 assert!(parse_sse_line("data: [DONE]", &mut aggregator).is_empty());
2946 }
2947
2948 #[tokio::test]
2951 async fn sse_stream_eof_flushes_pending_done() {
2952 let aggregator = Arc::new(Mutex::new(ToolCallAggregator::default()));
2953 let chunks = futures::stream::iter([
2954 Ok(b"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n".to_vec()),
2955 Ok(b"data: {\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":2,\"total_tokens\":3}}\n".to_vec()),
2956 ]);
2957 let mut stream = Box::pin(sse_stream(
2958 chunks,
2959 Duration::from_secs(60),
2960 Duration::from_secs(60),
2961 aggregator,
2962 ));
2963 let events: Vec<_> = stream.by_ref().map(|e| e.unwrap()).collect().await;
2964 assert_eq!(
2965 events,
2966 vec![StreamEvent::Done {
2967 reason: FinishReason::Stop,
2968 usage: Some(Usage::new(1, 2)),
2969 }]
2970 );
2971 }
2972
2973 #[tokio::test]
2976 async fn sse_stream_done_marker_flushes() {
2977 let aggregator = Arc::new(Mutex::new(ToolCallAggregator::default()));
2978 let chunks = futures::stream::iter([
2979 Ok(b"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n".to_vec()),
2980 Ok(b"data: [DONE]\n".to_vec()),
2981 ]);
2982 let mut stream = Box::pin(sse_stream(
2983 chunks,
2984 Duration::from_secs(60),
2985 Duration::from_secs(60),
2986 aggregator,
2987 ));
2988 let events: Vec<_> = stream.by_ref().map(|e| e.unwrap()).collect().await;
2989 assert_eq!(
2990 events,
2991 vec![StreamEvent::Done {
2992 reason: FinishReason::Stop,
2993 usage: None
2994 }]
2995 );
2996 }
2997
2998 #[tokio::test]
3001 async fn sse_stream_error_then_no_done() {
3002 let aggregator = Arc::new(Mutex::new(ToolCallAggregator::default()));
3003 let chunks = futures::stream::iter([
3004 Ok(b"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}".to_vec()),
3005 Ok(b"data: not-json".to_vec()),
3006 Ok(b"data: [DONE]".to_vec()),
3007 ]);
3008 let mut stream = Box::pin(sse_stream(
3009 chunks,
3010 Duration::from_secs(60),
3011 Duration::from_secs(60),
3012 aggregator,
3013 ));
3014 let events: Vec<Result<StreamEvent, ProviderError>> = stream.by_ref().collect().await;
3015 assert_eq!(events.len(), 1);
3016 assert!(matches!(events[0], Err(ProviderError::Decode { .. })));
3017 }
3018
3019 async fn one_shot_server(
3024 response: String,
3025 ) -> (std::net::SocketAddr, tokio::task::JoinHandle<String>) {
3026 use tokio::io::{AsyncReadExt, AsyncWriteExt};
3027 use tokio::net::TcpListener;
3028
3029 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3030 let addr = listener.local_addr().unwrap();
3031 let server = tokio::spawn(async move {
3032 let (mut socket, _) = listener.accept().await.unwrap();
3033 let mut bytes = Vec::new();
3034 let mut buf = [0u8; 8192];
3035 loop {
3036 let n = socket.read(&mut buf).await.unwrap();
3037 if n == 0 {
3038 break;
3039 }
3040 bytes.extend_from_slice(&buf[..n]);
3041 let Some(headers_end) = bytes
3042 .windows(4)
3043 .position(|window| window == b"\r\n\r\n")
3044 .map(|index| index + 4)
3045 else {
3046 continue;
3047 };
3048 let request = String::from_utf8_lossy(&bytes).to_string();
3049 let content_length = request
3050 .lines()
3051 .find_map(|line| {
3052 line.to_ascii_lowercase()
3053 .strip_prefix("content-length:")
3054 .and_then(|value| value.trim().parse::<usize>().ok())
3055 })
3056 .unwrap_or(0);
3057 if bytes.len() >= headers_end + content_length {
3058 break;
3059 }
3060 }
3061 socket.write_all(response.as_bytes()).await.unwrap();
3062 String::from_utf8_lossy(&bytes).to_string()
3063 });
3064 (addr, server)
3065 }
3066
3067 fn sse_response(content: &str) -> String {
3070 let body = format!(
3071 "data: {{\"choices\":[{{\"delta\":{{\"content\":\"{content}\"}}}}]}}\n\n\
3072 data: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"stop\"}}]}}\n\n\
3073 data: [DONE]\n\n"
3074 );
3075 format!(
3076 "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {}\r\n\r\n{}",
3077 body.len(),
3078 body
3079 )
3080 }
3081
3082 #[tokio::test]
3085 async fn chat_over_real_http_wire() {
3086 let body = r#"{"choices":[{"message":{"role":"assistant","content":"hi","reasoning_content":null,"tool_calls":null},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3}}"#;
3089 let response = format!(
3090 "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{}",
3091 body.len(),
3092 body
3093 );
3094 let (addr, server) = one_shot_server(response).await;
3095
3096 let provider = OpenAiProvider::new(format!("http://{addr}"), "sk-test", "deepseek-chat");
3097 let resp = provider
3098 .chat(ChatRequest {
3099 messages: vec![crate::message::Message::user("hi")],
3100 tools: vec![],
3101 options: Default::default(),
3102 })
3103 .await
3104 .unwrap();
3105 assert_eq!(resp.message, crate::message::Message::assistant("hi"));
3106 assert_eq!(resp.usage, Some(Usage::new(1, 2)));
3107
3108 let req = server.await.unwrap();
3110 assert!(
3111 req.starts_with("POST /chat/completions HTTP/1.1"),
3112 "method/path: {req}"
3113 );
3114 assert!(
3115 req.to_lowercase().contains("authorization: bearer sk-test"),
3116 "auth header: {req}"
3117 );
3118 assert!(
3119 req.contains(r#""model":"deepseek-chat""#),
3120 "body model: {req}"
3121 );
3122 }
3123
3124 #[tokio::test]
3129 async fn custom_headers_reach_the_wire() {
3130 let (addr, server) = one_shot_server(sse_response("hi")).await;
3131
3132 let provider = OpenAiProvider::new(format!("http://{addr}"), "sk-test", "deepseek-chat");
3133 let mut options = ModelOptions::default();
3134 options
3135 .headers
3136 .insert("x-trace-id".into(), "abc-123".into());
3137 options.headers.insert("api-key".into(), "secret".into());
3138 options
3139 .headers
3140 .insert("Authorization".into(), "Token custom".into());
3141 options
3142 .headers
3143 .insert("Content-Length".into(), "9999".into());
3144
3145 let mut stream = provider
3146 .stream_chat(ChatRequest {
3147 messages: vec![crate::message::Message::user("hi")],
3148 tools: vec![],
3149 options,
3150 })
3151 .await
3152 .unwrap();
3153 let mut text = String::new();
3154 while let Some(event) = stream.next().await {
3155 match event.unwrap() {
3156 StreamEvent::Delta(delta) => text.push_str(&delta),
3157 StreamEvent::Done { .. } => break,
3158 other => panic!("unexpected stream event: {other:?}"),
3159 }
3160 }
3161 assert_eq!(text, "hi");
3162
3163 let req = server.await.unwrap();
3164 let lower = req.to_lowercase();
3165 assert!(
3166 lower.contains("x-trace-id: abc-123"),
3167 "vendor header: {req}"
3168 );
3169 assert!(
3170 lower.contains("api-key: secret"),
3171 "vendor auth header: {req}"
3172 );
3173 assert!(
3176 lower.contains("authorization: token custom"),
3177 "caller auth: {req}"
3178 );
3179 assert!(
3180 !lower.contains("bearer sk-test"),
3181 "api key must not be appended: {req}"
3182 );
3183 assert_eq!(
3184 lower.matches("authorization:").count(),
3185 1,
3186 "exactly one Authorization: {req}"
3187 );
3188 let content_length = lower
3191 .lines()
3192 .find_map(|line| line.strip_prefix("content-length:"))
3193 .and_then(|value| value.trim().parse::<usize>().ok())
3194 .unwrap();
3195 let body = req.split("\r\n\r\n").nth(1).unwrap_or_default();
3196 assert_eq!(content_length, body.len(), "framing intact: {req}");
3197 assert!(body.contains(r#""stream":true"#), "streaming body: {req}");
3198 }
3199
3200 #[tokio::test]
3204 async fn invalid_header_fails_before_sending() {
3205 let provider = OpenAiProvider::new("http://127.0.0.1:1", "sk-test", "m");
3206 for (name, value) in [
3207 ("bad header", "value"),
3208 ("x-injected", "value\r\nx-evil: 1"),
3209 ] {
3210 let mut options = ModelOptions::default();
3211 options.headers.insert(name.into(), value.into());
3212 let err = provider
3213 .chat(ChatRequest {
3214 messages: vec![crate::message::Message::user("hi")],
3215 tools: vec![],
3216 options,
3217 })
3218 .await
3219 .unwrap_err();
3220
3221 assert!(
3222 matches!(&err, ProviderError::Protocol { message } if message.contains(name)),
3223 "expected a protocol error naming {name:?}, got: {err:?}"
3224 );
3225 }
3226 }
3227
3228 #[test]
3231 fn wire_headers_filter_framing_and_keep_content_type() {
3232 let mut options = ModelOptions::default();
3233 for name in [
3234 "content-length",
3235 "Transfer-Encoding",
3236 "connection",
3237 "Host",
3238 "upgrade",
3239 ] {
3240 options.headers.insert(name.into(), "x".into());
3241 }
3242 options.headers.insert(
3243 "Content-Type".into(),
3244 "application/json; charset=utf-8".into(),
3245 );
3246
3247 let headers = wire_headers(&options).unwrap();
3248
3249 assert_eq!(headers.len(), 1);
3250 assert_eq!(
3251 headers.get(reqwest::header::CONTENT_TYPE).unwrap(),
3252 "application/json; charset=utf-8"
3253 );
3254 }
3255
3256 #[tokio::test]
3260 async fn chat_request_timeout_fires() {
3261 use tokio::io::AsyncReadExt;
3262 use tokio::net::TcpListener;
3263
3264 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
3265 let addr = listener.local_addr().unwrap();
3266
3267 let server = tokio::spawn(async move {
3268 let (mut socket, _) = listener.accept().await.unwrap();
3271 let mut buf = [0u8; 8192];
3272 let _ = socket.read(&mut buf).await;
3273 std::future::pending::<()>().await;
3274 });
3275
3276 let provider = OpenAiProvider::new(format!("http://{addr}"), "sk-test", "m")
3277 .with_request_timeout(Duration::from_millis(100));
3278 let err = provider
3279 .chat(ChatRequest {
3280 messages: vec![crate::message::Message::user("hi")],
3281 tools: vec![],
3282 options: Default::default(),
3283 })
3284 .await
3285 .unwrap_err();
3286 assert!(matches!(err, ProviderError::Timeout(TimeoutStage::Request)));
3287 server.abort();
3288 }
3289
3290 #[tokio::test]
3294 async fn chat_connect_timeout_fires() {
3295 let provider = OpenAiProvider::new("http://192.0.2.1/mcp", "sk-test", "m")
3296 .with_connect_timeout(Duration::from_millis(200));
3297 let err = provider
3298 .chat(ChatRequest {
3299 messages: vec![crate::message::Message::user("hi")],
3300 tools: vec![],
3301 options: Default::default(),
3302 })
3303 .await
3304 .unwrap_err();
3305 assert!(
3309 matches!(&err, ProviderError::Timeout(_) | ProviderError::Network(_)),
3310 "connection failure should classify as Timeout or Network, got: {err:?}"
3311 );
3312 }
3313
3314 #[tokio::test]
3317 async fn sse_stream_empty_stream_is_error() {
3318 let aggregator = Arc::new(Mutex::new(ToolCallAggregator::default()));
3319 let mut stream = Box::pin(sse_stream(
3320 futures::stream::iter([Ok(b"\n".to_vec())]),
3321 Duration::from_secs(60),
3322 Duration::from_secs(60),
3323 aggregator,
3324 ));
3325 let events: Vec<Result<StreamEvent, ProviderError>> = stream.by_ref().collect().await;
3326 assert_eq!(events.len(), 1);
3327 assert!(matches!(
3328 events[0],
3329 Err(ProviderError::Protocol { ref message }) if message.contains("without finish_reason")
3330 ));
3331 }
3332
3333 #[tokio::test]
3336 async fn sse_stream_drops_data_after_done_marker() {
3337 let aggregator = Arc::new(Mutex::new(ToolCallAggregator::default()));
3338 let chunks = futures::stream::iter([
3339 Ok(b"data: {\"choices\":[{\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n".to_vec()),
3340 Ok(b"data: [DONE]\n".to_vec()),
3341 Ok(b"data: {\"choices\":[{\"delta\":{\"content\":\"x\"}}]}\n".to_vec()),
3343 ]);
3344 let mut stream = Box::pin(sse_stream(
3345 chunks,
3346 Duration::from_secs(60),
3347 Duration::from_secs(60),
3348 aggregator,
3349 ));
3350 let events: Vec<_> = stream.by_ref().map(|e| e.unwrap()).collect().await;
3351 assert_eq!(
3352 events,
3353 vec![
3354 StreamEvent::Delta("ok".into()),
3355 StreamEvent::Done {
3356 reason: FinishReason::Stop,
3357 usage: None
3358 },
3359 ]
3360 );
3361 }
3362
3363 #[tokio::test]
3366 async fn network_error_terminates_and_drops_buffered_tail() {
3367 let conn_err = reqwest::Client::new()
3370 .get("http://[::1")
3371 .build()
3372 .expect_err("invalid URL must fail to build");
3373 let mut stream = Box::pin(sse_lines(
3374 futures::stream::iter([Ok(b"data: partial".to_vec()), Err(conn_err)]),
3375 Duration::from_secs(60),
3376 Duration::from_secs(60),
3377 ));
3378 let first = stream.next().await.unwrap();
3379 assert!(matches!(first, Err(ProviderError::Network(_))));
3380 assert!(stream.next().await.is_none());
3383 }
3384}