1use std::time::Duration;
9
10use async_trait::async_trait;
11use serde::{Deserialize, Deserializer, Serialize, Serializer};
12use serde_json::{json, Value};
13
14use crate::error::{Result, RustyError};
15
16pub(crate) const ERROR_BODY_MAX_CHARS: usize = 512;
20
21pub(crate) fn truncate_body(body: &str) -> String {
23 body.chars().take(ERROR_BODY_MAX_CHARS).collect()
24}
25
26pub(crate) fn backoff_delay(base: Duration, attempt: u32) -> Duration {
31 let base = base.saturating_mul(2u32.saturating_pow(attempt.min(6)));
32 let nanos = std::time::SystemTime::now()
33 .duration_since(std::time::UNIX_EPOCH)
34 .map(|d| d.subsec_nanos())
35 .unwrap_or(0);
36 let jitter = 0.5 + f64::from(nanos % 1000) / 1000.0; let jittered = (base.as_nanos() as f64 * jitter) as u128;
38 Duration::from_nanos(jittered.min(u64::MAX as u128) as u64)
39}
40
41#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
43#[serde(rename_all = "lowercase")]
44pub enum Role {
45 System,
47 User,
49 Assistant,
51 Tool,
53}
54
55#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
61pub struct ChatMessage {
62 pub role: Role,
64
65 #[serde(default, skip_serializing_if = "Option::is_none")]
68 pub content: Option<String>,
69
70 #[serde(default, skip_serializing_if = "Vec::is_empty")]
72 pub tool_calls: Vec<ToolCall>,
73
74 #[serde(default, skip_serializing_if = "Option::is_none")]
76 pub tool_call_id: Option<String>,
77
78 #[serde(default, skip_serializing_if = "Option::is_none")]
80 pub name: Option<String>,
81}
82
83impl ChatMessage {
84 pub fn system(content: impl Into<String>) -> Self {
86 Self {
87 role: Role::System,
88 content: Some(content.into()),
89 tool_calls: Vec::new(),
90 tool_call_id: None,
91 name: None,
92 }
93 }
94
95 pub fn user(content: impl Into<String>) -> Self {
97 Self {
98 role: Role::User,
99 content: Some(content.into()),
100 tool_calls: Vec::new(),
101 tool_call_id: None,
102 name: None,
103 }
104 }
105
106 pub fn assistant(content: impl Into<String>) -> Self {
108 Self {
109 role: Role::Assistant,
110 content: Some(content.into()),
111 tool_calls: Vec::new(),
112 tool_call_id: None,
113 name: None,
114 }
115 }
116
117 pub fn assistant_tool_calls(tool_calls: Vec<ToolCall>) -> Self {
119 Self {
120 role: Role::Assistant,
121 content: None,
122 tool_calls,
123 tool_call_id: None,
124 name: None,
125 }
126 }
127
128 pub fn tool_result(tool_call_id: impl Into<String>, content: impl Into<String>) -> Self {
130 Self {
131 role: Role::Tool,
132 content: Some(content.into()),
133 tool_calls: Vec::new(),
134 tool_call_id: Some(tool_call_id.into()),
135 name: None,
136 }
137 }
138
139 pub fn has_tool_calls(&self) -> bool {
141 !self.tool_calls.is_empty()
142 }
143}
144
145#[derive(Debug, Clone, PartialEq)]
152pub struct ToolCall {
153 pub id: String,
155 pub name: String,
157 pub arguments: Value,
159}
160
161impl ToolCall {
162 pub fn new(id: impl Into<String>, name: impl Into<String>, arguments: Value) -> Self {
164 Self {
165 id: id.into(),
166 name: name.into(),
167 arguments,
168 }
169 }
170}
171
172impl Serialize for ToolCall {
173 fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
174 json!({
175 "id": self.id,
176 "type": "function",
177 "function": {
178 "name": self.name,
179 "arguments": self.arguments.to_string(),
180 }
181 })
182 .serialize(serializer)
183 }
184}
185
186impl<'de> Deserialize<'de> for ToolCall {
187 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
188 #[derive(Deserialize)]
189 struct FnArgs {
190 name: String,
191 arguments: Value,
192 }
193 #[derive(Deserialize)]
194 struct Wire {
195 id: String,
196 function: FnArgs,
197 }
198 let wire = Wire::deserialize(deserializer)?;
199 let arguments = match wire.function.arguments {
200 Value::String(s) => serde_json::from_str(&s).map_err(serde::de::Error::custom)?,
202 other => other,
204 };
205 Ok(ToolCall {
206 id: wire.id,
207 name: wire.function.name,
208 arguments,
209 })
210 }
211}
212
213#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
215pub struct Usage {
216 #[serde(default)]
218 pub prompt_tokens: u64,
219 #[serde(default)]
221 pub completion_tokens: u64,
222 #[serde(default)]
224 pub total_tokens: u64,
225}
226
227#[derive(Debug, Clone)]
230pub struct ChatResponse {
231 pub message: ChatMessage,
233 pub model: Option<String>,
235 pub usage: Option<Usage>,
237}
238
239#[derive(Debug, Clone)]
247pub struct TokenChunk {
248 pub delta: String,
252
253 pub finish: bool,
255
256 pub raw: Option<Value>,
260}
261
262#[async_trait]
310pub trait ChatModel: Send + Sync {
311 async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse>;
313
314 fn effect(&self) -> crate::record::Effect {
323 crate::record::Effect::NonIdempotent
324 }
325
326 async fn chat_stream(
335 &self,
336 messages: &[ChatMessage],
337 tools: &[Value],
338 on_token: &mut (dyn FnMut(TokenChunk) + Send),
339 ) -> Result<ChatResponse> {
340 let response = self.chat(messages, tools).await?;
341 on_token(TokenChunk {
342 delta: response.message.content.clone().unwrap_or_default(),
343 finish: true,
344 raw: None,
345 });
346 Ok(response)
347 }
348}
349
350pub const DEFAULT_MAX_RETRIES: u32 = 2;
353
354pub const DEFAULT_BASE_BACKOFF: Duration = Duration::from_millis(100);
357
358#[derive(Clone)]
371pub struct OpenAiCompatibleClient {
372 base_url: String,
373 api_key: Option<String>,
374 model: String,
375 client: reqwest::Client,
376 max_retries: u32,
378 base_backoff: Duration,
380}
381
382impl std::fmt::Debug for OpenAiCompatibleClient {
385 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
386 f.debug_struct("OpenAiCompatibleClient")
387 .field("base_url", &self.base_url)
388 .field("api_key", &self.api_key.as_ref().map(|_| "***"))
389 .field("model", &self.model)
390 .field("client", &self.client)
391 .field("max_retries", &self.max_retries)
392 .field("base_backoff", &self.base_backoff)
393 .finish()
394 }
395}
396
397impl OpenAiCompatibleClient {
398 pub fn new(
401 base_url: impl Into<String>,
402 api_key: Option<String>,
403 model: impl Into<String>,
404 ) -> Self {
405 Self {
406 base_url: base_url.into().trim_end_matches('/').to_string(),
407 api_key,
408 model: model.into(),
409 client: reqwest::Client::new(),
410 max_retries: DEFAULT_MAX_RETRIES,
411 base_backoff: DEFAULT_BASE_BACKOFF,
412 }
413 }
414
415 pub fn from_env(
421 base_url: impl Into<String>,
422 api_key_env: &str,
423 model: impl Into<String>,
424 ) -> Self {
425 let api_key = std::env::var(api_key_env).ok();
426 if api_key.is_none() {
427 tracing::warn!(
428 env_var = api_key_env,
429 "API key environment variable is not set; sending requests unauthenticated"
430 );
431 }
432 Self::new(base_url, api_key, model)
433 }
434
435 pub fn with_http_client(mut self, client: reqwest::Client) -> Self {
437 self.client = client;
438 self
439 }
440
441 pub fn with_retries(mut self, max_retries: u32) -> Self {
444 self.max_retries = max_retries;
445 self
446 }
447
448 pub fn with_backoff(mut self, base_backoff: Duration) -> Self {
451 self.base_backoff = base_backoff;
452 self
453 }
454
455 pub fn model(&self) -> &str {
457 &self.model
458 }
459
460 fn endpoint(&self) -> String {
461 format!("{}/chat/completions", self.base_url)
462 }
463
464 async fn send_with_retries(
468 &self,
469 build: impl Fn() -> reqwest::RequestBuilder,
470 ) -> Result<reqwest::Response> {
471 let mut attempt: u32 = 0;
472 loop {
473 match self.try_send(build()).await {
474 Ok(response) => return Ok(response),
475 Err(AttemptError::Fatal(e)) => return Err(e),
476 Err(AttemptError::Retryable { error, retry_after })
477 if attempt < self.max_retries =>
478 {
479 let mut delay = backoff_delay(self.base_backoff, attempt);
480 if let Some(floor) = retry_after {
481 delay = delay.max(floor);
484 }
485 tracing::warn!(
486 url = %self.base_url,
487 attempt = attempt + 1,
488 max_retries = self.max_retries,
489 backoff_ms = delay.as_millis() as u64,
490 error = %error,
491 "chat completions attempt failed; retrying"
492 );
493 tokio::time::sleep(delay).await;
494 attempt += 1;
495 }
496 Err(AttemptError::Retryable { error, .. }) => return Err(error),
497 }
498 }
499 }
500
501 async fn try_send(
504 &self,
505 request: reqwest::RequestBuilder,
506 ) -> std::result::Result<reqwest::Response, AttemptError> {
507 let response = request.send().await.map_err(|e| {
508 let err = RustyError::Llm(format!("request to {} failed: {e}", self.base_url));
509 if e.is_timeout() || e.is_connect() {
510 AttemptError::Retryable {
511 error: err,
512 retry_after: None,
513 }
514 } else {
515 AttemptError::Fatal(err)
516 }
517 })?;
518
519 let status = response.status();
520 if status.is_success() {
521 return Ok(response);
522 }
523 let retry_after = response
524 .headers()
525 .get(reqwest::header::RETRY_AFTER)
526 .and_then(|v| v.to_str().ok())
527 .and_then(|s| s.trim().parse::<u64>().ok())
528 .map(Duration::from_secs);
529 let body = response.text().await.unwrap_or_default();
530 let err = RustyError::Llm(format!(
531 "chat completions returned {status}: {}",
532 truncate_body(&body)
533 ));
534 let retryable = status.is_server_error()
537 || status == reqwest::StatusCode::REQUEST_TIMEOUT
538 || status == reqwest::StatusCode::TOO_MANY_REQUESTS;
539 Err(if retryable {
540 AttemptError::Retryable {
541 error: err,
542 retry_after,
543 }
544 } else {
545 AttemptError::Fatal(err)
546 })
547 }
548}
549
550#[derive(Debug)]
553enum AttemptError {
554 Retryable {
557 error: RustyError,
558 retry_after: Option<Duration>,
559 },
560 Fatal(RustyError),
562}
563
564#[derive(Deserialize)]
566struct WireChoice {
567 message: ChatMessage,
568}
569
570#[derive(Deserialize)]
572struct WireResponse {
573 choices: Vec<WireChoice>,
574 #[serde(default)]
575 model: Option<String>,
576 #[serde(default)]
577 usage: Option<Usage>,
578}
579
580#[derive(Deserialize)]
582struct WireStreamChunk {
583 #[serde(default)]
584 choices: Vec<WireStreamChoice>,
585 #[serde(default)]
586 model: Option<String>,
587 #[serde(default)]
588 usage: Option<Usage>,
589}
590
591#[derive(Deserialize)]
600struct WireStreamChoice {
601 delta: WireStreamDelta,
602}
603
604#[derive(Deserialize)]
606struct WireStreamDelta {
607 #[serde(default)]
608 content: Option<String>,
609 #[serde(default)]
610 tool_calls: Option<Vec<WireToolCallDelta>>,
611}
612
613#[derive(Deserialize)]
616struct WireToolCallDelta {
617 #[serde(default)]
618 index: usize,
619 #[serde(default)]
620 id: Option<String>,
621 #[serde(default)]
622 function: Option<WireFunctionDelta>,
623}
624
625#[derive(Deserialize)]
627struct WireFunctionDelta {
628 #[serde(default)]
629 name: Option<String>,
630 #[serde(default)]
631 arguments: Option<String>,
632}
633
634#[derive(Default)]
636struct StreamAccumulator {
637 content: String,
638 tool_calls: Vec<ToolCallAccumulator>,
639 model: Option<String>,
640 usage: Option<Usage>,
641}
642
643#[derive(Default)]
645struct ToolCallAccumulator {
646 id: String,
647 name: String,
648 arguments: String,
649}
650
651impl StreamAccumulator {
652 fn into_response(self) -> Result<ChatResponse> {
653 let mut tool_calls = Vec::with_capacity(self.tool_calls.len());
654 for (index, acc) in self.tool_calls.into_iter().enumerate() {
655 let arguments = if acc.arguments.trim().is_empty() {
656 Value::Object(serde_json::Map::new())
660 } else {
661 serde_json::from_str(&acc.arguments).map_err(|e| {
662 RustyError::Llm(format!(
663 "malformed tool-call arguments in stream (index {index}): {e}"
664 ))
665 })?
666 };
667 tool_calls.push(ToolCall::new(acc.id, acc.name, arguments));
668 }
669 let content = if self.content.is_empty() {
670 None
671 } else {
672 Some(self.content)
673 };
674 Ok(ChatResponse {
675 message: ChatMessage {
676 role: Role::Assistant,
677 content,
678 tool_calls,
679 tool_call_id: None,
680 name: None,
681 },
682 model: self.model,
683 usage: self.usage,
684 })
685 }
686}
687
688#[derive(Default)]
700struct SseDecoder {
701 pending: Vec<u8>,
703 buf: String,
705 data_lines: Vec<String>,
707 started: bool,
709}
710
711impl SseDecoder {
712 fn new() -> Self {
713 Self::default()
714 }
715
716 fn feed_bytes(&mut self, bytes: &[u8]) -> Vec<String> {
724 self.pending.extend_from_slice(bytes);
725 if !self.started {
726 self.started = true;
727 if self.pending.starts_with(&[0xEF, 0xBB, 0xBF]) {
731 self.pending.drain(..3);
732 }
733 }
734 let mut text = String::new();
735 let mut consumed = 0;
736 while consumed < self.pending.len() {
737 match std::str::from_utf8(&self.pending[consumed..]) {
738 Ok(s) => {
739 text.push_str(s);
740 consumed = self.pending.len();
741 }
742 Err(e) => {
743 let end = consumed + e.valid_up_to();
744 text.push_str(
745 std::str::from_utf8(&self.pending[consumed..end])
746 .expect("bytes up to valid_up_to are valid UTF-8"),
747 );
748 match e.error_len() {
749 Some(bad) => {
750 text.push('\u{FFFD}');
751 consumed = end + bad;
752 }
753 None => {
755 consumed = end;
756 break;
757 }
758 }
759 }
760 }
761 }
762 self.pending.drain(..consumed);
763 self.feed(&text)
764 }
765
766 fn feed(&mut self, chunk: &str) -> Vec<String> {
769 self.buf.push_str(chunk);
770 let mut events = Vec::new();
771 while let Some(pos) = self.buf.find('\n') {
772 let line: String = self.buf.drain(..=pos).collect();
773 if let Some(payload) = self.process_line(line.trim_end_matches(['\n', '\r'])) {
774 events.push(payload);
775 }
776 }
777 events
778 }
779
780 fn finish(&mut self) -> Vec<String> {
784 let mut events = Vec::new();
785 if !self.pending.is_empty() {
786 let tail = String::from_utf8_lossy(&self.pending).into_owned();
788 self.pending.clear();
789 events.extend(self.feed(&tail));
790 }
791 if !self.buf.is_empty() {
792 let line = std::mem::take(&mut self.buf);
793 if let Some(payload) = self.process_line(line.trim_end_matches('\r')) {
794 events.push(payload);
795 }
796 }
797 if !self.data_lines.is_empty() {
798 events.push(self.data_lines.join("\n"));
799 self.data_lines.clear();
800 }
801 events
802 }
803
804 fn process_line(&mut self, line: &str) -> Option<String> {
807 if line.is_empty() {
808 if self.data_lines.is_empty() {
809 return None;
810 }
811 let payload = self.data_lines.join("\n");
812 self.data_lines.clear();
813 return Some(payload);
814 }
815 if line.starts_with(':') {
816 return None; }
818 if let Some(data) = line.strip_prefix("data:") {
819 self.data_lines
821 .push(data.strip_prefix(' ').unwrap_or(data).to_owned());
822 } else if line == "data" {
823 self.data_lines.push(String::new());
825 }
826 None
827 }
828}
829
830fn handle_sse_payload(
839 payload: &str,
840 acc: &mut StreamAccumulator,
841 on_token: &mut (dyn FnMut(TokenChunk) + Send),
842) -> Result<bool> {
843 let trimmed = payload.trim();
844 if trimmed == "[DONE]" {
845 return Ok(true);
846 }
847 if trimmed.is_empty() {
848 return Ok(false);
849 }
850
851 let value: Value = serde_json::from_str(trimmed)
852 .map_err(|e| RustyError::Llm(format!("malformed stream chunk: {e}")))?;
853 let chunk: WireStreamChunk = serde_json::from_value(value.clone())
854 .map_err(|e| RustyError::Llm(format!("malformed stream chunk: {e}")))?;
855
856 if chunk.model.is_some() {
857 acc.model = chunk.model;
858 }
859 if chunk.usage.is_some() {
860 acc.usage = chunk.usage;
861 }
862
863 if let Some(choice) = chunk.choices.into_iter().next() {
864 let delta = choice.delta;
865 if let Some(content) = delta.content {
866 if !content.is_empty() {
867 acc.content.push_str(&content);
868 on_token(TokenChunk {
869 delta: content,
870 finish: false,
871 raw: Some(value),
872 });
873 }
874 }
875 if let Some(calls) = delta.tool_calls {
876 for call in calls {
877 if acc.tool_calls.len() <= call.index {
878 acc.tool_calls
879 .resize_with(call.index + 1, ToolCallAccumulator::default);
880 }
881 let slot = &mut acc.tool_calls[call.index];
882 if let Some(id) = call.id {
883 slot.id.push_str(&id);
884 }
885 if let Some(function) = call.function {
886 if let Some(name) = function.name {
887 slot.name.push_str(&name);
888 }
889 if let Some(arguments) = function.arguments {
890 slot.arguments.push_str(&arguments);
891 }
892 }
893 }
894 }
895 }
896 Ok(false)
897}
898
899#[async_trait]
900impl ChatModel for OpenAiCompatibleClient {
901 async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
902 let mut body = json!({
903 "model": self.model,
904 "messages": messages,
905 });
906 if !tools.is_empty() {
907 body["tools"] = Value::Array(tools.to_vec());
908 }
909
910 let response = self
911 .send_with_retries(|| {
912 let mut request = self.client.post(self.endpoint()).json(&body);
913 if let Some(key) = &self.api_key {
914 request = request.bearer_auth(key);
915 }
916 request
917 })
918 .await?;
919
920 let wire: WireResponse = response
921 .json()
922 .await
923 .map_err(|e| RustyError::Llm(format!("malformed chat completions response: {e}")))?;
924
925 let choice = wire
926 .choices
927 .into_iter()
928 .next()
929 .ok_or_else(|| RustyError::Llm("chat completions returned zero choices".into()))?;
930
931 Ok(ChatResponse {
932 message: choice.message,
933 model: wire.model,
934 usage: wire.usage,
935 })
936 }
937
938 async fn chat_stream(
939 &self,
940 messages: &[ChatMessage],
941 tools: &[Value],
942 on_token: &mut (dyn FnMut(TokenChunk) + Send),
943 ) -> Result<ChatResponse> {
944 let mut body = json!({
945 "model": self.model,
946 "messages": messages,
947 "stream": true,
948 "stream_options": {"include_usage": true},
951 });
952 if !tools.is_empty() {
953 body["tools"] = Value::Array(tools.to_vec());
954 }
955
956 let mut response = self
957 .send_with_retries(|| {
958 let mut request = self.client.post(self.endpoint()).json(&body);
959 if let Some(key) = &self.api_key {
960 request = request.bearer_auth(key);
961 }
962 request
963 })
964 .await?;
965
966 let mut decoder = SseDecoder::new();
970 let mut acc = StreamAccumulator::default();
971 let mut done = false;
972
973 while !done {
974 let bytes = match response.chunk().await {
975 Ok(Some(bytes)) => bytes,
976 Ok(None) => break, Err(e) => {
978 return Err(RustyError::Llm(format!(
979 "stream read from {} failed: {e}",
980 self.base_url
981 )))
982 }
983 };
984 for payload in decoder.feed_bytes(&bytes) {
985 if handle_sse_payload(&payload, &mut acc, on_token)? {
986 done = true;
987 break;
988 }
989 }
990 }
991 if !done {
992 for payload in decoder.finish() {
994 if handle_sse_payload(&payload, &mut acc, on_token)? {
995 break;
996 }
997 }
998 }
999
1000 on_token(TokenChunk {
1001 delta: String::new(),
1002 finish: true,
1003 raw: None,
1004 });
1005 acc.into_response()
1006 }
1007}
1008
1009#[cfg(test)]
1010mod tests {
1011 use super::*;
1012
1013 #[test]
1014 fn tool_call_wire_roundtrip() {
1015 let call = ToolCall::new("call_1", "search", json!({"q": "rust"}));
1016 let serialized = serde_json::to_value(&call).unwrap();
1017 assert_eq!(
1018 serialized,
1019 json!({
1020 "id": "call_1",
1021 "type": "function",
1022 "function": {"name": "search", "arguments": "{\"q\":\"rust\"}"}
1023 })
1024 );
1025 let back: ToolCall = serde_json::from_value(serialized).unwrap();
1026 assert_eq!(back, call);
1027 }
1028
1029 #[test]
1030 fn message_builders() {
1031 let m = ChatMessage::tool_result("call_1", "42");
1032 assert_eq!(m.role, Role::Tool);
1033 assert_eq!(m.tool_call_id.as_deref(), Some("call_1"));
1034
1035 let m = ChatMessage::assistant_tool_calls(vec![ToolCall::new("c", "t", json!({}))]);
1036 assert!(m.has_tool_calls());
1037 assert_eq!(m.content, None);
1038
1039 assert_eq!(
1041 serde_json::to_value(Role::Assistant).unwrap(),
1042 json!("assistant")
1043 );
1044 }
1045
1046 #[test]
1047 fn sse_decoder_handles_multi_chunk_delivery() {
1048 let mut decoder = SseDecoder::new();
1049 assert!(decoder.feed("data: {\"hel").is_empty());
1053 assert_eq!(
1054 decoder.feed("lo\"}\n: heartbeat\n\nda"),
1055 vec!["{\"hello\"}".to_string()]
1056 );
1057 assert_eq!(decoder.feed("ta: world\r\n\r\n"), vec!["world".to_string()]);
1058 }
1059
1060 #[test]
1061 fn sse_decoder_joins_multi_line_data_and_flushes_trailing_event() {
1062 let mut decoder = SseDecoder::new();
1063 assert_eq!(
1065 decoder.feed("data: a\ndata: b\n\n"),
1066 vec!["a\nb".to_string()]
1067 );
1068 assert!(decoder.feed("data: tail").is_empty());
1070 assert_eq!(decoder.finish(), vec!["tail".to_string()]);
1071 assert!(SseDecoder::new().process_line("").is_none());
1073 }
1074
1075 #[test]
1076 fn sse_done_sentinel_terminates_and_content_deltas_accumulate() {
1077 let mut acc = StreamAccumulator::default();
1078 let mut deltas: Vec<String> = Vec::new();
1079 let mut on_token = |chunk: TokenChunk| {
1080 assert!(!chunk.finish);
1081 assert!(chunk.raw.is_some(), "wire chunks carry the raw JSON");
1082 deltas.push(chunk.delta);
1083 };
1084
1085 let done = handle_sse_payload(
1086 r#"{"choices":[{"delta":{"content":"Hel"},"finish_reason":null}]}"#,
1087 &mut acc,
1088 &mut on_token,
1089 )
1090 .unwrap();
1091 assert!(!done);
1092 let done = handle_sse_payload(
1093 r#"{"choices":[{"delta":{"content":"lo"},"finish_reason":"stop"}],
1094 "model":"gpt-x","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3}}"#,
1095 &mut acc,
1096 &mut on_token,
1097 )
1098 .unwrap();
1099 assert!(!done);
1100 assert!(handle_sse_payload("[DONE]", &mut acc, &mut on_token).unwrap());
1102
1103 assert_eq!(deltas, ["Hel", "lo"]);
1104 let response = acc.into_response().unwrap();
1105 assert_eq!(response.message.content.as_deref(), Some("Hello"));
1106 assert_eq!(response.model.as_deref(), Some("gpt-x"));
1107 assert_eq!(response.usage.unwrap().total_tokens, 3);
1108 }
1109
1110 #[test]
1111 fn sse_stream_accumulates_tool_call_deltas() {
1112 let mut acc = StreamAccumulator::default();
1113 let mut on_token = |_chunk: TokenChunk| {};
1114 for payload in [
1116 r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"search","arguments":"{\"q\":"}}]}}]}"#,
1117 r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"rust\"}"}}]}}]}"#,
1118 ] {
1119 assert!(!handle_sse_payload(payload, &mut acc, &mut on_token).unwrap());
1120 }
1121 let response = acc.into_response().unwrap();
1122 assert_eq!(
1123 response.message.tool_calls,
1124 vec![ToolCall::new("call_1", "search", json!({"q": "rust"}))]
1125 );
1126 assert_eq!(response.message.content, None);
1127 }
1128
1129 struct NonStreamingMock;
1131
1132 #[async_trait]
1133 impl ChatModel for NonStreamingMock {
1134 async fn chat(&self, _messages: &[ChatMessage], _tools: &[Value]) -> Result<ChatResponse> {
1135 Ok(ChatResponse {
1136 message: ChatMessage::assistant("full answer"),
1137 model: Some("mock".into()),
1138 usage: None,
1139 })
1140 }
1141 }
1142
1143 #[tokio::test]
1144 async fn chat_stream_default_falls_back_to_single_chunk() {
1145 let model = NonStreamingMock;
1146 let mut chunks: Vec<TokenChunk> = Vec::new();
1147 let response = model
1148 .chat_stream(&[], &[], &mut |chunk| chunks.push(chunk))
1149 .await
1150 .unwrap();
1151
1152 assert_eq!(chunks.len(), 1);
1154 assert_eq!(chunks[0].delta, "full answer");
1155 assert!(chunks[0].finish);
1156 assert!(chunks[0].raw.is_none());
1157 assert_eq!(
1158 response.message.content.as_deref(),
1159 Some("full answer"),
1160 "the fallback returns the chat() response unchanged"
1161 );
1162 }
1163
1164 #[test]
1165 fn feed_bytes_preserves_multibyte_chars_split_across_chunks() {
1166 let mut decoder = SseDecoder::new();
1167 let mut first = b"data: {\"c\":\"".to_vec();
1169 first.push(0xE4);
1170 assert!(decoder.feed_bytes(&first).is_empty());
1171 assert!(decoder.feed_bytes(&[0xBD, 0xA0]).is_empty());
1172 let events = decoder.feed_bytes(b"\"}\n\n");
1173 assert_eq!(events, vec!["{\"c\":\"你\"}".to_string()]);
1174 }
1175
1176 #[test]
1177 fn sse_strips_leading_bom_and_handles_bare_data_line() {
1178 let mut decoder = SseDecoder::new();
1179 assert!(decoder.feed_bytes(&[0xEF, 0xBB, 0xBF]).is_empty());
1180 assert_eq!(decoder.feed("data\n\n"), vec![String::new()]);
1182 assert_eq!(decoder.feed("data: x\n\n"), vec!["x".to_string()]);
1183 }
1184
1185 #[test]
1186 fn empty_payload_is_skipped_not_parsed() {
1187 let mut acc = StreamAccumulator::default();
1188 let mut on_token = |_chunk: TokenChunk| {};
1189 assert!(!handle_sse_payload("", &mut acc, &mut on_token).unwrap());
1192 }
1193
1194 #[test]
1195 fn client_debug_redacts_the_api_key() {
1196 let client =
1197 OpenAiCompatibleClient::new("http://localhost", Some("sk-secret-123".into()), "m");
1198 let debug = format!("{client:?}");
1199 assert!(!debug.contains("sk-secret-123"), "got: {debug}");
1200 assert!(debug.contains("***"), "got: {debug}");
1201
1202 let no_key = OpenAiCompatibleClient::new("http://localhost", None, "m");
1203 assert!(format!("{no_key:?}").contains("api_key: None"));
1204 }
1205
1206 fn start_http_mock(
1211 handler: impl Fn(usize) -> (u16, String) + Send + Sync + 'static,
1212 ) -> (
1213 std::net::SocketAddr,
1214 std::sync::Arc<std::sync::atomic::AtomicUsize>,
1215 ) {
1216 use std::sync::atomic::{AtomicUsize, Ordering};
1217 use std::sync::Arc as StdArc;
1218 use tokio::io::{AsyncReadExt, AsyncWriteExt};
1219
1220 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
1221 listener.set_nonblocking(true).unwrap();
1222 let addr = listener.local_addr().unwrap();
1223 let attempts = StdArc::new(AtomicUsize::new(0));
1224 let attempts2 = attempts.clone();
1225 let handler = StdArc::new(handler);
1226 tokio::spawn(async move {
1227 let listener = tokio::net::TcpListener::from_std(listener).unwrap();
1228 loop {
1229 let Ok((mut stream, _)) = listener.accept().await else {
1230 continue;
1231 };
1232 let attempts = attempts2.clone();
1233 let handler = handler.clone();
1234 tokio::spawn(async move {
1235 let mut buf: Vec<u8> = Vec::new();
1238 let mut chunk = [0u8; 4096];
1239 loop {
1240 if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
1241 let headers = String::from_utf8_lossy(&buf[..pos]).to_lowercase();
1242 let len: usize = headers
1243 .lines()
1244 .find_map(|l| l.strip_prefix("content-length:"))
1245 .and_then(|v| v.trim().parse().ok())
1246 .unwrap_or(0);
1247 if buf.len() >= pos + 4 + len {
1248 break;
1249 }
1250 }
1251 let Ok(n) = stream.read(&mut chunk).await else {
1252 return;
1253 };
1254 if n == 0 {
1255 return;
1256 }
1257 buf.extend_from_slice(&chunk[..n]);
1258 }
1259 let n = attempts.fetch_add(1, Ordering::SeqCst) + 1;
1260 let (status, body) = handler(n);
1261 let reason = match status {
1262 200 => "OK",
1263 400 => "Bad Request",
1264 429 => "Too Many Requests",
1265 500 => "Internal Server Error",
1266 _ => "Status",
1267 };
1268 let response = format!(
1269 "HTTP/1.1 {status} {reason}\r\n\
1270 content-type: application/json\r\n\
1271 content-length: {}\r\n\
1272 connection: close\r\n\
1273 \r\n\
1274 {body}",
1275 body.len()
1276 );
1277 let _ = stream.write_all(response.as_bytes()).await;
1278 });
1279 }
1280 });
1281 (addr, attempts)
1282 }
1283
1284 #[tokio::test]
1285 async fn chat_retries_transient_5xx_then_succeeds() {
1286 let (addr, attempts) = start_http_mock(|n| {
1287 if n < 3 {
1288 (500, r#"{"error":"overloaded"}"#.to_string())
1289 } else {
1290 (
1291 200,
1292 r#"{"choices":[{"message":{"role":"assistant","content":"hi after retry"}}]}"#
1293 .to_string(),
1294 )
1295 }
1296 });
1297 let client = OpenAiCompatibleClient::new(format!("http://{addr}"), None, "m")
1298 .with_backoff(Duration::from_millis(1));
1299 let response = client.chat(&[ChatMessage::user("hi")], &[]).await.unwrap();
1300 assert_eq!(response.message.content.as_deref(), Some("hi after retry"));
1301 assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 3);
1303 }
1304
1305 #[tokio::test]
1306 async fn chat_does_not_retry_fatal_4xx() {
1307 let (addr, attempts) =
1308 start_http_mock(|_n| (400, r#"{"error":"bad request"}"#.to_string()));
1309 let client = OpenAiCompatibleClient::new(format!("http://{addr}"), None, "m")
1310 .with_backoff(Duration::from_millis(1));
1311 let err = client
1312 .chat(&[ChatMessage::user("hi")], &[])
1313 .await
1314 .unwrap_err();
1315 assert!(err.to_string().contains("400"), "got: {err}");
1316 assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);
1317 }
1318
1319 #[tokio::test]
1320 async fn chat_gives_up_after_retries_exhausted() {
1321 let (addr, attempts) =
1322 start_http_mock(|_n| (429, r#"{"error":"rate limited"}"#.to_string()));
1323 let client = OpenAiCompatibleClient::new(format!("http://{addr}"), None, "m")
1324 .with_retries(1)
1325 .with_backoff(Duration::from_millis(1));
1326 let err = client
1327 .chat(&[ChatMessage::user("hi")], &[])
1328 .await
1329 .unwrap_err();
1330 assert!(err.to_string().contains("429"), "got: {err}");
1331 assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 2);
1333 }
1334}