use std::time::Duration;
use async_trait::async_trait;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{json, Value};
use crate::error::{Result, RustyError};
pub(crate) const ERROR_BODY_MAX_CHARS: usize = 512;
pub(crate) fn truncate_body(body: &str) -> String {
body.chars().take(ERROR_BODY_MAX_CHARS).collect()
}
pub(crate) fn backoff_delay(base: Duration, attempt: u32) -> Duration {
let base = base.saturating_mul(2u32.saturating_pow(attempt.min(6)));
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.subsec_nanos())
.unwrap_or(0);
let jitter = 0.5 + f64::from(nanos % 1000) / 1000.0; let jittered = (base.as_nanos() as f64 * jitter) as u128;
Duration::from_nanos(jittered.min(u64::MAX as u128) as u64)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Role {
System,
User,
Assistant,
Tool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ChatMessage {
pub role: Role,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tool_calls: Vec<ToolCall>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tool_call_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
}
impl ChatMessage {
pub fn system(content: impl Into<String>) -> Self {
Self {
role: Role::System,
content: Some(content.into()),
tool_calls: Vec::new(),
tool_call_id: None,
name: None,
}
}
pub fn user(content: impl Into<String>) -> Self {
Self {
role: Role::User,
content: Some(content.into()),
tool_calls: Vec::new(),
tool_call_id: None,
name: None,
}
}
pub fn assistant(content: impl Into<String>) -> Self {
Self {
role: Role::Assistant,
content: Some(content.into()),
tool_calls: Vec::new(),
tool_call_id: None,
name: None,
}
}
pub fn assistant_tool_calls(tool_calls: Vec<ToolCall>) -> Self {
Self {
role: Role::Assistant,
content: None,
tool_calls,
tool_call_id: None,
name: None,
}
}
pub fn tool_result(tool_call_id: impl Into<String>, content: impl Into<String>) -> Self {
Self {
role: Role::Tool,
content: Some(content.into()),
tool_calls: Vec::new(),
tool_call_id: Some(tool_call_id.into()),
name: None,
}
}
pub fn has_tool_calls(&self) -> bool {
!self.tool_calls.is_empty()
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ToolCall {
pub id: String,
pub name: String,
pub arguments: Value,
}
impl ToolCall {
pub fn new(id: impl Into<String>, name: impl Into<String>, arguments: Value) -> Self {
Self {
id: id.into(),
name: name.into(),
arguments,
}
}
}
impl Serialize for ToolCall {
fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
json!({
"id": self.id,
"type": "function",
"function": {
"name": self.name,
"arguments": self.arguments.to_string(),
}
})
.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for ToolCall {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
#[derive(Deserialize)]
struct FnArgs {
name: String,
arguments: Value,
}
#[derive(Deserialize)]
struct Wire {
id: String,
function: FnArgs,
}
let wire = Wire::deserialize(deserializer)?;
let arguments = match wire.function.arguments {
Value::String(s) => serde_json::from_str(&s).map_err(serde::de::Error::custom)?,
other => other,
};
Ok(ToolCall {
id: wire.id,
name: wire.function.name,
arguments,
})
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Usage {
#[serde(default)]
pub prompt_tokens: u64,
#[serde(default)]
pub completion_tokens: u64,
#[serde(default)]
pub total_tokens: u64,
}
#[derive(Debug, Clone)]
pub struct ChatResponse {
pub message: ChatMessage,
pub model: Option<String>,
pub usage: Option<Usage>,
}
#[derive(Debug, Clone)]
pub struct TokenChunk {
pub delta: String,
pub finish: bool,
pub raw: Option<Value>,
}
#[async_trait]
pub trait ChatModel: Send + Sync {
async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse>;
fn effect(&self) -> crate::record::Effect {
crate::record::Effect::NonIdempotent
}
async fn chat_stream(
&self,
messages: &[ChatMessage],
tools: &[Value],
on_token: &mut (dyn FnMut(TokenChunk) + Send),
) -> Result<ChatResponse> {
let response = self.chat(messages, tools).await?;
on_token(TokenChunk {
delta: response.message.content.clone().unwrap_or_default(),
finish: true,
raw: None,
});
Ok(response)
}
}
pub const DEFAULT_MAX_RETRIES: u32 = 2;
pub const DEFAULT_BASE_BACKOFF: Duration = Duration::from_millis(100);
#[derive(Clone)]
pub struct OpenAiCompatibleClient {
base_url: String,
api_key: Option<String>,
model: String,
client: reqwest::Client,
max_retries: u32,
base_backoff: Duration,
}
impl std::fmt::Debug for OpenAiCompatibleClient {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OpenAiCompatibleClient")
.field("base_url", &self.base_url)
.field("api_key", &self.api_key.as_ref().map(|_| "***"))
.field("model", &self.model)
.field("client", &self.client)
.field("max_retries", &self.max_retries)
.field("base_backoff", &self.base_backoff)
.finish()
}
}
impl OpenAiCompatibleClient {
pub fn new(
base_url: impl Into<String>,
api_key: Option<String>,
model: impl Into<String>,
) -> Self {
Self {
base_url: base_url.into().trim_end_matches('/').to_string(),
api_key,
model: model.into(),
client: reqwest::Client::new(),
max_retries: DEFAULT_MAX_RETRIES,
base_backoff: DEFAULT_BASE_BACKOFF,
}
}
pub fn from_env(
base_url: impl Into<String>,
api_key_env: &str,
model: impl Into<String>,
) -> Self {
let api_key = std::env::var(api_key_env).ok();
if api_key.is_none() {
tracing::warn!(
env_var = api_key_env,
"API key environment variable is not set; sending requests unauthenticated"
);
}
Self::new(base_url, api_key, model)
}
pub fn with_http_client(mut self, client: reqwest::Client) -> Self {
self.client = client;
self
}
pub fn with_retries(mut self, max_retries: u32) -> Self {
self.max_retries = max_retries;
self
}
pub fn with_backoff(mut self, base_backoff: Duration) -> Self {
self.base_backoff = base_backoff;
self
}
pub fn model(&self) -> &str {
&self.model
}
fn endpoint(&self) -> String {
format!("{}/chat/completions", self.base_url)
}
async fn send_with_retries(
&self,
build: impl Fn() -> reqwest::RequestBuilder,
) -> Result<reqwest::Response> {
let mut attempt: u32 = 0;
loop {
match self.try_send(build()).await {
Ok(response) => return Ok(response),
Err(AttemptError::Fatal(e)) => return Err(e),
Err(AttemptError::Retryable { error, retry_after })
if attempt < self.max_retries =>
{
let mut delay = backoff_delay(self.base_backoff, attempt);
if let Some(floor) = retry_after {
delay = delay.max(floor);
}
tracing::warn!(
url = %self.base_url,
attempt = attempt + 1,
max_retries = self.max_retries,
backoff_ms = delay.as_millis() as u64,
error = %error,
"chat completions attempt failed; retrying"
);
tokio::time::sleep(delay).await;
attempt += 1;
}
Err(AttemptError::Retryable { error, .. }) => return Err(error),
}
}
}
async fn try_send(
&self,
request: reqwest::RequestBuilder,
) -> std::result::Result<reqwest::Response, AttemptError> {
let response = request.send().await.map_err(|e| {
let err = RustyError::Llm(format!("request to {} failed: {e}", self.base_url));
if e.is_timeout() || e.is_connect() {
AttemptError::Retryable {
error: err,
retry_after: None,
}
} else {
AttemptError::Fatal(err)
}
})?;
let status = response.status();
if status.is_success() {
return Ok(response);
}
let retry_after = response
.headers()
.get(reqwest::header::RETRY_AFTER)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.trim().parse::<u64>().ok())
.map(Duration::from_secs);
let body = response.text().await.unwrap_or_default();
let err = RustyError::Llm(format!(
"chat completions returned {status}: {}",
truncate_body(&body)
));
let retryable = status.is_server_error()
|| status == reqwest::StatusCode::REQUEST_TIMEOUT
|| status == reqwest::StatusCode::TOO_MANY_REQUESTS;
Err(if retryable {
AttemptError::Retryable {
error: err,
retry_after,
}
} else {
AttemptError::Fatal(err)
})
}
}
#[derive(Debug)]
enum AttemptError {
Retryable {
error: RustyError,
retry_after: Option<Duration>,
},
Fatal(RustyError),
}
#[derive(Deserialize)]
struct WireChoice {
message: ChatMessage,
}
#[derive(Deserialize)]
struct WireResponse {
choices: Vec<WireChoice>,
#[serde(default)]
model: Option<String>,
#[serde(default)]
usage: Option<Usage>,
}
#[derive(Deserialize)]
struct WireStreamChunk {
#[serde(default)]
choices: Vec<WireStreamChoice>,
#[serde(default)]
model: Option<String>,
#[serde(default)]
usage: Option<Usage>,
}
#[derive(Deserialize)]
struct WireStreamChoice {
delta: WireStreamDelta,
}
#[derive(Deserialize)]
struct WireStreamDelta {
#[serde(default)]
content: Option<String>,
#[serde(default)]
tool_calls: Option<Vec<WireToolCallDelta>>,
}
#[derive(Deserialize)]
struct WireToolCallDelta {
#[serde(default)]
index: usize,
#[serde(default)]
id: Option<String>,
#[serde(default)]
function: Option<WireFunctionDelta>,
}
#[derive(Deserialize)]
struct WireFunctionDelta {
#[serde(default)]
name: Option<String>,
#[serde(default)]
arguments: Option<String>,
}
#[derive(Default)]
struct StreamAccumulator {
content: String,
tool_calls: Vec<ToolCallAccumulator>,
model: Option<String>,
usage: Option<Usage>,
}
#[derive(Default)]
struct ToolCallAccumulator {
id: String,
name: String,
arguments: String,
}
impl StreamAccumulator {
fn into_response(self) -> Result<ChatResponse> {
let mut tool_calls = Vec::with_capacity(self.tool_calls.len());
for (index, acc) in self.tool_calls.into_iter().enumerate() {
let arguments = if acc.arguments.trim().is_empty() {
Value::Object(serde_json::Map::new())
} else {
serde_json::from_str(&acc.arguments).map_err(|e| {
RustyError::Llm(format!(
"malformed tool-call arguments in stream (index {index}): {e}"
))
})?
};
tool_calls.push(ToolCall::new(acc.id, acc.name, arguments));
}
let content = if self.content.is_empty() {
None
} else {
Some(self.content)
};
Ok(ChatResponse {
message: ChatMessage {
role: Role::Assistant,
content,
tool_calls,
tool_call_id: None,
name: None,
},
model: self.model,
usage: self.usage,
})
}
}
#[derive(Default)]
struct SseDecoder {
pending: Vec<u8>,
buf: String,
data_lines: Vec<String>,
started: bool,
}
impl SseDecoder {
fn new() -> Self {
Self::default()
}
fn feed_bytes(&mut self, bytes: &[u8]) -> Vec<String> {
self.pending.extend_from_slice(bytes);
if !self.started {
self.started = true;
if self.pending.starts_with(&[0xEF, 0xBB, 0xBF]) {
self.pending.drain(..3);
}
}
let mut text = String::new();
let mut consumed = 0;
while consumed < self.pending.len() {
match std::str::from_utf8(&self.pending[consumed..]) {
Ok(s) => {
text.push_str(s);
consumed = self.pending.len();
}
Err(e) => {
let end = consumed + e.valid_up_to();
text.push_str(
std::str::from_utf8(&self.pending[consumed..end])
.expect("bytes up to valid_up_to are valid UTF-8"),
);
match e.error_len() {
Some(bad) => {
text.push('\u{FFFD}');
consumed = end + bad;
}
None => {
consumed = end;
break;
}
}
}
}
}
self.pending.drain(..consumed);
self.feed(&text)
}
fn feed(&mut self, chunk: &str) -> Vec<String> {
self.buf.push_str(chunk);
let mut events = Vec::new();
while let Some(pos) = self.buf.find('\n') {
let line: String = self.buf.drain(..=pos).collect();
if let Some(payload) = self.process_line(line.trim_end_matches(['\n', '\r'])) {
events.push(payload);
}
}
events
}
fn finish(&mut self) -> Vec<String> {
let mut events = Vec::new();
if !self.pending.is_empty() {
let tail = String::from_utf8_lossy(&self.pending).into_owned();
self.pending.clear();
events.extend(self.feed(&tail));
}
if !self.buf.is_empty() {
let line = std::mem::take(&mut self.buf);
if let Some(payload) = self.process_line(line.trim_end_matches('\r')) {
events.push(payload);
}
}
if !self.data_lines.is_empty() {
events.push(self.data_lines.join("\n"));
self.data_lines.clear();
}
events
}
fn process_line(&mut self, line: &str) -> Option<String> {
if line.is_empty() {
if self.data_lines.is_empty() {
return None;
}
let payload = self.data_lines.join("\n");
self.data_lines.clear();
return Some(payload);
}
if line.starts_with(':') {
return None; }
if let Some(data) = line.strip_prefix("data:") {
self.data_lines
.push(data.strip_prefix(' ').unwrap_or(data).to_owned());
} else if line == "data" {
self.data_lines.push(String::new());
}
None
}
}
fn handle_sse_payload(
payload: &str,
acc: &mut StreamAccumulator,
on_token: &mut (dyn FnMut(TokenChunk) + Send),
) -> Result<bool> {
let trimmed = payload.trim();
if trimmed == "[DONE]" {
return Ok(true);
}
if trimmed.is_empty() {
return Ok(false);
}
let value: Value = serde_json::from_str(trimmed)
.map_err(|e| RustyError::Llm(format!("malformed stream chunk: {e}")))?;
let chunk: WireStreamChunk = serde_json::from_value(value.clone())
.map_err(|e| RustyError::Llm(format!("malformed stream chunk: {e}")))?;
if chunk.model.is_some() {
acc.model = chunk.model;
}
if chunk.usage.is_some() {
acc.usage = chunk.usage;
}
if let Some(choice) = chunk.choices.into_iter().next() {
let delta = choice.delta;
if let Some(content) = delta.content {
if !content.is_empty() {
acc.content.push_str(&content);
on_token(TokenChunk {
delta: content,
finish: false,
raw: Some(value),
});
}
}
if let Some(calls) = delta.tool_calls {
for call in calls {
if acc.tool_calls.len() <= call.index {
acc.tool_calls
.resize_with(call.index + 1, ToolCallAccumulator::default);
}
let slot = &mut acc.tool_calls[call.index];
if let Some(id) = call.id {
slot.id.push_str(&id);
}
if let Some(function) = call.function {
if let Some(name) = function.name {
slot.name.push_str(&name);
}
if let Some(arguments) = function.arguments {
slot.arguments.push_str(&arguments);
}
}
}
}
}
Ok(false)
}
#[async_trait]
impl ChatModel for OpenAiCompatibleClient {
async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
let mut body = json!({
"model": self.model,
"messages": messages,
});
if !tools.is_empty() {
body["tools"] = Value::Array(tools.to_vec());
}
let response = self
.send_with_retries(|| {
let mut request = self.client.post(self.endpoint()).json(&body);
if let Some(key) = &self.api_key {
request = request.bearer_auth(key);
}
request
})
.await?;
let wire: WireResponse = response
.json()
.await
.map_err(|e| RustyError::Llm(format!("malformed chat completions response: {e}")))?;
let choice = wire
.choices
.into_iter()
.next()
.ok_or_else(|| RustyError::Llm("chat completions returned zero choices".into()))?;
Ok(ChatResponse {
message: choice.message,
model: wire.model,
usage: wire.usage,
})
}
async fn chat_stream(
&self,
messages: &[ChatMessage],
tools: &[Value],
on_token: &mut (dyn FnMut(TokenChunk) + Send),
) -> Result<ChatResponse> {
let mut body = json!({
"model": self.model,
"messages": messages,
"stream": true,
"stream_options": {"include_usage": true},
});
if !tools.is_empty() {
body["tools"] = Value::Array(tools.to_vec());
}
let mut response = self
.send_with_retries(|| {
let mut request = self.client.post(self.endpoint()).json(&body);
if let Some(key) = &self.api_key {
request = request.bearer_auth(key);
}
request
})
.await?;
let mut decoder = SseDecoder::new();
let mut acc = StreamAccumulator::default();
let mut done = false;
while !done {
let bytes = match response.chunk().await {
Ok(Some(bytes)) => bytes,
Ok(None) => break, Err(e) => {
return Err(RustyError::Llm(format!(
"stream read from {} failed: {e}",
self.base_url
)))
}
};
for payload in decoder.feed_bytes(&bytes) {
if handle_sse_payload(&payload, &mut acc, on_token)? {
done = true;
break;
}
}
}
if !done {
for payload in decoder.finish() {
if handle_sse_payload(&payload, &mut acc, on_token)? {
break;
}
}
}
on_token(TokenChunk {
delta: String::new(),
finish: true,
raw: None,
});
acc.into_response()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tool_call_wire_roundtrip() {
let call = ToolCall::new("call_1", "search", json!({"q": "rust"}));
let serialized = serde_json::to_value(&call).unwrap();
assert_eq!(
serialized,
json!({
"id": "call_1",
"type": "function",
"function": {"name": "search", "arguments": "{\"q\":\"rust\"}"}
})
);
let back: ToolCall = serde_json::from_value(serialized).unwrap();
assert_eq!(back, call);
}
#[test]
fn message_builders() {
let m = ChatMessage::tool_result("call_1", "42");
assert_eq!(m.role, Role::Tool);
assert_eq!(m.tool_call_id.as_deref(), Some("call_1"));
let m = ChatMessage::assistant_tool_calls(vec![ToolCall::new("c", "t", json!({}))]);
assert!(m.has_tool_calls());
assert_eq!(m.content, None);
assert_eq!(
serde_json::to_value(Role::Assistant).unwrap(),
json!("assistant")
);
}
#[test]
fn sse_decoder_handles_multi_chunk_delivery() {
let mut decoder = SseDecoder::new();
assert!(decoder.feed("data: {\"hel").is_empty());
assert_eq!(
decoder.feed("lo\"}\n: heartbeat\n\nda"),
vec!["{\"hello\"}".to_string()]
);
assert_eq!(decoder.feed("ta: world\r\n\r\n"), vec!["world".to_string()]);
}
#[test]
fn sse_decoder_joins_multi_line_data_and_flushes_trailing_event() {
let mut decoder = SseDecoder::new();
assert_eq!(
decoder.feed("data: a\ndata: b\n\n"),
vec!["a\nb".to_string()]
);
assert!(decoder.feed("data: tail").is_empty());
assert_eq!(decoder.finish(), vec!["tail".to_string()]);
assert!(SseDecoder::new().process_line("").is_none());
}
#[test]
fn sse_done_sentinel_terminates_and_content_deltas_accumulate() {
let mut acc = StreamAccumulator::default();
let mut deltas: Vec<String> = Vec::new();
let mut on_token = |chunk: TokenChunk| {
assert!(!chunk.finish);
assert!(chunk.raw.is_some(), "wire chunks carry the raw JSON");
deltas.push(chunk.delta);
};
let done = handle_sse_payload(
r#"{"choices":[{"delta":{"content":"Hel"},"finish_reason":null}]}"#,
&mut acc,
&mut on_token,
)
.unwrap();
assert!(!done);
let done = handle_sse_payload(
r#"{"choices":[{"delta":{"content":"lo"},"finish_reason":"stop"}],
"model":"gpt-x","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3}}"#,
&mut acc,
&mut on_token,
)
.unwrap();
assert!(!done);
assert!(handle_sse_payload("[DONE]", &mut acc, &mut on_token).unwrap());
assert_eq!(deltas, ["Hel", "lo"]);
let response = acc.into_response().unwrap();
assert_eq!(response.message.content.as_deref(), Some("Hello"));
assert_eq!(response.model.as_deref(), Some("gpt-x"));
assert_eq!(response.usage.unwrap().total_tokens, 3);
}
#[test]
fn sse_stream_accumulates_tool_call_deltas() {
let mut acc = StreamAccumulator::default();
let mut on_token = |_chunk: TokenChunk| {};
for payload in [
r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"search","arguments":"{\"q\":"}}]}}]}"#,
r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"rust\"}"}}]}}]}"#,
] {
assert!(!handle_sse_payload(payload, &mut acc, &mut on_token).unwrap());
}
let response = acc.into_response().unwrap();
assert_eq!(
response.message.tool_calls,
vec![ToolCall::new("call_1", "search", json!({"q": "rust"}))]
);
assert_eq!(response.message.content, None);
}
struct NonStreamingMock;
#[async_trait]
impl ChatModel for NonStreamingMock {
async fn chat(&self, _messages: &[ChatMessage], _tools: &[Value]) -> Result<ChatResponse> {
Ok(ChatResponse {
message: ChatMessage::assistant("full answer"),
model: Some("mock".into()),
usage: None,
})
}
}
#[tokio::test]
async fn chat_stream_default_falls_back_to_single_chunk() {
let model = NonStreamingMock;
let mut chunks: Vec<TokenChunk> = Vec::new();
let response = model
.chat_stream(&[], &[], &mut |chunk| chunks.push(chunk))
.await
.unwrap();
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].delta, "full answer");
assert!(chunks[0].finish);
assert!(chunks[0].raw.is_none());
assert_eq!(
response.message.content.as_deref(),
Some("full answer"),
"the fallback returns the chat() response unchanged"
);
}
#[test]
fn feed_bytes_preserves_multibyte_chars_split_across_chunks() {
let mut decoder = SseDecoder::new();
let mut first = b"data: {\"c\":\"".to_vec();
first.push(0xE4);
assert!(decoder.feed_bytes(&first).is_empty());
assert!(decoder.feed_bytes(&[0xBD, 0xA0]).is_empty());
let events = decoder.feed_bytes(b"\"}\n\n");
assert_eq!(events, vec!["{\"c\":\"你\"}".to_string()]);
}
#[test]
fn sse_strips_leading_bom_and_handles_bare_data_line() {
let mut decoder = SseDecoder::new();
assert!(decoder.feed_bytes(&[0xEF, 0xBB, 0xBF]).is_empty());
assert_eq!(decoder.feed("data\n\n"), vec![String::new()]);
assert_eq!(decoder.feed("data: x\n\n"), vec!["x".to_string()]);
}
#[test]
fn empty_payload_is_skipped_not_parsed() {
let mut acc = StreamAccumulator::default();
let mut on_token = |_chunk: TokenChunk| {};
assert!(!handle_sse_payload("", &mut acc, &mut on_token).unwrap());
}
#[test]
fn client_debug_redacts_the_api_key() {
let client =
OpenAiCompatibleClient::new("http://localhost", Some("sk-secret-123".into()), "m");
let debug = format!("{client:?}");
assert!(!debug.contains("sk-secret-123"), "got: {debug}");
assert!(debug.contains("***"), "got: {debug}");
let no_key = OpenAiCompatibleClient::new("http://localhost", None, "m");
assert!(format!("{no_key:?}").contains("api_key: None"));
}
fn start_http_mock(
handler: impl Fn(usize) -> (u16, String) + Send + Sync + 'static,
) -> (
std::net::SocketAddr,
std::sync::Arc<std::sync::atomic::AtomicUsize>,
) {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc as StdArc;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let addr = listener.local_addr().unwrap();
let attempts = StdArc::new(AtomicUsize::new(0));
let attempts2 = attempts.clone();
let handler = StdArc::new(handler);
tokio::spawn(async move {
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
loop {
let Ok((mut stream, _)) = listener.accept().await else {
continue;
};
let attempts = attempts2.clone();
let handler = handler.clone();
tokio::spawn(async move {
let mut buf: Vec<u8> = Vec::new();
let mut chunk = [0u8; 4096];
loop {
if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
let headers = String::from_utf8_lossy(&buf[..pos]).to_lowercase();
let len: usize = headers
.lines()
.find_map(|l| l.strip_prefix("content-length:"))
.and_then(|v| v.trim().parse().ok())
.unwrap_or(0);
if buf.len() >= pos + 4 + len {
break;
}
}
let Ok(n) = stream.read(&mut chunk).await else {
return;
};
if n == 0 {
return;
}
buf.extend_from_slice(&chunk[..n]);
}
let n = attempts.fetch_add(1, Ordering::SeqCst) + 1;
let (status, body) = handler(n);
let reason = match status {
200 => "OK",
400 => "Bad Request",
429 => "Too Many Requests",
500 => "Internal Server Error",
_ => "Status",
};
let response = format!(
"HTTP/1.1 {status} {reason}\r\n\
content-type: application/json\r\n\
content-length: {}\r\n\
connection: close\r\n\
\r\n\
{body}",
body.len()
);
let _ = stream.write_all(response.as_bytes()).await;
});
}
});
(addr, attempts)
}
#[tokio::test]
async fn chat_retries_transient_5xx_then_succeeds() {
let (addr, attempts) = start_http_mock(|n| {
if n < 3 {
(500, r#"{"error":"overloaded"}"#.to_string())
} else {
(
200,
r#"{"choices":[{"message":{"role":"assistant","content":"hi after retry"}}]}"#
.to_string(),
)
}
});
let client = OpenAiCompatibleClient::new(format!("http://{addr}"), None, "m")
.with_backoff(Duration::from_millis(1));
let response = client.chat(&[ChatMessage::user("hi")], &[]).await.unwrap();
assert_eq!(response.message.content.as_deref(), Some("hi after retry"));
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 3);
}
#[tokio::test]
async fn chat_does_not_retry_fatal_4xx() {
let (addr, attempts) =
start_http_mock(|_n| (400, r#"{"error":"bad request"}"#.to_string()));
let client = OpenAiCompatibleClient::new(format!("http://{addr}"), None, "m")
.with_backoff(Duration::from_millis(1));
let err = client
.chat(&[ChatMessage::user("hi")], &[])
.await
.unwrap_err();
assert!(err.to_string().contains("400"), "got: {err}");
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 1);
}
#[tokio::test]
async fn chat_gives_up_after_retries_exhausted() {
let (addr, attempts) =
start_http_mock(|_n| (429, r#"{"error":"rate limited"}"#.to_string()));
let client = OpenAiCompatibleClient::new(format!("http://{addr}"), None, "m")
.with_retries(1)
.with_backoff(Duration::from_millis(1));
let err = client
.chat(&[ChatMessage::user("hi")], &[])
.await
.unwrap_err();
assert!(err.to_string().contains("429"), "got: {err}");
assert_eq!(attempts.load(std::sync::atomic::Ordering::SeqCst), 2);
}
}