use std::collections::HashMap;
use std::io::Read as _;
use std::sync::mpsc::Sender;
use std::sync::Arc;
use std::thread::sleep;
use std::time::Duration;
use serde_json::Value;
use crate::catalog::{self, ApiStyle, Auth};
use crate::encode::{self, Wire};
use crate::meter::Meter;
use crate::{Chunk, Error, OAuthTokenProvider, Provider, Request, Response, Result, Usage};
#[derive(Clone, Copy, Debug)]
pub struct Retry {
pub attempts: u32,
pub backoff: Duration,
}
impl Default for Retry {
fn default() -> Self {
Self {
attempts: 2,
backoff: Duration::from_millis(600),
}
}
}
impl Retry {
pub const NONE: Retry = Retry {
attempts: 0,
backoff: Duration::ZERO,
};
fn delay(&self, attempt: u32, asked: Option<Duration>) -> Duration {
asked.unwrap_or(self.backoff * 2_u32.pow(attempt.min(4)))
}
}
enum Credential {
None,
ApiKey(String),
Bearer(String),
OAuth(Arc<dyn OAuthTokenProvider>),
}
pub struct ClientBuilder {
provider: String,
api: ApiStyle,
url: String,
key_env: Option<String>,
key: Option<String>,
bearer_token: Option<String>,
oauth: Option<Arc<dyn OAuthTokenProvider>>,
model: String,
side_model: Option<String>,
reasoning_effort: Option<String>,
default_max_tokens: u32,
accepts_output_limit: bool,
headers: Vec<(String, String)>,
connect_timeout: Duration,
read_timeout: Duration,
retry: Retry,
}
impl ClientBuilder {
pub fn new(provider: &str, model: impl Into<String>) -> Result<Self> {
let spec = catalog::provider(provider).ok_or_else(|| {
Error::Config(format!(
"unknown provider `{provider}`; built in: {}",
catalog::PROVIDERS
.iter()
.map(|spec| spec.id)
.collect::<Vec<_>>()
.join(", ")
))
})?;
Ok(Self {
key_env: match spec.auth {
Auth::Key(name) => Some(name.to_string()),
Auth::Local => None,
},
..Self::custom(spec.id, spec.api, spec.url, model)
})
}
pub fn custom(
label: impl Into<String>,
api: ApiStyle,
url: impl Into<String>,
model: impl Into<String>,
) -> Self {
Self {
provider: label.into(),
api,
url: url.into(),
key_env: None,
key: None,
bearer_token: None,
oauth: None,
model: model.into(),
side_model: None,
reasoning_effort: None,
default_max_tokens: 1024,
accepts_output_limit: true,
headers: Vec::new(),
connect_timeout: Duration::from_secs(20),
read_timeout: Duration::from_secs(180),
retry: Retry::default(),
}
}
pub fn api_key(mut self, key: impl Into<String>) -> Self {
self.key = Some(key.into());
self
}
pub fn api_key_env(mut self, name: impl Into<String>) -> Self {
self.key_env = Some(name.into());
self
}
pub fn bearer_token(mut self, token: impl Into<String>) -> Self {
self.bearer_token = Some(token.into());
self
}
pub fn oauth(mut self, provider: Arc<dyn OAuthTokenProvider>) -> Self {
self.oauth = Some(provider);
self
}
pub fn side_model(mut self, model: impl Into<String>) -> Self {
self.side_model = Some(model.into());
self
}
pub fn reasoning_effort(mut self, effort: impl Into<String>) -> Self {
self.reasoning_effort = Some(effort.into());
self
}
pub fn default_max_tokens(mut self, tokens: u32) -> Self {
self.default_max_tokens = tokens;
self
}
pub fn accepts_output_limit(mut self, accepts: bool) -> Self {
self.accepts_output_limit = accepts;
self
}
pub fn header(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.headers.push((name.into(), value.into()));
self
}
pub fn connect_timeout(mut self, timeout: Duration) -> Self {
self.connect_timeout = timeout;
self
}
pub fn read_timeout(mut self, timeout: Duration) -> Self {
self.read_timeout = timeout;
self
}
pub fn retry(mut self, retry: Retry) -> Self {
self.retry = retry;
self
}
pub fn build(self) -> Result<Client> {
let alternate = self.oauth.is_some()
|| self
.bearer_token
.as_deref()
.is_some_and(|token| !token.trim().is_empty());
let key = match (self.key, self.key_env.as_deref(), alternate) {
(_, _, true) => None,
(Some(key), _, false) if !key.trim().is_empty() => Some(key),
(_, Some(name), false) => {
Some(std::env::var(name).map_err(|_| Error::Config(format!("{name} is not set")))?)
.filter(|key| !key.trim().is_empty())
}
_ => None,
};
if self.key_env.is_some() && key.is_none() && !alternate {
return Err(Error::Config(format!(
"{} is empty",
self.key_env.unwrap_or_default()
)));
}
let credential = if let Some(provider) = self.oauth {
Credential::OAuth(provider)
} else if let Some(token) = self.bearer_token.filter(|token| !token.trim().is_empty()) {
Credential::Bearer(token)
} else if let Some(key) = key {
Credential::ApiKey(key)
} else {
Credential::None
};
Ok(Client {
agent: ureq::AgentBuilder::new()
.timeout_connect(self.connect_timeout)
.timeout_read(self.read_timeout)
.build(),
provider: self.provider,
api: self.api,
url: self.url,
credential,
model: self.model,
side_model: self.side_model,
reasoning_effort: self.reasoning_effort,
default_max_tokens: self.default_max_tokens,
accepts_output_limit: self.accepts_output_limit,
headers: self.headers,
retry: self.retry,
meter: Meter::default(),
})
}
}
pub struct Client {
agent: ureq::Agent,
provider: String,
api: ApiStyle,
url: String,
credential: Credential,
model: String,
side_model: Option<String>,
reasoning_effort: Option<String>,
default_max_tokens: u32,
accepts_output_limit: bool,
headers: Vec<(String, String)>,
retry: Retry,
meter: Meter,
}
impl Client {
pub fn builder(provider: &str, model: impl Into<String>) -> Result<ClientBuilder> {
ClientBuilder::new(provider, model)
}
pub fn from_env(provider: &str, model: impl Into<String>) -> Result<Self> {
Self::builder(provider, model)?.build()
}
pub fn default_for(provider: &str) -> Result<Self> {
let spec = catalog::provider(provider)
.ok_or_else(|| Error::Config(format!("unknown provider `{provider}`")))?;
Self::from_env(provider, spec.default_model)
}
pub fn compatible(
url: impl Into<String>,
model: impl Into<String>,
key: impl Into<String>,
) -> Result<Self> {
ClientBuilder::custom("compatible", ApiStyle::ChatCompletions, url, model)
.api_key(key)
.build()
}
pub fn provider(&self) -> &str {
&self.provider
}
pub fn model(&self) -> &str {
&self.model
}
pub fn complete(&self, request: &Request) -> Result<Response> {
let before = self.meter.snapshot();
let mut response = crate::complete(self, request)?;
let after = self.meter.snapshot();
response.usage.cost_micros = after.cost_micros.saturating_sub(before.cost_micros);
response.usage.est_micros = after.est_micros.saturating_sub(before.est_micros);
response.usage.unpriced_calls = after.unpriced_calls.saturating_sub(before.unpriced_calls);
Ok(response)
}
pub fn complete_json(&self, request: &Request) -> Result<Value> {
let text = self.complete(request)?.text;
crate::extract_json(&text).ok_or_else(|| {
Error::Decode(format!(
"no JSON object in the reply: {}",
text.chars().take(160).collect::<String>()
))
})
}
pub fn request_body(&self, request: &Request) -> Value {
encode::body(&self.wire(request), request)
}
fn wire<'a>(&'a self, request: &'a Request) -> Wire<'a> {
Wire {
api: self.api,
model: self.resolved_model(request),
reasoning_effort: self.effort(request),
default_max_tokens: self.default_max_tokens,
accepts_output_limit: self.accepts_output_limit,
}
}
fn effort(&self, request: &Request) -> Option<&str> {
if request.side_call && self.api == ApiStyle::Anthropic {
return None;
}
self.reasoning_effort
.as_deref()
.filter(|effort| !effort.is_empty() && *effort != "none")
}
fn resolved_model<'a>(&'a self, request: &'a Request) -> &'a str {
if request.side_call {
self.side_model
.as_deref()
.filter(|model| !model.trim().is_empty())
.unwrap_or(&self.model)
} else {
request.model.as_deref().unwrap_or(&self.model)
}
}
fn send(&self, request: &Request, tx: &Sender<Chunk>) -> Result<()> {
let mut attempt = 0;
loop {
if request.cancelled() {
return Err(Error::Cancelled);
}
match self.attempt(request, tx, None) {
Ok(()) => return Ok(()),
Err(error) if error.is_auth() => return self.refresh_and_retry(request, tx, error),
Err(error) if error.retryable() && attempt < self.retry.attempts => {
sleep(self.retry.delay(attempt, error.retry_after()));
attempt += 1;
}
Err(error) => return Err(error),
}
}
}
fn refresh_and_retry(
&self,
request: &Request,
tx: &Sender<Chunk>,
rejected: Error,
) -> Result<()> {
let Credential::OAuth(provider) = &self.credential else {
return Err(rejected);
};
let token = provider
.refresh_access_token()
.map_err(|error| Error::Auth(format!("OAuth refresh failed: {error}")))?;
if token.trim().is_empty() {
return Err(Error::Auth(
"OAuth refresh returned an empty access token".into(),
));
}
self.attempt(request, tx, Some(&token)).map_err(|error| {
if error.is_auth() {
Error::Auth("credential rejected after one refresh".into())
} else {
error
}
})
}
fn attempt(
&self,
request: &Request,
tx: &Sender<Chunk>,
refreshed_token: Option<&str>,
) -> Result<()> {
let mut call = self
.agent
.post(&self.url)
.set("content-type", "application/json")
.set("accept", "text/event-stream");
if let Some((name, value)) = self.credential_header(refreshed_token)? {
call = call.set(name, &value);
}
if self.api == ApiStyle::Anthropic {
call = call.set("anthropic-version", ANTHROPIC_VERSION);
}
for (name, value) in &self.headers {
call = call.set(name, value);
}
match call.send_json(self.request_body(request)) {
Ok(response) => {
let cancel = request.cancel.as_deref();
let reader = response.into_reader();
let tally = match self.api {
ApiStyle::Responses => crate::stream::responses(reader, tx, cancel),
ApiStyle::Anthropic => crate::stream::anthropic(reader, tx, cancel),
ApiStyle::ChatCompletions => crate::stream::chat(reader, tx, cancel),
};
let model = self.resolved_model(request);
self.meter.record(
&format!("{}/{model}", self.provider),
model,
tally.input,
tally.output,
tally.cost_micros,
);
Ok(())
}
Err(ureq::Error::Status(status, response)) => {
let retry_after = response
.header("retry-after")
.and_then(|value| value.trim().parse::<u64>().ok())
.map(Duration::from_secs);
let mut body = String::new();
let _ = response.into_reader().take(4096).read_to_string(&mut body);
let message = summarize_error(&body);
Err(match status {
401 | 403 => Error::Auth(format!(
"HTTP {status}: credential rejected by {} ({message})",
self.provider
)),
429 => Error::RateLimited {
message,
retry_after,
},
_ => Error::Api { status, message },
})
}
Err(error) if self.provider == "ollama" => Err(Error::Network(format!(
"{error}. Is Ollama running? Try `ollama serve`."
))),
Err(error) => Err(Error::Network(error.to_string())),
}
}
fn credential_header(
&self,
refreshed_token: Option<&str>,
) -> Result<Option<(&'static str, String)>> {
if let Some(token) = refreshed_token {
return Ok(Some(("authorization", format!("Bearer {token}"))));
}
match &self.credential {
Credential::None => Ok(None),
Credential::ApiKey(key) if self.api == ApiStyle::Anthropic => {
Ok(Some(("x-api-key", key.clone())))
}
Credential::ApiKey(key) | Credential::Bearer(key) => {
Ok(Some(("authorization", format!("Bearer {key}"))))
}
Credential::OAuth(provider) => {
let token = provider
.access_token()
.map_err(|error| Error::Auth(format!("no OAuth access token: {error}")))?;
if token.trim().is_empty() {
Err(Error::Auth(
"OAuth provider returned an empty access token".into(),
))
} else {
Ok(Some(("authorization", format!("Bearer {token}"))))
}
}
}
}
}
const ANTHROPIC_VERSION: &str = "2023-06-01";
impl Provider for Client {
fn stream(&self, request: &Request, sender: &Sender<Chunk>) -> Result<()> {
self.send(request, sender)
}
fn label(&self) -> String {
format!("{}/{}", self.provider, self.model)
}
fn usage(&self) -> Usage {
self.meter.snapshot()
}
fn usage_by_model(&self) -> HashMap<String, Usage> {
self.meter.by_model()
}
}
fn summarize_error(body: &str) -> String {
if let Ok(value) = serde_json::from_str::<Value>(body) {
if let Some(message) = value
.pointer("/error/message")
.and_then(Value::as_str)
.or_else(|| value.get("detail").and_then(Value::as_str))
.or_else(|| value.get("message").and_then(Value::as_str))
{
return message.to_string();
}
}
let body = body.trim();
if body.is_empty() {
"no response body".into()
} else {
body.chars().take(300).collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::message::{Message, ToolCall};
use crate::request::{Cancel, ToolSchema};
use serde_json::json;
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
use std::sync::Mutex;
use std::thread;
struct RefreshingTokens {
token: Mutex<String>,
refreshes: AtomicUsize,
}
impl OAuthTokenProvider for RefreshingTokens {
fn access_token(&self) -> std::result::Result<String, String> {
Ok(self.token.lock().unwrap().clone())
}
fn refresh_access_token(&self) -> std::result::Result<String, String> {
self.refreshes.fetch_add(1, AtomicOrdering::Relaxed);
let mut token = self.token.lock().unwrap();
*token = "fresh-token".into();
Ok(token.clone())
}
}
fn request() -> Request {
Request::new("Be concise")
.user("Hello")
.max_output_tokens(50)
}
fn keyed(provider: &str) -> Client {
ClientBuilder::new(provider, "test")
.unwrap()
.api_key("key")
.build()
.unwrap()
}
const OK_STREAM: &str = concat!(
"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n",
"data: {\"choices\":[{\"delta\":{\"content\":\"Hello from Modelbridge\"}}]}\n\n",
"data: {\"choices\":[],\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":3}}\n\n",
"data: [DONE]\n\n",
);
fn read_request(socket: &mut TcpStream) -> String {
let mut received = Vec::new();
let mut buffer = [0_u8; 2048];
loop {
let count = socket.read(&mut buffer).unwrap();
if count == 0 {
break;
}
received.extend_from_slice(&buffer[..count]);
let request = String::from_utf8_lossy(&received);
if request.contains("\r\n\r\n") && request.contains("Hello") {
break;
}
}
String::from_utf8_lossy(&received).into_owned()
}
fn serve(responses: Vec<String>) -> (String, thread::JoinHandle<Vec<String>>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap().to_string();
let handle = thread::spawn(move || {
let mut seen = Vec::new();
for response in responses {
let (mut socket, _) = listener.accept().unwrap();
seen.push(read_request(&mut socket));
let _ = socket.write_all(response.as_bytes());
}
seen
});
(address, handle)
}
fn client_for(address: &str) -> ClientBuilder {
ClientBuilder::custom(
"test",
ApiStyle::ChatCompletions,
format!("http://{address}/v1/chat/completions"),
"test-model",
)
}
#[test]
fn built_in_routes_need_no_url_configuration() {
for provider in catalog::PROVIDERS {
assert!(!provider.url.is_empty());
assert!(!provider.default_model.is_empty());
}
}
#[test]
fn request_shapes_follow_each_api() {
assert_eq!(
keyed("openai").request_body(&request())["instructions"],
"Be concise"
);
assert_eq!(
keyed("anthropic").request_body(&request())["system"],
"Be concise"
);
assert_eq!(
keyed("groq").request_body(&request())["messages"][0]["role"],
"system"
);
}
#[test]
fn tool_schemas_are_translated_for_each_api() {
let request = request().tool(ToolSchema::new(
"weather",
"Get weather",
json!({"type": "object"}),
));
assert_eq!(
keyed("openai").request_body(&request)["tools"][0]["type"],
"function"
);
assert_eq!(
keyed("anthropic").request_body(&request)["tools"][0]["input_schema"]["type"],
"object"
);
assert_eq!(
keyed("groq").request_body(&request)["tools"][0]["function"]["name"],
"weather"
);
}
#[test]
fn anthropic_side_calls_take_the_side_model_and_omit_effort() {
let client = ClientBuilder::new("anthropic", "claude-opus-5")
.unwrap()
.api_key("key")
.side_model("claude-haiku-4-5")
.reasoning_effort("low")
.build()
.unwrap();
assert_eq!(
client.request_body(&request())["output_config"]["effort"],
"low"
);
let side = client.request_body(&request().side_call(true));
assert_eq!(side["model"], "claude-haiku-4-5");
assert!(side.get("output_config").is_none());
}
#[test]
fn arbitrary_models_are_allowed() {
let client = ClientBuilder::new("openai", "released-tomorrow")
.unwrap()
.api_key("key")
.build()
.unwrap();
assert_eq!(
client.request_body(&request())["model"],
"released-tomorrow"
);
}
#[test]
fn a_custom_client_completes_a_real_http_sse_round_trip() {
let (address, server) = serve(vec![OK_STREAM.into()]);
let client = Client::compatible(
format!("http://{address}/v1/chat/completions"),
"test-model",
"secret",
)
.unwrap();
let response = client.complete(&request()).unwrap();
server.join().unwrap();
assert_eq!(response.text, "Hello from Modelbridge");
assert_eq!(response.usage.input, 7);
assert_eq!(response.usage.output, 3);
assert_eq!(response.usage.unpriced_calls, 1);
}
#[test]
fn oauth_refreshes_once_after_unauthorized_and_retries() {
let unauthorized =
"HTTP/1.1 401 Unauthorized\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
let (address, server) = serve(vec![unauthorized.into(), OK_STREAM.into()]);
let tokens = Arc::new(RefreshingTokens {
token: Mutex::new("stale-token".into()),
refreshes: AtomicUsize::new(0),
});
let client = client_for(&address).oauth(tokens.clone()).build().unwrap();
assert!(client.complete(&request()).unwrap().text.contains("Hello"));
let seen = server.join().unwrap();
assert_eq!(tokens.refreshes.load(AtomicOrdering::Relaxed), 1);
assert!(seen[0].contains("authorization: Bearer stale-token"));
assert!(seen[1].contains("authorization: Bearer fresh-token"));
}
#[test]
fn a_rejected_api_key_is_not_retried() {
let unauthorized =
"HTTP/1.1 401 Unauthorized\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
let (address, server) = serve(vec![unauthorized.into()]);
let client = client_for(&address).api_key("bad").build().unwrap();
let error = client.complete(&request()).unwrap_err();
server.join().unwrap();
assert!(error.is_auth(), "{error}");
assert!(!error.retryable());
}
#[test]
fn a_rate_limit_is_retried_after_the_wait_the_server_named() {
let limited = "HTTP/1.1 429 Too Many Requests\r\nretry-after: 0\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
let (address, server) = serve(vec![limited.into(), OK_STREAM.into()]);
let client = client_for(&address)
.api_key("key")
.retry(Retry {
attempts: 1,
backoff: Duration::from_millis(1),
})
.build()
.unwrap();
assert!(client.complete(&request()).unwrap().text.contains("Hello"));
assert_eq!(server.join().unwrap().len(), 2);
}
#[test]
fn a_rate_limit_gives_up_once_the_attempts_are_spent() {
let limited = "HTTP/1.1 429 Too Many Requests\r\nretry-after: 0\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
let (address, server) = serve(vec![limited.into()]);
let client = client_for(&address)
.api_key("key")
.retry(Retry::NONE)
.build()
.unwrap();
let error = client.complete(&request()).unwrap_err();
server.join().unwrap();
assert!(matches!(error, Error::RateLimited { .. }));
}
#[test]
fn a_cancelled_request_never_leaves_the_process() {
let client = client_for("127.0.0.1:1").api_key("key").build().unwrap();
let flag = Cancel::default();
flag.store(true, AtomicOrdering::Relaxed);
assert_eq!(
client.complete(&request().cancel(flag)).unwrap_err(),
Error::Cancelled
);
}
#[test]
fn a_tool_round_trip_survives_a_second_turn() {
let call = ToolCall::new("weather", json!({"city": "Porto"})).with_id("call_1");
let request = request()
.message(Message::tool_calls("Checking.", vec![call.clone()]))
.message(Message::tool_results(vec![crate::ToolResult::new(
&call, 0, "20C",
)]));
let body = keyed("groq").request_body(&request);
let roles: Vec<&str> = body["messages"]
.as_array()
.unwrap()
.iter()
.map(|message| message["role"].as_str().unwrap())
.collect();
assert_eq!(roles, ["system", "user", "assistant", "tool"]);
}
#[test]
fn vendor_errors_are_reduced_to_safe_human_messages() {
assert_eq!(
summarize_error(r#"{"error":{"message":"quota exceeded","token":"secret"}}"#),
"quota exceeded"
);
assert_eq!(summarize_error(" "), "no response body");
}
}