use std::collections::HashMap;
use std::sync::Arc;
use std::time::SystemTime;
use async_trait::async_trait;
use axum::http::HeaderMap;
use serde::{Deserialize, Serialize};
use serde_json::Value;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatMessage {
pub role: String,
pub content: Value,
}
impl ChatMessage {
#[must_use]
pub fn text(role: impl Into<String>, content: impl Into<String>) -> Self {
Self {
role: role.into(),
content: Value::String(content.into()),
}
}
#[must_use]
pub fn text_view(&self) -> String {
match &self.content {
Value::String(s) => s.clone(),
Value::Array(blocks) => blocks
.iter()
.filter_map(|b| b.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("\n"),
_ => String::new(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelRequest {
pub model: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub system: Option<String>,
pub messages: Vec<ChatMessage>,
pub max_tokens: u32,
#[serde(default)]
pub tools: Value,
#[serde(default, skip_serializing_if = "Value::is_null")]
pub raw: Value,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub cache_prefix: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelResponse {
pub model: String,
pub text: String,
pub in_tokens: u64,
pub cache_write_tokens: u64,
pub cache_read_tokens: u64,
pub out_tokens: u64,
pub raw: Value,
}
#[derive(Debug, Clone, thiserror::Error)]
pub enum ProviderError {
#[error("transport error: {0}")]
Transport(String),
#[error("http {status}: {body}")]
Http {
status: u16,
body: String,
},
#[error("decode error: {0}")]
Decode(String),
}
impl ProviderError {
#[must_use]
pub fn is_failover_eligible(&self) -> bool {
match self {
ProviderError::Transport(_) => true,
ProviderError::Http { status, .. } => *status >= 500,
ProviderError::Decode(_) => false,
}
}
}
#[derive(Clone, Default)]
pub struct Auth {
pub anthropic_key: Option<String>,
pub openai_key: Option<String>,
}
impl std::fmt::Debug for Auth {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Auth")
.field("anthropic_key", &self.anthropic_key.as_ref().map(|_| "***"))
.field("openai_key", &self.openai_key.as_ref().map(|_| "***"))
.finish()
}
}
impl Auth {
#[must_use]
pub fn from_headers(headers: &HeaderMap) -> Self {
let anthropic_key = headers
.get("x-api-key")
.and_then(|v| v.to_str().ok())
.map(str::to_owned)
.or_else(|| std::env::var("ANTHROPIC_API_KEY").ok());
let openai_key = headers
.get(axum::http::header::AUTHORIZATION)
.and_then(|v| v.to_str().ok())
.and_then(|v| v.strip_prefix("Bearer "))
.map(str::to_owned)
.or_else(|| std::env::var("OPENAI_API_KEY").ok());
Self {
anthropic_key,
openai_key,
}
}
}
#[async_trait]
pub trait Provider: Send + Sync + std::fmt::Debug {
async fn complete(
&self,
req: &ModelRequest,
auth: &Auth,
) -> Result<ModelResponse, ProviderError>;
fn id(&self) -> &str;
fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
false
}
}
fn wire_model(model: &str) -> &str {
model.split_once('/').map_or(model, |(_, m)| m)
}
fn resolve_api_key(api_key_env: Option<&str>, byok_override: Option<&str>) -> String {
api_key_env
.and_then(|e| std::env::var(e).ok())
.or_else(|| byok_override.map(str::to_owned))
.unwrap_or_default()
}
#[must_use]
pub fn anthropic_wire_body(req: &ModelRequest) -> Value {
if let Value::Object(raw) = &req.raw {
let mut body = raw.clone();
body.insert(
"model".to_owned(),
Value::String(wire_model(&req.model).to_owned()),
);
body.remove("stream");
normalize_reasoning(&mut body, Wire::Anthropic);
if req.cache_prefix {
add_cache_breakpoints(&mut body);
}
return Value::Object(body);
}
let messages: Vec<Value> = req
.messages
.iter()
.map(|m| serde_json::json!({ "role": m.role, "content": m.content }))
.collect();
let mut body = serde_json::json!({
"model": wire_model(&req.model),
"max_tokens": req.max_tokens,
"messages": messages,
});
if let Some(system) = req.system.as_deref() {
body["system"] = serde_json::json!(system);
}
if !req.tools.is_null() {
body["tools"] = req.tools.clone();
}
if req.cache_prefix
&& let Some(map) = body.as_object_mut()
{
add_cache_breakpoints(map);
}
body
}
#[derive(Debug, Clone)]
pub struct AnthropicProvider {
pub id: String,
pub base_url: String,
pub api_key_env: Option<String>,
pub http: reqwest::Client,
}
#[async_trait]
impl Provider for AnthropicProvider {
fn id(&self) -> &str {
&self.id
}
fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
inbound == firstpass_core::Dialect::Anthropic
}
async fn complete(
&self,
req: &ModelRequest,
auth: &Auth,
) -> Result<ModelResponse, ProviderError> {
let key = resolve_api_key(self.api_key_env.as_deref(), auth.anthropic_key.as_deref());
let body = anthropic_wire_body(req);
let url = format!("{}/v1/messages", self.base_url.trim_end_matches('/'));
let resp = self
.http
.post(url)
.header("x-api-key", key)
.header("anthropic-version", "2023-06-01")
.json(&body)
.send()
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
let status = resp.status();
let bytes = resp
.bytes()
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
if !status.is_success() {
return Err(ProviderError::Http {
status: status.as_u16(),
body: String::from_utf8_lossy(&bytes).into_owned(),
});
}
let json: Value =
serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
let u = anthropic_parse_response(&json)?;
let (text, in_tokens, out_tokens) = (u.text, u.in_tokens, u.out_tokens);
let (cache_write_tokens, cache_read_tokens) = (u.cache_write_tokens, u.cache_read_tokens);
Ok(ModelResponse {
cache_write_tokens,
cache_read_tokens,
model: req.model.clone(),
text,
in_tokens,
out_tokens,
raw: json,
})
}
}
fn anthropic_parse_response(json: &Value) -> Result<AnthropicUsage, ProviderError> {
let text = json
.get("content")
.and_then(Value::as_array)
.map(|blocks| {
blocks
.iter()
.filter_map(|b| b.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("")
})
.ok_or_else(|| ProviderError::Decode("missing content[].text".to_owned()))?;
let in_tokens = json
.pointer("/usage/input_tokens")
.and_then(Value::as_u64)
.unwrap_or(0);
let out_tokens = json
.pointer("/usage/output_tokens")
.and_then(Value::as_u64)
.unwrap_or(0);
let cache_write_tokens = json
.pointer("/usage/cache_creation_input_tokens")
.and_then(Value::as_u64)
.unwrap_or(0);
let cache_read_tokens = json
.pointer("/usage/cache_read_input_tokens")
.and_then(Value::as_u64)
.unwrap_or(0);
Ok(AnthropicUsage {
text,
in_tokens,
cache_write_tokens,
cache_read_tokens,
out_tokens,
})
}
struct AnthropicUsage {
text: String,
in_tokens: u64,
cache_write_tokens: u64,
cache_read_tokens: u64,
out_tokens: u64,
}
fn anthropic_messages_body(req: &ModelRequest, anthropic_version: &str) -> Value {
if let Value::Object(raw) = &req.raw {
let mut body = raw.clone();
body.remove("model");
body.remove("stream");
body.insert(
"anthropic_version".to_owned(),
Value::String(anthropic_version.to_owned()),
);
return Value::Object(body);
}
let messages: Vec<Value> = req
.messages
.iter()
.map(|m| serde_json::json!({ "role": m.role, "content": m.content }))
.collect();
let mut body = serde_json::json!({
"anthropic_version": anthropic_version,
"max_tokens": req.max_tokens,
"messages": messages,
});
if let Some(system) = req.system.as_deref() {
body["system"] = serde_json::json!(system);
}
if !req.tools.is_null() {
body["tools"] = req.tools.clone();
}
body
}
#[must_use]
pub fn openai_wire_body(req: &ModelRequest) -> Value {
if let Value::Object(raw) = &req.raw {
let mut body = raw.clone();
body.insert(
"model".to_owned(),
Value::String(wire_model(&req.model).to_owned()),
);
body.remove("stream");
normalize_reasoning(&mut body, Wire::OpenAi);
return Value::Object(body);
}
let mut messages: Vec<Value> = Vec::with_capacity(req.messages.len() + 1);
if let Some(system) = req.system.as_deref() {
messages.push(serde_json::json!({ "role": "system", "content": system }));
}
messages.extend(
req.messages
.iter()
.map(|m| serde_json::json!({ "role": m.role, "content": m.content })),
);
let mut body = serde_json::json!({
"model": wire_model(&req.model),
"max_tokens": req.max_tokens,
"messages": messages,
});
if !req.tools.is_null() {
body["tools"] = req.tools.clone();
}
body
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Effort {
Low,
Medium,
High,
}
impl Effort {
const fn budget_tokens(self) -> u32 {
match self {
Self::Low => 1024, Self::Medium => 4096,
Self::High => 16384,
}
}
const fn from_budget(tokens: u64) -> Self {
if tokens < 2048 {
Self::Low
} else if tokens < 8192 {
Self::Medium
} else {
Self::High
}
}
fn as_str(self) -> &'static str {
match self {
Self::Low => "low",
Self::Medium => "medium",
Self::High => "high",
}
}
}
#[must_use]
pub fn effort_of(raw: &Value) -> Option<Effort> {
read_effort(raw.as_object()?)
}
fn read_effort(body: &serde_json::Map<String, Value>) -> Option<Effort> {
if let Some(s) = body.get("reasoning_effort").and_then(Value::as_str) {
return match s {
"low" | "minimal" => Some(Effort::Low),
"medium" => Some(Effort::Medium),
"high" => Some(Effort::High),
_ => None,
};
}
if let Some(t) = body.get("thinking").and_then(Value::as_object)
&& t.get("type").and_then(Value::as_str) != Some("disabled")
&& let Some(b) = t.get("budget_tokens").and_then(Value::as_u64)
{
return Some(Effort::from_budget(b));
}
if let Some(b) = body
.get("thinkingConfig")
.and_then(Value::as_object)
.and_then(|c| c.get("thinkingBudget"))
.and_then(Value::as_u64)
{
return Some(Effort::from_budget(b));
}
None
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Wire {
Anthropic,
OpenAi,
Gemini,
}
pub fn normalize_reasoning(body: &mut serde_json::Map<String, Value>, target: Wire) {
let native = match target {
Wire::OpenAi => "reasoning_effort",
Wire::Anthropic => "thinking",
Wire::Gemini => "thinkingConfig",
};
if body.contains_key(native) {
return;
}
let effort = read_effort(body);
body.remove("reasoning_effort");
body.remove("thinking");
body.remove("thinkingConfig");
let Some(effort) = effort else { return };
match target {
Wire::OpenAi => {
body.insert(
"reasoning_effort".to_owned(),
Value::String(effort.as_str().to_owned()),
);
}
Wire::Anthropic => {
let max = body.get("max_tokens").and_then(Value::as_u64).unwrap_or(0);
let budget = u64::from(effort.budget_tokens()).min(max.saturating_sub(1));
if budget >= 1024 {
body.insert(
"thinking".to_owned(),
serde_json::json!({ "type": "enabled", "budget_tokens": budget }),
);
body.remove("temperature");
body.remove("top_p");
}
}
Wire::Gemini => {
body.entry("generationConfig")
.or_insert_with(|| Value::Object(serde_json::Map::new()))
.as_object_mut()
.map(|c| {
c.insert(
"thinkingConfig".to_owned(),
serde_json::json!({ "thinkingBudget": effort.budget_tokens() }),
)
});
}
}
}
const MIN_CACHEABLE_PREFIX_CHARS: usize = 3000;
pub fn add_cache_breakpoints(body: &mut serde_json::Map<String, Value>) {
if body_has_cache_control(body) {
return;
}
let prefix_chars = body.get("system").map_or(0, |v| v.to_string().len())
+ body.get("tools").map_or(0, |v| v.to_string().len());
if prefix_chars < MIN_CACHEABLE_PREFIX_CHARS {
return;
}
let marker = serde_json::json!({ "type": "ephemeral" });
match body.get("system") {
Some(Value::String(text)) if !text.is_empty() => {
body.insert(
"system".to_owned(),
serde_json::json!([{
"type": "text", "text": text, "cache_control": marker,
}]),
);
}
Some(Value::Array(blocks)) if !blocks.is_empty() => {
let mut blocks = blocks.clone();
if let Some(last) = blocks.last_mut().and_then(Value::as_object_mut) {
last.insert("cache_control".to_owned(), marker.clone());
}
body.insert("system".to_owned(), Value::Array(blocks));
}
_ => {}
}
if let Some(Value::Array(tools)) = body.get("tools")
&& !tools.is_empty()
{
let mut tools = tools.clone();
if let Some(last) = tools.last_mut().and_then(Value::as_object_mut) {
last.insert("cache_control".to_owned(), marker);
}
body.insert("tools".to_owned(), Value::Array(tools));
}
}
fn body_has_cache_control(body: &serde_json::Map<String, Value>) -> bool {
fn any_in(v: &Value) -> bool {
match v {
Value::Array(items) => items.iter().any(any_in),
Value::Object(o) => o.contains_key("cache_control") || o.values().any(any_in),
_ => false,
}
}
body.values().any(any_in)
}
#[derive(Debug, Clone)]
pub struct OpenAiProvider {
pub id: String,
pub base_url: String,
pub api_key_env: Option<String>,
pub http: reqwest::Client,
}
#[async_trait]
impl Provider for OpenAiProvider {
fn id(&self) -> &str {
&self.id
}
fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
inbound == firstpass_core::Dialect::Openai
}
async fn complete(
&self,
req: &ModelRequest,
auth: &Auth,
) -> Result<ModelResponse, ProviderError> {
let key = resolve_api_key(self.api_key_env.as_deref(), auth.openai_key.as_deref());
let body = openai_wire_body(req);
let url = format!(
"{}/v1/chat/completions",
self.base_url.trim_end_matches('/')
);
let resp = self
.http
.post(url)
.header(axum::http::header::AUTHORIZATION, format!("Bearer {key}"))
.json(&body)
.send()
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
let status = resp.status();
let bytes = resp
.bytes()
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
if !status.is_success() {
return Err(ProviderError::Http {
status: status.as_u16(),
body: String::from_utf8_lossy(&bytes).into_owned(),
});
}
let json: Value =
serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
let text = json
.pointer("/choices/0/message/content")
.and_then(Value::as_str)
.unwrap_or("")
.to_owned();
let in_tokens = json
.pointer("/usage/prompt_tokens")
.and_then(Value::as_u64)
.unwrap_or(0);
let out_tokens = json
.pointer("/usage/completion_tokens")
.and_then(Value::as_u64)
.unwrap_or(0);
Ok(ModelResponse {
cache_write_tokens: 0,
cache_read_tokens: 0,
model: req.model.clone(),
text,
in_tokens,
out_tokens,
raw: json,
})
}
}
fn gemini_request_body(req: &ModelRequest) -> Value {
let contents: Vec<Value> = req
.messages
.iter()
.map(|m| {
let role = if m.role == "assistant" {
"model"
} else {
"user"
};
serde_json::json!({ "role": role, "parts": [{ "text": m.text_view() }] })
})
.collect();
let mut body = serde_json::json!({
"contents": contents,
"generationConfig": { "maxOutputTokens": req.max_tokens },
});
if let Some(system) = req.system.as_deref() {
body["system_instruction"] = serde_json::json!({ "parts": [{ "text": system }] });
}
if let (Some(effort), Some(map)) = (effort_of(&req.raw), body.as_object_mut()) {
let mut carry = serde_json::Map::new();
carry.insert(
"reasoning_effort".to_owned(),
Value::String(effort.as_str().to_owned()),
);
carry.insert(
"generationConfig".to_owned(),
map["generationConfig"].clone(),
);
normalize_reasoning(&mut carry, Wire::Gemini);
map.insert(
"generationConfig".to_owned(),
carry["generationConfig"].clone(),
);
}
body
}
fn gemini_parse_response(json: &Value) -> Result<(String, u64, u64), ProviderError> {
let parts = json
.pointer("/candidates/0/content/parts")
.and_then(Value::as_array)
.ok_or_else(|| ProviderError::Decode("missing candidates[0].content.parts".to_owned()))?;
let text = parts
.iter()
.filter_map(|p| p.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("");
let in_tokens = json
.pointer("/usageMetadata/promptTokenCount")
.and_then(Value::as_u64)
.unwrap_or(0);
let out_tokens = json
.pointer("/usageMetadata/candidatesTokenCount")
.and_then(Value::as_u64)
.unwrap_or(0);
Ok((text, in_tokens, out_tokens))
}
#[derive(Debug, Clone)]
pub struct GeminiProvider {
pub id: String,
pub base_url: String,
pub api_key_env: Option<String>,
pub http: reqwest::Client,
}
#[async_trait]
impl Provider for GeminiProvider {
fn id(&self) -> &str {
&self.id
}
async fn complete(
&self,
req: &ModelRequest,
auth: &Auth,
) -> Result<ModelResponse, ProviderError> {
let key = resolve_api_key(self.api_key_env.as_deref(), auth.openai_key.as_deref());
let body = gemini_request_body(req);
let url = format!(
"{}/v1beta/models/{}:generateContent",
self.base_url.trim_end_matches('/'),
wire_model(&req.model),
);
let resp = self
.http
.post(url)
.header("x-goog-api-key", key)
.json(&body)
.send()
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
let status = resp.status();
let bytes = resp
.bytes()
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
if !status.is_success() {
return Err(ProviderError::Http {
status: status.as_u16(),
body: String::from_utf8_lossy(&bytes).into_owned(),
});
}
let json: Value =
serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
let (text, in_tokens, out_tokens) = gemini_parse_response(&json)?;
Ok(ModelResponse {
cache_write_tokens: 0,
cache_read_tokens: 0,
model: req.model.clone(),
text,
in_tokens,
out_tokens,
raw: json,
})
}
}
struct AwsEnvCredentials {
access_key_id: String,
secret_access_key: String,
session_token: Option<String>,
}
impl AwsEnvCredentials {
fn from_env() -> Result<Self, ProviderError> {
let access_key_id = std::env::var("AWS_ACCESS_KEY_ID")
.map_err(|_| ProviderError::Transport("AWS_ACCESS_KEY_ID is not set".to_owned()))?;
let secret_access_key = std::env::var("AWS_SECRET_ACCESS_KEY")
.map_err(|_| ProviderError::Transport("AWS_SECRET_ACCESS_KEY is not set".to_owned()))?;
let session_token = std::env::var("AWS_SESSION_TOKEN").ok();
Ok(Self {
access_key_id,
secret_access_key,
session_token,
})
}
}
fn bedrock_url(region: &str, model: &str) -> String {
format!("https://bedrock-runtime.{region}.amazonaws.com/model/{model}/invoke")
}
fn sign_bedrock(
url: &str,
region: &str,
body: &[u8],
creds: &AwsEnvCredentials,
) -> Result<http::Request<Vec<u8>>, ProviderError> {
let host = url
.parse::<http::Uri>()
.ok()
.and_then(|u| u.host().map(str::to_owned))
.ok_or_else(|| ProviderError::Transport(format!("invalid bedrock URL: {url}")))?;
let identity: aws_smithy_runtime_api::client::identity::Identity =
aws_credential_types::Credentials::new(
creds.access_key_id.clone(),
creds.secret_access_key.clone(),
creds.session_token.clone(),
None,
"firstpass",
)
.into();
let signing_params: aws_sigv4::http_request::SigningParams<'_> =
aws_sigv4::sign::v4::SigningParams::builder()
.identity(&identity)
.region(region)
.name("bedrock")
.time(SystemTime::now())
.settings(aws_sigv4::http_request::SigningSettings::default())
.build()
.map_err(|e| ProviderError::Transport(format!("sigv4 signing params: {e}")))?
.into();
let headers = [
("host", host.as_str()),
("content-type", "application/json"),
];
let signable = aws_sigv4::http_request::SignableRequest::new(
"POST",
url,
headers.into_iter(),
aws_sigv4::http_request::SignableBody::Bytes(body),
)
.map_err(|e| ProviderError::Transport(format!("sigv4 signable request: {e}")))?;
let (instructions, _signature) = aws_sigv4::http_request::sign(signable, &signing_params)
.map_err(|e| ProviderError::Transport(format!("sigv4 sign: {e}")))?
.into_parts();
let mut req = http::Request::builder()
.method("POST")
.uri(url)
.header("host", host)
.header("content-type", "application/json")
.body(body.to_vec())
.map_err(|e| ProviderError::Transport(format!("build bedrock request: {e}")))?;
instructions.apply_to_request_http1x(&mut req);
Ok(req)
}
#[derive(Debug, Clone)]
pub struct BedrockProvider {
pub id: String,
pub region: Option<String>,
pub http: reqwest::Client,
}
#[async_trait]
impl Provider for BedrockProvider {
fn id(&self) -> &str {
&self.id
}
fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
inbound == firstpass_core::Dialect::Anthropic
}
async fn complete(
&self,
req: &ModelRequest,
_auth: &Auth,
) -> Result<ModelResponse, ProviderError> {
let region = self.region.as_deref().ok_or_else(|| {
ProviderError::Transport("bedrock provider requires a region".to_owned())
})?;
let model = wire_model(&req.model);
let url = bedrock_url(region, model);
let body = anthropic_messages_body(req, "bedrock-2023-05-31");
let body_bytes =
serde_json::to_vec(&body).map_err(|e| ProviderError::Decode(e.to_string()))?;
let creds = AwsEnvCredentials::from_env()?;
let signed = sign_bedrock(&url, region, &body_bytes, &creds)?;
let http_req = reqwest::Request::try_from(signed)
.map_err(|e| ProviderError::Transport(format!("build reqwest request: {e}")))?;
let resp = self
.http
.execute(http_req)
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
let status = resp.status();
let bytes = resp
.bytes()
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
if !status.is_success() {
return Err(ProviderError::Http {
status: status.as_u16(),
body: String::from_utf8_lossy(&bytes).into_owned(),
});
}
let json: Value =
serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
let u = anthropic_parse_response(&json)?;
let (text, in_tokens, out_tokens) = (u.text, u.in_tokens, u.out_tokens);
let (cache_write_tokens, cache_read_tokens) = (u.cache_write_tokens, u.cache_read_tokens);
Ok(ModelResponse {
cache_write_tokens,
cache_read_tokens,
model: req.model.clone(),
text,
in_tokens,
out_tokens,
raw: json,
})
}
}
fn vertex_url(region: &str, project: &str, model: &str) -> String {
format!(
"https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{region}/publishers/anthropic/models/{model}:rawPredict"
)
}
#[derive(Debug, Clone)]
pub struct VertexProvider {
pub id: String,
pub region: Option<String>,
pub project: Option<String>,
pub http: reqwest::Client,
}
#[async_trait]
impl Provider for VertexProvider {
fn id(&self) -> &str {
&self.id
}
fn carries_structured_verbatim(&self, inbound: firstpass_core::Dialect) -> bool {
inbound == firstpass_core::Dialect::Anthropic
}
async fn complete(
&self,
req: &ModelRequest,
_auth: &Auth,
) -> Result<ModelResponse, ProviderError> {
let region = self.region.as_deref().ok_or_else(|| {
ProviderError::Transport("vertex provider requires a region".to_owned())
})?;
let project = self.project.as_deref().ok_or_else(|| {
ProviderError::Transport("vertex provider requires a project".to_owned())
})?;
let model = wire_model(&req.model);
let url = vertex_url(region, project, model);
let body = anthropic_messages_body(req, "vertex-2023-10-16");
let provider = gcp_auth::provider()
.await
.map_err(|e| ProviderError::Transport(format!("gcp_auth provider: {e}")))?;
let token = provider
.token(&["https://www.googleapis.com/auth/cloud-platform"])
.await
.map_err(|e| ProviderError::Transport(format!("gcp_auth token: {e}")))?;
let resp = self
.http
.post(url)
.header(
axum::http::header::AUTHORIZATION,
format!("Bearer {}", token.as_str()),
)
.json(&body)
.send()
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
let status = resp.status();
let bytes = resp
.bytes()
.await
.map_err(|e| ProviderError::Transport(e.to_string()))?;
if !status.is_success() {
return Err(ProviderError::Http {
status: status.as_u16(),
body: String::from_utf8_lossy(&bytes).into_owned(),
});
}
let json: Value =
serde_json::from_slice(&bytes).map_err(|e| ProviderError::Decode(e.to_string()))?;
let u = anthropic_parse_response(&json)?;
let (text, in_tokens, out_tokens) = (u.text, u.in_tokens, u.out_tokens);
let (cache_write_tokens, cache_read_tokens) = (u.cache_write_tokens, u.cache_read_tokens);
Ok(ModelResponse {
cache_write_tokens,
cache_read_tokens,
model: req.model.clone(),
text,
in_tokens,
out_tokens,
raw: json,
})
}
}
#[derive(Clone)]
pub struct ProviderRegistry {
providers: HashMap<String, Arc<dyn Provider>>,
}
impl std::fmt::Debug for ProviderRegistry {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ProviderRegistry")
.field("providers", &self.providers.keys().collect::<Vec<_>>())
.finish()
}
}
impl ProviderRegistry {
fn build_http_client() -> reqwest::Client {
reqwest::Client::builder()
.connect_timeout(std::time::Duration::from_secs(10))
.timeout(std::time::Duration::from_secs(120))
.build()
.unwrap_or_else(|_| reqwest::Client::new())
}
#[must_use]
pub fn new(anthropic_base: impl Into<String>, openai_base: impl Into<String>) -> Self {
Self::from_config(&[], anthropic_base, openai_base)
}
#[must_use]
pub fn from_config(
defs: &[firstpass_core::ProviderDef],
anthropic_base: impl Into<String>,
openai_base: impl Into<String>,
) -> Self {
let http = Self::build_http_client();
let mut providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
providers.insert(
"anthropic".to_owned(),
Arc::new(AnthropicProvider {
id: "anthropic".to_owned(),
base_url: anthropic_base.into(),
api_key_env: None,
http: http.clone(),
}),
);
providers.insert(
"openai".to_owned(),
Arc::new(OpenAiProvider {
id: "openai".to_owned(),
base_url: openai_base.into(),
api_key_env: None,
http: http.clone(),
}),
);
for def in defs {
let provider: Arc<dyn Provider> = match def.auth {
firstpass_core::AuthScheme::AwsSigv4 => Arc::new(BedrockProvider {
id: def.id.clone(),
region: def.region.clone(),
http: http.clone(),
}),
firstpass_core::AuthScheme::GcpOauth => Arc::new(VertexProvider {
id: def.id.clone(),
region: def.region.clone(),
project: def.project.clone(),
http: http.clone(),
}),
firstpass_core::AuthScheme::ApiKey => match def.dialect {
firstpass_core::Dialect::Anthropic => Arc::new(AnthropicProvider {
id: def.id.clone(),
base_url: def.base_url.clone(),
api_key_env: def.api_key_env.clone(),
http: http.clone(),
}),
firstpass_core::Dialect::Openai => Arc::new(OpenAiProvider {
id: def.id.clone(),
base_url: def.base_url.clone(),
api_key_env: def.api_key_env.clone(),
http: http.clone(),
}),
firstpass_core::Dialect::Gemini => Arc::new(GeminiProvider {
id: def.id.clone(),
base_url: def.base_url.clone(),
api_key_env: def.api_key_env.clone(),
http: http.clone(),
}),
},
};
providers.insert(def.id.clone(), provider);
}
Self { providers }
}
#[must_use]
pub fn from_map(providers: HashMap<String, Arc<dyn Provider>>) -> Self {
Self { providers }
}
#[must_use]
pub fn get(&self, provider_id: &str) -> Option<Arc<dyn Provider>> {
self.providers.get(provider_id).cloned()
}
}
#[cfg(test)]
#[derive(Debug, Clone, Default)]
pub struct MockProvider {
id: String,
outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
calls: Arc<std::sync::Mutex<Vec<String>>>,
delay_ms: u64,
}
#[cfg(test)]
impl MockProvider {
#[must_use]
pub fn new(
id: impl Into<String>,
outcomes: HashMap<String, Result<ModelResponse, ProviderError>>,
) -> Self {
Self {
id: id.into(),
outcomes,
calls: Arc::default(),
delay_ms: 0,
}
}
#[must_use]
pub fn with_delay(mut self, ms: u64) -> Self {
self.delay_ms = ms;
self
}
#[must_use]
pub fn call_log(&self) -> Arc<std::sync::Mutex<Vec<String>>> {
Arc::clone(&self.calls)
}
}
#[cfg(test)]
#[async_trait]
impl Provider for MockProvider {
fn id(&self) -> &str {
&self.id
}
fn carries_structured_verbatim(&self, _inbound: firstpass_core::Dialect) -> bool {
true
}
async fn complete(
&self,
req: &ModelRequest,
_auth: &Auth,
) -> Result<ModelResponse, ProviderError> {
self.calls.lock().unwrap().push(req.model.clone());
if self.delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(self.delay_ms)).await;
}
self.outcomes.get(&req.model).cloned().unwrap_or_else(|| {
Err(ProviderError::Decode(format!(
"no mock outcome configured for {}",
req.model
)))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
fn obj(v: Value) -> serde_json::Map<String, Value> {
v.as_object().expect("test literal is an object").clone()
}
#[test]
fn openai_reasoning_effort_crosses_onto_an_anthropic_rung() {
let mut body = obj(serde_json::json!({
"model": "claude-sonnet-5", "max_tokens": 8192, "reasoning_effort": "high",
}));
normalize_reasoning(&mut body, Wire::Anthropic);
assert!(
!body.contains_key("reasoning_effort"),
"the foreign spelling must not survive: {body:?}"
);
assert_eq!(body["thinking"]["type"], "enabled");
assert_eq!(body["thinking"]["budget_tokens"], 8191);
}
#[test]
fn anthropic_thinking_budget_crosses_onto_an_openai_rung_as_a_word() {
let mut body = obj(serde_json::json!({
"model": "gpt-5.5",
"thinking": { "type": "enabled", "budget_tokens": 16384 },
}));
normalize_reasoning(&mut body, Wire::OpenAi);
assert!(!body.contains_key("thinking"));
assert_eq!(body["reasoning_effort"], "high");
}
#[test]
fn thinking_is_dropped_rather_than_sent_invalid_when_max_tokens_cannot_fit_it() {
let mut body = obj(serde_json::json!({
"model": "claude-sonnet-5", "max_tokens": 500, "reasoning_effort": "high",
}));
normalize_reasoning(&mut body, Wire::Anthropic);
assert!(
!body.contains_key("thinking"),
"must not emit a budget max_tokens cannot satisfy: {body:?}"
);
assert!(!body.contains_key("reasoning_effort"));
}
#[test]
fn enabling_thinking_clears_a_temperature_the_caller_chose_without_knowing() {
let mut body = obj(serde_json::json!({
"model": "claude-sonnet-5", "max_tokens": 8192,
"reasoning_effort": "medium", "temperature": 0.2, "top_p": 0.9,
}));
normalize_reasoning(&mut body, Wire::Anthropic);
assert_eq!(body["thinking"]["budget_tokens"], 4096);
assert!(!body.contains_key("temperature"), "{body:?}");
assert!(!body.contains_key("top_p"), "{body:?}");
}
#[test]
fn a_body_that_asked_for_no_reasoning_is_left_alone() {
let before = serde_json::json!({
"model": "claude-sonnet-5", "max_tokens": 1024, "temperature": 0.7,
});
let mut body = obj(before.clone());
normalize_reasoning(&mut body, Wire::Anthropic);
assert_eq!(
Value::Object(body),
before,
"no reasoning request, no rewrite"
);
}
#[test]
fn gemini_gets_a_nested_budget_not_a_top_level_one() {
let mut body = obj(serde_json::json!({
"contents": [], "generationConfig": { "maxOutputTokens": 4096 },
"reasoning_effort": "low",
}));
normalize_reasoning(&mut body, Wire::Gemini);
assert_eq!(
body["generationConfig"]["thinkingConfig"]["thinkingBudget"],
1024
);
assert_eq!(
body["generationConfig"]["maxOutputTokens"], 4096,
"must not clobber the rest of generationConfig"
);
assert!(
!body.contains_key("thinkingConfig"),
"top level is wrong for Gemini"
);
}
#[test]
fn a_same_dialect_body_is_never_rewritten_even_when_the_caller_got_it_wrong() {
let before = serde_json::json!({
"max_tokens": 512,
"thinking": { "type": "enabled", "budget_tokens": 1024 },
});
let mut body = obj(before.clone());
normalize_reasoning(&mut body, Wire::Anthropic);
assert_eq!(Value::Object(body), before);
}
#[test]
fn disabled_thinking_is_not_read_as_a_request_to_think() {
let body = obj(serde_json::json!({
"thinking": { "type": "disabled", "budget_tokens": 16384 },
}));
assert_eq!(read_effort(&body), None);
}
#[test]
fn a_cached_response_reports_its_cache_tokens_not_just_the_remainder() {
let json = serde_json::json!({
"content": [{ "type": "text", "text": "ok" }],
"usage": {
"input_tokens": 20,
"cache_creation_input_tokens": 190_000,
"cache_read_input_tokens": 1_234,
"output_tokens": 400,
},
});
let u = anthropic_parse_response(&json).expect("parses");
assert_eq!(u.in_tokens, 20);
assert_eq!(u.cache_write_tokens, 190_000);
assert_eq!(u.cache_read_tokens, 1_234);
assert_eq!(u.out_tokens, 400);
}
#[test]
fn an_uncached_response_reports_zero_cache_traffic() {
let json = serde_json::json!({
"content": [{ "type": "text", "text": "ok" }],
"usage": { "input_tokens": 1000, "output_tokens": 200 },
});
let u = anthropic_parse_response(&json).expect("parses");
assert_eq!(u.in_tokens, 1000);
assert_eq!(u.cache_write_tokens, 0);
assert_eq!(u.cache_read_tokens, 0);
}
#[test]
fn cache_breakpoints_land_on_the_stable_prefix_only() {
let mut body = obj(serde_json::json!({
"model": "claude-sonnet-5",
"system": "you are a careful assistant. ".repeat(200),
"tools": [{ "name": "a" }, { "name": "b" }],
"messages": [{ "role": "user", "content": "hi" }],
}));
add_cache_breakpoints(&mut body);
assert_eq!(body["system"][0]["type"], "text");
assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
assert!(
body["system"][0]["text"]
.as_str()
.unwrap()
.starts_with("you are a careful")
);
assert!(body["tools"][0].get("cache_control").is_none());
assert_eq!(body["tools"][1]["cache_control"]["type"], "ephemeral");
assert!(
body["messages"][0].get("cache_control").is_none(),
"the conversation is not the stable prefix"
);
}
#[test]
fn a_prefix_too_small_to_qualify_is_not_marked() {
let mut body = obj(serde_json::json!({
"model": "claude-sonnet-5",
"system": "be terse",
"messages": [{ "role": "user", "content": "hi" }],
}));
let before = body.clone();
add_cache_breakpoints(&mut body);
assert_eq!(body, before, "a two-word system prompt cannot be cached");
}
#[test]
fn a_caller_that_placed_its_own_breakpoints_is_left_alone() {
let before = serde_json::json!({
"system": [{ "type": "text", "text": "s", "cache_control": { "type": "ephemeral" } }],
"tools": [{ "name": "a" }],
});
let mut body = obj(before.clone());
add_cache_breakpoints(&mut body);
assert_eq!(Value::Object(body), before);
}
#[test]
fn nothing_is_marked_when_there_is_no_stable_prefix() {
let mut body = obj(serde_json::json!({
"model": "claude-sonnet-5",
"messages": [{ "role": "user", "content": "one shot" }],
}));
let before = body.clone();
add_cache_breakpoints(&mut body);
assert_eq!(body, before, "no system, no tools — nothing to cache");
}
#[test]
fn breakpoints_are_off_unless_the_request_asks_for_them() {
let raw = serde_json::json!({
"model": "claude-sonnet-5", "max_tokens": 64,
"system": "a long stable system prefix. ".repeat(200),
"messages": [{ "role": "user", "content": "hi" }],
});
let req = ModelRequest {
model: "anthropic/claude-sonnet-5".to_owned(),
system: Some("a long stable system prefix. ".repeat(200)),
messages: vec![ChatMessage::text("user", "hi")],
max_tokens: 64,
tools: Value::Null,
raw: raw.clone(),
cache_prefix: false,
};
let body = anthropic_wire_body(&req);
assert!(body["system"].is_string(), "unchanged when opted out");
let opted = ModelRequest {
cache_prefix: true,
..req
};
let body = anthropic_wire_body(&opted);
assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
}
#[test]
fn breakpoints_apply_on_the_normalized_path_too() {
let req = ModelRequest {
model: "anthropic/claude-sonnet-5".to_owned(),
system: Some("stable prefix. ".repeat(300)),
messages: vec![ChatMessage::text("user", "hi")],
max_tokens: 64,
tools: serde_json::json!([{ "name": "a" }]),
raw: Value::Null, cache_prefix: true,
};
let body = anthropic_wire_body(&req);
assert_eq!(body["system"][0]["cache_control"]["type"], "ephemeral");
assert_eq!(body["tools"][0]["cache_control"]["type"], "ephemeral");
}
#[test]
fn wire_model_strips_the_provider_prefix() {
assert_eq!(wire_model("anthropic/claude-haiku-4-5"), "claude-haiku-4-5");
assert_eq!(wire_model("openai/gpt-5.5"), "gpt-5.5");
assert_eq!(wire_model("claude-opus-4-8"), "claude-opus-4-8"); }
#[test]
fn gemini_request_maps_roles_and_system_instruction() {
let req = ModelRequest {
model: "gemini/gemini-2.0-flash".to_owned(),
system: Some("be terse".to_owned()),
messages: vec![
ChatMessage::text("user", "hi"),
ChatMessage::text("assistant", "hello"),
],
max_tokens: 256,
tools: Value::Null,
raw: Value::Null,
cache_prefix: false,
};
let body = gemini_request_body(&req);
assert_eq!(body["system_instruction"]["parts"][0]["text"], "be terse");
assert_eq!(body["generationConfig"]["maxOutputTokens"], 256);
assert_eq!(body["contents"][0]["role"], "user");
assert_eq!(body["contents"][0]["parts"][0]["text"], "hi");
assert_eq!(body["contents"][1]["role"], "model");
assert_eq!(body["contents"][1]["parts"][0]["text"], "hello");
}
#[test]
fn gemini_response_parses_text_and_usage() {
let json = serde_json::json!({
"candidates": [{ "content": { "role": "model", "parts": [
{ "text": "the answer " }, { "text": "is 42" }
] } }],
"usageMetadata": { "promptTokenCount": 11, "candidatesTokenCount": 4 }
});
let (text, in_tok, out_tok) = gemini_parse_response(&json).unwrap();
assert_eq!(text, "the answer is 42");
assert_eq!(in_tok, 11);
assert_eq!(out_tok, 4);
assert!(gemini_parse_response(&serde_json::json!({ "candidates": [] })).is_err());
}
#[test]
fn from_config_wires_the_gemini_dialect() {
let defs = vec![firstpass_core::ProviderDef {
id: "gemini".to_owned(),
dialect: firstpass_core::Dialect::Gemini,
base_url: "https://generativelanguage.googleapis.com".to_owned(),
api_key_env: Some("GEMINI_API_KEY".to_owned()),
auth: firstpass_core::AuthScheme::ApiKey,
region: None,
project: None,
}];
let reg = ProviderRegistry::from_config(
&defs,
"https://api.anthropic.com",
"https://api.openai.com",
);
assert_eq!(reg.get("gemini").unwrap().id(), "gemini");
}
#[test]
fn from_config_registers_custom_providers_alongside_builtins() {
let defs = vec![
firstpass_core::ProviderDef {
id: "groq".to_owned(),
dialect: firstpass_core::Dialect::Openai,
base_url: "https://api.groq.com/openai".to_owned(),
api_key_env: Some("GROQ_API_KEY".to_owned()),
auth: firstpass_core::AuthScheme::ApiKey,
region: None,
project: None,
},
firstpass_core::ProviderDef {
id: "openai".to_owned(),
dialect: firstpass_core::Dialect::Openai,
base_url: "https://my-azure.openai.azure.com".to_owned(),
api_key_env: Some("AZURE_OPENAI_KEY".to_owned()),
auth: firstpass_core::AuthScheme::ApiKey,
region: None,
project: None,
},
];
let reg = ProviderRegistry::from_config(
&defs,
"https://api.anthropic.com",
"https://api.openai.com",
);
assert_eq!(reg.get("anthropic").unwrap().id(), "anthropic");
assert_eq!(reg.get("groq").unwrap().id(), "groq");
assert!(reg.get("does-not-exist").is_none());
}
#[test]
fn anthropic_messages_body_omits_model_and_includes_system_only_when_set() {
let req = ModelRequest {
model: "bedrock/anthropic.claude-3-5-haiku".to_owned(),
system: Some("be terse".to_owned()),
messages: vec![
ChatMessage::text("user", "hi"),
ChatMessage {
role: "assistant".to_owned(),
content: serde_json::json!([{ "type": "text", "text": "hello" }]),
},
],
max_tokens: 128,
tools: Value::Null,
raw: Value::Null,
cache_prefix: false,
};
let body = anthropic_messages_body(&req, "bedrock-2023-05-31");
assert!(body.get("model").is_none());
assert_eq!(body["anthropic_version"], "bedrock-2023-05-31");
assert_eq!(body["max_tokens"], 128);
assert_eq!(body["system"], "be terse");
assert_eq!(body["messages"][0]["content"], serde_json::json!("hi"));
assert_eq!(
body["messages"][1]["content"],
serde_json::json!([{ "type": "text", "text": "hello" }])
);
let req_no_system = ModelRequest {
system: None,
..req
};
let body2 = anthropic_messages_body(&req_no_system, "vertex-2023-10-16");
assert!(body2.get("system").is_none());
}
#[test]
fn bedrock_url_construction_and_missing_region() {
assert_eq!(
bedrock_url("us-east-1", "anthropic.claude-3-5-haiku-20241022-v1:0"),
"https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke"
);
}
#[tokio::test]
async fn bedrock_complete_errors_without_a_region() {
let provider = BedrockProvider {
id: "bedrock".to_owned(),
region: None,
http: reqwest::Client::new(),
};
let req = ModelRequest {
model: "bedrock/anthropic.claude-3-5-haiku".to_owned(),
system: None,
messages: vec![],
max_tokens: 16,
tools: Value::Null,
raw: Value::Null,
cache_prefix: false,
};
let err = provider.complete(&req, &Auth::default()).await.unwrap_err();
assert!(matches!(err, ProviderError::Transport(_)));
}
#[test]
fn bedrock_signing_produces_a_sigv4_authorization_header() {
let creds = AwsEnvCredentials {
access_key_id: "AKIDEXAMPLE".to_owned(),
secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_owned(),
session_token: None,
};
let url = bedrock_url("us-east-1", "anthropic.claude-3-5-haiku");
let body = br#"{"anthropic_version":"bedrock-2023-05-31"}"#;
let signed = sign_bedrock(&url, "us-east-1", body, &creds).unwrap();
let auth_header = signed
.headers()
.get("authorization")
.and_then(|v| v.to_str().ok())
.expect("authorization header present");
assert!(auth_header.starts_with("AWS4-HMAC-SHA256"));
let host_header = signed
.headers()
.get("host")
.and_then(|v| v.to_str().ok())
.expect("host header present");
assert_eq!(host_header, "bedrock-runtime.us-east-1.amazonaws.com");
}
#[test]
fn vertex_url_construction_and_missing_project() {
assert_eq!(
vertex_url("us-central1", "my-project", "claude-3-5-sonnet"),
"https://us-central1-aiplatform.googleapis.com/v1/projects/my-project/locations/us-central1/publishers/anthropic/models/claude-3-5-sonnet:rawPredict"
);
}
#[tokio::test]
async fn vertex_complete_errors_without_a_project() {
let provider = VertexProvider {
id: "vertex".to_owned(),
region: Some("us-central1".to_owned()),
project: None,
http: reqwest::Client::new(),
};
let req = ModelRequest {
model: "vertex/claude-3-5-sonnet".to_owned(),
system: None,
messages: vec![],
max_tokens: 16,
tools: Value::Null,
raw: Value::Null,
cache_prefix: false,
};
let err = provider.complete(&req, &Auth::default()).await.unwrap_err();
assert!(matches!(err, ProviderError::Transport(_)));
}
#[test]
fn from_config_wires_bedrock_and_vertex_auth_schemes() {
let defs = vec![
firstpass_core::ProviderDef {
id: "bedrock".to_owned(),
dialect: firstpass_core::Dialect::Anthropic,
base_url: String::new(),
api_key_env: None,
auth: firstpass_core::AuthScheme::AwsSigv4,
region: Some("us-east-1".to_owned()),
project: None,
},
firstpass_core::ProviderDef {
id: "vertex".to_owned(),
dialect: firstpass_core::Dialect::Anthropic,
base_url: String::new(),
api_key_env: None,
auth: firstpass_core::AuthScheme::GcpOauth,
region: Some("us-central1".to_owned()),
project: Some("my-project".to_owned()),
},
];
let reg = ProviderRegistry::from_config(
&defs,
"https://api.anthropic.com",
"https://api.openai.com",
);
assert_eq!(reg.get("bedrock").unwrap().id(), "bedrock");
assert_eq!(reg.get("vertex").unwrap().id(), "vertex");
}
#[test]
fn resolve_api_key_prefers_configured_env_then_byok() {
let path = std::env::var("PATH").expect("PATH is set");
assert_eq!(resolve_api_key(Some("PATH"), Some("byok")), path);
assert_eq!(
resolve_api_key(Some("FIRSTPASS_DEFINITELY_UNSET_KEY"), Some("byok")),
"byok"
);
assert_eq!(resolve_api_key(None, Some("byok")), "byok");
assert_eq!(resolve_api_key(None, None), "");
}
#[test]
fn anthropic_wire_forwards_tool_and_image_content_verbatim() {
let messages = vec![
ChatMessage::text("user", "hi"),
ChatMessage {
role: "assistant".to_owned(),
content: serde_json::json!([
{ "type": "tool_use", "id": "t1", "name": "calc", "input": { "x": 1 } }
]),
},
ChatMessage {
role: "user".to_owned(),
content: serde_json::json!([
{ "type": "tool_result", "tool_use_id": "t1", "content": "2" },
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "AA==" } }
]),
},
];
let wire = anthropic_wire_body(&ModelRequest {
model: "anthropic/claude-haiku-4-5".to_owned(),
system: None,
messages,
max_tokens: 64,
tools: Value::Null,
raw: Value::Null,
cache_prefix: false,
});
assert_eq!(wire["messages"][0]["content"], serde_json::json!("hi"));
assert_eq!(
wire["messages"][1]["content"],
serde_json::json!([{ "type": "tool_use", "id": "t1", "name": "calc", "input": { "x": 1 } }])
);
assert_eq!(
wire["messages"][2]["content"],
serde_json::json!([
{ "type": "tool_result", "tool_use_id": "t1", "content": "2" },
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "AA==" } }
])
);
}
fn resp(model: &str, text: &str) -> ModelResponse {
ModelResponse {
model: model.to_owned(),
text: text.to_owned(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
raw: Value::Null,
}
}
#[test]
fn transport_and_5xx_are_failover_eligible() {
assert!(ProviderError::Transport("boom".into()).is_failover_eligible());
assert!(
ProviderError::Http {
status: 503,
body: String::new()
}
.is_failover_eligible()
);
}
#[test]
fn client_errors_and_decode_failures_are_hard() {
assert!(
!ProviderError::Http {
status: 400,
body: String::new()
}
.is_failover_eligible()
);
assert!(!ProviderError::Decode("bad json".into()).is_failover_eligible());
}
#[test]
fn auth_debug_never_prints_key_material() {
let auth = Auth {
anthropic_key: Some("sk-ant-super-secret".to_owned()),
openai_key: Some("sk-oai-super-secret".to_owned()),
};
let debug = format!("{auth:?}");
assert!(!debug.contains("super-secret"));
}
#[tokio::test]
async fn mock_provider_returns_configured_outcome() {
let mut outcomes = HashMap::new();
outcomes.insert(
"anthropic/claude-haiku-4-5".to_owned(),
Ok(resp("anthropic/claude-haiku-4-5", "hello")),
);
let provider = MockProvider::new("anthropic", outcomes);
let req = ModelRequest {
model: "anthropic/claude-haiku-4-5".to_owned(),
system: None,
messages: vec![],
max_tokens: 100,
tools: Value::Null,
raw: Value::Null,
cache_prefix: false,
};
let out = provider.complete(&req, &Auth::default()).await.unwrap();
assert_eq!(out.text, "hello");
}
#[test]
fn anthropic_wire_body_carries_raw_request_verbatim() {
let raw = serde_json::json!({
"model": "claude-opus-4-8",
"stream": true,
"max_tokens": 512,
"temperature": 0.2,
"tool_choice": { "type": "auto" },
"tools": [{ "name": "get_weather", "input_schema": { "type": "object" } }],
"thinking": { "type": "enabled", "budget_tokens": 1024 },
"messages": [{ "role": "user", "content": "hi" }]
});
let req = ModelRequest {
model: "anthropic/claude-haiku-4-5".to_owned(),
system: None,
messages: vec![ChatMessage::text("user", "hi")],
max_tokens: 512,
tools: raw["tools"].clone(),
raw: raw.clone(),
cache_prefix: false,
};
let wire = anthropic_wire_body(&req);
assert_eq!(wire["model"], "claude-haiku-4-5", "rung model swapped in");
assert!(wire.get("stream").is_none(), "stream stripped");
for field in [
"temperature",
"tool_choice",
"tools",
"thinking",
"max_tokens",
"messages",
] {
assert_eq!(
wire[field], raw[field],
"field {field} must survive verbatim"
);
}
}
#[test]
fn bedrock_vertex_body_carries_raw_minus_model_plus_version() {
let raw = serde_json::json!({
"model": "claude-haiku-4-5",
"stream": true,
"max_tokens": 64,
"tools": [{ "name": "t" }],
"messages": [{ "role": "user", "content": "hi" }]
});
let req = ModelRequest {
model: "bedrock/anthropic.claude-haiku".to_owned(),
system: None,
messages: vec![ChatMessage::text("user", "hi")],
max_tokens: 64,
tools: raw["tools"].clone(),
raw: raw.clone(),
cache_prefix: false,
};
let body = anthropic_messages_body(&req, "bedrock-2023-05-31");
assert!(body.get("model").is_none(), "model lives in the URL");
assert!(body.get("stream").is_none());
assert_eq!(body["anthropic_version"], "bedrock-2023-05-31");
assert_eq!(body["tools"], raw["tools"]);
assert_eq!(body["messages"], raw["messages"]);
}
#[test]
fn verbatim_carriers_are_dialect_aware() {
use firstpass_core::Dialect;
let reg = ProviderRegistry::new("http://localhost", "http://localhost");
assert!(
reg.get("anthropic")
.unwrap()
.carries_structured_verbatim(Dialect::Anthropic)
);
assert!(
!reg.get("anthropic")
.unwrap()
.carries_structured_verbatim(Dialect::Openai)
);
assert!(
reg.get("openai")
.unwrap()
.carries_structured_verbatim(Dialect::Openai)
);
assert!(
!reg.get("openai")
.unwrap()
.carries_structured_verbatim(Dialect::Anthropic)
);
}
#[test]
fn openai_wire_body_carries_raw_request_verbatim() {
let raw = serde_json::json!({
"model": "gpt-4o",
"stream": true,
"max_tokens": 256,
"temperature": 0.7,
"response_format": { "type": "json_object" },
"tool_choice": "auto",
"tools": [{ "type": "function", "function": { "name": "get_weather", "parameters": {} } }],
"messages": [{ "role": "user", "content": "hi" }]
});
let req = ModelRequest {
model: "openai/gpt-4.1-mini".to_owned(),
system: None,
messages: vec![ChatMessage::text("user", "hi")],
max_tokens: 256,
tools: Value::Null,
raw: raw.clone(),
cache_prefix: false,
};
let wire = openai_wire_body(&req);
assert_eq!(wire["model"], "gpt-4.1-mini", "rung model swapped in");
assert!(wire.get("stream").is_none(), "stream stripped");
for field in [
"temperature",
"tool_choice",
"tools",
"response_format",
"messages",
"max_tokens",
] {
assert_eq!(
wire[field], raw[field],
"field {field} must survive verbatim"
);
}
}
#[test]
fn openai_wire_body_falls_back_to_normalized_fields() {
let req = ModelRequest {
model: "openai/gpt-4.1-mini".to_owned(),
system: Some("be terse".to_owned()),
messages: vec![
ChatMessage::text("user", "hello"),
ChatMessage::text("assistant", "hi"),
],
max_tokens: 128,
tools: Value::Null,
raw: Value::Null,
cache_prefix: false,
};
let wire = openai_wire_body(&req);
assert_eq!(wire["model"], "gpt-4.1-mini");
assert_eq!(wire["messages"][0]["role"], "system");
assert_eq!(wire["messages"][0]["content"], "be terse");
assert_eq!(wire["messages"][1]["role"], "user");
assert_eq!(wire["messages"][2]["role"], "assistant");
}
}