use std::collections::HashMap;
use std::sync::Arc;
use crate::{
builder::{LLMBackend, SystemContent, SystemPrompt},
chat::{
ChatMessage, ChatProvider, ChatResponse, ChatRole, MessageType, StreamChunk, Tool,
ToolChoice, Usage,
},
completion::{CompletionProvider, CompletionRequest, CompletionResponse},
embedding::EmbeddingProvider,
error::LLMError,
models::{ModelListRawEntry, ModelListRequest, ModelListResponse, ModelsProvider},
stt::SpeechToTextProvider,
tts::TextToSpeechProvider,
FunctionCall, ToolCall,
};
use async_trait::async_trait;
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use chrono::{DateTime, Utc};
use futures::stream::Stream;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::Value;
#[derive(Debug)]
pub struct AnthropicConfig {
pub api_key: String,
pub model: String,
pub max_tokens: u32,
pub temperature: f32,
pub timeout_seconds: u64,
pub system: SystemPrompt,
pub top_p: Option<f32>,
pub top_k: Option<u32>,
pub tools: Option<Vec<Tool>>,
pub tool_choice: Option<ToolChoice>,
pub reasoning: bool,
pub thinking_budget_tokens: Option<u32>,
}
#[derive(Debug, Clone)]
pub struct Anthropic {
pub config: Arc<AnthropicConfig>,
pub client: Client,
}
#[derive(Serialize, Debug)]
struct AnthropicTool<'a> {
name: &'a str,
description: &'a str,
#[serde(rename = "input_schema")]
schema: &'a serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none")]
cache_control: Option<&'a serde_json::Value>,
}
#[derive(Serialize, Debug)]
struct ThinkingConfig {
#[serde(rename = "type")]
thinking_type: String,
budget_tokens: u32,
}
#[derive(Serialize, Debug)]
#[serde(untagged)]
enum RequestSystemPrompt<'a> {
String(&'a str),
Messages(&'a [SystemContent]),
}
#[derive(Serialize, Debug)]
struct AnthropicCompleteRequest<'a> {
messages: Vec<AnthropicMessage<'a>>,
model: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
max_tokens: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
temperature: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
system: Option<RequestSystemPrompt<'a>>,
#[serde(skip_serializing_if = "Option::is_none")]
stream: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
top_p: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
top_k: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
tools: Option<Vec<AnthropicTool<'a>>>,
#[serde(skip_serializing_if = "Option::is_none")]
tool_choice: Option<HashMap<String, String>>,
#[serde(skip_serializing_if = "Option::is_none")]
thinking: Option<ThinkingConfig>,
}
#[derive(Serialize, Debug)]
struct AnthropicMessage<'a> {
role: &'a str,
content: Vec<MessageContent<'a>>,
}
#[derive(Serialize, Debug)]
struct MessageContent<'a> {
#[serde(rename = "type")]
message_type: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
text: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
image_url: Option<ImageUrlContent<'a>>,
#[serde(skip_serializing_if = "Option::is_none")]
source: Option<ImageSource<'a>>,
#[serde(skip_serializing_if = "Option::is_none", rename = "id")]
tool_use_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none", rename = "name")]
tool_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none", rename = "input")]
tool_input: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none", rename = "tool_use_id")]
tool_result_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none", rename = "content")]
tool_output: Option<String>,
}
#[derive(Serialize, Debug)]
struct ImageUrlContent<'a> {
url: &'a str,
}
#[derive(Serialize, Debug)]
struct ImageSource<'a> {
#[serde(rename = "type")]
source_type: &'a str,
media_type: &'a str,
data: String,
}
#[derive(Deserialize, Debug)]
struct AnthropicCompleteResponse {
content: Vec<AnthropicContent>,
usage: Option<AnthropicUsage>,
}
#[derive(Deserialize, Debug)]
struct AnthropicUsage {
input_tokens: u32,
output_tokens: u32,
#[serde(skip_serializing_if = "Option::is_none")]
cache_creation_input_tokens: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
cache_read_input_tokens: Option<u32>,
}
#[derive(Serialize, Deserialize, Debug)]
struct AnthropicContent {
text: Option<String>,
#[serde(rename = "type")]
content_type: Option<String>,
thinking: Option<String>,
name: Option<String>,
input: Option<serde_json::Value>,
id: Option<String>,
}
#[derive(Deserialize, Debug)]
struct AnthropicStreamResponse {
#[serde(rename = "type")]
response_type: String,
index: Option<usize>,
content_block: Option<AnthropicStreamContentBlock>,
delta: Option<AnthropicDelta>,
}
#[derive(Deserialize, Debug)]
struct AnthropicStreamContentBlock {
#[serde(rename = "type")]
block_type: String,
id: Option<String>,
name: Option<String>,
#[allow(dead_code)]
text: Option<String>,
}
#[derive(Deserialize, Debug)]
struct AnthropicDelta {
#[serde(rename = "type")]
delta_type: Option<String>,
text: Option<String>,
partial_json: Option<String>,
stop_reason: Option<String>,
}
impl std::fmt::Display for AnthropicCompleteResponse {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
for content in self.content.iter() {
match content.content_type {
Some(ref t) if t == "tool_use" => write!(
f,
"{{\n \"name\": {}, \"input\": {}\n}}",
content.name.clone().unwrap_or_default(),
content.input.clone().unwrap_or(serde_json::Value::Null)
)?,
Some(ref t) if t == "thinking" => {
write!(f, "{}", content.thinking.clone().unwrap_or_default())?
}
_ => write!(
f,
"{}",
self.content
.iter()
.map(|c| c.text.clone().unwrap_or_default())
.collect::<Vec<_>>()
.join("\n")
)?,
}
}
Ok(())
}
}
impl ChatResponse for AnthropicCompleteResponse {
fn text(&self) -> Option<String> {
Some(
self.content
.iter()
.filter_map(|c| {
if c.content_type == Some("text".to_string()) || c.content_type.is_none() {
c.text.clone()
} else {
None
}
})
.collect::<Vec<_>>()
.join("\n"),
)
}
fn thinking(&self) -> Option<String> {
self.content
.iter()
.find(|c| c.content_type == Some("thinking".to_string()))
.and_then(|c| c.thinking.clone())
}
fn tool_calls(&self) -> Option<Vec<ToolCall>> {
match self
.content
.iter()
.filter_map(|c| {
if c.content_type == Some("tool_use".to_string()) {
Some(ToolCall {
id: c.id.clone().unwrap_or_default(),
call_type: "function".to_string(),
function: FunctionCall {
name: c.name.clone().unwrap_or_default(),
arguments: serde_json::to_string(
&c.input.clone().unwrap_or(serde_json::Value::Null),
)
.unwrap_or_default(),
},
})
} else {
None
}
})
.collect::<Vec<ToolCall>>()
{
v if v.is_empty() => None,
v => Some(v),
}
}
fn usage(&self) -> Option<Usage> {
self.usage.as_ref().map(|anthropic_usage| {
let cached_tokens = anthropic_usage.cache_creation_input_tokens.unwrap_or(0)
+ anthropic_usage.cache_read_input_tokens.unwrap_or(0);
Usage {
prompt_tokens: anthropic_usage.input_tokens,
completion_tokens: anthropic_usage.output_tokens,
total_tokens: anthropic_usage.input_tokens + anthropic_usage.output_tokens,
completion_tokens_details: None,
prompt_tokens_details: if cached_tokens > 0 {
Some(crate::chat::PromptTokensDetails {
cached_tokens: Some(cached_tokens),
audio_tokens: None,
})
} else {
None
},
}
})
}
}
impl Anthropic {
fn convert_messages_to_anthropic<'a>(messages: &'a [ChatMessage]) -> Vec<AnthropicMessage<'a>> {
messages
.iter()
.map(|m| AnthropicMessage {
role: match m.role {
ChatRole::User => "user",
ChatRole::Assistant => "assistant",
},
content: match &m.message_type {
MessageType::Text => vec![MessageContent {
message_type: Some("text"),
text: Some(&m.content),
image_url: None,
source: None,
tool_use_id: None,
tool_input: None,
tool_name: None,
tool_result_id: None,
tool_output: None,
}],
MessageType::Pdf(raw_bytes) => {
vec![MessageContent {
message_type: Some("document"),
text: None,
image_url: None,
source: Some(ImageSource {
source_type: "base64",
media_type: "application/pdf",
data: BASE64.encode(raw_bytes),
}),
tool_use_id: None,
tool_input: None,
tool_name: None,
tool_result_id: None,
tool_output: None,
}]
}
MessageType::Image((image_mime, raw_bytes)) => {
vec![MessageContent {
message_type: Some("image"),
text: None,
image_url: None,
source: Some(ImageSource {
source_type: "base64",
media_type: image_mime.mime_type(),
data: BASE64.encode(raw_bytes),
}),
tool_use_id: None,
tool_input: None,
tool_name: None,
tool_result_id: None,
tool_output: None,
}]
}
MessageType::ImageURL(ref url) => vec![MessageContent {
message_type: Some("image_url"),
text: None,
image_url: Some(ImageUrlContent { url }),
source: None,
tool_use_id: None,
tool_input: None,
tool_name: None,
tool_result_id: None,
tool_output: None,
}],
MessageType::Audio(_) => vec![],
MessageType::ToolUse(calls) => calls
.iter()
.map(|c| MessageContent {
message_type: Some("tool_use"),
text: None,
image_url: None,
source: None,
tool_use_id: Some(c.id.clone()),
tool_input: Some(
serde_json::from_str(&c.function.arguments)
.unwrap_or(c.function.arguments.clone().into()),
),
tool_name: Some(c.function.name.clone()),
tool_result_id: None,
tool_output: None,
})
.collect(),
MessageType::ToolResult(responses) => responses
.iter()
.map(|r| MessageContent {
message_type: Some("tool_result"),
text: None,
image_url: None,
source: None,
tool_use_id: None,
tool_input: None,
tool_name: None,
tool_result_id: Some(r.id.clone()),
tool_output: Some(r.function.arguments.clone()),
})
.collect(),
},
})
.collect()
}
fn prepare_tools_and_choice<'a>(
tools: Option<&'a [Tool]>,
instance_tools: Option<&'a [Tool]>,
tool_choice: &Option<ToolChoice>,
) -> (
Option<Vec<AnthropicTool<'a>>>,
Option<HashMap<String, String>>,
) {
let maybe_tool_slice: Option<&[Tool]> = tools.or(instance_tools);
let anthropic_tools = maybe_tool_slice.map(|slice| {
slice
.iter()
.map(|tool| AnthropicTool {
name: &tool.function.name,
description: &tool.function.description,
schema: &tool.function.parameters,
cache_control: tool.cache_control.as_ref(),
})
.collect::<Vec<_>>()
});
let tool_choice_map = match tool_choice {
Some(ToolChoice::Auto) => {
Some(HashMap::from([("type".to_string(), "auto".to_string())]))
}
Some(ToolChoice::Any) => Some(HashMap::from([("type".to_string(), "any".to_string())])),
Some(ToolChoice::Tool(ref tool_name)) => Some(HashMap::from([
("type".to_string(), "tool".to_string()),
("name".to_string(), tool_name.clone()),
])),
Some(ToolChoice::None) => {
Some(HashMap::from([("type".to_string(), "none".to_string())]))
}
None => None,
};
let final_tool_choice = if anthropic_tools.is_some() {
tool_choice_map
} else {
None
};
(anthropic_tools, final_tool_choice)
}
fn system_to_request(system: &SystemPrompt) -> RequestSystemPrompt<'_> {
match system {
SystemPrompt::String(s) => RequestSystemPrompt::String(s),
SystemPrompt::Messages(msgs) => RequestSystemPrompt::Messages(msgs),
}
}
#[allow(clippy::too_many_arguments)]
pub fn new(
api_key: impl Into<String>,
model: Option<String>,
max_tokens: Option<u32>,
temperature: Option<f32>,
timeout_seconds: Option<u64>,
system: Option<SystemPrompt>,
top_p: Option<f32>,
top_k: Option<u32>,
tools: Option<Vec<Tool>>,
tool_choice: Option<ToolChoice>,
reasoning: Option<bool>,
thinking_budget_tokens: Option<u32>,
) -> Self {
let timeout = timeout_seconds.unwrap_or(30);
let mut builder = Client::builder();
if timeout > 0 {
builder = builder.timeout(std::time::Duration::from_secs(timeout));
}
Self::with_client(
builder.build().expect("Failed to build reqwest Client"),
api_key,
model,
max_tokens,
temperature,
timeout_seconds,
system,
top_p,
top_k,
tools,
tool_choice,
reasoning,
thinking_budget_tokens,
)
}
#[allow(clippy::too_many_arguments)]
pub fn with_client(
client: Client,
api_key: impl Into<String>,
model: Option<String>,
max_tokens: Option<u32>,
temperature: Option<f32>,
timeout_seconds: Option<u64>,
system: Option<SystemPrompt>,
top_p: Option<f32>,
top_k: Option<u32>,
tools: Option<Vec<Tool>>,
tool_choice: Option<ToolChoice>,
reasoning: Option<bool>,
thinking_budget_tokens: Option<u32>,
) -> Self {
Self {
config: Arc::new(AnthropicConfig {
api_key: api_key.into(),
model: model.unwrap_or_else(|| "claude-3-sonnet-20240229".to_string()),
max_tokens: max_tokens.unwrap_or(300),
temperature: temperature.unwrap_or(0.7),
system: system.unwrap_or_else(|| {
SystemPrompt::String("You are a helpful assistant.".to_string())
}),
timeout_seconds: timeout_seconds.unwrap_or(30),
top_p,
top_k,
tools,
tool_choice,
reasoning: reasoning.unwrap_or(false),
thinking_budget_tokens,
}),
client,
}
}
pub fn api_key(&self) -> &str {
&self.config.api_key
}
pub fn model(&self) -> &str {
&self.config.model
}
pub fn max_tokens(&self) -> u32 {
self.config.max_tokens
}
pub fn temperature(&self) -> f32 {
self.config.temperature
}
pub fn timeout_seconds(&self) -> u64 {
self.config.timeout_seconds
}
pub fn system(&self) -> &SystemPrompt {
&self.config.system
}
pub fn top_p(&self) -> Option<f32> {
self.config.top_p
}
pub fn top_k(&self) -> Option<u32> {
self.config.top_k
}
pub fn tools(&self) -> Option<&[Tool]> {
self.config.tools.as_deref()
}
pub fn tool_choice(&self) -> Option<&ToolChoice> {
self.config.tool_choice.as_ref()
}
pub fn reasoning(&self) -> bool {
self.config.reasoning
}
pub fn thinking_budget_tokens(&self) -> Option<u32> {
self.config.thinking_budget_tokens
}
pub fn client(&self) -> &Client {
&self.client
}
}
const AUDIO_UNSUPPORTED: &str = "Audio messages are not supported by Anthropic chat";
#[async_trait]
impl ChatProvider for Anthropic {
async fn chat_with_tools(
&self,
messages: &[ChatMessage],
tools: Option<&[Tool]>,
) -> Result<Box<dyn ChatResponse>, LLMError> {
crate::chat::ensure_no_audio(messages, AUDIO_UNSUPPORTED)?;
if self.config.api_key.is_empty() {
return Err(LLMError::AuthError("Missing Anthropic API key".to_string()));
}
let anthropic_messages = Self::convert_messages_to_anthropic(messages);
let (anthropic_tools, final_tool_choice) = Self::prepare_tools_and_choice(
tools,
self.config.tools.as_deref(),
&self.config.tool_choice,
);
let thinking = if self.config.reasoning {
Some(ThinkingConfig {
thinking_type: "enabled".to_string(),
budget_tokens: self.config.thinking_budget_tokens.unwrap_or(16000),
})
} else {
None
};
let system_prompt = Self::system_to_request(&self.config.system);
let req_body = AnthropicCompleteRequest {
messages: anthropic_messages,
model: &self.config.model,
max_tokens: Some(self.config.max_tokens),
temperature: Some(self.config.temperature),
system: Some(system_prompt),
stream: Some(false),
top_p: self.config.top_p,
top_k: self.config.top_k,
tools: anthropic_tools,
tool_choice: final_tool_choice,
thinking,
};
let mut request = self
.client
.post("https://api.anthropic.com/v1/messages")
.header("x-api-key", &self.config.api_key)
.header("Content-Type", "application/json")
.header("anthropic-version", "2023-06-01")
.json(&req_body);
if self.config.timeout_seconds > 0 {
request = request.timeout(std::time::Duration::from_secs(self.config.timeout_seconds));
}
if log::log_enabled!(log::Level::Trace) {
if let Ok(json) = serde_json::to_string(&req_body) {
log::trace!("Anthropic request payload: {}", json);
}
}
log::debug!("Anthropic request: POST /v1/messages");
let resp = request.send().await?;
log::debug!("Anthropic HTTP status: {}", resp.status());
let resp = resp.error_for_status()?;
let body = resp.text().await?;
let json_resp: AnthropicCompleteResponse = serde_json::from_str(&body)
.map_err(|e| LLMError::HttpError(format!("Failed to parse JSON: {e}")))?;
Ok(Box::new(json_resp))
}
async fn chat(&self, messages: &[ChatMessage]) -> Result<Box<dyn ChatResponse>, LLMError> {
self.chat_with_tools(messages, None).await
}
async fn chat_stream(
&self,
messages: &[ChatMessage],
) -> Result<std::pin::Pin<Box<dyn Stream<Item = Result<String, LLMError>> + Send>>, LLMError>
{
crate::chat::ensure_no_audio(messages, AUDIO_UNSUPPORTED)?;
if self.config.api_key.is_empty() {
return Err(LLMError::AuthError("Missing Anthropic API key".to_string()));
}
let anthropic_messages: Vec<AnthropicMessage> = messages
.iter()
.map(|m| AnthropicMessage {
role: match m.role {
ChatRole::User => "user",
ChatRole::Assistant => "assistant",
},
content: match &m.message_type {
MessageType::Text => vec![MessageContent {
message_type: Some("text"),
text: Some(&m.content),
image_url: None,
source: None,
tool_use_id: None,
tool_input: None,
tool_name: None,
tool_result_id: None,
tool_output: None,
}],
MessageType::Pdf(_) => unimplemented!(),
MessageType::Image((image_mime, raw_bytes)) => {
vec![MessageContent {
message_type: Some("image"),
text: None,
image_url: None,
source: Some(ImageSource {
source_type: "base64",
media_type: image_mime.mime_type(),
data: BASE64.encode(raw_bytes),
}),
tool_use_id: None,
tool_input: None,
tool_name: None,
tool_result_id: None,
tool_output: None,
}]
}
_ => vec![MessageContent {
message_type: Some("text"),
text: Some(&m.content),
image_url: None,
source: None,
tool_use_id: None,
tool_input: None,
tool_name: None,
tool_result_id: None,
tool_output: None,
}],
},
})
.collect();
let system_prompt = Self::system_to_request(&self.config.system);
let req_body = AnthropicCompleteRequest {
messages: anthropic_messages,
model: &self.config.model,
max_tokens: Some(self.config.max_tokens),
temperature: Some(self.config.temperature),
system: Some(system_prompt),
stream: Some(true),
top_p: self.config.top_p,
top_k: self.config.top_k,
tools: None,
tool_choice: None,
thinking: None,
};
let mut request = self
.client
.post("https://api.anthropic.com/v1/messages")
.header("x-api-key", &self.config.api_key)
.header("Content-Type", "application/json")
.header("anthropic-version", "2023-06-01")
.json(&req_body);
if self.config.timeout_seconds > 0 {
request = request.timeout(std::time::Duration::from_secs(self.config.timeout_seconds));
}
let response = request.send().await?;
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await?;
return Err(LLMError::ResponseFormatError {
message: format!("Anthropic API returned error status: {status}"),
raw_response: error_text,
});
}
Ok(crate::chat::create_sse_stream(
response,
parse_anthropic_sse_chunk,
))
}
async fn chat_stream_with_tools(
&self,
messages: &[ChatMessage],
tools: Option<&[Tool]>,
) -> Result<std::pin::Pin<Box<dyn Stream<Item = Result<StreamChunk, LLMError>> + Send>>, LLMError>
{
if self.config.api_key.is_empty() {
return Err(LLMError::AuthError("Missing Anthropic API key".to_string()));
}
let anthropic_messages = Self::convert_messages_to_anthropic(messages);
let (anthropic_tools, final_tool_choice) = Self::prepare_tools_and_choice(
tools,
self.config.tools.as_deref(),
&self.config.tool_choice,
);
let system_prompt = Self::system_to_request(&self.config.system);
let req_body = AnthropicCompleteRequest {
messages: anthropic_messages,
model: &self.config.model,
max_tokens: Some(self.config.max_tokens),
temperature: Some(self.config.temperature),
system: Some(system_prompt),
stream: Some(true),
top_p: self.config.top_p,
top_k: self.config.top_k,
tools: anthropic_tools,
tool_choice: final_tool_choice,
thinking: None, };
let mut request = self
.client
.post("https://api.anthropic.com/v1/messages")
.header("x-api-key", &self.config.api_key)
.header("Content-Type", "application/json")
.header("anthropic-version", "2023-06-01")
.json(&req_body);
if self.config.timeout_seconds > 0 {
request = request.timeout(std::time::Duration::from_secs(self.config.timeout_seconds));
}
if log::log_enabled!(log::Level::Trace) {
if let Ok(json) = serde_json::to_string(&req_body) {
log::trace!("Anthropic streaming request payload: {}", json);
}
}
log::debug!("Anthropic request: POST /v1/messages (streaming with tools)");
let response = request.send().await?;
log::debug!("Anthropic HTTP status: {}", response.status());
if !response.status().is_success() {
let status = response.status();
let error_text = response.text().await?;
return Err(LLMError::ResponseFormatError {
message: format!("Anthropic API returned error status: {status}"),
raw_response: error_text,
});
}
Ok(create_anthropic_tool_stream(response))
}
}
fn create_anthropic_tool_stream(
response: reqwest::Response,
) -> std::pin::Pin<Box<dyn Stream<Item = Result<StreamChunk, LLMError>> + Send>> {
use futures::stream::StreamExt;
let stream = response
.bytes_stream()
.scan(
(String::new(), Vec::new(), HashMap::new()),
move |(buffer, utf8_buffer, tool_states), chunk| {
let result = match chunk {
Ok(bytes) => {
utf8_buffer.extend_from_slice(&bytes);
match String::from_utf8(utf8_buffer.clone()) {
Ok(text) => {
buffer.push_str(&text);
utf8_buffer.clear();
}
Err(e) => {
let valid_up_to = e.utf8_error().valid_up_to();
if valid_up_to > 0 {
let valid =
String::from_utf8_lossy(&utf8_buffer[..valid_up_to]);
buffer.push_str(&valid);
utf8_buffer.drain(..valid_up_to);
}
}
}
let mut results = Vec::new();
while let Some(pos) = buffer.find("\n\n") {
let event = buffer[..pos + 2].to_string();
buffer.drain(..pos + 2);
match parse_anthropic_sse_chunk_with_tools(&event, tool_states) {
Ok(Some(chunk)) => results.push(Ok(chunk)),
Ok(None) => {}
Err(e) => results.push(Err(e)),
}
}
Some(results)
}
Err(e) => Some(vec![Err(LLMError::HttpError(e.to_string()))]),
};
async move { result }
},
)
.flat_map(futures::stream::iter);
Box::pin(stream)
}
#[async_trait]
impl CompletionProvider for Anthropic {
async fn complete(&self, _req: &CompletionRequest) -> Result<CompletionResponse, LLMError> {
unimplemented!()
}
}
#[async_trait]
impl EmbeddingProvider for Anthropic {
async fn embed(&self, _text: Vec<String>) -> Result<Vec<Vec<f32>>, LLMError> {
Err(LLMError::ProviderError(
"Embedding not supported".to_string(),
))
}
}
#[async_trait]
impl SpeechToTextProvider for Anthropic {
async fn transcribe(&self, _audio: Vec<u8>) -> Result<String, LLMError> {
Err(LLMError::ProviderError(
"Speech to text not supported".to_string(),
))
}
}
#[async_trait]
impl TextToSpeechProvider for Anthropic {}
#[derive(Clone, Debug, Deserialize)]
pub struct AnthropicModelListResponse {
data: Vec<AnthropicModelEntry>,
}
impl ModelListResponse for AnthropicModelListResponse {
fn get_models(&self) -> Vec<String> {
self.data.iter().map(|m| m.id.clone()).collect()
}
fn get_models_raw(&self) -> Vec<Box<dyn ModelListRawEntry>> {
self.data
.iter()
.map(|e| Box::new(e.clone()) as Box<dyn ModelListRawEntry>)
.collect()
}
fn get_backend(&self) -> LLMBackend {
LLMBackend::Anthropic
}
}
#[derive(Clone, Debug, Deserialize)]
pub struct AnthropicModelEntry {
created_at: DateTime<Utc>,
id: String,
#[serde(flatten)]
extra: Value,
}
impl ModelListRawEntry for AnthropicModelEntry {
fn get_id(&self) -> String {
self.id.clone()
}
fn get_created_at(&self) -> DateTime<Utc> {
self.created_at
}
fn get_raw(&self) -> Value {
self.extra.clone()
}
}
#[async_trait]
impl ModelsProvider for Anthropic {
async fn list_models(
&self,
_request: Option<&ModelListRequest>,
) -> Result<Box<dyn ModelListResponse>, LLMError> {
let resp = self
.client
.get("https://api.anthropic.com/v1/models")
.header("x-api-key", &self.config.api_key)
.header("Content-Type", "application/json")
.header("anthropic-version", "2023-06-01")
.send()
.await?;
let result: AnthropicModelListResponse = resp.json().await?;
Ok(Box::new(result))
}
}
impl crate::LLMProvider for Anthropic {
fn tools(&self) -> Option<&[Tool]> {
self.config.tools.as_deref()
}
}
fn parse_anthropic_sse_chunk(chunk: &str) -> Result<Option<String>, LLMError> {
for line in chunk.lines() {
let line = line.trim();
if let Some(data) = line.strip_prefix("data: ") {
match serde_json::from_str::<AnthropicStreamResponse>(data) {
Ok(response) => {
if response.response_type == "content_block_delta" {
if let Some(delta) = response.delta {
if let Some(text) = delta.text {
return Ok(Some(text));
}
}
}
return Ok(None);
}
Err(_) => continue,
}
}
}
Ok(None)
}
#[derive(Debug, Default)]
struct ToolUseState {
id: String,
name: String,
json_buffer: String,
}
fn parse_anthropic_sse_chunk_with_tools(
chunk: &str,
tool_states: &mut HashMap<usize, ToolUseState>,
) -> Result<Option<StreamChunk>, LLMError> {
for line in chunk.lines() {
let line = line.trim();
if let Some(data) = line.strip_prefix("data: ") {
match serde_json::from_str::<AnthropicStreamResponse>(data) {
Ok(response) => {
match response.response_type.as_str() {
"content_block_start" => {
if let (Some(index), Some(content_block)) =
(response.index, response.content_block)
{
if content_block.block_type == "tool_use" {
let id = content_block.id.unwrap_or_default();
let name = content_block.name.unwrap_or_default();
tool_states.insert(
index,
ToolUseState {
id: id.clone(),
name: name.clone(),
json_buffer: String::new(),
},
);
return Ok(Some(StreamChunk::ToolUseStart { index, id, name }));
}
}
}
"content_block_delta" => {
if let (Some(index), Some(delta)) = (response.index, response.delta) {
match delta.delta_type.as_deref() {
Some("text_delta") => {
if let Some(text) = delta.text {
return Ok(Some(StreamChunk::Text(text)));
}
}
Some("input_json_delta") => {
if let Some(partial_json) = delta.partial_json {
if let Some(state) = tool_states.get_mut(&index) {
state.json_buffer.push_str(&partial_json);
}
return Ok(Some(StreamChunk::ToolUseInputDelta {
index,
partial_json,
}));
}
}
_ => {}
}
}
}
"content_block_stop" => {
if let Some(index) = response.index {
if let Some(state) = tool_states.remove(&index) {
let arguments = if state.json_buffer.is_empty() {
"{}".to_string()
} else {
state.json_buffer
};
let tool_call = ToolCall {
id: state.id,
call_type: "function".to_string(),
function: FunctionCall {
name: state.name,
arguments,
},
};
return Ok(Some(StreamChunk::ToolUseComplete {
index,
tool_call,
}));
}
}
}
"message_delta" => {
if let Some(delta) = response.delta {
if let Some(stop_reason) = delta.stop_reason {
return Ok(Some(StreamChunk::Done { stop_reason }));
}
}
}
_ => {}
}
return Ok(None);
}
Err(_) => continue,
}
}
}
Ok(None)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_stream_text_delta() {
let chunk = r#"event: content_block_delta
data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello"}}
"#;
let mut tool_states = HashMap::new();
let result = parse_anthropic_sse_chunk_with_tools(chunk, &mut tool_states).unwrap();
match result {
Some(StreamChunk::Text(text)) => assert_eq!(text, "Hello"),
_ => panic!("Expected Text chunk, got {:?}", result),
}
}
#[test]
fn test_parse_stream_tool_use_start() {
let chunk = r#"event: content_block_start
data: {"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "toolu_01ABC", "name": "get_weather", "input": {}}}
"#;
let mut tool_states = HashMap::new();
let result = parse_anthropic_sse_chunk_with_tools(chunk, &mut tool_states).unwrap();
match result {
Some(StreamChunk::ToolUseStart { index, id, name }) => {
assert_eq!(index, 1);
assert_eq!(id, "toolu_01ABC");
assert_eq!(name, "get_weather");
}
_ => panic!("Expected ToolUseStart chunk, got {:?}", result),
}
assert!(tool_states.contains_key(&1));
assert_eq!(tool_states[&1].id, "toolu_01ABC");
assert_eq!(tool_states[&1].name, "get_weather");
}
#[test]
fn test_parse_stream_tool_use_input_delta() {
let chunk = r#"event: content_block_delta
data: {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": "{\"location\":"}}
"#;
let mut tool_states = HashMap::new();
tool_states.insert(
1,
ToolUseState {
id: "toolu_01ABC".to_string(),
name: "get_weather".to_string(),
json_buffer: String::new(),
},
);
let result = parse_anthropic_sse_chunk_with_tools(chunk, &mut tool_states).unwrap();
match result {
Some(StreamChunk::ToolUseInputDelta {
index,
partial_json,
}) => {
assert_eq!(index, 1);
assert_eq!(partial_json, "{\"location\":");
}
_ => panic!("Expected ToolUseInputDelta chunk, got {:?}", result),
}
assert_eq!(tool_states[&1].json_buffer, "{\"location\":");
}
#[test]
fn test_parse_stream_tool_use_complete() {
let chunk = r#"event: content_block_stop
data: {"type": "content_block_stop", "index": 1}
"#;
let mut tool_states = HashMap::new();
tool_states.insert(
1,
ToolUseState {
id: "toolu_01ABC".to_string(),
name: "get_weather".to_string(),
json_buffer: r#"{"location": "Paris"}"#.to_string(),
},
);
let result = parse_anthropic_sse_chunk_with_tools(chunk, &mut tool_states).unwrap();
match result {
Some(StreamChunk::ToolUseComplete { index, tool_call }) => {
assert_eq!(index, 1);
assert_eq!(tool_call.id, "toolu_01ABC");
assert_eq!(tool_call.function.name, "get_weather");
assert_eq!(tool_call.function.arguments, r#"{"location": "Paris"}"#);
}
_ => panic!("Expected ToolUseComplete chunk, got {:?}", result),
}
assert!(!tool_states.contains_key(&1));
}
#[test]
fn test_parse_stream_tool_use_complete_empty_arguments() {
let chunk = r#"event: content_block_stop
data: {"type": "content_block_stop", "index": 1}
"#;
let mut tool_states = HashMap::new();
tool_states.insert(
1,
ToolUseState {
id: "toolu_01XYZ".to_string(),
name: "get_current_time".to_string(),
json_buffer: String::new(), },
);
let result = parse_anthropic_sse_chunk_with_tools(chunk, &mut tool_states).unwrap();
match result {
Some(StreamChunk::ToolUseComplete { index, tool_call }) => {
assert_eq!(index, 1);
assert_eq!(tool_call.id, "toolu_01XYZ");
assert_eq!(tool_call.function.name, "get_current_time");
assert_eq!(
tool_call.function.arguments, "{}",
"Empty arguments should default to '{{}}' not empty string"
);
}
_ => panic!("Expected ToolUseComplete chunk, got {:?}", result),
}
assert!(!tool_states.contains_key(&1));
}
#[test]
fn test_parse_stream_done_tool_use() {
let chunk = r#"event: message_delta
data: {"type": "message_delta", "delta": {"stop_reason": "tool_use"}}
"#;
let mut tool_states = HashMap::new();
let result = parse_anthropic_sse_chunk_with_tools(chunk, &mut tool_states).unwrap();
match result {
Some(StreamChunk::Done { stop_reason }) => {
assert_eq!(stop_reason, "tool_use");
}
_ => panic!("Expected Done chunk, got {:?}", result),
}
}
#[test]
fn test_parse_stream_done_end_turn() {
let chunk = r#"event: message_delta
data: {"type": "message_delta", "delta": {"stop_reason": "end_turn"}}
"#;
let mut tool_states = HashMap::new();
let result = parse_anthropic_sse_chunk_with_tools(chunk, &mut tool_states).unwrap();
match result {
Some(StreamChunk::Done { stop_reason }) => {
assert_eq!(stop_reason, "end_turn");
}
_ => panic!("Expected Done chunk, got {:?}", result),
}
}
#[test]
fn test_parse_stream_full_tool_use_sequence() {
let mut tool_states = HashMap::new();
let start_chunk = r#"event: content_block_start
data: {"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "toolu_01ABC", "name": "get_weather", "input": {}}}
"#;
let result = parse_anthropic_sse_chunk_with_tools(start_chunk, &mut tool_states).unwrap();
assert!(matches!(result, Some(StreamChunk::ToolUseStart { .. })));
let delta1 = r#"event: content_block_delta
data: {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": "{\"loc"}}
"#;
let _ = parse_anthropic_sse_chunk_with_tools(delta1, &mut tool_states).unwrap();
let delta2 = r#"event: content_block_delta
data: {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": "ation\": \"Paris\"}"}}
"#;
let _ = parse_anthropic_sse_chunk_with_tools(delta2, &mut tool_states).unwrap();
assert_eq!(tool_states[&1].json_buffer, "{\"location\": \"Paris\"}");
let stop_chunk = r#"event: content_block_stop
data: {"type": "content_block_stop", "index": 1}
"#;
let result = parse_anthropic_sse_chunk_with_tools(stop_chunk, &mut tool_states).unwrap();
match result {
Some(StreamChunk::ToolUseComplete { tool_call, .. }) => {
assert_eq!(tool_call.function.arguments, "{\"location\": \"Paris\"}");
}
_ => panic!("Expected ToolUseComplete"),
}
let done_chunk = r#"event: message_delta
data: {"type": "message_delta", "delta": {"stop_reason": "tool_use"}}
"#;
let result = parse_anthropic_sse_chunk_with_tools(done_chunk, &mut tool_states).unwrap();
assert!(matches!(
result,
Some(StreamChunk::Done {
stop_reason
}) if stop_reason == "tool_use"
));
}
#[test]
fn test_parse_stream_mixed_text_and_tool() {
let mut tool_states = HashMap::new();
let text_chunk = r#"event: content_block_delta
data: {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "I'll check the weather"}}
"#;
let result = parse_anthropic_sse_chunk_with_tools(text_chunk, &mut tool_states).unwrap();
assert!(matches!(result, Some(StreamChunk::Text(t)) if t == "I'll check the weather"));
let tool_start = r#"event: content_block_start
data: {"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "toolu_01XYZ", "name": "weather", "input": {}}}
"#;
let result = parse_anthropic_sse_chunk_with_tools(tool_start, &mut tool_states).unwrap();
assert!(
matches!(result, Some(StreamChunk::ToolUseStart { name, .. }) if name == "weather")
);
}
#[test]
fn test_parse_stream_ignores_message_start() {
let chunk = r#"event: message_start
data: {"type": "message_start", "message": {"id": "msg_123", "type": "message", "role": "assistant"}}
"#;
let mut tool_states = HashMap::new();
let result = parse_anthropic_sse_chunk_with_tools(chunk, &mut tool_states).unwrap();
assert!(result.is_none());
}
#[test]
fn test_parse_stream_ignores_ping() {
let chunk = r#"event: ping
data: {"type": "ping"}
"#;
let mut tool_states = HashMap::new();
let result = parse_anthropic_sse_chunk_with_tools(chunk, &mut tool_states).unwrap();
assert!(result.is_none());
}
#[test]
fn test_anthropic_tool_serialization_without_cache_control() {
let schema = serde_json::json!({"type": "object", "properties": {}});
let tool = AnthropicTool {
name: "my_tool",
description: "desc",
schema: &schema,
cache_control: None,
};
let json = serde_json::to_value(&tool).unwrap();
assert_eq!(json.get("name").unwrap(), "my_tool");
assert!(json.get("cache_control").is_none());
}
#[test]
fn test_anthropic_tool_serialization_with_cache_control() {
let schema = serde_json::json!({"type": "object", "properties": {}});
let cc = serde_json::json!({"type": "ephemeral"});
let tool = AnthropicTool {
name: "my_tool",
description: "desc",
schema: &schema,
cache_control: Some(&cc),
};
let json = serde_json::to_value(&tool).unwrap();
assert_eq!(
json.get("cache_control").unwrap(),
&serde_json::json!({"type": "ephemeral"})
);
}
#[test]
fn test_prepare_tools_maps_cache_control() {
let tools = vec![Tool {
tool_type: "function".to_string(),
function: crate::chat::FunctionTool {
name: "get_weather".to_string(),
description: "Get weather".to_string(),
parameters: serde_json::json!({"type": "object", "properties": {}}),
},
cache_control: Some(serde_json::json!({"type": "ephemeral"})),
}];
let (anthropic_tools, _) =
Anthropic::prepare_tools_and_choice(Some(&tools), None, &None);
let anthropic_tools = anthropic_tools.expect("tools should be present");
assert_eq!(anthropic_tools.len(), 1);
assert_eq!(
anthropic_tools[0].cache_control,
Some(&serde_json::json!({"type": "ephemeral"}))
);
}
#[test]
fn test_prepare_tools_none_cache_control() {
let tools = vec![Tool {
tool_type: "function".to_string(),
function: crate::chat::FunctionTool {
name: "get_weather".to_string(),
description: "Get weather".to_string(),
parameters: serde_json::json!({"type": "object", "properties": {}}),
},
cache_control: None,
}];
let (anthropic_tools, _) =
Anthropic::prepare_tools_and_choice(Some(&tools), None, &None);
let anthropic_tools = anthropic_tools.expect("tools should be present");
assert!(anthropic_tools[0].cache_control.is_none());
}
}