use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use super::tool_schema;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
#[derive(Default)]
pub enum Role {
System,
#[default]
User,
Assistant,
Tool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum ContentPart {
Text { text: String },
Image { image_url: ImageUrl },
Audio { audio: AudioData },
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ImageUrl {
pub url: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub detail: Option<ImageDetail>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ImageDetail {
Low,
High,
Auto,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AudioData {
pub data: String,
pub format: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatMessage {
pub role: Role,
#[serde(skip_serializing_if = "Option::is_none")]
pub content: Option<MessageContent>,
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_calls: Option<Vec<ToolCall>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_call_id: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum MessageContent {
Text(String),
Parts(Vec<ContentPart>),
}
impl ChatMessage {
pub fn system(content: impl Into<String>) -> Self {
Self {
role: Role::System,
content: Some(MessageContent::Text(content.into())),
name: None,
tool_calls: None,
tool_call_id: None,
}
}
pub fn user(content: impl Into<String>) -> Self {
Self {
role: Role::User,
content: Some(MessageContent::Text(content.into())),
name: None,
tool_calls: None,
tool_call_id: None,
}
}
pub fn user_with_content(content: MessageContent) -> Self {
Self {
role: Role::User,
content: Some(content),
name: None,
tool_calls: None,
tool_call_id: None,
}
}
pub fn user_with_parts(parts: Vec<ContentPart>) -> Self {
Self::user_with_content(MessageContent::Parts(parts))
}
pub fn assistant(content: impl Into<String>) -> Self {
Self {
role: Role::Assistant,
content: Some(MessageContent::Text(content.into())),
name: None,
tool_calls: None,
tool_call_id: None,
}
}
pub fn assistant_with_tool_calls(tool_calls: Vec<ToolCall>) -> Self {
Self {
role: Role::Assistant,
content: None,
name: None,
tool_calls: Some(tool_calls),
tool_call_id: None,
}
}
pub fn tool_result(tool_call_id: impl Into<String>, content: impl Into<String>) -> Self {
Self {
role: Role::Tool,
content: Some(MessageContent::Text(content.into())),
name: None,
tool_calls: None,
tool_call_id: Some(tool_call_id.into()),
}
}
pub fn user_with_image(text: impl Into<String>, image_url: impl Into<String>) -> Self {
Self {
role: Role::User,
content: Some(MessageContent::Parts(vec![
ContentPart::Text { text: text.into() },
ContentPart::Image {
image_url: ImageUrl {
url: image_url.into(),
detail: None,
},
},
])),
name: None,
tool_calls: None,
tool_call_id: None,
}
}
pub fn text_content(&self) -> Option<&str> {
match &self.content {
Some(MessageContent::Text(s)) => Some(s),
Some(MessageContent::Parts(parts)) => {
for part in parts {
if let ContentPart::Text { text } = part {
return Some(text);
}
}
None
}
None => None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolCall {
pub id: String,
#[serde(rename = "type")]
pub call_type: String,
pub function: FunctionCall,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FunctionCall {
pub name: String,
pub arguments: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Tool {
#[serde(rename = "type")]
pub tool_type: String,
pub function: FunctionDefinition,
}
impl Tool {
pub fn function(
name: impl Into<String>,
description: impl Into<String>,
parameters: serde_json::Value,
) -> Self {
let parameters = tool_schema::normalize_schema(parameters);
Self {
tool_type: "function".to_string(),
function: FunctionDefinition {
name: name.into(),
description: Some(description.into()),
parameters: Some(parameters),
strict: None,
},
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FunctionDefinition {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub parameters: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub strict: Option<bool>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum ToolChoice {
Auto,
None,
Required,
Specific {
#[serde(rename = "type")]
choice_type: String,
function: ToolChoiceFunction,
},
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolChoiceFunction {
pub name: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ChatCompletionRequest {
pub model: String,
pub messages: Vec<ChatMessage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub temperature: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub top_p: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stop: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stream: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tools: Option<Vec<Tool>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_choice: Option<ToolChoice>,
#[serde(skip_serializing_if = "Option::is_none")]
pub frequency_penalty: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub presence_penalty: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub user: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub response_format: Option<ResponseFormat>,
#[serde(flatten)]
pub extra: HashMap<String, serde_json::Value>,
}
impl ChatCompletionRequest {
pub fn new(model: impl Into<String>) -> Self {
Self {
model: model.into(),
..Default::default()
}
}
pub fn message(mut self, message: ChatMessage) -> Self {
self.messages.push(message);
self
}
pub fn system(mut self, content: impl Into<String>) -> Self {
self.messages.push(ChatMessage::system(content));
self
}
pub fn user(mut self, content: impl Into<String>) -> Self {
self.messages.push(ChatMessage::user(content));
self
}
pub fn temperature(mut self, temp: f32) -> Self {
self.temperature = Some(temp);
self
}
pub fn max_tokens(mut self, tokens: u32) -> Self {
self.max_tokens = Some(tokens);
self
}
pub fn tool(mut self, tool: Tool) -> Self {
self.tools.get_or_insert_with(Vec::new).push(tool);
self
}
pub fn tools(mut self, tools: Vec<Tool>) -> Self {
self.tools = Some(tools);
self
}
pub fn stream(mut self) -> Self {
self.stream = Some(true);
self
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResponseFormat {
#[serde(rename = "type")]
pub format_type: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub json_schema: Option<serde_json::Value>,
}
impl ResponseFormat {
pub fn text() -> Self {
Self {
format_type: "text".to_string(),
json_schema: None,
}
}
pub fn json() -> Self {
Self {
format_type: "json_object".to_string(),
json_schema: None,
}
}
pub fn json_schema(schema: serde_json::Value) -> Self {
Self {
format_type: "json_schema".to_string(),
json_schema: Some(schema),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatCompletionResponse {
pub id: String,
pub object: String,
pub created: u64,
pub model: String,
pub choices: Vec<Choice>,
#[serde(skip_serializing_if = "Option::is_none")]
pub usage: Option<Usage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub system_fingerprint: Option<String>,
}
impl ChatCompletionResponse {
pub fn content(&self) -> Option<&str> {
self.choices.first()?.message.text_content()
}
pub fn tool_calls(&self) -> Option<&Vec<ToolCall>> {
self.choices.first()?.message.tool_calls.as_ref()
}
pub fn has_tool_calls(&self) -> bool {
self.tool_calls().map(|t| !t.is_empty()).unwrap_or(false)
}
pub fn finish_reason(&self) -> Option<&FinishReason> {
self.choices.first()?.finish_reason.as_ref()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Choice {
pub index: u32,
pub message: ChatMessage,
#[serde(skip_serializing_if = "Option::is_none")]
pub finish_reason: Option<FinishReason>,
#[serde(skip_serializing_if = "Option::is_none")]
pub logprobs: Option<serde_json::Value>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FinishReason {
Stop,
Length,
ToolCalls,
ContentFilter,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Usage {
pub prompt_tokens: u32,
pub completion_tokens: u32,
pub total_tokens: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LLMResponseMetadata {
pub id: String,
pub model: String,
pub prompt_tokens: u32,
pub completion_tokens: u32,
pub total_tokens: u32,
}
impl From<&ChatCompletionResponse> for LLMResponseMetadata {
fn from(resp: &ChatCompletionResponse) -> Self {
let usage = resp.usage.as_ref();
Self {
id: resp.id.clone(),
model: resp.model.clone(),
prompt_tokens: usage.map(|u| u.prompt_tokens).unwrap_or(0),
completion_tokens: usage.map(|u| u.completion_tokens).unwrap_or(0),
total_tokens: usage.map(|u| u.total_tokens).unwrap_or(0),
}
}
}
impl From<&ChatCompletionChunk> for LLMResponseMetadata {
fn from(chunk: &ChatCompletionChunk) -> Self {
let usage = chunk.usage.as_ref();
Self {
id: chunk.id.clone(),
model: chunk.model.clone(),
prompt_tokens: usage.map(|u| u.prompt_tokens).unwrap_or(0),
completion_tokens: usage.map(|u| u.completion_tokens).unwrap_or(0),
total_tokens: usage.map(|u| u.total_tokens).unwrap_or(0),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatCompletionChunk {
pub id: String,
pub object: String,
pub created: u64,
pub model: String,
pub choices: Vec<ChunkChoice>,
#[serde(skip_serializing_if = "Option::is_none")]
pub usage: Option<Usage>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChunkChoice {
pub index: u32,
pub delta: ChunkDelta,
#[serde(skip_serializing_if = "Option::is_none")]
pub finish_reason: Option<FinishReason>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ChunkDelta {
#[serde(skip_serializing_if = "Option::is_none")]
pub role: Option<Role>,
#[serde(skip_serializing_if = "Option::is_none")]
pub content: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_calls: Option<Vec<ToolCallDelta>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolCallDelta {
pub index: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
#[serde(rename = "type", skip_serializing_if = "Option::is_none")]
pub call_type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub function: Option<FunctionCallDelta>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FunctionCallDelta {
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub arguments: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EmbeddingRequest {
pub model: String,
pub input: EmbeddingInput,
#[serde(skip_serializing_if = "Option::is_none")]
pub encoding_format: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dimensions: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub user: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum EmbeddingInput {
Single(String),
Multiple(Vec<String>),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EmbeddingResponse {
pub object: String,
pub model: String,
pub data: Vec<EmbeddingData>,
pub usage: EmbeddingUsage,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EmbeddingData {
pub object: String,
pub index: u32,
pub embedding: Vec<f32>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EmbeddingUsage {
pub prompt_tokens: u32,
pub total_tokens: u32,
}
#[derive(Debug, Clone, thiserror::Error)]
pub enum LLMError {
#[error("API error: {message} (code: {code:?})")]
ApiError {
code: Option<String>,
message: String,
},
#[error("Authentication failed: {0}")]
AuthError(String),
#[error("Rate limited: {0}")]
RateLimited(String),
#[error("Quota exceeded: {0}")]
QuotaExceeded(String),
#[error("Model not found: {0}")]
ModelNotFound(String),
#[error("Context length exceeded: {0}")]
ContextLengthExceeded(String),
#[error("Content filtered: {0}")]
ContentFiltered(String),
#[error("Network error: {0}")]
NetworkError(String),
#[error("Request timeout: {0}")]
Timeout(String),
#[error("Serialization error: {0}")]
SerializationError(String),
#[error("Configuration error: {0}")]
ConfigError(String),
#[error("Provider not supported: {0}")]
ProviderNotSupported(String),
#[error("LLM error: {0}")]
Other(String),
}
pub type LLMResult<T> = Result<T, LLMError>;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
pub enum RetryStrategy {
NoRetry,
#[default]
DirectRetry,
PromptRetry,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum BackoffStrategy {
Fixed { delay_ms: u64 },
Linear {
initial_delay_ms: u64,
increment_ms: u64,
},
Exponential {
initial_delay_ms: u64,
max_delay_ms: u64,
},
ExponentialWithJitter {
initial_delay_ms: u64,
max_delay_ms: u64,
jitter_ms: u64,
},
}
impl Default for BackoffStrategy {
fn default() -> Self {
Self::ExponentialWithJitter {
initial_delay_ms: 1000,
max_delay_ms: 30000,
jitter_ms: 500,
}
}
}
impl BackoffStrategy {
pub fn delay(&self, attempt: u32) -> std::time::Duration {
match self {
Self::Fixed { delay_ms } => std::time::Duration::from_millis(*delay_ms),
Self::Linear {
initial_delay_ms,
increment_ms,
} => {
let delay = *initial_delay_ms + (*increment_ms * attempt as u64);
std::time::Duration::from_millis(delay)
}
Self::Exponential {
initial_delay_ms,
max_delay_ms,
} => {
let delay = *initial_delay_ms * 2u64.pow(attempt.min(10));
let capped = delay.min(*max_delay_ms);
std::time::Duration::from_millis(capped)
}
Self::ExponentialWithJitter {
initial_delay_ms,
max_delay_ms,
jitter_ms,
} => {
let base_delay = *initial_delay_ms * 2u64.pow(attempt.min(10));
let capped = base_delay.min(*max_delay_ms);
let jitter = if *jitter_ms > 0 {
use rand::Rng;
let mut rng = rand::thread_rng();
rng.gen_range(0..*jitter_ms) as i64 - (*jitter_ms as i64 / 2)
} else {
0
};
let final_delay = (capped as i64 + jitter).max(0) as u64;
std::time::Duration::from_millis(final_delay)
}
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub enum RetryableErrorType {
Network,
RateLimit,
Serialization,
Authentication,
ServerError,
ContextLength,
ContentFiltered,
}
impl RetryableErrorType {
pub fn from_error(error: &LLMError) -> Option<Self> {
match error {
LLMError::NetworkError(_) => Some(Self::Network),
LLMError::Timeout(_) => Some(Self::Network),
LLMError::RateLimited(_) => Some(Self::RateLimit),
LLMError::SerializationError(_) => Some(Self::Serialization),
LLMError::AuthError(_) => Some(Self::Authentication),
LLMError::ApiError { code, .. } => {
if let Some(c) = code {
if c.starts_with('5') {
return Some(Self::ServerError);
}
}
None
}
LLMError::ContextLengthExceeded(_) => Some(Self::ContextLength),
LLMError::ContentFiltered(_) => Some(Self::ContentFiltered),
LLMError::QuotaExceeded(_)
| LLMError::ModelNotFound(_)
| LLMError::ConfigError(_)
| LLMError::ProviderNotSupported(_)
| LLMError::Other(_) => None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LLMRetryPolicy {
pub max_attempts: u32,
pub backoff: BackoffStrategy,
pub default_strategy: RetryStrategy,
pub error_strategies: std::collections::HashMap<RetryableErrorType, RetryStrategy>,
pub retry_on: Vec<RetryableErrorType>,
}
impl Default for LLMRetryPolicy {
fn default() -> Self {
Self {
max_attempts: 3,
backoff: BackoffStrategy::default(),
default_strategy: RetryStrategy::PromptRetry,
error_strategies: Self::default_strategies(),
retry_on: vec![
RetryableErrorType::Network,
RetryableErrorType::RateLimit,
RetryableErrorType::Serialization,
RetryableErrorType::ServerError,
],
}
}
}
impl LLMRetryPolicy {
fn default_strategies() -> std::collections::HashMap<RetryableErrorType, RetryStrategy> {
let mut map = std::collections::HashMap::new();
map.insert(
RetryableErrorType::Serialization,
RetryStrategy::PromptRetry,
);
map.insert(RetryableErrorType::Network, RetryStrategy::DirectRetry);
map.insert(RetryableErrorType::RateLimit, RetryStrategy::DirectRetry);
map.insert(RetryableErrorType::ServerError, RetryStrategy::DirectRetry);
map.insert(RetryableErrorType::Authentication, RetryStrategy::NoRetry);
map.insert(RetryableErrorType::ContextLength, RetryStrategy::NoRetry);
map.insert(RetryableErrorType::ContentFiltered, RetryStrategy::NoRetry);
map
}
pub fn no_retry() -> Self {
Self {
max_attempts: 1,
backoff: BackoffStrategy::Fixed { delay_ms: 0 },
default_strategy: RetryStrategy::NoRetry,
error_strategies: Self::default_strategies(),
retry_on: vec![],
}
}
pub fn with_max_attempts(max: u32) -> Self {
Self {
max_attempts: max.max(1),
..Default::default()
}
}
pub fn with_backoff(mut self, backoff: BackoffStrategy) -> Self {
self.backoff = backoff;
self
}
pub fn strategy_for_error(&self, error: &LLMError) -> RetryStrategy {
RetryableErrorType::from_error(error)
.and_then(|error_type| self.error_strategies.get(&error_type).cloned())
.unwrap_or_else(|| self.default_strategy.clone())
}
pub fn should_retry_error(&self, error: &LLMError) -> bool {
if let Some(error_type) = RetryableErrorType::from_error(error) {
self.retry_on.contains(&error_type)
} else {
false
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JSONValidationError {
pub raw_content: String,
pub parse_error: String,
pub expected_schema: Option<serde_json::Value>,
}
impl std::fmt::Display for JSONValidationError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"JSON validation failed: {}. Raw content (first 200 chars): {}",
self.parse_error,
self.raw_content.chars().take(200).collect::<String>()
)
}
}
impl std::error::Error for JSONValidationError {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_backoff_fixed() {
let strategy = BackoffStrategy::Fixed { delay_ms: 1000 };
assert_eq!(strategy.delay(0).as_millis(), 1000);
assert_eq!(strategy.delay(5).as_millis(), 1000);
}
#[test]
fn test_backoff_linear() {
let strategy = BackoffStrategy::Linear {
initial_delay_ms: 1000,
increment_ms: 500,
};
assert_eq!(strategy.delay(0).as_millis(), 1000);
assert_eq!(strategy.delay(1).as_millis(), 1500);
assert_eq!(strategy.delay(2).as_millis(), 2000);
}
#[test]
fn test_backoff_exponential_capping() {
let strategy = BackoffStrategy::Exponential {
initial_delay_ms: 1000,
max_delay_ms: 5000,
};
assert_eq!(strategy.delay(0).as_millis(), 1000);
assert_eq!(strategy.delay(1).as_millis(), 2000);
assert_eq!(strategy.delay(2).as_millis(), 4000);
assert_eq!(strategy.delay(3).as_millis(), 5000); assert_eq!(strategy.delay(10).as_millis(), 5000); }
#[test]
fn test_backoff_jitter_range() {
let strategy = BackoffStrategy::ExponentialWithJitter {
initial_delay_ms: 1000,
max_delay_ms: 10000,
jitter_ms: 200,
};
let delay = strategy.delay(1).as_millis();
assert!(
delay >= 1800 && delay <= 2200,
"Delay {} out of range",
delay
);
}
#[test]
fn test_strategy_selection() {
let policy = LLMRetryPolicy::default();
let serde_err = LLMError::SerializationError("Invalid JSON".to_string());
assert_eq!(
policy.strategy_for_error(&serde_err),
RetryStrategy::PromptRetry
);
let net_err = LLMError::NetworkError("Connection failed".to_string());
assert_eq!(
policy.strategy_for_error(&net_err),
RetryStrategy::DirectRetry
);
}
#[test]
fn test_retryable_error_type_mapping() {
let net_err = LLMError::NetworkError("error".to_string());
assert_eq!(
RetryableErrorType::from_error(&net_err),
Some(RetryableErrorType::Network)
);
let rate_err = LLMError::RateLimited("error".to_string());
assert_eq!(
RetryableErrorType::from_error(&rate_err),
Some(RetryableErrorType::RateLimit)
);
let auth_err = LLMError::AuthError("error".to_string());
assert_eq!(
RetryableErrorType::from_error(&auth_err),
Some(RetryableErrorType::Authentication)
);
let quota_err = LLMError::QuotaExceeded("error".to_string());
assert_eq!(RetryableErrorType::from_error("a_err), None);
}
#[test]
fn test_no_retry_policy() {
let policy = LLMRetryPolicy::no_retry();
assert_eq!(policy.max_attempts, 1);
assert!(!policy.should_retry_error(&LLMError::NetworkError("error".to_string())));
}
}