litellm-rs 0.1.1

A high-performance AI Gateway written in Rust, providing OpenAI-compatible APIs with intelligent routing, load balancing, and enterprise features
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

//! Provider configuration

use super::*;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;

/// Provider configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProviderConfig {
    /// Provider name
    pub name: String,
    /// Provider type (openai, anthropic, etc.)
    pub provider_type: String,
    /// API key
    pub api_key: String,
    /// Base URL
    pub base_url: Option<String>,
    /// API version
    pub api_version: Option<String>,
    /// Organization ID
    pub organization: Option<String>,
    /// Project ID
    pub project: Option<String>,
    /// Provider weight for load balancing
    #[serde(default = "default_weight")]
    pub weight: f32,
    /// Maximum requests per minute
    #[serde(default = "default_rpm")]
    pub rpm: u32,
    /// Maximum tokens per minute
    #[serde(default = "default_tpm")]
    pub tpm: u32,
    /// Maximum concurrent requests
    #[serde(default = "default_max_connections")]
    pub max_concurrent_requests: u32,
    /// Request timeout in seconds
    #[serde(default = "default_timeout")]
    pub timeout: u64,
    /// Maximum retries
    #[serde(default = "default_max_retries")]
    pub max_retries: u32,
    /// Retry configuration
    #[serde(default)]
    pub retry: RetryConfig,
    /// Health check configuration
    #[serde(default)]
    pub health_check: HealthCheckConfig,
    /// Provider-specific settings
    #[serde(default)]
    pub settings: HashMap<String, serde_json::Value>,
    /// Supported models
    #[serde(default)]
    pub models: Vec<String>,
    /// Tags for grouping providers
    #[serde(default)]
    pub tags: Vec<String>,
    /// Whether provider is enabled
    #[serde(default = "default_true")]
    pub enabled: bool,
}

impl Default for ProviderConfig {
    fn default() -> Self {
        Self {
            name: String::new(),
            provider_type: String::new(),
            api_key: String::new(),
            base_url: None,
            api_version: None,
            organization: None,
            project: None,
            weight: default_weight(),
            rpm: default_rpm(),
            tpm: default_tpm(),
            max_concurrent_requests: default_max_connections(),
            timeout: default_timeout(),
            max_retries: default_max_retries(),
            retry: RetryConfig::default(),
            health_check: HealthCheckConfig::default(),
            settings: HashMap::new(),
            models: Vec::new(),
            tags: Vec::new(),
            enabled: true,
        }
    }
}

/// Retry configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RetryConfig {
    /// Base delay in milliseconds
    #[serde(default = "default_base_delay")]
    pub base_delay: u64,
    /// Maximum delay in milliseconds
    #[serde(default = "default_max_delay")]
    pub max_delay: u64,
    /// Backoff multiplier
    #[serde(default = "default_backoff_multiplier")]
    pub backoff_multiplier: f64,
    /// Jitter factor (0.0 to 1.0)
    #[serde(default)]
    pub jitter: f64,
}

impl Default for RetryConfig {
    fn default() -> Self {
        Self {
            base_delay: default_base_delay(),
            max_delay: default_max_delay(),
            backoff_multiplier: default_backoff_multiplier(),
            jitter: 0.1,
        }
    }
}

/// Health check configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthCheckConfig {
    /// Health check interval in seconds
    #[serde(default = "default_health_check_interval")]
    pub interval: u64,
    /// Failure threshold before marking unhealthy
    #[serde(default = "default_failure_threshold")]
    pub failure_threshold: u32,
    /// Recovery timeout in seconds
    #[serde(default = "default_recovery_timeout")]
    pub recovery_timeout: u64,
    /// Health check endpoint path
    pub endpoint: Option<String>,
    /// Expected status codes for healthy response
    #[serde(default)]
    pub expected_codes: Vec<u16>,
}

impl Default for HealthCheckConfig {
    fn default() -> Self {
        Self {
            interval: default_health_check_interval(),
            failure_threshold: default_failure_threshold(),
            recovery_timeout: default_recovery_timeout(),
            endpoint: None,
            expected_codes: vec![200],
        }
    }
}

fn default_true() -> bool {
    true
}