litellm-rs 0.6.0

A high-performance AI Gateway written in Rust, providing OpenAI-compatible APIs with intelligent routing, load balancing, and enterprise features
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

use serde::{Deserialize, Serialize};

use super::{Message, MessageDelta, Tool, ToolChoice, Usage};

/// Chat request
#[derive(Debug, Clone)]
pub struct SdkChatRequest {
    /// Model name
    pub model: String,
    /// Message list
    pub messages: Vec<Message>,
    /// Request options
    pub options: ChatOptions,
}

/// Chat options
#[derive(Debug, Clone, Default)]
pub struct ChatOptions {
    /// Temperature parameter
    pub temperature: Option<f32>,
    /// Maximum token count
    pub max_tokens: Option<u32>,
    /// Top-p parameter
    pub top_p: Option<f32>,
    /// Frequency penalty
    pub frequency_penalty: Option<f32>,
    /// Presence penalty
    pub presence_penalty: Option<f32>,
    /// Stop sequences
    pub stop: Option<Vec<String>>,
    /// Stream response
    pub stream: bool,
    /// Tool list
    pub tools: Option<Vec<Tool>>,
    /// Tool choice
    pub tool_choice: Option<ToolChoice>,
}

/// Chat response
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatResponse {
    /// Response ID
    pub id: String,
    /// Model name
    pub model: String,
    /// Choice list
    pub choices: Vec<ChatChoice>,
    /// Usage statistics
    pub usage: Usage,
    /// Creation timestamp
    pub created: u64,
}

/// Chat choice
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatChoice {
    /// Choice index
    pub index: u32,
    /// Message content
    pub message: Message,
    /// Finish reason
    pub finish_reason: Option<String>,
}

/// Chat chunk (streaming)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatChunk {
    /// Response ID
    pub id: String,
    /// Model name
    pub model: String,
    /// Choice list
    pub choices: Vec<ChunkChoice>,
}

/// Streaming choice
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChunkChoice {
    /// Choice index
    pub index: u32,
    /// Delta message
    pub delta: MessageDelta,
    /// Finish reason
    pub finish_reason: Option<String>,
}