litellm-rust 0.3.0

Minimal Rust SDK port of LiteLLM – unified interface for chat, embeddings, images, and video across OpenAI-compatible, Anthropic, Gemini, and xAI providers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148

use crate::config::{Config, ProviderConfig, ProviderKind};
use crate::error::{LiteLLMError, Result};
use crate::providers::{anthropic, gemini, openai_compat};
use crate::registry::Registry;
use crate::router::{resolve_model, ResolvedModel};
use crate::stream::ChatStream;
use crate::types::{
    ChatRequest, ChatResponse, EmbeddingRequest, EmbeddingResponse, ImageEditRequest, ImageRequest,
    ImageResponse, VideoRequest, VideoResponse,
};
use reqwest::Client;

#[derive(Debug, Clone)]
pub struct LiteLLM {
    config: Config,
    client: Client,
    registry: Registry,
}

impl LiteLLM {
    pub fn new() -> Result<Self> {
        let registry = Registry::load_embedded()?;
        Ok(Self {
            config: Config::default(),
            client: Client::builder()
                .timeout(std::time::Duration::from_secs(60))
                .build()
                .map_err(LiteLLMError::from)?,
            registry,
        })
    }

    pub fn with_default_provider(mut self, provider: impl Into<String>) -> Self {
        self.config.default_provider = Some(provider.into());
        self
    }

    pub fn with_provider(mut self, name: impl Into<String>, config: ProviderConfig) -> Self {
        self.config.providers.insert(name.into(), config);
        self
    }

    pub fn with_client(mut self, client: Client) -> Self {
        self.client = client;
        self
    }

    pub fn registry(&self) -> &Registry {
        &self.registry
    }

    pub async fn completion(&self, mut req: ChatRequest) -> Result<ChatResponse> {
        let resolved = resolve_model(&req.model, &self.config)?;
        req.model = resolved.model.clone();
        dispatch_chat(&self.client, resolved, req).await
    }

    pub async fn stream_completion(&self, mut req: ChatRequest) -> Result<ChatStream> {
        let resolved = resolve_model(&req.model, &self.config)?;
        req.model = resolved.model.clone();
        match resolved.config.kind {
            ProviderKind::OpenAICompatible => {
                openai_compat::chat_stream(&self.client, &resolved.config, req).await
            }
            ProviderKind::Anthropic => {
                anthropic::chat_stream(&self.client, &resolved.config, req).await
            }
            _ => Err(LiteLLMError::Unsupported(
                "streaming not supported for provider".into(),
            )),
        }
    }

    pub async fn embedding(&self, mut req: EmbeddingRequest) -> Result<EmbeddingResponse> {
        let resolved = resolve_model(&req.model, &self.config)?;
        req.model = resolved.model.clone();
        match resolved.config.kind {
            ProviderKind::OpenAICompatible => {
                openai_compat::embeddings(&self.client, &resolved.config, req).await
            }
            _ => Err(LiteLLMError::Unsupported(
                "embeddings not supported for provider".into(),
            )),
        }
    }

    pub async fn image_generation(&self, mut req: ImageRequest) -> Result<ImageResponse> {
        let resolved = resolve_model(&req.model, &self.config)?;
        req.model = resolved.model.clone();
        match resolved.config.kind {
            ProviderKind::OpenAICompatible => {
                openai_compat::image_generation(&self.client, &resolved.config, req).await
            }
            ProviderKind::Gemini => {
                gemini::image_generation(&self.client, &resolved.config, req).await
            }
            _ => Err(LiteLLMError::Unsupported(
                "image generation not supported for provider".into(),
            )),
        }
    }

    pub async fn image_editing(&self, mut req: ImageEditRequest) -> Result<ImageResponse> {
        let resolved = resolve_model(&req.model, &self.config)?;
        req.model = resolved.model.clone();
        match resolved.config.kind {
            ProviderKind::Gemini => {
                gemini::image_editing(&self.client, &resolved.config, req).await
            }
            _ => Err(LiteLLMError::Unsupported(
                "image editing not supported for provider".into(),
            )),
        }
    }

    pub async fn video_generation(&self, mut req: VideoRequest) -> Result<VideoResponse> {
        let resolved = resolve_model(&req.model, &self.config)?;
        req.model = resolved.model.clone();
        match resolved.config.kind {
            ProviderKind::OpenAICompatible => {
                openai_compat::video_generation(&self.client, &resolved.config, req).await
            }
            ProviderKind::Gemini => {
                gemini::video_generation(&self.client, &resolved.config, req).await
            }
            _ => Err(LiteLLMError::Unsupported(
                "video generation not supported for provider".into(),
            )),
        }
    }

    pub fn estimate_cost(&self, model: &str, input_tokens: u32, output_tokens: u32) -> Option<f64> {
        self.registry
            .estimate_cost(model, input_tokens, output_tokens)
    }
}

async fn dispatch_chat(
    client: &Client,
    resolved: ResolvedModel,
    req: ChatRequest,
) -> Result<ChatResponse> {
    match resolved.config.kind {
        ProviderKind::OpenAICompatible => openai_compat::chat(client, &resolved.config, req).await,
        ProviderKind::Anthropic => anthropic::chat(client, &resolved.config, req).await,
        ProviderKind::Gemini => gemini::chat(client, &resolved.config, req).await,
    }
}