litellm-rs 0.6.0

A high-performance AI Gateway written in Rust, providing OpenAI-compatible APIs with intelligent routing, load balancing, and enterprise features
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

//! Policy-aware HTTP client for Azure AI Foundry.

use std::sync::Arc;

use reqwest::{Method, header::HeaderMap};

use super::config::AzureAIConfig;
use crate::core::providers::base::{BaseHttpClient, ProviderRequestBuilder};
use crate::core::providers::unified_provider::ProviderError;

#[derive(Debug, Clone)]
pub(crate) struct AzureAIClient {
    inner: Arc<AzureAIClientInner>,
}

#[derive(Debug)]
struct AzureAIClientInner {
    config: AzureAIConfig,
    headers: HeaderMap,
    http_client: BaseHttpClient,
    streaming_client: BaseHttpClient,
}

impl AzureAIClient {
    pub(crate) fn new(config: AzureAIConfig) -> Result<Self, ProviderError> {
        config
            .validate_policy_client_settings()
            .map_err(|error| ProviderError::configuration("azure_ai", error))?;
        let headers = build_headers(&config)?;
        let http_client = BaseHttpClient::new_for_provider("azure_ai", config.base.clone())?;
        let streaming_client =
            BaseHttpClient::new_for_provider_streaming("azure_ai", config.base.clone())?;

        Ok(Self {
            inner: Arc::new(AzureAIClientInner {
                config,
                headers,
                http_client,
                streaming_client,
            }),
        })
    }

    pub(crate) fn get_config(&self) -> &AzureAIConfig {
        &self.inner.config
    }

    pub(crate) fn request(
        &self,
        method: Method,
        url: &str,
    ) -> Result<ProviderRequestBuilder, ProviderError> {
        Ok(self
            .inner
            .http_client
            .request(method, url)?
            .headers(self.inner.headers.clone()))
    }

    pub(crate) fn streaming_request(
        &self,
        method: Method,
        url: &str,
    ) -> Result<ProviderRequestBuilder, ProviderError> {
        Ok(self
            .inner
            .streaming_client
            .request(method, url)?
            .headers(self.inner.headers.clone()))
    }
}

fn build_headers(config: &AzureAIConfig) -> Result<HeaderMap, ProviderError> {
    let mut headers = HeaderMap::new();
    for (key, value) in config
        .create_default_headers()
        .map_err(|error| ProviderError::configuration("azure_ai", error))?
    {
        let name = reqwest::header::HeaderName::from_bytes(key.as_bytes()).map_err(|error| {
            ProviderError::configuration("azure_ai", format!("invalid header name {key}: {error}"))
        })?;
        let value = reqwest::header::HeaderValue::from_str(&value).map_err(|error| {
            ProviderError::configuration(
                "azure_ai",
                format!("invalid header value for {key}: {error}"),
            )
        })?;
        headers.insert(name, value);
    }
    Ok(headers)
}