litellm-rs 0.8.2

A high-performance AI Gateway written in Rust, providing OpenAI-compatible APIs with intelligent routing, load balancing, and enterprise features
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

use crate::core::providers::unified_provider::ProviderError;
use crate::core::types::responses::{EmbeddingData, EmbeddingResponse, Usage};

pub(super) fn parse_embedding_response(
    response: &serde_json::Value,
    expected_embedding_count: usize,
    model: &str,
) -> Result<EmbeddingResponse, ProviderError> {
    let embeddings = response
        .get("embeddings")
        .and_then(serde_json::Value::as_array)
        .ok_or_else(|| {
            ProviderError::response_parsing("ollama", "Missing embeddings in response")
        })?;

    if embeddings.len() != expected_embedding_count {
        return Err(ProviderError::response_parsing(
            "ollama",
            format!(
                "Ollama returned {} embeddings for {expected_embedding_count} inputs",
                embeddings.len()
            ),
        ));
    }

    let mut expected_dimension = None;
    let mut data = Vec::with_capacity(embeddings.len());
    for (i, emb) in embeddings.iter().enumerate() {
        let values = emb.as_array().ok_or_else(|| {
            ProviderError::response_parsing(
                "ollama",
                format!("Ollama embedding at index {i} is not an array"),
            )
        })?;
        if values.is_empty() {
            return Err(ProviderError::response_parsing(
                "ollama",
                format!("Ollama embedding at index {i} is empty"),
            ));
        }
        if expected_dimension.is_some_and(|dimension| dimension != values.len()) {
            return Err(ProviderError::response_parsing(
                "ollama",
                format!("Ollama embedding at index {i} has an inconsistent dimension"),
            ));
        }
        expected_dimension.get_or_insert(values.len());
        let embedding = values
            .iter()
            .enumerate()
            .map(|(coordinate, value)| {
                value
                    .as_f64()
                    .filter(|value| value.is_finite() && (*value as f32).is_finite())
                    .map(|value| value as f32)
                    .ok_or_else(|| {
                        ProviderError::response_parsing(
                            "ollama",
                            format!(
                                "Ollama embedding at index {i} has an invalid coordinate at {coordinate}"
                            ),
                        )
                    })
            })
            .collect::<Result<Vec<_>, _>>()?;
        let index = u32::try_from(i).map_err(|_| {
            ProviderError::response_parsing(
                "ollama",
                "Ollama embedding response contains too many rows",
            )
        })?;

        data.push(EmbeddingData {
            object: "embedding".to_string(),
            embedding,
            index,
        });
    }

    let prompt_tokens = match response.get("prompt_eval_count") {
        None => 0,
        Some(count) => count
            .as_u64()
            .and_then(|count| u32::try_from(count).ok())
            .ok_or_else(|| {
                ProviderError::response_parsing(
                    "ollama",
                    "Ollama embedding response has an invalid prompt_eval_count",
                )
            })?,
    };

    Ok(EmbeddingResponse {
        object: "list".to_string(),
        data,
        model: format!("ollama/{model}"),
        usage: Some(Usage {
            prompt_tokens,
            completion_tokens: 0,
            total_tokens: prompt_tokens,
            prompt_tokens_details: None,
            completion_tokens_details: None,
            thinking_usage: None,
        }),
        embeddings: None,
    })
}