litellm-rs 0.4.16

A high-performance AI Gateway written in Rust, providing OpenAI-compatible APIs with intelligent routing, load balancing, and enterprise features
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

//! Model listing and retrieval endpoints

use crate::core::models::openai::{Model, ModelListResponse};
use crate::core::router::UnifiedRouter;
use crate::server::state::AppState;
use crate::utils::error::gateway_error::GatewayError;
use actix_web::{HttpResponse, ResponseError, Result as ActixResult, web};
use tracing::{debug, error};

/// List available models
///
/// Returns a list of available AI models across all configured providers.
pub async fn list_models(state: web::Data<AppState>) -> ActixResult<HttpResponse> {
    debug!("Listing available models");

    let unified_router = &state.unified_router;

    match get_models_from_router(unified_router).await {
        Ok(models) => {
            let response = ModelListResponse {
                object: "list".to_string(),
                data: models,
            };
            Ok(HttpResponse::Ok().json(response))
        }
        Err(e) => {
            error!("Failed to list models: {}", e);
            Ok(e.error_response())
        }
    }
}

/// Get specific model information
///
/// Returns detailed information about a specific model.
pub async fn get_model(
    state: web::Data<AppState>,
    model_id: web::Path<String>,
) -> ActixResult<HttpResponse> {
    debug!("Getting model info for: {}", model_id);

    let unified_router = &state.unified_router;

    match get_model_from_router(unified_router, &model_id).await {
        Ok(Some(model)) => Ok(HttpResponse::Ok().json(model)),
        Ok(None) => {
            Ok(GatewayError::not_found(format!("Model not found: {}", model_id)).error_response())
        }
        Err(e) => {
            error!("Failed to get model {}: {}", model_id, e);
            Ok(e.error_response())
        }
    }
}

/// Get all models from UnifiedRouter
pub async fn get_models_from_router(router: &UnifiedRouter) -> Result<Vec<Model>, GatewayError> {
    let mut models = Vec::new();

    for model_name in router.list_models() {
        let owned_by = router
            .get_deployments_for_model(&model_name)
            .into_iter()
            .find_map(|deployment_id| {
                router
                    .get_deployment(&deployment_id)
                    .map(|deployment| deployment.provider.name().to_string())
            })
            .unwrap_or_else(|| "unknown".to_string());

        models.push(Model {
            id: model_name,
            object: "model".to_string(),
            created: chrono::Utc::now().timestamp() as u64,
            owned_by,
        });
    }

    models.sort_by(|a, b| a.id.cmp(&b.id));
    Ok(models)
}

/// Get specific model from UnifiedRouter
pub async fn get_model_from_router(
    router: &UnifiedRouter,
    model_id: &str,
) -> Result<Option<Model>, GatewayError> {
    let deployment_ids = router.get_deployments_for_model(model_id);
    if let Some(owner) = deployment_ids.into_iter().find_map(|deployment_id| {
        router
            .get_deployment(&deployment_id)
            .map(|deployment| deployment.provider.name().to_string())
    }) {
        return Ok(Some(Model {
            id: model_id.to_string(),
            object: "model".to_string(),
            created: chrono::Utc::now().timestamp() as u64,
            owned_by: owner,
        }));
    }

    Ok(None)
}