llmproxy 0.2.2

A simple HTTP proxy server for llm api requests
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

use crate::models::ProxyServerInfo;
use crate::state::AppState;
use axum::{extract::State, response::IntoResponse, Json};

pub async fn list_servers(State(state): State<AppState>) -> impl IntoResponse {
    let servers = state.servers.lock().await;

    let server_list_display: Vec<ProxyServerInfo> = servers
        .iter()
        .map(|server| ProxyServerInfo {
            model_name: server.model_name.clone(),
            addr: server.addr.clone(),
            avg_response_time_ms: Some(server.metrics.avg_response_time_ms),
            total_requests: Some(server.metrics.total_requests),
            recent_failures: Some(server.metrics.recent_failures),
        })
        .collect();
    Json(server_list_display)
}