llmproxy 0.2.2

A simple HTTP proxy server for llm api requests
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

use crate::models::{ResponseStatus, ServerResponse, TestRequest};
use crate::state::AppState;
use axum::{extract::State, http::StatusCode, response::IntoResponse, Json};
use hyper::Uri;

pub async fn test_server(
    State(state): State<AppState>,
    Json(payload): Json<TestRequest>,
) -> impl IntoResponse {
    let servers = state.servers.lock().await;

    let server_addr = payload.addr.trim().to_string();

    if let Some(_server) = servers.iter().find(|s| s.addr == server_addr) {
        let uri = format!("http://{}/health", server_addr)
            .parse::<Uri>()
            .expect("Failed to parse URI");

        match state.http_client.get(uri).await {
            Ok(response) => {
                if response.status().is_success() {
                    (
                        StatusCode::OK,
                        Json(ServerResponse {
                            status: ResponseStatus::Success,
                            message: format!("Service at {} is reachable", server_addr),
                        }),
                    )
                } else {
                    (
                        StatusCode::OK,
                        Json(ServerResponse {
                            status: ResponseStatus::Error,
                            message: format!(
                                "Service at {} returned status {}",
                                server_addr,
                                response.status()
                            ),
                        }),
                    )
                }
            }
            Err(e) => (
                StatusCode::OK,
                Json(ServerResponse {
                    status: ResponseStatus::Error,
                    message: format!("Failed to connect to service at {}: {}", server_addr, e),
                }),
            ),
        }
    } else {
        (
            StatusCode::NOT_FOUND,
            Json(ServerResponse {
                status: ResponseStatus::Error,
                message: "Service not found".to_string(),
            }),
        )
    }
}