llmproxy 0.2.2

A simple HTTP proxy server for llm api requests
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

use crate::models::ModelInfo;
use axum::body::Body;
use hyper_util::client::legacy::Client;
use std::collections::VecDeque;
use std::sync::Arc;
use tokio::sync::Mutex;

#[derive(Clone, Debug)]
pub struct ProxyServer {
    pub model_name: String,
    pub addr: String,
    pub metrics: ServerMetrics,
}

#[derive(Clone, Debug)]
pub struct ServerMetrics {
    // Exponential moving average of response time in milliseconds
    pub avg_response_time_ms: f64,
    // Total requests handled
    pub total_requests: u64,
    // Recent failures (last 10 requests)
    pub recent_failures: u64,
    // Last request timestamp
    pub last_request_time: Option<std::time::SystemTime>,

    // --- UCB (Upper Confidence Bound) fields ---
    // Sliding window of recent rewards (1.0 / response_time)
    // Used for UCB load balancing algorithm
    pub recent_rewards: VecDeque<f64>,
    // Maximum window size (number of recent requests to track)
    pub window_size: usize,
}

impl Default for ServerMetrics {
    fn default() -> Self {
        Self {
            // Start with a neutral value (100ms)
            avg_response_time_ms: 100.0,
            total_requests: 0,
            recent_failures: 0,
            last_request_time: None,
            // Initialize UCB fields
            recent_rewards: VecDeque::new(),
            window_size: 1000, // Default window size
        }
    }
}

#[derive(Clone, Debug)]
pub struct EndpointModelsCache {
    pub endpoint: String,
    pub models: Vec<ModelInfo>,
    pub discovered_at: std::time::SystemTime,
}

#[derive(Clone)]
pub struct AppState {
    pub servers: Arc<Mutex<Vec<ProxyServer>>>,
    pub http_client: Client<hyper_util::client::legacy::connect::HttpConnector, Body>,
    pub models_cache: Arc<Mutex<Vec<EndpointModelsCache>>>,
}

impl AppState {
    pub fn new(
        http_client: Client<hyper_util::client::legacy::connect::HttpConnector, Body>,
    ) -> Self {
        Self {
            servers: Arc::new(Mutex::new(vec![])),
            http_client,
            models_cache: Arc::new(Mutex::new(vec![])),
        }
    }
}