octoroute 1.0.0

Intelligent multi-model router for self-hosted LLMs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

//! Models endpoint handler
//!
//! Exposes model endpoint health status via GET /models

use crate::handlers::AppState;
use crate::handlers::chat::ModelTier;
use axum::{Json, extract::State};
use serde::Serialize;

/// Response for GET /models endpoint
#[derive(Serialize)]
pub struct ModelsResponse {
    pub models: Vec<ModelStatus>,
}

/// Health status for a single model endpoint
#[derive(Serialize)]
pub struct ModelStatus {
    pub name: String,
    pub tier: ModelTier,
    pub endpoint: String,
    pub healthy: bool,
    pub last_check_seconds_ago: u64,
    pub consecutive_failures: u32,
}

/// GET /models handler
///
/// Returns health status of all model endpoints across all tiers.
pub async fn handler(State(state): State<AppState>) -> Json<ModelsResponse> {
    let health_statuses = state.selector().health_checker().get_all_statuses().await;

    // Determine tier for each endpoint by checking which tier it belongs to
    let config = state.config();
    let models: Vec<ModelStatus> = health_statuses
        .into_iter()
        .map(|h| {
            // Determine tier by checking config
            let tier = if config.models.fast.iter().any(|e| e.name() == h.name()) {
                ModelTier::Fast
            } else if config.models.balanced.iter().any(|e| e.name() == h.name()) {
                ModelTier::Balanced
            } else if config.models.deep.iter().any(|e| e.name() == h.name()) {
                ModelTier::Deep
            } else {
                // Default to Balanced if not found (shouldn't happen in practice)
                tracing::warn!(
                    endpoint_name = %h.name(),
                    "Endpoint not found in any tier, defaulting to Balanced"
                );
                ModelTier::Balanced
            };

            ModelStatus {
                name: h.name().to_string(),
                tier,
                endpoint: h.base_url().to_string(),
                healthy: h.is_healthy(),
                last_check_seconds_ago: h.last_check().elapsed().as_secs(),
                consecutive_failures: h.consecutive_failures(),
            }
        })
        .collect();

    tracing::debug!(
        total_models = models.len(),
        healthy_count = models.iter().filter(|m| m.healthy).count(),
        "Retrieved model status for /models endpoint"
    );

    Json(ModelsResponse { models })
}