octoroute 1.0.0

Intelligent multi-model router for self-hosted LLMs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

//! OpenAI-compatible models list handler
//!
//! Handles GET /v1/models requests.

use crate::handlers::AppState;
use axum::{Json, extract::State, response::IntoResponse};

use super::types::{ModelObject, ModelsListResponse};

/// GET /v1/models handler
///
/// Returns a list of available models in OpenAI-compatible format.
///
/// # Response Format
///
/// Returns an object with:
/// - `object`: "list"
/// - `data`: Array of model objects, each with:
///   - `id`: Model identifier (tier name or endpoint name)
///   - `object`: "model"
///   - `created`: Unix timestamp
///   - `owned_by`: "octoroute" for tiers, "user" for configured endpoints
///
/// # Available Models
///
/// Tier-based models (use Octoroute routing):
/// - `auto` - Automatic tier selection via LLM/hybrid routing
/// - `fast` - Route to fast tier (smallest models, lowest latency)
/// - `balanced` - Route to balanced tier (medium models)
/// - `deep` - Route to deep tier (largest models, best quality)
///
/// Plus all configured endpoint names from config.toml, which bypass
/// routing and directly use that specific endpoint.
pub async fn handler(State(state): State<AppState>) -> impl IntoResponse {
    // Start with tier-based virtual models
    let mut models = vec![
        ModelObject::new("auto", "octoroute"),
        ModelObject::new("fast", "octoroute"),
        ModelObject::new("balanced", "octoroute"),
        ModelObject::new("deep", "octoroute"),
    ];

    // Add configured endpoint names from each tier
    for endpoint in &state.config().models.fast {
        models.push(ModelObject::new(endpoint.name(), "user"));
    }
    for endpoint in &state.config().models.balanced {
        models.push(ModelObject::new(endpoint.name(), "user"));
    }
    for endpoint in &state.config().models.deep {
        models.push(ModelObject::new(endpoint.name(), "user"));
    }

    Json(ModelsListResponse::new(models))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_model_object_tier() {
        let model = ModelObject::new("fast", "octoroute");
        assert_eq!(model.id, "fast");
        assert_eq!(model.object, "model");
        assert_eq!(model.owned_by, "octoroute");
        // OpenAI uses 0 for many models
        assert_eq!(model.created, 0);
    }

    #[test]
    fn test_model_object_endpoint() {
        let model = ModelObject::new("qwen3-8b", "user");
        assert_eq!(model.id, "qwen3-8b");
        assert_eq!(model.object, "model");
        assert_eq!(model.owned_by, "user");
    }

    #[test]
    fn test_models_list_response() {
        let models = vec![
            ModelObject::new("auto", "octoroute"),
            ModelObject::new("fast", "octoroute"),
        ];
        let response = ModelsListResponse::new(models);
        assert_eq!(response.object, "list");
        assert_eq!(response.data.len(), 2);
    }
}