cortiq-gateway 0.2.46

Universal LLM gateway with intelligent routing and an embedded multilingual admin console
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

//! Inbound OpenAI Models adapter: `GET /v1/models`.
//! Lists everything a client can address: the virtual `cortiq-auto`, the
//! configured `[[models]]` pool, and the managed local `[[cmf.servers]]`.
//!
//! The managed CMF servers belong here because they are routable model ids like
//! any other — omitting them left OpenAI clients (Open WebUI among them) unable
//! to pin a local model even though the gateway would happily serve it.

use crate::error::{GatewayError, Result};
use crate::state::SharedState;
use axum::{extract::State, response::IntoResponse, routing::get, Json, Router};

pub fn routes() -> Router<SharedState> {
    Router::new().route("/v1/models", get(handler))
}

fn now_secs() -> u64 {
    std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap_or_default()
        .as_secs()
}

async fn handler(State(state): State<SharedState>) -> Result<impl IntoResponse> {
    let live = state.live();
    if !live.cfg.protocols.openai_models {
        return Err(GatewayError::InvalidRequest(
            "openai_models protocol is disabled".into(),
        ));
    }
    let created = now_secs();
    let mut data = vec![serde_json::json!({
        "id": "cortiq-auto",
        "object": "model",
        "created": created,
        "owned_by": "cortiq",
    })];
    let mut seen: std::collections::HashSet<String> =
        std::collections::HashSet::from(["cortiq-auto".to_string()]);

    for m in &live.cfg.models {
        if !seen.insert(m.id.clone()) {
            continue;
        }
        data.push(serde_json::json!({
            "id": m.id,
            "object": "model",
            "created": created,
            "owned_by": m.provider,
        }));
    }

    // Managed local CMF servers. Health is reported alongside (a non-standard
    // field clients ignore) rather than used as a filter: a large `.cmf` takes
    // minutes to load, and a model that vanishes from the list while it warms up
    // is worse than one marked "not ready yet".
    let cmf = state.cmf.status();
    for s in live.cfg.cmf.effective_servers() {
        if !seen.insert(s.id.clone()) {
            continue; // a static [[models]] entry with the same id wins
        }
        let status = cmf.servers.iter().find(|st| st.id == s.id);
        data.push(serde_json::json!({
            "id": s.id,
            "object": "model",
            "created": created,
            "owned_by": "cortiq-cmf",
            "cortiq": {
                "kind": "cmf",
                "local": true,
                "running": status.map(|st| st.running).unwrap_or(false),
                "healthy": status.map(|st| st.healthy).unwrap_or(false),
            },
        }));
    }

    Ok(Json(serde_json::json!({ "object": "list", "data": data })))
}