ghost-link 2.3.0

CLI and API runtime for Ghostlink, a distributed inference fabric for routing and scheduling custom LLM workloads across heterogeneous hardware.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

// Add this to the module declarations in main.rs:

mod runtime;

// Add these new routes in the router setup (in start_openai_api_server):

async fn handle_runtime_detection(
    State(state): State<Arc<Mutex<BackendState>>>,
) -> Json<serde_json::Value> {
    use crate::runtime::{RuntimeDetector, Runtime};
    
    let runtimes = RuntimeDetector::detect();
    let primary = RuntimeDetector::detect_primary();
    
    let runtime_data: Vec<_> = runtimes.iter().map(|rt| {
        serde_json::json!({
            "runtime": rt.detected_runtime.to_string(),
            "available": rt.is_available,
            "compute_capability": rt.compute_capability,
            "memory_gb": rt.memory_gb,
            "device_count": rt.device_count,
        })
    }).collect();
    
    Json(serde_json::json!({
        "available_runtimes": runtime_data,
        "primary_runtime": primary.to_string(),
        "auto_detected": true,
    }))
}

async fn handle_models_by_runtime(
    Query(params): Query<HashMap<String, String>>,
) -> Json<serde_json::Value> {
    use crate::runtime::{ModelRegistry, Runtime};
    
    let runtime_str = params.get("runtime").map(|s| s.as_str()).unwrap_or("CPU");
    
    let runtime = match runtime_str {
        "cuda" | "CUDA" => Runtime::CUDA,
        "metal" | "Metal" => Runtime::Metal,
        "rocm" | "ROCm" => Runtime::ROCm,
        "npu" | "NPU" => Runtime::NPU,
        _ => Runtime::CPU,
    };
    
    let models = ModelRegistry::models_for_runtime(runtime);
    
    let model_data: Vec<_> = models.iter().map(|m| {
        serde_json::json!({
            "name": m.name,
            "parameters": m.parameters,
            "size_gb": m.size_gb,
            "memory_required_gb": m.memory_required_gb,
            "quality_tier": format!("{:?}", m.quality_tier),
            "inference_speed": format!("{:?}", m.inference_speed),
            "use_cases": m.use_cases,
        })
    }).collect();
    
    let best = ModelRegistry::best_for_runtime(runtime);
    
    Json(serde_json::json!({
        "runtime": runtime.to_string(),
        "model_count": model_data.len(),
        "models": model_data,
        "best_model": best.map(|m| serde_json::json!({
            "name": m.name,
            "parameters": m.parameters,
            "recommended_reason": "Best balance of quality and performance for this runtime",
        })),
    }))
}

async fn handle_model_recommendations(
    Query(params): Query<HashMap<String, String>>,
) -> Json<serde_json::Value> {
    use crate::runtime::{RuntimeDetector, ModelRegistry};
    
    let memory_gb = params
        .get("memory_gb")
        .and_then(|s| s.parse::<f32>().ok())
        .unwrap_or_else(|| RuntimeDetector::detect_primary(); 8.0);
    
    let runtime = RuntimeDetector::detect_primary();
    let recommended = ModelRegistry::recommend_models(runtime, memory_gb);
    
    let model_data: Vec<_> = recommended.iter().map(|m| {
        serde_json::json!({
            "name": m.name,
            "parameters": m.parameters,
            "size_gb": m.size_gb,
            "memory_required_gb": m.memory_required_gb,
            "quality_tier": format!("{:?}", m.quality_tier),
            "inference_speed": format!("{:?}", m.inference_speed),
            "reason": format!("Fits in {:.1}GB available memory", memory_gb),
        })
    }).collect();
    
    Json(serde_json::json!({
        "detected_runtime": runtime.to_string(),
        "available_memory_gb": memory_gb,
        "recommended_models": model_data,
        "count": model_data.len(),
    }))
}

// Add these routes to the router:
.route("/api/runtime/detect", get(handle_runtime_detection))
.route("/api/runtime/models", get(handle_models_by_runtime))
.route("/api/runtime/recommend", get(handle_model_recommendations))