ferrum-interfaces 0.6.0

Core trait contracts for the Ferrum LLM inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

use ferrum_types::EngineStatus;
use ferrum_types::{MemoryUsage, ModelId};
use serde_json as json;

#[test]
fn engine_status_serde_roundtrip() {
    let st = EngineStatus {
        is_ready: true,
        loaded_models: vec![ModelId::from("m")],
        active_requests: 1,
        queued_requests: 0,
        memory_usage: MemoryUsage {
            total_bytes: 1_000,
            used_bytes: 100,
            free_bytes: 900,
            gpu_memory_bytes: None,
            cpu_memory_bytes: Some(100),
            cache_memory_bytes: 0,
            utilization_percent: 10.0,
        },
        uptime_seconds: 5,
        last_heartbeat: chrono::Utc::now(),
        version: "0.1".into(),
    };
    let s = json::to_string(&st).unwrap();
    let back: EngineStatus = json::from_str(&s).unwrap();
    assert_eq!(back.is_ready, true);
}