cortiq-gateway 0.2.38

Universal LLM gateway with intelligent routing and an embedded multilingual admin console
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

//! Host metrics sampler for the admin dashboard: CPU load, RAM, free disk at
//! the models dir, and the gateway's own RSS. One background task refreshes a
//! snapshot every 2 s; the API endpoint only reads it.

use serde::Serialize;
use std::sync::{Arc, Mutex};

#[derive(Clone, Default, Serialize)]
pub struct Snapshot {
    pub ts: u64,
    /// Whole-machine CPU usage 0..100 (average over all cores).
    pub cpu_percent: f32,
    pub cores: usize,
    pub load1: f64,
    pub ram_total: u64,
    pub ram_used: u64,
    /// Disk that holds models_dir.
    pub disk_total: u64,
    pub disk_free: u64,
    /// Resident memory of the gateway process itself.
    pub process_rss: u64,
}

pub struct SysMon {
    snap: Mutex<Snapshot>,
}

impl SysMon {
    pub fn snapshot(&self) -> Snapshot {
        self.snap.lock().unwrap().clone()
    }
}

fn now() -> u64 {
    std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap_or_default()
        .as_secs()
}

/// Start the sampler. `models_dir` picks which disk's free space is reported.
pub fn spawn(models_dir: String) -> Arc<SysMon> {
    let mon = Arc::new(SysMon {
        snap: Mutex::new(Snapshot::default()),
    });
    let ret = mon.clone();
    tokio::task::spawn_blocking(move || {
        use sysinfo::{Disks, ProcessesToUpdate, System};
        let mut sys = System::new();
        let pid = sysinfo::Pid::from_u32(std::process::id());
        let models_abs = std::fs::canonicalize(&models_dir)
            .unwrap_or_else(|_| std::path::PathBuf::from(&models_dir));
        loop {
            sys.refresh_cpu_usage();
            sys.refresh_memory();
            let _ = sys.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
            let disks = Disks::new_with_refreshed_list();
            // the disk whose mount point is the longest prefix of models_dir
            let mut best: Option<(usize, u64, u64)> = None;
            for d in disks.list() {
                let mp = d.mount_point();
                if models_abs.starts_with(mp) {
                    let len = mp.as_os_str().len();
                    if best.map(|(l, _, _)| len > l).unwrap_or(true) {
                        best = Some((len, d.total_space(), d.available_space()));
                    }
                }
            }
            let (disk_total, disk_free) = best.map(|(_, t, f)| (t, f)).unwrap_or((0, 0));
            let snap = Snapshot {
                ts: now(),
                cpu_percent: sys.global_cpu_usage(),
                cores: sys.cpus().len(),
                load1: System::load_average().one,
                ram_total: sys.total_memory(),
                ram_used: sys.used_memory(),
                disk_total,
                disk_free,
                process_rss: sys.process(pid).map(|p| p.memory()).unwrap_or(0),
            };
            *mon.snap.lock().unwrap() = snap;
            std::thread::sleep(std::time::Duration::from_secs(2));
        }
    });
    ret
}