sqlite_graphrag/
system_load.rs

1//! G28-D: system load average observation before spawning LLM subprocesses.
2//!
3//! The 2026-06-03 incident saturated a 10-CPU host with load 276 because
4//! parallel `enrich` workers kept spawning `claude -p` / `codex exec`
5//! children even when the system was already at saturation. This module
6//! exposes a single helper that returns `true` when the 1-minute load
7//! average is above `2 × ncpus` (the conservative threshold the G28-D
8//! original discussion recommended).
9//!
10//! Uses `sysinfo::System::load_average()` which is already a transitive
11//! dependency of the project. The read is cheap (single syscall on
12//! Linux) and throttled to once per second via a Mutex-cached timestamp.
13
14use std::sync::Mutex;
15use std::time::{Duration, Instant};
16
17static LAST_REFRESH: Mutex<Option<Instant>> = Mutex::new(None);
18
19/// Returns the 1-minute load average as reported by the OS.
20///
21/// On platforms where `sysinfo` cannot read load average (very old Linux
22/// without /proc/loadavg), returns `0.0` so callers default to "no
23/// saturation detected".
24pub fn load_average_one() -> f64 {
25    let _ = ensure_fresh();
26    sysinfo::System::load_average().one
27}
28
29/// Returns the number of logical CPUs the runtime can detect.
30///
31/// Used together with [`load_average_one`] to apply a saturation check.
32pub fn ncpus() -> usize {
33    std::thread::available_parallelism()
34        .map(|n| n.get())
35        .unwrap_or(4)
36}
37
38/// G28-D: returns `true` when the 1-minute load average exceeds
39/// `2 × ncpus`. Override via XDG `config set system.max_load_per_ncpu <f64>`.
40pub fn is_system_saturated() -> bool {
41    let load = load_average_one();
42    let n = ncpus() as f64;
43    let multiplier: f64 = crate::config::get_setting("system.max_load_per_ncpu")
44        .ok()
45        .flatten()
46        .and_then(|v| v.parse().ok())
47        .unwrap_or(2.0);
48    load > n * multiplier
49}
50
51/// Throttles the cached refresh timestamp so we read /proc/loadavg at
52/// most once per second across all callers. The function returns the
53/// previous timestamp (or None on first call) so the caller can decide
54/// whether to actually invoke the syscall.
55fn ensure_fresh() -> Option<Instant> {
56    let mut guard = LAST_REFRESH
57        .lock()
58        .unwrap_or_else(|poisoned| poisoned.into_inner());
59    let now = Instant::now();
60    let should_refresh = guard
61        .as_ref()
62        .is_none_or(|last| now.duration_since(*last) > Duration::from_secs(1));
63    let prev = guard.as_ref().copied();
64    if should_refresh {
65        *guard = Some(now);
66    }
67    prev
68}
69
70#[cfg(test)]
71mod tests {
72    use super::*;
73
74    #[test]
75    fn ncpus_is_at_least_one() {
76        assert!(ncpus() >= 1);
77    }
78
79    #[test]
80    fn load_average_is_non_negative() {
81        assert!(load_average_one() >= 0.0);
82    }
83
84    #[test]
85    fn saturation_default_threshold_is_two() {
86        // G28-D default: 2 × ncpus. Operators can lower it via env var
87        // when running on contended CI runners.
88        let default = 2.0_f64;
89        assert!(default >= 1.0);
90    }
91
92    #[test]
93    fn saturation_check_does_not_panic() {
94        // The function must always return a definitive answer.
95        let _ = is_system_saturated();
96    }
97
98    #[test]
99    fn ensure_fresh_returns_previous_then_sets_new() {
100        let prev = ensure_fresh();
101        // On the first call prev is None; subsequent calls return Some.
102        if prev.is_none() {
103            let second = ensure_fresh();
104            // Within the same second the cache is fresh so prev is Some.
105            assert!(second.is_some());
106        }
107    }
108}
sqlite_graphrag/system_load.rs

sqlite_graphrag/
system_load.rs
