Skip to main content

crabshell_services/
stats.rs

1//! CPU, memory and temperature stats from procfs and sysfs, whose files
2//! change without an inotify event, so they are sampled on a timer that
3//! lives only while a config shows them. The reads run on a thread of
4//! their own, as some sensors take 100 ms to answer.
5
6#[cfg(test)]
7mod tests;
8
9use std::cell::{Cell, RefCell};
10use std::collections::{HashMap, HashSet};
11use std::fmt::Display;
12use std::io::Read;
13use std::panic::{self, AssertUnwindSafe};
14use std::path::{Path, PathBuf};
15use std::rc::Rc;
16use std::sync::{Once, mpsc};
17use std::thread::JoinHandle;
18use std::time::{Duration, Instant};
19
20use calloop::RegistrationToken;
21use calloop::timer::TimeoutAction;
22use crabshell_reactive::{ReadSignal, Signal};
23use crabshell_ui::Cx;
24
25use crate::Shared;
26use crate::file::{FileError, FileErrorKind, open_file, set};
27use crate::lines::panic_message;
28
29/// The shortest period, so stats can't keep the shell busy.
30const MIN_PERIOD: Duration = Duration::from_millis(250);
31/// The longest period, as `cx.interval`'s, so adding it to now can't
32/// overflow.
33const MAX_PERIOD: Duration = Duration::from_secs(u32::MAX as u64);
34const MAX_CPUS: usize = 256;
35const MAX_TEMPERATURES: usize = 64;
36/// The most read from one file.
37const MAX_READ: u64 = 64 * 1024;
38/// The most entries looked at in one sysfs directory.
39const MAX_LISTED: usize = 1024;
40
41/// The system's load, from `cx.system_stats(period)`.
42#[derive(Clone, Debug, Default, PartialEq)]
43pub struct SystemStats {
44  /// The share of time every CPU was busy over the last period, from 0
45  /// to 1; 0 until the second sample.
46  pub cpu: f32,
47  /// Each online CPU's share, as `cpu`, in CPU number order, up to 256
48  /// CPUs. A CPU that came online since the last sample shows 0.
49  pub cpus: Vec<f32>,
50  /// Memory in bytes.
51  pub memory_total: u64,
52  /// Memory a program could take without swapping, in bytes.
53  pub memory_available: u64,
54  /// Swap in bytes.
55  pub swap_total: u64,
56  /// Swap free, in bytes.
57  pub swap_free: u64,
58  /// Each hwmon sensor's temperature, up to 64, except those of a chip
59  /// whose device sleeps. The sensors are listed at the first sample and
60  /// again only once one is gone.
61  pub temperatures: Vec<Temperature>,
62}
63
64/// A temperature sensor's reading, from [`SystemStats::temperatures`].
65#[derive(Clone, Debug, PartialEq)]
66pub struct Temperature {
67  /// The chip's name and the sensor's label, such as `coretemp Core 0`,
68  /// or its number, such as `acpitz temp1`, when it has no label.
69  pub name: String,
70  /// The reading in degrees Celsius.
71  pub celsius: f32,
72}
73
74/// What `cx.system_stats` needs from the shell: the loop and the roots,
75/// and the one thread that samples while any context shows stats.
76pub(crate) struct Service {
77  shared: Rc<Shared>,
78  /// Each live call's signal, with its sampler while no read runs.
79  live: RefCell<HashMap<u64, Live>>,
80  next: Cell<u64>,
81  worker: RefCell<Option<Worker>>,
82  /// Whether the worker couldn't start, so samples are read on the loop
83  /// thread instead.
84  inline: Cell<bool>,
85  /// How many samples were asked for, for the tests.
86  #[cfg(test)]
87  requests: Cell<usize>,
88  /// The last worker's thread, once stopped, for the tests to join.
89  #[cfg(test)]
90  ended: RefCell<Option<JoinHandle<()>>>,
91}
92
93struct Live {
94  stats: Signal<SystemStats>,
95  /// `None` while the worker reads with it.
96  sampler: Option<Sampler>,
97}
98
99/// A finished sample: whose it is, its sampler back, and what it read.
100pub(crate) type Answer = (u64, Sampler, SystemStats);
101
102/// The thread that reads, and the channel it answers on.
103struct Worker {
104  requests: mpsc::Sender<(u64, Sampler)>,
105  token: RegistrationToken,
106  /// Detached outside tests, which join it.
107  #[cfg_attr(not(test), allow(dead_code))]
108  thread: JoinHandle<()>,
109}
110
111impl Service {
112  pub(crate) fn new(shared: Rc<Shared>) -> Service {
113    Service {
114      shared,
115      live: RefCell::default(),
116      next: Cell::new(0),
117      worker: RefCell::new(None),
118      inline: Cell::new(false),
119      #[cfg(test)]
120      requests: Cell::new(0),
121      #[cfg(test)]
122      ended: RefCell::new(None),
123    }
124  }
125
126  /// Hands sampler `id` to the worker, starting it if none runs. A
127  /// sampler still reading is skipped, not queued.
128  fn request(self: &Rc<Self>, id: u64) {
129    let taken = self
130      .live
131      .borrow_mut()
132      .get_mut(&id)
133      .and_then(|live| live.sampler.take());
134    let Some(sampler) = taken else {
135      return;
136    };
137    #[cfg(test)]
138    self.requests.set(self.requests.get() + 1);
139    if self.worker.borrow().is_none() && !self.inline.get() {
140      let worker = self.start();
141      self.inline.set(worker.is_none());
142      self.worker.replace(worker);
143    }
144    if self.inline.get() {
145      let mut sampler = sampler;
146      let stats = sample_caught(&mut sampler);
147      return self.answered(id, sampler, stats);
148    }
149    let unsent = match self.worker.borrow().as_ref() {
150      Some(worker) => worker
151        .requests
152        .send((id, sampler))
153        .err()
154        .map(|error| error.0.1),
155      None => Some(sampler),
156    };
157    if let Some(sampler) = unsent
158      && let Some(live) = self.live.borrow_mut().get_mut(&id)
159    {
160      live.sampler = Some(sampler);
161    }
162  }
163
164  /// The worker, answering on a channel in the loop. `None`, logged, if
165  /// either couldn't start; samples are read on the loop thread then.
166  fn start(self: &Rc<Self>) -> Option<Worker> {
167    let (answers, channel) = calloop::channel::channel::<Answer>();
168    let weak = Rc::downgrade(self);
169    let answered = Box::new(move |(id, sampler, stats)| {
170      if let Some(service) = weak.upgrade() {
171        service.answered(id, sampler, stats);
172      }
173    });
174    let event_loop = self.shared.event_loop()?;
175    let token = event_loop.insert_stats_channel(channel, answered)?;
176    let (requests, received) = mpsc::channel::<(u64, Sampler)>();
177    let spawned = std::thread::Builder::new()
178      .name(String::from("crabshell-stats"))
179      .spawn(move || {
180        // Ends once the requests' sender or the loop's channel is gone.
181        for (id, mut sampler) in received {
182          let stats = sample_caught(&mut sampler);
183          if answers.send((id, sampler, stats)).is_err() {
184            break;
185          }
186        }
187      });
188    match spawned {
189      Ok(thread) => Some(Worker {
190        requests,
191        token,
192        thread,
193      }),
194      Err(error) => {
195        tracing::warn!(%error, "couldn't start the stats' thread, so they are read on the loop thread");
196        event_loop.remove(token);
197        None
198      }
199    }
200  }
201
202  fn answered(&self, id: u64, sampler: Sampler, stats: SystemStats) {
203    let signal = {
204      let mut live = self.live.borrow_mut();
205      let Some(live) = live.get_mut(&id) else {
206        return;
207      };
208      live.sampler = Some(sampler);
209      live.stats
210    };
211    set(signal, stats);
212  }
213
214  /// Takes the worker out of the loop. Its thread ends once a read it is
215  /// in finishes.
216  fn stop(&self) {
217    let Some(worker) = self.worker.take() else {
218      return;
219    };
220    if let Some(event_loop) = self.shared.event_loop() {
221      event_loop.remove(worker.token);
222    }
223    drop(worker.requests);
224    #[cfg(test)]
225    self.ended.replace(Some(worker.thread));
226  }
227}
228
229/// `cx.system_stats(period)`.
230pub(crate) fn system_stats(cx: &Cx, period: Duration) -> ReadSignal<SystemStats> {
231  let stats = cx.signal(SystemStats::default());
232  let Some(service) = cx.service::<Service>() else {
233    static LOGGED: Once = Once::new();
234    LOGGED.call_once(|| {
235      tracing::warn!(
236        "cx.system_stats() needs the services the shell provides; these stats stand still"
237      );
238    });
239    return stats.read_only();
240  };
241  let Some(event_loop) = service.shared.event_loop() else {
242    return stats.read_only();
243  };
244  let id = service.next.get();
245  service.next.set(id + 1);
246  let config = &service.shared.config;
247  #[cfg_attr(not(test), allow(unused_mut))]
248  let mut sampler = Sampler::new(&config.proc, &config.sys);
249  #[cfg(test)]
250  {
251    sampler.delay = config.stats_delay;
252    sampler.panic_once = config.stats_panic;
253  }
254  let live = Live {
255    stats,
256    sampler: Some(sampler),
257  };
258  service.live.borrow_mut().insert(id, live);
259  service.request(id);
260  let period = period.clamp(MIN_PERIOD, MAX_PERIOD);
261  let weak = Rc::downgrade(&service);
262  let tick = Box::new(move || {
263    let Some(service) = weak.upgrade() else {
264      return TimeoutAction::Drop;
265    };
266    service.request(id);
267    TimeoutAction::ToDuration(period)
268  });
269  let timer = event_loop.insert_timer(Instant::now() + period, tick);
270  cx.owner().signal(Unschedule { service, id, timer });
271  stats.read_only()
272}
273
274/// Reads a sample. A bug in the reading gives an empty one, logged once,
275/// rather than taking the sampler and freezing the stats.
276fn sample_caught(sampler: &mut Sampler) -> SystemStats {
277  panic::catch_unwind(AssertUnwindSafe(|| sampler.sample())).unwrap_or_else(|payload| {
278    static LOGGED: Once = Once::new();
279    let message = panic_message(payload.as_ref());
280    LOGGED.call_once(|| tracing::error!("reading the system stats panicked: {message}"));
281    SystemStats::default()
282  })
283}
284
285/// Takes a call's timer out of the loop when its context goes, and the
286/// worker with the last one.
287struct Unschedule {
288  service: Rc<Service>,
289  id: u64,
290  timer: Option<RegistrationToken>,
291}
292
293impl Drop for Unschedule {
294  fn drop(&mut self) {
295    if let (Some(timer), Some(event_loop)) = (self.timer, self.service.shared.event_loop()) {
296      event_loop.remove(timer);
297    }
298    let empty = {
299      let mut live = self.service.live.borrow_mut();
300      live.remove(&self.id);
301      live.is_empty()
302    };
303    if empty {
304      self.service.stop();
305    }
306  }
307}
308
309/// A CPU's time counters, in clock ticks since boot.
310#[derive(Clone, Copy, Default)]
311struct Times {
312  busy: u64,
313  total: u64,
314}
315
316/// A hwmon chip's temperature sensors, listed once.
317struct Chip {
318  /// The chip's device's power state, `suspended` while it sleeps.
319  runtime_status: PathBuf,
320  /// Each sensor's name and the file holding its reading.
321  sensors: Vec<(String, PathBuf)>,
322}
323
324/// Reads the stats, keeping the last CPU counters for the next sample's
325/// share. It moves to the worker for each read and back with the result.
326pub(crate) struct Sampler {
327  proc: PathBuf,
328  sys: PathBuf,
329  /// Every CPU's counters at the last sample, then each CPU's by number.
330  last: Option<(Times, Vec<(u32, Times)>)>,
331  /// The temperature sensors, listed at the first sample and again after
332  /// one was gone.
333  chips: Option<Vec<Chip>>,
334  /// The problems logged already, by kind or by file.
335  logged: HashSet<String>,
336  /// A pause before each read, so a test's sensors can be slow.
337  #[cfg(test)]
338  delay: Duration,
339  /// Whether the next read panics, as a bug would.
340  #[cfg(test)]
341  panic_once: bool,
342}
343
344impl Sampler {
345  fn new(proc: &Path, sys: &Path) -> Sampler {
346    Sampler {
347      proc: proc.to_owned(),
348      sys: sys.to_owned(),
349      last: None,
350      chips: None,
351      logged: HashSet::new(),
352      #[cfg(test)]
353      delay: Duration::ZERO,
354      #[cfg(test)]
355      panic_once: false,
356    }
357  }
358
359  fn sample(&mut self) -> SystemStats {
360    #[cfg(test)]
361    {
362      std::thread::sleep(self.delay);
363      assert!(
364        !std::mem::take(&mut self.panic_once),
365        "a test's sensor panicked"
366      );
367    }
368    let mut stats = SystemStats::default();
369    self.cpu(&mut stats);
370    self.memory(&mut stats);
371    self.temperatures(&mut stats);
372    stats
373  }
374
375  /// Logs a problem the first time it comes.
376  fn once(&mut self, key: impl Into<String>, detail: impl Display) {
377    if self.logged.insert(key.into()) {
378      tracing::info!("system stats: {detail}; it is skipped");
379    }
380  }
381
382  fn read(&mut self, path: &Path) -> Option<String> {
383    match read_capped(path) {
384      Ok(text) => Some(text),
385      Err(error) => {
386        let path = path.display().to_string();
387        let detail = format!("can't read {path}: {}", error.message);
388        self.once(path, detail);
389        None
390      }
391    }
392  }
393
394  fn cpu(&mut self, stats: &mut SystemStats) {
395    let Some(text) = self.read(&self.proc.join("stat")) else {
396      return;
397    };
398    let mut all = None;
399    let mut cpus = Vec::new();
400    for line in text.lines() {
401      let mut words = line.split_ascii_whitespace();
402      let Some(label) = words.next().filter(|label| label.starts_with("cpu")) else {
403        continue;
404      };
405      let number = label[3..].parse::<u32>();
406      let Some(times) = parse_times(words).filter(|_| label == "cpu" || number.is_ok()) else {
407        let detail = format!("a CPU line doesn't parse: {line:.80}");
408        self.once("cpu line", detail);
409        continue;
410      };
411      match number {
412        Err(_) => all = Some(times),
413        Ok(_) if cpus.len() == MAX_CPUS => {
414          self.once("cpus", "there are over 256 CPUs; the rest are left out");
415        }
416        Ok(number) => cpus.push((number, times)),
417      }
418    }
419    cpus.sort_unstable_by_key(|(number, _)| *number);
420    let all = all.unwrap_or_default();
421    let last = self.last.take();
422    stats.cpu = last.as_ref().map_or(0.0, |(then, _)| share(*then, all));
423    // A CPU that came online since starts from 0.
424    stats.cpus = cpus
425      .iter()
426      .map(|(number, now)| {
427        let then = last.as_ref().and_then(|(_, then)| {
428          let at = then.binary_search_by_key(number, |(number, _)| *number);
429          at.ok().map(|at| then[at].1)
430        });
431        then.map_or(0.0, |then| share(then, *now))
432      })
433      .collect();
434    self.last = Some((all, cpus));
435  }
436
437  fn memory(&mut self, stats: &mut SystemStats) {
438    let Some(text) = self.read(&self.proc.join("meminfo")) else {
439      return;
440    };
441    for line in text.lines() {
442      let Some((key, value)) = line.split_once(':') else {
443        continue;
444      };
445      let field = match key {
446        "MemTotal" => &mut stats.memory_total,
447        "MemAvailable" => &mut stats.memory_available,
448        "SwapTotal" => &mut stats.swap_total,
449        "SwapFree" => &mut stats.swap_free,
450        _ => continue,
451      };
452      match parse_bytes(value) {
453        Some(bytes) => *field = bytes,
454        None => self.once(
455          "meminfo line",
456          format_args!("a meminfo line doesn't parse: {line:.80}"),
457        ),
458      }
459    }
460  }
461
462  /// Reads each listed sensor, except a sleeping chip's, so a sampled
463  /// dGPU isn't woken. A sensor gone has the sensors listed again at the
464  /// next sample.
465  fn temperatures(&mut self, stats: &mut SystemStats) {
466    let chips = match self.chips.take() {
467      Some(chips) => chips,
468      None => self.list_sensors(),
469    };
470    let mut gone = false;
471    for chip in &chips {
472      let asleep =
473        read_capped(&chip.runtime_status).is_ok_and(|status| status.trim() == "suspended");
474      if asleep {
475        continue;
476      }
477      for (name, input) in &chip.sensors {
478        let text = match read_capped(input) {
479          Ok(text) => text,
480          Err(error) if error.kind == FileErrorKind::NotFound => {
481            gone = true;
482            continue;
483          }
484          Err(error) => {
485            let path = input.display().to_string();
486            let detail = format!("can't read {path}: {}", error.message);
487            self.once(path, detail);
488            continue;
489          }
490        };
491        match text.trim().parse::<i64>() {
492          Ok(millidegrees) => stats.temperatures.push(Temperature {
493            name: name.clone(),
494            celsius: millidegrees as f32 / 1000.0,
495          }),
496          Err(_) => {
497            let path = input.display().to_string();
498            let detail = format!("{path} doesn't hold a temperature");
499            self.once(path, detail);
500          }
501        }
502      }
503    }
504    self.chips = (!gone).then_some(chips);
505  }
506
507  /// The hwmon chips and their temperature sensors, named by the chip's
508  /// name and each sensor's label, in number order, up to
509  /// [`MAX_TEMPERATURES`] sensors.
510  fn list_sensors(&mut self) -> Vec<Chip> {
511    let class = self.sys.join("class/hwmon");
512    let Some(mut numbered) = self.numbered(&class, "hwmon", "") else {
513      return Vec::new();
514    };
515    numbered.sort_unstable();
516    let mut chips = Vec::new();
517    let mut count = 0;
518    for (number, dir) in numbered {
519      let name = read_capped(&dir.join("name"))
520        .ok()
521        .map(|name| name.trim().to_owned())
522        .filter(|name| !name.is_empty())
523        .unwrap_or_else(|| format!("hwmon{number}"));
524      let Some(mut inputs) = self.numbered(&dir, "temp", "_input") else {
525        continue;
526      };
527      inputs.sort_unstable();
528      let mut sensors = Vec::new();
529      for (sensor, input) in inputs {
530        if count == MAX_TEMPERATURES {
531          self.once(
532            "temperatures",
533            "there are over 64 temperatures; the rest are left out",
534          );
535          break;
536        }
537        let label = read_capped(&dir.join(format!("temp{sensor}_label")))
538          .ok()
539          .map(|label| label.trim().to_owned())
540          .filter(|label| !label.is_empty())
541          .unwrap_or_else(|| format!("temp{sensor}"));
542        sensors.push((format!("{name} {label}"), input));
543        count += 1;
544      }
545      chips.push(Chip {
546        runtime_status: dir.join("device/power/runtime_status"),
547        sensors,
548      });
549    }
550    chips
551  }
552
553  /// The entries in `dir` named `<prefix><number><suffix>`, with their
554  /// numbers, among its first [`MAX_LISTED`].
555  fn numbered(&mut self, dir: &Path, prefix: &str, suffix: &str) -> Option<Vec<(u32, PathBuf)>> {
556    let entries = match std::fs::read_dir(dir) {
557      Ok(entries) => entries,
558      Err(error) => {
559        self.once(
560          "list",
561          format_args!("can't list {}: {error}", dir.display()),
562        );
563        return None;
564      }
565    };
566    let numbered = entries
567      .take(MAX_LISTED)
568      .filter_map(|entry| {
569        let entry = entry.ok()?;
570        let name = entry.file_name();
571        let number = name.to_str()?.strip_prefix(prefix)?.strip_suffix(suffix)?;
572        Some((number.parse().ok()?, entry.path()))
573      })
574      .collect();
575    Some(numbered)
576  }
577}
578
579/// A `/proc/stat` CPU line's counters after its label: user, nice,
580/// system, idle, then optionally iowait, irq, softirq and steal. Guest
581/// time is in user's already, so it is left out.
582fn parse_times<'a>(words: impl Iterator<Item = &'a str>) -> Option<Times> {
583  let counters: Vec<u64> = words
584    .take(8)
585    .map(str::parse)
586    .collect::<Result<_, _>>()
587    .ok()?;
588  let idle = counters.get(3)? + counters.get(4).unwrap_or(&0);
589  let total = counters
590    .iter()
591    .try_fold(0_u64, |sum, n| sum.checked_add(*n))?;
592  Some(Times {
593    busy: total.checked_sub(idle)?,
594    total,
595  })
596}
597
598/// The share of `now`'s time since `then` that was busy, from 0 to 1.
599fn share(then: Times, now: Times) -> f32 {
600  let total = now.total.saturating_sub(then.total);
601  if total == 0 {
602    return 0.0;
603  }
604  // iowait may go back, so busy may move more than total.
605  (now.busy.saturating_sub(then.busy) as f64 / total as f64).clamp(0.0, 1.0) as f32
606}
607
608/// A meminfo value, such as `  16318060 kB`, in bytes.
609fn parse_bytes(value: &str) -> Option<u64> {
610  let mut words = value.split_ascii_whitespace();
611  let number: u64 = words.next()?.parse().ok()?;
612  match (words.next(), words.next()) {
613    (Some("kB"), None) => number.checked_mul(1024),
614    (None, None) => Some(number),
615    _ => None,
616  }
617}
618
619/// Up to [`MAX_READ`] of the file at `path`, which must be a regular
620/// file, as procfs's and sysfs's are.
621fn read_capped(path: &Path) -> Result<String, FileError> {
622  let file = open_file(path)?;
623  let mut bytes = Vec::new();
624  file.take(MAX_READ).read_to_end(&mut bytes)?;
625  Ok(String::from_utf8_lossy(&bytes).into_owned())
626}