1use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
9
10use crate::config::Config;
11use crate::gpu::NvmlCollector;
12use crate::public_ip::{PublicIpChecker, PublicIpLookup};
13use crate::smart::SmartReader;
14#[cfg(target_os = "linux")]
15use crate::snapshot::device_matches_disk;
16use crate::snapshot::{PowerStats, Snapshot};
17use crate::system::SystemCollector;
18
19#[cfg(target_os = "linux")]
20use crate::linux::{
21 self, SysRoots, diskstats::DiskStatsMeter, net::NetLinkReader, rapl::RaplMeter,
22};
23
24pub const MIN_POLL_INTERVAL: Duration = sysinfo::MINIMUM_CPU_UPDATE_INTERVAL;
27
28#[derive(Debug, thiserror::Error)]
30pub enum SourceError {
31 #[error("I/O error: {0}")]
33 Io(#[from] std::io::Error),
34 #[error("source disconnected: {0}")]
36 Disconnected(String),
37 #[error("remote: {0}")]
39 Remote(String),
40 #[error("cannot signal process {pid}: {source}")]
42 Signal {
43 pid: u32,
45 source: std::io::Error,
47 },
48}
49
50#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
52pub enum ProcessSignal {
53 Terminate,
55 Kill,
57 Stop,
60 Continue,
62 Nice(i32),
65}
66
67impl ProcessSignal {
68 pub fn is_control(self) -> bool {
70 matches!(self, Self::Stop | Self::Continue | Self::Nice(_))
71 }
72}
73
74pub trait DataSource {
76 fn poll(&mut self) -> Result<Snapshot, SourceError>;
82
83 fn signal_process(&mut self, pid: u32, signal: ProcessSignal) -> Result<(), SourceError>;
86
87 fn set_interval(&mut self, _interval: Duration) {}
91
92 fn set_process_limit(&mut self, _limit: Option<usize>) {}
95
96 fn watch_processes(&mut self, _pids: Vec<u32>) {}
100
101 fn watch_process_names(&mut self, _names: Vec<String>) {}
106
107 fn set_public_ip(&mut self, _lookup: Option<PublicIpLookup>) {}
111}
112
113impl<T: DataSource + ?Sized> DataSource for Box<T> {
115 fn poll(&mut self) -> Result<Snapshot, SourceError> {
116 (**self).poll()
117 }
118
119 fn signal_process(&mut self, pid: u32, signal: ProcessSignal) -> Result<(), SourceError> {
120 (**self).signal_process(pid, signal)
121 }
122
123 fn set_interval(&mut self, interval: Duration) {
124 (**self).set_interval(interval);
125 }
126
127 fn set_process_limit(&mut self, limit: Option<usize>) {
128 (**self).set_process_limit(limit);
129 }
130
131 fn watch_processes(&mut self, pids: Vec<u32>) {
132 (**self).watch_processes(pids);
133 }
134
135 fn watch_process_names(&mut self, names: Vec<String>) {
136 (**self).watch_process_names(names);
137 }
138
139 fn set_public_ip(&mut self, lookup: Option<PublicIpLookup>) {
140 (**self).set_public_ip(lookup);
141 }
142}
143
144pub struct LocalSource {
146 system: SystemCollector,
147 nvml: NvmlCollector,
148 smart: SmartReader,
149 last_poll: Option<Instant>,
150 watched_pids: Vec<u32>,
152 watched_names: Vec<String>,
155 name_matches: Vec<u32>,
156 fill: crate::fill::FillTracker,
157 public_ip: Option<PublicIpChecker>,
159 #[cfg(target_os = "linux")]
160 roots: SysRoots,
161 #[cfg(target_os = "linux")]
162 rapl: RaplMeter,
163 #[cfg(target_os = "linux")]
164 diskstats: DiskStatsMeter,
165 #[cfg(target_os = "linux")]
166 net_links: NetLinkReader,
167 #[cfg(target_os = "linux")]
168 failed_units: linux::systemd::FailedUnitsReader,
169 #[cfg(target_os = "linux")]
170 zpool: linux::zpool::ZpoolReader,
171 #[cfg(target_os = "linux")]
172 containers: linux::containers::ContainerTracker,
173 #[cfg(target_os = "linux")]
174 oom: linux::oom::OomTracker,
175 #[cfg(target_os = "linux")]
176 battery: linux::battery::BatteryMeter,
177 #[cfg(target_os = "linux")]
180 core_locations: Vec<Option<linux::hwmon::CoreLocation>>,
181 #[cfg(target_os = "linux")]
183 scrub_dirs: Vec<std::path::PathBuf>,
184 #[cfg(target_os = "linux")]
186 drm_gpus: linux::drm::DrmGpuReader,
187 #[cfg(windows)]
189 lhm: crate::lhm::LhmReader,
190}
191
192impl LocalSource {
193 pub fn new() -> Self {
195 Self::with_config(&Config::default())
196 }
197
198 pub fn with_config(config: &Config) -> Self {
205 let mut source = Self {
206 system: SystemCollector::new(),
207 nvml: NvmlCollector::new(),
208 smart: SmartReader::new(&config.smart.dir),
209 last_poll: None,
210 watched_pids: Vec::new(),
211 watched_names: Vec::new(),
212 name_matches: Vec::new(),
213 fill: crate::fill::FillTracker::new(),
214 public_ip: None,
215 #[cfg(target_os = "linux")]
216 roots: SysRoots::default(),
217 #[cfg(target_os = "linux")]
218 rapl: RaplMeter::new(),
219 #[cfg(target_os = "linux")]
220 diskstats: DiskStatsMeter::new(),
221 #[cfg(target_os = "linux")]
222 net_links: NetLinkReader::new(),
223 #[cfg(target_os = "linux")]
224 failed_units: linux::systemd::FailedUnitsReader::new(),
225 #[cfg(target_os = "linux")]
226 zpool: linux::zpool::ZpoolReader::new(),
227 #[cfg(target_os = "linux")]
228 containers: linux::containers::ContainerTracker::new(),
229 #[cfg(target_os = "linux")]
230 oom: linux::oom::OomTracker::new(),
231 #[cfg(target_os = "linux")]
232 battery: linux::battery::BatteryMeter::default(),
233 #[cfg(target_os = "linux")]
234 core_locations: Vec::new(),
235 #[cfg(target_os = "linux")]
236 drm_gpus: linux::drm::DrmGpuReader::new(),
237 #[cfg(windows)]
240 lhm: crate::lhm::LhmReader::new(&config.lhm.url),
241 #[cfg(target_os = "linux")]
242 scrub_dirs: vec![
243 config.btrfs.scrub_dir.clone(),
244 linux::btrfs::BTRFS_PROGS_STATUS_DIR.into(),
245 ],
246 };
247 let _ = source.collect();
249 source
250 }
251
252 pub fn gpu_unavailable_reason(&self) -> Option<&str> {
254 self.nvml.init_error()
255 }
256
257 fn collect(&mut self) -> Snapshot {
258 let now = Instant::now();
259 let elapsed = self
260 .last_poll
261 .map(|prev| now.saturating_duration_since(prev).as_secs_f64());
262 self.last_poll = Some(now);
263
264 self.system.refresh();
265 let processes = self.system.processes(elapsed);
266 let gpus = self.nvml.collect();
267 let gpus_w = sum_present(gpus.iter().map(|g| g.power_w));
268
269 let mut snapshot = Snapshot {
270 timestamp_ms: SystemTime::now()
271 .duration_since(UNIX_EPOCH)
272 .map(|d| d.as_millis() as u64)
273 .unwrap_or(0),
274 host: self.system.host(),
275 cpu: self.system.cpu(),
276 memory: self.system.memory(),
277 networks: self.system.networks(elapsed),
278 filesystems: self.system.filesystems(),
279 process_count: processes.len(),
280 processes,
281 watched_pids: self.watched_pids.clone(),
282 gpus,
283 smart: self.smart.read(),
284 power: PowerStats {
285 gpus_w,
286 ..Default::default()
287 },
288 ..Default::default()
289 };
290
291 #[cfg(target_os = "linux")]
292 self.collect_linux(&mut snapshot, now);
293 #[cfg(not(target_os = "linux"))]
294 self.collect_portable(&mut snapshot, elapsed);
295
296 self.fill.annotate(&mut snapshot.filesystems, now);
297 if !(self.watched_names.is_empty() && self.name_matches.is_empty()) {
298 self.name_matches =
299 report_name_matches(&self.watched_names, &self.name_matches, &mut snapshot);
300 }
301 let power = &mut snapshot.power;
302 power.total_w = sum_present([power.cpu_or_soc_w().1, power.gpus_w]);
303 snapshot.public_ip = self
304 .public_ip
305 .as_mut()
306 .and_then(|checker| checker.poll().map(str::to_owned));
307 snapshot
308 }
309
310 #[cfg(not(target_os = "linux"))]
313 fn collect_portable(&mut self, snapshot: &mut Snapshot, elapsed: Option<f64>) {
314 snapshot.disks = self.system.volume_io(elapsed);
315 let (chips, cpu_temperatures) = self.system.components();
316 snapshot.sensors = chips;
317 snapshot.cpu.temperatures = cpu_temperatures;
318 #[cfg(windows)]
319 {
320 crate::windows::annotate_links(&mut snapshot.networks);
321 match self.lhm.read(Instant::now()) {
324 Some(lhm) => {
325 if !lhm.chips.is_empty() {
326 snapshot.sensors = lhm.chips;
327 }
328 if !lhm.cpu_temperatures.is_empty() {
329 snapshot.cpu.temperatures = lhm.cpu_temperatures;
330 }
331 snapshot.power.cpu_package_w = lhm.cpu_package_w;
332 if !lhm.smart.is_empty() {
333 snapshot.smart = lhm.smart;
334 }
335 }
336 None => snapshot.lhm_error = self.lhm.problem().map(str::to_owned),
337 }
338 }
339 }
340
341 #[cfg(target_os = "linux")]
343 fn collect_linux(&mut self, snapshot: &mut Snapshot, now: Instant) {
344 let chips = linux::hwmon::read_chips(&self.roots);
345
346 let cores = snapshot.cpu.per_core_percent.len();
347 if self.core_locations.len() != cores {
348 self.core_locations = linux::hwmon::read_core_locations(&self.roots, cores);
349 }
350 (snapshot.cpu.temperatures, snapshot.cpu.per_core_celsius) =
351 linux::hwmon::split_core_temperatures(&chips, &self.core_locations);
352 snapshot.memory.cached_bytes = linux::meminfo::read_cached_bytes(&self.roots);
353 snapshot.memory.oom_kills =
354 self.oom
355 .sample(&self.roots, &snapshot.processes, snapshot.timestamp_ms);
356 snapshot.power.cpu_package_w = self.rapl.sample(&self.roots, now);
357 snapshot.power.igpu_w = sum_present(
360 chips
361 .iter()
362 .filter(|c| c.is_apu)
363 .flat_map(|c| c.power_w.iter().map(|(_, w)| Some(*w))),
364 );
365 snapshot.power.battery = self.battery.read(&self.roots);
366
367 let first = snapshot.gpus.len() as u32;
369 snapshot.gpus.extend(self.drm_gpus.read(&self.roots, first));
370 snapshot.power.gpus_w = sum_present(snapshot.gpus.iter().map(|g| g.power_w));
371
372 let mut disks = self.diskstats.sample(&self.roots, now);
373 for disk in &mut disks {
374 disk.temperature_c = nvme_temperature(&chips, &disk.name);
375 }
376 snapshot.disks = disks;
377
378 let mounts = linux::mounts::read_mounts(&self.roots);
379 snapshot.raid.btrfs = linux::btrfs::read_filesystems(&self.roots, &mounts);
380 for fs in &mut snapshot.raid.btrfs {
381 fs.scrub = linux::btrfs::read_scrub(&self.scrub_dirs, &fs.fsid);
382 }
383 self.net_links
384 .annotate(&self.roots, &mut snapshot.networks, now);
385 snapshot.raid.md = linux::mdstat::read_arrays(&self.roots);
386 snapshot.raid.zfs = linux::zfs::read_pools(&self.roots);
387 self.zpool.annotate(&mut snapshot.raid.zfs, now);
388 snapshot.failed_units = self.failed_units.read(now);
389 snapshot.pressure = linux::pressure::read_pressure(&self.roots);
390 let release = std::fs::read_to_string(self.roots.proc.join("sys/kernel/osrelease"))
391 .unwrap_or_default();
392 snapshot.host.reboot_required = linux::reboot::reboot_required(
393 std::path::Path::new("/run"),
394 std::path::Path::new("/usr/lib/modules"),
395 &release,
396 );
397 for process in &mut snapshot.processes {
399 if let Some(stat) = linux::procstat::read(&self.roots, process.pid) {
400 process.threads = stat.threads;
401 process.nice = stat.nice;
402 } else {
403 process.threads = None;
404 }
405 }
406 self.containers
407 .annotate(&self.roots, &mut snapshot.processes, now);
408
409 snapshot.sensors = chips.into_iter().map(|c| c.chip).collect();
410 }
411}
412
413impl Default for LocalSource {
414 fn default() -> Self {
415 Self::new()
416 }
417}
418
419impl DataSource for LocalSource {
420 fn poll(&mut self) -> Result<Snapshot, SourceError> {
421 Ok(self.collect())
422 }
423
424 fn signal_process(&mut self, pid: u32, signal: ProcessSignal) -> Result<(), SourceError> {
425 send_signal(pid, signal).map_err(|source| SourceError::Signal { pid, source })
426 }
427
428 fn set_process_limit(&mut self, limit: Option<usize>) {
431 self.nvml.set_process_utilization(limit.is_none());
432 }
433
434 fn watch_processes(&mut self, pids: Vec<u32>) {
435 self.watched_pids = pids;
436 }
437
438 fn watch_process_names(&mut self, names: Vec<String>) {
439 self.watched_names = names;
440 }
441
442 fn set_public_ip(&mut self, lookup: Option<PublicIpLookup>) {
443 if self.public_ip.as_ref().map(PublicIpChecker::lookup) != lookup.as_ref() {
445 self.public_ip = lookup.map(PublicIpChecker::new);
446 }
447 }
448}
449
450fn report_name_matches(names: &[String], previous: &[u32], snapshot: &mut Snapshot) -> Vec<u32> {
455 let matches: Vec<u32> = snapshot
456 .processes
457 .iter()
458 .filter(|p| {
459 !p.kernel_thread
460 && names
461 .iter()
462 .any(|name| crate::view::process_matches(p, name))
463 })
464 .map(|p| p.pid)
465 .collect();
466 let gone = previous
467 .iter()
468 .filter(|pid| !snapshot.processes.iter().any(|p| p.pid == **pid));
469 for pid in matches.iter().chain(gone) {
470 if !snapshot.watched_pids.contains(pid) {
471 snapshot.watched_pids.push(*pid);
472 }
473 }
474 matches
475}
476
477#[cfg(unix)]
479fn send_signal(pid: u32, signal: ProcessSignal) -> std::io::Result<()> {
480 let pid = libc::pid_t::try_from(pid)
483 .ok()
484 .filter(|p| *p > 0)
485 .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::InvalidInput))?;
486 let sig = match signal {
487 ProcessSignal::Terminate => libc::SIGTERM,
488 ProcessSignal::Kill => libc::SIGKILL,
489 ProcessSignal::Stop => libc::SIGSTOP,
490 ProcessSignal::Continue => libc::SIGCONT,
491 ProcessSignal::Nice(nice) => return set_nice(pid, nice),
492 };
493 if unsafe { libc::kill(pid, sig) } == 0 {
495 Ok(())
496 } else {
497 Err(std::io::Error::last_os_error())
498 }
499}
500
501#[cfg(unix)]
503fn set_nice(pid: libc::pid_t, nice: i32) -> std::io::Result<()> {
504 let nice = nice.clamp(-20, 19);
505 if unsafe { libc::setpriority(libc::PRIO_PROCESS as _, pid as _, nice) } == 0 {
508 Ok(())
509 } else {
510 Err(std::io::Error::last_os_error())
511 }
512}
513
514#[cfg(windows)]
515use crate::windows::send_signal;
516
517#[cfg(not(any(unix, windows)))]
519fn send_signal(_pid: u32, _signal: ProcessSignal) -> std::io::Result<()> {
520 Err(std::io::Error::from(std::io::ErrorKind::Unsupported))
521}
522
523fn sum_present(values: impl IntoIterator<Item = Option<f32>>) -> Option<f32> {
525 values
526 .into_iter()
527 .flatten()
528 .fold(None, |acc, v| Some(acc.unwrap_or(0.0) + v))
529}
530
531#[cfg(target_os = "linux")]
534fn nvme_temperature(chips: &[linux::hwmon::HwmonChip], disk: &str) -> Option<f32> {
535 chips.iter().find_map(|c| {
536 let controller = c.chip.key.strip_prefix("nvme:")?;
537 device_matches_disk(controller, disk).then_some(())?;
538 c.chip.temperatures.first().map(|t| t.celsius)
540 })
541}
542
543#[cfg(test)]
544mod tests {
545 use super::*;
546 use crate::snapshot::ProcessInfo;
547
548 #[test]
549 fn named_processes_are_reported_until_seen_gone() {
550 let process = |pid, name: &str, command: &str| ProcessInfo {
551 pid,
552 name: name.to_owned(),
553 command: command.to_owned(),
554 ..Default::default()
555 };
556 let names = vec!["train.py".to_owned()];
557 let mut snapshot = Snapshot {
558 processes: vec![
559 process(10, "python3", "python3 jobs/train.py --epochs 3"),
560 process(11, "vim", "vim notes.txt"),
561 ],
562 watched_pids: vec![99],
563 ..Default::default()
564 };
565 let matches = report_name_matches(&names, &[], &mut snapshot);
566 assert_eq!(matches, [10]);
567 assert_eq!(snapshot.watched_pids, [99, 10]);
568
569 let mut later = Snapshot {
571 processes: vec![process(11, "vim", "vim notes.txt")],
572 ..Default::default()
573 };
574 let matches = report_name_matches(&names, &matches, &mut later);
575 assert!(matches.is_empty());
576 assert_eq!(later.watched_pids, [10]);
577 let mut after = Snapshot::default();
578 assert!(report_name_matches(&names, &matches, &mut after).is_empty());
579 assert!(after.watched_pids.is_empty());
580 }
581
582 #[cfg(unix)]
583 #[test]
584 fn signal_rejects_group_pids_and_reports_missing_process() {
585 assert_eq!(
586 send_signal(0, ProcessSignal::Terminate).unwrap_err().kind(),
587 std::io::ErrorKind::InvalidInput
588 );
589 assert_eq!(
590 send_signal(u32::MAX, ProcessSignal::Kill)
591 .unwrap_err()
592 .kind(),
593 std::io::ErrorKind::InvalidInput
594 );
595 let mut child = std::process::Command::new("true").spawn().unwrap();
597 let pid = child.id();
598 child.wait().unwrap();
599 let err = send_signal(pid, ProcessSignal::Terminate).unwrap_err();
600 assert_eq!(err.raw_os_error(), Some(libc::ESRCH));
601 }
602
603 #[cfg(unix)]
604 #[test]
605 fn signal_terminates_a_child_process() {
606 let mut child = std::process::Command::new("sleep")
607 .arg("30")
608 .spawn()
609 .unwrap();
610 send_signal(child.id(), ProcessSignal::Kill).unwrap();
611 let status = child.wait().unwrap();
612 assert!(!status.success());
613 }
614
615 #[cfg(target_os = "linux")]
616 #[test]
617 fn pause_resume_and_renice_a_child_process() {
618 let mut child = std::process::Command::new("sleep")
619 .arg("30")
620 .spawn()
621 .unwrap();
622 let pid = child.id();
623 let roots = SysRoots::default();
624 let state = || {
625 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).unwrap();
626 let after = &stat[stat.rfind(')').unwrap() + 2..];
627 after.chars().next().unwrap()
628 };
629 send_signal(pid, ProcessSignal::Stop).unwrap();
630 for _ in 0..100 {
632 if state() == 'T' {
633 break;
634 }
635 std::thread::sleep(std::time::Duration::from_millis(10));
636 }
637 assert_eq!(state(), 'T');
638 send_signal(pid, ProcessSignal::Continue).unwrap();
639 send_signal(pid, ProcessSignal::Nice(7)).unwrap();
641 assert_eq!(linux::procstat::read(&roots, pid).unwrap().nice, Some(7));
642 child.kill().unwrap();
643 child.wait().unwrap();
644 }
645
646 #[test]
647 fn sum_present_ignores_missing_values() {
648 assert_eq!(sum_present([None, None]), None);
649 assert_eq!(sum_present([Some(1.5), None, Some(2.0)]), Some(3.5));
650 }
651
652 #[cfg(target_os = "linux")]
653 #[test]
654 fn nvme_temperature_matches_controller_exactly() {
655 use crate::linux::hwmon::HwmonChip;
656 use crate::snapshot::{SensorChip, Temperature};
657
658 let chip = |key: &str, celsius: f32| HwmonChip {
659 chip: SensorChip {
660 key: key.to_owned(),
661 name: "nvme".to_owned(),
662 temperatures: vec![Temperature {
663 label: "Composite".to_owned(),
664 celsius,
665 ..Default::default()
666 }],
667 fans: Vec::new(),
668 },
669 power_w: Vec::new(),
670 is_apu: false,
671 };
672 let chips = vec![chip("nvme:nvme1", 40.0), chip("nvme:nvme10", 50.0)];
673 assert_eq!(nvme_temperature(&chips, "nvme10n1"), Some(50.0));
674 assert_eq!(nvme_temperature(&chips, "nvme1n1"), Some(40.0));
675 assert_eq!(nvme_temperature(&chips, "sda"), None);
676 }
677}