1use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
9
10use crate::config::Config;
11use crate::gpu::NvmlCollector;
12use crate::public_ip::{PublicIpChecker, PublicIpLookup};
13use crate::smart::SmartReader;
14#[cfg(target_os = "linux")]
15use crate::snapshot::device_matches_disk;
16use crate::snapshot::{PowerStats, Snapshot};
17use crate::system::SystemCollector;
18
19#[cfg(target_os = "linux")]
20use crate::linux::{
21 self, SysRoots, diskstats::DiskStatsMeter, net::NetLinkReader, rapl::RaplMeter,
22};
23
24pub const MIN_POLL_INTERVAL: Duration = sysinfo::MINIMUM_CPU_UPDATE_INTERVAL;
27
28#[derive(Debug, thiserror::Error)]
30pub enum SourceError {
31 #[error("I/O error: {0}")]
33 Io(#[from] std::io::Error),
34 #[error("source disconnected: {0}")]
36 Disconnected(String),
37 #[error("remote: {0}")]
39 Remote(String),
40 #[error("cannot signal process {pid}: {source}")]
42 Signal {
43 pid: u32,
45 source: std::io::Error,
47 },
48}
49
50#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
52pub enum ProcessSignal {
53 Terminate,
55 Kill,
57 Stop,
60 Continue,
62 Nice(i32),
65}
66
67impl ProcessSignal {
68 pub fn is_control(self) -> bool {
70 matches!(self, Self::Stop | Self::Continue | Self::Nice(_))
71 }
72}
73
74pub trait DataSource {
76 fn poll(&mut self) -> Result<Snapshot, SourceError>;
82
83 fn signal_process(&mut self, pid: u32, signal: ProcessSignal) -> Result<(), SourceError>;
86
87 fn set_interval(&mut self, _interval: Duration) {}
91
92 fn set_process_limit(&mut self, _limit: Option<usize>) {}
95
96 fn watch_processes(&mut self, _pids: Vec<u32>) {}
100
101 fn watch_process_names(&mut self, _names: Vec<String>) {}
106
107 fn set_public_ip(&mut self, _lookup: Option<PublicIpLookup>) {}
111}
112
113impl<T: DataSource + ?Sized> DataSource for Box<T> {
115 fn poll(&mut self) -> Result<Snapshot, SourceError> {
116 (**self).poll()
117 }
118
119 fn signal_process(&mut self, pid: u32, signal: ProcessSignal) -> Result<(), SourceError> {
120 (**self).signal_process(pid, signal)
121 }
122
123 fn set_interval(&mut self, interval: Duration) {
124 (**self).set_interval(interval);
125 }
126
127 fn set_process_limit(&mut self, limit: Option<usize>) {
128 (**self).set_process_limit(limit);
129 }
130
131 fn watch_processes(&mut self, pids: Vec<u32>) {
132 (**self).watch_processes(pids);
133 }
134
135 fn watch_process_names(&mut self, names: Vec<String>) {
136 (**self).watch_process_names(names);
137 }
138
139 fn set_public_ip(&mut self, lookup: Option<PublicIpLookup>) {
140 (**self).set_public_ip(lookup);
141 }
142}
143
144pub struct LocalSource {
146 system: SystemCollector,
147 nvml: NvmlCollector,
148 smart: SmartReader,
149 last_poll: Option<Instant>,
150 watched_pids: Vec<u32>,
152 watched_names: Vec<String>,
155 name_matches: Vec<u32>,
156 fill: crate::fill::FillTracker,
157 public_ip: Option<PublicIpChecker>,
159 #[cfg(target_os = "linux")]
160 roots: SysRoots,
161 #[cfg(target_os = "linux")]
162 rapl: RaplMeter,
163 #[cfg(target_os = "linux")]
164 diskstats: DiskStatsMeter,
165 #[cfg(target_os = "linux")]
166 net_links: NetLinkReader,
167 #[cfg(target_os = "linux")]
168 failed_units: linux::systemd::FailedUnitsReader,
169 #[cfg(target_os = "linux")]
170 zpool: linux::zpool::ZpoolReader,
171 #[cfg(target_os = "linux")]
172 containers: linux::containers::ContainerTracker,
173 #[cfg(target_os = "linux")]
174 oom: linux::oom::OomTracker,
175 #[cfg(target_os = "linux")]
176 battery: linux::battery::BatteryMeter,
177 #[cfg(target_os = "linux")]
180 core_locations: Vec<Option<linux::hwmon::CoreLocation>>,
181 #[cfg(target_os = "linux")]
183 scrub_dirs: Vec<std::path::PathBuf>,
184 #[cfg(target_os = "linux")]
186 drm_gpus: linux::drm::DrmGpuReader,
187 #[cfg(target_os = "linux")]
189 drm_clients: linux::drm_clients::DrmClientTracker,
190 #[cfg(windows)]
192 lhm: crate::lhm::LhmReader,
193 nut: crate::nut::NutReader,
195}
196
197impl LocalSource {
198 pub fn new() -> Self {
200 Self::with_config(&Config::default())
201 }
202
203 pub fn with_config(config: &Config) -> Self {
210 let mut source = Self {
211 system: SystemCollector::new(),
212 nvml: NvmlCollector::new(),
213 smart: SmartReader::new(&config.smart.dir),
214 last_poll: None,
215 watched_pids: Vec::new(),
216 watched_names: Vec::new(),
217 name_matches: Vec::new(),
218 fill: crate::fill::FillTracker::new(),
219 public_ip: None,
220 nut: crate::nut::NutReader::new(&config.ups.address),
221 #[cfg(target_os = "linux")]
222 roots: SysRoots::default(),
223 #[cfg(target_os = "linux")]
224 rapl: RaplMeter::new(),
225 #[cfg(target_os = "linux")]
226 diskstats: DiskStatsMeter::new(),
227 #[cfg(target_os = "linux")]
228 net_links: NetLinkReader::new(),
229 #[cfg(target_os = "linux")]
230 failed_units: linux::systemd::FailedUnitsReader::new(),
231 #[cfg(target_os = "linux")]
232 zpool: linux::zpool::ZpoolReader::new(),
233 #[cfg(target_os = "linux")]
234 containers: linux::containers::ContainerTracker::new(),
235 #[cfg(target_os = "linux")]
236 oom: linux::oom::OomTracker::new(),
237 #[cfg(target_os = "linux")]
238 battery: linux::battery::BatteryMeter::default(),
239 #[cfg(target_os = "linux")]
240 core_locations: Vec::new(),
241 #[cfg(target_os = "linux")]
242 drm_gpus: linux::drm::DrmGpuReader::new(),
243 #[cfg(target_os = "linux")]
244 drm_clients: linux::drm_clients::DrmClientTracker::new(),
245 #[cfg(windows)]
248 lhm: crate::lhm::LhmReader::new(&config.lhm.url),
249 #[cfg(target_os = "linux")]
250 scrub_dirs: vec![
251 config.btrfs.scrub_dir.clone(),
252 linux::btrfs::BTRFS_PROGS_STATUS_DIR.into(),
253 ],
254 };
255 let _ = source.collect();
257 source
258 }
259
260 pub fn gpu_unavailable_reason(&self) -> Option<&str> {
262 self.nvml.init_error()
263 }
264
265 fn collect(&mut self) -> Snapshot {
266 let now = Instant::now();
267 let elapsed = self
268 .last_poll
269 .map(|prev| now.saturating_duration_since(prev).as_secs_f64());
270 self.last_poll = Some(now);
271
272 self.system.refresh();
273 let processes = self.system.processes(elapsed);
274 let gpus = self.nvml.collect();
275 let gpus_w = sum_present(gpus.iter().map(|g| g.power_w));
276
277 let mut snapshot = Snapshot {
278 timestamp_ms: SystemTime::now()
279 .duration_since(UNIX_EPOCH)
280 .map(|d| d.as_millis() as u64)
281 .unwrap_or(0),
282 host: self.system.host(),
283 cpu: self.system.cpu(),
284 memory: self.system.memory(),
285 networks: self.system.networks(elapsed),
286 filesystems: self.system.filesystems(),
287 process_count: processes.len(),
288 processes,
289 watched_pids: self.watched_pids.clone(),
290 gpus,
291 smart: self.smart.read(),
292 power: PowerStats {
293 gpus_w,
294 ..Default::default()
295 },
296 ..Default::default()
297 };
298
299 #[cfg(target_os = "linux")]
300 self.collect_linux(&mut snapshot, now);
301 #[cfg(not(target_os = "linux"))]
302 self.collect_portable(&mut snapshot, elapsed);
303
304 self.fill.annotate(&mut snapshot.filesystems, now);
305 snapshot.ups = self.nut.read(now);
306 if !(self.watched_names.is_empty() && self.name_matches.is_empty()) {
307 self.name_matches =
308 report_name_matches(&self.watched_names, &self.name_matches, &mut snapshot);
309 }
310 let power = &mut snapshot.power;
311 power.total_w = sum_present([power.cpu_or_soc_w().1, power.gpus_w]);
312 snapshot.public_ip = self
313 .public_ip
314 .as_mut()
315 .and_then(|checker| checker.poll().map(str::to_owned));
316 snapshot
317 }
318
319 #[cfg(not(target_os = "linux"))]
322 fn collect_portable(&mut self, snapshot: &mut Snapshot, elapsed: Option<f64>) {
323 snapshot.disks = self.system.volume_io(elapsed);
324 let (chips, cpu_temperatures) = self.system.components();
325 snapshot.sensors = chips;
326 snapshot.cpu.temperatures = cpu_temperatures;
327 #[cfg(windows)]
328 {
329 crate::windows::annotate_links(&mut snapshot.networks);
330 match self.lhm.read(Instant::now()) {
333 Some(lhm) => {
334 if !lhm.chips.is_empty() {
335 snapshot.sensors = lhm.chips;
336 }
337 if !lhm.cpu_temperatures.is_empty() {
338 snapshot.cpu.temperatures = lhm.cpu_temperatures;
339 }
340 snapshot.power.cpu_package_w = lhm.cpu_package_w;
341 if !lhm.smart.is_empty() {
342 snapshot.smart = lhm.smart;
343 }
344 }
345 None => snapshot.lhm_error = self.lhm.problem().map(str::to_owned),
346 }
347 }
348 }
349
350 #[cfg(target_os = "linux")]
352 fn collect_linux(&mut self, snapshot: &mut Snapshot, now: Instant) {
353 let chips = linux::hwmon::read_chips(&self.roots);
354
355 let cores = snapshot.cpu.per_core_percent.len();
356 if self.core_locations.len() != cores {
357 self.core_locations = linux::hwmon::read_core_locations(&self.roots, cores);
358 }
359 (snapshot.cpu.temperatures, snapshot.cpu.per_core_celsius) =
360 linux::hwmon::split_core_temperatures(&chips, &self.core_locations);
361 snapshot.memory.cached_bytes = linux::meminfo::read_cached_bytes(&self.roots);
362 if let Some(arc) = linux::zfs::read_arc(&self.roots) {
363 linux::zfs::account_arc(&mut snapshot.memory, arc);
364 }
365 snapshot.memory.oom_kills =
366 self.oom
367 .sample(&self.roots, &snapshot.processes, snapshot.timestamp_ms);
368 snapshot.power.cpu_package_w = self.rapl.sample(&self.roots, now);
369 snapshot.power.igpu_w = sum_present(
372 chips
373 .iter()
374 .filter(|c| c.is_apu)
375 .flat_map(|c| c.power_w.iter().map(|(_, w)| Some(*w))),
376 );
377 snapshot.power.battery = self.battery.read(&self.roots);
378
379 let first = snapshot.gpus.len();
381 snapshot
382 .gpus
383 .extend(self.drm_gpus.read(&self.roots, first as u32));
384 self.drm_clients.annotate(
385 &self.roots,
386 &snapshot.processes,
387 &mut snapshot.gpus[first..],
388 self.drm_gpus.pci_addresses(),
389 now,
390 );
391 snapshot.power.gpus_w = sum_present(snapshot.gpus.iter().map(|g| g.power_w));
392
393 let mut disks = self.diskstats.sample(&self.roots, now);
394 for disk in &mut disks {
395 disk.temperature_c = nvme_temperature(&chips, &disk.name);
396 }
397 snapshot.disks = disks;
398
399 let mounts = linux::mounts::read_mounts(&self.roots);
400 snapshot.raid.btrfs = linux::btrfs::read_filesystems(&self.roots, &mounts);
401 for fs in &mut snapshot.raid.btrfs {
402 fs.scrub = linux::btrfs::read_scrub(&self.scrub_dirs, &fs.fsid);
403 }
404 self.net_links
405 .annotate(&self.roots, &mut snapshot.networks, now);
406 snapshot.raid.md = linux::mdstat::read_arrays(&self.roots);
407 snapshot.raid.zfs = linux::zfs::read_pools(&self.roots);
408 self.zpool.annotate(&mut snapshot.raid.zfs, now);
409 snapshot.failed_units = self.failed_units.read(now);
410 snapshot.pressure = linux::pressure::read_pressure(&self.roots);
411 let release = std::fs::read_to_string(self.roots.proc.join("sys/kernel/osrelease"))
412 .unwrap_or_default();
413 snapshot.host.reboot_required = linux::reboot::reboot_required(
414 std::path::Path::new("/run"),
415 std::path::Path::new("/usr/lib/modules"),
416 &release,
417 );
418 for process in &mut snapshot.processes {
420 if let Some(stat) = linux::procstat::read(&self.roots, process.pid) {
421 process.threads = stat.threads;
422 process.nice = stat.nice;
423 } else {
424 process.threads = None;
425 }
426 }
427 self.containers
428 .annotate(&self.roots, &mut snapshot.processes, now);
429
430 snapshot.sensors = chips.into_iter().map(|c| c.chip).collect();
431 }
432}
433
434impl Default for LocalSource {
435 fn default() -> Self {
436 Self::new()
437 }
438}
439
440impl DataSource for LocalSource {
441 fn poll(&mut self) -> Result<Snapshot, SourceError> {
442 Ok(self.collect())
443 }
444
445 fn signal_process(&mut self, pid: u32, signal: ProcessSignal) -> Result<(), SourceError> {
446 send_signal(pid, signal).map_err(|source| SourceError::Signal { pid, source })
447 }
448
449 fn set_process_limit(&mut self, limit: Option<usize>) {
452 self.nvml.set_process_utilization(limit.is_none());
453 }
454
455 fn watch_processes(&mut self, pids: Vec<u32>) {
456 self.watched_pids = pids;
457 }
458
459 fn watch_process_names(&mut self, names: Vec<String>) {
460 self.watched_names = names;
461 }
462
463 fn set_public_ip(&mut self, lookup: Option<PublicIpLookup>) {
464 if self.public_ip.as_ref().map(PublicIpChecker::lookup) != lookup.as_ref() {
466 self.public_ip = lookup.map(PublicIpChecker::new);
467 }
468 }
469}
470
471fn report_name_matches(names: &[String], previous: &[u32], snapshot: &mut Snapshot) -> Vec<u32> {
476 let matches: Vec<u32> = snapshot
477 .processes
478 .iter()
479 .filter(|p| {
480 !p.kernel_thread
481 && names
482 .iter()
483 .any(|name| crate::view::process_matches(p, name))
484 })
485 .map(|p| p.pid)
486 .collect();
487 let gone = previous
488 .iter()
489 .filter(|pid| !snapshot.processes.iter().any(|p| p.pid == **pid));
490 for pid in matches.iter().chain(gone) {
491 if !snapshot.watched_pids.contains(pid) {
492 snapshot.watched_pids.push(*pid);
493 }
494 }
495 matches
496}
497
498#[cfg(unix)]
500fn send_signal(pid: u32, signal: ProcessSignal) -> std::io::Result<()> {
501 let pid = libc::pid_t::try_from(pid)
504 .ok()
505 .filter(|p| *p > 0)
506 .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::InvalidInput))?;
507 let sig = match signal {
508 ProcessSignal::Terminate => libc::SIGTERM,
509 ProcessSignal::Kill => libc::SIGKILL,
510 ProcessSignal::Stop => libc::SIGSTOP,
511 ProcessSignal::Continue => libc::SIGCONT,
512 ProcessSignal::Nice(nice) => return set_nice(pid, nice),
513 };
514 if unsafe { libc::kill(pid, sig) } == 0 {
516 Ok(())
517 } else {
518 Err(std::io::Error::last_os_error())
519 }
520}
521
522#[cfg(unix)]
524fn set_nice(pid: libc::pid_t, nice: i32) -> std::io::Result<()> {
525 let nice = nice.clamp(-20, 19);
526 if unsafe { libc::setpriority(libc::PRIO_PROCESS as _, pid as _, nice) } == 0 {
529 Ok(())
530 } else {
531 Err(std::io::Error::last_os_error())
532 }
533}
534
535#[cfg(windows)]
536use crate::windows::send_signal;
537
538#[cfg(not(any(unix, windows)))]
540fn send_signal(_pid: u32, _signal: ProcessSignal) -> std::io::Result<()> {
541 Err(std::io::Error::from(std::io::ErrorKind::Unsupported))
542}
543
544fn sum_present(values: impl IntoIterator<Item = Option<f32>>) -> Option<f32> {
546 values
547 .into_iter()
548 .flatten()
549 .fold(None, |acc, v| Some(acc.unwrap_or(0.0) + v))
550}
551
552#[cfg(target_os = "linux")]
555fn nvme_temperature(chips: &[linux::hwmon::HwmonChip], disk: &str) -> Option<f32> {
556 chips.iter().find_map(|c| {
557 let controller = c.chip.key.strip_prefix("nvme:")?;
558 device_matches_disk(controller, disk).then_some(())?;
559 c.chip.temperatures.first().map(|t| t.celsius)
561 })
562}
563
564#[cfg(test)]
565mod tests {
566 use super::*;
567 use crate::snapshot::ProcessInfo;
568
569 #[test]
570 fn named_processes_are_reported_until_seen_gone() {
571 let process = |pid, name: &str, command: &str| ProcessInfo {
572 pid,
573 name: name.to_owned(),
574 command: command.to_owned(),
575 ..Default::default()
576 };
577 let names = vec!["train.py".to_owned()];
578 let mut snapshot = Snapshot {
579 processes: vec![
580 process(10, "python3", "python3 jobs/train.py --epochs 3"),
581 process(11, "vim", "vim notes.txt"),
582 ],
583 watched_pids: vec![99],
584 ..Default::default()
585 };
586 let matches = report_name_matches(&names, &[], &mut snapshot);
587 assert_eq!(matches, [10]);
588 assert_eq!(snapshot.watched_pids, [99, 10]);
589
590 let mut later = Snapshot {
592 processes: vec![process(11, "vim", "vim notes.txt")],
593 ..Default::default()
594 };
595 let matches = report_name_matches(&names, &matches, &mut later);
596 assert!(matches.is_empty());
597 assert_eq!(later.watched_pids, [10]);
598 let mut after = Snapshot::default();
599 assert!(report_name_matches(&names, &matches, &mut after).is_empty());
600 assert!(after.watched_pids.is_empty());
601 }
602
603 #[cfg(unix)]
604 #[test]
605 fn signal_rejects_group_pids_and_reports_missing_process() {
606 assert_eq!(
607 send_signal(0, ProcessSignal::Terminate).unwrap_err().kind(),
608 std::io::ErrorKind::InvalidInput
609 );
610 assert_eq!(
611 send_signal(u32::MAX, ProcessSignal::Kill)
612 .unwrap_err()
613 .kind(),
614 std::io::ErrorKind::InvalidInput
615 );
616 let mut child = std::process::Command::new("true").spawn().unwrap();
618 let pid = child.id();
619 child.wait().unwrap();
620 let err = send_signal(pid, ProcessSignal::Terminate).unwrap_err();
621 assert_eq!(err.raw_os_error(), Some(libc::ESRCH));
622 }
623
624 #[cfg(unix)]
625 #[test]
626 fn signal_terminates_a_child_process() {
627 let mut child = std::process::Command::new("sleep")
628 .arg("30")
629 .spawn()
630 .unwrap();
631 send_signal(child.id(), ProcessSignal::Kill).unwrap();
632 let status = child.wait().unwrap();
633 assert!(!status.success());
634 }
635
636 #[cfg(target_os = "linux")]
637 #[test]
638 fn pause_resume_and_renice_a_child_process() {
639 let mut child = std::process::Command::new("sleep")
640 .arg("30")
641 .spawn()
642 .unwrap();
643 let pid = child.id();
644 let roots = SysRoots::default();
645 let state = || {
646 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).unwrap();
647 let after = &stat[stat.rfind(')').unwrap() + 2..];
648 after.chars().next().unwrap()
649 };
650 send_signal(pid, ProcessSignal::Stop).unwrap();
651 for _ in 0..100 {
653 if state() == 'T' {
654 break;
655 }
656 std::thread::sleep(std::time::Duration::from_millis(10));
657 }
658 assert_eq!(state(), 'T');
659 send_signal(pid, ProcessSignal::Continue).unwrap();
660 send_signal(pid, ProcessSignal::Nice(7)).unwrap();
662 assert_eq!(linux::procstat::read(&roots, pid).unwrap().nice, Some(7));
663 child.kill().unwrap();
664 child.wait().unwrap();
665 }
666
667 #[test]
668 fn sum_present_ignores_missing_values() {
669 assert_eq!(sum_present([None, None]), None);
670 assert_eq!(sum_present([Some(1.5), None, Some(2.0)]), Some(3.5));
671 }
672
673 #[cfg(target_os = "linux")]
674 #[test]
675 fn nvme_temperature_matches_controller_exactly() {
676 use crate::linux::hwmon::HwmonChip;
677 use crate::snapshot::{SensorChip, Temperature};
678
679 let chip = |key: &str, celsius: f32| HwmonChip {
680 chip: SensorChip {
681 key: key.to_owned(),
682 name: "nvme".to_owned(),
683 temperatures: vec![Temperature {
684 label: "Composite".to_owned(),
685 celsius,
686 ..Default::default()
687 }],
688 fans: Vec::new(),
689 },
690 power_w: Vec::new(),
691 is_apu: false,
692 };
693 let chips = vec![chip("nvme:nvme1", 40.0), chip("nvme:nvme10", 50.0)];
694 assert_eq!(nvme_temperature(&chips, "nvme10n1"), Some(50.0));
695 assert_eq!(nvme_temperature(&chips, "nvme1n1"), Some(40.0));
696 assert_eq!(nvme_temperature(&chips, "sda"), None);
697 }
698}