use crate::{
config::{MemoryConfigParsed, RuntimeContext},
events::{LogLevel, SentinelEvent},
logging::{self, get_log_level},
psi::read_psi_total,
swap,
};
use std::time::Instant;
use sysinfo::{MemoryRefreshKind, RefreshKind, System};
pub struct Monitor {
system: System,
last_psi_total: Option<u64>,
last_psi_time: Instant,
last_warn_time: Option<Instant>,
pub ram_bytes: Option<u64>,
pub ram_percent: Option<f64>,
pub swap_bytes: Option<u64>,
pub swap_percent: Option<f64>,
pub zram_bytes: Option<u64>,
pub zram_percent: Option<f64>,
pub psi_pressure: Option<f64>,
}
pub enum MonitorStatus {
Normal,
Warn, Kill(SentinelEvent<'static>), }
impl Monitor {
pub fn new() -> Self {
let mut system = System::new_with_specifics(
RefreshKind::nothing().with_memory(MemoryRefreshKind::everything()),
);
system.refresh_memory();
let total = Self::read_psi();
Self {
system,
last_psi_total: total,
last_psi_time: Instant::now(),
last_warn_time: None,
ram_bytes: None,
ram_percent: None,
swap_bytes: None,
swap_percent: None,
zram_bytes: None,
zram_percent: None,
psi_pressure: None,
}
}
pub fn check(&mut self, ctx: &RuntimeContext) -> MonitorStatus {
self.system.refresh_memory();
let now = Instant::now();
let mut pending_warn: Option<SentinelEvent<'static>> = None;
if let Some(ram_config) = &ctx.ram {
let available = self.system.available_memory();
let total = self.system.total_memory();
if total > 0 {
let percent_free = (available as f64 / total as f64) * 100.0;
self.ram_bytes = Some(available);
self.ram_percent = Some(percent_free);
if let Some((threshold, type_str)) =
check_kill(ram_config, available, percent_free as f32)
{
let amount_needed = calc_needed(ram_config, available, total);
return MonitorStatus::Kill(SentinelEvent::KillTriggered {
trigger: "LowMemory",
observed_value: if type_str == "bytes" {
available as f64
} else {
percent_free
},
threshold_value: threshold,
threshold_type: type_str,
amount_needed,
});
}
if let Some((threshold, type_str)) =
check_warn(ram_config, available, percent_free as f32)
{
if pending_warn.is_none() {
pending_warn = Some(SentinelEvent::LowMemoryWarn {
available_bytes: available,
available_percent: percent_free,
threshold_type: type_str,
threshold_value: threshold,
});
}
}
}
}
let has_swap = ctx.swap.is_some();
let has_zram = ctx.zram.is_some();
let swap_stats = if has_swap || has_zram {
swap::get_aggregated_swap_stats(has_swap, has_zram)
} else {
swap::AggregatedSwapStats::default()
};
if let Some(swap_config) = &ctx.swap {
let free = swap_stats.disk_free;
let total = swap_stats.disk_total;
if total > 0 {
let percent_free = (free as f64 / total as f64) * 100.0;
self.swap_bytes = Some(free);
self.swap_percent = Some(percent_free);
if let Some((threshold, type_str)) =
check_kill(swap_config, free, percent_free as f32)
{
let amount_needed = calc_needed(swap_config, free, total);
return MonitorStatus::Kill(SentinelEvent::KillTriggered {
trigger: "LowSwap",
observed_value: if type_str == "bytes" {
free as f64
} else {
percent_free
},
threshold_value: threshold,
threshold_type: type_str,
amount_needed,
});
}
if let Some((threshold, type_str)) =
check_warn(swap_config, free, percent_free as f32)
{
if pending_warn.is_none() {
pending_warn = Some(SentinelEvent::LowSwapWarn {
free_bytes: free,
free_percent: percent_free,
threshold_type: type_str,
threshold_value: threshold,
});
}
}
}
}
if let Some(zram_config) = &ctx.zram {
let free = swap_stats.zram_free;
let total = swap_stats.zram_total;
if total > 0 {
let percent_free = (free as f64 / total as f64) * 100.0;
self.zram_bytes = Some(free);
self.zram_percent = Some(percent_free);
if let Some((threshold, type_str)) =
check_kill(zram_config, free, percent_free as f32)
{
let amount_needed = calc_needed(zram_config, free, total);
return MonitorStatus::Kill(SentinelEvent::KillTriggered {
trigger: "LowZram",
observed_value: if type_str == "bytes" {
free as f64
} else {
percent_free
},
threshold_value: threshold,
threshold_type: type_str,
amount_needed,
});
}
if let Some((threshold, type_str)) =
check_warn(zram_config, free, percent_free as f32)
{
if pending_warn.is_none() {
pending_warn = Some(SentinelEvent::LowZramWarn {
free_bytes: free,
free_percent: percent_free,
threshold_type: type_str,
threshold_value: threshold,
});
}
}
}
}
if let Some(psi_config) = &ctx.psi {
if now.duration_since(self.last_psi_time).as_millis() as u64
>= psi_config.check_interval_ms
{
if let Some(current_total) = Self::read_psi() {
if let Some(last_total) = self.last_psi_total {
let time_delta_us =
now.duration_since(self.last_psi_time).as_micros() as f64;
let total_delta = (current_total.saturating_sub(last_total)) as f64;
let pressure = if time_delta_us > 0.0 {
(total_delta / time_delta_us) * 100.0
} else {
0.0
};
self.last_psi_total = Some(current_total);
self.last_psi_time = now;
self.psi_pressure = Some(pressure);
if let Some(kill_max) = psi_config.kill_max_percent {
if pressure as f32 > kill_max {
let amount = psi_config.amount_to_free.expect("validated");
return MonitorStatus::Kill(SentinelEvent::KillTriggered {
trigger: "PsiPressure",
observed_value: pressure,
threshold_value: kill_max as f64,
threshold_type: "percent",
amount_needed: Some(amount),
});
}
}
if pending_warn.is_none() {
if let Some(warn_max) = psi_config.warn_max_percent {
if pressure as f32 > warn_max {
pending_warn = Some(SentinelEvent::PsiPressureWarn {
pressure_curr: pressure,
threshold: warn_max as f64,
});
}
}
}
} else {
self.last_psi_total = Some(current_total);
self.last_psi_time = now;
}
}
}
}
if get_log_level() >= LogLevel::Debug {
logging::emit(&SentinelEvent::Monitor {
memory_available_bytes: self.ram_bytes,
memory_available_percent: self.ram_percent,
swap_free_bytes: self.swap_bytes,
swap_free_percent: self.swap_percent,
zram_free_bytes: self.zram_bytes,
zram_free_percent: self.zram_percent,
psi_pressure: self.psi_pressure,
});
}
if let Some(event) = pending_warn {
if self.can_warn(ctx) {
logging::emit(&event);
self.last_warn_time = Some(now);
return MonitorStatus::Warn;
}
}
MonitorStatus::Normal
}
fn can_warn(&self, ctx: &RuntimeContext) -> bool {
match self.last_warn_time {
Some(last) => {
(Instant::now().duration_since(last).as_millis() as u64) >= ctx.warn_reset_ms
}
None => true,
}
}
fn read_psi() -> Option<u64> {
read_psi_total().ok()
}
}
fn check_kill(
config: &MemoryConfigParsed,
free_bytes: u64,
free_percent: f32,
) -> Option<(f64, &'static str)> {
if let Some(limit) = config.kill_min_free_bytes {
if free_bytes < limit {
return Some((limit as f64, "bytes"));
}
return None;
}
if let Some(limit_percent) = config.kill_min_free_percent {
if free_percent < limit_percent {
return Some((limit_percent as f64, "percent"));
}
}
None
}
fn check_warn(
config: &MemoryConfigParsed,
free_bytes: u64,
free_percent: f32,
) -> Option<(f64, &'static str)> {
if let Some(limit) = config.warn_min_free_bytes {
if free_bytes < limit {
return Some((limit as f64, "bytes"));
}
return None;
}
if let Some(limit_percent) = config.warn_min_free_percent {
if free_percent < limit_percent {
return Some((limit_percent as f64, "percent"));
}
}
None
}
fn calc_needed(config: &MemoryConfigParsed, current_free: u64, total: u64) -> Option<u64> {
let target = if let Some(bytes) = config.kill_min_free_bytes {
bytes
} else if let Some(percent) = config.kill_min_free_percent {
(total as f64 * (percent as f64 / 100.0)) as u64
} else {
0
};
if target > current_free {
Some(target - current_free)
} else {
None
}
}