use chrono::Utc;
use log::{error, warn};
use metrics::{describe_counter, describe_gauge, describe_histogram, gauge};
use metrics_exporter_prometheus::PrometheusBuilder;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
use sysinfo::{CpuRefreshKind, MemoryRefreshKind, RefreshKind, System};
fn new_system() -> Option<System> {
match System::new_with_specifics(
RefreshKind::nothing()
.with_cpu(CpuRefreshKind::everything())
.with_memory(MemoryRefreshKind::everything()),
) {
Ok(sys) => {
Some(sys)
}
Err(e) => {
log::error!("sysinfo System 初始化失败,系统监控 fail-closed 降级(读取返回 100%,内存准入拒绝新任务): {e}");
None
}
}
}
pub trait SystemMonitorTrait: Send + Sync {
fn cpu_usage(&self) -> f64;
fn memory_usage(&self) -> f64;
fn is_metrics_stale(&self) -> bool;
}
pub struct SystemMonitorComponent {
system: Arc<Mutex<Option<System>>>,
last_update: Arc<AtomicU64>,
cpu_usage: Arc<AtomicU64>,
memory_usage: Arc<AtomicU64>,
}
impl SystemMonitorComponent {
pub fn new() -> Self {
let sys = new_system();
Self {
system: Arc::new(Mutex::new(sys)),
last_update: Arc::new(AtomicU64::new(0)),
cpu_usage: Arc::new(AtomicU64::new(0)),
memory_usage: Arc::new(AtomicU64::new(0)),
}
}
#[cfg(test)]
fn with_system(system: Option<System>) -> Self {
Self {
system: Arc::new(Mutex::new(system)),
last_update: Arc::new(AtomicU64::new(0)),
cpu_usage: Arc::new(AtomicU64::new(0)),
memory_usage: Arc::new(AtomicU64::new(0)),
}
}
fn update(&mut self) {
let mut guard = match self.system.lock() {
Ok(guard) => guard,
Err(_) => {
log::error!("System monitor mutex poisoned, cannot update metrics");
return;
}
};
let Some(sys) = guard.as_mut() else {
return;
};
sys.refresh_cpu_all();
sys.refresh_memory();
}
pub fn refresh(&mut self) {
self.update();
let cpu_usage = {
let guard = match self.system.lock() {
Ok(guard) => guard,
Err(_) => {
log::error!(
"Failed to acquire system monitor lock for CPU usage; fail-closed 100%"
);
self.cpu_usage.store(100, Ordering::Relaxed);
self.memory_usage.store(100, Ordering::Relaxed);
return;
}
};
match guard.as_ref() {
Some(sys) => f64::from(sys.global_cpu_usage()),
None => 100.0,
}
};
let memory_usage = {
let guard = match self.system.lock() {
Ok(guard) => guard,
Err(_) => {
log::error!(
"Failed to acquire system monitor lock for memory usage; fail-closed 100%"
);
self.cpu_usage.store(100, Ordering::Relaxed);
self.memory_usage.store(100, Ordering::Relaxed);
return;
}
};
match guard.as_ref() {
Some(sys) => {
let total_mem = sys.total_memory();
if total_mem > 0 {
sys.used_memory() as f64 / total_mem as f64 * 100.0
} else {
0.0
}
}
None => 100.0,
}
};
let now = Utc::now().timestamp() as u64;
self.last_update.store(now, Ordering::Relaxed);
self.cpu_usage.store((cpu_usage) as u64, Ordering::Relaxed);
self.memory_usage
.store(memory_usage as u64, Ordering::Relaxed);
}
}
impl Default for SystemMonitorComponent {
fn default() -> Self {
Self::new()
}
}
impl SystemMonitorTrait for SystemMonitorComponent {
fn cpu_usage(&self) -> f64 {
self.cpu_usage.load(Ordering::Relaxed) as f64 / 100.0
}
fn memory_usage(&self) -> f64 {
self.memory_usage.load(Ordering::Relaxed) as f64 / 100.0
}
fn is_metrics_stale(&self) -> bool {
let last_update = self.last_update.load(Ordering::Relaxed);
if last_update == 0 {
return true;
}
let now = Utc::now().timestamp() as u64;
now - last_update > 2
}
}
#[derive(Clone)]
pub struct SystemMonitor {
system: Arc<Mutex<Option<System>>>,
}
impl SystemMonitor {
pub fn new() -> Self {
let sys = new_system();
Self {
system: Arc::new(Mutex::new(sys)),
}
}
#[cfg(test)]
fn with_system(system: Option<System>) -> Self {
Self {
system: Arc::new(Mutex::new(system)),
}
}
fn update(&mut self) {
let mut guard = match self.system.lock() {
Ok(guard) => guard,
Err(_) => {
log::error!("System monitor mutex poisoned, cannot update metrics");
return;
}
};
let Some(sys) = guard.as_mut() else {
return;
};
sys.refresh_cpu_all();
sys.refresh_memory();
}
pub fn cpu_usage(&mut self) -> f64 {
self.update();
let guard = match self.system.lock() {
Ok(guard) => guard,
Err(_) => {
log::error!(
"System monitor mutex poisoned, returning fail-closed 1.0 for CPU usage"
);
return 1.0;
}
};
match guard.as_ref() {
Some(sys) => f64::from(sys.global_cpu_usage()) / 100.0,
None => 1.0,
}
}
pub fn memory_usage(&mut self) -> f64 {
self.update();
let guard = match self.system.lock() {
Ok(guard) => guard,
Err(_) => {
log::error!(
"System monitor mutex poisoned, returning fail-closed 1.0 for memory usage"
);
return 1.0;
}
};
match guard.as_ref() {
Some(sys) => {
let total_mem = sys.total_memory();
if total_mem > 0 {
sys.used_memory() as f64 / total_mem as f64
} else {
0.0
}
}
None => 1.0,
}
}
}
impl Default for SystemMonitor {
fn default() -> Self {
Self::new()
}
}
pub struct MutableSystemMonitor {
system: Arc<Mutex<Option<System>>>,
}
impl MutableSystemMonitor {
fn new() -> Self {
let sys = new_system();
Self {
system: Arc::new(Mutex::new(sys)),
}
}
#[cfg(test)]
fn with_system(system: Option<System>) -> Self {
Self {
system: Arc::new(Mutex::new(system)),
}
}
pub fn refresh(&self) {
let mut guard = match self.system.lock() {
Ok(guard) => guard,
Err(_) => {
log::error!("MutableSystemMonitor mutex poisoned, cannot refresh");
return;
}
};
let Some(sys) = guard.as_mut() else {
return;
};
sys.refresh_cpu_all();
sys.refresh_memory();
}
pub fn cpu_usage(&self) -> f64 {
let guard = match self.system.lock() {
Ok(guard) => guard,
Err(_) => {
log::error!(
"MutableSystemMonitor mutex poisoned, returning fail-closed 1.0 for CPU usage"
);
return 1.0;
}
};
match guard.as_ref() {
Some(sys) => f64::from(sys.global_cpu_usage()) / 100.0,
None => 1.0,
}
}
pub fn memory_usage(&self) -> f64 {
let guard = match self.system.lock() {
Ok(guard) => guard,
Err(_) => {
log::error!("MutableSystemMonitor mutex poisoned, returning fail-closed 1.0 for memory usage");
return 1.0;
}
};
match guard.as_ref() {
Some(sys) => {
let total_mem = sys.total_memory();
if total_mem > 0 {
sys.used_memory() as f64 / total_mem as f64
} else {
0.0
}
}
None => 1.0,
}
}
}
impl MutableSystemMonitor {
pub fn is_degraded(&self) -> bool {
match self.system.lock() {
Ok(guard) => guard.is_none(),
Err(_) => true,
}
}
}
impl SystemMonitorTrait for MutableSystemMonitor {
fn cpu_usage(&self) -> f64 {
MutableSystemMonitor::cpu_usage(self)
}
fn memory_usage(&self) -> f64 {
MutableSystemMonitor::memory_usage(self)
}
fn is_metrics_stale(&self) -> bool {
match self.system.lock() {
Ok(guard) => guard.is_none(),
Err(_) => true,
}
}
}
static SHARED_MONITOR: OnceLock<Arc<MutableSystemMonitor>> = OnceLock::new();
pub fn shared_system_monitor() -> Arc<MutableSystemMonitor> {
SHARED_MONITOR
.get_or_init(|| {
let monitor = Arc::new(MutableSystemMonitor::new());
let monitor_clone = Arc::clone(&monitor);
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
monitor_clone.refresh();
}
});
monitor
})
.clone()
}
static PROMETHEUS_HANDLE: OnceLock<metrics_exporter_prometheus::PrometheusHandle> = OnceLock::new();
pub fn prometheus_handle() -> Option<&'static metrics_exporter_prometheus::PrometheusHandle> {
PROMETHEUS_HANDLE.get()
}
fn install_prometheus_recorder() -> bool {
match PrometheusBuilder::new().install_recorder() {
Ok(handle) => {
let _ = PROMETHEUS_HANDLE.set(handle);
true
}
Err(e) => {
log::error!(
"Failed to install Prometheus recorder: {}. Metrics will be disabled.",
e
);
false
}
}
}
fn spawn_system_metrics_updater(monitor: Arc<MutableSystemMonitor>) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
update_system_metrics(&monitor);
}
});
}
pub fn init_metrics() {
if !install_prometheus_recorder() {
return;
}
let monitor = shared_system_monitor();
spawn_system_metrics_updater(monitor);
describe_gauge!(
"system_cpu_usage_ratio",
"Current CPU usage ratio (0.0 to 1.0)"
);
describe_gauge!(
"system_memory_usage_ratio",
"Current memory usage ratio (0.0 to 1.0)"
);
describe_counter!("crawl_tasks_total", "Total number of crawl tasks submitted");
describe_counter!(
"crawl_tasks_completed_total",
"Total number of crawl tasks completed"
);
describe_counter!(
"crawl_tasks_failed_total",
"Total number of crawl tasks failed"
);
describe_histogram!(
"crawl_duration_seconds",
"Duration of crawl tasks in seconds"
);
describe_counter!(
"circuit_breaker_requests_total",
"Total number of requests processed by circuit breaker"
);
describe_counter!(
"circuit_breaker_failures_total",
"Total number of failed requests recorded by circuit breaker"
);
describe_counter!(
"circuit_breaker_successes_total",
"Total number of successful requests recorded by circuit breaker"
);
describe_counter!(
"circuit_breaker_rejected_total",
"Total number of requests rejected by open circuit breaker"
);
describe_gauge!(
"circuit_breaker_status",
"Current status of circuit breaker (0=Closed, 0.5=HalfOpen, 1=Open)"
);
describe_gauge!(
"crawlrs_queue_depth",
"Current number of items in the task queue"
);
describe_counter!(
"crawlrs_engine_success_total",
"Total engine routing results by engine and outcome"
);
describe_histogram!(
"crawlrs_engine_duration_seconds",
"Engine routing duration in seconds by engine"
);
describe_counter!(
"crawlrs_cache_hit_total",
"Total cache lookups by cache_type and result (hit/miss)"
);
describe_counter!(
"crawlrs_webhook_delivery_total",
"Total webhook delivery attempts by result and status_code"
);
describe_gauge!(
"crawlrs_system_monitor_degraded",
"Whether system monitoring is degraded (1 = sysinfo unavailable, readings are fail-closed sentinels)"
);
}
fn update_system_metrics(monitor: &MutableSystemMonitor) {
monitor.refresh();
gauge!("crawlrs_system_monitor_degraded").set(if monitor.is_degraded() { 1.0 } else { 0.0 });
let cpu_usage = monitor.cpu_usage();
gauge!("system_cpu_usage_ratio").set(cpu_usage);
if cpu_usage > 0.9 {
error!(
"CRITICAL: System CPU usage is extremely high: {:.2}%",
cpu_usage * 100.0
);
} else if cpu_usage > 0.8 {
warn!("ALARM: System CPU usage is high: {:.2}%", cpu_usage * 100.0);
}
let mem_usage = monitor.memory_usage();
gauge!("system_memory_usage_ratio").set(mem_usage);
if mem_usage > 0.9 {
error!(
"CRITICAL: System memory usage is extremely high: {:.2}%",
mem_usage * 100.0
);
} else if mem_usage > 0.8 {
warn!(
"ALARM: System memory usage is high: {:.2}%",
mem_usage * 100.0
);
}
}
pub fn check_alerts() {
}
pub fn get_cpu_usage_with_monitor(monitor: &dyn SystemMonitorTrait) -> f64 {
monitor.cpu_usage()
}
pub fn get_memory_usage_with_monitor(monitor: &dyn SystemMonitorTrait) -> f64 {
monitor.memory_usage()
}
pub fn is_metrics_stale_with_monitor(monitor: &dyn SystemMonitorTrait) -> bool {
monitor.is_metrics_stale()
}
#[cfg(test)]
mod tests {
use super::*;
struct MockSystemMonitor {
cpu: f64,
memory: f64,
stale: bool,
}
impl SystemMonitorTrait for MockSystemMonitor {
fn cpu_usage(&self) -> f64 {
self.cpu
}
fn memory_usage(&self) -> f64 {
self.memory
}
fn is_metrics_stale(&self) -> bool {
self.stale
}
}
#[test]
fn degraded_component_reports_fail_closed_sentinel() {
let mut c = SystemMonitorComponent::with_system(None);
c.refresh();
assert!((SystemMonitorTrait::cpu_usage(&c) - 1.0).abs() < f64::EPSILON);
assert!((SystemMonitorTrait::memory_usage(&c) - 1.0).abs() < f64::EPSILON);
assert!(!SystemMonitorTrait::is_metrics_stale(&c));
}
#[test]
fn degraded_monitor_reports_fail_closed_sentinel() {
let mut m = SystemMonitor::with_system(None);
assert!((m.cpu_usage() - 1.0).abs() < f64::EPSILON);
assert!((m.memory_usage() - 1.0).abs() < f64::EPSILON);
}
#[test]
fn degraded_mutable_monitor_reports_fail_closed_and_stale() {
let m = MutableSystemMonitor::with_system(None);
assert!((m.memory_usage() - 1.0).abs() < f64::EPSILON);
assert!((m.cpu_usage() - 1.0).abs() < f64::EPSILON);
assert!(SystemMonitorTrait::is_metrics_stale(&m));
}
#[test]
fn test_get_cpu_usage_with_monitor_returns_value() {
let monitor = MockSystemMonitor {
cpu: 0.75,
memory: 0.5,
stale: false,
};
let result = get_cpu_usage_with_monitor(&monitor);
assert!((result - 0.75).abs() < f64::EPSILON);
}
#[test]
fn test_get_cpu_usage_with_monitor_zero() {
let monitor = MockSystemMonitor {
cpu: 0.0,
memory: 0.0,
stale: false,
};
assert!((get_cpu_usage_with_monitor(&monitor) - 0.0).abs() < f64::EPSILON);
}
#[test]
fn test_get_cpu_usage_with_monitor_full() {
let monitor = MockSystemMonitor {
cpu: 1.0,
memory: 0.0,
stale: false,
};
assert!((get_cpu_usage_with_monitor(&monitor) - 1.0).abs() < f64::EPSILON);
}
#[test]
fn test_get_memory_usage_with_monitor_returns_value() {
let monitor = MockSystemMonitor {
cpu: 0.3,
memory: 0.65,
stale: false,
};
let result = get_memory_usage_with_monitor(&monitor);
assert!((result - 0.65).abs() < f64::EPSILON);
}
#[test]
fn test_get_memory_usage_with_monitor_zero() {
let monitor = MockSystemMonitor {
cpu: 0.0,
memory: 0.0,
stale: false,
};
assert!((get_memory_usage_with_monitor(&monitor) - 0.0).abs() < f64::EPSILON);
}
#[test]
fn test_get_memory_usage_with_monitor_full() {
let monitor = MockSystemMonitor {
cpu: 0.0,
memory: 1.0,
stale: false,
};
assert!((get_memory_usage_with_monitor(&monitor) - 1.0).abs() < f64::EPSILON);
}
#[test]
fn test_is_metrics_stale_with_monitor_true() {
let monitor = MockSystemMonitor {
cpu: 0.0,
memory: 0.0,
stale: true,
};
assert!(is_metrics_stale_with_monitor(&monitor));
}
#[test]
fn test_is_metrics_stale_with_monitor_false() {
let monitor = MockSystemMonitor {
cpu: 0.0,
memory: 0.0,
stale: false,
};
assert!(!is_metrics_stale_with_monitor(&monitor));
}
#[test]
fn test_system_monitor_component_new_initializes_zero() {
let component = SystemMonitorComponent::new();
assert!((component.cpu_usage() - 0.0).abs() < f64::EPSILON);
assert!((component.memory_usage() - 0.0).abs() < f64::EPSILON);
}
#[test]
fn test_system_monitor_component_default_equals_new() {
let component = SystemMonitorComponent::default();
assert!((component.cpu_usage() - 0.0).abs() < f64::EPSILON);
assert!((component.memory_usage() - 0.0).abs() < f64::EPSILON);
}
#[test]
fn test_system_monitor_component_is_stale_before_refresh() {
let component = SystemMonitorComponent::new();
assert!(
component.is_metrics_stale(),
"metrics should be stale before refresh"
);
}
#[test]
fn test_system_monitor_component_refresh_updates_timestamp() {
let mut component = SystemMonitorComponent::new();
assert!(component.is_metrics_stale());
component.refresh();
assert!(
!component.is_metrics_stale(),
"metrics should not be stale immediately after refresh"
);
}
#[test]
fn test_system_monitor_component_refresh_updates_cpu_and_memory() {
let mut component = SystemMonitorComponent::new();
assert!((component.cpu_usage() - 0.0).abs() < f64::EPSILON);
assert!((component.memory_usage() - 0.0).abs() < f64::EPSILON);
component.refresh();
let cpu = component.cpu_usage();
let mem = component.memory_usage();
assert!(
(0.0..=1.0).contains(&cpu),
"cpu usage should be in [0, 1], got {}",
cpu
);
assert!(
(0.0..=1.0).contains(&mem),
"memory usage should be in [0, 1], got {}",
mem
);
}
#[test]
fn test_system_monitor_component_cpu_usage_divides_by_100() {
let mut component = SystemMonitorComponent::new();
component.refresh();
let cpu = component.cpu_usage();
assert!(cpu >= 0.0, "cpu should be non-negative");
assert!(cpu <= 1.0, "cpu should be at most 1.0, got {}", cpu);
}
#[test]
fn test_system_monitor_new_does_not_panic() {
let _monitor = SystemMonitor::new();
}
#[test]
fn test_system_monitor_default_equals_new() {
let _monitor = SystemMonitor::default();
}
#[test]
fn test_system_monitor_cpu_usage_in_valid_range() {
let mut monitor = SystemMonitor::new();
let cpu = monitor.cpu_usage();
assert!(cpu >= 0.0, "cpu should be non-negative");
assert!(cpu <= 1.0, "cpu should be at most 1.0, got {}", cpu);
}
#[test]
fn test_system_monitor_memory_usage_in_valid_range() {
let mut monitor = SystemMonitor::new();
let mem = monitor.memory_usage();
assert!(mem >= 0.0, "memory should be non-negative");
assert!(mem <= 1.0, "memory should be at most 1.0, got {}", mem);
}
#[test]
fn test_system_monitor_clone() {
let monitor = SystemMonitor::new();
let _cloned = monitor.clone();
}
#[test]
fn test_system_monitor_multiple_cpu_reads() {
let mut monitor = SystemMonitor::new();
let cpu1 = monitor.cpu_usage();
let cpu2 = monitor.cpu_usage();
assert!((0.0..=1.0).contains(&cpu1));
assert!((0.0..=1.0).contains(&cpu2));
}
#[test]
fn test_check_alerts_does_not_panic() {
check_alerts();
}
#[test]
fn test_mock_monitor_implements_trait() {
let monitor = MockSystemMonitor {
cpu: 0.42,
memory: 0.58,
stale: false,
};
let trait_ref: &dyn SystemMonitorTrait = &monitor;
assert!((trait_ref.cpu_usage() - 0.42).abs() < f64::EPSILON);
assert!((trait_ref.memory_usage() - 0.58).abs() < f64::EPSILON);
assert!(!trait_ref.is_metrics_stale());
}
#[test]
fn test_helper_functions_with_mock_via_trait() {
let monitor = MockSystemMonitor {
cpu: 0.80,
memory: 0.90,
stale: true,
};
let trait_ref: &dyn SystemMonitorTrait = &monitor;
assert!((get_cpu_usage_with_monitor(trait_ref) - 0.80).abs() < f64::EPSILON);
assert!((get_memory_usage_with_monitor(trait_ref) - 0.90).abs() < f64::EPSILON);
assert!(is_metrics_stale_with_monitor(trait_ref));
}
#[test]
fn test_mutable_system_monitor_new_does_not_panic() {
let _monitor = MutableSystemMonitor::new();
}
#[test]
fn test_mutable_system_monitor_refresh_does_not_panic() {
let monitor = MutableSystemMonitor::new();
monitor.refresh();
}
#[test]
fn test_mutable_system_monitor_cpu_usage_in_valid_range() {
let monitor = MutableSystemMonitor::new();
let cpu = monitor.cpu_usage();
assert!(cpu >= 0.0, "cpu should be non-negative");
assert!(cpu <= 1.0, "cpu should be at most 1.0, got {}", cpu);
}
#[test]
fn test_mutable_system_monitor_memory_usage_in_valid_range() {
let monitor = MutableSystemMonitor::new();
let mem = monitor.memory_usage();
assert!(mem >= 0.0, "memory should be non-negative");
assert!(mem <= 1.0, "memory should be at most 1.0, got {}", mem);
}
#[test]
fn test_mutable_system_monitor_cpu_usage_after_refresh() {
let monitor = MutableSystemMonitor::new();
monitor.refresh();
let cpu = monitor.cpu_usage();
assert!((0.0..=1.0).contains(&cpu));
}
#[test]
fn test_mutable_system_monitor_memory_usage_after_refresh() {
let monitor = MutableSystemMonitor::new();
monitor.refresh();
let mem = monitor.memory_usage();
assert!((0.0..=1.0).contains(&mem));
}
#[test]
fn test_mutable_system_monitor_multiple_refresh_cycles() {
let monitor = MutableSystemMonitor::new();
for _ in 0..3 {
monitor.refresh();
let cpu = monitor.cpu_usage();
let mem = monitor.memory_usage();
assert!((0.0..=1.0).contains(&cpu));
assert!((0.0..=1.0).contains(&mem));
}
}
#[test]
fn test_update_system_metrics_does_not_panic() {
let monitor = MutableSystemMonitor::new();
update_system_metrics(&monitor);
}
#[test]
fn test_update_system_metrics_after_refresh() {
let monitor = MutableSystemMonitor::new();
monitor.refresh();
update_system_metrics(&monitor);
let cpu = monitor.cpu_usage();
let mem = monitor.memory_usage();
assert!((0.0..=1.0).contains(&cpu));
assert!((0.0..=1.0).contains(&mem));
}
#[test]
fn test_update_system_metrics_multiple_calls() {
let monitor = MutableSystemMonitor::new();
for _ in 0..5 {
update_system_metrics(&monitor);
}
let cpu = monitor.cpu_usage();
let mem = monitor.memory_usage();
assert!((0.0..=1.0).contains(&cpu));
assert!((0.0..=1.0).contains(&mem));
}
#[test]
fn test_system_monitor_component_mutex_poisoned_refresh_no_panic() {
let mut component = SystemMonitorComponent::new();
assert!(component.is_metrics_stale());
let system_arc = component.system.clone();
let handle = std::thread::spawn(move || {
let _guard = system_arc.lock().unwrap();
panic!("poisoning mutex for test");
});
let _ = handle.join();
component.refresh();
assert_eq!(SystemMonitorTrait::cpu_usage(&component), 1.0);
assert_eq!(SystemMonitorTrait::memory_usage(&component), 1.0);
assert!(
component.is_metrics_stale(),
"last_update should remain 0 after poisoned refresh"
);
}
#[test]
fn test_system_monitor_mutex_poisoned_cpu_fails_closed() {
let mut monitor = SystemMonitor::new();
let system_arc = monitor.system.clone();
let handle = std::thread::spawn(move || {
let _guard = system_arc.lock().unwrap();
panic!("poisoning mutex for test");
});
let _ = handle.join();
let cpu = monitor.cpu_usage();
assert_eq!(cpu, 1.0, "poisoned mutex must fail closed (100%)");
}
#[test]
fn test_system_monitor_mutex_poisoned_memory_fails_closed() {
let mut monitor = SystemMonitor::new();
let system_arc = monitor.system.clone();
let handle = std::thread::spawn(move || {
let _guard = system_arc.lock().unwrap();
panic!("poisoning mutex for test");
});
let _ = handle.join();
let mem = monitor.memory_usage();
assert_eq!(mem, 1.0, "poisoned mutex must fail closed (100%)");
}
#[test]
fn test_mutable_system_monitor_mutex_poisoned_refresh_no_panic() {
let monitor = MutableSystemMonitor::new();
let system_arc = monitor.system.clone();
let handle = std::thread::spawn(move || {
let _guard = system_arc.lock().unwrap();
panic!("poisoning mutex for test");
});
let _ = handle.join();
monitor.refresh();
}
#[test]
fn test_mutable_system_monitor_mutex_poisoned_cpu_fails_closed() {
let monitor = MutableSystemMonitor::new();
let system_arc = monitor.system.clone();
let handle = std::thread::spawn(move || {
let _guard = system_arc.lock().unwrap();
panic!("poisoning mutex for test");
});
let _ = handle.join();
let cpu = monitor.cpu_usage();
assert_eq!(cpu, 1.0);
}
#[test]
fn test_mutable_system_monitor_mutex_poisoned_memory_fails_closed() {
let monitor = MutableSystemMonitor::new();
let system_arc = monitor.system.clone();
let handle = std::thread::spawn(move || {
let _guard = system_arc.lock().unwrap();
panic!("poisoning mutex for test");
});
let _ = handle.join();
let mem = monitor.memory_usage();
assert_eq!(mem, 1.0);
}
#[tokio::test]
async fn test_init_metrics_does_not_panic() {
init_metrics();
}
}