use std::sync::{Arc, RwLock};
use std::time::{Duration, Instant};
use tokio::sync::watch;
use crate::infrastructure::observability::metrics::SystemMonitorTrait;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MemoryState {
Normal,
Pressure,
Critical,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Admission {
Proceed,
Defer,
Reschedule,
}
#[derive(Debug)]
struct MemorySchedulerInner {
state: MemoryState,
critical_since: Option<Instant>,
}
impl Default for MemorySchedulerInner {
fn default() -> Self {
Self {
state: MemoryState::Normal,
critical_since: None,
}
}
}
pub struct MemoryScheduler {
monitor: Arc<dyn SystemMonitorTrait>,
pressure_threshold: f64,
critical_threshold: f64,
critical_timeout: Duration,
inner: Arc<RwLock<MemorySchedulerInner>>,
shutdown_tx: watch::Sender<bool>,
}
impl MemoryScheduler {
pub fn new(
monitor: Arc<dyn SystemMonitorTrait>,
pressure_threshold: f64,
critical_threshold: f64,
critical_timeout: Duration,
) -> Self {
let (shutdown_tx, _) = watch::channel(false);
Self {
monitor,
pressure_threshold,
critical_threshold,
critical_timeout,
inner: Arc::new(RwLock::new(MemorySchedulerInner::default())),
shutdown_tx,
}
}
pub fn state(&self) -> MemoryState {
let inner = self
.inner
.read()
.expect("MemoryScheduler inner RwLock poisoned");
inner.state
}
pub fn update_state(&self) {
let usage = self.monitor.memory_usage();
let new_state = if usage >= self.critical_threshold {
MemoryState::Critical
} else if usage >= self.pressure_threshold {
MemoryState::Pressure
} else {
MemoryState::Normal
};
let mut inner = self
.inner
.write()
.expect("MemoryScheduler inner RwLock poisoned");
match new_state {
MemoryState::Critical => {
if inner.critical_since.is_none() {
inner.critical_since = Some(Instant::now());
}
}
_ => inner.critical_since = None,
}
inner.state = new_state;
}
pub async fn admit(&self) -> Admission {
match self.state() {
MemoryState::Normal => Admission::Proceed,
MemoryState::Pressure => Admission::Defer,
MemoryState::Critical => Admission::Reschedule,
}
}
pub fn check_critical_timeout(&self) -> bool {
let inner = self
.inner
.read()
.expect("MemoryScheduler inner RwLock poisoned");
match inner.critical_since {
Some(since) => since.elapsed() >= self.critical_timeout,
None => false,
}
}
pub fn shutdown_signal(&self) -> watch::Receiver<bool> {
self.shutdown_tx.subscribe()
}
pub fn spawn_monitor(self: Arc<Self>) -> (watch::Receiver<bool>, tokio::task::JoinHandle<()>) {
let rx = self.shutdown_signal();
let me = Arc::clone(&self);
let handle = tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(1));
loop {
interval.tick().await;
me.update_state();
if me.check_critical_timeout() {
let _ = me.shutdown_tx.send(true);
break;
}
}
});
(rx, handle)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU64, Ordering};
#[derive(Clone)]
struct MockHandle {
memory_bits: Arc<AtomicU64>,
}
impl MockHandle {
fn set_memory(&self, value: f64) {
self.memory_bits.store(value.to_bits(), Ordering::Relaxed);
}
}
struct MockSystemMonitor {
memory_bits: Arc<AtomicU64>,
}
impl MockSystemMonitor {
fn pair(memory: f64) -> (Arc<Self>, MockHandle) {
let bits = Arc::new(AtomicU64::new(memory.to_bits()));
(
Arc::new(Self {
memory_bits: bits.clone(),
}),
MockHandle { memory_bits: bits },
)
}
}
impl SystemMonitorTrait for MockSystemMonitor {
fn cpu_usage(&self) -> f64 {
0.0
}
fn memory_usage(&self) -> f64 {
f64::from_bits(self.memory_bits.load(Ordering::Relaxed))
}
fn is_metrics_stale(&self) -> bool {
false
}
}
fn make_scheduler(
memory: f64,
pressure: f64,
critical: f64,
timeout: Duration,
) -> (MemoryScheduler, MockHandle) {
let (mock, handle) = MockSystemMonitor::pair(memory);
(
MemoryScheduler::new(mock, pressure, critical, timeout),
handle,
)
}
fn make_static_scheduler(
memory: f64,
pressure: f64,
critical: f64,
timeout: Duration,
) -> MemoryScheduler {
make_scheduler(memory, pressure, critical, timeout).0
}
#[test]
fn test_state_normal_below_pressure_threshold() {
let scheduler = make_static_scheduler(0.5, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.state(), MemoryState::Normal);
}
#[test]
fn test_state_pressure_between_thresholds() {
let scheduler = make_static_scheduler(0.85, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.state(), MemoryState::Pressure);
}
#[test]
fn test_state_pressure_at_pressure_threshold_boundary() {
let scheduler = make_static_scheduler(0.8, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.state(), MemoryState::Pressure);
}
#[test]
fn test_state_critical_at_critical_threshold_boundary() {
let scheduler = make_static_scheduler(0.9, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.state(), MemoryState::Critical);
}
#[test]
fn test_state_critical_above_critical_threshold() {
let scheduler = make_static_scheduler(0.95, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.state(), MemoryState::Critical);
}
#[test]
fn test_state_defaults_to_normal_before_first_sample() {
let scheduler = make_static_scheduler(0.95, 0.8, 0.9, Duration::from_secs(30));
assert_eq!(scheduler.state(), MemoryState::Normal);
}
#[test]
fn test_state_transition_critical_to_normal_clears_critical_since() {
let (scheduler, handle) = make_scheduler(0.95, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.state(), MemoryState::Critical);
assert!(!scheduler.check_critical_timeout());
handle.set_memory(0.3);
scheduler.update_state();
assert_eq!(scheduler.state(), MemoryState::Normal);
assert!(!scheduler.check_critical_timeout());
}
#[test]
fn test_critical_since_persists_across_consecutive_critical_samples() {
let (scheduler, _handle) = make_scheduler(0.95, 0.8, 0.9, Duration::from_millis(10));
scheduler.update_state();
assert!(!scheduler.check_critical_timeout());
std::thread::sleep(Duration::from_millis(5));
scheduler.update_state();
std::thread::sleep(Duration::from_millis(10));
assert!(
scheduler.check_critical_timeout(),
"critical_since 必须保持首次进入时刻"
);
}
#[tokio::test]
async fn test_admit_proceed_when_normal() {
let scheduler = make_static_scheduler(0.5, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.admit().await, Admission::Proceed);
}
#[tokio::test]
async fn test_admit_defer_when_pressure() {
let scheduler = make_static_scheduler(0.85, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.admit().await, Admission::Defer);
}
#[tokio::test]
async fn test_admit_reschedule_when_critical() {
let scheduler = make_static_scheduler(0.95, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.admit().await, Admission::Reschedule);
}
#[tokio::test]
async fn test_admit_follows_live_state_changes() {
let (scheduler, handle) = make_scheduler(0.5, 0.8, 0.9, Duration::from_secs(30));
scheduler.update_state();
assert_eq!(scheduler.admit().await, Admission::Proceed);
handle.set_memory(0.85);
scheduler.update_state();
assert_eq!(scheduler.admit().await, Admission::Defer);
handle.set_memory(0.95);
scheduler.update_state();
assert_eq!(scheduler.admit().await, Admission::Reschedule);
}
#[test]
fn test_check_critical_timeout_false_when_not_critical() {
let scheduler = make_static_scheduler(0.5, 0.8, 0.9, Duration::from_millis(10));
scheduler.update_state();
assert!(!scheduler.check_critical_timeout());
}
#[test]
fn test_check_critical_timeout_false_immediately_after_entering_critical() {
let scheduler = make_static_scheduler(0.95, 0.8, 0.9, Duration::from_secs(60));
scheduler.update_state();
assert!(!scheduler.check_critical_timeout());
}
#[test]
fn test_check_critical_timeout_true_after_elapsed_exceeds_timeout() {
let scheduler = make_static_scheduler(0.95, 0.8, 0.9, Duration::from_millis(20));
scheduler.update_state();
std::thread::sleep(Duration::from_millis(30));
assert!(scheduler.check_critical_timeout());
}
#[tokio::test]
async fn test_shutdown_signal_initially_false() {
let scheduler = make_static_scheduler(0.5, 0.8, 0.9, Duration::from_secs(30));
let rx = scheduler.shutdown_signal();
assert!(!(*rx.borrow()));
}
#[tokio::test]
async fn test_spawn_monitor_emits_shutdown_on_critical_timeout() {
let scheduler = Arc::new(make_static_scheduler(
0.95,
0.8,
0.9,
Duration::from_millis(50),
));
let (mut rx, _handle) = scheduler.clone().spawn_monitor();
let changed = tokio::time::timeout(Duration::from_secs(5), rx.changed()).await;
assert!(changed.is_ok(), "spawn_monitor 必须在超时后发送关闭信号");
assert!(*rx.borrow());
}
#[tokio::test]
async fn test_spawn_monitor_no_shutdown_when_memory_recovers() {
let scheduler = Arc::new(make_static_scheduler(
0.5,
0.8,
0.9,
Duration::from_millis(50),
));
let (mut rx, _handle) = scheduler.clone().spawn_monitor();
let changed = tokio::time::timeout(Duration::from_millis(500), rx.changed()).await;
assert!(changed.is_err(), "内存正常时不应发送 shutdown 信号");
}
}