use core::time::Duration;
use std::collections::HashMap;
use std::sync::{Arc, Mutex, PoisonError};
use tokio::sync::mpsc;
use shep_core::values::MemSize;
pub mod sample;
pub(crate) mod stats;
use sample::{MemorySampler, TreeIndex};
use stats::StatsState;
pub(crate) const MEMORY_POLL_INTERVAL: Duration = Duration::from_secs(15);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct LimitBreach {
pub id: u32,
pub root_pid: u32,
pub observed: MemSize,
pub limit: MemSize,
}
pub trait LimitEnforcer: Send + Sync + 'static {
fn arm(&self, id: u32, root_pid: u32, limit: MemSize);
fn disarm(&self, id: u32);
}
#[derive(Debug, Clone, Copy)]
struct Armed {
root_pid: u32,
limit: MemSize,
}
#[derive(Debug)]
pub(crate) struct PollingEnforcer {
armed: Arc<Mutex<HashMap<u32, Armed>>>,
task: tokio::task::JoinHandle<()>,
}
impl PollingEnforcer {
#[must_use]
pub(crate) fn start(
sampler: Arc<dyn MemorySampler>,
breaches: mpsc::Sender<LimitBreach>,
stats: Arc<StatsState>,
) -> Self {
let armed: Arc<Mutex<HashMap<u32, Armed>>> = Arc::new(Mutex::new(HashMap::new()));
let loop_armed = Arc::clone(&armed);
let task = tokio::spawn(async move {
loop {
tokio::time::sleep(MEMORY_POLL_INTERVAL).await;
let table = sampler.sample();
let index = TreeIndex::build(&table);
stats.record_baseline(&index, tokio::time::Instant::now());
let mut breached = Vec::new();
{
let mut guard = loop_armed.lock().unwrap_or_else(PoisonError::into_inner);
guard.retain(|&id, entry| {
let observed = index.sum_from(entry.root_pid);
let over_limit = observed > entry.limit.bytes();
if over_limit {
breached.push(LimitBreach {
id,
root_pid: entry.root_pid,
observed: MemSize::from_bytes(observed),
limit: entry.limit,
});
}
!over_limit
});
}
for breach in breached {
if breaches.send(breach).await.is_err() {
return;
}
}
}
});
Self { armed, task }
}
}
impl LimitEnforcer for PollingEnforcer {
fn arm(&self, id: u32, root_pid: u32, limit: MemSize) {
let mut guard = self.armed.lock().unwrap_or_else(PoisonError::into_inner);
guard.insert(id, Armed { root_pid, limit });
}
fn disarm(&self, id: u32) {
let mut guard = self.armed.lock().unwrap_or_else(PoisonError::into_inner);
guard.remove(&id);
}
}
impl Drop for PollingEnforcer {
fn drop(&mut self) {
self.task.abort();
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use tokio::sync::mpsc;
use super::*;
use crate::testing::{ScriptedSampler, rss, rss_cpu};
const BREACH_WAIT: Duration = Duration::from_secs(120);
async fn start_and_settle(
sampler: Arc<dyn MemorySampler>,
breaches: mpsc::Sender<LimitBreach>,
) -> PollingEnforcer {
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
let enforcer = PollingEnforcer::start(sampler, breaches, stats);
tokio::task::yield_now().await;
enforcer
}
fn ticks(n: u32) -> Duration {
MEMORY_POLL_INTERVAL * n + Duration::from_millis(1)
}
async fn expect_breach(rx: &mut mpsc::Receiver<LimitBreach>, id: u32) -> LimitBreach {
match tokio::time::timeout(BREACH_WAIT, rx.recv()).await {
Ok(Some(breach)) => breach,
Ok(None) => panic!("breach channel closed before a breach for id {id} arrived"),
Err(_) => panic!("timed out waiting for a breach for id {id}"),
}
}
async fn assert_no_breach_within(rx: &mut mpsc::Receiver<LimitBreach>, window: Duration) {
match tokio::time::timeout(window, rx.recv()).await {
Err(_) => {} Ok(Some(breach)) => panic!("unexpected breach observed: {breach:?}"),
Ok(None) => panic!("breach channel disconnected while checking for no breach"),
}
}
#[tokio::test(start_paused = true)]
async fn one_tick_records_the_sampling_baseline_for_a_sheep_nothing_is_armed_against() {
let sampler: Arc<dyn MemorySampler> = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(1, None, 100, 1_000)],
vec![rss_cpu(1, None, 100, 2_500)],
]));
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
stats.watch(9, 1);
let (tx, mut rx) = mpsc::channel(1);
let enforcer = PollingEnforcer::start(Arc::clone(&sampler), tx, Arc::clone(&stats));
tokio::task::yield_now().await;
assert_no_breach_within(&mut rx, ticks(1)).await;
assert_no_breach_within(&mut rx, Duration::from_millis(7_499)).await;
let cpu = stats.sample_now()[&1]
.cpu_percent
.expect("the tick must have recorded a baseline");
assert!(
(cpu - 20.0).abs() < 0.01,
"1500 CPU-ms over the 7.5 s since the tick is 20%, got {cpu}"
);
drop(enforcer);
}
#[tokio::test(start_paused = true)]
async fn polling_enforcer_is_dyn_compatible() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss(1, None, 0)]]));
let (tx, _rx) = mpsc::channel(1);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
let _: &dyn LimitEnforcer = &enforcer;
}
#[tokio::test(start_paused = true)]
async fn three_ticks_under_limit_produce_no_breach_and_three_samples() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss(1, None, 100)]]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(1, 1, MemSize::from_bytes(200));
assert_no_breach_within(&mut rx, ticks(3)).await;
assert_eq!(
sampler.calls(),
3,
"expected exactly one sample() call per elapsed tick"
);
}
#[tokio::test(start_paused = true)]
async fn breach_arrives_on_exactly_the_third_tick() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss(1, None, 100)], vec![rss(1, None, 100)], vec![rss(1, None, 900)], ]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(7, 1, MemSize::from_bytes(500));
let start = tokio::time::Instant::now();
assert_no_breach_within(&mut rx, ticks(2)).await; let breach = expect_breach(&mut rx, 7).await;
assert_eq!(breach.id, 7);
assert_eq!(
tokio::time::Instant::now() - start,
MEMORY_POLL_INTERVAL * 3,
"breach should arrive at exactly the third tick, not before or after"
);
}
#[tokio::test(start_paused = true)]
async fn breach_observed_is_the_tree_sum_and_root_pid_is_the_armed_pid() {
let table = vec![rss(10, None, 100), rss(11, Some(10), 100)];
let sampler = Arc::new(ScriptedSampler::new(vec![table]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(3, 10, MemSize::from_bytes(150));
let breach = expect_breach(&mut rx, 3).await;
assert_eq!(
breach.root_pid, 10,
"root_pid must be the pid arm() was given"
);
assert_eq!(
breach.observed.bytes(),
200,
"observed must be the tree sum (100 + 100), not the root's own 100 bytes"
);
assert_eq!(breach.limit.bytes(), 150);
}
#[tokio::test(start_paused = true)]
async fn a_breach_self_disarms_so_two_more_ticks_report_nothing() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss(1, None, 900)]]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(9, 1, MemSize::from_bytes(500));
expect_breach(&mut rx, 9).await;
assert_no_breach_within(&mut rx, ticks(2)).await;
}
#[tokio::test(start_paused = true)]
async fn only_the_id_over_its_own_limit_breaches() {
let table = vec![
rss(1, None, 300), rss(2, None, 700), ];
let sampler = Arc::new(ScriptedSampler::new(vec![table]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(101, 1, MemSize::from_bytes(1000));
enforcer.arm(102, 2, MemSize::from_bytes(500));
let breach = expect_breach(&mut rx, 102).await;
assert_eq!(
breach.id, 102,
"only the id over its own limit should breach"
);
assert_no_breach_within(&mut rx, ticks(1)).await;
}
#[tokio::test(start_paused = true)]
async fn disarm_before_the_next_tick_produces_no_breach() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss(1, None, 900)]]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(4, 1, MemSize::from_bytes(500));
enforcer.disarm(4);
assert_no_breach_within(&mut rx, ticks(1)).await;
}
#[tokio::test(start_paused = true)]
async fn missing_root_pid_produces_no_breach_even_against_a_zero_limit() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss(1, None, 100)]]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(5, 999, MemSize::from_bytes(0));
assert_no_breach_within(&mut rx, ticks(1)).await;
}
#[tokio::test(start_paused = true)]
async fn exactly_at_the_limit_does_not_breach_but_one_byte_over_does() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss(1, None, 500)], vec![rss(1, None, 501)], ]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(6, 1, MemSize::from_bytes(500));
assert_no_breach_within(&mut rx, ticks(1)).await;
let breach = expect_breach(&mut rx, 6).await;
assert_eq!(breach.observed.bytes(), 501);
}
#[tokio::test(start_paused = true)]
async fn a_limit_below_any_plausible_reading_breaches_on_the_very_first_tick() {
let limit = MemSize::from_bytes(1);
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss(1, None, 4096)]]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(11, 1, limit);
let start = tokio::time::Instant::now();
let breach = expect_breach(&mut rx, 11).await;
assert_eq!(
tokio::time::Instant::now() - start,
MEMORY_POLL_INTERVAL,
"a tree already over its ceiling must breach on the first tick, not the second"
);
assert_eq!(breach.observed.bytes(), 4096);
assert_eq!(breach.limit, limit);
assert_eq!(sampler.calls(), 1, "one tick, one sample");
}
#[tokio::test(start_paused = true)]
async fn a_tree_summing_to_exactly_the_limit_does_not_breach() {
let table = vec![rss(10, None, 300), rss(11, Some(10), 200)];
let sampler = Arc::new(ScriptedSampler::new(vec![table]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(12, 10, MemSize::from_bytes(500));
assert_no_breach_within(&mut rx, ticks(3)).await;
assert_eq!(
sampler.calls(),
3,
"expected exactly one sample() call per elapsed tick"
);
}
#[test]
fn memory_poll_interval_is_fifteen_seconds() {
assert_eq!(MEMORY_POLL_INTERVAL, Duration::from_secs(15));
}
#[tokio::test(start_paused = true)]
async fn two_ids_breaching_the_same_tick_both_deliver() {
let table = vec![rss(1, None, 900), rss(2, None, 900)];
let sampler = Arc::new(ScriptedSampler::new(vec![table]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(21, 1, MemSize::from_bytes(500));
enforcer.arm(22, 2, MemSize::from_bytes(500));
let first = expect_breach(&mut rx, 21).await;
let second = expect_breach(&mut rx, 22).await;
let mut ids = [first.id, second.id];
ids.sort_unstable();
assert_eq!(
ids,
[21, 22],
"both ids over their own limit on the same tick must be delivered, in either order"
);
}
#[tokio::test(start_paused = true)]
async fn rearming_an_already_armed_id_replaces_its_root_pid() {
let table = vec![rss(20, None, 900)];
let sampler = Arc::new(ScriptedSampler::new(vec![table]));
let (tx, mut rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(30, 10, MemSize::from_bytes(500));
enforcer.arm(30, 20, MemSize::from_bytes(500));
let breach = expect_breach(&mut rx, 30).await;
assert_eq!(
breach.root_pid, 20,
"re-arming the same id must replace the previous root_pid, not keep it"
);
}
#[tokio::test(start_paused = true)]
async fn dropping_the_enforcer_stops_further_sampling() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss(1, None, 100)]]));
let (tx, _rx) = mpsc::channel(8);
let enforcer = start_and_settle(Arc::clone(&sampler) as Arc<dyn MemorySampler>, tx).await;
enforcer.arm(40, 1, MemSize::from_bytes(1000));
tokio::time::sleep(ticks(1)).await;
let calls_before_drop = sampler.calls();
assert!(
calls_before_drop > 0,
"expected at least one sample before drop"
);
drop(enforcer);
tokio::time::sleep(ticks(3)).await;
assert_eq!(
sampler.calls(),
calls_before_drop,
"sample() must not be called again once the enforcer is dropped"
);
}
}