use core::fmt;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex, PoisonError};
use shep_core::protocol::Lamb;
use tokio::time::Instant;
use super::sample::{MemorySampler, ProcessIdentity, TreeIndex};
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) struct SheepStats {
pub cpu_percent: Option<f32>,
pub memory_bytes: u64,
}
#[derive(Debug, Clone, Copy)]
struct Baseline {
cpu_ms: u64,
at: Instant,
}
pub(crate) struct StatsState {
sampler: Arc<dyn MemorySampler>,
watched: Mutex<HashMap<u32, u32>>,
baselines: Mutex<HashMap<u32, Baseline>>,
}
#[derive(Debug)]
pub(crate) struct LambIndex {
by_pid: HashMap<u32, ProcessIdentity>,
children_of: HashMap<u32, Vec<u32>>,
}
impl StatsState {
pub(crate) fn new(sampler: Arc<dyn MemorySampler>) -> Self {
Self {
sampler,
watched: Mutex::new(HashMap::new()),
baselines: Mutex::new(HashMap::new()),
}
}
pub(crate) fn watch(&self, id: u32, root_pid: u32) {
self.watched
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(id, root_pid);
self.baselines
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(&root_pid);
}
pub(crate) fn unwatch(&self, id: u32) {
self.watched
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(&id);
}
pub(crate) fn record_baseline(&self, index: &TreeIndex, now: Instant) {
let fresh: HashMap<u32, Baseline> = self
.watched_pids()
.into_iter()
.map(|root_pid| {
let baseline = Baseline {
cpu_ms: index.cpu_from(root_pid),
at: now,
};
(root_pid, baseline)
})
.collect();
*self
.baselines
.lock()
.unwrap_or_else(PoisonError::into_inner) = fresh;
}
#[allow(dead_code, reason = "called only by this crate's tests")]
pub(crate) fn record_baseline_now(&self, now: Instant) {
let table = self.sampler.sample();
self.record_baseline(&TreeIndex::build(&table), now);
}
pub(crate) fn sample_now(&self) -> HashMap<u32, SheepStats> {
let roots = self.watched_pids();
let table = self.sampler.sample();
let index = TreeIndex::build(&table);
let now = Instant::now();
let baselines = self
.baselines
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
roots
.into_iter()
.map(|root_pid| {
let observed_cpu_ms = index.cpu_from(root_pid);
let stats = SheepStats {
cpu_percent: baselines
.get(&root_pid)
.and_then(|baseline| cpu_percent(*baseline, observed_cpu_ms, now)),
memory_bytes: index.sum_from(root_pid),
};
(root_pid, stats)
})
.collect()
}
fn watched_pids(&self) -> Vec<u32> {
self.watched
.lock()
.unwrap_or_else(PoisonError::into_inner)
.values()
.copied()
.collect()
}
#[cfg(test)]
pub(crate) fn watched_for_test(&self) -> Vec<(u32, u32)> {
let mut watched: Vec<(u32, u32)> = self
.watched
.lock()
.unwrap_or_else(PoisonError::into_inner)
.iter()
.map(|(&id, &root_pid)| (id, root_pid))
.collect();
watched.sort_unstable();
watched
}
pub(crate) fn lamb_index(&self) -> LambIndex {
let table = self.sampler.identify();
let mut by_pid = HashMap::with_capacity(table.len());
let mut children_of: HashMap<u32, Vec<u32>> = HashMap::new();
for entry in table {
if let Some(parent) = entry.parent {
children_of.entry(parent).or_default().push(entry.pid);
}
by_pid.insert(entry.pid, entry);
}
LambIndex {
by_pid,
children_of,
}
}
pub(crate) fn lambs_of(&self, index: &LambIndex, root_pid: u32) -> Vec<Lamb> {
let mut visited: HashSet<u32> = HashSet::from([root_pid]);
let mut stack = vec![root_pid];
let mut lambs = Vec::new();
while let Some(pid) = stack.pop() {
for child in index.children_of.get(&pid).into_iter().flatten() {
if !visited.insert(*child) {
continue;
}
if let Some(entry) = index.by_pid.get(child) {
lambs.push(Lamb::new(entry.pid, entry.name.clone()));
}
stack.push(*child);
}
}
lambs.sort_unstable_by_key(|lamb| lamb.pid);
lambs
}
}
impl fmt::Debug for StatsState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("StatsState")
.field("sampler", &"<dyn MemorySampler>")
.finish_non_exhaustive()
}
}
fn cpu_percent(baseline: Baseline, cpu_ms: u64, now: Instant) -> Option<f32> {
let window = now.saturating_duration_since(baseline.at);
if window.is_zero() {
return None;
}
let elapsed_cpu_ms = cpu_ms.saturating_sub(baseline.cpu_ms);
let percent = elapsed_cpu_ms as f64 / window.as_secs_f64() / 10.0;
Some(percent as f32)
}
#[cfg(test)]
mod tests {
use core::time::Duration;
use super::super::MEMORY_POLL_INTERVAL;
use super::super::sample::ProcessRss;
use super::*;
use crate::testing::{ScriptedSampler, identity};
fn rss_cpu(pid: u32, parent: Option<u32>, bytes: u64, cpu_ms: u64) -> ProcessRss {
ProcessRss {
pid,
parent,
bytes,
cpu_ms,
}
}
#[tokio::test(start_paused = true)]
async fn a_sheep_with_no_baseline_reports_no_cpu_but_still_reports_memory() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss_cpu(
100, None, 4096, 1000,
)]]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
let now = stats.sample_now();
assert_eq!(now[&100].cpu_percent, None);
assert_eq!(now[&100].memory_bytes, 4096, "memory is always current");
}
#[tokio::test(start_paused = true)]
async fn cpu_is_the_delta_since_the_periodic_baseline() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 2_500)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
tokio::time::advance(MEMORY_POLL_INTERVAL).await;
let now = stats.sample_now();
let cpu = now[&100].cpu_percent.expect("a baseline exists");
assert!((cpu - 10.0).abs() < 0.01, "expected ~10%, got {cpu}");
}
#[tokio::test(start_paused = true)]
async fn a_second_read_a_moment_later_still_measures_from_the_periodic_baseline() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 2_500)],
vec![rss_cpu(100, None, 4096, 2_501)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
tokio::time::advance(MEMORY_POLL_INTERVAL).await;
let first = stats.sample_now()[&100].cpu_percent.unwrap();
tokio::time::advance(Duration::from_millis(1)).await;
let second = stats.sample_now()[&100].cpu_percent.unwrap();
assert!(
(first - 10.0).abs() < 0.01,
"1500 CPU-ms over 15 s is 10%, got {first}"
);
assert!(
(second - 10.0).abs() < 0.02,
"the second read divided by the gap between the two READS rather \
than by the window since the tick: {first} then {second}"
);
}
#[tokio::test(start_paused = true)]
async fn an_unwatched_sheep_is_no_longer_sampled() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 2_500)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
tokio::time::advance(MEMORY_POLL_INTERVAL).await;
stats.unwatch(1);
assert!(
stats.sample_now().is_empty(),
"an unwatched sheep must not still be sampled"
);
}
#[tokio::test(start_paused = true)]
async fn a_recycled_pid_starts_from_no_baseline_rather_than_the_dead_process_counter() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 9_000)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
tokio::time::advance(MEMORY_POLL_INTERVAL).await;
stats.unwatch(1);
stats.watch(2, 100);
assert_eq!(
stats.sample_now()[&100].cpu_percent,
None,
"the new process on pid 100 must not be measured against the dead one's counter"
);
}
#[tokio::test(start_paused = true)]
async fn a_read_in_the_same_instant_as_the_baseline_reports_no_cpu() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 9_000)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
assert_eq!(stats.sample_now()[&100].cpu_percent, None);
}
#[tokio::test(start_paused = true)]
async fn stats_state_debug_names_the_sampler_by_role_and_prints_no_map() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss_cpu(
100, None, 4096, 1000,
)]]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
assert_eq!(
format!("{stats:?}"),
r#"StatsState { sampler: "<dyn MemorySampler>", .. }"#
);
}
#[tokio::test(start_paused = true)]
async fn watching_reports_each_id_once_against_its_current_pid() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss_cpu(
100, None, 4096, 1000,
)]]));
let stats = StatsState::new(sampler);
stats.watch(7, 4242);
stats.watch(2, 100);
assert_eq!(stats.watched_for_test(), vec![(2, 100), (7, 4242)]);
stats.watch(7, 5353);
assert_eq!(
stats.watched_for_test(),
vec![(2, 100), (7, 5353)],
"a respawn replaces the id's pid rather than adding a second one"
);
stats.unwatch(2);
stats.unwatch(99);
assert_eq!(
stats.watched_for_test(),
vec![(7, 5353)],
"unwatching an id never watched must leave the rest alone"
);
}
#[test]
fn the_lamb_walk_excludes_the_sheeps_own_pid() {
let table = vec![
identity(100, None, "srv"),
identity(101, Some(100), "node"),
identity(102, Some(101), "sh"),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
let lambs = stats.lambs_of(&stats.lamb_index(), 100);
assert_eq!(
lambs,
vec![Lamb::new(101, "node"), Lamb::new(102, "sh")],
"the root pid must not appear among its own lambs"
);
}
#[test]
fn the_lamb_walk_reaches_every_generation() {
let table = vec![
identity(100, None, "sh"),
identity(101, Some(100), "node"),
identity(102, Some(101), "node"),
identity(103, Some(102), "node"),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
assert_eq!(stats.lambs_of(&stats.lamb_index(), 100).len(), 3);
}
#[test]
fn a_sibling_subtree_is_not_this_sheeps() {
let table = vec![
identity(100, None, "srv"),
identity(101, Some(100), "mine"),
identity(200, None, "srv"),
identity(201, Some(200), "theirs"),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
let index = stats.lamb_index();
assert_eq!(stats.lambs_of(&index, 100), vec![Lamb::new(101, "mine")]);
assert_eq!(stats.lambs_of(&index, 200), vec![Lamb::new(201, "theirs")]);
}
#[test]
fn a_parent_link_cycle_terminates_and_reports_each_pid_once() {
let table = vec![identity(100, Some(101), "a"), identity(101, Some(100), "b")];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
let lambs = stats.lambs_of(&stats.lamb_index(), 100);
assert_eq!(
lambs,
vec![Lamb::new(101, "b")],
"the root is its own descendant through the cycle and must not be listed"
);
}
#[test]
fn lambs_come_back_in_pid_order() {
let table = vec![
identity(100, None, "srv"),
identity(103, Some(100), "c"),
identity(101, Some(100), "a"),
identity(102, Some(100), "b"),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
assert_eq!(
stats
.lambs_of(&stats.lamb_index(), 100)
.iter()
.map(|l| l.pid)
.collect::<Vec<_>>(),
vec![101, 102, 103]
);
}
#[test]
fn a_sampler_that_cannot_identify_reports_no_lambs() {
let stats = StatsState::new(Arc::new(ScriptedSampler::new(vec![vec![]])));
assert!(stats.lambs_of(&stats.lamb_index(), 100).is_empty());
}
}