use core::fmt;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex, PoisonError};
use shep_core::protocol::{Lamb, LambLabel};
use shep_core::values;
use tokio::time::Instant;
use super::labels::LambLabels;
use super::sample::{MemorySampler, ProcessIdentity, TreeIndex};
use crate::proc_table::ProcInstant;
use crate::sweep::LambSnapshot;
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) struct SheepStats {
pub cpu_percent: Option<f32>,
pub memory_bytes: u64,
pub cpu_ms: u64,
}
#[derive(Debug, Clone, Copy)]
struct Baseline {
cpu_ms: u64,
at: Instant,
}
pub(crate) struct StatsState {
sampler: Arc<dyn MemorySampler>,
watched: Mutex<HashMap<u32, u32>>,
baselines: Mutex<HashMap<u32, Baseline>>,
lamb_snapshots: Mutex<HashMap<u32, LambSnapshot>>,
labels: LambLabels,
}
#[derive(Debug)]
pub(crate) struct LambIndex {
by_pid: HashMap<u32, ProcessIdentity>,
children_of: HashMap<u32, Vec<u32>>,
}
impl StatsState {
pub(crate) fn new(sampler: Arc<dyn MemorySampler>) -> Self {
Self {
sampler,
watched: Mutex::new(HashMap::new()),
baselines: Mutex::new(HashMap::new()),
lamb_snapshots: Mutex::new(HashMap::new()),
labels: LambLabels::default(),
}
}
pub(crate) fn watch(&self, id: u32, root_pid: u32) {
self.watched
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(id, root_pid);
self.baselines
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(&root_pid);
self.lamb_snapshots
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(&root_pid);
}
pub(crate) fn unwatch(&self, id: u32) {
let root_pid = self
.watched
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(&id);
if let Some(root_pid) = root_pid {
self.lamb_snapshots
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(&root_pid);
}
}
pub(crate) fn record_baseline(&self, index: &TreeIndex, now: Instant) {
let fresh: HashMap<u32, Baseline> = self
.watched_pids()
.into_iter()
.map(|root_pid| {
let baseline = Baseline {
cpu_ms: index.cpu_from(root_pid),
at: now,
};
(root_pid, baseline)
})
.collect();
*self
.baselines
.lock()
.unwrap_or_else(PoisonError::into_inner) = fresh;
}
pub(crate) fn record_lamb_snapshots(
&self,
index: &TreeIndex,
taken_at: ProcInstant,
shepherd: u32,
) {
let ours = index.descendants_of(shepherd);
let fresh: HashMap<u32, LambSnapshot> = self
.watched_pids()
.into_iter()
.filter(|root_pid| ours.contains(root_pid))
.map(|root_pid| {
let lambs = index.descendants_of(root_pid);
(root_pid, LambSnapshot::new(lambs, taken_at))
})
.collect();
*self
.lamb_snapshots
.lock()
.unwrap_or_else(PoisonError::into_inner) = fresh;
}
pub(crate) fn last_lamb_snapshot(&self, root_pid: u32) -> Option<LambSnapshot> {
self.lamb_snapshots
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(&root_pid)
.cloned()
}
pub(crate) fn lamb_snapshot_now(&self, root_pid: u32, shepherd: u32) -> LambSnapshot {
let taken_at = ProcInstant::now();
let table = self.sampler.sample();
let index = TreeIndex::build(&table);
if !index.descendants_of(shepherd).contains(&root_pid) {
return LambSnapshot::default();
}
LambSnapshot::new(index.descendants_of(root_pid), taken_at)
}
pub(crate) fn lamb_walk_from(&self, roots: &[u32]) -> LambSnapshot {
let taken_at = ProcInstant::now();
let table = self.sampler.sample();
let index = TreeIndex::build(&table);
let lambs = roots
.iter()
.flat_map(|&root| index.descendants_of(root))
.filter(|pid| !roots.contains(pid));
LambSnapshot::new(lambs, taken_at)
}
#[cfg(test)]
pub(crate) fn record_baseline_now(&self, now: Instant) {
let table = self.sampler.sample();
self.record_baseline(&TreeIndex::build(&table), now);
}
pub(crate) fn sample_now(&self) -> HashMap<u32, SheepStats> {
let roots = self.watched_pids();
let table = self.sampler.sample();
let index = TreeIndex::build(&table);
let now = Instant::now();
let baselines = self
.baselines
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone();
roots
.into_iter()
.map(|root_pid| {
let observed_cpu_ms = index.cpu_from(root_pid);
let stats = SheepStats {
cpu_percent: baselines
.get(&root_pid)
.and_then(|baseline| cpu_percent(*baseline, observed_cpu_ms, now)),
memory_bytes: index.sum_from(root_pid),
cpu_ms: observed_cpu_ms,
};
(root_pid, stats)
})
.collect()
}
fn watched_pids(&self) -> Vec<u32> {
self.watched
.lock()
.unwrap_or_else(PoisonError::into_inner)
.values()
.copied()
.collect()
}
#[cfg(test)]
pub(crate) fn watched_for_test(&self) -> Vec<(u32, u32)> {
let mut watched: Vec<(u32, u32)> = self
.watched
.lock()
.unwrap_or_else(PoisonError::into_inner)
.iter()
.map(|(&id, &root_pid)| (id, root_pid))
.collect();
watched.sort_unstable();
watched
}
pub(crate) fn lamb_index(&self) -> LambIndex {
let table = self.sampler.identify();
let mut by_pid = HashMap::with_capacity(table.len());
let mut children_of: HashMap<u32, Vec<u32>> = HashMap::new();
for entry in table {
if let Some(parent) = entry.parent {
children_of.entry(parent).or_default().push(entry.pid);
}
by_pid.insert(entry.pid, entry);
}
LambIndex {
by_pid,
children_of,
}
}
pub(crate) fn lambs_of(&self, index: &LambIndex, root_pid: u32) -> Vec<Lamb> {
let mut visited: HashSet<u32> = HashSet::from([root_pid]);
let mut stack = vec![root_pid];
let mut lambs = Vec::new();
while let Some(pid) = stack.pop() {
for child in index.children_of.get(&pid).into_iter().flatten() {
if !visited.insert(*child) {
continue;
}
if let Some(entry) = index.by_pid.get(child) {
lambs.push(Lamb::new(entry.pid, entry.name.clone()));
}
stack.push(*child);
}
}
lambs.sort_unstable_by_key(|lamb| lamb.pid);
let mut labels = self.labels.of(root_pid);
for lamb in &mut lambs {
lamb.label = labels.remove(&lamb.pid);
}
lambs
}
pub(crate) fn label_lamb(&self, root_pid: u32, pid: u32, label: &LambLabel, now: Instant) {
self.labels.set(root_pid, pid, label, now);
}
pub(crate) fn prune_labels(&self, index: &TreeIndex, taken_at: Instant) {
self.labels.prune(index, taken_at);
}
}
impl fmt::Debug for StatsState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("StatsState")
.field("sampler", &"<dyn MemorySampler>")
.finish_non_exhaustive()
}
}
fn cpu_percent(baseline: Baseline, cpu_ms: u64, now: Instant) -> Option<f32> {
let elapsed_cpu_ms = cpu_ms.saturating_sub(baseline.cpu_ms);
values::cpu_percent(elapsed_cpu_ms, now.saturating_duration_since(baseline.at))
}
#[cfg(test)]
mod tests {
use core::time::Duration;
use super::super::MEMORY_POLL_INTERVAL;
use super::*;
use crate::testing::{ScriptedSampler, identity, rss, rss_cpu};
fn at(raw: u64) -> ProcInstant {
ProcInstant::from_raw(raw)
}
const SHEPHERD: u32 = 50;
#[tokio::test(start_paused = true)]
async fn a_sheep_with_no_baseline_reports_no_cpu_but_still_reports_memory() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss_cpu(
100, None, 4096, 1000,
)]]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
let now = stats.sample_now();
assert_eq!(now[&100].cpu_percent, None);
assert_eq!(now[&100].memory_bytes, 4096, "memory is always current");
}
#[tokio::test(start_paused = true)]
async fn a_sample_carries_the_counter_without_a_baseline() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss_cpu(
100, None, 1024, 5_000,
)]]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
let now = stats.sample_now();
assert_eq!(now[&100].cpu_ms, 5_000);
assert_eq!(now[&100].cpu_percent, None);
}
#[tokio::test(start_paused = true)]
async fn cpu_is_the_delta_since_the_periodic_baseline() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 2_500)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
tokio::time::advance(MEMORY_POLL_INTERVAL).await;
let now = stats.sample_now();
let cpu = now[&100].cpu_percent.expect("a baseline exists");
assert!((cpu - 10.0).abs() < 0.01, "expected ~10%, got {cpu}");
}
#[tokio::test(start_paused = true)]
async fn a_second_read_a_moment_later_still_measures_from_the_periodic_baseline() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 2_500)],
vec![rss_cpu(100, None, 4096, 2_501)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
tokio::time::advance(MEMORY_POLL_INTERVAL).await;
let first = stats.sample_now()[&100].cpu_percent.unwrap();
tokio::time::advance(Duration::from_millis(1)).await;
let second = stats.sample_now()[&100].cpu_percent.unwrap();
assert!(
(first - 10.0).abs() < 0.01,
"1500 CPU-ms over 15 s is 10%, got {first}"
);
assert!(
(second - 10.0).abs() < 0.02,
"the second read divided by the gap between the two READS rather \
than by the window since the tick: {first} then {second}"
);
}
#[tokio::test(start_paused = true)]
async fn an_unwatched_sheep_is_no_longer_sampled() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 2_500)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
tokio::time::advance(MEMORY_POLL_INTERVAL).await;
stats.unwatch(1);
assert!(
stats.sample_now().is_empty(),
"an unwatched sheep must not still be sampled"
);
}
#[tokio::test(start_paused = true)]
async fn a_recycled_pid_starts_from_no_baseline_rather_than_the_dead_process_counter() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 9_000)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
tokio::time::advance(MEMORY_POLL_INTERVAL).await;
stats.unwatch(1);
stats.watch(2, 100);
assert_eq!(
stats.sample_now()[&100].cpu_percent,
None,
"the new process on pid 100 must not be measured against the dead one's counter"
);
}
#[tokio::test(start_paused = true)]
async fn a_read_in_the_same_instant_as_the_baseline_reports_no_cpu() {
let sampler = Arc::new(ScriptedSampler::new(vec![
vec![rss_cpu(100, None, 4096, 1_000)],
vec![rss_cpu(100, None, 4096, 9_000)],
]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
stats.record_baseline_now(Instant::now());
assert_eq!(stats.sample_now()[&100].cpu_percent, None);
}
#[tokio::test(start_paused = true)]
async fn stats_state_debug_names_the_sampler_by_role_and_prints_no_map() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss_cpu(
100, None, 4096, 1000,
)]]));
let stats = StatsState::new(sampler);
stats.watch(1, 100);
assert_eq!(
format!("{stats:?}"),
r#"StatsState { sampler: "<dyn MemorySampler>", .. }"#
);
}
#[tokio::test(start_paused = true)]
async fn watching_reports_each_id_once_against_its_current_pid() {
let sampler = Arc::new(ScriptedSampler::new(vec![vec![rss_cpu(
100, None, 4096, 1000,
)]]));
let stats = StatsState::new(sampler);
stats.watch(7, 4242);
stats.watch(2, 100);
assert_eq!(stats.watched_for_test(), vec![(2, 100), (7, 4242)]);
stats.watch(7, 5353);
assert_eq!(
stats.watched_for_test(),
vec![(2, 100), (7, 5353)],
"a respawn replaces the id's pid rather than adding a second one"
);
stats.unwatch(2);
stats.unwatch(99);
assert_eq!(
stats.watched_for_test(),
vec![(7, 5353)],
"unwatching an id never watched must leave the rest alone"
);
}
#[test]
fn a_tick_records_each_watched_roots_lambs_and_drops_the_unwatched() {
let index = TreeIndex::build(&[
rss(100, Some(SHEPHERD), 0),
rss(101, Some(100), 0),
rss(102, Some(101), 0),
rss(200, Some(SHEPHERD), 0),
rss(201, Some(200), 0),
]);
let stats = StatsState::new(Arc::new(ScriptedSampler::new(vec![vec![]])));
stats.watch(1, 100);
stats.watch(2, 200);
stats.record_lamb_snapshots(&index, at(77), SHEPHERD);
assert_eq!(
stats.last_lamb_snapshot(100),
Some(LambSnapshot::new([101, 102], at(77)))
);
stats.unwatch(2);
stats.record_lamb_snapshots(&index, at(78), SHEPHERD);
assert_eq!(stats.last_lamb_snapshot(200), None);
assert_eq!(
stats.last_lamb_snapshot(100),
Some(LambSnapshot::new([101, 102], at(78)))
);
}
#[test]
fn an_unwatched_root_leaves_no_lamb_snapshot_behind() {
let index = TreeIndex::build(&[rss(100, Some(SHEPHERD), 0), rss(101, Some(100), 0)]);
let stats = StatsState::new(Arc::new(ScriptedSampler::new(vec![vec![]])));
stats.watch(1, 100);
stats.record_lamb_snapshots(&index, at(77), SHEPHERD);
assert!(
stats.last_lamb_snapshot(100).is_some(),
"sanity: one was recorded"
);
stats.unwatch(1);
assert_eq!(stats.last_lamb_snapshot(100), None);
}
#[test]
fn a_recycled_pid_starts_with_no_lamb_snapshot() {
let index = TreeIndex::build(&[rss(100, Some(SHEPHERD), 0), rss(101, Some(100), 0)]);
let stats = StatsState::new(Arc::new(ScriptedSampler::new(vec![vec![]])));
stats.watch(1, 100);
stats.record_lamb_snapshots(&index, at(77), SHEPHERD);
assert!(
stats.last_lamb_snapshot(100).is_some(),
"sanity: one was recorded"
);
stats.watch(2, 100);
assert_eq!(
stats.last_lamb_snapshot(100),
None,
"the new process on pid 100 must not inherit the old one's lambs"
);
}
#[test]
fn a_walk_from_survivors_finds_their_children_and_not_the_survivors() {
let table = vec![
rss(1, None, 0),
rss(101, Some(1), 0),
rss(102, Some(101), 0),
rss(103, Some(102), 0),
rss(201, Some(1), 0),
rss(202, Some(201), 0),
rss(300, Some(1), 0),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::new(vec![table])));
let snapshot = stats.lamb_walk_from(&[101, 201]);
assert_eq!(snapshot.pids().collect::<Vec<_>>(), vec![102, 103, 202]);
}
#[test]
fn a_fresh_walk_snapshots_every_descendant_and_not_the_root() {
let table = vec![
rss(100, Some(SHEPHERD), 0),
rss(101, Some(100), 0),
rss(102, Some(101), 0),
rss(200, Some(SHEPHERD), 0),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::new(vec![table])));
let before = ProcInstant::now();
let snapshot = stats.lamb_snapshot_now(100, SHEPHERD);
let after = ProcInstant::now();
assert_eq!(snapshot.pids().collect::<Vec<_>>(), vec![101, 102]);
let taken = snapshot.seen_at(101).expect("101 was seen");
assert!(
before <= taken && taken <= after,
"{before:?} <= {taken:?} <= {after:?}"
);
}
#[test]
fn a_root_outside_the_shepherds_tree_is_never_snapshotted() {
let table = vec![
rss(100, Some(SHEPHERD), 0),
rss(101, Some(100), 0),
rss(200, Some(SHEPHERD + 1), 0),
rss(201, Some(200), 0),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::new(vec![table.clone()])));
stats.watch(1, 100);
stats.watch(2, 200);
stats.record_lamb_snapshots(&TreeIndex::build(&table), at(77), SHEPHERD);
assert_eq!(
stats.last_lamb_snapshot(100),
Some(LambSnapshot::new([101], at(77)))
);
assert_eq!(stats.last_lamb_snapshot(200), None);
assert!(stats.lamb_snapshot_now(200, SHEPHERD).is_empty());
}
#[test]
fn the_lamb_walk_excludes_the_sheeps_own_pid() {
let table = vec![
identity(100, None, "srv"),
identity(101, Some(100), "node"),
identity(102, Some(101), "sh"),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
let lambs = stats.lambs_of(&stats.lamb_index(), 100);
assert_eq!(
lambs,
vec![Lamb::new(101, "node"), Lamb::new(102, "sh")],
"the root pid must not appear among its own lambs"
);
}
#[test]
fn the_lamb_walk_reaches_every_generation() {
let table = vec![
identity(100, None, "sh"),
identity(101, Some(100), "node"),
identity(102, Some(101), "node"),
identity(103, Some(102), "node"),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
assert_eq!(stats.lambs_of(&stats.lamb_index(), 100).len(), 3);
}
#[test]
fn a_sibling_subtree_is_not_this_sheeps() {
let table = vec![
identity(100, None, "srv"),
identity(101, Some(100), "mine"),
identity(200, None, "srv"),
identity(201, Some(200), "theirs"),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
let index = stats.lamb_index();
assert_eq!(stats.lambs_of(&index, 100), vec![Lamb::new(101, "mine")]);
assert_eq!(stats.lambs_of(&index, 200), vec![Lamb::new(201, "theirs")]);
}
#[test]
fn a_parent_link_cycle_terminates_and_reports_each_pid_once() {
let table = vec![identity(100, Some(101), "a"), identity(101, Some(100), "b")];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
let lambs = stats.lambs_of(&stats.lamb_index(), 100);
assert_eq!(
lambs,
vec![Lamb::new(101, "b")],
"the root is its own descendant through the cycle and must not be listed"
);
}
#[test]
fn lambs_come_back_in_pid_order() {
let table = vec![
identity(100, None, "srv"),
identity(103, Some(100), "c"),
identity(101, Some(100), "a"),
identity(102, Some(100), "b"),
];
let stats = StatsState::new(Arc::new(ScriptedSampler::identifying(vec![table])));
assert_eq!(
stats
.lambs_of(&stats.lamb_index(), 100)
.iter()
.map(|l| l.pid)
.collect::<Vec<_>>(),
vec![101, 102, 103]
);
}
#[test]
fn a_sampler_that_cannot_identify_reports_no_lambs() {
let stats = StatsState::new(Arc::new(ScriptedSampler::new(vec![vec![]])));
assert!(stats.lambs_of(&stats.lamb_index(), 100).is_empty());
}
}