use std::collections::BTreeMap;
use std::path::PathBuf;
use std::time::Duration;
use jiff::Timestamp;
use crate::admit::BUSY_WINDOW_MS;
use crate::error::Error;
use crate::job::{Capacity, Job, State};
use crate::log::{self, Event};
use crate::machine;
use crate::state::StateDir;
pub const SAMPLE_EVERY_MS: i64 = 30_000;
const CHECK: Duration = Duration::from_secs(10);
fn now_ms() -> i64 {
Timestamp::now().as_millisecond()
}
pub fn sample_once(state: &StateDir, cwd: &std::path::Path, cap: &Capacity) -> Result<bool, Error> {
let due = |now: i64| {
(
now - state.last_sample_ms() >= SAMPLE_EVERY_MS,
busy_due(state, now),
)
};
if due(now_ms()) == (false, false) {
return Ok(false);
}
let running: Vec<_> = state
.read_jobs()
.into_iter()
.filter(|j| j.state == State::Running)
.collect();
let mut live = machine::live(cwd, cap, &running)?;
let _lock = state.lock()?;
let now = now_ms();
let (sample_due, reading_due) = due(now);
let jobs = state.live_jobs();
if jobs.is_empty() {
return Ok(false);
}
if reading_due {
state.push_busy(now, live.cpu_pct, BUSY_WINDOW_MS);
}
if !sample_due {
return Ok(false);
}
live.stopped = machine::stopped_ids(&jobs);
let (waiting, blocked) = tally(&jobs, &live.stopped);
let running = u32::try_from(jobs.iter().filter(|j| j.state == State::Running).count())
.unwrap_or(u32::MAX);
log::record(
state.base(),
&Event::Sample {
ts: Timestamp::from_millisecond(now).unwrap_or_default(),
cpu_pct: live.cpu_pct,
mem_pct: live.mem_used_pct,
load1: live.load1,
load5: live.load5,
ncpu: cap.cpu,
running,
waiting,
blocked,
},
);
state.set_last_sample_ms(now);
Ok(true)
}
fn busy_due(state: &StateDir, now: i64) -> bool {
let every = i64::try_from(CHECK.as_millis()).unwrap_or(i64::MAX);
state
.busy_readings()
.last()
.is_none_or(|(ts, _)| now - ts >= every)
}
fn tally(jobs: &[Job], stopped: &[String]) -> (u32, BTreeMap<String, u32>) {
let mut blocked: BTreeMap<String, u32> = BTreeMap::new();
let mut waiting = 0;
for j in jobs
.iter()
.filter(|j| j.state == State::Queued && !stopped.contains(&j.id))
{
let Some(b) = &j.blocked_by else { continue };
waiting += 1;
*blocked.entry(b.clone()).or_default() += 1;
}
(waiting, blocked)
}
pub fn spawn(state: &StateDir, cwd: PathBuf, cap: Capacity) {
let state = state.clone();
std::thread::spawn(move || {
loop {
let _ = sample_once(&state, &cwd, &cap);
std::thread::sleep(CHECK);
}
});
}
#[cfg(test)]
mod tests {
use super::*;
use crate::job::Weights;
fn job(id: &str, state: State, by: Option<&str>) -> Job {
Job {
id: id.into(),
pid: 1,
start_time: 0,
weights: Weights::default(),
cmd: vec!["x".into()],
cwd: "/".into(),
state,
enqueued_ms: 1,
started_ms: None,
mem_declared: false,
mem_peak_at_ms: None,
blocked_by: by.map(Into::into),
}
}
#[test]
fn the_sampler_reads_the_reasons_the_waiters_wrote() {
let jobs = [
job("r", State::Running, None),
job("a", State::Queued, Some("cpu")),
job("b", State::Queued, Some("ramp")),
job("c", State::Queued, Some("cpu")),
];
let (waiting, blocked) = tally(&jobs, &[]);
assert_eq!(waiting, 3);
assert_eq!(
blocked,
BTreeMap::from([("cpu".to_string(), 2), ("ramp".to_string(), 1)])
);
}
#[test]
fn unevaluated_waiters_are_not_counted() {
let jobs = [
job("a", State::Queued, None),
job("b", State::Queued, Some("cpu")),
];
let (waiting, blocked) = tally(&jobs, &[]);
assert_eq!(waiting, 1);
assert_eq!(blocked, BTreeMap::from([("cpu".to_string(), 1)]));
}
#[test]
fn stopped_jobs_do_not_wait() {
let jobs = [
job("a", State::Stopped, Some("cpu")),
job("b", State::Queued, Some("cpu")),
job("c", State::Queued, Some("ramp")),
];
let (waiting, blocked) = tally(&jobs, &["c".to_string()]);
assert_eq!(waiting, 1);
assert_eq!(blocked, BTreeMap::from([("cpu".to_string(), 1)]));
}
}