use std::os::unix::process::{CommandExt, ExitStatusExt};
use std::path::Path;
use std::process::{Command, ExitStatus};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel};
use std::time::{Duration, Instant};
use jiff::Timestamp;
use nix::errno::Errno;
use nix::sys::signal::{Signal, kill, killpg};
use nix::sys::wait::{WaitPidFlag, WaitStatus, waitpid};
use nix::unistd::Pid;
use signal_hook::consts::{SIGHUP, SIGINT, SIGTERM, SIGTSTP};
use signal_hook::iterator::Signals;
use crate::admit::{Decision, Smoother, evaluate};
use crate::cli::Flags;
use crate::error::Error;
use crate::history::{self, Entry};
use crate::job::{Capacity, Job, START_UNKNOWN, State, Usage, Weights};
use crate::log::{self, Event, JobInfo, WaitTally};
use crate::machine::{
self, OtherSampler, TreeSampler, cpu_times, effective_mem_peak, effective_peak, net_bytes,
process_start, vram,
};
use crate::sample;
use crate::state::{StateDir, is_alive};
use crate::stats::{Stats, children_usage, compute, format_stats};
use crate::tty::{Terminal, TerminalGuard};
const DEFAULT_MEM: u64 = 4 << 30;
const LIVE_RESAMPLE: Duration = Duration::from_secs(10);
const SWEEP: Duration = Duration::from_secs(30);
const RAMP_MIN_SLEEP_MS: u64 = 20;
const DEFAULT_GRACE_MS: u64 = 10_000;
const TIMEOUT_EXIT: u8 = 124;
const DISK_FULL_EXIT: u8 = 75;
const TREE_SAMPLE: Duration = Duration::from_millis(500);
const OTHER_SAMPLE: Duration = Duration::from_secs(1);
enum WaitEvent {
Fs,
Signal(i32),
Stop,
}
#[derive(Default)]
struct Sig {
pgid: AtomicI32,
last: AtomicI32,
waiting: AtomicBool,
}
fn forward(pgid: i32, sig: i32) {
if let Ok(s) = Signal::try_from(sig) {
let _ = killpg(Pid::from_raw(pgid), s);
if s != Signal::SIGKILL {
let _ = killpg(Pid::from_raw(pgid), Signal::SIGCONT);
}
}
}
fn spawn_signal_thread(tx: Sender<WaitEvent>) -> Result<Arc<Sig>, Error> {
let sig = Arc::new(Sig::default());
let mut signals = Signals::new([SIGINT, SIGTERM, SIGHUP, SIGTSTP])?;
let shared = sig.clone();
std::thread::spawn(move || {
for s in signals.forever() {
if s == SIGTSTP {
if shared.waiting.load(Ordering::SeqCst) {
let _ = tx.send(WaitEvent::Stop);
} else if let Err(e) = signal_hook::low_level::emulate_default_handler(s) {
eprintln!("stokehold: cannot stop: {e}");
}
continue;
}
shared.last.store(s, Ordering::SeqCst);
let pg = shared.pgid.load(Ordering::SeqCst);
if pg > 0 {
forward(pg, s);
}
let _ = tx.send(WaitEvent::Signal(s));
}
});
Ok(sig)
}
struct JobGuard<'a> {
state: &'a StateDir,
id: String,
}
impl Drop for JobGuard<'_> {
fn drop(&mut self) {
self.state.remove_job(&self.id);
}
}
fn now_ms() -> i64 {
Timestamp::now().as_millisecond()
}
fn default_cpu(_cmd: &[String], cap: &Capacity) -> u32 {
cap.cpu
}
fn resolve_weights(
state: &StateDir,
flags: &Flags,
cmd: &[String],
cwd: &str,
cap: &Capacity,
) -> (Weights, Option<i64>) {
let hist = history::defaults_for(
&state.history_path(),
cmd,
cwd,
history::repo_key(Path::new(cwd)).as_deref(),
);
let weights = cap.clamp(Weights {
cpu: flags
.cpu
.or(hist.map(|d| d.cpu))
.unwrap_or_else(|| default_cpu(cmd, cap)),
mem: flags.mem.or(hist.map(|d| d.mem)).unwrap_or(DEFAULT_MEM),
vram: flags.vram.or(hist.and_then(|d| d.vram)).unwrap_or(0),
disk: flags.disk.unwrap_or(0),
});
(
weights,
hist.and_then(|d| d.mem_peak_at_ms)
.and_then(|ms| i64::try_from(ms).ok()),
)
}
fn exec_nested(state: &StateDir, cmd: &[String]) -> Option<Error> {
let id = crate::state::env_var("STOKEHOLD_JOB")?.into_string().ok()?;
let job = state.read_job(&id)?;
if job.state != State::Running || !is_alive(&job) {
return None;
}
let (first, rest) = cmd.split_first()?;
Some(Error::Io(Command::new(first).args(rest).exec()))
}
pub fn run(state: &StateDir, flags: &Flags, cmd: &[String]) -> Result<u8, Error> {
if let Some(e) = exec_nested(state, cmd) {
eprintln!(
"stokehold: cannot run {}: {e}",
cmd.first().map_or("", String::as_str)
);
return Ok(127);
}
state.ensure()?;
let cwd = std::env::current_dir()?;
let cwd_str = cwd.to_string_lossy().into_owned();
let cap = machine::capacity(&cwd)?;
let (weights, mem_peak_at_ms) = resolve_weights(state, flags, cmd, &cwd_str, &cap);
let (tx, rx) = channel();
let sig = spawn_signal_thread(tx.clone())?;
let _watcher = watch(&state.jobs_dir(), tx)?;
let pid = std::process::id();
let enqueued_ms = now_ms();
let mut job = Job {
id: format!("{enqueued_ms:013}-{pid}"),
pid,
start_time: probe_start(pid),
weights,
cmd: cmd.to_vec(),
cwd: cwd_str,
state: State::Queued,
enqueued_ms,
started_ms: None,
blocked_by: None,
mem_declared: flags.mem.is_some(),
mem_peak_at_ms: if flags.mem.is_some() {
None
} else {
mem_peak_at_ms
},
};
let _guard = JobGuard {
state,
id: job.id.clone(),
};
{
let _lock = state.lock()?;
state.write_job(&job)?;
}
let info = JobInfo::new(&job.id, pid, &job.cwd, cmd, weights);
log::prune(state.base());
log::record(
state.base(),
&Event::Queued {
ts: Timestamp::now(),
job: info.clone(),
},
);
sample::spawn(state, cwd.clone(), cap);
sig.waiting.store(true, Ordering::SeqCst);
let mut tally = WaitTally::default();
let acquired = acquire(state, &cwd, &cap, &mut job, &rx, &mut tally);
sig.waiting.store(false, Ordering::SeqCst);
match acquired {
Ok(()) => {}
Err(Error::Interrupted(s)) => return Ok(u8::try_from(128 + s).unwrap_or(130)),
Err(Error::DiskFull(why)) => {
eprintln!("stokehold: {why}");
return Ok(DISK_FULL_EXIT);
}
Err(e) => return Err(e),
}
let queued_ms = u64::try_from(now_ms() - enqueued_ms).unwrap_or(0);
log::record(
state.base(),
&Event::Started {
ts: Timestamp::now(),
job: info.clone(),
wait_ms: queued_ms,
reasons: tally.reasons(),
},
);
execute(state, &cap, &job, &info, &sig, flags.timeout(), queued_ms)
}
fn probe_start(pid: u32) -> u64 {
for _ in 0..5 {
if let Some(t) = process_start(pid) {
return t;
}
std::thread::sleep(Duration::from_millis(50));
}
START_UNKNOWN
}
fn watch(dir: &Path, tx: Sender<WaitEvent>) -> Result<notify::RecommendedWatcher, Error> {
use notify::Watcher;
let mut w = notify::recommended_watcher(move |_res: notify::Result<notify::Event>| {
let _ = tx.send(WaitEvent::Fs);
})?;
w.watch(dir, notify::RecursiveMode::NonRecursive)?;
Ok(w)
}
fn wait_timeout(live_blocked: bool, ramp_end_ms: Option<i64>, now_ms: i64) -> Duration {
if let Some(end) = ramp_end_ms {
let left = u64::try_from(end - now_ms + 1).unwrap_or(0);
return Duration::from_millis(left.max(RAMP_MIN_SLEEP_MS)).min(LIVE_RESAMPLE);
}
if live_blocked { LIVE_RESAMPLE } else { SWEEP }
}
fn acquire(
state: &StateDir,
cwd: &Path,
cap: &Capacity,
job: &mut Job,
rx: &Receiver<WaitEvent>,
tally: &mut WaitTally,
) -> Result<(), Error> {
let mut smoother = Smoother::default();
let mut said = String::new();
loop {
let running: Vec<Job> = state
.read_jobs()
.into_iter()
.filter(|j| j.state == State::Running)
.collect();
let mut live = machine::live(cwd, cap, &running)?;
live.non_heavy = smoother.push(live.non_heavy);
let decision = {
let _lock = state.lock()?;
let jobs = state.live_jobs();
live.stopped = machine::stopped_ids(&jobs);
let now = now_ms();
state.push_busy(now, live.cpu_pct, crate::admit::BUSY_WINDOW_MS);
live.cpu_sustained_pct =
crate::admit::sustained_pct(live.cpu_pct, &state.busy_readings(), now);
let d = evaluate(job, &jobs, cap, &live, now);
let by = match &d {
Decision::Wait { block, .. } => Some(block.name().to_string()),
_ => None,
};
if d != Decision::Start && job.blocked_by != by {
job.blocked_by = by;
state.write_job(job)?;
}
if d == Decision::Start {
job.blocked_by = None;
job.state = State::Running;
job.started_ms = Some(now_ms());
state.write_job(job)?;
}
d
};
let (reason, live_blocked, block, ramp_end_ms) = match decision {
Decision::Start => {
tally.note(now_ms(), None);
return Ok(());
}
Decision::Fail(why) => return Err(Error::DiskFull(why)),
Decision::Wait {
reason,
live,
block,
ramp_end_ms,
} => (reason, live, block, ramp_end_ms),
};
tally.note(now_ms(), Some(block));
if said != reason {
eprintln!("stokehold: waiting ({reason})");
said = reason;
}
match rx.recv_timeout(wait_timeout(live_blocked, ramp_end_ms, now_ms())) {
Ok(WaitEvent::Signal(s)) => return Err(Error::Interrupted(s)),
Ok(WaitEvent::Stop) => park(state, job)?,
Ok(WaitEvent::Fs) => while rx.try_recv().is_ok_and(|e| matches!(e, WaitEvent::Fs)) {},
Err(RecvTimeoutError::Timeout) => {}
Err(RecvTimeoutError::Disconnected) => {
return Err(Error::Probe("event channel closed".into()));
}
}
}
}
fn park(state: &StateDir, job: &mut Job) -> Result<(), Error> {
set_state(state, job, State::Stopped)?;
kill(Pid::this(), Signal::SIGSTOP)?;
set_state(state, job, State::Queued)
}
fn set_state(state: &StateDir, job: &mut Job, to: State) -> Result<(), Error> {
let _lock = state.lock()?;
job.state = to;
state.write_job(job)
}
fn exit_code(status: ExitStatus) -> i32 {
status
.code()
.unwrap_or_else(|| 128 + status.signal().unwrap_or(0))
}
fn vram_sampler(cap: &Capacity) -> Option<(Sender<()>, std::thread::JoinHandle<u64>)> {
if cap.vram == 0 || cap.vram_is_ram {
return None;
}
let (stop_tx, stop_rx) = channel::<()>();
let handle = std::thread::spawn(move || {
let mut peak = 0;
loop {
peak = peak.max(vram().map_or(0, |(_, used)| used));
if !matches!(
stop_rx.recv_timeout(Duration::from_secs(2)),
Err(RecvTimeoutError::Timeout)
) {
return peak;
}
}
});
Some((stop_tx, handle))
}
#[derive(Default)]
struct TreePeak {
usage: Usage,
mem_at_ms: u64,
}
fn tree_sampler(root: u32) -> Option<(Sender<()>, std::thread::JoinHandle<TreePeak>)> {
let mut sampler = TreeSampler::new(root).ok()?;
let (stop_tx, stop_rx) = channel::<()>();
let started = Instant::now();
let handle = std::thread::spawn(move || {
let mut peak = TreePeak::default();
loop {
let s = sampler.sample();
if s.mem > peak.usage.mem {
peak.mem_at_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
}
peak.usage = peak.usage.max(s);
if !matches!(
stop_rx.recv_timeout(TREE_SAMPLE),
Err(RecvTimeoutError::Timeout)
) {
return peak;
}
}
});
Some((stop_tx, handle))
}
fn other_sampler(dir: &StateDir, own_pid: u32, cores: u32) -> Option<OtherHandle> {
let mut sampler = OtherSampler::new(cores).ok()?;
let roots = move |d: &StateDir| {
let mut r: Vec<u32> = d
.read_jobs()
.iter()
.filter(|j| j.state == State::Running)
.map(|j| j.pid)
.collect();
if !r.contains(&own_pid) {
r.push(own_pid);
}
r
};
sampler.sample(&roots(dir));
let mut sum = Usage::default();
for _ in 0..4 {
std::thread::sleep(Duration::from_millis(250));
let s = sampler.sample(&roots(dir));
sum = Usage {
cpu: sum.cpu + s.cpu,
mem: sum.mem + s.mem,
vram: 0,
};
}
let baseline = Usage {
cpu: sum.cpu / 4,
mem: sum.mem / 4,
vram: 0,
};
let (stop_tx, stop_rx) = channel::<()>();
let dir = dir.clone();
let started = Instant::now();
let handle = std::thread::spawn(move || {
let (mut cpu, mut mem) = (vec![], vec![]);
let (mut max_mem, mut mem_at_ms) = (0, 0);
loop {
let s = sampler.sample(&roots(&dir));
cpu.push(s.cpu.saturating_sub(baseline.cpu));
let rise = s.mem.saturating_sub(baseline.mem);
if rise > max_mem {
max_mem = rise;
mem_at_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
}
mem.push(rise);
if !matches!(
stop_rx.recv_timeout(OTHER_SAMPLE),
Err(RecvTimeoutError::Timeout)
) {
return OtherPeak {
cpu: p90(&mut cpu),
mem: p90(&mut mem),
mem_at_ms,
};
}
}
});
Some(OtherHandle {
stop: stop_tx,
handle,
})
}
fn p90(samples: &mut [u64]) -> u64 {
if samples.is_empty() {
return 0;
}
samples.sort_unstable();
samples[(samples.len() * 9).div_ceil(10) - 1]
}
pub(crate) fn hosts_vm_work(cmd: &[String]) -> bool {
cmd.iter().any(|a| a.contains("docker"))
}
struct OtherPeak {
cpu: u64,
mem: u64,
mem_at_ms: u64,
}
struct OtherHandle {
stop: Sender<()>,
handle: std::thread::JoinHandle<OtherPeak>,
}
fn wait_child(pid: Pid, term: Option<Terminal>) -> Result<ExitStatus, nix::Error> {
loop {
match waitpid(pid, Some(WaitPidFlag::WUNTRACED)) {
Ok(WaitStatus::Exited(_, code)) => return Ok(ExitStatus::from_raw(code << 8)),
Ok(WaitStatus::Signaled(_, sig, _)) => return Ok(ExitStatus::from_raw(sig as i32)),
Ok(WaitStatus::Stopped(..)) => {
if let Some(t) = term {
t.take_back();
}
let _ = kill(Pid::this(), Signal::SIGSTOP);
if let Some(t) = term {
t.give_to_child();
}
let _ = killpg(pid, Signal::SIGCONT);
}
Ok(_) | Err(Errno::EINTR) => {}
Err(e) => return Err(e),
}
}
}
fn record_finished(
dir: &StateDir,
job: &Job,
info: &JobInfo,
stats: Stats,
exit: ExitStatus,
timed_out: bool,
) -> i32 {
let code = if timed_out {
i32::from(TIMEOUT_EXIT)
} else {
exit_code(exit)
};
let entry = Entry {
ts: Timestamp::now(),
cmd: job.cmd.clone(),
cwd: job.cwd.clone(),
requested: job.weights,
measured: stats,
exit_code: code,
v: crate::history::VERSION,
repo: history::repo_key(Path::new(&job.cwd)),
};
if let Err(e) = history::append(&dir.history_path(), &entry) {
eprintln!("stokehold: could not write history: {e}");
}
log::record(
dir.base(),
&Event::Finished {
ts: Timestamp::now(),
job: info.clone(),
exit: code,
wall_ms: stats.run_ms,
cpu_ms: stats.cpu_user_ms + stats.cpu_sys_ms,
peak_rss: stats.peak_tree_rss.max(stats.peak_rss),
},
);
code
}
fn execute(
dir: &StateDir,
cap: &Capacity,
job: &Job,
info: &JobInfo,
sig: &Sig,
timeout: Duration,
queued_ms: u64,
) -> Result<u8, Error> {
let (first, rest) = job
.cmd
.split_first()
.ok_or_else(|| Error::Usage("no command".into()))?;
let mut command = Command::new(first);
command
.args(rest)
.process_group(0)
.env("STOKEHOLD_JOB", &job.id)
.env("HEAVY_JOB", &job.id);
let usage_before = children_usage()?;
let net_before = net_bytes();
let steal_before = cpu_times().map(|t| t.steal);
let sampler = vram_sampler(cap);
let tree = tree_sampler(job.pid);
let other = if hosts_vm_work(&job.cmd) {
other_sampler(dir, job.pid, cap.cpu)
} else {
None
};
let started = Instant::now();
let child = match command.spawn() {
Ok(c) => c,
Err(e) => {
eprintln!("stokehold: cannot run {first}: {e}");
return Ok(127);
}
};
let pgid = i32::try_from(child.id()).map_err(|_| Error::Probe("pid out of range".into()))?;
sig.pgid.store(pgid, Ordering::SeqCst);
let pending = sig.last.load(Ordering::SeqCst);
if pending != 0 {
forward(pgid, pending);
}
let term = Terminal::claim(Pid::from_raw(pgid));
let tty = TerminalGuard(term);
let (wtx, wrx) = channel();
std::thread::spawn(move || {
let _ = wtx.send(wait_child(Pid::from_raw(pgid), term));
});
let grace = Duration::from_millis(
crate::state::env_var("STOKEHOLD_KILL_GRACE_MS")
.and_then(|v| v.into_string().ok())
.and_then(|v| v.parse().ok())
.unwrap_or(DEFAULT_GRACE_MS),
);
let (result, timed_out) = if let Ok(r) = wrx.recv_timeout(timeout) {
(r, false)
} else {
forward(pgid, SIGTERM);
let r = wrx.recv_timeout(grace).unwrap_or_else(|_| {
forward(pgid, 9);
wrx.recv().unwrap_or_else(|_| Ok(ExitStatus::from_raw(0)))
});
(r, true)
};
forward(pgid, 9);
drop(tty);
let exit = result?;
let run_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX);
let mut stats = compute(queued_ms, run_ms, usage_before, children_usage()?);
let net_after = net_bytes();
stats.net_in = net_after.0.saturating_sub(net_before.0);
stats.net_out = net_after.1.saturating_sub(net_before.1);
stats.steal_ms = steal_before
.zip(cpu_times())
.map(|(a, b)| b.steal.saturating_sub(a) * 10);
stats.peak_vram = sampler.and_then(|(stop, h)| {
drop(stop);
h.join().ok().filter(|v| *v > 0)
});
let tree_peak = tree.and_then(|(stop, h)| {
drop(stop);
h.join().ok()
});
let other_peak = other.and_then(|o| {
drop(o.stop);
o.handle.join().ok()
});
if let Some(peak) = tree_peak {
stats.peak_tree_rss = peak.usage.mem;
stats.peak_centi_cores = peak.usage.cpu;
stats.peak_mem_at_ms = peak.mem_at_ms;
}
if let Some(other) = other_peak {
stats.peak_other_centi_cores = other.cpu;
stats.peak_centi_cores = effective_peak(stats.peak_centi_cores, other.cpu, 0, cap.cpu);
stats.peak_other_mem = other.mem;
if other.mem > stats.peak_tree_rss {
stats.peak_mem_at_ms = other.mem_at_ms;
}
stats.peak_tree_rss = effective_mem_peak(stats.peak_tree_rss, other.mem, cap.mem);
}
stats.timed_out = timed_out;
eprintln!("{}", format_stats(&stats));
let code = record_finished(dir, job, info, stats, exit, timed_out);
Ok(u8::try_from(code & 0xff).unwrap_or(1))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cold_and_docker_jobs_ask_for_the_whole_machine() {
let cap = Capacity {
cpu: 16,
mem: 64 << 30,
vram: 0,
disk_total: 1 << 40,
vram_is_ram: true,
};
let cold = |c: &[&str]| {
default_cpu(
&c.iter().map(|s| (*s).to_string()).collect::<Vec<_>>(),
&cap,
)
};
assert_eq!(cold(&["cargo", "build"]), 16);
assert_eq!(cold(&["docker", "build", "."]), 16);
assert_eq!(cold(&["docker", "compose", "up"]), 16);
assert_eq!(cold(&["make", "docker-build"]), 16);
}
#[test]
fn p90_is_sustained_not_a_spike_and_only_docker_uses_the_machine() {
let mut s: Vec<u64> = (1..=10).map(|i| if i == 10 { 1500 } else { 100 }).collect();
assert_eq!(p90(&mut s), 100);
assert_eq!(p90(&mut []), 0);
let c = |v: &[&str]| v.iter().map(|x| (*x).to_string()).collect::<Vec<_>>();
assert!(hosts_vm_work(&c(&["docker", "build"])));
assert!(hosts_vm_work(&c(&["make", "docker-build"])));
assert!(!hosts_vm_work(&c(&["cargo", "test"])));
}
#[test]
fn a_ramp_sleeps_until_its_end_not_a_fixed_ten_seconds() {
let ms = Duration::from_millis;
assert_eq!(wait_timeout(true, Some(13_000), 10_000), ms(3001));
assert_eq!(wait_timeout(true, Some(70_000), 10_000), LIVE_RESAMPLE);
assert_eq!(
wait_timeout(true, Some(10_000), 10_000),
ms(RAMP_MIN_SLEEP_MS)
);
assert_eq!(
wait_timeout(true, Some(9_000), 10_000),
ms(RAMP_MIN_SLEEP_MS)
);
assert_eq!(wait_timeout(true, None, 0), LIVE_RESAMPLE);
assert_eq!(wait_timeout(false, None, 0), SWEEP);
}
}