use std::collections::VecDeque;
use crate::job::{Capacity, Job, Live, State, Usage, Weights};
pub const BACKFILL_WINDOW_MS: i64 = 60_000;
const MEM_PCT: u64 = 95;
const VRAM_PCT: u64 = 90;
const CPU_PCT: u64 = 95;
const WARN_MEM_PCT: u64 = 85;
const TINY_MEM: (u64, u64) = (1, 32);
pub const TINY_BYPASS_MS: i64 = 5 * 60_000;
pub const BUSY_WINDOW_MS: i64 = 20_000;
pub const CPU_PINNED_PCT: u64 = 98;
pub const PINNED_WINDOW_MS: i64 = 10_000;
const LOAD_SOFT: f64 = 2.0;
const LOAD_BUSY_CPU_PCT: u64 = 90;
const LOAD_RISING_CPU_PCT: u64 = 75;
const LOAD_HARD: f64 = 4.0;
const LOAD_HARD_CPU_PCT: u64 = 50;
const LOAD_EXTREME: f64 = 6.0;
const CPU_PSI_PCT: u64 = 25;
pub const RAMP_MS: i64 = 10_000;
pub const CPU_FALLBACK_HOLD_MS: i64 = 30_000;
pub const CPU_MAX_HOLD_MS: i64 = 120_000;
const CPU_SHOWN: (u64, u64) = (3, 4);
pub const RAMP_FULL_MS: i64 = 3_000;
const FULL_HOLD_FRACTION: (u64, u64) = (3, 4);
pub const MEM_FALLBACK_HOLD_MS: i64 = 60_000;
pub const MEM_DECAY_MS: i64 = 60_000;
const MEM_HELD_CAP: (u64, u64) = (3, 4);
const RAMP_IDLE_CPU_PCT: u64 = 50;
const RAMP_STEP_MS: i64 = 250;
const CPU_ASK: u64 = 100;
const BACKFILL_CPU: u64 = 200;
pub const SMOOTH_WINDOW: usize = 3;
pub const PLATEAU_MS: u64 = 10_000;
const PLATEAU_SLACK_CPU: u64 = 100;
const PROJECTED_PEAK: (u64, u64) = (2, 5);
const UNDERUSED_CPU_PCT: u64 = 75;
pub const SUSTAINED_FULL_MS: i64 = RAMP_MS;
pub const BACKFILL_OVERRUN_MS: i64 = 30_000;
pub const SUSTAINED_WINDOW_MS: u64 = 10_000;
pub const FAST_RUN_MS: u64 = 10_000;
pub const FAST_MAX_CPU: u32 = 2;
pub const FAST_MAX_JOBS: usize = 2;
#[derive(
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize,
)]
#[serde(rename_all = "lowercase")]
pub enum Block {
Cpu,
Mem,
Vram,
Load1,
Load5,
Queue,
Ramp,
}
impl Block {
#[must_use]
pub const fn name(self) -> &'static str {
match self {
Self::Cpu => "cpu",
Self::Mem => "mem",
Self::Vram => "vram",
Self::Load1 => "load1",
Self::Load5 => "load5",
Self::Queue => "queue",
Self::Ramp => "ramp",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Decision {
Start,
Fail(String),
Wait {
reason: String,
live: bool,
block: Block,
ramp_end_ms: Option<i64>,
},
}
struct Blocked {
reason: String,
live: bool,
block: Block,
ramp_end_ms: Option<i64>,
}
fn blocked(reason: impl Into<String>, live: bool, block: Block) -> Blocked {
Blocked {
reason: reason.into(),
live,
block,
ramp_end_ms: None,
}
}
fn wait(reason: impl Into<String>, block: Block) -> Decision {
Decision::Wait {
reason: reason.into(),
live: false,
block,
ramp_end_ms: None,
}
}
#[derive(Debug, Default)]
pub struct Smoother {
window: VecDeque<Usage>,
}
impl Smoother {
pub fn push(&mut self, sample: Usage) -> Usage {
self.window.push_back(sample);
while self.window.len() > SMOOTH_WINDOW {
self.window.pop_front();
}
let n = u64::try_from(self.window.len().max(1)).unwrap_or(1);
let sum = self.window.iter().fold(Usage::default(), |a, b| Usage {
cpu: a.cpu + b.cpu,
mem: a.mem + b.mem,
vram: a.vram + b.vram,
});
Usage {
cpu: (sum.cpu / n).max(sample.cpu),
mem: (sum.mem / n).max(sample.mem),
vram: (sum.vram / n).max(sample.vram),
}
}
}
#[must_use]
pub fn non_heavy(machine: Usage, heavy_trees: Usage) -> Usage {
machine.saturating_sub(heavy_trees)
}
fn centi(cores: u32) -> u64 {
u64::from(cores) * 100
}
fn cores(centi: u64) -> String {
format!("{}.{:02}", centi / 100, centi % 100)
}
fn tiny(w: Weights, cap: &Capacity) -> bool {
centi(w.cpu) <= backfill(cap) && w.mem <= cap.mem / TINY_MEM.1 * TINY_MEM.0
}
fn ceiling(cap: &Capacity) -> Usage {
Usage {
cpu: centi(cap.cpu) * CPU_PCT / 100,
mem: cap.mem / 100 * MEM_PCT,
vram: cap.vram / 100 * VRAM_PCT,
}
}
fn backfill(cap: &Capacity) -> u64 {
(centi(cap.cpu) / 8).max(BACKFILL_CPU)
}
fn request(w: Weights) -> Usage {
Usage {
cpu: if w.cpu > 0 { CPU_ASK } else { 0 },
mem: w.mem,
vram: w.vram,
}
}
fn fits(ask: Usage, room: Usage) -> bool {
ask.cpu <= room.cpu && ask.mem <= room.mem && ask.vram <= room.vram
}
#[must_use]
pub fn fits_reservations(req: Weights, running: &[&Job], cap: &Capacity) -> bool {
let used: u64 = running.iter().map(|j| j.weights.vram).sum();
req.vram <= ceiling(cap).vram.saturating_sub(used)
}
#[must_use]
pub fn mem_unmet(
reserved: u64,
live: u64,
elapsed_ms: i64,
declared: bool,
peak_at_ms: Option<i64>,
) -> u64 {
let (full, tail) = mem_unmet_split(reserved, live, elapsed_ms, declared, peak_at_ms);
full + tail
}
#[must_use]
pub fn mem_unmet_split(
reserved: u64,
live: u64,
elapsed_ms: i64,
declared: bool,
peak_at_ms: Option<i64>,
) -> (u64, u64) {
let unmet = reserved.saturating_sub(live);
if declared {
return (unmet, 0);
}
let hold = peak_at_ms.unwrap_or(MEM_FALLBACK_HOLD_MS).max(RAMP_MS);
if elapsed_ms <= hold {
(unmet, 0)
} else if elapsed_ms >= hold + MEM_DECAY_MS {
(0, 0)
} else {
(
0,
unmet * u64::try_from(hold + MEM_DECAY_MS - elapsed_ms).unwrap_or(0)
/ u64::try_from(MEM_DECAY_MS).unwrap_or(1),
)
}
}
#[must_use]
pub fn ramp_hold(peak: u64, machine: u64, elapsed_ms: i64, cpu_pct: u64, vm: bool) -> u64 {
if elapsed_ms >= RAMP_MS {
return 0;
}
if elapsed_ms < RAMP_FULL_MS
|| vm
|| peak * FULL_HOLD_FRACTION.1 >= machine * FULL_HOLD_FRACTION.0
{
return peak;
}
if cpu_pct < RAMP_IDLE_CPU_PCT {
return 0;
}
let left = u64::try_from(RAMP_MS - elapsed_ms).unwrap_or(0);
let span = u64::try_from(RAMP_MS - RAMP_FULL_MS).unwrap_or(1);
peak * left / span
}
#[must_use]
pub fn cpu_hold_ms(peak_at_ms: Option<u64>) -> i64 {
peak_at_ms
.and_then(|ms| i64::try_from(ms.saturating_add(ms / 4)).ok())
.unwrap_or(CPU_FALLBACK_HOLD_MS)
.clamp(RAMP_MS, CPU_MAX_HOLD_MS)
}
#[must_use]
#[expect(
clippy::too_many_arguments,
reason = "one job's hold inputs: its size, age, live use and learned schedule"
)]
pub fn cpu_hold(
peak: u64,
machine: u64,
elapsed_ms: i64,
cpu_pct: u64,
vm: bool,
live_cpu: u64,
hold_ms: Option<i64>,
settled: bool,
) -> u64 {
match hold_ms {
Some(hold) if vm => {
if elapsed_ms < hold.max(RAMP_MS) {
peak
} else {
0
}
}
Some(hold) => {
if elapsed_ms < RAMP_FULL_MS {
peak
} else if settled || elapsed_ms >= hold || live_cpu * CPU_SHOWN.1 >= peak * CPU_SHOWN.0
{
0
} else {
peak
}
}
_ => ramp_hold(peak, machine, elapsed_ms, cpu_pct, vm),
}
}
fn cpu_hold_end(j: &Job, started_ms: i64) -> i64 {
started_ms
+ match j.cpu_hold_ms {
Some(h) if crate::run::hosts_vm_work(&j.cmd) => h.max(RAMP_MS),
Some(h) => h.max(RAMP_FULL_MS),
None => RAMP_MS,
}
}
#[must_use]
pub fn cpu_plateaued(samples: &[(u64, u64)], slowest_ms: u64) -> bool {
let Some(&(now, _)) = samples.last() else {
return false;
};
let from = slowest_ms.max(u64::try_from(RAMP_MS).unwrap_or(0)) + PLATEAU_MS;
if now < from {
return false;
}
let cut = now - PLATEAU_MS;
let recent: Vec<u64> = samples
.iter()
.filter(|(ms, _)| *ms > cut)
.map(|(_, c)| *c)
.collect();
let (Some(&hi), Some(&lo)) = (recent.iter().max(), recent.iter().min()) else {
return false;
};
let first = samples
.iter()
.find(|(ms, _)| *ms > cut)
.map_or(now, |s| s.0);
first <= cut + PLATEAU_MS / 4 && hi - lo <= (hi / 8).max(PLATEAU_SLACK_CPU)
}
#[must_use]
pub fn sustained_pct(latest: u64, readings: &[(i64, u64)], now_ms: i64) -> u64 {
let recent: Vec<u64> = readings
.iter()
.filter(|(ts, _)| *ts <= now_ms && now_ms - *ts <= BUSY_WINDOW_MS)
.map(|(_, pct)| *pct)
.collect();
let n = u64::try_from(recent.len()).unwrap_or(u64::MAX);
recent
.iter()
.sum::<u64>()
.checked_div(n)
.unwrap_or(0)
.max(latest)
}
#[must_use]
pub fn recent_peak_pct(readings: &[(i64, u64)], now_ms: i64) -> u64 {
readings
.iter()
.filter(|(ts, _)| *ts <= now_ms && now_ms - *ts <= PINNED_WINDOW_MS)
.map(|(_, pct)| *pct)
.max()
.unwrap_or(0)
}
#[cfg(test)]
#[must_use]
pub fn headroom(running: &[&Job], cap: &Capacity, live: &Live, now_ms: i64) -> Usage {
headroom_for(running, cap, live, now_ms, false)
}
fn running_hold(j: &Job, cap: &Capacity, live: &Live, now_ms: i64, vm: bool, used: u64) -> u64 {
let peak = centi(j.weights.cpu).min(centi(cap.cpu));
j.started_ms.map_or(0, |t| {
let elapsed = now_ms.saturating_sub(t).max(0);
let hold = sustained_hold(
cpu_hold(
peak,
centi(cap.cpu),
elapsed,
live.cpu_pct,
vm,
used,
j.cpu_hold_ms,
j.cpu_settled,
),
j.cpu_sustained,
elapsed,
vm,
j.cpu_hold_ms,
);
sustained_floor(hold, peak, j.cpu_sustained, vm)
})
}
#[must_use]
pub fn sustained_floor(hold: u64, peak: u64, sustained: Option<u64>, vm: bool) -> u64 {
match sustained {
Some(level) if !vm => hold.max(level.min(peak)),
_ => hold,
}
}
#[must_use]
pub fn headroom_for(
running: &[&Job],
cap: &Capacity,
live: &Live,
now_ms: i64,
small: bool,
) -> Usage {
let (mut held, (mut unmet, mut unmet_small), (pre_peak, tail), vm_own, vm_base) =
running.iter().fold(
(Usage::default(), (0, 0), (0u64, 0u64), 0u64, 0u64),
|(a, unmet, learned, vm_own, vm_base), j| {
let l = live
.per_job
.iter()
.find(|(id, _)| *id == j.id)
.map_or_else(Usage::default, |(_, u)| *u);
let vm = crate::run::hosts_vm_work(&j.cmd);
let hold = running_hold(j, cap, live, now_ms, vm, l.cpu);
let cpu = l.cpu.max(hold);
let (mem, learned_unmet) = match j.started_ms {
Some(t) if !j.mem_declared => (
l.mem,
mem_unmet_split(
j.weights.mem,
l.mem,
now_ms.saturating_sub(t).max(0),
false,
j.mem_peak_at_ms,
),
),
Some(t) => (
l.mem
+ mem_unmet(
j.weights.mem,
l.mem,
now_ms.saturating_sub(t).max(0),
true,
None,
),
(0, 0),
),
None => (l.mem + j.weights.mem.saturating_sub(l.mem), (0, 0)),
};
(
Usage {
cpu: a.cpu + cpu,
mem: a.mem + mem,
vram: a.vram + l.vram.max(j.weights.vram),
},
{
let own = hold.saturating_sub(l.cpu);
let is_small =
small && centi(j.weights.cpu).min(centi(cap.cpu)) <= backfill(cap);
(unmet.0 + own, unmet.1 + if is_small { own } else { 0 })
},
(learned.0 + learned_unmet.0, learned.1 + learned_unmet.1),
vm_own
+ u64::from(vm && j.started_ms.is_some_and(|t| now_ms - t >= RAMP_MS))
* hold.saturating_sub(l.cpu),
vm_base + u64::from(vm) * j.non_heavy_cpu_at_start,
)
},
);
let vm_shown = vm_own.min(live.non_heavy.cpu.saturating_sub(vm_base));
held.cpu = held.cpu.saturating_sub(vm_shown);
unmet = unmet.saturating_sub(vm_shown);
unmet_small = unmet_small.min(unmet);
let free = ceiling(cap)
.mem
.saturating_sub(live.non_heavy.mem)
.saturating_sub(held.mem);
held.mem += pre_peak;
let free = free.saturating_sub(pre_peak);
held.mem += tail.min(free / MEM_HELD_CAP.1 * MEM_HELD_CAP.0);
let limit = if small {
ceiling(cap).cpu.saturating_sub(backfill(cap))
} else {
u64::MAX
};
let live_jobs = held.cpu.saturating_sub(unmet);
let held_cpu = held
.cpu
.saturating_sub(unmet_small)
.min(limit.saturating_sub(live.non_heavy.cpu))
.max(live_jobs)
+ unmet_small;
let measured = live.cpu_sustained_pct * centi(cap.cpu) / 100;
let instant = live.non_heavy.cpu + held_cpu;
let sustained = (measured + unmet - unmet_small).min(limit).max(measured) + unmet_small;
let room = ceiling(cap)
.saturating_sub(live.non_heavy)
.saturating_sub(held);
Usage {
cpu: ceiling(cap).cpu.saturating_sub(instant.max(sustained)),
..room
}
}
#[must_use]
pub fn disk_full(req: Weights, cap: &Capacity, live: &Live) -> Option<String> {
let need = u128::from(req.disk) + u128::from(cap.disk_total) / 10;
(u128::from(live.disk_free) < need).then(|| {
format!(
"the disk is full: {} free, need {} (the request plus 10% of the disk). Clean up, or ask Todd",
crate::sizes::fmt_size(live.disk_free),
crate::sizes::fmt_size(u64::try_from(need).unwrap_or(u64::MAX))
)
})
}
const RISING_MARGIN: f64 = 1.1;
#[must_use]
pub fn guard_block(cap: &Capacity, live: &Live) -> Option<(String, Block)> {
let cores = f64::from(cap.cpu);
let (soft, hard) = (LOAD_SOFT * cores, LOAD_HARD * cores);
let cpu = live.cpu_pct.max(live.cpu_sustained_pct);
let peak = live.cpu_pct.max(live.cpu_recent_peak_pct);
if peak >= CPU_PINNED_PCT {
return Some((
format!(
"cpu pinned: {peak}% busy in the last {} s",
PINNED_WINDOW_MS / 1_000
),
Block::Cpu,
));
}
let psi = live.cpu_pressure.filter(|p| *p >= CPU_PSI_PCT);
let psi_note = psi.map_or_else(String::new, |p| format!(", cpu pressure {p}%"));
if live.load1 >= LOAD_EXTREME * cores {
return Some((
format!(
"1-minute load {:.1} >= {:.0} (6 x {} cores)",
live.load1,
LOAD_EXTREME * cores,
cap.cpu
),
Block::Load1,
));
}
let unconfirmed = live.cpu_pressure.is_none();
if live.load1 >= hard && (unconfirmed || cpu >= LOAD_HARD_CPU_PCT || psi.is_some()) {
return Some((
format!(
"1-minute load {:.1} >= {hard:.0} (4 x {} cores), cpu {cpu}% busy{psi_note}",
live.load1, cap.cpu
),
Block::Load1,
));
}
if live.mem_pressure {
return Some(("critical memory pressure".into(), Block::Mem));
}
if live.load1 >= soft {
if live.load1 > live.load5 * RISING_MARGIN && cpu >= LOAD_RISING_CPU_PCT {
return Some((
format!(
"1-minute load {:.1} >= {soft:.0} (2 x {} cores) and still rising (5-minute {:.1}), cpu {cpu}% busy",
live.load1, cap.cpu, live.load5
),
Block::Load1,
));
}
if cpu >= LOAD_BUSY_CPU_PCT || psi.is_some() {
return Some((
format!(
"1-minute load {:.1} >= {soft:.0} (2 x {} cores) and cpu {cpu}% busy{psi_note}",
live.load1, cap.cpu
),
Block::Load1,
));
}
}
if live.mem_used_pct >= MEM_PCT {
return Some((
format!("memory {}% used >= {MEM_PCT}%", live.mem_used_pct),
Block::Mem,
));
}
if live.mem_pressure_warn && live.mem_used_pct >= WARN_MEM_PCT {
return Some((
format!(
"memory pressure warn with memory {}% used >= {WARN_MEM_PCT}%",
live.mem_used_pct
),
Block::Mem,
));
}
None
}
fn blocker(
w: Weights,
sustained: Option<u64>,
running: &[&Job],
cap: &Capacity,
live: &Live,
now_ms: i64,
waited_ms: i64,
) -> Option<Blocked> {
if !running.is_empty() {
if !fits_reservations(w, running, cap) {
return Some(blocked("vram reservations full", false, Block::Vram));
}
let small = centi(w.cpu) <= backfill(cap);
let mut ask = request(w);
if small {
ask.cpu = ask.cpu.max(centi(w.cpu));
} else {
if waited_ms < BACKFILL_WINDOW_MS || w.cpu < cap.cpu {
let over_band = centi(cap.cpu) - ceiling(cap).cpu;
ask.cpu = ask.cpu.max(
projected_peak(centi(w.cpu).min(centi(cap.cpu)), sustained)
.saturating_sub(over_band),
);
}
}
let room_at = |t: i64| headroom_for(running, cap, live, t, small);
if !fits(ask, room_at(now_ms)) {
if fits(ask, room_at(i64::MAX)) {
let last = running
.iter()
.filter_map(|j| j.started_ms.map(|t| cpu_hold_end(j, t)))
.filter(|e| *e > now_ms)
.max();
let end = last.map(|last| {
let floors = running
.iter()
.filter_map(|j| j.started_ms)
.map(|t| t + RAMP_FULL_MS);
let mut at: Vec<i64> = (1..=(last - now_ms) / RAMP_STEP_MS)
.map(|k| now_ms + k * RAMP_STEP_MS)
.chain(floors)
.filter(|t| *t > now_ms && *t < last)
.collect();
at.sort_unstable();
at.into_iter()
.find(|t| fits(ask, room_at(*t)))
.unwrap_or(last)
});
return Some(Blocked {
reason: format!(
"cpu held for jobs still ramping up: needs {} cores, {} free now under the {CPU_PCT}% band",
cores(ask.cpu),
cores(room_at(now_ms).cpu)
),
live: true,
block: Block::Ramp,
ramp_end_ms: end,
});
}
let room = room_at(now_ms);
let (reason, block) = if ask.cpu > room.cpu {
(
format!(
"cpu: needs {} cores, {} free under the {CPU_PCT}% band (machine {}% busy)",
cores(ask.cpu),
cores(room.cpu),
live.cpu_pct.max(live.cpu_sustained_pct)
),
Block::Cpu,
)
} else if ask.mem > room.mem {
(
format!(
"mem: needs {}, {} free under {MEM_PCT}% of RAM after running jobs' holds",
crate::sizes::fmt_size(ask.mem),
crate::sizes::fmt_size(room.mem)
),
Block::Mem,
)
} else {
(
format!(
"vram: needs {}, {} free",
crate::sizes::fmt_size(ask.vram),
crate::sizes::fmt_size(room.vram)
),
Block::Vram,
)
};
return Some(blocked(reason, true, block));
}
}
guard_block(cap, live).map(|(why, block)| blocked(why, true, block))
}
#[must_use]
pub fn projected_peak(peak: u64, sustained: Option<u64>) -> u64 {
(peak * PROJECTED_PEAK.0 / PROJECTED_PEAK.1).min(sustained.unwrap_or(u64::MAX))
}
#[must_use]
pub fn sustained_hold(
hold: u64,
sustained: Option<u64>,
elapsed_ms: i64,
vm: bool,
hold_ms: Option<i64>,
) -> u64 {
match (sustained, hold_ms) {
(Some(level), Some(peak_by)) if !vm && elapsed_ms >= peak_by.max(SUSTAINED_FULL_MS) => {
hold.min(level)
}
_ => hold,
}
}
#[must_use]
pub fn sustained_level(samples: &[(u64, u64)]) -> u64 {
let Some(&(last, _)) = samples.last() else {
return 0;
};
let mean = |w: &[(u64, u64)]| {
w.iter().map(|s| s.1).sum::<u64>() / u64::try_from(w.len().max(1)).unwrap_or(1)
};
if last < SUSTAINED_WINDOW_MS {
return mean(samples);
}
let mut best = 0;
let mut from = 0;
for (i, &(at, _)) in samples.iter().enumerate() {
while samples[from].0 + SUSTAINED_WINDOW_MS <= at {
from += 1;
}
if at >= SUSTAINED_WINDOW_MS {
best = best.max(mean(&samples[from..=i]));
}
}
best
}
#[must_use]
pub fn fast_lane_eligible(w: Weights, cap: &Capacity, slowest_run_ms: Option<u64>) -> bool {
slowest_run_ms.is_some_and(|ms| ms <= FAST_RUN_MS)
&& w.cpu <= FAST_MAX_CPU
&& w.mem <= cap.mem / TINY_MEM.1 * TINY_MEM.0
&& w.vram == 0
}
fn fast_job_outgrew_lane(running: &[&Job], live: &Live) -> bool {
running.iter().filter(|j| j.fast).any(|j| {
live.per_job
.iter()
.any(|(id, u)| *id == j.id && u.cpu > centi(FAST_MAX_CPU))
})
}
fn fast_cpu_room(cap: &Capacity, live: &Live) -> bool {
let busy = live.cpu_pct.max(live.cpu_sustained_pct);
let free_centi = u64::from(cap.cpu) * 100 * 100u64.saturating_sub(busy) / 100;
busy < 100 && free_centi >= centi(FAST_MAX_CPU)
}
fn fast_lane(job: &Job, running: &[&Job], cap: &Capacity, live: &Live, now_ms: i64) -> bool {
job.fast
&& running.iter().filter(|j| j.fast).count() < FAST_MAX_JOBS
&& !fast_job_outgrew_lane(running, live)
&& fast_cpu_room(cap, live)
&& guard_block(cap, live).is_none()
&& fits_reservations(job.weights, running, cap)
&& job.weights.mem <= headroom_for(running, cap, live, now_ms, true).mem
}
fn outlasts_backfill(job: &Job, head: &Job, now_ms: i64) -> bool {
let by = head.enqueued_ms + TINY_BYPASS_MS + BACKFILL_OVERRUN_MS;
job.run_p90_ms
.and_then(|d| i64::try_from(d).ok())
.is_none_or(|d| now_ms.saturating_add(d) > by)
}
#[must_use]
pub fn evaluate(job: &Job, jobs: &[Job], cap: &Capacity, live: &Live, now_ms: i64) -> Decision {
let running: Vec<&Job> = jobs.iter().filter(|j| j.state == State::Running).collect();
let mut queued: Vec<&Job> = jobs
.iter()
.filter(|j| j.state == State::Queued && (j.id == job.id || !live.stopped.contains(&j.id)))
.collect();
queued.sort_by(|a, b| (a.enqueued_ms, &a.id).cmp(&(b.enqueued_ms, &b.id)));
if let Some(why) = disk_full(job.weights, cap, live) {
return Decision::Fail(why);
}
if fast_lane(job, &running, cap, live, now_ms) {
return Decision::Start;
}
if let Some(head) = queued.first().filter(|h| h.id != job.id) {
let head_blocker = |rs: &[&Job]| {
blocker(
head.weights,
head.cpu_sustained,
rs,
cap,
live,
now_ms,
now_ms - head.enqueued_ms,
)
};
let head_block = head_blocker(&running);
let me = Job {
state: State::Running,
started_ms: Some(now_ms),
..job.clone()
};
let mut with_me = running.clone();
with_me.push(&me);
if head_block.is_none() {
if head_blocker(&with_me).is_some() {
return wait("queued behind a job that fits now", Block::Queue);
}
} else if !tiny(job.weights, cap)
|| head_block.as_ref().is_some_and(|b| match b.block {
Block::Cpu => true,
Block::Ramp => head_blocker(&with_me).is_none_or(|after| {
after.block != Block::Ramp
|| after.ramp_end_ms.unwrap_or(i64::MAX) > b.ramp_end_ms.unwrap_or(i64::MAX)
}),
_ => now_ms - head.enqueued_ms >= TINY_BYPASS_MS,
})
{
let backfills_idle_cpu = now_ms - head.enqueued_ms < TINY_BYPASS_MS
&& live.cpu_pct.max(live.cpu_sustained_pct) < UNDERUSED_CPU_PCT
&& head_block
.as_ref()
.is_some_and(|b| matches!(b.block, Block::Cpu | Block::Ramp));
if now_ms - head.enqueued_ms >= BACKFILL_WINDOW_MS && !backfills_idle_cpu {
return wait(
format!("queued behind {} (waited over 60 s)", head.id),
Block::Queue,
);
}
let (a, h) = (job.weights, head.weights);
if a.cpu > h.cpu || a.mem > h.mem || a.vram > h.vram {
return wait("queued behind a smaller-or-equal head", Block::Queue);
}
if !tiny(a, cap) && outlasts_backfill(job, head, now_ms) {
return wait(
format!(
"queued behind {} (its run would outlast the head's backfill time)",
head.id
),
Block::Queue,
);
}
}
}
match blocker(
job.weights,
job.cpu_sustained,
&running,
cap,
live,
now_ms,
now_ms - job.enqueued_ms,
) {
Some(b) => Decision::Wait {
reason: b.reason,
live: b.live,
block: b.block,
ramp_end_ms: b.ramp_end_ms,
},
None => Decision::Start,
}
}
#[cfg(test)]
mod tests {
use super::*;
const G: u64 = 1 << 30;
fn cap() -> Capacity {
Capacity {
cpu: 10,
mem: 100 * G,
vram: 0,
disk_total: 1000 * G,
vram_is_ram: false,
}
}
fn calm() -> Live {
Live {
load1: 0.5,
mem_used_pct: 10,
disk_free: 900 * G,
..Live::default()
}
}
fn job(id: &str, cpu: u32, mem_g: u64, state: State, enq: i64) -> Job {
Job {
id: id.into(),
pid: 1,
start_time: 0,
weights: Weights {
cpu,
mem: mem_g * G,
vram: 0,
disk: 0,
},
cmd: vec!["x".into()],
cwd: "/".into(),
state,
enqueued_ms: enq,
started_ms: None,
blocked_by: None,
mem_declared: false,
mem_peak_at_ms: None,
cpu_hold_ms: None,
cpu_settled: false,
non_heavy_cpu_at_start: 0,
fast: false,
cpu_sustained: None,
run_p90_ms: None,
}
}
fn waits(d: &Decision) -> bool {
matches!(d, Decision::Wait { .. })
}
#[test]
fn alone_always_runs() {
let me = job("a", 10, 100, State::Queued, 0);
let busy = Live {
non_heavy: Usage {
cpu: 900,
mem: 90 * G,
vram: 0,
},
..calm()
};
assert_eq!(
evaluate(&me, std::slice::from_ref(&me), &cap(), &busy, 0),
Decision::Start
);
}
#[test]
fn ninety_percent_vram_reservation_ceiling() {
let c = Capacity {
vram: 100 * G,
..cap()
};
let with_vram = |id: &str, g: u64, state| {
let j = job(id, 1, 1, state, 0);
Job {
weights: Weights {
vram: g * G,
..j.weights
},
..j
}
};
let run = with_vram("r", 50, State::Running);
let fits = with_vram("a", 40, State::Queued);
let over = with_vram("b", 41, State::Queued);
assert_eq!(
evaluate(&fits, &[run.clone(), fits.clone()], &c, &calm(), 2),
Decision::Start
);
let d = evaluate(&over, &[run, over.clone()], &c, &calm(), 2);
assert!(
matches!(
d,
Decision::Wait {
live: false,
block: Block::Vram,
..
}
),
"{d:?}"
);
}
#[test]
fn declared_reservations_over_the_ceiling_admit_when_measured_use_is_low() {
let c = cap();
let live = Live {
mem_used_pct: 43,
non_heavy: Usage {
cpu: 0,
mem: 20 * G,
vram: 0,
},
per_job: vec![
(
"r1".into(),
Usage {
cpu: 100,
mem: 12 * G,
vram: 0,
},
),
(
"r2".into(),
Usage {
cpu: 100,
mem: 11 * G,
vram: 0,
},
),
],
..calm()
};
let old = |id: &str| Job {
started_ms: Some(0),
..job(id, 2, 44, State::Running, 0)
};
let me = job("w", 2, 20, State::Queued, 1);
let jobs = [old("r1"), old("r2"), me.clone()];
assert_eq!(
evaluate(&me, &jobs, &c, &live, 100_000),
Decision::Start,
"reservations 88G + 20G > 90G, measured 43G"
);
let d = evaluate(&me, &jobs, &c, &live, 1_000);
assert!(
matches!(
d,
Decision::Wait {
block: Block::Ramp,
ramp_end_ms: Some(_),
..
}
),
"{d:?}"
);
let full = Live {
non_heavy: Usage {
cpu: 0,
mem: 60 * G,
vram: 0,
},
..live.clone()
};
let d = evaluate(&me, &jobs, &c, &full, 100_000);
assert!(
matches!(
d,
Decision::Wait {
block: Block::Mem,
live: true,
..
}
),
"{d:?}"
);
}
#[test]
fn mem_unmet_is_held_to_the_learned_peak_then_decays() {
let unmet = |at: i64, declared, peak| mem_unmet(10 * G, 3 * G, at, declared, peak);
assert_eq!(
mem_unmet(10 * G, 12 * G, 0, false, None),
0,
"live use over the reservation leaves nothing unmet"
);
assert_eq!(unmet(RAMP_MS, false, Some(90_000)), 7 * G);
assert_eq!(unmet(90_000, false, Some(90_000)), 7 * G);
assert_eq!(
unmet(90_000 + MEM_DECAY_MS / 2, false, Some(90_000)),
7 * G / 2
);
assert_eq!(unmet(90_000 + MEM_DECAY_MS, false, Some(90_000)), 0);
assert_eq!(unmet(RAMP_MS, false, Some(2_000)), 7 * G);
assert_eq!(unmet(MEM_FALLBACK_HOLD_MS, false, None), 7 * G);
assert_eq!(unmet(MEM_FALLBACK_HOLD_MS + MEM_DECAY_MS, false, None), 0);
assert_eq!(unmet(10_000_000, true, None), 7 * G);
}
fn big_mac() -> Capacity {
Capacity {
cpu: 16,
mem: 64 * G,
vram: 0,
disk_total: 1000 * G,
vram_is_ram: false,
}
}
fn admitted_of_twelve(declared: bool, now: impl Fn(usize) -> i64) -> usize {
let cap = big_mac();
let mut running: Vec<Job> = vec![];
for i in 0..12_i64 {
let w = Job {
mem_declared: declared,
mem_peak_at_ms: Some(120_000),
..job(&format!("w{i:02}"), 1, 12, State::Queued, i)
};
let live = Live {
non_heavy: Usage {
cpu: 0,
mem: 10 * G,
vram: 0,
},
per_job: running
.iter()
.map(|j: &Job| {
(
j.id.clone(),
Usage {
cpu: 50,
mem: G,
vram: 0,
},
)
})
.collect(),
..calm()
};
let mut all = running.clone();
all.push(w.clone());
if evaluate(&w, &all, &cap, &live, now(usize::try_from(i).unwrap_or(0)))
== Decision::Start
{
running.push(Job {
state: State::Running,
started_ms: Some(0),
..w
});
}
}
running.len()
}
#[test]
fn twelve_late_peaking_jobs_do_not_all_start_on_a_64g_mac() {
assert_eq!(admitted_of_twelve(false, |_| 30_000), 4);
assert_eq!(admitted_of_twelve(false, |_| 180_000), 12);
assert_eq!(admitted_of_twelve(true, |_| 3_600_000), 4);
}
#[test]
fn the_free_memory_cap_binds_only_the_decaying_tail_not_the_hold_before_the_peak() {
let cap = big_mac();
let live = Live {
non_heavy: Usage {
cpu: 0,
mem: 10 * G,
vram: 0,
},
per_job: vec![(
"r".into(),
Usage {
cpu: 50,
mem: 0,
vram: 0,
},
)],
..calm()
};
let r = Job {
started_ms: Some(0),
mem_peak_at_ms: Some(120_000),
..job("r", 1, 40, State::Running, 0)
};
let usable = ceiling(&cap).mem;
let at = |ms| headroom(&[&r], &cap, &live, ms).mem;
assert_eq!(at(30_000), usable - 50 * G);
assert_eq!(at(120_000), usable - 50 * G);
assert_eq!(at(150_000), usable - 30 * G);
let big = Job {
weights: Weights {
mem: 80 * G,
..r.weights
},
..r.clone()
};
let free = usable - 10 * G;
let tail = 80 * G / 2;
assert!(tail > free / 4 * 3);
assert_eq!(
headroom(&[&big], &cap, &live, 150_000).mem,
free - free / 4 * 3
);
}
#[test]
fn a_declared_mem_is_held_for_the_whole_run() {
let cap = big_mac();
let live = Live {
per_job: vec![(
"r".into(),
Usage {
cpu: 100,
mem: 2 * G,
vram: 0,
},
)],
..calm()
};
let me = job("w", 1, 30, State::Queued, 1);
let held = |declared: bool, now: i64| {
let r = Job {
started_ms: Some(0),
mem_declared: declared,
mem_peak_at_ms: Some(5_000),
..job("r", 1, 40, State::Running, 0)
};
evaluate(&me, &[r, me.clone()], &cap, &live, now)
};
assert!(
matches!(
held(true, 3_600_000),
Decision::Wait {
block: Block::Mem,
..
}
),
"{:?}",
held(true, 3_600_000)
);
assert_eq!(held(false, 3_600_000), Decision::Start);
}
#[test]
fn headroom_math_with_fake_non_heavy_usage() {
let run = job("r", 2, 10, State::Running, 0);
let live = Live {
non_heavy: Usage {
cpu: 200,
mem: 30 * G,
vram: 0,
},
per_job: vec![(
"r".into(),
Usage {
cpu: 300,
mem: 20 * G,
vram: 0,
},
)],
..calm()
};
let room = headroom(&[&run], &cap(), &live, 100_000);
assert_eq!(
room,
Usage {
cpu: 950 - 200 - 300,
mem: 95 * G - 30 * G - 20 * G,
vram: 0
}
);
let edge = job("e", 4, 45, State::Queued, 1);
assert_eq!(
evaluate(&edge, &[run.clone(), edge.clone()], &cap(), &live, 2),
Decision::Start
);
let over = job("o", 5, 46, State::Queued, 1);
let d = evaluate(&over, &[run.clone(), over.clone()], &cap(), &live, 2);
assert!(matches!(d, Decision::Wait { live: true, .. }), "{d:?}");
let idle = Live {
per_job: vec![("r".into(), Usage::default())],
non_heavy: Usage::default(),
..calm()
};
let run = Job {
started_ms: Some(0),
..run
};
let room = headroom(&[&run], &cap(), &idle, 200_000);
assert_eq!((room.cpu, room.mem), (950, 95 * G));
let room = headroom(&[&run], &cap(), &idle, 1_000);
assert_eq!(room.mem, 85 * G);
}
#[test]
fn non_heavy_is_machine_minus_heavy_trees() {
let machine = Usage {
cpu: 800,
mem: 50 * G,
vram: 2 * G,
};
let trees = Usage {
cpu: 600,
mem: 20 * G,
vram: 5 * G,
};
assert_eq!(
non_heavy(machine, trees),
Usage {
cpu: 200,
mem: 30 * G,
vram: 0
}
);
}
#[test]
fn smoother_is_fast_to_rise_and_slow_to_fall() {
let mut s = Smoother::default();
let low = Usage {
cpu: 100,
mem: 2 * G,
vram: 0,
};
let hi = Usage {
cpu: 900,
mem: 12 * G,
vram: 0,
};
assert_eq!(s.push(low), low);
assert_eq!(s.push(low), low);
assert_eq!(s.push(hi), hi, "a rise is not averaged away");
assert_eq!(
s.push(low),
Usage {
cpu: 366,
mem: 16 * G / 3,
vram: 0
},
"a fall is released slowly"
);
assert_eq!(
s.push(low),
Usage {
cpu: 366,
mem: 16 * G / 3,
vram: 0
}
);
for _ in 0..SMOOTH_WINDOW {
s.push(Usage::default());
}
assert_eq!(
s.push(Usage::default()),
Usage::default(),
"old peaks age out"
);
}
#[test]
fn a_fresh_waiter_sees_a_jump_at_once() {
let mut s = Smoother::default();
let hot = Usage {
cpu: 1500,
mem: 30 * G,
vram: 0,
};
assert_eq!(s.push(hot), hot);
}
#[test]
fn many_tiny_jobs_run_together() {
let mut all: Vec<Job> = (0..50)
.map(|i| job(&format!("r{i:02}"), 0, 0, State::Running, 0))
.collect();
let me = job("a", 0, 0, State::Queued, 1);
all.push(me.clone());
assert_eq!(
evaluate(&me, &all, &cap(), &calm(), 2),
Decision::Start,
"no cap on the number of jobs"
);
}
#[test]
fn guards_veto() {
let me = job("a", 1, 1, State::Queued, 0);
let one = std::slice::from_ref(&me);
let busy = Live {
load1: 21.0,
load5: 22.0,
cpu_pct: 95,
..calm()
};
assert!(matches!(
evaluate(&me, one, &cap(), &busy, 0),
Decision::Wait {
live: true,
block: Block::Load1,
..
}
));
let calm_1m = Live {
load1: 19.0,
load5: 5.0,
cpu_pct: 95,
..calm()
};
assert_eq!(
evaluate(&me, one, &cap(), &calm_1m, 0),
Decision::Start,
"macOS load reads high: 2 x cores"
);
let io_bound = Live {
load1: 35.0,
load5: 36.0,
cpu_pct: 40,
..calm()
};
assert_eq!(
evaluate(&me, one, &cap(), &io_bound, 0),
Decision::Start,
"high, falling load with a calm CPU is threads waiting on I/O"
);
let runaway = Live {
load1: 40.0,
cpu_pct: 50,
..calm()
};
assert!(
matches!(
evaluate(&me, one, &cap(), &runaway, 0),
Decision::Wait {
block: Block::Load1,
..
}
),
"4 x cores blocks with the CPU half busy"
);
let io_storm = Live {
cpu_pct: 10,
cpu_pressure: Some(0),
..runaway.clone()
};
assert_eq!(
evaluate(&me, one, &cap(), &io_storm, 0),
Decision::Start,
"0.4.8: 4 x cores at 10% CPU with PSI showing no CPU pressure is I/O wait"
);
let no_psi = Live {
cpu_pressure: None,
..io_storm
};
assert!(
waits(&evaluate(&me, one, &cap(), &no_psi, 0)),
"without PSI (macOS) 4 x cores blocks whatever the CPU says"
);
let mem = Live {
mem_used_pct: 95,
..calm()
};
let run = job("r", 1, 1, State::Running, 0);
assert!(waits(&evaluate(&me, &[run, me.clone()], &cap(), &mem, 0)));
assert!(waits(&evaluate(&me, one, &cap(), &mem, 0)), "even alone");
let thrash = Live {
mem_used_pct: 97,
mem_pressure_warn: true,
..calm()
};
let big = job("b", 1, 8, State::Queued, 0);
assert!(waits(&evaluate(
&big,
std::slice::from_ref(&big),
&cap(),
&thrash,
0
)));
}
#[test]
fn rising_load_blocks_at_two_times_cores_but_falling_load_admits() {
let me = job("a", 1, 1, State::Queued, 0);
let one = std::slice::from_ref(&me);
let rising = Live {
load1: 30.0,
load5: 22.0,
cpu_pct: 80,
..calm()
};
let d = evaluate(&me, one, &cap(), &rising, 0);
assert!(
matches!(&d, Decision::Wait { block: Block::Load1, live: true, reason, .. } if reason.contains("rising")),
"{d:?}"
);
let rising_io = Live {
cpu_pct: 30,
..rising.clone()
};
assert_eq!(evaluate(&me, one, &cap(), &rising_io, 0), Decision::Start);
let falling = Live {
load1: 30.0,
load5: 38.0,
cpu_pct: 30,
..calm()
};
assert_eq!(
evaluate(&me, one, &cap(), &falling, 0),
Decision::Start,
"falling and CPU under 90%"
);
let level = Live {
load1: 30.0,
load5: 30.0,
cpu_pct: 30,
..calm()
};
assert_eq!(
evaluate(&me, one, &cap(), &level, 0),
Decision::Start,
"load1 <= load5 is not rising"
);
let jitter = Live {
load1: 31.5,
load5: 30.0,
cpu_pct: 30,
..calm()
};
assert_eq!(
evaluate(&me, one, &cap(), &jitter, 0),
Decision::Start,
"load1 = load5 x 1.05 is jitter, not rising"
);
let falling_busy = Live {
cpu_pct: 92,
..falling.clone()
};
assert!(waits(&evaluate(&me, one, &cap(), &falling_busy, 0)));
let ramping_up = Live {
load1: 15.0,
load5: 5.0,
cpu_pct: 30,
..calm()
};
assert_eq!(evaluate(&me, one, &cap(), &ramping_up, 0), Decision::Start);
let hard = Live {
load1: 41.0,
load5: 80.0,
cpu_pct: 55,
..calm()
};
assert!(waits(&evaluate(&me, one, &cap(), &hard, 0)));
}
#[test]
fn memory_pressure_blocks() {
let me = job("a", 1, 1, State::Queued, 0);
let one = std::slice::from_ref(&me);
let pressured = Live {
mem_pressure: true,
..calm()
};
assert!(matches!(
evaluate(&me, one, &cap(), &pressured, 0),
Decision::Wait {
block: Block::Mem,
live: true,
..
}
));
assert_eq!(evaluate(&me, one, &cap(), &calm(), 0), Decision::Start);
}
#[test]
fn eight_cold_builds_start_one_ramp_apart() {
let c = Capacity { cpu: 16, ..cap() };
let mut jobs: Vec<Job> = (0..8)
.map(|i| job(&format!("j{i}"), 16, 1, State::Queued, i))
.collect();
let mut starts = Vec::new();
let mut t = 100;
while t < 100 + 8 * RAMP_MS && starts.len() < 8 {
for i in 0..jobs.len() {
if jobs[i].state == State::Queued
&& evaluate(&jobs[i].clone(), &jobs, &c, &calm(), t) == Decision::Start
{
jobs[i].state = State::Running;
jobs[i].started_ms = Some(t);
starts.push(t);
}
}
t += 100;
}
assert_eq!(starts.len(), 8);
assert!(
starts.windows(2).all(|w| w[1] - w[0] >= RAMP_MS),
"one cold build per ramp: {starts:?}"
);
let busy = Live {
cpu_pct: 40,
non_heavy: Usage {
cpu: 640,
mem: 0,
vram: 0,
},
..calm()
};
let mut jobs: Vec<Job> = (0..3)
.map(|i| job(&format!("k{i}"), 16, 1, State::Queued, i))
.collect();
let mut starts = Vec::new();
let mut t = 100;
while t < 100 + 4 * RAMP_MS && starts.len() < 3 {
for i in 0..jobs.len() {
if jobs[i].state == State::Queued
&& evaluate(&jobs[i].clone(), &jobs, &c, &busy, t) == Decision::Start
{
jobs[i].state = State::Running;
jobs[i].started_ms = Some(t);
starts.push(t);
}
}
t += 100;
}
assert_eq!(starts.len(), 3);
assert!(
starts.windows(2).all(|w| w[1] - w[0] >= RAMP_MS),
"busy: one cold build per ramp: {starts:?}"
);
let mut now = jobs.clone();
for j in &mut now {
j.state = State::Queued;
j.started_ms = None;
}
let first = evaluate(&now[0], &now, &c, &calm(), 5000);
assert_eq!(first, Decision::Start);
}
#[test]
fn disk_guard() {
let mut me = job("a", 1, 1, State::Queued, 0);
me.weights.disk = 500 * G;
let low = Live {
disk_free: 550 * G,
..calm()
};
assert!(
matches!(evaluate(&me, std::slice::from_ref(&me), &cap(), &low, 0), Decision::Fail(m) if m.contains("disk is full"))
);
let ok = Live {
disk_free: 600 * G,
..calm()
};
assert_eq!(
evaluate(&me, std::slice::from_ref(&me), &cap(), &ok, 0),
Decision::Start
);
}
#[test]
fn fifo_head_blocks_non_fitting_follower() {
let run = job("r", 8, 50, State::Running, 0);
let head = job("h", 5, 30, State::Queued, 1);
let big = job("z", 6, 40, State::Queued, 2);
assert!(waits(&evaluate(
&big,
&[run, head, big.clone()],
&cap(),
&calm(),
3
)));
}
#[test]
fn backfill_only_while_head_is_young_and_follower_smaller() {
let run = job("r", 8, 70, State::Running, 0);
let head = job("h", 5, 30, State::Queued, 1);
let small = Job {
run_p90_ms: Some(30_000),
..job("s", 1, 5, State::Queued, 2)
};
let all = [run, head, small.clone()];
assert_eq!(
evaluate(&small, &all, &cap(), &calm(), 1000),
Decision::Start
);
assert!(waits(&evaluate(
&small,
&all,
&cap(),
&calm(),
BACKFILL_WINDOW_MS + 1
)));
}
#[test]
fn follower_starts_when_it_fits_together_with_the_head() {
let head = job("h", 5, 10, State::Queued, 1);
let small = job("s", 1, 5, State::Queued, 2);
let all = [head.clone(), small.clone()];
assert_eq!(evaluate(&small, &all, &cap(), &calm(), 3), Decision::Start);
assert_eq!(evaluate(&head, &all, &cap(), &calm(), 3), Decision::Start);
}
#[test]
fn follower_waits_when_it_would_stop_the_head_fitting() {
let head = job("h", 5, 50, State::Queued, 1);
let other = job("o", 5, 50, State::Queued, 2);
let all = [head.clone(), other.clone()];
assert!(waits(&evaluate(&other, &all, &cap(), &calm(), 3)));
assert_eq!(evaluate(&head, &all, &cap(), &calm(), 3), Decision::Start);
let run = job("r", 3, 30, State::Running, 0);
let follow = job("f", 2, 20, State::Queued, 3);
let all = [run, head.clone(), follow.clone()];
assert!(waits(&evaluate(&follow, &all, &cap(), &calm(), 4)));
}
#[test]
fn a_ramp_blocker_names_the_moment_it_ends() {
let c = Capacity { cpu: 16, ..cap() };
let live = Live {
non_heavy: Usage {
cpu: 320,
mem: 8 * G,
vram: 0,
},
..calm()
};
let ra = Job {
state: State::Running,
started_ms: Some(10),
..job("a", 16, 8, State::Running, 0)
};
let b = job("b", 16, 8, State::Queued, 2);
let d = evaluate(&b, &[ra.clone(), b.clone()], &c, &live, 500);
assert!(
matches!(d, Decision::Wait { block: Block::Ramp, ramp_end_ms: Some(e), live: true, .. } if e == 10 + RAMP_MS),
"{d:?}"
);
let warm = Live {
cpu_pct: 60,
..live.clone()
};
let d = evaluate(&b, &[ra, b.clone()], &c, &warm, 500);
assert!(
matches!(d, Decision::Wait { block: Block::Ramp, ramp_end_ms: Some(e), .. } if e == 10 + RAMP_MS),
"{d:?}"
);
let live = Live {
non_heavy: Usage {
cpu: 800,
mem: 8 * G,
vram: 0,
},
..calm()
};
let warm = Live {
cpu_pct: 60,
..live.clone()
};
let rl = Job {
state: State::Running,
started_ms: Some(10),
..job("l", 8, 8, State::Running, 0)
};
let b = job("b", 4, 8, State::Queued, 2);
let d = evaluate(&b, &[rl.clone(), b.clone()], &c, &live, 500);
assert!(
matches!(d, Decision::Wait { block: Block::Ramp, ramp_end_ms: Some(e), .. } if e == 10 + RAMP_FULL_MS),
"{d:?}"
);
let d = evaluate(&b, &[rl.clone(), b.clone()], &c, &warm, 500);
assert!(
matches!(
d,
Decision::Wait {
block: Block::Ramp,
ramp_end_ms: Some(4750),
..
}
),
"{d:?}"
);
assert_eq!(
evaluate(&b, &[rl, b.clone()], &c, &warm, 4750),
Decision::Start
);
let busy = Live {
non_heavy: Usage {
cpu: 1550,
mem: 8 * G,
vram: 0,
},
..calm()
};
let run = Job {
state: State::Running,
started_ms: Some(0),
..job("r", 1, 8, State::Running, 0)
};
let d = evaluate(&b, &[run, b.clone()], &c, &busy, 100_000);
assert!(
matches!(
d,
Decision::Wait {
block: Block::Cpu,
ramp_end_ms: None,
..
}
),
"{d:?}"
);
}
#[test]
fn peak_core_waiters_start_on_a_mostly_idle_machine() {
let c = Capacity { cpu: 16, ..cap() };
let live = Live {
non_heavy: Usage {
cpu: 320,
mem: 8 * G,
vram: 0,
},
..calm()
};
let a = job("a", 16, 8, State::Queued, 1);
let b = job("b", 16, 8, State::Queued, 2);
assert_eq!(
evaluate(&a, &[a.clone(), b.clone()], &c, &live, 10),
Decision::Start
);
let ra = Job {
state: State::Running,
started_ms: Some(10),
..a
};
assert!(
waits(&evaluate(&b, &[ra.clone(), b.clone()], &c, &live, 11)),
"blocked by a's ramp hold"
);
assert!(
waits(&evaluate(
&b,
&[ra.clone(), b.clone()],
&c,
&live,
10 + RAMP_FULL_MS
)),
"a whole-machine hold is not released early"
);
assert_eq!(
evaluate(&b, &[ra.clone(), b.clone()], &c, &live, 10 + RAMP_MS),
Decision::Start,
"the second starts once the first one's ramp has ended"
);
let ca = job("c", 16, 8, State::Queued, 3);
let busy = Live {
non_heavy: Usage {
cpu: 1550,
mem: 8 * G,
vram: 0,
},
..calm()
};
let run = Job {
state: State::Running,
started_ms: Some(0),
..job("r", 16, 8, State::Running, 0)
};
assert!(waits(&evaluate(
&ca,
&[run, ca.clone()],
&c,
&busy,
100_000
)));
}
#[test]
fn burst_is_held_back_by_ramp_holds() {
let c = Capacity { cpu: 16, ..cap() };
let mut jobs: Vec<Job> = (0..8)
.map(|i| job(&format!("j{i}"), 4, 1, State::Queued, i))
.collect();
let mut started = 0;
for i in 0..jobs.len() {
let me = jobs[i].clone();
if evaluate(&me, &jobs, &c, &calm(), 100) == Decision::Start {
jobs[i].state = State::Running;
jobs[i].started_ms = Some(100);
started += 1;
}
}
assert_eq!(
started, 4,
"holds of 400 each leave 320 for the 4th and none for the 5th: not a burst of 8"
);
let next = jobs
.iter()
.find(|j| j.state == State::Queued)
.unwrap()
.clone();
assert_eq!(
evaluate(&next, &jobs, &c, &calm(), 100 + RAMP_MS),
Decision::Start
);
let ramped = Job {
state: State::Running,
started_ms: Some(0),
..job("r", 16, 1, State::Running, 0)
};
let room = headroom(&[&ramped], &c, &calm(), RAMP_MS + 1);
assert_eq!(room.cpu, 1520);
}
#[test]
fn stopped_waiter_file_does_not_block_the_next_waiter() {
let run = job("r", 1, 50, State::Running, 0);
let head = job("h", 1, 30, State::Queued, 1);
let next = job("n", 1, 40, State::Queued, 2);
let t = 3 + BACKFILL_WINDOW_MS;
assert!(waits(&evaluate(
&next,
&[run.clone(), head.clone(), next.clone()],
&cap(),
&calm(),
t
)));
let stopped = Job {
state: State::Stopped,
..head
};
assert_eq!(
evaluate(&next, &[run, stopped, next.clone()], &cap(), &calm(), t),
Decision::Start
);
}
#[test]
fn stopped_head_does_not_block_the_next_waiter() {
let run = job("r", 1, 50, State::Running, 0);
let head = job("h", 1, 30, State::Queued, 1);
let next = job("n", 1, 40, State::Queued, 2);
let all = [run, head, next.clone()];
assert!(
waits(&evaluate(
&next,
&all,
&cap(),
&calm(),
3 + BACKFILL_WINDOW_MS
)),
"a live head blocks the bigger follower"
);
let live = Live {
stopped: vec!["h".into()],
..calm()
};
assert_eq!(
evaluate(&next, &all, &cap(), &live, 3 + BACKFILL_WINDOW_MS),
Decision::Start
);
}
fn mac() -> Capacity {
Capacity {
cpu: 16,
mem: 64 * G,
vram: 0,
disk_total: 1000 * G,
vram_is_ram: true,
}
}
#[test]
fn a_lagging_five_minute_load_never_blocks_an_idle_machine() {
let me = job("a", 4, 4, State::Queued, 0);
let live = Live {
load1: 22.6,
load5: 34.2,
cpu_pct: 12,
mem_used_pct: 46,
non_heavy: Usage {
cpu: 192,
mem: 29 * G,
vram: 0,
},
..calm()
};
assert_eq!(
evaluate(&me, std::slice::from_ref(&me), &mac(), &live, 0),
Decision::Start
);
let run = Job {
started_ms: Some(0),
..job("r", 4, 4, State::Running, 0)
};
let live = Live {
load1: 21.3,
load5: 32.9,
cpu_pct: 16,
per_job: vec![(
"r".into(),
Usage {
cpu: 100,
mem: 2 * G,
vram: 0,
},
)],
non_heavy: Usage {
cpu: 156,
mem: 24 * G,
vram: 0,
},
..live
};
assert_eq!(
evaluate(&me, &[run, me.clone()], &mac(), &live, 100_000),
Decision::Start
);
}
#[test]
fn a_small_job_starts_at_seventy_percent_cpu() {
let run = Job {
started_ms: Some(0),
..job("r", 8, 4, State::Running, 0)
};
let me = job("a", 4, 4, State::Queued, 1);
let live = Live {
load1: 26.0,
load5: 31.0,
cpu_pct: 72,
per_job: vec![(
"r".into(),
Usage {
cpu: 800,
mem: 4 * G,
vram: 0,
},
)],
non_heavy: Usage {
cpu: 352,
mem: 22 * G,
vram: 0,
},
..calm()
};
assert_eq!(
evaluate(&me, &[run.clone(), me.clone()], &mac(), &live, 100_000),
Decision::Start
);
let full = Live {
per_job: vec![(
"r".into(),
Usage {
cpu: 1100,
mem: 4 * G,
vram: 0,
},
)],
non_heavy: Usage {
cpu: 420,
mem: 22 * G,
vram: 0,
},
cpu_pct: 95,
..live
};
assert!(matches!(
evaluate(&me, &[run, me.clone()], &mac(), &full, 100_000),
Decision::Wait {
block: Block::Cpu,
..
}
));
}
#[test]
fn ramp_hold_is_released_while_measured_cpu_is_low() {
let run = Job {
started_ms: Some(0),
..job("r", 8, 4, State::Running, 0)
};
let me = job("a", 4, 4, State::Queued, 1);
let all = [run.clone(), me.clone()];
let live = Live {
load1: 29.4,
load5: 23.7,
cpu_pct: 20,
mem_used_pct: 38,
per_job: vec![(
"r".into(),
Usage {
cpu: 150,
mem: 2 * G,
vram: 0,
},
)],
non_heavy: Usage {
cpu: 800,
mem: 22 * G,
vram: 0,
},
..calm()
};
let d = evaluate(&me, &all, &mac(), &live, 1_000);
assert!(
matches!(
d,
Decision::Wait {
block: Block::Ramp,
ramp_end_ms: Some(RAMP_FULL_MS),
..
}
),
"{d:?}"
);
assert_eq!(evaluate(&me, &all, &mac(), &live, 4_000), Decision::Start);
assert_eq!(
evaluate(&me, &all, &mac(), &live, RAMP_FULL_MS),
Decision::Start
);
let cold = Job {
started_ms: Some(0),
..job("r", 16, 4, State::Running, 0)
};
let all_cold = [cold, me.clone()];
let d = evaluate(&me, &all_cold, &mac(), &live, 4_000);
assert!(
matches!(
d,
Decision::Wait {
block: Block::Ramp,
ramp_end_ms: Some(RAMP_MS),
..
}
),
"{d:?}"
);
assert!(waits(&evaluate(&me, &all_cold, &mac(), &live, RAMP_MS - 1)));
assert_eq!(
evaluate(&me, &all_cold, &mac(), &live, RAMP_MS),
Decision::Start
);
}
#[test]
fn ramp_hold_shrinks_for_a_recent_job_with_a_learned_peak() {
let me = job("a", 4, 4, State::Queued, 1);
let mut all: Vec<Job> = (0..3)
.map(|i| Job {
started_ms: Some(0),
..job(&format!("d{i}"), 4, 2, State::Running, 0)
})
.collect();
all.push(Job {
started_ms: Some(95_000),
..job("new", 8, 2, State::Running, 1)
});
all.push(me.clone());
let live = Live {
load1: 15.5,
load5: 14.3,
cpu_pct: 24,
mem_used_pct: 46,
per_job: vec![
(
"d0".into(),
Usage {
cpu: 80,
mem: G,
vram: 0,
},
),
(
"d1".into(),
Usage {
cpu: 60,
mem: G,
vram: 0,
},
),
(
"d2".into(),
Usage {
cpu: 40,
mem: G,
vram: 0,
},
),
(
"new".into(),
Usage {
cpu: 50,
mem: G,
vram: 0,
},
),
],
non_heavy: Usage {
cpu: 150,
mem: 25 * G,
vram: 0,
},
..calm()
};
assert_eq!(evaluate(&me, &all, &mac(), &live, 100_000), Decision::Start);
let busy = Live {
cpu_pct: 70,
non_heavy: Usage {
cpu: 1180 - 230,
mem: 25 * G,
vram: 0,
},
..live
};
assert!(matches!(
evaluate(&me, &all, &mac(), &busy, 100_000),
Decision::Wait {
block: Block::Ramp,
..
}
));
}
#[test]
fn big_learned_peaks_and_docker_keep_the_full_hold() {
let c = Capacity { cpu: 16, ..cap() };
let idle = Live {
non_heavy: Usage {
cpu: 100,
mem: 8 * G,
vram: 0,
},
..calm()
};
let b = job("b", 15, 8, State::Queued, 2);
let ra = Job {
state: State::Running,
started_ms: Some(10),
..job("a", 15, 8, State::Running, 0)
};
assert!(
waits(&evaluate(
&b,
&[ra.clone(), b.clone()],
&c,
&idle,
10 + RAMP_FULL_MS
)),
"no early release at 94%"
);
assert!(waits(&evaluate(
&b,
&[ra.clone(), b.clone()],
&c,
&idle,
10 + RAMP_MS - 1
)));
assert_eq!(
evaluate(&b, &[ra, b.clone()], &c, &idle, 10 + RAMP_MS),
Decision::Start
);
let busy_other = Live {
non_heavy: Usage {
cpu: 800,
mem: 8 * G,
vram: 0,
},
..calm()
};
let rh = Job {
state: State::Running,
started_ms: Some(10),
..job("h", 8, 8, State::Running, 0)
};
assert!(waits(&evaluate(
&b,
&[rh.clone(), b.clone()],
&c,
&busy_other,
11
)));
assert_eq!(
evaluate(&b, &[rh, b.clone()], &c, &busy_other, 10 + RAMP_FULL_MS),
Decision::Start
);
let rd = Job {
state: State::Running,
started_ms: Some(10),
cmd: vec!["docker".into(), "build".into()],
..job("d", 8, 8, State::Running, 0)
};
assert!(waits(&evaluate(
&b,
&[rd.clone(), b.clone()],
&c,
&busy_other,
10 + RAMP_FULL_MS
)));
assert_eq!(
evaluate(&b, &[rd, b.clone()], &c, &busy_other, 10 + RAMP_MS),
Decision::Start
);
assert_eq!(ramp_hold(1200, 1600, RAMP_FULL_MS, 0, false), 1200);
assert_eq!(ramp_hold(1100, 1600, RAMP_FULL_MS, 0, false), 0);
assert_eq!(ramp_hold(800, 1600, RAMP_FULL_MS, 0, true), 800);
}
#[test]
fn ramp_hold_schedule() {
let (peak, machine) = (800, 1600);
assert_eq!(
ramp_hold(peak, machine, 0, 0, false),
peak,
"full at admission"
);
assert_eq!(
ramp_hold(peak, machine, RAMP_FULL_MS - 1, 0, false),
peak,
"full through the floor, however idle"
);
assert_eq!(
ramp_hold(peak, machine, RAMP_FULL_MS, 49, false),
0,
"released on an idle machine"
);
assert_eq!(
ramp_hold(peak, machine, RAMP_FULL_MS, 50, false),
peak,
"shrinks from the full peak on a busy one"
);
assert_eq!(
ramp_hold(peak, machine, 6_500, 80, false),
400,
"halfway down at the middle of the shrink"
);
assert_eq!(
ramp_hold(peak, machine, RAMP_MS, 100, false),
0,
"gone when the ramp ends"
);
assert_eq!(ramp_hold(machine, machine, RAMP_FULL_MS, 0, false), machine);
assert_eq!(ramp_hold(machine, machine, RAMP_MS - 1, 0, false), machine);
assert_eq!(ramp_hold(machine, machine, RAMP_MS, 0, false), 0);
}
#[test]
fn rising_load_with_idle_cpu_is_io_wait_not_overload() {
let me = job("a", 4, 4, State::Queued, 0);
let one = std::slice::from_ref(&me);
for (load1, load5, cpu_pct) in [(57.85, 24.18, 26), (53.44, 24.32, 28), (46.16, 25.83, 16)]
{
let live = Live {
load1,
load5,
cpu_pct,
mem_used_pct: 36,
non_heavy: Usage {
cpu: cpu_pct * 16,
mem: 23 * G,
vram: 0,
},
..calm()
};
assert_eq!(
evaluate(&me, one, &mac(), &live, 0),
Decision::Start,
"load1 {load1} cpu {cpu_pct}%"
);
let busy = Live {
cpu_pct: 80,
..live.clone()
};
assert!(
matches!(
evaluate(&me, one, &mac(), &busy, 0),
Decision::Wait {
block: Block::Load1,
..
}
),
"load1 {load1} busy"
);
}
let hard = Live {
load1: 64.0,
load5: 24.0,
cpu_pct: 16,
..calm()
};
assert!(matches!(
evaluate(&me, one, &mac(), &hard, 0),
Decision::Wait {
block: Block::Load1,
..
}
));
let hard = Live {
cpu_pressure: Some(0),
..hard
};
assert_eq!(evaluate(&me, one, &mac(), &hard, 0), Decision::Start);
let hard = Live {
cpu_pressure: Some(30),
..hard
};
assert!(matches!(
evaluate(&me, one, &mac(), &hard, 0),
Decision::Wait {
block: Block::Load1,
..
}
));
let pressured = Live {
load1: 50.0,
load5: 24.0,
cpu_pct: 16,
mem_pressure: true,
..calm()
};
assert!(matches!(
evaluate(&me, one, &mac(), &pressured, 0),
Decision::Wait {
block: Block::Mem,
..
}
));
}
#[test]
fn nothing_starts_while_the_cpu_reads_pinned() {
let me = job("a", 1, 0, State::Queued, 0);
let fast = Job {
fast: true,
..me.clone()
};
let dip = Live {
cpu_pct: 40,
cpu_sustained_pct: 70,
cpu_recent_peak_pct: CPU_PINNED_PCT,
..calm()
};
for j in [&me, &fast] {
assert!(matches!(
evaluate(j, std::slice::from_ref(j), &mac(), &dip, 0),
Decision::Wait {
block: Block::Cpu,
..
}
));
}
let calm_again = Live {
cpu_recent_peak_pct: CPU_PINNED_PCT - 1,
..dip
};
assert_eq!(
evaluate(&me, std::slice::from_ref(&me), &mac(), &calm_again, 0),
Decision::Start
);
assert_eq!(
recent_peak_pct(&[(0, 100), (5_000, 60), (12_000, 50)], 12_000),
60,
"only the last 10 s count"
);
assert_eq!(recent_peak_pct(&[], 0), 0);
}
#[test]
fn a_bursty_job_keeps_its_sustained_level_after_its_hold_is_released() {
assert_eq!(sustained_floor(0, 1_000, Some(400), false), 400);
assert_eq!(sustained_floor(900, 1_000, Some(400), false), 900);
assert_eq!(
sustained_floor(0, 300, Some(400), false),
300,
"never over the peak"
);
assert_eq!(
sustained_floor(0, 1_000, Some(400), true),
0,
"a VM job shows as other programs"
);
assert_eq!(sustained_floor(0, 1_000, None, false), 0);
let c = mac();
let run = Job {
started_ms: Some(0),
cpu_hold_ms: Some(RAMP_MS),
cpu_sustained: Some(1_100),
..job("bursty", 11, 2, State::Running, 0)
};
let dip = Live {
cpu_pct: 10,
cpu_sustained_pct: 10,
per_job: vec![(
"bursty".into(),
Usage {
cpu: 100,
mem: 2 * G,
vram: 0,
},
)],
..calm()
};
let now = 10 * 60_000;
let free = headroom(&[&run], &c, &dip, now).cpu;
assert!(free <= ceiling(&c).cpu - 1_100, "free {free}");
let next = job("next", 12, 2, State::Queued, now);
let jobs = [run, next.clone()];
assert!(waits(&evaluate(&next, &jobs, &c, &dip, now)));
}
#[test]
fn a_saturated_machine_still_blocks() {
let me = job("a", 4, 4, State::Queued, 0);
let live = Live {
load1: 55.3,
load5: 39.6,
cpu_pct: 100,
..calm()
};
assert!(matches!(
evaluate(&me, std::slice::from_ref(&me), &mac(), &live, 0),
Decision::Wait {
block: Block::Cpu,
..
}
));
let loaded = Live {
cpu_pct: 90,
..live
};
assert!(matches!(
evaluate(&me, std::slice::from_ref(&me), &mac(), &loaded, 0),
Decision::Wait {
block: Block::Load1,
..
}
));
let storm = Live {
load1: 120.0,
cpu_pct: 5,
..calm()
};
assert!(matches!(
evaluate(&me, std::slice::from_ref(&me), &mac(), &storm, 0),
Decision::Wait {
block: Block::Load1,
..
}
));
}
#[test]
fn clamp_to_machine_and_fold_vram() {
let c = cap();
let w = c.clamp(Weights {
cpu: 64,
mem: 999 * G,
vram: 5 * G,
disk: 9999 * G,
});
assert_eq!(
w,
Weights {
cpu: 10,
mem: 100 * G,
vram: 0,
disk: 1000 * G
}
);
let mac = Capacity {
vram_is_ram: true,
..c
};
assert_eq!(
mac.clamp(Weights {
cpu: 1,
mem: 4 * G,
vram: 2 * G,
disk: 0
})
.mem,
6 * G
);
}
fn pinned(jobs: u64, non_heavy: u64) -> (Vec<Job>, Live) {
let a = Job {
started_ms: Some(0),
..job("a", 8, 2, State::Running, 0)
};
let b = Job {
started_ms: Some(0),
..job("b", 8, 2, State::Running, 0)
};
let live = Live {
cpu_pct: (jobs + non_heavy) / 16,
per_job: vec![
(
"a".into(),
Usage {
cpu: jobs / 2,
mem: 2 * G,
vram: 0,
},
),
(
"b".into(),
Usage {
cpu: jobs / 2,
mem: 2 * G,
vram: 0,
},
),
],
non_heavy: Usage {
cpu: non_heavy,
mem: 20 * G,
vram: 0,
},
..calm()
};
(vec![a, b], live)
}
#[test]
fn a_dip_between_build_steps_does_not_admit_onto_a_pinned_machine() {
let (mut jobs, dip) = pinned(1_100, 180);
let me = job("c", 4, 2, State::Queued, 50_000);
jobs.push(me.clone());
assert_eq!(dip.cpu_pct, 80);
let readings = [(40_000, 100), (45_000, 100), (52_000, 98), (60_000, 80)];
let live = Live {
cpu_sustained_pct: sustained_pct(dip.cpu_pct, &readings, 60_000),
..dip.clone()
};
assert_eq!(live.cpu_sustained_pct, 94);
let d = evaluate(&me, &jobs, &mac(), &live, 60_000);
assert!(
matches!(
d,
Decision::Wait {
block: Block::Cpu,
ramp_end_ms: None,
..
}
),
"{d:?}"
);
let settled = [(45_000, 81), (52_000, 79), (60_000, 80)];
let live = Live {
cpu_sustained_pct: sustained_pct(80, &settled, 60_000),
..dip
};
assert_eq!(evaluate(&me, &jobs, &mac(), &live, 60_000), Decision::Start);
}
#[test]
fn jobs_start_up_to_the_band_and_wait_inside_it() {
let me = job("c", 4, 2, State::Queued, 1);
for (busy, starts) in [
(60, true),
(75, true),
(83, true),
(88, true),
(89, false),
(93, false),
(100, false),
] {
let (mut jobs, live) = pinned(busy * 16 - 160, 160);
jobs.push(me.clone());
let live = Live {
cpu_sustained_pct: busy,
..live
};
assert_eq!(
evaluate(&me, &jobs, &mac(), &live, 100_000) == Decision::Start,
starts,
"{busy}% busy"
);
}
}
#[test]
fn sustained_cpu_rises_at_once_and_falls_slowly() {
assert_eq!(sustained_pct(70, &[], 0), 70, "no readings: the latest");
assert_eq!(
sustained_pct(10, &[(0, 10), (90_000, 100)], 5_000),
10,
"a reading from the future (clock skew) is dropped"
);
assert_eq!(
sustained_pct(100, &[(0, 40), (5_000, 40)], 5_000),
100,
"a rise counts at once"
);
assert_eq!(
sustained_pct(40, &[(0, 100), (10_000, 100), (20_000, 40)], 20_000),
80,
"a fall is averaged"
);
assert_eq!(
sustained_pct(40, &[(0, 100), (25_000, 40)], 25_000),
40,
"readings over 20 s old are dropped"
);
}
#[test]
fn a_rising_load_in_the_underused_zone_admits() {
let me = job("c", 4, 2, State::Queued, 1);
let (mut jobs, live) = pinned(1_000, 120);
jobs.push(me.clone());
let rising = Live {
load1: 40.0,
load5: 30.0,
cpu_sustained_pct: 70,
..live
};
assert_eq!(rising.cpu_pct, 70);
assert_eq!(
evaluate(&me, &jobs, &mac(), &rising, 100_000),
Decision::Start
);
let busy = Live {
cpu_sustained_pct: 80,
..rising
};
assert!(matches!(
evaluate(&me, &jobs, &mac(), &busy, 100_000),
Decision::Wait {
block: Block::Load1,
..
}
));
}
#[test]
fn thread_inflated_memory_reservations_no_longer_starve_an_idle_rig() {
let rig = Capacity {
cpu: 32,
mem: 64 * G,
vram: 0,
disk_total: 1000 * G,
vram_is_ram: false,
};
let live = Live {
cpu_pct: 21,
cpu_sustained_pct: 21,
mem_used_pct: 6,
..calm()
};
let me = job("w", 4, 3, State::Queued, 1);
let inflated = |id: &str| Job {
started_ms: Some(0),
..job(id, 32, 46, State::Running, 0)
};
let d = evaluate(
&me,
&[inflated("r"), inflated("s"), me.clone()],
&rig,
&live,
100_000,
);
assert_eq!(d, Decision::Start, "{d:?}");
let real = |id: &str| Job {
started_ms: Some(0),
..job(id, 4, 3, State::Running, 0)
};
assert_eq!(
evaluate(
&me,
&[real("r"), real("s"), me.clone()],
&rig,
&live,
100_000
),
Decision::Start
);
}
fn burst_of_six(cpu_hold: Option<i64>, ttp: i64, run: i64) -> (Vec<i64>, usize) {
let c = mac();
let mut jobs: Vec<Job> = (0..6)
.map(|i| Job {
cpu_hold_ms: cpu_hold,
..job(&format!("b{i}"), 16, 2, State::Queued, i)
})
.collect();
let (mut starts, mut most_ramping) = (Vec::new(), 0);
let mut t = 0;
while t <= 700_000 && starts.len() < 6 {
jobs.retain(|j| j.started_ms.is_none_or(|s| t - s < run));
let demand: Vec<(String, u64)> = jobs
.iter()
.filter_map(|j| {
j.started_ms
.map(|s| (j.id.clone(), if t - s < ttp { 200 } else { 1600 }))
})
.collect();
let total: u64 = demand.iter().map(|d| d.1).sum();
let pct = total.min(1600) * 100 / 1600;
let live = Live {
cpu_pct: pct,
cpu_sustained_pct: pct,
per_job: demand
.iter()
.map(|(id, d)| {
let cpu = if total > 1600 { d * 1600 / total } else { *d };
(
id.clone(),
Usage {
cpu,
mem: 2 * G,
vram: 0,
},
)
})
.collect(),
..calm()
};
for i in 0..jobs.len() {
if jobs[i].state == State::Queued
&& evaluate(&jobs[i].clone(), &jobs, &c, &live, t) == Decision::Start
{
jobs[i].state = State::Running;
jobs[i].started_ms = Some(t);
starts.push(t);
}
}
let ramping = jobs
.iter()
.filter(|j| j.started_ms.is_some_and(|s| t - s < ttp))
.count();
most_ramping = most_ramping.max(ramping);
t += 500;
}
(starts, most_ramping)
}
#[test]
fn a_burst_of_six_16_core_cargo_builds_starts_one_at_a_time_as_their_peaks_show() {
let (old, old_ramping) = burst_of_six(None, 30_000, 90_000);
assert!(
old.iter().filter(|t| **t < 30_000).count() >= 3 && old_ramping >= 3,
"the 0.4.2 schedule admits a burst: {old:?}"
);
let (starts, ramping) = burst_of_six(Some(cpu_hold_ms(Some(30_000))), 30_000, 90_000);
assert_eq!(ramping, 1, "never two builds ramping at once: {starts:?}");
assert_eq!(
starts.len(),
6,
"all six still run (no starvation): {starts:?}"
);
assert!(
starts.windows(2).all(|w| w[1] - w[0] >= 30_000),
"each start waits for the previous peak: {starts:?}"
);
let (starts, ramping) = burst_of_six(Some(cpu_hold_ms(None)), 20_000, 60_000);
assert_eq!((ramping, starts.len()), (1, 6), "{starts:?}");
let (starts, ramping) = burst_of_six(Some(CPU_FALLBACK_HOLD_MS), 25_000, 60_000);
assert_eq!((ramping, starts.len()), (1, 6), "{starts:?}");
assert!(
starts.windows(2).all(|w| w[1] - w[0] >= 25_000),
"cold jobs wait for each other's peak: {starts:?}"
);
}
#[test]
fn a_hold_from_the_slowest_ramp_outlasts_a_median_length_ramp() {
let median_hold = 30_000;
let hold = cpu_hold_ms(Some(40_000));
assert!(hold > 40_000, "{hold}");
let (peak, machine) = (1_600, 1_600);
let still_ramping = |at, h| cpu_hold(peak, machine, at, 10, false, 200, Some(h), false);
assert_eq!(still_ramping(38_000, median_hold), 0);
assert_eq!(still_ramping(38_000, hold), peak);
assert_eq!(still_ramping(hold, hold), 0, "still bounded");
}
#[test]
fn a_learned_cpu_peak_is_held_until_it_shows_or_its_time_passes() {
let (peak, machine) = (1_600, 1_600);
let hold = Some(cpu_hold_ms(Some(32_000)));
assert_eq!(hold, Some(40_000), "the slowest ramp plus 25%");
assert_eq!(
cpu_hold(peak, machine, 1_000, 10, false, 1_600, hold, false),
peak
);
assert_eq!(
cpu_hold(peak, machine, 20_000, 10, false, 200, hold, false),
peak
);
assert_eq!(
cpu_hold(peak, machine, 20_000, 90, false, 1_200, hold, false),
0
);
assert_eq!(
cpu_hold(peak, machine, 40_000, 10, false, 200, hold, false),
0
);
assert_eq!(
cpu_hold(peak, machine, 9_000, 10, true, 0, hold, false),
peak
);
assert_eq!(
cpu_hold(peak, machine, 39_999, 10, true, 1_600, hold, true),
peak
);
assert_eq!(cpu_hold(peak, machine, 40_000, 10, true, 0, hold, false), 0);
assert_eq!(
cpu_hold(peak, machine, RAMP_MS, 10, true, 0, None, false),
0
);
assert_eq!(
cpu_hold(peak, machine, RAMP_MS, 10, false, 0, None, false),
0
);
assert_eq!(cpu_hold_ms(None), CPU_FALLBACK_HOLD_MS);
assert_eq!(cpu_hold_ms(Some(500)), RAMP_MS);
assert_eq!(cpu_hold_ms(Some(3_600_000)), CPU_MAX_HOLD_MS);
}
#[test]
fn a_small_waiter_starts_under_a_held_16_core_peak_when_the_band_has_room() {
let c = mac();
let run = Job {
state: State::Running,
started_ms: Some(0),
cpu_hold_ms: Some(40_000),
..job("r", 16, 2, State::Running, 0)
};
let live = Live {
per_job: vec![(
"r".into(),
Usage {
cpu: 200,
mem: 2 * G,
vram: 0,
},
)],
cpu_pct: 13,
cpu_sustained_pct: 13,
..calm()
};
let one = job("one", 1, 1, State::Queued, 1);
assert_eq!(
evaluate(&one, &[run.clone(), one.clone()], &c, &live, 20_000),
Decision::Start,
"a 1-core waiter fits in the band minus the held reservation"
);
let big = job("big", 16, 1, State::Queued, 1);
assert!(waits(&evaluate(
&big,
&[run.clone(), big.clone()],
&c,
&live,
20_000
)));
let busy = Live {
cpu_pct: 90,
cpu_sustained_pct: 90,
..live
};
assert!(waits(&evaluate(
&one,
&[run, one.clone()],
&c,
&busy,
20_000
)));
}
fn six_small_waiters(live: &Live, run: &Job, now: i64) -> usize {
let c = mac();
let mut running = vec![run.clone()];
for i in 0..6 {
let id = format!("s{i}");
let w = job(&id, 2, 1, State::Queued, 1);
let mut jobs = running.clone();
jobs.push(w.clone());
if evaluate(&w, &jobs, &c, live, now) == Decision::Start {
running.push(Job {
state: State::Running,
started_ms: Some(now),
cpu_hold_ms: Some(40_000),
..w
});
}
}
running.len() - 1
}
fn held_big() -> (Job, Live) {
let run = Job {
state: State::Running,
started_ms: Some(0),
cpu_hold_ms: Some(40_000),
..job("r", 16, 2, State::Running, 0)
};
let live = Live {
per_job: vec![(
"r".into(),
Usage {
cpu: 200,
mem: 2 * G,
vram: 0,
},
)],
cpu_pct: 13,
cpu_sustained_pct: 13,
..calm()
};
(run, live)
}
#[test]
fn small_waiters_share_one_backfill_budget_under_a_held_peak() {
let (run, live) = held_big();
assert_eq!(six_small_waiters(&live, &run, 20_000), 1);
}
#[test]
fn the_rest_start_once_the_big_peak_shows_or_its_hold_expires() {
let (run, live) = held_big();
let shown = Live {
per_job: vec![(
"r".into(),
Usage {
cpu: 1_200,
mem: 2 * G,
vram: 0,
},
)],
cpu_pct: 75,
cpu_sustained_pct: 75,
..live.clone()
};
assert_eq!(six_small_waiters(&shown, &run, 20_000), 1);
assert_eq!(six_small_waiters(&live, &run, 41_000), 6);
}
#[test]
fn a_ramp_blocker_on_a_learned_hold_names_the_hold_end() {
let c = mac();
let run = Job {
state: State::Running,
started_ms: Some(0),
cpu_hold_ms: Some(40_000),
..job("r", 16, 2, State::Running, 0)
};
let me = job("w", 8, 2, State::Queued, 1);
let live = Live {
per_job: vec![(
"r".into(),
Usage {
cpu: 200,
mem: 2 * G,
vram: 0,
},
)],
cpu_pct: 13,
cpu_sustained_pct: 13,
..calm()
};
let d = evaluate(&me, &[run.clone(), me.clone()], &c, &live, 20_000);
assert!(
matches!(
d,
Decision::Wait {
block: Block::Ramp,
ramp_end_ms: Some(40_000),
..
}
),
"{d:?}"
);
assert_eq!(
evaluate(&me, &[run, me.clone()], &c, &live, 40_000),
Decision::Start
);
}
#[test]
fn sustained_overload_is_blocked_by_the_two_times_cores_rule() {
let c = mac();
let over = Live {
load1: 49.0,
load5: 60.0,
cpu_pct: 96,
cpu_sustained_pct: 97,
..calm()
};
let (why, block) = guard_block(&c, &over).unwrap();
assert_eq!(block, Block::Load1);
assert!(
why.contains("2 x 16 cores") && why.contains("cpu 97% busy"),
"{why}"
);
let me = job("a", 1, 1, State::Queued, 0);
assert!(waits(&evaluate(
&me,
std::slice::from_ref(&me),
&c,
&over,
0
)));
let io = Live {
cpu_pct: 30,
cpu_sustained_pct: 30,
..over
};
assert_eq!(guard_block(&c, &io), None);
}
#[test]
fn a_run_plateaus_once_its_cpu_is_flat_past_its_slowest_time_to_peak() {
let run = |f: &dyn Fn(u64) -> u64, until: u64| -> Vec<(u64, u64)> {
(0..=until / 500).map(|k| (k * 500, f(k * 500))).collect()
};
let flat = |_| 150;
assert!(!cpu_plateaued(&run(&flat, 19_500), 9_000));
assert!(cpu_plateaued(&run(&flat, 20_000), 9_000));
let rising = |ms| 100 + ms / 50;
assert!(!cpu_plateaued(&run(&rising, 60_000), 9_000));
let cargo = |ms| if ms < 30_000 { 200 } else { 1_600 };
assert!(!cpu_plateaued(&run(&cargo, 29_500), 30_000));
assert!(!cpu_plateaued(&run(&cargo, 35_000), 30_000));
assert!(!cpu_plateaued(&run(&cargo, 39_500), 30_000));
let late = |ms| if ms < 25_000 { 200 } else { 1_600 };
assert!(!cpu_plateaued(&run(&late, 25_500), 15_000));
let lull = |ms| {
if (1_000..40_000).contains(&ms) {
150
} else {
1_500
}
};
assert!(!cpu_plateaued(&run(&lull, 30_000), 40_000));
assert!(!cpu_plateaued(&run(&lull, 45_000), 40_000));
let again = |ms| if ms < 30_000 { 150 } else { 1_500 };
assert!(cpu_plateaued(&run(&again, 29_500), 5_000));
assert!(!cpu_plateaued(&run(&again, 33_000), 5_000));
let jitter = |ms| if (ms / 500) % 2 == 0 { 800 } else { 880 };
assert!(cpu_plateaued(&run(&jitter, 20_000), 0));
assert!(!cpu_plateaued(&[], 0));
}
#[test]
fn a_cold_head_that_waited_starts_beside_one_long_job_under_the_band() {
let c = mac();
let run = Job {
state: State::Running,
started_ms: Some(0),
cpu_hold_ms: Some(cpu_hold_ms(Some(4_000))),
..job("long", 10, 2, State::Running, 0)
};
let head = job("cold", 16, 2, State::Queued, 1_000);
let behind = Job {
run_p90_ms: Some(60_000),
..job("small", 4, 2, State::Queued, 2_000)
};
let live = Live {
per_job: vec![(
"long".into(),
Usage {
cpu: 1_000,
mem: 2 * G,
vram: 0,
},
)],
cpu_pct: 62,
cpu_sustained_pct: 62,
..calm()
};
let all = [run, head.clone(), behind.clone()];
let t0 = 1_000 + 10_000;
assert!(waits(&evaluate(&head, &all, &c, &live, t0)));
let t = 1_000 + BACKFILL_WINDOW_MS;
assert_eq!(evaluate(&head, &all, &c, &live, t), Decision::Start);
let hot = Live {
load1: 70.0,
load5: 70.0,
..live.clone()
};
assert!(waits(&evaluate(&head, &all, &c, &hot, t)));
assert_eq!(evaluate(&behind, &all, &c, &live, t0), Decision::Start);
}
fn low_cpu_hold(settled: bool) -> Decision {
let c = mac();
let run = Job {
state: State::Running,
started_ms: Some(0),
cpu_hold_ms: Some(CPU_MAX_HOLD_MS),
cpu_settled: settled,
..job("r", 15, 2, State::Running, 0)
};
let me = job("w", 8, 2, State::Queued, 1);
let live = Live {
per_job: vec![(
"r".into(),
Usage {
cpu: 150,
mem: 2 * G,
vram: 0,
},
)],
non_heavy: Usage {
cpu: 100,
..Usage::default()
},
cpu_pct: 15,
cpu_sustained_pct: 15,
..calm()
};
evaluate(&me, &[run, me.clone()], &c, &live, 30_000)
}
#[test]
fn a_plateaued_hold_is_released_while_the_machine_reads_low() {
assert!(
matches!(
low_cpu_hold(false),
Decision::Wait {
block: Block::Ramp,
..
}
),
"an unsettled hold still holds"
);
assert_eq!(
low_cpu_hold(true),
Decision::Start,
"settled: the job's real use counts"
);
assert_eq!(
cpu_hold(
1_500,
1_600,
5_000,
15,
true,
0,
Some(CPU_MAX_HOLD_MS),
true
),
1_500
);
assert_eq!(
cpu_hold(
1_500,
1_600,
1_000,
15,
false,
0,
Some(CPU_MAX_HOLD_MS),
true
),
1_500
);
}
fn onto_busy_mac(busy: u64, cpu: u32) -> Decision {
let c = mac();
let run = Job {
state: State::Running,
started_ms: Some(0),
cpu_hold_ms: Some(RAMP_MS),
..job("r", 13, 2, State::Running, 0)
};
let me = job("w", cpu, 2, State::Queued, 1);
let pct = busy * 100 / 1_600;
let live = Live {
per_job: vec![(
"r".into(),
Usage {
cpu: busy,
mem: 2 * G,
vram: 0,
},
)],
cpu_pct: pct,
cpu_sustained_pct: pct,
..calm()
};
evaluate(&me, &[run, me.clone()], &c, &live, 60_000)
}
#[test]
fn a_big_waiter_starts_only_where_two_fifths_of_its_peak_fit_on_the_projected_cpu() {
assert!(
matches!(
onto_busy_mac(1_100, 13),
Decision::Wait {
block: Block::Cpu,
..
}
),
"68% busy + 5.2 cores > 16"
);
assert_eq!(onto_busy_mac(1_080, 13), Decision::Start, "67% + 5.2 fits");
assert_eq!(
onto_busy_mac(900, 13),
Decision::Start,
"56% fits (blocked in 0.4.4 above 59%)"
);
assert_eq!(
onto_busy_mac(1_300, 2),
Decision::Start,
"a small waiter asks its own peak"
);
assert_eq!(onto_busy_mac(1_400, 4), Decision::Start);
assert!(waits(&onto_busy_mac(1_440, 4)));
assert!(waits(&onto_busy_mac(1_000, 16)));
assert_eq!(onto_busy_mac(960, 16), Decision::Start);
}
fn bursty_eight() -> (u64, u64, i64) {
let c = mac();
let mut jobs: Vec<Job> = (0..8)
.map(|i| Job {
cpu_hold_ms: Some(cpu_hold_ms(Some(4_000))),
..job(&format!("b{i}"), 13, 2, State::Queued, i)
})
.collect();
let demand_at = |since: i64| -> u64 {
if since < 3_000 {
100
} else if (since - 3_000) % 40_000 < 15_000 {
1_300
} else {
200
}
};
let (mut samples, mut over, mut under, mut last) = (0u64, 0u64, 0u64, 0);
let mut readings: Vec<(i64, u64)> = Vec::new();
let mut t = 0;
while t <= 1_200_000 && !jobs.is_empty() {
jobs.retain(|j| j.started_ms.is_none_or(|s| t - s < 150_000));
let demand: Vec<(String, u64)> = jobs
.iter()
.filter_map(|j| j.started_ms.map(|s| (j.id.clone(), demand_at(t - s))))
.collect();
let total: u64 = demand.iter().map(|d| d.1).sum();
let pct = total.min(1_600) * 100 / 1_600;
readings.push((t, pct));
readings.retain(|(ts, _)| t - ts <= BUSY_WINDOW_MS);
let live = Live {
cpu_pct: pct,
cpu_sustained_pct: sustained_pct(pct, &readings, t),
per_job: demand
.iter()
.map(|(id, d)| {
let cpu = if total > 1_600 { d * 1_600 / total } else { *d };
(
id.clone(),
Usage {
cpu,
mem: 2 * G,
vram: 0,
},
)
})
.collect(),
..calm()
};
let waiting = jobs.iter().any(|j| j.state == State::Queued);
samples += 1;
over += u64::from(total > 1_600);
under += u64::from(waiting && pct < 75);
for i in 0..jobs.len() {
if jobs[i].state == State::Queued
&& evaluate(&jobs[i].clone(), &jobs, &c, &live, t) == Decision::Start
{
jobs[i].state = State::Running;
jobs[i].started_ms = Some(t);
last = t;
}
}
t += 500;
}
assert!(jobs.is_empty(), "every job ran");
(over * 100 / samples, under * 100 / samples, last)
}
#[test]
fn bursty_big_jobs_start_one_at_a_time_without_pinning_the_machine() {
let (over, under, last) = bursty_eight();
assert!(over <= 10, "demand over the machine {over}% of the time");
assert!(
under <= 30,
"jobs waited at under 75% CPU {under}% of the time"
);
assert!(last <= 600_000, "the last job started at {last} ms");
}
fn stuck_queue() -> (Vec<Job>, Live) {
let mi = 1 << 20;
let jobs = vec![
Job {
weights: Weights {
cpu: 1,
mem: 148 * mi,
vram: 0,
disk: 0,
},
mem_peak_at_ms: Some(2_190),
cpu_hold_ms: Some(RAMP_MS),
..job("lint", 1, 0, State::Queued, 1_000)
},
Job {
mem_peak_at_ms: Some(106_250),
cpu_hold_ms: Some(CPU_MAX_HOLD_MS),
..job("ui-update-ios", 11, 7, State::Queued, 331_000)
},
job("xcodebuild", 8, 8, State::Queued, 1_111_000),
Job {
weights: Weights {
cpu: 1,
mem: 310 * mi,
vram: 0,
disk: 0,
},
..job("typecheck", 1, 0, State::Queued, 3_534_000)
},
];
let live = Live {
load1: 8.0,
load5: 8.0,
mem_used_pct: 49,
cpu_pct: 27,
cpu_sustained_pct: 27,
mem_pressure_warn: true,
non_heavy: Usage {
cpu: 430,
mem: 31 * G,
vram: 0,
},
..calm()
};
(jobs, live)
}
#[test]
fn replay_the_stuck_queue_of_2026_11_21() {
let c = mac();
let (mut jobs, live) = stuck_queue();
let now = 3_600_000;
assert_eq!(evaluate(&jobs[0], &jobs, &c, &live, now), Decision::Start);
let critical = Live {
mem_pressure: true,
..live.clone()
};
assert!(matches!(
evaluate(&jobs[0], &jobs, &c, &critical, now),
Decision::Wait {
block: Block::Mem,
..
}
));
let mut t = now;
while jobs.iter().any(|j| j.state == State::Queued) && t < now + 600_000 {
let order: Vec<usize> = (0..jobs.len()).collect();
for i in order {
if jobs[i].state == State::Queued
&& evaluate(&jobs[i].clone(), &jobs, &c, &live, t) == Decision::Start
{
jobs[i].state = State::Running;
jobs[i].started_ms = Some(t);
}
}
t += 500;
}
let started: Vec<i64> = jobs.iter().map(|j| j.started_ms.unwrap_or(-1)).collect();
assert!(
started.iter().all(|s| *s >= 0),
"every job started: {started:?}"
);
assert_eq!(started[0], now, "the head at once");
assert_eq!(started[3], now, "the tiny typecheck beside it");
assert!(
started[2] <= now + CPU_MAX_HOLD_MS,
"the xcodebuild once the 11-core hold ends: {started:?}"
);
}
#[test]
fn warn_pressure_blocks_only_with_memory_nearly_used_up() {
let me = job("a", 1, 1, State::Queued, 1);
let run = Job {
started_ms: Some(0),
..job("r", 1, 1, State::Running, 0)
};
let both = [run, me.clone()];
let warn = |mem_used_pct| Live {
mem_pressure_warn: true,
mem_used_pct,
..calm()
};
assert_eq!(
evaluate(&me, &both, &cap(), &warn(84), 100_000),
Decision::Start
);
assert!(matches!(
evaluate(&me, &both, &cap(), &warn(85), 100_000),
Decision::Wait {
block: Block::Mem,
..
}
));
let used = |mem_used_pct| Live {
mem_used_pct,
..calm()
};
assert_eq!(
evaluate(&me, &both, &cap(), &used(94), 100_000),
Decision::Start
);
assert!(waits(&evaluate(&me, &both, &cap(), &used(95), 100_000)));
}
#[test]
fn a_lone_job_is_held_by_the_memory_and_runaway_guards_but_not_by_warn_alone() {
let me = job("a", 16, 8, State::Queued, 0);
let one = std::slice::from_ref(&me);
let c = mac();
let warn = Live {
mem_used_pct: 49,
mem_pressure_warn: true,
cpu_pct: 27,
..calm()
};
assert_eq!(evaluate(&me, one, &c, &warn, 0), Decision::Start);
for warn_on in [false, true] {
let full = Live {
mem_used_pct: 97,
mem_pressure_warn: warn_on,
..calm()
};
assert!(matches!(
evaluate(&me, one, &c, &full, 0),
Decision::Wait {
block: Block::Mem,
..
}
));
}
let critical = Live {
mem_pressure: true,
..warn.clone()
};
assert!(waits(&evaluate(&me, one, &c, &critical, 0)));
let runaway = Live {
load1: 64.0,
load5: 30.0,
cpu_pct: 55,
..warn
};
assert!(matches!(
evaluate(&me, one, &c, &runaway, 0),
Decision::Wait {
block: Block::Load1,
..
}
));
}
#[test]
fn tiny_jobs_stop_passing_a_memory_blocked_head_after_the_bypass_window() {
let c = mac();
let run = Job {
started_ms: Some(0),
mem_declared: true,
..job("r", 2, 40, State::Running, 0)
};
let head = job("head", 4, 30, State::Queued, 1_000);
let tiny_one = job("tiny", 1, 1, State::Queued, 2_000);
let all = [run, head.clone(), tiny_one.clone()];
let live = Live {
non_heavy: Usage {
cpu: 200,
mem: 15 * G,
vram: 0,
},
..calm()
};
let before = 1_000 + TINY_BYPASS_MS - 1;
assert_eq!(
evaluate(&tiny_one, &all, &c, &live, before),
Decision::Start
);
let after = 1_000 + TINY_BYPASS_MS;
assert!(matches!(
evaluate(&tiny_one, &all, &c, &live, after),
Decision::Wait {
block: Block::Queue,
..
}
));
}
#[test]
fn a_tiny_job_passes_a_vram_blocked_head_until_the_bypass_window_ends() {
let c = Capacity {
vram: 100 * G,
..cap()
};
let vr = |id: &str, g: u64, state| {
let j = job(id, 1, 1, state, 0);
Job {
weights: Weights {
vram: g * G,
..j.weights
},
started_ms: Some(0),
..j
}
};
let run = vr("r", 50, State::Running);
let head = Job {
enqueued_ms: 1_000,
..vr("head", 41, State::Queued)
};
let tiny_one = Job {
enqueued_ms: 2_000,
..vr("tiny", 0, State::Queued)
};
let all = [run, head.clone(), tiny_one.clone()];
let t = 1_000 + BACKFILL_WINDOW_MS + 1;
assert!(matches!(
evaluate(&head, &all, &c, &calm(), t),
Decision::Wait {
block: Block::Vram,
..
}
));
assert_eq!(evaluate(&tiny_one, &all, &c, &calm(), t), Decision::Start);
assert!(waits(&evaluate(
&tiny_one,
&all,
&c,
&calm(),
1_000 + TINY_BYPASS_MS
)));
}
#[test]
fn a_tiny_job_passes_a_ramp_blocked_head_only_if_the_head_is_not_delayed() {
let c = mac();
let ra = Job {
started_ms: Some(10),
..job("a", 16, 8, State::Running, 0)
};
let head = job("b", 16, 8, State::Queued, 2);
let tiny_one = job("t", 1, 1, State::Queued, 3);
let all = [ra, head.clone(), tiny_one.clone()];
let live = Live {
non_heavy: Usage {
cpu: 320,
mem: 8 * G,
vram: 0,
},
..calm()
};
let d = evaluate(&head, &all, &c, &live, 500);
assert!(
matches!(
d,
Decision::Wait {
block: Block::Ramp,
..
}
),
"{d:?}"
);
assert_eq!(evaluate(&tiny_one, &all, &c, &live, 500), Decision::Start);
}
#[test]
fn a_tiny_job_passes_an_old_head_blocked_by_memory_but_not_by_cpu() {
let c = mac();
let run = Job {
started_ms: Some(0),
mem_declared: true,
..job("r", 2, 40, State::Running, 0)
};
let head = job("head", 4, 30, State::Queued, 1_000);
let tiny_one = job("tiny", 1, 1, State::Queued, 2_000);
let mid = job("mid", 4, 1, State::Queued, 3_000);
let all = [run, head.clone(), tiny_one.clone(), mid.clone()];
let live = Live {
non_heavy: Usage {
cpu: 200,
mem: 15 * G,
vram: 0,
},
..calm()
};
let t = 1_000 + BACKFILL_WINDOW_MS + 1;
assert!(matches!(
evaluate(&head, &all, &c, &live, t),
Decision::Wait {
block: Block::Mem,
..
}
));
assert_eq!(evaluate(&tiny_one, &all, &c, &live, t), Decision::Start);
assert!(
matches!(
evaluate(&mid, &all, &c, &live, t),
Decision::Wait {
block: Block::Queue,
..
}
),
"4 cores is not tiny: it keeps the 60 s rule"
);
let busy = Live {
cpu_pct: 90,
cpu_sustained_pct: 90,
non_heavy: Usage {
cpu: 1_440,
mem: 0,
vram: 0,
},
..calm()
};
assert!(matches!(
evaluate(&tiny_one, &all, &c, &busy, t),
Decision::Wait {
block: Block::Queue,
..
}
));
}
#[test]
fn a_blocked_waiter_says_why_with_the_numbers() {
let d = onto_busy_mac(1_100, 13);
let Decision::Wait { reason, .. } = d else {
panic!("{d:?}")
};
assert_eq!(
reason,
"cpu: needs 4.40 cores, 4.20 free under the 95% band (machine 68% busy)"
);
}
fn docker_build_at(other_cpu: u64, pct: u64) -> (Vec<Job>, Live) {
let build = |id: &str, state, enq| Job {
cmd: vec!["docker".into(), "build".into(), ".".into()],
cpu_hold_ms: Some(cpu_hold_ms(None)),
..job(id, 16, 4, state, enq)
};
let run = Job {
started_ms: Some(0),
..build("b1", State::Running, 0)
};
let live = Live {
cpu_pct: pct,
cpu_sustained_pct: pct,
non_heavy: Usage {
cpu: other_cpu,
mem: 20 * G,
vram: 0,
},
per_job: vec![(
"b1".into(),
Usage {
cpu: 10,
mem: G / 10,
vram: 0,
},
)],
..calm()
};
(vec![run, build("b2", State::Queued, 1_000)], live)
}
#[test]
fn a_docker_build_holds_the_machine_until_its_vm_cpu_shows() {
let c = mac();
let (jobs, live) = docker_build_at(300, 20);
assert!(matches!(
evaluate(&jobs[1], &jobs, &c, &live, 15_000),
Decision::Wait {
block: Block::Ramp,
..
}
));
let small = job("lint", 4, 2, State::Queued, 2_000);
let with_small = [jobs[0].clone(), small.clone()];
assert!(waits(&evaluate(&small, &with_small, &c, &live, 15_000)));
assert_eq!(
evaluate(&jobs[1], &jobs, &c, &live, CPU_FALLBACK_HOLD_MS),
Decision::Start
);
let d8 = Job {
started_ms: Some(0),
cmd: vec!["docker".into(), "build".into(), ".".into()],
cpu_hold_ms: Some(cpu_hold_ms(None)),
..job("d8", 8, 4, State::Running, 0)
};
let showing = Live {
non_heavy: Usage {
cpu: 600,
mem: 20 * G,
vram: 0,
},
per_job: vec![],
..calm()
};
assert_eq!(headroom(&[&d8], &c, &showing, 20_000).cpu, 1_520 - 800);
assert_eq!(headroom(&[&d8], &c, &showing, 5_000).cpu, 1_520 - 1_400);
}
#[test]
fn a_docker_build_with_history_holds_until_its_learned_vm_cpu_time() {
let c = mac();
let d8 = Job {
started_ms: Some(0),
cmd: vec!["docker".into(), "build".into(), ".".into()],
cpu_hold_ms: Some(cpu_hold_ms(Some(20_000))),
..job("d8", 8, 4, State::Running, 0)
};
assert_eq!(d8.cpu_hold_ms, Some(25_000));
let live = Live {
per_job: vec![],
..calm()
};
assert_eq!(headroom(&[&d8], &c, &live, 15_000).cpu, 1_520 - 800);
assert_eq!(headroom(&[&d8], &c, &live, 24_000).cpu, 1_520 - 800);
assert_eq!(headroom(&[&d8], &c, &live, 26_000).cpu, 1_520);
}
#[test]
fn a_docker_hold_is_netted_only_against_the_rise_above_its_start_baseline() {
let c = mac();
let d8 = Job {
started_ms: Some(0),
cmd: vec!["docker".into(), "build".into(), ".".into()],
cpu_hold_ms: Some(cpu_hold_ms(Some(20_000))),
non_heavy_cpu_at_start: 400,
..job("d8", 8, 4, State::Running, 0)
};
let live = |other| Live {
non_heavy: Usage {
cpu: other,
mem: 20 * G,
vram: 0,
},
per_job: vec![],
..calm()
};
assert_eq!(headroom(&[&d8], &c, &live(400), 11_000).cpu, 1_520 - 1_200);
assert_eq!(headroom(&[&d8], &c, &live(700), 11_000).cpu, 1_520 - 1_200);
}
fn big_head_at(pct: u64) -> (Vec<Job>, Live) {
let run = Job {
started_ms: Some(0),
cpu_hold_ms: Some(cpu_hold_ms(Some(4_000))),
..job("long", 10, 6, State::Running, 0)
};
let head = Job {
cpu_hold_ms: Some(CPU_MAX_HOLD_MS),
..job("ui-update", 13, 4, State::Queued, 100_000)
};
let small = Job {
cpu_hold_ms: Some(RAMP_MS),
run_p90_ms: Some(30_000),
..job("typecheck", 2, 4, State::Queued, 101_000)
};
let live = Live {
cpu_pct: pct,
cpu_sustained_pct: pct,
non_heavy: Usage {
cpu: 100,
mem: 20 * G,
vram: 0,
},
per_job: vec![(
"long".into(),
Usage {
cpu: 900,
mem: 6 * G,
vram: 0,
},
)],
..calm()
};
(vec![run, head, small], live)
}
#[test]
fn a_small_job_passes_a_cpu_blocked_head_on_an_underused_machine() {
let c = mac();
let (jobs, live) = big_head_at(70);
let past = 100_000 + BACKFILL_WINDOW_MS + 1;
assert!(matches!(
evaluate(&jobs[1], &jobs, &c, &live, past),
Decision::Wait {
block: Block::Cpu,
..
}
));
assert_eq!(evaluate(&jobs[2], &jobs, &c, &live, past), Decision::Start);
let (jobs, warm) = big_head_at(UNDERUSED_CPU_PCT);
assert!(matches!(
evaluate(&jobs[2], &jobs, &c, &warm, past),
Decision::Wait {
block: Block::Queue,
..
}
));
assert!(matches!(
evaluate(&jobs[2], &jobs, &c, &live, 100_000 + TINY_BYPASS_MS),
Decision::Wait {
block: Block::Queue,
..
}
));
let mid = job("check", 4, 4, State::Queued, 102_000);
let all = [jobs[0].clone(), jobs[1].clone(), mid.clone()];
assert!(matches!(
evaluate(&mid, &all, &c, &live, past),
Decision::Wait {
block: Block::Queue,
..
}
));
let mid = Job {
run_p90_ms: Some(60_000),
..mid
};
let all = [jobs[0].clone(), jobs[1].clone(), mid.clone()];
assert_eq!(evaluate(&mid, &all, &c, &live, past), Decision::Start);
}
fn busy_rig() -> (Capacity, Vec<Job>, Live) {
let c = Capacity {
cpu: 16,
mem: 64 * G,
vram: 10 * G,
disk_total: 1000 * G,
vram_is_ram: false,
};
let big = Job {
started_ms: Some(0),
cpu_hold_ms: Some(RAMP_MS),
..job("tests", 14, 12, State::Running, 0)
};
let live = Live {
load1: 20.0,
load5: 20.0,
cpu_pct: 97,
cpu_sustained_pct: 97,
per_job: vec![(
"tests".into(),
Usage {
cpu: 1_400,
mem: 8 * G,
vram: 0,
},
)],
non_heavy: Usage {
cpu: 150,
mem: 10 * G,
vram: 2 * G,
},
..calm()
};
(c, vec![big], live)
}
fn roomy(live: Live) -> Live {
Live {
cpu_pct: 85,
cpu_sustained_pct: 85,
..live
}
}
fn lint(id: &str, enq: i64, fast: bool, state: State) -> Job {
let mut j = job(id, 1, 0, state, enq);
j.weights.mem = 72 << 20;
j.fast = fast;
if state == State::Running {
j.started_ms = Some(enq);
}
j
}
#[test]
fn a_learned_short_small_job_starts_in_the_fast_lane_on_a_busy_machine() {
let (c, mut jobs, live) = busy_rig();
let live = roomy(live);
jobs.push(job("head", 8, 4, State::Queued, 50_000));
let now = 60_000;
let slow = lint("lint", 55_000, false, State::Queued);
let mut all = jobs.clone();
all.push(slow.clone());
assert!(
waits(&evaluate(&slow, &all, &c, &live, now)),
"no fast lane: it waits"
);
let quick = lint("lint", 55_000, true, State::Queued);
let mut all = jobs.clone();
all.push(quick.clone());
assert_eq!(evaluate(&quick, &all, &c, &live, now), Decision::Start);
}
#[test]
fn the_fast_lane_is_bounded_and_keeps_the_guards() {
let (c, jobs, live) = busy_rig();
let live = roomy(live);
let now = 60_000;
let with = |extra: &[Job]| {
let mut v = jobs.clone();
v.extend_from_slice(extra);
v
};
let me = lint("me", 59_000, true, State::Queued);
let running = [
lint("f1", 1_000, true, State::Running),
lint("f2", 58_000, true, State::Running),
];
let all = with(&[running[0].clone(), running[1].clone(), me.clone()]);
assert!(waits(&evaluate(&me, &all, &c, &live, now)));
let all = with(&[running[1].clone(), me.clone()]);
assert_eq!(evaluate(&me, &all, &c, &live, now), Decision::Start);
let all = with(std::slice::from_ref(&me));
for bad in [
Live {
load1: 64.0,
..live.clone()
},
Live {
mem_pressure: true,
..live.clone()
},
Live {
mem_used_pct: 96,
..live.clone()
},
] {
assert!(waits(&evaluate(&me, &all, &c, &bad, now)), "{bad:?}");
}
let full = Live {
non_heavy: Usage {
mem: 52 * G,
..live.non_heavy
},
..live.clone()
};
assert!(waits(&evaluate(&me, &all, &c, &full, now)));
}
#[test]
fn a_running_fast_job_using_16_cores_closes_the_lane() {
let (c, jobs, live) = busy_rig();
let live = roomy(live);
let now = 60_000;
let me = lint("me", 59_000, true, State::Queued);
let runner = lint("f1", 58_000, true, State::Running);
let mut all = jobs;
all.extend([runner, me.clone()]);
assert_eq!(evaluate(&me, &all, &c, &live, now), Decision::Start);
let mut hot = live.clone();
hot.per_job.push((
"f1".into(),
Usage {
cpu: 1_600,
mem: 72 << 20,
vram: 0,
},
));
assert!(waits(&evaluate(&me, &all, &c, &hot, now)));
}
#[test]
fn a_machine_with_no_free_cores_refuses_the_fast_lane() {
let (c, mut jobs, live) = busy_rig();
let now = 60_000;
let me = lint("me", 59_000, true, State::Queued);
jobs.push(me.clone());
for pct in [97, 100] {
let full = Live {
cpu_pct: pct,
cpu_sustained_pct: pct,
..live.clone()
};
assert!(waits(&evaluate(&me, &jobs, &c, &full, now)), "{pct}%");
}
assert!(!fast_cpu_room(&c, &live));
assert!(fast_cpu_room(&c, &roomy(live)));
}
#[test]
fn only_short_small_jobs_without_vram_are_fast_lane_eligible() {
let c = Capacity {
cpu: 16,
mem: 64 * G,
vram: 10 * G,
disk_total: 1000 * G,
vram_is_ram: false,
};
let w = |cpu, mem, vram| Weights {
cpu,
mem,
vram,
disk: 0,
};
assert!(fast_lane_eligible(w(1, 72 << 20, 0), &c, Some(1_285)));
assert!(fast_lane_eligible(
w(FAST_MAX_CPU, 2 * G, 0),
&c,
Some(FAST_RUN_MS)
));
assert!(!fast_lane_eligible(w(1, 0, 0), &c, None), "cold or pooled");
assert!(
!fast_lane_eligible(w(1, 0, 0), &c, Some(FAST_RUN_MS + 1)),
"one recent run was long"
);
assert!(!fast_lane_eligible(
w(FAST_MAX_CPU + 1, 0, 0),
&c,
Some(1_000)
));
assert!(
!fast_lane_eligible(w(1, 3 * G, 0), &c, Some(1_000)),
"over 1/32 of RAM"
);
assert!(
!fast_lane_eligible(w(1, 0, 1 << 30), &c, Some(1_000)),
"uses VRAM"
);
}
#[test]
fn a_burst_of_fast_jobs_adds_at_most_the_lane_cap_to_a_pinned_machine() {
let (c, jobs, live) = busy_rig();
let live = roomy(live);
let mut all = jobs.clone();
let mut started = 0;
for k in 0..12 {
let me = lint(&format!("l{k:02}"), 60_000 + k, true, State::Queued);
all.push(me.clone());
if evaluate(&me, &all, &c, &live, 61_000) == Decision::Start {
started += 1;
let last = all.len() - 1;
all[last].state = State::Running;
all[last].started_ms = Some(61_000);
}
}
assert_eq!(started, FAST_MAX_JOBS);
}
#[test]
fn a_high_load_blocks_only_when_the_cpu_or_cpu_pressure_confirms_it() {
let c = Capacity { cpu: 16, ..cap() };
let me = job("a", 1, 1, State::Queued, 0);
let one = std::slice::from_ref(&me);
let rig = Live {
load1: 35.4,
load5: 30.0,
cpu_pct: 15,
cpu_pressure: Some(0),
..calm()
};
assert_eq!(evaluate(&me, one, &c, &rig, 0), Decision::Start);
let storm = Live {
load1: 70.0,
load5: 60.0,
..rig.clone()
};
assert_eq!(evaluate(&me, one, &c, &storm, 0), Decision::Start);
for confirmed in [
Live {
cpu_pressure: Some(CPU_PSI_PCT),
..storm.clone()
},
Live {
cpu_pct: LOAD_HARD_CPU_PCT,
..storm.clone()
},
] {
let (why, block) = guard_block(&c, &confirmed).expect("blocked");
assert_eq!(block, Block::Load1);
assert!(why.contains("4 x 16 cores"), "{why}");
}
let queued = Live {
cpu_pressure: Some(40),
..rig.clone()
};
let (why, _) = guard_block(&c, &queued).expect("blocked");
assert!(why.contains("cpu pressure 40%"), "{why}");
assert_eq!(
guard_block(
&c,
&Live {
load1: 20.0,
..queued
}
),
None,
"under 2 x cores PSI alone never blocks"
);
let mac_meltdown = Live {
load1: 120.0,
load5: 90.0,
cpu_pct: 100,
..calm()
};
assert!(guard_block(&c, &mac_meltdown).is_some());
}
fn mid_behind_head(pct: u64) -> (Vec<Job>, Live) {
let run = Job {
started_ms: Some(0),
cpu_hold_ms: Some(RAMP_MS),
..job("long", 10, 6, State::Running, 0)
};
let head = Job {
cpu_hold_ms: Some(CPU_MAX_HOLD_MS),
..job("lint", 13, 8, State::Queued, 0)
};
let mid = Job {
cpu_hold_ms: Some(RAMP_MS),
run_p90_ms: Some(20_000),
..job("clippy", 6, 4, State::Queued, 1_000)
};
let busy = pct * 1_600 / 100;
let live = Live {
cpu_pct: pct,
cpu_sustained_pct: pct,
per_job: vec![(
"long".into(),
Usage {
cpu: busy,
mem: 6 * G,
vram: 0,
},
)],
..calm()
};
(vec![run, head, mid], live)
}
#[test]
fn a_mid_size_job_backfills_a_young_cpu_blocked_head_on_an_underused_machine() {
let c = mac();
let (jobs, live) = mid_behind_head(72);
let past = BACKFILL_WINDOW_MS + 1;
assert!(matches!(
evaluate(&jobs[1], &jobs, &c, &live, past),
Decision::Wait {
block: Block::Cpu,
..
}
));
assert_eq!(evaluate(&jobs[2], &jobs, &c, &live, past), Decision::Start);
let (_, warm) = mid_behind_head(UNDERUSED_CPU_PCT);
assert!(waits(&evaluate(&jobs[2], &jobs, &c, &warm, past)));
assert!(matches!(
evaluate(&jobs[2], &jobs, &c, &live, TINY_BYPASS_MS),
Decision::Wait {
block: Block::Queue,
..
}
));
let bigger = Job {
weights: Weights {
cpu: 14,
..jobs[2].weights
},
..jobs[2].clone()
};
let all = [jobs[0].clone(), jobs[1].clone(), bigger.clone()];
assert!(matches!(
evaluate(&bigger, &all, &c, &live, past),
Decision::Wait {
block: Block::Queue,
..
}
));
}
#[test]
fn a_long_backfiller_cannot_hold_the_head_past_its_backfill_time() {
let c = mac();
let (mut jobs, live) = mid_behind_head(72);
let at = TINY_BYPASS_MS - 1_000;
jobs[2] = Job {
run_p90_ms: Some(70 * 60_000),
..job("long-test", 12, 4, State::Queued, at)
};
assert!(matches!(
evaluate(&jobs[2], &jobs, &c, &live, at),
Decision::Wait {
block: Block::Queue,
..
}
));
jobs[2] = Job {
cpu_hold_ms: Some(RAMP_MS),
..job("clippy", 6, 4, State::Queued, at)
};
jobs[2].run_p90_ms = Some(u64::try_from(1_000 + BACKFILL_OVERRUN_MS).unwrap_or(0));
assert_eq!(evaluate(&jobs[2], &jobs, &c, &live, at), Decision::Start);
jobs[2].run_p90_ms = Some(u64::try_from(2_000 + BACKFILL_OVERRUN_MS).unwrap_or(0));
assert!(
waits(&evaluate(&jobs[2], &jobs, &c, &live, at)),
"past the bound"
);
jobs[2].run_p90_ms = None;
assert!(
waits(&evaluate(&jobs[2], &jobs, &c, &live, at)),
"no learned run time"
);
let tiny_job = job("fmt", 1, 1, State::Queued, at);
let all = [jobs[0].clone(), jobs[1].clone(), tiny_job.clone()];
assert_eq!(evaluate(&tiny_job, &all, &c, &live, at), Decision::Start);
}
#[test]
fn a_head_that_holds_the_queue_is_never_passed_and_projects_its_sustained_level() {
assert_eq!(projected_peak(1_300, None), 520, "2/5 of the peak");
assert_eq!(
projected_peak(1_300, Some(300)),
300,
"a lower sustained level"
);
assert_eq!(
projected_peak(1_300, Some(900)),
520,
"never above 2/5 of the peak"
);
let c = mac();
let (jobs, live) = mid_behind_head(72);
let held = TINY_BYPASS_MS;
assert_eq!(
evaluate(&jobs[2], &jobs, &c, &live, held - 1),
Decision::Start
);
assert!(matches!(
evaluate(&jobs[2], &jobs, &c, &live, held),
Decision::Wait {
block: Block::Queue,
..
}
));
assert!(waits(&evaluate(&jobs[1], &jobs, &c, &live, held)));
let mut lower = jobs.clone();
lower[1].cpu_sustained = Some(300);
assert_eq!(
evaluate(&lower[1], &lower, &c, &live, held),
Decision::Start
);
}
#[test]
fn a_held_peak_drops_to_the_sustained_level_only_after_its_learned_peak_time() {
let by = Some(cpu_hold_ms(Some(60_000)));
assert_eq!(
sustained_hold(1_100, Some(300), 0, false, by),
1_100,
"full peak while it ramps"
);
assert_eq!(
sustained_hold(1_100, Some(300), SUSTAINED_FULL_MS, false, by),
1_100,
"its learned peak (60 s plus margin) is still ahead"
);
assert_eq!(
sustained_hold(1_100, Some(300), 75_000, false, by),
300,
"past the learned peak time and margin"
);
assert_eq!(
sustained_hold(1_100, Some(300), SUSTAINED_FULL_MS, false, Some(RAMP_MS)),
300,
"an early peak: from SUSTAINED_FULL_MS"
);
assert_eq!(sustained_hold(1_100, Some(1_500), 75_000, false, by), 1_100);
assert_eq!(
sustained_hold(0, Some(300), 75_000, false, by),
0,
"a released hold stays released"
);
assert_eq!(
sustained_hold(1_100, Some(300), 200_000, true, by),
1_100,
"Docker/VM keeps its hold"
);
assert_eq!(
sustained_hold(1_100, None, 200_000, false, by),
1_100,
"nothing learned"
);
assert_eq!(
sustained_hold(1_100, Some(300), 200_000, false, None),
1_100,
"no learned time to peak"
);
let c = mac();
let lint = Job {
started_ms: Some(0),
cpu_hold_ms: Some(cpu_hold_ms(Some(15_000))),
cpu_sustained: Some(300),
..job("lint", 11, 4, State::Running, 0)
};
let me = job("clippy", 6, 4, State::Queued, 1);
let live = Live {
cpu_pct: 32,
cpu_sustained_pct: 32,
non_heavy: Usage {
cpu: 400,
mem: 10 * G,
vram: 0,
},
per_job: vec![(
"lint".into(),
Usage {
cpu: 120,
mem: 2 * G,
vram: 0,
},
)],
..calm()
};
let at = |t| evaluate(&me, &[lint.clone(), me.clone()], &c, &live, t);
assert!(waits(&at(5_000)), "the full peak during the ramp");
assert!(
waits(&at(12_000)),
"the full peak before its learned peak time"
);
assert_eq!(at(cpu_hold_ms(Some(15_000))), Decision::Start);
}
#[test]
fn late_spiking_jobs_are_not_let_in_together_by_their_sustained_level() {
let c = mac();
let shape = |since: i64| -> u64 {
if (60_000..62_000).contains(&since) {
1_600
} else {
100
}
};
let samples: Vec<(u64, u64)> = (0..240)
.map(|i| (i * 500, shape(i64::try_from(i * 500).unwrap_or(0))))
.collect();
let level = sustained_level(&samples);
assert!(level < 500, "the learned sustained level {level}");
let mut jobs: Vec<Job> = (0..3)
.map(|i| Job {
cpu_hold_ms: Some(cpu_hold_ms(Some(60_000))),
cpu_sustained: Some(level),
run_p90_ms: Some(120_000),
..job(&format!("s{i}"), 16, 2, State::Queued, i)
})
.collect();
let (mut worst, mut t) = (0u64, 0);
while t <= 900_000 && !jobs.is_empty() {
jobs.retain(|j| j.started_ms.is_none_or(|s| t - s < 120_000));
let demand: Vec<(String, u64)> = jobs
.iter()
.filter_map(|j| j.started_ms.map(|s| (j.id.clone(), shape(t - s))))
.collect();
let total: u64 = demand.iter().map(|d| d.1).sum();
worst = worst.max(total);
let pct = total.min(1_600) * 100 / 1_600;
let live = Live {
cpu_pct: pct,
cpu_sustained_pct: pct,
per_job: demand
.iter()
.map(|(id, d)| {
(
id.clone(),
Usage {
cpu: (*d).min(1_600),
mem: 2 * G,
vram: 0,
},
)
})
.collect(),
..calm()
};
for i in 0..jobs.len() {
if jobs[i].state == State::Queued
&& evaluate(&jobs[i].clone(), &jobs, &c, &live, t) == Decision::Start
{
jobs[i].state = State::Running;
jobs[i].started_ms = Some(t);
}
}
t += 500;
}
assert!(jobs.is_empty(), "every job ran");
assert!(
worst <= 1_700,
"demand peaked at {worst} centi-cores on a 16-core machine"
);
}
#[test]
fn the_sustained_level_is_the_best_ten_second_mean() {
assert_eq!(sustained_level(&[]), 0);
let mut spike: Vec<(u64, u64)> = (0..60).map(|i| (i * 500, 100)).collect();
spike[30].1 = 1_000;
assert_eq!(sustained_level(&spike), 145);
let burst: Vec<(u64, u64)> = (0..80)
.map(|i| (i * 500, if i < 30 { 800 } else { 100 }))
.collect();
assert_eq!(sustained_level(&burst), 800);
assert_eq!(sustained_level(&[(0, 100), (500, 300)]), 200);
}
}