use super::capacity::{self, Capacity, Held};
use super::task_board::{TaskOffer, TaskReject};
use super::BrokerState;
use crate::policy::admit::{Reason, Refusal, Source};
use crate::policy::schedule::{self, LocalTime};
use crate::policy::{MinPrice, Policy};
use crate::sandbox::{Allocation, Limits};
use serde_json::{json, Value};
use std::collections::{BTreeMap, HashMap};
use std::path::Path;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
const SNAPSHOT_FILE: &str = "revenue.json";
const DEFAULT_RANK_WINDOW_MS: u64 = 20;
pub const PUSH_FLOOR: Duration = Duration::from_secs(15);
pub const PUSH_CEILING: Duration = Duration::from_secs(60);
const HOLD_GRACE: Duration = Duration::from_secs(120);
const DEFAULT_HOLD: Duration = Duration::from_secs(300);
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Need {
pub cpus: f64,
pub memory_bytes: u64,
pub gpus: u32,
}
impl Need {
pub fn of_offer(o: &TaskOffer) -> Self {
Self {
cpus: o.cpus.max(0.0),
memory_bytes: o.memory_bytes,
gpus: o.gpus,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct BudgetTotals {
pub cpus: f64,
pub memory_bytes: u64,
pub gpus: u32,
}
impl BudgetTotals {
pub fn of(l: &Limits, gpus: u32) -> Self {
Self {
cpus: l.cpu_millis as f64 / 1000.0,
memory_bytes: l.memory_bytes,
gpus,
}
}
}
pub fn revenue_per_budget_hour(
price_per_hour: f64,
min_charge: f64,
expected_secs: f64,
need: &Need,
budget: &BudgetTotals,
) -> f64 {
let hours = expected_secs.max(1.0) / 3600.0;
let revenue = (price_per_hour.max(0.0) * hours).max(min_charge.max(0.0));
let frac = |want: f64, have: f64| if have > 0.0 { want / have } else { 0.0 };
let share = frac(need.cpus, budget.cpus)
.max(frac(need.memory_bytes as f64, budget.memory_bytes as f64))
.max(frac(need.gpus as f64, budget.gpus as f64))
.max(1e-6);
revenue / (share * hours)
}
#[derive(Debug, Clone)]
struct Hold {
need: Need,
expires: Instant,
}
#[derive(Debug, Clone)]
struct Waiter {
id: String,
need: Need,
score: f64,
seq: u64,
}
fn remaining(budget: &Limits, held: &Held, nothing_held: bool) -> (u64, u64) {
let cpu = (budget.cpu_millis as u64).saturating_sub((held.cpus * 1000.0).round() as u64);
let mem = budget.memory_bytes.saturating_sub(held.memory_bytes);
if nothing_held {
(cpu.max(1000), mem.max(1 << 30))
} else {
(cpu, mem)
}
}
fn fits(need: &Need, left: (u64, u64)) -> bool {
((need.cpus * 1000.0).round() as u64) <= left.0 && need.memory_bytes <= left.1
}
pub fn pick_ranked(waiting: &[(String, Need, f64, u64)], mut left: (u64, u64)) -> Vec<String> {
let mut order: Vec<&(String, Need, f64, u64)> = waiting.iter().collect();
order.sort_by(|a, b| {
b.2.partial_cmp(&a.2)
.unwrap_or(std::cmp::Ordering::Equal)
.then(a.3.cmp(&b.3))
});
let mut chosen = Vec::new();
for (id, need, _, _) in order {
if fits(need, left) {
left.0 -= ((need.cpus * 1000.0).round() as u64).min(left.0);
left.1 -= need.memory_bytes.min(left.1);
chosen.push(id.clone());
}
}
chosen
}
#[derive(Debug, Default)]
struct Idle {
seconds: f64,
by_source: BTreeMap<&'static str, f64>,
covered_to: Option<Instant>,
used_weighted: f64,
observed: f64,
last_used: f64,
}
#[derive(Debug, Default)]
pub struct PushGate {
pub(crate) last: Option<(Instant, u64)>,
}
impl PushGate {
pub fn due(&mut self, now: Instant, signature: u64) -> bool {
let due = match self.last {
None => true,
Some((at, sig)) => {
let age = now.saturating_duration_since(at);
age >= PUSH_CEILING || (sig != signature && age >= PUSH_FLOOR)
}
};
if due {
self.last = Some((now, signature));
}
due
}
}
struct Inner {
holds: Mutex<HashMap<String, Hold>>,
waiting: Mutex<Vec<Waiter>>,
seq: AtomicU64,
idle: Mutex<Idle>,
gate: Mutex<PushGate>,
since: std::time::SystemTime,
window_override: Mutex<Option<Duration>>,
workloads: Mutex<WorkloadScan>,
preempted: Mutex<BTreeMap<&'static str, u64>>,
room_wanted: Mutex<Option<RoomWanted>>,
}
const WORKLOAD_SCAN_TTL: Duration = Duration::from_secs(2);
const ROOM_WANTED_TTL: Duration = Duration::from_secs(30);
pub type AllocSource = Arc<dyn Fn() -> Vec<Allocation> + Send + Sync>;
#[derive(Default)]
struct WorkloadScan {
at: Option<Instant>,
list: Vec<Allocation>,
source: Option<AllocSource>,
}
fn scan_workloads() -> Vec<Allocation> {
if cfg!(test) {
return Vec::new();
}
let mut all = crate::sandbox::engine::allocations().unwrap_or_default();
if let Ok(more) =
crate::broker::deploy::ContainerRuntime::allocations(&crate::broker::deploy::DockerCli)
{
for a in more {
if !all.iter().any(|x| x.name == a.name) {
all.push(a);
}
}
}
all
}
#[derive(Debug, Clone, Copy)]
pub struct RoomWanted {
pub need: Need,
pub price_per_hour: f64,
at: Instant,
}
#[derive(Clone)]
pub struct Revenue(Arc<Inner>);
impl Default for Revenue {
fn default() -> Self {
Self::new()
}
}
#[must_use]
pub struct HoldGuard {
rev: Revenue,
id: String,
keep: bool,
}
impl HoldGuard {
pub fn keep(mut self) {
self.keep = true;
}
}
impl Drop for HoldGuard {
fn drop(&mut self) {
if !self.keep {
self.rev.release(&self.id);
}
}
}
impl Revenue {
pub fn new() -> Self {
Self(Arc::new(Inner {
holds: Mutex::default(),
waiting: Mutex::default(),
seq: AtomicU64::new(0),
idle: Mutex::default(),
gate: Mutex::default(),
since: std::time::SystemTime::now(),
window_override: Mutex::new(None),
workloads: Mutex::default(),
preempted: Mutex::default(),
room_wanted: Mutex::new(None),
}))
}
#[cfg(test)]
pub fn with_window(self, w: Duration) -> Self {
*self.0.window_override.lock().unwrap() = Some(w);
self
}
fn window(&self) -> Duration {
if let Some(w) = *self.0.window_override.lock().unwrap() {
return w;
}
let ms = std::env::var("ZAKURO_OFFER_RANK_WINDOW_MS")
.ok()
.and_then(|v| v.trim().parse().ok())
.unwrap_or(DEFAULT_RANK_WINDOW_MS);
Duration::from_millis(ms.min(500))
}
fn holds(&self) -> std::sync::MutexGuard<'_, HashMap<String, Hold>> {
self.0.holds.lock().unwrap_or_else(|e| e.into_inner())
}
pub fn held(&self) -> Held {
held_of(&self.holds())
}
pub fn held_count(&self) -> usize {
self.holds().len()
}
pub fn workloads(&self) -> Vec<Allocation> {
let mut w = self.0.workloads.lock().unwrap_or_else(|e| e.into_inner());
if w.at.is_none_or(|t| t.elapsed() >= WORKLOAD_SCAN_TTL) {
let list = match &w.source {
Some(f) => f(),
None => scan_workloads(),
};
w.list = list;
w.at = Some(Instant::now());
}
w.list.clone()
}
pub fn held_all(&self) -> Held {
let mut h = self.held();
for a in self.workloads() {
h.cpus += a.cpu_millis as f64 / 1000.0;
h.memory_bytes = h.memory_bytes.saturating_add(a.memory_bytes);
}
h
}
#[cfg(test)]
pub fn with_workloads(self, f: AllocSource) -> Self {
self.0.workloads.lock().unwrap().source = Some(f);
self
}
pub fn record_preempted(&self, reason: &'static str, leases: u64) {
*self
.0
.preempted
.lock()
.unwrap_or_else(|e| e.into_inner())
.entry(reason)
.or_insert(0) += leases;
}
pub fn preempted(&self) -> Value {
let p = self.0.preempted.lock().unwrap_or_else(|e| e.into_inner());
let get = |k: &str| p.get(k).copied().unwrap_or(0);
json!({ "owner_active": get("owner_active"), "better_offer": get("better_offer") })
}
pub fn want_room(&self, need: Need, price_per_hour: f64) {
let mut w = self.0.room_wanted.lock().unwrap_or_else(|e| e.into_inner());
let keep = w.as_ref().is_some_and(|c| {
c.at.elapsed() < ROOM_WANTED_TTL && c.price_per_hour >= price_per_hour
});
if !keep {
*w = Some(RoomWanted {
need,
price_per_hour,
at: Instant::now(),
});
}
}
pub fn take_room_wanted(&self) -> Option<RoomWanted> {
self.0
.room_wanted
.lock()
.unwrap_or_else(|e| e.into_inner())
.take()
.filter(|w| w.at.elapsed() < ROOM_WANTED_TTL)
}
pub fn release(&self, id: &str) {
self.holds().remove(id);
}
pub fn release_on_drop(&self, id: &str) -> HoldGuard {
HoldGuard {
rev: self.clone(),
id: id.to_string(),
keep: false,
}
}
pub fn sweep(&self, now: Instant) {
self.holds().retain(|_, h| h.expires > now);
}
fn insert(&self, holds: &mut HashMap<String, Hold>, id: &str, need: Need, timeout_secs: f64) {
let ttl = if timeout_secs > 0.0 {
Duration::from_secs_f64(timeout_secs.min(86_400.0))
} else {
DEFAULT_HOLD
};
holds.insert(
id.to_string(),
Hold {
need,
expires: Instant::now() + ttl + HOLD_GRACE,
},
);
}
#[allow(clippy::too_many_arguments)]
pub fn acquire(
&self,
id: &str,
need: Need,
score: f64,
timeout_secs: f64,
budget: Option<&Limits>,
check: &dyn Fn(&[Allocation]) -> Result<(), Refusal>,
outranked: &dyn Fn(String) -> Refusal,
) -> Result<HoldGuard, Refusal> {
let guard = |me: &Self| HoldGuard {
rev: me.clone(),
id: id.to_string(),
keep: false,
};
let Some(budget) = budget else {
let mut holds = self.holds();
check(&allocations_of(&holds))?;
self.insert(&mut holds, id, need, timeout_secs);
return Ok(guard(self));
};
let contended = {
let holds = self.holds();
let held = held_of(&holds);
let left = remaining(budget, &held, holds.is_empty());
fits(&need, left) && {
let after = (
left.0.saturating_sub((need.cpus * 1000.0).round() as u64),
left.1.saturating_sub(need.memory_bytes),
);
!fits(&need, after)
}
};
let window = self.window();
let mut registered = false;
if contended && !window.is_zero() {
let seq = self.0.seq.fetch_add(1, Ordering::SeqCst);
self.0
.waiting
.lock()
.unwrap_or_else(|e| e.into_inner())
.push(Waiter {
id: id.to_string(),
need,
score,
seq,
});
registered = true;
std::thread::sleep(window);
}
let unregister = |me: &Self| {
if registered {
me.0.waiting
.lock()
.unwrap_or_else(|e| e.into_inner())
.retain(|w| w.id != id);
}
};
let waiters: Vec<(String, Need, f64, u64)> = self
.0
.waiting
.lock()
.unwrap_or_else(|e| e.into_inner())
.iter()
.map(|w| (w.id.clone(), w.need, w.score, w.seq))
.collect();
let mut holds = self.holds();
if registered {
let left = remaining(budget, &held_of(&holds), holds.is_empty());
let chosen = pick_ranked(&waiters, left);
if !chosen.iter().any(|c| c == id) && fits(&need, left) {
drop(holds);
unregister(self);
return Err(outranked(
"a better-paying offer took the capacity (revenue per budget-hour)".into(),
));
}
}
let verdict = check(&allocations_of(&holds));
if verdict.is_ok() {
self.insert(&mut holds, id, need, timeout_secs);
}
drop(holds);
unregister(self);
verdict.map(|_| guard(self))
}
pub fn eligible_idle(&self, source: Source, covers: Duration) {
self.eligible_idle_at(source, covers, Instant::now());
}
pub fn eligible_idle_at(&self, source: Source, covers: Duration, now: Instant) {
let mut i = self.0.idle.lock().unwrap_or_else(|e| e.into_inner());
*i.by_source.entry(source.as_str()).or_insert(0.0) += covers.as_secs_f64();
let start = now.checked_sub(covers).unwrap_or(now);
let start = match i.covered_to {
Some(end) if end > start => end,
_ => start,
};
if now > start {
i.seconds += now.duration_since(start).as_secs_f64();
}
i.covered_to = Some(i.covered_to.map_or(now, |e| e.max(now)));
}
pub fn observe_use(&self, dt: Duration, used: f64) {
let used = used.clamp(0.0, 1.0);
let mut i = self.0.idle.lock().unwrap_or_else(|e| e.into_inner());
i.used_weighted += used * dt.as_secs_f64();
i.observed += dt.as_secs_f64();
i.last_used = used;
}
pub fn eligible_idle_seconds(&self) -> f64 {
self.0
.idle
.lock()
.unwrap_or_else(|e| e.into_inner())
.seconds
}
pub fn counters(&self) -> Value {
let i = self.0.idle.lock().unwrap_or_else(|e| e.into_inner());
let by_source: serde_json::Map<String, Value> = i
.by_source
.iter()
.map(|(k, v)| (k.to_string(), json!(round1(*v))))
.collect();
let avg = if i.observed > 0.0 {
i.used_weighted / i.observed
} else {
0.0
};
let since = self
.0
.since
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
json!({
"eligible_idle_seconds": round1(i.seconds),
"eligible_idle_by_source": by_source,
"utilisation": { "now": round3(i.last_used), "avg": round3(avg), "observed_seconds": round1(i.observed) },
"since": since,
"preempted": self.preempted(),
})
}
pub fn push_sent(&self, now: Instant, signature: u64) {
self.0.gate.lock().unwrap_or_else(|e| e.into_inner()).last = Some((now, signature));
}
pub fn push_due(&self, now: Instant, signature: u64) -> bool {
self.0
.gate
.lock()
.unwrap_or_else(|e| e.into_inner())
.due(now, signature)
}
}
fn round1(v: f64) -> f64 {
(v * 10.0).round() / 10.0
}
fn round3(v: f64) -> f64 {
(v * 1000.0).round() / 1000.0
}
fn held_of(holds: &HashMap<String, Hold>) -> Held {
holds.values().fold(Held::default(), |mut h, x| {
h.cpus += x.need.cpus;
h.memory_bytes = h.memory_bytes.saturating_add(x.need.memory_bytes);
h.gpus += x.need.gpus;
h
})
}
fn allocations_of(holds: &HashMap<String, Hold>) -> Vec<Allocation> {
holds
.iter()
.map(|(id, h)| Allocation {
name: id.clone(),
cpu_millis: (h.need.cpus * 1000.0).round() as u32,
memory_bytes: h.need.memory_bytes,
})
.collect()
}
pub fn sharing_allowed(p: &Policy, now: LocalTime) -> bool {
!p.sharing.squads.is_empty()
&& p.sharing.min_price != MinPrice::Withdrawn
&& (p.sharing.when != "schedule" || schedule::is_open(&p.sharing.schedule, now))
}
pub fn capacity(state: &BrokerState) -> Capacity {
capacity::compute(
&state.workers.list(),
state.admission.budget_limits().as_ref(),
state.revenue.held_all(),
)
}
pub fn advert_price(state: &BrokerState, cap: &Capacity) -> f64 {
cap.price_per_hour.unwrap_or_else(|| state.price.get())
}
fn quote(state: &BrokerState, offer: &TaskOffer) -> (f64, f64) {
state
.workers
.healthy()
.into_iter()
.filter(|w| w.source_node.is_none() && w.pricing.price_per_hour <= offer.max_price_per_hour)
.map(|w| (w.pricing.price_per_hour, w.pricing.min_charge))
.min_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal))
.unwrap_or((offer.max_price_per_hour.min(1e6), 0.0))
}
pub fn admit_peer_offer(state: &BrokerState, offer: &TaskOffer) -> Result<HoldGuard, TaskReject> {
let req = super::admission::offer_request(offer, state.config.owner_user_id.as_deref());
let need = Need::of_offer(offer);
let budget = state.admission.budget_limits();
let workloads = state.revenue.workloads();
let cap = capacity(state);
let (price, min_charge) = quote(state, offer);
let score = match &budget {
Some(b) => {
let gpus = cap.gpu.iter().map(|c| c.gpus.budget).sum();
revenue_per_budget_hour(
price,
min_charge,
offer.estimated_duration_secs,
&need,
&BudgetTotals::of(b, gpus),
)
}
None => 0.0,
};
let now = LocalTime::now();
let admitted = state.revenue.acquire(
&offer.task_id,
need,
score,
offer.timeout_secs,
budget.as_ref(),
&|running| {
let mut all = running.to_vec();
all.extend(workloads.iter().cloned());
state.admission.check_with(&req, &all, now)?;
match vram_shortfall(&cap, offer) {
Some(why) => Err(state.admission.refuse(&req, Reason::Budget, why)),
None => Ok(()),
}
},
&|why| state.admission.refuse(&req, Reason::Budget, why),
);
admitted.map_err(|r| {
if r.reason == Reason::Budget && !req.own {
state.revenue.want_room(need, price);
}
TaskReject {
task_id: offer.task_id.clone(),
reason: format!("policy refused ({}): {}", r.reason.as_str(), r.detail),
}
})
}
pub fn vram_shortfall(cap: &Capacity, offer: &TaskOffer) -> Option<String> {
let min = offer.min_vram_gb?;
if offer.gpus == 0 || cap.free_gpus(Some(min)) >= offer.gpus {
return None;
}
Some(format!(
"vram: asks {} GPU(s) of at least {min} GiB, {} free",
offer.gpus,
cap.free_gpus(Some(min))
))
}
pub fn refuse_no_capacity(state: &BrokerState, offer: &TaskOffer, detail: &str) -> TaskReject {
let req = super::admission::offer_request(offer, state.config.owner_user_id.as_deref());
let r = state
.admission
.refuse(&req, Reason::Budget, detail.to_string());
TaskReject {
task_id: offer.task_id.clone(),
reason: format!("policy refused ({}): {}", r.reason.as_str(), r.detail),
}
}
pub fn snapshot(state: &BrokerState) -> Value {
let cap = capacity(state);
let mut v = state.revenue.counters();
let admission = state.admission.snapshot();
v["capacity"] = serde_json::to_value(&cap).unwrap_or(Value::Null);
v["held_offers"] = json!(state.revenue.held_count());
v["refusals"] = admission["refused"].clone();
v["admitted"] = admission["admitted"].clone();
let policy = state.admission.policy();
let owner_blocks = state.admission.owner().blocks(&policy.sharing.when);
v["sharing_allowed"] =
json!(sharing_allowed(&policy, LocalTime::now()) && owner_blocks.is_none());
v["owner_yield"] = json!(owner_blocks);
v["held_workloads"] = json!(state.revenue.workloads().len());
v["price_per_hour"] = json!(advert_price(state, &cap));
v
}
pub fn tick(state: &BrokerState, dt: Duration, home: Option<&Path>, periodic: bool) -> bool {
let rev = &state.revenue;
rev.sweep(Instant::now());
let cap = capacity(state);
let used = if cap.cpu_cores.budget > 0.0 {
1.0 - cap.cpu_cores.free / cap.cpu_cores.budget
} else {
0.0
};
rev.observe_use(dt, used);
if let Some(home) = home {
let tmp = home.join(".revenue.json.tmp");
if std::fs::write(&tmp, snapshot(state).to_string()).is_ok() {
let _ = std::fs::rename(&tmp, home.join(SNAPSHOT_FILE));
}
}
let now = Instant::now();
let due = rev.push_due(now, cap.signature());
if periodic && !due {
rev.push_sent(now, cap.signature());
}
due || periodic
}
pub fn read_snapshot(home: &Path) -> Option<Value> {
serde_json::from_str(&std::fs::read_to_string(home.join(SNAPSHOT_FILE)).ok()?).ok()
}
pub const HUB_MAX_LEASES: usize = 16;
pub fn lease_slots(
budget: &Limits,
held_cpu_millis: u64,
held_memory: u64,
unit: &Need,
ceiling: usize,
) -> usize {
let left_cpu = (budget.cpu_millis as u64).saturating_sub(held_cpu_millis);
let left_mem = budget.memory_bytes.saturating_sub(held_memory);
let unit_cpu = ((unit.cpus * 1000.0).round() as u64).max(1000);
let unit_mem = unit.memory_bytes.max(1 << 30);
let by_cpu = left_cpu / unit_cpu;
let by_mem = left_mem / unit_mem;
(by_cpu.min(by_mem) as usize).min(ceiling)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::policy::admit::Request;
use crate::policy::{Admission, Policy};
use crate::sandbox::Gpus;
fn limits(cpu: u32, mem_gib: u64) -> Limits {
Limits {
cpu_millis: cpu,
memory_bytes: mem_gib << 30,
gpus: Gpus::None,
}
}
fn need(cpus: f64, mem_gib: u64) -> Need {
Need {
cpus,
memory_bytes: mem_gib << 30,
gpus: 0,
}
}
fn peer_req() -> Request {
Request::new(Source::PeerOffer, crate::policy::admit::Workload::Function)
}
fn acquire(
rev: &Revenue,
a: &Admission,
id: &str,
n: Need,
score: f64,
b: &Limits,
) -> Result<HoldGuard, Refusal> {
let mut req = peer_req();
req.cpus = n.cpus;
req.memory_bytes = n.memory_bytes;
rev.acquire(
id,
n,
score,
10.0,
Some(b),
&|running| a.check_with(&req, running, LocalTime::now()),
&|why| a.refuse(&req, Reason::Budget, why),
)
}
#[test]
fn revenue_per_budget_hour_prefers_the_smaller_share_at_the_same_price() {
let b = BudgetTotals {
cpus: 8.0,
memory_bytes: 32 << 30,
gpus: 0,
};
let small = revenue_per_budget_hour(540.0, 0.0, 600.0, &need(1.0, 2), &b);
let big = revenue_per_budget_hour(540.0, 0.0, 600.0, &need(4.0, 2), &b);
assert!(small > big * 3.9 && small < big * 4.1, "{small} vs {big}");
let rich = revenue_per_budget_hour(1080.0, 0.0, 600.0, &need(4.0, 2), &b);
assert!((rich / big - 2.0).abs() < 1e-9);
}
#[test]
fn the_dominant_share_and_the_minimum_charge_count() {
let b = BudgetTotals {
cpus: 8.0,
memory_bytes: 32 << 30,
gpus: 1,
};
let cpu_only = revenue_per_budget_hour(100.0, 0.0, 3600.0, &need(1.0, 1), &b);
let whole_gpu = revenue_per_budget_hour(
100.0,
0.0,
3600.0,
&Need {
cpus: 1.0,
memory_bytes: 1 << 30,
gpus: 1,
},
&b,
);
assert!(whole_gpu < cpu_only, "holding the only GPU is a full share");
let metered = revenue_per_budget_hour(540.0, 0.0, 1.0, &need(1.0, 1), &b);
let floor = revenue_per_budget_hour(540.0, 1.0, 1.0, &need(1.0, 1), &b);
assert!(floor > metered);
assert!(revenue_per_budget_hour(1.0, 0.0, 1.0, &need(0.0, 0), &b).is_finite());
}
#[test]
fn ranked_picking_takes_the_best_payer_that_fits() {
let w = |id: &str, cpus, score, seq| (id.to_string(), need(cpus, 1), score, seq);
let waiting = [
w("low", 2.0, 10.0, 0),
w("high", 2.0, 99.0, 1),
w("mid", 2.0, 50.0, 2),
];
assert_eq!(pick_ranked(&waiting, (4000, 8 << 30)), vec!["high", "mid"]);
assert_eq!(pick_ranked(&waiting, (2000, 8 << 30)), vec!["high"]);
let waiting = [w("huge", 8.0, 99.0, 0), w("small", 1.0, 5.0, 1)];
assert_eq!(pick_ranked(&waiting, (2000, 8 << 30)), vec!["small"]);
let waiting = [w("b", 2.0, 5.0, 1), w("a", 2.0, 5.0, 0)];
assert_eq!(pick_ranked(&waiting, (2000, 8 << 30)), vec!["a"]);
}
#[test]
fn a_lone_offer_is_admitted_at_once_and_holds_its_share_until_dropped() {
let b = limits(4000, 8);
let a = Admission::pinned(Policy::default(), Some(b.clone()));
let rev = Revenue::new().with_window(Duration::from_secs(5));
let t = Instant::now();
let g = acquire(&rev, &a, "o1", need(1.0, 1), 1.0, &b).unwrap();
assert!(
t.elapsed() < Duration::from_secs(2),
"a lone offer must not wait"
);
assert_eq!(rev.held().cpus, 1.0);
assert_eq!(rev.held_count(), 1);
drop(g);
assert_eq!(rev.held_count(), 0);
assert_eq!(rev.held().cpus, 0.0);
}
#[test]
fn what_is_held_comes_off_the_budget_and_a_refusal_says_budget() {
let b = limits(4000, 8);
let a = Admission::pinned(Policy::default(), Some(b.clone()));
let rev = Revenue::new().with_window(Duration::ZERO);
let g1 = acquire(&rev, &a, "o1", need(3.0, 2), 1.0, &b).unwrap();
let e = acquire(&rev, &a, "o2", need(2.0, 2), 1.0, &b)
.err()
.unwrap();
assert_eq!(e.reason, Reason::Budget);
assert!(e.detail.contains("cpu"), "{}", e.detail);
assert_eq!(a.snapshot()["refused"]["budget"], 1);
assert_eq!(rev.held_count(), 1, "a refusal holds nothing");
let g2 = acquire(&rev, &a, "o3", need(1.0, 1), 1.0, &b).unwrap();
assert_eq!(rev.held().cpus, 4.0);
drop(g1);
assert!(acquire(&rev, &a, "o4", need(3.0, 1), 1.0, &b).is_ok());
drop(g2);
}
#[test]
fn a_kept_hold_lives_until_released_or_swept() {
let b = limits(4000, 8);
let a = Admission::pinned(Policy::default(), Some(b.clone()));
let rev = Revenue::new().with_window(Duration::ZERO);
acquire(&rev, &a, "two-phase", need(2.0, 1), 1.0, &b)
.unwrap()
.keep();
assert_eq!(rev.held_count(), 1, "reserved, not yet committed");
{
let _commit = rev.release_on_drop("two-phase");
assert_eq!(rev.held_count(), 1, "held while the committed work runs");
}
assert_eq!(rev.held_count(), 0);
acquire(&rev, &a, "lost", need(1.0, 1), 1.0, &b)
.unwrap()
.keep();
rev.sweep(Instant::now());
assert_eq!(rev.held_count(), 1);
rev.sweep(Instant::now() + Duration::from_secs(10 + 121));
assert_eq!(rev.held_count(), 0);
}
#[test]
fn the_better_payer_wins_the_last_slot_when_two_arrive_together() {
let b = limits(2000, 4);
let a = Admission::pinned(Policy::default(), Some(b.clone()));
let rev = Revenue::new().with_window(Duration::from_millis(150));
let hold = acquire(&rev, &a, "base", need(1.0, 1), 1.0, &b).unwrap();
let r1 = rev.clone();
let (a1, b1) = (a.clone(), b.clone());
let cheap = std::thread::spawn(move || {
acquire(&r1, &a1, "cheap", need(1.0, 1), 1.0, &b1).map(|g| {
g.keep();
})
});
std::thread::sleep(Duration::from_millis(20));
let rich = acquire(&rev, &a, "rich", need(1.0, 1), 500.0, &b);
let cheap = cheap.join().unwrap();
assert!(rich.is_ok(), "the better payer is admitted");
let e = cheap.unwrap_err();
assert_eq!(e.reason, Reason::Budget);
assert!(e.detail.contains("better-paying"), "{}", e.detail);
assert_eq!(a.snapshot()["refused"]["budget"], 1);
drop(hold);
}
#[test]
fn eligible_idle_time_is_a_union_with_a_line_per_source() {
let rev = Revenue::new();
let t0 = Instant::now();
let s = |n| Duration::from_secs(n);
rev.eligible_idle_at(Source::SpacesLease, s(15), t0 + s(15));
assert!((rev.eligible_idle_seconds() - 15.0).abs() < 1e-6);
rev.eligible_idle_at(Source::HubAllocation, s(10), t0 + s(15));
assert!((rev.eligible_idle_seconds() - 15.0).abs() < 1e-6);
rev.eligible_idle_at(Source::SpacesLease, s(15), t0 + s(30));
assert!((rev.eligible_idle_seconds() - 30.0).abs() < 1e-6);
let c = rev.counters();
assert_eq!(c["eligible_idle_seconds"], 30.0);
assert_eq!(c["eligible_idle_by_source"]["spaces_lease"], 30.0);
assert_eq!(c["eligible_idle_by_source"]["hub_allocation"], 10.0);
}
#[test]
fn utilisation_is_time_weighted() {
let rev = Revenue::new();
rev.observe_use(Duration::from_secs(30), 0.0);
rev.observe_use(Duration::from_secs(10), 1.0);
let u = rev.counters()["utilisation"].clone();
assert_eq!(u["avg"], 0.25);
assert_eq!(u["now"], 1.0);
}
#[test]
fn the_push_gate_floors_changes_and_ceilings_silence() {
let mut g = PushGate::default();
let t = Instant::now();
let s = |n| Duration::from_secs(n);
assert!(g.due(t, 1), "the first view goes out");
assert!(!g.due(t + s(5), 2), "a change inside the floor waits");
assert!(g.due(t + s(15), 2), "and goes at the floor");
assert!(!g.due(t + s(40), 2), "nothing changed");
assert!(g.due(t + s(75), 2), "the ceiling sends it anyway");
}
#[test]
fn sharing_is_allowed_by_squads_price_and_schedule() {
let noon = LocalTime {
weekday: 2,
minute: 12 * 60,
};
let mut p = Policy::default();
assert!(sharing_allowed(&p, noon));
p.sharing.schedule = vec!["mon-fri 19:00-08:00".into()];
p.sharing.when = "schedule".into();
assert!(!sharing_allowed(&p, noon));
let night = LocalTime {
weekday: 2,
minute: 22 * 60,
};
assert!(sharing_allowed(&p, night));
p.sharing.when = "always".into();
p.sharing.min_price = MinPrice::Withdrawn;
assert!(!sharing_allowed(&p, noon));
let mut p = Policy::default();
p.sharing.squads = Vec::new();
assert!(!sharing_allowed(&p, noon));
}
#[test]
fn lease_slots_follow_the_free_budget() {
let b = limits(16_000, 64);
let unit = need(2.0, 4);
assert_eq!(lease_slots(&b, 0, 0, &unit, 16), 8, "cpu bound: 16 / 2");
assert_eq!(lease_slots(&b, 6000, 0, &unit, 16), 5);
assert_eq!(
lease_slots(&b, 0, 40 << 30, &unit, 16),
6,
"memory bound: 24 / 4"
);
assert_eq!(lease_slots(&b, 0, 0, &unit, 3), 3, "the ceiling holds");
assert_eq!(lease_slots(&b, 16_000, 0, &unit, 16), 0, "budget used up");
assert_eq!(lease_slots(&b, 0, 0, &need(0.0, 0), 16), 16);
assert_eq!(lease_slots(&limits(500, 1), 0, 0, &need(1.0, 1), 16), 0);
}
#[test]
fn preempted_leases_are_counted_by_reason_in_the_counters() {
let rev = Revenue::new();
assert_eq!(rev.counters()["preempted"]["owner_active"], 0);
rev.record_preempted("owner_active", 2);
rev.record_preempted("better_offer", 1);
rev.record_preempted("owner_active", 1);
let c = rev.counters();
assert_eq!(c["preempted"]["owner_active"], 3);
assert_eq!(c["preempted"]["better_offer"], 1);
}
#[test]
fn the_best_refused_offer_is_remembered_once() {
let rev = Revenue::new();
rev.want_room(need(1.0, 1), 10.0);
rev.want_room(need(2.0, 2), 5.0); rev.want_room(need(4.0, 4), 40.0); let w = rev.take_room_wanted().unwrap();
assert_eq!((w.need.cpus, w.price_per_hour), (4.0, 40.0));
assert!(rev.take_room_wanted().is_none());
}
#[test]
fn the_workload_scan_is_reused_for_a_moment_and_feeds_the_held_total() {
let calls = Arc::new(AtomicU64::new(0));
let c = calls.clone();
let rev = Revenue::new().with_workloads(Arc::new(move || {
c.fetch_add(1, Ordering::SeqCst);
vec![Allocation::from_label("zk-sbx", "2000m,1073741824").unwrap()]
}));
assert_eq!(rev.workloads().len(), 1);
let h = rev.held_all();
assert_eq!((h.cpus, h.memory_bytes), (2.0, 1 << 30));
assert_eq!(calls.load(Ordering::SeqCst), 1, "one scan serves both");
}
}