pub mod admit;
pub mod cli;
pub mod migrate;
pub mod owner;
pub mod schedule;
pub mod sync;
use crate::sandbox::{self, Amount, Gpus, HostResources};
use serde::de::Deserializer;
use serde::{Deserialize, Serialize, Serializer};
use serde_json::{json, Value};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime};
pub use admit::Admission;
pub const PUBLIC_SQUAD: &str = "public";
pub const POLICY_FILE: &str = "policy.toml";
const LOCK_FILE: &str = "policy.lock";
pub const ATELIER_BACKENDS: [&str; 2] = ["peft", "ludwig"];
pub fn path(home: &Path) -> PathBuf {
home.join(POLICY_FILE)
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Policy {
pub version: u32,
pub revision: u64,
pub sharing: Sharing,
pub budget: BudgetSpec,
pub workloads: Workloads,
pub runtime: RuntimeCfg,
pub cleanup: Cleanup,
pub upstreams: Upstreams,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Sharing {
pub squads: Vec<String>,
pub when: String,
pub schedule: Vec<String>,
pub min_price: MinPrice,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct BudgetSpec {
#[serde(deserialize_with = "string_or_number")]
pub cpu: String,
#[serde(deserialize_with = "string_or_number")]
pub memory: String,
pub gpu: bool,
#[serde(deserialize_with = "string_or_number")]
pub vram: String,
#[serde(deserialize_with = "string_or_number")]
pub disk: String,
pub on_ac_only: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Workloads {
pub environments: bool,
pub spaces: bool,
pub hub_serving: bool,
pub atelier: Atelier,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct RuntimeCfg {
pub prefer: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Cleanup {
pub idle_ttl: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Upstreams {
pub hub: bool,
pub peers: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum MinPrice {
Default,
PerHour(f64),
Withdrawn,
}
impl Serialize for MinPrice {
fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
match self {
MinPrice::Default => s.serialize_str("default"),
MinPrice::PerHour(p) => s.serialize_f64(*p),
MinPrice::Withdrawn => s.serialize_i64(-1),
}
}
}
impl<'de> Deserialize<'de> for MinPrice {
fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum Raw {
S(String),
I(i64),
F(f64),
}
match Raw::deserialize(d)? {
Raw::S(s) if s.trim().eq_ignore_ascii_case("default") => Ok(MinPrice::Default),
Raw::S(s) => Err(serde::de::Error::custom(format!(
"min_price {s:?}: use \"default\", a price per hour, or -1"
))),
Raw::I(-1) => Ok(MinPrice::Withdrawn),
Raw::I(n) => Ok(MinPrice::PerHour(n as f64)),
Raw::F(-1.0) => Ok(MinPrice::Withdrawn),
Raw::F(f) => Ok(MinPrice::PerHour(f)),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum Atelier {
On,
Off,
GpuOnly,
Backends(Vec<String>),
}
impl Serialize for Atelier {
fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
match self {
Atelier::On => s.serialize_bool(true),
Atelier::Off => s.serialize_bool(false),
Atelier::GpuOnly => s.serialize_str("gpu-only"),
Atelier::Backends(v) => v.serialize(s),
}
}
}
impl<'de> Deserialize<'de> for Atelier {
fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum Raw {
B(bool),
S(String),
L(Vec<String>),
}
match Raw::deserialize(d)? {
Raw::B(true) => Ok(Atelier::On),
Raw::B(false) => Ok(Atelier::Off),
Raw::S(s) if s == "gpu-only" => Ok(Atelier::GpuOnly),
Raw::S(s) => Err(serde::de::Error::custom(format!(
"workloads.atelier {s:?}: use true, false, \"gpu-only\" or a list of backends"
))),
Raw::L(v) => Ok(Atelier::Backends(v)),
}
}
}
fn string_or_number<'de, D: Deserializer<'de>>(d: D) -> Result<String, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum Raw {
S(String),
I(i64),
F(f64),
}
Ok(match Raw::deserialize(d)? {
Raw::S(s) => s,
Raw::I(n) => n.to_string(),
Raw::F(f) => f.to_string(),
})
}
fn default_prefer() -> Vec<String> {
let v: &[&str] = if cfg!(target_os = "macos") {
&["lima", "docker"]
} else {
&["lxc", "docker"]
};
v.iter().map(|s| s.to_string()).collect()
}
impl Default for Policy {
fn default() -> Self {
Self {
version: 1,
revision: 0,
sharing: Sharing::default(),
budget: BudgetSpec::default(),
workloads: Workloads::default(),
runtime: RuntimeCfg::default(),
cleanup: Cleanup::default(),
upstreams: Upstreams::default(),
}
}
}
impl Default for Sharing {
fn default() -> Self {
Self {
squads: vec![PUBLIC_SQUAD.to_string()],
when: "idle".into(),
schedule: Vec::new(),
min_price: MinPrice::Default,
}
}
}
impl Default for BudgetSpec {
fn default() -> Self {
Self {
cpu: "50%".into(),
memory: "50%".into(),
gpu: true,
vram: "0".into(),
disk: "50G".into(),
on_ac_only: true,
}
}
}
impl Default for Workloads {
fn default() -> Self {
Self {
environments: true,
spaces: true,
hub_serving: true,
atelier: Atelier::On,
}
}
}
impl Default for RuntimeCfg {
fn default() -> Self {
Self {
prefer: default_prefer(),
}
}
}
impl Default for Cleanup {
fn default() -> Self {
Self {
idle_ttl: "7d".into(),
}
}
}
impl Default for Upstreams {
fn default() -> Self {
Self {
hub: true,
peers: Vec::new(),
}
}
}
fn parse_vram(v: &str) -> Result<u64, String> {
let t = v.trim();
if t == "0" || t.is_empty() {
return Ok(0);
}
match sandbox::parse_memory(t)? {
Amount::Absolute(n) => Ok(n as u64),
Amount::Percent(_) => Err(format!("{t:?}: give a size such as 8G, not a percentage")),
}
}
pub fn parse_ttl(v: &str) -> Result<Duration, String> {
let v = v.trim();
let (num, unit) = v.split_at(v.len().saturating_sub(1));
let mult = match unit {
"d" => 86_400,
"h" => 3_600,
"m" => 60,
"s" => 1,
_ => return Err(format!("{v:?}: use 7d, 36h, 90m or 600s")),
};
num.parse::<u64>()
.ok()
.filter(|n| *n > 0)
.map(|n| Duration::from_secs(n * mult))
.ok_or_else(|| format!("{v:?}: use 7d, 36h, 90m or 600s"))
}
impl Policy {
pub fn validate(&self) -> Result<(), String> {
if self.version != 1 {
return Err(format!(
"version = {}: only version 1 is known",
self.version
));
}
let mut seen = std::collections::HashSet::new();
for s in &self.sharing.squads {
if s.trim().is_empty() || s.len() > 64 || s.contains(char::is_whitespace) {
return Err(format!("sharing.squads: {s:?} is not a squad id"));
}
if !seen.insert(s) {
return Err(format!("sharing.squads: {s:?} is listed twice"));
}
}
if !matches!(self.sharing.when.as_str(), "idle" | "always" | "schedule") {
return Err(format!(
"sharing.when = {:?}: use idle, always or schedule",
self.sharing.when
));
}
for w in &self.sharing.schedule {
schedule::parse_window(w).map_err(|e| format!("sharing.schedule: {e}"))?;
}
if let MinPrice::PerHour(p) = self.sharing.min_price {
if !p.is_finite() || p < 0.0 {
return Err(format!(
"sharing.min_price = {p}: a price per hour is >= 0 (or -1 to withdraw)"
));
}
}
sandbox::parse_cpu(&self.budget.cpu).map_err(|e| format!("budget.cpu: {e}"))?;
sandbox::parse_memory(&self.budget.memory).map_err(|e| format!("budget.memory: {e}"))?;
parse_vram(&self.budget.vram).map_err(|e| format!("budget.vram: {e}"))?;
match sandbox::parse_memory(&self.budget.disk).map_err(|e| format!("budget.disk: {e}"))? {
Amount::Absolute(n) if n >= (1u64 << 30) as f64 => {}
Amount::Absolute(_) => {
return Err("budget.disk: below the 1 GiB minimum for the sandbox disk".into())
}
Amount::Percent(_) => {
return Err("budget.disk: give a size such as 50G, not a percentage".into())
}
}
if let Atelier::Backends(list) = &self.workloads.atelier {
for b in list {
if !ATELIER_BACKENDS.contains(&b.as_str()) {
return Err(format!(
"workloads.atelier: unknown backend {b:?} (known: {})",
ATELIER_BACKENDS.join(", ")
));
}
}
}
for r in &self.runtime.prefer {
if sandbox::RuntimeKind::parse(r).is_none() {
return Err(format!("runtime.prefer: unknown runtime {r:?}"));
}
}
parse_ttl(&self.cleanup.idle_ttl).map_err(|e| format!("cleanup.idle_ttl: {e}"))?;
for p in &self.upstreams.peers {
if p.trim().is_empty() {
return Err("upstreams.peers: an empty entry".into());
}
}
Ok(())
}
pub fn validate_for(&self, host: &HostResources) -> Result<(), String> {
self.validate()?;
if let Amount::Absolute(c) = sandbox::parse_cpu(&self.budget.cpu)? {
if c > host.logical_cpus.max(1) as f64 {
return Err(format!(
"budget.cpu = {}: this machine has {} logical CPUs",
self.budget.cpu, host.logical_cpus
));
}
}
if let (Amount::Absolute(m), Some(total)) = (
sandbox::parse_memory(&self.budget.memory)?,
host.memory_bytes,
) {
if m > total as f64 {
return Err(format!(
"budget.memory = {}: this machine has {} bytes",
self.budget.memory, total
));
}
}
Ok(())
}
pub fn sandbox_budget(&self) -> Result<sandbox::Budget, String> {
let disk_bytes = match sandbox::parse_memory(&self.budget.disk)
.map_err(|e| format!("budget.disk: {e}"))?
{
Amount::Absolute(n) => n as u64,
Amount::Percent(_) => {
return Err("budget.disk: give a size such as 50G, not a percentage".into())
}
};
Ok(sandbox::Budget {
cpu: sandbox::parse_cpu(&self.budget.cpu).map_err(|e| format!("budget.cpu: {e}"))?,
memory: sandbox::parse_memory(&self.budget.memory)
.map_err(|e| format!("budget.memory: {e}"))?,
gpu: if self.budget.gpu {
Gpus::All
} else {
Gpus::None
},
disk_bytes,
})
}
pub fn effective_budget(
&self,
get: &dyn Fn(&str) -> Option<String>,
) -> Result<sandbox::Budget, String> {
sandbox::Budget::from_env_over(self.sandbox_budget()?, get)
}
pub fn atelier_backends(&self, supported: &[String]) -> Vec<String> {
match &self.workloads.atelier {
Atelier::Off => Vec::new(),
Atelier::On | Atelier::GpuOnly => supported.to_vec(),
Atelier::Backends(v) => v.clone(),
}
}
pub fn to_hub_json(&self) -> Value {
let atelier = match &self.workloads.atelier {
Atelier::Backends(v) if v.is_empty() => json!(false),
Atelier::Backends(_) => json!(true),
other => serde_json::to_value(other).unwrap_or(json!(false)),
};
json!({
"version": self.version,
"revision": self.revision,
"sharing": {
"squads": self.sharing.squads,
"when": self.sharing.when,
"schedule": self.sharing.schedule,
"min_price": self.sharing.min_price,
},
"budget": {
"cpu": self.budget.cpu,
"memory": self.budget.memory,
"gpu": self.budget.gpu,
"vram": self.budget.vram,
"disk": self.budget.disk,
},
"workloads": {
"environments": self.workloads.environments,
"spaces": self.workloads.spaces,
"hub_serving": self.workloads.hub_serving,
"atelier": atelier,
},
"runtime": { "prefer": self.runtime.prefer },
})
}
pub fn idle_ttl(&self) -> Result<Duration, String> {
parse_ttl(&self.cleanup.idle_ttl).map_err(|e| format!("cleanup.idle_ttl={e}"))
}
pub fn merged(&self, patch: &Value) -> Result<Policy, String> {
let mut base = serde_json::to_value(self).map_err(|e| e.to_string())?;
let mut patch = patch.clone();
if let Some(o) = patch.as_object_mut() {
o.remove("revision");
}
merge_json(&mut base, &patch);
serde_json::from_value(base).map_err(|e| e.to_string())
}
pub fn same_content(&self, other: &Policy) -> bool {
let mut a = self.clone();
a.revision = other.revision;
a == *other
}
}
fn merge_json(base: &mut Value, patch: &Value) {
match (base, patch) {
(Value::Object(b), Value::Object(p)) => {
for (k, v) in p {
match b.get_mut(k) {
Some(slot) if slot.is_object() && v.is_object() => merge_json(slot, v),
_ => {
b.insert(k.clone(), v.clone());
}
}
}
}
(b, p) => *b = p.clone(),
}
}
pub fn known_keys_only(body: &Value) -> Value {
fn walk(template: &Value, v: &Value) -> Value {
match (template, v) {
(Value::Object(t), Value::Object(o)) => {
let mut out = serde_json::Map::new();
for (k, val) in o {
if let Some(sub) = t.get(k) {
out.insert(k.clone(), walk(sub, val));
}
}
Value::Object(out)
}
_ => v.clone(),
}
}
let template = serde_json::to_value(Policy::default()).unwrap_or(Value::Null);
walk(&template, body)
}
pub struct Lock {
#[cfg(unix)]
file: std::fs::File,
}
#[cfg(unix)]
impl Drop for Lock {
fn drop(&mut self) {
use std::os::unix::io::AsRawFd;
unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) };
}
}
pub fn lock(home: &Path) -> Result<Lock, String> {
std::fs::create_dir_all(home).map_err(|e| format!("{}: {e}", home.display()))?;
#[cfg(unix)]
{
use std::os::unix::io::AsRawFd;
let p = home.join(LOCK_FILE);
let file = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(&p)
.map_err(|e| format!("{}: {e}", p.display()))?;
let deadline = Instant::now() + Duration::from_secs(5);
loop {
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
if rc == 0 {
return Ok(Lock { file });
}
if Instant::now() >= deadline {
return Err(format!(
"{}: another zc process holds the policy lock",
p.display()
));
}
std::thread::sleep(Duration::from_millis(25));
}
}
#[cfg(not(unix))]
{
Ok(Lock {})
}
}
fn read_file(home: &Path) -> Result<Option<Policy>, String> {
let p = path(home);
match std::fs::read_to_string(&p) {
Ok(s) => toml::from_str::<Policy>(&s)
.map(Some)
.map_err(|e| format!("{}: {e}", p.display())),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(format!("{}: {e}", p.display())),
}
}
pub fn peek(home: &Path) -> Option<Policy> {
read_file(home).ok().flatten()
}
const HEADER: &str =
"# Zakuro sharing policy (ADR ZC-0017). One file decides what this node runs\n\
# for other people. Change it with `zc policy set <key>=<value>`, `zc budget`\n\
# or the app; `revision` is bumped by every change.\n\n";
fn write_file(home: &Path, p: &Policy) -> Result<(), String> {
let dest = path(home);
let body = toml::to_string_pretty(p).map_err(|e| e.to_string())?;
let tmp = home.join(format!(".policy.toml.{}", std::process::id()));
{
use std::io::Write;
let mut opts = std::fs::OpenOptions::new();
opts.write(true).create(true).truncate(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.mode(0o600);
}
let mut f = opts
.open(&tmp)
.map_err(|e| format!("{}: {e}", tmp.display()))?;
f.write_all(HEADER.as_bytes()).map_err(|e| e.to_string())?;
f.write_all(body.as_bytes()).map_err(|e| e.to_string())?;
f.sync_all().map_err(|e| e.to_string())?;
}
std::fs::rename(&tmp, &dest).map_err(|e| format!("{}: {e}", dest.display()))
}
pub fn load(home: &Path) -> Result<Policy, String> {
if let Some(p) = read_file(home)? {
return Ok(p);
}
let _guard = lock(home)?;
if let Some(p) = read_file(home)? {
return Ok(p);
}
let mut p = migrate::from_legacy(&migrate::Legacy::read(home, &|k| std::env::var(k).ok()))?;
p.revision = 1;
write_file(home, &p)?;
migrate::retire_hosting_toml(home);
Ok(p)
}
#[derive(Debug, Clone, PartialEq)]
pub enum UpdateError {
Stale {
current: u64,
},
Invalid(String),
Io(String),
}
impl std::fmt::Display for UpdateError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
UpdateError::Stale { current } => write!(
f,
"the policy changed since you read it (revision is now {current}); read it again"
),
UpdateError::Invalid(m) | UpdateError::Io(m) => f.write_str(m),
}
}
}
pub fn update(
home: &Path,
expected: Option<u64>,
host: Option<&HostResources>,
f: impl FnOnce(&mut Policy) -> Result<(), String>,
) -> Result<(Policy, bool), UpdateError> {
let current = load(home).map_err(UpdateError::Io)?;
let _guard = lock(home).map_err(UpdateError::Io)?;
let current = read_file(home).map_err(UpdateError::Io)?.unwrap_or(current);
if let Some(e) = expected {
if e != current.revision {
return Err(UpdateError::Stale {
current: current.revision,
});
}
}
let mut next = current.clone();
f(&mut next).map_err(UpdateError::Invalid)?;
match host {
Some(h) => next.validate_for(h),
None => next.validate(),
}
.map_err(UpdateError::Invalid)?;
if next.same_content(¤t) {
return Ok((current, false));
}
next.revision = current.revision + 1;
write_file(home, &next).map_err(UpdateError::Io)?;
Ok((next, true))
}
pub fn update_partial(
home: &Path,
patch: &Value,
host: Option<&HostResources>,
) -> Result<(Policy, bool), UpdateError> {
let expected = patch.get("revision").and_then(Value::as_u64);
update(home, expected, host, |p| {
*p = p.merged(patch)?;
Ok(())
})
}
pub fn apply_desired(
home: &Path,
desired: &Value,
host: &HostResources,
) -> Result<Option<Policy>, String> {
let Some(rev) = desired.get("revision").and_then(Value::as_u64) else {
return Err("the staged policy has no revision".into());
};
let current = load(home)?;
if rev <= current.revision {
return Ok(None);
}
let _guard = lock(home)?;
let current = read_file(home)?.unwrap_or(current);
if rev <= current.revision {
return Ok(None);
}
let mut next = current.merged(&known_keys_only(desired))?;
next.revision = rev;
next.validate_for(host)?;
write_file(home, &next)?;
Ok(Some(next))
}
pub struct Live {
home: Mutex<PathBuf>,
inner: Mutex<LiveInner>,
pinned: Mutex<Option<Arc<Policy>>>,
}
struct LiveInner {
policy: Option<Arc<Policy>>,
mtime: Option<SystemTime>,
home: Option<PathBuf>,
checked: Option<Instant>,
warned: bool,
}
const RECHECK: Duration = Duration::from_secs(1);
impl Live {
pub fn process() -> Self {
Self::at(PathBuf::new())
}
pub fn at(home: PathBuf) -> Self {
Self {
home: Mutex::new(home),
inner: Mutex::new(LiveInner {
policy: None,
mtime: None,
home: None,
checked: None,
warned: false,
}),
pinned: Mutex::new(None),
}
}
pub fn pin(&self, p: Policy) {
*self.pinned.lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(p));
}
fn home(&self) -> Option<PathBuf> {
let h = self.home.lock().unwrap_or_else(|e| e.into_inner()).clone();
if h.as_os_str().is_empty() {
crate::credentials::dir()
} else {
Some(h)
}
}
pub fn current(&self) -> Arc<Policy> {
if let Some(p) = self
.pinned
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone()
{
return p;
}
let Some(home) = self.home() else {
return Arc::new(Policy::default());
};
let mut g = self.inner.lock().unwrap_or_else(|e| e.into_inner());
let moved = g.home.as_deref() != Some(home.as_path());
let due = g.checked.is_none_or(|t| t.elapsed() >= RECHECK);
if g.policy.is_some() && !moved && !due {
return g.policy.clone().unwrap();
}
g.checked = Some(Instant::now());
let mtime = std::fs::metadata(path(&home))
.and_then(|m| m.modified())
.ok();
if g.policy.is_some() && !moved && mtime == g.mtime {
return g.policy.clone().unwrap();
}
match load(&home) {
Ok(p) => {
g.policy = Some(Arc::new(p));
g.mtime = std::fs::metadata(path(&home))
.and_then(|m| m.modified())
.ok();
g.home = Some(home);
g.warned = false;
}
Err(e) => {
if !g.warned {
eprintln!(" [POLICY] {e} -- keeping the last good policy");
g.warned = true;
}
if g.policy.is_none() || moved {
g.policy = Some(Arc::new(Policy::default()));
g.home = Some(home);
}
}
}
g.policy.clone().unwrap()
}
}
pub fn node_budget(
home: Option<&Path>,
get: &dyn Fn(&str) -> Option<String>,
) -> Result<sandbox::Budget, String> {
let base = match home {
Some(h) => load(h)?,
None => Policy::default(),
};
warn_env_budget_once(get);
base.effective_budget(get)
}
pub const ENV_BUDGET: [&str; 4] = [
"ZAKURO_BUDGET_CPU",
"ZAKURO_BUDGET_MEMORY",
"ZAKURO_BUDGET_GPU",
"ZAKURO_BUDGET_DISK",
];
pub fn env_budget_overrides(get: &dyn Fn(&str) -> Option<String>) -> Vec<&'static str> {
ENV_BUDGET
.iter()
.copied()
.filter(|k| get(k).is_some_and(|v| !v.trim().is_empty()))
.collect()
}
fn warn_env_budget_once(get: &dyn Fn(&str) -> Option<String>) {
static ONCE: std::sync::Once = std::sync::Once::new();
let set = env_budget_overrides(get);
if !set.is_empty() {
ONCE.call_once(|| {
eprintln!(
" [POLICY] {} set: deprecated, [budget] in policy.toml is the setting now \
(the variable still wins while it is set)",
set.join(", ")
);
});
}
}
#[cfg(test)]
pub(crate) fn test_home() -> PathBuf {
let d = std::env::temp_dir().join(format!(
"zc-policy-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&d).unwrap();
d
}
#[cfg(test)]
mod tests {
use super::*;
fn host() -> HostResources {
HostResources {
logical_cpus: 8,
memory_bytes: Some(16 << 30),
}
}
#[test]
fn a_fresh_node_gets_the_adr_defaults() {
let home = test_home();
let p = load(&home).unwrap();
assert_eq!(p.revision, 1);
assert_eq!(p.sharing.squads, vec!["public".to_string()]);
assert_eq!(p.sharing.when, "idle");
assert_eq!(p.sharing.min_price, MinPrice::Default);
assert_eq!(
(p.budget.cpu.as_str(), p.budget.memory.as_str()),
("50%", "50%")
);
assert!(p.budget.gpu, "design D2: GPU sharing on by default");
assert!(p.workloads.environments && p.workloads.spaces && p.workloads.hub_serving);
assert_eq!(p.workloads.atelier, Atelier::On);
assert!(p.upstreams.hub && p.upstreams.peers.is_empty());
assert_eq!(read_file(&home).unwrap().unwrap(), p);
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = std::fs::metadata(path(&home)).unwrap().permissions().mode();
assert_eq!(mode & 0o777, 0o600);
}
}
#[test]
fn the_defaults_file_round_trips_as_toml() {
let text = toml::to_string_pretty(&Policy::default()).unwrap();
let back: Policy = toml::from_str(&text).unwrap();
assert_eq!(back, Policy::default());
assert!(text.contains("atelier = true"), "{text}");
assert!(text.contains("min_price = \"default\""), "{text}");
}
#[test]
fn an_unknown_key_is_an_error_naming_it() {
for (doc, key) in [
("version = 1\nfoo = 2\n", "foo"),
("[sharing]\nsquad = [\"public\"]\n", "squad"),
("[budget]\ncores = 2\n", "cores"),
("[upstreams]\nmesh = true\n", "mesh"),
] {
let e = toml::from_str::<Policy>(doc).unwrap_err().to_string();
assert!(e.contains(key), "{e}");
}
let home = test_home();
std::fs::write(path(&home), "[workloads]\nbogus = true\n").unwrap();
let e = load(&home).unwrap_err();
assert!(e.contains("bogus") && e.contains("policy.toml"), "{e}");
}
#[test]
fn min_price_and_atelier_accept_every_documented_form() {
let p: Policy =
toml::from_str("[sharing]\nmin_price = 2.5\n[workloads]\natelier = [\"peft\"]\n")
.unwrap();
assert_eq!(p.sharing.min_price, MinPrice::PerHour(2.5));
assert_eq!(p.workloads.atelier, Atelier::Backends(vec!["peft".into()]));
let p: Policy =
toml::from_str("[sharing]\nmin_price = -1\n[workloads]\natelier = false\n").unwrap();
assert_eq!(p.sharing.min_price, MinPrice::Withdrawn);
assert_eq!(p.workloads.atelier, Atelier::Off);
let p: Policy =
toml::from_str("[sharing]\nmin_price = 3\n[workloads]\natelier = true\n").unwrap();
assert_eq!(p.sharing.min_price, MinPrice::PerHour(3.0));
assert_eq!(p.workloads.atelier, Atelier::On);
assert!(toml::from_str::<Policy>("[sharing]\nmin_price = \"cheap\"\n").is_err());
assert!(toml::from_str::<Policy>("[workloads]\natelier = \"sometimes\"\n").is_err());
}
#[test]
fn a_budget_given_as_a_number_is_kept_as_text() {
let p: Policy = toml::from_str("[budget]\ncpu = 4\nmemory = 8589934592\n").unwrap();
assert_eq!(p.budget.cpu, "4");
assert!(p.validate().is_ok());
}
#[test]
fn validation_names_the_key() {
let mut p = Policy::default();
p.sharing.when = "sometimes".into();
assert!(p.validate().unwrap_err().contains("sharing.when"));
let mut p = Policy::default();
p.sharing.schedule = vec!["mon 25:00-26:00".into()];
assert!(p.validate().unwrap_err().contains("sharing.schedule"));
let mut p = Policy::default();
p.budget.cpu = "lots".into();
assert!(p.validate().unwrap_err().contains("budget.cpu"));
let mut p = Policy::default();
p.budget.disk = "50%".into();
assert!(p.validate().unwrap_err().contains("budget.disk"));
let mut p = Policy::default();
p.sharing.min_price = MinPrice::PerHour(-3.0);
assert!(p.validate().unwrap_err().contains("min_price"));
let mut p = Policy::default();
p.workloads.atelier = Atelier::Backends(vec!["tensorflow".into()]);
assert!(p.validate().unwrap_err().contains("tensorflow"));
let mut p = Policy::default();
p.sharing.squads = vec!["public".into(), "public".into()];
assert!(p.validate().unwrap_err().contains("twice"));
}
#[test]
fn a_budget_past_the_hardware_is_refused_for_this_host() {
let mut p = Policy::default();
p.budget.cpu = "64".into();
assert!(p.validate().is_ok());
assert!(p.validate_for(&host()).unwrap_err().contains("8 logical"));
p.budget.cpu = "4".into();
p.budget.memory = "64G".into();
assert!(p
.validate_for(&host())
.unwrap_err()
.contains("budget.memory"));
p.budget.memory = "8G".into();
assert!(p.validate_for(&host()).is_ok());
}
#[test]
fn update_bumps_the_revision_only_when_something_changed() {
let home = test_home();
let r0 = load(&home).unwrap().revision;
let (p, changed) = update(&home, None, None, |p| {
p.sharing.when = "always".into();
Ok(())
})
.unwrap();
assert!(changed);
assert_eq!(p.revision, r0 + 1);
let (p2, changed) = update(&home, None, None, |p| {
p.sharing.when = "always".into();
Ok(())
})
.unwrap();
assert!(!changed);
assert_eq!(p2.revision, p.revision);
assert_eq!(load(&home).unwrap().sharing.when, "always");
}
#[test]
fn a_stale_revision_is_refused_and_nothing_is_written() {
let home = test_home();
let r = load(&home).unwrap().revision;
update(&home, Some(r), None, |p| {
p.budget.gpu = false;
Ok(())
})
.unwrap();
let e = update(&home, Some(r), None, |p| {
p.budget.gpu = true;
Ok(())
})
.unwrap_err();
assert_eq!(e, UpdateError::Stale { current: r + 1 });
assert!(!load(&home).unwrap().budget.gpu);
}
#[test]
fn an_invalid_edit_is_refused_and_leaves_the_file_alone() {
let home = test_home();
let before = load(&home).unwrap();
let e = update(&home, None, None, |p| {
p.budget.memory = "plenty".into();
Ok(())
})
.unwrap_err();
assert!(matches!(e, UpdateError::Invalid(m) if m.contains("budget.memory")));
assert_eq!(load(&home).unwrap(), before);
}
#[test]
fn a_partial_document_merges_and_an_unknown_key_is_named() {
let home = test_home();
let r = load(&home).unwrap().revision;
let (p, changed) = update_partial(
&home,
&json!({"revision": r, "sharing": {"min_price": 2.0}, "workloads": {"spaces": false}}),
None,
)
.unwrap();
assert!(changed);
assert_eq!(p.sharing.min_price, MinPrice::PerHour(2.0));
assert!(!p.workloads.spaces);
assert_eq!(
p.sharing.squads,
vec!["public".to_string()],
"untouched keys stay"
);
let e = update_partial(&home, &json!({"sharing": {"squad": []}}), None).unwrap_err();
assert!(matches!(e, UpdateError::Invalid(m) if m.contains("squad")));
let e = update_partial(&home, &json!({"revision": r}), None).unwrap_err();
assert!(matches!(e, UpdateError::Stale { .. }));
}
#[test]
fn concurrent_writers_never_lose_an_update() {
let home = test_home();
load(&home).unwrap();
let handles: Vec<_> = (0..8)
.map(|i| {
let home = home.clone();
std::thread::spawn(move || {
update(&home, None, None, |p| {
p.sharing.squads = vec![format!("squad-{i}")];
Ok(())
})
.unwrap()
})
})
.collect();
for h in handles {
h.join().unwrap();
}
assert_eq!(load(&home).unwrap().revision, 9);
}
#[test]
fn a_staged_policy_applies_only_when_newer_and_valid() {
let home = test_home();
let r = load(&home).unwrap().revision;
let desired = json!({"revision": r + 5, "sharing": {"min_price": 4.0},
"from_the_future": {"x": 1}});
let p = apply_desired(&home, &desired, &host()).unwrap().unwrap();
assert_eq!(p.revision, r + 5, "the hub's revision is kept");
assert_eq!(p.sharing.min_price, MinPrice::PerHour(4.0));
assert!(apply_desired(&home, &json!({"revision": r + 5}), &host())
.unwrap()
.is_none());
assert!(apply_desired(&home, &json!({"revision": 1}), &host())
.unwrap()
.is_none());
let e = apply_desired(
&home,
&json!({"revision": r + 9, "budget": {"cpu": "99"}}),
&host(),
)
.unwrap_err();
assert!(e.contains("budget.cpu"), "{e}");
assert_eq!(load(&home).unwrap().revision, r + 5);
}
#[test]
fn the_live_view_follows_the_file_and_keeps_the_last_good_policy() {
let home = test_home();
let live = Live::at(home.clone());
assert_eq!(live.current().sharing.when, "idle");
update(&home, None, None, |p| {
p.sharing.when = "always".into();
Ok(())
})
.unwrap();
live.inner.lock().unwrap().checked = None;
assert_eq!(live.current().sharing.when, "always");
std::fs::write(path(&home), "this is = not [valid").unwrap();
live.inner.lock().unwrap().checked = None;
live.inner.lock().unwrap().mtime = None;
assert_eq!(live.current().sharing.when, "always", "last good kept");
}
#[test]
fn env_budget_overrides_the_file_and_is_listed() {
let mut p = Policy::default();
p.budget.memory = "8G".into();
let env = |k: &str| match k {
"ZAKURO_BUDGET_MEMORY" => Some("2g".to_string()),
_ => None,
};
let b = p.effective_budget(&env).unwrap();
assert_eq!(b.memory, Amount::Absolute(2.0 * 1024.0 * 1024.0 * 1024.0));
assert_eq!(b.cpu, Amount::Percent(50.0));
assert_eq!(b.disk_bytes, 50 << 30, "the file's disk is the base");
let none = |_: &str| None;
assert_eq!(
p.effective_budget(&none).unwrap().memory,
Amount::Absolute(8.0 * 1024.0 * 1024.0 * 1024.0)
);
assert_eq!(env_budget_overrides(&env), vec!["ZAKURO_BUDGET_MEMORY"]);
let bad = |k: &str| (k == "ZAKURO_BUDGET_CPU").then(|| "lots".to_string());
assert!(p.effective_budget(&bad).is_err());
}
#[test]
fn gpu_in_the_file_becomes_all_gpus_and_a_deploy_pin_still_wins() {
let mut p = Policy::default();
p.budget.gpu = true;
let none = |_: &str| None;
assert_eq!(p.effective_budget(&none).unwrap().gpu, Gpus::All);
let pin = |k: &str| (k == "ZAKURO_DEPLOY_GPUS").then(|| "device=0".to_string());
assert_eq!(
p.effective_budget(&pin).unwrap().gpu,
Gpus::Spec("device=0".into())
);
let off = |k: &str| (k == "ZAKURO_BUDGET_GPU").then(|| "0".to_string());
assert_eq!(p.effective_budget(&off).unwrap().gpu, Gpus::None);
}
#[test]
fn the_hub_object_has_the_documented_shape() {
let mut p = Policy {
revision: 7,
..Default::default()
};
p.workloads.atelier = Atelier::Backends(vec!["peft".into()]);
p.sharing.min_price = MinPrice::Withdrawn;
let v = p.to_hub_json();
assert_eq!(v["revision"], 7);
assert_eq!(v["sharing"]["squads"], json!(["public"]));
assert_eq!(v["sharing"]["min_price"], -1);
assert_eq!(v["budget"]["vram"], "0");
assert_eq!(v["workloads"]["atelier"], true, "a list is sent as on");
assert!(v["runtime"]["prefer"].is_array());
assert!(v.get("upstreams").is_none() && v.get("cleanup").is_none());
}
}