use chrono::{DateTime, Duration, Utc};
use dashmap::DashMap;
use serde::{Deserialize, Serialize};
use std::collections::{HashSet, VecDeque};
use std::sync::Mutex;
pub const MODEL_WILDCARD: &str = "*";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ProviderType {
General,
#[default]
Specialized,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum WorkerStatus {
Healthy,
Busy,
Unhealthy,
Draining,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkerResources {
pub cpus_total: f64,
pub cpus_available: f64,
pub memory_total: u64,
pub memory_available: u64,
pub gpus_total: u32,
pub gpus_available: u32,
}
impl Default for WorkerResources {
fn default() -> Self {
Self {
cpus_total: 1.0,
cpus_available: 1.0,
memory_total: 1024 * 1024 * 1024, memory_available: 1024 * 1024 * 1024,
gpus_total: 0,
gpus_available: 0,
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct BrokerResources {
pub cpus_available: u64,
pub memory_available: u64,
pub gpus_available: u32,
pub healthy_workers: u32,
}
pub(crate) fn default_price_per_hour() -> f64 {
540.0
}
fn default_min_charge() -> f64 {
0.15
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkerPricing {
#[serde(default = "default_price_per_hour")]
pub price_per_hour: f64,
#[serde(default = "default_min_charge")]
pub min_charge: f64,
}
impl Default for WorkerPricing {
fn default() -> Self {
Self {
price_per_hour: default_price_per_hour(), min_charge: default_min_charge(), }
}
}
impl WorkerPricing {
pub fn estimate_cost(&self, duration_secs: f64) -> f64 {
(self.price_per_hour / 3600.0 * duration_secs).max(self.min_charge)
}
pub fn price_score(&self) -> f64 {
self.estimate_cost(10.0)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkerQuotas {
pub per_5h: u64,
pub per_week: u64,
pub per_month: u64,
}
impl Default for WorkerQuotas {
fn default() -> Self {
Self {
per_5h: std::env::var("ZAKURO_QUOTA_5H")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(0),
per_week: std::env::var("ZAKURO_QUOTA_WEEK")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(0),
per_month: std::env::var("ZAKURO_QUOTA_MONTH")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(0),
}
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct HardwareInfo {
#[serde(default)]
pub gpu_model: Option<String>,
#[serde(default)]
pub gpu_vram_gb: Option<u32>,
#[serde(default)]
pub cpu_model: Option<String>,
#[serde(default)]
pub storage_gb: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ram_gb: Option<f64>,
#[serde(default)]
pub python_version: Option<String>,
#[serde(default)]
pub python_implementation: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WorkerName<'a>(&'a str);
impl<'a> WorkerName<'a> {
pub fn of(worker: &'a Worker) -> Self {
Self(&worker.name)
}
pub fn reported(name: &'a str) -> Self {
Self(name)
}
pub fn none() -> Self {
Self("")
}
pub fn as_str(&self) -> &'a str {
self.0
}
}
impl std::fmt::Display for WorkerName<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.0)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Worker {
pub id: String,
pub name: String,
pub uri: String,
pub worker_type: String,
pub status: WorkerStatus,
pub resources: WorkerResources,
pub pricing: WorkerPricing,
pub last_heartbeat: DateTime<Utc>,
pub active_requests: u32,
pub total_requests: u64,
pub avg_latency_ms: f64,
#[serde(skip)]
pub stats: crate::broker::estimator::Stats,
#[serde(skip)]
pub warmup_pending: u32,
pub tags: Vec<String>,
#[serde(default)]
pub max_timeout_secs: f64,
#[serde(default)]
pub hardware: HardwareInfo,
#[serde(default)]
pub wireguard_ip: Option<String>,
#[serde(default)]
pub is_docker: Option<bool>,
#[serde(default)]
pub source_node: Option<String>,
#[serde(default)]
pub explicit_local: bool,
#[serde(default)]
pub node_fp: String,
#[serde(default)]
pub slot: String,
#[serde(default)]
pub provider_type: ProviderType,
#[serde(default)]
pub served_models: Vec<String>,
#[serde(default)]
pub price_per_mtok: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reported_price_per_hour: Option<f64>,
}
impl Worker {
pub fn zc_uri(&self) -> String {
format!("zc://worker-{}-{}", self.node_fp, self.slot)
}
pub fn new(id: String, name: String, uri: String, worker_type: String) -> Self {
Self {
stats: crate::broker::estimator::Stats::new(),
warmup_pending: 0,
id,
name,
uri,
worker_type,
status: WorkerStatus::Healthy,
resources: WorkerResources::default(),
pricing: WorkerPricing::default(),
last_heartbeat: Utc::now(),
active_requests: 0,
total_requests: 0,
avg_latency_ms: 0.0,
tags: Vec::new(),
max_timeout_secs: 0.0, hardware: HardwareInfo::default(),
wireguard_ip: None,
is_docker: None,
source_node: None,
explicit_local: false,
node_fp: String::new(),
slot: String::new(),
provider_type: ProviderType::default(),
served_models: Vec::new(),
price_per_mtok: 0.0,
reported_price_per_hour: None,
}
}
pub fn can_handle(&self, cpus: f64, memory_bytes: u64, gpus: u32) -> bool {
self.status == WorkerStatus::Healthy
&& self.resources.cpus_available >= cpus
&& self.resources.memory_available >= memory_bytes
&& self.resources.gpus_available >= gpus
}
pub fn heartbeat(&mut self) {
self.last_heartbeat = Utc::now();
}
pub fn is_stale(&self, timeout_secs: i64) -> bool {
let elapsed = Utc::now().signed_duration_since(self.last_heartbeat);
elapsed.num_seconds() > timeout_secs
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkerRegistration {
pub name: String,
pub uri: String,
pub worker_type: String,
#[serde(default)]
pub resources: WorkerResources,
#[serde(default)]
pub pricing: WorkerPricing,
#[serde(default)]
pub tags: Vec<String>,
#[serde(default)]
pub max_timeout_secs: f64,
#[serde(default)]
pub hardware: HardwareInfo,
#[serde(default)]
pub wireguard_ip: Option<String>,
#[serde(default)]
pub is_docker: Option<bool>,
#[serde(default)]
pub source_node: Option<String>,
#[serde(default)]
pub explicit_local: bool,
#[serde(default)]
pub provider_type: ProviderType,
#[serde(default)]
pub served_models: Vec<String>,
#[serde(default)]
pub price_per_mtok: f64,
}
#[cfg(test)]
pub(crate) fn test_registration(name: &str, uri: String, price: f64) -> WorkerRegistration {
WorkerRegistration {
name: name.to_string(),
uri,
worker_type: "zakuro".to_string(),
resources: WorkerResources::default(),
pricing: WorkerPricing {
price_per_hour: price,
min_charge: 0.001,
},
tags: vec![],
max_timeout_secs: 0.0,
hardware: HardwareInfo::default(),
wireguard_ip: None,
is_docker: None,
source_node: None,
explicit_local: false,
provider_type: ProviderType::Specialized,
served_models: vec![],
price_per_mtok: 0.0,
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkerHeartbeat {
pub worker_id: String,
#[serde(default)]
pub resources: Option<WorkerResources>,
#[serde(default)]
pub active_requests: Option<u32>,
#[serde(default)]
pub status: Option<WorkerStatus>,
#[serde(default)]
pub max_timeout_secs: Option<f64>,
}
const STATS_DECAY: f64 = 0.98;
const POSTERIOR_TTL_SECS: i64 = 600;
const MAX_PARKED_POSTERIORS: usize = 4096;
#[derive(Debug, Clone, Copy)]
struct ParkedPosterior {
stats: crate::broker::estimator::Stats,
last_seen: DateTime<Utc>,
}
#[derive(Debug, Default)]
pub(crate) struct PosteriorCache {
parked: DashMap<String, ParkedPosterior>,
}
fn posterior_gap_decay(elapsed_secs: f64) -> f64 {
STATS_DECAY.powf(elapsed_secs.max(0.0))
}
fn posterior_key(source_node: Option<&str>, uri: &str) -> String {
let node = source_node
.map(|sn| sn.strip_prefix("node-").unwrap_or(sn))
.unwrap_or("");
format!("{node}|{uri}")
}
impl PosteriorCache {
pub(crate) fn new() -> Self {
Self {
parked: DashMap::new(),
}
}
fn park(
&self,
key: String,
stats: crate::broker::estimator::Stats,
last_seen: DateTime<Utc>,
now: DateTime<Utc>,
) {
if stats == crate::broker::estimator::Stats::new() {
return;
}
if self.parked.len() >= MAX_PARKED_POSTERIORS {
self.evict(now);
}
self.parked
.insert(key, ParkedPosterior { stats, last_seen });
}
fn take(&self, key: &str, now: DateTime<Utc>) -> Option<crate::broker::estimator::Stats> {
let (_, entry) = self.parked.remove(key)?;
let elapsed = now.signed_duration_since(entry.last_seen);
if elapsed.num_seconds() > POSTERIOR_TTL_SECS {
return None;
}
let mut stats = entry.stats;
stats.decay(posterior_gap_decay(
elapsed.num_milliseconds() as f64 / 1000.0,
));
Some(stats)
}
fn forget(&self, key: &str) {
self.parked.remove(key);
}
fn evict(&self, now: DateTime<Utc>) {
self.parked.retain(|_, e| {
now.signed_duration_since(e.last_seen).num_seconds() <= POSTERIOR_TTL_SECS
});
while self.parked.len() >= MAX_PARKED_POSTERIORS {
let Some(oldest) = self
.parked
.iter()
.min_by_key(|e| e.value().last_seen)
.map(|e| e.key().clone())
else {
return;
};
self.parked.remove(&oldest);
}
}
fn len(&self) -> usize {
self.parked.len()
}
}
#[derive(Debug)]
pub struct WorkerRegistry {
workers: DashMap<String, Worker>,
request_history: DashMap<String, Mutex<VecDeque<DateTime<Utc>>>>,
quotas: DashMap<String, WorkerQuotas>,
model_index: DashMap<String, HashSet<String>>,
general_workers: DashMap<String, ()>,
hub_prices: DashMap<String, f64>,
posteriors: PosteriorCache,
}
impl WorkerRegistry {
pub fn new() -> Self {
Self {
workers: DashMap::new(),
request_history: DashMap::new(),
quotas: DashMap::new(),
model_index: DashMap::new(),
general_workers: DashMap::new(),
hub_prices: DashMap::new(),
posteriors: PosteriorCache::new(),
}
}
fn index_worker(&self, worker: &Worker) {
match worker.provider_type {
ProviderType::General => {
self.general_workers.insert(worker.id.clone(), ());
}
ProviderType::Specialized => {
for model_uuid in &worker.served_models {
if model_uuid == MODEL_WILDCARD {
self.general_workers.insert(worker.id.clone(), ());
continue;
}
self.model_index
.entry(model_uuid.clone())
.or_default()
.insert(worker.id.clone());
}
}
}
}
fn deindex_worker(&self, worker_id: &str) {
self.general_workers.remove(worker_id);
self.model_index.retain(|_, ids| {
ids.remove(worker_id);
!ids.is_empty()
});
}
pub fn workers_serving(&self, model_uuid: &str) -> Vec<Worker> {
let mut ids: HashSet<String> = self
.model_index
.get(model_uuid)
.map(|entry| entry.clone())
.unwrap_or_default();
for entry in self.general_workers.iter() {
ids.insert(entry.key().clone());
}
ids.into_iter().filter_map(|id| self.get(&id)).collect()
}
pub fn register(&self, registration: WorkerRegistration) -> Worker {
self.register_at(registration, Utc::now())
}
pub(crate) fn register_at(
&self,
registration: WorkerRegistration,
now: DateTime<Utc>,
) -> Worker {
let id = uuid::Uuid::new_v4().to_string();
let wireguard_ip = registration.wireguard_ip.clone().or_else(|| {
let uri_ip = registration
.uri
.strip_prefix("http://")
.or_else(|| registration.uri.strip_prefix("https://"))
.and_then(|rest| {
let host = rest.split('/').next().unwrap_or(rest);
let ip = host.split(':').next().unwrap_or(host);
if ip.is_empty() {
None
} else {
Some(ip.to_string())
}
});
match uri_ip.as_deref() {
Some("127.0.0.1") | Some("::1") | Some("localhost") => {
super::discovery::get_mesh_ip().or(uri_ip)
}
_ => uri_ip,
}
});
let mut worker = Worker::new(
id.clone(),
registration.name,
registration.uri,
registration.worker_type,
);
worker.resources = registration.resources;
worker.pricing = registration.pricing;
worker.tags = registration.tags;
worker.max_timeout_secs = registration.max_timeout_secs;
worker.hardware = registration.hardware;
worker.wireguard_ip = wireguard_ip;
worker.is_docker = registration
.is_docker
.or_else(|| Some(std::path::Path::new("/.dockerenv").exists()));
worker.source_node = registration.source_node;
if worker.node_fp.is_empty() {
if let Some(fp) = worker
.source_node
.as_deref()
.map(super::node_identity::strip_node_arg)
.filter(|fp| !fp.is_empty())
{
worker.node_fp = fp.to_string();
}
}
worker.explicit_local = registration.explicit_local;
worker.provider_type = registration.provider_type;
worker.served_models = registration.served_models;
worker.price_per_mtok = registration.price_per_mtok;
worker.slot = worker
.uri
.strip_prefix("http://")
.or_else(|| worker.uri.strip_prefix("https://"))
.and_then(|rest| rest.split('/').next())
.and_then(|host| host.rsplit(':').next())
.unwrap_or_default()
.to_string();
if worker.source_node.is_none() {
if let Some(hub_price) = self.hub_prices.get(&worker.name).map(|p| *p) {
worker.reported_price_per_hour = Some(worker.pricing.price_per_hour);
worker.pricing.price_per_hour = hub_price;
}
}
if let Some(stats) = self.posteriors.take(
&posterior_key(worker.source_node.as_deref(), &worker.uri),
now,
) {
worker.stats = stats;
}
self.workers.insert(id.clone(), worker.clone());
self.request_history
.insert(id.clone(), Mutex::new(VecDeque::new()));
self.quotas.insert(id.clone(), WorkerQuotas::default());
self.index_worker(&worker);
worker
}
pub fn set_node_fp(&self, worker_id: &str, node_fp: &str) -> Option<Worker> {
self.workers.get_mut(worker_id).map(|mut w| {
w.node_fp = node_fp.to_string();
w.clone()
})
}
pub fn requests_in_window(&self, worker_id: &str, window: Duration) -> u64 {
let cutoff = Utc::now() - window;
self.request_history
.get(worker_id)
.map(|entry| {
let ring = entry.lock().unwrap();
ring.iter().filter(|&&ts| ts >= cutoff).count() as u64
})
.unwrap_or(0)
}
pub fn get_quotas(&self, worker_id: &str) -> WorkerQuotas {
self.quotas
.get(worker_id)
.map(|q| q.clone())
.unwrap_or_default()
}
pub fn heartbeat(&self, heartbeat: WorkerHeartbeat) -> Option<Worker> {
self.workers.get_mut(&heartbeat.worker_id).map(|mut w| {
w.heartbeat();
if let Some(resources) = heartbeat.resources {
w.resources = resources;
}
if let Some(active) = heartbeat.active_requests {
w.active_requests = active;
}
if let Some(status) = heartbeat.status {
if w.status != WorkerStatus::Draining {
w.status = status;
}
}
if let Some(max_timeout) = heartbeat.max_timeout_secs {
w.max_timeout_secs = max_timeout;
}
w.clone()
})
}
pub fn refresh_heartbeat(&self, id: &str) {
if let Some(mut w) = self.workers.get_mut(id) {
w.heartbeat();
if w.status == WorkerStatus::Unhealthy {
w.status = WorkerStatus::Healthy;
}
}
}
pub fn update_resources(&self, id: &str, resources: WorkerResources, hardware: HardwareInfo) {
if let Some(mut w) = self.workers.get_mut(id) {
w.heartbeat();
w.resources = resources;
if hardware.storage_gb.is_some() {
w.hardware.storage_gb = hardware.storage_gb;
}
if w.status == WorkerStatus::Unhealthy {
w.status = WorkerStatus::Healthy;
}
}
}
pub fn get(&self, id: &str) -> Option<Worker> {
self.workers.get(id).map(|w| w.clone())
}
pub fn remove(&self, id: &str) -> Option<Worker> {
self.deindex_worker(id);
let removed = self.workers.remove(id).map(|(_, w)| w);
if let Some(w) = &removed {
self.posteriors
.forget(&posterior_key(w.source_node.as_deref(), &w.uri));
}
removed
}
pub fn list(&self) -> Vec<Worker> {
self.workers.iter().map(|w| w.clone()).collect()
}
pub fn aggregate_available(&self) -> BrokerResources {
self.healthy()
.into_iter()
.fold(BrokerResources::default(), |mut acc, w| {
let cpus = w.resources.cpus_available.max(0.0) as u64;
acc.cpus_available = acc.cpus_available.saturating_add(cpus);
acc.memory_available = acc
.memory_available
.saturating_add(w.resources.memory_available);
acc.gpus_available = acc
.gpus_available
.saturating_add(w.resources.gpus_available);
acc.healthy_workers = acc.healthy_workers.saturating_add(1);
acc
})
}
pub fn healthy(&self) -> Vec<Worker> {
self.workers
.iter()
.filter(|w| w.status == WorkerStatus::Healthy)
.map(|w| w.clone())
.collect()
}
pub fn healthy_ids(&self) -> Vec<String> {
self.workers
.iter()
.filter(|w| w.status == WorkerStatus::Healthy)
.map(|w| w.key().clone())
.collect()
}
pub fn find_capable(
&self,
cpus: f64,
memory_bytes: u64,
gpus: u32,
min_vram_gb: Option<u32>,
) -> Vec<Worker> {
self.workers
.iter()
.filter(|w| {
w.can_handle(cpus, memory_bytes, gpus)
&& min_vram_gb
.is_none_or(|min| w.hardware.gpu_vram_gb.is_some_and(|v| v >= min))
})
.map(|w| w.clone())
.collect()
}
pub fn mark_stale(&self, timeout_secs: i64) {
for mut entry in self.workers.iter_mut() {
if entry.is_stale(timeout_secs) && entry.status == WorkerStatus::Healthy {
entry.status = WorkerStatus::Unhealthy;
}
}
}
pub fn remove_stale(&self, timeout_secs: i64) -> Vec<String> {
let now = Utc::now();
let to_remove: Vec<String> = self
.workers
.iter()
.filter(|w| w.status == WorkerStatus::Unhealthy && w.is_stale(timeout_secs))
.map(|w| w.id.clone())
.collect();
for id in &to_remove {
self.deindex_worker(id);
if let Some((_, w)) = self.workers.remove(id) {
self.posteriors.park(
posterior_key(w.source_node.as_deref(), &w.uri),
w.stats,
w.last_heartbeat,
now,
);
}
self.request_history.remove(id);
self.quotas.remove(id);
}
to_remove
}
pub fn try_reserve_quota(&self, worker_id: &str) -> bool {
let quotas = self.get_quotas(worker_id);
let all_unlimited = quotas.per_5h == 0 && quotas.per_week == 0 && quotas.per_month == 0;
match self.request_history.get(worker_id) {
None => true, Some(entry) => {
let mut ring = entry.lock().unwrap();
let now = Utc::now();
if !all_unlimited {
if quotas.per_5h > 0 {
let cutoff = now - Duration::hours(5);
let count = ring.iter().filter(|&&ts| ts >= cutoff).count() as u64;
if count >= quotas.per_5h {
return false;
}
}
if quotas.per_week > 0 {
let cutoff = now - Duration::weeks(1);
let count = ring.iter().filter(|&&ts| ts >= cutoff).count() as u64;
if count >= quotas.per_week {
return false;
}
}
if quotas.per_month > 0 {
let cutoff = now - Duration::days(30);
let count = ring.iter().filter(|&&ts| ts >= cutoff).count() as u64;
if count >= quotas.per_month {
return false;
}
}
}
ring.push_back(now);
let prune_cutoff = now - Duration::days(30);
while ring.front().map(|&ts| ts < prune_cutoff).unwrap_or(false) {
ring.pop_front();
}
true
}
}
}
pub fn cancel_quota_reservation(&self, worker_id: &str) {
if let Some(entry) = self.request_history.get(worker_id) {
let mut ring = entry.lock().unwrap();
ring.pop_back();
}
}
pub fn record_request(&self, worker_id: &str, duration_ms: f64, _success: bool) {
if let Some(mut worker) = self.workers.get_mut(worker_id) {
worker.total_requests += 1;
let alpha = 0.1;
worker.avg_latency_ms = alpha * duration_ms + (1.0 - alpha) * worker.avg_latency_ms;
if worker.warmup_pending > 0 {
worker.warmup_pending -= 1;
worker.stats.observe_outcome(_success);
} else {
worker.stats.decay(STATS_DECAY);
worker.stats.observe(duration_ms, _success);
}
if worker.active_requests > 0 {
worker.active_requests -= 1;
}
}
}
pub fn observe_latency_by_name(&self, name: &str, duration_ms: f64, success: bool) -> bool {
let Some(id) = self
.workers
.iter()
.find(|e| e.value().name == name)
.map(|e| e.key().clone())
else {
return false;
};
let Some(mut worker) = self.workers.get_mut(&id) else {
return false;
};
worker.total_requests += 1;
let alpha = 0.1;
worker.avg_latency_ms = alpha * duration_ms + (1.0 - alpha) * worker.avg_latency_ms;
worker.stats.decay(STATS_DECAY);
worker.stats.observe(duration_ms, success);
true
}
pub fn increment_active(&self, worker_id: &str) {
if let Some(mut worker) = self.workers.get_mut(worker_id) {
worker.active_requests += 1;
if worker.total_requests == 0 {
worker.warmup_pending += 1;
}
}
}
pub fn mark_busy(&self, worker_id: &str) {
if let Some(mut worker) = self.workers.get_mut(worker_id) {
worker.active_requests = worker.active_requests.max(1);
worker.resources.gpus_available = 0;
worker.resources.cpus_available = 0.0;
}
}
pub fn decrement_active(&self, worker_id: &str) {
if let Some(mut worker) = self.workers.get_mut(worker_id) {
if worker.active_requests > 0 {
worker.active_requests -= 1;
}
worker.warmup_pending = worker.warmup_pending.saturating_sub(1);
}
}
pub fn active_guard(&self, worker_id: &str) -> ActiveGuard<'_> {
self.increment_active(worker_id);
ActiveGuard {
registry: self,
worker_id: worker_id.to_string(),
armed: true,
}
}
pub fn count(&self) -> usize {
self.workers.len()
}
pub fn mark_unhealthy(&self, worker_id: &str) {
if let Some(mut worker) = self.workers.get_mut(worker_id) {
if worker.status != WorkerStatus::Draining {
worker.status = WorkerStatus::Unhealthy;
}
}
}
pub fn drain(&self, id: &str) -> Option<Worker> {
self.workers.get_mut(id).map(|mut w| {
w.status = WorkerStatus::Draining;
w.clone()
})
}
pub fn apply_hub_prices(&self, prices: &std::collections::HashMap<String, f64>) -> usize {
for (name, &price) in prices.iter() {
self.hub_prices.insert(name.clone(), price);
}
let mut changed = 0;
for mut w in self.workers.iter_mut() {
if w.source_node.is_some() {
continue;
}
if let Some(&price) = prices.get(&w.name) {
if w.reported_price_per_hour.is_none() {
w.reported_price_per_hour = Some(w.pricing.price_per_hour);
}
if w.pricing.price_per_hour != price {
w.pricing.price_per_hour = price;
changed += 1;
}
}
}
changed
}
pub fn resolve_model(&self, model_uuid: &str) -> Result<Worker, ModelResolveError> {
let candidates: Vec<Worker> = self
.workers_serving(model_uuid)
.into_iter()
.filter(|w| w.status == WorkerStatus::Healthy)
.collect();
let (specialized, general): (Vec<Worker>, Vec<Worker>) = candidates
.into_iter()
.partition(|w| w.provider_type == ProviderType::Specialized);
let pick_cheapest = |mut tier: Vec<Worker>| -> Option<Worker> {
tier.sort_by(|a, b| {
a.price_per_mtok
.partial_cmp(&b.price_per_mtok)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| a.name.cmp(&b.name))
});
tier.into_iter().next()
};
if !specialized.is_empty() {
return Ok(pick_cheapest(specialized).expect("non-empty specialized tier"));
}
if !general.is_empty() {
return Ok(pick_cheapest(general).expect("non-empty general tier"));
}
Err(ModelResolveError::NoProvider {
model_uuid: model_uuid.to_string(),
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ModelResolveError {
NoProvider { model_uuid: String },
}
impl std::fmt::Display for ModelResolveError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::NoProvider { model_uuid } => {
write!(f, "no provider serving zc://{}", model_uuid)
}
}
}
}
impl std::error::Error for ModelResolveError {}
impl Default for WorkerRegistry {
fn default() -> Self {
Self::new()
}
}
pub struct ActiveGuard<'a> {
registry: &'a WorkerRegistry,
worker_id: String,
armed: bool,
}
impl<'a> ActiveGuard<'a> {
pub fn disarm(&mut self) {
self.armed = false;
}
}
impl<'a> Drop for ActiveGuard<'a> {
fn drop(&mut self) {
if self.armed {
self.registry.decrement_active(&self.worker_id);
}
}
}
#[cfg(test)]
mod peer_worker_identity_tests {
#[test]
fn hardware_carries_ram_gb_through_serde() {
let hw: HardwareInfo =
serde_json::from_str(r#"{"gpu_vram_gb": 0, "ram_gb": 31.3, "cpu_model": "x"}"#)
.unwrap();
assert_eq!(hw.ram_gb, Some(31.3));
assert_eq!(
serde_json::to_value(&hw).unwrap()["ram_gb"],
serde_json::json!(31.3)
);
let old: HardwareInfo = serde_json::from_str(r#"{"gpu_vram_gb": 8}"#).unwrap();
assert_eq!(old.ram_gb, None);
}
use super::*;
fn peer_registration(source_node: &str) -> WorkerRegistration {
let mut r = test_registration("peer-w0", "http://127.0.0.1:39617".to_string(), 36.0);
r.source_node = Some(source_node.to_string());
r
}
#[test]
fn a_registration_that_names_an_owning_node_carries_its_fingerprint() {
let registry = WorkerRegistry::new();
let w = registry.register(peer_registration("zc://node-feedfacedeadbeef"));
assert_eq!(w.node_fp, "feedfacedeadbeef");
}
#[test]
fn a_bare_fingerprint_is_accepted_too() {
let registry = WorkerRegistry::new();
let w = registry.register(peer_registration("feedfacedeadbeef"));
assert_eq!(w.node_fp, "feedfacedeadbeef");
}
#[test]
fn a_local_registration_claims_no_node() {
let registry = WorkerRegistry::new();
let w = registry.register(test_registration(
"local-w0",
"http://127.0.0.1:3960".to_string(),
3.6,
));
assert!(w.node_fp.is_empty());
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn worker_zc_uri_and_source_node_default() {
let mut w = Worker::new(
"id1".into(),
"worker-abc123".into(),
"http://10.13.13.21:3960".into(),
"zakuro".into(),
);
w.node_fp = "abc123".into();
w.slot = "3960".into();
assert_eq!(w.zc_uri(), "zc://worker-abc123-3960");
assert!(w.source_node.is_none());
w.source_node = Some("node-i9".into());
assert_eq!(w.source_node.as_deref(), Some("node-i9"));
}
#[test]
fn worker_zc_uri_is_key_derived_and_collision_free() {
let mut w1 = Worker::new(
"id1".into(),
"worker-lxd".into(),
"http://127.0.0.1:3960".into(),
"zakuro".into(),
);
let mut w2 = Worker::new(
"id2".into(),
"worker-lxd".into(),
"http://127.0.0.1:3960".into(),
"zakuro".into(),
);
w1.node_fp = "aaaaaaaaaaaaaaaa".into();
w1.slot = "3960".into();
w2.node_fp = "bbbbbbbbbbbbbbbb".into();
w2.slot = "3960".into();
assert_eq!(w1.zc_uri(), "zc://worker-aaaaaaaaaaaaaaaa-3960");
assert_ne!(w1.zc_uri(), w2.zc_uri()); }
fn registration(name: &str, uri: &str) -> WorkerRegistration {
WorkerRegistration {
name: name.to_string(),
uri: uri.to_string(),
worker_type: "zakuro".to_string(),
resources: WorkerResources::default(),
pricing: WorkerPricing::default(),
tags: vec![],
max_timeout_secs: 0.0,
hardware: HardwareInfo::default(),
wireguard_ip: None,
is_docker: None,
source_node: None,
explicit_local: false,
provider_type: ProviderType::default(),
served_models: vec![],
price_per_mtok: 0.0,
}
}
fn model_registration(
name: &str,
uri: &str,
provider_type: ProviderType,
served_models: Vec<&str>,
) -> WorkerRegistration {
let mut r = registration(name, uri);
r.provider_type = provider_type;
r.served_models = served_models.into_iter().map(String::from).collect();
r
}
#[test]
fn test_workers_serving_specialized_matches_only_its_model() {
let reg = WorkerRegistry::new();
let w = reg.register(model_registration(
"specialist",
"http://127.0.0.1:4001",
ProviderType::Specialized,
vec!["uuid-a"],
));
let matches = reg.workers_serving("uuid-a");
assert_eq!(matches.len(), 1);
assert_eq!(matches[0].id, w.id);
let no_matches = reg.workers_serving("uuid-b");
assert!(no_matches.is_empty());
}
#[test]
fn test_workers_serving_general_matches_any_model() {
let reg = WorkerRegistry::new();
let w = reg.register(model_registration(
"generalist",
"http://127.0.0.1:4002",
ProviderType::General,
vec!["*"],
));
for uuid in ["uuid-a", "uuid-b", "any-random-uuid"] {
let matches = reg.workers_serving(uuid);
assert_eq!(matches.len(), 1, "general worker should serve {uuid}");
assert_eq!(matches[0].id, w.id);
}
}
#[test]
fn test_workers_serving_unions_specialized_and_general() {
let reg = WorkerRegistry::new();
let specialist = reg.register(model_registration(
"specialist",
"http://127.0.0.1:4003",
ProviderType::Specialized,
vec!["uuid-a"],
));
let generalist = reg.register(model_registration(
"generalist",
"http://127.0.0.1:4004",
ProviderType::General,
vec![],
));
let mut ids: Vec<String> = reg
.workers_serving("uuid-a")
.into_iter()
.map(|w| w.id)
.collect();
ids.sort();
let mut expected = vec![specialist.id.clone(), generalist.id.clone()];
expected.sort();
assert_eq!(ids, expected);
}
#[test]
fn test_reregister_updates_model_index() {
let reg = WorkerRegistry::new();
let w1 = reg.register(model_registration(
"flip",
"http://127.0.0.1:4005",
ProviderType::Specialized,
vec!["uuid-old"],
));
assert_eq!(reg.workers_serving("uuid-old").len(), 1);
reg.remove(&w1.id);
assert!(
reg.workers_serving("uuid-old").is_empty(),
"removed worker must drop out of the old model's index entry"
);
let w2 = reg.register(model_registration(
"flip",
"http://127.0.0.1:4005",
ProviderType::Specialized,
vec!["uuid-new"],
));
assert!(reg.workers_serving("uuid-old").is_empty());
let matches = reg.workers_serving("uuid-new");
assert_eq!(matches.len(), 1);
assert_eq!(matches[0].id, w2.id);
}
#[test]
fn test_remove_drops_worker_from_general_index() {
let reg = WorkerRegistry::new();
let w = reg.register(model_registration(
"generalist",
"http://127.0.0.1:4006",
ProviderType::General,
vec![],
));
assert_eq!(reg.workers_serving("any-uuid").len(), 1);
reg.remove(&w.id);
assert!(reg.workers_serving("any-uuid").is_empty());
}
#[test]
fn test_legacy_worker_json_without_new_fields_defaults_back_compat() {
let legacy_json = r#"{
"id": "legacy-id",
"name": "legacy-worker",
"uri": "http://127.0.0.1:3960",
"worker_type": "zakuro",
"status": "healthy",
"resources": {
"cpus_total": 1.0,
"cpus_available": 1.0,
"memory_total": 1073741824,
"memory_available": 1073741824,
"gpus_total": 0,
"gpus_available": 0
},
"pricing": {
"price_per_hour": 3.6,
"min_charge": 0.001
},
"last_heartbeat": "2026-01-01T00:00:00Z",
"active_requests": 0,
"total_requests": 0,
"avg_latency_ms": 0.0,
"tags": []
}"#;
let w: Worker = serde_json::from_str(legacy_json).expect("legacy JSON must deserialize");
assert_eq!(w.provider_type, ProviderType::Specialized);
assert!(w.served_models.is_empty());
assert_eq!(w.price_per_mtok, 0.0);
}
#[test]
fn test_legacy_worker_registration_json_without_new_fields_defaults_back_compat() {
let legacy_json = r#"{
"name": "legacy-worker",
"uri": "http://127.0.0.1:3960",
"worker_type": "zakuro"
}"#;
let r: WorkerRegistration =
serde_json::from_str(legacy_json).expect("legacy JSON must deserialize");
assert_eq!(r.provider_type, ProviderType::Specialized);
assert!(r.served_models.is_empty());
assert_eq!(r.price_per_mtok, 0.0);
}
#[test]
fn test_register_assigns_unique_id() {
let reg = WorkerRegistry::new();
let w1 = reg.register(registration("w1", "http://127.0.0.1:3960"));
let w2 = reg.register(registration("w2", "http://127.0.0.1:3961"));
assert!(!w1.id.is_empty());
assert_ne!(w1.id, w2.id);
assert_eq!(reg.count(), 2);
}
#[test]
fn test_register_stores_fields() {
let reg = WorkerRegistry::new();
let worker = reg.register(registration("my-worker", "http://10.0.0.1:3960"));
let got = reg.get(&worker.id).unwrap();
assert_eq!(got.name, "my-worker");
assert_eq!(got.uri, "http://10.0.0.1:3960");
assert_eq!(got.status, WorkerStatus::Healthy);
assert_eq!(got.active_requests, 0);
}
#[test]
fn test_remove_worker() {
let reg = WorkerRegistry::new();
let w = reg.register(registration("w", "http://127.0.0.1:3960"));
assert!(reg.get(&w.id).is_some());
let removed = reg.remove(&w.id);
assert!(removed.is_some());
assert!(reg.get(&w.id).is_none());
assert_eq!(reg.count(), 0);
}
#[test]
fn test_heartbeat_updates_resources_and_status() {
let reg = WorkerRegistry::new();
let w = reg.register(registration("w", "http://127.0.0.1:3960"));
let hb = WorkerHeartbeat {
worker_id: w.id.clone(),
resources: Some(WorkerResources {
cpus_total: 8.0,
cpus_available: 4.0,
memory_total: 16 * 1024 * 1024 * 1024,
memory_available: 8 * 1024 * 1024 * 1024,
gpus_total: 2,
gpus_available: 1,
}),
active_requests: Some(5),
status: Some(WorkerStatus::Busy),
max_timeout_secs: Some(120.0),
};
let updated = reg.heartbeat(hb).unwrap();
assert_eq!(updated.resources.cpus_available, 4.0);
assert_eq!(updated.resources.gpus_available, 1);
assert_eq!(updated.active_requests, 5);
assert_eq!(updated.status, WorkerStatus::Busy);
assert_eq!(updated.max_timeout_secs, 120.0);
}
#[test]
fn test_heartbeat_unknown_worker_returns_none() {
let reg = WorkerRegistry::new();
let hb = WorkerHeartbeat {
worker_id: "nonexistent".to_string(),
resources: None,
active_requests: None,
status: None,
max_timeout_secs: None,
};
assert!(reg.heartbeat(hb).is_none());
}
#[test]
fn test_mark_stale_sets_unhealthy() {
let reg = WorkerRegistry::new();
let w = reg.register(registration("w", "http://127.0.0.1:3960"));
reg.mark_stale(-1);
let got = reg.get(&w.id).unwrap();
assert_eq!(got.status, WorkerStatus::Unhealthy);
}
#[test]
fn test_refresh_heartbeat_restores_healthy() {
let reg = WorkerRegistry::new();
let w = reg.register(registration("w", "http://127.0.0.1:3960"));
reg.mark_stale(-1);
assert_eq!(reg.get(&w.id).unwrap().status, WorkerStatus::Unhealthy);
reg.refresh_heartbeat(&w.id);
assert_eq!(reg.get(&w.id).unwrap().status, WorkerStatus::Healthy);
}
#[test]
fn aggregate_available_sums_healthy_workers() {
let reg = WorkerRegistry::new();
let mut r1 = registration("w1", "http://127.0.0.1:3960");
r1.resources = WorkerResources {
cpus_total: 4.0,
cpus_available: 4.0,
memory_total: 8_000_000_000,
memory_available: 8_000_000_000,
gpus_total: 1,
gpus_available: 1,
};
let mut r2 = registration("w2", "http://127.0.0.1:3961");
r2.resources = WorkerResources {
cpus_total: 2.0,
cpus_available: 2.0,
memory_total: 4_000_000_000,
memory_available: 4_000_000_000,
gpus_total: 0,
gpus_available: 0,
};
reg.register(r1);
reg.register(r2);
let agg = reg.aggregate_available();
assert_eq!(agg.cpus_available, 6);
assert_eq!(agg.memory_available, 12_000_000_000);
assert_eq!(agg.gpus_available, 1);
assert_eq!(agg.healthy_workers, 2);
}
#[test]
fn aggregate_available_excludes_unhealthy_and_ignores_empty() {
let reg = WorkerRegistry::new();
let empty = reg.aggregate_available();
assert_eq!(empty, BrokerResources::default());
let w = reg.register(registration("sick", "http://127.0.0.1:3962"));
reg.mark_unhealthy(&w.id);
let agg = reg.aggregate_available();
assert_eq!(agg.healthy_workers, 0);
assert_eq!(agg.cpus_available, 0);
}
#[test]
fn test_healthy_list_excludes_unhealthy() {
let reg = WorkerRegistry::new();
let w1 = reg.register(registration("healthy", "http://127.0.0.1:3960"));
let w2 = reg.register(registration("sick", "http://127.0.0.1:3961"));
reg.mark_unhealthy(&w2.id);
let healthy = reg.healthy();
assert_eq!(healthy.len(), 1);
assert_eq!(healthy[0].id, w1.id);
}
#[test]
fn test_increment_active_and_decrement_on_record() {
let reg = WorkerRegistry::new();
let w = reg.register(registration("w", "http://127.0.0.1:3960"));
reg.increment_active(&w.id);
reg.increment_active(&w.id);
assert_eq!(reg.get(&w.id).unwrap().active_requests, 2);
reg.record_request(&w.id, 100.0, true);
assert_eq!(reg.get(&w.id).unwrap().active_requests, 1);
}
#[test]
fn warm_up_replies_are_left_out_of_the_latency_posterior() {
let reg = WorkerRegistry::new();
let w = reg.register(registration("w", "http://127.0.0.1:3960"));
for _ in 0..3 {
reg.increment_active(&w.id);
}
reg.record_request(&w.id, 1000.0, true); reg.increment_active(&w.id);
reg.record_request(&w.id, 990.0, true); reg.record_request(&w.id, 985.0, true);
let got = reg.get(&w.id).unwrap();
assert_eq!(
got.stats.weight(),
0.0,
"no warm-up timing may reach the posterior"
);
assert!(
got.stats.success_rate() > 0.7,
"but the successes still count"
);
assert_eq!(got.total_requests, 3);
reg.record_request(&w.id, 4.0, true);
let got = reg.get(&w.id).unwrap();
assert_eq!(got.stats.weight(), 1.0);
assert!((got.stats.median_service_ms().unwrap() - 4.0).abs() < 1e-9);
assert_eq!(got.active_requests, 0);
let v = reg.register(registration("v", "http://127.0.0.1:3961"));
{
let _failed = reg.active_guard(&v.id);
}
reg.increment_active(&v.id);
reg.record_request(&v.id, 1000.0, true);
reg.increment_active(&v.id);
reg.record_request(&v.id, 5.0, true);
let got = reg.get(&v.id).unwrap();
assert_eq!(got.stats.weight(), 1.0);
assert!((got.stats.median_service_ms().unwrap() - 5.0).abs() < 1e-9);
}
#[test]
fn active_guard_decrements_on_drop_even_without_record() {
let reg = WorkerRegistry::new();
let w = reg.register(registration("w1", "http://127.0.0.1:3960"));
{
let _g = reg.active_guard(&w.id);
assert_eq!(reg.get(&w.id).unwrap().active_requests, 1);
} assert_eq!(
reg.get(&w.id).unwrap().active_requests,
0,
"active_requests must return to 0 when the guard drops on an error path"
);
}
#[test]
fn active_guard_disarm_skips_decrement() {
let reg = WorkerRegistry::new();
let w = reg.register(registration("w2", "http://127.0.0.1:3960"));
{
let mut g = reg.active_guard(&w.id);
assert_eq!(reg.get(&w.id).unwrap().active_requests, 1);
reg.record_request(&w.id, 10.0, true);
g.disarm();
}
assert_eq!(
reg.get(&w.id).unwrap().active_requests,
0,
"disarmed guard must not double-decrement after record_request"
);
}
#[test]
fn test_record_request_latency_ema() {
let reg = WorkerRegistry::new();
let w = reg.register(registration("w", "http://127.0.0.1:3960"));
reg.record_request(&w.id, 200.0, true);
let got = reg.get(&w.id).unwrap();
assert!((got.avg_latency_ms - 20.0).abs() < 0.001);
assert_eq!(got.total_requests, 1);
}
#[test]
fn test_find_capable_filters_by_resources() {
let reg = WorkerRegistry::new();
let mut r_small = registration("small", "http://127.0.0.1:3960");
r_small.resources = WorkerResources {
cpus_total: 2.0,
cpus_available: 2.0,
memory_total: 2 * 1024 * 1024 * 1024,
memory_available: 2 * 1024 * 1024 * 1024,
gpus_total: 0,
gpus_available: 0,
};
let mut r_large = registration("large", "http://127.0.0.1:3961");
r_large.resources = WorkerResources {
cpus_total: 32.0,
cpus_available: 32.0,
memory_total: 128 * 1024 * 1024 * 1024,
memory_available: 128 * 1024 * 1024 * 1024,
gpus_total: 4,
gpus_available: 4,
};
reg.register(r_small);
reg.register(r_large);
let capable = reg.find_capable(16.0, 1024 * 1024 * 1024, 0, None);
assert_eq!(capable.len(), 1);
assert_eq!(capable[0].name, "large");
assert_eq!(reg.find_capable(1.0, 512 * 1024 * 1024, 0, None).len(), 2);
assert_eq!(reg.find_capable(1.0, 512 * 1024 * 1024, 8, None).len(), 0);
}
#[test]
fn test_find_capable_filters_by_min_vram_gb() {
let reg = WorkerRegistry::new();
let mut small_vram = registration("small-vram", "http://127.0.0.1:3962");
small_vram.hardware.gpu_vram_gb = Some(8);
let mut big_vram = registration("big-vram", "http://127.0.0.1:3963");
big_vram.hardware.gpu_vram_gb = Some(24);
let no_vram = registration("no-vram", "http://127.0.0.1:3964");
reg.register(small_vram);
reg.register(big_vram);
reg.register(no_vram);
let capable = reg.find_capable(0.1, 1024, 0, Some(16));
assert_eq!(
capable.len(),
1,
"only the 24 GiB worker clears a 16 GiB floor"
);
assert_eq!(capable[0].name, "big-vram");
assert_eq!(reg.find_capable(0.1, 1024, 0, None).len(), 3);
}
#[test]
fn test_can_handle_exact_match() {
let w = Worker::new(
"id".to_string(),
"w".to_string(),
"http://x".to_string(),
"zakuro".to_string(),
);
assert!(w.can_handle(1.0, 1024 * 1024 * 1024, 0));
}
#[test]
fn test_can_handle_over_cpu_fails() {
let w = Worker::new(
"id".to_string(),
"w".to_string(),
"http://x".to_string(),
"zakuro".to_string(),
);
assert!(!w.can_handle(2.0, 512 * 1024 * 1024, 0));
}
#[test]
fn test_can_handle_over_memory_fails() {
let w = Worker::new(
"id".to_string(),
"w".to_string(),
"http://x".to_string(),
"zakuro".to_string(),
);
assert!(!w.can_handle(0.5, 2 * 1024 * 1024 * 1024, 0));
}
#[test]
fn test_can_handle_gpu_required_but_none_fails() {
let w = Worker::new(
"id".to_string(),
"w".to_string(),
"http://x".to_string(),
"zakuro".to_string(),
);
assert!(!w.can_handle(0.5, 512 * 1024 * 1024, 1));
}
#[test]
fn test_can_handle_unhealthy_worker_fails() {
let mut w = Worker::new(
"id".to_string(),
"w".to_string(),
"http://x".to_string(),
"zakuro".to_string(),
);
w.status = WorkerStatus::Unhealthy;
assert!(!w.can_handle(0.1, 1024, 0));
}
#[test]
fn test_pricing_cost_formula() {
let p = WorkerPricing {
price_per_hour: 3.6,
min_charge: 0.001,
};
let cost = p.estimate_cost(10.0);
assert!((cost - 0.010).abs() < 0.0001);
}
#[test]
fn test_pricing_min_charge_enforced() {
let p = WorkerPricing {
price_per_hour: 0.0,
min_charge: 0.005,
};
let cost = p.estimate_cost(0.001);
assert_eq!(cost, 0.005);
}
fn priced_model_registration(
name: &str,
uri: &str,
provider_type: ProviderType,
served_models: Vec<&str>,
price_per_mtok: f64,
) -> WorkerRegistration {
let mut r = model_registration(name, uri, provider_type, served_models);
r.price_per_mtok = price_per_mtok;
r
}
#[test]
fn test_resolve_model_prefers_specialized_over_general() {
let reg = WorkerRegistry::new();
let specialized = reg.register(priced_model_registration(
"specialist",
"http://127.0.0.1:5001",
ProviderType::Specialized,
vec!["uuid-a"],
10.0,
));
reg.register(priced_model_registration(
"generalist",
"http://127.0.0.1:5002",
ProviderType::General,
vec![],
1.0, ));
let resolved = reg.resolve_model("uuid-a").unwrap();
assert_eq!(resolved.id, specialized.id);
}
#[test]
fn test_resolve_model_cheapest_among_specialized() {
let reg = WorkerRegistry::new();
reg.register(priced_model_registration(
"expensive",
"http://127.0.0.1:5003",
ProviderType::Specialized,
vec!["uuid-a"],
20.0,
));
let cheap = reg.register(priced_model_registration(
"cheap",
"http://127.0.0.1:5004",
ProviderType::Specialized,
vec!["uuid-a"],
5.0,
));
let resolved = reg.resolve_model("uuid-a").unwrap();
assert_eq!(resolved.id, cheap.id);
}
#[test]
fn test_resolve_model_general_fallback_cheapest() {
let reg = WorkerRegistry::new();
reg.register(priced_model_registration(
"general-expensive",
"http://127.0.0.1:5005",
ProviderType::General,
vec![],
8.0,
));
let cheap_general = reg.register(priced_model_registration(
"general-cheap",
"http://127.0.0.1:5006",
ProviderType::General,
vec![],
2.0,
));
let resolved = reg.resolve_model("uuid-z").unwrap();
assert_eq!(resolved.id, cheap_general.id);
}
#[test]
fn test_resolve_model_excludes_unhealthy_specialized_falls_back_to_general() {
let reg = WorkerRegistry::new();
let stale_specialist = reg.register(priced_model_registration(
"stale-specialist",
"http://127.0.0.1:5007",
ProviderType::Specialized,
vec!["uuid-a"],
1.0, ));
reg.mark_unhealthy(&stale_specialist.id);
let healthy_general = reg.register(priced_model_registration(
"healthy-general",
"http://127.0.0.1:5008",
ProviderType::General,
vec![],
5.0,
));
let resolved = reg.resolve_model("uuid-a").unwrap();
assert_eq!(resolved.id, healthy_general.id);
}
#[test]
fn test_resolve_model_no_provider_when_no_healthy_candidate() {
let reg = WorkerRegistry::new();
let w = reg.register(priced_model_registration(
"only-specialist",
"http://127.0.0.1:5009",
ProviderType::Specialized,
vec!["uuid-a"],
1.0,
));
reg.mark_unhealthy(&w.id);
let err = reg.resolve_model("uuid-a").unwrap_err();
assert_eq!(
err,
ModelResolveError::NoProvider {
model_uuid: "uuid-a".to_string()
}
);
assert_eq!(err.to_string(), "no provider serving zc://uuid-a");
}
#[test]
fn test_pricing_price_score_weighted() {
let p = WorkerPricing::default();
let score = p.price_score();
assert!((score - 1.5).abs() < 0.0001);
}
}
#[cfg(test)]
mod drain_and_price_tests {
use super::*;
#[test]
fn drain_marks_draining_and_leaves_the_routable_set() {
let r = WorkerRegistry::new();
let w = r.register(test_registration(
"fp-w0",
"http://127.0.0.1:3960".to_string(),
3.6,
));
assert_eq!(r.drain(&w.id).unwrap().status, WorkerStatus::Draining);
assert!(
r.healthy_ids().is_empty(),
"a draining worker is never selected"
);
assert!(r.healthy().is_empty());
}
#[test]
fn discovery_refresh_does_not_undo_a_drain() {
let r = WorkerRegistry::new();
let w = r.register(test_registration(
"fp-w0",
"http://127.0.0.1:3960".to_string(),
3.6,
));
r.drain(&w.id);
r.refresh_heartbeat(&w.id);
r.update_resources(&w.id, WorkerResources::default(), HardwareInfo::default());
assert_eq!(r.get(&w.id).unwrap().status, WorkerStatus::Draining);
}
#[test]
fn a_worker_heartbeat_does_not_undo_a_drain() {
let r = WorkerRegistry::new();
let w = r.register(test_registration(
"fp-w0",
"http://127.0.0.1:3960".to_string(),
3.6,
));
r.drain(&w.id);
r.heartbeat(WorkerHeartbeat {
worker_id: w.id.clone(),
resources: None,
active_requests: Some(1),
status: Some(WorkerStatus::Healthy),
max_timeout_secs: None,
});
let got = r.get(&w.id).unwrap();
assert_eq!(got.status, WorkerStatus::Draining);
assert_eq!(
got.active_requests, 1,
"the rest of the heartbeat still applies"
);
}
#[test]
fn draining_an_unknown_worker_is_none() {
assert!(WorkerRegistry::new().drain("nope").is_none());
}
#[test]
fn mark_unhealthy_does_not_undo_a_drain() {
let r = WorkerRegistry::new();
let w = r.register(test_registration(
"fp-w0",
"http://127.0.0.1:3960".to_string(),
3.6,
));
r.drain(&w.id);
r.mark_unhealthy(&w.id);
assert_eq!(r.get(&w.id).unwrap().status, WorkerStatus::Draining);
}
#[test]
fn hub_price_overrides_billing_and_remembers_the_workers_own_price() {
use std::collections::HashMap;
let r = WorkerRegistry::new();
let w = r.register(test_registration(
"fp-w0",
"http://127.0.0.1:3960".to_string(),
3.6,
));
assert_eq!(
r.apply_hub_prices(&HashMap::from([("fp-w0".to_string(), 18.0)])),
1
);
let got = r.get(&w.id).unwrap();
assert_eq!(got.pricing.price_per_hour, 18.0);
assert_eq!(got.reported_price_per_hour, Some(3.6));
r.apply_hub_prices(&HashMap::from([("fp-w0".to_string(), 25.0)]));
let got = r.get(&w.id).unwrap();
assert_eq!(got.pricing.price_per_hour, 25.0);
assert_eq!(got.reported_price_per_hour, Some(3.6));
assert_eq!(
r.apply_hub_prices(&HashMap::from([("fp-w0".to_string(), 25.0)])),
0
);
}
#[test]
fn discovery_refresh_keeps_the_hub_price() {
use std::collections::HashMap;
let r = WorkerRegistry::new();
let w = r.register(test_registration(
"fp-w0",
"http://127.0.0.1:3960".to_string(),
3.6,
));
r.apply_hub_prices(&HashMap::from([("fp-w0".to_string(), 18.0)]));
r.update_resources(&w.id, WorkerResources::default(), HardwareInfo::default());
r.refresh_heartbeat(&w.id);
assert_eq!(r.get(&w.id).unwrap().pricing.price_per_hour, 18.0);
}
#[test]
fn peer_workers_are_never_repriced() {
use std::collections::HashMap;
let r = WorkerRegistry::new();
let mut peer = test_registration("fp-w0", "http://127.0.0.1:3960".to_string(), 3.6);
peer.source_node = Some("node-other".to_string());
let w = r.register(peer);
assert_eq!(
r.apply_hub_prices(&HashMap::from([("fp-w0".to_string(), 18.0)])),
0
);
assert_eq!(r.get(&w.id).unwrap().pricing.price_per_hour, 3.6);
}
#[test]
fn hub_price_survives_re_registration_with_no_sync_in_between() {
use std::collections::HashMap;
let r = WorkerRegistry::new();
let w1 = r.register(test_registration(
"fp-w0",
"http://127.0.0.1:3960".to_string(),
3.6,
));
r.apply_hub_prices(&HashMap::from([("fp-w0".to_string(), 18.0)]));
assert_eq!(r.get(&w1.id).unwrap().pricing.price_per_hour, 18.0);
let w2 = r.register(test_registration(
"fp-w0",
"http://127.0.0.1:3961".to_string(),
4.2,
));
assert_ne!(w1.id, w2.id, "re-registration creates a new record");
let got = r.get(&w2.id).unwrap();
assert_eq!(
got.pricing.price_per_hour, 18.0,
"billing must use the hub price immediately, no sync needed"
);
assert_eq!(
got.reported_price_per_hour,
Some(4.2),
"sync must keep reporting the freshly-probed price, not the hub price"
);
}
}
#[cfg(test)]
mod posterior_ttl_tests {
use super::*;
use crate::broker::estimator::Stats;
use crate::broker::policy::{self, Candidate, Weights};
fn measured(median_ms: f64, n: usize) -> Stats {
let mut s = Stats::new();
for _ in 0..n {
s.observe(median_ms, true);
}
s
}
fn at(secs: i64) -> DateTime<Utc> {
DateTime::from_timestamp(1_700_000_000 + secs, 0).expect("valid timestamp")
}
#[test]
fn a_posterior_taken_back_at_once_is_the_one_that_was_parked() {
let cache = PosteriorCache::new();
let stats = measured(40.0, 30);
cache.park("k".into(), stats, at(0), at(0));
let back = cache.take("k", at(0)).expect("parked a moment ago");
assert!(back.close_to(&stats), "{back:?} vs {stats:?}");
assert!(
cache.take("k", at(0)).is_none(),
"taken means taken: the entry is not handed out twice"
);
}
#[test]
fn a_posterior_loses_confidence_in_proportion_to_how_long_it_was_gone() {
let cache = PosteriorCache::new();
let stats = measured(40.0, 100);
cache.park("k".into(), stats, at(0), at(0));
let back = cache.take("k", at(240)).expect("inside the ttl");
assert!(
(back.median_service_ms().unwrap() - 40.0).abs() < 1e-9,
"the median it learned is still the median: {back:?}"
);
let expected = 100.0 * STATS_DECAY.powi(240);
assert!(
(back.weight() - expected).abs() < 1e-9,
"four minutes away should leave {expected}, got {}",
back.weight()
);
assert!(
back.weight() < 1.0,
"four minutes is long enough that the policy re-explores it"
);
}
#[test]
fn the_gap_is_measured_from_when_the_worker_was_last_seen() {
let cache = PosteriorCache::new();
let stats = measured(40.0, 100);
cache.park("k".into(), stats, at(0), at(60));
let back = cache.take("k", at(120)).expect("inside the ttl");
let expected = 100.0 * STATS_DECAY.powi(120);
assert!(
(back.weight() - expected).abs() < 1e-9,
"120s of silence, not 60s since removal: want {expected}, got {}",
back.weight()
);
}
#[test]
fn a_posterior_older_than_the_ttl_is_not_restored() {
let cache = PosteriorCache::new();
cache.park("k".into(), measured(40.0, 100), at(0), at(0));
assert!(
cache.take("k", at(POSTERIOR_TTL_SECS)).is_some(),
"on the boundary it still counts"
);
cache.park("k".into(), measured(40.0, 100), at(0), at(0));
assert!(cache.take("k", at(POSTERIOR_TTL_SECS + 1)).is_none());
}
#[test]
fn nothing_survives_to_the_ttl_that_the_policy_would_still_believe() {
let cache = PosteriorCache::new();
cache.park("k".into(), measured(40.0, 100_000), at(0), at(0));
let back = cache
.take("k", at(POSTERIOR_TTL_SECS))
.expect("on the boundary");
assert!(
back.weight() < 1.0,
"a hundred thousand observations still decay below one in {POSTERIOR_TTL_SECS}s, \
got {}",
back.weight()
);
}
#[test]
fn the_parked_cache_does_not_grow_without_bound() {
let cache = PosteriorCache::new();
for i in 0..(MAX_PARKED_POSTERIORS * 2) {
cache.park(
format!("k{i}"),
measured(40.0, 5),
at(i as i64),
at(i as i64),
);
}
assert!(
cache.len() <= MAX_PARKED_POSTERIORS,
"{} entries parked, cap is {MAX_PARKED_POSTERIORS}",
cache.len()
);
assert!(
cache.take("k0", at(0)).is_none(),
"the first one went first"
);
let newest = format!("k{}", MAX_PARKED_POSTERIORS * 2 - 1);
assert!(cache.take(&newest, at(0)).is_some(), "the newest is kept");
}
#[test]
fn an_unmeasured_worker_has_no_posterior_worth_parking() {
let cache = PosteriorCache::new();
cache.park("k".into(), Stats::new(), at(0), at(0));
assert_eq!(cache.len(), 0);
assert!(cache.take("k", at(0)).is_none());
}
#[test]
fn a_worker_that_flaps_comes_back_with_what_it_had_learned() {
let reg = WorkerRegistry::new();
let w = reg.register(test_registration(
"w0",
"http://10.13.13.21:3960".to_string(),
3.6,
));
for _ in 0..40 {
reg.record_request(&w.id, 12.0, true);
}
let learned = reg.get(&w.id).unwrap().stats;
assert!(learned.weight() > 1.0);
reg.mark_stale(-1);
assert_eq!(reg.remove_stale(-1), vec![w.id.clone()]);
let again = reg.register(test_registration(
"w0",
"http://10.13.13.21:3960".to_string(),
3.6,
));
assert_ne!(again.id, w.id, "the id is a fresh uuid every registration");
assert!(
(again.stats.median_service_ms().unwrap() - 12.0).abs() < 1e-6,
"it came back knowing what it had measured: {:?}",
again.stats
);
assert!(
again.stats.weight() > 1.0,
"and enough weight to be believed"
);
}
#[test]
fn a_worker_deleted_on_purpose_comes_back_a_stranger() {
let reg = WorkerRegistry::new();
let w = reg.register(test_registration(
"w0",
"http://10.13.13.21:3960".to_string(),
3.6,
));
for _ in 0..40 {
reg.record_request(&w.id, 12.0, true);
}
reg.remove(&w.id).expect("removed");
let again = reg.register(test_registration(
"w0",
"http://10.13.13.21:3960".to_string(),
3.6,
));
assert_eq!(
again.stats.median_service_ms(),
None,
"an explicit delete forgets the posterior too"
);
}
#[test]
fn deleting_a_worker_clears_a_posterior_parked_before_it() {
let reg = WorkerRegistry::new();
let first = reg.register(test_registration(
"w0",
"http://10.13.13.21:3960".to_string(),
3.6,
));
for _ in 0..40 {
reg.record_request(&first.id, 12.0, true);
}
reg.mark_stale(-1);
reg.remove_stale(-1);
let second = reg.register(test_registration(
"w0",
"http://10.13.13.21:3960".to_string(),
3.6,
));
assert!(second.stats.weight() > 1.0, "restored by the flap path");
reg.remove(&second.id).expect("removed");
let third = reg.register(test_registration(
"w0",
"http://10.13.13.21:3960".to_string(),
3.6,
));
assert_eq!(third.stats.median_service_ms(), None);
}
#[test]
fn a_posterior_is_never_restored_onto_a_different_worker() {
let reg = WorkerRegistry::new();
let w = reg.register(test_registration(
"worker-lxd",
"http://10.13.13.21:3960".to_string(),
3.6,
));
for _ in 0..40 {
reg.record_request(&w.id, 12.0, true);
}
reg.mark_stale(-1);
reg.remove_stale(-1);
let elsewhere = reg.register(test_registration(
"worker-lxd",
"http://10.13.13.99:3960".to_string(),
3.6,
));
assert_eq!(
elsewhere.stats.median_service_ms(),
None,
"same name, different box: not the same worker"
);
}
#[test]
fn a_posterior_does_not_cross_from_a_local_worker_to_a_peer_one() {
let reg = WorkerRegistry::new();
let w = reg.register(test_registration(
"w0",
"http://10.13.13.21:3960".to_string(),
3.6,
));
for _ in 0..40 {
reg.record_request(&w.id, 12.0, true);
}
reg.mark_stale(-1);
reg.remove_stale(-1);
let mut peer = test_registration("w0", "http://10.13.13.21:3960".to_string(), 3.6);
peer.source_node = Some("node-i9".to_string());
let as_peer = reg.register(peer);
assert_eq!(as_peer.stats.median_service_ms(), None);
}
#[test]
fn a_returning_worker_is_believed_less_the_longer_it_was_away() {
let after = |gap_secs: i64| {
let reg = WorkerRegistry::new();
let w = reg.register(test_registration(
"w0",
"http://10.13.13.21:3960".to_string(),
3.6,
));
for _ in 0..100 {
reg.record_request(&w.id, 12.0, true);
}
let last_seen = reg.get(&w.id).unwrap().last_heartbeat;
reg.mark_stale(-1);
reg.remove_stale(-1);
reg.register_at(
test_registration("w0", "http://10.13.13.21:3960".to_string(), 3.6),
last_seen + Duration::seconds(gap_secs),
)
.stats
};
let straight_back = after(1).weight();
let four_minutes = after(240).weight();
let past_the_ttl = after(POSTERIOR_TTL_SECS + 1);
assert!(
straight_back > four_minutes,
"{straight_back} vs {four_minutes}"
);
assert!(four_minutes < 1.0, "four minutes away costs its confidence");
assert!(four_minutes > 0.0, "but not its shape");
assert_eq!(
past_the_ttl.median_service_ms(),
None,
"past the ttl it is a stranger again"
);
}
#[test]
fn a_restored_posterior_beats_the_class_prior_it_would_otherwise_start_from() {
let cache = PosteriorCache::new();
cache.park("k".into(), measured(12.0, 100), at(0), at(0));
let restored = cache.take("k", at(120)).expect("inside the ttl");
let w = Weights::default();
let returning = Candidate {
stats: restored,
in_flight: 0,
price_per_hour: 3.6,
local: false,
};
let stranger = Candidate {
stats: Stats::new(),
in_flight: 0,
price_per_hour: 3.6,
local: false,
};
assert!(
policy::table_weight(&returning, &w) > policy::table_weight(&stranger, &w),
"a worker measured at 12ms two minutes ago must outrank one merely \
assumed at {}ms",
w.unmeasured_ms + w.mesh_hop_ms
);
}
}