use crate::agent::config::AgentConfig;
use crate::agent::events::EventBus;
use crate::agent::files::{self, DesiredState, CHILDREN_FILE, HISTORY_FILE, STATE_FILE};
use crate::agent::history::History;
use crate::agent::hub::{self, CredWatch, Credentials, HubClient, HubError};
use crate::agent::hubwire::HubSummary;
use crate::agent::jobs::{self, Journal};
use crate::agent::merge::{self, LocalView, MergeInput};
use crate::agent::offers::{self, GpuProbe, OfferApiError};
use crate::agent::plan::{self, Desired};
use crate::agent::summary::{HubStatus, MeshStatus, NetworkCapacity, Problem, Summary};
use crate::agent::summary_v2::{self, Ledger, LedgerPeriod, SummaryV2};
use crate::agent::supervisor::{self, HttpBrokerApi, SpawnSpec, Supervisor};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::sync::Notify;
#[derive(Debug, Clone, PartialEq)]
pub struct ApiError {
pub status: u16,
pub code: &'static str,
pub message: String,
}
impl ApiError {
fn new(status: u16, code: &'static str, message: impl Into<String>) -> Self {
Self {
status,
code,
message: message.into(),
}
}
pub fn invalid(m: impl Into<String>) -> Self {
Self::new(400, "invalid_input", m)
}
pub fn conflict(m: impl Into<String>) -> Self {
Self::new(409, "conflict", m)
}
pub fn hub(m: impl Into<String>) -> Self {
Self::new(502, "hub_error", m)
}
pub fn prerequisite(m: impl Into<String>) -> Self {
Self::new(503, "prerequisite_missing", m)
}
pub fn internal(m: impl Into<String>) -> Self {
Self::new(500, "internal", m)
}
}
pub const HANDOFF_SEEN_FILE: &str = "handoff_seen.json";
const LEDGER_POLL_INTERVAL: Duration = Duration::from_secs(60);
#[derive(Default)]
struct LedgerCache {
last_good: Option<Ledger>,
attempted_at: Option<Instant>,
}
#[derive(Default)]
struct Published {
summary: Option<serde_json::Value>,
problems: Vec<String>,
policy_rev: Option<u64>,
}
const NETWORK_POLL_INTERVAL: Duration = Duration::from_secs(60);
const NETWORK_STALE_AFTER: Duration = Duration::from_secs(5 * 60);
#[derive(Default)]
struct NetworkCache {
last_good: Option<NetworkCapacity>,
fetched_at: Option<Instant>,
attempted_at: Option<Instant>,
}
const ADMIN_FRESH: Duration = Duration::from_secs(60);
#[derive(Default)]
struct HubCache {
last_good: Option<HubSummary>,
fetched_at: Option<Instant>,
last_ok: Option<String>,
error: Option<String>,
error_detail: Option<String>,
failures: u32,
reachable: bool,
unauthorized: bool,
too_old: bool,
}
pub struct Core {
pub cfg: AgentConfig,
node_pubkey: String,
node_name: String,
desired: Mutex<DesiredState>,
hub: Mutex<HubCache>,
network: Mutex<NetworkCache>,
creds: Mutex<CredWatch>,
local: Mutex<LocalView>,
local_problems: Mutex<Vec<Problem>>,
vpn_problem: Mutex<Option<Problem>>,
mesh: Mutex<MeshStatus>,
history: Mutex<History>,
supervisor: Mutex<Supervisor>,
tz_source: Mutex<Box<dyn Fn() -> String + Send + Sync>>,
tz_override: Mutex<Option<String>>,
pub(crate) broker_unmanaged: AtomicBool,
pub wake_reconcile: Notify,
pub wake_hub: Notify,
stop_requested: Arc<AtomicBool>,
container_workers: bool,
pub events: EventBus,
pub outbox: crate::agent::outbox::Outbox,
jobs: Mutex<Journal>,
pub atelier_jobs: crate::sandbox::job::JobManager<crate::sandbox::job::DockerJobCli>,
pub hosting: Mutex<crate::agent::hosting_loop::HostingRuntime>,
pub admission: crate::policy::Admission,
pub hosting_status: Mutex<crate::agent::hosting_loop::HostingStatus>,
ledger: Mutex<LedgerCache>,
published: Mutex<Published>,
gpu_probe: Arc<GpuProbe>,
gpus: Mutex<Option<Vec<crate::bench_node::fingerprint::GpuId>>>,
handoffs: Mutex<crate::deeplink::Handoffs>,
#[cfg(test)]
handoff_test: Mutex<Option<(Vec<crate::deeplink::TrustedKey>, Option<String>)>>,
}
impl Core {
pub fn new(cfg: AgentConfig) -> std::io::Result<Arc<Self>> {
files::ensure_dir(&cfg.dir)?;
let desired =
files::load_json::<DesiredState>(&cfg.dir.join(STATE_FILE)).unwrap_or_default();
let spawn_spec = SpawnSpec::from_config(&cfg);
let container_workers = spawn_spec.worker_backend != supervisor::WorkerBackend::Host;
let mut sup = Supervisor::new(spawn_spec);
if let Some(children) = files::load_json(&cfg.dir.join(CHILDREN_FILE)) {
sup.adopt(&children, &supervisor::proc_start_time);
}
if container_workers {
sup.sweep_orphans();
}
let stop_requested = sup.stop_flag();
Ok(Arc::new(Self {
node_pubkey: cfg.node_pubkey(),
node_name: crate::broker::node_name_or_default(),
desired: Mutex::new(desired),
hub: Mutex::new(HubCache::default()),
network: Mutex::new(NetworkCache::default()),
creds: Mutex::new(CredWatch::new(cfg.state_dir.join("credentials"))),
local: Mutex::new(LocalView::default()),
local_problems: Mutex::new(vec![]),
vpn_problem: Mutex::new(None),
mesh: Mutex::new(MeshStatus::default()),
history: Mutex::new(History::load(
&cfg.dir.join(HISTORY_FILE),
chrono::Utc::now(),
)),
supervisor: Mutex::new(sup),
tz_source: Mutex::new(Box::new(hub::system_tz)),
tz_override: Mutex::new(None),
broker_unmanaged: AtomicBool::new(false),
wake_reconcile: Notify::new(),
wake_hub: Notify::new(),
stop_requested,
container_workers,
events: EventBus::new(),
outbox: crate::agent::outbox::Outbox::new(&cfg.state_dir),
jobs: Mutex::new(Journal::load(&cfg.dir)),
atelier_jobs: crate::sandbox::job::JobManager::new(
crate::sandbox::job::JobRuntime::new(
crate::sandbox::job::DockerJobCli,
cfg.dir.join("atelier-docker"),
crate::sandbox::job::Admission::node(),
),
),
hosting: Mutex::new(Default::default()),
admission: crate::policy::Admission::at(cfg.state_dir.clone()),
hosting_status: Mutex::new(Default::default()),
ledger: Mutex::new(LedgerCache::default()),
published: Mutex::new(Published::default()),
gpu_probe: Arc::new(offers::real_probe()),
gpus: Mutex::new(None),
handoffs: Mutex::new(crate::deeplink::Handoffs::new(Some(
cfg.dir.join(HANDOFF_SEEN_FILE),
))),
#[cfg(test)]
handoff_test: Mutex::new(None),
cfg,
}))
}
pub fn desired(&self) -> DesiredState {
self.desired.lock().unwrap().clone()
}
pub fn hub_failures(&self) -> u32 {
self.hub.lock().unwrap().failures
}
fn credentials(&self) -> Option<Credentials> {
self.creds.lock().unwrap().current()
}
fn api_url(&self) -> String {
self.credentials()
.and_then(|c| c.api_url)
.unwrap_or_else(|| self.cfg.api_url.clone())
}
fn client(&self) -> Option<HubClient> {
let c = self.credentials()?;
let base = c
.api_url
.clone()
.unwrap_or_else(|| self.cfg.api_url.clone());
Some(HubClient::new(
&base,
&c.api_key,
self.cfg.timings.hub_timeout,
))
}
fn save_desired(&self, d: &DesiredState) -> Result<(), ApiError> {
files::save_json(&self.cfg.dir.join(STATE_FILE), d)
.map_err(|e| ApiError::internal(e.to_string()))
}
pub fn summary(&self) -> Summary {
let logged_in = self.credentials().is_some();
let api_url = self.api_url();
let desired = self.desired();
let mut local = self.local.lock().unwrap().clone();
local.sharing = desired.sharing;
local.workers.desired = desired.workers;
local.workers.max = self.cfg.max_workers;
let hub = self.hub.lock().unwrap();
let mut problems = Vec::new();
if !logged_in || hub.unauthorized {
problems.push(Problem::new("not_logged_in"));
} else if hub.too_old {
problems.push(Problem::new("hub_too_old"));
} else if hub.failures > 0 {
let mut p = Problem::hub_unreachable(hub.last_good.is_some());
p.detail = hub.error_detail.clone();
problems.push(p);
}
for p in self.local_problems.lock().unwrap().iter() {
if !problems.iter().any(|q| q.code == p.code) {
problems.push(p.clone());
}
}
if let Some(p) = self.vpn_problem.lock().unwrap().clone() {
problems.push(p);
}
let mut s = merge::build_summary(MergeInput {
agent_version: env!("CARGO_PKG_VERSION"),
now: chrono::Utc::now(),
api_url: &api_url,
hub: hub.last_good.as_ref(),
hub_status: HubStatus {
reachable: hub.reachable,
last_ok: hub.last_ok.clone(),
error: hub.error.clone(),
},
problems,
node_pubkey: &self.node_pubkey,
node_name: &self.node_name,
local: &local,
mesh: self.mesh.lock().unwrap().clone(),
});
s.history = self.history.lock().unwrap().to_wire();
s.network = self.network_for_summary();
s
}
pub fn summary_v2(&self) -> SummaryV2 {
let since = chrono::Utc::now() - chrono::Duration::hours(24);
let jobs = {
let j = self.jobs.lock().unwrap();
summary_v2::Jobs {
live: j.live(),
done_24h: j.done_since(since) as u32,
}
};
summary_v2::build(
self.summary(),
summary_v2::Extras {
backend: if self.container_workers {
"container"
} else {
"host"
},
jobs,
ledger: self.ledger.lock().unwrap().last_good.clone(),
policy: crate::policy::peek(&self.cfg.state_dir)
.and_then(|p| serde_json::to_value(p).ok()),
admission: crate::policy::admit::read_snapshot(&self.cfg.state_dir),
revenue: crate::broker::revenue::read_snapshot(&self.cfg.state_dir),
},
)
}
pub(crate) fn account_sub(&self) -> Option<String> {
self.credentials()?;
let hub = self.hub.lock().unwrap();
hub.last_good
.as_ref()
.and_then(|h| h.account.zakuro_user_id.clone())
.filter(|s| !s.trim().is_empty())
}
pub fn handoff(
&self,
link: &str,
) -> Result<crate::deeplink::Preview, crate::deeplink::LinkError> {
use crate::deeplink::{self, Context, LinkError};
let env = deeplink::env_for_api_url(Some(&self.api_url()));
#[allow(unused_mut)]
let mut keys = deeplink::keys_for(env).map_err(LinkError::KeyConfig)?;
#[allow(unused_mut)]
let mut sub = self.account_sub();
#[cfg(test)]
if let Some((k, s)) = self.handoff_test.lock().unwrap().clone() {
keys = k;
sub = s;
}
let node_fp = crate::broker::node_identity::fingerprint_of_pubkey_b64(&self.node_pubkey)
.unwrap_or_default();
let ctx = Context {
keys: &keys,
env: env.name(),
account_sub: sub.as_deref(),
node_fp: &node_fp,
node_pubkey: &self.node_pubkey,
now: chrono::Utc::now().timestamp(),
};
self.handoffs
.lock()
.unwrap()
.open(link, &ctx, &deeplink::ISSUES)
}
#[cfg(test)]
pub(crate) fn set_handoff_test(
&self,
keys: Vec<crate::deeplink::TrustedKey>,
sub: Option<&str>,
) {
*self.handoff_test.lock().unwrap() = Some((keys, sub.map(str::to_string)));
}
pub fn node_pubkey(&self) -> &str {
&self.node_pubkey
}
pub fn handoff_confirm(
&self,
jti: &str,
) -> Result<crate::deeplink::Confirmed, crate::deeplink::ConfirmError> {
self.handoffs
.lock()
.unwrap()
.confirm(jti, chrono::Utc::now().timestamp())
}
pub async fn ledger(
&self,
by: &str,
from: Option<&str>,
to: Option<&str>,
) -> Result<serde_json::Value, OfferApiError> {
let q = crate::agent::ledger::Query::parse(by, from, to)
.map_err(|m| OfferApiError::new(400, "invalid_input", m))?;
let client = self.offers_client()?;
let tz = self.tz_for_summary();
let body = client
.provide_ledger(q.from.as_deref(), q.to.as_deref(), &tz)
.await
.map_err(|e| match e {
hub::HubError::Unauthorized => OfferApiError::new(
409,
"not_logged_in",
"the hub rejected this Mac's key: run `zc login`",
),
hub::HubError::NotFound => OfferApiError::new(
502,
"hub_too_old",
"the hub does not have the ledger route yet",
),
hub::HubError::Transport(m) => OfferApiError::new(
502,
"hub_unreachable",
format!("could not reach the hub: {m}"),
),
other => OfferApiError::new(502, "hub_error", other.message()),
})?;
Ok(crate::agent::ledger::view(&q, &tz, &body))
}
pub fn set_vpn_problem(&self, detail: Option<String>) {
*self.vpn_problem.lock().unwrap() = detail.map(|d| {
let mut p = Problem::new("vpn_down");
p.detail = Some(d);
p
});
}
pub fn sharing_wanted(&self) -> bool {
self.desired().sharing
}
pub fn jobs(&self, q: &jobs::Query) -> Vec<jobs::Job> {
self.jobs.lock().unwrap().query(q)
}
pub fn publish_changes(&self) {
let summary = self.summary_v2();
let mut wire = serde_json::to_value(&summary).unwrap_or_default();
let codes: Vec<String> = summary.problems.iter().map(|p| p.code.clone()).collect();
let mut published = self.published.lock().unwrap();
for p in &summary.problems {
if !published.problems.contains(&p.code) {
self.events.publish(
"problem.raised",
serde_json::to_value(p).unwrap_or_default(),
);
}
}
for code in &published.problems {
if !codes.contains(code) {
self.events
.publish("problem.cleared", serde_json::json!({ "code": code }));
}
}
published.problems = codes;
if let Some(rev) = crate::policy::peek(&self.cfg.state_dir).map(|p| p.revision) {
if published.policy_rev.is_some_and(|seen| seen != rev) {
self.events
.publish("policy.changed", serde_json::json!({ "revision": rev }));
}
published.policy_rev = Some(rev);
}
let full = wire.clone();
if let Some(o) = wire.as_object_mut() {
o.remove("as_of");
if let Some(hub) = o.get_mut("hub").and_then(|h| h.as_object_mut()) {
hub.remove("last_ok");
}
if let Some(l) = o.get_mut("ledger").and_then(|l| l.as_object_mut()) {
l.remove("as_of");
}
}
if published.summary.as_ref() != Some(&wire) {
published.summary = Some(wire);
self.events.publish("summary.changed", full);
}
}
pub async fn refresh_ledger(&self) {
{
let l = self.ledger.lock().unwrap();
if l.attempted_at
.is_some_and(|t| t.elapsed() < LEDGER_POLL_INTERVAL)
{
return;
}
}
let Some(client) = self.client() else {
self.ledger.lock().unwrap().attempted_at = Some(Instant::now());
return;
};
let result = client.earnings(&self.tz_for_summary()).await;
let mut l = self.ledger.lock().unwrap();
l.attempted_at = Some(Instant::now());
if let Ok(e) = result {
l.last_good = Some(Ledger {
unit: "jpy".into(),
today: LedgerPeriod {
earned: e.today,
spent: None,
by_source: None,
},
month: LedgerPeriod {
earned: e.month.earned,
spent: Some(e.month.spent),
by_source: None,
},
tz: e.tz,
days: e
.week
.iter()
.map(|d| summary_v2::LedgerDay {
date: d.date.clone(),
earned: d.earned,
})
.collect(),
prev_month_earned: e.month.prev_month_earned,
as_of: chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true),
});
}
}
fn network_for_summary(&self) -> Option<NetworkCapacity> {
let n = self.network.lock().unwrap();
let fresh = n
.fetched_at
.is_some_and(|t| t.elapsed() <= NETWORK_STALE_AFTER);
fresh.then(|| n.last_good).flatten()
}
pub fn set_sharing(&self, on: bool) -> Result<Summary, ApiError> {
if on && self.broker_unmanaged.load(Ordering::Relaxed) {
return Err(ApiError::conflict(
"a broker the agent didn't start is running on this Mac; run `zc down` so the agent can manage sharing",
));
}
{
let mut d = self.desired.lock().unwrap();
d.sharing = on;
self.save_desired(&d)?;
}
self.wake_reconcile.notify_one();
self.wake_hub.notify_one(); self.publish_changes();
Ok(self.summary())
}
pub fn is_draining(&self) -> bool {
self.local.lock().unwrap().workers.draining > 0
}
pub fn set_workers(&self, count: u32) -> Result<Summary, ApiError> {
if count > self.cfg.max_workers {
return Err(ApiError::invalid(format!(
"count must be between 0 and {}",
self.cfg.max_workers
)));
}
{
let mut d = self.desired.lock().unwrap();
d.workers = count;
self.save_desired(&d)?;
}
self.wake_reconcile.notify_one();
self.wake_hub.notify_one();
self.publish_changes();
Ok(self.summary())
}
async fn gpus(&self) -> Option<Vec<crate::bench_node::fingerprint::GpuId>> {
if let Some(g) = self.gpus.lock().unwrap().clone() {
return Some(g);
}
let probe = self.gpu_probe.clone();
let found = tokio::task::spawn_blocking(move || probe()).await.ok()??;
*self.gpus.lock().unwrap() = Some(found.clone());
Some(found)
}
fn offers_client(&self) -> Result<HubClient, OfferApiError> {
self.client().ok_or_else(|| {
OfferApiError::new(
409,
"not_logged_in",
"this Mac is not signed in: run `zc login`",
)
})
}
fn offer_error(e: hub::OfferError) -> OfferApiError {
match e {
hub::OfferError::Unreachable(m) => OfferApiError::new(
502,
"hub_unreachable",
format!("could not reach the hub: {m}"),
),
hub::OfferError::Rejected { status: 401, .. } => OfferApiError::new(
409,
"not_logged_in",
"the hub rejected this Mac's key: run `zc login`",
),
hub::OfferError::Rejected {
status,
code,
message,
} => OfferApiError::new(
status,
code.unwrap_or_else(|| "hub_error".to_string()),
message,
),
}
}
pub async fn is_admin(&self) -> bool {
if self.credentials().is_none() {
return false;
}
let fresh_admin = |c: &Core| -> Option<bool> {
let h = c.hub.lock().unwrap();
if h.unauthorized {
return Some(false);
}
let fresh = h.fetched_at.is_some_and(|t| t.elapsed() < ADMIN_FRESH);
fresh.then(|| h.last_good.as_ref().is_some_and(|s| s.account.is_admin))
};
if let Some(a) = fresh_admin(self) {
return a;
}
self.refresh_hub().await;
fresh_admin(self).unwrap_or(false)
}
pub async fn offers(&self) -> Result<serde_json::Value, OfferApiError> {
let client = self.offers_client()?;
let mut body = client
.offers(&self.node_pubkey)
.await
.map_err(Self::offer_error)?;
offers::annotate_all(&mut body, self.gpus().await.as_deref());
Ok(body)
}
pub async fn put_offer(
&self,
template_id: &str,
mut body: serde_json::Value,
) -> Result<serde_json::Value, OfferApiError> {
let client = self.offers_client()?;
body["node"] = serde_json::Value::String(self.node_pubkey.clone());
let mut offer = client
.put_offer(template_id, &body)
.await
.map_err(Self::offer_error)?;
offers::annotate_offer(&mut offer, self.gpus().await.as_deref());
Ok(offer)
}
pub async fn set_price(&self, scope: &str, price: Option<f64>) -> Result<Summary, ApiError> {
if scope != "device" && scope != "default" {
return Err(ApiError::invalid(format!(
r#"scope must be "device" or "default", got {scope:?}"#
)));
}
let bounds = self
.hub
.lock()
.unwrap()
.last_good
.as_ref()
.map(HubSummary::bounds)
.unwrap_or_default();
if let Some(p) = price {
if !p.is_finite() || p < bounds.min as f64 || p > bounds.max as f64 {
return Err(ApiError::invalid(format!(
"price_per_hour must be between {} and {}",
bounds.min, bounds.max
)));
}
}
let client = self
.client()
.ok_or_else(|| ApiError::prerequisite("this Mac is not signed in: run `zc login`"))?;
if scope == "default" {
client
.put_default_price(price)
.await
.map_err(|e| ApiError::hub(e.message()))?;
} else {
let stale = self
.hub
.lock()
.unwrap()
.fetched_at
.is_none_or(|t| t.elapsed() > Duration::from_secs(120));
if stale {
self.refresh_hub().await;
}
let (ids, hub_error): (Vec<i64>, Option<String>) = {
let h = self.hub.lock().unwrap();
match h.last_good.as_ref() {
Some(summary) => (
summary
.this_mac_workers(&self.node_pubkey)
.iter()
.map(|w| w.id)
.collect(),
None,
),
None => (vec![], h.error.clone()),
}
};
if ids.is_empty() {
if let Some(msg) = hub_error {
return Err(ApiError::hub(msg));
}
return Err(ApiError::conflict(
"this Mac has no workers on the hub yet: start sharing first",
));
}
hub::set_device_price(&client, &ids, price)
.await
.map_err(|failed| {
let detail: Vec<String> =
failed.iter().map(|(id, e)| format!("{id}: {e}")).collect();
ApiError::hub(format!(
"could not price {} worker(s): {}",
failed.len(),
detail.join("; ")
))
})?;
}
self.refresh_hub().await;
self.publish_changes();
Ok(self.summary())
}
fn tz_for_summary(&self) -> String {
if let Some(tz) = self.tz_override.lock().unwrap().clone() {
return tz;
}
(self.tz_source.lock().unwrap())()
}
#[cfg(test)]
fn set_tz_source_for_test(&self, tz: &'static str) {
*self.tz_source.lock().unwrap() = Box::new(move || tz.to_string());
}
#[cfg(test)]
fn clear_network_poll_gate_for_test(&self) {
self.network.lock().unwrap().attempted_at = None;
}
#[cfg(test)]
fn age_network_fetch_for_test(&self, age: Duration) {
if let Some(t) = self.network.lock().unwrap().fetched_at.as_mut() {
*t = Instant::now() - age;
}
}
pub async fn refresh_hub(&self) {
let Some(client) = self.client() else {
let mut h = self.hub.lock().unwrap();
h.reachable = false;
h.error = Some("not signed in".to_string());
return;
};
let tz = self.tz_for_summary();
let mut result = client.summary(&tz).await;
if tz != "UTC" && matches!(result, Err(HubError::Status(422, _))) {
*self.tz_override.lock().unwrap() = Some("UTC".to_string());
result = client.summary("UTC").await;
}
{
let mut h = self.hub.lock().unwrap();
match result {
Ok(summary) => {
let requests_5h = self.local.lock().unwrap().requests.last_5h;
let earned_today = summary.earnings.today;
let path = self.cfg.dir.join(HISTORY_FILE);
let mut history = self.history.lock().unwrap();
history.record(chrono::Utc::now(), requests_5h, earned_today);
let _ = history.save(&path);
drop(history);
h.last_good = Some(summary);
h.fetched_at = Some(Instant::now());
h.last_ok =
Some(chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true));
h.error = None;
h.error_detail = None;
h.failures = 0;
h.reachable = true;
h.unauthorized = false;
h.too_old = false;
}
Err(e) => {
h.unauthorized = e == HubError::Unauthorized;
h.too_old = e == HubError::NotFound;
h.reachable = h.too_old; h.error = Some(e.message());
h.error_detail = e.detail();
h.failures += 1;
}
}
}
self.seed_new_workers(&client).await;
}
pub async fn refresh_network(&self) {
{
let n = self.network.lock().unwrap();
if n.attempted_at
.is_some_and(|t| t.elapsed() < NETWORK_POLL_INTERVAL)
{
return;
}
}
let Some(client) = self.client() else {
self.network.lock().unwrap().attempted_at = Some(Instant::now());
return;
};
let result = client.network().await;
let mut n = self.network.lock().unwrap();
n.attempted_at = Some(Instant::now());
if let Some(status) = result {
n.last_good = Some(status);
n.fetched_at = Some(Instant::now());
}
}
async fn seed_new_workers(&self, client: &HubClient) {
let (targets, all_ids) = {
let h = self.hub.lock().unwrap();
let Some(summary) = h.last_good.as_ref() else {
return;
};
let mine = summary.this_mac_workers(&self.node_pubkey);
let known = self.desired.lock().unwrap().known_worker_ids.clone();
let min = summary.price_bounds.min;
(
hub::seed_targets(&mine, &known, min),
mine.iter().map(|w| w.id).collect::<Vec<_>>(),
)
};
let mut seeded = Vec::new();
for (id, price) in &targets {
match client.put_worker_price(*id, Some(*price)).await {
Ok(()) => seeded.push(*id),
Err(e) => eprintln!(
" [AGENT] seeding price on worker row {id}: {}",
e.message()
),
}
}
let target_ids: Vec<i64> = targets.iter().map(|(id, _)| *id).collect();
let mut d = self.desired.lock().unwrap();
let before = d.known_worker_ids.len();
for id in all_ids {
let now_known = !target_ids.contains(&id) || seeded.contains(&id);
if now_known && !d.known_worker_ids.contains(&id) {
d.known_worker_ids.push(id);
}
}
if d.known_worker_ids.len() != before {
let _ = files::save_json(&self.cfg.dir.join(STATE_FILE), &*d);
}
drop(d);
if !seeded.is_empty() {
self.wake_hub.notify_one(); }
}
pub fn reconcile_once(&self) {
if self.stop_requested.load(Ordering::Relaxed) {
return;
}
let d = self.desired();
let desired = Desired {
sharing: d.sharing,
workers: d.workers,
};
let path = std::env::var("PATH").unwrap_or_default();
let zakuro_dir = crate::up::resolve_zakuro_dir_for(
self.cfg.worker_dir_env.clone(),
Some(&self.cfg.state_dir),
&self.cfg.worker_dir_legacy_candidates,
);
let prerequisites = plan::prerequisite_problems(
plan::find_in_path("uv", &path).is_some(),
zakuro_dir.is_ok(),
zakuro_dir.as_ref().err().and_then(|m| m.detail()),
self.credentials().is_some(),
self.cfg.worker_template_overridden,
self.container_workers
.then(crate::up::docker_cli_available_cached),
);
let route = crate::vpn::route();
let now = Instant::now();
let mut sup = self.supervisor.lock().unwrap();
sup.set_worker_cwd(zakuro_dir.ok());
sup.set_route(route.kind());
let api = HttpBrokerApi {
port: sup.broker_port(),
worker_key: crate::worker_key::resolve(
std::env::var(crate::worker_key::ENV).ok(),
Some(&self.cfg.state_dir),
),
};
sup.reconcile(&desired, &api, prerequisites.is_empty(), now);
let observed = sup.observed_workers();
let mesh = crate::agent::mesh::mesh_status(&route, sup.broker_running(), sup.broker_port());
let _ = files::save_json(&self.cfg.dir.join(CHILDREN_FILE), &sup.children());
let mut problems = prerequisites;
problems.extend(sup.problems(now));
let view = sup.local_view(&desired, self.cfg.max_workers);
self.broker_unmanaged
.store(sup.broker_unmanaged(), Ordering::Relaxed);
drop(sup);
*self.local.lock().unwrap() = view;
*self.local_problems.lock().unwrap() = problems;
*self.mesh.lock().unwrap() = mesh;
let job_events = self
.jobs
.lock()
.unwrap()
.observe(observed.as_deref(), chrono::Utc::now());
for e in job_events {
self.events
.publish(e.kind, serde_json::to_value(&e.job).unwrap_or_default());
}
self.publish_changes();
}
pub fn request_stop(&self) {
self.stop_requested.store(true, Ordering::Relaxed);
}
pub fn shutdown_children(&self) {
self.request_stop();
let mut sup = self
.supervisor
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
sup.terminate_all(self.cfg.timings.shutdown_wait);
let _ = files::save_json(&self.cfg.dir.join(CHILDREN_FILE), &sup.children());
}
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use crate::agent::files::tests::tmp;
use crate::agent::hub::tests::{mock_hub, Seen};
fn block_on<F: std::future::Future>(f: F) -> F::Output {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(f)
}
pub(crate) fn set_gpu_probe(core: &mut Arc<Core>, p: GpuProbe) {
Arc::get_mut(core).expect("core not shared yet").gpu_probe = Arc::new(p);
}
pub(crate) fn core_with(max_workers: u32) -> Arc<Core> {
let dir = tmp("core");
std::fs::create_dir_all(&dir).unwrap();
let mut cfg = AgentConfig::for_dirs(dir);
cfg.max_workers = max_workers;
Core::new(cfg).unwrap()
}
fn admin_then(later: (u16, bool)) -> Arc<Core> {
let core = core_with(0);
let pk = core.cfg.node_pubkey();
let body = move |admin: bool| {
hub_json(&pk, "").replace(
r#""balance":1240.5}"#,
&format!(r#""balance":1240.5,"is_admin":{admin}}}"#),
)
};
let (hub, _) = crate::agent::hub::tests::mock_hub(move |_, url, nth| {
if !url.starts_with("/api/provide/summary") {
return (404, "{}".into());
}
match nth {
1 => (200, body(true)),
_ => (
later.0,
if later.0 == 200 {
body(later.1)
} else {
"{}".into()
},
),
}
});
sign_in(&core, &hub);
core
}
fn age_hub(core: &Core, by: Duration) {
let mut h = core.hub.lock().unwrap();
h.fetched_at = h.fetched_at.map(|t| t - by);
}
#[test]
fn a_demoted_admin_loses_access_once_the_summary_is_a_minute_old() {
let core = admin_then((200, false));
assert!(block_on(core.is_admin()));
assert!(block_on(core.is_admin()), "a fresh summary is reused");
age_hub(&core, ADMIN_FRESH);
assert!(!block_on(core.is_admin()));
}
#[test]
fn a_revoked_key_or_an_unreachable_hub_is_not_an_admin() {
for (status, what) in [(401, "revoked key"), (503, "hub down")] {
let core = admin_then((status, true));
assert!(block_on(core.is_admin()), "{what}");
age_hub(&core, ADMIN_FRESH);
assert!(
!block_on(core.is_admin()),
"{what}: stale cache must not count"
);
}
}
#[test]
fn signing_out_ends_admin_access_at_once() {
let core = admin_then((200, true));
assert!(block_on(core.is_admin()));
std::fs::remove_file(core.cfg.state_dir.join("credentials")).unwrap();
assert!(!block_on(core.is_admin()));
}
pub(crate) fn sign_in(core: &Core, hub: &str) {
std::fs::write(
core.cfg.state_dir.join("credentials"),
format!("api_key=zk_1_test\napi_url={hub}\n"),
)
.unwrap();
}
pub(crate) fn hub_json(pk: &str, worker_rows: &str) -> String {
format!(
r#"{{"account":{{"username":"jean","email":"jean@zakuro-ai.com","balance":1240.5}},"default_price_per_hour":null,"price_bounds":{{"min":1,"max":120,"step":1}},"earnings":{{"today":1.0,"last_7d":2.0,"tz":"UTC"}},"devices":[{{"node_pubkey":"{pk}","name":"mac","freshness":"fresh","last_seen_at":null,"workers":[{worker_rows}]}}],"web_base":"https://stg.hub.zakuro-ai.com"}}"#
)
}
pub(crate) fn row(id: i64, price: Option<f64>) -> String {
let p = price.map_or("null".to_string(), |p| p.to_string());
let e = price.unwrap_or(3.6);
format!(
r#"{{"id":{id},"worker_id":"fp-w{id}","status":"online","last_seen":null,"price_per_hour":{p},"reported_price_per_hour":3.6,"effective_price_per_hour":{e},"disabled":false}}"#
)
}
fn puts(seen: &Seen) -> Vec<(String, String)> {
seen.lock()
.unwrap()
.iter()
.filter(|e| e.0 == "PUT")
.map(|e| (e.1.clone(), e.2.clone()))
.collect()
}
#[test]
fn the_ledger_comes_from_the_hubs_earnings_and_survives_a_failure() {
let core = core_with(3);
assert!(
core.summary_v2().ledger.is_none(),
"null until the hub answers"
);
let fail = Arc::new(AtomicBool::new(false));
let f = fail.clone();
let (hub, seen) = mock_hub(move |_, url, _| {
if url.starts_with("/api/provide/earnings") && !f.load(Ordering::Relaxed) {
(
200,
r#"{"goal_per_day": null, "today": 4.5, "streak_days": 0, "always_on_days": 0,
"week": [], "month": {"earned": 120.25, "spent": 30.0, "balance": 900.0,
"prev_month_earned": 0.0}, "did": [], "tz": "UTC"}"#
.to_string(),
)
} else {
(500, "{}".into())
}
});
sign_in(&core, &hub);
core.set_tz_source_for_test("UTC");
block_on(core.refresh_ledger());
let l = core
.summary_v2()
.ledger
.expect("ledger after a good answer");
assert_eq!((l.today.earned, l.today.spent), (4.5, None));
assert_eq!((l.month.earned, l.month.spent), (120.25, Some(30.0)));
assert_eq!((l.unit.as_str(), l.tz.as_str()), ("jpy", "UTC"));
assert!(seen
.lock()
.unwrap()
.iter()
.any(|e| e.1 == "/api/provide/earnings?tz=UTC"));
fail.store(true, Ordering::Relaxed);
core.ledger.lock().unwrap().attempted_at = None;
block_on(core.refresh_ledger());
assert_eq!(
core.summary_v2().ledger.map(|l| l.month.spent),
Some(Some(30.0)),
"a failed poll keeps the last good ledger"
);
}
#[test]
fn workers_are_persisted_and_bounded_by_max() {
let core = core_with(3);
let s = core.set_workers(2).unwrap();
assert_eq!((s.this_mac.workers.desired, s.this_mac.workers.max), (2, 3));
let on_disk: DesiredState = files::load_json(&core.cfg.dir.join(STATE_FILE)).unwrap();
assert_eq!(on_disk.workers, 2);
let err = core.set_workers(4).unwrap_err();
assert_eq!((err.status, err.code), (400, "invalid_input"));
}
#[test]
fn sharing_on_is_refused_while_an_unmanaged_broker_runs() {
let core = core_with(3);
core.broker_unmanaged.store(true, Ordering::Relaxed);
assert_eq!(core.set_sharing(true).unwrap_err().status, 409);
assert!(!core.set_sharing(false).unwrap().this_mac.sharing);
core.broker_unmanaged.store(false, Ordering::Relaxed);
assert!(core.set_sharing(true).unwrap().this_mac.sharing);
}
#[test]
fn before_sign_in_the_summary_says_not_logged_in() {
let s = core_with(3).summary();
assert!(s.account.is_none());
assert_eq!(s.problems[0].code, "not_logged_in");
}
#[test]
fn refresh_keeps_the_last_snapshot_and_names_the_problem() {
let core = core_with(3);
let pk = core.node_pubkey.clone();
let status = Arc::new(Mutex::new(200u16));
let st = status.clone();
let (hub, _) = mock_hub(move |_, _, _| {
let s = *st.lock().unwrap();
(
s,
if s == 200 {
hub_json(&pk, &row(11, None))
} else {
"{}".into()
},
)
});
sign_in(&core, &hub);
block_on(core.refresh_hub());
let s = core.summary();
assert_eq!(s.account.as_ref().unwrap().username, "jean");
assert!(s.devices.as_ref().unwrap()[0].this_mac);
assert!(s.hub.reachable && s.problems.is_empty(), "{:?}", s.problems);
*status.lock().unwrap() = 500;
block_on(core.refresh_hub());
let s = core.summary();
assert!(!s.hub.reachable);
assert!(s.account.is_some(), "the last good snapshot is kept");
assert!(s.problems.iter().any(|p| p.code == "hub_unreachable"));
assert_eq!(core.hub_failures(), 1);
*status.lock().unwrap() = 404;
block_on(core.refresh_hub());
let s = core.summary();
assert!(s.hub.reachable, "an old hub is still a reachable hub");
assert!(s.problems.iter().any(|p| p.code == "hub_too_old"));
*status.lock().unwrap() = 401;
block_on(core.refresh_hub());
assert!(core
.summary()
.problems
.iter()
.any(|p| p.code == "not_logged_in"));
}
#[test]
fn an_unreachable_hub_carries_its_cause_and_its_next_step() {
let core = core_with(3);
sign_in(&core, "http://127.0.0.1:1");
block_on(core.refresh_hub());
let s = core.summary();
let p = s
.problems
.iter()
.find(|p| p.code == "hub_unreachable")
.expect("the hub could not be reached");
let detail = p.detail.as_deref().expect("a next step, not null");
assert!(detail.to_lowercase().contains("vpn"), "{detail}");
let error = s.hub.error.expect("the failure is recorded");
assert!(
error.to_lowercase().contains("connection refused"),
"the cause, not just the url: {error}"
);
let answering = core_with(3);
let (hub, _) = mock_hub(|_, _, _| (500, r#"{"detail":"boom"}"#.into()));
sign_in(&answering, &hub);
block_on(answering.refresh_hub());
let s = answering.summary();
let p = s
.problems
.iter()
.find(|p| p.code == "hub_unreachable")
.expect("a 500 is still an unreachable hub");
assert_eq!(p.detail, None, "the hub answered: no tunnel to blame");
assert_eq!(
s.hub.error.as_deref(),
Some("hub answered HTTP 500: boom"),
"unchanged"
);
}
#[test]
fn the_unreachable_hint_promises_cached_values_only_when_there_are_some() {
let fresh = core_with(3);
sign_in(&fresh, "http://127.0.0.1:1");
block_on(fresh.refresh_hub());
let s = fresh.summary();
let hint = s
.problems
.iter()
.find(|p| p.code == "hub_unreachable")
.and_then(|p| p.hint.clone())
.expect("the hub could not be reached");
assert!(
hint.to_lowercase().contains("no cached values"),
"nothing has ever been cached: {hint}"
);
assert!(
s.account.is_none() && s.earnings.is_none() && s.devices.is_none(),
"and there is indeed nothing to show"
);
let cached = core_with(3);
let pk = cached.node_pubkey.clone();
let status = Arc::new(Mutex::new(200u16));
let st = status.clone();
let (hub, _) = mock_hub(move |_, _, _| {
let s = *st.lock().unwrap();
(
s,
if s == 200 {
hub_json(&pk, &row(11, None))
} else {
"{}".into()
},
)
});
sign_in(&cached, &hub);
block_on(cached.refresh_hub());
*status.lock().unwrap() = 500;
block_on(cached.refresh_hub());
let s = cached.summary();
assert_eq!(
s.problems
.iter()
.find(|p| p.code == "hub_unreachable")
.and_then(|p| p.hint.as_deref()),
Some("Showing the last known values"),
"one good answer landed, and it is still shown"
);
assert!(s.account.is_some() && s.devices.is_some());
}
#[test]
fn network_is_polled_at_most_once_per_interval() {
let core = core_with(3);
let calls = Arc::new(Mutex::new(0u32));
let c = calls.clone();
let (hub, _) = mock_hub(move |_, url, _| match url {
"/api/status/network" => {
*c.lock().unwrap() += 1;
(
200,
r#"{"workers": 5, "workers_busy": 2, "cpu_cores": 40, "cpu_cores_busy": 10,
"ram_gb": 200, "ram_gb_busy": 50, "gpus": 4, "gpus_busy": 1}"#
.to_string(),
)
}
_ => (404, "{}".into()),
});
sign_in(&core, &hub);
block_on(core.refresh_network());
let network = core
.summary()
.network
.expect("a successful poll populates network");
assert_eq!((network.workers, network.workers_busy), (5, 2));
assert_eq!((network.gpus, network.gpus_busy), (4, 1));
block_on(core.refresh_network());
assert_eq!(*calls.lock().unwrap(), 1, "the second call is within 60s");
}
#[test]
fn a_network_401_is_silently_no_data_and_never_a_problem() {
let core = core_with(3);
let (hub, _) = mock_hub(|_, _, _| (401, "{}".into()));
sign_in(&core, &hub);
block_on(core.refresh_network());
let s = core.summary();
assert_eq!(s.network, None);
assert!(!s.problems.iter().any(|p| p.code == "not_logged_in"));
}
#[test]
fn network_holds_the_last_good_value_then_drops_it_once_stale() {
let core = core_with(3);
let healthy = Arc::new(Mutex::new(true));
let h = healthy.clone();
let (hub, _) = mock_hub(move |_, url, _| match url {
"/api/status/network" if *h.lock().unwrap() => (
200,
r#"{"workers": 10, "workers_busy": 3, "cpu_cores": 80, "cpu_cores_busy": 20,
"ram_gb": 400, "ram_gb_busy": 100, "gpus": 6, "gpus_busy": 2}"#
.to_string(),
),
_ => (401, "{}".into()),
});
sign_in(&core, &hub);
block_on(core.refresh_network());
assert!(core.summary().network.is_some());
*healthy.lock().unwrap() = false;
core.clear_network_poll_gate_for_test();
block_on(core.refresh_network());
let network = core
.summary()
.network
.expect("a transient failure holds the last good value");
assert_eq!(network.workers, 10);
core.age_network_fetch_for_test(Duration::from_secs(301));
assert_eq!(
core.summary().network,
None,
"a stale figure is worse than no number"
);
}
#[test]
fn device_price_fans_out_then_new_rows_are_seeded() {
let core = core_with(3);
let pk = core.node_pubkey.clone();
let rows = Arc::new(Mutex::new(format!("{},{}", row(11, None), row(12, None))));
let r = rows.clone();
let (hub, seen) = mock_hub(move |method, url, _| match method {
"GET" if url.starts_with("/api/provide/summary") => {
(200, hub_json(&pk, &r.lock().unwrap()))
}
"PUT" => (200, "{}".into()),
_ => (404, "{}".into()),
});
sign_in(&core, &hub);
block_on(core.refresh_hub());
assert_eq!(core.desired().known_worker_ids, vec![11, 12]);
block_on(core.set_price("device", Some(18.0))).unwrap();
let body = r#"{"price_per_hour":18.0}"#.to_string();
let mut fanned_out = puts(&seen);
fanned_out.sort();
assert_eq!(
fanned_out,
vec![
("/api/workers/11/price".to_string(), body.clone()),
("/api/workers/12/price".to_string(), body.clone()),
]
);
*rows.lock().unwrap() = format!(
"{},{},{}",
row(11, Some(18.0)),
row(12, Some(18.0)),
row(13, None)
);
block_on(core.refresh_hub());
assert_eq!(
puts(&seen).last().unwrap(),
&("/api/workers/13/price".to_string(), body)
);
assert_eq!(core.desired().known_worker_ids, vec![11, 12, 13]);
}
#[test]
fn price_writes_are_validated_before_touching_the_hub() {
let core = core_with(3);
let status = |r: Result<Summary, ApiError>| r.unwrap_err().status;
assert_eq!(status(block_on(core.set_price("bogus", Some(10.0)))), 400);
assert_eq!(
status(block_on(core.set_price("device", Some(500.0)))),
400,
"outside {{1,120}}"
);
assert_eq!(status(block_on(core.set_price("default", Some(0.5)))), 400);
assert_eq!(
status(block_on(core.set_price("default", Some(10.0)))),
503,
"not signed in"
);
let (hub, _) = mock_hub(|_, _, _| (200, "{}".into()));
sign_in(&core, &hub);
assert_eq!(
status(block_on(core.set_price("device", Some(10.0)))),
502,
"the hub never produced a usable snapshot"
);
}
#[test]
fn a_price_write_gets_409_only_once_the_hub_confirms_this_mac_has_no_workers() {
let core = core_with(3);
let pk = core.node_pubkey.clone();
let (hub, _) = mock_hub(move |_, _, _| (200, hub_json(&pk, "")));
sign_in(&core, &hub);
let err = block_on(core.set_price("device", Some(10.0))).unwrap_err();
assert_eq!(err.status, 409);
}
#[test]
fn a_failed_seed_put_keeps_the_row_unknown_until_it_succeeds() {
let core = core_with(3);
let pk = core.node_pubkey.clone();
let rows = Arc::new(Mutex::new(format!(
"{},{}",
row(11, Some(18.0)),
row(12, Some(18.0))
)));
let r = rows.clone();
let (hub, seen) = mock_hub(move |method, url, nth| match method {
"GET" if url.starts_with("/api/provide/summary") => {
(200, hub_json(&pk, &r.lock().unwrap()))
}
"PUT" if url == "/api/workers/13/price" && nth == 1 => (500, "{}".into()),
"PUT" => (200, "{}".into()),
_ => (404, "{}".into()),
});
sign_in(&core, &hub);
block_on(core.refresh_hub());
assert_eq!(core.desired().known_worker_ids, vec![11, 12]);
*rows.lock().unwrap() = format!(
"{},{},{}",
row(11, Some(18.0)),
row(12, Some(18.0)),
row(13, None)
);
block_on(core.refresh_hub());
assert_eq!(
core.desired().known_worker_ids,
vec![11, 12],
"the failed seed keeps row 13 unknown"
);
assert!(puts(&seen)
.iter()
.any(|(u, _)| u == "/api/workers/13/price"));
block_on(core.refresh_hub());
assert_eq!(
core.desired().known_worker_ids,
vec![11, 12, 13],
"the retried seed succeeds and 13 becomes known"
);
}
#[test]
fn a_422_on_the_local_zone_retries_once_with_utc_then_sticks_to_it() {
let core = core_with(3);
core.set_tz_source_for_test("Asia/Tokyo");
let pk = core.node_pubkey.clone();
let requested_tz: TzLog = Default::default();
let seen = requested_tz.clone();
let (hub, _) = mock_hub(move |_, url, _| {
let raw = url.split("tz=").nth(1).unwrap_or("");
let tz = raw.replace("%2F", "/");
seen.lock().unwrap().push(tz.clone());
if tz == "UTC" {
(200, hub_json(&pk, &row(11, None)))
} else {
(422, "{}".into())
}
});
sign_in(&core, &hub);
block_on(core.refresh_hub());
let s = core.summary();
assert!(s.hub.reachable, "{:?}", s.problems);
assert!(s.account.is_some());
assert_eq!(
*requested_tz.lock().unwrap(),
vec!["Asia/Tokyo".to_string(), "UTC".to_string()],
"exactly one retry, with UTC"
);
requested_tz.lock().unwrap().clear();
block_on(core.refresh_hub());
assert_eq!(
*requested_tz.lock().unwrap(),
vec!["UTC".to_string()],
"later polls go straight to UTC"
);
}
#[test]
fn a_422_on_utc_itself_is_not_retried() {
let core = core_with(3);
core.set_tz_source_for_test("UTC");
let calls = Arc::new(Mutex::new(0u32));
let c = calls.clone();
let (hub, _) = mock_hub(move |_, _, _| {
*c.lock().unwrap() += 1;
(422, "{}".into())
});
sign_in(&core, &hub);
block_on(core.refresh_hub());
assert_eq!(
*calls.lock().unwrap(),
1,
"no retry when the zone is already UTC"
);
assert!(!core.summary().hub.reachable);
}
type TzLog = Arc<Mutex<Vec<String>>>;
#[test]
fn zakuro_dir_missing_clears_on_the_next_tick_once_the_directory_appears() {
let dir = tmp("zakuro-dir-appears");
std::fs::create_dir_all(&dir).unwrap();
let cfg = AgentConfig::for_dirs(dir.clone());
let core = Core::new(cfg).unwrap();
core.reconcile_once();
assert!(
core.summary()
.problems
.iter()
.any(|p| p.code == "zakuro_dir_missing"),
"no zak-zakuro directory exists yet"
);
std::fs::create_dir_all(dir.join("zak-zakuro/zakuro/worker")).unwrap();
std::fs::write(dir.join("zak-zakuro/zakuro/worker/server.py"), b"").unwrap();
core.reconcile_once();
assert!(
!core
.summary()
.problems
.iter()
.any(|p| p.code == "zakuro_dir_missing"),
"the very next tick must see the freshly cloned directory"
);
}
#[test]
fn container_runtime_does_not_require_the_zakuro_dir() {
let dir = tmp("zakuro-dir-container");
std::fs::create_dir_all(&dir).unwrap();
let mut cfg = AgentConfig::for_dirs(dir);
cfg.child_env
.push(("ZAKURO_SHARE_RUNTIME".into(), "container".into()));
let core = Core::new(cfg).unwrap();
core.reconcile_once();
let codes: Vec<_> = core
.summary()
.problems
.iter()
.map(|p| p.code.clone())
.collect();
assert!(
!codes
.iter()
.any(|c| c == "zakuro_dir_missing" || c == "uv_missing"),
"{codes:?}"
);
}
}