use crate::agent::core::{ApiError, Core};
use crate::agent::summary::Summary;
use axum::extract::rejection::JsonRejection;
use axum::extract::{Path, Request, State};
use axum::http::{header, HeaderMap, StatusCode};
use axum::middleware::{self, Next};
use axum::response::{IntoResponse, Response};
use axum::routing::{delete, get, post, put};
use axum::{Json, Router};
use serde::{Deserialize, Deserializer};
use std::sync::Arc;
#[derive(Clone)]
struct Guard {
token: Arc<String>,
port: u16,
}
pub fn router(core: Arc<Core>, token: String, port: u16) -> Router {
let guard = Guard {
token: Arc::new(token),
port,
};
Router::new()
.route("/v1/summary", get(get_summary))
.route("/v1/refresh", post(post_refresh))
.route("/v1/sharing", put(put_sharing))
.route("/v1/workers", put(put_workers))
.route("/v1/price", put(put_price))
.merge(v2_routes())
.fallback(not_found)
.method_not_allowed_fallback(method_not_allowed)
.layer(middleware::from_fn_with_state(guard, guard_layer))
.with_state(core)
}
fn v2_routes() -> Router<Arc<Core>> {
Router::new()
.route("/v2/summary", get(get_summary_v2))
.route("/v2/jobs", get(get_jobs))
.route("/v2/atelier/jobs", post(post_atelier_job))
.route("/v2/atelier/jobs/reconcile", post(post_atelier_reconcile))
.route(
"/v2/atelier/jobs/:id",
get(get_atelier_job).delete(delete_atelier_job),
)
.route("/v2/atelier/jobs/:id/logs", get(get_atelier_logs))
.route("/v2/events", get(crate::agent::server::sse::get_events))
.route("/v2/offers", get(get_offers))
.route("/v2/offers/:template_id", put(put_offer))
.route("/v2/logs", get(get_logs))
.route("/v2/logs/:name", get(get_log))
.route("/v2/policy", get(get_policy).put(put_policy))
.route("/v2/ledger", get(get_ledger))
.route(
"/v2/sandbox",
get(|| not_implemented("sandbox", ISSUE_SANDBOX)),
)
.route(
"/v2/sandbox/resize",
post(|| not_implemented("sandbox resize", ISSUE_BUDGET)),
)
.route(
"/v2/squads",
get(|| not_implemented("squads", ISSUE_SQUADS)),
)
.route(
"/v2/squads/join",
post(|| not_implemented("squads", ISSUE_SQUADS)),
)
.route(
"/v2/squads/:id/leave",
post(|| not_implemented("squads", ISSUE_SQUADS)),
)
.route(
"/v2/envs",
get(|| not_implemented("environments", ISSUE_ENVS))
.post(|| not_implemented("environments", ISSUE_ENVS)),
)
.route(
"/v2/envs/:id/connect",
post(|| not_implemented("environments", ISSUE_ENVS)),
)
.route(
"/v2/envs/:id",
delete(|| not_implemented("environments", ISSUE_ENVS)),
)
.route("/v2/handoff", post(post_handoff))
.route("/v2/handoff/:jti/confirm", post(post_handoff_confirm))
.route("/v2/deployments", get(get_deployments))
.route("/v2/deployments/:kind", put(put_deployment))
.route("/v2/images", get(get_images))
.route("/v2/images/:id", delete(delete_image))
.route("/v2/hosting", get(get_hosting).put(put_hosting))
.route("/v2/resources", get(get_resources))
.route(
"/v2/sandbox/provision",
get(get_provision).post(post_provision),
)
.route(
"/v2/local-nodes",
get(get_local_nodes).post(post_local_node),
)
.route(
"/v2/local-nodes/:id",
put(put_local_node).delete(delete_local_node),
)
.merge(crate::agent::outbox::routes::routes())
}
struct HostingSnapshot {
cat: Vec<crate::agent::runners::CatalogueEntry>,
unified: bool,
reason: Option<&'static str>,
serving: bool,
status: crate::agent::hosting_loop::HostingStatus,
}
fn hosting_snapshot(core: &Core) -> HostingSnapshot {
let mut st = core
.hosting_status
.lock()
.unwrap_or_else(|p| p.into_inner());
HostingSnapshot {
cat: st.catalogue(),
unified: st.unified(),
reason: st.reason,
serving: st.serving,
status: st.clone(),
}
}
impl HostingSnapshot {
fn ctx(&self) -> crate::agent::hosting_api::Ctx<'_> {
crate::agent::hosting_api::Ctx {
cat: &self.cat,
host: crate::sandbox::HostResources::detect(),
unified: self.unified,
host_reason: self.reason,
serving: self.serving,
}
}
}
fn load_hosting(
core: &Core,
snap: &HostingSnapshot,
) -> Result<crate::agent::hosting::Hosting, Box<Response>> {
crate::agent::hosting::load_or_default(&core.cfg.state_dir, &snap.cat)
.map_err(|e| Box::new(error(500, "hosting_settings", &e)))
}
fn deployments_response(core: &Core, snap: &HostingSnapshot) -> Response {
let h = match load_hosting(core, snap) {
Ok(h) => h,
Err(r) => return *r,
};
match crate::agent::hosting_api::deployments(&snap.ctx(), &h) {
Ok(v) => Json(v).into_response(),
Err(e) => error(500, "hosting_settings", &e),
}
}
async fn get_deployments(State(core): State<Arc<Core>>) -> Response {
tokio::task::spawn_blocking(move || {
let snap = hosting_snapshot(&core);
deployments_response(&core, &snap)
})
.await
.unwrap_or_else(|_| error(500, "internal", "deployments view failed"))
}
#[derive(Deserialize)]
struct EnabledBody {
enabled: bool,
}
async fn put_deployment(
State(core): State<Arc<Core>>,
Path(kind): Path<String>,
body: Result<Json<EnabledBody>, JsonRejection>,
) -> Response {
let b = match body {
Ok(Json(b)) => b,
Err(e) => return bad_body(e),
};
tokio::task::spawn_blocking(move || {
let snap = hosting_snapshot(&core);
let h = match load_hosting(&core, &snap) {
Ok(h) => h,
Err(r) => return *r,
};
match crate::agent::hosting_api::set_enabled(&snap.ctx(), &h, &kind, b.enabled) {
Ok(next) => match crate::agent::hosting::save(&core.cfg.state_dir, &next, &snap.cat) {
Ok(()) => deployments_response(&core, &snap),
Err(e) => error(500, "hosting_settings", &e),
},
Err(e) => error(e.status, e.code, &e.message),
}
})
.await
.unwrap_or_else(|_| error(500, "internal", "deployment update failed"))
}
fn image_store(
snap: &HostingSnapshot,
h: &crate::agent::hosting::Hosting,
) -> crate::agent::images::HostStore {
snap.status.image_store(h)
}
async fn get_images(State(core): State<Arc<Core>>) -> Response {
tokio::task::spawn_blocking(move || {
use crate::agent::images::Store;
let snap = hosting_snapshot(&core);
let h = match load_hosting(&core, &snap) {
Ok(h) => h,
Err(r) => return *r,
};
let store = image_store(&snap, &h);
let images: Vec<serde_json::Value> = store
.list()
.into_iter()
.map(|i| {
let secs = |t: std::time::SystemTime| {
t.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
};
serde_json::json!({
"id": i.id,
"kind": i.kind,
"size_bytes": i.size_bytes,
"last_used": i.last_used.map(secs),
"created": secs(i.created),
"earnings": i.earnings,
"in_use": store.in_use(&i.id),
})
})
.collect();
let ttl_days = h.idle_ttl().map(|d| d.as_secs() / 86_400).unwrap_or(7);
Json(serde_json::json!({ "images": images, "ttl_days": ttl_days })).into_response()
})
.await
.unwrap_or_else(|_| error(500, "internal", "images listing failed"))
}
async fn delete_image(State(core): State<Arc<Core>>, Path(id): Path<String>) -> Response {
tokio::task::spawn_blocking(move || {
use crate::agent::images::RemoveError;
let snap = hosting_snapshot(&core);
let h = match load_hosting(&core, &snap) {
Ok(h) => h,
Err(r) => return *r,
};
match crate::agent::images::remove_now(&image_store(&snap, &h), &id) {
Ok(freed) => Json(serde_json::json!({ "freed_bytes": freed })).into_response(),
Err(RemoveError::InUse) => error(409, "in_use", "a job or the owner is using it"),
Err(RemoveError::Timeout) => error(504, "timeout", "removal did not finish in time"),
Err(RemoveError::NotFound) => error(404, "not_found", "no such image"),
Err(RemoveError::Other(e)) => error(500, "remove_failed", &e),
}
})
.await
.unwrap_or_else(|_| error(500, "internal", "image removal failed"))
}
async fn get_resources() -> Response {
tokio::task::spawn_blocking(|| Json(crate::agent::resources::snapshot()).into_response())
.await
.unwrap_or_else(|_| error(500, "internal", "resources failed"))
}
fn local_nodes_view(core: &Core) -> crate::agent::local_nodes::View {
let home = &core.cfg.state_dir;
let this = core.summary().this_mac;
crate::agent::local_nodes::build(
home,
&this,
core.main_broker_port(),
&crate::nodes::Registry::load(home),
crate::nodes::MachineLimit::load(home).ok().as_ref(),
crate::agent::local_nodes::cached_usage(),
)
}
async fn get_local_nodes(State(core): State<Arc<Core>>) -> Response {
tokio::task::spawn_blocking(move || Json(local_nodes_view(&core)).into_response())
.await
.unwrap_or_else(|_| error(500, "internal", "local nodes failed"))
}
async fn local_node_change(
core: Arc<Core>,
f: impl FnOnce(&std::path::Path) -> Result<crate::nodes::NodeSpec, String> + Send + 'static,
) -> Response {
tokio::task::spawn_blocking(move || match f(&core.cfg.state_dir) {
Ok(_) => {
core.wake_nodes.notify_one();
Json(local_nodes_view(&core)).into_response()
}
Err(e) if e.starts_with("no node ") => error(404, "not_found", &e),
Err(e) => error(409, "node_refused", &e),
})
.await
.unwrap_or_else(|_| error(500, "internal", "local node change failed"))
}
async fn post_local_node(
State(core): State<Arc<Core>>,
body: Result<Json<crate::nodes::NewNode>, JsonRejection>,
) -> Response {
let req = match body {
Ok(Json(b)) => b,
Err(e) => return bad_body(e),
};
local_node_change(core, move |home| crate::nodes::add(home, &req)).await
}
async fn put_local_node(
State(core): State<Arc<Core>>,
Path(id): Path<String>,
body: Result<Json<crate::nodes::NodePatch>, JsonRejection>,
) -> Response {
let patch = match body {
Ok(Json(b)) => b,
Err(e) => return bad_body(e),
};
if id == crate::nodes::MAIN {
return error(
409,
"node_refused",
"the main node's size is the limit on Resources less the other nodes; its workers are set there too",
);
}
local_node_change(core, move |home| crate::nodes::patch(home, &id, &patch)).await
}
async fn delete_local_node(State(core): State<Arc<Core>>, Path(id): Path<String>) -> Response {
if id == crate::nodes::MAIN {
return error(
409,
"node_refused",
"the main node cannot be removed: pause Sharing instead",
);
}
local_node_change(core, move |home| crate::nodes::remove(home, &id)).await
}
async fn get_hosting(State(core): State<Arc<Core>>) -> Response {
tokio::task::spawn_blocking(move || {
let snap = hosting_snapshot(&core);
match load_hosting(&core, &snap) {
Ok(h) => Json(h).into_response(),
Err(r) => *r,
}
})
.await
.unwrap_or_else(|_| error(500, "internal", "hosting settings failed"))
}
async fn put_hosting(
State(core): State<Arc<Core>>,
body: Result<Json<crate::agent::hosting::Hosting>, JsonRejection>,
) -> Response {
let h = match body {
Ok(Json(h)) => h,
Err(e) => return bad_body(e),
};
if let Err(e) = crate::agent::hosting_api::validate(&h) {
return error(400, "invalid_input", &e);
}
tokio::task::spawn_blocking(move || {
let snap = hosting_snapshot(&core);
match crate::agent::hosting::save(&core.cfg.state_dir, &h, &snap.cat) {
Ok(()) => deployments_response(&core, &snap),
Err(e) => error(500, "hosting_settings", &e),
}
})
.await
.unwrap_or_else(|_| error(500, "internal", "hosting update failed"))
}
async fn get_policy(State(core): State<Arc<Core>>) -> Response {
tokio::task::spawn_blocking(move || {
match crate::agent::policy_api::view(
&core.cfg.state_dir,
&crate::sandbox::HostResources::detect(),
&|k| std::env::var(k).ok(),
) {
Ok(v) => Json(v).into_response(),
Err(e) => error(e.status, e.code, &e.message),
}
})
.await
.unwrap_or_else(|_| error(500, "internal", "policy view failed"))
}
async fn put_policy(
State(core): State<Arc<Core>>,
body: Result<Json<serde_json::Value>, JsonRejection>,
) -> Response {
let body = match body {
Ok(Json(b)) => b,
Err(e) => return bad_body(e),
};
tokio::task::spawn_blocking(move || {
let r = crate::agent::policy_api::put(
&core.cfg.state_dir,
&body,
&crate::sandbox::HostResources::detect(),
&|k| std::env::var(k).ok(),
);
match r {
Ok(v) => {
if v["changed"] == true {
core.publish_changes();
}
Json(v).into_response()
}
Err(e) => error(e.status, e.code, &e.message),
}
})
.await
.unwrap_or_else(|_| error(500, "internal", "policy update failed"))
}
#[derive(Deserialize)]
struct LedgerParams {
by: Option<String>,
from: Option<String>,
to: Option<String>,
}
async fn get_ledger(
State(core): State<Arc<Core>>,
params: Result<axum::extract::Query<LedgerParams>, axum::extract::rejection::QueryRejection>,
) -> Response {
let Ok(axum::extract::Query(p)) = params else {
return error(400, "invalid_input", "unreadable query string");
};
offer_response(
core.ledger(
p.by.as_deref().unwrap_or("day"),
p.from.as_deref(),
p.to.as_deref(),
)
.await,
)
}
pub const ISSUE_SANDBOX: &str = "zakuro-ai/zc#510";
pub const ISSUE_BUDGET: &str = "zakuro-ai/zc#513";
pub const ISSUE_SQUADS: &str = "zakuro-ai/zc#517";
pub const ISSUE_ENVS: &str = "zakuro-ai/zc#519";
async fn not_implemented(what: &'static str, until: &'static str) -> Response {
(
StatusCode::NOT_IMPLEMENTED,
Json(serde_json::json!({ "error": {
"code": "not_implemented",
"message": format!("{what} is not available in this zc yet"),
"until": until,
} })),
)
.into_response()
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct HandoffBody {
link: String,
}
async fn post_handoff(
State(core): State<Arc<Core>>,
body: Result<Json<HandoffBody>, JsonRejection>,
) -> Response {
let link = match body {
Ok(Json(b)) => b.link,
Err(e) => return bad_body(e),
};
match core.handoff(&link) {
Ok(p) => Json(p).into_response(),
Err(e) => error(e.status(), e.code(), &e.message()),
}
}
async fn post_handoff_confirm(
State(core): State<Arc<Core>>,
axum::extract::Path(jti): axum::extract::Path<String>,
) -> Response {
use crate::deeplink::ConfirmError;
match core.handoff_confirm(&jti) {
Ok(done) => Json(done).into_response(),
Err(ConfirmError::NotFound) => error(
404,
"not_found",
"no verified link with this id is waiting; open the link again",
),
Err(ConfirmError::Link(e)) => error(e.status(), e.code(), &e.message()),
Err(ConfirmError::NotAvailable { until, view }) => (
StatusCode::NOT_IMPLEMENTED,
Json(serde_json::json!({ "error": {
"code": "not_implemented",
"message": "this action is not available in this zc yet",
"until": until,
"view": view,
} })),
)
.into_response(),
}
}
async fn get_summary_v2(State(core): State<Arc<Core>>) -> Response {
Json(core.summary_v2()).into_response()
}
async fn get_offers(State(core): State<Arc<Core>>) -> Response {
offer_response(core.offers().await)
}
const ADMIN_ONLY: &str = "Log access is for admin accounts";
async fn get_logs(State(core): State<Arc<Core>>) -> Response {
if !core.is_admin().await {
return error(403, "admin_only", ADMIN_ONLY);
}
Json(serde_json::json!({ "files": crate::agent::logs::list(&core.cfg.dir) })).into_response()
}
#[derive(Deserialize)]
struct TailParams {
tail: Option<usize>,
}
async fn get_log(
State(core): State<Arc<Core>>,
Path(name): Path<String>,
params: Result<axum::extract::Query<TailParams>, axum::extract::rejection::QueryRejection>,
) -> Response {
if !core.is_admin().await {
return error(403, "admin_only", ADMIN_ONLY);
}
let Ok(axum::extract::Query(p)) = params else {
return error(400, "invalid_input", "tail must be a number");
};
let n = crate::agent::logs::clamp_tail(p.tail);
let dir = core.cfg.dir.clone();
let result = tokio::task::spawn_blocking(move || crate::agent::logs::tail(&dir, &name, n))
.await
.ok()
.flatten();
match result {
Some(t) => Json(t).into_response(),
None => error(404, "not_found", "no such log file"),
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct OfferBody {
subscribed: bool,
#[serde(default)]
price: Option<serde_json::Value>,
}
async fn put_offer(
State(core): State<Arc<Core>>,
Path(template_id): Path<String>,
body: Result<Json<OfferBody>, JsonRejection>,
) -> Response {
let b = match body {
Ok(Json(b)) => b,
Err(e) => return bad_body(e),
};
let mut fwd = serde_json::json!({ "subscribed": b.subscribed });
match b.price {
None | Some(serde_json::Value::Null) => {}
Some(p @ (serde_json::Value::String(_) | serde_json::Value::Number(_))) => {
fwd["price"] = p;
}
Some(_) => {
return error(
400,
"invalid_input",
r#"price must be "default" or a number of credits per hour"#,
)
}
}
offer_response(core.put_offer(&template_id, fwd).await)
}
fn offer_response(
result: Result<serde_json::Value, crate::agent::offers::OfferApiError>,
) -> Response {
match result {
Ok(v) => Json(v).into_response(),
Err(e) => error(e.status, &e.code, &e.message),
}
}
#[derive(Deserialize)]
struct JobsParams {
state: Option<String>,
source: Option<String>,
squad: Option<String>,
since: Option<String>,
}
async fn get_jobs(
State(core): State<Arc<Core>>,
params: Result<axum::extract::Query<JobsParams>, axum::extract::rejection::QueryRejection>,
) -> Response {
let Ok(axum::extract::Query(p)) = params else {
return error(400, "invalid_input", "unreadable query string");
};
match crate::agent::jobs::Query::parse(p.state, p.source, p.squad, p.since) {
Ok(q) => Json(serde_json::json!({ "jobs": core.jobs(&q) })).into_response(),
Err(m) => error(400, "invalid_input", &m),
}
}
async fn post_atelier_job(
State(core): State<Arc<Core>>,
body: Result<Json<crate::sandbox::job::JobSpec>, JsonRejection>,
) -> Response {
let Ok(Json(spec)) = body else {
return error(400, "invalid_input", "body is not a job spec");
};
let name = spec.name.clone();
if let Err(r) = core.admission.check(&atelier_request(&spec)) {
return error(403, "policy_refused", &r.to_string());
}
match core.atelier_jobs.submit(spec) {
Ok(()) => (
StatusCode::ACCEPTED,
Json(serde_json::json!({ "id": name })),
)
.into_response(),
Err(m) if m.contains("exists") => error(409, "conflict", &m),
Err(m) => error(400, "invalid_input", &m),
}
}
pub fn atelier_request(spec: &crate::sandbox::job::JobSpec) -> crate::policy::admit::Request {
use crate::policy::admit::{Request, Source, Workload};
let gpu = !spec.gpus.trim().is_empty();
let mut r = Request::new(Source::AtelierJob, Workload::Atelier { backend: None, gpu });
r.own = true;
r.gpus = u32::from(gpu);
r
}
async fn get_atelier_job(State(core): State<Arc<Core>>, Path(id): Path<String>) -> Response {
match core.atelier_jobs.status(&id) {
Some(st) => Json(st).into_response(),
None => error(404, "not_found", "no such atelier job"),
}
}
#[derive(Deserialize)]
struct LogsSince {
#[serde(default)]
since: usize,
}
async fn get_atelier_logs(
State(core): State<Arc<Core>>,
Path(id): Path<String>,
q: Result<axum::extract::Query<LogsSince>, axum::extract::rejection::QueryRejection>,
) -> Response {
let since = q.map(|axum::extract::Query(q)| q.since).unwrap_or(0);
match core.atelier_jobs.logs(&id, since) {
None => error(404, "not_found", "no such atelier job"),
Some(Ok((lines, next))) => {
Json(serde_json::json!({ "lines": lines, "next": next })).into_response()
}
Some(Err(m)) => error(502, "runtime_error", &m),
}
}
async fn delete_atelier_job(State(core): State<Arc<Core>>, Path(id): Path<String>) -> Response {
match core.atelier_jobs.stop(&id) {
Ok(()) => StatusCode::NO_CONTENT.into_response(),
Err(m) => error(502, "runtime_error", &m),
}
}
#[derive(Deserialize)]
struct Known {
known: Vec<String>,
}
async fn post_atelier_reconcile(
State(core): State<Arc<Core>>,
body: Result<Json<Known>, JsonRejection>,
) -> Response {
let Ok(Json(k)) = body else {
return error(400, "invalid_input", "body must be {\"known\": [...]}");
};
match core.atelier_jobs.reconcile(&k.known) {
Ok(stopped) => Json(serde_json::json!({ "stopped": stopped })).into_response(),
Err(m) => error(502, "runtime_error", &m),
}
}
pub mod sse {
use crate::agent::core::Core;
use crate::agent::events::{Event as BusEvent, Replay};
use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::sse::{Event, KeepAlive, Sse};
use axum::response::{IntoResponse, Response};
use futures_util::stream::{self, Stream, StreamExt};
use std::convert::Infallible;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::broadcast::error::RecvError;
pub const KEEPALIVE: Duration = Duration::from_secs(15);
pub fn to_sse(e: &BusEvent) -> Event {
Event::default()
.id(e.id.to_string())
.event(e.kind.clone())
.data(serde_json::to_string(e).unwrap_or_default())
}
fn resync(last_id: u64) -> Event {
Event::default()
.id(last_id.to_string())
.event("resync")
.data(serde_json::json!({ "type": "resync", "id": last_id }).to_string())
}
pub fn last_event_id(headers: &HeaderMap) -> Option<u64> {
headers
.get("last-event-id")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.trim().parse().ok())
}
pub async fn get_events(State(core): State<Arc<Core>>, headers: HeaderMap) -> Response {
let (replay, rx) = core.events.subscribe(last_event_id(&headers));
let first: Vec<Event> = match replay {
Replay::Events(ev) => ev.iter().map(to_sse).collect(),
Replay::Resync => vec![resync(core.events.last_id())],
};
let live = stream::unfold((rx, core), |(mut rx, core)| async move {
let ev = match rx.recv().await {
Ok(e) => to_sse(&e),
Err(RecvError::Lagged(_)) => resync(core.events.last_id()),
Err(RecvError::Closed) => return None,
};
Some((ev, (rx, core)))
});
let body: std::pin::Pin<Box<dyn Stream<Item = Result<Event, Infallible>> + Send>> =
Box::pin(stream::iter(first).chain(live).map(Ok));
Sse::new(body)
.keep_alive(KeepAlive::new().interval(KEEPALIVE))
.into_response()
}
}
fn error(status: u16, code: &str, message: &str) -> Response {
(
StatusCode::from_u16(status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
Json(serde_json::json!({ "error": { "code": code, "message": message } })),
)
.into_response()
}
pub use crate::broker::key_auth::ct_eq;
#[allow(clippy::result_large_err)] pub fn check_request(headers: &HeaderMap, token: &str, port: u16) -> Result<(), Response> {
if headers.contains_key(header::ORIGIN) {
return Err(error(
403,
"forbidden",
"requests from a browser are refused",
));
}
let host = headers
.get(header::HOST)
.and_then(|h| h.to_str().ok())
.unwrap_or("");
if host != format!("127.0.0.1:{port}") && host != format!("localhost:{port}") {
return Err(error(403, "forbidden", "unexpected Host header"));
}
let presented = headers
.get(header::AUTHORIZATION)
.and_then(|h| h.to_str().ok())
.and_then(|h| h.strip_prefix("Bearer "))
.unwrap_or("");
if token.is_empty() || !ct_eq(presented.as_bytes(), token.as_bytes()) {
return Err(error(401, "unauthorized", "missing or invalid agent token"));
}
Ok(())
}
async fn guard_layer(State(g): State<Guard>, req: Request, next: Next) -> Response {
match check_request(req.headers(), &g.token, g.port) {
Ok(()) => next.run(req).await,
Err(resp) => resp,
}
}
fn respond(result: Result<Summary, ApiError>) -> Response {
match result {
Ok(s) => Json(s).into_response(),
Err(e) => error(e.status, e.code, &e.message),
}
}
fn bad_body(e: JsonRejection) -> Response {
error(400, "invalid_input", &e.body_text())
}
async fn get_summary(State(core): State<Arc<Core>>) -> Response {
Json(core.summary()).into_response()
}
async fn get_provision(State(core): State<Arc<Core>>) -> Response {
Json(core.provision.state()).into_response()
}
async fn post_provision(State(core): State<Arc<Core>>) -> Response {
let events = core.clone();
let state = core.provision.start(
Arc::new(crate::sandbox::provision::Host),
Arc::new(move |s| {
events
.events
.publish("sandbox.state", serde_json::to_value(s).unwrap_or_default());
}),
);
(StatusCode::ACCEPTED, Json(state)).into_response()
}
async fn post_refresh(State(core): State<Arc<Core>>) -> Response {
core.refresh_hub().await;
core.wake_reconcile.notify_one();
Json(core.summary()).into_response()
}
#[derive(Deserialize)]
struct SharingBody {
on: bool,
}
async fn put_sharing(
State(core): State<Arc<Core>>,
body: Result<Json<SharingBody>, JsonRejection>,
) -> Response {
match body {
Ok(Json(b)) => respond(core.set_sharing(b.on)),
Err(e) => bad_body(e),
}
}
#[derive(Deserialize)]
struct WorkersBody {
count: u32,
}
async fn put_workers(
State(core): State<Arc<Core>>,
body: Result<Json<WorkersBody>, JsonRejection>,
) -> Response {
match body {
Ok(Json(b)) => respond(core.set_workers(b.count)),
Err(e) => bad_body(e),
}
}
fn deserialize_present<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
where
T: Deserialize<'de>,
D: Deserializer<'de>,
{
T::deserialize(deserializer).map(Some)
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct PriceBody {
scope: String,
#[serde(default, deserialize_with = "deserialize_present")]
price_per_hour: Option<Option<f64>>,
}
async fn put_price(
State(core): State<Arc<Core>>,
body: Result<Json<PriceBody>, JsonRejection>,
) -> Response {
match body {
Ok(Json(b)) => match b.price_per_hour {
Some(price) => respond(core.set_price(&b.scope, price).await),
None => error(400, "invalid_input", "price_per_hour is required"),
},
Err(e) => bad_body(e),
}
}
async fn not_found() -> Response {
error(404, "not_found", "no such route")
}
async fn method_not_allowed() -> Response {
error(
405,
"method_not_allowed",
"method not allowed on this route",
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::agent::core::tests::core_with;
use std::time::Duration;
fn headers(pairs: &[(&str, &str)]) -> axum::http::HeaderMap {
let mut h = axum::http::HeaderMap::new();
for (k, v) in pairs {
h.insert(
axum::http::HeaderName::from_bytes(k.as_bytes()).unwrap(),
v.parse().unwrap(),
);
}
h
}
#[test]
fn guards_refuse_browsers_foreign_hosts_and_bad_tokens() {
assert!(check_request(
&headers(&[("host", "127.0.0.1:4720"), ("authorization", "Bearer tok")]),
"tok",
4720
)
.is_ok());
assert!(check_request(
&headers(&[("host", "localhost:4720"), ("authorization", "Bearer tok")]),
"tok",
4720
)
.is_ok());
let status = |pairs: &[(&str, &str)]| {
check_request(&headers(pairs), "tok", 4720)
.unwrap_err()
.status()
.as_u16()
};
assert_eq!(status(&[("host", "127.0.0.1:4720")]), 401, "no token");
assert_eq!(
status(&[("host", "127.0.0.1:4720"), ("authorization", "Bearer nope")]),
401,
"wrong token"
);
assert_eq!(
status(&[("host", "127.0.0.1:4720"), ("authorization", "tok")]),
401,
"Bearer scheme required"
);
assert_eq!(
status(&[
("host", "evil.example:4720"),
("authorization", "Bearer tok")
]),
403,
"DNS rebinding"
);
assert_eq!(
status(&[("host", "127.0.0.1:9999"), ("authorization", "Bearer tok")]),
403
);
assert_eq!(
status(&[
("host", "127.0.0.1:4720"),
("authorization", "Bearer tok"),
("origin", "https://evil.example")
]),
403,
"CSRF"
);
assert!(ct_eq(b"abc", b"abc") && !ct_eq(b"abc", b"abd") && !ct_eq(b"abc", b"ab"));
assert_eq!(
check_request(
&headers(&[("host", "127.0.0.1:4720"), ("authorization", "Bearer ")]),
"",
4720
)
.unwrap_err()
.status()
.as_u16(),
401,
"an empty configured token never authenticates"
);
}
fn serve(core: std::sync::Arc<crate::agent::core::Core>, token: &str) -> u16 {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
listener.set_nonblocking(true).unwrap();
let app = router(core, token.to_string(), port);
std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
rt.block_on(async move {
let l = tokio::net::TcpListener::from_std(listener).unwrap();
axum::serve(l, app).await.unwrap();
});
});
port
}
fn call(
method: &str,
port: u16,
path: &str,
token: Option<&str>,
body: &str,
) -> (u16, serde_json::Value, bool) {
let agent = ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.http_status_as_error(false)
.timeout_global(Some(Duration::from_secs(5)))
.build(),
);
let url = format!("http://127.0.0.1:{port}{path}");
let auth = token.map(|t| format!("Bearer {t}")).unwrap_or_default();
let resp = match method {
"GET" => agent
.get(&url)
.header("Authorization", auth.as_str())
.call(),
"DELETE" => agent
.delete(&url)
.header("Authorization", auth.as_str())
.call(),
"PUT" => agent
.put(&url)
.header("Authorization", auth.as_str())
.header("Content-Type", "application/json")
.send(body),
_ => agent
.post(&url)
.header("Authorization", auth.as_str())
.header("Content-Type", "application/json")
.send(body),
}
.unwrap();
let status = resp.status().as_u16();
let cors = resp
.headers()
.keys()
.any(|k| k.as_str().starts_with("access-control"));
let json = resp
.into_body()
.read_json::<serde_json::Value>()
.unwrap_or(serde_json::Value::Null);
(status, json, cors)
}
#[test]
fn job_api_rejects_without_token_and_bad_specs() {
let core = core_with(3);
let mut open = crate::policy::Policy::default();
open.workloads.atelier = crate::policy::Atelier::On;
core.admission.pin_policy(open);
let port = serve(core, "tok");
let body = r#"{"name":"job-1","image":"img","workdir":"/tmp/w","cmd":["true"]}"#;
let (s, v, _) = call("POST", port, "/v2/atelier/jobs", None, body);
assert_eq!(
(s, v["error"]["code"].as_str()),
(401, Some("unauthorized"))
);
let (s, _, _) = call("GET", port, "/v2/atelier/jobs/job-1", None, "");
assert_eq!(s, 401);
let bad = r#"{"name":"../x","image":"img","workdir":"/tmp/w","cmd":["true"]}"#;
let (s, v, _) = call("POST", port, "/v2/atelier/jobs", Some("tok"), bad);
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input"))
);
let layer = r#"{"name":"job-2","image":"img","workdir":"/tmp/w","cmd":["true"],
"layer":{"pip":["git+https://x/y"],"venv":"/opt/v"}}"#;
let (s, v, _) = call("POST", port, "/v2/atelier/jobs", Some("tok"), layer);
assert_eq!(s, 400, "{v}");
let (s, _, _) = call("GET", port, "/v2/atelier/jobs/nope", Some("tok"), "");
assert_eq!(s, 404);
}
#[test]
fn the_local_api_round_trip() {
let core = core_with(3);
let port = serve(core, "tok");
let (s, v, cors) = call("GET", port, "/v1/summary", Some("tok"), "");
assert_eq!(
(s, v["v"].as_u64(), v["this_mac"]["workers"]["max"].as_u64()),
(200, Some(1), Some(3))
);
assert!(!cors, "no CORS headers, ever");
let (s, v, _) = call("GET", port, "/v1/summary", None, "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(401, Some("unauthorized"))
);
let (s, v, _) = call("PUT", port, "/v1/workers", Some("tok"), r#"{"count":2}"#);
assert_eq!(
(s, v["this_mac"]["workers"]["desired"].as_u64()),
(200, Some(2))
);
let (s, v, _) = call("PUT", port, "/v1/workers", Some("tok"), r#"{"count":9}"#);
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input"))
);
let (s, v, _) = call(
"PUT",
port,
"/v1/workers",
Some("tok"),
r#"{"count":"two"}"#,
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input"))
);
let (s, v, _) = call("PUT", port, "/v1/sharing", Some("tok"), r#"{"on":true}"#);
assert_eq!((s, v["this_mac"]["sharing"].as_bool()), (200, Some(true)));
let (s, v, _) = call(
"PUT",
port,
"/v1/price",
Some("tok"),
r#"{"scope":"device","price_per_hour":10}"#,
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(503, Some("prerequisite_missing"))
);
let (s, v, _) = call("POST", port, "/v1/refresh", Some("tok"), "");
assert_eq!((s, v["v"].as_u64()), (200, Some(1)));
let (s, _, _) = call("GET", port, "/v1/nope", Some("tok"), "");
assert_eq!(s, 404);
let (s, v, _) = call("GET", port, "/v1/nope", None, "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(401, Some("unauthorized")),
"the guard runs before the 404 too"
);
let (s, v, _) = call(
"PUT",
port,
"/v1/price",
Some("tok"),
r#"{"scope":"device","price":18}"#,
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input")),
"an unknown field is rejected"
);
let (s, v, _) = call(
"PUT",
port,
"/v1/price",
Some("tok"),
r#"{"scope":"device"}"#,
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input")),
"price_per_hour is required"
);
let (s, v, _) = call(
"PUT",
port,
"/v1/price",
Some("tok"),
r#"{"scope":"device","price_per_hour":null}"#,
);
assert_ne!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input")),
"an explicit null still clears, it's not treated as missing"
);
}
#[test]
fn method_not_allowed_is_json_and_the_guard_runs_first() {
let core = core_with(3);
let port = serve(core, "tok");
let (s, v, _) = call("POST", port, "/v1/summary", Some("tok"), "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(405, Some("method_not_allowed"))
);
let (s, v, _) = call("POST", port, "/v1/summary", None, "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(401, Some("unauthorized")),
"the guard runs before the 405"
);
}
fn mac_core() -> std::sync::Arc<crate::agent::core::Core> {
let core = core_with(0);
core.hosting_status.lock().unwrap().root = Some(core.cfg.state_dir.join("runner-root"));
core.hosting_status.lock().unwrap().catalogue = Some(crate::agent::runners::catalogue(
&crate::agent::runners::Hardware {
apple_silicon: true,
nvidia: vec![],
host: crate::sandbox::HostResources {
logical_cpus: 10,
memory_bytes: Some(16 << 30),
},
},
));
core
}
#[test]
fn deployments_requires_token() {
let port = serve(mac_core(), "tok");
for (m, p) in [
("GET", "/v2/deployments"),
("PUT", "/v2/deployments/spaces"),
("GET", "/v2/images"),
("GET", "/v2/resources"),
("GET", "/v2/local-nodes"),
("POST", "/v2/local-nodes"),
("PUT", "/v2/local-nodes/x"),
("DELETE", "/v2/local-nodes/x"),
("DELETE", "/v2/images/env%2Fpeft"),
("PUT", "/v2/hosting"),
] {
let (s, v, _) = call(m, port, p, None, "{}");
assert_eq!(
(s, v["error"]["code"].as_str()),
(401, Some("unauthorized")),
"{m} {p}"
);
}
}
#[test]
fn deployments_view_and_toggle() {
let port = serve(mac_core(), "tok");
let (s, v, _) = call("GET", port, "/v2/deployments", Some("tok"), "");
assert_eq!(s, 200, "{v}");
assert_eq!(v["unified_memory"], true);
assert!(v["deployments"]
.as_array()
.unwrap()
.iter()
.any(|d| d["kind"] == "atelier-peft"));
let (s, v, _) = call(
"PUT",
port,
"/v2/deployments/atelier-ludwig",
Some("tok"),
r#"{"enabled":true}"#,
);
assert_eq!((s, v["error"]["code"].as_str()), (409, Some("hardware")));
let (s, _, _) = call(
"PUT",
port,
"/v2/deployments/spaces",
Some("tok"),
r#"{"enabled":false}"#,
);
assert_eq!(s, 200);
let (_, v, _) = call("GET", port, "/v2/deployments", Some("tok"), "");
let spaces = v["deployments"]
.as_array()
.unwrap()
.iter()
.find(|d| d["kind"] == "spaces")
.unwrap()
.clone();
assert_eq!(spaces["state"], "disabled");
}
#[test]
fn hosting_put_validates_and_saves() {
let port = serve(mac_core(), "tok");
let mut h = crate::agent::hosting::defaults(&[]);
h.cleanup.idle_ttl = "soon".into();
let (s, v, _) = call(
"PUT",
port,
"/v2/hosting",
Some("tok"),
&serde_json::to_string(&h).unwrap(),
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input"))
);
h.cleanup.idle_ttl = "3d".into();
let (s, v, _) = call(
"PUT",
port,
"/v2/hosting",
Some("tok"),
&serde_json::to_string(&h).unwrap(),
);
assert_eq!(s, 200, "{v}");
let (_, v, _) = call("GET", port, "/v2/images", Some("tok"), "");
assert_eq!(v["ttl_days"], 3);
}
#[test]
fn delete_image_with_encoded_slash_removes_it() {
let core = mac_core();
let root = core.hosting_status.lock().unwrap().root();
std::fs::create_dir_all(root.join("envs/old/bin")).unwrap();
std::fs::write(root.join("envs/old/bin/python"), b"12345").unwrap();
let port = serve(core, "tok");
let (s, v, _) = call("GET", port, "/v2/images", Some("tok"), "");
assert_eq!(s, 200);
assert!(
v["images"]
.as_array()
.unwrap()
.iter()
.any(|i| i["id"] == "env/old"),
"{v}"
);
let (s, v, _) = call("DELETE", port, "/v2/images/env%2Fold", Some("tok"), "");
assert_eq!((s, v["freed_bytes"].as_u64()), (200, Some(5)), "{v}");
assert!(!root.join("envs/old").exists());
let (s, _, _) = call(
"PUT",
port,
"/v2/policy",
Some("tok"),
r#"{"budget":{"gpu":true}}"#,
);
assert_eq!(s, 200);
std::fs::create_dir_all(root.join("envs/peft")).unwrap();
let (s, v, _) = call("DELETE", port, "/v2/images/env%2Fpeft", Some("tok"), "");
assert_eq!((s, v["error"]["code"].as_str()), (409, Some("in_use")));
}
#[test]
fn delete_unknown_image_is_404() {
let port = serve(mac_core(), "tok");
let (s, v, _) = call("DELETE", port, "/v2/images/env%2Fnope", Some("tok"), "");
assert_eq!((s, v["error"]["code"].as_str()), (404, Some("not_found")));
}
#[test]
fn v2_is_additive_and_guarded_like_v1() {
let core = core_with(3);
let port = serve(core, "tok");
let (s, v, cors) = call("GET", port, "/v2/summary", Some("tok"), "");
assert_eq!((s, v["summary_version"].as_u64()), (200, Some(2)));
assert!(!cors);
assert!(v["nodes"][0]["this_node"].as_bool().unwrap());
assert_eq!(v["jobs"]["live"], serde_json::json!([]));
for k in ["squads", "sandbox", "policy"] {
assert!(v[k].is_null(), "{k}");
}
let (s, v, _) = call("GET", port, "/v1/summary", Some("tok"), "");
assert_eq!((s, v["v"].as_u64()), (200, Some(1)), "v1 unchanged");
let (s, v, _) = call("GET", port, "/v2/summary", None, "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(401, Some("unauthorized"))
);
let (s, _, _) = call("GET", port, "/v2/squads", None, "");
assert_eq!(s, 401, "the guard runs before a 501 too");
let (s, v, _) = call("GET", port, "/v2/jobs", Some("tok"), "");
assert_eq!((s, v["jobs"].clone()), (200, serde_json::json!([])));
let (s, v, _) = call("GET", port, "/v2/jobs?state=nope", Some("tok"), "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input"))
);
let (s, _, _) = call(
"GET",
port,
"/v2/jobs?state=done&since=2026-01-01T00:00:00Z",
Some("tok"),
"",
);
assert_eq!(s, 200);
for (method, path, until) in [
("GET", "/v2/sandbox", ISSUE_SANDBOX),
("POST", "/v2/sandbox/resize", ISSUE_BUDGET),
("GET", "/v2/squads", ISSUE_SQUADS),
("POST", "/v2/squads/join", ISSUE_SQUADS),
("POST", "/v2/squads/s1/leave", ISSUE_SQUADS),
("GET", "/v2/envs", ISSUE_ENVS),
("POST", "/v2/envs", ISSUE_ENVS),
("POST", "/v2/envs/e1/connect", ISSUE_ENVS),
("DELETE", "/v2/envs/e1", ISSUE_ENVS),
] {
let (s, v, _) = call(method, port, path, Some("tok"), "{}");
assert_eq!(
(s, v["error"]["code"].as_str(), v["error"]["until"].as_str()),
(501, Some("not_implemented"), Some(until)),
"{method} {path}"
);
assert!(v["error"]["message"].is_string());
}
let (s, v, _) = call("POST", port, "/v2/summary", Some("tok"), "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(405, Some("method_not_allowed"))
);
let (s, _, _) = call("GET", port, "/v2/nope", Some("tok"), "");
assert_eq!(s, 404);
}
#[test]
fn deep_links_verify_then_run_only_on_confirm() {
use crate::deeplink::tests::{claims, Signer, NODE_FP};
let core = core_with(3);
let port = serve(core.clone(), "tok");
let post = |path: &str, body: &str| call("POST", port, path, Some("tok"), body);
let (s, v, _) = post(
"/v2/handoff",
r#"{"link":"zakuro://open?view=contribution"}"#,
);
assert_eq!(
(s, v["view"].as_str(), v["jti"].is_null()),
(200, Some("contribution"), true)
);
let (s, v, _) = post("/v2/handoff", r#"{"link":"https://example.com"}"#);
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_link"))
);
let (s, _, _) = post("/v2/handoff", r#"{"url":"zakuro://open?view=jobs"}"#);
assert_eq!(s, 400);
let hub = Signer::new(11);
let node_fp =
crate::broker::node_identity::fingerprint_of_pubkey_b64(core.node_pubkey()).unwrap();
let now = chrono::Utc::now().timestamp();
let mint = |act: &str, jti: &str, args: serde_json::Value| {
let mut c = claims(act, jti);
c["node"] = serde_json::json!(node_fp);
c["iat"] = serde_json::json!(now);
c["exp"] = serde_json::json!(now + 600);
c["args"] = args;
hub.mint(c)
};
let budget = mint(
"share/budget",
"b1",
serde_json::json!({"node_pubkey": core.node_pubkey(), "proposed": {"price_per_hour": 120}}),
);
let body = format!(r#"{{"link":"zakuro://share/budget?t={budget}"}}"#);
if std::env::var(crate::deeplink::ENV_ACTION_PUBKEYS).is_err() {
let (s, v, _) = post("/v2/handoff", &body);
assert_eq!(
(s, v["error"]["code"].as_str()),
(503, Some("no_action_key"))
);
}
core.set_handoff_test(vec![hub.key()], Some("someone-else"));
let (s, v, _) = post("/v2/handoff", &body);
assert_eq!(
(s, v["error"]["code"].as_str()),
(409, Some("wrong_account"))
);
core.set_handoff_test(vec![hub.key()], Some("1234567890"));
let (s, v, _) = post("/v2/handoff", &body);
assert_eq!(
(s, v["jti"].as_str(), v["available"].as_bool()),
(200, Some("b1"), Some(true))
);
assert_eq!(v["title"].as_str(), Some("Review a budget change"));
let (s, v, _) = post("/v2/handoff/b1/confirm", "");
assert_eq!(
(s, v["status"].as_str(), v["view"].as_str()),
(200, Some("done"), Some("budget"))
);
assert_eq!(v["prefill"]["price_per_hour"].as_u64(), Some(120));
let (s, v, _) = post("/v2/handoff/b1/confirm", "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(410, Some("already_used"))
);
let (s, v, _) = post("/v2/handoff", &body);
assert_eq!(
(s, v["error"]["code"].as_str()),
(410, Some("already_used"))
);
let mut c = claims("squad/join", "s0");
c["node"] = serde_json::json!(NODE_FP);
c["iat"] = serde_json::json!(now);
c["exp"] = serde_json::json!(now + 600);
let other = hub.mint(c);
let (s, v, _) = post(
"/v2/handoff",
&format!(r#"{{"link":"zakuro://squad/join?t={other}"}}"#),
);
assert_eq!((s, v["error"]["code"].as_str()), (403, Some("wrong_node")));
let squad = mint(
"squad/join",
"s1",
serde_json::json!({"name": "Kyoto Vision", "role": "member"}),
);
let (s, v, _) = post(
"/v2/handoff",
&format!(r#"{{"link":"zakuro://squad/join?t={squad}"}}"#),
);
assert_eq!(
(s, v["available"].as_bool(), v["view"].as_str()),
(200, Some(false), Some("squads"))
);
let (s, v, _) = post("/v2/handoff/s1/confirm", "");
assert_eq!(
(
s,
v["error"]["code"].as_str(),
v["error"]["until"].as_str(),
v["error"]["view"].as_str()
),
(
501,
Some("not_implemented"),
Some(ISSUE_SQUADS),
Some("squads")
)
);
let (s, v, _) = post("/v2/handoff/nope/confirm", "");
assert_eq!((s, v["error"]["code"].as_str()), (404, Some("not_found")));
let (s, _, _) = call("POST", port, "/v2/handoff", None, &body);
assert_eq!(s, 401);
}
fn offers_core(
gpus: Option<Vec<crate::bench_node::fingerprint::GpuId>>,
) -> std::sync::Arc<crate::agent::core::Core> {
let mut core = core_with(3);
crate::agent::core::tests::set_gpu_probe(&mut core, Box::new(move || gpus.clone()));
core
}
fn nvidia(gib: u64) -> crate::bench_node::fingerprint::GpuId {
crate::bench_node::fingerprint::GpuId {
vendor: "nvidia".into(),
model: "m".into(),
uuid: "u".into(),
vram_bytes: gib << 30,
}
}
const OFFERS: &str = r#"{"node":"N","categories":["Machine learning & AI","Developer tools"],"offers":[
{"template":{"id":"pytorch","gpu_required":true,"min_vram_gb":8},"hosts":3,"subscribed":false},
{"template":{"id":"python","gpu_required":false,"min_vram_gb":null},"hosts":1,"subscribed":true}]}"#;
fn admin_core(is_admin: Option<bool>) -> std::sync::Arc<crate::agent::core::Core> {
let core = core_with(3);
if let Some(a) = is_admin {
let body = crate::agent::core::tests::hub_json(&core.cfg.node_pubkey(), "").replace(
r#""balance":1240.5}"#,
&format!(r#""balance":1240.5,"is_admin":{a}}}"#),
);
let (hub, _) = crate::agent::hub::tests::mock_hub(move |_, _, _| (200, body.clone()));
crate::agent::core::tests::sign_in(&core, &hub);
}
let d = &core.cfg.dir;
std::fs::create_dir_all(d.join("workers")).unwrap();
std::fs::write(
d.join("agent.log"),
"one\n\u{1b}[31mtwo zk_9_secret\u{1b}[0m\nthree Bearer abc.def\n",
)
.unwrap();
std::fs::write(d.join("broker.log"), "b\n").unwrap();
std::fs::write(d.join("workers/w2.log"), "w2 line\n").unwrap();
std::fs::write(d.join("workers/w10.log"), "w10 line\n").unwrap();
std::fs::write(d.parent().unwrap().join("credentials-x.log"), "no\n").unwrap();
core
}
#[test]
fn logs_need_the_guard_and_an_admin_account() {
let port = serve(admin_core(Some(true)), "tok");
assert_eq!(call("GET", port, "/v2/logs", None, "").0, 401);
for (core, what) in [
(admin_core(Some(false)), "non-admin"),
(admin_core(None), "no account"),
] {
let port = serve(core, "tok");
for path in ["/v2/logs", "/v2/logs/agent.log"] {
let (s, v, _) = call("GET", port, path, Some("tok"), "");
assert_eq!(s, 403, "{what} {path}");
assert_eq!(v["error"]["code"], "admin_only");
assert_eq!(v["error"]["message"], "Log access is for admin accounts");
}
}
}
#[test]
fn admin_lists_logs_in_order() {
let port = serve(admin_core(Some(true)), "tok");
let (s, v, _) = call("GET", port, "/v2/logs", Some("tok"), "");
assert_eq!(s, 200);
let names: Vec<_> = v["files"]
.as_array()
.unwrap()
.iter()
.map(|f| f["name"].as_str().unwrap().to_string())
.collect();
assert_eq!(
names,
[
"agent.log",
"broker.log",
"workers/w2.log",
"workers/w10.log"
]
);
assert!(v["files"][0]["size"].as_u64().unwrap() > 0);
assert!(v["files"][0]["modified"].as_str().unwrap().ends_with('Z'));
}
#[test]
fn admin_tails_with_redaction_ansi_strip_and_clamp() {
let port = serve(admin_core(Some(true)), "tok");
let (s, v, _) = call("GET", port, "/v2/logs/agent.log?tail=2", Some("tok"), "");
assert_eq!(s, 200);
assert_eq!(v["name"], "agent.log");
assert_eq!(v["lines"][0], "two …redacted");
assert_eq!(v["lines"][1], "three …redacted");
assert_eq!(v["truncated"], true);
let (_, v, _) = call("GET", port, "/v2/logs/agent.log", Some("tok"), "");
assert_eq!(v["lines"].as_array().unwrap().len(), 3);
assert_eq!(v["truncated"], false);
let (_, v, _) = call("GET", port, "/v2/logs/agent.log?tail=0", Some("tok"), "");
assert_eq!(v["lines"].as_array().unwrap().len(), 1, "clamped up to 1");
let (_, v, _) = call("GET", port, "/v2/logs/workers%2Fw2.log", Some("tok"), "");
assert_eq!(v["name"], "workers/w2.log");
assert_eq!(v["lines"][0], "w2 line");
let (s, _, _) = call("GET", port, "/v2/logs/agent.log?tail=x", Some("tok"), "");
assert_eq!(s, 400);
}
#[test]
fn log_traversal_names_are_not_found() {
let port = serve(admin_core(Some(true)), "tok");
for n in [
"..%2Fcredentials-x.log",
"..%2Fcredentials",
"%2Fetc%2Fpasswd",
"workers%2F..%2F..%2Fx",
"%2e%2e",
"..",
"nope.log",
"workers",
] {
let (s, v, _) = call("GET", port, &format!("/v2/logs/{n}"), Some("tok"), "");
assert_eq!(s, 404, "{n}");
assert_eq!(v["error"]["code"], "not_found", "{n}");
}
}
#[test]
fn offers_are_guarded_and_need_a_login() {
let port = serve(offers_core(None), "tok");
let (s, _, _) = call("GET", port, "/v2/offers", None, "");
assert_eq!(s, 401, "the guard runs first");
let (s, v, _) = call("GET", port, "/v2/offers", Some("tok"), "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(409, Some("not_logged_in"))
);
let (s, v, _) = call(
"PUT",
port,
"/v2/offers/py",
Some("tok"),
r#"{"subscribed":true}"#,
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(409, Some("not_logged_in"))
);
}
#[test]
fn get_offers_proxies_the_hub_and_adds_fits() {
let core = offers_core(Some(vec![nvidia(4)]));
let pk = core.cfg.node_pubkey();
let (hub, seen) = crate::agent::hub::tests::mock_hub(|_, _, _| (200, OFFERS.to_string()));
crate::agent::core::tests::sign_in(&core, &hub);
let port = serve(core, "tok");
let (s, v, _) = call("GET", port, "/v2/offers", Some("tok"), "");
assert_eq!(s, 200);
assert_eq!(v["node"], "N", "hub body verbatim");
assert_eq!(v["offers"][0]["hosts"], 3);
assert_eq!(v["offers"][0]["fits"], false);
assert_eq!(v["offers"][0]["fit_reason"], "needs 8 GB VRAM");
assert_eq!(v["offers"][1]["fits"], true);
assert!(v["offers"][1]["fit_reason"].is_null());
let (m, url, _, auth) = seen.lock().unwrap()[0].clone();
assert_eq!(m, "GET");
assert_eq!(auth.as_deref(), Some("Bearer zk_1_test"));
let parsed = reqwest::Url::parse(&format!("http://hub{url}")).unwrap();
assert_eq!(parsed.path(), "/api/provide/offers");
let q = parsed
.query_pairs()
.find(|(k, _)| k == "node")
.unwrap()
.1
.into_owned();
assert_eq!(q, pk, "this node's pubkey");
}
#[test]
fn fits_is_null_when_the_hardware_is_unknown() {
let core = offers_core(None);
let (hub, _) = crate::agent::hub::tests::mock_hub(|_, _, _| (200, OFFERS.to_string()));
crate::agent::core::tests::sign_in(&core, &hub);
let port = serve(core, "tok");
let (_, v, _) = call("GET", port, "/v2/offers", Some("tok"), "");
for i in 0..2 {
assert!(v["offers"][i]["fits"].is_null(), "{i}");
assert!(v["offers"][i]["fit_reason"].is_null(), "{i}");
}
}
#[test]
fn put_offer_adds_the_node_and_passes_hub_errors_through() {
let core = offers_core(Some(vec![nvidia(24)]));
let pk = core.cfg.node_pubkey();
let (hub, seen) = crate::agent::hub::tests::mock_hub(|m, url, _| {
match (m, url) {
("PUT", "/api/provide/offers/pytorch") => (
200,
r#"{"template":{"id":"pytorch","gpu_required":true,"min_vram_gb":8},"subscribed":true,"price_mode":"fixed","price_per_hour":4.5}"#.into(),
),
("PUT", "/api/provide/offers/bad") => (422, r#"{"detail":"price must be > 0"}"#.into()),
("PUT", "/api/provide/offers/coded") => {
(403, r#"{"detail":{"code":"not_your_node","message":"not your node"}}"#.into())
}
("PUT", "/api/provide/offers/nokey") => (401, "{}".into()),
_ => (404, r#"{"detail":"unknown template"}"#.into()),
}
});
crate::agent::core::tests::sign_in(&core, &hub);
let port = serve(core, "tok");
let (s, v, _) = call(
"PUT",
port,
"/v2/offers/pytorch",
Some("tok"),
r#"{"subscribed":true,"price":4.5}"#,
);
assert_eq!(s, 200);
assert_eq!(
(v["price_mode"].as_str(), v["fits"].as_bool()),
(Some("fixed"), Some(true))
);
let sent: serde_json::Value = serde_json::from_str(&seen.lock().unwrap()[0].2).unwrap();
assert_eq!(
sent,
serde_json::json!({"node": pk, "subscribed": true, "price": 4.5})
);
let (s, _, _) = call(
"PUT",
port,
"/v2/offers/pytorch",
Some("tok"),
r#"{"subscribed":true,"price":"default"}"#,
);
assert_eq!(s, 200);
assert_eq!(
serde_json::from_str::<serde_json::Value>(&seen.lock().unwrap()[1].2).unwrap()["price"],
"default"
);
let (s, _, _) = call(
"PUT",
port,
"/v2/offers/pytorch",
Some("tok"),
r#"{"subscribed":false}"#,
);
assert_eq!(s, 200);
assert!(
serde_json::from_str::<serde_json::Value>(&seen.lock().unwrap()[2].2)
.unwrap()
.get("price")
.is_none()
);
for (path, status, code, msg) in [
("bad", 422, "hub_error", "price must be > 0"),
("coded", 403, "not_your_node", "not your node"),
("nokey", 409, "not_logged_in", ""),
("unknown", 404, "hub_error", "unknown template"),
] {
let (s, v, _) = call(
"PUT",
port,
&format!("/v2/offers/{path}"),
Some("tok"),
r#"{"subscribed":true}"#,
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(status, Some(code)),
"{path}"
);
assert!(
v["error"]["message"].as_str().unwrap().contains(msg),
"{path}"
);
}
for body in [
r#"{"price":1}"#,
r#"{"subscribed":"yes"}"#,
r#"{"subscribed":true,"price":[1]}"#,
r#"{"subscribed":true,"x":1}"#,
] {
let (s, v, _) = call("PUT", port, "/v2/offers/pytorch", Some("tok"), body);
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input")),
"{body}"
);
}
let (s, v, _) = call("POST", port, "/v2/offers", Some("tok"), "{}");
assert_eq!(
(s, v["error"]["code"].as_str()),
(405, Some("method_not_allowed"))
);
}
#[test]
fn an_unreachable_hub_is_502_hub_unreachable() {
let core = offers_core(None);
crate::agent::core::tests::sign_in(&core, "http://127.0.0.1:1");
let port = serve(core, "tok");
let (s, v, _) = call("GET", port, "/v2/offers", Some("tok"), "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(502, Some("hub_unreachable"))
);
let (s, v, _) = call(
"PUT",
port,
"/v2/offers/x",
Some("tok"),
r#"{"subscribed":false}"#,
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(502, Some("hub_unreachable"))
);
}
fn sse(
port: u16,
last_id: Option<u64>,
n: usize,
) -> std::sync::mpsc::Receiver<crate::agent::client::SseMessage> {
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let dir = crate::agent::files::tests::tmp("sse-client");
crate::agent::files::ensure_dir(&dir).unwrap();
crate::agent::files::save_json(
&dir.join(crate::agent::files::AGENT_FILE),
&crate::agent::files::AgentFile {
v: 1,
port,
token: "tok".into(),
pid: 1,
version: "x".into(),
rev: 1,
},
)
.unwrap();
let c = crate::agent::client::AgentClient::from_dir(&dir).unwrap();
let mut seen = 0;
let _ = c.events(last_id, |m| {
seen += 1;
let _ = tx.send(m);
seen < n
});
});
rx
}
#[test]
fn events_stream_live_and_replay_on_reconnect() {
let core = core_with(3);
let port = serve(core.clone(), "tok");
let first = core
.events
.publish("job.started", serde_json::json!({"id": "j1"}));
let rx = sse(port, None, 1);
std::thread::sleep(Duration::from_millis(300));
core.events
.publish("job.finished", serde_json::json!({"id": "j1"}));
let m = rx.recv_timeout(Duration::from_secs(5)).unwrap();
assert_eq!(m.event, "job.finished");
let body: serde_json::Value = serde_json::from_str(&m.data).unwrap();
assert_eq!(
(body["type"].as_str(), body["data"]["id"].as_str()),
(Some("job.finished"), Some("j1"))
);
assert_eq!(m.id, Some(body["id"].to_string()));
let rx = sse(port, Some(first), 1);
let m = rx.recv_timeout(Duration::from_secs(5)).unwrap();
assert_eq!(
(m.event.as_str(), m.id),
("job.finished", Some((first + 1).to_string()))
);
let rx = sse(port, Some(9_999), 1);
let m = rx.recv_timeout(Duration::from_secs(5)).unwrap();
assert_eq!(m.event, "resync");
let (s, _, _) = call("GET", port, "/v2/events", None, "");
assert_eq!(s, 401);
}
#[test]
fn a_write_publishes_summary_changed() {
let core = core_with(3);
let (_, mut rx) = core.events.subscribe(None);
core.set_workers(2).unwrap();
let mut kinds = vec![];
while let Ok(e) = rx.try_recv() {
kinds.push(e.kind.clone());
if e.kind == "summary.changed" {
assert_eq!(e.data["summary_version"], 2);
assert_eq!(e.data["nodes"][0]["local"]["workers"]["desired"], 2);
}
}
assert!(kinds.contains(&"summary.changed".to_string()), "{kinds:?}");
assert!(
kinds.contains(&"problem.raised".to_string()),
"not signed in: {kinds:?}"
);
core.publish_changes();
assert!(rx.try_recv().is_err());
}
#[test]
fn an_atelier_job_the_node_policy_refuses_is_a_403_with_the_reason() {
let core = core_with(3);
crate::policy::update(&core.cfg.state_dir, None, None, |p| {
p.workloads.atelier = crate::policy::Atelier::Off;
Ok(())
})
.unwrap();
let port = serve(core.clone(), "tok");
let body = r#"{"name":"job-1","image":"img","workdir":"/tmp/w","cmd":["true"]}"#;
let (s, v, _) = call("POST", port, "/v2/atelier/jobs", Some("tok"), body);
assert_eq!(
(s, v["error"]["code"].as_str()),
(403, Some("policy_refused"))
);
assert!(
v["error"]["message"]
.as_str()
.unwrap()
.contains("(workload)"),
"{v}"
);
assert_eq!(core.admission.snapshot()["refused"]["workload"], 1);
let mut p = crate::policy::Policy::default();
p.workloads.atelier = crate::policy::Atelier::Off;
core.admission.pin_policy(p);
let (s, _, _) = call("POST", port, "/v2/atelier/jobs", Some("tok"), body);
assert_eq!(s, 403);
}
#[test]
fn policy_is_read_and_changed_through_the_agent_with_a_revision_check() {
let core = core_with(3);
let port = serve(core.clone(), "tok");
let (s, _, _) = call("GET", port, "/v2/policy", None, "");
assert_eq!(s, 401, "behind the same guard");
let (s, v0, _) = call("GET", port, "/v2/policy", Some("tok"), "");
assert_eq!(s, 200, "{v0}");
assert_eq!(
v0["policy"]["sharing"]["squads"],
serde_json::json!(["public"])
);
let rev = v0["revision"].as_u64().unwrap();
core.publish_changes();
let before = core.events.last_id();
let (s, v, _) = call(
"PUT",
port,
"/v2/policy",
Some("tok"),
&format!(r#"{{"revision":{rev},"sharing":{{"squads":[]}}}}"#),
);
assert_eq!(s, 200, "{v}");
assert_eq!(v["changed"], true);
assert_eq!(v["revision"].as_u64(), Some(rev + 1));
assert_eq!(v["policy"]["sharing"]["squads"], serde_json::json!([]));
let (replay, _rx) = core.events.subscribe(Some(before));
let crate::agent::events::Replay::Events(evs) = replay else {
panic!("resync")
};
assert!(
evs.iter()
.any(|e| e.kind == "policy.changed" && e.data["revision"] == rev + 1),
"{evs:?}"
);
let (_, sum, _) = call("GET", port, "/v2/summary", Some("tok"), "");
assert_eq!(sum["policy"]["revision"].as_u64(), Some(rev + 1));
let (s, v, _) = call(
"PUT",
port,
"/v2/policy",
Some("tok"),
&format!(r#"{{"revision":{rev},"sharing":{{"when":"always"}}}}"#),
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(409, Some("stale_revision"))
);
let (s, v, _) = call(
"PUT",
port,
"/v2/policy",
Some("tok"),
r#"{"sharing":{"squad":[]}}"#,
);
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input"))
);
let (_, v, _) = call("GET", port, "/v2/policy", Some("tok"), "");
assert_eq!(v["policy"]["sharing"]["when"], "idle");
}
#[test]
fn the_ledger_route_answers_from_the_hub_and_says_when_it_cannot() {
let core = core_with(3);
let port = serve(core.clone(), "tok");
let (s, v, _) = call("GET", port, "/v2/ledger", Some("tok"), "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(409, Some("not_logged_in"))
);
let (s, v, _) = call("GET", port, "/v2/ledger?by=week", Some("tok"), "");
assert_eq!(
(s, v["error"]["code"].as_str()),
(400, Some("invalid_input"))
);
let (s, _, _) = call("GET", port, "/v2/ledger?from=soon", Some("tok"), "");
assert_eq!(s, 400);
let (hub, seen) = crate::agent::hub::tests::mock_hub(|_, url, _| {
if url.starts_with("/api/provide/ledger") {
(
200,
r#"{"rows":[],"by_day":[{"day":"2026-11-09","earned":5.0,"spent":1.0}],
"by_node":[],"by_product":[],
"totals":{"earned":5.0,"spent":1.0,"fake_earned":0.0}}"#
.into(),
)
} else {
(404, "{}".into())
}
});
crate::agent::core::tests::sign_in(&core, &hub);
let (s, v, _) = call(
"GET",
port,
"/v2/ledger?by=day&from=2026-11-01&to=2026-11-10",
Some("tok"),
"",
);
assert_eq!(s, 200, "{v}");
assert_eq!(
(v["unit"].as_str(), v["by"].as_str()),
(Some("jpy"), Some("day"))
);
assert_eq!(v["groups"][0]["earned"], 5.0);
assert_eq!(v["totals"]["spent"], 1.0);
let urls: Vec<String> = seen.lock().unwrap().iter().map(|e| e.1.clone()).collect();
let u = urls
.iter()
.find(|u| u.starts_with("/api/provide/ledger"))
.unwrap();
assert!(
u.contains("from=2026-11-01") && u.contains("to=2026-11-10"),
"{u}"
);
let (dead, _) = crate::agent::hub::tests::mock_hub(|_, _, _| (500, "{}".into()));
crate::agent::core::tests::sign_in(&core, &dead);
let (s, v, _) = call("GET", port, "/v2/ledger", Some("tok"), "");
assert_eq!((s, v["error"]["code"].as_str()), (502, Some("hub_error")));
}
}