use crate::agent::config::AgentConfig;
use crate::agent::files::{ensure_dir, ChildRecord, Children, WORKER_LOG_DIR};
use crate::agent::merge::LocalView;
use crate::agent::plan::{self, Action, BrokerObs, Desired, Observed, WorkerObs};
use crate::agent::summary::{Problem, Requests, WorkerCounts};
use crate::vpn::RouteKind;
use std::collections::{BTreeMap, HashSet};
use std::io::{Read as _, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
pub const WORKER_LOG_MAX_BYTES: u64 = 5 * 1024 * 1024;
const DRAIN_TIMEOUT_SHOWN_FOR: Duration = Duration::from_secs(300);
const CRASH_STABLE_AFTER: Duration = Duration::from_secs(300);
const CRASHLOOP_WINDOW: Duration = Duration::from_secs(300);
pub(crate) const BROKER_CHECK_TIMEOUT: Duration = Duration::from_secs(2);
#[derive(Debug, Clone, PartialEq, serde::Deserialize)]
pub struct BrokerWorker {
pub id: String,
pub name: String,
pub status: String,
#[serde(default)]
pub active_requests: u32,
#[serde(default)]
pub requests_5h: u64,
#[serde(default)]
pub requests_1w: u64,
}
pub trait BrokerApi {
fn is_broker(&self) -> bool;
fn workers(&self) -> Result<Vec<BrokerWorker>, String>;
fn drain(&self, id: &str) -> Result<(), String>;
fn delete(&self, id: &str) -> Result<(), String>;
fn is_broker_at(&self, _port: u16) -> bool {
false
}
fn port_free(&self, port: u16) -> bool;
fn os_assigned_port(&self) -> Option<u16>;
}
pub const BROKER_FALLBACK_START: u16 = 9001;
pub const BROKER_FALLBACK_END: u16 = 9010;
pub fn choose_broker_port(
preferred: u16,
fallback_start: u16,
fallback_end: u16,
mut is_free: impl FnMut(u16) -> bool,
os_pick: impl FnOnce() -> Option<u16>,
) -> Option<u16> {
if is_free(preferred) {
return Some(preferred);
}
for p in fallback_start..=fallback_end {
if is_free(p) {
return Some(p);
}
}
os_pick()
}
pub struct HttpBrokerApi {
pub port: u16,
pub worker_key: Option<String>,
}
impl HttpBrokerApi {
fn base(&self) -> String {
format!("http://127.0.0.1:{}", self.port)
}
fn http() -> ureq::Agent {
ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.timeout_global(Some(BROKER_CHECK_TIMEOUT))
.http_status_as_error(false)
.build(),
)
}
}
fn is_success_status(status: u16) -> bool {
(200..300).contains(&status)
}
impl BrokerApi for HttpBrokerApi {
fn is_broker(&self) -> bool {
self.is_broker_at(self.port)
}
fn is_broker_at(&self, port: u16) -> bool {
Self::http()
.get(&format!("http://127.0.0.1:{port}/health"))
.call()
.ok()
.filter(|r| r.status().as_u16() == 200)
.and_then(|r| r.into_body().read_to_string().ok())
.is_some_and(|b| b.contains("zakuro-broker"))
}
fn port_free(&self, port: u16) -> bool {
!crate::up::is_port_open(port) && std::net::TcpListener::bind(("0.0.0.0", port)).is_ok()
}
fn os_assigned_port(&self) -> Option<u16> {
std::net::TcpListener::bind(("127.0.0.1", 0))
.ok()
.and_then(|l| l.local_addr().ok())
.map(|a| a.port())
}
fn workers(&self) -> Result<Vec<BrokerWorker>, String> {
#[derive(serde::Deserialize)]
struct List {
workers: Vec<BrokerWorker>,
}
let resp = Self::http()
.get(&format!("{}/workers", self.base()))
.call()
.map_err(|e| e.to_string())?;
let status = resp.status().as_u16();
if !is_success_status(status) {
return Err(format!("workers: HTTP {status}"));
}
let body = resp
.into_body()
.read_to_string()
.map_err(|e| e.to_string())?;
serde_json::from_str::<List>(&body)
.map(|l| l.workers)
.map_err(|e| e.to_string())
}
fn drain(&self, id: &str) -> Result<(), String> {
let resp = Self::http()
.post(&format!("{}/workers/{id}/drain", self.base()))
.send_empty()
.map_err(|e| e.to_string())?;
match resp.status().as_u16() {
200 => Ok(()),
s => Err(format!("drain {id}: HTTP {s}")),
}
}
fn delete(&self, id: &str) -> Result<(), String> {
let mut req = Self::http().delete(&format!("{}/workers/{id}", self.base()));
if let Some(k) = &self.worker_key {
req = req.header("X-Worker-Key", k.as_str());
}
let resp = req.call().map_err(|e| e.to_string())?;
match resp.status().as_u16() {
200 | 404 => Ok(()),
s => Err(format!("delete {id}: HTTP {s}")),
}
}
}
pub fn render_argv(template: &[String], port: u16, name: &str) -> Vec<String> {
template
.iter()
.map(|a| {
a.replace("{port}", &port.to_string())
.replace("{name}", name)
})
.collect()
}
pub fn spawn_in_group(
argv: &[String],
cwd: Option<&Path>,
env: &[(String, String)],
log: &Path,
) -> std::io::Result<Child> {
use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::process::CommandExt;
let (prog, rest) = argv
.split_first()
.ok_or_else(|| std::io::Error::other("empty command"))?;
if let Some(parent) = log.parent() {
ensure_dir(parent)?;
}
let out = std::fs::OpenOptions::new()
.create(true)
.append(true)
.mode(0o644)
.open(log)?;
let err = out.try_clone()?;
let mut cmd = Command::new(prog);
cmd.args(rest)
.stdin(Stdio::null())
.stdout(Stdio::from(out))
.stderr(Stdio::from(err))
.process_group(0);
if let Some(d) = cwd {
cmd.current_dir(d);
}
for (k, v) in env {
cmd.env(k, v);
}
cmd.spawn()
}
pub fn signal_group(pid: u32, sig: libc::c_int) -> bool {
if pid <= 1 {
return false;
}
unsafe { libc::killpg(pid as libc::pid_t, sig) == 0 }
}
fn process_state(pid: u32) -> Option<char> {
#[cfg(target_os = "linux")]
{
let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
let after = &stat[stat.rfind(')')? + 1..];
after.split_whitespace().next()?.chars().next()
}
#[cfg(target_os = "macos")]
{
let out = Command::new("ps")
.args(["-o", "state=", "-p", &pid.to_string()])
.output()
.ok()?;
if !out.status.success() {
return None;
}
String::from_utf8_lossy(&out.stdout).trim().chars().next()
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
{
let _ = pid;
None
}
}
fn all_processes() -> Vec<(u32, u32, char)> {
#[cfg(target_os = "linux")]
{
let mut out = Vec::new();
if let Ok(entries) = std::fs::read_dir("/proc") {
for entry in entries.flatten() {
let Some(name) = entry.file_name().to_str().map(str::to_owned) else {
continue;
};
let Ok(pid) = name.parse::<u32>() else {
continue;
};
let Ok(stat) = std::fs::read_to_string(entry.path().join("stat")) else {
continue;
};
let Some(after) = stat.rfind(')').map(|i| &stat[i + 1..]) else {
continue;
};
let mut fields = after.split_whitespace();
let Some(state) = fields.next().and_then(|s| s.chars().next()) else {
continue;
};
let Some(pgrp) = fields.nth(1).and_then(|s| s.parse::<u32>().ok()) else {
continue;
};
out.push((pid, pgrp, state));
}
}
out
}
#[cfg(target_os = "macos")]
{
let mut out = Vec::new();
if let Ok(o) = Command::new("ps")
.args(["-axo", "pid=,pgid=,state="])
.output()
{
if o.status.success() {
for line in String::from_utf8_lossy(&o.stdout).lines() {
let mut it = line.split_whitespace();
let pid = it.next().and_then(|s| s.parse::<u32>().ok());
let pgid = it.next().and_then(|s| s.parse::<u32>().ok());
let state = it.next().and_then(|s| s.chars().next());
if let (Some(pid), Some(pgid), Some(state)) = (pid, pgid, state) {
out.push((pid, pgid, state));
}
}
}
}
out
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
{
Vec::new()
}
}
fn group_alive(pid: u32) -> bool {
if pid <= 1 {
return false;
}
if unsafe { libc::killpg(pid as libc::pid_t, 0) } != 0 {
return false;
}
match process_state(pid) {
Some(state) if state != 'Z' => return true,
None => {
}
Some(_) => {
}
}
let procs = all_processes();
if procs.is_empty() {
return true;
}
procs
.iter()
.any(|&(_, pgrp, state)| pgrp == pid && state != 'Z')
}
pub fn is_alive(pid: u32) -> bool {
if unsafe { libc::kill(pid as libc::pid_t, 0) } != 0 {
return false;
}
process_state(pid) != Some('Z')
}
#[cfg(target_os = "linux")]
fn linux_start_ticks(pid: u32) -> Option<String> {
let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
let after = &stat[stat.rfind(')')? + 1..];
after
.split_whitespace()
.nth(19)
.map(|t| format!("ticks:{t}"))
}
pub fn proc_start_time(pid: u32) -> Option<String> {
#[cfg(target_os = "linux")]
{
if let Some(t) = linux_start_ticks(pid) {
return Some(t);
}
}
let out = Command::new("ps")
.args(["-o", "lstart=", "-p", &pid.to_string()])
.output()
.ok()?;
let s = String::from_utf8_lossy(&out.stdout).trim().to_string();
(out.status.success() && !s.is_empty()).then_some(s)
}
pub fn truncate_if_large(path: &Path, max: u64) -> std::io::Result<bool> {
match std::fs::metadata(path) {
Ok(m) if m.len() > max => {
std::fs::OpenOptions::new()
.write(true)
.truncate(true)
.open(path)?;
Ok(true)
}
_ => Ok(false),
}
}
pub fn tail_lines(path: &Path, n: usize) -> String {
const CAP: u64 = 64 * 1024;
let mut f = match std::fs::File::open(path) {
Ok(f) => f,
Err(_) => return String::new(),
};
let len = f.metadata().map(|m| m.len()).unwrap_or(0);
let start = len.saturating_sub(CAP);
if start > 0 && f.seek(SeekFrom::Start(start)).is_err() {
return String::new();
}
let mut buf = Vec::new();
let _ = f.read_to_end(&mut buf);
let text = String::from_utf8_lossy(&buf).to_string();
let lines: Vec<&str> = text.lines().collect();
lines[lines.len().saturating_sub(n)..].join("\n")
}
pub trait DockerCli: Send + Sync {
fn run(&self, argv: &[String]) -> Result<String, String>;
}
pub struct DockerShell;
impl DockerCli for DockerShell {
fn run(&self, argv: &[String]) -> Result<String, String> {
let (prog, rest) = argv.split_first().ok_or("empty docker command")?;
let out = Command::new(prog)
.args(rest)
.output()
.map_err(|e| format!("docker not available: {e}"))?;
if out.status.success() {
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
} else {
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WorkerBackend {
Host,
Container { image: String },
}
#[derive(Debug, Clone)]
pub struct SpawnSpec {
pub broker_argv: Vec<String>,
pub broker_env: Vec<(String, String)>,
pub worker_template: Vec<String>,
pub worker_env: Vec<(String, String)>,
pub worker_cwd: Option<PathBuf>,
pub worker_backend: WorkerBackend,
pub worker_price: Option<String>,
pub base_worker_port: u16,
pub preferred_broker_port: u16,
pub fp8: String,
pub log_dir: PathBuf,
pub kill_grace: Duration,
pub broker_stop_wait: Duration,
pub drain_timeout: Duration,
}
impl SpawnSpec {
pub fn from_config(cfg: &AgentConfig) -> Self {
Self::from_config_with(cfg, &crate::up::docker_cli_available)
}
pub(crate) fn from_config_with(cfg: &AgentConfig, docker_available: &dyn Fn() -> bool) -> Self {
let scan_end = cfg.base_worker_port as u32 + cfg.max_workers + 19;
let mut broker_env = vec![
(
"ZAKURO_SCAN_RANGE".to_string(),
format!("{}-{scan_end}", cfg.base_worker_port),
),
("ZAKURO_SCAN_INTERVAL".to_string(), "3".to_string()),
];
broker_env.extend(cfg.child_env.iter().cloned());
let mut worker_env = vec![
("ZAKURO_WORKER_TYPE".to_string(), "zakuro".to_string()),
("UV_NO_PROGRESS".to_string(), "1".to_string()),
];
worker_env.extend(cfg.child_env.iter().cloned());
let effective = |k: &str| {
cfg.child_env
.iter()
.rev()
.find(|(ek, _)| ek == k)
.map(|(_, v)| v.clone())
.or_else(|| std::env::var(k).ok())
};
if let Some((k, v)) = crate::up::worker_bind_env(&effective) {
worker_env.push((k.to_string(), v.to_string()));
}
let runtime = match crate::up::select_share_runtime(&effective, docker_available) {
Ok(choice) => {
if let Some(w) = &choice.warning {
eprintln!("zc agent: {w}");
}
choice.runtime
}
Err(e) if e.starts_with("ZAKURO_SHARE_RUNTIME=container") => {
crate::up::ShareRuntime::Container
}
Err(e) => {
eprintln!("zc agent: {e}; using the host process");
crate::up::ShareRuntime::Host
}
};
let worker_backend = if runtime == crate::up::ShareRuntime::Container {
WorkerBackend::Container {
image: crate::up::worker_container_image(&effective),
}
} else {
WorkerBackend::Host
};
Self {
broker_argv: cfg.broker_argv.clone(),
broker_env,
worker_template: cfg.worker_template.clone(),
worker_env,
worker_cwd: None,
worker_backend,
worker_price: crate::up::worker_price_env(&effective, None).map(|(_, v)| v),
base_worker_port: cfg.base_worker_port,
preferred_broker_port: cfg.broker_port,
fp8: cfg.node_fp8(),
log_dir: cfg.dir.clone(),
kill_grace: cfg.timings.kill_grace,
broker_stop_wait: cfg.timings.broker_stop_wait,
drain_timeout: cfg.timings.drain_timeout,
}
}
}
struct HostProc {
pid: u32,
started_at: String,
child: Option<Child>,
}
struct ContainerProc {
name: String,
started_at: String,
docker: Arc<dyn DockerCli>,
last_alive: Option<(Instant, bool)>,
}
const CONTAINER_POLL: Duration = Duration::from_millis(250);
const CRASH_LOG_LINES: usize = 20;
const CONTAINER_LIVENESS_TTL: Duration = Duration::from_secs(5);
enum Proc {
Host(HostProc),
Container(ContainerProc),
}
impl Proc {
fn spawned(child: Child) -> Self {
let pid = child.id();
Proc::Host(HostProc {
pid,
started_at: proc_start_time(pid).unwrap_or_default(),
child: Some(child),
})
}
fn pid(&self) -> u32 {
match self {
Proc::Host(h) => h.pid,
Proc::Container(_) => 0,
}
}
fn alive(&mut self, now: Instant) -> bool {
match self {
Proc::Host(h) => match &mut h.child {
Some(c) => matches!(c.try_wait(), Ok(None)),
None => is_alive(h.pid),
},
Proc::Container(c) => {
if let Some((at, answer)) = c.last_alive {
if now.saturating_duration_since(at) < CONTAINER_LIVENESS_TTL {
return answer;
}
}
let answer = c.docker.run(&crate::up::worker_alive_argv(&c.name)).is_ok();
c.last_alive = Some((now, answer));
answer
}
}
}
fn record(&self, port: Option<u16>) -> ChildRecord {
match self {
Proc::Host(h) => ChildRecord {
pid: h.pid,
started_at: h.started_at.clone(),
port,
container: None,
},
Proc::Container(c) => ChildRecord {
pid: 0,
started_at: c.started_at.clone(),
port,
container: Some(c.name.clone()),
},
}
}
fn signal_term(&self) {
match self {
Proc::Host(h) => {
signal_group(h.pid, libc::SIGTERM);
}
Proc::Container(c) => {
let _ = c
.docker
.run(&crate::up::docker_signal_argv(&c.name, "TERM"));
}
}
}
fn is_gone(&mut self) -> bool {
match self {
Proc::Host(h) => {
if let Some(c) = h.child.as_mut() {
let _ = c.try_wait();
}
!group_alive(h.pid)
}
Proc::Container(c) => match c.docker.run(&crate::up::docker_state_argv(&c.name)) {
Ok(out) => crate::up::parse_container_state(&out).is_none_or(|s| !s.running),
Err(_) => true,
},
}
}
fn force_kill(&self) {
match self {
Proc::Host(h) => {
if group_alive(h.pid) {
signal_group(h.pid, libc::SIGKILL);
}
}
Proc::Container(c) => {
let _ = c
.docker
.run(&crate::up::docker_signal_argv(&c.name, "KILL"));
}
}
}
fn reap(&mut self) {
if let Proc::Host(h) = self {
if let Some(c) = h.child.as_mut() {
let _ = c.wait();
}
}
}
fn poll_interval(&self, base: Duration) -> Duration {
match self {
Proc::Host(_) => base,
Proc::Container(_) => base.max(CONTAINER_POLL),
}
}
fn kill_stragglers(&self) {
match self {
Proc::Host(h) => {
if group_alive(h.pid) {
signal_group(h.pid, libc::SIGKILL);
}
}
Proc::Container(c) => {
let _ = c.docker.run(&crate::up::docker_rm_argv(&c.name));
}
}
}
fn captured_log_tail(&self, n: usize) -> Option<String> {
match self {
Proc::Host(_) => None,
Proc::Container(c) => {
let headline = match container_state(&c.docker, &c.name) {
Some(s) if s.running => format!(
"the worker process is gone from container {}, which is still up",
c.name
),
Some(s) => format!("container {} stopped, exit code {}", c.name, s.exit_code),
None => format!("container {} no longer exists", c.name),
};
let tail = container_log_tail(&c.docker, &c.name, n);
Some(format!("{headline}\n{tail}").trim_end().to_string())
}
}
}
}
fn container_log_tail(docker: &Arc<dyn DockerCli>, name: &str, n: usize) -> String {
docker
.run(&crate::up::worker_log_tail_argv(name, n))
.ok()
.filter(|s| !s.trim().is_empty())
.or_else(|| docker.run(&crate::up::docker_logs_argv(name, n)).ok())
.unwrap_or_default()
}
fn container_state(docker: &Arc<dyn DockerCli>, name: &str) -> Option<crate::up::ContainerState> {
let out = docker.run(&crate::up::docker_state_argv(name)).ok()?;
crate::up::parse_container_state(&out)
}
fn terminate(mut p: Proc, grace: Duration, stop: &AtomicBool) -> Option<Proc> {
p.signal_term();
let deadline = Instant::now() + grace;
let poll = p.poll_interval(Duration::from_millis(50));
let mut gone = false;
while Instant::now() < deadline {
if stop.load(Ordering::Relaxed) {
return Some(p);
}
if p.is_gone() {
gone = true;
break;
}
std::thread::sleep(poll);
}
if !gone {
p.force_kill();
}
p.reap();
None
}
#[derive(Default)]
struct Slot {
proc_: Option<Proc>,
draining_since: Option<Instant>,
failures: Vec<Instant>,
consecutive: u32,
restart_at: Option<Instant>,
up_since: Option<Instant>,
crash_log: Option<String>,
}
pub struct Supervisor {
spec: SpawnSpec,
broker: Option<Proc>,
broker_port: u16,
broker_stopping: bool,
broker_failures: Vec<Instant>,
broker_consecutive: u32,
broker_restart_at: Option<Instant>,
broker_up_since: Option<Instant>,
broker_route: Option<RouteKind>,
slots: BTreeMap<u32, Slot>,
last_workers: Vec<BrokerWorker>,
workers_fresh: bool,
workers_err_streak: bool,
last_broker: BrokerObs,
last_drain_timeout: Option<Instant>,
stop: Arc<AtomicBool>,
route: RouteKind,
docker: Arc<dyn DockerCli>,
}
fn broker_label(b: BrokerObs) -> &'static str {
match b {
BrokerObs::Running => "running",
BrokerObs::Starting => "starting",
BrokerObs::Stopping => "stopping",
BrokerObs::Stopped => "stopped",
BrokerObs::Unmanaged => "unmanaged",
BrokerObs::PortBlocked => "error",
}
}
impl Supervisor {
pub fn new(spec: SpawnSpec) -> Self {
let broker_port = spec.preferred_broker_port;
Self {
spec,
broker: None,
broker_port,
broker_stopping: false,
broker_failures: vec![],
broker_consecutive: 0,
broker_restart_at: None,
broker_up_since: None,
broker_route: None,
slots: BTreeMap::new(),
last_workers: vec![],
workers_fresh: true,
workers_err_streak: false,
last_broker: BrokerObs::Stopped,
last_drain_timeout: None,
stop: Arc::new(AtomicBool::new(false)),
route: RouteKind::None,
docker: Arc::new(DockerShell),
}
}
pub(crate) fn set_docker(&mut self, docker: Arc<dyn DockerCli>) {
self.docker = docker;
}
pub fn stop_flag(&self) -> Arc<AtomicBool> {
self.stop.clone()
}
pub fn broker_port(&self) -> u16 {
self.broker_port
}
pub fn set_worker_cwd(&mut self, dir: Option<PathBuf>) {
self.spec.worker_cwd = dir;
}
pub fn set_route(&mut self, route: RouteKind) {
self.route = route;
}
fn worker_name(&self, i: u32) -> String {
crate::up::worker_name(&self.spec.fp8, i as usize)
}
fn worker_log(&self, i: u32) -> PathBuf {
self.spec
.log_dir
.join(WORKER_LOG_DIR)
.join(format!("w{i}.log"))
}
fn broker_log(&self) -> PathBuf {
self.spec.log_dir.join("broker.log")
}
pub fn adopt(&mut self, recorded: &Children, start_time: &dyn Fn(u32) -> Option<String>) {
let docker = self.docker.clone();
let adopted = |r: &ChildRecord| -> Option<Proc> {
match &r.container {
None => (is_alive(r.pid)
&& start_time(r.pid).as_deref() == Some(r.started_at.as_str()))
.then(|| {
Proc::Host(HostProc {
pid: r.pid,
started_at: r.started_at.clone(),
child: None,
})
}),
Some(name) => {
let state = container_state(&docker, name)?;
(state.running && state.started_at == r.started_at).then(|| {
Proc::Container(ContainerProc {
name: name.clone(),
started_at: state.started_at,
docker: docker.clone(),
last_alive: None,
})
})
}
}
};
if let Some(b) = recorded.broker.as_ref() {
if let Some(p) = adopted(b) {
self.broker = Some(p);
self.broker_port = b.port.unwrap_or(self.spec.preferred_broker_port);
}
}
for (k, r) in &recorded.workers {
if let (Ok(i), Some(p)) = (k.parse::<u32>(), adopted(r)) {
self.slots.entry(i).or_default().proc_ = Some(p);
}
}
}
pub fn sweep_orphans(&self) -> Vec<String> {
let listing = match self.docker.run(&crate::up::docker_ps_share_argv()) {
Ok(o) => o,
Err(_) => return vec![],
};
let known: Vec<&str> = self
.slots
.values()
.filter_map(|s| match s.proc_.as_ref() {
Some(Proc::Container(c)) => Some(c.name.as_str()),
_ => None,
})
.collect();
let mut removed = vec![];
for (id, name) in crate::up::parse_share_ps(&listing) {
if known.contains(&name.as_str()) {
continue;
}
let short: String = id.chars().take(12).collect();
match self.docker.run(&crate::up::docker_rm_argv(&id)) {
Ok(_) => {
eprintln!(" [AGENT] removed orphaned share container {name} ({short})");
removed.push(name);
}
Err(e) => eprintln!(
" [AGENT] could not remove orphaned share container {name} ({short}): {e}"
),
}
}
removed
}
pub fn children(&self) -> Children {
let broker_port = self.broker_port;
Children {
broker: self.broker.as_ref().map(|p| p.record(Some(broker_port))),
workers: self
.slots
.iter()
.filter_map(|(i, s)| s.proc_.as_ref().map(|p| (i.to_string(), p.record(None))))
.collect(),
}
}
pub fn broker_unmanaged(&self) -> bool {
self.last_broker == BrokerObs::Unmanaged
}
pub fn broker_running(&self) -> bool {
self.last_broker == BrokerObs::Running
}
fn observe(
&mut self,
api: &dyn BrokerApi,
prerequisites_ok: bool,
sharing: bool,
now: Instant,
) -> Observed {
let broker_alive = self.broker.as_mut().map(|b| b.alive(now));
if self.broker.is_some() && self.broker_route.is_none() {
self.broker_route = Some(self.route);
}
let broker = match broker_alive {
Some(true) if self.broker_stopping => BrokerObs::Stopping,
Some(true) if api.is_broker() => {
match self.broker_up_since {
None => self.broker_up_since = Some(now),
Some(since) if now.saturating_duration_since(since) >= CRASH_STABLE_AFTER => {
self.broker_consecutive = 0;
}
_ => {}
}
BrokerObs::Running
}
Some(true) => BrokerObs::Starting,
Some(false) | None => {
if broker_alive == Some(false) {
if let Some(b) = self.broker.as_ref() {
b.kill_stragglers();
}
if !self.broker_stopping {
self.broker_failures.push(now);
self.broker_failures
.retain(|t| now.saturating_duration_since(*t) <= CRASHLOOP_WINDOW);
self.broker_consecutive += 1;
self.broker_restart_at =
Some(now + plan::restart_backoff(self.broker_consecutive));
}
}
self.broker = None;
self.broker_stopping = false;
self.broker_up_since = None;
self.broker_route = None;
let preferred = self.spec.preferred_broker_port;
let restart_ready = self.broker_restart_at.is_none_or(|t| now >= t);
if api.is_broker_at(preferred) {
BrokerObs::Unmanaged
} else if sharing && restart_ready {
let relayed_only = self.route == RouteKind::Proxy;
match choose_broker_port(
preferred,
BROKER_FALLBACK_START,
BROKER_FALLBACK_END,
|p| (!relayed_only || crate::vpn::relay_covers(p)) && api.port_free(p),
|| {
if relayed_only {
None
} else {
api.os_assigned_port()
}
},
) {
Some(p) => {
self.broker_port = p;
BrokerObs::Stopped
}
None => BrokerObs::PortBlocked,
}
} else {
BrokerObs::Stopped
}
}
};
self.last_broker = broker;
match broker {
BrokerObs::Running => match api.workers() {
Ok(list) => {
if self.workers_err_streak {
eprintln!(" [AGENT] GET /workers: recovered");
self.workers_err_streak = false;
}
self.last_workers = list;
self.workers_fresh = true;
}
Err(e) => {
if !self.workers_err_streak {
eprintln!(" [AGENT] GET /workers: {e}");
self.workers_err_streak = true;
}
self.workers_fresh = false;
}
},
BrokerObs::Stopped | BrokerObs::Unmanaged | BrokerObs::PortBlocked => {
self.last_workers = vec![];
self.workers_fresh = true;
self.workers_err_streak = false;
}
BrokerObs::Starting | BrokerObs::Stopping => {
self.workers_fresh = false;
}
}
let broker_restart_ready = self.broker_restart_at.is_none_or(|t| now >= t);
let route_restart = self.broker.is_some()
&& (self
.broker_route
.is_some_and(|r| plan::route_change_needs_restart(r, self.route))
|| (self.route == RouteKind::Proxy && !crate::vpn::relay_covers(self.broker_port)));
let fresh = self.workers_fresh;
let Self {
slots,
last_workers,
spec,
..
} = self;
let mut workers = Vec::with_capacity(slots.len());
for (&i, slot) in slots.iter_mut() {
let running = slot.proc_.as_mut().is_some_and(|p| p.alive(now));
if !running {
if let Some(p) = slot.proc_.take() {
if slot.draining_since.is_none() {
slot.crash_log = p.captured_log_tail(CRASH_LOG_LINES);
}
p.kill_stragglers();
if slot.draining_since.is_none() {
slot.failures.push(now);
slot.failures
.retain(|t| now.saturating_duration_since(*t) <= CRASHLOOP_WINDOW);
slot.consecutive += 1;
slot.restart_at = Some(now + plan::restart_backoff(slot.consecutive));
slot.up_since = None;
}
}
} else {
match slot.up_since {
None => slot.up_since = Some(now),
Some(since) if now.saturating_duration_since(since) >= CRASH_STABLE_AFTER => {
slot.consecutive = 0;
}
_ => {}
}
}
let name = crate::up::worker_name(&spec.fp8, i as usize);
let seen = last_workers.iter().find(|w| w.name == name);
let active_requests = if slot.draining_since.is_some() && !fresh {
seen.map_or(1, |w| w.active_requests.max(1))
} else {
seen.map_or(0, |w| w.active_requests)
};
workers.push(WorkerObs {
index: i,
running,
draining: slot.draining_since.is_some(),
active_requests,
drain_expired: slot
.draining_since
.is_some_and(|t| now.saturating_duration_since(t) >= spec.drain_timeout),
restart_ready: slot.restart_at.is_none_or(|t| now >= t),
});
}
Observed {
broker,
workers,
prerequisites_ok,
broker_restart_ready,
route_restart,
}
}
fn spawn_worker(&mut self, i: u32, now: Instant) -> Result<(), String> {
let port = self.spec.base_worker_port + i as u16;
let name = self.worker_name(i);
let proc_ = match self.spec.worker_backend.clone() {
WorkerBackend::Host => {
let argv = render_argv(&self.spec.worker_template, port, &name);
let mut env = self.spec.worker_env.clone();
env.push(("ZAKURO_AGENT_CHILD_PORT".to_string(), port.to_string()));
env.push(("ZAKURO_AGENT_CHILD_NAME".to_string(), name));
let child = spawn_in_group(
&argv,
self.spec.worker_cwd.as_deref(),
&env,
&self.worker_log(i),
)
.map_err(|e| e.to_string())?;
Proc::spawned(child)
}
WorkerBackend::Container { image } => {
self.spawn_container_worker(port, &name, &image)?
}
};
let slot = self.slots.entry(i).or_default();
slot.proc_ = Some(proc_);
slot.draining_since = None;
slot.up_since = Some(now);
Ok(())
}
fn spawn_container_worker(&self, port: u16, name: &str, image: &str) -> Result<Proc, String> {
let docker = self.docker.clone();
let container = crate::up::worker_container_name(name);
let network = crate::up::worker_network_name(name);
let _ = docker.run(&crate::up::docker_rm_argv(&container));
let _ = docker.run(&crate::up::docker_network_create_argv(&network));
docker.run(&crate::up::docker_run_argv(
port, image, &container, &network, name,
))?;
if let Err(e) = docker.run(&crate::up::worker_exec_argv(
&container,
port,
name,
self.spec.worker_price.as_deref(),
)) {
let _ = docker.run(&crate::up::docker_rm_argv(&container));
return Err(e);
}
let started_at = container_state(&docker, &container)
.map(|s| s.started_at)
.unwrap_or_default();
Ok(Proc::Container(ContainerProc {
name: container,
started_at,
docker,
last_alive: None,
}))
}
fn broker_id(&self, i: u32) -> Option<String> {
let name = self.worker_name(i);
self.last_workers
.iter()
.find(|w| w.name == name)
.map(|w| w.id.clone())
}
fn apply(
&mut self,
actions: &[Action],
observed: &Observed,
api: &dyn BrokerApi,
now: Instant,
) {
for action in actions {
match *action {
Action::StartBroker => {
if self.broker_port != self.spec.preferred_broker_port {
let preferred = self.spec.preferred_broker_port;
let why = if self.route == RouteKind::Proxy
&& !crate::vpn::relay_covers(preferred)
{
"isn't a port the mesh relay carries"
} else {
"is held by another program"
};
eprintln!(
" [AGENT] broker: 127.0.0.1:{preferred} {why}; starting the broker on 127.0.0.1:{}",
self.broker_port
);
}
let log = self.broker_log();
let argv = render_argv(&self.spec.broker_argv, self.broker_port, "");
let mut env = self.spec.broker_env.clone();
env.push((
"ZAKURO_AGENT_CHILD_PORT".to_string(),
self.broker_port.to_string(),
));
if !env.iter().any(|(k, _)| k == "ZAKURO_P2P") {
env.push((
"ZAKURO_P2P".to_string(),
(self.route != RouteKind::None).to_string(),
));
}
match spawn_in_group(&argv, None, &env, &log) {
Ok(c) => {
self.broker = Some(Proc::spawned(c));
self.broker_up_since = Some(now);
self.broker_route = Some(self.route);
}
Err(e) => eprintln!(" [AGENT] could not start the broker: {e}"),
}
}
Action::StopBroker => {
if let Some(b) = self.broker.take() {
if observed.route_restart {
eprintln!(
" [AGENT] mesh route changed ({} → {}); restarting the broker",
self.broker_route.map_or("?", RouteKind::label),
self.route.label()
);
}
self.broker_stopping = true;
let leftover = terminate(b, self.spec.broker_stop_wait, &self.stop);
self.broker_stopping = false;
self.broker_up_since = None;
self.broker = leftover;
if self.broker.is_none() {
self.broker_route = None;
}
}
}
Action::StartWorker(i) | Action::RestartWorker(i) => {
if let Err(e) = self.spawn_worker(i, now) {
eprintln!(" [AGENT] could not start worker {i}: {e}");
}
}
Action::DrainWorker(i) => {
if let Some(id) = self.broker_id(i) {
if let Err(e) = api.drain(&id) {
eprintln!(" [AGENT] drain {id}: {e}");
}
}
self.slots.entry(i).or_default().draining_since = Some(now);
}
Action::KillWorker(i) => {
if observed
.workers
.iter()
.any(|w| w.index == i && w.drain_expired && w.active_requests > 0)
{
self.last_drain_timeout = Some(now);
}
if let Some(mut slot) = self.slots.remove(&i) {
if let Some(p) = slot.proc_.take() {
if let Some(p) = terminate(p, self.spec.kill_grace, &self.stop) {
slot.proc_ = Some(p);
self.slots.insert(i, slot);
continue;
}
}
}
if let Some(id) = self.broker_id(i) {
if let Err(e) = api.delete(&id) {
eprintln!(" [AGENT] delete {id}: {e}");
}
}
let name = self.worker_name(i);
self.last_workers.retain(|w| w.name != name);
}
}
}
}
fn resend_drains(&self, api: &dyn BrokerApi, actions: &[Action]) {
if !self.workers_fresh || self.last_broker != BrokerObs::Running {
return;
}
for (&i, slot) in &self.slots {
if slot.draining_since.is_none() || actions.contains(&Action::KillWorker(i)) {
continue;
}
let name = self.worker_name(i);
let Some(w) = self.last_workers.iter().find(|w| w.name == name) else {
continue;
};
if w.status != "draining" {
if let Err(e) = api.drain(&w.id) {
eprintln!(" [AGENT] drain {} (again): {e}", w.id);
}
}
}
}
pub fn reconcile(
&mut self,
desired: &Desired,
api: &dyn BrokerApi,
prerequisites_ok: bool,
now: Instant,
) -> Vec<Action> {
let observed = self.observe(api, prerequisites_ok, desired.sharing, now);
let actions = plan::plan_actions(desired, &observed);
self.resend_drains(api, &actions);
self.apply(&actions, &observed, api, now);
let logs: Vec<PathBuf> = self.slots.keys().map(|&i| self.worker_log(i)).collect();
for log in logs {
let _ = truncate_if_large(&log, WORKER_LOG_MAX_BYTES);
}
let _ = truncate_if_large(&self.broker_log(), WORKER_LOG_MAX_BYTES);
actions
}
pub fn problems(&self, now: Instant) -> Vec<Problem> {
let mut out = Vec::new();
match self.last_broker {
BrokerObs::Unmanaged => out.push(Problem::new("unmanaged_broker")),
BrokerObs::PortBlocked if self.route == RouteKind::Proxy => {
out.push(Problem::new("broker_port_unrelayable"))
}
BrokerObs::PortBlocked => out.push(Problem::new("port_in_use")),
_ => {}
}
if plan::is_crashloop(&self.broker_failures, now) {
let log = self.broker_log();
let mut p = Problem::new("broker_crashloop");
if log.exists() {
p = p.with_detail(tail_lines(&log, 20));
}
out.push(p);
}
for (&i, slot) in &self.slots {
if plan::is_crashloop(&slot.failures, now) {
let detail = slot
.crash_log
.clone()
.unwrap_or_else(|| tail_lines(&self.worker_log(i), CRASH_LOG_LINES));
out.push(Problem::new("worker_crashloop").with_detail(detail));
}
}
if self
.last_drain_timeout
.is_some_and(|t| now.saturating_duration_since(t) < DRAIN_TIMEOUT_SHOWN_FOR)
{
out.push(Problem::new("drain_timeout"));
}
out
}
pub fn local_view(&self, desired: &Desired, max: u32) -> LocalView {
let names: HashSet<String> = self.slots.keys().map(|&i| self.worker_name(i)).collect();
let mine: Vec<&BrokerWorker> = self
.last_workers
.iter()
.filter(|w| names.contains(&w.name))
.collect();
let count = |f: &dyn Fn(&Slot) -> bool| self.slots.values().filter(|s| f(s)).count() as u32;
LocalView {
sharing: desired.sharing,
broker: broker_label(self.last_broker).to_string(),
workers: WorkerCounts {
desired: desired.workers,
running: count(&|s| s.proc_.is_some() && s.draining_since.is_none()),
busy: mine.iter().filter(|w| w.active_requests > 0).count() as u32,
draining: count(&|s| s.proc_.is_some() && s.draining_since.is_some()),
max,
},
requests: Requests {
last_5h: mine.iter().map(|w| w.requests_5h).sum(),
last_1w: mine.iter().map(|w| w.requests_1w).sum(),
},
}
}
pub fn terminate_all(&mut self, wait: Duration) {
let mut procs: Vec<Proc> = self
.slots
.values_mut()
.filter_map(|s| s.proc_.take())
.collect();
procs.extend(self.broker.take());
for p in &procs {
p.signal_term();
}
let mut gone = vec![false; procs.len()];
let poll = procs
.iter()
.map(|p| p.poll_interval(Duration::from_millis(100)))
.max()
.unwrap_or(Duration::from_millis(100));
let deadline = Instant::now() + wait;
while Instant::now() < deadline && gone.iter().any(|g| !g) {
for (p, g) in procs.iter_mut().zip(gone.iter_mut()) {
if *g {
continue;
}
if p.is_gone() {
*g = true;
}
}
if gone.iter().any(|g| !g) {
std::thread::sleep(poll);
}
}
for (p, g) in procs.iter_mut().zip(gone.iter_mut()) {
if !*g {
p.force_kill();
}
p.reap();
}
self.slots.clear();
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::agent::files::tests::tmp;
use std::cell::{Cell, RefCell};
use std::ops::{Deref, DerefMut};
#[derive(Default)]
struct FakeBroker {
up: Cell<bool>,
health_override: Cell<Option<bool>>,
workers: RefCell<Vec<BrokerWorker>>,
workers_err: Cell<bool>,
drain_fails: Cell<bool>,
drained: RefCell<Vec<String>>,
deleted: RefCell<Vec<String>>,
foreign_broker_port: Cell<Option<u16>>,
blocked_ports: RefCell<std::collections::HashSet<u16>>,
os_pick: Cell<Option<u16>>,
port_free_calls: RefCell<u32>,
os_assigned_port_calls: RefCell<u32>,
}
impl BrokerApi for FakeBroker {
fn is_broker(&self) -> bool {
self.health_override.get().unwrap_or_else(|| self.up.get())
}
fn is_broker_at(&self, port: u16) -> bool {
self.foreign_broker_port.get() == Some(port)
}
fn port_free(&self, port: u16) -> bool {
*self.port_free_calls.borrow_mut() += 1;
!self.blocked_ports.borrow().contains(&port)
}
fn os_assigned_port(&self) -> Option<u16> {
*self.os_assigned_port_calls.borrow_mut() += 1;
self.os_pick.get()
}
fn workers(&self) -> Result<Vec<BrokerWorker>, String> {
if self.workers_err.get() {
return Err("simulated GET /workers failure".into());
}
Ok(self.workers.borrow().clone())
}
fn drain(&self, id: &str) -> Result<(), String> {
self.drained.borrow_mut().push(id.into());
if self.drain_fails.get() {
return Err("simulated POST /drain failure".into());
}
Ok(())
}
fn delete(&self, id: &str) -> Result<(), String> {
self.deleted.borrow_mut().push(id.into());
Ok(())
}
}
fn sh(script: &str) -> Vec<String> {
vec!["sh".into(), "-c".into(), script.into()]
}
fn spec(dir: &Path, worker: Vec<String>) -> SpawnSpec {
SpawnSpec {
broker_argv: sh("exec sleep 60"),
broker_env: vec![],
worker_template: worker,
worker_env: vec![],
worker_cwd: None,
worker_backend: WorkerBackend::Host,
worker_price: None,
base_worker_port: 3960,
preferred_broker_port: 9000,
fp8: "testfp00".into(),
log_dir: dir.to_path_buf(),
kill_grace: Duration::from_secs(2),
broker_stop_wait: Duration::from_secs(2),
drain_timeout: Duration::from_secs(300),
}
}
fn bw_n(id: &str, name: &str, active_requests: u32) -> BrokerWorker {
BrokerWorker {
id: id.into(),
name: name.into(),
status: "healthy".into(),
active_requests,
requests_5h: 1,
requests_1w: 2,
}
}
fn bw(id: &str, name: &str) -> BrokerWorker {
bw_n(id, name, 0)
}
fn wait_until(mut pred: impl FnMut() -> bool) -> bool {
for _ in 0..100 {
if pred() {
return true;
}
std::thread::sleep(Duration::from_millis(50));
}
pred()
}
fn wait_until_worker_exited(sup: &mut Supervisor, i: u32) -> bool {
wait_until(|| {
sup.slots
.get_mut(&i)
.and_then(|s| s.proc_.as_mut())
.map(|p| !p.alive(Instant::now()))
.unwrap_or(true)
})
}
fn wait_until_broker_exited(sup: &mut Supervisor) -> bool {
wait_until(|| {
sup.broker
.as_mut()
.map(|p| !p.alive(Instant::now()))
.unwrap_or(true)
})
}
struct PgGuard(u32);
impl Drop for PgGuard {
fn drop(&mut self) {
signal_group(self.0, libc::SIGKILL);
}
}
struct ChildGuard(Child);
impl ChildGuard {
fn id(&self) -> u32 {
self.0.id()
}
}
impl Drop for ChildGuard {
fn drop(&mut self) {
let pid = self.0.id();
let leader_reaped = matches!(self.0.try_wait(), Ok(Some(_)));
if leader_reaped && !group_alive(pid) {
return;
}
signal_group(pid, libc::SIGKILL);
let _ = self.0.wait();
}
}
struct ProcGuard(Proc);
impl ProcGuard {
fn into_inner(self) -> Proc {
let this = std::mem::ManuallyDrop::new(self);
unsafe { std::ptr::read(&this.0) }
}
}
impl Drop for ProcGuard {
fn drop(&mut self) {
if self.0.is_gone() {
return;
}
self.0.force_kill();
self.0.reap();
}
}
struct PidReapGuard(u32);
impl Drop for PidReapGuard {
fn drop(&mut self) {
signal_group(self.0, libc::SIGKILL);
unsafe {
libc::waitpid(self.0 as libc::pid_t, std::ptr::null_mut(), 0);
}
}
}
struct SupGuard(Supervisor);
impl Deref for SupGuard {
type Target = Supervisor;
fn deref(&self) -> &Supervisor {
&self.0
}
}
impl DerefMut for SupGuard {
fn deref_mut(&mut self) -> &mut Supervisor {
&mut self.0
}
}
impl Drop for SupGuard {
fn drop(&mut self) {
self.0.terminate_all(Duration::from_secs(2));
}
}
#[test]
fn dropping_a_plain_supervisor_leaves_its_child_running() {
let dir = tmp("plain-drop");
let pid = {
let mut sup = Supervisor::new(spec(&dir, sh("exec sleep 60")));
sup.spawn_worker(0, Instant::now()).unwrap();
sup.children().workers["0"].pid
}; let _reap = PidReapGuard(pid); assert!(
is_alive(pid),
"a plain Supervisor must not kill its children on drop"
);
}
#[test]
fn reconcile_starts_scales_drains_and_pauses_real_processes() {
use Action::*;
let dir = tmp("sup");
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let t = Instant::now();
let on2 = Desired {
sharing: true,
workers: 2,
};
assert_eq!(sup.reconcile(&on2, &api, true, t), vec![StartBroker]);
api.up.set(true); assert_eq!(
sup.reconcile(&on2, &api, true, t),
vec![StartWorker(0), StartWorker(1)]
);
*api.workers.borrow_mut() = vec![bw("id0", "testfp00-w0"), bw("id1", "testfp00-w1")];
assert_eq!(sup.reconcile(&on2, &api, true, t), vec![]);
let view = sup.local_view(&on2, 5);
assert_eq!(
(
view.workers.running,
view.workers.desired,
view.broker.as_str()
),
(2, 2, "running")
);
assert_eq!(
view.requests,
Requests {
last_5h: 2,
last_1w: 4
}
);
assert_eq!(sup.children().workers.len(), 2);
assert!(sup.children().broker.is_some());
let on1 = Desired {
sharing: true,
workers: 1,
};
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![DrainWorker(1)]);
assert_eq!(*api.drained.borrow(), vec!["id1".to_string()]);
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![KillWorker(1)]);
assert_eq!(*api.deleted.borrow(), vec!["id1".to_string()]);
assert_eq!(
sup.children().workers.keys().cloned().collect::<Vec<_>>(),
vec!["0".to_string()]
);
let off = Desired {
sharing: false,
workers: 1,
};
assert_eq!(sup.reconcile(&off, &api, true, t), vec![DrainWorker(0)]);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![KillWorker(0)]);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![StopBroker]);
assert!(sup.children().broker.is_none());
assert!(sup.children().workers.is_empty());
}
#[test]
fn a_crashing_worker_backs_off_and_is_flagged_as_a_crashloop() {
use Action::*;
let dir = tmp("crash");
let api = FakeBroker::default();
api.up.set(true);
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("echo boom; exit 1"))));
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
let on1 = Desired {
sharing: true,
workers: 1,
};
let broker_child =
spawn_in_group(&sh("exec sleep 60"), None, &[], &dir.join("b.log")).unwrap();
let _guard = PgGuard(broker_child.id()); sup.broker = Some(Proc::spawned(broker_child));
assert_eq!(sup.reconcile(&on1, &api, true, at(0)), vec![StartWorker(0)]);
assert!(
wait_until_worker_exited(&mut sup, 0),
"worker slot 0 did not exit in time"
);
assert_eq!(
sup.reconcile(&on1, &api, true, at(100)),
vec![],
"1st failure: 1 s backoff"
);
assert_eq!(
sup.reconcile(&on1, &api, true, at(1200)),
vec![RestartWorker(0)]
);
assert!(
wait_until_worker_exited(&mut sup, 0),
"worker slot 0 did not exit in time"
);
assert_eq!(
sup.reconcile(&on1, &api, true, at(1300)),
vec![],
"2nd failure: 2 s backoff"
);
assert_eq!(
sup.reconcile(&on1, &api, true, at(3400)),
vec![RestartWorker(0)]
);
assert!(
wait_until_worker_exited(&mut sup, 0),
"worker slot 0 did not exit in time"
);
sup.reconcile(&on1, &api, true, at(3500));
let problems = sup.problems(at(3500));
let crash = problems
.iter()
.find(|p| p.code == "worker_crashloop")
.expect("3 failures in 5 min");
assert!(
crash.detail.as_deref().unwrap_or("").contains("boom"),
"detail carries the log tail"
);
}
#[test]
fn a_crashing_worker_that_stabilizes_resets_its_backoff() {
use Action::*;
let dir = tmp("worker-stable-reset");
let api = FakeBroker::default();
api.up.set(true);
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
let on1 = Desired {
sharing: true,
workers: 1,
};
let broker_child =
spawn_in_group(&sh("exec sleep 60"), None, &[], &dir.join("b.log")).unwrap();
let _guard = PgGuard(broker_child.id());
sup.broker = Some(Proc::spawned(broker_child));
assert_eq!(sup.reconcile(&on1, &api, true, at(0)), vec![StartWorker(0)]);
let pid0 = sup.children().workers["0"].pid;
signal_group(pid0, libc::SIGKILL);
assert!(
wait_until_worker_exited(&mut sup, 0),
"worker slot 0 did not exit in time"
);
assert_eq!(
sup.reconcile(&on1, &api, true, at(100)),
vec![],
"1st failure: 1 s backoff"
);
assert_eq!(
sup.reconcile(&on1, &api, true, at(1200)),
vec![RestartWorker(0)]
);
assert_eq!(sup.reconcile(&on1, &api, true, at(1200 + 301_000)), vec![]);
let pid1 = sup.children().workers["0"].pid;
signal_group(pid1, libc::SIGKILL);
assert!(
wait_until_worker_exited(&mut sup, 0),
"worker slot 0 did not exit in time"
);
let t2 = at(1200 + 301_000 + 100);
assert_eq!(
sup.reconcile(&on1, &api, true, t2),
vec![],
"post-reset failure: backoff restarts at 1 s"
);
assert_eq!(
sup.reconcile(&on1, &api, true, t2 + Duration::from_millis(900)),
vec![],
"not ready yet at 0.9 s"
);
assert_eq!(
sup.reconcile(&on1, &api, true, t2 + Duration::from_millis(1100)),
vec![RestartWorker(0)],
"ready at 1.1 s: proves the backoff reset to 1 s, not 2 s"
);
}
#[test]
fn killpg_reaches_the_whole_process_group() {
let dir = tmp("pg");
let log = dir.join("pg.log");
let child = spawn_in_group(&sh("sleep 30 & echo $!; wait"), None, &[], &log).unwrap();
let p = ProcGuard(Proc::spawned(child));
let pid = p.0.pid();
let grandchild = (0..40)
.find_map(|_| {
std::thread::sleep(Duration::from_millis(50));
std::fs::read_to_string(&log)
.ok()?
.trim()
.parse::<u32>()
.ok()
})
.expect("sh printed the sleep pid");
assert!(is_alive(grandchild));
let leftover = terminate(
p.into_inner(),
Duration::from_secs(2),
&AtomicBool::new(false),
);
assert!(
leftover.is_none(),
"an uninterrupted terminate always finishes the group off"
);
assert!(!is_alive(pid));
let gone = wait_until(|| !is_alive(grandchild));
assert!(
gone,
"SIGTERM to the group must reach the grandchild, as python under uv"
);
}
#[test]
fn a_group_member_that_ignores_term_is_sigkilled() {
let dir = tmp("ignore-term");
let log = dir.join("ignore.log");
let child = spawn_in_group(
&sh("sh -c 'trap \"\" TERM; echo $$; exec sleep 30' & wait"),
None,
&[],
&log,
)
.unwrap();
let p = ProcGuard(Proc::spawned(child));
let pid = p.0.pid();
let grandchild = (0..40)
.find_map(|_| {
std::thread::sleep(Duration::from_millis(50));
std::fs::read_to_string(&log)
.ok()?
.trim()
.parse::<u32>()
.ok()
})
.expect("sh printed the sleep pid");
assert!(is_alive(grandchild));
let leftover = terminate(
p.into_inner(),
Duration::from_millis(300),
&AtomicBool::new(false),
);
assert!(
leftover.is_none(),
"an uninterrupted terminate always finishes the group off"
);
assert!(!is_alive(pid));
let gone = wait_until(|| !is_alive(grandchild));
assert!(gone, "SIGKILL must reach a group member that ignores TERM");
}
#[test]
fn a_stop_flag_set_mid_wait_hands_the_group_back_instead_of_sigkilling_it() {
let dir = tmp("interrupt-terminate");
let log = dir.join("ignore.log");
let child = spawn_in_group(
&sh("sh -c 'trap \"\" TERM; echo $$; exec sleep 30' & wait"),
None,
&[],
&log,
)
.unwrap();
let p = ProcGuard(Proc::spawned(child));
let pid = p.0.pid();
let grandchild = (0..40)
.find_map(|_| {
std::thread::sleep(Duration::from_millis(50));
std::fs::read_to_string(&log)
.ok()?
.trim()
.parse::<u32>()
.ok()
})
.expect("sh printed the sleep pid");
assert!(is_alive(grandchild));
let stop = Arc::new(AtomicBool::new(false));
let setter = stop.clone();
std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(50));
setter.store(true, Ordering::Relaxed);
});
let began = Instant::now();
let leftover = terminate(p.into_inner(), Duration::from_secs(30), &stop);
let elapsed = began.elapsed();
let mut leftover = leftover.map(ProcGuard);
assert!(
elapsed < Duration::from_secs(2),
"terminate took {elapsed:?} to notice the stop flag \
(still far below the 30 s grace this must interrupt)"
);
let leftover = leftover
.take()
.expect("an interrupted terminate must hand the group back, not kill it");
assert!(
group_alive(pid),
"an interrupted terminate must not SIGKILL the group \
(the leader itself may already have exited on the plain SIGTERM \
it never trapped; the group member that traps it must survive)"
);
assert!(
is_alive(grandchild),
"an interrupted terminate must not touch a group member either"
);
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exit 0"))));
sup.broker = Some(leftover.into_inner());
sup.terminate_all(Duration::from_secs(2));
assert!(
wait_until(|| !group_alive(pid)),
"terminate_all must still finish off a group terminate() handed back"
);
let gone = wait_until(|| !is_alive(grandchild));
assert!(
gone,
"terminate_all must still finish off a group terminate() handed back"
);
}
#[test]
fn group_alive_and_is_alive_treat_an_unreaped_zombie_as_gone() {
let dir = tmp("zombie-alive");
let log = dir.join("z.log");
let mut child = spawn_in_group(&sh("exec sleep 30"), None, &[], &log).unwrap();
let pid = child.id();
unsafe {
libc::kill(pid as libc::pid_t, libc::SIGKILL);
}
let became_zombie = (0..200).any(|_| {
std::thread::sleep(Duration::from_millis(20));
(unsafe { libc::kill(pid as libc::pid_t, 0) } == 0) && !is_alive(pid)
});
assert!(
became_zombie,
"the killed, unreaped child must become a zombie before we assert on it \
(if this fails, the platform's zombie detection route needs a look)"
);
assert!(
!group_alive(pid),
"a group whose only member is an unreaped zombie must read as gone"
);
assert!(
!is_alive(pid),
"is_alive must already treat a zombie leader as gone"
);
unsafe {
libc::waitpid(pid as libc::pid_t, std::ptr::null_mut(), 0);
}
let _ = child.try_wait();
}
#[test]
fn terminate_all_also_reaches_a_group_member_that_ignores_term() {
use Action::*;
let dir = tmp("ignore-term-all");
let api = FakeBroker::default();
api.up.set(true);
let mut sup = SupGuard(Supervisor::new(spec(
&dir,
sh("sh -c 'trap \"\" TERM; echo $$; exec sleep 30' & wait"),
)));
let on1 = Desired {
sharing: true,
workers: 1,
};
let broker_child =
spawn_in_group(&sh("exec sleep 60"), None, &[], &dir.join("b.log")).unwrap();
let _guard = PgGuard(broker_child.id());
sup.broker = Some(Proc::spawned(broker_child));
assert_eq!(
sup.reconcile(&on1, &api, true, Instant::now()),
vec![StartWorker(0)]
);
let log = sup.worker_log(0);
let grandchild = (0..40)
.find_map(|_| {
std::thread::sleep(Duration::from_millis(50));
std::fs::read_to_string(&log)
.ok()?
.trim()
.parse::<u32>()
.ok()
})
.expect("worker script printed the sleep pid");
assert!(is_alive(grandchild));
sup.terminate_all(Duration::from_millis(300));
let gone = wait_until(|| !is_alive(grandchild));
assert!(
gone,
"terminate_all's SIGKILL must reach a group member that ignores TERM"
);
}
#[test]
fn adoption_needs_a_live_pid_with_the_same_start_time() {
let dir = tmp("adopt");
let mut child = ChildGuard(
spawn_in_group(&sh("exec sleep 30"), None, &[], &dir.join("a.log")).unwrap(),
);
let pid = child.id();
let started = proc_start_time(pid).expect("ps/proc reports a start time");
assert_eq!(
proc_start_time(pid).as_deref(),
Some(started.as_str()),
"stable"
);
let mut recorded = Children::default();
recorded.workers.insert(
"0".into(),
ChildRecord {
pid,
started_at: started.clone(),
port: None,
container: None,
},
);
recorded.workers.insert(
"1".into(),
ChildRecord {
pid,
started_at: "reused pid".into(),
port: None,
container: None,
},
);
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
sup.adopt(&recorded, &proc_start_time);
assert_eq!(
sup.children().workers.keys().cloned().collect::<Vec<_>>(),
vec!["0".to_string()]
);
sup.terminate_all(Duration::from_secs(2));
let _ = child.0.wait();
assert_eq!(proc_start_time(pid), None, "a dead pid has no start time");
}
#[test]
fn logs_are_truncated_past_the_cap_and_tailed() {
let dir = tmp("logs");
std::fs::create_dir_all(&dir).unwrap();
let log = dir.join("w0.log");
std::fs::write(&log, "a\nb\nc\nd\n").unwrap();
assert_eq!(tail_lines(&log, 2), "c\nd");
assert!(!truncate_if_large(&log, 1024).unwrap());
assert!(truncate_if_large(&log, 3).unwrap());
assert_eq!(std::fs::metadata(&log).unwrap().len(), 0);
}
#[test]
fn argv_templates_and_broker_env_follow_the_spec() {
let t: Vec<String> = ["uv", "--port", "{port}", "--worker-name", "{name}"]
.iter()
.map(|s| s.to_string())
.collect();
assert_eq!(
render_argv(&t, 3961, "fp-w1"),
vec!["uv", "--port", "3961", "--worker-name", "fp-w1"]
);
let mut cfg = crate::agent::config::AgentConfig::for_dirs(tmp("spec"));
cfg.max_workers = 5;
let s = SpawnSpec::from_config(&cfg);
assert!(s
.broker_env
.contains(&("ZAKURO_SCAN_RANGE".into(), "3960-3984".into())));
assert!(s
.broker_env
.contains(&("ZAKURO_SCAN_INTERVAL".into(), "3".into())));
assert!(
!s.broker_env.iter().any(|(k, _)| k == "ZAKURO_P2P"),
"ZAKURO_P2P is added at each broker start, from the mesh route of that moment"
);
assert!(
!s.broker_env
.iter()
.any(|(k, _)| k == "ZAKURO_AGENT_CHILD_PORT"),
"the broker's port env is added at spawn time, once the port is chosen"
);
assert_eq!(
&s.broker_argv[1..],
&["broker".to_string(), "{port}".to_string()]
);
assert_eq!(s.preferred_broker_port, 9000);
assert!(s
.worker_env
.contains(&("ZAKURO_WORKER_TYPE".into(), "zakuro".into())));
assert!(s
.worker_env
.contains(&("UV_NO_PROGRESS".into(), "1".into())));
}
#[test]
fn workers_bind_loopback_unless_the_config_picks_a_host_or_caller_auth() {
use crate::agent::config::AgentConfig;
let host = |cfg: &AgentConfig| {
SpawnSpec::from_config(cfg)
.worker_env
.iter()
.rev()
.find(|(k, _)| k == "ZAKURO_HOST")
.map(|(_, v)| v.clone())
};
let plain = AgentConfig::for_dirs(tmp("bind-plain"));
assert_eq!(host(&plain).as_deref(), Some("127.0.0.1"));
let mut insecure = AgentConfig::for_dirs(tmp("bind-insecure"));
insecure
.child_env
.push(("ZAKURO_INSECURE_BIND".into(), "1".into()));
assert_eq!(host(&insecure), None);
let mut chosen = AgentConfig::for_dirs(tmp("bind-host"));
chosen
.child_env
.push(("ZAKURO_HOST".into(), "0.0.0.0".into()));
assert_eq!(host(&chosen).as_deref(), Some("0.0.0.0"));
}
#[test]
fn a_crashing_broker_backs_off_then_restarts() {
use Action::*;
let dir = tmp("brk-backoff");
let api = FakeBroker::default();
let mut s = spec(&dir, sh("exec sleep 60"));
s.broker_argv = sh("echo boom; exit 1");
let mut sup = SupGuard(Supervisor::new(s));
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
let on = Desired {
sharing: true,
workers: 0,
};
assert_eq!(sup.reconcile(&on, &api, true, at(0)), vec![StartBroker]);
assert!(
wait_until_broker_exited(&mut sup),
"the broker did not exit in time"
);
assert_eq!(
sup.reconcile(&on, &api, true, at(100)),
vec![],
"1st failure: 1 s backoff, broker_restart_ready must be false"
);
assert_eq!(
sup.reconcile(&on, &api, true, at(1200)),
vec![StartBroker],
"backoff elapsed: broker_restart_ready is true again"
);
}
#[test]
fn three_broker_crashes_within_five_minutes_raise_a_crashloop_problem() {
use Action::*;
let dir = tmp("brk-crashloop");
let api = FakeBroker::default();
let mut s = spec(&dir, sh("exec sleep 60"));
s.broker_argv = sh("echo boom; exit 1");
let mut sup = SupGuard(Supervisor::new(s));
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
let on = Desired {
sharing: true,
workers: 0,
};
assert_eq!(sup.reconcile(&on, &api, true, at(0)), vec![StartBroker]);
assert!(
wait_until_broker_exited(&mut sup),
"the broker did not exit in time"
);
assert_eq!(
sup.reconcile(&on, &api, true, at(100)),
vec![],
"1st failure: 1 s backoff"
);
assert_eq!(sup.reconcile(&on, &api, true, at(1200)), vec![StartBroker]);
assert!(
wait_until_broker_exited(&mut sup),
"the broker did not exit in time"
);
assert_eq!(
sup.reconcile(&on, &api, true, at(1300)),
vec![],
"2nd failure: 2 s backoff"
);
assert_eq!(sup.reconcile(&on, &api, true, at(3400)), vec![StartBroker]);
assert!(
wait_until_broker_exited(&mut sup),
"the broker did not exit in time"
);
sup.reconcile(&on, &api, true, at(3500));
let problems = sup.problems(at(3500));
let crash = problems
.iter()
.find(|p| p.code == "broker_crashloop")
.expect("3 failures in 5 min");
assert!(
crash.detail.as_deref().unwrap_or("").contains("boom"),
"detail carries the broker log tail"
);
}
#[test]
fn a_crashing_broker_that_stabilizes_resets_its_backoff() {
use Action::*;
let dir = tmp("broker-stable-reset");
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
let on = Desired {
sharing: true,
workers: 0,
};
assert_eq!(sup.reconcile(&on, &api, true, at(0)), vec![StartBroker]);
api.up.set(true);
let pid0 = sup.children().broker.unwrap().pid;
signal_group(pid0, libc::SIGKILL);
assert!(
wait_until_broker_exited(&mut sup),
"the broker did not exit in time"
);
api.up.set(false);
assert_eq!(
sup.reconcile(&on, &api, true, at(100)),
vec![],
"1st failure: 1 s backoff"
);
assert_eq!(sup.reconcile(&on, &api, true, at(1200)), vec![StartBroker]);
api.up.set(true);
assert_eq!(sup.reconcile(&on, &api, true, at(1200 + 301_000)), vec![]);
let pid1 = sup.children().broker.unwrap().pid;
signal_group(pid1, libc::SIGKILL);
assert!(
wait_until_broker_exited(&mut sup),
"the broker did not exit in time"
);
api.up.set(false);
let t2 = at(1200 + 301_000 + 100);
assert_eq!(
sup.reconcile(&on, &api, true, t2),
vec![],
"post-reset failure: backoff restarts at 1 s"
);
assert_eq!(
sup.reconcile(&on, &api, true, t2 + Duration::from_millis(900)),
vec![],
"not ready yet at 0.9 s"
);
assert_eq!(
sup.reconcile(&on, &api, true, t2 + Duration::from_millis(1100)),
vec![StartBroker],
"ready at 1.1 s: proves the backoff reset to 1 s, not 2 s"
);
}
#[test]
fn a_deliberate_broker_stop_is_not_counted_as_a_crash_failure() {
use Action::*;
let dir = tmp("brk-stop");
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let t = Instant::now();
let on = Desired {
sharing: true,
workers: 0,
};
let off = Desired {
sharing: false,
workers: 0,
};
assert_eq!(sup.reconcile(&on, &api, true, t), vec![StartBroker]);
api.up.set(true);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![StopBroker]);
api.up.set(false); assert!(sup.children().broker.is_none());
let problems = sup.problems(t);
assert!(
!problems.iter().any(|p| p.code == "broker_crashloop"),
"a clean stop is not a failure"
);
assert_eq!(sup.reconcile(&on, &api, true, t), vec![StartBroker]);
}
fn stub_http_once(status: u16, body: &'static str) -> u16 {
use std::io::{Read, Write};
use std::net::TcpListener;
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
std::thread::spawn(move || {
if let Ok((mut s, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = s.read(&mut buf);
let _ = write!(
s,
"HTTP/1.1 {status} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
}
});
port
}
#[test]
fn workers_over_http_errs_on_a_non_2xx_response() {
let port = stub_http_once(500, r#"{"workers":[]}"#);
let api = HttpBrokerApi {
port,
worker_key: None,
};
assert!(
api.workers().is_err(),
"a non-2xx GET /workers must be an error, not an empty list"
);
}
#[test]
fn workers_over_http_parses_a_valid_2xx_response() {
let body = r#"{"workers":[{"id":"id0","name":"fp-w0","status":"healthy","active_requests":1,"requests_5h":2,"requests_1w":3}]}"#;
let port = stub_http_once(200, body);
let api = HttpBrokerApi {
port,
worker_key: None,
};
let list = api.workers().expect("a 2xx body should parse");
assert_eq!(list.len(), 1);
assert_eq!(list[0].id, "id0");
assert_eq!(list[0].active_requests, 1);
}
#[test]
fn a_workers_fetch_error_keeps_a_zero_reading_drain_from_completing() {
use Action::*;
let dir = tmp("stale-workers-zero");
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let t = Instant::now();
let on1 = Desired {
sharing: true,
workers: 1,
};
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![StartBroker]);
api.up.set(true);
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![StartWorker(0)]);
*api.workers.borrow_mut() = vec![bw("id0", "testfp00-w0")]; assert_eq!(sup.reconcile(&on1, &api, true, t), vec![]);
let off = Desired {
sharing: false,
workers: 1,
};
assert_eq!(sup.reconcile(&off, &api, true, t), vec![DrainWorker(0)]);
assert_eq!(*api.drained.borrow(), vec!["id0".to_string()]);
api.workers_err.set(true);
assert_eq!(
sup.reconcile(&off, &api, true, t),
vec![],
"an unknown observation must not read a zero-reading drain as idle"
);
assert_eq!(
sup.broker_id(0).as_deref(),
Some("id0"),
"the retained id must survive the fetch-error tick"
);
api.workers_err.set(false);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![KillWorker(0)]);
assert_eq!(*api.deleted.borrow(), vec!["id0".to_string()]);
}
#[test]
fn a_health_check_timeout_keeps_a_busy_drain_from_completing() {
use Action::*;
let dir = tmp("stale-workers-health-timeout");
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let t = Instant::now();
let on1 = Desired {
sharing: true,
workers: 1,
};
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![StartBroker]);
api.up.set(true);
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![StartWorker(0)]);
*api.workers.borrow_mut() = vec![bw_n("id0", "testfp00-w0", 2)];
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![]);
let off = Desired {
sharing: false,
workers: 1,
};
assert_eq!(sup.reconcile(&off, &api, true, t), vec![DrainWorker(0)]);
assert_eq!(*api.drained.borrow(), vec!["id0".to_string()]);
api.health_override.set(Some(false));
assert_eq!(
sup.reconcile(&off, &api, true, t),
vec![],
"a health-check timeout must not read the drain as idle"
);
assert_eq!(
sup.broker_id(0).as_deref(),
Some("id0"),
"the retained id must survive the Starting tick"
);
api.health_override.set(None);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![]);
*api.workers.borrow_mut() = vec![bw("id0", "testfp00-w0")];
assert_eq!(sup.reconcile(&off, &api, true, t), vec![KillWorker(0)]);
assert_eq!(*api.deleted.borrow(), vec!["id0".to_string()]);
}
#[test]
fn killing_a_worker_during_an_error_streak_forgets_its_retained_entry() {
use Action::*;
let dir = tmp("kill-forgets-entry");
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let t = Instant::now();
let on1 = Desired {
sharing: true,
workers: 1,
};
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![StartBroker]);
api.up.set(true);
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![StartWorker(0)]);
*api.workers.borrow_mut() = vec![bw_n("dead-id", "testfp00-w0", 9)];
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![]);
let scale0 = Desired {
sharing: true,
workers: 0,
};
assert_eq!(sup.reconcile(&scale0, &api, true, t), vec![DrainWorker(0)]);
assert_eq!(*api.drained.borrow(), vec!["dead-id".to_string()]);
api.workers_err.set(true);
let pid0 = sup.children().workers["0"].pid;
signal_group(pid0, libc::SIGKILL);
assert!(wait_until_worker_exited(&mut sup, 0));
assert_eq!(sup.reconcile(&scale0, &api, true, t), vec![KillWorker(0)]);
assert_eq!(*api.deleted.borrow(), vec!["dead-id".to_string()]);
assert_eq!(
sup.broker_id(0),
None,
"the dead worker's id must not linger"
);
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![StartWorker(0)]);
let view = sup.local_view(&on1, 5);
assert_eq!(
view.requests,
Requests {
last_5h: 0,
last_1w: 0
},
"no stale stats for the new worker"
);
assert_eq!(view.workers.busy, 0, "no stale busy count");
assert_eq!(
sup.broker_id(0),
None,
"no stale id for the new worker either"
);
assert_eq!(sup.reconcile(&scale0, &api, true, t), vec![DrainWorker(0)]);
assert_eq!(
*api.drained.borrow(),
vec!["dead-id".to_string()],
"no id to drain yet: the dead id must not be reused for the new worker"
);
}
#[test]
fn a_drain_is_resent_until_the_broker_reports_the_worker_draining() {
use Action::*;
let dir = tmp("drain-resend");
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let t = Instant::now();
let on1 = Desired {
sharing: true,
workers: 1,
};
let off = Desired {
sharing: false,
workers: 1,
};
let drained = || api.drained.borrow().clone();
let with_status = |id: &str, status: &str| BrokerWorker {
status: status.into(),
..bw_n(id, "testfp00-w0", 1)
};
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![StartBroker]);
api.up.set(true);
assert_eq!(sup.reconcile(&on1, &api, true, t), vec![StartWorker(0)]);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![DrainWorker(0)]);
assert!(drained().is_empty());
*api.workers.borrow_mut() = vec![with_status("id0", "healthy")];
api.drain_fails.set(true);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![]);
assert_eq!(drained(), vec!["id0".to_string()], "sent once it has an id");
api.drain_fails.set(false);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![]);
assert_eq!(drained().len(), 2, "re-sent after the failed POST");
*api.workers.borrow_mut() = vec![with_status("id0", "draining")];
assert_eq!(sup.reconcile(&off, &api, true, t), vec![]);
assert_eq!(drained().len(), 2);
*api.workers.borrow_mut() = vec![with_status("id0-b", "healthy")];
assert_eq!(sup.reconcile(&off, &api, true, t), vec![]);
assert_eq!(
drained().last().map(String::as_str),
Some("id0-b"),
"re-sent after a broker restart"
);
let before = drained().len();
api.workers_err.set(true);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![]);
assert_eq!(drained().len(), before);
}
#[test]
fn a_worker_leader_that_exits_early_has_its_surviving_group_member_killed() {
use Action::*;
let dir = tmp("leader-exits-early");
let api = FakeBroker::default();
api.up.set(true);
let mut sup = SupGuard(Supervisor::new(spec(
&dir,
sh("sleep 30 & echo $!; exit 0"),
)));
let on1 = Desired {
sharing: true,
workers: 1,
};
let broker_child =
spawn_in_group(&sh("exec sleep 60"), None, &[], &dir.join("b.log")).unwrap();
let _guard = PgGuard(broker_child.id());
sup.broker = Some(Proc::spawned(broker_child));
assert_eq!(
sup.reconcile(&on1, &api, true, Instant::now()),
vec![StartWorker(0)]
);
let pid0 = sup.children().workers["0"].pid; let _pg_guard = PgGuard(pid0);
let log = sup.worker_log(0);
let member = (0..40)
.find_map(|_| {
std::thread::sleep(Duration::from_millis(50));
std::fs::read_to_string(&log)
.ok()?
.trim()
.parse::<u32>()
.ok()
})
.expect("the worker script printed the backgrounded sleep's pid");
assert!(
wait_until_worker_exited(&mut sup, 0),
"the leader exits on its own almost immediately"
);
assert!(
is_alive(member),
"the backgrounded `sleep 30` must still be alive right after the leader exits"
);
sup.reconcile(&on1, &api, true, Instant::now());
assert!(
wait_until(|| !is_alive(member)),
"the surviving group member must be killed once the leader is found dead"
);
}
#[test]
fn port_free_is_false_when_something_holds_the_wildcard_address() {
let holder = std::net::TcpListener::bind(("0.0.0.0", 0)).unwrap();
let port = holder.local_addr().unwrap().port();
let api = HttpBrokerApi {
port: 0,
worker_key: None,
};
assert!(
!api.port_free(port),
"a *:{port} holder must not read as free"
);
drop(holder);
}
mod choose_broker_port_tests {
use super::choose_broker_port;
#[test]
fn preferred_free() {
assert_eq!(
choose_broker_port(9000, 9001, 9010, |p| p == 9000, || None),
Some(9000)
);
}
#[test]
fn preferred_held_falls_back_to_9001() {
assert_eq!(
choose_broker_port(9000, 9001, 9010, |p| p == 9001, || None),
Some(9001)
);
}
#[test]
fn preferred_and_9001_through_9003_held_falls_back_to_9004() {
assert_eq!(
choose_broker_port(9000, 9001, 9010, |p| p == 9004, || None),
Some(9004)
);
}
#[test]
fn the_whole_range_held_falls_back_to_an_os_pick() {
assert_eq!(
choose_broker_port(9000, 9001, 9010, |_| false, || Some(54321)),
Some(54321)
);
}
#[test]
fn nothing_available_is_none() {
assert_eq!(
choose_broker_port(9000, 9001, 9010, |_| false, || None),
None
);
}
}
#[test]
fn a_foreign_broker_on_the_preferred_port_is_unmanaged() {
let dir = tmp("foreign-preferred");
let api = FakeBroker::default();
api.foreign_broker_port.set(Some(9000));
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let on = Desired {
sharing: true,
workers: 1,
};
assert_eq!(sup.reconcile(&on, &api, true, Instant::now()), vec![]);
assert!(sup.broker_unmanaged());
assert_eq!(sup.broker_port(), 9000, "never relocated: never touched");
}
#[test]
fn a_blocked_preferred_port_falls_back_and_renders_the_chosen_port() {
use Action::*;
let dir = tmp("fallback-9001");
let api = FakeBroker::default();
api.blocked_ports.borrow_mut().insert(9000);
let mut spawn_spec = spec(&dir, sh("exec sleep 60"));
spawn_spec.broker_argv = vec!["sh".into(), "-c".into(), "exec sleep 60 {port}".into()];
let mut sup = SupGuard(Supervisor::new(spawn_spec));
let on = Desired {
sharing: true,
workers: 0,
};
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker]
);
assert_eq!(sup.broker_port(), 9001);
assert_eq!(sup.children().broker.unwrap().port, Some(9001));
}
#[test]
fn an_idle_tick_never_probes_or_binds_a_fallback_port() {
let dir = tmp("fallback-while-paused");
let api = FakeBroker::default();
api.blocked_ports.borrow_mut().insert(9000); let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let off = Desired {
sharing: false,
workers: 0,
};
assert_eq!(sup.reconcile(&off, &api, true, Instant::now()), vec![]);
assert_eq!(
*api.port_free_calls.borrow(),
0,
"an idle tick must never probe a fallback port"
);
assert_eq!(
*api.os_assigned_port_calls.borrow(),
0,
"an idle tick must never bind an OS-assigned port either"
);
assert_eq!(
sup.broker_port(),
9000,
"nothing was chosen, so it stays at the preferred port"
);
}
#[test]
fn an_idle_tick_still_detects_an_unmanaged_broker_on_the_preferred_port() {
let dir = tmp("unmanaged-while-paused");
let api = FakeBroker::default();
api.foreign_broker_port.set(Some(9000));
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let off = Desired {
sharing: false,
workers: 0,
};
sup.reconcile(&off, &api, true, Instant::now());
assert!(sup.broker_unmanaged());
}
#[test]
fn turning_sharing_on_probes_and_starts_on_the_very_next_tick() {
use Action::*;
let dir = tmp("fallback-then-share-on");
let api = FakeBroker::default();
api.blocked_ports.borrow_mut().insert(9000);
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let off = Desired {
sharing: false,
workers: 0,
};
sup.reconcile(&off, &api, true, Instant::now());
assert_eq!(*api.port_free_calls.borrow(), 0);
let on = Desired {
sharing: true,
workers: 0,
};
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker]
);
assert!(*api.port_free_calls.borrow() > 0);
assert_eq!(sup.broker_port(), 9001);
}
#[test]
fn everything_blocked_and_no_os_pick_is_port_in_use() {
let dir = tmp("all-blocked");
let api = FakeBroker::default();
for p in 9000..=9010 {
api.blocked_ports.borrow_mut().insert(p);
}
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let on = Desired {
sharing: true,
workers: 0,
};
sup.reconcile(&on, &api, true, Instant::now());
let problems = sup.problems(Instant::now());
assert!(problems.iter().any(|p| p.code == "port_in_use"));
}
#[test]
fn everything_blocked_falls_back_to_an_os_assigned_port() {
use Action::*;
let dir = tmp("os-pick");
let api = FakeBroker::default();
for p in 9000..=9010 {
api.blocked_ports.borrow_mut().insert(p);
}
api.os_pick.set(Some(54321));
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let on = Desired {
sharing: true,
workers: 0,
};
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker]
);
assert_eq!(sup.broker_port(), 54321);
}
#[test]
fn adopt_restores_the_brokers_recorded_port_defaulting_to_9000() {
let dir = tmp("adopt-port");
let broker_child =
spawn_in_group(&sh("exec sleep 60"), None, &[], &dir.join("b.log")).unwrap();
let _guard = PgGuard(broker_child.id());
let pid = broker_child.id();
let started = proc_start_time(pid).unwrap_or_default();
let child_guard = ChildGuard(broker_child);
let recorded = Children {
broker: Some(ChildRecord {
pid,
started_at: started.clone(),
port: Some(9007),
container: None,
}),
..Default::default()
};
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
sup.adopt(&recorded, &proc_start_time);
assert_eq!(sup.broker_port(), 9007);
let recorded_no_port = Children {
broker: Some(ChildRecord {
pid,
started_at: started,
port: None,
container: None,
}),
..Default::default()
};
let mut sup2 = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
sup2.adopt(&recorded_no_port, &proc_start_time);
assert_eq!(sup2.broker_port(), 9000, "a missing port field means 9000");
drop(child_guard);
}
#[test]
fn set_worker_cwd_changes_where_the_next_spawned_worker_runs() {
use Action::*;
let dir = tmp("worker-cwd");
let cwd_a = dir.join("cwd-a");
let cwd_b = dir.join("cwd-b");
std::fs::create_dir_all(&cwd_a).unwrap();
std::fs::create_dir_all(&cwd_b).unwrap();
let api = FakeBroker::default();
api.up.set(true);
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("pwd"))));
let broker_child =
spawn_in_group(&sh("exec sleep 60"), None, &[], &dir.join("b.log")).unwrap();
let _guard = PgGuard(broker_child.id());
sup.broker = Some(Proc::spawned(broker_child));
sup.set_worker_cwd(Some(cwd_a.clone()));
assert_eq!(
sup.reconcile(
&Desired {
sharing: true,
workers: 1
},
&api,
true,
Instant::now()
),
vec![StartWorker(0)]
);
sup.set_worker_cwd(Some(cwd_b.clone()));
assert_eq!(
sup.reconcile(
&Desired {
sharing: true,
workers: 2
},
&api,
true,
Instant::now()
),
vec![StartWorker(1)]
);
assert!(wait_until_worker_exited(&mut sup, 0));
assert!(wait_until_worker_exited(&mut sup, 1));
let printed_a = std::fs::read_to_string(sup.worker_log(0)).unwrap();
let printed_b = std::fs::read_to_string(sup.worker_log(1)).unwrap();
assert_eq!(
printed_a.trim(),
cwd_a.canonicalize().unwrap().to_str().unwrap()
);
assert_eq!(
printed_b.trim(),
cwd_b.canonicalize().unwrap().to_str().unwrap()
);
}
const PRINTS_ZAKURO_P2P: &str = r#"echo "ZAKURO_P2P=${ZAKURO_P2P-unset}"; exec sleep 60"#;
fn broker_log_lines(sup: &Supervisor, n: usize) -> Vec<String> {
let log = sup.broker_log();
let text = || std::fs::read_to_string(&log).unwrap_or_default();
assert!(
wait_until(|| text().matches('\n').count() >= n),
"the broker should have printed {n} line(s)"
);
text().lines().map(str::to_string).collect()
}
fn p2p_of_a_broker_started_on(route: RouteKind, s: SpawnSpec) -> String {
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(s));
sup.set_route(route);
let on = Desired {
sharing: true,
workers: 0,
};
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![Action::StartBroker]
);
broker_log_lines(&sup, 1).remove(0)
}
fn p2p_printing_spec(tag: &str) -> SpawnSpec {
let mut s = spec(&tmp(tag), sh("exec sleep 60"));
s.broker_argv = sh(PRINTS_ZAKURO_P2P);
s
}
#[test]
fn a_started_broker_runs_p2p_on_either_mesh_route_only() {
for (route, want) in [
(RouteKind::Proxy, "ZAKURO_P2P=true"),
(RouteKind::Host, "ZAKURO_P2P=true"),
(RouteKind::None, "ZAKURO_P2P=false"),
] {
let tag = format!("p2p-{}", route.label());
assert_eq!(
p2p_of_a_broker_started_on(route, p2p_printing_spec(&tag)),
want,
"{route:?}"
);
}
}
#[test]
fn an_explicit_zakuro_p2p_in_child_env_wins_over_the_route() {
for (route, explicit) in [(RouteKind::Proxy, "false"), (RouteKind::None, "true")] {
let mut cfg = crate::agent::config::AgentConfig::for_dirs(tmp("p2p-explicit"));
cfg.broker_argv = sh(PRINTS_ZAKURO_P2P);
cfg.child_env = vec![("ZAKURO_P2P".to_string(), explicit.to_string())];
assert_eq!(
p2p_of_a_broker_started_on(route, SpawnSpec::from_config(&cfg)),
format!("ZAKURO_P2P={explicit}"),
"{route:?}"
);
}
}
#[test]
fn each_broker_start_takes_zakuro_p2p_from_the_route_of_that_moment() {
use Action::*;
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(p2p_printing_spec("p2p-each-start")));
let on = Desired {
sharing: true,
workers: 0,
};
let off = Desired {
sharing: false,
workers: 0,
};
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker]
);
assert_eq!(broker_log_lines(&sup, 1), ["ZAKURO_P2P=false"]);
assert_eq!(
sup.reconcile(&off, &api, true, Instant::now()),
vec![StopBroker]
);
sup.set_route(RouteKind::Host);
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker]
);
assert_eq!(
broker_log_lines(&sup, 2),
["ZAKURO_P2P=false", "ZAKURO_P2P=true"]
);
}
#[test]
fn on_the_proxy_route_the_last_relayed_port_is_still_used() {
use Action::*;
let dir = tmp("proxy-range-end");
let api = FakeBroker::default();
for p in 9000..=9009 {
api.blocked_ports.borrow_mut().insert(p);
}
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
sup.set_route(RouteKind::Proxy);
let on = Desired {
sharing: true,
workers: 0,
};
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker]
);
assert_eq!(sup.broker_port(), 9010);
}
#[test]
fn on_the_proxy_route_a_full_relay_range_is_broker_port_unrelayable() {
let dir = tmp("proxy-range-full");
let api = FakeBroker::default();
for p in 9000..=9010 {
api.blocked_ports.borrow_mut().insert(p);
}
api.os_pick.set(Some(54321));
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
sup.set_route(RouteKind::Proxy);
let on = Desired {
sharing: true,
workers: 0,
};
let now = Instant::now();
assert_eq!(sup.reconcile(&on, &api, true, now), vec![]);
assert_eq!(
*api.os_assigned_port_calls.borrow(),
0,
"no OS-assigned port on the proxy route"
);
let codes: Vec<String> = sup.problems(now).into_iter().map(|p| p.code).collect();
assert_eq!(codes, vec!["broker_port_unrelayable".to_string()]);
}
#[test]
fn the_broker_port_range_is_the_relay_range() {
assert_eq!(
crate::agent::config::DEFAULT_BROKER_PORT,
crate::vpn::RELAY_FIRST_PORT
);
assert_eq!(BROKER_FALLBACK_START, crate::vpn::RELAY_FIRST_PORT + 1);
assert_eq!(BROKER_FALLBACK_END, crate::vpn::RELAY_LAST_PORT);
}
#[test]
fn on_the_proxy_route_an_unrelayed_preferred_port_is_skipped() {
use Action::*;
let on = Desired {
sharing: true,
workers: 0,
};
for route in [RouteKind::Proxy, RouteKind::Host, RouteKind::None] {
let api = FakeBroker::default();
let mut s = spec(&tmp("unrelayed-preferred"), sh("exec sleep 60"));
s.preferred_broker_port = 4000;
let mut sup = SupGuard(Supervisor::new(s));
sup.set_route(route);
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker],
"{route:?}"
);
let port = sup.broker_port();
if route == RouteKind::Proxy {
assert!(crate::vpn::relay_covers(port), "{port} on {route:?}");
} else {
assert_eq!(port, 4000, "{route:?}");
}
}
}
#[test]
fn the_agents_own_zakuro_p2p_false_wins_on_the_proxy_route() {
let mut cfg = crate::agent::config::AgentConfig::from_env_with(|k| {
(k == "ZAKURO_P2P").then(|| "false".to_string())
})
.unwrap();
cfg.state_dir = tmp("p2p-agent-env");
cfg.dir = cfg.state_dir.join("agent");
cfg.broker_argv = sh(PRINTS_ZAKURO_P2P);
assert_eq!(
p2p_of_a_broker_started_on(RouteKind::Proxy, SpawnSpec::from_config(&cfg)),
"ZAKURO_P2P=false"
);
}
#[test]
fn route_kind_changes_restart_the_broker_address_changes_dont() {
use Action::*;
for (from, to, restarts) in [
(RouteKind::None, RouteKind::Host, true),
(RouteKind::Host, RouteKind::None, true),
(RouteKind::Host, RouteKind::Proxy, true),
(RouteKind::Proxy, RouteKind::Host, true),
(RouteKind::Proxy, RouteKind::None, true),
(RouteKind::Host, RouteKind::Host, false),
(RouteKind::Proxy, RouteKind::Proxy, false),
] {
let tag = format!("route-kind-{}-{}", from.label(), to.label());
let dir = tmp(&tag);
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let on = Desired {
sharing: true,
workers: 0,
};
sup.set_route(from);
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker],
"{tag}"
);
api.up.set(true);
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![],
"{tag}"
);
sup.set_route(to);
let want = if restarts { vec![StopBroker] } else { vec![] };
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
want,
"{tag}"
);
}
}
#[test]
fn a_route_change_onto_the_proxy_route_restarts_the_broker() {
use Action::*;
let dir = tmp("route-change-proxy");
let api = FakeBroker::default();
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
let on = Desired {
sharing: true,
workers: 0,
};
sup.set_route(RouteKind::Host);
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker]
);
api.up.set(true);
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![],
"running on the host route"
);
sup.set_route(RouteKind::Proxy);
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StopBroker]
);
api.up.set(false);
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker],
"started again, on the proxy route"
);
assert!(crate::vpn::relay_covers(sup.broker_port()));
}
#[test]
fn an_adopted_broker_on_an_unrelayed_port_restarts_on_the_proxy_route() {
use Action::*;
let dir = tmp("adopt-unrelayed");
let broker_child =
spawn_in_group(&sh("exec sleep 60"), None, &[], &dir.join("b.log")).unwrap();
let _guard = PgGuard(broker_child.id());
let pid = broker_child.id();
let started = proc_start_time(pid).unwrap_or_default();
let child_guard = ChildGuard(broker_child);
let recorded = Children {
broker: Some(ChildRecord {
pid,
started_at: started,
port: Some(54321),
container: None,
}),
..Default::default()
};
let api = FakeBroker::default();
api.up.set(true);
let mut sup = SupGuard(Supervisor::new(spec(&dir, sh("exec sleep 60"))));
sup.adopt(&recorded, &proc_start_time);
assert_eq!(sup.broker_port(), 54321);
sup.set_route(RouteKind::Proxy);
let on = Desired {
sharing: true,
workers: 0,
};
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StopBroker],
"the adopted port isn't relayed"
);
api.up.set(false);
assert_eq!(
sup.reconcile(&on, &api, true, Instant::now()),
vec![StartBroker]
);
assert!(crate::vpn::relay_covers(sup.broker_port()));
drop(child_guard);
}
#[derive(Default)]
struct FakeDocker {
calls: std::sync::Mutex<Vec<Vec<String>>>,
state: std::sync::Mutex<Option<String>>,
worker_alive: std::sync::Mutex<bool>,
log: std::sync::Mutex<String>,
image_log: std::sync::Mutex<String>,
term_stops: std::sync::Mutex<bool>,
run_fails: std::sync::Mutex<bool>,
}
impl FakeDocker {
fn healthy() -> Self {
let d = Self::default();
*d.state.lock().unwrap() = Some("true|2026-09-18T07:00:00Z|0".into());
*d.worker_alive.lock().unwrap() = true;
*d.term_stops.lock().unwrap() = true;
*d.log.lock().unwrap() = "worker log line\n".into();
*d.image_log.lock().unwrap() = "s6-rc: service legacy-services started\n".into();
d
}
fn stop(&self) {
*self.state.lock().unwrap() = Some("false|2026-09-18T07:00:00Z|137".into());
*self.worker_alive.lock().unwrap() = false;
}
fn calls(&self) -> Vec<String> {
self.calls
.lock()
.unwrap()
.iter()
.map(|a| a.join(" "))
.collect()
}
fn ran(&self, needle: &str) -> bool {
self.calls().iter().any(|c| c.contains(needle))
}
fn count(&self, needle: &str) -> usize {
self.calls().iter().filter(|c| c.contains(needle)).count()
}
}
impl DockerCli for FakeDocker {
fn run(&self, argv: &[String]) -> Result<String, String> {
self.calls.lock().unwrap().push(argv.to_vec());
let joined = argv.join(" ");
let rest: Vec<&str> = argv[1..].iter().map(String::as_str).collect();
match rest.first().copied() {
Some("inspect") => self
.state
.lock()
.unwrap()
.clone()
.ok_or_else(|| "Error: No such object".to_string()),
Some("network") => Ok(String::new()),
Some("rm") => {
*self.state.lock().unwrap() = None;
*self.worker_alive.lock().unwrap() = false;
Ok(String::new())
}
Some("run") => {
if *self.run_fails.lock().unwrap() {
return Err("Unable to find image".into());
}
*self.state.lock().unwrap() = Some("true|2026-09-18T07:00:00Z|0".into());
Ok("deadbeef".into())
}
Some("logs") => Ok(self.image_log.lock().unwrap().clone()),
Some("kill") => {
let sig = rest.get(2).copied().unwrap_or("");
if self.state.lock().unwrap().is_none() {
return Err("No such container".into());
}
if sig == "KILL" || *self.term_stops.lock().unwrap() {
self.stop();
}
Ok(String::new())
}
Some("exec") if joined.contains("-d ") => {
*self.worker_alive.lock().unwrap() = true;
Ok(String::new())
}
Some("exec")
if !self
.state
.lock()
.unwrap()
.as_deref()
.is_some_and(|s| s.starts_with("true")) =>
{
Err("Container is not running".into())
}
Some("exec") if joined.contains("kill -0") => {
if *self.worker_alive.lock().unwrap() {
Ok(String::new())
} else {
Err("exit status 1".into())
}
}
Some("exec") if joined.contains("tail -n") => Ok(self.log.lock().unwrap().clone()),
_ => Ok(String::new()),
}
}
}
#[derive(Default)]
struct SweepDocker {
ps: String,
rm_fails: Vec<String>,
fail_all: bool,
calls: std::sync::Mutex<Vec<String>>,
}
impl DockerCli for SweepDocker {
fn run(&self, argv: &[String]) -> Result<String, String> {
let joined = argv.join(" ");
self.calls.lock().unwrap().push(joined.clone());
if self.fail_all {
return Err("docker not available: no such file".into());
}
match argv.get(1).map(String::as_str) {
Some("ps") => Ok(self.ps.clone()),
Some("inspect") => Ok("true|2026-09-18T07:00:00Z|0".into()),
Some("rm") if self.rm_fails.iter().any(|id| joined.ends_with(id.as_str())) => {
Err("removal in progress".into())
}
_ => Ok(String::new()),
}
}
}
fn removed_ids(d: &SweepDocker) -> Vec<String> {
d.calls
.lock()
.unwrap()
.iter()
.filter(|c| c.starts_with("docker rm -f "))
.map(|c| c.trim_start_matches("docker rm -f ").to_string())
.collect()
}
fn sweeper(dir: &Path, d: Arc<SweepDocker>, adopt: &[&str]) -> Supervisor {
let mut sup = Supervisor::new(container_spec(dir));
sup.set_docker(d);
let mut c = Children::default();
for (i, name) in adopt.iter().enumerate() {
c.workers.insert(
i.to_string(),
ChildRecord {
pid: 0,
started_at: "2026-09-18T07:00:00Z".into(),
port: None,
container: Some((*name).into()),
},
);
}
sup.adopt(&c, &proc_start_time);
sup
}
#[test]
fn sweep_removes_an_orphaned_share_container() {
let dir = tmp("sweep-orphan");
let d = Arc::new(SweepDocker {
ps: "aaaa1111bbbb2222cccc\tzc-share-old-w0\n".into(),
..Default::default()
});
let sup = sweeper(&dir, d.clone(), &[]);
assert_eq!(sup.sweep_orphans(), vec!["zc-share-old-w0".to_string()]);
assert_eq!(removed_ids(&d), vec!["aaaa1111bbbb2222cccc".to_string()]);
}
#[test]
fn sweep_keeps_an_adopted_container() {
let dir = tmp("sweep-adopted");
let d = Arc::new(SweepDocker {
ps: "id-kept\tzc-share-testfp00-w0\nid-orphan\tzc-share-testfp00-w1\n".into(),
..Default::default()
});
let sup = sweeper(&dir, d.clone(), &["zc-share-testfp00-w0"]);
assert_eq!(sup.children().workers.len(), 1);
assert_eq!(
sup.sweep_orphans(),
vec!["zc-share-testfp00-w1".to_string()]
);
assert_eq!(removed_ids(&d), vec!["id-orphan".to_string()]);
}
#[test]
fn sweep_only_lists_by_the_share_label_so_unlabelled_containers_are_untouched() {
let dir = tmp("sweep-label");
let d = Arc::new(SweepDocker::default());
let sup = sweeper(&dir, d.clone(), &[]);
assert!(sup.sweep_orphans().is_empty());
assert!(removed_ids(&d).is_empty());
let calls = d.calls.lock().unwrap().clone();
let ps = calls.iter().find(|c| c.starts_with("docker ps")).unwrap();
assert!(ps.contains("label=zakuro.share=1"), "{ps}");
}
#[test]
fn sweep_is_skipped_without_docker() {
let dir = tmp("sweep-nodocker");
let d = Arc::new(SweepDocker {
fail_all: true,
..Default::default()
});
let sup = sweeper(&dir, d.clone(), &[]);
assert!(sup.sweep_orphans().is_empty());
assert!(removed_ids(&d).is_empty());
}
#[test]
fn a_failed_removal_does_not_stop_the_sweep() {
let dir = tmp("sweep-rmfail");
let d = Arc::new(SweepDocker {
ps: "id-a\tzc-share-x-w0\nid-b\tzc-share-x-w1\n".into(),
rm_fails: vec!["id-a".into()],
..Default::default()
});
let sup = sweeper(&dir, d.clone(), &[]);
assert_eq!(sup.sweep_orphans(), vec!["zc-share-x-w1".to_string()]);
assert_eq!(removed_ids(&d).len(), 2);
}
fn container_spec(dir: &Path) -> SpawnSpec {
let mut s = spec(dir, vec![]);
s.worker_backend = WorkerBackend::Container {
image: "zakuro-image-compute:latest".into(),
};
s.kill_grace = Duration::from_millis(400);
s
}
fn container_sup(dir: &Path, docker: Arc<FakeDocker>) -> (SupGuard, PgGuard) {
let mut sup = SupGuard(Supervisor::new(container_spec(dir)));
sup.set_docker(docker);
let broker_child =
spawn_in_group(&sh("exec sleep 60"), None, &[], &dir.join("b.log")).unwrap();
let guard = PgGuard(broker_child.id());
sup.broker = Some(Proc::spawned(broker_child));
(sup, guard)
}
const ON1: Desired = Desired {
sharing: true,
workers: 1,
};
#[test]
fn a_container_backed_worker_starts_from_the_argv_zc_share_already_uses() {
let dir = tmp("ctr-start");
let docker = Arc::new(FakeDocker::healthy());
let mut sup = SupGuard(Supervisor::new(container_spec(&dir)));
sup.set_docker(docker.clone());
sup.spawn_worker(0, Instant::now()).unwrap();
let calls = docker.calls();
assert_eq!(
calls,
vec![
"docker rm -f zc-share-testfp00-w0".to_string(),
"docker network create zc-worker-testfp00".to_string(),
crate::up::docker_run_argv(
3960,
"zakuro-image-compute:latest",
"zc-share-testfp00-w0",
"zc-worker-testfp00",
"testfp00-w0"
)
.join(" "),
crate::up::worker_exec_argv("zc-share-testfp00-w0", 3960, "testfp00-w0", None)
.join(" "),
crate::up::docker_state_argv("zc-share-testfp00-w0").join(" "),
]
);
assert_eq!(
sup.children().workers["0"],
ChildRecord {
pid: 0,
started_at: "2026-09-18T07:00:00Z".into(),
port: None,
container: Some("zc-share-testfp00-w0".into()),
}
);
}
#[test]
fn a_container_that_cannot_start_leaves_no_slot_and_no_container() {
let dir = tmp("ctr-start-fail");
let docker = Arc::new(FakeDocker::healthy());
*docker.run_fails.lock().unwrap() = true;
let mut sup = SupGuard(Supervisor::new(container_spec(&dir)));
sup.set_docker(docker.clone());
assert!(sup.spawn_worker(0, Instant::now()).is_err());
assert!(sup.children().workers.is_empty());
assert!(!docker.ran("docker exec -d"), "{:?}", docker.calls());
}
#[test]
fn a_worker_that_dies_inside_a_still_running_container_is_a_crash() {
use Action::*;
let dir = tmp("ctr-crash");
let docker = Arc::new(FakeDocker::healthy());
let (mut sup, _pg) = container_sup(&dir, docker.clone());
let api = FakeBroker::default();
api.up.set(true);
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
assert_eq!(sup.reconcile(&ON1, &api, true, at(0)), vec![StartWorker(0)]);
assert_eq!(
sup.reconcile(&ON1, &api, true, at(100)),
vec![],
"the worker is alive inside the container"
);
*docker.worker_alive.lock().unwrap() = false;
let removals_before = docker.count("docker rm -f");
assert_eq!(
sup.reconcile(&ON1, &api, true, at(5200)),
vec![],
"1st failure: 1 s backoff, not an instant restart"
);
assert_eq!(
docker.count("docker rm -f"),
removals_before + 1,
"a container whose worker died must not keep publishing the \
worker's port: {:?}",
docker.calls()
);
assert_eq!(
sup.reconcile(&ON1, &api, true, at(6300)),
vec![RestartWorker(0)]
);
}
#[test]
fn a_container_is_not_re_probed_on_every_reconcile_tick() {
let dir = tmp("ctr-probe-rate");
let docker = Arc::new(FakeDocker::healthy());
let (mut sup, _pg) = container_sup(&dir, docker.clone());
let api = FakeBroker::default();
api.up.set(true);
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
sup.reconcile(&ON1, &api, true, at(0)); let probes = |d: &FakeDocker| d.count("kill -0");
for ms in [100, 2100, 4100] {
sup.reconcile(&ON1, &api, true, at(ms));
}
assert_eq!(
probes(&docker),
1,
"four ticks inside one TTL ask Docker once: {:?}",
docker.calls()
);
sup.reconcile(&ON1, &api, true, at(5200));
assert_eq!(probes(&docker), 2, "and again once the TTL has passed");
}
#[test]
fn a_container_crashloop_reports_the_workers_own_log_tail() {
let dir = tmp("ctr-crashloop");
let docker = Arc::new(FakeDocker::healthy());
*docker.log.lock().unwrap() =
"Traceback (most recent call last)\nRuntimeError: bind refused\n".into();
let (mut sup, _pg) = container_sup(&dir, docker.clone());
let api = FakeBroker::default();
api.up.set(true);
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
for (n, ms) in [0u64, 1300, 3500].iter().enumerate() {
*docker.worker_alive.lock().unwrap() = true;
*docker.state.lock().unwrap() = Some("true|2026-09-18T07:00:00Z|0".into());
sup.reconcile(&ON1, &api, true, at(*ms));
*docker.worker_alive.lock().unwrap() = false;
sup.reconcile(&ON1, &api, true, at(ms + 100));
assert!(n < 3);
}
let problems = sup.problems(at(4000));
let crash = problems
.iter()
.find(|p| p.code == "worker_crashloop")
.expect("3 failures in 5 min");
let detail = crash.detail.as_deref().unwrap_or("");
assert!(
detail.contains("RuntimeError: bind refused"),
"detail carries the worker's own log, captured before the \
container that held it was removed: {detail:?}"
);
assert!(
!detail.contains("s6-rc"),
"and not `docker logs`, which is the image's stream: {detail:?}"
);
assert!(
detail.contains("still up"),
"and says what the container itself was doing: {detail:?}"
);
}
#[test]
fn a_container_crashloop_still_says_why_once_the_slot_has_restarted() {
let dir = tmp("ctr-crashloop-up");
let docker = Arc::new(FakeDocker::healthy());
*docker.log.lock().unwrap() =
"Traceback (most recent call last)\nRuntimeError: bind refused\n".into();
let (mut sup, _pg) = container_sup(&dir, docker.clone());
let api = FakeBroker::default();
api.up.set(true);
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
for ms in [0u64, 1300, 3500] {
*docker.worker_alive.lock().unwrap() = true;
*docker.state.lock().unwrap() = Some("true|2026-09-18T07:00:00Z|0".into());
sup.reconcile(&ON1, &api, true, at(ms));
*docker.worker_alive.lock().unwrap() = false;
sup.reconcile(&ON1, &api, true, at(ms + 100));
}
*docker.worker_alive.lock().unwrap() = true;
*docker.state.lock().unwrap() = Some("true|2026-09-18T07:00:00Z|0".into());
sup.reconcile(&ON1, &api, true, at(8000));
assert!(
sup.slots[&0].proc_.is_some(),
"the slot must be running again for this test to be about anything"
);
let problems = sup.problems(at(8100));
let crash = problems
.iter()
.find(|p| p.code == "worker_crashloop")
.expect("3 failures in 5 min is still a crash loop while the slot is back up");
let detail = crash.detail.as_deref().unwrap_or("");
assert!(
detail.contains("RuntimeError: bind refused"),
"a crash loop that has restarted must still carry the last crash's \
log, not an empty detail: {detail:?}"
);
}
#[test]
fn a_container_that_died_outright_reports_its_exit_code() {
let dir = tmp("ctr-exitcode");
let docker = Arc::new(FakeDocker::healthy());
let (mut sup, _pg) = container_sup(&dir, docker.clone());
let api = FakeBroker::default();
api.up.set(true);
let t = Instant::now();
let at = |ms: u64| t + Duration::from_millis(ms);
for ms in [0u64, 1300, 3500] {
*docker.state.lock().unwrap() = Some("true|2026-09-18T07:00:00Z|0".into());
*docker.worker_alive.lock().unwrap() = true;
sup.reconcile(&ON1, &api, true, at(ms));
*docker.state.lock().unwrap() = Some("false|2026-09-18T07:00:00Z|137".into());
*docker.worker_alive.lock().unwrap() = false;
sup.reconcile(&ON1, &api, true, at(ms + 100));
}
let detail = sup
.problems(at(4000))
.into_iter()
.find(|p| p.code == "worker_crashloop")
.and_then(|p| p.detail)
.expect("3 failures in 5 min");
assert!(detail.contains("exit code 137"), "{detail:?}");
assert!(
detail.contains("s6-rc"),
"a stopped container can't be exec'd into, so the tail falls \
back to `docker logs`: {detail:?}"
);
}
#[test]
fn killing_a_container_worker_sigterms_it_and_deletes_it_from_the_broker() {
use Action::*;
let dir = tmp("ctr-kill");
let docker = Arc::new(FakeDocker::healthy());
let (mut sup, _pg) = container_sup(&dir, docker.clone());
let api = FakeBroker::default();
api.up.set(true);
let t = Instant::now();
assert_eq!(sup.reconcile(&ON1, &api, true, t), vec![StartWorker(0)]);
*api.workers.borrow_mut() = vec![bw("id0", "testfp00-w0")];
let off = Desired {
sharing: false,
workers: 1,
};
assert_eq!(sup.reconcile(&off, &api, true, t), vec![DrainWorker(0)]);
assert_eq!(*api.drained.borrow(), vec!["id0".to_string()]);
assert_eq!(sup.reconcile(&off, &api, true, t), vec![KillWorker(0)]);
assert!(
docker.ran("docker kill --signal TERM zc-share-testfp00-w0"),
"{:?}",
docker.calls()
);
assert!(
!docker.ran("--signal KILL"),
"a container that stops on TERM is never SIGKILLed: {:?}",
docker.calls()
);
assert_eq!(*api.deleted.borrow(), vec!["id0".to_string()]);
assert!(sup.children().workers.is_empty());
}
#[test]
fn a_container_that_ignores_term_is_sigkilled_after_the_grace_period() {
let dir = tmp("ctr-stubborn");
let docker = Arc::new(FakeDocker::healthy());
*docker.term_stops.lock().unwrap() = false;
let (mut sup, _pg) = container_sup(&dir, docker.clone());
sup.spawn_worker(0, Instant::now()).unwrap();
sup.terminate_all(Duration::from_millis(300));
assert!(
docker.ran("docker kill --signal TERM zc-share-testfp00-w0"),
"{:?}",
docker.calls()
);
assert!(
docker.ran("docker kill --signal KILL zc-share-testfp00-w0"),
"{:?}",
docker.calls()
);
}
#[test]
fn a_stop_flag_set_mid_wait_hands_a_container_back_instead_of_sigkilling_it() {
let dir = tmp("ctr-stopflag");
let docker = Arc::new(FakeDocker::healthy());
*docker.term_stops.lock().unwrap() = false;
let mut sup = SupGuard(Supervisor::new(container_spec(&dir)));
sup.set_docker(docker.clone());
let stop = sup.stop_flag();
stop.store(true, Ordering::Relaxed);
sup.spawn_worker(0, Instant::now()).unwrap();
let p = sup.0.slots.get_mut(&0).unwrap().proc_.take().unwrap();
let handed_back = terminate(p, Duration::from_secs(30), &stop);
assert!(handed_back.is_some(), "the container is handed back");
assert!(docker.ran("--signal TERM"), "{:?}", docker.calls());
assert!(
!docker.ran("--signal KILL"),
"a shutdown must not turn a graceful stop into a kill: {:?}",
docker.calls()
);
sup.0.slots.get_mut(&0).unwrap().proc_ = handed_back;
}
#[test]
fn a_container_worker_is_re_adopted_only_when_it_is_the_same_run() {
let dir = tmp("ctr-adopt");
let recorded = |started_at: &str| {
let mut c = Children::default();
c.workers.insert(
"0".into(),
ChildRecord {
pid: 0,
started_at: started_at.into(),
port: None,
container: Some("zc-share-testfp00-w0".into()),
},
);
c
};
let adopted = |docker: Arc<FakeDocker>, started_at: &str| {
let mut sup = Supervisor::new(container_spec(&dir));
sup.set_docker(docker);
sup.adopt(&recorded(started_at), &proc_start_time);
sup.children().workers.len()
};
let up = Arc::new(FakeDocker::healthy());
assert_eq!(adopted(up.clone(), "2026-09-18T07:00:00Z"), 1);
assert_eq!(
adopted(up, "2026-09-18T06:00:00Z"),
0,
"a container restarted under the same name is not the one we recorded"
);
let stopped = Arc::new(FakeDocker::healthy());
stopped.stop();
assert_eq!(adopted(stopped, "2026-09-18T07:00:00Z"), 0);
let gone = Arc::new(FakeDocker::healthy());
*gone.state.lock().unwrap() = None;
assert_eq!(adopted(gone, "2026-09-18T07:00:00Z"), 0);
}
#[test]
fn terminate_all_stops_container_workers_too() {
let dir = tmp("ctr-termall");
let docker = Arc::new(FakeDocker::healthy());
let mut sup = Supervisor::new(container_spec(&dir));
sup.set_docker(docker.clone());
sup.spawn_worker(0, Instant::now()).unwrap();
sup.spawn_worker(1, Instant::now()).unwrap();
sup.terminate_all(Duration::from_millis(500));
assert_eq!(
docker.count("docker kill --signal TERM"),
2,
"{:?}",
docker.calls()
);
assert!(sup.children().workers.is_empty());
}
#[test]
fn auto_runtime_picks_the_container_only_when_docker_is_available() {
let mut cfg = AgentConfig::for_dirs(tmp("ctr-auto"));
cfg.child_env.clear();
assert_eq!(
SpawnSpec::from_config_with(&cfg, &|| true).worker_backend,
WorkerBackend::Container {
image: "zakuro-image-compute:latest".into()
}
);
assert_eq!(
SpawnSpec::from_config_with(&cfg, &|| false).worker_backend,
WorkerBackend::Host
);
}
#[test]
fn the_worker_backend_follows_zakuro_share_runtime() {
let dir = tmp("ctr-cfg");
let mut cfg = AgentConfig::for_dirs(dir.clone());
cfg.child_env
.push(("ZAKURO_SHARE_RUNTIME".into(), "host".into()));
assert_eq!(
SpawnSpec::from_config(&cfg).worker_backend,
WorkerBackend::Host
);
cfg.child_env
.push(("ZAKURO_SHARE_RUNTIME".into(), "container".into()));
assert_eq!(
SpawnSpec::from_config(&cfg).worker_backend,
WorkerBackend::Container {
image: "zakuro-image-compute:latest".into()
}
);
cfg.child_env.push((
"ZC_SHARE_WORKER_IMAGE".into(),
"zakuroai/compute@sha256:deadbeef".into(),
));
assert_eq!(
SpawnSpec::from_config(&cfg).worker_backend,
WorkerBackend::Container {
image: "zakuroai/compute@sha256:deadbeef".into()
}
);
cfg.child_env
.push(("ZAKURO_PRICE_PER_HOUR".into(), "0.75".into()));
assert_eq!(
SpawnSpec::from_config(&cfg).worker_price.as_deref(),
Some("0.75")
);
}
#[test]
fn a_container_backed_workers_price_is_written_into_its_exec_line() {
let dir = tmp("ctr-price");
let docker = Arc::new(FakeDocker::healthy());
let mut spec = container_spec(&dir);
spec.worker_price = Some("0.75".into());
let mut sup = SupGuard(Supervisor::new(spec));
sup.set_docker(docker.clone());
sup.spawn_worker(0, Instant::now()).unwrap();
assert!(
docker.ran("ZAKURO_PRICE_PER_HOUR=0.75"),
"{:?}",
docker.calls()
);
}
fn e2e_var(key: &str) -> String {
std::env::var(key).unwrap_or_else(|_| {
panic!(
"{key} is not set -- these tests are driven by \
scripts/verify-container-supervision.sh, not run directly"
)
})
}
fn e2e_dir() -> PathBuf {
PathBuf::from(e2e_var("ZC_E2E_DIR"))
}
fn e2e_port(key: &str) -> u16 {
e2e_var(key)
.parse()
.unwrap_or_else(|e| panic!("{key} is not a port number: {e}"))
}
fn e2e_spec() -> SpawnSpec {
let dir = e2e_dir();
let base_worker_port = e2e_port("ZC_E2E_WORKER_PORT");
SpawnSpec {
broker_argv: vec![e2e_var("ZC_E2E_ZC_BIN"), "broker".into(), "{port}".into()],
broker_env: vec![
(
"ZAKURO_SCAN_RANGE".into(),
format!("{base_worker_port}-{}", base_worker_port + 4),
),
("ZAKURO_SCAN_INTERVAL".into(), "3".into()),
("ZAKURO_P2P".into(), "false".into()),
("ZAKURO_HOME".into(), dir.display().to_string()),
(
"ZAKURO_WAL_PATH".into(),
dir.join("wal.jsonl").display().to_string(),
),
],
worker_template: vec![],
worker_env: vec![],
worker_cwd: None,
worker_backend: WorkerBackend::Container {
image: e2e_var("ZC_E2E_IMAGE"),
},
worker_price: None,
base_worker_port,
preferred_broker_port: e2e_port("ZC_E2E_BROKER_PORT"),
fp8: e2e_var("ZC_E2E_FP8"),
log_dir: dir,
kill_grace: Duration::from_secs(30),
broker_stop_wait: Duration::from_secs(20),
drain_timeout: Duration::from_secs(300),
}
}
fn e2e_step(what: &str) {
static START: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
let t = START.get_or_init(Instant::now).elapsed().as_secs_f32();
println!(" [e2e] {t:6.1}s {what}");
}
fn e2e_docker(args: &[&str]) -> Result<String, String> {
let argv: Vec<String> = std::iter::once("docker")
.chain(args.iter().copied())
.map(str::to_string)
.collect();
DockerShell.run(&argv)
}
fn e2e_containers(prefix: &str) -> Vec<String> {
e2e_docker(&["ps", "-a", "--format", "{{.Names}}"])
.unwrap_or_default()
.lines()
.map(str::trim)
.filter(|n| n.starts_with(prefix))
.map(str::to_string)
.collect()
}
fn e2e_health(port: u16) -> bool {
ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.timeout_global(Some(Duration::from_secs(3)))
.http_status_as_error(false)
.build(),
)
.get(&format!("http://127.0.0.1:{port}/health"))
.call()
.is_ok_and(|r| r.status().as_u16() == 200)
}
fn e2e_tick(sup: &mut Supervisor) -> Vec<Action> {
let api = HttpBrokerApi {
port: sup.broker_port(),
worker_key: None,
};
let prerequisites_ok = plan::prerequisite_problems(
false,
false,
None,
true,
false,
Some(crate::up::docker_cli_available_cached()),
)
.is_empty();
sup.reconcile(&ON1, &api, prerequisites_ok, Instant::now())
}
fn e2e_tick_until(
sup: &mut Supervisor,
within: Duration,
what: &str,
mut done: impl FnMut(&mut Supervisor, &[Action]) -> bool,
) {
let deadline = Instant::now() + within;
loop {
let actions = e2e_tick(sup);
if !actions.is_empty() {
println!(" [e2e] tick planned {actions:?}");
}
if done(sup, &actions) {
return;
}
assert!(
Instant::now() < deadline,
"timed out after {within:?} waiting for {what}; last tick planned {actions:?}"
);
std::thread::sleep(Duration::from_millis(500));
}
}
#[test]
#[ignore = "real Docker daemon and image; driven by scripts/verify-container-supervision.sh"]
fn a_container_backed_worker_is_spawned_and_restarted_when_the_worker_dies_inside_its_container(
) {
let spec = e2e_spec();
let worker_port = spec.base_worker_port;
let name = crate::up::worker_name(&spec.fp8, 0);
let container = crate::up::worker_container_name(&name);
let prefix = format!("zc-share-{}", spec.fp8);
let mut sup = Supervisor::new(spec);
e2e_step("ASSERT the supervisor starts a broker that answers /health");
e2e_tick_until(&mut sup, Duration::from_secs(90), "the broker", |s, _| {
s.broker_running()
});
assert!(!sup.broker_unmanaged(), "this broker must be ours to stop");
e2e_step("ASSERT the worker slot comes up as a container that answers /health");
e2e_tick_until(
&mut sup,
Duration::from_secs(240),
"the container worker's /health",
|_, _| e2e_health(worker_port),
);
assert_eq!(
e2e_containers(&prefix),
vec![container.clone()],
"exactly one container, named the way `zc share` names it"
);
let first = sup.children().workers["0"].clone();
assert_eq!(first.container.as_deref(), Some(container.as_str()));
assert_eq!(
first.pid, 0,
"a container-backed worker is identified by name, not by pid"
);
assert!(
!first.started_at.is_empty(),
"the daemon's own StartedAt must be recorded: it is what adoption compares against"
);
let mut previous_run = first.started_at.clone();
for round in 1..=3u32 {
e2e_step(&format!(
"ASSERT round {round}: killing the worker INSIDE the container leaves the container itself running"
));
let pid = e2e_docker(&["exec", &container, "cat", crate::up::WORKER_CONTAINER_PID])
.expect("read the worker's recorded pid inside the container");
let pid = pid.trim().to_string();
assert!(!pid.is_empty(), "the exec line must have recorded a pid");
e2e_docker(&["exec", &container, "kill", "-9", &pid])
.expect("kill the worker process inside the container");
let raw = e2e_docker(
&crate::up::docker_state_argv(&container)[1..]
.iter()
.map(String::as_str)
.collect::<Vec<_>>(),
)
.expect("inspect the container");
let state =
crate::up::parse_container_state(&raw).expect("a parseable container state");
assert!(
state.running,
"the container must still be running after the worker inside it died: \
that is precisely the case a `.State.Running` liveness check misses"
);
assert!(
!e2e_health(worker_port),
"...while the worker itself answers nothing, on a port the container \
still publishes"
);
e2e_step(&format!(
"ASSERT round {round}: the supervisor sees the crash and keeps the worker's OWN log"
));
e2e_tick_until(
&mut sup,
Duration::from_secs(120),
"the crash to be detected",
|s, _| s.slots.get(&0).is_some_and(|slot| slot.proc_.is_none()),
);
let detail = sup.slots[&0].crash_log.clone().unwrap_or_default();
assert!(
detail.contains(&format!(
"the worker process is gone from container {container}, which is still up"
)),
"the detail must say what the container itself was doing: {detail:?}"
);
assert!(
detail.contains("Uvicorn running on"),
"...and carry the worker's own /tmp/worker.log, read back with \
`docker exec tail` before the container is removed: {detail:?}"
);
assert!(
!detail.contains("s6-rc"),
"...which is a different stream from `docker logs`, the image's own: {detail:?}"
);
e2e_step(&format!(
"ASSERT round {round}: the slot restarts and answers /health from a new container run"
));
let mut restarted = false;
e2e_tick_until(
&mut sup,
Duration::from_secs(180),
"a RestartWorker(0)",
|_, actions| {
restarted |= actions.contains(&Action::RestartWorker(0));
restarted
},
);
e2e_tick_until(
&mut sup,
Duration::from_secs(240),
"the restarted worker's /health",
|_, _| e2e_health(worker_port),
);
let now_up = sup.children().workers["0"].clone();
assert_eq!(now_up.container.as_deref(), Some(container.as_str()));
assert_ne!(
now_up.started_at, previous_run,
"a restart is a new run of that container, not the old one re-read"
);
assert_eq!(
e2e_containers(&prefix),
vec![container.clone()],
"the restart must leave no second container behind"
);
previous_run = now_up.started_at;
}
e2e_step(
"ASSERT three crashes in five minutes are reported as a crash loop that can say why",
);
let problems = sup.problems(Instant::now());
let crash = problems
.iter()
.find(|p| p.code == "worker_crashloop")
.unwrap_or_else(|| panic!("three failures inside five minutes: {problems:?}"));
let detail = crash.detail.as_deref().unwrap_or("");
assert!(
detail.contains("Uvicorn running on"),
"a crash loop is read while the slot is back up between its crashes, and it \
must still carry the last crash's log there, not an empty detail: {detail:?}"
);
e2e_step(
"hand over to phase two: write children.json and exit without terminating anything",
);
let dir = e2e_dir();
ensure_dir(&dir).expect("the scratch dir");
crate::agent::files::save_json(
&dir.join(crate::agent::files::CHILDREN_FILE),
&sup.children(),
)
.expect("write children.json");
}
#[test]
#[ignore = "real Docker daemon and image; driven by scripts/verify-container-supervision.sh"]
fn a_restarted_agent_re_adopts_the_container_it_left_running_and_then_tears_it_down() {
let dir = e2e_dir();
let recorded: Children =
crate::agent::files::load_json(&dir.join(crate::agent::files::CHILDREN_FILE))
.expect("phase one's children.json");
let spec = e2e_spec();
let worker_port = spec.base_worker_port;
let name = crate::up::worker_name(&spec.fp8, 0);
let container = crate::up::worker_container_name(&name);
let prefix = format!("zc-share-{}", spec.fp8);
let was = recorded.workers["0"].clone();
let mut sup = Supervisor::new(spec);
e2e_step("ASSERT a fresh supervisor re-adopts the container the last one left running");
sup.adopt(&recorded, &proc_start_time);
let adopted = sup.children();
assert_eq!(
adopted.workers.get("0").and_then(|r| r.container.clone()),
Some(container.clone()),
"the container must be re-adopted, not orphaned"
);
assert_eq!(
adopted.workers["0"].started_at, was.started_at,
"and it must be the same run of it"
);
assert!(
adopted.broker.is_some(),
"the broker must be re-adopted too, or this agent would start a second one"
);
e2e_step("ASSERT a reconcile tick starts nothing a second time");
let actions = e2e_tick(&mut sup);
assert!(
sup.broker_running(),
"the re-adopted broker must answer /health (unmanaged={})",
sup.broker_unmanaged()
);
assert!(
!actions
.iter()
.any(|a| matches!(a, Action::StartWorker(_) | Action::RestartWorker(_))),
"a re-adopted worker must not be started again: {actions:?}"
);
assert_eq!(
e2e_containers(&prefix),
vec![container.clone()],
"still exactly one container"
);
assert!(
e2e_health(worker_port),
"the re-adopted worker must still be the one answering /health"
);
e2e_step("ASSERT terminate_all leaves no container and nothing on either port");
let broker_port = sup.broker_port();
sup.terminate_all(Duration::from_secs(90));
assert!(
wait_until(|| e2e_containers(&prefix).is_empty()),
"containers left behind: {:?}",
e2e_containers(&prefix)
);
assert!(
wait_until(|| !crate::up::is_port_open(worker_port)),
"something still holds the worker port {worker_port}"
);
assert!(
wait_until(|| !crate::up::is_port_open(broker_port)),
"something still holds the broker port {broker_port}"
);
}
}