use std::net::TcpStream;
use std::path::{Path, PathBuf};
use std::process::{Child, Command};
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use std::time::{Duration, Instant};
use colored::Colorize;
pub(crate) const BASE_WORKER_PORT: u16 = 3960;
const DEFAULT_BROKER_PORT: u16 = 9000;
const WORKER_STARTUP_TIMEOUT_SECS: u64 = 20;
struct WorkerHandle {
port: u16,
name: String,
backend: WorkerBackend,
}
impl WorkerHandle {
fn identity(&self) -> String {
match &self.backend {
WorkerBackend::Host(c) => format!("PID {}", c.id()),
WorkerBackend::Container(name) => format!("container {name}"),
}
}
fn exited(&mut self) -> Option<String> {
match &mut self.backend {
WorkerBackend::Host(c) => match c.try_wait() {
Ok(Some(status)) => Some(status.to_string()),
_ => None,
},
WorkerBackend::Container(name) => {
let out = Command::new("docker")
.args(["inspect", "-f", "{{.State.Running}}", name])
.output();
match out {
Ok(o) if o.status.success() => {
let running = String::from_utf8_lossy(&o.stdout).trim() == "true";
(!running).then(|| "container stopped".to_string())
}
_ => Some("container no longer exists".to_string()),
}
}
}
}
}
impl Drop for WorkerHandle {
fn drop(&mut self) {
match &mut self.backend {
WorkerBackend::Host(child) => {
let _ = child.kill();
}
WorkerBackend::Container(name) => {
let _ = Command::new("docker")
.args(["stop", "-t", "5", name])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
}
}
}
}
pub fn parse_args(args: &[String]) -> (usize, u16, u16, bool) {
let mut workers = 1usize;
let mut base_port = BASE_WORKER_PORT;
let mut broker_port = DEFAULT_BROKER_PORT;
let mut daemon = false;
let mut i = 0;
while i < args.len() {
match args[i].as_str() {
"--workers" | "-w" | "-n" => {
if let Some(v) = args.get(i + 1) {
workers = v.parse().unwrap_or(1);
i += 2;
continue;
}
}
"--port" | "-p" => {
if let Some(v) = args.get(i + 1) {
base_port = v.parse().unwrap_or(BASE_WORKER_PORT);
i += 2;
continue;
}
}
"--broker-port" | "-b" => {
if let Some(v) = args.get(i + 1) {
broker_port = v.parse().unwrap_or(DEFAULT_BROKER_PORT);
i += 2;
continue;
}
}
"-d" | "--daemon" | "--background" => {
daemon = true;
}
s if s.starts_with("--workers=") => {
workers = s.trim_start_matches("--workers=").parse().unwrap_or(1);
}
_ => {}
}
i += 1;
}
(workers, base_port, broker_port, daemon)
}
pub fn is_port_open(port: u16) -> bool {
TcpStream::connect_timeout(
&format!("127.0.0.1:{}", port).parse().unwrap(),
Duration::from_millis(150),
)
.is_ok()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ZakuroDirMissing {
pub invalid_setting: Option<String>,
}
impl ZakuroDirMissing {
pub fn detail(&self) -> Option<String> {
self.invalid_setting
.as_ref()
.map(|p| format!("ZAKURO_WORKER_DIR={p} has no zakuro/worker/server.py"))
}
}
fn is_valid_zakuro_dir(p: &Path) -> bool {
p.join("zakuro/worker/server.py").exists()
}
pub fn resolve_zakuro_dir(
env_worker_dir: Option<String>,
file_worker_dir: Option<String>,
zc_dir: Option<&Path>,
legacy_candidates: &[PathBuf],
) -> Result<PathBuf, ZakuroDirMissing> {
let mut invalid_setting = None;
for v in [env_worker_dir, file_worker_dir].into_iter().flatten() {
if v.is_empty() {
continue;
}
let p = PathBuf::from(&v);
if is_valid_zakuro_dir(&p) {
return Ok(p);
}
invalid_setting.get_or_insert(v);
}
if let Some(zc) = zc_dir {
let p = zc.join("zak-zakuro");
if is_valid_zakuro_dir(&p) {
return Ok(p);
}
}
legacy_candidates
.iter()
.find(|c| is_valid_zakuro_dir(c))
.cloned()
.ok_or(ZakuroDirMissing { invalid_setting })
}
pub fn legacy_zakuro_dir_candidates_from_env() -> Vec<PathBuf> {
let mut out = Vec::new();
if let Some(parent) = std::env::current_dir()
.ok()
.and_then(|d| d.parent().map(PathBuf::from))
{
out.push(parent.join("zak-zakuro"));
}
out.push(PathBuf::from("/opt/code/ZAK/zak-zakuro"));
if let Some(h) = std::env::var_os("HOME") {
out.push(PathBuf::from(h).join("zak-zakuro"));
}
out
}
pub(crate) fn resolve_zakuro_dir_for(
env_worker_dir: Option<String>,
zc_dir: Option<&Path>,
legacy_candidates: &[PathBuf],
) -> Result<PathBuf, ZakuroDirMissing> {
let file_worker_dir = zc_dir
.and_then(|d| std::fs::read_to_string(d.join("env")).ok())
.and_then(|text| {
crate::credentials::parse(&text)
.get("ZAKURO_WORKER_DIR")
.cloned()
});
resolve_zakuro_dir(env_worker_dir, file_worker_dir, zc_dir, legacy_candidates)
}
pub(crate) fn find_zakuro_dir() -> Option<PathBuf> {
let zc_dir = crate::credentials::dir();
resolve_zakuro_dir_for(
std::env::var("ZAKURO_WORKER_DIR").ok(),
zc_dir.as_deref(),
&legacy_zakuro_dir_candidates_from_env(),
)
.ok()
}
pub(crate) fn worker_name(node_fingerprint: &str, index: usize) -> String {
let fp8: String = node_fingerprint.chars().take(8).collect();
format!("{fp8}-w{index}")
}
pub(crate) fn worker_argv(port: u16, name: &str) -> Vec<String> {
[
"uv",
"run",
"--extra",
"worker",
"python",
"-m",
"zakuro.worker.server",
"--port",
]
.iter()
.map(|s| s.to_string())
.chain([
port.to_string(),
"--worker-name".to_string(),
name.to_string(),
])
.collect()
}
pub(crate) fn worker_bind_env(
get: &dyn Fn(&str) -> Option<String>,
) -> Option<(&'static str, &'static str)> {
let val = |k: &str| get(k).map(|v| v.trim().to_lowercase()).unwrap_or_default();
let truthy = |k: &str| matches!(val(k).as_str(), "1" | "true" | "yes");
let chosen = !val("ZAKURO_HOST").is_empty()
|| truthy("ZAKURO_INSECURE_BIND")
|| truthy("ZAKURO_AUTH_REQUIRED")
|| !val("ZAKURO_CERT_DIR").is_empty()
|| val("ZAKURO_WIRE") == "v1";
(!chosen).then_some(("ZAKURO_HOST", "127.0.0.1"))
}
pub fn parse_price_arg(args: &[String]) -> Result<Option<f64>, String> {
let mut raw: Option<&str> = None;
let mut i = 0;
while i < args.len() {
let a = args[i].as_str();
if a == "--price" {
raw = Some(args.get(i + 1).map(String::as_str).unwrap_or(""));
i += 2;
continue;
}
if let Some(v) = a.strip_prefix("--price=") {
raw = Some(v);
}
i += 1;
}
let Some(raw) = raw else { return Ok(None) };
let v: f64 = raw
.trim()
.parse()
.map_err(|_| format!("invalid --price {raw:?}: expected a number of zkcr per hour"))?;
if !v.is_finite() || v < 0.0 {
return Err(format!("invalid --price {raw:?}: must be zero or more"));
}
Ok(Some(v))
}
pub fn apply_price_arg(args: &[String]) -> Option<f64> {
match parse_price_arg(args) {
Err(msg) => {
eprintln!("{msg}");
std::process::exit(2);
}
Ok(None) => None,
Ok(Some(v)) => {
std::env::set_var("ZAKURO_PRICE_PER_HOUR", format!("{v}"));
println!("Offering this machine at {v} zkcr/hour.");
Some(v)
}
}
}
pub(crate) fn worker_price_env(
get: &dyn Fn(&str) -> Option<String>,
flag: Option<f64>,
) -> Option<(&'static str, String)> {
let key = "ZAKURO_PRICE_PER_HOUR";
if let Some(v) = flag {
return Some((key, format!("{v}")));
}
get(key)
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
.map(|v| (key, v))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ShareRuntime {
Host,
Container,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RuntimeChoice {
pub runtime: ShareRuntime,
pub warning: Option<String>,
}
pub(crate) fn select_share_runtime(
get: &dyn Fn(&str) -> Option<String>,
docker_available: &dyn Fn() -> bool,
) -> Result<RuntimeChoice, String> {
let norm = |k: &str| {
get(k)
.map(|v| v.trim().to_lowercase())
.filter(|v| !v.is_empty())
};
let mode = match norm("ZAKURO_SHARE_RUNTIME").as_deref() {
Some("auto") => "auto",
Some("host") => "host",
Some("container") => "container",
Some(other) => {
return Err(format!(
"ZAKURO_SHARE_RUNTIME={other:?} is not one of auto, host, container"
))
}
None => match norm("ZC_SHARE_CONTAINER").as_deref() {
Some("1") | Some("true") | Some("yes") => "container",
Some("0") | Some("false") | Some("no") => "host",
_ => "auto",
},
};
match mode {
"host" => Ok(RuntimeChoice {
runtime: ShareRuntime::Host,
warning: Some(
"worker runtime: host process (opted out of the container with \
ZAKURO_SHARE_RUNTIME=host)"
.to_string(),
),
}),
"container" => {
if docker_available() {
Ok(RuntimeChoice {
runtime: ShareRuntime::Container,
warning: None,
})
} else {
Err(container_worker_refusal_message())
}
}
_ => {
if docker_available() {
Ok(RuntimeChoice {
runtime: ShareRuntime::Container,
warning: None,
})
} else {
Ok(RuntimeChoice {
runtime: ShareRuntime::Host,
warning: Some(
"worker runtime: host process (Docker is not available: `docker info` \
failed); set ZAKURO_SHARE_RUNTIME=container to require it"
.to_string(),
),
})
}
}
}
}
pub(crate) fn worker_container_image(get: &dyn Fn(&str) -> Option<String>) -> String {
get("ZC_SHARE_WORKER_IMAGE").unwrap_or_else(|| crate::mesh::DEFAULT_COMPUTE_IMAGE.to_string())
}
pub(crate) fn worker_container_name(name: &str) -> String {
format!("zc-share-{name}")
}
pub(crate) fn worker_network_name(name: &str) -> String {
let fp8: String = name.chars().take(8).collect();
format!("zc-worker-{fp8}")
}
pub(crate) fn docker_network_create_argv(name: &str) -> Vec<String> {
vec![
"docker".to_string(),
"network".to_string(),
"create".to_string(),
name.to_string(),
]
}
pub(crate) fn docker_run_argv(
port: u16,
image: &str,
container_name: &str,
network_name: &str,
worker_name: &str,
) -> Vec<String> {
[
"docker",
"run",
"--rm",
"-d",
"--network",
network_name,
"--name",
container_name,
"--label",
"zakuro.share=1",
"--label",
]
.into_iter()
.map(str::to_string)
.chain([format!("zakuro.worker={worker_name}")])
.chain(["-p".to_string(), format!("127.0.0.1:{port}:{port}")])
.chain([image.to_string()])
.collect()
}
pub(crate) const WORKER_CONTAINER_LOG: &str = "/tmp/worker.log";
pub(crate) const WORKER_CONTAINER_PID: &str = "/tmp/worker.pid";
pub(crate) fn worker_exec_argv(
container_name: &str,
port: u16,
name: &str,
price: Option<&str>,
) -> Vec<String> {
let price_prefix = price
.map(|v| format!("ZAKURO_PRICE_PER_HOUR={v} "))
.unwrap_or_default();
let inner = format!(
"echo $$ >{WORKER_CONTAINER_PID}; \
exec env {price_prefix}ZAKURO_WORKER_NAME={name} ZAKURO_INSECURE_BIND=1 \
python -m zakuro.worker.server \
--host 0.0.0.0 --port {port} --worker-name {name} >{WORKER_CONTAINER_LOG} 2>&1"
);
vec![
"docker".to_string(),
"exec".to_string(),
"-d".to_string(),
container_name.to_string(),
"sh".to_string(),
"-c".to_string(),
inner,
]
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ContainerState {
pub running: bool,
pub started_at: String,
pub exit_code: i64,
}
pub(crate) fn docker_state_argv(container_name: &str) -> Vec<String> {
vec![
"docker".to_string(),
"inspect".to_string(),
"-f".to_string(),
"{{.State.Running}}|{{.State.StartedAt}}|{{.State.ExitCode}}".to_string(),
container_name.to_string(),
]
}
pub(crate) fn parse_container_state(out: &str) -> Option<ContainerState> {
let mut fields = out.trim().splitn(3, '|');
let running = match fields.next()?.trim() {
"true" => true,
"false" => false,
_ => return None,
};
let started_at = fields.next()?.trim().to_string();
let exit_code = fields.next()?.trim().parse::<i64>().ok()?;
(!started_at.is_empty()).then_some(ContainerState {
running,
started_at,
exit_code,
})
}
pub(crate) fn docker_signal_argv(container_name: &str, signal: &str) -> Vec<String> {
vec![
"docker".to_string(),
"kill".to_string(),
"--signal".to_string(),
signal.to_string(),
container_name.to_string(),
]
}
pub(crate) fn docker_rm_argv(container_name: &str) -> Vec<String> {
vec![
"docker".to_string(),
"rm".to_string(),
"-f".to_string(),
container_name.to_string(),
]
}
pub(crate) fn docker_ps_share_argv() -> Vec<String> {
[
"docker",
"ps",
"-a",
"--no-trunc",
"--filter",
"label=zakuro.share=1",
"--format",
"{{.ID}}\t{{.Names}}",
]
.into_iter()
.map(str::to_string)
.collect()
}
pub(crate) fn parse_share_ps(out: &str) -> Vec<(String, String)> {
out.lines()
.filter_map(|l| {
let (id, name) = l.split_once('\t')?;
let (id, name) = (id.trim(), name.trim());
(!id.is_empty() && !name.is_empty()).then(|| (id.to_string(), name.to_string()))
})
.collect()
}
pub(crate) fn worker_alive_argv(container_name: &str) -> Vec<String> {
vec![
"docker".to_string(),
"exec".to_string(),
container_name.to_string(),
"sh".to_string(),
"-c".to_string(),
format!("kill -0 \"$(cat {WORKER_CONTAINER_PID} 2>/dev/null)\" 2>/dev/null"),
]
}
pub(crate) fn worker_log_tail_argv(container_name: &str, n: usize) -> Vec<String> {
vec![
"docker".to_string(),
"exec".to_string(),
container_name.to_string(),
"sh".to_string(),
"-c".to_string(),
format!("tail -n {n} {WORKER_CONTAINER_LOG} 2>/dev/null"),
]
}
pub(crate) fn docker_logs_argv(container_name: &str, n: usize) -> Vec<String> {
vec![
"docker".to_string(),
"logs".to_string(),
"--tail".to_string(),
n.to_string(),
container_name.to_string(),
]
}
pub(crate) fn container_worker_refusal_message() -> String {
"ZAKURO_SHARE_RUNTIME=container but Docker isn't available on this host \
(checked `docker info`). Install/start Docker, or set \
ZAKURO_SHARE_RUNTIME=host (or unset it) to share with the host process."
.to_string()
}
pub(crate) fn docker_cli_available() -> bool {
Command::new("docker")
.arg("info")
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
}
const DOCKER_PROBE_TTL: Duration = Duration::from_secs(30);
pub(crate) fn docker_cli_available_cached() -> bool {
static CACHE: std::sync::Mutex<Option<(Instant, bool)>> = std::sync::Mutex::new(None);
let mut cache = match CACHE.lock() {
Ok(c) => c,
Err(poisoned) => poisoned.into_inner(),
};
if let Some((at, ok)) = *cache {
if at.elapsed() < DOCKER_PROBE_TTL {
return ok;
}
}
let ok = docker_cli_available();
*cache = Some((Instant::now(), ok));
ok
}
fn run_docker(argv: &[String], what: &str) -> Result<(), String> {
let out = Command::new(&argv[0])
.args(&argv[1..])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::piped())
.output()
.map_err(|e| format!("Failed to run docker for {what}: {e}"))?;
if !out.status.success() {
return Err(format!(
"docker failed for {what}: {}",
String::from_utf8_lossy(&out.stderr).trim()
));
}
Ok(())
}
fn pull_image_if_missing(image: &str) -> Result<(), String> {
let present = Command::new("docker")
.args(["image", "inspect", image])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false);
if present {
return Ok(());
}
eprintln!(" Pulling {image} (first run only; this can take a few minutes) ...");
let ok = Command::new("docker")
.args(["pull", image])
.stdout(std::process::Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false);
if ok {
Ok(())
} else {
Err(format!("docker pull {image} failed"))
}
}
fn spawn_container_worker(port: u16, name: String) -> Result<WorkerHandle, String> {
if !docker_cli_available() {
return Err(container_worker_refusal_message());
}
let image = worker_container_image(&|k| std::env::var(k).ok());
pull_image_if_missing(&image)?;
let container_name = worker_container_name(&name);
let network_name = worker_network_name(&name);
let _ = Command::new("docker")
.args(&docker_network_create_argv(&network_name)[1..])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
run_docker(
&docker_run_argv(port, &image, &container_name, &network_name, &name),
&format!("worker container {name}"),
)?;
if let Err(e) = run_docker(
&worker_exec_argv(
&container_name,
port,
&name,
worker_price_env(&|k| std::env::var(k).ok(), None)
.as_ref()
.map(|(_, v)| v.as_str()),
),
&format!("worker {name}"),
) {
let _ = Command::new("docker")
.args(["rm", "-f", &container_name])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
return Err(e);
}
Ok(WorkerHandle {
port,
name,
backend: WorkerBackend::Container(container_name),
})
}
enum WorkerBackend {
Host(Child),
Container(String),
}
fn spawn_worker(
port: u16,
name: String,
zakuro_dir: &PathBuf,
runtime: ShareRuntime,
) -> Result<WorkerHandle, String> {
if runtime == ShareRuntime::Container {
return spawn_container_worker(port, name);
}
let argv = worker_argv(port, &name);
let mut cmd = Command::new(&argv[0]);
cmd.args(&argv[1..])
.current_dir(zakuro_dir)
.env("ZAKURO_WORKER_TYPE", "zakuro")
.env("UV_NO_PROGRESS", "1")
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null());
if let Some((k, v)) = worker_bind_env(&|k| std::env::var(k).ok()) {
cmd.env(k, v);
}
let child = cmd
.spawn()
.map_err(|e| format!("Failed to spawn worker on port {}: {}", port, e))?;
Ok(WorkerHandle {
port,
name,
backend: WorkerBackend::Host(child),
})
}
fn wait_for_worker(port: u16, timeout_secs: u64) -> bool {
let deadline = Instant::now() + Duration::from_secs(timeout_secs);
let url = format!("http://127.0.0.1:{}/health", port);
while Instant::now() < deadline {
if let Ok(resp) = ureq::get(&url)
.config()
.timeout_global(Some(Duration::from_millis(500)))
.build()
.call()
{
if resp.status().as_u16() == 200 {
return true;
}
}
std::thread::sleep(Duration::from_millis(300));
}
false
}
pub fn start_up(workers: usize, base_port: u16, broker_port: u16, daemon: bool) {
println!();
if daemon {
println!(
" {} {} worker(s) + broker on :{} {}\n",
"zc up".bold().cyan(),
workers,
broker_port,
"(background)".dimmed()
);
} else {
println!(
" {} {} worker(s) + broker on :{}\n",
"zc up".bold().cyan(),
workers,
broker_port
);
}
let runtime = match select_share_runtime(&|k| std::env::var(k).ok(), &docker_cli_available) {
Ok(choice) => {
if let Some(w) = &choice.warning {
eprintln!(" {} {w}", "Warning:".yellow());
}
choice.runtime
}
Err(e) => {
eprintln!(" {} {e}", "Error:".red());
return;
}
};
let zakuro_dir = if runtime == ShareRuntime::Host {
match find_zakuro_dir() {
Some(d) => {
println!(" {} {}", "Worker src:".bold(), d.display());
d
}
None => {
eprintln!(
" {} Cannot find zak-zakuro directory.\n Clone it into ~/.zakuro/zak-zakuro, or set ZAKURO_WORKER_DIR",
"Error:".red()
);
return;
}
}
} else {
PathBuf::new()
};
let broker_already_up = is_port_open(broker_port);
if broker_already_up {
println!(
" {} Broker already running on :{} (reusing)",
"Broker:".bold(),
broker_port
);
} else {
let scan_end = base_port + (workers as u16).max(1) + 19;
let scan_range = format!("{}-{}", base_port, scan_end);
broker::apply_user_broker_defaults();
if daemon {
let exe = std::env::current_exe().unwrap_or_else(|_| PathBuf::from("zc"));
let broker_child = Command::new(&exe)
.args(["broker", &broker_port.to_string()])
.env("ZAKURO_SCAN_RANGE", &scan_range)
.env("ZAKURO_SCAN_INTERVAL", "3")
.env(
"ZAKURO_P2P",
if broker::p2p_default() {
"true"
} else {
"false"
},
)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn();
match broker_child {
Ok(_child) => { }
Err(e) => {
eprintln!(
" {} Failed to spawn broker subprocess: {}",
"Error:".red(),
e
);
return;
}
}
} else {
let discovery = broker::DiscoveryConfig {
worker_port: base_port,
extra_ports: vec![],
scan_port_range: Some((base_port, scan_end)),
enable_scan: true,
enable_dns: false,
interval_secs: 5,
..Default::default()
};
let config = broker::BrokerConfig {
host: "0.0.0.0".to_string(),
port: broker_port,
daemon: true,
verbose: false,
enable_discovery: true,
discovery,
enable_p2p: broker::p2p_default(),
..Default::default()
};
std::thread::Builder::new()
.name("broker".into())
.spawn(move || {
if let Err(e) = broker::start_server(config) {
eprintln!(" [BROKER] Error: {}", e);
}
})
.expect("Failed to spawn broker thread");
}
print!(" {} Starting...", "Broker:".bold());
let start = Instant::now();
loop {
if is_port_open(broker_port) {
println!(
"\r {} http://127.0.0.1:{}{}",
"Broker:".bold(),
broker_port,
" ".repeat(20)
);
break;
}
if start.elapsed().as_secs() > 5 {
println!();
eprintln!(" {} Broker didn't start in time", "Error:".red());
return;
}
std::thread::sleep(Duration::from_millis(100));
}
}
println!();
let node_fp = broker::node_identity::NodeKey::load_or_create().fingerprint();
let mut handles: Vec<WorkerHandle> = Vec::with_capacity(workers);
let mut ready_from_existing: usize = 0;
for i in 0..workers {
let port = base_port + i as u16;
if is_port_open(port) {
print!(
" {} Port {} already in use — checking ...",
format!("Worker {}:", i).bold(),
port
);
if wait_for_worker(port, 3) {
println!(" {}", "ready".green());
ready_from_existing += 1;
} else {
println!(" {}", "no response (not a worker?)".yellow());
}
continue;
}
match spawn_worker(port, worker_name(&node_fp, i), &zakuro_dir, runtime) {
Ok(handle) => {
println!(
" {} Spawned on :{} ({})",
format!("Worker {}:", i).bold(),
port,
handle.identity()
);
handles.push(handle);
}
Err(e) => {
eprintln!(" {} Failed to start worker {}: {}", "Error:".red(), i, e);
}
}
}
println!();
let mut ready = ready_from_existing;
for handle in &handles {
print!(
" Waiting for {} on :{} ...",
handle.name.cyan(),
handle.port
);
if wait_for_worker(handle.port, WORKER_STARTUP_TIMEOUT_SECS) {
println!(" {}", "ready".green());
ready += 1;
} else {
println!(" {}", "timeout".red());
}
}
if ready == 0 && (workers > 0) {
eprintln!(
"\n {} No workers became healthy. Aborting.",
"Error:".red()
);
return;
}
std::thread::sleep(Duration::from_secs(2));
println!();
println!(
" {}",
"─────────────────────────────────────────────".dimmed()
);
println!(" {} {}/{} ready", "Workers:".bold(), ready, workers);
println!(
" {} http://127.0.0.1:{} (route requests here)",
"Broker:".bold(),
broker_port
);
println!();
println!(
" Benchmark: {}",
format!("zc bench http://127.0.0.1:{}", broker_port).cyan()
);
if daemon {
println!(" Stop with: {}", "zc down".cyan());
println!(" Monitor: {}", "zc workers".cyan());
println!(
" {}",
"─────────────────────────────────────────────".dimmed()
);
println!();
for handle in handles {
std::mem::forget(handle);
}
std::process::exit(0);
} else {
println!(
" Press {} to stop all workers + broker.",
"Ctrl+C".yellow().bold()
);
println!(
" {}",
"─────────────────────────────────────────────".dimmed()
);
println!();
let running = Arc::new(AtomicBool::new(true));
let r = running.clone();
#[cfg(unix)]
unsafe {
libc_sigint(r);
}
#[cfg(not(unix))]
{
let _ = r;
}
while running.load(Ordering::SeqCst) {
for handle in &mut handles {
if let Some(desc) = handle.exited() {
eprintln!(
"\n {} Worker {} (:{}) exited unexpectedly: {}",
"Warning:".yellow(),
handle.name,
handle.port,
desc
);
}
}
std::thread::sleep(Duration::from_secs(2));
}
println!("\n Shutting down {} worker(s)...", handles.len());
drop(handles);
println!(" Done.");
}
}
#[cfg(unix)]
unsafe fn libc_sigint(flag: Arc<AtomicBool>) {
use std::sync::Mutex;
static HANDLER: std::sync::OnceLock<Mutex<Option<Arc<AtomicBool>>>> =
std::sync::OnceLock::new();
HANDLER.get_or_init(|| Mutex::new(None));
if let Some(m) = HANDLER.get() {
*m.lock().unwrap() = Some(flag);
}
extern "C" fn handler(_: libc::c_int) {
use std::sync::OnceLock;
static HANDLER: OnceLock<Mutex<Option<Arc<AtomicBool>>>> = OnceLock::new();
println!("\n Received Ctrl+C");
std::process::exit(0);
}
libc::signal(libc::SIGINT, handler as *const () as libc::sighandler_t);
}
pub fn stop_up(broker_port: u16, worker_port_start: u16, worker_port_end: u16) {
println!();
println!(" {} Stopping cluster...", "zc down".bold().cyan());
println!();
let mut killed = 0usize;
let open_ports: Vec<u16> = {
let (tx, rx) = std::sync::mpsc::channel();
let mut handles = Vec::new();
for port in worker_port_start..=worker_port_end {
let tx = tx.clone();
handles.push(std::thread::spawn(move || {
let addr: std::net::SocketAddr = format!("127.0.0.1:{}", port).parse().unwrap();
if TcpStream::connect_timeout(&addr, Duration::from_millis(50)).is_ok() {
let _ = tx.send(port);
}
}));
}
drop(tx);
let mut open: Vec<u16> = rx.iter().collect();
for h in handles {
let _ = h.join();
}
open.sort_unstable();
open
};
for port in open_ports {
if port == broker_port {
continue;
} let url = format!("http://127.0.0.1:{}/health", port);
let is_worker = ureq::get(&url)
.config()
.timeout_global(Some(Duration::from_millis(300)))
.build()
.call()
.map(|r| r.status().as_u16() == 200)
.unwrap_or(false);
if is_worker {
print!(" Stopping worker on :{} ...", port);
if kill_port(port) {
println!(" {}", "done".green());
killed += 1;
} else {
println!(" {}", "failed (not found)".yellow());
}
}
}
if is_port_open(broker_port) {
print!(" Stopping broker on :{} ...", broker_port);
if kill_port(broker_port) {
println!(" {}", "done".green());
killed += 1;
} else {
println!(" {}", "failed (not found)".yellow());
}
}
println!();
if killed > 0 {
println!(" {} {} process(es) stopped.", "✓".green(), killed);
} else {
println!(
" {} Nothing was running on the expected ports.",
"Info:".bold()
);
}
println!();
}
fn kill_port(port: u16) -> bool {
let status = std::process::Command::new("fuser")
.args(["-k", &format!("{}/tcp", port)])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
if status.map(|s| s.success()).unwrap_or(false) {
return true;
}
let lsof = std::process::Command::new("lsof")
.args(["-ti", &format!("tcp:{}", port)])
.output();
if let Ok(out) = lsof {
let pids = String::from_utf8_lossy(&out.stdout);
for pid_str in pids.split_whitespace() {
if let Ok(pid) = pid_str.parse::<u32>() {
let _ = std::process::Command::new("kill")
.arg(pid.to_string())
.status();
return true;
}
}
}
false
}
use crate::broker;
#[cfg(test)]
mod tests {
use super::{
container_worker_refusal_message, docker_logs_argv, docker_network_create_argv,
docker_rm_argv, docker_run_argv, docker_signal_argv, docker_state_argv,
parse_container_state, parse_price_arg, resolve_zakuro_dir, select_share_runtime,
worker_alive_argv, worker_argv, worker_bind_env, worker_container_image,
worker_container_name, worker_exec_argv, worker_log_tail_argv, worker_name,
worker_network_name, worker_price_env, ContainerState, ShareRuntime, ZakuroDirMissing,
};
use crate::agent::files::tests::tmp;
use crate::broker::node_identity::NodeKey;
use std::path::PathBuf;
fn valid_dir(tag: &str) -> PathBuf {
let d = tmp(&format!("up-{tag}"));
std::fs::create_dir_all(d.join("zakuro/worker")).unwrap();
std::fs::write(d.join("zakuro/worker/server.py"), b"").unwrap();
d
}
#[test]
fn env_var_wins_when_valid() {
let d = valid_dir("env");
assert_eq!(
resolve_zakuro_dir(Some(d.display().to_string()), None, None, &[]),
Ok(d)
);
}
#[test]
fn an_invalid_env_setting_falls_through_to_the_file_setting() {
let d = valid_dir("file");
assert_eq!(
resolve_zakuro_dir(
Some("/nonexistent/bogus".to_string()),
Some(d.display().to_string()),
None,
&[]
),
Ok(d)
);
}
#[test]
fn both_settings_invalid_falls_through_to_the_zc_dir() {
let zc = valid_dir("zc-dir-parent");
let zak = zc.join("zak-zakuro");
std::fs::create_dir_all(zak.join("zakuro/worker")).unwrap();
std::fs::write(zak.join("zakuro/worker/server.py"), b"").unwrap();
assert_eq!(
resolve_zakuro_dir(
Some("/nonexistent/a".to_string()),
Some("/nonexistent/b".to_string()),
Some(&zc),
&[]
),
Ok(zak)
);
}
#[test]
fn legacy_candidates_are_tried_in_the_given_order() {
let zc = valid_dir("zc-dir-no-zak");
let first = tmp("legacy-first"); std::fs::create_dir_all(&first).unwrap();
let second_parent = valid_dir("legacy-second");
let second = second_parent.join("zak-zakuro");
std::fs::create_dir_all(second.join("zakuro/worker")).unwrap();
std::fs::write(second.join("zakuro/worker/server.py"), b"").unwrap();
let third_parent = valid_dir("legacy-third");
let third = third_parent.join("zak-zakuro");
std::fs::create_dir_all(third.join("zakuro/worker")).unwrap();
std::fs::write(third.join("zakuro/worker/server.py"), b"").unwrap();
assert_eq!(
resolve_zakuro_dir(
None,
None,
Some(&zc),
&[first.clone(), second.clone(), third.clone()]
),
Ok(second),
"the first candidate is invalid, so the second (not the third) wins"
);
}
#[test]
fn nothing_found_reports_the_first_invalid_setting() {
assert_eq!(
resolve_zakuro_dir(
Some("/nonexistent/env-val".to_string()),
Some("/nonexistent/file-val".to_string()),
None,
&[]
),
Err(ZakuroDirMissing {
invalid_setting: Some("/nonexistent/env-val".to_string())
})
);
}
#[test]
fn nothing_found_and_no_setting_given_has_no_detail() {
let missing = resolve_zakuro_dir(None, None, None, &[]).unwrap_err();
assert_eq!(missing.invalid_setting, None);
assert_eq!(missing.detail(), None);
}
#[test]
fn an_invalid_setting_produces_the_expected_detail_text() {
let missing =
resolve_zakuro_dir(Some("/bad/path".to_string()), None, None, &[]).unwrap_err();
assert_eq!(
missing.detail().as_deref(),
Some("ZAKURO_WORKER_DIR=/bad/path has no zakuro/worker/server.py")
);
}
#[test]
fn legacy_zakuro_dir_candidates_from_env_orders_cwd_dev_path_then_home() {
let candidates = super::legacy_zakuro_dir_candidates_from_env();
assert!(candidates.contains(&PathBuf::from("/opt/code/ZAK/zak-zakuro")));
let dev_path_index = candidates
.iter()
.position(|p| p == &PathBuf::from("/opt/code/ZAK/zak-zakuro"))
.unwrap();
if let Some(home) = std::env::var_os("HOME") {
let home_path = PathBuf::from(home).join("zak-zakuro");
if let Some(home_index) = candidates.iter().position(|p| p == &home_path) {
assert!(
dev_path_index < home_index,
"the dev path comes before $HOME/zak-zakuro"
);
}
}
}
#[test]
fn worker_name_is_fp8_dash_index() {
assert_eq!(worker_name("a1b2c3d4e5f60718", 0), "a1b2c3d4-w0");
assert_eq!(worker_name("a1b2c3d4e5f60718", 12), "a1b2c3d4-w12");
}
#[test]
fn two_macs_never_share_a_worker_name() {
let (a, b) = (NodeKey::generate(), NodeKey::generate());
assert_ne!(
worker_name(&a.fingerprint(), 0),
worker_name(&b.fingerprint(), 0)
);
}
#[test]
fn worker_argv_is_the_python_entrypoint() {
assert_eq!(
worker_argv(3961, "a1b2c3d4-w1"),
vec![
"uv",
"run",
"--extra",
"worker",
"python",
"-m",
"zakuro.worker.server",
"--port",
"3961",
"--worker-name",
"a1b2c3d4-w1",
]
);
}
#[test]
fn a_worker_with_no_caller_auth_binds_loopback() {
let unset = |_: &str| None;
assert_eq!(worker_bind_env(&unset), Some(("ZAKURO_HOST", "127.0.0.1")));
for (k, v) in [
("ZAKURO_HOST", " "),
("ZAKURO_INSECURE_BIND", "0"),
("ZAKURO_AUTH_REQUIRED", "off"),
("ZAKURO_CERT_DIR", " "),
("ZAKURO_WIRE", "legacy"),
] {
let env = move |q: &str| (q == k).then(|| v.to_string());
assert_eq!(
worker_bind_env(&env),
Some(("ZAKURO_HOST", "127.0.0.1")),
"{k}={v:?}"
);
}
}
#[test]
fn an_operator_host_or_caller_auth_is_left_alone() {
for (k, v) in [
("ZAKURO_HOST", "0.0.0.0"),
("ZAKURO_INSECURE_BIND", "1"),
("ZAKURO_INSECURE_BIND", "YES"),
("ZAKURO_AUTH_REQUIRED", "true"),
("ZAKURO_CERT_DIR", "/etc/zakuro/certs"),
("ZAKURO_WIRE", "v1"),
] {
let env = move |q: &str| (q == k).then(|| v.to_string());
assert_eq!(worker_bind_env(&env), None, "{k}={v:?}");
}
}
fn env_of(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
let m: std::collections::HashMap<String, String> = pairs
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
move |k: &str| m.get(k).cloned()
}
#[test]
fn runtime_auto_picks_the_container_when_docker_is_up() {
let c = select_share_runtime(&env_of(&[]), &|| true).unwrap();
assert_eq!(c.runtime, ShareRuntime::Container);
assert_eq!(c.warning, None);
let c =
select_share_runtime(&env_of(&[("ZAKURO_SHARE_RUNTIME", "AUTO")]), &|| true).unwrap();
assert_eq!(c.runtime, ShareRuntime::Container);
}
#[test]
fn runtime_auto_falls_back_to_host_with_a_warning_when_docker_is_absent() {
let c = select_share_runtime(&env_of(&[]), &|| false).unwrap();
assert_eq!(c.runtime, ShareRuntime::Host);
let w = c.warning.expect("fallback must warn");
assert!(w.contains("Docker is not available"), "{w}");
assert!(!w.contains('\n'), "one line: {w}");
}
#[test]
fn runtime_host_opt_out_never_probes_docker() {
let probe = || -> bool { panic!("must not probe docker when opted out") };
let c = select_share_runtime(&env_of(&[("ZAKURO_SHARE_RUNTIME", "host")]), &probe).unwrap();
assert_eq!(c.runtime, ShareRuntime::Host);
assert!(c.warning.unwrap().contains("ZAKURO_SHARE_RUNTIME=host"));
}
#[test]
fn runtime_container_forces_it_and_fails_clearly_without_docker() {
let env = env_of(&[("ZAKURO_SHARE_RUNTIME", "container")]);
let c = select_share_runtime(&env, &|| true).unwrap();
assert_eq!(c.runtime, ShareRuntime::Container);
let e = select_share_runtime(&env, &|| false).unwrap_err();
assert!(e.contains("ZAKURO_SHARE_RUNTIME=container"), "{e}");
}
#[test]
fn runtime_rejects_an_unknown_value() {
let e = select_share_runtime(&env_of(&[("ZAKURO_SHARE_RUNTIME", "podman")]), &|| true)
.unwrap_err();
assert!(e.contains("podman"), "{e}");
}
#[test]
fn legacy_zc_share_container_still_works_and_the_new_var_wins() {
let on = env_of(&[("ZC_SHARE_CONTAINER", "yes")]);
assert!(select_share_runtime(&on, &|| false).is_err());
let off = env_of(&[("ZC_SHARE_CONTAINER", "0")]);
assert_eq!(
select_share_runtime(&off, &|| true).unwrap().runtime,
ShareRuntime::Host
);
let both = env_of(&[
("ZC_SHARE_CONTAINER", "1"),
("ZAKURO_SHARE_RUNTIME", "host"),
]);
assert_eq!(
select_share_runtime(&both, &|| true).unwrap().runtime,
ShareRuntime::Host
);
}
#[test]
fn container_worker_is_loopback_only_and_binds_deliberately() {
let run = docker_run_argv(3960, "img", "zc-share-x-w0", "zc-worker-x", "x-w0");
let p = run.iter().position(|a| a == "-p").unwrap();
assert_eq!(run[p + 1], "127.0.0.1:3960:3960");
assert!(!run.iter().any(|a| a.starts_with("0.0.0.0")));
let exec = worker_exec_argv("zc-share-x-w0", 3960, "x-w0", None).join(" ");
assert!(exec.contains("--host 0.0.0.0"), "{exec}");
assert!(exec.contains("ZAKURO_INSECURE_BIND=1"), "{exec}");
}
#[test]
fn worker_container_image_defaults_to_the_mesh_compute_image() {
let env = |_: &str| None;
assert_eq!(worker_container_image(&env), "zakuro-image-compute:latest");
}
#[test]
fn worker_container_image_is_overridable_for_pinning() {
let env = |k: &str| {
(k == "ZC_SHARE_WORKER_IMAGE").then(|| "zakuroai/compute@sha256:deadbeef".to_string())
};
assert_eq!(
worker_container_image(&env),
"zakuroai/compute@sha256:deadbeef"
);
}
#[test]
fn worker_container_name_is_deterministic_and_namespaced() {
assert_eq!(worker_container_name("a1b2c3d4-w0"), "zc-share-a1b2c3d4-w0");
}
#[test]
fn docker_run_argv_publishes_only_to_loopback_and_does_not_override_cmd() {
let argv = docker_run_argv(
3960,
"zakuro-image-compute:latest",
"zc-share-a1b2c3d4-w0",
"zc-worker-a1b2c3d4",
"a1b2c3d4-w0",
);
assert_eq!(
argv,
vec![
"docker",
"run",
"--rm",
"-d",
"--network",
"zc-worker-a1b2c3d4",
"--name",
"zc-share-a1b2c3d4-w0",
"--label",
"zakuro.share=1",
"--label",
"zakuro.worker=a1b2c3d4-w0",
"-p",
"127.0.0.1:3960:3960",
"zakuro-image-compute:latest",
]
);
}
#[test]
fn worker_network_name_is_scoped_per_node_fingerprint() {
assert_eq!(worker_network_name("a1b2c3d4-w0"), "zc-worker-a1b2c3d4");
assert_eq!(worker_network_name("a1b2c3d4-w1"), "zc-worker-a1b2c3d4");
}
#[test]
fn docker_network_create_argv_is_a_plain_idempotent_create() {
assert_eq!(
docker_network_create_argv("zc-worker-a1b2c3d4"),
vec!["docker", "network", "create", "zc-worker-a1b2c3d4"]
);
}
#[test]
fn no_price_flag_leaves_the_worker_default_alone() {
assert_eq!(parse_price_arg(&[]).unwrap(), None);
assert_eq!(
parse_price_arg(&["--workers".into(), "2".into()]).unwrap(),
None
);
}
#[test]
fn a_price_is_read_in_both_spellings() {
let split = vec!["--price".to_string(), "0.75".to_string()];
let joined = vec!["--price=0.75".to_string()];
assert_eq!(parse_price_arg(&split).unwrap(), Some(0.75));
assert_eq!(parse_price_arg(&joined).unwrap(), Some(0.75));
}
#[test]
fn free_is_allowed_but_negative_is_not() {
assert_eq!(
parse_price_arg(&["--price=0".to_string()]).unwrap(),
Some(0.0)
);
assert!(parse_price_arg(&["--price=-1".to_string()]).is_err());
}
#[test]
fn a_malformed_price_is_an_error_not_a_default() {
for bad in ["0.5o", "", "abc", "NaN", "inf"] {
assert!(
parse_price_arg(&[format!("--price={bad}")]).is_err(),
"{bad:?} should be refused, not defaulted"
);
}
}
#[test]
fn the_flag_beats_the_environment() {
let env = |k: &str| (k == "ZAKURO_PRICE_PER_HOUR").then(|| "9.9".to_string());
assert_eq!(
worker_price_env(&env, Some(0.5)),
Some(("ZAKURO_PRICE_PER_HOUR", "0.5".to_string()))
);
}
#[test]
fn an_environment_price_is_passed_through_for_the_container() {
let env = |k: &str| (k == "ZAKURO_PRICE_PER_HOUR").then(|| " 2.5 ".to_string());
assert_eq!(
worker_price_env(&env, None),
Some(("ZAKURO_PRICE_PER_HOUR", "2.5".to_string()))
);
let unset = |_: &str| None;
assert_eq!(worker_price_env(&unset, None), None);
}
#[test]
fn the_container_exec_line_carries_the_price() {
let argv = worker_exec_argv("zc-share-a1b2c3d4-w0", 3960, "a1b2c3d4-w0", Some("0.75"));
let inner = argv.last().unwrap();
assert!(
inner.contains("exec env ZAKURO_PRICE_PER_HOUR=0.75 "),
"price must reach the container: {inner}"
);
let unpriced = worker_exec_argv("zc-share-a1b2c3d4-w0", 3960, "a1b2c3d4-w0", None);
assert!(!unpriced.last().unwrap().contains("ZAKURO_PRICE_PER_HOUR"));
}
#[test]
fn worker_exec_argv_binds_0_0_0_0_inside_via_the_trusted_mesh_escape_hatch() {
let argv = worker_exec_argv("zc-share-a1b2c3d4-w0", 3960, "a1b2c3d4-w0", None);
assert_eq!(argv[0], "docker");
assert_eq!(argv[1], "exec");
assert_eq!(argv[2], "-d");
assert_eq!(argv[3], "zc-share-a1b2c3d4-w0");
assert_eq!(argv[4], "sh");
assert_eq!(argv[5], "-c");
let inner = &argv[6];
assert!(inner.contains("ZAKURO_INSECURE_BIND=1"), "{inner}");
assert!(inner.contains("--host 0.0.0.0"), "{inner}");
assert!(inner.contains("--port 3960"), "{inner}");
assert!(inner.contains("--worker-name a1b2c3d4-w0"), "{inner}");
assert!(inner.contains("zakuro.worker.server"), "{inner}");
}
#[test]
fn container_worker_refusal_message_names_the_fix() {
let msg = container_worker_refusal_message();
assert!(msg.contains("Docker"), "{msg}");
assert!(msg.contains("ZAKURO_SHARE_RUNTIME"), "{msg}");
}
#[test]
fn worker_exec_argv_records_the_pid_the_worker_runs_as() {
let argv = worker_exec_argv("zc-share-a1b2c3d4-w0", 3960, "a1b2c3d4-w0", None);
let inner = &argv[6];
assert!(inner.contains("echo $$ >/tmp/worker.pid"), "{inner}");
assert!(inner.contains("exec env "), "{inner}");
assert!(inner.contains(">/tmp/worker.log 2>&1"), "{inner}");
assert!(inner.contains("ZAKURO_INSECURE_BIND=1"), "{inner}");
assert!(inner.contains("--host 0.0.0.0"), "{inner}");
}
#[test]
fn docker_state_argv_asks_for_running_started_at_and_exit_code() {
assert_eq!(
docker_state_argv("zc-share-a1b2c3d4-w0"),
vec![
"docker",
"inspect",
"-f",
"{{.State.Running}}|{{.State.StartedAt}}|{{.State.ExitCode}}",
"zc-share-a1b2c3d4-w0",
]
);
}
#[test]
fn a_containers_state_parses_into_running_started_at_and_exit_code() {
assert_eq!(
parse_container_state("true|2026-09-18T07:01:02.3456789Z|0\n"),
Some(ContainerState {
running: true,
started_at: "2026-09-18T07:01:02.3456789Z".to_string(),
exit_code: 0,
})
);
assert_eq!(
parse_container_state("false|2026-09-18T07:01:02Z|137"),
Some(ContainerState {
running: false,
started_at: "2026-09-18T07:01:02Z".to_string(),
exit_code: 137,
})
);
}
#[test]
fn an_unparseable_container_state_is_none_not_a_default() {
for out in ["", "\n", "true", "true|2026-09-18T07:01:02Z", "yes|t|0"] {
assert_eq!(parse_container_state(out), None, "{out:?}");
}
}
#[test]
fn docker_signal_argv_sends_one_named_signal_to_the_container() {
assert_eq!(
docker_signal_argv("zc-share-a1b2c3d4-w0", "TERM"),
vec!["docker", "kill", "--signal", "TERM", "zc-share-a1b2c3d4-w0"]
);
assert_eq!(
docker_signal_argv("zc-share-a1b2c3d4-w0", "KILL"),
vec!["docker", "kill", "--signal", "KILL", "zc-share-a1b2c3d4-w0"]
);
}
#[test]
fn docker_rm_argv_force_removes_the_container_by_name() {
assert_eq!(
docker_rm_argv("zc-share-a1b2c3d4-w0"),
vec!["docker", "rm", "-f", "zc-share-a1b2c3d4-w0"]
);
}
#[test]
fn worker_alive_argv_probes_the_recorded_pid_inside_the_container() {
let argv = worker_alive_argv("zc-share-a1b2c3d4-w0");
assert_eq!(argv[..4], ["docker", "exec", "zc-share-a1b2c3d4-w0", "sh"]);
assert_eq!(argv[4], "-c");
let inner = &argv[5];
assert!(inner.contains("/tmp/worker.pid"), "{inner}");
assert!(inner.contains("kill -0"), "{inner}");
}
#[test]
fn worker_log_tail_argv_reads_the_workers_own_log_inside_the_container() {
let argv = worker_log_tail_argv("zc-share-a1b2c3d4-w0", 20);
assert_eq!(argv[..3], ["docker", "exec", "zc-share-a1b2c3d4-w0"]);
let inner = argv.last().unwrap();
assert!(inner.contains("tail -n 20"), "{inner}");
assert!(inner.contains("/tmp/worker.log"), "{inner}");
}
#[test]
fn docker_logs_argv_is_the_fallback_tail() {
assert_eq!(
docker_logs_argv("zc-share-a1b2c3d4-w0", 20),
vec!["docker", "logs", "--tail", "20", "zc-share-a1b2c3d4-w0"]
);
}
}