use std::cell::RefCell;
use std::collections::HashSet;
use std::time::{Duration, Instant};
use super::expose::{self, MeshExposer};
use super::runtime::{container_name, deployment_labels, ContainerRuntime, RunSpec};
use super::state::{DeploymentRecord, LocalState};
use super::{Desired, Healthcheck};
#[derive(Debug, Clone, PartialEq)]
pub enum Action {
Pull {
id: String,
},
Start {
id: String,
},
Health {
id: String,
},
StopOld {
id: String,
},
Stop {
id: String,
},
Remove {
id: String,
container_id: String,
},
Forget {
id: String,
},
Report {
id: String,
phase: String,
message: Option<String>,
},
}
impl Action {
fn deployment_id(&self) -> &str {
match self {
Action::Pull { id }
| Action::Start { id }
| Action::Health { id }
| Action::StopOld { id }
| Action::Stop { id }
| Action::Remove { id, .. }
| Action::Forget { id }
| Action::Report { id, .. } => id,
}
}
}
pub fn plan(desired: &[Desired], local: &LocalState) -> Vec<Action> {
let mut actions = Vec::new();
for d in desired {
let record = local.deployments.get(&d.id);
if d.desired_state == "stopped" {
if let Some(rec) = record {
if rec.container_id.is_some() {
actions.push(Action::Stop { id: d.id.clone() });
}
}
continue;
}
let deploying = match record {
None => {
actions.push(Action::Pull { id: d.id.clone() });
actions.push(Action::Start { id: d.id.clone() });
actions.push(Action::Health { id: d.id.clone() });
true
}
Some(rec) if failed_at(rec, d.version) => false,
Some(rec) if rec.version != d.version => {
actions.push(Action::Pull { id: d.id.clone() });
actions.push(Action::Start { id: d.id.clone() });
actions.push(Action::Health { id: d.id.clone() });
actions.push(Action::StopOld { id: d.id.clone() });
true
}
Some(rec) if rec.phase != "healthy" || rec.container_id.is_none() => {
actions.push(Action::Start { id: d.id.clone() });
actions.push(Action::Health { id: d.id.clone() });
true
}
Some(_) => false, };
if deploying {
if let Some(prev) = &d.previous {
if prev.id != d.id {
actions.push(Action::Stop {
id: prev.id.clone(),
});
}
}
}
}
let handed_off: HashSet<&str> = desired
.iter()
.filter_map(|d| d.previous.as_ref().map(|p| p.id.as_str()))
.collect();
let mut unlisted: Vec<&String> = local
.deployments
.keys()
.filter(|id| !desired.iter().any(|d| &d.id == *id) && !handed_off.contains(id.as_str()))
.collect();
unlisted.sort();
for id in unlisted {
actions.push(Action::Forget { id: id.clone() });
}
actions
}
fn failed_at(rec: &DeploymentRecord, version: u64) -> bool {
rec.failed_version == Some(version)
|| (rec.version == version && matches!(rec.phase.as_str(), "failed" | "rolled_back"))
}
pub fn plan_failure(
id: &str,
failed_container_id: Option<&str>,
had_previous: bool,
) -> Vec<Action> {
let mut actions = Vec::new();
if let Some(cid) = failed_container_id {
actions.push(Action::Remove {
id: id.to_string(),
container_id: cid.to_string(),
});
}
actions.push(Action::Report {
id: id.to_string(),
phase: "failed".to_string(),
message: None,
});
if had_previous {
actions.push(Action::Report {
id: id.to_string(),
phase: "rolled_back".to_string(),
message: None,
});
}
actions
}
#[derive(Debug, Clone, PartialEq)]
pub struct FailureDetail {
pub code: &'static str,
pub message: String,
pub logs: Option<String>,
pub probes_failed: Option<u32>,
}
impl FailureDetail {
pub fn new(code: &'static str, message: impl Into<String>) -> Self {
Self {
code,
message: message.into(),
logs: None,
probes_failed: None,
}
}
}
fn last_line(text: &str) -> String {
text.lines()
.map(str::trim)
.rfind(|l| !l.is_empty())
.unwrap_or("")
.to_string()
}
pub trait StatusReporter {
#[allow(clippy::too_many_arguments)]
fn report(
&self,
id: &str,
version: u64,
phase: &str,
message: &str,
attempt: u32,
container_id: Option<&str>,
endpoint: Option<&str>,
mesh_endpoint: Option<&str>,
);
fn report_healthy(
&self,
id: &str,
version: u64,
attempt: u32,
container_id: Option<&str>,
endpoint: Option<&str>,
_logs: Option<&str>,
) {
self.report(
id,
version,
"healthy",
"",
attempt,
container_id,
endpoint,
None,
);
}
fn report_failure(
&self,
id: &str,
version: u64,
attempt: u32,
container_id: Option<&str>,
failure: &FailureDetail,
) {
self.report(
id,
version,
"failed",
&failure.message,
attempt,
container_id,
None,
None,
);
}
}
struct HealthFailure {
last_error: String,
probes_failed: u32,
}
fn wait_healthy(
rt: &dyn ContainerRuntime,
hc: &Option<Healthcheck>,
container_id: &str,
) -> Result<(), HealthFailure> {
let Some(hc) = hc else { return Ok(()) };
let deadline = Instant::now() + Duration::from_secs(hc.timeout_s.max(1));
let mut probes_failed = 0u32;
loop {
let last_error = match rt.health_probe(container_id, hc) {
Ok(()) => return Ok(()),
Err(e) => e,
};
probes_failed += 1;
if Instant::now() >= deadline {
return Err(HealthFailure {
last_error,
probes_failed,
});
}
std::thread::sleep(Duration::from_secs(2));
}
}
const LOG_TAIL_LINES: u32 = 50;
fn deployment_admission(
rt: &dyn ContainerRuntime,
id: &str,
) -> Result<crate::sandbox::Limits, String> {
let budget = rt.budget()?;
let limits = crate::sandbox::deployment_limits(&budget, &|k| std::env::var(k).ok())?;
let running = rt
.allocations()
.map_err(|e| format!("cannot read the node's allocations: {e}"))?;
let prefix = format!("zk-dep-{id}-v");
let others: Vec<_> = running
.into_iter()
.filter(|a| {
!a.name
.strip_prefix(&prefix)
.is_some_and(|v| !v.is_empty() && v.bytes().all(|b| b.is_ascii_digit()))
})
.collect();
crate::sandbox::admit(&budget, &others, &limits)?;
Ok(limits)
}
pub fn drive(
rt: &dyn ContainerRuntime,
reporter: &dyn StatusReporter,
desired: &[Desired],
local: &mut LocalState,
) {
let actions = plan(desired, local);
let mut failed: std::collections::HashSet<String> = Default::default();
let attempt: u32 = 1;
for action in actions {
let id = action.deployment_id().to_string();
if failed.contains(&id) {
continue;
}
let Some(d) = desired.iter().find(|d| d.id == id) else {
match &action {
Action::Stop { id } => {
let version = local.deployments.get(id).map(|r| r.version).unwrap_or(0);
do_stop(rt, reporter, local, id, version);
}
Action::Forget { id } => do_forget(rt, local, id),
_ => {}
}
continue;
};
match action {
Action::Pull { id } => {
reporter.report(&id, d.version, "pulling", "", attempt, None, None, None);
if let Err(e) = rt.pull(&d.image) {
let failure = FailureDetail::new("pull_failed", last_line(&e));
run_failure(reporter, local, &id, d.version, attempt, None, &failure);
failed.insert(id);
}
}
Action::Start { id } => {
reporter.report(&id, d.version, "starting", "", attempt, None, None, None);
if !d.warm {
println!(" [DEPLOY] warm=false not implemented, keeping warm ({id})");
}
let name = container_name(&id, d.version);
let labels = deployment_labels(&id, d.version);
let mut env = d.env.clone();
if let Some(sa) = &d.service_account {
env.insert("ZAKURO_SERVICE_ACCOUNT".to_string(), sa.name.clone());
env.insert("ZAKURO_GRANTS".to_string(), sa.grants.join(","));
}
let net = super::policy::network_access(d.service_account.as_ref());
let network = if net.isolated() {
println!(
" [DEPLOY] service-account policy denies network (mesh={}, internet={}): \
isolating {id}",
net.mesh, net.internet
);
Some(super::runtime::isolated_network())
} else {
None
};
let mut ports = d.ports.clone();
if let Some(hc) = &d.healthcheck {
if !ports.iter().any(|p| p.container == hc.port) {
ports.push(super::Port {
container: hc.port,
protocol: hc.kind.clone(),
});
}
}
if let Err(r) = rt.admit_guest(d) {
let code = if r.reason == crate::policy::admit::Reason::Budget {
"budget"
} else {
"start_failed"
};
let failure = FailureDetail::new(code, r.to_string());
run_failure(reporter, local, &id, d.version, attempt, None, &failure);
failed.insert(id);
continue;
}
let admission = crate::sandbox::admission_lock();
let limits = match deployment_admission(rt, &id) {
Ok(l) => l,
Err(e) => {
drop(admission);
let failure = FailureDetail::new("budget", last_line(&e));
run_failure(reporter, local, &id, d.version, attempt, None, &failure);
failed.insert(id);
continue;
}
};
let spec = RunSpec {
name,
image: &d.image,
cmd: d.cmd.as_deref(),
ports: &ports,
env: &env,
labels,
mounts: Vec::new(),
limits,
network,
hardened: false,
};
let ran = rt.run(&spec);
drop(admission);
match ran {
Ok(container_id) => {
let rec = local.deployments.entry(id.clone()).or_default();
rec.previous_container_id = rec.container_id.take();
rec.container_id = Some(container_id);
rec.image = d.image.clone();
rec.version = d.version;
rec.price_per_second = d.price_per_second;
rec.service_account = d.service_account.clone();
rec.warm = d.warm;
rec.phase = "starting".to_string();
let _ = local.save();
}
Err(e) => {
let failure = FailureDetail::new("start_failed", last_line(&e));
run_failure(reporter, local, &id, d.version, attempt, None, &failure);
failed.insert(id);
}
}
}
Action::Health { id } => {
let container_id = local
.deployments
.get(&id)
.and_then(|r| r.container_id.clone());
let Some(container_id) = container_id else {
failed.insert(id);
continue;
};
reporter.report(
&id,
d.version,
"health_check",
"",
attempt,
Some(&container_id),
None,
None,
);
let health = wait_healthy(rt, &d.healthcheck, &container_id);
if health.is_ok() {
let container_port = d
.healthcheck
.as_ref()
.map(|hc| hc.port)
.or_else(|| d.ports.first().map(|p| p.container));
let addr = match container_port {
Some(port) => rt.container_addr(&container_id, port).ok(),
None => None,
};
let endpoint = addr.as_ref().map(|(ip, port)| format!("{ip}:{port}"));
let (ip, port) = match addr {
Some((ip, port)) => (Some(ip), Some(port)),
None => (rt.container_ip(&container_id).ok(), container_port),
};
if let Some(rec) = local.deployments.get_mut(&id) {
rec.phase = "healthy".to_string();
rec.failed_version = None;
rec.ip = ip;
rec.port = port;
rec.endpoint = endpoint.clone();
}
let _ = local.save();
let logs = rt.logs_tail(&container_id, LOG_TAIL_LINES);
reporter.report_healthy(
&id,
d.version,
attempt,
Some(&container_id),
endpoint.as_deref(),
Some(logs.as_str()).filter(|l| !l.is_empty()),
);
} else if let Err(h) = health {
let mut failure = FailureDetail::new("health_failed", h.last_error);
failure.logs = Some(rt.logs_tail(&container_id, LOG_TAIL_LINES));
failure.probes_failed = Some(h.probes_failed);
let _ = rt.stop(&container_id);
let _ = rt.rm(&container_id);
let previous = local
.deployments
.get_mut(&id)
.and_then(|rec| rec.previous_container_id.take());
run_failure(
reporter,
local,
&id,
d.version,
attempt,
Some(&container_id),
&failure,
);
if let Some(prev) = previous {
let _ = rt.restart(&prev);
restore_previous_container(reporter, local, &id, d.version, attempt, prev);
}
failed.insert(id);
}
}
Action::StopOld { id } => {
if let Some(rec) = local.deployments.get_mut(&id) {
if let Some(old) = rec.previous_container_id.take() {
let _ = rt.stop(&old);
let _ = rt.rm(&old);
}
}
let _ = local.save();
}
Action::Stop { id } => do_stop(rt, reporter, local, &id, d.version),
Action::Forget { .. } => {}
Action::Remove { container_id, .. } => {
let _ = rt.stop(&container_id);
let _ = rt.rm(&container_id);
}
Action::Report { id, phase, message } => {
reporter.report(
&id,
d.version,
&phase,
message.as_deref().unwrap_or(""),
attempt,
None,
None,
None,
);
}
}
}
}
pub fn drive_and_expose(
rt: &dyn ContainerRuntime,
reporter: &dyn StatusReporter,
exposer: &dyn MeshExposer,
mesh_ip: Option<&str>,
desired: &[Desired],
local: &mut LocalState,
) {
let tracked = ReportedIds {
inner: reporter,
ids: RefCell::default(),
};
drive(rt, &tracked, desired, local);
let mut changed = false;
for id in tracked.ids.into_inner() {
if let Some(rec) = local.deployments.get_mut(&id) {
changed |= rec.mesh_endpoint.take().is_some();
}
}
let synced = expose::sync(local, exposer, mesh_ip, reporter);
if synced || changed {
let _ = local.save();
}
}
struct ReportedIds<'a> {
inner: &'a dyn StatusReporter,
ids: RefCell<HashSet<String>>,
}
impl StatusReporter for ReportedIds<'_> {
fn report(
&self,
id: &str,
version: u64,
phase: &str,
message: &str,
attempt: u32,
container_id: Option<&str>,
endpoint: Option<&str>,
mesh_endpoint: Option<&str>,
) {
self.ids.borrow_mut().insert(id.to_string());
self.inner.report(
id,
version,
phase,
message,
attempt,
container_id,
endpoint,
mesh_endpoint,
);
}
fn report_healthy(
&self,
id: &str,
version: u64,
attempt: u32,
container_id: Option<&str>,
endpoint: Option<&str>,
logs: Option<&str>,
) {
self.ids.borrow_mut().insert(id.to_string());
self.inner
.report_healthy(id, version, attempt, container_id, endpoint, logs);
}
fn report_failure(
&self,
id: &str,
version: u64,
attempt: u32,
container_id: Option<&str>,
failure: &FailureDetail,
) {
self.ids.borrow_mut().insert(id.to_string());
self.inner
.report_failure(id, version, attempt, container_id, failure);
}
}
fn do_stop(
rt: &dyn ContainerRuntime,
reporter: &dyn StatusReporter,
local: &mut LocalState,
id: &str,
version: u64,
) {
if let Some(rec) = local.deployments.get_mut(id) {
if let Some(cid) = rec.container_id.take() {
let _ = rt.stop(&cid);
let _ = rt.rm(&cid);
}
rec.phase = "stopped".to_string();
rec.endpoint = None;
rec.port = None;
}
let _ = local.save();
reporter.report(id, version, "stopped", "", 1, None, None, None);
}
fn do_forget(rt: &dyn ContainerRuntime, local: &mut LocalState, id: &str) {
if let Some(rec) = local.deployments.remove(id) {
for cid in rec
.container_id
.iter()
.chain(rec.previous_container_id.iter())
{
let _ = rt.stop(cid);
let _ = rt.rm(cid);
}
}
let _ = local.save();
}
fn run_failure(
reporter: &dyn StatusReporter,
local: &mut LocalState,
id: &str,
version: u64,
attempt: u32,
failed_container_id: Option<&str>,
failure: &FailureDetail,
) {
reporter.report_failure(id, version, attempt, failed_container_id, failure);
let rec = local.deployments.entry(id.to_string()).or_default();
rec.failed_version = Some(version);
if failed_container_id.is_some() || rec.container_id.is_none() {
rec.container_id = None;
rec.phase = "failed".to_string();
rec.version = version;
}
let _ = local.save();
}
fn restore_previous_container(
reporter: &dyn StatusReporter,
local: &mut LocalState,
id: &str,
version: u64,
attempt: u32,
previous_container_id: String,
) {
let endpoint = local.deployments.get(id).and_then(|r| r.endpoint.clone());
if let Some(rec) = local.deployments.get_mut(id) {
rec.container_id = Some(previous_container_id.clone());
rec.phase = "rolled_back".to_string();
}
let _ = local.save();
reporter.report(
id,
version,
"rolled_back",
"",
attempt,
Some(&previous_container_id),
endpoint.as_deref(),
None,
);
}
#[cfg(test)]
mod tests {
use super::*;
use crate::broker::deploy::state::DeploymentRecord;
use std::collections::HashMap;
fn desired(id: &str, version: u64, desired_state: &str) -> Desired {
Desired {
id: id.to_string(),
version,
name: format!("name-{id}"),
image: "python:3.12-slim".to_string(),
cmd: None,
ports: vec![super::super::Port {
container: 8000,
protocol: "http".to_string(),
}],
env: HashMap::new(),
healthcheck: None,
price_per_second: 0.0001,
desired_state: desired_state.to_string(),
service_account: None,
warm: true,
previous: None,
}
}
fn record(version: u64, phase: &str, container_id: Option<&str>) -> DeploymentRecord {
DeploymentRecord {
version,
container_id: container_id.map(|s| s.to_string()),
image: "python:3.12-slim".to_string(),
endpoint: None,
ip: None,
port: None,
price_per_second: 0.0001,
service_account: None,
warm: true,
phase: phase.to_string(),
previous_container_id: None,
mesh_endpoint: None,
mesh_port: None,
failed_version: None,
}
}
#[test]
fn plan_new_deployment_pulls_starts_and_checks_health() {
let d = desired("dep_1", 1, "running");
let local = LocalState::default();
let actions = plan(&[d], &local);
assert_eq!(
actions,
vec![
Action::Pull { id: "dep_1".into() },
Action::Start { id: "dep_1".into() },
Action::Health { id: "dep_1".into() },
]
);
}
#[test]
fn plan_version_bump_appends_stop_old_after_health() {
let d = desired("dep_1", 2, "running");
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("old-container")));
let actions = plan(&[d], &local);
assert_eq!(
actions,
vec![
Action::Pull { id: "dep_1".into() },
Action::Start { id: "dep_1".into() },
Action::Health { id: "dep_1".into() },
Action::StopOld { id: "dep_1".into() },
]
);
}
#[test]
fn plan_stopped_desired_state_stops_running_container() {
let d = desired("dep_1", 1, "stopped");
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("c1")));
let actions = plan(&[d], &local);
assert_eq!(actions, vec![Action::Stop { id: "dep_1".into() }]);
}
#[test]
fn plan_stopped_desired_state_with_nothing_running_is_a_noop() {
let d = desired("dep_1", 1, "stopped");
let local = LocalState::default();
assert_eq!(plan(&[d], &local), vec![]);
}
#[test]
fn plan_unchanged_and_healthy_is_a_noop() {
let d = desired("dep_1", 1, "running");
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("c1")));
assert_eq!(plan(&[d], &local), vec![]);
}
#[test]
fn plan_resumes_a_stuck_mid_flight_deployment() {
let d = desired("dep_1", 1, "running");
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "starting", Some("c1")));
let actions = plan(&[d], &local);
assert_eq!(
actions,
vec![
Action::Start { id: "dep_1".into() },
Action::Health { id: "dep_1".into() },
]
);
}
#[test]
fn plan_does_not_retry_a_failed_record_at_the_same_version() {
let d = desired("dep_1", 1, "running");
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "failed", None));
assert_eq!(plan(std::slice::from_ref(&d), &local), vec![]);
local
.deployments
.insert("dep_1".into(), record(1, "rolled_back", Some("old")));
assert_eq!(plan(&[d], &local), vec![]);
}
#[test]
fn plan_retries_a_failed_record_once_the_version_is_bumped() {
let d = desired("dep_1", 2, "running");
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "failed", None));
assert_eq!(
plan(&[d], &local),
vec![
Action::Pull { id: "dep_1".into() },
Action::Start { id: "dep_1".into() },
Action::Health { id: "dep_1".into() },
Action::StopOld { id: "dep_1".into() },
]
);
}
#[test]
fn plan_does_not_retry_a_failed_version_whose_record_is_the_old_healthy_one() {
let d = desired("dep_1", 2, "running");
let mut rec = record(1, "healthy", Some("old"));
rec.failed_version = Some(2);
let mut local = LocalState::default();
local.deployments.insert("dep_1".into(), rec);
assert_eq!(plan(&[d], &local), vec![]);
}
#[test]
fn plan_still_resumes_records_stuck_in_every_in_flight_phase() {
for phase in ["pulling", "starting", "health_check"] {
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, phase, Some("c1")));
assert_eq!(
plan(&[desired("dep_1", 1, "running")], &local),
vec![
Action::Start { id: "dep_1".into() },
Action::Health { id: "dep_1".into() },
],
"{phase}"
);
}
}
#[test]
fn last_line_skips_trailing_blank_lines() {
assert_eq!(last_line("a\nb\n\n"), "b");
assert_eq!(last_line(""), "");
}
#[test]
fn plan_previous_naming_another_deployment_stops_it_on_handoff() {
let mut d = desired("dep_2", 1, "running");
d.previous = Some(Box::new(super::super::PreviousDeployment {
id: "dep_1".into(),
image: "old-image".into(),
cmd: None,
ports: vec![],
env: HashMap::new(),
healthcheck: None,
}));
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("old-container")));
let actions = plan(&[d], &local);
assert_eq!(
actions,
vec![
Action::Pull { id: "dep_2".into() },
Action::Start { id: "dep_2".into() },
Action::Health { id: "dep_2".into() },
Action::Stop { id: "dep_1".into() },
]
);
}
#[test]
fn plan_previous_naming_the_same_deployment_does_not_double_stop() {
let mut d = desired("dep_1", 2, "running");
d.previous = Some(Box::new(super::super::PreviousDeployment {
id: "dep_1".into(),
image: "python:3.12-slim".into(),
cmd: None,
ports: vec![],
env: HashMap::new(),
healthcheck: None,
}));
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("old-container")));
let actions = plan(&[d], &local);
assert_eq!(
actions,
vec![
Action::Pull { id: "dep_1".into() },
Action::Start { id: "dep_1".into() },
Action::Health { id: "dep_1".into() },
Action::StopOld { id: "dep_1".into() },
]
);
}
#[test]
fn plan_failure_with_container_and_previous_removes_reports_failed_then_rolled_back() {
let actions = plan_failure("dep_1", Some("new-c"), true);
assert_eq!(
actions,
vec![
Action::Remove {
id: "dep_1".into(),
container_id: "new-c".into(),
},
Action::Report {
id: "dep_1".into(),
phase: "failed".into(),
message: None,
},
Action::Report {
id: "dep_1".into(),
phase: "rolled_back".into(),
message: None,
},
]
);
}
#[test]
fn plan_failure_without_container_or_previous_is_just_a_failed_report() {
let actions = plan_failure("dep_1", None, false);
assert_eq!(
actions,
vec![Action::Report {
id: "dep_1".into(),
phase: "failed".into(),
message: None,
}]
);
}
use std::sync::Mutex;
#[derive(Default)]
struct FakeRuntime {
next_container_id: Mutex<u32>,
run_should_fail: Mutex<bool>,
pull_error: Mutex<Option<String>>,
health_should_fail: Mutex<bool>,
stopped: Mutex<Vec<String>>,
removed: Mutex<Vec<String>>,
restarted: Mutex<Vec<String>>,
last_run_env: Mutex<HashMap<String, String>>,
last_run_network: Mutex<Option<String>>,
last_run_ports: Mutex<Vec<u16>>,
published: Mutex<Option<(String, u16)>>,
budget_error: Mutex<Option<String>>,
run_limits: Mutex<Vec<crate::sandbox::Limits>>,
running: Mutex<Vec<crate::sandbox::Allocation>>,
refuse_guest: Mutex<Option<crate::policy::admit::Reason>>,
}
impl ContainerRuntime for FakeRuntime {
fn available(&self) -> bool {
true
}
fn admit_guest(&self, _d: &Desired) -> Result<(), crate::policy::admit::Refusal> {
match *self.refuse_guest.lock().unwrap() {
Some(reason) => Err(crate::policy::admit::Refusal {
reason,
detail: "the test policy says no".into(),
source: crate::policy::admit::Source::EnvGuest,
squad: "public".into(),
at: 0,
}),
None => Ok(()),
}
}
fn budget(&self) -> Result<crate::sandbox::Limits, String> {
match self.budget_error.lock().unwrap().clone() {
Some(e) => Err(e),
None => Ok(crate::sandbox::Limits::new(
2.0,
4.0,
crate::sandbox::Gpus::None,
)),
}
}
fn allocations(&self) -> Result<Vec<crate::sandbox::Allocation>, String> {
Ok(self.running.lock().unwrap().clone())
}
fn pull(&self, _image: &str) -> Result<(), String> {
match self.pull_error.lock().unwrap().clone() {
Some(e) => Err(e),
None => Ok(()),
}
}
fn run(&self, spec: &RunSpec) -> Result<String, String> {
self.run_limits.lock().unwrap().push(spec.limits.clone());
*self.last_run_env.lock().unwrap() = spec.env.clone();
*self.last_run_network.lock().unwrap() = spec.network.clone();
*self.last_run_ports.lock().unwrap() = spec.ports.iter().map(|p| p.container).collect();
if *self.run_should_fail.lock().unwrap() {
return Err(
"docker: Error response from daemon\nno space left on device\n".to_string(),
);
}
let mut n = self.next_container_id.lock().unwrap();
*n += 1;
Ok(format!("c{n}"))
}
fn stop(&self, container_id: &str) -> Result<(), String> {
self.stopped.lock().unwrap().push(container_id.to_string());
Ok(())
}
fn rm(&self, container_id: &str) -> Result<(), String> {
self.removed.lock().unwrap().push(container_id.to_string());
Ok(())
}
fn restart(&self, container_id: &str) -> Result<(), String> {
self.restarted
.lock()
.unwrap()
.push(container_id.to_string());
Ok(())
}
fn is_running(&self, _container_id: &str) -> bool {
true
}
fn container_ip(&self, _container_id: &str) -> Result<String, String> {
Ok("172.17.0.5".to_string())
}
fn container_addr(&self, container_id: &str, port: u16) -> Result<(String, u16), String> {
match self.published.lock().unwrap().clone() {
Some(addr) => Ok(addr),
None => self.container_ip(container_id).map(|ip| (ip, port)),
}
}
fn health_check(&self, _container_id: &str, _hc: &Healthcheck) -> bool {
!*self.health_should_fail.lock().unwrap()
}
fn health_probe(&self, _container_id: &str, _hc: &Healthcheck) -> Result<(), String> {
if *self.health_should_fail.lock().unwrap() {
Err("GET http://172.17.0.5:8000/ → connection refused".to_string())
} else {
Ok(())
}
}
fn logs_tail(&self, _container_id: &str, lines: u32) -> String {
format!("boom (tail {lines})")
}
fn list_by_label(&self, _id: &str) -> Vec<String> {
vec![]
}
}
#[derive(Default)]
struct RecordingReporter {
calls: Mutex<Vec<(String, u64, String)>>, meshes: Mutex<Vec<Option<String>>>, failures: Mutex<Vec<(Option<String>, FailureDetail)>>, healthy_logs: Mutex<Vec<Option<String>>>, }
impl StatusReporter for RecordingReporter {
fn report(
&self,
id: &str,
version: u64,
phase: &str,
_message: &str,
_attempt: u32,
_container_id: Option<&str>,
_endpoint: Option<&str>,
mesh_endpoint: Option<&str>,
) {
self.calls
.lock()
.unwrap()
.push((id.to_string(), version, phase.to_string()));
self.meshes
.lock()
.unwrap()
.push(mesh_endpoint.map(str::to_string));
}
fn report_healthy(
&self,
id: &str,
version: u64,
attempt: u32,
container_id: Option<&str>,
endpoint: Option<&str>,
logs: Option<&str>,
) {
self.healthy_logs
.lock()
.unwrap()
.push(logs.map(str::to_string));
self.report(
id,
version,
"healthy",
"",
attempt,
container_id,
endpoint,
None,
);
}
fn report_failure(
&self,
id: &str,
version: u64,
attempt: u32,
container_id: Option<&str>,
failure: &FailureDetail,
) {
self.failures
.lock()
.unwrap()
.push((container_id.map(str::to_string), failure.clone()));
self.report(
id,
version,
"failed",
&failure.message,
attempt,
container_id,
None,
None,
);
}
}
#[test]
fn drive_new_deployment_goes_healthy_and_persists_endpoint() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let d = desired("dep_1", 1, "running");
let mut local = LocalState::default();
drive(&rt, &reporter, &[d], &mut local);
let rec = local.deployments.get("dep_1").expect("recorded");
assert_eq!(rec.phase, "healthy");
assert_eq!(rec.container_id.as_deref(), Some("c1"));
assert_eq!(rec.endpoint.as_deref(), Some("172.17.0.5:8000"));
assert_eq!(rec.ip.as_deref(), Some("172.17.0.5"));
let calls = reporter.calls.lock().unwrap();
let phases: Vec<&str> = calls.iter().map(|(_, _, p)| p.as_str()).collect();
assert_eq!(
phases,
vec!["pulling", "starting", "health_check", "healthy"]
);
}
#[test]
fn drive_passes_service_account_to_the_container_as_env_and_persists_it() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let mut d = desired("dep_1", 1, "running");
d.service_account = Some(super::super::ServiceAccount {
name: "billing-svc".to_string(),
grants: vec!["kv:read".to_string(), "queue:publish".to_string()],
policy: None,
});
let mut local = LocalState::default();
drive(&rt, &reporter, &[d], &mut local);
let env = rt.last_run_env.lock().unwrap();
assert_eq!(
env.get("ZAKURO_SERVICE_ACCOUNT").map(String::as_str),
Some("billing-svc")
);
assert_eq!(
env.get("ZAKURO_GRANTS").map(String::as_str),
Some("kv:read,queue:publish")
);
let rec = local.deployments.get("dep_1").unwrap();
assert_eq!(
rec.service_account.as_ref().map(|sa| sa.name.as_str()),
Some("billing-svc")
);
}
fn run_with_policy(grants: &[&str], policy: Option<serde_json::Value>) -> Option<String> {
let rt = FakeRuntime::default();
let mut d = desired("dep_1", 1, "running");
d.service_account = Some(super::super::ServiceAccount {
name: "svc".to_string(),
grants: grants.iter().map(|s| s.to_string()).collect(),
policy,
});
drive(
&rt,
&RecordingReporter::default(),
&[d],
&mut LocalState::default(),
);
let n = rt.last_run_network.lock().unwrap().clone();
n
}
#[test]
fn drive_isolates_a_workload_whose_policy_denies_network() {
let _home = expose::TempZakuroHome::new();
let deny = serde_json::json!({"Statement":[
{"Effect":"Allow","Action":"datasets:*","Resource":"arn:zakuro:datasets:::*"}]});
assert_eq!(
run_with_policy(&["network:mesh"], Some(deny)),
Some(crate::broker::deploy::runtime::isolated_network())
);
assert!(run_with_policy(&[], Some(serde_json::json!("garbage"))).is_some());
let p = serde_json::json!({"Statement":[
{"Effect":"Allow","Action":"*:*","Resource":"arn:zakuro:*:::*"},
{"Effect":"Deny","Action":"network:*","Resource":"arn:zakuro:network:::*"}]});
assert!(run_with_policy(&[], Some(p)).is_some());
}
#[test]
fn drive_keeps_the_default_network_without_an_enforced_policy() {
let _home = expose::TempZakuroHome::new();
assert_eq!(run_with_policy(&["network:mesh"], None), None);
let legacy = serde_json::json!({"Version":"2026-01-01","Statement":[
{"Sid":"LegacyGrant0","Effect":"Allow","Action":"network:mesh",
"Resource":"arn:zakuro:network:::*"}]});
assert_eq!(run_with_policy(&["network:mesh"], Some(legacy)), None);
let allow = serde_json::json!({"Statement":[
{"Effect":"Allow","Action":"network:Connect","Resource":"arn:zakuro:network:::*"}]});
assert_eq!(run_with_policy(&[], Some(allow)), None);
}
#[test]
fn desired_without_policy_still_parses_and_policy_round_trips() {
let old: super::super::ServiceAccount =
serde_json::from_str(r#"{"name":"default","grants":["network:mesh"]}"#).unwrap();
assert!(old.policy.is_none());
let new: super::super::ServiceAccount = serde_json::from_str(
r#"{"name":"a","grants":[],"policy":{"Statement":"not even valid"}}"#,
)
.unwrap();
assert!(new.policy.is_some());
}
#[test]
fn drive_records_the_published_loopback_address_when_the_runtime_has_one() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.published.lock().unwrap() = Some(("127.0.0.1".to_string(), 49153));
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
drive(
&rt,
&reporter,
&[desired("dep_1", 1, "running")],
&mut local,
);
let rec = local.deployments.get("dep_1").expect("recorded");
assert_eq!(rec.phase, "healthy");
assert_eq!(rec.endpoint.as_deref(), Some("127.0.0.1:49153"));
assert_eq!(rec.ip.as_deref(), Some("127.0.0.1"));
assert_eq!(rec.port, Some(49153));
}
#[test]
fn drive_runs_with_the_healthcheck_port_even_when_ports_omit_it() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let mut d = desired("dep_1", 1, "running");
d.healthcheck = Some(Healthcheck {
kind: "http".to_string(),
port: 8888,
path: Some("/api".to_string()),
timeout_s: 5,
});
let mut local = LocalState::default();
drive(&rt, &reporter, &[d], &mut local);
assert_eq!(*rt.last_run_ports.lock().unwrap(), vec![8000, 8888]);
}
#[test]
fn drive_version_bump_stops_old_container_after_healthy() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let d = desired("dep_1", 2, "running");
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("old-container")));
drive(&rt, &reporter, &[d], &mut local);
assert!(rt
.stopped
.lock()
.unwrap()
.contains(&"old-container".to_string()));
assert!(rt
.removed
.lock()
.unwrap()
.contains(&"old-container".to_string()));
let rec = local.deployments.get("dep_1").unwrap();
assert_eq!(rec.phase, "healthy");
assert_eq!(rec.version, 2);
assert!(rec.previous_container_id.is_none());
}
#[test]
fn drive_start_failure_leaves_existing_container_untouched() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.run_should_fail.lock().unwrap() = true;
let reporter = RecordingReporter::default();
let d = desired("dep_1", 2, "running");
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("old-container")));
drive(&rt, &reporter, &[d], &mut local);
let rec = local.deployments.get("dep_1").unwrap();
assert_eq!(rec.phase, "healthy");
assert_eq!(rec.version, 1);
assert_eq!(rec.container_id.as_deref(), Some("old-container"));
assert!(rt.restarted.lock().unwrap().is_empty());
assert!(rt.stopped.lock().unwrap().is_empty());
assert!(rt.removed.lock().unwrap().is_empty());
let calls = reporter.calls.lock().unwrap();
let phases: Vec<&str> = calls.iter().map(|(_, _, p)| p.as_str()).collect();
assert_eq!(phases, vec!["pulling", "starting", "failed"]);
}
#[test]
fn drive_health_failure_rolls_back_to_previous_container() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.health_should_fail.lock().unwrap() = true;
let reporter = RecordingReporter::default();
let mut d = desired("dep_1", 2, "running");
d.healthcheck = Some(Healthcheck {
kind: "http".to_string(),
port: 8000,
path: None,
timeout_s: 1,
});
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("old-container")));
drive(&rt, &reporter, &[d], &mut local);
let rec = local.deployments.get("dep_1").unwrap();
assert_eq!(rec.phase, "rolled_back");
assert_eq!(rec.container_id.as_deref(), Some("old-container"));
assert!(rt
.restarted
.lock()
.unwrap()
.contains(&"old-container".to_string()));
assert!(rt.stopped.lock().unwrap().contains(&"c1".to_string()));
assert!(rt.removed.lock().unwrap().contains(&"c1".to_string()));
assert!(!rt
.removed
.lock()
.unwrap()
.contains(&"old-container".to_string()));
let calls = reporter.calls.lock().unwrap();
let phases: Vec<&str> = calls.iter().map(|(_, _, p)| p.as_str()).collect();
assert_eq!(
phases,
vec![
"pulling",
"starting",
"health_check",
"failed",
"rolled_back"
]
);
drop(calls);
let failures = reporter.failures.lock().unwrap();
assert_eq!(failures.len(), 1);
let (cid, f) = &failures[0];
assert_eq!(cid.as_deref(), Some("c1"));
assert_eq!(f.code, "health_failed");
assert_eq!(
f.message,
"GET http://172.17.0.5:8000/ → connection refused"
);
assert_eq!(f.logs.as_deref(), Some("boom (tail 50)"));
assert!(f.probes_failed.unwrap() >= 1);
}
#[test]
fn drive_run_failure_with_no_previous_leaves_deployment_failed() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.run_should_fail.lock().unwrap() = true;
let reporter = RecordingReporter::default();
let d = desired("dep_1", 1, "running");
let mut local = LocalState::default();
drive(&rt, &reporter, &[d], &mut local);
let rec = local.deployments.get("dep_1").unwrap();
assert_eq!(rec.phase, "failed");
assert_eq!(rec.version, 1);
assert!(rec.container_id.is_none());
let calls = reporter.calls.lock().unwrap();
let phases: Vec<&str> = calls.iter().map(|(_, _, p)| p.as_str()).collect();
assert_eq!(phases, vec!["pulling", "starting", "failed"]);
drop(calls);
let failures = reporter.failures.lock().unwrap();
let (cid, f) = &failures[0];
assert!(cid.is_none());
assert_eq!(f.code, "start_failed");
assert_eq!(f.message, "no space left on device");
assert!(f.logs.is_none() && f.probes_failed.is_none());
}
#[test]
fn drive_runs_every_deployment_at_its_slice_of_the_budget() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
drive(
&rt,
&reporter,
&[desired("dep_1", 1, "running")],
&mut local,
);
assert_eq!(
rt.run_limits.lock().unwrap().as_slice(),
[crate::sandbox::Limits::new(
0.5,
1.0,
crate::sandbox::Gpus::None
)]
);
}
fn running(name: &str, cpus: f64, gib: f64) -> crate::sandbox::Allocation {
let l = crate::sandbox::Limits::new(cpus, gib, crate::sandbox::Gpus::None);
crate::sandbox::Allocation::from_label(name, &l.budget_label()).unwrap()
}
#[test]
fn drive_refuses_a_deployment_past_the_node_budget() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.running.lock().unwrap() = vec![
running("zakuro-share-w0", 1.0, 1.0),
running("zk-space-x", 0.75, 1.0),
];
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
drive(
&rt,
&reporter,
&[desired("dep_1", 1, "running")],
&mut local,
);
assert!(rt.run_limits.lock().unwrap().is_empty(), "no docker run");
assert_eq!(local.deployments.get("dep_1").unwrap().phase, "failed");
let failures = reporter.failures.lock().unwrap();
let (cid, f) = &failures[0];
assert!(cid.is_none());
assert_eq!(f.code, "budget");
assert!(
f.message.contains("cpu: asks 0.5 of the 0.25 left"),
"{}",
f.message
);
}
#[test]
fn drive_refuses_a_guest_the_node_policy_does_not_allow() {
use crate::policy::admit::Reason;
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.refuse_guest.lock().unwrap() = Some(Reason::Workload);
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
drive(
&rt,
&reporter,
&[desired("dep_1", 1, "running")],
&mut local,
);
assert!(rt.run_limits.lock().unwrap().is_empty(), "no docker run");
assert_eq!(local.deployments.get("dep_1").unwrap().phase, "failed");
let failures = reporter.failures.lock().unwrap();
assert_eq!(failures[0].1.code, "start_failed");
assert!(
failures[0].1.message.contains("refused (workload)"),
"{}",
failures[0].1.message
);
drop(failures);
*rt.refuse_guest.lock().unwrap() = Some(Reason::Budget);
let mut local = LocalState::default();
drive(
&rt,
&reporter,
&[desired("dep_2", 1, "running")],
&mut local,
);
assert_eq!(reporter.failures.lock().unwrap()[1].1.code, "budget");
}
#[test]
fn drive_admits_once_what_held_the_budget_has_stopped() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.running.lock().unwrap() = vec![running("zk-space-x", 2.0, 4.0)];
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
drive(
&rt,
&reporter,
&[desired("dep_1", 1, "running")],
&mut local,
);
assert!(rt.run_limits.lock().unwrap().is_empty());
rt.running.lock().unwrap().clear();
drive(
&rt,
&reporter,
&[desired("dep_1", 2, "running")],
&mut local,
);
assert_eq!(rt.run_limits.lock().unwrap().len(), 1);
assert_eq!(local.deployments.get("dep_1").unwrap().phase, "healthy");
}
#[test]
fn a_deployments_own_versions_do_not_count_against_its_upgrade() {
let rt = FakeRuntime::default();
*rt.running.lock().unwrap() = vec![
running("zk-dep-dep_1-v1", 0.5, 1.0),
running("zk-dep-dep_10-v3", 0.5, 1.0),
running("zakuro-share-w0", 1.0, 2.0),
];
assert!(deployment_admission(&rt, "dep_1").is_ok());
let e = deployment_admission(&rt, "dep_2").unwrap_err();
assert!(e.contains("node budget exhausted"), "{e}");
}
#[test]
fn drive_an_invalid_budget_fails_the_start_without_running_anything() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.budget_error.lock().unwrap() =
Some("ZAKURO_BUDGET_CPU=\"lots\" is not a cpu count or a percentage".into());
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
drive(
&rt,
&reporter,
&[desired("dep_1", 1, "running")],
&mut local,
);
assert!(rt.run_limits.lock().unwrap().is_empty(), "no docker run");
let rec = local.deployments.get("dep_1").unwrap();
assert_eq!(rec.phase, "failed");
assert!(rec.container_id.is_none());
let failures = reporter.failures.lock().unwrap();
let (cid, f) = &failures[0];
assert!(cid.is_none());
assert_eq!(f.code, "budget");
assert!(f.message.contains("ZAKURO_BUDGET_CPU"), "{}", f.message);
}
#[test]
fn drive_pull_failure_reports_pull_failed_and_stops_the_pass() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.pull_error.lock().unwrap() =
Some("Pulling from x\n\ndenied: requested access is denied\n".to_string());
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
drive(
&rt,
&reporter,
&[desired("dep_1", 1, "running")],
&mut local,
);
let calls = reporter.calls.lock().unwrap();
let phases: Vec<&str> = calls.iter().map(|(_, _, p)| p.as_str()).collect();
assert_eq!(phases, vec!["pulling", "failed"]);
drop(calls);
let failures = reporter.failures.lock().unwrap();
assert_eq!(failures[0].1.code, "pull_failed");
assert_eq!(failures[0].1.message, "denied: requested access is denied");
assert_eq!(local.deployments["dep_1"].phase, "failed");
}
#[test]
fn drive_does_not_retry_a_failed_version_on_the_next_tick() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.run_should_fail.lock().unwrap() = true;
let reporter = RecordingReporter::default();
let d = desired("dep_1", 1, "running");
let mut local = LocalState::default();
drive(&rt, &reporter, std::slice::from_ref(&d), &mut local);
let first = reporter.calls.lock().unwrap().len();
drive(&rt, &reporter, &[d], &mut local);
assert_eq!(reporter.calls.lock().unwrap().len(), first);
}
#[test]
fn drive_retries_once_the_hub_bumps_the_version() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
*rt.run_should_fail.lock().unwrap() = true;
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
drive(
&rt,
&reporter,
&[desired("dep_1", 1, "running")],
&mut local,
);
*rt.run_should_fail.lock().unwrap() = false;
drive(
&rt,
&reporter,
&[desired("dep_1", 2, "running")],
&mut local,
);
let rec = &local.deployments["dep_1"];
assert_eq!(rec.phase, "healthy");
assert_eq!(rec.version, 2);
assert_eq!(rec.failed_version, None);
}
#[test]
fn drive_stopped_desired_state_stops_and_reports() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let d = desired("dep_1", 1, "stopped");
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("c1")));
drive(&rt, &reporter, &[d], &mut local);
assert!(rt.stopped.lock().unwrap().contains(&"c1".to_string()));
let rec = local.deployments.get("dep_1").unwrap();
assert_eq!(rec.phase, "stopped");
assert!(rec.container_id.is_none());
let calls = reporter.calls.lock().unwrap();
let phases: Vec<&str> = calls.iter().map(|(_, _, p)| p.as_str()).collect();
assert_eq!(phases, vec!["stopped"]);
}
fn exposed_record(ip: &str, container_id: &str) -> DeploymentRecord {
let mut rec = record(1, "healthy", Some(container_id));
rec.ip = Some(ip.to_string());
rec.port = Some(8000);
rec.endpoint = Some(format!("{ip}:8000"));
rec.mesh_endpoint = Some("10.13.13.22:8000".to_string());
rec.mesh_port = Some(8000);
rec
}
#[test]
fn drive_and_expose_reports_a_new_deployment_healthy_then_its_mesh_endpoint() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let exposer = expose::FakeExposer::default();
let mut local = LocalState::default();
drive_and_expose(
&rt,
&reporter,
&exposer,
Some("10.13.13.22"),
&[desired("dep_1", 1, "running")],
&mut local,
);
let calls = reporter.calls.lock().unwrap();
let phases: Vec<&str> = calls.iter().map(|(_, _, p)| p.as_str()).collect();
assert_eq!(
phases,
vec!["pulling", "starting", "health_check", "healthy", "healthy"]
);
assert_eq!(
*reporter.meshes.lock().unwrap(),
vec![None, None, None, None, Some("10.13.13.22:8000".to_string())]
);
assert_eq!(
exposer.target_of("dep_1").as_deref(),
Some("172.17.0.5:8000")
);
assert_eq!(
local.deployments["dep_1"].mesh_endpoint.as_deref(),
Some("10.13.13.22:8000")
);
}
#[test]
fn drive_and_expose_re_reports_the_mesh_endpoint_after_a_version_bump() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let exposer = expose::FakeExposer::default();
exposer.expose("dep_1", "10.13.13.22", "172.17.0.4:8000", 8000);
let mut local = LocalState::default();
local.deployments.insert(
"dep_1".into(),
exposed_record("172.17.0.4", "old-container"),
);
drive_and_expose(
&rt,
&reporter,
&exposer,
Some("10.13.13.22"),
&[desired("dep_1", 2, "running")],
&mut local,
);
let last = reporter.calls.lock().unwrap().last().cloned();
assert_eq!(last, Some(("dep_1".to_string(), 2, "healthy".to_string())));
let last_mesh = reporter.meshes.lock().unwrap().last().cloned().flatten();
assert_eq!(last_mesh.as_deref(), Some("10.13.13.22:8000"));
assert_eq!(
exposer.target_of("dep_1").as_deref(),
Some("172.17.0.5:8000")
);
}
#[test]
fn drive_and_expose_releases_a_stopped_deployment() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let exposer = expose::FakeExposer::default();
exposer.expose("dep_1", "10.13.13.22", "172.17.0.4:8000", 8000);
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), exposed_record("172.17.0.4", "c1"));
drive_and_expose(
&rt,
&reporter,
&exposer,
Some("10.13.13.22"),
&[desired("dep_1", 1, "stopped")],
&mut local,
);
assert!(exposer.exposed_ids().is_empty());
assert_eq!(local.deployments["dep_1"].mesh_endpoint, None);
assert_eq!(*reporter.meshes.lock().unwrap(), vec![None]);
}
#[test]
fn drive_stop_reports_the_desired_version_not_the_local_one() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("c1")));
drive(
&rt,
&reporter,
&[desired("dep_1", 2, "stopped")],
&mut local,
);
assert_eq!(
*reporter.calls.lock().unwrap(),
vec![("dep_1".to_string(), 2, "stopped".to_string())]
);
}
#[test]
fn drive_handoff_stop_of_an_unlisted_deployment_reports_its_local_version() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let mut d = desired("dep_2", 1, "running");
d.previous = Some(Box::new(super::super::PreviousDeployment {
id: "dep_1".into(),
image: "old-image".into(),
cmd: None,
ports: vec![],
env: HashMap::new(),
healthcheck: None,
}));
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(3, "healthy", Some("old-container")));
drive(&rt, &reporter, &[d], &mut local);
let calls = reporter.calls.lock().unwrap();
assert!(
calls.contains(&("dep_1".to_string(), 3, "stopped".to_string())),
"{calls:?}"
);
}
#[test]
fn plan_empty_desired_forgets_every_local_record() {
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("c1")));
assert_eq!(
plan(&[], &local),
vec![Action::Forget { id: "dep_1".into() }]
);
}
#[test]
fn plan_only_forgets_the_unlisted_record() {
let mut local = LocalState::default();
local
.deployments
.insert("dep_a".into(), record(1, "healthy", Some("ca")));
local
.deployments
.insert("dep_b".into(), record(1, "healthy", Some("cb")));
let actions = plan(&[desired("dep_a", 1, "running")], &local);
assert_eq!(actions, vec![Action::Forget { id: "dep_b".into() }]);
}
#[test]
fn plan_forgets_an_unlisted_record_that_is_already_stopped() {
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "stopped", None));
assert_eq!(
plan(&[], &local),
vec![Action::Forget { id: "dep_1".into() }]
);
}
#[test]
fn plan_leaves_a_handoff_previous_to_the_handoff_stop() {
let mut d = desired("dep_2", 1, "running");
d.previous = Some(Box::new(super::super::PreviousDeployment {
id: "dep_1".into(),
image: "old-image".into(),
cmd: None,
ports: vec![],
env: HashMap::new(),
healthcheck: None,
}));
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "healthy", Some("c1")));
let actions = plan(&[d], &local);
assert!(!actions.iter().any(|a| matches!(a, Action::Forget { .. })));
assert!(actions.contains(&Action::Stop { id: "dep_1".into() }));
}
#[test]
fn drive_unlisted_record_is_stopped_removed_and_forgotten_silently() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
local
.deployments
.insert("dep_a".into(), record(1, "healthy", Some("ca")));
local
.deployments
.insert("dep_b".into(), record(1, "healthy", Some("cb")));
drive(
&rt,
&reporter,
&[desired("dep_a", 1, "running")],
&mut local,
);
assert_eq!(*rt.stopped.lock().unwrap(), vec!["cb".to_string()]);
assert_eq!(*rt.removed.lock().unwrap(), vec!["cb".to_string()]);
assert!(!local.deployments.contains_key("dep_b"));
assert!(local.deployments.contains_key("dep_a"));
assert!(reporter.calls.lock().unwrap().is_empty());
}
#[test]
fn drive_unlisted_stopped_record_is_just_forgotten() {
let _home = expose::TempZakuroHome::new();
let rt = FakeRuntime::default();
let reporter = RecordingReporter::default();
let mut local = LocalState::default();
local
.deployments
.insert("dep_1".into(), record(1, "stopped", None));
drive(&rt, &reporter, &[], &mut local);
assert!(local.deployments.is_empty());
assert!(rt.stopped.lock().unwrap().is_empty());
assert!(rt.removed.lock().unwrap().is_empty());
}
}