use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use crate::backend::{SandboxBackend, SandboxHandle};
use crate::backends;
use crate::checkpoint::{self, Checkpoint};
use crate::cleanup::{self, CleanupJob};
use crate::error::{Result, RightsizeError};
use crate::free_ports::FreePorts;
use crate::model::{ContainerSpec, ExecResult, FileMount};
use crate::mountable_file::MountableFile;
use crate::network::{Network, NetworkMember};
use crate::run_id::RunId;
use crate::wait::{Wait, WaitStrategy, WaitTarget};
const PORT_BIND_ATTEMPTS: usize = 5;
static NAME_COUNTER: AtomicU64 = AtomicU64::new(0);
static FREE_PORTS: std::sync::OnceLock<FreePorts> = std::sync::OnceLock::new();
fn free_ports() -> &'static FreePorts {
FREE_PORTS.get_or_init(FreePorts::new)
}
type SpecCustomizer = dyn Fn(ContainerSpec, &dyn Fn(u16) -> u16) -> ContainerSpec + Send + Sync;
type PostStartHook = dyn Fn(&ContainerGuard) -> crate::BoxFuture<'_, Result<()>> + Send + Sync;
pub struct Container {
image: String,
env: Vec<(String, String)>,
exposed_ports: Vec<u16>,
command: Option<Vec<String>>,
network: Option<Arc<Network>>,
aliases: Vec<String>,
mounts: Vec<FileMount>,
wait_strategy: Arc<dyn WaitStrategy>,
memory_limit_mb: Option<u64>,
backend_override: Option<Arc<dyn SandboxBackend>>,
spec_customizer: Option<Arc<SpecCustomizer>>,
post_start: Option<Arc<PostStartHook>>,
reuse: bool,
cache_dir_override: Option<std::path::PathBuf>,
reuse_env_override: Option<bool>,
require_isolation: bool,
reaper_cache_dir_override: Option<std::path::PathBuf>,
checkpoint_ref: Option<String>,
checkpoint_backend: Option<String>,
checkpoint_cache_dir_override: Option<std::path::PathBuf>,
}
impl Container {
pub fn new(image: &str) -> Container {
Container {
image: image.to_string(),
env: Vec::new(),
exposed_ports: Vec::new(),
command: None,
network: None,
aliases: Vec::new(),
mounts: Vec::new(),
wait_strategy: Arc::from(Wait::for_listening_port()),
memory_limit_mb: None,
backend_override: None,
spec_customizer: None,
post_start: None,
reuse: false,
cache_dir_override: None,
reuse_env_override: None,
require_isolation: false,
reaper_cache_dir_override: None,
checkpoint_ref: None,
checkpoint_backend: None,
checkpoint_cache_dir_override: None,
}
}
pub fn from_checkpoint(cp: &Checkpoint) -> Container {
let mut c = Container::new(&cp.checkpoint_ref);
c.env = cp.spec.env.clone();
c.command = cp.spec.command.clone();
c.exposed_ports = cp.spec.ports.iter().map(|p| p.guest_port).collect();
c.memory_limit_mb = cp.spec.memory_limit_mb;
c.checkpoint_ref = Some(cp.checkpoint_ref.clone());
c.checkpoint_backend = Some(cp.backend.clone());
c
}
pub fn with_env(mut self, k: &str, v: &str) -> Self {
self.env.push((k.to_string(), v.to_string()));
self
}
pub fn remove_env(mut self, key: &str) -> Self {
self.env.retain(|(k, _)| k != key);
self
}
pub fn env(&self) -> &[(String, String)] {
&self.env
}
pub fn with_exposed_ports(mut self, ports: &[u16]) -> Self {
self.exposed_ports.extend_from_slice(ports);
self
}
pub fn with_command(mut self, cmd: &[&str]) -> Self {
self.command = Some(cmd.iter().map(|s| s.to_string()).collect());
self
}
pub fn with_network(mut self, net: &Arc<Network>) -> Self {
self.network = Some(net.clone());
self
}
pub fn with_network_aliases(mut self, names: &[&str]) -> Self {
self.aliases.extend(names.iter().map(|s| s.to_string()));
self
}
pub fn with_copy_file_to_container(mut self, file: MountableFile, guest_path: &str) -> Self {
self.mounts.push(FileMount::new(file.path(), guest_path));
self
}
pub fn waiting_for(mut self, strategy: impl WaitStrategy + 'static) -> Self {
self.wait_strategy = Arc::new(strategy);
self
}
pub fn with_memory_limit(mut self, megabytes: u64) -> Self {
self.memory_limit_mb = Some(megabytes);
self
}
pub fn reuse(mut self, enabled: bool) -> Self {
self.reuse = enabled;
self
}
pub fn require_isolation(mut self, enabled: bool) -> Self {
self.require_isolation = enabled;
self
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn with_backend(mut self, b: Arc<dyn SandboxBackend>) -> Self {
self.backend_override = Some(b);
self
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn with_cache_dir_override(mut self, dir: std::path::PathBuf) -> Self {
self.cache_dir_override = Some(dir);
self
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn with_reuse_env_override(mut self, enabled: bool) -> Self {
self.reuse_env_override = Some(enabled);
self
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn with_reaper_cache_dir_override(mut self, dir: std::path::PathBuf) -> Self {
self.reaper_cache_dir_override = Some(dir);
self
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn with_checkpoint_cache_dir_override(mut self, dir: std::path::PathBuf) -> Self {
self.checkpoint_cache_dir_override = Some(dir);
self
}
pub fn with_spec_customizer(
mut self,
f: impl Fn(ContainerSpec, &dyn Fn(u16) -> u16) -> ContainerSpec + Send + Sync + 'static,
) -> Self {
self.spec_customizer = Some(Arc::new(f));
self
}
pub fn with_post_start(
mut self,
f: impl for<'a> Fn(&'a ContainerGuard) -> crate::BoxFuture<'a, Result<()>>
+ Send
+ Sync
+ 'static,
) -> Self {
self.post_start = Some(Arc::new(f));
self
}
fn active_backend(&self) -> Arc<dyn SandboxBackend> {
match &self.backend_override {
Some(b) => b.clone(),
None => backends::active(),
}
}
fn describe(id: &str, image: &str) -> String {
format!("container(image={image}, id={id})")
}
pub async fn start(self) -> Result<ContainerGuard> {
let backend = self.active_backend();
if let Some(creator) = &self.checkpoint_backend {
if creator != backend.name() {
return Err(RightsizeError::CheckpointBackendMismatch {
active_backend: backend.name().to_string(),
checkpoint_backend: creator.clone(),
});
}
}
if self.require_isolation && !backend.capabilities().hardware_isolated {
return Err(RightsizeError::IsolationRequired {
backend: backend.name().to_string(),
});
}
if self.reuse {
let reuse_env_enabled = self
.reuse_env_override
.unwrap_or_else(crate::reuse::env_enabled);
if reuse_env_enabled {
if self.checkpoint_ref.is_some() {
return Err(RightsizeError::ReuseCheckpointConflict);
}
if let Some(net) = &self.network {
return Err(RightsizeError::ReuseNetworkConflict {
network_id: net.id().to_string(),
});
}
return start_reuse(self, backend).await;
}
eprintln!(
"rightsize: .reuse(true) was requested but RIGHTSIZE_REUSE is not enabled (set \
it to \"true\" or \"1\") — starting an ordinary, non-reused container instead."
);
}
if let Some(net) = &self.network {
crate::reaper::before_ensure_network(
net.id(),
self.reaper_cache_dir_override.as_deref(),
);
if let Err(e) = backend.ensure_network(net.id()).await {
crate::reaper::after_remove_network(
net.id(),
self.reaper_cache_dir_override.as_deref(),
);
return Err(e);
}
}
let (handle, mapped_ports) = create_started_container(
&backend,
&self.image,
&self.env,
&self.command,
&self.exposed_ports,
&self.mounts,
self.network.as_deref(),
&self.aliases,
self.memory_limit_mb,
self.spec_customizer.as_deref(),
self.reaper_cache_dir_override.as_deref(),
self.checkpoint_ref.as_deref(),
)
.await?;
let name = handle.id().to_string();
let ledger_name = handle.spec().name.clone();
let keep_alive = handle.spec().keep_alive;
let diagnostics_handle_id = handle.id().to_string();
let diagnostics_spec = handle.spec().clone();
let diagnostics_ports = mapped_ports.clone();
let mut guard = ContainerGuard {
handle: Some(handle),
backend: backend.clone(),
mapped_ports: Mutex::new(mapped_ports),
network: self.network.clone(),
image: self.image.clone(),
exposed_ports: self.exposed_ports.clone(),
name,
ledger_name,
keep_alive,
reaper_cache_dir_override: self.reaper_cache_dir_override.clone(),
checkpoint_cache_dir_override: self.checkpoint_cache_dir_override.clone(),
wait_strategy: self.wait_strategy.clone(),
network_links: Vec::new(),
};
match link_register_and_wait(
&guard,
&backend,
self.network.as_deref(),
&self.aliases,
guard.wait_strategy.as_ref(),
)
.await
{
Ok(links) => guard.network_links = links,
Err(e) => {
let _ = guard.stop().await;
return Err(e);
}
}
crate::diagnostics::register(
&guard.ledger_name,
&guard.image,
guard.host(),
diagnostics_ports,
backend.clone(),
&diagnostics_handle_id,
diagnostics_spec,
);
if let Some(post_start) = &self.post_start {
if let Err(e) = post_start(&guard).await {
let _ = guard.stop().await;
return Err(e);
}
}
Ok(guard)
}
}
#[allow(clippy::too_many_arguments)]
async fn create_started_container(
backend: &Arc<dyn SandboxBackend>,
image: &str,
env: &[(String, String)],
command: &Option<Vec<String>>,
exposed_ports: &[u16],
mounts: &[FileMount],
network: Option<&Network>,
aliases: &[String],
memory_limit_mb: Option<u64>,
spec_customizer: Option<&SpecCustomizer>,
reaper_cache_dir_override: Option<&std::path::Path>,
checkpoint_ref: Option<&str>,
) -> Result<(Box<dyn SandboxHandle>, Vec<(u16, u16)>)> {
let mut last_conflict: Option<RightsizeError> = None;
for _ in 0..PORT_BIND_ATTEMPTS {
let mapped_ports = allocate_ports(exposed_ports)?;
let seq = NAME_COUNTER.fetch_add(1, Ordering::SeqCst);
let name = format!("rz-{}-{seq}", RunId::value());
let mut spec = ContainerSpec {
name: name.clone(),
image: image.to_string(),
env: env.to_vec(),
command: command.clone(),
ports: mapped_ports
.iter()
.map(|&(guest_port, host_port)| crate::model::PortBinding {
host_port,
guest_port,
})
.collect(),
mounts: mounts.to_vec(),
network_id: network.map(|n| n.id().to_string()),
aliases: aliases.to_vec(),
run_id: RunId::value().to_string(),
memory_limit_mb,
keep_alive: false,
checkpoint_ref: checkpoint_ref.map(ToString::to_string),
};
if let Some(customizer) = spec_customizer {
let lookup: std::collections::HashMap<u16, u16> =
mapped_ports.iter().copied().collect();
let mapped_fn = move |guest: u16| -> u16 {
*lookup
.get(&guest)
.expect("customizer looked up an unexposed port")
};
spec = customizer(spec, &mapped_fn);
}
spec.env = dedup_env_last_wins(spec.env);
crate::reaper::before_create(
backend,
&spec.name,
spec.keep_alive,
reaper_cache_dir_override,
);
let attempt_name = spec.name.clone();
let attempt_keep_alive = spec.keep_alive;
let handle = match backend.create(spec).await {
Ok(h) => h,
Err(e) => {
crate::reaper::after_stop(
&attempt_name,
attempt_keep_alive,
reaper_cache_dir_override,
);
return Err(e);
}
};
match backend.start(handle.as_ref()).await {
Ok(()) => return Ok((handle, mapped_ports)),
Err(e) => {
let _ = backend.stop(handle.as_ref()).await;
let _ = backend.remove(handle.as_ref()).await;
release_ports(&mapped_ports);
crate::reaper::after_stop(
&attempt_name,
attempt_keep_alive,
reaper_cache_dir_override,
);
if is_port_bind_conflict(&e) {
last_conflict = Some(e);
continue;
}
return Err(e);
}
}
}
Err(RightsizeError::Backend(format!(
"Could not bind free host ports for {} after {PORT_BIND_ATTEMPTS} attempts — another \
process kept grabbing the allocated ports first; if this persists, check for a port \
scanner/leaked process racing the allocator on this host{}",
Container::describe("<unstarted>", image),
last_conflict
.map(|c| format!(" (last conflict: {c})"))
.unwrap_or_default(),
)))
}
fn dedup_env_last_wins(env: Vec<(String, String)>) -> Vec<(String, String)> {
let mut order: Vec<String> = Vec::new();
let mut values: std::collections::HashMap<String, String> = std::collections::HashMap::new();
for (k, v) in env {
if !values.contains_key(&k) {
order.push(k.clone());
}
values.insert(k, v);
}
order
.into_iter()
.map(|k| {
let v = values.remove(&k).expect("key was just recorded in order");
(k, v)
})
.collect()
}
fn allocate_ports(exposed_ports: &[u16]) -> Result<Vec<(u16, u16)>> {
let mut mapped = Vec::with_capacity(exposed_ports.len());
for &guest_port in exposed_ports {
match free_ports().allocate() {
Ok(host_port) => mapped.push((guest_port, host_port)),
Err(e) => {
release_ports(&mapped);
return Err(e);
}
}
}
Ok(mapped)
}
fn release_ports(mapped_ports: &[(u16, u16)]) {
for &(_, host_port) in mapped_ports {
free_ports().release(host_port);
}
}
pub(crate) fn is_port_bind_conflict(e: &RightsizeError) -> bool {
let mut current: Option<&RightsizeError> = Some(e);
while let Some(err) = current {
if matches!(err, RightsizeError::PortBindConflict { .. }) {
return true;
}
let msg = err.to_string().to_lowercase();
if msg.contains("address already in use") || msg.contains("already allocated") {
return true;
}
current = match err {
RightsizeError::PortBindConflict {
source: Some(s), ..
} => Some(s.as_ref()),
_ => None,
};
}
false
}
async fn link_register_and_wait(
guard: &ContainerGuard,
backend: &Arc<dyn SandboxBackend>,
network: Option<&Network>,
aliases: &[String],
wait_strategy: &dyn WaitStrategy,
) -> Result<Vec<crate::backend::NetworkLink>> {
let links = if let Some(net) = network {
let links = net.links_for_new_member();
backend
.install_network_links(guard.handle_ref(), &links)
.await?;
links
} else {
Vec::new()
};
if let Some(net) = network {
net.register(guard.as_network_member(), aliases.to_vec(), backend.clone());
}
let target = GuardWaitTarget { guard };
wait_strategy.wait_until_ready(&target).await?;
Ok(links)
}
struct GuardWaitTarget<'a> {
guard: &'a ContainerGuard,
}
#[async_trait::async_trait]
impl WaitTarget for GuardWaitTarget<'_> {
fn host(&self) -> &str {
self.guard.host()
}
fn mapped_port(&self, guest_port: u16) -> u16 {
self.guard.get_mapped_port(guest_port).unwrap_or(guest_port)
}
fn exposed_guest_ports(&self) -> Vec<u16> {
self.guard.exposed_ports.clone()
}
async fn current_logs(&self) -> String {
self.guard.logs().await.unwrap_or_default()
}
fn describe(&self) -> String {
self.guard.describe()
}
}
struct RawWaitTarget<'a> {
host: &'a str,
mapped_ports: &'a [(u16, u16)],
exposed_ports: &'a [u16],
backend: &'a Arc<dyn SandboxBackend>,
handle: &'a dyn SandboxHandle,
name: &'a str,
image: &'a str,
}
#[async_trait::async_trait]
impl WaitTarget for RawWaitTarget<'_> {
fn host(&self) -> &str {
self.host
}
fn mapped_port(&self, guest_port: u16) -> u16 {
self.mapped_ports
.iter()
.find(|&&(g, _)| g == guest_port)
.map(|&(_, h)| h)
.unwrap_or(guest_port)
}
fn exposed_guest_ports(&self) -> Vec<u16> {
self.exposed_ports.to_vec()
}
async fn current_logs(&self) -> String {
self.backend.logs(self.handle).await.unwrap_or_default()
}
fn describe(&self) -> String {
Container::describe(self.name, self.image)
}
}
async fn start_reuse(
container: Container,
backend: Arc<dyn SandboxBackend>,
) -> Result<ContainerGuard> {
let env = dedup_env_last_wins(container.env.clone());
let identity = crate::reuse::compute_identity(
&container.image,
&env,
&container.command,
&container.exposed_ports,
container.memory_limit_mb,
&container.mounts,
)?;
let cache_dir = container
.cache_dir_override
.clone()
.unwrap_or_else(crate::cache_dir::dir);
let registry = crate::reuse::Registry::new(&cache_dir, &identity.hash_hex);
if registry.exists() {
match registry.read() {
Some(entry) => {
if let Some(guard) = try_adopt(&container, &backend, &identity, &entry).await {
return Ok(guard);
}
backend.remove_by_name(&entry.name);
registry.delete();
}
None => {
backend.remove_by_name(&identity.name);
registry.delete();
}
}
}
if find_running_by_name(&backend, &identity.name, &container.image)
.await
.is_some()
{
backend.remove_by_name(&identity.name);
}
match create_fresh_reuse(&container, &backend, &identity, &env, ®istry).await {
Ok(guard) => Ok(guard),
Err(e) if crate::reuse::is_name_conflict(&e) => {
match registry.read() {
Some(entry) => match try_adopt(&container, &backend, &identity, &entry).await {
Some(guard) => Ok(guard),
None => Err(e),
},
None => Err(e),
}
}
Err(e) => Err(e),
}
}
async fn find_running_by_name(
backend: &Arc<dyn SandboxBackend>,
name: &str,
image: &str,
) -> Option<Box<dyn SandboxHandle>> {
let probe = ContainerSpec::new(name, image, RunId::value());
backend.find_running(&probe).await.ok().flatten()
}
async fn try_adopt(
container: &Container,
backend: &Arc<dyn SandboxBackend>,
identity: &crate::reuse::Identity,
entry: &crate::reuse::RegistryEntry,
) -> Option<ContainerGuard> {
let mut mapped_ports = Vec::with_capacity(container.exposed_ports.len());
for &guest_port in &container.exposed_ports {
let host_port = *entry.ports.get(&guest_port.to_string())?;
mapped_ports.push((guest_port, host_port));
}
let adopted_spec = ContainerSpec {
name: identity.name.clone(),
image: container.image.clone(),
env: dedup_env_last_wins(container.env.clone()),
command: container.command.clone(),
ports: mapped_ports
.iter()
.map(|&(guest_port, host_port)| crate::model::PortBinding {
host_port,
guest_port,
})
.collect(),
mounts: container.mounts.clone(),
network_id: None,
aliases: container.aliases.clone(),
run_id: RunId::value().to_string(),
memory_limit_mb: container.memory_limit_mb,
keep_alive: true,
checkpoint_ref: None,
};
let Ok(Some(handle)) = backend.find_running(&adopted_spec).await else {
return None;
};
let raw = RawWaitTarget {
host: "127.0.0.1",
mapped_ports: &mapped_ports,
exposed_ports: &container.exposed_ports,
backend,
handle: handle.as_ref(),
name: &identity.name,
image: &container.image,
};
if container
.wait_strategy
.wait_until_ready(&raw)
.await
.is_err()
{
return None;
}
let diagnostics_handle_id = handle.id().to_string();
let diagnostics_spec = handle.spec().clone();
let diagnostics_ports = mapped_ports.clone();
let guard = ContainerGuard {
handle: Some(handle),
backend: backend.clone(),
mapped_ports: Mutex::new(mapped_ports),
network: None,
image: container.image.clone(),
exposed_ports: container.exposed_ports.clone(),
name: identity.name.clone(),
ledger_name: identity.name.clone(),
keep_alive: true,
reaper_cache_dir_override: container.reaper_cache_dir_override.clone(),
checkpoint_cache_dir_override: container.checkpoint_cache_dir_override.clone(),
wait_strategy: container.wait_strategy.clone(),
network_links: Vec::new(),
};
crate::diagnostics::register(
&guard.ledger_name,
&guard.image,
guard.host(),
diagnostics_ports,
backend.clone(),
&diagnostics_handle_id,
diagnostics_spec,
);
Some(guard)
}
async fn create_and_start_reuse_sandbox(
backend: &Arc<dyn SandboxBackend>,
container: &Container,
identity: &crate::reuse::Identity,
env: &[(String, String)],
) -> Result<(Box<dyn SandboxHandle>, Vec<(u16, u16)>)> {
let mut last_conflict: Option<RightsizeError> = None;
for _ in 0..PORT_BIND_ATTEMPTS {
let mapped_ports = allocate_ports(&container.exposed_ports)?;
let mut spec = ContainerSpec {
name: identity.name.clone(),
image: container.image.clone(),
env: env.to_vec(),
command: container.command.clone(),
ports: mapped_ports
.iter()
.map(|&(guest_port, host_port)| crate::model::PortBinding {
host_port,
guest_port,
})
.collect(),
mounts: container.mounts.clone(),
network_id: None,
aliases: container.aliases.clone(),
run_id: RunId::value().to_string(),
memory_limit_mb: container.memory_limit_mb,
keep_alive: true,
checkpoint_ref: None,
};
if let Some(customizer) = &container.spec_customizer {
let lookup: std::collections::HashMap<u16, u16> =
mapped_ports.iter().copied().collect();
let mapped_fn = move |guest: u16| -> u16 {
*lookup
.get(&guest)
.expect("customizer looked up an unexposed port")
};
spec = customizer(spec, &mapped_fn);
}
spec.env = dedup_env_last_wins(spec.env);
let handle = match backend.create(spec).await {
Ok(h) => h,
Err(e) => {
release_ports(&mapped_ports);
return Err(e);
}
};
match backend.start(handle.as_ref()).await {
Ok(()) => return Ok((handle, mapped_ports)),
Err(e) => {
let _ = backend.stop(handle.as_ref()).await;
let _ = backend.remove(handle.as_ref()).await;
release_ports(&mapped_ports);
if is_port_bind_conflict(&e) {
last_conflict = Some(e);
continue;
}
return Err(e);
}
}
}
Err(RightsizeError::Backend(format!(
"Could not bind free host ports for {} after {PORT_BIND_ATTEMPTS} attempts — another \
process kept grabbing the allocated ports first; if this persists, check for a port \
scanner/leaked process racing the allocator on this host{}",
Container::describe(&identity.name, &container.image),
last_conflict
.map(|c| format!(" (last conflict: {c})"))
.unwrap_or_default(),
)))
}
async fn create_fresh_reuse(
container: &Container,
backend: &Arc<dyn SandboxBackend>,
identity: &crate::reuse::Identity,
env: &[(String, String)],
registry: &crate::reuse::Registry,
) -> Result<ContainerGuard> {
let (handle, mapped_ports) =
create_and_start_reuse_sandbox(backend, container, identity, env).await?;
let raw = RawWaitTarget {
host: "127.0.0.1",
mapped_ports: &mapped_ports,
exposed_ports: &container.exposed_ports,
backend,
handle: handle.as_ref(),
name: &identity.name,
image: &container.image,
};
if let Err(e) = container.wait_strategy.wait_until_ready(&raw).await {
let _ = backend.stop(handle.as_ref()).await;
let _ = backend.remove(handle.as_ref()).await;
release_ports(&mapped_ports);
return Err(e);
}
let entry = crate::reuse::RegistryEntry {
name: identity.name.clone(),
image: container.image.clone(),
ports: mapped_ports
.iter()
.map(|&(guest_port, host_port)| (guest_port.to_string(), host_port))
.collect(),
created_iso: crate::reuse::now_iso8601(),
backend: backend.name().to_string(),
};
let _ = registry.write_atomic(&entry);
let diagnostics_handle_id = handle.id().to_string();
let diagnostics_spec = handle.spec().clone();
let diagnostics_ports = mapped_ports.clone();
let guard = ContainerGuard {
handle: Some(handle),
backend: backend.clone(),
mapped_ports: Mutex::new(mapped_ports),
network: None,
image: container.image.clone(),
exposed_ports: container.exposed_ports.clone(),
name: identity.name.clone(),
ledger_name: identity.name.clone(),
keep_alive: true,
reaper_cache_dir_override: container.reaper_cache_dir_override.clone(),
checkpoint_cache_dir_override: container.checkpoint_cache_dir_override.clone(),
wait_strategy: container.wait_strategy.clone(),
network_links: Vec::new(),
};
crate::diagnostics::register(
&guard.ledger_name,
&guard.image,
guard.host(),
diagnostics_ports,
backend.clone(),
&diagnostics_handle_id,
diagnostics_spec,
);
if let Some(post_start) = &container.post_start {
if let Err(e) = post_start(&guard).await {
let handle_ref = guard.handle_ref();
let _ = backend.stop(handle_ref).await;
let _ = backend.remove(handle_ref).await;
registry.delete();
for &(_, host_port) in guard
.mapped_ports
.lock()
.expect("mapped_ports mutex poisoned")
.iter()
{
free_ports().release(host_port);
}
return Err(e);
}
}
Ok(guard)
}
pub struct ContainerGuard {
handle: Option<Box<dyn SandboxHandle>>,
backend: Arc<dyn SandboxBackend>,
mapped_ports: Mutex<Vec<(u16, u16)>>,
network: Option<Arc<Network>>,
image: String,
exposed_ports: Vec<u16>,
name: String,
ledger_name: String,
keep_alive: bool,
reaper_cache_dir_override: Option<std::path::PathBuf>,
checkpoint_cache_dir_override: Option<std::path::PathBuf>,
wait_strategy: Arc<dyn WaitStrategy>,
network_links: Vec<crate::backend::NetworkLink>,
}
impl ContainerGuard {
fn handle_ref(&self) -> &dyn SandboxHandle {
self.handle
.as_deref()
.expect("ContainerGuard invariant: handle is only None after being consumed by stop()")
}
fn as_network_member(&self) -> Arc<dyn NetworkMember> {
Arc::new(GuardMemberSnapshot {
mapped_ports: self
.mapped_ports
.lock()
.expect("mapped_ports mutex poisoned")
.clone(),
})
}
pub fn network(&self) -> Option<&Arc<Network>> {
self.network.as_ref()
}
pub fn host(&self) -> &str {
"127.0.0.1"
}
#[allow(clippy::misnamed_getters)]
pub fn name(&self) -> &str {
&self.ledger_name
}
pub fn get_mapped_port(&self, guest_port: u16) -> Result<u16> {
let mapped = self
.mapped_ports
.lock()
.expect("mapped_ports mutex poisoned");
if let Some(&(_, host_port)) = mapped.iter().find(|&&(g, _)| g == guest_port) {
return Ok(host_port);
}
if !self.is_running() {
Err(RightsizeError::Backend(format!(
"Cannot get mapped port {guest_port} on {}: the container is not running — call \
start() first, or check that it did not stop/fail after start()",
self.describe()
)))
} else {
Err(RightsizeError::Backend(format!(
"Port {guest_port} is not exposed on {} — call with_exposed_ports({guest_port}) \
before start(), or check exposed_ports for the port you actually declared",
self.describe()
)))
}
}
pub async fn logs(&self) -> Result<String> {
self.backend.logs(self.require_handle()?).await
}
pub async fn exec(&self, cmd: &[&str]) -> Result<ExecResult> {
let cmd: Vec<String> = cmd.iter().map(|s| s.to_string()).collect();
self.backend.exec(self.require_handle()?, &cmd).await
}
pub async fn copy_file_to_container(
&self,
host_path: impl AsRef<std::path::Path>,
container_path: &str,
) -> Result<()> {
let handle = self.require_handle()?;
require_absolute_container_path(container_path)?;
if let Some(parent) = container_parent_dir(container_path) {
let mkdir = vec!["mkdir".to_string(), "-p".to_string(), parent.to_string()];
self.backend.exec(handle, &mkdir).await?;
}
self.backend
.copy_to_container(handle, host_path.as_ref(), container_path)
.await
}
pub async fn copy_content_to_container(
&self,
content: impl AsRef<[u8]>,
container_path: &str,
) -> Result<()> {
let temp = TempCopyFile::create(content.as_ref())?;
self.copy_file_to_container(&temp.path, container_path)
.await
}
pub async fn copy_file_from_container(
&self,
container_path: &str,
host_path: impl AsRef<std::path::Path>,
) -> Result<()> {
let handle = self.require_handle()?;
require_absolute_container_path(container_path)?;
let host_path = host_path.as_ref();
if let Some(parent) = host_path.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)?;
}
}
self.backend
.copy_from_container(handle, container_path, host_path)
.await
}
pub async fn checkpoint(&self) -> Result<Checkpoint> {
self.ensure_checkpoint_capable()?;
let handle = self.require_handle()?;
let nonce = checkpoint::generate_ref_nonce();
let (checkpoint_ref, spec) = self.checkpoint_core(handle, &nonce).await?;
Ok(Checkpoint {
checkpoint_ref,
backend: self.backend.name().to_string(),
spec,
})
}
pub async fn checkpoint_named(&self, name: &str) -> Result<Checkpoint> {
checkpoint::validate_name(name)?;
self.ensure_checkpoint_capable()?;
let handle = self.require_handle()?;
let cache_dir = self
.checkpoint_cache_dir_override
.clone()
.unwrap_or_else(crate::cache_dir::dir);
let registry = checkpoint::Registry::new(&cache_dir, name);
if let Some(previous) = registry.read() {
if previous.backend == self.backend.name() {
let _ = self
.backend
.remove_checkpoint(&previous.checkpoint_ref)
.await;
}
}
let (checkpoint_ref, spec) = self.checkpoint_core(handle, name).await?;
let entry = checkpoint::NamedRegistryEntry {
name: name.to_string(),
checkpoint_ref: checkpoint_ref.clone(),
backend: self.backend.name().to_string(),
created_iso: crate::reuse::now_iso8601(),
spec: checkpoint::NamedRegistrySpec::from_container_spec(&spec),
};
registry.write_atomic(&entry)?;
Ok(Checkpoint {
checkpoint_ref,
backend: self.backend.name().to_string(),
spec,
})
}
fn ensure_checkpoint_capable(&self) -> Result<()> {
if !self.backend.capabilities().checkpoint {
return Err(RightsizeError::CheckpointUnsupported {
backend: self.backend.name().to_string(),
});
}
Ok(())
}
async fn checkpoint_core(
&self,
handle: &dyn SandboxHandle,
nonce_or_name: &str,
) -> Result<(String, ContainerSpec)> {
let checkpoint_ref = self
.backend
.create_checkpoint(handle, nonce_or_name)
.await?;
if self.backend.capabilities().checkpoint_restarts_workload {
if !self.network_links.is_empty() {
self.backend
.install_network_links(handle, &self.network_links)
.await?;
}
let target = GuardWaitTarget { guard: self };
self.wait_strategy.wait_until_ready(&target).await?;
}
Ok((checkpoint_ref, handle.spec().clone()))
}
pub async fn follow_output(
&self,
consumer: impl Fn(String) + Send + Sync + 'static,
) -> Result<crate::backend::FollowHandle> {
self.backend
.follow_logs(self.require_handle()?, Box::new(consumer))
.await
}
pub fn is_running(&self) -> bool {
self.handle.is_some()
}
fn require_handle(&self) -> Result<&dyn SandboxHandle> {
self.handle.as_deref().ok_or_else(|| {
RightsizeError::Backend(format!(
"{} is not running — call start() first",
self.describe()
))
})
}
fn describe(&self) -> String {
Container::describe(&self.name, &self.image)
}
pub async fn stop(mut self) -> Result<()> {
self.stop_inner().await;
Ok(())
}
async fn stop_inner(&mut self) {
let Some(handle) = self.handle.take() else {
return; };
crate::diagnostics::deregister(&self.ledger_name);
if self.keep_alive {
self.mapped_ports
.lock()
.expect("mapped_ports mutex poisoned")
.clear();
return;
}
let _ = self.backend.stop(handle.as_ref()).await;
let _ = self.backend.remove(handle.as_ref()).await;
crate::reaper::after_stop(
&self.ledger_name,
self.keep_alive,
self.reaper_cache_dir_override.as_deref(),
);
let mut mapped = self
.mapped_ports
.lock()
.expect("mapped_ports mutex poisoned");
for &(_, host_port) in mapped.iter() {
free_ports().release(host_port);
}
mapped.clear();
}
}
struct GuardMemberSnapshot {
mapped_ports: Vec<(u16, u16)>,
}
impl NetworkMember for GuardMemberSnapshot {
fn is_running(&self) -> bool {
true }
fn mapped_ports(&self) -> Vec<(u16, u16)> {
self.mapped_ports.clone()
}
}
impl Drop for ContainerGuard {
fn drop(&mut self) {
let Some(handle) = self.handle.take() else {
return; };
crate::diagnostics::deregister(&self.ledger_name);
if self.keep_alive {
return;
}
if let Ok(mut mapped) = self.mapped_ports.lock() {
for &(_, host_port) in mapped.iter() {
free_ports().release(host_port);
}
mapped.clear();
}
let ledger_name = self.ledger_name.clone();
let reaper_cache_dir_override = self.reaper_cache_dir_override.clone();
cleanup::enqueue(CleanupJob {
backend: self.backend.clone(),
container_id: handle.id().to_string(),
after_teardown: Some(Box::new(move || {
crate::reaper::after_stop(
&ledger_name,
false,
reaper_cache_dir_override.as_deref(),
);
})),
});
}
}
impl Checkpoint {
pub async fn find(name: &str) -> Result<Option<Checkpoint>> {
checkpoint::validate_name(name)?;
find_named(name, &backends::active(), &crate::cache_dir::dir()).await
}
pub fn list() -> Result<Vec<Checkpoint>> {
list_named(&crate::cache_dir::dir())
}
pub async fn remove(name: &str) -> Result<bool> {
checkpoint::validate_name(name)?;
remove_named(name, &backends::active(), &crate::cache_dir::dir()).await
}
pub async fn export_to(&self, path: impl AsRef<std::path::Path>) -> Result<()> {
export_to_impl(
self,
path.as_ref(),
&backends::active(),
&crate::cache_dir::dir(),
&std::env::temp_dir(),
)
.await
}
pub async fn import_from(path: impl AsRef<std::path::Path>) -> Result<Checkpoint> {
import_from_impl(
path.as_ref(),
&backends::active(),
&crate::cache_dir::dir(),
&std::env::temp_dir(),
)
.await
}
}
async fn find_named(
name: &str,
backend: &Arc<dyn SandboxBackend>,
cache_dir: &std::path::Path,
) -> Result<Option<Checkpoint>> {
let registry = checkpoint::Registry::new(cache_dir, name);
let Some(entry) = registry.read() else {
registry.delete();
return Ok(None);
};
if entry.backend != backend.name() {
return Ok(Some(checkpoint_from_entry(entry)));
}
match backend.has_checkpoint(&entry.checkpoint_ref).await {
Ok(true) => Ok(Some(checkpoint_from_entry(entry))),
Ok(false) => {
registry.delete();
Ok(None)
}
Err(e) => Err(e),
}
}
fn list_named(cache_dir: &std::path::Path) -> Result<Vec<Checkpoint>> {
let entries = checkpoint::list_registry_entries(cache_dir)?;
Ok(entries.into_iter().map(checkpoint_from_entry).collect())
}
async fn remove_named(
name: &str,
backend: &Arc<dyn SandboxBackend>,
cache_dir: &std::path::Path,
) -> Result<bool> {
let registry = checkpoint::Registry::new(cache_dir, name);
let existed = registry.exists();
if let Some(entry) = registry.read() {
if entry.backend == backend.name() {
let _ = backend.remove_checkpoint(&entry.checkpoint_ref).await;
}
}
registry.delete();
Ok(existed)
}
async fn export_to_impl(
cp: &Checkpoint,
dest: &std::path::Path,
backend: &Arc<dyn SandboxBackend>,
cache_dir: &std::path::Path,
staging_parent: &std::path::Path,
) -> Result<()> {
if cp.backend != backend.name() {
return Err(RightsizeError::CheckpointBackendMismatch {
active_backend: backend.name().to_string(),
checkpoint_backend: cp.backend.clone(),
});
}
if !backend.has_checkpoint(&cp.checkpoint_ref).await? {
return Err(RightsizeError::CheckpointArtifactMissing {
checkpoint_ref: cp.checkpoint_ref.clone(),
backend: cp.backend.clone(),
});
}
let staging = crate::archive::TempStagingDir::create_in(staging_parent, "export")?;
backend
.export_checkpoint(
&cp.checkpoint_ref,
&crate::archive::artifact_path(staging.path()),
)
.await?;
let name = find_registered_name(cache_dir, &cp.backend, &cp.checkpoint_ref);
let manifest = crate::archive::ArchiveManifest {
rightsize_archive: crate::archive::FORMAT_VERSION,
name,
checkpoint_ref: cp.checkpoint_ref.clone(),
backend: cp.backend.clone(),
created_iso: crate::reuse::now_iso8601(),
spec: checkpoint::NamedRegistrySpec::from_container_spec(&cp.spec),
};
crate::archive::write_manifest(&crate::archive::manifest_path(staging.path()), &manifest)?;
if let Some(parent) = dest.parent() {
if !parent.as_os_str().is_empty() {
std::fs::create_dir_all(parent)?;
}
}
crate::archive::tar_create(dest, staging.path()).await
}
fn find_registered_name(
cache_dir: &std::path::Path,
backend: &str,
checkpoint_ref: &str,
) -> Option<String> {
checkpoint::list_registry_entries(cache_dir)
.ok()?
.into_iter()
.find(|entry| entry.backend == backend && entry.checkpoint_ref == checkpoint_ref)
.map(|entry| entry.name)
}
async fn import_from_impl(
src: &std::path::Path,
backend: &Arc<dyn SandboxBackend>,
cache_dir: &std::path::Path,
staging_parent: &std::path::Path,
) -> Result<Checkpoint> {
let staging = crate::archive::TempStagingDir::create_in(staging_parent, "import")?;
crate::archive::tar_extract(src, staging.path()).await?;
let manifest =
crate::archive::read_manifest(src, &crate::archive::manifest_path(staging.path()))?;
if manifest.rightsize_archive != crate::archive::FORMAT_VERSION {
return Err(RightsizeError::MalformedArchive {
path: src.to_path_buf(),
reason: format!(
"unsupported rightsizeArchive version {} (this port understands only {})",
manifest.rightsize_archive,
crate::archive::FORMAT_VERSION
),
});
}
if let Some(name) = &manifest.name {
checkpoint::validate_name(name)?;
}
if manifest.backend != backend.name() {
return Err(RightsizeError::CheckpointBackendMismatch {
active_backend: backend.name().to_string(),
checkpoint_backend: manifest.backend.clone(),
});
}
let effective_ref = backend
.import_checkpoint(
&crate::archive::artifact_path(staging.path()),
&manifest.checkpoint_ref,
)
.await?;
if let Some(name) = &manifest.name {
let registry = checkpoint::Registry::new(cache_dir, name);
if let Some(previous) = registry.read() {
if previous.backend == backend.name() && previous.checkpoint_ref != effective_ref {
let _ = backend.remove_checkpoint(&previous.checkpoint_ref).await;
}
}
let entry = checkpoint::NamedRegistryEntry {
name: name.clone(),
checkpoint_ref: effective_ref.clone(),
backend: backend.name().to_string(),
created_iso: crate::reuse::now_iso8601(),
spec: manifest.spec.clone(),
};
registry.write_atomic(&entry)?;
}
Ok(checkpoint_from_reduced(
manifest.name.as_deref().unwrap_or("imported"),
effective_ref,
backend.name().to_string(),
manifest.spec,
))
}
fn checkpoint_from_reduced(
display_name: &str,
checkpoint_ref: String,
backend: String,
spec: checkpoint::NamedRegistrySpec,
) -> Checkpoint {
let ports = spec
.exposed_ports
.iter()
.map(|&guest_port| crate::model::PortBinding {
host_port: 0,
guest_port,
})
.collect();
Checkpoint {
checkpoint_ref: checkpoint_ref.clone(),
backend,
spec: ContainerSpec {
name: format!("rz-checkpoint-{display_name}"),
image: checkpoint_ref,
env: spec.env.into_iter().collect(),
command: spec.command,
ports,
mounts: Vec::new(),
network_id: None,
aliases: Vec::new(),
run_id: String::new(),
memory_limit_mb: spec.memory_limit_mb,
keep_alive: false,
checkpoint_ref: None,
},
}
}
fn checkpoint_from_entry(entry: checkpoint::NamedRegistryEntry) -> Checkpoint {
checkpoint_from_reduced(&entry.name, entry.checkpoint_ref, entry.backend, entry.spec)
}
fn require_absolute_container_path(path: &str) -> Result<()> {
if path.starts_with('/') {
Ok(())
} else {
Err(RightsizeError::Backend(format!(
"container path '{path}' must be absolute — both msb copy and docker cp require a \
NAME:/abs/path shape"
)))
}
}
fn container_parent_dir(path: &str) -> Option<&str> {
let trimmed = path.trim_end_matches('/');
if trimmed.is_empty() {
return None; }
match trimmed.rfind('/') {
Some(0) => Some("/"),
Some(idx) => Some(&trimmed[..idx]),
None => None, }
}
struct TempCopyFile {
path: std::path::PathBuf,
}
impl TempCopyFile {
fn create(content: &[u8]) -> Result<Self> {
use std::sync::atomic::{AtomicU64, Ordering};
static SEQ: AtomicU64 = AtomicU64::new(0);
let seq = SEQ.fetch_add(1, Ordering::SeqCst);
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let path = std::env::temp_dir().join(format!(
"rightsize-copy-content-{}-{seq}-{nanos}",
std::process::id()
));
write_private_temp_file(&path, content)?;
Ok(Self { path })
}
}
impl Drop for TempCopyFile {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
#[cfg(unix)]
fn write_private_temp_file(path: &std::path::Path, content: &[u8]) -> std::io::Result<()> {
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(path)?;
f.write_all(content)
}
#[cfg(not(unix))]
fn write_private_temp_file(path: &std::path::Path, content: &[u8]) -> std::io::Result<()> {
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(path)?;
use std::io::Write;
f.write_all(content)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::wait::{WaitStrategy, WaitTarget};
use std::sync::Mutex as StdMutex;
use std::time::Duration;
fn expect_start_err(result: Result<ContainerGuard>, msg: &str) -> RightsizeError {
match result {
Ok(_) => panic!("{msg}: expected an error, got Ok"),
Err(e) => e,
}
}
struct FakeHandle {
id: String,
spec: ContainerSpec,
}
impl SandboxHandle for FakeHandle {
fn id(&self) -> &str {
&self.id
}
fn spec(&self) -> &ContainerSpec {
&self.spec
}
}
struct ReadyImmediately;
#[async_trait::async_trait]
impl WaitStrategy for ReadyImmediately {
async fn wait_until_ready(&self, _target: &dyn WaitTarget) -> Result<()> {
Ok(())
}
fn with_startup_timeout(self: Box<Self>, _timeout: Duration) -> Box<dyn WaitStrategy> {
self
}
}
struct NeverReady {
precaptured_port: Arc<StdMutex<Option<u16>>>,
}
#[async_trait::async_trait]
impl WaitStrategy for NeverReady {
async fn wait_until_ready(&self, target: &dyn WaitTarget) -> Result<()> {
*self.precaptured_port.lock().unwrap() = Some(target.mapped_port(6379));
Err(RightsizeError::ContainerLaunch("never ready".to_string()))
}
fn with_startup_timeout(self: Box<Self>, _timeout: Duration) -> Box<dyn WaitStrategy> {
self
}
}
#[derive(Default)]
struct FakeBackendState {
created: Vec<ContainerSpec>,
started: Vec<String>,
stopped: Vec<String>,
installed_links: Vec<(String, Vec<crate::backend::NetworkLink>)>,
committed: Vec<(String, String)>,
copied_in: Vec<(String, std::path::PathBuf, String, bool)>,
copied_out: Vec<(String, String, std::path::PathBuf)>,
removed_checkpoints: Vec<String>,
probed_checkpoints: Vec<String>,
exported_checkpoints: Vec<(String, std::path::PathBuf)>,
imported_checkpoints: Vec<(std::path::PathBuf, String, Vec<u8>)>,
}
struct FakeBackend {
state: StdMutex<FakeBackendState>,
fail_install_network_links: bool,
fail_ensure_network: bool,
hardware_isolated: bool,
checkpoint_capable: bool,
checkpoint_restarts_workload: bool,
fail_create_checkpoint: bool,
fail_export_checkpoint: bool,
probe_result: StdMutex<Option<bool>>,
import_effective_ref: StdMutex<Option<String>>,
name: &'static str,
}
impl FakeBackend {
fn new() -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(FakeBackendState::default()),
fail_install_network_links: false,
fail_ensure_network: false,
hardware_isolated: false,
checkpoint_capable: false,
checkpoint_restarts_workload: false,
fail_create_checkpoint: false,
fail_export_checkpoint: false,
probe_result: StdMutex::new(Some(true)),
import_effective_ref: StdMutex::new(None),
name: "fake",
})
}
fn failing_install_network_links() -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(FakeBackendState::default()),
fail_install_network_links: true,
fail_ensure_network: false,
hardware_isolated: false,
checkpoint_capable: false,
checkpoint_restarts_workload: false,
fail_create_checkpoint: false,
fail_export_checkpoint: false,
probe_result: StdMutex::new(Some(true)),
import_effective_ref: StdMutex::new(None),
name: "fake",
})
}
fn failing_ensure_network() -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(FakeBackendState::default()),
fail_install_network_links: false,
fail_ensure_network: true,
hardware_isolated: false,
checkpoint_capable: false,
checkpoint_restarts_workload: false,
fail_create_checkpoint: false,
fail_export_checkpoint: false,
probe_result: StdMutex::new(Some(true)),
import_effective_ref: StdMutex::new(None),
name: "fake",
})
}
fn hardware_isolated() -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(FakeBackendState::default()),
fail_install_network_links: false,
fail_ensure_network: false,
hardware_isolated: true,
checkpoint_capable: false,
checkpoint_restarts_workload: false,
fail_create_checkpoint: false,
fail_export_checkpoint: false,
probe_result: StdMutex::new(Some(true)),
import_effective_ref: StdMutex::new(None),
name: "fake",
})
}
fn checkpoint_capable() -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(FakeBackendState::default()),
fail_install_network_links: false,
fail_ensure_network: false,
hardware_isolated: false,
checkpoint_capable: true,
checkpoint_restarts_workload: false,
fail_create_checkpoint: false,
fail_export_checkpoint: false,
probe_result: StdMutex::new(Some(true)),
import_effective_ref: StdMutex::new(None),
name: "fake",
})
}
fn checkpoint_capable_restarts_workload() -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(FakeBackendState::default()),
fail_install_network_links: false,
fail_ensure_network: false,
hardware_isolated: false,
checkpoint_capable: true,
checkpoint_restarts_workload: true,
fail_create_checkpoint: false,
fail_export_checkpoint: false,
probe_result: StdMutex::new(Some(true)),
import_effective_ref: StdMutex::new(None),
name: "fake",
})
}
fn named(name: &'static str) -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(FakeBackendState::default()),
fail_install_network_links: false,
fail_ensure_network: false,
hardware_isolated: false,
checkpoint_capable: true,
checkpoint_restarts_workload: false,
fail_create_checkpoint: false,
fail_export_checkpoint: false,
probe_result: StdMutex::new(Some(true)),
import_effective_ref: StdMutex::new(None),
name,
})
}
fn checkpoint_capable_that_fails_create() -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(FakeBackendState::default()),
fail_install_network_links: false,
fail_ensure_network: false,
hardware_isolated: false,
checkpoint_capable: true,
checkpoint_restarts_workload: false,
fail_create_checkpoint: true,
fail_export_checkpoint: false,
probe_result: StdMutex::new(Some(true)),
import_effective_ref: StdMutex::new(None),
name: "fake",
})
}
fn checkpoint_capable_that_fails_export() -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(FakeBackendState::default()),
fail_install_network_links: false,
fail_ensure_network: false,
hardware_isolated: false,
checkpoint_capable: true,
checkpoint_restarts_workload: false,
fail_create_checkpoint: false,
fail_export_checkpoint: true,
probe_result: StdMutex::new(Some(true)),
import_effective_ref: StdMutex::new(None),
name: "fake",
})
}
fn set_probe_result(&self, result: Option<bool>) {
*self.probe_result.lock().unwrap() = result;
}
fn set_import_effective_ref(&self, effective_ref: impl Into<String>) {
*self.import_effective_ref.lock().unwrap() = Some(effective_ref.into());
}
}
#[async_trait::async_trait]
impl SandboxBackend for FakeBackend {
fn name(&self) -> &str {
self.name
}
fn supports_native_networks(&self) -> bool {
false
}
fn capabilities(&self) -> crate::backend::Capabilities {
crate::backend::Capabilities {
hardware_isolated: self.hardware_isolated,
checkpoint: self.checkpoint_capable,
checkpoint_restarts_workload: self.checkpoint_restarts_workload,
}
}
async fn create(&self, spec: ContainerSpec) -> Result<Box<dyn SandboxHandle>> {
self.state.lock().unwrap().created.push(spec.clone());
Ok(Box::new(FakeHandle {
id: spec.name.clone(),
spec,
}))
}
async fn start(&self, handle: &dyn SandboxHandle) -> Result<()> {
self.state
.lock()
.unwrap()
.started
.push(handle.id().to_string());
Ok(())
}
async fn stop(&self, handle: &dyn SandboxHandle) -> Result<()> {
self.state
.lock()
.unwrap()
.stopped
.push(handle.id().to_string());
Ok(())
}
async fn remove(&self, _handle: &dyn SandboxHandle) -> Result<()> {
Ok(())
}
async fn exec(&self, _handle: &dyn SandboxHandle, cmd: &[String]) -> Result<ExecResult> {
Ok(ExecResult {
exit_code: 0,
stdout: cmd.join(" "),
stderr: String::new(),
})
}
async fn logs(&self, _handle: &dyn SandboxHandle) -> Result<String> {
Ok(String::new())
}
async fn follow_logs(
&self,
_handle: &dyn SandboxHandle,
_consumer: Box<dyn Fn(String) + Send + Sync>,
) -> Result<crate::backend::FollowHandle> {
unimplemented!("not exercised by this test suite")
}
async fn ensure_network(&self, _network_id: &str) -> Result<()> {
if self.fail_ensure_network {
return Err(RightsizeError::Backend("ensure_network failed".to_string()));
}
Ok(())
}
async fn remove_network(&self, _network_id: &str) -> Result<()> {
Ok(())
}
async fn install_network_links(
&self,
handle: &dyn SandboxHandle,
links: &[crate::backend::NetworkLink],
) -> Result<()> {
if self.fail_install_network_links {
return Err(RightsizeError::unsupported_with_remedy(
"network links (no nc in image; try docker)",
"fake",
"run with a different backend",
));
}
if !links.is_empty() {
self.state
.lock()
.unwrap()
.installed_links
.push((handle.id().to_string(), links.to_vec()));
}
Ok(())
}
fn cleanup_sync(&self, _container_id: &str) {}
fn remove_by_name(&self, _name: &str) {}
fn watchdog_kill_command(&self) -> Vec<String> {
vec!["true".to_string()]
}
async fn create_checkpoint(
&self,
handle: &dyn SandboxHandle,
nonce: &str,
) -> Result<String> {
if self.fail_create_checkpoint {
return Err(RightsizeError::Backend(
"create_checkpoint failed".to_string(),
));
}
let checkpoint_ref = format!("fake-checkpoint:{nonce}");
self.state
.lock()
.unwrap()
.committed
.push((handle.id().to_string(), checkpoint_ref.clone()));
Ok(checkpoint_ref)
}
async fn remove_checkpoint(&self, checkpoint_ref: &str) -> Result<()> {
self.state
.lock()
.unwrap()
.removed_checkpoints
.push(checkpoint_ref.to_string());
Ok(())
}
async fn has_checkpoint(&self, checkpoint_ref: &str) -> Result<bool> {
self.state
.lock()
.unwrap()
.probed_checkpoints
.push(checkpoint_ref.to_string());
match *self.probe_result.lock().unwrap() {
Some(exists) => Ok(exists),
None => Err(RightsizeError::Backend(
"has_checkpoint probe failed".to_string(),
)),
}
}
async fn export_checkpoint(
&self,
checkpoint_ref: &str,
dest: &std::path::Path,
) -> Result<()> {
if self.fail_export_checkpoint {
return Err(RightsizeError::Backend(
"export_checkpoint failed".to_string(),
));
}
std::fs::write(dest, format!("fake-artifact-payload:{checkpoint_ref}"))?;
self.state
.lock()
.unwrap()
.exported_checkpoints
.push((checkpoint_ref.to_string(), dest.to_path_buf()));
Ok(())
}
async fn import_checkpoint(
&self,
src_file: &std::path::Path,
ref_hint: &str,
) -> Result<String> {
let bytes = std::fs::read(src_file).unwrap_or_default();
self.state.lock().unwrap().imported_checkpoints.push((
src_file.to_path_buf(),
ref_hint.to_string(),
bytes,
));
Ok(self
.import_effective_ref
.lock()
.unwrap()
.clone()
.unwrap_or_else(|| format!("effective:{ref_hint}")))
}
async fn copy_to_container(
&self,
handle: &dyn SandboxHandle,
host_path: &std::path::Path,
container_path: &str,
) -> Result<()> {
let existed = host_path.exists();
self.state.lock().unwrap().copied_in.push((
handle.id().to_string(),
host_path.to_path_buf(),
container_path.to_string(),
existed,
));
Ok(())
}
async fn copy_from_container(
&self,
handle: &dyn SandboxHandle,
container_path: &str,
host_path: &std::path::Path,
) -> Result<()> {
self.state.lock().unwrap().copied_out.push((
handle.id().to_string(),
container_path.to_string(),
host_path.to_path_buf(),
));
Ok(())
}
}
fn container_on(backend: &Arc<FakeBackend>) -> Container {
Container::new("redis:8.6-alpine")
.with_backend(backend.clone())
.waiting_for(ReadyImmediately)
}
#[tokio::test]
async fn u1_start_allocates_ports_creates_spec_waits_and_maps_ports() {
let backend = FakeBackend::new();
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.with_env("A", "1");
let guard = c.start().await.expect("start must succeed");
let spec = {
let state = backend.state.lock().unwrap();
assert_eq!(state.created.len(), 1);
state.created[0].clone()
};
assert_eq!(spec.image, "redis:8.6-alpine");
assert_eq!(spec.env, vec![("A".to_string(), "1".to_string())]);
assert_eq!(spec.ports.len(), 1);
assert_eq!(spec.ports[0].guest_port, 6379);
assert!(spec.ports[0].host_port > 0);
assert_eq!(
guard.get_mapped_port(6379).unwrap(),
spec.ports[0].host_port
);
assert!(guard.is_running());
guard.stop().await.unwrap();
}
#[tokio::test]
async fn require_isolation_on_a_non_isolated_backend_errors_before_any_create() {
let backend = FakeBackend::new(); let c = container_on(&backend)
.with_exposed_ports(&[6379])
.require_isolation(true);
let err = expect_start_err(c.start().await, "require_isolation must refuse to start");
assert!(
matches!(err, RightsizeError::IsolationRequired { .. }),
"{err}"
);
let msg = err.to_string();
assert!(msg.contains("fake"), "{msg}");
assert!(msg.contains("RIGHTSIZE_BACKEND=microsandbox"), "{msg}");
assert!(
backend.state.lock().unwrap().created.is_empty(),
"no sandbox may be created when isolation is required and unavailable"
);
}
#[tokio::test]
async fn require_isolation_on_a_hardware_isolated_backend_starts_normally() {
let backend = FakeBackend::hardware_isolated();
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.require_isolation(true);
let guard = c.start().await.expect("start must succeed");
assert!(guard.is_running());
assert_eq!(backend.state.lock().unwrap().created.len(), 1);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn require_isolation_defaults_to_false_and_does_not_gate_a_normal_start() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.expect("start must succeed");
guard.stop().await.unwrap();
}
#[tokio::test]
async fn checkpoint_refuses_before_any_backend_call_when_capability_is_false() {
let backend = FakeBackend::new(); let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.expect("start must succeed");
let err = guard
.checkpoint()
.await
.expect_err("checkpoint must refuse on a non-checkpoint-capable backend");
assert!(
matches!(err, RightsizeError::CheckpointUnsupported { .. }),
"{err}"
);
let msg = err.to_string();
assert!(msg.contains("fake"), "{msg}");
assert!(msg.contains("capabilities().checkpoint"), "{msg}");
assert!(
backend.state.lock().unwrap().committed.is_empty(),
"create_checkpoint must never be called once capability gating refuses"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn checkpoint_on_a_non_running_container_is_a_state_error() {
let backend = FakeBackend::checkpoint_capable();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let mut guard = c.start().await.unwrap();
guard.stop_inner().await;
let err = guard
.checkpoint()
.await
.expect_err("checkpoint must refuse on a stopped guard");
assert!(err.to_string().contains("not running"), "{err}");
assert!(
backend.state.lock().unwrap().committed.is_empty(),
"create_checkpoint must never be called on a non-running guard"
);
}
#[tokio::test]
async fn checkpoint_returns_the_ref_backend_and_the_full_source_spec() {
let backend = FakeBackend::checkpoint_capable();
let c = container_on(&backend)
.with_env("A", "1")
.with_exposed_ports(&[6379])
.with_command(&["redis-server"])
.with_memory_limit(256);
let guard = c.start().await.unwrap();
let cp = guard.checkpoint().await.expect("checkpoint must succeed");
let tag = cp
.checkpoint_ref
.strip_prefix("fake-checkpoint:")
.unwrap_or_else(|| {
panic!(
"expected the fake-checkpoint: prefix, got {}",
cp.checkpoint_ref
)
});
assert_eq!(tag.len(), 12, "{}", cp.checkpoint_ref);
assert!(
tag.chars()
.all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()),
"{}",
cp.checkpoint_ref
);
assert_eq!(cp.backend, "fake");
assert_eq!(cp.spec.env, vec![("A".to_string(), "1".to_string())]);
assert_eq!(cp.spec.command, Some(vec!["redis-server".to_string()]));
assert_eq!(cp.spec.ports.len(), 1);
assert_eq!(cp.spec.ports[0].guest_port, 6379);
assert_eq!(cp.spec.memory_limit_mb, Some(256));
let committed = backend.state.lock().unwrap().committed.clone();
assert_eq!(committed.len(), 1);
assert_eq!(committed[0].0, guard.name());
assert_eq!(committed[0].1, cp.checkpoint_ref);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn from_checkpoint_under_a_different_backend_refuses_before_any_backend_call() {
let source_backend = FakeBackend::named("fake-a");
let source = container_on(&source_backend).with_exposed_ports(&[6379]);
let source_guard = source.start().await.unwrap();
let cp = source_guard.checkpoint().await.unwrap();
source_guard.stop().await.unwrap();
assert_eq!(cp.backend, "fake-a");
let restore_backend = FakeBackend::named("fake-b");
let restored = Container::from_checkpoint(&cp).with_backend(restore_backend.clone());
let err = expect_start_err(
restored.start().await,
"restoring under a different backend than the creator must refuse",
);
assert!(
matches!(err, RightsizeError::CheckpointBackendMismatch { .. }),
"{err}"
);
let msg = err.to_string();
assert!(msg.contains("fake-a"), "{msg}");
assert!(msg.contains("fake-b"), "{msg}");
assert!(msg.contains("RIGHTSIZE_BACKEND=fake-a"), "{msg}");
assert!(
restore_backend.state.lock().unwrap().created.is_empty(),
"no backend call may happen once the mismatch is detected"
);
}
#[tokio::test]
async fn reuse_combined_with_from_checkpoint_is_a_config_error() {
let source_backend = FakeBackend::checkpoint_capable();
let source = container_on(&source_backend).with_exposed_ports(&[6379]);
let source_guard = source.start().await.unwrap();
let cp = source_guard.checkpoint().await.unwrap();
source_guard.stop().await.unwrap();
let restore_backend = FakeBackend::checkpoint_capable();
let restored = Container::from_checkpoint(&cp)
.with_backend(restore_backend.clone())
.with_reuse_env_override(true)
.reuse(true);
let err = expect_start_err(
restored.start().await,
"reuse combined with from_checkpoint must refuse before any backend work",
);
assert!(
matches!(err, RightsizeError::ReuseCheckpointConflict),
"{err}"
);
assert!(
restore_backend.state.lock().unwrap().created.is_empty(),
"no backend call may happen once the conflict is detected"
);
}
struct CountingWait {
calls: Arc<StdMutex<usize>>,
}
#[async_trait::async_trait]
impl WaitStrategy for CountingWait {
async fn wait_until_ready(&self, _target: &dyn WaitTarget) -> Result<()> {
*self.calls.lock().unwrap() += 1;
Ok(())
}
fn with_startup_timeout(self: Box<Self>, _timeout: Duration) -> Box<dyn WaitStrategy> {
self
}
}
#[tokio::test]
async fn checkpoint_reruns_the_wait_strategy_when_the_backend_restarts_the_workload() {
let calls = Arc::new(StdMutex::new(0usize));
let backend = FakeBackend::checkpoint_capable_restarts_workload();
let c = Container::new("redis:8.6-alpine")
.with_backend(backend.clone())
.with_exposed_ports(&[6379])
.waiting_for(CountingWait {
calls: calls.clone(),
});
let guard = c.start().await.unwrap();
assert_eq!(*calls.lock().unwrap(), 1, "start() runs the wait once");
guard.checkpoint().await.expect("checkpoint must succeed");
assert_eq!(
*calls.lock().unwrap(),
2,
"a backend whose checkpoint restarts the workload must re-run the wait strategy \
before checkpoint() returns"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn checkpoint_does_not_rerun_the_wait_strategy_when_the_container_is_left_undisturbed() {
let calls = Arc::new(StdMutex::new(0usize));
let backend = FakeBackend::checkpoint_capable(); let c = Container::new("redis:8.6-alpine")
.with_backend(backend.clone())
.with_exposed_ports(&[6379])
.waiting_for(CountingWait {
calls: calls.clone(),
});
let guard = c.start().await.unwrap();
assert_eq!(*calls.lock().unwrap(), 1);
guard.checkpoint().await.expect("checkpoint must succeed");
assert_eq!(
*calls.lock().unwrap(),
1,
"a backend whose checkpoint leaves the container running must not re-run the wait"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn checkpoint_reinstalls_network_links_when_the_backend_restarts_the_workload() {
let backend = FakeBackend::checkpoint_capable_restarts_workload();
let net = Arc::new(Network::new_network());
let stub = container_on(&backend)
.with_exposed_ports(&[8888])
.with_network(&net)
.with_network_aliases(&["configuration-stub"]);
let stub_guard = stub.start().await.unwrap();
let app = container_on(&backend)
.with_exposed_ports(&[8080])
.with_network(&net)
.waiting_for(ReadyImmediately);
let app_guard = app.start().await.unwrap();
let links_at_start = {
let state = backend.state.lock().unwrap();
assert_eq!(
state.installed_links.len(),
1,
"start() installs the app's link to the already-running stub once"
);
state.installed_links[0].1.clone()
};
app_guard
.checkpoint()
.await
.expect("checkpoint must succeed");
{
let state = backend.state.lock().unwrap();
assert_eq!(
state.installed_links.len(),
2,
"a backend whose checkpoint restarts the workload must re-install the network \
links a second time before checkpoint() returns: {:?}",
state.installed_links
);
assert_eq!(
state.installed_links[1].1, links_at_start,
"the re-installed links must be the exact same links installed at start()"
);
}
app_guard.stop().await.unwrap();
stub_guard.stop().await.unwrap();
}
#[tokio::test]
async fn checkpoint_does_not_reinstall_network_links_when_the_container_is_left_undisturbed() {
let backend = FakeBackend::checkpoint_capable(); let net = Arc::new(Network::new_network());
let stub = container_on(&backend)
.with_exposed_ports(&[8888])
.with_network(&net)
.with_network_aliases(&["configuration-stub"]);
let stub_guard = stub.start().await.unwrap();
let app = container_on(&backend)
.with_exposed_ports(&[8080])
.with_network(&net)
.waiting_for(ReadyImmediately);
let app_guard = app.start().await.unwrap();
assert_eq!(backend.state.lock().unwrap().installed_links.len(), 1);
app_guard
.checkpoint()
.await
.expect("checkpoint must succeed");
assert_eq!(
backend.state.lock().unwrap().installed_links.len(),
1,
"a backend whose checkpoint leaves the container running must not re-install \
network links"
);
app_guard.stop().await.unwrap();
stub_guard.stop().await.unwrap();
}
#[tokio::test]
async fn from_checkpoint_applies_the_spec_defaults_and_allows_overrides() {
let source_backend = FakeBackend::checkpoint_capable();
let source = container_on(&source_backend)
.with_env("A", "1")
.with_exposed_ports(&[6379])
.with_command(&["redis-server"])
.with_memory_limit(256);
let source_guard = source.start().await.unwrap();
let cp = source_guard.checkpoint().await.unwrap();
source_guard.stop().await.unwrap();
let restore_backend = FakeBackend::new();
let restored = Container::from_checkpoint(&cp)
.with_backend(restore_backend.clone())
.waiting_for(ReadyImmediately);
let restored_guard = restored.start().await.expect("restore must start");
let created = restore_backend.state.lock().unwrap().created[0].clone();
assert_eq!(created.image, cp.checkpoint_ref);
assert_eq!(created.checkpoint_ref, Some(cp.checkpoint_ref.clone()));
assert_eq!(created.env, vec![("A".to_string(), "1".to_string())]);
assert_eq!(created.command, Some(vec!["redis-server".to_string()]));
assert_eq!(created.ports.len(), 1);
assert_eq!(created.ports[0].guest_port, 6379);
assert_eq!(created.memory_limit_mb, Some(256));
restored_guard.stop().await.unwrap();
let override_backend = FakeBackend::new();
let overridden = Container::from_checkpoint(&cp)
.with_backend(override_backend.clone())
.waiting_for(ReadyImmediately)
.with_command(&["redis-server", "--appendonly", "yes"]);
let overridden_guard = overridden.start().await.expect("restore must start");
let created = override_backend.state.lock().unwrap().created[0].clone();
assert_eq!(
created.command,
Some(vec![
"redis-server".to_string(),
"--appendonly".to_string(),
"yes".to_string()
])
);
overridden_guard.stop().await.unwrap();
}
fn sample_named_entry(
name: &str,
checkpoint_ref: &str,
backend: &str,
) -> checkpoint::NamedRegistryEntry {
checkpoint::NamedRegistryEntry {
name: name.to_string(),
checkpoint_ref: checkpoint_ref.to_string(),
backend: backend.to_string(),
created_iso: "2025-01-01T00:00:00Z".to_string(),
spec: checkpoint::NamedRegistrySpec {
env: std::collections::BTreeMap::from([("A".to_string(), "1".to_string())]),
command: Some(vec!["redis-server".to_string()]),
exposed_ports: vec![6379],
memory_limit_mb: Some(256),
},
}
}
#[tokio::test]
async fn checkpoint_named_rejects_an_invalid_name_before_any_backend_call() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("checkpoint-named-invalid");
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.with_checkpoint_cache_dir_override(cache_dir);
let guard = c.start().await.unwrap();
let err = guard
.checkpoint_named("Bad Name!")
.await
.expect_err("an invalid name must be rejected");
assert!(
matches!(err, RightsizeError::InvalidCheckpointName { .. }),
"{err}"
);
assert!(
backend.state.lock().unwrap().committed.is_empty(),
"create_checkpoint must never be called once name validation refuses"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn checkpoint_named_writes_the_pinned_registry_json_after_backend_success() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("checkpoint-named-write");
let c = container_on(&backend)
.with_env("A", "1")
.with_exposed_ports(&[6379])
.with_command(&["redis-server"])
.with_memory_limit(256)
.with_checkpoint_cache_dir_override(cache_dir.clone());
let guard = c.start().await.unwrap();
let cp = guard
.checkpoint_named("seeded-db")
.await
.expect("checkpoint_named must succeed");
assert_eq!(cp.checkpoint_ref, "fake-checkpoint:seeded-db");
assert_eq!(cp.backend, "fake");
let raw = std::fs::read_to_string(cache_dir.join("checkpoints").join("seeded-db.json"))
.expect("the registry file must exist");
for pinned in [
"\"name\": \"seeded-db\"",
"\"ref\": \"fake-checkpoint:seeded-db\"",
"\"backend\": \"fake\"",
"\"createdIso\"",
"\"spec\"",
"\"env\"",
"\"command\"",
"\"exposedPorts\"",
"\"memoryLimitMb\": 256",
] {
assert!(raw.contains(pinned), "{pinned} missing from {raw}");
}
guard.stop().await.unwrap();
}
#[tokio::test]
async fn checkpoint_named_writes_no_registry_entry_when_the_backend_call_fails() {
let backend = FakeBackend::checkpoint_capable_that_fails_create();
let cache_dir = temp_cache_dir("checkpoint-named-fail");
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.with_checkpoint_cache_dir_override(cache_dir.clone());
let guard = c.start().await.unwrap();
guard
.checkpoint_named("seeded-db")
.await
.expect_err("a failing backend checkpoint must propagate");
assert!(
!cache_dir
.join("checkpoints")
.join("seeded-db.json")
.exists(),
"no registry entry may be written when the backend call fails"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn checkpoint_named_replaces_an_existing_name_removing_the_old_ref_first() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("checkpoint-named-replace");
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.with_checkpoint_cache_dir_override(cache_dir.clone());
let guard = c.start().await.unwrap();
let first = guard.checkpoint_named("seeded-db").await.unwrap();
assert_eq!(first.checkpoint_ref, "fake-checkpoint:seeded-db");
let second = guard.checkpoint_named("seeded-db").await.unwrap();
assert_eq!(second.checkpoint_ref, "fake-checkpoint:seeded-db");
assert_eq!(
backend.state.lock().unwrap().removed_checkpoints,
vec![first.checkpoint_ref.clone()],
"re-checkpointing an existing name must best-effort remove the OLD ref first"
);
let entry = checkpoint::Registry::new(&cache_dir, "seeded-db")
.read()
.expect("the registry entry must still exist");
assert_eq!(entry.checkpoint_ref, second.checkpoint_ref);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn checkpoint_named_replace_never_removes_a_different_backend_entry() {
let backend = FakeBackend::checkpoint_capable(); let cache_dir = temp_cache_dir("checkpoint-named-replace-cross-backend");
checkpoint::Registry::new(&cache_dir, "seeded-db")
.write_atomic(&sample_named_entry(
"seeded-db",
"other-backend-ref",
"fake-other",
))
.unwrap();
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.with_checkpoint_cache_dir_override(cache_dir.clone());
let guard = c.start().await.unwrap();
let replaced = guard.checkpoint_named("seeded-db").await.unwrap();
assert_eq!(replaced.checkpoint_ref, "fake-checkpoint:seeded-db");
assert!(
backend.state.lock().unwrap().removed_checkpoints.is_empty(),
"a different-backend entry's ref must never reach remove_checkpoint"
);
let entry = checkpoint::Registry::new(&cache_dir, "seeded-db")
.read()
.expect("the registry entry must be overwritten with the new one");
assert_eq!(entry.checkpoint_ref, replaced.checkpoint_ref);
assert_eq!(entry.backend, "fake");
guard.stop().await.unwrap();
}
#[tokio::test]
async fn find_named_returns_none_when_no_entry_exists() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("find-named-absent");
let found = find_named(
"seeded-db",
&(backend as Arc<dyn SandboxBackend>),
&cache_dir,
)
.await
.unwrap();
assert!(found.is_none());
}
#[tokio::test]
async fn find_named_returns_the_checkpoint_when_the_artifact_still_exists() {
let backend = FakeBackend::checkpoint_capable(); let cache_dir = temp_cache_dir("find-named-present");
checkpoint::Registry::new(&cache_dir, "seeded-db")
.write_atomic(&sample_named_entry(
"seeded-db",
"fake-checkpoint:abc",
"fake",
))
.unwrap();
let found = find_named(
"seeded-db",
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
)
.await
.unwrap()
.expect("an entry whose artifact exists must be returned");
assert_eq!(found.checkpoint_ref, "fake-checkpoint:abc");
assert_eq!(found.backend, "fake");
assert_eq!(found.spec.command, Some(vec!["redis-server".to_string()]));
assert_eq!(found.spec.ports[0].guest_port, 6379);
assert_eq!(found.spec.memory_limit_mb, Some(256));
assert_eq!(
backend.state.lock().unwrap().probed_checkpoints,
vec!["fake-checkpoint:abc".to_string()]
);
}
#[tokio::test]
async fn find_named_treats_a_gone_artifact_as_stale_and_deletes_the_entry() {
let backend = FakeBackend::checkpoint_capable();
backend.set_probe_result(Some(false)); let cache_dir = temp_cache_dir("find-named-stale");
let registry = checkpoint::Registry::new(&cache_dir, "seeded-db");
registry
.write_atomic(&sample_named_entry(
"seeded-db",
"fake-checkpoint:gone",
"fake",
))
.unwrap();
let found = find_named(
"seeded-db",
&(backend as Arc<dyn SandboxBackend>),
&cache_dir,
)
.await
.unwrap();
assert!(found.is_none());
assert!(
!registry.exists(),
"a stale entry (artifact confirmed gone) must be deleted"
);
}
#[tokio::test]
async fn find_named_treats_corrupt_json_as_absent_without_probing() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("find-named-corrupt");
std::fs::create_dir_all(cache_dir.join("checkpoints")).unwrap();
std::fs::write(
cache_dir.join("checkpoints").join("seeded-db.json"),
b"not json",
)
.unwrap();
let found = find_named(
"seeded-db",
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
)
.await
.unwrap();
assert!(found.is_none());
assert!(
backend.state.lock().unwrap().probed_checkpoints.is_empty(),
"a corrupt entry must never reach the probe"
);
}
#[tokio::test]
async fn find_named_returns_a_different_backend_entry_unprobed() {
let backend = FakeBackend::named("fake-active");
let cache_dir = temp_cache_dir("find-named-other-backend");
checkpoint::Registry::new(&cache_dir, "seeded-db")
.write_atomic(&sample_named_entry("seeded-db", "other-ref", "fake-other"))
.unwrap();
let found = find_named(
"seeded-db",
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
)
.await
.unwrap()
.expect("a different-backend entry must still be returned");
assert_eq!(found.backend, "fake-other");
assert!(
backend.state.lock().unwrap().probed_checkpoints.is_empty(),
"a different-backend entry must never be probed via the active backend"
);
}
#[tokio::test]
async fn find_named_propagates_a_probe_failure_as_an_error() {
let backend = FakeBackend::checkpoint_capable();
backend.set_probe_result(None); let cache_dir = temp_cache_dir("find-named-probe-error");
checkpoint::Registry::new(&cache_dir, "seeded-db")
.write_atomic(&sample_named_entry(
"seeded-db",
"fake-checkpoint:x",
"fake",
))
.unwrap();
let err = find_named(
"seeded-db",
&(backend as Arc<dyn SandboxBackend>),
&cache_dir,
)
.await
.expect_err("a probe failure must surface, never resolve to Ok(None)");
assert!(matches!(err, RightsizeError::Backend(_)), "{err}");
}
#[tokio::test]
async fn list_named_skips_corrupt_entries() {
let cache_dir = temp_cache_dir("list-named-mixed");
assert!(list_named(&cache_dir).unwrap().is_empty());
checkpoint::Registry::new(&cache_dir, "good")
.write_atomic(&sample_named_entry("good", "fake-checkpoint:good", "fake"))
.unwrap();
std::fs::write(
cache_dir.join("checkpoints").join("corrupt.json"),
b"not json",
)
.unwrap();
let listed = list_named(&cache_dir).unwrap();
assert_eq!(listed.len(), 1, "{listed:?}");
assert_eq!(listed[0].checkpoint_ref, "fake-checkpoint:good");
}
#[tokio::test]
async fn remove_named_is_idempotent_and_reports_correctly() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("remove-named");
let registry = checkpoint::Registry::new(&cache_dir, "seeded-db");
registry
.write_atomic(&sample_named_entry(
"seeded-db",
"fake-checkpoint:x",
"fake",
))
.unwrap();
let removed = remove_named(
"seeded-db",
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
)
.await
.unwrap();
assert!(removed, "a name with a registry entry must report true");
assert!(!registry.exists());
assert_eq!(
backend.state.lock().unwrap().removed_checkpoints,
vec!["fake-checkpoint:x".to_string()]
);
let removed_again = remove_named(
"seeded-db",
&(backend as Arc<dyn SandboxBackend>),
&cache_dir,
)
.await
.unwrap();
assert!(
!removed_again,
"removing a name with no registry entry must report false, not error"
);
}
#[tokio::test]
async fn remove_named_deletes_a_different_backend_entry_without_calling_remove_checkpoint() {
let backend = FakeBackend::named("fake-active");
let cache_dir = temp_cache_dir("remove-named-cross-backend");
let registry = checkpoint::Registry::new(&cache_dir, "seeded-db");
registry
.write_atomic(&sample_named_entry(
"seeded-db",
"other-backend-ref",
"fake-other",
))
.unwrap();
let removed = remove_named(
"seeded-db",
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
)
.await
.unwrap();
assert!(
removed,
"a different-backend entry still counts as something existed to remove"
);
assert!(
!registry.exists(),
"the registry entry must be deleted regardless of the entry's backend"
);
assert!(
backend.state.lock().unwrap().removed_checkpoints.is_empty(),
"a different-backend entry's ref must never reach remove_checkpoint"
);
}
fn temp_archive_path(label: &str) -> std::path::PathBuf {
temp_cache_dir(label).join("cp.archive")
}
fn archive_manifest(
name: Option<&str>,
backend: &str,
checkpoint_ref: &str,
) -> crate::archive::ArchiveManifest {
crate::archive::ArchiveManifest {
rightsize_archive: crate::archive::FORMAT_VERSION,
name: name.map(str::to_string),
checkpoint_ref: checkpoint_ref.to_string(),
backend: backend.to_string(),
created_iso: "2025-01-01T00:00:00Z".to_string(),
spec: checkpoint::NamedRegistrySpec {
env: std::collections::BTreeMap::from([("A".to_string(), "1".to_string())]),
command: Some(vec!["redis-server".to_string()]),
exposed_ports: vec![6379],
memory_limit_mb: Some(256),
},
}
}
async fn write_archive(
dest: &std::path::Path,
manifest: &crate::archive::ArchiveManifest,
artifact_bytes: &[u8],
) {
let staging = crate::archive::TempStagingDir::create("test-fixture").unwrap();
crate::archive::write_manifest(&crate::archive::manifest_path(staging.path()), manifest)
.unwrap();
std::fs::write(
crate::archive::artifact_path(staging.path()),
artifact_bytes,
)
.unwrap();
crate::archive::tar_create(dest, staging.path())
.await
.unwrap();
}
async fn write_archive_missing_manifest(dest: &std::path::Path) {
let staging = crate::archive::TempStagingDir::create("test-fixture-no-manifest").unwrap();
std::fs::write(crate::archive::artifact_path(staging.path()), b"artifact").unwrap();
let output = tokio::process::Command::new("tar")
.arg("-cf")
.arg(dest)
.arg("-C")
.arg(staging.path())
.arg("artifact")
.output()
.await
.unwrap();
assert!(output.status.success(), "{output:?}");
}
async fn write_archive_malformed_json(dest: &std::path::Path) {
let staging = crate::archive::TempStagingDir::create("test-fixture-bad-json").unwrap();
std::fs::write(crate::archive::manifest_path(staging.path()), b"not json").unwrap();
std::fs::write(crate::archive::artifact_path(staging.path()), b"artifact").unwrap();
crate::archive::tar_create(dest, staging.path())
.await
.unwrap();
}
#[tokio::test]
async fn export_to_refuses_on_a_backend_mismatch_before_any_backend_call() {
let cp = Checkpoint {
checkpoint_ref: "fake-checkpoint:abc".to_string(),
backend: "fake-a".to_string(),
spec: ContainerSpec::new("rz-x-0", "redis:8.6-alpine", "deadbeef"),
};
let active_backend = FakeBackend::named("fake-b");
let dest = temp_archive_path("export-mismatch");
let err = export_to_impl(
&cp,
&dest,
&(active_backend.clone() as Arc<dyn SandboxBackend>),
&temp_cache_dir("export-mismatch-cache"),
&std::env::temp_dir(),
)
.await
.expect_err("a backend mismatch must refuse before any work");
assert!(
matches!(err, RightsizeError::CheckpointBackendMismatch { .. }),
"{err}"
);
assert!(!dest.exists(), "no archive may be written on a mismatch");
assert!(
active_backend
.state
.lock()
.unwrap()
.exported_checkpoints
.is_empty(),
"no backend call may happen once the mismatch is detected"
);
}
#[tokio::test]
async fn export_to_refuses_when_the_artifact_is_stale() {
let backend = FakeBackend::checkpoint_capable();
backend.set_probe_result(Some(false)); let cp = Checkpoint {
checkpoint_ref: "fake-checkpoint:gone".to_string(),
backend: "fake".to_string(),
spec: ContainerSpec::new("rz-x-0", "redis:8.6-alpine", "deadbeef"),
};
let dest = temp_archive_path("export-stale");
let err = export_to_impl(
&cp,
&dest,
&(backend.clone() as Arc<dyn SandboxBackend>),
&temp_cache_dir("export-stale-cache"),
&std::env::temp_dir(),
)
.await
.expect_err("a stale artifact must refuse");
assert!(
matches!(err, RightsizeError::CheckpointArtifactMissing { .. }),
"{err}"
);
assert!(!dest.exists());
assert!(
backend
.state
.lock()
.unwrap()
.exported_checkpoints
.is_empty(),
"export_checkpoint must never be called once the staleness probe refuses"
);
}
#[tokio::test]
async fn export_to_cleans_up_its_staging_dir_on_success_and_on_failure() {
let staging_parent = temp_cache_dir("export-cleanup-staging-parent");
let matching_staging_dirs = |label: &str| -> Vec<std::path::PathBuf> {
std::fs::read_dir(&staging_parent)
.map(|rd| {
rd.filter_map(std::result::Result::ok)
.map(|e| e.path())
.filter(|p| {
p.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.contains(label))
})
.collect()
})
.unwrap_or_default()
};
let backend = FakeBackend::checkpoint_capable();
let cp = Checkpoint {
checkpoint_ref: "fake-checkpoint:ok".to_string(),
backend: "fake".to_string(),
spec: ContainerSpec::new("rz-x-0", "redis:8.6-alpine", "deadbeef"),
};
export_to_impl(
&cp,
&temp_archive_path("export-cleanup-ok"),
&(backend as Arc<dyn SandboxBackend>),
&temp_cache_dir("export-cleanup-ok-cache"),
&staging_parent,
)
.await
.expect("export must succeed");
assert!(
matching_staging_dirs("rightsize-archive-export-").is_empty(),
"a successful export must leave no staging dir behind"
);
let failing_backend = FakeBackend::checkpoint_capable_that_fails_export();
let cp = Checkpoint {
checkpoint_ref: "fake-checkpoint:fail".to_string(),
backend: "fake".to_string(),
spec: ContainerSpec::new("rz-x-0", "redis:8.6-alpine", "deadbeef"),
};
export_to_impl(
&cp,
&temp_archive_path("export-cleanup-fail"),
&(failing_backend as Arc<dyn SandboxBackend>),
&temp_cache_dir("export-cleanup-fail-cache"),
&staging_parent,
)
.await
.expect_err("export_checkpoint must fail");
assert!(
matching_staging_dirs("rightsize-archive-export-").is_empty(),
"a failed export must ALSO leave no staging dir behind"
);
}
#[tokio::test]
async fn export_then_import_round_trips_payload_and_metadata_and_propagates_the_effective_ref()
{
let export_backend = FakeBackend::checkpoint_capable();
let export_cache = temp_cache_dir("archive-roundtrip-export-cache");
checkpoint::Registry::new(&export_cache, "seeded-db")
.write_atomic(&sample_named_entry(
"seeded-db",
"fake-checkpoint:seeded-db",
"fake",
))
.unwrap();
let mut spec = ContainerSpec::new("rz-x-0", "redis:8.6-alpine", "deadbeef");
spec.env = vec![("A".to_string(), "1".to_string())];
spec.command = Some(vec!["redis-server".to_string()]);
spec.ports = vec![crate::model::PortBinding {
host_port: 0,
guest_port: 6379,
}];
spec.memory_limit_mb = Some(256);
let cp = Checkpoint {
checkpoint_ref: "fake-checkpoint:seeded-db".to_string(),
backend: "fake".to_string(),
spec,
};
let dest = temp_archive_path("archive-roundtrip");
export_to_impl(
&cp,
&dest,
&(export_backend.clone() as Arc<dyn SandboxBackend>),
&export_cache,
&std::env::temp_dir(),
)
.await
.expect("export must succeed");
assert_eq!(
export_backend.state.lock().unwrap().exported_checkpoints[0].0,
"fake-checkpoint:seeded-db"
);
let import_backend = FakeBackend::checkpoint_capable();
let import_cache = temp_cache_dir("archive-roundtrip-import-cache");
let imported = import_from_impl(
&dest,
&(import_backend.clone() as Arc<dyn SandboxBackend>),
&import_cache,
&std::env::temp_dir(),
)
.await
.expect("import must succeed");
assert_eq!(
imported.checkpoint_ref,
"effective:fake-checkpoint:seeded-db"
);
assert_eq!(imported.backend, "fake");
assert_eq!(
imported.spec.command,
Some(vec!["redis-server".to_string()])
);
assert_eq!(imported.spec.ports[0].guest_port, 6379);
assert_eq!(imported.spec.memory_limit_mb, Some(256));
let state = import_backend.state.lock().unwrap();
assert_eq!(state.imported_checkpoints.len(), 1);
assert_eq!(state.imported_checkpoints[0].1, "fake-checkpoint:seeded-db");
assert_eq!(
state.imported_checkpoints[0].2,
b"fake-artifact-payload:fake-checkpoint:seeded-db".to_vec(),
"the imported artifact bytes must match exactly what export_checkpoint wrote"
);
drop(state);
let entry = checkpoint::Registry::new(&import_cache, "seeded-db")
.read()
.expect("a named archive must write a registry entry on import");
assert_eq!(entry.checkpoint_ref, imported.checkpoint_ref);
assert_eq!(entry.backend, "fake");
}
#[tokio::test]
async fn an_unnamed_checkpoint_exports_and_imports_with_a_null_name_and_no_registry_write() {
let export_backend = FakeBackend::checkpoint_capable();
let cp = Checkpoint {
checkpoint_ref: "fake-checkpoint:anon".to_string(),
backend: "fake".to_string(),
spec: ContainerSpec::new("rz-x-0", "redis:8.6-alpine", "deadbeef"),
};
let dest = temp_archive_path("archive-unnamed");
export_to_impl(
&cp,
&dest,
&(export_backend as Arc<dyn SandboxBackend>),
&temp_cache_dir("archive-unnamed-export-cache"), &std::env::temp_dir(),
)
.await
.unwrap();
let staging = crate::archive::TempStagingDir::create("archive-unnamed-inspect").unwrap();
crate::archive::tar_extract(&dest, staging.path())
.await
.unwrap();
let raw = std::fs::read_to_string(crate::archive::manifest_path(staging.path())).unwrap();
assert!(raw.contains("\"name\": null"), "{raw}");
let import_backend = FakeBackend::checkpoint_capable();
let import_cache = temp_cache_dir("archive-unnamed-import-cache");
let imported = import_from_impl(
&dest,
&(import_backend as Arc<dyn SandboxBackend>),
&import_cache,
&std::env::temp_dir(),
)
.await
.unwrap();
assert_eq!(imported.checkpoint_ref, "effective:fake-checkpoint:anon");
assert!(
checkpoint::list_registry_entries(&import_cache)
.unwrap()
.is_empty(),
"an unnamed archive must write no registry entry at all"
);
}
#[tokio::test]
async fn import_from_a_missing_file_is_a_typed_error_before_any_backend_call() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("import-missing-file");
let err = import_from_impl(
std::path::Path::new("/definitely/not/a/real/archive"),
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
&std::env::temp_dir(),
)
.await
.expect_err("a missing archive file must be a typed error");
assert!(
matches!(err, RightsizeError::MalformedArchive { .. }),
"{err}"
);
assert!(
backend
.state
.lock()
.unwrap()
.imported_checkpoints
.is_empty(),
"no backend call may happen once extraction fails"
);
assert!(list_named(&cache_dir).unwrap().is_empty());
}
#[tokio::test]
async fn import_from_an_archive_missing_checkpoint_json_is_a_typed_error() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("import-missing-manifest");
let archive = temp_archive_path("import-missing-manifest-archive");
write_archive_missing_manifest(&archive).await;
let err = import_from_impl(
&archive,
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
&std::env::temp_dir(),
)
.await
.expect_err("a missing checkpoint.json must be a typed error");
assert!(
matches!(err, RightsizeError::MalformedArchive { .. }),
"{err}"
);
assert!(
backend
.state
.lock()
.unwrap()
.imported_checkpoints
.is_empty()
);
assert!(list_named(&cache_dir).unwrap().is_empty());
}
#[tokio::test]
async fn import_from_malformed_json_is_a_typed_error() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("import-malformed-json");
let archive = temp_archive_path("import-malformed-json-archive");
write_archive_malformed_json(&archive).await;
let err = import_from_impl(
&archive,
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
&std::env::temp_dir(),
)
.await
.expect_err("malformed JSON must be a typed error");
assert!(
matches!(err, RightsizeError::MalformedArchive { .. }),
"{err}"
);
assert!(
backend
.state
.lock()
.unwrap()
.imported_checkpoints
.is_empty()
);
assert!(list_named(&cache_dir).unwrap().is_empty());
}
#[tokio::test]
async fn import_from_an_unsupported_archive_version_is_a_typed_error_naming_the_value() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("import-bad-version");
let archive = temp_archive_path("import-bad-version-archive");
let mut manifest = archive_manifest(Some("seeded-db"), "fake", "fake-checkpoint:x");
manifest.rightsize_archive = 2;
write_archive(&archive, &manifest, b"artifact").await;
let err = import_from_impl(
&archive,
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
&std::env::temp_dir(),
)
.await
.expect_err("an unsupported version must be a typed error");
match err {
RightsizeError::MalformedArchive { reason, .. } => {
assert!(reason.contains('2'), "{reason}");
}
other => panic!("expected MalformedArchive, got {other:?}"),
}
assert!(
backend
.state
.lock()
.unwrap()
.imported_checkpoints
.is_empty()
);
assert!(list_named(&cache_dir).unwrap().is_empty());
}
#[tokio::test]
async fn import_from_an_invalid_name_is_a_typed_error() {
let backend = FakeBackend::checkpoint_capable();
let cache_dir = temp_cache_dir("import-invalid-name");
let archive = temp_archive_path("import-invalid-name-archive");
let manifest = archive_manifest(Some("Bad Name!"), "fake", "fake-checkpoint:x");
write_archive(&archive, &manifest, b"artifact").await;
let err = import_from_impl(
&archive,
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
&std::env::temp_dir(),
)
.await
.expect_err("an invalid name must be a typed error");
assert!(
matches!(err, RightsizeError::InvalidCheckpointName { .. }),
"{err}"
);
assert!(
backend
.state
.lock()
.unwrap()
.imported_checkpoints
.is_empty()
);
assert!(list_named(&cache_dir).unwrap().is_empty());
}
#[tokio::test]
async fn import_from_a_backend_mismatch_is_a_typed_error() {
let backend = FakeBackend::named("fake-active"); let cache_dir = temp_cache_dir("import-backend-mismatch");
let archive = temp_archive_path("import-backend-mismatch-archive");
let manifest = archive_manifest(Some("seeded-db"), "fake-other", "other-ref");
write_archive(&archive, &manifest, b"artifact").await;
let err = import_from_impl(
&archive,
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
&std::env::temp_dir(),
)
.await
.expect_err("a backend mismatch must be a typed error");
assert!(
matches!(err, RightsizeError::CheckpointBackendMismatch { .. }),
"{err}"
);
assert!(
backend
.state
.lock()
.unwrap()
.imported_checkpoints
.is_empty()
);
assert!(list_named(&cache_dir).unwrap().is_empty());
}
#[tokio::test]
async fn import_replaces_an_existing_same_backend_entry_removing_the_old_ref_first() {
let backend = FakeBackend::checkpoint_capable();
backend.set_import_effective_ref("fake-checkpoint:new-digest");
let cache_dir = temp_cache_dir("import-replace-same-backend");
checkpoint::Registry::new(&cache_dir, "seeded-db")
.write_atomic(&sample_named_entry(
"seeded-db",
"fake-checkpoint:old-digest",
"fake",
))
.unwrap();
let archive = temp_archive_path("import-replace-same-backend-archive");
let manifest = archive_manifest(Some("seeded-db"), "fake", "fake-checkpoint:whatever");
write_archive(&archive, &manifest, b"artifact").await;
let imported = import_from_impl(
&archive,
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
&std::env::temp_dir(),
)
.await
.expect("import must succeed");
assert_eq!(imported.checkpoint_ref, "fake-checkpoint:new-digest");
assert_eq!(
backend.state.lock().unwrap().removed_checkpoints,
vec!["fake-checkpoint:old-digest".to_string()],
"the OLD ref must be best-effort removed before the registry is rewritten"
);
let entry = checkpoint::Registry::new(&cache_dir, "seeded-db")
.read()
.expect("the registry entry must still exist");
assert_eq!(entry.checkpoint_ref, "fake-checkpoint:new-digest");
}
#[tokio::test]
async fn import_never_removes_the_old_artifact_when_the_effective_ref_is_unchanged() {
let backend = FakeBackend::checkpoint_capable();
backend.set_import_effective_ref("fake-checkpoint:same-digest");
let cache_dir = temp_cache_dir("import-replace-same-ref");
checkpoint::Registry::new(&cache_dir, "seeded-db")
.write_atomic(&sample_named_entry(
"seeded-db",
"fake-checkpoint:same-digest",
"fake",
))
.unwrap();
let archive = temp_archive_path("import-replace-same-ref-archive");
let manifest = archive_manifest(Some("seeded-db"), "fake", "fake-checkpoint:whatever");
write_archive(&archive, &manifest, b"artifact").await;
let imported = import_from_impl(
&archive,
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
&std::env::temp_dir(),
)
.await
.expect("import must succeed");
assert_eq!(imported.checkpoint_ref, "fake-checkpoint:same-digest");
assert!(
backend.state.lock().unwrap().removed_checkpoints.is_empty(),
"re-importing an archive that resolves to the SAME ref must never remove it"
);
}
#[tokio::test]
async fn import_never_calls_remove_checkpoint_on_a_different_backend_entry() {
let backend = FakeBackend::named("fake-active");
let cache_dir = temp_cache_dir("import-replace-cross-backend");
checkpoint::Registry::new(&cache_dir, "seeded-db")
.write_atomic(&sample_named_entry(
"seeded-db",
"other-backend-ref",
"fake-other",
))
.unwrap();
let archive = temp_archive_path("import-replace-cross-backend-archive");
let manifest = archive_manifest(Some("seeded-db"), "fake-active", "fake-checkpoint:x");
write_archive(&archive, &manifest, b"artifact").await;
let imported = import_from_impl(
&archive,
&(backend.clone() as Arc<dyn SandboxBackend>),
&cache_dir,
&std::env::temp_dir(),
)
.await
.expect("import must succeed");
assert!(
backend.state.lock().unwrap().removed_checkpoints.is_empty(),
"a different-backend entry's ref must never reach remove_checkpoint"
);
let entry = checkpoint::Registry::new(&cache_dir, "seeded-db")
.read()
.expect("the registry entry must be overwritten with the new one");
assert_eq!(entry.checkpoint_ref, imported.checkpoint_ref);
assert_eq!(entry.backend, "fake-active");
}
#[tokio::test]
async fn copy_file_to_container_on_a_non_running_container_is_a_state_error() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let mut guard = c.start().await.unwrap();
guard.stop_inner().await;
let err = guard
.copy_file_to_container("/tmp/does-not-matter", "/dst")
.await
.expect_err("copy must refuse on a stopped guard");
assert!(err.to_string().contains("not running"), "{err}");
assert!(
backend.state.lock().unwrap().copied_in.is_empty(),
"the backend must never be called once the running check refuses"
);
}
#[tokio::test]
async fn copy_file_from_container_on_a_non_running_container_is_a_state_error() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let mut guard = c.start().await.unwrap();
guard.stop_inner().await;
let err = guard
.copy_file_from_container("/src", "/tmp/does-not-matter")
.await
.expect_err("copy must refuse on a stopped guard");
assert!(err.to_string().contains("not running"), "{err}");
assert!(backend.state.lock().unwrap().copied_out.is_empty());
}
#[tokio::test]
async fn copy_file_to_container_with_a_relative_path_is_a_typed_error() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap();
let err = guard
.copy_file_to_container("/tmp/does-not-matter", "relative/path")
.await
.expect_err("a relative container path must be refused");
assert!(err.to_string().contains("must be absolute"), "{err}");
assert!(backend.state.lock().unwrap().copied_in.is_empty());
guard.stop().await.unwrap();
}
#[tokio::test]
async fn copy_file_from_container_with_a_relative_path_is_a_typed_error() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap();
let err = guard
.copy_file_from_container("relative/path", "/tmp/does-not-matter")
.await
.expect_err("a relative container path must be refused");
assert!(err.to_string().contains("must be absolute"), "{err}");
assert!(backend.state.lock().unwrap().copied_out.is_empty());
guard.stop().await.unwrap();
}
#[tokio::test]
async fn copy_file_to_container_issues_a_mkdir_p_for_the_parent_before_the_backend_copy() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap();
guard
.copy_file_to_container("/tmp/host-src.txt", "/mnt/nested/dst.txt")
.await
.expect("copy must succeed against the fake backend");
{
let state = backend.state.lock().unwrap();
assert_eq!(state.copied_in.len(), 1);
assert_eq!(state.copied_in[0].2, "/mnt/nested/dst.txt");
assert_eq!(
state.copied_in[0].1,
std::path::PathBuf::from("/tmp/host-src.txt")
);
}
guard.stop().await.unwrap();
}
#[tokio::test]
async fn copy_content_to_container_creates_and_removes_its_temp_file() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap();
guard
.copy_content_to_container(b"hello from memory".as_slice(), "/dst/from-memory.txt")
.await
.expect("copy must succeed against the fake backend");
let (temp_path, existed_during_the_call) = {
let state = backend.state.lock().unwrap();
assert_eq!(state.copied_in.len(), 1);
assert_eq!(state.copied_in[0].2, "/dst/from-memory.txt");
(state.copied_in[0].1.clone(), state.copied_in[0].3)
};
assert!(
existed_during_the_call,
"the temp file must exist at the moment the backend is called: {temp_path:?}"
);
assert!(
!temp_path.exists(),
"the temp file must be removed once the copy call returns: {temp_path:?}"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn restored_container_is_registered_in_the_reaping_ledger_like_any_other() {
let cache_dir = temp_cache_dir("restored-ledger");
let source_backend = FakeBackend::checkpoint_capable();
let source = container_on(&source_backend)
.with_exposed_ports(&[6379])
.with_reaper_cache_dir_override(cache_dir.clone());
let source_guard = source.start().await.unwrap();
let cp = source_guard.checkpoint().await.unwrap();
source_guard.stop().await.unwrap();
let restore_backend = FakeBackend::new();
let restored = Container::from_checkpoint(&cp)
.with_backend(restore_backend.clone())
.with_reaper_cache_dir_override(cache_dir.clone())
.waiting_for(ReadyImmediately);
let restored_guard = restored.start().await.expect("restore must start");
let ledger_name = restore_backend.state.lock().unwrap().created[0]
.name
.clone();
let ledger = crate::reaper::Ledger::new(&cache_dir, crate::RunId::value());
assert!(
ledger.sandbox_names().contains(&ledger_name),
"a restored container must be listed in the reaping ledger like any other"
);
restored_guard.stop().await.unwrap();
}
#[tokio::test]
async fn a_started_container_is_registered_for_diagnostics_and_deregistered_on_stop() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap();
let name = guard.name().to_string();
let header = format!("-- {name} (");
let report = crate::diagnostics().await;
assert!(report.contains(&header), "{report}");
guard.stop().await.unwrap();
let report_after_stop = crate::diagnostics().await;
assert!(
!report_after_stop.contains(&header),
"stop() must deregister this container from the diagnostics report"
);
}
struct CaptureDiagnosticsThenReady {
mid_wait_report: Arc<StdMutex<Option<String>>>,
}
#[async_trait::async_trait]
impl WaitStrategy for CaptureDiagnosticsThenReady {
async fn wait_until_ready(&self, _target: &dyn WaitTarget) -> Result<()> {
*self.mid_wait_report.lock().unwrap() = Some(crate::diagnostics().await);
Ok(())
}
fn with_startup_timeout(self: Box<Self>, _timeout: Duration) -> Box<dyn WaitStrategy> {
self
}
}
#[tokio::test]
async fn a_container_is_not_diagnosable_until_its_readiness_wait_succeeds() {
let backend = FakeBackend::new();
let mid_wait_report = Arc::new(StdMutex::new(None));
let c = Container::new("redis:8.6-alpine")
.with_backend(backend.clone())
.waiting_for(CaptureDiagnosticsThenReady {
mid_wait_report: mid_wait_report.clone(),
})
.with_exposed_ports(&[6379]);
let guard = c.start().await.expect("this wait strategy always succeeds");
let name = guard.name().to_string();
let header = format!("-- {name} (");
let captured = mid_wait_report
.lock()
.unwrap()
.clone()
.expect("wait strategy must have run");
assert!(
!captured.contains(&header),
"a container must not be diagnosable while its readiness wait is still \
running: {captured}"
);
let report_after_start = crate::diagnostics().await;
assert!(report_after_start.contains(&header), "{report_after_start}");
guard.stop().await.unwrap();
}
#[tokio::test]
async fn a_container_that_never_becomes_ready_is_never_diagnosable() {
let backend = FakeBackend::new();
let precaptured_port = Arc::new(StdMutex::new(None));
let c = Container::new("redis:8.6-alpine")
.with_backend(backend.clone())
.waiting_for(NeverReady {
precaptured_port: precaptured_port.clone(),
})
.with_exposed_ports(&[6379]);
let err = expect_start_err(c.start().await, "wait strategy must fail start()");
assert!(err.to_string().contains("never ready"), "{err}");
let name = backend.state.lock().unwrap().created[0].name.clone();
let header = format!("-- {name} (");
let report = crate::diagnostics().await;
assert!(
!report.contains(&header),
"a container that never became ready must never appear in the \
diagnostics report: {report}"
);
}
#[tokio::test]
async fn get_mapped_port_reports_not_running_after_stop_clears_the_mappings() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap();
assert!(guard.get_mapped_port(6379).unwrap() > 0);
guard.stop().await.unwrap();
let backend2 = FakeBackend::new();
let c2 = container_on(&backend2).with_exposed_ports(&[6379]);
let mut guard2 = c2.start().await.unwrap();
guard2.stop_inner().await;
let err = guard2.get_mapped_port(6379).unwrap_err().to_string();
assert!(err.contains("not running"), "{err}");
assert!(!err.contains("not exposed"), "{err}");
}
#[tokio::test]
async fn get_mapped_port_reports_not_exposed_for_an_undeclared_port() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap();
let err = guard.get_mapped_port(9999).unwrap_err().to_string();
assert!(err.contains("not exposed"), "{err}");
guard.stop().await.unwrap();
}
#[tokio::test]
async fn u2_starting_on_a_network_installs_links_to_running_siblings() {
let backend = FakeBackend::new();
let net = Arc::new(Network::new_network());
let stub = container_on(&backend)
.with_exposed_ports(&[8888])
.with_network(&net)
.with_network_aliases(&["configuration-stub"]);
let stub_guard = stub.start().await.unwrap();
let app = container_on(&backend)
.with_exposed_ports(&[8080])
.with_network(&net);
let app_guard = app.start().await.unwrap();
let (consumer_id, links) = {
let state = backend.state.lock().unwrap();
assert_eq!(state.installed_links.len(), 1);
state.installed_links[0].clone()
};
assert_eq!(
consumer_id,
backend.state.lock().unwrap().created.last().unwrap().name
);
assert_eq!(
links,
vec![crate::backend::NetworkLink {
alias: "configuration-stub".to_string(),
guest_port: 8888,
target_host_port: stub_guard.get_mapped_port(8888).unwrap(),
}]
);
assert_eq!(
net.resolve("configuration-stub", 8888).unwrap(),
"configuration-stub:8888"
);
assert!(net.resolve("nope", 1).is_err());
app_guard.stop().await.unwrap();
stub_guard.stop().await.unwrap();
}
#[tokio::test]
async fn single_container_on_network_installs_no_links_but_is_registered() {
let backend = FakeBackend::new();
let net = Arc::new(Network::new_network());
let solo = container_on(&backend)
.with_exposed_ports(&[9999])
.with_network(&net)
.with_network_aliases(&["solo"]);
let solo_guard = solo.start().await.unwrap();
assert!(
backend.state.lock().unwrap().installed_links.is_empty(),
"a lone container must not link to itself"
);
let joiner = container_on(&backend)
.with_exposed_ports(&[8080])
.with_network(&net);
let joiner_guard = joiner.start().await.unwrap();
let (_, links) = {
let state = backend.state.lock().unwrap();
assert_eq!(state.installed_links.len(), 1);
state.installed_links[0].clone()
};
assert_eq!(
links,
vec![crate::backend::NetworkLink {
alias: "solo".to_string(),
guest_port: 9999,
target_host_port: solo_guard.get_mapped_port(9999).unwrap(),
}]
);
joiner_guard.stop().await.unwrap();
solo_guard.stop().await.unwrap();
}
#[tokio::test]
async fn u5_wait_strategy_failure_stops_the_container_and_releases_ports() {
let backend = FakeBackend::new();
let precaptured_port = Arc::new(StdMutex::new(None));
let c = Container::new("redis:8.6-alpine")
.with_backend(backend.clone())
.waiting_for(NeverReady {
precaptured_port: precaptured_port.clone(),
})
.with_exposed_ports(&[6379]);
let err = expect_start_err(c.start().await, "wait strategy must fail start()");
assert!(err.to_string().contains("never ready"), "{err}");
let name = backend.state.lock().unwrap().created[0].name.clone();
assert!(
backend.state.lock().unwrap().stopped.contains(&name),
"started container must be stopped when the wait strategy fails"
);
let port = precaptured_port
.lock()
.unwrap()
.expect("wait strategy must have observed a real mapped port");
assert!(port > 0);
assert!(
!free_ports().issued_view().contains(&port),
"port {port} must be released by the wait-strategy-failure cleanup path"
);
}
#[tokio::test]
async fn u5_install_network_links_failure_stops_the_container() {
let backend = FakeBackend::failing_install_network_links();
let net = Arc::new(Network::new_network());
let c = container_on(&backend)
.with_exposed_ports(&[8080])
.with_network(&net);
let err = expect_start_err(
c.start().await,
"install_network_links failure must propagate",
);
assert!(err.to_string().contains("nc"), "{err}");
let created = backend.state.lock().unwrap().created[0].clone();
assert!(
backend
.state
.lock()
.unwrap()
.stopped
.contains(&created.name),
"started container must be stopped on link-install failure"
);
let port = created
.ports
.first()
.expect("spec must carry the allocated host port")
.host_port;
assert!(
!free_ports().issued_view().contains(&port),
"port {port} must be released when install_network_links fails, \
not just when the wait strategy fails"
);
}
#[tokio::test]
async fn ensure_network_failure_does_not_leave_a_phantom_networks_ledger_entry() {
let cache_dir = temp_cache_dir("ensure-network-failure-ledger");
let backend = FakeBackend::failing_ensure_network();
let net = Arc::new(Network::new_network());
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.with_network(&net)
.with_reaper_cache_dir_override(cache_dir.clone());
let err = expect_start_err(c.start().await, "ensure_network failure must propagate");
assert!(err.to_string().contains("ensure_network failed"), "{err}");
let ledger = crate::reaper::Ledger::new(&cache_dir, crate::RunId::value());
assert!(
!ledger.network_ids().contains(&net.id().to_string()),
"a failed ensure_network must not leave a phantom .networks entry behind"
);
}
#[tokio::test]
async fn u7_stop_is_idempotent_across_the_stop_then_drop_sequence() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap();
let name = backend.state.lock().unwrap().created[0].name.clone();
let mapped_port = guard.get_mapped_port(6379).unwrap();
guard.stop().await.unwrap();
assert_eq!(
backend
.state
.lock()
.unwrap()
.stopped
.iter()
.filter(|n| **n == name)
.count(),
1,
"backend.stop must be called exactly once"
);
assert!(
!free_ports().issued_view().contains(&mapped_port),
"port must be released by stop()"
);
}
#[tokio::test]
async fn stop_before_start_is_a_no_op() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let mut guard = c.start().await.unwrap();
let name = backend.state.lock().unwrap().created[0].name.clone();
guard.stop_inner().await;
assert_eq!(
backend
.state
.lock()
.unwrap()
.stopped
.iter()
.filter(|n| **n == name)
.count(),
1
);
guard.stop_inner().await; assert_eq!(
backend
.state
.lock()
.unwrap()
.stopped
.iter()
.filter(|n| **n == name)
.count(),
1,
"a second stop must not re-call backend.stop"
);
assert!(!guard.is_running());
}
struct PortConflictBackend {
fail_first: usize,
conflict: Box<dyn Fn(u16) -> RightsizeError + Send + Sync>,
created: StdMutex<Vec<ContainerSpec>>,
started_ports: StdMutex<Vec<u16>>,
start_attempts: std::sync::atomic::AtomicUsize,
}
impl PortConflictBackend {
fn new(fail_first: usize) -> Arc<Self> {
Self::with_conflict(fail_first, |port| {
RightsizeError::Backend(format!(
"driver failed programming external connectivity: failed to bind host port 127.0.0.1:{port}/tcp: address already in use"
))
})
}
fn with_conflict(
fail_first: usize,
conflict: impl Fn(u16) -> RightsizeError + Send + Sync + 'static,
) -> Arc<Self> {
Arc::new(Self {
fail_first,
conflict: Box::new(conflict),
created: StdMutex::new(Vec::new()),
started_ports: StdMutex::new(Vec::new()),
start_attempts: std::sync::atomic::AtomicUsize::new(0),
})
}
fn create_count(&self) -> usize {
self.created.lock().unwrap().len()
}
}
#[async_trait::async_trait]
impl SandboxBackend for PortConflictBackend {
fn name(&self) -> &str {
"port-conflict"
}
fn supports_native_networks(&self) -> bool {
false
}
async fn create(&self, spec: ContainerSpec) -> Result<Box<dyn SandboxHandle>> {
self.created.lock().unwrap().push(spec.clone());
Ok(Box::new(FakeHandle {
id: spec.name.clone(),
spec,
}))
}
async fn start(&self, handle: &dyn SandboxHandle) -> Result<()> {
let attempt = self.start_attempts.fetch_add(1, Ordering::SeqCst) + 1;
let port = handle.spec().ports[0].host_port;
self.started_ports.lock().unwrap().push(port);
if attempt <= self.fail_first {
return Err((self.conflict)(port));
}
Ok(())
}
async fn stop(&self, _handle: &dyn SandboxHandle) -> Result<()> {
Ok(())
}
async fn remove(&self, _handle: &dyn SandboxHandle) -> Result<()> {
Ok(())
}
async fn exec(&self, _handle: &dyn SandboxHandle, _cmd: &[String]) -> Result<ExecResult> {
Ok(ExecResult {
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
})
}
async fn logs(&self, _handle: &dyn SandboxHandle) -> Result<String> {
Ok(String::new())
}
async fn follow_logs(
&self,
_handle: &dyn SandboxHandle,
_consumer: Box<dyn Fn(String) + Send + Sync>,
) -> Result<crate::backend::FollowHandle> {
unimplemented!()
}
async fn ensure_network(&self, _network_id: &str) -> Result<()> {
Ok(())
}
async fn remove_network(&self, _network_id: &str) -> Result<()> {
Ok(())
}
fn cleanup_sync(&self, _container_id: &str) {}
fn remove_by_name(&self, _name: &str) {}
fn watchdog_kill_command(&self) -> Vec<String> {
vec!["true".to_string()]
}
}
#[tokio::test]
async fn u6_start_retries_with_fresh_host_ports_on_a_bind_conflict() {
let backend = PortConflictBackend::new(2);
let c = Container::new("redis:8.6-alpine")
.with_backend(backend.clone())
.waiting_for(ReadyImmediately)
.with_exposed_ports(&[6379]);
let guard = c.start().await.expect("must eventually succeed");
assert!(guard.is_running());
assert_eq!(
backend.create_count(),
3,
"each attempt recreates the container"
);
let started_ports = backend.started_ports.lock().unwrap().clone();
assert_eq!(started_ports.len(), 3, "start attempted three times");
let distinct: std::collections::HashSet<u16> = started_ports.iter().copied().collect();
assert_eq!(
distinct.len(),
started_ports.len(),
"ports are reallocated per attempt, not reused after a conflict"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn u6_reuse_fresh_create_retries_with_fresh_host_ports_on_a_bind_conflict() {
let backend = PortConflictBackend::new(2);
let cache_dir = temp_cache_dir("fresh-create-port-conflict");
let c = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_cache_dir_override(cache_dir)
.with_reuse_env_override(true)
.waiting_for(ReadyImmediately)
.with_exposed_ports(&[6379])
.reuse(true);
let guard = c
.start()
.await
.expect("reuse fresh-create must retry and eventually succeed");
assert!(guard.is_running());
assert_eq!(
backend.create_count(),
3,
"each attempt recreates the container"
);
let started_ports = backend.started_ports.lock().unwrap().clone();
assert_eq!(started_ports.len(), 3, "start attempted three times");
let distinct: std::collections::HashSet<u16> = started_ports.iter().copied().collect();
assert_eq!(
distinct.len(),
started_ports.len(),
"ports are reallocated per attempt, not reused after a conflict"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn u6_start_retries_on_the_typed_port_bind_conflict_bare_or_nested() {
let bare = PortConflictBackend::with_conflict(1, |port| RightsizeError::PortBindConflict {
message: format!("could not bind host port {port}"),
source: None,
});
let c = Container::new("redis:8.6-alpine")
.with_backend(bare.clone())
.waiting_for(ReadyImmediately)
.with_exposed_ports(&[6379]);
let guard = c
.start()
.await
.expect("must retry exactly once for the typed exception");
assert!(guard.is_running());
assert_eq!(bare.create_count(), 2);
guard.stop().await.unwrap();
let nested =
PortConflictBackend::with_conflict(1, |port| RightsizeError::PortBindConflict {
message: "start failed".to_string(),
source: Some(Box::new(RightsizeError::PortBindConflict {
message: "io error".to_string(),
source: Some(Box::new(RightsizeError::PortBindConflict {
message: format!("could not bind host port {port}"),
source: None,
})),
})),
});
let c = Container::new("redis:8.6-alpine")
.with_backend(nested.clone())
.waiting_for(ReadyImmediately)
.with_exposed_ports(&[6379]);
let guard = c
.start()
.await
.expect("must unwrap to find a nested typed exception");
assert!(guard.is_running());
assert_eq!(nested.create_count(), 2);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn u6_truth_table_known_phrasings_retry_negative_does_not() {
let phrasings = [
"address already in use",
"port is already allocated",
"bind: address already in use",
"Bind for 0.0.0.0:32770 failed: PORT IS ALREADY ALLOCATED",
];
for phrasing in phrasings {
let backend = PortConflictBackend::with_conflict(1, move |_port| {
RightsizeError::Backend(phrasing.to_string())
});
let c = Container::new("redis:8.6-alpine")
.with_backend(backend.clone())
.waiting_for(ReadyImmediately)
.with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap_or_else(|e| {
panic!("must retry and succeed for phrasing '{phrasing}': {e}")
});
assert!(guard.is_running());
assert_eq!(
backend.create_count(),
2,
"must retry exactly once for phrasing: {phrasing}"
);
guard.stop().await.unwrap();
}
let boom = PortConflictBackend::with_conflict(99, |_port| {
RightsizeError::Backend("boom".to_string())
});
let c = Container::new("redis:8.6-alpine")
.with_backend(boom.clone())
.waiting_for(ReadyImmediately)
.with_exposed_ports(&[6379]);
let err = expect_start_err(
c.start().await,
"a non-conflict exception must fail fast, no retry",
);
assert_eq!(err.to_string(), "boom");
assert_eq!(
boom.create_count(),
1,
"a non-conflict exception must fail fast, no retry"
);
}
#[tokio::test]
async fn u4_and_u6_start_gives_up_after_the_retry_budget_is_exhausted_and_releases_every_attempts_ports()
{
let backend = PortConflictBackend::new(99);
let c = Container::new("redis:8.6-alpine")
.with_backend(backend.clone())
.waiting_for(ReadyImmediately)
.with_exposed_ports(&[6379]);
let err = expect_start_err(c.start().await, "must give up after the retry budget");
assert!(err.to_string().contains("free host ports"), "{err}");
let started_ports = backend.started_ports.lock().unwrap().clone();
assert_eq!(
started_ports.len(),
PORT_BIND_ATTEMPTS,
"all attempts must have been tried"
);
let distinct: std::collections::HashSet<u16> = started_ports.iter().copied().collect();
assert_eq!(
distinct.len(),
PORT_BIND_ATTEMPTS,
"each retry must allocate a fresh port"
);
let issued = free_ports().issued_view();
for port in started_ports {
assert!(
!issued.contains(&port),
"port {port} from a discarded attempt must be released — mutation-verified: this is the U4 port-release gate"
);
}
}
#[tokio::test]
async fn with_memory_limit_carries_through_to_the_container_spec() {
let backend = FakeBackend::new();
let limited = container_on(&backend)
.with_exposed_ports(&[6379])
.with_memory_limit(1024);
let guard = limited.start().await.unwrap();
assert_eq!(
backend.state.lock().unwrap().created[0].memory_limit_mb,
Some(1024)
);
guard.stop().await.unwrap();
let unset = container_on(&backend).with_exposed_ports(&[6379]);
let guard = unset.start().await.unwrap();
assert_eq!(
backend
.state
.lock()
.unwrap()
.created
.last()
.unwrap()
.memory_limit_mb,
None
);
guard.stop().await.unwrap();
}
#[test]
fn dedup_env_last_wins_keeps_first_position_but_last_value() {
let env = vec![
("A".to_string(), "1".to_string()),
("B".to_string(), "2".to_string()),
("A".to_string(), "override".to_string()),
("C".to_string(), "3".to_string()),
("B".to_string(), "final".to_string()),
];
let deduped = dedup_env_last_wins(env);
assert_eq!(
deduped,
vec![
("A".to_string(), "override".to_string()),
("B".to_string(), "final".to_string()),
("C".to_string(), "3".to_string()),
],
"A and B must keep their FIRST-occurrence position; each must carry its LAST value"
);
}
#[test]
fn dedup_env_last_wins_is_a_no_op_on_already_unique_keys() {
let env = vec![
("X".to_string(), "1".to_string()),
("Y".to_string(), "2".to_string()),
];
assert_eq!(dedup_env_last_wins(env.clone()), env);
}
#[tokio::test]
async fn duplicate_with_env_calls_resolve_last_wins_in_the_spec_reaching_the_backend() {
let backend = FakeBackend::new();
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.with_env("MODE", "first")
.with_env("OTHER", "x")
.with_env("MODE", "second");
let guard = c.start().await.unwrap();
let spec = backend.state.lock().unwrap().created[0].clone();
assert_eq!(
spec.env,
vec![
("MODE".to_string(), "second".to_string()),
("OTHER".to_string(), "x".to_string()),
],
"MODE must appear exactly once, in its first-occurrence position, with its last value"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn a_spec_customizer_overriding_an_existing_key_also_resolves_last_wins() {
let backend = FakeBackend::new();
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.with_env("KAFKA_ADVERTISED_LISTENERS", "PLACEHOLDER")
.with_spec_customizer(|mut spec, _mapped| {
spec.env.push((
"KAFKA_ADVERTISED_LISTENERS".to_string(),
"PLAINTEXT://127.0.0.1:9999".to_string(),
));
spec
});
let guard = c.start().await.unwrap();
let spec = backend.state.lock().unwrap().created[0].clone();
assert_eq!(
spec.env,
vec![(
"KAFKA_ADVERTISED_LISTENERS".to_string(),
"PLAINTEXT://127.0.0.1:9999".to_string()
)],
"the customizer's later push must win, deduped to a single entry"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn u3_exec_and_mapped_port_require_a_running_container() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let mut guard = c.start().await.unwrap();
guard.stop_inner().await;
assert!(guard.exec(&["ls"]).await.is_err());
assert!(guard.get_mapped_port(6379).is_err());
}
#[tokio::test]
async fn exec_returns_the_backends_result() {
let backend = FakeBackend::new();
let c = container_on(&backend);
let guard = c.start().await.unwrap();
let result = guard.exec(&["ls", "-la"]).await.unwrap();
assert_eq!(result.stdout, "ls -la");
guard.stop().await.unwrap();
}
#[tokio::test]
async fn dropping_a_guard_without_stop_releases_its_ports_synchronously() {
let backend = FakeBackend::new();
let c = container_on(&backend).with_exposed_ports(&[6379]);
let guard = c.start().await.unwrap();
let port = guard.get_mapped_port(6379).unwrap();
assert!(free_ports().issued_view().contains(&port));
drop(guard);
assert!(
!free_ports().issued_view().contains(&port),
"Drop must release mapped ports synchronously even without an explicit stop()"
);
}
#[tokio::test]
async fn dropping_a_guard_without_stop_removes_it_from_the_reaping_ledger() {
let cache_dir = temp_cache_dir("dropping-guard-ledger");
let backend = FakeBackend::new();
let c = container_on(&backend)
.with_exposed_ports(&[6379])
.with_reaper_cache_dir_override(cache_dir.clone());
let guard = c.start().await.unwrap();
let ledger_name = backend.state.lock().unwrap().created[0].name.clone();
let ledger = crate::reaper::Ledger::new(&cache_dir, crate::RunId::value());
assert!(
ledger.sandbox_names().contains(&ledger_name),
"before_create must have listed the sandbox in the reaping ledger"
);
drop(guard);
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
while ledger.sandbox_names().contains(&ledger_name) && std::time::Instant::now() < deadline
{
std::thread::sleep(std::time::Duration::from_millis(10));
}
assert!(
!ledger.sandbox_names().contains(&ledger_name),
"Drop's cleanup-thread fallback must remove the sandbox from the reaping ledger \
too, not just tear it down on the backend"
);
}
fn temp_cache_dir(label: &str) -> std::path::PathBuf {
let dir = std::env::temp_dir().join(format!(
"rz-reuse-container-test-{label}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[derive(Default)]
struct ReuseFakeState {
created: Vec<ContainerSpec>,
started: Vec<String>,
stopped: Vec<String>,
removed: Vec<String>,
removed_by_name: Vec<String>,
running: std::collections::HashSet<String>,
find_running_calls: usize,
}
struct ReuseFakeBackend {
state: StdMutex<ReuseFakeState>,
conflict_once_for_name: StdMutex<Option<String>>,
on_conflict: StdMutex<Option<Box<dyn FnMut() + Send>>>,
}
impl ReuseFakeBackend {
fn new() -> Arc<Self> {
Arc::new(Self {
state: StdMutex::new(ReuseFakeState::default()),
conflict_once_for_name: StdMutex::new(None),
on_conflict: StdMutex::new(None),
})
}
fn mark_running(&self, name: &str) {
self.state.lock().unwrap().running.insert(name.to_string());
}
fn fail_next_create_with_conflict(
&self,
name: &str,
on_conflict: impl FnMut() + Send + 'static,
) {
*self.conflict_once_for_name.lock().unwrap() = Some(name.to_string());
*self.on_conflict.lock().unwrap() = Some(Box::new(on_conflict));
}
}
#[async_trait::async_trait]
impl SandboxBackend for ReuseFakeBackend {
fn name(&self) -> &str {
"reuse-fake"
}
fn supports_native_networks(&self) -> bool {
false
}
async fn create(&self, spec: ContainerSpec) -> Result<Box<dyn SandboxHandle>> {
{
let mut conflict = self.conflict_once_for_name.lock().unwrap();
if conflict.as_deref() == Some(spec.name.as_str()) {
*conflict = None;
drop(conflict);
if let Some(cb) = self.on_conflict.lock().unwrap().as_mut() {
cb();
}
return Err(RightsizeError::NameConflict {
message: format!("sandbox '{}' already exists", spec.name),
source: None,
});
}
}
self.state.lock().unwrap().created.push(spec.clone());
Ok(Box::new(FakeHandle {
id: spec.name.clone(),
spec,
}))
}
async fn start(&self, handle: &dyn SandboxHandle) -> Result<()> {
let id = handle.id().to_string();
let mut state = self.state.lock().unwrap();
state.started.push(id.clone());
state.running.insert(id);
Ok(())
}
async fn stop(&self, handle: &dyn SandboxHandle) -> Result<()> {
let id = handle.id().to_string();
let mut state = self.state.lock().unwrap();
state.stopped.push(id.clone());
state.running.remove(&id);
Ok(())
}
async fn remove(&self, handle: &dyn SandboxHandle) -> Result<()> {
self.state
.lock()
.unwrap()
.removed
.push(handle.id().to_string());
Ok(())
}
async fn exec(&self, _handle: &dyn SandboxHandle, cmd: &[String]) -> Result<ExecResult> {
Ok(ExecResult {
exit_code: 0,
stdout: cmd.join(" "),
stderr: String::new(),
})
}
async fn logs(&self, _handle: &dyn SandboxHandle) -> Result<String> {
Ok(String::new())
}
async fn follow_logs(
&self,
_handle: &dyn SandboxHandle,
_consumer: Box<dyn Fn(String) + Send + Sync>,
) -> Result<crate::backend::FollowHandle> {
unimplemented!("not exercised by the reuse test suite")
}
async fn ensure_network(&self, _network_id: &str) -> Result<()> {
Ok(())
}
async fn remove_network(&self, _network_id: &str) -> Result<()> {
Ok(())
}
fn cleanup_sync(&self, _container_id: &str) {}
fn remove_by_name(&self, name: &str) {
let mut state = self.state.lock().unwrap();
state.removed_by_name.push(name.to_string());
state.running.remove(name);
}
fn watchdog_kill_command(&self) -> Vec<String> {
vec!["true".to_string()]
}
async fn find_running(
&self,
spec: &ContainerSpec,
) -> Result<Option<Box<dyn SandboxHandle>>> {
let mut state = self.state.lock().unwrap();
state.find_running_calls += 1;
if state.running.contains(&spec.name) {
Ok(Some(Box::new(FakeHandle {
id: spec.name.clone(),
spec: spec.clone(),
})))
} else {
Ok(None)
}
}
}
fn sample_registry_entry(
identity: &crate::reuse::Identity,
host_port: u16,
) -> crate::reuse::RegistryEntry {
crate::reuse::RegistryEntry {
name: identity.name.clone(),
image: "redis:7-alpine".to_string(),
ports: std::collections::BTreeMap::from([("6379".to_string(), host_port)]),
created_iso: "2025-01-01T00:00:00Z".to_string(),
backend: "reuse-fake".to_string(),
}
}
#[tokio::test]
async fn reuse_double_opt_in_gating_all_four_combinations() {
async fn is_reuse_name(marker: bool, env_enabled: bool) -> bool {
let backend = ReuseFakeBackend::new();
let cache_dir = temp_cache_dir("gating");
let guard = Container::new("redis:7-alpine")
.with_backend(backend)
.with_cache_dir_override(cache_dir)
.with_reuse_env_override(env_enabled)
.with_exposed_ports(&[6379])
.waiting_for(ReadyImmediately)
.reuse(marker)
.start()
.await
.expect("start must succeed regardless of the gating outcome");
let reused = guard.name().starts_with("rz-reuse-");
guard.stop().await.unwrap();
reused
}
assert!(
is_reuse_name(true, true).await,
"marker on + env on must produce a reuse name"
);
assert!(
!is_reuse_name(true, false).await,
"marker on, env off: ordinary container"
);
assert!(
!is_reuse_name(false, true).await,
"env on, marker off: reuse never considered"
);
assert!(
!is_reuse_name(false, false).await,
"both off: ordinary container"
);
}
#[tokio::test]
async fn adopt_path_registry_hit_running_and_wait_ok_skips_create_and_uses_registry_ports() {
let backend = ReuseFakeBackend::new();
let cache_dir = temp_cache_dir("adopt-hit");
let identity =
crate::reuse::compute_identity("redis:7-alpine", &[], &None, &[6379], None, &[])
.unwrap();
backend.mark_running(&identity.name);
crate::reuse::Registry::new(&cache_dir, &identity.hash_hex)
.write_atomic(&sample_registry_entry(&identity, 40321))
.unwrap();
let guard = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_cache_dir_override(cache_dir)
.with_reuse_env_override(true)
.with_exposed_ports(&[6379])
.waiting_for(ReadyImmediately)
.reuse(true)
.start()
.await
.expect("adopt must succeed");
assert_eq!(guard.name(), identity.name);
assert_eq!(
guard.get_mapped_port(6379).unwrap(),
40321,
"must use the REGISTRY's port, not a freshly allocated one"
);
{
let state = backend.state.lock().unwrap();
assert!(
state.created.is_empty(),
"adopt must not call backend.create"
);
assert!(state.find_running_calls >= 1);
}
guard.stop().await.unwrap();
}
#[tokio::test]
async fn stale_registry_not_running_removes_and_creates_fresh_and_rewrites_registry() {
let backend = ReuseFakeBackend::new();
let cache_dir = temp_cache_dir("adopt-stale");
let identity =
crate::reuse::compute_identity("redis:7-alpine", &[], &None, &[6379], None, &[])
.unwrap();
crate::reuse::Registry::new(&cache_dir, &identity.hash_hex)
.write_atomic(&sample_registry_entry(&identity, 40321))
.unwrap();
let guard = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_cache_dir_override(cache_dir.clone())
.with_reuse_env_override(true)
.with_exposed_ports(&[6379])
.waiting_for(ReadyImmediately)
.reuse(true)
.start()
.await
.expect("a stale registry must fall back to a fresh create");
assert_eq!(guard.name(), identity.name);
{
let state = backend.state.lock().unwrap();
assert_eq!(state.created.len(), 1, "must create exactly once");
assert_eq!(
state.removed_by_name,
vec![identity.name.clone()],
"the stale sandbox must be best-effort removed by name"
);
}
let rewritten = crate::reuse::Registry::new(&cache_dir, &identity.hash_hex)
.read()
.expect("the registry must be rewritten after the fresh create");
let new_port = *rewritten.ports.get("6379").unwrap();
assert_eq!(guard.get_mapped_port(6379).unwrap(), new_port);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn corrupted_registry_json_falls_back_to_fresh_create() {
let backend = ReuseFakeBackend::new();
let cache_dir = temp_cache_dir("adopt-corrupt");
let identity =
crate::reuse::compute_identity("redis:7-alpine", &[], &None, &[6379], None, &[])
.unwrap();
let reuse_dir = cache_dir.join("reuse");
std::fs::create_dir_all(&reuse_dir).unwrap();
std::fs::write(
reuse_dir.join(format!("{}.json", identity.hash_hex)),
b"not json",
)
.unwrap();
let guard = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_cache_dir_override(cache_dir.clone())
.with_reuse_env_override(true)
.with_exposed_ports(&[6379])
.waiting_for(ReadyImmediately)
.reuse(true)
.start()
.await
.expect("corrupt registry JSON must fall back to a fresh create, not fail start()");
{
let state = backend.state.lock().unwrap();
assert_eq!(state.created.len(), 1);
assert_eq!(state.removed_by_name, vec![identity.name.clone()]);
}
assert!(
crate::reuse::Registry::new(&cache_dir, &identity.hash_hex)
.read()
.is_some(),
"a valid registry must exist after the fresh create"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn stop_on_a_reuse_container_leaves_the_sandbox_running_and_never_touches_the_ledger() {
let backend = ReuseFakeBackend::new();
let cache_dir = temp_cache_dir("stop-semantics");
let guard = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_cache_dir_override(cache_dir.clone())
.with_reuse_env_override(true)
.with_reaper_cache_dir_override(cache_dir.clone())
.with_exposed_ports(&[6379])
.waiting_for(ReadyImmediately)
.reuse(true)
.start()
.await
.unwrap();
let name = guard.name().to_string();
assert!(name.starts_with("rz-reuse-"));
guard.stop().await.unwrap();
{
let state = backend.state.lock().unwrap();
assert!(
state.stopped.is_empty(),
"stop() must not call backend.stop for a reuse container"
);
assert!(
state.removed.is_empty(),
"stop() must not call backend.remove for a reuse container"
);
}
let ledger = crate::reaper::Ledger::new(&cache_dir, crate::RunId::value());
assert!(
!ledger.sandbox_names().contains(&name),
"a reuse container must never appear in the reaping ledger, before or after stop()"
);
}
#[tokio::test]
async fn reuse_plus_custom_network_is_a_typed_error() {
let backend = ReuseFakeBackend::new();
let net = Arc::new(Network::new_network());
let start_result = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_reuse_env_override(true)
.with_network(&net)
.reuse(true)
.start()
.await;
let err = expect_start_err(start_result, "reuse + a custom network must fail fast");
assert!(
matches!(err, RightsizeError::ReuseNetworkConflict { .. }),
"{err}"
);
assert!(
backend.state.lock().unwrap().created.is_empty(),
"must fail before any create() call"
);
}
#[tokio::test]
async fn name_collision_on_create_retries_the_adopt_path_once() {
let backend = ReuseFakeBackend::new();
let cache_dir = temp_cache_dir("collision");
let identity =
crate::reuse::compute_identity("redis:7-alpine", &[], &None, &[6379], None, &[])
.unwrap();
let entry = sample_registry_entry(&identity, 41111);
let winner_cache_dir = cache_dir.clone();
let winner_hash = identity.hash_hex.clone();
let winner_backend = backend.clone();
let winner_name = identity.name.clone();
backend.fail_next_create_with_conflict(&identity.name, move || {
winner_backend.mark_running(&winner_name);
let _ =
crate::reuse::Registry::new(&winner_cache_dir, &winner_hash).write_atomic(&entry);
});
let guard = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_cache_dir_override(cache_dir)
.with_reuse_env_override(true)
.with_exposed_ports(&[6379])
.waiting_for(ReadyImmediately)
.reuse(true)
.start()
.await
.expect("a name collision must retry the adopt path and succeed");
assert_eq!(guard.name(), identity.name);
assert_eq!(guard.get_mapped_port(6379).unwrap(), 41111);
assert_eq!(
backend.state.lock().unwrap().created.len(),
0,
"the losing create() attempt must not count as a successful create"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn fresh_create_removes_a_running_but_unregistered_orphan_before_creating() {
let backend = ReuseFakeBackend::new();
let cache_dir = temp_cache_dir("orphan-recovery");
let identity =
crate::reuse::compute_identity("redis:7-alpine", &[], &None, &[6379], None, &[])
.unwrap();
backend.mark_running(&identity.name);
let guard = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_cache_dir_override(cache_dir)
.with_reuse_env_override(true)
.with_exposed_ports(&[6379])
.waiting_for(ReadyImmediately)
.reuse(true)
.start()
.await
.expect("an orphaned running sandbox must not fail start(), just be replaced");
assert_eq!(guard.name(), identity.name);
{
let state = backend.state.lock().unwrap();
assert_eq!(
state.removed_by_name,
vec![identity.name.clone()],
"the orphaned sandbox must be best-effort removed before the fresh create"
);
assert_eq!(state.created.len(), 1, "must still create exactly once");
assert!(state.find_running_calls >= 1);
}
guard.stop().await.unwrap();
}
#[tokio::test]
async fn adopt_with_a_verified_registry_never_calls_remove_by_name() {
let backend = ReuseFakeBackend::new();
let cache_dir = temp_cache_dir("orphan-recovery-adopt");
let identity =
crate::reuse::compute_identity("redis:7-alpine", &[], &None, &[6379], None, &[])
.unwrap();
backend.mark_running(&identity.name);
crate::reuse::Registry::new(&cache_dir, &identity.hash_hex)
.write_atomic(&sample_registry_entry(&identity, 40321))
.unwrap();
let guard = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_cache_dir_override(cache_dir)
.with_reuse_env_override(true)
.with_exposed_ports(&[6379])
.waiting_for(ReadyImmediately)
.reuse(true)
.start()
.await
.expect("a verified registry entry must adopt");
assert_eq!(guard.name(), identity.name);
assert!(
backend.state.lock().unwrap().removed_by_name.is_empty(),
"adopting a verified registry entry must never call remove_by_name"
);
guard.stop().await.unwrap();
}
#[tokio::test]
async fn fresh_create_with_nothing_running_never_calls_remove_by_name() {
let backend = ReuseFakeBackend::new();
let cache_dir = temp_cache_dir("orphan-recovery-clean");
let guard = Container::new("redis:7-alpine")
.with_backend(backend.clone())
.with_cache_dir_override(cache_dir)
.with_reuse_env_override(true)
.with_exposed_ports(&[6379])
.waiting_for(ReadyImmediately)
.reuse(true)
.start()
.await
.expect("a genuinely fresh identity must create normally");
{
let state = backend.state.lock().unwrap();
assert!(
state.removed_by_name.is_empty(),
"nothing was running, so remove_by_name must never be called"
);
assert!(
state.find_running_calls >= 1,
"the orphan-recovery check must still run"
);
assert_eq!(state.created.len(), 1);
}
guard.stop().await.unwrap();
}
}