use std::sync::Arc;
use std::time::Duration;
use crate::control::{
ControlPlane, CreateImageRequest, Image, Microvm, RunHookPayload, RunMicrovmRequest, WaitOpts,
};
use crate::error::{Error, ErrorKind};
use crate::region::Region;
use crate::session::{Session, TokenMinter};
pub const DEFAULT_READY_TIMEOUT: Duration = Duration::from_secs(300);
pub const DEFAULT_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(300);
pub const DEFAULT_DELETE_ATTEMPTS: u32 = 20;
pub const DEFAULT_DELETE_BACKOFF: Duration = Duration::from_secs(15);
pub const DENY_EGRESS_PROXY_URL: &str = "http://127.0.0.1:1";
pub const DENY_EGRESS_ENV_KEYS: [&str; 6] = [
"http_proxy",
"https_proxy",
"all_proxy",
"HTTP_PROXY",
"HTTPS_PROXY",
"ALL_PROXY",
];
const LIFECYCLE_POLL_INTERVAL: Duration = Duration::from_secs(5);
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Lifecycle {
Pending,
Running,
Suspending,
Suspended,
Terminating,
Terminated,
}
impl Lifecycle {
pub fn as_str(self) -> &'static str {
match self {
Lifecycle::Pending => "PENDING",
Lifecycle::Running => "RUNNING",
Lifecycle::Suspending => "SUSPENDING",
Lifecycle::Suspended => "SUSPENDED",
Lifecycle::Terminating => "TERMINATING",
Lifecycle::Terminated => "TERMINATED",
}
}
pub fn from_service(state: &str) -> Option<Self> {
Some(match state {
"PENDING" => Lifecycle::Pending,
"RUNNING" => Lifecycle::Running,
"SUSPENDING" => Lifecycle::Suspending,
"SUSPENDED" => Lifecycle::Suspended,
"TERMINATING" => Lifecycle::Terminating,
"TERMINATED" => Lifecycle::Terminated,
_ => return None,
})
}
pub fn is_live(self) -> bool {
matches!(
self,
Lifecycle::Pending | Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended
)
}
}
impl std::fmt::Display for Lifecycle {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Clone, Debug)]
pub struct RunRequest {
pub image_identifier: Option<String>,
pub image_version: Option<String>,
pub execution_role_arn: Option<String>,
pub agent_token: Option<String>,
pub client_token: Option<String>,
pub launch_env: std::collections::HashMap<String, String>,
pub identity: bool,
pub egress: bool,
pub egress_network_connectors: Vec<String>,
pub deny_egress: bool,
pub shell: bool,
pub max_idle_sec: u32,
pub suspended_sec: u32,
pub auto_resume: bool,
pub max_duration_sec: u32,
pub ready_timeout: Duration,
pub token_scope: Option<String>,
pub logging: Option<crate::control::ops::Logging>,
pub wait: bool,
}
impl Default for RunRequest {
fn default() -> Self {
Self {
image_identifier: None,
image_version: None,
execution_role_arn: None,
agent_token: None,
client_token: None,
launch_env: std::collections::HashMap::new(),
identity: false,
egress: false,
egress_network_connectors: Vec::new(),
deny_egress: false,
shell: false,
max_idle_sec: 600,
suspended_sec: 600,
auto_resume: false,
max_duration_sec: 3_600,
ready_timeout: DEFAULT_READY_TIMEOUT,
token_scope: None,
logging: None,
wait: true,
}
}
}
impl RunRequest {
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn with_image(mut self, identifier: impl Into<String>) -> Self {
self.image_identifier = Some(identifier.into());
self
}
#[must_use]
pub fn with_egress(mut self) -> Self {
self.egress = true;
self
}
#[must_use]
pub fn with_egress_network_connector(mut self, arn: impl Into<String>) -> Self {
self.egress_network_connectors.push(arn.into());
self
}
#[must_use]
pub fn with_vpc_egress(mut self, arns: Vec<String>) -> Self {
if !arns.is_empty() {
self.egress = false;
self.egress_network_connectors = arns;
}
self
}
#[must_use]
pub fn with_deny_egress(mut self) -> Self {
self.deny_egress = true;
self
}
pub fn egress_posture(&self) -> crate::control::EgressPosture {
crate::control::EgressPosture::for_launch(self.egress, self.deny_egress)
}
pub fn effective_launch_env(&self) -> std::collections::HashMap<String, String> {
let mut env = self.launch_env.clone();
if self.deny_egress {
for key in DENY_EGRESS_ENV_KEYS {
env.entry(key.to_string())
.or_insert_with(|| DENY_EGRESS_PROXY_URL.to_string());
}
}
env
}
#[must_use]
pub fn with_shell(mut self) -> Self {
self.shell = true;
self
}
#[must_use]
pub fn with_image_version(mut self, version: impl Into<String>) -> Self {
self.image_version = Some(version.into());
self
}
#[must_use]
pub fn with_suspended_sec(mut self, seconds: u32) -> Self {
self.suspended_sec = seconds;
self
}
#[must_use]
pub fn with_agent_token(mut self, token: impl Into<String>) -> Self {
self.agent_token = Some(token.into());
self
}
#[must_use]
pub fn with_identity(mut self) -> Self {
self.identity = true;
self
}
#[must_use]
pub fn with_launch_env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.launch_env.insert(key.into(), value.into());
self
}
}
#[derive(Clone, Copy, Debug)]
pub struct TeardownOpts {
pub delete_image: bool,
pub delete_log_group: bool,
pub delete_attempts: u32,
pub delete_backoff: Duration,
pub wait_for_terminated: Option<Duration>,
}
impl Default for TeardownOpts {
fn default() -> Self {
Self {
delete_image: false,
delete_log_group: false,
delete_attempts: DEFAULT_DELETE_ATTEMPTS,
delete_backoff: DEFAULT_DELETE_BACKOFF,
wait_for_terminated: None,
}
}
}
impl TeardownOpts {
#[must_use]
pub fn deleting_image(mut self) -> Self {
self.delete_image = true;
self
}
#[must_use]
pub fn deleting_log_group(mut self) -> Self {
self.delete_log_group = true;
self
}
#[must_use]
pub fn waiting_for_terminated(mut self) -> Self {
self.wait_for_terminated = Some(DEFAULT_LIFECYCLE_TIMEOUT);
self
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct TeardownReport {
pub undeleted: Vec<String>,
pub terminate_accepted: bool,
pub image_deleted: Option<bool>,
pub lifecycle: Option<Lifecycle>,
pub failures: Vec<String>,
}
impl TeardownReport {
pub fn leaked(&self) -> bool {
!self.undeleted.is_empty()
}
}
pub(crate) struct ControlPlaneMinter {
pub(crate) control: Arc<ControlPlane>,
pub(crate) microvm_id: String,
}
impl TokenMinter for ControlPlaneMinter {
fn mint(
&self,
) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
Box::pin(async move {
let minted = self.control.mint_auth_token(&self.microvm_id).await?;
Ok(minted.into())
})
}
fn mint_for_ports(
&self,
ports: &[u16],
) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
let specs: Vec<crate::control::ops::PortSpecification> = ports
.iter()
.map(|port| crate::control::ops::PortSpecification::port(*port))
.collect();
Box::pin(async move {
let minted = self
.control
.mint_auth_token_for(&self.microvm_id, &specs)
.await?;
Ok(minted.into())
})
}
}
pub struct Sandbox {
control: Arc<ControlPlane>,
image: Option<Image>,
microvm: Option<Microvm>,
session: Option<Session>,
session_backend: Option<crate::session::SharedBackend>,
lifecycle: Lifecycle,
lifecycle_watch: tokio::sync::watch::Sender<Lifecycle>,
token_installed: bool,
image_exists: bool,
was_terminated: bool,
bootstrap_count: u32,
suspended_window: Option<Duration>,
launch_adoptable: bool,
launch_posture: Option<crate::control::EgressPosture>,
pending_agent_token: Option<String>,
idle_window: Option<Duration>,
suspended_at: Option<Duration>,
torn_down: bool,
adopted: bool,
detached: bool,
tunnel_identity: Option<crate::identity::TunnelIdentity>,
build_services: Option<Arc<dyn crate::control::BuildServices>>,
account: Option<String>,
}
impl std::fmt::Debug for Sandbox {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Sandbox")
.field("lifecycle", &self.lifecycle)
.field("microvm", &self.microvm.as_ref().map(|vm| &vm.id))
.field("image", &self.image.as_ref().map(|image| &image.identifier))
.field("token_installed", &self.token_installed)
.field("bootstrap_count", &self.bootstrap_count)
.field("was_terminated", &self.was_terminated)
.field("suspended_window", &self.suspended_window)
.field("adopted", &self.adopted)
.field("detached", &self.detached)
.finish_non_exhaustive()
}
}
#[derive(Clone)]
pub struct Detached {
pub microvm_id: String,
pub endpoint: String,
pub region: Region,
pub port: u16,
agent_token: String,
}
impl Detached {
pub fn agent_token(&self) -> &str {
&self.agent_token
}
}
impl std::fmt::Debug for Detached {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Detached")
.field("microvm_id", &self.microvm_id)
.field("endpoint", &self.endpoint)
.field("region", &self.region)
.field("port", &self.port)
.field("agent_token", &"<redacted>")
.finish()
}
}
impl Sandbox {
pub async fn new(region: Region) -> Result<Self, Error> {
Ok(Self::with_control_plane(ControlPlane::new(region).await?))
}
pub fn with_control_plane(control: ControlPlane) -> Self {
Self {
control: Arc::new(control),
image: None,
microvm: None,
session: None,
session_backend: None,
lifecycle: Lifecycle::Pending,
lifecycle_watch: tokio::sync::watch::Sender::new(Lifecycle::Pending),
token_installed: false,
image_exists: false,
was_terminated: false,
bootstrap_count: 0,
suspended_window: None,
launch_adoptable: false,
launch_posture: None,
pending_agent_token: None,
idle_window: None,
suspended_at: None,
torn_down: false,
adopted: false,
detached: false,
tunnel_identity: None,
build_services: None,
account: None,
}
}
pub fn with_build_services(mut self, services: Arc<dyn crate::control::BuildServices>) -> Self {
self.build_services = Some(services);
self
}
pub fn with_session_backend(mut self, backend: crate::session::SharedBackend) -> Self {
self.session_backend = Some(backend);
self
}
pub fn lifecycle(&self) -> Lifecycle {
self.lifecycle
}
pub fn token_installed(&self) -> bool {
self.token_installed
}
pub fn image_exists(&self) -> bool {
self.image_exists
}
pub fn was_terminated(&self) -> bool {
self.was_terminated
}
pub fn bootstrap_count(&self) -> u32 {
self.bootstrap_count
}
pub fn image(&self) -> Option<&Image> {
self.image.as_ref()
}
pub fn microvm(&self) -> Option<&Microvm> {
self.microvm.as_ref()
}
pub fn session(&self) -> Option<&Session> {
self.session.as_ref()
}
pub fn suspended_window(&self) -> Option<Duration> {
self.suspended_window
}
pub fn watch_lifecycle(&self) -> tokio::sync::watch::Receiver<Lifecycle> {
self.lifecycle_watch.subscribe()
}
fn set_lifecycle(&mut self, lifecycle: Lifecycle) {
self.lifecycle = lifecycle;
self.lifecycle_watch.send_replace(lifecycle);
}
pub fn idle_window(&self) -> Option<Duration> {
self.idle_window
}
pub fn tunnel_identity(&self) -> Option<&crate::identity::TunnelIdentity> {
self.tunnel_identity.as_ref()
}
pub fn egress_posture(&self) -> crate::control::EgressPosture {
self.launch_posture.unwrap_or_default()
}
pub fn port(&self) -> u16 {
self.control.port()
}
pub async fn build_image(&mut self, request: CreateImageRequest) -> Result<&Image, Error> {
let size = request.size;
let created = self.control.create_image(request).await?;
let mut built = self
.control
.wait_for_image(&created.identifier, size, WaitOpts::default())
.await?;
built.log_group = created.log_group;
built.log_stream = created.log_stream;
self.image_exists = true;
self.image = Some(built);
Ok(self.image.as_ref().expect("just assigned"))
}
pub async fn ensure_image(
&mut self,
request: crate::control::EnsureImageRequest,
) -> Result<crate::control::EnsuredImage, Error> {
let prepared = crate::control::ensure::prepare(&self.control, request)?;
let services = match &self.build_services {
Some(services) => Arc::clone(services),
None => {
let built: Arc<dyn crate::control::BuildServices> = Arc::new(
crate::control::SignedBuildServices::new(self.control.region().clone()).await?,
);
self.build_services = Some(Arc::clone(&built));
built
}
};
let account = match &self.account {
Some(account) => account.clone(),
None => {
let account = services.caller_account().await?;
self.account = Some(account.clone());
account
}
};
let ensured =
crate::control::ensure::ensure(&self.control, services.as_ref(), &account, prepared)
.await?;
self.image_exists = true;
self.image = Some(ensured.image.clone());
Ok(ensured)
}
pub fn build_artifact_for(&self, request: &CreateImageRequest) -> Result<Vec<u8>, Error> {
self.control.build_artifact_for(request)
}
pub fn preflight(&self, request: &CreateImageRequest) -> Result<(), Error> {
self.control.preflight(request)
}
pub async fn resolve_image_arn(&self, identifier: &str) -> Result<String, Error> {
self.control.resolve_image_arn(identifier).await
}
pub async fn find_image_by_name(
&self,
name: &str,
) -> Result<Option<crate::control::ops::MicrovmImageSummaryWire>, Error> {
self.control.find_image_by_name(name).await
}
pub fn artifact_content_hash_for(&self, request: &CreateImageRequest) -> String {
self.control.artifact_content_hash_for(request)
}
pub async fn managed_base_versions(
&self,
base_image_arn: &str,
) -> Result<Vec<crate::control::ops::ManagedMicrovmImageVersionWire>, Error> {
self.control.managed_base_versions(base_image_arn).await
}
pub async fn image_version(
&self,
identifier: &str,
version: &str,
) -> Result<crate::control::ops::MicrovmImageVersionSummaryWire, Error> {
self.control.get_image_version(identifier, version).await
}
pub async fn run(&mut self, request: RunRequest) -> Result<&mut Session, Error> {
self.refuse_detached("run")?;
if self.bootstrap_count > 0 || self.microvm.is_some() {
return Err(Error::invalid_arg(format!(
"this sandbox has already launched a VM ({} bootstrap(s), lifecycle {}), and the \
agent token is installed at most once per VM lifetime (STATE-3). A second VM \
needs a second Sandbox — reusing this one would either re-deliver a run-hook \
payload to a daemon whose one-shot bootstrap refuses it, or silently address \
two guests through one handle.",
self.bootstrap_count, self.lifecycle,
)));
}
let posture = crate::control::egress_posture_for(
request.egress,
&request.egress_network_connectors,
request.deny_egress,
Some(self.control.region()),
)?;
let Some(identifier) = request
.image_identifier
.clone()
.or_else(|| self.image.as_ref().map(|image| image.identifier.clone()))
else {
return Err(Error::new(
ErrorKind::Precondition,
"no image to launch: pass RunRequest::with_image or call build_image first."
.to_string(),
));
};
if request.client_token.is_some() && (request.agent_token.is_none() || request.identity) {
return Err(Error::invalid_arg(
"a stable client_token requires an explicit agent_token and identity=false; persist identical launch parameters before retrying",
));
}
let agent_token = request.agent_token.clone().unwrap_or_else(mint_agent_token);
let launch_identity = if request.identity {
Some(crate::identity::LaunchIdentity::generate()?)
} else {
None
};
let payload = RunHookPayload::for_launch_with_identity(
&agent_token,
&request.effective_launch_env(),
launch_identity.as_ref(),
)?;
let mut wire = RunMicrovmRequest::new(&identifier, payload);
wire.image_version = request.image_version.clone();
wire.execution_role_arn = request.execution_role_arn.clone();
wire.egress_network_connectors = request.egress_network_connectors.clone();
wire.max_idle_sec = request.max_idle_sec;
wire.suspended_sec = request.suspended_sec;
wire.auto_resume = request.auto_resume;
wire.max_duration_sec = request.max_duration_sec;
wire.token_scope = request.token_scope.clone();
wire.client_token = request.client_token.clone();
wire.logging = request.logging.clone();
if request.egress {
wire = wire.with_egress();
}
if request.shell {
wire = wire.with_shell();
}
let launched = self.control.run_microvm(wire).await?;
self.tunnel_identity = launch_identity.map(crate::identity::LaunchIdentity::keep);
self.set_lifecycle(Lifecycle::Pending);
self.image_exists = true;
self.suspended_window = Some(Duration::from_secs(u64::from(request.suspended_sec)));
self.idle_window = Some(Duration::from_secs(u64::from(request.max_idle_sec)));
self.microvm = Some(launched);
self.launch_adoptable = request.client_token.is_some();
self.launch_posture = Some(posture);
self.pending_agent_token = Some(agent_token);
if request.wait {
return self.wait_until_running(request.ready_timeout).await;
}
self.build_session()?;
Ok(self.session.as_mut().expect("just assigned"))
}
fn build_session(&mut self) -> Result<(), Error> {
let (Some(vm), Some(agent_token)) = (&self.microvm, &self.pending_agent_token) else {
return Err(Error::new(
ErrorKind::Precondition,
"no launch to build a session for",
));
};
let minter = Arc::new(ControlPlaneMinter {
control: Arc::clone(&self.control),
microvm_id: vm.id.clone(),
});
let mut builder = Session::builder(vm.endpoint.clone(), agent_token.clone())
.with_minter(minter)
.with_port(self.control.port())
.with_egress_posture(self.egress_posture());
if let Some(backend) = &self.session_backend {
builder = builder.with_backend(Arc::clone(backend));
}
self.session = Some(builder.build()?);
Ok(())
}
pub async fn wait_until_running(&mut self, timeout: Duration) -> Result<&mut Session, Error> {
self.refuse_detached("wait until running")?;
let id = self.require_microvm("wait_until_running")?;
if self.lifecycle != Lifecycle::Pending {
return Err(Error::invalid_arg(format!(
"microvm {id} is {}; wait_until_running finishes a launch that is still PENDING",
self.lifecycle,
)));
}
let opts = WaitOpts {
timeout,
poll_interval: LIFECYCLE_POLL_INTERVAL,
stall_grace: Duration::MAX,
};
let running = if self.launch_adoptable {
self.control.wait_for_launch(&id, opts).await?
} else {
self.control.wait_for_running(&id, opts).await?
};
self.set_lifecycle(Lifecycle::Running);
self.token_installed = true;
self.bootstrap_count += 1;
self.microvm = Some(running);
if self.session.is_none() {
self.build_session()?;
}
Ok(self.session.as_mut().expect("just assigned"))
}
pub async fn adopt(
control: ControlPlane,
microvm_id: impl Into<String>,
endpoint: impl Into<String>,
agent_token: impl Into<String>,
) -> Result<Self, Error> {
let microvm_id = microvm_id.into();
let endpoint = endpoint.into();
let agent_token = agent_token.into();
if agent_token.is_empty() {
return Err(Error::invalid_arg(
"adopt needs the agent token the VM was launched with: its daemon accepts only \
that bearer, and adopting without it would build a handle every exec refuses.",
));
}
let mut vm = control.get_microvm(µvm_id).await?;
if !endpoint.is_empty() && !vm.endpoint.is_empty() && vm.endpoint != endpoint {
return Err(Error::invalid_arg(format!(
"microvm {microvm_id} reports endpoint {}, not {endpoint}: the id and endpoint \
came from different records, and a session built from them would address one \
VM while its lifecycle calls reached another.",
vm.endpoint,
)));
}
if vm.endpoint.is_empty() {
vm.endpoint = endpoint;
}
let Some(lifecycle) = Lifecycle::from_service(&vm.state) else {
return Err(Error::new(
ErrorKind::Platform,
format!(
"microvm {microvm_id} reports state {}, which this client does not know; \
refusing to guess which lifecycle guards apply.",
vm.state,
),
));
};
let mut sandbox = Self::with_control_plane(control);
sandbox.adopted = true;
sandbox.set_lifecycle(lifecycle);
sandbox.idle_window = vm
.idle_policy
.as_ref()
.map(|policy| Duration::from_secs(u64::from(policy.max_idle_duration_seconds)));
sandbox.image_exists = true;
sandbox.launch_adoptable = true;
sandbox.pending_agent_token = Some(agent_token);
match lifecycle {
Lifecycle::Pending => {}
Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended => {
sandbox.token_installed = true;
sandbox.bootstrap_count = 1;
}
Lifecycle::Terminating | Lifecycle::Terminated => {
sandbox.bootstrap_count = 1;
sandbox.was_terminated = true;
}
}
sandbox.microvm = Some(vm);
if lifecycle.is_live() {
sandbox.build_session()?;
}
Ok(sandbox)
}
pub async fn adopt_in(
region: Region,
microvm_id: impl Into<String>,
endpoint: impl Into<String>,
agent_token: impl Into<String>,
port: Option<u16>,
) -> Result<Self, Error> {
Self::adopt(
Self::plane_for(region, port).await?,
microvm_id,
endpoint,
agent_token,
)
.await
}
pub fn name_record(&self, name: &str) -> Result<crate::names::NameRecord, Error> {
let Some(vm) = self.microvm() else {
return Err(Error::new(
ErrorKind::Precondition,
"there is no VM to name yet: launch or adopt one first",
));
};
let Some(session) = self.session() else {
return Err(Error::new(
ErrorKind::Precondition,
format!(
"microvm {} has no session to take its agent token from",
vm.id
),
));
};
crate::names::NameRecord::new(
name,
vm.id.as_str(),
vm.endpoint.as_str(),
session.agent_token(),
self.control.region().as_str(),
)
}
pub async fn from_name(
store: &dyn crate::names::NameStore,
name: &str,
region: Option<Region>,
port: Option<u16>,
) -> Result<Self, Error> {
let record = crate::names::resolve(store, name, region.as_ref())?;
let control = Self::plane_for(record.region(), port).await?;
Self::adopt_record(control, record).await
}
pub async fn adopt_record(
control: ControlPlane,
record: crate::names::NameRecord,
) -> Result<Self, Error> {
Self::adopt(
control,
record.microvm_id,
record.endpoint,
record.agent_token,
)
.await
}
pub(crate) async fn plane_for(
region: Region,
port: Option<u16>,
) -> Result<ControlPlane, Error> {
let control = ControlPlane::new(region).await?;
match port {
Some(port) => control.with_port(port),
None => Ok(control),
}
}
pub fn detach(&mut self) -> Result<Detached, Error> {
self.refuse_detached("detach")?;
let Some(vm) = self.microvm.as_ref() else {
return Err(Error::new(
ErrorKind::Precondition,
"nothing to detach: this sandbox has not launched a VM.",
));
};
if self.torn_down || !self.lifecycle.is_live() {
return Err(Error::new(
ErrorKind::Precondition,
format!(
"microvm {} is {} and there is no live VM to hand off; only a PENDING, \
RUNNING, or SUSPENDED VM can be adopted.",
vm.id, self.lifecycle,
),
));
}
let agent_token = self
.session
.as_ref()
.map(|session| session.agent_token().to_string())
.or_else(|| self.pending_agent_token.clone())
.ok_or_else(|| {
Error::new(
ErrorKind::Precondition,
format!(
"microvm {} has no agent token to hand off; an adopter could not \
reach its daemon.",
vm.id
),
)
})?;
let detached = Detached {
microvm_id: vm.id.clone(),
endpoint: vm.endpoint.clone(),
region: self.control.region().clone(),
port: self.control.port(),
agent_token,
};
self.detached = true;
self.session = None;
self.pending_agent_token = None;
Ok(detached)
}
pub fn detached(&self) -> bool {
self.detached
}
fn refuse_detached(&self, what: &str) -> Result<(), Error> {
if !self.detached {
return Ok(());
}
let id = self.microvm.as_ref().map_or("?", |vm| vm.id.as_str());
Err(Error::new(
ErrorKind::Precondition,
format!(
"cannot {what}: microvm {id} was detached and belongs to whoever adopts it. \
Adopt it with `Sandbox::adopt` and {what} through that handle."
),
))
}
pub fn adopted(&self) -> bool {
self.adopted
}
pub async fn suspend(&mut self) -> Result<(), Error> {
self.refuse_detached("suspend")?;
let id = self.require_microvm("suspend")?;
if self.lifecycle != Lifecycle::Running {
return Err(Error::invalid_arg(format!(
"microvm {id} is {} and a suspend is only issued from RUNNING (STATE-5). Refused \
here rather than by the service, because the service's answer about a \
non-running id does not say which of the two things went wrong — and a suspend \
issued from SUSPENDED is a caller who believes they resumed.",
self.lifecycle,
)));
}
self.control.suspend(&id).await?;
self.set_lifecycle(Lifecycle::Suspending);
self.suspended_at = Some(self.control.clock().elapsed());
let settled = self
.control
.wait_for_state(
&id,
&crate::control::microvm::SUSPEND_WANTED,
&[],
self.lifecycle_wait(),
)
.await?;
let settled_lifecycle = match settled.state.as_str() {
"SUSPENDED" => Lifecycle::Suspended,
"TERMINATED" => {
self.was_terminated = true;
Lifecycle::Terminated
}
other => {
return Err(Error::new(
ErrorKind::Platform,
format!(
"the suspend wait returned {other}, which is neither SUSPENDED nor \
TERMINATED — and those two are what this client asked for."
),
));
}
};
self.set_lifecycle(settled_lifecycle);
self.microvm = Some(settled);
Ok(())
}
pub async fn resume(&mut self) -> Result<&mut Session, Error> {
self.refuse_detached("resume")?;
let id = self.require_microvm("resume")?;
if self.was_terminated || self.lifecycle == Lifecycle::Terminated {
return Err(Error::invalid_arg(format!(
"microvm {id} was terminated, and a terminated VM never returns to RUNNING \
(STATE-11). There is nothing to resume: the guest's memory is gone, so even a \
call the service accepted would hand back a different machine."
)));
}
if self.lifecycle != Lifecycle::Suspended {
return Err(Error::invalid_arg(format!(
"microvm {id} is {} and a resume is only issued from SUSPENDED (STATE-7).",
self.lifecycle,
)));
}
self.require_open_suspended_window(&id)?;
self.control.resume(&id).await?;
let running = self
.control
.wait_for_state(
&id,
&["RUNNING"],
&crate::constants::DEAD_STATES,
self.lifecycle_wait(),
)
.await?;
self.set_lifecycle(Lifecycle::Running);
let endpoint = running.endpoint.clone();
self.microvm = Some(running);
if let Some(session) = self.session.as_mut() {
session.rebind(endpoint);
}
self.suspended_at = None;
self.session.as_mut().ok_or_else(|| {
Error::new(
ErrorKind::Unexpected,
"the VM resumed but this sandbox holds no session, which run() always builds"
.to_string(),
)
})
}
fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> {
let reported = self
.microvm
.as_ref()
.and_then(|vm| vm.idle_policy.as_ref())
.map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds)));
let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at)
else {
return Ok(());
};
let elapsed = self.control.clock().elapsed().saturating_sub(since);
if elapsed <= window {
return Ok(());
}
Err(Error::new(
ErrorKind::WindowClosed,
format!(
"microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \
window set at launch — the idlePolicy terminates a suspended VM once that window \
passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \
Refused before ResumeMicrovm, because calling would cost the full poll timeout \
to learn the same thing less clearly. A longer window has to be set at launch \
on the next VM; there is no call that extends this one.",
elapsed.as_secs(),
window.as_secs(),
),
))
}
pub async fn terminate(&mut self, opts: TeardownOpts) -> TeardownReport {
let mut report = TeardownReport::default();
if let Err(error) = self.refuse_detached("terminate") {
report.failures.push(error.to_string());
report.lifecycle = Some(self.lifecycle);
return report;
}
self.torn_down = true;
self.session = None;
if let Some(id) = self.microvm.as_ref().map(|vm| vm.id.clone()) {
self.set_lifecycle(Lifecycle::Terminating);
self.was_terminated = true;
match self.control.terminate(&id).await {
Ok(()) => report.terminate_accepted = true,
Err(error) => {
report.failures.push(format!("terminate {id}: {error}"));
report.undeleted.push(id.clone());
}
}
if report.terminate_accepted
&& let Some(timeout) = opts.wait_for_terminated
{
let wait = WaitOpts {
timeout,
poll_interval: LIFECYCLE_POLL_INTERVAL,
stall_grace: Duration::MAX,
};
match self
.control
.wait_for_state(&id, &["TERMINATED"], &[], wait)
.await
{
Ok(settled) => {
self.set_lifecycle(Lifecycle::Terminated);
self.microvm = Some(settled);
}
Err(error) => report
.failures
.push(format!("waiting for {id} to reach TERMINATED: {error}")),
}
}
}
if opts.delete_image {
match self.image.as_ref().map(|image| image.identifier.clone()) {
Some(identifier) => {
let deleted = self
.control
.delete_image(&identifier, opts.delete_attempts, opts.delete_backoff)
.await;
report.image_deleted = Some(deleted);
if deleted {
self.image_exists = false;
} else {
report.failures.push(format!(
"the image {identifier} survived {} delete attempts",
opts.delete_attempts.max(1)
));
report.undeleted.push(identifier);
}
}
None => report.image_deleted = Some(false),
}
}
if opts.delete_log_group
&& let Some(group) = self.image.as_ref().map(Image::build_log_group)
{
report.failures.push(format!(
"the build log group {group} was not deleted: CloudWatch Logs is not in this \
crate's dependency set, so it is reported rather than removed. It is \
service-created, which means no Terraform stack owns it and `terraform destroy` \
leaves it behind — six accumulated before anyone noticed."
));
report.undeleted.push(group);
}
report.lifecycle = Some(self.lifecycle);
report
}
fn require_microvm(&self, what: &str) -> Result<String, Error> {
self.microvm
.as_ref()
.map(|vm| vm.id.clone())
.ok_or_else(|| {
Error::new(
ErrorKind::Precondition,
format!("nothing to {what}: this sandbox has not launched a VM."),
)
})
}
fn lifecycle_wait(&self) -> WaitOpts {
WaitOpts {
timeout: DEFAULT_LIFECYCLE_TIMEOUT,
poll_interval: LIFECYCLE_POLL_INTERVAL,
stall_grace: Duration::MAX,
}
}
}
impl Sandbox {
fn drop_warning(&self) -> Option<String> {
if self.torn_down || self.adopted || self.detached {
return None;
}
let vm = self.microvm.as_ref()?;
self.lifecycle.is_live().then(|| {
format!(
"warning: the Sandbox for microvm {} was dropped in {} without terminate() or \
detach(). Nothing was torn down — Drop cannot await, so a teardown here would \
deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \
ceiling: terminate it with `microvm terminate {}`.",
vm.id, self.lifecycle, vm.id,
)
})
}
}
impl Drop for Sandbox {
fn drop(&mut self) {
use std::io::Write as _;
if let Some(warning) = self.drop_warning() {
let _ = writeln!(std::io::stderr(), "{warning}");
}
}
}
fn mint_agent_token() -> String {
let mut bytes = [0u8; 32];
getrandom::fill(&mut bytes).expect("the OS random pool is available");
const_hex::encode(bytes)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::control::fake::{self as fake, Answer, FakeControlPlane, TestClock};
use crate::control::transport::Method;
fn planted() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
let recorder = Arc::new(FakeControlPlane::new());
let clock = Arc::new(TestClock::new());
let plane = ControlPlane::with_transport(
Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
Region::UsEast1,
Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
);
(Sandbox::with_control_plane(plane), recorder, clock)
}
fn answer_launch(recorder: &FakeControlPlane) {
recorder
.answer(
"RunMicrovm",
Answer::ok(fake::microvm_response("PENDING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer(
"CreateMicrovmAuthToken",
Answer::ok(fake::auth_token_response("proxy-token")),
);
}
#[tokio::test]
async fn vpc_egress_configuration_survives_the_sandbox_launch() {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
let request = RunRequest::new()
.with_image("arn:image")
.with_egress_network_connector(arn);
assert_eq!(
request.egress_posture(),
crate::control::EgressPosture::Unsealed
);
sandbox.run(request).await.expect("launch");
let body = recorder.first_body("RunMicrovm");
assert_eq!(body["egressNetworkConnectors"], serde_json::json!([arn]));
}
#[tokio::test]
async fn a_launched_session_reports_its_requests_posture() {
use crate::control::EgressPosture;
let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
let rows = [
(RunRequest::new(), EgressPosture::Unsealed),
(RunRequest::new().with_egress(), EgressPosture::Open),
(
RunRequest::new().with_deny_egress(),
EgressPosture::BestEffort,
),
(
RunRequest::new().with_egress_network_connector(arn),
EgressPosture::Unsealed,
),
];
for (request, expected) in rows {
let answered = crate::control::egress_posture_for(
request.egress,
&request.egress_network_connectors,
request.deny_egress,
Some(&Region::UsEast1),
)
.expect("a launchable request");
assert_eq!(answered, expected);
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
let session = sandbox
.run(request.with_image("arn:image"))
.await
.expect("the launch reaches RUNNING");
assert_eq!(session.egress_posture(), expected, "BIND-12");
assert_eq!(sandbox.egress_posture(), expected);
sandbox.detach().expect("hand the scripted VM off quietly");
}
}
#[tokio::test]
async fn a_session_without_its_launch_options_reports_unsealed() {
use crate::control::EgressPosture;
let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
assert_eq!(
sandbox.session().expect("RUNNING").egress_posture(),
EgressPosture::Unsealed
);
assert_eq!(sandbox.egress_posture(), EgressPosture::Unsealed);
let direct = Session::direct(ADOPT_ENDPOINT, ADOPT_TOKEN).expect("a direct session");
assert_eq!(direct.egress_posture(), EgressPosture::Unsealed);
}
async fn launched() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
let (mut sandbox, recorder, clock) = planted();
answer_launch(&recorder);
sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.expect("the launch reaches RUNNING");
(sandbox, recorder, clock)
}
#[tokio::test]
async fn a_detached_sandbox_hands_off_its_vm_and_stays_quiet() {
let (mut sandbox, recorder, _clock) = launched().await;
assert!(
sandbox.drop_warning().is_some(),
"a launched, live VM warns on drop"
);
let token = sandbox
.session()
.expect("launched")
.agent_token()
.to_string();
let vm = sandbox.microvm().expect("launched").clone();
let detached = sandbox.detach().expect("a RUNNING VM detaches");
assert_eq!(detached.microvm_id, vm.id);
assert_eq!(detached.endpoint, vm.endpoint);
assert_eq!(detached.agent_token(), token);
assert_eq!(detached.region, Region::UsEast1);
let printed = format!("{detached:?} {sandbox:?}");
assert!(!printed.contains(&token), "{printed}");
assert!(printed.contains("detached: true"), "{printed}");
assert!(sandbox.detached());
assert!(
sandbox.drop_warning().is_none(),
"detached is not abandoned"
);
assert!(
sandbox.session().is_none(),
"the session went to the adopter"
);
let before = recorder.calls().len();
for error in [
sandbox.suspend().await.expect_err("detached"),
sandbox.resume().await.map(|_| ()).expect_err("detached"),
sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.map(|_| ())
.expect_err("detached"),
sandbox
.wait_until_running(Duration::from_secs(1))
.await
.map(|_| ())
.expect_err("detached"),
sandbox.detach().map(|_| ()).expect_err("already detached"),
] {
assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
assert!(error.to_string().contains("detached"), "{error}");
assert!(!error.to_string().contains(&token), "{error}");
}
let report = sandbox.terminate(TeardownOpts::default()).await;
assert!(!report.terminate_accepted);
assert!(report.failures[0].contains("detached"), "{report:?}");
assert_eq!(recorder.calls().len(), before, "detached makes no AWS call");
assert!(
sandbox.drop_warning().is_none(),
"a refused terminate is still quiet"
);
}
#[tokio::test]
async fn detach_refuses_without_a_live_vm() {
let (mut unlaunched, _recorder, _clock) = planted();
let error = unlaunched.detach().expect_err("nothing launched");
assert_eq!(error.kind(), ErrorKind::Precondition);
assert!(!unlaunched.detached());
let (mut sandbox, recorder, _clock) = launched().await;
recorder
.answer("TerminateMicrovm", Answer::ok("{}"))
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("TERMINATED", None)),
);
sandbox.terminate(TeardownOpts::default()).await;
let error = sandbox.detach().expect_err("torn down");
assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
assert!(!sandbox.detached());
}
async fn suspended_with_window(
suspended_sec: u32,
) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
let (mut sandbox, recorder, clock) = planted();
recorder
.answer(
"RunMicrovm",
Answer::ok(fake::microvm_response("PENDING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer(
"CreateMicrovmAuthToken",
Answer::ok(fake::auth_token_response("proxy-token")),
)
.answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
sandbox
.run(
RunRequest::new()
.with_image("arn:image")
.with_suspended_sec(suspended_sec),
)
.await
.expect("launches");
sandbox.suspend().await.expect("suspends");
assert_eq!(
clock.now(),
Duration::ZERO,
"no wait may have slept, or the window arithmetic below is measuring the fake's \
poll interval as well as the elapsed suspension"
);
(sandbox, recorder, clock)
}
async fn suspended() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
suspended_with_window(600).await
}
const ADOPT_TOKEN: &str = "adopt-canary-token-9f1c";
const ADOPT_ENDPOINT: &str = "https://mvm-abc123.microvm.us-east-1.amazonaws.com";
fn adopt_plane() -> (ControlPlane, Arc<FakeControlPlane>, Arc<TestClock>) {
let recorder = Arc::new(FakeControlPlane::new());
let clock = Arc::new(TestClock::new());
let plane = ControlPlane::with_transport(
Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
Region::UsEast1,
Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
);
(plane, recorder, clock)
}
async fn adopted_in(state: &str) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
let (plane, recorder, clock) = adopt_plane();
recorder.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response(state, None)),
);
let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
.await
.expect("adopts");
(sandbox, recorder, clock)
}
#[tokio::test]
async fn an_adopted_sandbox_reports_the_services_idle_window_and_watches_its_lifecycle() {
let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
assert_eq!(sandbox.idle_window(), Some(Duration::from_secs(1800)));
assert_eq!(*sandbox.watch_lifecycle().borrow(), Lifecycle::Running);
}
#[test]
fn every_service_state_maps_onto_one_lifecycle() {
for state in crate::constants::MICROVM_STATES {
let lifecycle = Lifecycle::from_service(state).expect("a known state");
assert_eq!(lifecycle.as_str(), state);
}
assert_eq!(Lifecycle::from_service("Running"), None);
}
#[tokio::test]
async fn an_adopted_vm_takes_its_lifecycle_from_the_service_and_refuses_run() {
let (mut sandbox, recorder, _) = adopted_in("RUNNING").await;
assert!(sandbox.adopted());
assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
assert!(sandbox.token_installed());
assert_eq!(sandbox.bootstrap_count(), 1);
assert!(sandbox.image_exists());
let session = sandbox.session().expect("a live VM gets a session");
assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
assert_eq!(session.agent_token(), ADOPT_TOKEN);
let error = sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.expect_err("an adopted VM was bootstrapped by its own launch");
assert_eq!(error.kind(), ErrorKind::InvalidArg);
assert!(error.to_string().contains("STATE-3"), "{error}");
assert_eq!(recorder.call_count("RunMicrovm"), 0);
assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
}
#[tokio::test]
async fn a_name_adopts_its_records_vm_and_foreign_names_are_refused_locally() {
struct One(crate::names::NameRecord);
impl crate::names::NameStore for One {
fn get(&self, name: &str) -> Result<Option<crate::names::NameRecord>, Error> {
Ok((name == self.0.name).then(|| self.0.clone()))
}
fn put(&self, _: &crate::names::NameRecord) -> Result<(), Error> {
unreachable!()
}
fn delete(&self, _: &str) -> Result<bool, Error> {
unreachable!()
}
fn list(&self) -> Result<Vec<crate::names::NameRecord>, Error> {
Ok(vec![self.0.clone()])
}
}
let record = crate::names::NameRecord::new(
"ci",
"mvm-abc123",
ADOPT_ENDPOINT,
ADOPT_TOKEN,
"us-west-2",
)
.expect("record");
let store = One(record.clone());
let foreign = Sandbox::from_name(&store, "ci", Some(Region::UsEast1), None)
.await
.expect_err("registered in another region");
assert_eq!(foreign.kind(), ErrorKind::InvalidArg);
let missing = Sandbox::from_name(&store, "nope", None, None)
.await
.expect_err("no such name");
assert_eq!(missing.kind(), ErrorKind::Precondition);
assert!(!format!("{foreign} {missing}").contains(ADOPT_TOKEN));
let (plane, recorder, _) = adopt_plane();
recorder.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
);
let resolved = crate::names::resolve(&store, "ci", Some(&Region::UsWest2)).expect("ok");
let sandbox = Sandbox::adopt_record(plane, resolved)
.await
.expect("adopts");
assert!(sandbox.adopted());
assert_eq!(
sandbox.microvm().map(|vm| vm.id.as_str()),
Some("mvm-abc123")
);
assert_eq!(
sandbox.session().expect("session").agent_token(),
ADOPT_TOKEN
);
let named = sandbox
.name_record("again")
.expect("an adopted VM can be named");
assert_eq!(
(
named.microvm_id.as_str(),
named.agent_token.as_str(),
named.region.as_str()
),
("mvm-abc123", ADOPT_TOKEN, "us-east-1")
);
let unlaunched = Sandbox::with_control_plane(adopt_plane().0);
assert_eq!(
unlaunched
.name_record("x")
.expect_err("nothing to name")
.kind(),
ErrorKind::Precondition
);
}
#[tokio::test]
async fn an_adopted_vm_suspends_resumes_and_terminates_through_its_handle() {
let (plane, recorder, _) = adopt_plane();
recorder
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("TERMINATED", None)),
)
.answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
.answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
.await
.expect("adopts");
sandbox.suspend().await.expect("suspends from RUNNING");
assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
sandbox.resume().await.expect("resumes from SUSPENDED");
assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
let report = sandbox
.terminate(TeardownOpts::default().waiting_for_terminated())
.await;
assert!(report.terminate_accepted, "{report:?}");
assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
assert!(sandbox.was_terminated());
assert_eq!(recorder.call_count("RunMicrovm"), 0);
assert!(
recorder
.bodies_as_text()
.iter()
.all(|body| !body.contains("runHookPayload") && !body.contains(ADOPT_TOKEN)),
"no launch payload or agent token may reach the control plane"
);
assert_eq!(sandbox.bootstrap_count(), 1, "STATE-3: never above one");
}
#[tokio::test]
async fn an_adopted_vm_refuses_a_resume_past_the_reported_window() {
let (plane, recorder, clock) = adopt_plane();
recorder
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
.await
.expect("adopts");
assert_eq!(sandbox.suspended_window(), None, "no request of its own");
sandbox.suspend().await.expect("suspends");
clock.advance(Duration::from_secs(601));
let error = sandbox
.resume()
.await
.expect_err("past the 600 s window the service reported");
assert_eq!(error.kind(), ErrorKind::WindowClosed);
assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
}
#[tokio::test]
async fn an_adopted_suspended_vm_resumes_through_the_service() {
let (plane, recorder, _) = adopt_plane();
recorder
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
.await
.expect("adopts");
assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
let session = sandbox
.resume()
.await
.expect("the service answers the resume");
assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
}
#[tokio::test]
async fn an_adopted_terminated_vm_refuses_every_transition_locally() {
let (mut sandbox, recorder, _) = adopted_in("TERMINATED").await;
assert!(sandbox.was_terminated());
assert!(sandbox.session().is_none());
let resume = sandbox.resume().await.expect_err("STATE-11");
assert!(resume.to_string().contains("STATE-11"), "{resume}");
let suspend = sandbox.suspend().await.expect_err("STATE-5");
assert!(suspend.to_string().contains("STATE-5"), "{suspend}");
assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
}
#[tokio::test]
async fn an_adopted_pending_vm_counts_its_bootstrap_once_on_running() {
let (plane, recorder, _) = adopt_plane();
recorder
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("PENDING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
);
let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
.await
.expect("adopts");
assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
assert_eq!(sandbox.bootstrap_count(), 0);
sandbox
.wait_until_running(Duration::from_secs(60))
.await
.expect("reaches RUNNING");
assert_eq!(sandbox.bootstrap_count(), 1);
assert!(sandbox.token_installed());
assert!(
sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.is_err(),
"still one bootstrap per VM"
);
assert_eq!(recorder.call_count("RunMicrovm"), 0);
}
#[tokio::test]
async fn adopt_refuses_bad_records_and_never_prints_the_token() {
let (plane, recorder, _) = adopt_plane();
let empty = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, "")
.await
.expect_err("no token");
assert_eq!(empty.kind(), ErrorKind::InvalidArg);
assert_eq!(recorder.calls().len(), 0, "refused before any call");
let (plane, recorder, _) = adopt_plane();
recorder.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
);
let mismatch = Sandbox::adopt(plane, "mvm-abc123", "https://other.example", ADOPT_TOKEN)
.await
.expect_err("the endpoint belongs to another record");
assert_eq!(mismatch.kind(), ErrorKind::InvalidArg);
assert!(!mismatch.to_string().contains(ADOPT_TOKEN));
let (plane, recorder, _) = adopt_plane();
recorder.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("REBOOTING", None)),
);
let unknown = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
.await
.expect_err("an unknown state");
assert_eq!(unknown.kind(), ErrorKind::Platform);
assert!(!unknown.to_string().contains(ADOPT_TOKEN));
let (sandbox, _, _) = adopted_in("RUNNING").await;
let printed = format!("{sandbox:?}");
assert!(!printed.contains(ADOPT_TOKEN), "{printed}");
assert!(printed.contains("adopted: true"), "{printed}");
}
#[tokio::test]
async fn a_launch_records_the_image_and_the_running_report_installs_the_token() {
let (mut sandbox, recorder, _) = planted();
assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
assert!(
!sandbox.image_exists(),
"nothing is recorded before a launch"
);
assert!(!sandbox.token_installed());
answer_launch(&recorder);
let session = sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.expect("launches");
assert_eq!(
session.endpoint(),
"https://mvm-abc123.microvm.us-east-1.amazonaws.com"
);
assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
assert!(sandbox.image_exists(), "STATE-1: the image is recorded");
assert!(sandbox.token_installed(), "STATE-2: the token is installed");
assert_eq!(sandbox.bootstrap_count(), 1);
assert_eq!(sandbox.microvm().expect("launched").id, "mvm-abc123");
let body = recorder.first_body("RunMicrovm");
let payload = body["runHookPayload"].as_str().expect("a string");
assert!(payload.starts_with(r#"{"agent_token":"#), "{payload}");
assert_eq!(body["idlePolicy"]["suspendedDurationSeconds"], 600);
}
#[tokio::test]
async fn a_launch_env_reaches_the_run_hook_payload_on_the_wire() {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
sandbox
.run(
RunRequest::new()
.with_image("arn:image")
.with_launch_env("ANTHROPIC_BASE_URL", "https://gateway.example")
.with_launch_env("PATH", "/usr/local/bin:/usr/bin:/bin"),
)
.await
.expect("launches");
let body = recorder.first_body("RunMicrovm");
let payload = body["runHookPayload"].as_str().expect("a string");
let inner: serde_json::Value =
serde_json::from_str(payload).expect("the payload is itself JSON");
assert!(
inner["agent_token"].as_str().is_some_and(|t| !t.is_empty()),
"the token still rides alongside: {payload}"
);
assert_eq!(
inner["env"]["ANTHROPIC_BASE_URL"],
"https://gateway.example"
);
assert_eq!(inner["env"]["PATH"], "/usr/local/bin:/usr/bin:/bin");
}
#[tokio::test]
async fn the_advisory_deny_reaches_the_launch_environment_in_both_spellings() {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
let request = RunRequest::new().with_image("arn:image").with_deny_egress();
assert_eq!(
request.egress_posture(),
crate::control::EgressPosture::BestEffort
);
sandbox.run(request).await.expect("launches");
let body = recorder.first_body("RunMicrovm");
let inner: serde_json::Value =
serde_json::from_str(body["runHookPayload"].as_str().expect("a string"))
.expect("the payload is itself JSON");
for key in crate::sandbox::DENY_EGRESS_ENV_KEYS {
assert_eq!(
inner["env"][key], DENY_EGRESS_PROXY_URL,
"{key} must point at the black hole: {inner}"
);
}
assert!(
body.get("egressNetworkConnectors").is_none(),
"the advisory deny asks the platform for nothing: {body}"
);
}
#[test]
fn the_deny_never_overwrites_a_callers_own_proxy_and_is_absent_by_default() {
let kept = RunRequest::new()
.with_deny_egress()
.with_launch_env("https_proxy", "http://proxy.internal:3128")
.effective_launch_env();
assert_eq!(kept["https_proxy"], "http://proxy.internal:3128");
assert_eq!(kept["http_proxy"], DENY_EGRESS_PROXY_URL);
let plain = RunRequest::new().effective_launch_env();
assert!(
plain.is_empty(),
"a launch that did not ask for the deny sends byte-for-byte what it always sent: \
{plain:?}"
);
assert_eq!(
RunRequest::new().egress_posture(),
crate::control::EgressPosture::Unsealed,
"and its posture is the measured one, not a seal"
);
}
#[tokio::test]
async fn asking_for_egress_and_the_deny_together_is_refused_before_any_call() {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
let mut request = RunRequest::new().with_image("arn:image").with_egress();
request.deny_egress = true;
let error = sandbox
.run(request)
.await
.expect_err("opposite intents are not resolved by a precedence rule");
assert_eq!(error.kind(), ErrorKind::InvalidArg);
assert!(error.to_string().contains("opposite things"), "{error}");
assert_eq!(
recorder.calls().len(),
0,
"a locally-refused launch costs nothing"
);
}
#[tokio::test]
async fn a_pinned_image_version_is_forwarded_to_the_launch_request() {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
sandbox
.run(
RunRequest::new()
.with_image("arn:image")
.with_image_version("2.0"),
)
.await
.expect("launches");
assert_eq!(
recorder.first_body("RunMicrovm")["imageVersion"],
"2.0",
"a canary has to launch against the version it means to test"
);
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.expect("launches");
assert!(
recorder
.first_body("RunMicrovm")
.get("imageVersion")
.is_none(),
"an unpinned launch sends what this client always sent"
);
}
#[tokio::test]
async fn an_over_budget_launch_env_is_refused_before_any_control_plane_call() {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
let error = sandbox
.run(
RunRequest::new()
.with_image("arn:image")
.with_launch_env("AWS_SESSION_TOKEN", "t".repeat(4096)),
)
.await
.expect_err("a credential-scale launch env does not fit the payload");
assert_eq!(error.kind(), ErrorKind::InvalidArg);
let message = error.to_string();
assert!(message.contains("ceiling of 4096"), "{message}");
assert!(message.contains("launch env contributed"), "{message}");
assert_eq!(
recorder.calls().len(),
0,
"the refusal has to be local: a launch that reached AWS costs a VM and a bill"
);
assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
assert_eq!(sandbox.bootstrap_count(), 0);
assert!(sandbox.microvm().is_none());
}
#[tokio::test]
async fn the_bindings_layer_refuses_an_illegal_launch_with_zero_calls() {
let cases: [(&str, RunRequest, &str); 3] = [
(
"max_idle_sec=59, which is what a Python keyword argument passes",
RunRequest {
max_idle_sec: 59,
..RunRequest::new().with_image("arn:image")
},
"maxIdleDurationSeconds",
),
(
"an execution role that is a bare name",
RunRequest {
execution_role_arn: Some("execution-role".to_string()),
..RunRequest::new().with_image("arn:image")
},
"role *name*",
),
(
"a pinned version with a trailing newline",
RunRequest::new()
.with_image("arn:image")
.with_image_version("2.0\n"),
"contains whitespace",
),
];
for (label, request, expected) in cases {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
let Err(error) = sandbox.run(request).await else {
panic!("{label} must be refused");
};
assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
assert!(
error.to_string().contains(expected),
"{label}: wanted {expected:?}, got {error}"
);
assert_eq!(
recorder.calls().len(),
0,
"{label}: the refusal has to be local, because a launch that reached AWS costs a \
VM and a bill"
);
assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
assert!(sandbox.microvm().is_none());
}
}
#[tokio::test]
async fn a_launch_that_dies_during_startup_installs_no_token() {
let (mut sandbox, recorder, _) = planted();
recorder
.answer(
"RunMicrovm",
Answer::ok(fake::microvm_response("PENDING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response(
"TERMINATED",
Some("run hook returned 500"),
)),
);
let error = sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.expect_err("a VM that died during startup is not a launch");
assert_eq!(error.kind(), ErrorKind::LaunchDied);
assert!(
error.to_string().contains("run hook returned 500"),
"{error}"
);
assert!(
!sandbox.token_installed(),
"the run hook never answered, so nothing was installed"
);
assert_eq!(
sandbox.bootstrap_count(),
0,
"a bootstrap counted here would make STATE-3's ceiling unreachable for a retry"
);
assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
assert!(
sandbox.image_exists(),
"STATE-1 still holds: the launch was accepted against a real image"
);
}
#[tokio::test]
async fn a_second_run_on_one_sandbox_is_refused_before_any_call() {
let (mut sandbox, recorder, _) = launched().await;
assert_eq!(sandbox.bootstrap_count(), 1);
let before = recorder.calls().len();
let error = sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.expect_err("the token is installed at most once per VM lifetime");
assert_eq!(error.kind(), ErrorKind::InvalidArg);
assert!(error.to_string().contains("STATE-3"), "{error}");
assert!(
error.to_string().contains("needs a second Sandbox"),
"the remedy has to be nameable: {error}"
);
assert_eq!(
sandbox.bootstrap_count(),
1,
"bootstrap_count <= 1 is the Z3-proved invariant"
);
assert_eq!(
recorder.calls().len(),
before,
"the refusal must cost no control-plane call"
);
assert_eq!(recorder.call_count("RunMicrovm"), 1);
}
#[tokio::test]
async fn a_suspend_from_running_passes_through_suspending_to_suspended() {
let (sandbox, recorder, _) = suspended().await;
assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
assert!(
sandbox.token_installed(),
"a freeze keeps the guest's memory, so the token survives"
);
assert_eq!(sandbox.bootstrap_count(), 1, "no re-bootstrap on a freeze");
let calls = recorder.calls();
let suspend = calls
.iter()
.find(|call| call.operation == "SuspendMicrovm")
.expect("the suspend went out");
assert_eq!(suspend.method, Method::Post);
assert_eq!(suspend.path, "/2025-09-09/microvms/mvm-abc123/suspend");
}
#[tokio::test]
async fn a_vm_that_dies_while_suspending_is_recorded_rather_than_raised() {
let (mut sandbox, recorder, _) = launched().await;
recorder
.answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("TERMINATED", Some("idle policy"))),
);
sandbox
.suspend()
.await
.expect("a death mid-suspend is a state, not an error");
assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
assert!(
sandbox.was_terminated(),
"STATE-11's precondition is recorded"
);
}
#[tokio::test]
async fn a_suspend_whose_call_fails_stays_running_and_can_be_retried() {
let (mut sandbox, recorder, _) = planted();
recorder
.answer(
"RunMicrovm",
Answer::ok(fake::microvm_response("PENDING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer(
"CreateMicrovmAuthToken",
Answer::ok(fake::auth_token_response("proxy-token")),
)
.answer("SuspendMicrovm", Answer::failure(409, "ConflictException"))
.answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.expect("launches");
sandbox
.suspend()
.await
.expect_err("the control plane refused the suspend");
assert_eq!(
sandbox.lifecycle(),
Lifecycle::Running,
"a refused suspend must not move the lifecycle: SUSPENDING accepts neither a \
suspend nor a resume, so recording it here would brick the handle"
);
sandbox.suspend().await.expect("the retry suspends");
assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
assert_eq!(recorder.call_count("SuspendMicrovm"), 2);
}
#[tokio::test]
async fn a_suspend_from_a_non_running_state_reaches_no_control_plane_call() {
let (mut sandbox, recorder, _) = suspended().await;
let before = recorder.call_count("SuspendMicrovm");
let error = sandbox
.suspend()
.await
.expect_err("a suspend is only issued from RUNNING");
assert_eq!(error.kind(), ErrorKind::InvalidArg);
assert!(error.to_string().contains("STATE-5"), "{error}");
assert!(error.to_string().contains("SUSPENDED"), "{error}");
assert_eq!(
recorder.call_count("SuspendMicrovm"),
before,
"the refusal must be local: no second suspend went to the wire"
);
let (mut fresh, fresh_recorder, _) = planted();
let error = fresh.suspend().await.expect_err("nothing to suspend");
assert_eq!(error.kind(), ErrorKind::Precondition);
assert_eq!(fresh_recorder.calls().len(), 0);
}
#[tokio::test]
async fn a_resume_reuses_the_token_redelivers_nothing_and_drops_the_proxy_token() {
let (mut sandbox, recorder, _) = suspended().await;
let auth = Arc::clone(
sandbox
.session()
.expect("launched")
.proxy_auth()
.expect("the launch wired a minter"),
);
auth.headers().await.expect("mints");
assert!(
auth.is_cached(),
"there must be a token for the drop to matter"
);
assert_eq!(auth.mint_count(), 1);
recorder
.answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
);
let session = sandbox.resume().await.expect("resumes");
assert_eq!(
session.endpoint(),
"https://mvm-abc123.microvm.us-east-1.amazonaws.com"
);
assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
assert!(
sandbox.token_installed(),
"STATE-7: the installed token is reused"
);
assert_eq!(
sandbox.bootstrap_count(),
1,
"STATE-7: a resume re-delivers no run-hook payload"
);
assert_eq!(
recorder.call_count("RunMicrovm"),
1,
"a resume is not a second launch"
);
assert!(
!auth.is_cached(),
"STATE-8: the cached proxy token survived the resume"
);
}
#[tokio::test]
async fn the_request_after_a_resume_mints_a_fresh_proxy_token() {
let (mut sandbox, recorder, _) = suspended().await;
let auth = Arc::clone(
sandbox
.session()
.expect("launched")
.proxy_auth()
.expect("wired"),
);
auth.headers().await.expect("mints");
assert_eq!(auth.mint_count(), 1);
recorder
.answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
);
sandbox.resume().await.expect("resumes");
auth.headers().await.expect("re-mints");
assert_eq!(
auth.mint_count(),
2,
"STATE-8: the resume did not force a fresh mint"
);
}
#[tokio::test]
async fn the_lifecycle_watch_follows_every_transition() {
let (mut sandbox, recorder, _clock) = planted();
let watch = sandbox.watch_lifecycle();
assert_eq!(*watch.borrow(), Lifecycle::Pending);
recorder
.answer(
"RunMicrovm",
Answer::ok(fake::microvm_response("PENDING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer(
"CreateMicrovmAuthToken",
Answer::ok(fake::auth_token_response("proxy-token")),
)
.answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
sandbox
.run(RunRequest::new().with_image("arn:image"))
.await
.expect("launches");
assert_eq!(*watch.borrow(), Lifecycle::Running);
sandbox.suspend().await.expect("suspends");
assert_eq!(*watch.borrow(), Lifecycle::Suspended);
assert_eq!(*watch.borrow(), sandbox.lifecycle());
}
#[tokio::test]
async fn a_resume_past_the_suspended_window_is_refused_before_the_wire() {
let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
assert_eq!(sandbox.suspended_window(), Some(Duration::from_secs(60)));
assert_eq!(
sandbox.idle_window(),
Some(Duration::from_secs(600)),
"the launch's own maxIdleDurationSeconds, which a keepalive checks its cadence against"
);
clock.advance(Duration::from_secs(61));
recorder
.answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response(
"TERMINATED",
Some("suspended window elapsed"),
)),
);
let elapsed_before = clock.now();
let resumes_before = recorder.call_count("ResumeMicrovm");
let error = sandbox
.resume()
.await
.expect_err("the window the launch set has closed");
assert_eq!(error.kind(), ErrorKind::WindowClosed);
assert_eq!(error.code(), "ERR_WINDOW_CLOSED");
let message = error.to_string();
assert!(message.contains("61s"), "the elapsed time: {message}");
assert!(
message.contains("60s suspendedDurationSeconds"),
"the window has to be named: {message}"
);
assert!(message.contains("idlePolicy"), "the finding: {message}");
assert!(
message.contains("Refused before ResumeMicrovm"),
"why the refusal is local: {message}"
);
assert_eq!(
recorder.call_count("ResumeMicrovm"),
resumes_before,
"the refusal must come before ResumeMicrovm"
);
assert_eq!(
clock.now(),
elapsed_before,
"the refusal must be immediate rather than polled to the deadline"
);
}
#[tokio::test]
async fn the_window_falls_back_to_the_idle_policy_the_service_reported() {
let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
sandbox.suspended_window = None;
recorder.answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
clock.advance(Duration::from_secs(601));
let error = sandbox
.resume()
.await
.expect_err("past the reported window");
assert_eq!(error.kind(), ErrorKind::WindowClosed);
assert!(error.to_string().contains("600s"), "{error}");
assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
}
#[tokio::test]
async fn a_client_token_launch_resumes_the_vm_it_adopted() {
let (mut sandbox, recorder, _) = planted();
recorder
.answer(
"RunMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
.answer(
"CreateMicrovmAuthToken",
Answer::ok(fake::auth_token_response("proxy-token")),
);
let mut request = RunRequest::new().with_image("arn:image");
request.client_token = Some("job-195".into());
request.agent_token = Some("agent-token".into());
sandbox
.run(request)
.await
.expect("the adopted VM is resumed");
assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
assert_eq!(sandbox.bootstrap_count(), 1);
assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
let (mut fresh, recorder, _) = planted();
recorder
.answer(
"RunMicrovm",
Answer::ok(fake::microvm_response("PENDING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", Some("hook failed"))),
);
let error = fresh
.run(RunRequest::new().with_image("arn:image"))
.await
.expect_err("a fresh launch that reads SUSPENDED died during startup");
assert_eq!(error.kind(), ErrorKind::LaunchDied);
assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
}
#[tokio::test]
async fn a_launch_that_does_not_wait_is_finished_by_wait_until_running() {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
let mut request = RunRequest::new().with_image("arn:image");
request.wait = false;
let endpoint = sandbox
.run(request)
.await
.expect("accepted")
.endpoint()
.to_string();
assert!(endpoint.starts_with("https://"), "{endpoint}");
assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
assert_eq!(sandbox.bootstrap_count(), 0);
assert_eq!(recorder.call_count("GetMicrovm"), 0, "nothing polled yet");
sandbox
.wait_until_running(DEFAULT_READY_TIMEOUT)
.await
.expect("reaches RUNNING");
assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
assert_eq!(sandbox.bootstrap_count(), 1);
let error = sandbox
.wait_until_running(DEFAULT_READY_TIMEOUT)
.await
.expect_err("a running launch has nothing to wait for");
assert_eq!(error.kind(), ErrorKind::InvalidArg);
assert_eq!(sandbox.bootstrap_count(), 1, "counted once (STATE-3)");
}
#[tokio::test]
async fn a_resume_inside_the_window_is_allowed_and_the_boundary_is_inclusive() {
let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
clock.advance(Duration::from_secs(60));
recorder
.answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
);
sandbox
.resume()
.await
.expect("60s == the 60s window is open");
assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
}
#[tokio::test]
async fn a_successful_resume_clears_the_stamp_so_a_second_cycle_measures_its_own_window() {
let (mut sandbox, recorder, clock) = planted();
recorder
.answer(
"RunMicrovm",
Answer::ok(fake::microvm_response("PENDING", None)),
)
.answer(
"CreateMicrovmAuthToken",
Answer::ok(fake::auth_token_response("proxy-token")),
)
.answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
.answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("SUSPENDED", None)),
)
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("RUNNING", None)),
);
sandbox
.run(
RunRequest::new()
.with_image("arn:image")
.with_suspended_sec(60),
)
.await
.expect("launches");
for cycle in 0..2 {
sandbox
.suspend()
.await
.unwrap_or_else(|error| panic!("cycle {cycle} suspend: {error}"));
assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended, "cycle {cycle}");
clock.advance(Duration::from_secs(45));
sandbox.resume().await.unwrap_or_else(|error| {
panic!("cycle {cycle} resume, whose own window is wide open: {error}")
});
assert_eq!(sandbox.lifecycle(), Lifecycle::Running, "cycle {cycle}");
assert_eq!(
sandbox.bootstrap_count(),
1,
"cycle {cycle}: no re-bootstrap"
);
}
assert_eq!(
clock.now(),
Duration::from_secs(90),
"ninety seconds of accumulated suspension against a sixty-second window: this is \
the total an implementation that never cleared the stamp would have compared"
);
assert_eq!(
recorder.call_count("ResumeMicrovm"),
2,
"both cycles resumed"
);
}
#[tokio::test]
async fn a_resume_after_terminate_records_zero_resume_calls() {
let (mut sandbox, recorder, _) = launched().await;
recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
let report = sandbox.terminate(TeardownOpts::default()).await;
assert!(report.terminate_accepted);
assert!(sandbox.was_terminated());
let error = sandbox
.resume()
.await
.expect_err("a terminated VM never returns to RUNNING");
assert_eq!(error.kind(), ErrorKind::InvalidArg);
assert!(error.to_string().contains("STATE-11"), "{error}");
assert_eq!(
recorder.call_count("ResumeMicrovm"),
0,
"no resume may reach the wire once the VM was terminated"
);
assert_ne!(
sandbox.lifecycle(),
Lifecycle::Running,
"the lifecycle must not return to RUNNING"
);
}
#[tokio::test]
async fn a_terminate_records_terminating_and_the_completion_report_marks_terminated() {
let (mut sandbox, recorder, _) = launched().await;
recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
let report = sandbox.terminate(TeardownOpts::default()).await;
assert!(report.terminate_accepted, "STATE-9: the call was accepted");
assert!(sandbox.was_terminated(), "STATE-9: recorded terminated");
assert_eq!(
report.lifecycle,
Some(Lifecycle::Terminating),
"the default teardown does not wait, so TERMINATED is not claimed"
);
assert!(!report.leaked(), "{report:?}");
let (mut sandbox, recorder, _) = launched().await;
recorder
.answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
.answer(
"GetMicrovm",
Answer::ok(fake::microvm_response("TERMINATED", None)),
);
let report = sandbox
.terminate(TeardownOpts::default().waiting_for_terminated())
.await;
assert_eq!(report.lifecycle, Some(Lifecycle::Terminated));
assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
}
#[tokio::test]
async fn teardown_deletes_the_vm_then_the_image_and_names_the_log_group_last() {
let (mut sandbox, recorder, _) = planted();
recorder
.answer(
"CreateMicrovmImage",
Answer::created(fake::create_image_response("agentd-conformance")),
)
.answer(
"GetMicrovmImage",
Answer::ok(fake::get_image_response("agentd-conformance", "CREATED")),
);
sandbox
.build_image(CreateImageRequest::new(
"agentd-conformance",
b"binary".to_vec(),
"s3://bucket/img.zip",
"arn:aws:iam::123456789012:role/build",
))
.await
.expect("builds");
answer_launch(&recorder);
sandbox
.run(RunRequest::new())
.await
.expect("launches from the built image");
recorder
.answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
.answer(
"ListMicrovmImageVersions",
Answer::ok(fake::list_versions_response("1")),
)
.answer(
"DeleteMicrovmImage",
Answer::ok(fake::delete_image_response()),
);
let report = sandbox
.terminate(
TeardownOpts::default()
.deleting_image()
.deleting_log_group(),
)
.await;
assert!(report.terminate_accepted);
assert_eq!(report.image_deleted, Some(true));
let order: Vec<&str> = recorder
.operations()
.into_iter()
.filter(|operation| matches!(*operation, "TerminateMicrovm" | "DeleteMicrovmImage"))
.collect();
assert_eq!(
order,
["TerminateMicrovm", "DeleteMicrovmImage"],
"the VM goes before the image: a terminating VM holds a reference to it"
);
assert_eq!(
report.undeleted,
["/aws/lambda-microvms/agentd-conformance"],
"{report:?}"
);
assert!(report.leaked());
assert!(
report
.failures
.last()
.expect("a failure")
.contains("log group"),
"the log group's failure has to be the last one recorded: {report:?}"
);
assert!(
report
.failures
.last()
.expect("a failure")
.contains("terraform destroy"),
"the reason it leaks has to be named: {report:?}"
);
}
#[tokio::test]
async fn a_failing_image_delete_names_what_it_left_behind() {
let (mut sandbox, recorder, _) = planted();
recorder
.answer(
"CreateMicrovmImage",
Answer::created(fake::create_image_response("img")),
)
.answer(
"GetMicrovmImage",
Answer::ok(fake::get_image_response("img", "CREATED")),
);
sandbox
.build_image(CreateImageRequest::new(
"img",
b"binary".to_vec(),
"s3://bucket/img.zip",
"arn:aws:iam::123456789012:role/build",
))
.await
.expect("builds");
answer_launch(&recorder);
sandbox.run(RunRequest::new()).await.expect("launches");
recorder
.answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
.answer(
"ListMicrovmImageVersions",
Answer::ok(fake::list_versions_response("1")),
)
.answer(
"DeleteMicrovmImage",
Answer::failure(409, "image is in CREATING"),
);
let mut opts = TeardownOpts::default().deleting_image();
opts.delete_attempts = 3;
opts.delete_backoff = Duration::from_secs(1);
let report = sandbox.terminate(opts).await;
assert_eq!(report.image_deleted, Some(false));
assert_eq!(
report.undeleted,
["arn:aws:lambda:us-east-1:123456789012:microvm-image:img"],
"{report:?}"
);
assert!(
sandbox.image_exists(),
"the image really is still there, so the flag must say so"
);
assert_eq!(
recorder.call_count("DeleteMicrovmImage"),
3,
"every attempt ran"
);
}
#[tokio::test]
async fn a_terminate_whose_call_fails_still_records_the_intent_and_the_leak() {
let (mut sandbox, recorder, _) = launched().await;
recorder.answer(
"TerminateMicrovm",
Answer::failure(500, "InternalServerException"),
);
let report = sandbox.terminate(TeardownOpts::default()).await;
assert!(!report.terminate_accepted);
assert_eq!(report.undeleted, ["mvm-abc123"], "{report:?}");
assert!(sandbox.was_terminated());
assert_eq!(report.lifecycle, Some(Lifecycle::Terminating));
assert!(
!report.failures.is_empty(),
"the swallowed failure is recorded"
);
}
#[tokio::test]
async fn a_teardown_before_any_launch_is_an_empty_report() {
let (mut sandbox, recorder, _) = planted();
let report = sandbox.terminate(TeardownOpts::default()).await;
assert!(!report.terminate_accepted);
assert!(!report.leaked());
assert_eq!(report.lifecycle, Some(Lifecycle::Pending));
assert_eq!(recorder.calls().len(), 0);
}
#[tokio::test]
async fn a_run_with_no_image_is_refused_before_any_call() {
let (mut sandbox, recorder, _) = planted();
let error = sandbox
.run(RunRequest::new())
.await
.expect_err("there is nothing to launch");
assert_eq!(error.kind(), ErrorKind::Precondition);
assert!(error.to_string().contains("build_image first"), "{error}");
assert_eq!(recorder.calls().len(), 0);
}
#[test]
fn the_lifecycle_states_are_the_state_models_domain_in_the_services_spelling() {
let all = [
Lifecycle::Pending,
Lifecycle::Running,
Lifecycle::Suspending,
Lifecycle::Suspended,
Lifecycle::Terminating,
Lifecycle::Terminated,
];
assert_eq!(
all.map(Lifecycle::as_str),
[
"PENDING",
"RUNNING",
"SUSPENDING",
"SUSPENDED",
"TERMINATING",
"TERMINATED"
]
);
for state in all {
assert_eq!(state.to_string(), state.as_str());
}
assert_eq!(
all.iter().filter(|state| state.is_live()).count(),
4,
"a state added to the enum defaults to not-live, which would silence the warning"
);
assert!(!Lifecycle::Terminating.is_live());
assert!(!Lifecycle::Terminated.is_live());
}
#[test]
fn a_minted_agent_token_is_sixty_four_hex_characters_and_never_repeats() {
let minted: std::collections::HashSet<String> =
(0..200).map(|_| mint_agent_token()).collect();
assert_eq!(minted.len(), 200, "a repeated token is a shared guest");
for token in &minted {
assert_eq!(token.len(), 64, "{token}");
assert!(token.chars().all(|c| c.is_ascii_hexdigit()), "{token}");
assert_ne!(
token,
&"0".repeat(64),
"an all-zero draw is a read that did nothing"
);
}
}
#[tokio::test]
async fn a_sandbox_debug_does_not_print_the_agent_token() {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
sandbox
.run(
RunRequest::new()
.with_image("arn:image")
.with_agent_token("super-secret-agent-token"),
)
.await
.expect("launches");
let rendered = format!("{sandbox:?}");
assert!(rendered.contains("mvm-abc123"), "{rendered}");
assert!(rendered.contains("Running"), "{rendered}");
assert!(
!rendered.contains("super-secret"),
"the agent token reached a Debug string: {rendered}"
);
}
#[tokio::test]
async fn a_torn_down_sandbox_is_distinguishable_from_an_abandoned_one() {
let (mut sandbox, recorder, _) = launched().await;
assert!(
sandbox.lifecycle().is_live() && !sandbox.torn_down,
"an abandoned live VM is what the drop warning is for"
);
recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
sandbox.terminate(TeardownOpts::default()).await;
assert!(
sandbox.torn_down,
"terminate must mark the sandbox torn down"
);
assert!(
!sandbox.lifecycle().is_live(),
"and the lifecycle must no longer read as live"
);
}
#[tokio::test]
async fn stable_launch_key_and_payload_survive_a_fresh_supervisor() {
let mut bodies = Vec::new();
for _ in 0..2 {
let (mut sandbox, recorder, _) = planted();
answer_launch(&recorder);
let mut request = RunRequest::new().with_image("arn:image");
request.client_token = Some("one-unique-job-launch".into());
request.agent_token = Some("persisted-private-secret".into());
sandbox.run(request).await.expect("accepted");
bodies.push(recorder.first_body("RunMicrovm"));
}
assert_eq!(bodies[0]["clientToken"], "one-unique-job-launch");
assert_eq!(bodies[0], bodies[1]);
}
#[tokio::test]
async fn unsafe_or_malformed_stable_launch_is_refused_without_calls() {
for (token, secret, identity) in [
("key".to_string(), None, false),
("key".to_string(), Some("secret".into()), true),
("".to_string(), Some("secret".into()), false),
("x".repeat(129), Some("secret".into()), false),
("line\nbreak".to_string(), Some("secret".into()), false),
] {
let (mut sandbox, recorder, _) = planted();
let mut request = RunRequest::new().with_image("arn:image");
request.client_token = Some(token);
request.agent_token = secret;
request.identity = identity;
let error = sandbox.run(request).await.expect_err("invalid");
assert_eq!(error.kind(), ErrorKind::InvalidArg);
assert!(recorder.calls().is_empty());
}
}
}