microvms_core/sandbox.rs
1// SPDX-License-Identifier: Apache-2.0
2//! The `Sandbox` lifecycle: one VM's whole life, with the state machine the symspec
3//! model describes made true of the code.
4//!
5//! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the
6//! bindings use: build, run, suspend, resume, terminate, the launch-time suspended
7//! window (STATE-12), and teardown in the order that does not leak.
8//!
9//! # The lifecycle is a field, and every transition is guarded
10//!
11//! [`Lifecycle`] is the symspec's `vm_state` verbatim, and [`Sandbox`] carries the other
12//! four variables beside it — `token_installed`, `image_exists`, `was_terminated`,
13//! `bootstrap_count`. The Z3 proofs over that model (bootstrap at most once, suspend from
14//! non-RUNNING unreachable, TERMINATED never returns to RUNNING) are proofs about *this*
15//! struct's reachable states, which is only worth something if the transitions here are
16//! the only way to move it. They are: every one of those fields is private, and every
17//! mutation happens in one of the five methods below.
18//!
19//! # Runtime-checked rather than typestate, deliberately
20//!
21//! The packet offered `Sandbox<Running>` returning a `Suspended` handle, which would make
22//! STATE-5's wrong call a compile error rather than a local refusal — strictly stronger on
23//! the ladder in [`crate`]. It is not what landed, for the reason the packet names as
24//! acceptable: T-W3-8 wraps **one** object for PyO3 and napi-rs, and a type whose Rust
25//! identity changes on every transition cannot be one `#[pyclass]`. A typestate sandbox
26//! would be re-erased into a runtime-checked enum at the binding boundary, so the check
27//! would exist twice with the binding's copy being the one most callers actually hit.
28//!
29//! What is kept from the typestate idea is the part that costs nothing: the check happens
30//! **before** the wire call, so a suspend from SUSPENDED is refused with zero
31//! control-plane calls rather than answered by AWS. The test asserts the call count, which
32//! is the observable that distinguishes the two.
33//!
34//! # The suspended window is checked before the call (STATE-12)
35//!
36//! The launch-time `idlePolicy` *terminates* a suspended VM once `suspendedDurationSeconds`
37//! passes, which means "resume later" silently stops working. A resume past the window is
38//! refused locally, before `ResumeMicrovm`, because the alternative is calling and reading
39//! the failure: the service answers about a terminated id, which is not the same statement
40//! as "the window you set at launch closed", and getting there costs the full poll timeout
41//! first. The window comes from this sandbox's own `RunMicrovm` request, falling back to the
42//! `idlePolicy` that `GetMicrovm` reports (measured 2026-08-15, `docs/PLATFORM.md`). Only
43//! the suspend's start time is the client's alone.
44//!
45//! # Teardown never raises, and the log group is last
46//!
47//! [`Sandbox::terminate`] returns a [`TeardownReport`] rather than a `Result`. It runs
48//! where a caller's `finally` would, and an error raised there replaces the real failure
49//! with a teardown failure — the real one being the one worth reading.
50//!
51//! The order is VM, then image (retrying), then the log group **last**, because the
52//! service can recreate a group deleted before its image. See
53//! [`TeardownReport::undeleted`] for what this crate can and cannot delete.
54//!
55//! # There is no `Drop` that tears down
56//!
57//! Rust has no context manager and `Drop` cannot await. A `Drop` that blocked on a runtime
58//! would deadlock inside one; a `Drop` that spawned would race the process exit. So
59//! [`Sandbox`]'s `Drop` only **warns** about a live VM, naming the id, and the rule is
60//! that a caller calls [`Sandbox::terminate`] explicitly.
61
62use std::sync::Arc;
63use std::time::Duration;
64
65use crate::control::{
66 ControlPlane, CreateImageRequest, Image, Microvm, RunHookPayload, RunMicrovmRequest, WaitOpts,
67};
68use crate::error::{Error, ErrorKind};
69use crate::region::Region;
70use crate::session::{Session, TokenMinter};
71
72/// The default launch wait: five minutes, matching the Python client's `ready_timeout_sec`.
73pub const DEFAULT_READY_TIMEOUT: Duration = Duration::from_secs(300);
74
75/// The default lifecycle wait for suspend, resume, and terminate.
76pub const DEFAULT_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(300);
77
78/// How many times a teardown retries the image delete.
79///
80/// Twenty, from the Python client: an image in `CREATING` refuses deletion and a VM still
81/// terminating holds a reference, so this is the difference between a clean account and a
82/// billed leak rather than politeness.
83pub const DEFAULT_DELETE_ATTEMPTS: u32 = 20;
84
85/// The gap between image-delete attempts.
86pub const DEFAULT_DELETE_BACKOFF: Duration = Duration::from_secs(15);
87
88/// Where the advisory deny points a well-behaved client: a loopback port nothing serves.
89///
90/// Loopback rather than an unroutable public address, because a refused connection is
91/// immediate and a black-holed one costs the workload a connect timeout per request. Port 1
92/// is privileged, so a demoted workload cannot answer it by accident.
93pub const DENY_EGRESS_PROXY_URL: &str = "http://127.0.0.1:1";
94
95/// The environment variables the advisory deny sets, in both spellings clients read.
96///
97/// Both cases on purpose: `curl` and most Unix clients read the lowercase names, Python's
98/// `requests`/`urllib3`, npm and the AWS SDKs read the uppercase ones, and a client that
99/// reads only the spelling this list omits is a hole in a mechanism that is advisory to
100/// begin with. `no_proxy` is deliberately absent: the launch environment starts empty
101/// (`agentd` clears it and applies only what the hook delivered), so there is no inherited
102/// exemption to overwrite, and each key spends payload budget.
103pub const DENY_EGRESS_ENV_KEYS: [&str; 6] = [
104 "http_proxy",
105 "https_proxy",
106 "all_proxy",
107 "HTTP_PROXY",
108 "HTTPS_PROXY",
109 "ALL_PROXY",
110];
111
112/// How often a lifecycle wait polls.
113const LIFECYCLE_POLL_INTERVAL: Duration = Duration::from_secs(5);
114
115/// The symspec's `vm_state`, verbatim.
116///
117/// Six states and no others, which is the S1 half of this module: a lifecycle held as a
118/// `String` would let `"RUNNING "` and `"Running"` both exist, and every guard below would
119/// have to decide which it meant.
120#[derive(Clone, Copy, Debug, Eq, PartialEq)]
121pub enum Lifecycle {
122 /// The initial state, and the state a launch is accepted into (STATE-1).
123 Pending,
124 /// The run hook answered with a success status and the token is installed (STATE-2).
125 Running,
126 /// A suspend was accepted and the platform has not yet reported it complete (STATE-4).
127 Suspending,
128 /// The platform reported suspension complete (STATE-6).
129 Suspended,
130 /// A terminate was accepted (STATE-9).
131 Terminating,
132 /// The platform reported termination complete (STATE-10).
133 Terminated,
134}
135
136impl Lifecycle {
137 /// The name the service uses for this state, which is also what an error message says.
138 pub fn as_str(self) -> &'static str {
139 match self {
140 Lifecycle::Pending => "PENDING",
141 Lifecycle::Running => "RUNNING",
142 Lifecycle::Suspending => "SUSPENDING",
143 Lifecycle::Suspended => "SUSPENDED",
144 Lifecycle::Terminating => "TERMINATING",
145 Lifecycle::Terminated => "TERMINATED",
146 }
147 }
148
149 /// The lifecycle the service's `state` names, or `None` for a spelling this client does
150 /// not know. `constants::MICROVM_STATES` is the closed set, and a test ties the two.
151 pub fn from_service(state: &str) -> Option<Self> {
152 Some(match state {
153 "PENDING" => Lifecycle::Pending,
154 "RUNNING" => Lifecycle::Running,
155 "SUSPENDING" => Lifecycle::Suspending,
156 "SUSPENDED" => Lifecycle::Suspended,
157 "TERMINATING" => Lifecycle::Terminating,
158 "TERMINATED" => Lifecycle::Terminated,
159 _ => return None,
160 })
161 }
162
163 /// Whether a VM in this state is still billing, which is what a `Drop` warning is for.
164 pub fn is_live(self) -> bool {
165 matches!(
166 self,
167 Lifecycle::Pending | Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended
168 )
169 }
170}
171
172impl std::fmt::Display for Lifecycle {
173 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
174 f.write_str(self.as_str())
175 }
176}
177
178/// Everything a launch needs, with the defaults the Python client measured.
179///
180/// `agent_token` is optional because the common case is a per-VM secret nobody needs to
181/// see; a caller who has one already — a harness minting its own, or a retry that must
182/// reuse the first attempt's — passes it.
183#[derive(Clone, Debug)]
184pub struct RunRequest {
185 /// The image to launch, or `None` for the one [`Sandbox::build_image`] built.
186 pub image_identifier: Option<String>,
187 /// `imageVersion`, or `None` for the image's own latest active version.
188 ///
189 /// Pinning is what makes a canary a canary: without it a launch takes whatever version is
190 /// latest at the moment the call lands, which is not necessarily the one the caller just
191 /// built. Paired with [`crate::control::ControlPlane::set_image_version_status`] it is also
192 /// the rollback — set the bad version INACTIVE, re-pin here to the good one — and a version
193 /// set INACTIVE refuses to launch when named here.
194 ///
195 /// See [`crate::control::RunMicrovmRequest::image_version`], which this forwards to.
196 pub image_version: Option<String>,
197 /// The execution role. Optional in the model; every real launch needs one.
198 pub execution_role_arn: Option<String>,
199 /// The bearer token the daemon will accept, or `None` to mint one.
200 pub agent_token: Option<String>,
201 /// Persist this unique launch key before sending a request; reuse only for that same
202 /// launch and identical parameters. None mints a fresh key. A stable key requires
203 /// an explicit agent_token and identity=false so retries carry identical payloads.
204 pub client_token: Option<String>,
205 /// Base environment for every exec in the launched VM, delivered in the same
206 /// `runHookPayload` as the token.
207 ///
208 /// The daemon applies this *under* each request's own `env`, so a per-exec value
209 /// wins on a key both set. Empty by default, and an empty map produces byte-for-byte
210 /// the payload this client always sent — a caller who never touches this field
211 /// cannot be affected by the field existing.
212 ///
213 /// It shares the token's 4096-byte payload budget, and the check is local:
214 /// [`crate::control::RunHookPayload::for_launch`] refuses an over-ceiling payload
215 /// before any call, naming the byte count and how much of it the env is. That
216 /// matters here more than for the token, because one bearer token has always fit
217 /// with room to spare and a map of credentials does not.
218 pub launch_env: std::collections::HashMap<String, String>,
219 /// Whether to generate and deliver a tunnel identity (#70 layer 3).
220 ///
221 /// On, the launch generates two fresh x25519 seeds, delivers the VM's seed and the
222 /// host's public key in the same payload as the token, and [`RunOutcome`]-equivalent
223 /// state on the session keeps the [`crate::identity::TunnelIdentity`] — the host secret
224 /// and the VM's public pin — for `tunnel --verify-identity` to prove the far end.
225 /// Off (the default) sends byte-for-byte the payload this client always sent.
226 ///
227 /// A flag rather than always-on, deliberately: the material spends 137 bytes of the
228 /// payload's measured 4096-byte budget, and a caller near the ceiling with a launch env
229 /// deserves to choose which feature gets the room.
230 pub identity: bool,
231 /// Whether to request the egress connector. Off omits it from the request; measured
232 /// 2026-09-12 the platform gave a connector-less VM outbound network anyway
233 /// (`docs/PLATFORM.md`).
234 pub egress: bool,
235 /// Customer-managed Lambda VPC egress connector ARNs. See
236 /// [`RunMicrovmRequest::egress_network_connectors`].
237 pub egress_network_connectors: Vec<String>,
238 /// Sets advisory proxy-deny environment variables for clients that honor them.
239 /// Workloads can bypass these variables. Internet isolation requires a VPC egress
240 /// connector using a VPC without an internet gateway or NAT gateway.
241 ///
242 /// Refused with [`RunRequest::egress`]. Existing caller-supplied proxy variables win;
243 /// the added variables share the 4096-byte run-hook payload budget.
244 pub deny_egress: bool,
245 /// Whether to launch shell-capable: the ingress set becomes the measured pair
246 /// `[HTTP_INGRESS, SHELL_INGRESS]` instead of `ALL_INGRESS`, which is what
247 /// `CreateMicrovmShellAuthToken` requires and what `microvm shell` attaches to.
248 /// Off (the default) launches byte-for-byte what this client always sent.
249 pub shell: bool,
250 /// `idlePolicy.maxIdleDurationSeconds`.
251 pub max_idle_sec: u32,
252 /// `idlePolicy.suspendedDurationSeconds` — the window STATE-12 refuses past.
253 pub suspended_sec: u32,
254 /// `idlePolicy.autoResumeEnabled`.
255 pub auto_resume: bool,
256 /// `maximumDurationInSeconds`, checked against 1..=28800 before the call.
257 pub max_duration_sec: u32,
258 /// How long to wait for RUNNING.
259 pub ready_timeout: Duration,
260 /// A label for the run token (TRAP-1). Never the token.
261 pub token_scope: Option<String>,
262 /// Per-VM `logging` (#201). See [`RunMicrovmRequest::logging`].
263 pub logging: Option<crate::control::ops::Logging>,
264 /// Whether [`Sandbox::run`] waits for RUNNING before returning (the default).
265 ///
266 /// Off returns as soon as `RunMicrovm` is accepted, with the lifecycle still PENDING:
267 /// the session is addressable, and [`Sandbox::wait_until_running`] finishes the wait
268 /// later — from this process, or after a durable workflow's next step.
269 pub wait: bool,
270}
271
272impl Default for RunRequest {
273 fn default() -> Self {
274 Self {
275 image_identifier: None,
276 image_version: None,
277 execution_role_arn: None,
278 agent_token: None,
279 client_token: None,
280 launch_env: std::collections::HashMap::new(),
281 identity: false,
282 egress: false,
283 egress_network_connectors: Vec::new(),
284 deny_egress: false,
285 shell: false,
286 max_idle_sec: 600,
287 suspended_sec: 600,
288 auto_resume: false,
289 max_duration_sec: 3_600,
290 ready_timeout: DEFAULT_READY_TIMEOUT,
291 token_scope: None,
292 logging: None,
293 wait: true,
294 }
295 }
296}
297
298impl RunRequest {
299 /// A launch with the measured defaults: ingress only, ten-minute idle and suspended
300 /// windows, a one-hour maximum duration, no auto-resume.
301 pub fn new() -> Self {
302 Self::default()
303 }
304
305 /// Launches `identifier` rather than the built image.
306 #[must_use]
307 pub fn with_image(mut self, identifier: impl Into<String>) -> Self {
308 self.image_identifier = Some(identifier.into());
309 self
310 }
311
312 /// Requests the managed internet egress connector. Omission does not block internet access.
313 #[must_use]
314 pub fn with_egress(mut self) -> Self {
315 self.egress = true;
316 self
317 }
318
319 /// Routes egress through a customer-managed Lambda network connector.
320 #[must_use]
321 pub fn with_egress_network_connector(mut self, arn: impl Into<String>) -> Self {
322 self.egress_network_connectors.push(arn.into());
323 self
324 }
325
326 /// Replaces managed internet egress with customer-managed VPC connectors, when any are
327 /// given; an empty list leaves the request unchanged.
328 ///
329 /// The two cannot be combined, so a launch surface that requests managed egress by
330 /// default (an agent VM) uses this to switch rather than failing the launch.
331 #[must_use]
332 pub fn with_vpc_egress(mut self, arns: Vec<String>) -> Self {
333 if !arns.is_empty() {
334 self.egress = false;
335 self.egress_network_connectors = arns;
336 }
337 self
338 }
339
340 /// Applies the advisory in-guest deny. See [`RunRequest::deny_egress`] for what it is
341 /// and, more importantly, for what it is not.
342 #[must_use]
343 pub fn with_deny_egress(mut self) -> Self {
344 self.deny_egress = true;
345 self
346 }
347
348 /// What this launch's outbound network **is**, as opposed to what it asked for.
349 ///
350 /// The one derivation, so the CLI's envelope, the CLI's human line and any embedding
351 /// consumer cannot disagree about the same launch.
352 pub fn egress_posture(&self) -> crate::control::EgressPosture {
353 crate::control::EgressPosture::for_launch(self.egress, self.deny_egress)
354 }
355
356 /// The launch environment as the guest will see it: the caller's, plus the advisory
357 /// proxy deny when [`RunRequest::deny_egress`] is on.
358 ///
359 /// The caller's own value wins on a shared key. A launch that already routes through a
360 /// proxy of its own has said where its traffic goes, and overwriting that with a black
361 /// hole would break a working configuration in the name of a posture this client
362 /// already declines to call a seal.
363 pub fn effective_launch_env(&self) -> std::collections::HashMap<String, String> {
364 let mut env = self.launch_env.clone();
365 if self.deny_egress {
366 for key in DENY_EGRESS_ENV_KEYS {
367 env.entry(key.to_string())
368 .or_insert_with(|| DENY_EGRESS_PROXY_URL.to_string());
369 }
370 }
371 env
372 }
373
374 /// Requests a shell-capable launch. See [`RunRequest::shell`].
375 #[must_use]
376 pub fn with_shell(mut self) -> Self {
377 self.shell = true;
378 self
379 }
380
381 /// Pins the launch to one `imageVersion`. See the field for why.
382 #[must_use]
383 pub fn with_image_version(mut self, version: impl Into<String>) -> Self {
384 self.image_version = Some(version.into());
385 self
386 }
387
388 /// Sets the suspended window this sandbox will refuse a resume past (STATE-12).
389 #[must_use]
390 pub fn with_suspended_sec(mut self, seconds: u32) -> Self {
391 self.suspended_sec = seconds;
392 self
393 }
394
395 /// Reuses a caller-supplied agent token rather than minting one.
396 #[must_use]
397 pub fn with_agent_token(mut self, token: impl Into<String>) -> Self {
398 self.agent_token = Some(token.into());
399 self
400 }
401
402 /// Generates and delivers a tunnel identity with the launch (#70 layer 3).
403 ///
404 /// After a `run` with this set, [`Sandbox::tunnel_identity`] holds what
405 /// `tunnel --verify-identity` needs: the host's secret and the VM's public pin.
406 #[must_use]
407 pub fn with_identity(mut self) -> Self {
408 self.identity = true;
409 self
410 }
411
412 /// Adds one launch-environment variable, which every exec in the VM starts with.
413 ///
414 /// One pair per call rather than a whole map, because that is how a caller builds
415 /// one — from flags, from a config file, one credential at a time — and a
416 /// map-taking setter makes the second call silently discard the first. The field is
417 /// public for a caller who really does hold a map.
418 #[must_use]
419 pub fn with_launch_env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
420 self.launch_env.insert(key.into(), value.into());
421 self
422 }
423}
424
425/// What a teardown should delete beyond the VM itself.
426///
427/// Both deletions are opt-in, because both destroy something a caller may still want: the
428/// image is reusable across runs, and the log group is where a failed build's only evidence
429/// lives.
430#[derive(Clone, Copy, Debug)]
431pub struct TeardownOpts {
432 /// Whether to delete the image the sandbox built.
433 pub delete_image: bool,
434 /// Whether the build log group should be deleted.
435 ///
436 /// This crate **cannot** delete it — CloudWatch is not in the dependency set — so
437 /// asking names the group in [`TeardownReport::undeleted`] rather than removing it.
438 /// See that field for why naming is the honest answer rather than a silent success.
439 pub delete_log_group: bool,
440 /// How many times the image delete is retried.
441 pub delete_attempts: u32,
442 /// The gap between image-delete attempts.
443 pub delete_backoff: Duration,
444 /// How long to wait for TERMINATED, or `None` to return as soon as the terminate call
445 /// is accepted.
446 ///
447 /// `None` by default, matching the Python client: the caller is on the way out, and a
448 /// teardown that blocked five minutes on a state nobody reads is five minutes of a CI
449 /// job. A caller that needs STATE-10 *observed* passes
450 /// [`TeardownOpts::waiting_for_terminated`].
451 pub wait_for_terminated: Option<Duration>,
452}
453
454impl Default for TeardownOpts {
455 fn default() -> Self {
456 Self {
457 delete_image: false,
458 delete_log_group: false,
459 delete_attempts: DEFAULT_DELETE_ATTEMPTS,
460 delete_backoff: DEFAULT_DELETE_BACKOFF,
461 wait_for_terminated: None,
462 }
463 }
464}
465
466impl TeardownOpts {
467 /// Deletes the image as well as the VM.
468 #[must_use]
469 pub fn deleting_image(mut self) -> Self {
470 self.delete_image = true;
471 self
472 }
473
474 /// Asks for the log group too, which names it rather than deleting it.
475 #[must_use]
476 pub fn deleting_log_group(mut self) -> Self {
477 self.delete_log_group = true;
478 self
479 }
480
481 /// Waits for TERMINATED before returning (STATE-10).
482 #[must_use]
483 pub fn waiting_for_terminated(mut self) -> Self {
484 self.wait_for_terminated = Some(DEFAULT_LIFECYCLE_TIMEOUT);
485 self
486 }
487}
488
489/// What a teardown did, and what it left behind.
490///
491/// Returned rather than raised. See the module docs: this runs where a `finally` would.
492#[derive(Clone, Debug, Default, Eq, PartialEq)]
493pub struct TeardownReport {
494 /// Identifiers of everything a caller asked to have deleted that still exists.
495 ///
496 /// The CLI emits these (CLI-6), which is the whole reason they are identifiers rather
497 /// than a boolean: a leak nobody can name is a leak nobody can clean up.
498 ///
499 /// Two things land here. A delete that was attempted and failed — an image whose twenty
500 /// attempts all hit a conflict. And the build **log group**, which this crate cannot
501 /// delete at all: no CloudWatch client exists anywhere in this workspace, and a logs
502 /// API for one delete call is a bigger surface than the leak it would hide (an
503 /// `aws-sdk-cloudwatchlogs` edge is a reasonable future add if teardown grows more
504 /// log-group work). Naming it is strictly better than the
505 /// alternatives: deleting it is unavailable, and silently succeeding would report a
506 /// clean teardown over six accumulated log groups — which is how the leak was found in
507 /// the first place.
508 pub undeleted: Vec<String>,
509 /// Whether the terminate call was accepted.
510 pub terminate_accepted: bool,
511 /// Whether the image was deleted, or `None` when deletion was not asked for.
512 pub image_deleted: Option<bool>,
513 /// The lifecycle state the sandbox ended in.
514 pub lifecycle: Option<Lifecycle>,
515 /// Every failure the teardown swallowed, in the order it hit them.
516 ///
517 /// Kept because a teardown that never raises is a teardown whose failures are invisible
518 /// otherwise, and the first one is usually the cause of the rest.
519 pub failures: Vec<String>,
520}
521
522impl TeardownReport {
523 /// Whether anything a caller asked for was left behind.
524 pub fn leaked(&self) -> bool {
525 !self.undeleted.is_empty()
526 }
527}
528
529/// Mints proxy tokens for one MicroVM through the control plane.
530///
531/// The bridge between the two lanes: `ControlPlane::mint_auth_token` answers a
532/// `control::ProxyToken`, [`TokenMinter`] wants a `session::ProxyToken`, and the `From`
533/// impl at `session/proxy.rs` is the conversion — one `.into()` at the boundary, which is
534/// why that impl is `From` rather than a named function.
535///
536/// The plane sits behind an `Arc` rather than a reference because the minter must outlive
537/// the call that built the session: minting happens inside the request path on every later
538/// request, which is what makes it happen at all (TRAP-9).
539pub(crate) struct ControlPlaneMinter {
540 pub(crate) control: Arc<ControlPlane>,
541 pub(crate) microvm_id: String,
542}
543
544impl TokenMinter for ControlPlaneMinter {
545 fn mint(
546 &self,
547 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
548 Box::pin(async move {
549 let minted = self.control.mint_auth_token(&self.microvm_id).await?;
550 Ok(minted.into())
551 })
552 }
553
554 /// Overrides the default, which ignores the ports and delegates.
555 ///
556 /// This is the minter that has a control plane behind it, so it is the one that can
557 /// actually widen a token's scope — and without this override
558 /// `Session::connect_headers(8080)` would keep answering a header pair behind a token
559 /// scoped to 9000 only, which the proxy refuses with 403 `Access to port denied`. See
560 /// [`TokenMinter::mint_for_ports`] for the measurement.
561 fn mint_for_ports(
562 &self,
563 ports: &[u16],
564 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
565 let specs: Vec<crate::control::ops::PortSpecification> = ports
566 .iter()
567 .map(|port| crate::control::ops::PortSpecification::port(*port))
568 .collect();
569 Box::pin(async move {
570 let minted = self
571 .control
572 .mint_auth_token_for(&self.microvm_id, &specs)
573 .await?;
574 Ok(minted.into())
575 })
576 }
577}
578
579/// One MicroVM's whole life.
580///
581/// See the module docs for the state machine, the window, and why teardown is explicit.
582pub struct Sandbox {
583 control: Arc<ControlPlane>,
584 image: Option<Image>,
585 microvm: Option<Microvm>,
586 session: Option<Session>,
587 /// A backend for the session [`Sandbox::run`] builds, or `None` for the real HTTP one.
588 ///
589 /// The daemon-side sibling of `with_control_plane`'s transport seam: a test that
590 /// scripts the control plane can launch, but the session `run` then builds would dial
591 /// a real endpoint. This lets the same test script the daemon too. Never set in
592 /// production — `Session::builder` picks reqwest when no backend is given.
593 session_backend: Option<crate::session::SharedBackend>,
594
595 // ── the symspec's five variables ─────────────────────────────────────────
596 lifecycle: Lifecycle,
597 /// Every transition of `lifecycle`, for a reader that must not take the caller's lock
598 /// (a keepalive task running while the same sandbox is busy inside a long exec).
599 lifecycle_watch: tokio::sync::watch::Sender<Lifecycle>,
600 token_installed: bool,
601 image_exists: bool,
602 was_terminated: bool,
603 bootstrap_count: u32,
604
605 /// The window from *our own* `RunMicrovm` request. `GetMicrovm` also reports it (in
606 /// `idlePolicy`), which is the fallback when this sandbox has no request of its own.
607 suspended_window: Option<Duration>,
608 /// Whether the accepted launch carried a caller client token, and so may have adopted
609 /// a VM an earlier attempt launched (#195).
610 launch_adoptable: bool,
611 /// The launch's agent token, held until the session that carries it is built. Kept out
612 /// of `Debug` like every other credential here.
613 pending_agent_token: Option<String>,
614 /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence
615 /// is checked against it.
616 idle_window: Option<Duration>,
617 /// The clock reading when the suspend call was accepted, or `None` when not suspended.
618 suspended_at: Option<Duration>,
619 /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down
620 /// one.
621 torn_down: bool,
622 /// Built by [`Sandbox::adopt`] around a VM another process launched. That process owns
623 /// the VM's teardown, so dropping this handle is not a leak and `Drop` stays quiet.
624 adopted: bool,
625 /// The launch's tunnel identity, when [`RunRequest::identity`] asked for one.
626 ///
627 /// Holds the host secret and the VM's *public* pin — the VM's own secret was dropped
628 /// the moment the payload was built ([`crate::identity::LaunchIdentity::keep`]), so
629 /// nothing on the host side can impersonate the VM it launched.
630 tunnel_identity: Option<crate::identity::TunnelIdentity>,
631}
632
633impl std::fmt::Debug for Sandbox {
634 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
635 // No agent token: it is a credential, and a `Debug` that printed it would put it in
636 // every log line that formats a sandbox.
637 f.debug_struct("Sandbox")
638 .field("lifecycle", &self.lifecycle)
639 .field("microvm", &self.microvm.as_ref().map(|vm| &vm.id))
640 .field("image", &self.image.as_ref().map(|image| &image.identifier))
641 .field("token_installed", &self.token_installed)
642 .field("bootstrap_count", &self.bootstrap_count)
643 .field("was_terminated", &self.was_terminated)
644 .field("suspended_window", &self.suspended_window)
645 .field("adopted", &self.adopted)
646 .finish_non_exhaustive()
647 }
648}
649
650impl Sandbox {
651 /// Resolves credentials for `region` and returns a sandbox with nothing launched.
652 ///
653 /// TRAP-6 is closed by [`Region`] before this is reached, which is why there is no
654 /// region check here.
655 pub async fn new(region: Region) -> Result<Self, Error> {
656 Ok(Self::with_control_plane(ControlPlane::new(region).await?))
657 }
658
659 /// A sandbox over an already-built [`ControlPlane`].
660 ///
661 /// The seam every test uses, and public for the same reason `ControlPlane::with_transport`
662 /// is: a caller who wants a non-default port, or a fake transport, has already built the
663 /// plane, and a sandbox constructible only from a [`Region`] would be a sandbox no test
664 /// can drive without AWS.
665 ///
666 /// The plane's own clock is what times the suspended window, rather than a second one
667 /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil
668 /// tests record: a window measured on one clock while the poll loop sleeps on another
669 /// calls a closed window open.
670 pub fn with_control_plane(control: ControlPlane) -> Self {
671 Self {
672 control: Arc::new(control),
673 image: None,
674 microvm: None,
675 session: None,
676 session_backend: None,
677 lifecycle: Lifecycle::Pending,
678 lifecycle_watch: tokio::sync::watch::Sender::new(Lifecycle::Pending),
679 token_installed: false,
680 image_exists: false,
681 was_terminated: false,
682 bootstrap_count: 0,
683 suspended_window: None,
684 launch_adoptable: false,
685 pending_agent_token: None,
686 idle_window: None,
687 suspended_at: None,
688 torn_down: false,
689 adopted: false,
690 tunnel_identity: None,
691 }
692 }
693
694 /// Routes the session [`Sandbox::run`] builds through `backend` instead of real HTTP.
695 ///
696 /// The daemon-side half of the test seam `with_control_plane` opens: scripting the
697 /// control plane gets a test through the launch, and this gets it through everything
698 /// the launched session then does — uploads, execs, downloads — without a daemon.
699 pub fn with_session_backend(mut self, backend: crate::session::SharedBackend) -> Self {
700 self.session_backend = Some(backend);
701 self
702 }
703
704 // ── the symspec's five variables, readable ───────────────────────────────
705
706 /// The lifecycle state, which is the symspec's `vm_state`.
707 pub fn lifecycle(&self) -> Lifecycle {
708 self.lifecycle
709 }
710
711 /// Whether the agent token has been installed (STATE-2).
712 pub fn token_installed(&self) -> bool {
713 self.token_installed
714 }
715
716 /// Whether an image is recorded as existing (STATE-1).
717 pub fn image_exists(&self) -> bool {
718 self.image_exists
719 }
720
721 /// Whether this VM was ever terminated (STATE-11).
722 pub fn was_terminated(&self) -> bool {
723 self.was_terminated
724 }
725
726 /// How many times the token has been installed. Never above one (STATE-3).
727 pub fn bootstrap_count(&self) -> u32 {
728 self.bootstrap_count
729 }
730
731 /// The image, once built.
732 pub fn image(&self) -> Option<&Image> {
733 self.image.as_ref()
734 }
735
736 /// The VM as the service last described it.
737 pub fn microvm(&self) -> Option<&Microvm> {
738 self.microvm.as_ref()
739 }
740
741 /// The session, once launched.
742 pub fn session(&self) -> Option<&Session> {
743 self.session.as_ref()
744 }
745
746 /// The suspended window this sandbox asked for at launch, once it has launched.
747 pub fn suspended_window(&self) -> Option<Duration> {
748 self.suspended_window
749 }
750
751 /// A receiver that sees every lifecycle transition without taking this sandbox's lock.
752 ///
753 /// A binding's keepalive reads it between polls: a sandbox busy inside a long exec
754 /// holds the lock the whole time, which is exactly when the keepalive must keep polling,
755 /// and a suspend or terminate must end the keepalive before its next poll auto-resumes
756 /// the VM.
757 pub fn watch_lifecycle(&self) -> tokio::sync::watch::Receiver<Lifecycle> {
758 self.lifecycle_watch.subscribe()
759 }
760
761 fn set_lifecycle(&mut self, lifecycle: Lifecycle) {
762 self.lifecycle = lifecycle;
763 self.lifecycle_watch.send_replace(lifecycle);
764 }
765
766 /// The idle window this sandbox asked for at launch, once it has launched.
767 pub fn idle_window(&self) -> Option<Duration> {
768 self.idle_window
769 }
770
771 /// The tunnel identity, when the launch asked for one ([`RunRequest::identity`]).
772 ///
773 /// What `tunnel --verify-identity` verifies with: the host's secret and the VM's public
774 /// pin. `None` on a launch that did not ask, and the tunnel route then refuses an
775 /// identity request with close code 4401 rather than downgrading.
776 pub fn tunnel_identity(&self) -> Option<&crate::identity::TunnelIdentity> {
777 self.tunnel_identity.as_ref()
778 }
779
780 /// The agent port the control plane was built with: the hooks port on every image
781 /// this sandbox builds, and the port the daemon listens on in every VM it launches.
782 /// Read-only; `agents::AgentVm` needs it to derive a Dockerfile whose `AGENTD_PORT`
783 /// agrees with the launch.
784 pub fn port(&self) -> u16 {
785 self.control.port()
786 }
787
788 // ── build ────────────────────────────────────────────────────────────────
789
790 /// Builds an image and waits for it to become usable.
791 ///
792 /// Every local guard runs inside [`ControlPlane::create_image`], before the call —
793 /// which matters because the create happens *after* the caller's artifact upload, so a
794 /// rejection AWS raises costs the upload first.
795 pub async fn build_image(&mut self, request: CreateImageRequest) -> Result<&Image, Error> {
796 let size = request.size;
797 let created = self.control.create_image(request).await?;
798 let mut built = self
799 .control
800 .wait_for_image(&created.identifier, size, WaitOpts::default())
801 .await?;
802 // The logging config survives the wait from the create's own record, because the
803 // readback cannot carry it: `GetMicrovmImage` reports no logging member, and the
804 // resolved log stream's discriminator was minted inside `create_image` — this is
805 // the only copy, and dropping it here would leave the caller unable to name the
806 // stream their build wrote to.
807 built.log_group = created.log_group;
808 built.log_stream = created.log_stream;
809 // Recorded before any launch, because a built image exists whether or not anything
810 // is ever run from it — and the teardown has to be able to name it either way.
811 self.image_exists = true;
812 self.image = Some(built);
813 Ok(self.image.as_ref().expect("just assigned"))
814 }
815
816 /// The artifact bytes to upload to the request's `code_artifact_uri`.
817 ///
818 /// The upload is the caller's: S3 is not in this crate's dependency set.
819 pub fn build_artifact_for(&self, request: &CreateImageRequest) -> Result<Vec<u8>, Error> {
820 self.control.build_artifact_for(request)
821 }
822
823 /// Every local guard [`Self::build_image`] runs, callable **before** the artifact upload.
824 ///
825 /// A delegation to [`ControlPlane::preflight`], on this type so the CLI's build path
826 /// reaches it through the one sandbox it already holds. Local; zero calls. The point is
827 /// ordering: `build_image`'s guards run after the caller's upload, so a request this
828 /// client itself refuses would still cost the S3 PUT unless the caller checks first.
829 pub fn preflight(&self, request: &CreateImageRequest) -> Result<(), Error> {
830 self.control.preflight(request)
831 }
832
833 /// The ARN for `identifier`: an ARN passes through with zero calls, a bare name is
834 /// resolved through the image listing by exact match.
835 ///
836 /// A read-only delegation to [`ControlPlane::resolve_image_arn`], on this type so the
837 /// CLI's launch path reaches it through the one sandbox it already holds. It touches
838 /// none of the five state-machine variables — resolution is a question about the
839 /// account, not about this VM's lifecycle.
840 pub async fn resolve_image_arn(&self, identifier: &str) -> Result<String, Error> {
841 self.control.resolve_image_arn(identifier).await
842 }
843
844 /// The image with exactly `name`, or `None`. Read-only; see
845 /// [`ControlPlane::find_image_by_name`] for the pagination and exact-match rules.
846 pub async fn find_image_by_name(
847 &self,
848 name: &str,
849 ) -> Result<Option<crate::control::ops::MicrovmImageSummaryWire>, Error> {
850 self.control.find_image_by_name(name).await
851 }
852
853 /// The content hash `build --reuse` keys an image name to. Local; zero calls.
854 pub fn artifact_content_hash_for(&self, request: &CreateImageRequest) -> String {
855 self.control.artifact_content_hash_for(request)
856 }
857
858 /// The versions of a managed base image, for pinning
859 /// [`CreateImageRequest::base_image_version`].
860 ///
861 /// A read-only delegation to [`crate::control::ControlPlane::managed_base_versions`], on
862 /// this type for the reason [`Self::resolve_image_arn`] gives: it touches none of the five
863 /// state-machine variables, because a question about AWS's published bases is not a
864 /// question about this VM's lifecycle.
865 pub async fn managed_base_versions(
866 &self,
867 base_image_arn: &str,
868 ) -> Result<Vec<crate::control::ops::ManagedMicrovmImageVersionWire>, Error> {
869 self.control.managed_base_versions(base_image_arn).await
870 }
871
872 /// One version's configuration, state, and availability status. Read-only.
873 pub async fn image_version(
874 &self,
875 identifier: &str,
876 version: &str,
877 ) -> Result<crate::control::ops::MicrovmImageVersionSummaryWire, Error> {
878 self.control.get_image_version(identifier, version).await
879 }
880
881 // ── run (STATE-1, STATE-2, STATE-3) ──────────────────────────────────────
882
883 /// Launches a MicroVM, waits for RUNNING, and returns its session.
884 ///
885 /// # The three state requirements this is
886 ///
887 /// STATE-1: the accepted launch moves the lifecycle to PENDING and records the image as
888 /// existing. STATE-2: the platform reporting RUNNING is what marks the token installed
889 /// — not the launch call, because the run hook is what delivers it and a launch that
890 /// dies during startup delivered nothing. STATE-3: `bootstrap_count` is incremented
891 /// exactly here, and a second `run` on the same sandbox is refused, which is what makes
892 /// "at most once per VM lifetime" a property of the type rather than of a caller's
893 /// discipline.
894 ///
895 /// The agent token rides in `runHookPayload`, which is what keeps it out of the shared
896 /// image snapshot. That is safe because the platform forwards no external traffic until
897 /// the run hook returns 200, so a per-VM secret delivered at launch wins the
898 /// first-writer race through the endpoint.
899 pub async fn run(&mut self, request: RunRequest) -> Result<&mut Session, Error> {
900 // STATE-3's local half. A sandbox that has already bootstrapped cannot bootstrap
901 // again, and the refusal is here rather than in a comment because `run` twice is
902 // the plausible mistake — a retry loop around a launch that timed out.
903 if self.bootstrap_count > 0 || self.microvm.is_some() {
904 return Err(Error::invalid_arg(format!(
905 "this sandbox has already launched a VM ({} bootstrap(s), lifecycle {}), and the \
906 agent token is installed at most once per VM lifetime (STATE-3). A second VM \
907 needs a second Sandbox — reusing this one would either re-deliver a run-hook \
908 payload to a daemon whose one-shot bootstrap refuses it, or silently address \
909 two guests through one handle.",
910 self.bootstrap_count, self.lifecycle,
911 )));
912 }
913
914 // Opposite intents, refused before the call rather than resolved by a precedence
915 // rule nobody would find: `--egress` asks the platform for outbound network and
916 // `--deny-egress` asks the guest's own clients to refuse it, so a launch carrying
917 // both would report `open` while its workload's tools failed closed.
918 if request.egress && request.deny_egress {
919 return Err(Error::invalid_arg(
920 "egress and deny_egress ask for opposite things: egress puts the \
921 INTERNET_EGRESS connector on the launch, and deny_egress sets the guest's \
922 proxy variables to a black hole so a well-behaved client refuses to leave \
923 the VM. Pick one. Neither seals the VM — omitting the connector is measured \
924 not to (docs/PLATFORM.md), and the deny is advisory."
925 .to_string(),
926 ));
927 }
928
929 let Some(identifier) = request
930 .image_identifier
931 .clone()
932 .or_else(|| self.image.as_ref().map(|image| image.identifier.clone()))
933 else {
934 return Err(Error::new(
935 ErrorKind::Precondition,
936 "no image to launch: pass RunRequest::with_image or call build_image first."
937 .to_string(),
938 ));
939 };
940
941 if request.client_token.is_some() && (request.agent_token.is_none() || request.identity) {
942 return Err(Error::invalid_arg(
943 "a stable client_token requires an explicit agent_token and identity=false; persist identical launch parameters before retrying",
944 ));
945 }
946 let agent_token = request.agent_token.clone().unwrap_or_else(mint_agent_token);
947 // Generated before the payload because the payload carries its two public-facing
948 // fields. The VM's secret half lives exactly as long as this binding: `keep()` below
949 // drops it before the launch call returns.
950 let launch_identity = if request.identity {
951 Some(crate::identity::LaunchIdentity::generate()?)
952 } else {
953 None
954 };
955 // Checked even though this builds the JSON itself, because neither half is ours:
956 // the token may be caller-supplied, so someone passing a signed blob rather than a
957 // bearer token is exactly who this catches (TRAP-5), and the launch env is entirely
958 // the caller's. This is the pre-flight refusal — over-ceiling fails here with the
959 // byte count, before the launch, rather than as a `ValidationException` on a member
960 // the caller did not know they were filling.
961 let payload = RunHookPayload::for_launch_with_identity(
962 &agent_token,
963 &request.effective_launch_env(),
964 launch_identity.as_ref(),
965 )?;
966
967 let mut wire = RunMicrovmRequest::new(&identifier, payload);
968 wire.image_version = request.image_version.clone();
969 wire.execution_role_arn = request.execution_role_arn.clone();
970 wire.egress_network_connectors = request.egress_network_connectors.clone();
971 wire.max_idle_sec = request.max_idle_sec;
972 wire.suspended_sec = request.suspended_sec;
973 wire.auto_resume = request.auto_resume;
974 wire.max_duration_sec = request.max_duration_sec;
975 wire.token_scope = request.token_scope.clone();
976 wire.client_token = request.client_token.clone();
977 wire.logging = request.logging.clone();
978 if request.egress {
979 wire = wire.with_egress();
980 }
981 if request.shell {
982 wire = wire.with_shell();
983 }
984
985 let launched = self.control.run_microvm(wire).await?;
986
987 // The durable half of the identity, kept the moment the launch is accepted: the host
988 // secret and the VM's public pin. The VM's own secret ends here — `keep` drops it —
989 // so from this line on the host can verify its VM and impersonate it never.
990 self.tunnel_identity = launch_identity.map(crate::identity::LaunchIdentity::keep);
991
992 // STATE-1: the launch was accepted.
993 self.set_lifecycle(Lifecycle::Pending);
994 self.image_exists = true;
995 // Recorded here rather than after the wait, because the window the idlePolicy
996 // enforces was set by *this* request and a launch that then fails still leaves a VM
997 // the caller may have to reason about.
998 self.suspended_window = Some(Duration::from_secs(u64::from(request.suspended_sec)));
999 self.idle_window = Some(Duration::from_secs(u64::from(request.max_idle_sec)));
1000 self.microvm = Some(launched);
1001 // A caller-supplied client token can adopt the VM an earlier attempt launched, which
1002 // may have idle-suspended since (#195); a minted token always launches afresh.
1003 self.launch_adoptable = request.client_token.is_some();
1004 self.pending_agent_token = Some(agent_token);
1005
1006 if request.wait {
1007 return self.wait_until_running(request.ready_timeout).await;
1008 }
1009 // Not waiting: the endpoint is in the launch reply, so the session is addressable
1010 // now, and a daemon request before RUNNING fails like any other unready request.
1011 self.build_session()?;
1012 Ok(self.session.as_mut().expect("just assigned"))
1013 }
1014
1015 /// The session for the launched VM, from its endpoint and the launch's agent token.
1016 fn build_session(&mut self) -> Result<(), Error> {
1017 let (Some(vm), Some(agent_token)) = (&self.microvm, &self.pending_agent_token) else {
1018 return Err(Error::new(
1019 ErrorKind::Precondition,
1020 "no launch to build a session for",
1021 ));
1022 };
1023 let minter = Arc::new(ControlPlaneMinter {
1024 control: Arc::clone(&self.control),
1025 microvm_id: vm.id.clone(),
1026 });
1027 let mut builder = Session::builder(vm.endpoint.clone(), agent_token.clone())
1028 .with_minter(minter)
1029 .with_port(self.control.port());
1030 if let Some(backend) = &self.session_backend {
1031 builder = builder.with_backend(Arc::clone(backend));
1032 }
1033 self.session = Some(builder.build()?);
1034 Ok(())
1035 }
1036
1037 /// Waits for a launch accepted by [`Sandbox::run`] to reach RUNNING.
1038 ///
1039 /// `run` calls this itself unless [`RunRequest::wait`] was off. A launch that adopted an
1040 /// existing VM through its client token resumes it if it idle-suspended (#195); a fresh
1041 /// launch that reaches any terminal state first fails fast with `stateReason` (TRAP-8).
1042 pub async fn wait_until_running(&mut self, timeout: Duration) -> Result<&mut Session, Error> {
1043 let id = self.require_microvm("wait_until_running")?;
1044 if self.lifecycle != Lifecycle::Pending {
1045 return Err(Error::invalid_arg(format!(
1046 "microvm {id} is {}; wait_until_running finishes a launch that is still PENDING",
1047 self.lifecycle,
1048 )));
1049 }
1050 let opts = WaitOpts {
1051 timeout,
1052 poll_interval: LIFECYCLE_POLL_INTERVAL,
1053 stall_grace: Duration::MAX,
1054 };
1055 let running = if self.launch_adoptable {
1056 self.control.wait_for_launch(&id, opts).await?
1057 } else {
1058 self.control.wait_for_running(&id, opts).await?
1059 };
1060
1061 // STATE-2. The platform reported the run hook succeeded, so the token is in the
1062 // guest's memory — and this is the one place that counts it (STATE-3).
1063 self.set_lifecycle(Lifecycle::Running);
1064 self.token_installed = true;
1065 self.bootstrap_count += 1;
1066 self.microvm = Some(running);
1067 if self.session.is_none() {
1068 self.build_session()?;
1069 }
1070 Ok(self.session.as_mut().expect("just assigned"))
1071 }
1072
1073 // ── adopt (STATE-3, with the lifecycle read from the service) ────────────
1074
1075 /// A sandbox for a VM another process launched, rebuilt from its private record.
1076 ///
1077 /// The durable-workflow shape: every step runs in a fresh process, and each one needs
1078 /// suspend, resume, and terminate, not just the exec and file surface
1079 /// [`Session::attach`] gives. The lifecycle is read from `GetMicrovm` rather than
1080 /// assumed, so every guard below starts from what the service reports.
1081 ///
1082 /// # Bootstrap is counted, never repeated (STATE-3)
1083 ///
1084 /// The VM was bootstrapped by the launch that created it, so an adopted sandbox refuses
1085 /// [`Sandbox::run`] and never sends a run-hook payload. A VM adopted while still PENDING
1086 /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once
1087 /// when the service reports RUNNING, the same as a launch this sandbox made itself.
1088 ///
1089 /// # The suspended window (STATE-12)
1090 ///
1091 /// The window comes from the `idlePolicy` `GetMicrovm` reports. A VM adopted while
1092 /// already SUSPENDED has no suspend time this client observed, so the local check has
1093 /// nothing to measure and the service answers the resume instead.
1094 ///
1095 /// `endpoint` must agree with the service's when the service reports one: a mismatch
1096 /// means the identifiers came from two different records. `agent_token` is the bearer
1097 /// credential the VM was launched with; it never appears in `Debug` or an error.
1098 pub async fn adopt(
1099 control: ControlPlane,
1100 microvm_id: impl Into<String>,
1101 endpoint: impl Into<String>,
1102 agent_token: impl Into<String>,
1103 ) -> Result<Self, Error> {
1104 let microvm_id = microvm_id.into();
1105 let endpoint = endpoint.into();
1106 let agent_token = agent_token.into();
1107 if agent_token.is_empty() {
1108 return Err(Error::invalid_arg(
1109 "adopt needs the agent token the VM was launched with: its daemon accepts only \
1110 that bearer, and adopting without it would build a handle every exec refuses.",
1111 ));
1112 }
1113 let mut vm = control.get_microvm(µvm_id).await?;
1114 if !endpoint.is_empty() && !vm.endpoint.is_empty() && vm.endpoint != endpoint {
1115 return Err(Error::invalid_arg(format!(
1116 "microvm {microvm_id} reports endpoint {}, not {endpoint}: the id and endpoint \
1117 came from different records, and a session built from them would address one \
1118 VM while its lifecycle calls reached another.",
1119 vm.endpoint,
1120 )));
1121 }
1122 if vm.endpoint.is_empty() {
1123 vm.endpoint = endpoint;
1124 }
1125 let Some(lifecycle) = Lifecycle::from_service(&vm.state) else {
1126 return Err(Error::new(
1127 ErrorKind::Platform,
1128 format!(
1129 "microvm {microvm_id} reports state {}, which this client does not know; \
1130 refusing to guess which lifecycle guards apply.",
1131 vm.state,
1132 ),
1133 ));
1134 };
1135
1136 let mut sandbox = Self::with_control_plane(control);
1137 sandbox.adopted = true;
1138 sandbox.set_lifecycle(lifecycle);
1139 // The window the VM was launched with, so a keepalive on the adopted session paces
1140 // itself against the real idle policy rather than the platform minimum.
1141 sandbox.idle_window = vm
1142 .idle_policy
1143 .as_ref()
1144 .map(|policy| Duration::from_secs(u64::from(policy.max_idle_duration_seconds)));
1145 // A VM exists, so the image it was launched from did (STATE-1).
1146 sandbox.image_exists = true;
1147 // PENDING may still idle-suspend before this handle sees RUNNING (#195).
1148 sandbox.launch_adoptable = true;
1149 sandbox.pending_agent_token = Some(agent_token);
1150 match lifecycle {
1151 Lifecycle::Pending => {}
1152 Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended => {
1153 sandbox.token_installed = true;
1154 sandbox.bootstrap_count = 1;
1155 }
1156 Lifecycle::Terminating | Lifecycle::Terminated => {
1157 sandbox.bootstrap_count = 1;
1158 sandbox.was_terminated = true;
1159 }
1160 }
1161 sandbox.microvm = Some(vm);
1162 if lifecycle.is_live() {
1163 sandbox.build_session()?;
1164 }
1165 Ok(sandbox)
1166 }
1167
1168 /// [`Sandbox::adopt`] over a plane resolved for `region`, on `port` when the image's
1169 /// daemon listens somewhere other than the default. The bindings' entry point.
1170 pub async fn adopt_in(
1171 region: Region,
1172 microvm_id: impl Into<String>,
1173 endpoint: impl Into<String>,
1174 agent_token: impl Into<String>,
1175 port: Option<u16>,
1176 ) -> Result<Self, Error> {
1177 Self::adopt(
1178 Self::plane_for(region, port).await?,
1179 microvm_id,
1180 endpoint,
1181 agent_token,
1182 )
1183 .await
1184 }
1185
1186 /// A [`crate::names::NameRecord`] for this VM, to register under `name`.
1187 ///
1188 /// The egress posture is left unknown: the sandbox does not keep its launch request, and
1189 /// a record must not claim a network it cannot vouch for.
1190 pub fn name_record(&self, name: &str) -> Result<crate::names::NameRecord, Error> {
1191 let Some(vm) = self.microvm() else {
1192 return Err(Error::new(
1193 ErrorKind::Precondition,
1194 "there is no VM to name yet: launch or adopt one first",
1195 ));
1196 };
1197 let Some(session) = self.session() else {
1198 return Err(Error::new(
1199 ErrorKind::Precondition,
1200 format!(
1201 "microvm {} has no session to take its agent token from",
1202 vm.id
1203 ),
1204 ));
1205 };
1206 crate::names::NameRecord::new(
1207 name,
1208 vm.id.as_str(),
1209 vm.endpoint.as_str(),
1210 session.agent_token(),
1211 self.control.region().as_str(),
1212 )
1213 }
1214
1215 /// Adopts the VM registered as `name` in `store`: [`Sandbox::adopt`] from a record.
1216 ///
1217 /// `region`, when given, must match the record's; see [`crate::names::resolve`].
1218 pub async fn from_name(
1219 store: &dyn crate::names::NameStore,
1220 name: &str,
1221 region: Option<Region>,
1222 port: Option<u16>,
1223 ) -> Result<Self, Error> {
1224 let record = crate::names::resolve(store, name, region.as_ref())?;
1225 let control = Self::plane_for(record.region(), port).await?;
1226 Self::adopt_record(control, record).await
1227 }
1228
1229 /// [`Sandbox::adopt`] from a [`crate::names::NameRecord`] kept in any store.
1230 pub async fn adopt_record(
1231 control: ControlPlane,
1232 record: crate::names::NameRecord,
1233 ) -> Result<Self, Error> {
1234 Self::adopt(
1235 control,
1236 record.microvm_id,
1237 record.endpoint,
1238 record.agent_token,
1239 )
1240 .await
1241 }
1242
1243 /// A control plane for `region`, on `port` when one is given.
1244 pub(crate) async fn plane_for(
1245 region: Region,
1246 port: Option<u16>,
1247 ) -> Result<ControlPlane, Error> {
1248 let control = ControlPlane::new(region).await?;
1249 match port {
1250 Some(port) => control.with_port(port),
1251 None => Ok(control),
1252 }
1253 }
1254
1255 /// Whether this sandbox was built by [`Sandbox::adopt`] rather than by its own launch.
1256 pub fn adopted(&self) -> bool {
1257 self.adopted
1258 }
1259
1260 // ── suspend (STATE-4, STATE-5, STATE-6) ──────────────────────────────────
1261
1262 /// Freezes the VM and waits for the platform to report it.
1263 ///
1264 /// A freeze and restore rather than a stop and start: the guest keeps its memory, so the
1265 /// token, the filesystem, and every exec record survive. The one thing that does not is
1266 /// the guest's view of time — it observes the whole suspension as a single jump, so any
1267 /// timeout, lease, or TLS session a running command holds expires at once on resume.
1268 ///
1269 /// # STATE-5 is checked before the wire, not after
1270 ///
1271 /// A suspend from anything but RUNNING is refused here with **zero** control-plane
1272 /// calls. That is the observable difference between this and a client that lets AWS
1273 /// answer, and it is what the test asserts on.
1274 pub async fn suspend(&mut self) -> Result<(), Error> {
1275 let id = self.require_microvm("suspend")?;
1276
1277 // STATE-5.
1278 if self.lifecycle != Lifecycle::Running {
1279 return Err(Error::invalid_arg(format!(
1280 "microvm {id} is {} and a suspend is only issued from RUNNING (STATE-5). Refused \
1281 here rather than by the service, because the service's answer about a \
1282 non-running id does not say which of the two things went wrong — and a suspend \
1283 issued from SUSPENDED is a caller who believes they resumed.",
1284 self.lifecycle,
1285 )));
1286 }
1287
1288 // STATE-4: accepted, so the lifecycle moves before the wait. Acceptance is the
1289 // wire call succeeding, so the assignment comes after it: moving to SUSPENDING
1290 // first would leave a failed call (a throttle, a dead transport) stuck in a state
1291 // neither suspend nor resume accepts, bricking the handle over one bad request.
1292 self.control.suspend(&id).await?;
1293 self.set_lifecycle(Lifecycle::Suspending);
1294 // Stamped after the call and before the wait, not after the wait: the idlePolicy's
1295 // window starts when the platform begins suspending, so timing it from SUSPENDED
1296 // would under-count the transition and call a closed window open.
1297 self.suspended_at = Some(self.control.clock().elapsed());
1298
1299 let settled = self
1300 .control
1301 .wait_for_state(
1302 &id,
1303 &crate::control::microvm::SUSPEND_WANTED,
1304 &[],
1305 self.lifecycle_wait(),
1306 )
1307 .await?;
1308
1309 // STATE-6, and the TERMINATED case beside it. A VM that dies while suspending is a
1310 // state to report rather than an exception out of the middle of a teardown, so the
1311 // wait *wants* TERMINATED — and recording it here is what stops a resume from being
1312 // offered afterwards (STATE-11).
1313 let settled_lifecycle = match settled.state.as_str() {
1314 "SUSPENDED" => Lifecycle::Suspended,
1315 "TERMINATED" => {
1316 self.was_terminated = true;
1317 Lifecycle::Terminated
1318 }
1319 other => {
1320 return Err(Error::new(
1321 ErrorKind::Platform,
1322 format!(
1323 "the suspend wait returned {other}, which is neither SUSPENDED nor \
1324 TERMINATED — and those two are what this client asked for."
1325 ),
1326 ));
1327 }
1328 };
1329 self.set_lifecycle(settled_lifecycle);
1330 self.microvm = Some(settled);
1331 Ok(())
1332 }
1333
1334 // ── resume (STATE-7, STATE-8, STATE-12) ──────────────────────────────────
1335
1336 /// Thaws the VM and returns a usable session.
1337 ///
1338 /// # What is deliberately not re-delivered (STATE-7)
1339 ///
1340 /// Nothing. No run-hook payload, no token, no bootstrap. The in-memory token survived
1341 /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be
1342 /// refused — a 409 that reads like a broken VM.
1343 ///
1344 /// # The window is checked first (STATE-12)
1345 ///
1346 /// Before any wire call, because the answer is already known and calling costs the poll
1347 /// timeout to learn something worse. The falsification is a service reporting
1348 /// TERMINATED: without this check the resume burns the full deadline and then reports a
1349 /// state the client could have named at once.
1350 ///
1351 /// # The proxy token is dropped (STATE-8)
1352 ///
1353 /// [`Session::rebind`] invalidates it. The endpoint URL does not change across
1354 /// suspend/resume, so the rebind is usually a no-op on the URL — but a token minted
1355 /// against the pre-suspend instance may no longer validate, and that rejection reads
1356 /// exactly like a dead daemon.
1357 pub async fn resume(&mut self) -> Result<&mut Session, Error> {
1358 let id = self.require_microvm("resume")?;
1359
1360 // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal
1361 // comes before the window check and before any call.
1362 if self.was_terminated || self.lifecycle == Lifecycle::Terminated {
1363 return Err(Error::invalid_arg(format!(
1364 "microvm {id} was terminated, and a terminated VM never returns to RUNNING \
1365 (STATE-11). There is nothing to resume: the guest's memory is gone, so even a \
1366 call the service accepted would hand back a different machine."
1367 )));
1368 }
1369 if self.lifecycle != Lifecycle::Suspended {
1370 return Err(Error::invalid_arg(format!(
1371 "microvm {id} is {} and a resume is only issued from SUSPENDED (STATE-7).",
1372 self.lifecycle,
1373 )));
1374 }
1375
1376 // STATE-12, first and locally.
1377 self.require_open_suspended_window(&id)?;
1378
1379 self.control.resume(&id).await?;
1380 // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the
1381 // state this call was made from, so failing on it would fail every resume. A VM the
1382 // idlePolicy terminated during suspension never reaches RUNNING, and waiting only
1383 // for RUNNING there burns the full timeout and then reports a timeout message
1384 // hiding a cause the service had already stated in `stateReason`.
1385 let running = self
1386 .control
1387 .wait_for_state(
1388 &id,
1389 &["RUNNING"],
1390 &crate::constants::DEAD_STATES,
1391 self.lifecycle_wait(),
1392 )
1393 .await?;
1394
1395 self.set_lifecycle(Lifecycle::Running);
1396 // STATE-8, through the endpoint the service just reported rather than the one held:
1397 // the URL is measured not to change, and reading it from the response is what makes
1398 // that a fact this code depends on rather than an assumption it encodes.
1399 let endpoint = running.endpoint.clone();
1400 self.microvm = Some(running);
1401 if let Some(session) = self.session.as_mut() {
1402 session.rebind(endpoint);
1403 }
1404 // Cleared on success so the next cycle's window is measured from the next suspend.
1405 // Leaving it set would accumulate every suspension's elapsed time into one total and
1406 // reject a resume whose own window is wide open.
1407 self.suspended_at = None;
1408
1409 self.session.as_mut().ok_or_else(|| {
1410 Error::new(
1411 ErrorKind::Unexpected,
1412 "the VM resumed but this sandbox holds no session, which run() always builds"
1413 .to_string(),
1414 )
1415 })
1416 }
1417
1418 /// Rejects a resume the launch-time `idlePolicy` has already made impossible (STATE-12).
1419 ///
1420 /// The message names the elapsed time, the window, and the `idlePolicy` finding, because
1421 /// "cannot resume" alone sends a reader looking for the flag that reopens it.
1422 fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> {
1423 let reported = self
1424 .microvm
1425 .as_ref()
1426 .and_then(|vm| vm.idle_policy.as_ref())
1427 .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds)));
1428 let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at)
1429 else {
1430 // No window or no suspend time means this sandbox cannot know how long the VM
1431 // has been suspended, and guessing would refuse a resume the service would honour.
1432 return Ok(());
1433 };
1434 let elapsed = self.control.clock().elapsed().saturating_sub(since);
1435 if elapsed <= window {
1436 return Ok(());
1437 }
1438 Err(Error::new(
1439 ErrorKind::WindowClosed,
1440 format!(
1441 "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \
1442 window set at launch — the idlePolicy terminates a suspended VM once that window \
1443 passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \
1444 Refused before ResumeMicrovm, because calling would cost the full poll timeout \
1445 to learn the same thing less clearly. A longer window has to be set at launch \
1446 on the next VM; there is no call that extends this one.",
1447 elapsed.as_secs(),
1448 window.as_secs(),
1449 ),
1450 ))
1451 }
1452
1453 // ── terminate (STATE-9, STATE-10) ────────────────────────────────────────
1454
1455 /// Tears down, best-effort, never erroring.
1456 ///
1457 /// Order: VM, then image, then the log group **last**. The log group is last because the
1458 /// service can recreate a group deleted before its image — and see
1459 /// [`TeardownReport::undeleted`] for why this crate names it rather than deleting it.
1460 pub async fn terminate(&mut self, opts: TeardownOpts) -> TeardownReport {
1461 let mut report = TeardownReport::default();
1462 self.torn_down = true;
1463
1464 // The session first: it holds a cached proxy token whose only remaining use would be
1465 // a request against a VM that is going away.
1466 self.session = None;
1467
1468 // 1. The VM.
1469 if let Some(id) = self.microvm.as_ref().map(|vm| vm.id.clone()) {
1470 // STATE-9. Recorded before the call, so a terminate whose call fails still marks
1471 // the VM as one this client asked to destroy — which is what stops a later
1472 // resume (STATE-11) rather than leaving the sandbox looking resumable.
1473 self.set_lifecycle(Lifecycle::Terminating);
1474 self.was_terminated = true;
1475
1476 match self.control.terminate(&id).await {
1477 Ok(()) => report.terminate_accepted = true,
1478 Err(error) => {
1479 report.failures.push(format!("terminate {id}: {error}"));
1480 report.undeleted.push(id.clone());
1481 }
1482 }
1483
1484 if report.terminate_accepted
1485 && let Some(timeout) = opts.wait_for_terminated
1486 {
1487 let wait = WaitOpts {
1488 timeout,
1489 poll_interval: LIFECYCLE_POLL_INTERVAL,
1490 stall_grace: Duration::MAX,
1491 };
1492 match self
1493 .control
1494 .wait_for_state(&id, &["TERMINATED"], &[], wait)
1495 .await
1496 {
1497 // STATE-10.
1498 Ok(settled) => {
1499 self.set_lifecycle(Lifecycle::Terminated);
1500 self.microvm = Some(settled);
1501 }
1502 // Not a leak: the platform accepted the terminate, so the VM is on its
1503 // way out and the lifecycle stays TERMINATING honestly.
1504 Err(error) => report
1505 .failures
1506 .push(format!("waiting for {id} to reach TERMINATED: {error}")),
1507 }
1508 }
1509 }
1510
1511 // 2. The image, retrying — an image in CREATING refuses deletion and a VM still
1512 // terminating holds a reference.
1513 if opts.delete_image {
1514 match self.image.as_ref().map(|image| image.identifier.clone()) {
1515 Some(identifier) => {
1516 let deleted = self
1517 .control
1518 .delete_image(&identifier, opts.delete_attempts, opts.delete_backoff)
1519 .await;
1520 report.image_deleted = Some(deleted);
1521 if deleted {
1522 self.image_exists = false;
1523 } else {
1524 report.failures.push(format!(
1525 "the image {identifier} survived {} delete attempts",
1526 opts.delete_attempts.max(1)
1527 ));
1528 report.undeleted.push(identifier);
1529 }
1530 }
1531 None => report.image_deleted = Some(false),
1532 }
1533 }
1534
1535 // 3. The log group, LAST. Named rather than deleted; see TeardownReport::undeleted.
1536 if opts.delete_log_group
1537 && let Some(group) = self.image.as_ref().map(Image::build_log_group)
1538 {
1539 report.failures.push(format!(
1540 "the build log group {group} was not deleted: CloudWatch Logs is not in this \
1541 crate's dependency set, so it is reported rather than removed. It is \
1542 service-created, which means no Terraform stack owns it and `terraform destroy` \
1543 leaves it behind — six accumulated before anyone noticed."
1544 ));
1545 report.undeleted.push(group);
1546 }
1547
1548 report.lifecycle = Some(self.lifecycle);
1549 report
1550 }
1551
1552 // ── internals ────────────────────────────────────────────────────────────
1553
1554 /// The VM id, or a precondition error naming what was attempted.
1555 fn require_microvm(&self, what: &str) -> Result<String, Error> {
1556 self.microvm
1557 .as_ref()
1558 .map(|vm| vm.id.clone())
1559 .ok_or_else(|| {
1560 Error::new(
1561 ErrorKind::Precondition,
1562 format!("nothing to {what}: this sandbox has not launched a VM."),
1563 )
1564 })
1565 }
1566
1567 /// Five minutes at five-second polls, for suspend, resume, and terminate.
1568 fn lifecycle_wait(&self) -> WaitOpts {
1569 WaitOpts {
1570 timeout: DEFAULT_LIFECYCLE_TIMEOUT,
1571 poll_interval: LIFECYCLE_POLL_INTERVAL,
1572 stall_grace: Duration::MAX,
1573 }
1574 }
1575}
1576
1577/// Warns about a live VM rather than tearing it down.
1578///
1579/// See the module docs: `Drop` cannot await, so a teardown here would either deadlock
1580/// inside a runtime or race the process exit. The warning names the id, because the only
1581/// useful thing a drop can do is tell whoever reads stderr what to go delete.
1582///
1583/// `eprintln!` rather than a log macro because this crate has no logging dependency, and
1584/// taking one on to warn about a leak would be a dependency for a diagnostic.
1585impl Drop for Sandbox {
1586 fn drop(&mut self) {
1587 if self.torn_down || self.adopted {
1588 return;
1589 }
1590 if let Some(vm) = self.microvm.as_ref()
1591 && self.lifecycle.is_live()
1592 {
1593 eprintln!(
1594 "warning: the Sandbox for microvm {} was dropped in {} without terminate(). \
1595 Nothing was torn down — Drop cannot await, so a teardown here would deadlock \
1596 inside a runtime. The VM bills until its maximumDurationInSeconds ceiling: \
1597 terminate it with `microvm terminate {}`.",
1598 vm.id, self.lifecycle, vm.id,
1599 );
1600 }
1601 }
1602}
1603
1604/// A fresh per-VM bearer token: 32 bytes of randomness as 64 hex characters.
1605///
1606/// `getrandom` reaches the kernel CSPRNG directly. The hand-rolled `/dev/urandom`
1607/// read this replaces carried a clock-mixing fallback for a failed read; getrandom
1608/// fails only when the OS pool is genuinely unavailable, and minting a bearer token
1609/// from a clock in that state would be worse than refusing the launch.
1610fn mint_agent_token() -> String {
1611 let mut bytes = [0u8; 32];
1612 getrandom::fill(&mut bytes).expect("the OS random pool is available");
1613 const_hex::encode(bytes)
1614}
1615
1616#[cfg(test)]
1617mod tests {
1618 use super::*;
1619 use crate::control::fake::{self as fake, Answer, FakeControlPlane, TestClock};
1620 use crate::control::transport::Method;
1621
1622 /// A sandbox over the contract recorder, plus the two handles a test asserts through.
1623 ///
1624 /// The recorder is T-W2-4's, deliberately: its answers are literal JSON in the service
1625 /// model's spelling rather than values serialized from this crate's types, so a member
1626 /// this lane misreads cannot be misread identically by the fake.
1627 fn planted() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
1628 let recorder = Arc::new(FakeControlPlane::new());
1629 let clock = Arc::new(TestClock::new());
1630 let plane = ControlPlane::with_transport(
1631 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
1632 Region::UsEast1,
1633 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
1634 );
1635 (Sandbox::with_control_plane(plane), recorder, clock)
1636 }
1637
1638 /// Queues everything a launch to RUNNING needs.
1639 fn answer_launch(recorder: &FakeControlPlane) {
1640 recorder
1641 .answer(
1642 "RunMicrovm",
1643 Answer::ok(fake::microvm_response("PENDING", None)),
1644 )
1645 .answer(
1646 "GetMicrovm",
1647 Answer::ok(fake::microvm_response("RUNNING", None)),
1648 )
1649 .answer(
1650 "CreateMicrovmAuthToken",
1651 Answer::ok(fake::auth_token_response("proxy-token")),
1652 );
1653 }
1654
1655 #[tokio::test]
1656 async fn vpc_egress_configuration_survives_the_sandbox_launch() {
1657 let (mut sandbox, recorder, _) = planted();
1658 answer_launch(&recorder);
1659 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
1660 let request = RunRequest::new()
1661 .with_image("arn:image")
1662 .with_egress_network_connector(arn);
1663 assert_eq!(
1664 request.egress_posture(),
1665 crate::control::EgressPosture::Unsealed
1666 );
1667 sandbox.run(request).await.expect("launch");
1668 let body = recorder.first_body("RunMicrovm");
1669 assert_eq!(body["egressNetworkConnectors"], serde_json::json!([arn]));
1670 }
1671
1672 /// A launched sandbox in RUNNING, which is where four of the twelve keys start.
1673 async fn launched() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
1674 let (mut sandbox, recorder, clock) = planted();
1675 answer_launch(&recorder);
1676 sandbox
1677 .run(RunRequest::new().with_image("arn:image"))
1678 .await
1679 .expect("the launch reaches RUNNING");
1680 (sandbox, recorder, clock)
1681 }
1682
1683 /// Drives a launched sandbox to SUSPENDED, which is where resume starts.
1684 ///
1685 /// Both `GetMicrovm` answers are queued **before** the launch rather than one before
1686 /// each wait, and that is not a style choice. The recorder repeats its last queued
1687 /// answer, so a queue left at `[RUNNING]` after the launch and then appended to becomes
1688 /// `[RUNNING, SUSPENDED]` — the suspend wait pops the stale RUNNING, sleeps a poll
1689 /// interval, and the fake's `sleep` **advances the clock**. That five seconds is
1690 /// invisible until it lands in the middle of the STATE-12 window arithmetic, which is
1691 /// exactly where it first showed up. Queueing both up front means every wait matches on
1692 /// its first poll and no test clock moves except when a test moves it.
1693 async fn suspended_with_window(
1694 suspended_sec: u32,
1695 ) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
1696 let (mut sandbox, recorder, clock) = planted();
1697 recorder
1698 .answer(
1699 "RunMicrovm",
1700 Answer::ok(fake::microvm_response("PENDING", None)),
1701 )
1702 .answer(
1703 "GetMicrovm",
1704 Answer::ok(fake::microvm_response("RUNNING", None)),
1705 )
1706 .answer(
1707 "GetMicrovm",
1708 Answer::ok(fake::microvm_response("SUSPENDED", None)),
1709 )
1710 .answer(
1711 "CreateMicrovmAuthToken",
1712 Answer::ok(fake::auth_token_response("proxy-token")),
1713 )
1714 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
1715
1716 sandbox
1717 .run(
1718 RunRequest::new()
1719 .with_image("arn:image")
1720 .with_suspended_sec(suspended_sec),
1721 )
1722 .await
1723 .expect("launches");
1724 sandbox.suspend().await.expect("suspends");
1725 assert_eq!(
1726 clock.now(),
1727 Duration::ZERO,
1728 "no wait may have slept, or the window arithmetic below is measuring the fake's \
1729 poll interval as well as the elapsed suspension"
1730 );
1731 (sandbox, recorder, clock)
1732 }
1733
1734 /// A suspended sandbox at the default ten-minute window.
1735 async fn suspended() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
1736 suspended_with_window(600).await
1737 }
1738
1739 // ── adopt (#196) ─────────────────────────────────────────────────────────
1740
1741 /// A canary agent token, so a test can assert it appears in no `Debug` or error.
1742 const ADOPT_TOKEN: &str = "adopt-canary-token-9f1c";
1743 const ADOPT_ENDPOINT: &str = "https://mvm-abc123.microvm.us-east-1.amazonaws.com";
1744
1745 /// A plane over the recorder, whose `GetMicrovm` answers the caller queues.
1746 fn adopt_plane() -> (ControlPlane, Arc<FakeControlPlane>, Arc<TestClock>) {
1747 let recorder = Arc::new(FakeControlPlane::new());
1748 let clock = Arc::new(TestClock::new());
1749 let plane = ControlPlane::with_transport(
1750 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
1751 Region::UsEast1,
1752 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
1753 );
1754 (plane, recorder, clock)
1755 }
1756
1757 async fn adopted_in(state: &str) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
1758 let (plane, recorder, clock) = adopt_plane();
1759 recorder.answer(
1760 "GetMicrovm",
1761 Answer::ok(fake::microvm_response(state, None)),
1762 );
1763 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
1764 .await
1765 .expect("adopts");
1766 (sandbox, recorder, clock)
1767 }
1768
1769 /// **An adopted sandbox feeds its keepalive.** The idle window is the one `GetMicrovm`
1770 /// reports, and the lifecycle watch starts at the adopted state, so a keepalive on an
1771 /// adopted session paces against the real policy and ends when that sandbox suspends.
1772 ///
1773 /// **Falsification** — 2026-09-24. Assign `sandbox.lifecycle` directly instead of
1774 /// through `set_lifecycle`, or drop the `idle_window` line, and this goes red; restored.
1775 #[tokio::test]
1776 async fn an_adopted_sandbox_reports_the_services_idle_window_and_watches_its_lifecycle() {
1777 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
1778 assert_eq!(sandbox.idle_window(), Some(Duration::from_secs(1800)));
1779 assert_eq!(*sandbox.watch_lifecycle().borrow(), Lifecycle::Running);
1780 }
1781
1782 /// Every service state has a lifecycle, and nothing else parses.
1783 #[test]
1784 fn every_service_state_maps_onto_one_lifecycle() {
1785 for state in crate::constants::MICROVM_STATES {
1786 let lifecycle = Lifecycle::from_service(state).expect("a known state");
1787 assert_eq!(lifecycle.as_str(), state);
1788 }
1789 assert_eq!(Lifecycle::from_service("Running"), None);
1790 }
1791
1792 /// **STATE-3 for an adopted VM.** The lifecycle is the service's, the bootstrap is
1793 /// counted once, and `run` is refused before any wire call, so no run-hook payload is
1794 /// ever sent.
1795 ///
1796 /// **Falsification** — 2026-09-24. Leave the adopted lifecycle at PENDING instead of
1797 /// reading it from `GetMicrovm` and the lifecycle and `token_installed` assertions here
1798 /// go red, as does the suspend in the next test; restored after.
1799 #[tokio::test]
1800 async fn an_adopted_vm_takes_its_lifecycle_from_the_service_and_refuses_run() {
1801 let (mut sandbox, recorder, _) = adopted_in("RUNNING").await;
1802 assert!(sandbox.adopted());
1803 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
1804 assert!(sandbox.token_installed());
1805 assert_eq!(sandbox.bootstrap_count(), 1);
1806 assert!(sandbox.image_exists());
1807 let session = sandbox.session().expect("a live VM gets a session");
1808 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
1809 assert_eq!(session.agent_token(), ADOPT_TOKEN);
1810
1811 let error = sandbox
1812 .run(RunRequest::new().with_image("arn:image"))
1813 .await
1814 .expect_err("an adopted VM was bootstrapped by its own launch");
1815 assert_eq!(error.kind(), ErrorKind::InvalidArg);
1816 assert!(error.to_string().contains("STATE-3"), "{error}");
1817 assert_eq!(recorder.call_count("RunMicrovm"), 0);
1818 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
1819 }
1820
1821 /// A name resolves to the record's VM and adopts it; a missing name or a record from
1822 /// another region is refused before any AWS call.
1823 #[tokio::test]
1824 async fn a_name_adopts_its_records_vm_and_foreign_names_are_refused_locally() {
1825 struct One(crate::names::NameRecord);
1826 impl crate::names::NameStore for One {
1827 fn get(&self, name: &str) -> Result<Option<crate::names::NameRecord>, Error> {
1828 Ok((name == self.0.name).then(|| self.0.clone()))
1829 }
1830 fn put(&self, _: &crate::names::NameRecord) -> Result<(), Error> {
1831 unreachable!()
1832 }
1833 fn delete(&self, _: &str) -> Result<bool, Error> {
1834 unreachable!()
1835 }
1836 fn list(&self) -> Result<Vec<crate::names::NameRecord>, Error> {
1837 Ok(vec![self.0.clone()])
1838 }
1839 }
1840 let record = crate::names::NameRecord::new(
1841 "ci",
1842 "mvm-abc123",
1843 ADOPT_ENDPOINT,
1844 ADOPT_TOKEN,
1845 "us-west-2",
1846 )
1847 .expect("record");
1848 let store = One(record.clone());
1849 let foreign = Sandbox::from_name(&store, "ci", Some(Region::UsEast1), None)
1850 .await
1851 .expect_err("registered in another region");
1852 assert_eq!(foreign.kind(), ErrorKind::InvalidArg);
1853 let missing = Sandbox::from_name(&store, "nope", None, None)
1854 .await
1855 .expect_err("no such name");
1856 assert_eq!(missing.kind(), ErrorKind::Precondition);
1857 assert!(!format!("{foreign} {missing}").contains(ADOPT_TOKEN));
1858
1859 let (plane, recorder, _) = adopt_plane();
1860 recorder.answer(
1861 "GetMicrovm",
1862 Answer::ok(fake::microvm_response("RUNNING", None)),
1863 );
1864 let resolved = crate::names::resolve(&store, "ci", Some(&Region::UsWest2)).expect("ok");
1865 let sandbox = Sandbox::adopt_record(plane, resolved)
1866 .await
1867 .expect("adopts");
1868 assert!(sandbox.adopted());
1869 assert_eq!(
1870 sandbox.microvm().map(|vm| vm.id.as_str()),
1871 Some("mvm-abc123")
1872 );
1873 assert_eq!(
1874 sandbox.session().expect("session").agent_token(),
1875 ADOPT_TOKEN
1876 );
1877 let named = sandbox
1878 .name_record("again")
1879 .expect("an adopted VM can be named");
1880 assert_eq!(
1881 (
1882 named.microvm_id.as_str(),
1883 named.agent_token.as_str(),
1884 named.region.as_str()
1885 ),
1886 ("mvm-abc123", ADOPT_TOKEN, "us-east-1")
1887 );
1888 let unlaunched = Sandbox::with_control_plane(adopt_plane().0);
1889 assert_eq!(
1890 unlaunched
1891 .name_record("x")
1892 .expect_err("nothing to name")
1893 .kind(),
1894 ErrorKind::Precondition
1895 );
1896 }
1897
1898 /// Suspend, resume, and terminate all work through an adopted handle with the usual
1899 /// guards, and nothing about the launch is re-delivered (STATE-7).
1900 #[tokio::test]
1901 async fn an_adopted_vm_suspends_resumes_and_terminates_through_its_handle() {
1902 let (plane, recorder, _) = adopt_plane();
1903 recorder
1904 .answer(
1905 "GetMicrovm",
1906 Answer::ok(fake::microvm_response("RUNNING", None)),
1907 )
1908 .answer(
1909 "GetMicrovm",
1910 Answer::ok(fake::microvm_response("SUSPENDED", None)),
1911 )
1912 .answer(
1913 "GetMicrovm",
1914 Answer::ok(fake::microvm_response("RUNNING", None)),
1915 )
1916 .answer(
1917 "GetMicrovm",
1918 Answer::ok(fake::microvm_response("TERMINATED", None)),
1919 )
1920 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
1921 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
1922 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
1923 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
1924 .await
1925 .expect("adopts");
1926
1927 sandbox.suspend().await.expect("suspends from RUNNING");
1928 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
1929 sandbox.resume().await.expect("resumes from SUSPENDED");
1930 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
1931 let report = sandbox
1932 .terminate(TeardownOpts::default().waiting_for_terminated())
1933 .await;
1934 assert!(report.terminate_accepted, "{report:?}");
1935 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
1936 assert!(sandbox.was_terminated());
1937
1938 assert_eq!(recorder.call_count("RunMicrovm"), 0);
1939 assert!(
1940 recorder
1941 .bodies_as_text()
1942 .iter()
1943 .all(|body| !body.contains("runHookPayload") && !body.contains(ADOPT_TOKEN)),
1944 "no launch payload or agent token may reach the control plane"
1945 );
1946 assert_eq!(sandbox.bootstrap_count(), 1, "STATE-3: never above one");
1947 }
1948
1949 /// **STATE-12 for an adopted VM.** With no window from a request of its own, the
1950 /// sandbox refuses a late resume using the `idlePolicy` window `GetMicrovm` reported.
1951 ///
1952 /// **Falsification** — 2026-09-24. Drop the reported-window fallback in
1953 /// `require_open_suspended_window` and this resume reaches `ResumeMicrovm`; restored
1954 /// after.
1955 #[tokio::test]
1956 async fn an_adopted_vm_refuses_a_resume_past_the_reported_window() {
1957 let (plane, recorder, clock) = adopt_plane();
1958 recorder
1959 .answer(
1960 "GetMicrovm",
1961 Answer::ok(fake::microvm_response("RUNNING", None)),
1962 )
1963 .answer(
1964 "GetMicrovm",
1965 Answer::ok(fake::microvm_response("SUSPENDED", None)),
1966 )
1967 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
1968 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
1969 .await
1970 .expect("adopts");
1971 assert_eq!(sandbox.suspended_window(), None, "no request of its own");
1972 sandbox.suspend().await.expect("suspends");
1973
1974 clock.advance(Duration::from_secs(601));
1975 let error = sandbox
1976 .resume()
1977 .await
1978 .expect_err("past the 600 s window the service reported");
1979 assert_eq!(error.kind(), ErrorKind::WindowClosed);
1980 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
1981 }
1982
1983 /// A VM adopted while already SUSPENDED has no locally observed suspend time, so the
1984 /// resume goes to the service rather than being refused on a guess.
1985 #[tokio::test]
1986 async fn an_adopted_suspended_vm_resumes_through_the_service() {
1987 let (plane, recorder, _) = adopt_plane();
1988 recorder
1989 .answer(
1990 "GetMicrovm",
1991 Answer::ok(fake::microvm_response("SUSPENDED", None)),
1992 )
1993 .answer(
1994 "GetMicrovm",
1995 Answer::ok(fake::microvm_response("RUNNING", None)),
1996 )
1997 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
1998 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
1999 .await
2000 .expect("adopts");
2001 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
2002 let session = sandbox
2003 .resume()
2004 .await
2005 .expect("the service answers the resume");
2006 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
2007 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
2008 }
2009
2010 /// **STATE-11 for an adopted VM.** A terminated VM is offered neither a resume nor a
2011 /// suspend, and gets no session.
2012 #[tokio::test]
2013 async fn an_adopted_terminated_vm_refuses_every_transition_locally() {
2014 let (mut sandbox, recorder, _) = adopted_in("TERMINATED").await;
2015 assert!(sandbox.was_terminated());
2016 assert!(sandbox.session().is_none());
2017 let resume = sandbox.resume().await.expect_err("STATE-11");
2018 assert!(resume.to_string().contains("STATE-11"), "{resume}");
2019 let suspend = sandbox.suspend().await.expect_err("STATE-5");
2020 assert!(suspend.to_string().contains("STATE-5"), "{suspend}");
2021 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
2022 }
2023
2024 /// A VM adopted while PENDING finishes through `wait_until_running`, which counts the
2025 /// bootstrap exactly once.
2026 #[tokio::test]
2027 async fn an_adopted_pending_vm_counts_its_bootstrap_once_on_running() {
2028 let (plane, recorder, _) = adopt_plane();
2029 recorder
2030 .answer(
2031 "GetMicrovm",
2032 Answer::ok(fake::microvm_response("PENDING", None)),
2033 )
2034 .answer(
2035 "GetMicrovm",
2036 Answer::ok(fake::microvm_response("RUNNING", None)),
2037 );
2038 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2039 .await
2040 .expect("adopts");
2041 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
2042 assert_eq!(sandbox.bootstrap_count(), 0);
2043 sandbox
2044 .wait_until_running(Duration::from_secs(60))
2045 .await
2046 .expect("reaches RUNNING");
2047 assert_eq!(sandbox.bootstrap_count(), 1);
2048 assert!(sandbox.token_installed());
2049 assert!(
2050 sandbox
2051 .run(RunRequest::new().with_image("arn:image"))
2052 .await
2053 .is_err(),
2054 "still one bootstrap per VM"
2055 );
2056 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2057 }
2058
2059 /// The refusals: an empty token before any call, a mismatched endpoint, and a state
2060 /// this client does not know. None of them, and no `Debug`, prints the token.
2061 ///
2062 /// **Falsification** — 2026-09-24. Remove the endpoint comparison and the mismatch case
2063 /// adopts; remove the empty-token check and that case reaches `GetMicrovm`; print the
2064 /// held token in `Debug` and the canary assertion fails. Each restored after.
2065 #[tokio::test]
2066 async fn adopt_refuses_bad_records_and_never_prints_the_token() {
2067 let (plane, recorder, _) = adopt_plane();
2068 let empty = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, "")
2069 .await
2070 .expect_err("no token");
2071 assert_eq!(empty.kind(), ErrorKind::InvalidArg);
2072 assert_eq!(recorder.calls().len(), 0, "refused before any call");
2073
2074 let (plane, recorder, _) = adopt_plane();
2075 recorder.answer(
2076 "GetMicrovm",
2077 Answer::ok(fake::microvm_response("RUNNING", None)),
2078 );
2079 let mismatch = Sandbox::adopt(plane, "mvm-abc123", "https://other.example", ADOPT_TOKEN)
2080 .await
2081 .expect_err("the endpoint belongs to another record");
2082 assert_eq!(mismatch.kind(), ErrorKind::InvalidArg);
2083 assert!(!mismatch.to_string().contains(ADOPT_TOKEN));
2084
2085 let (plane, recorder, _) = adopt_plane();
2086 recorder.answer(
2087 "GetMicrovm",
2088 Answer::ok(fake::microvm_response("REBOOTING", None)),
2089 );
2090 let unknown = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2091 .await
2092 .expect_err("an unknown state");
2093 assert_eq!(unknown.kind(), ErrorKind::Platform);
2094 assert!(!unknown.to_string().contains(ADOPT_TOKEN));
2095
2096 let (sandbox, _, _) = adopted_in("RUNNING").await;
2097 let printed = format!("{sandbox:?}");
2098 assert!(!printed.contains(ADOPT_TOKEN), "{printed}");
2099 assert!(printed.contains("adopted: true"), "{printed}");
2100 }
2101
2102 /// **STATE-1 and STATE-2.** A launch accepted is PENDING with the image recorded as
2103 /// existing; the platform reporting the run hook succeeded is what marks the token
2104 /// installed.
2105 ///
2106 /// The two halves are asserted with the *same* fake because they are the same call, and
2107 /// what distinguishes them is which fact is recorded when: `image_exists` is true from
2108 /// the accepted launch, `token_installed` only from the RUNNING report. A client that
2109 /// set both at the launch call would pass an end-state assertion and be wrong about a
2110 /// VM that died during startup — which is the next test.
2111 #[tokio::test]
2112 async fn a_launch_records_the_image_and_the_running_report_installs_the_token() {
2113 let (mut sandbox, recorder, _) = planted();
2114 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
2115 assert!(
2116 !sandbox.image_exists(),
2117 "nothing is recorded before a launch"
2118 );
2119 assert!(!sandbox.token_installed());
2120
2121 answer_launch(&recorder);
2122 let session = sandbox
2123 .run(RunRequest::new().with_image("arn:image"))
2124 .await
2125 .expect("launches");
2126 assert_eq!(
2127 session.endpoint(),
2128 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
2129 );
2130
2131 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2132 assert!(sandbox.image_exists(), "STATE-1: the image is recorded");
2133 assert!(sandbox.token_installed(), "STATE-2: the token is installed");
2134 assert_eq!(sandbox.bootstrap_count(), 1);
2135 assert_eq!(sandbox.microvm().expect("launched").id, "mvm-abc123");
2136
2137 // The payload really carried the token, read off the wire member rather than from
2138 // the request type — a field on a struct proves nothing about what was emitted.
2139 let body = recorder.first_body("RunMicrovm");
2140 let payload = body["runHookPayload"].as_str().expect("a string");
2141 assert!(payload.starts_with(r#"{"agent_token":"#), "{payload}");
2142 assert_eq!(body["idlePolicy"]["suspendedDurationSeconds"], 600);
2143 }
2144
2145 /// A launch env reaches the wire member the daemon parses, and it is the only thing
2146 /// that changes about the request.
2147 ///
2148 /// Read off `runHookPayload` rather than off the request struct: a field on a struct
2149 /// proves nothing about what was emitted, which is the same argument the test above
2150 /// makes about the token.
2151 #[tokio::test]
2152 async fn a_launch_env_reaches_the_run_hook_payload_on_the_wire() {
2153 let (mut sandbox, recorder, _) = planted();
2154 answer_launch(&recorder);
2155 sandbox
2156 .run(
2157 RunRequest::new()
2158 .with_image("arn:image")
2159 .with_launch_env("ANTHROPIC_BASE_URL", "https://gateway.example")
2160 .with_launch_env("PATH", "/usr/local/bin:/usr/bin:/bin"),
2161 )
2162 .await
2163 .expect("launches");
2164
2165 let body = recorder.first_body("RunMicrovm");
2166 let payload = body["runHookPayload"].as_str().expect("a string");
2167 // One parse deeper, because that is where the daemon reads it from too.
2168 let inner: serde_json::Value =
2169 serde_json::from_str(payload).expect("the payload is itself JSON");
2170 assert!(
2171 inner["agent_token"].as_str().is_some_and(|t| !t.is_empty()),
2172 "the token still rides alongside: {payload}"
2173 );
2174 assert_eq!(
2175 inner["env"]["ANTHROPIC_BASE_URL"],
2176 "https://gateway.example"
2177 );
2178 assert_eq!(inner["env"]["PATH"], "/usr/local/bin:/usr/bin:/bin");
2179 }
2180
2181 /// **The advisory deny reaches the guest's environment, and it changes nothing else
2182 /// about the request.**
2183 ///
2184 /// Read off `runHookPayload` for the reason the launch-env test gives, and the connector
2185 /// member is asserted absent in the same body: `deny_egress` must not quietly ask the
2186 /// platform for anything, because the whole claim is that the platform has nothing to
2187 /// ask for.
2188 ///
2189 /// **Falsification** — 2026-09-13. Make `run` build the payload from
2190 /// `request.launch_env` instead of `request.effective_launch_env()` and the two
2191 /// `https_proxy` assertions go red while every other launch test stays green; restored
2192 /// after.
2193 #[tokio::test]
2194 async fn the_advisory_deny_reaches_the_launch_environment_in_both_spellings() {
2195 let (mut sandbox, recorder, _) = planted();
2196 answer_launch(&recorder);
2197 let request = RunRequest::new().with_image("arn:image").with_deny_egress();
2198 assert_eq!(
2199 request.egress_posture(),
2200 crate::control::EgressPosture::BestEffort
2201 );
2202 sandbox.run(request).await.expect("launches");
2203
2204 let body = recorder.first_body("RunMicrovm");
2205 let inner: serde_json::Value =
2206 serde_json::from_str(body["runHookPayload"].as_str().expect("a string"))
2207 .expect("the payload is itself JSON");
2208 for key in crate::sandbox::DENY_EGRESS_ENV_KEYS {
2209 assert_eq!(
2210 inner["env"][key], DENY_EGRESS_PROXY_URL,
2211 "{key} must point at the black hole: {inner}"
2212 );
2213 }
2214 assert!(
2215 body.get("egressNetworkConnectors").is_none(),
2216 "the advisory deny asks the platform for nothing: {body}"
2217 );
2218 }
2219
2220 /// A caller's own proxy value survives the deny, and a launch that never asked for the
2221 /// deny carries none of the keys.
2222 #[test]
2223 fn the_deny_never_overwrites_a_callers_own_proxy_and_is_absent_by_default() {
2224 let kept = RunRequest::new()
2225 .with_deny_egress()
2226 .with_launch_env("https_proxy", "http://proxy.internal:3128")
2227 .effective_launch_env();
2228 assert_eq!(kept["https_proxy"], "http://proxy.internal:3128");
2229 assert_eq!(kept["http_proxy"], DENY_EGRESS_PROXY_URL);
2230
2231 let plain = RunRequest::new().effective_launch_env();
2232 assert!(
2233 plain.is_empty(),
2234 "a launch that did not ask for the deny sends byte-for-byte what it always sent: \
2235 {plain:?}"
2236 );
2237 assert_eq!(
2238 RunRequest::new().egress_posture(),
2239 crate::control::EgressPosture::Unsealed,
2240 "and its posture is the measured one, not a seal"
2241 );
2242 }
2243
2244 /// **`egress` and `deny_egress` together are refused locally, with zero calls.**
2245 ///
2246 /// The same shape as the over-budget refusal below: a launch carrying both would report
2247 /// `open` while the workload's own tools failed closed, and AWS has no opinion to
2248 /// return about it.
2249 ///
2250 /// **Falsification** — 2026-09-13. Delete the refusal in `run` and this test fails on
2251 /// `expect_err`; the launch then succeeds and reports posture `open`. Restored after.
2252 #[tokio::test]
2253 async fn asking_for_egress_and_the_deny_together_is_refused_before_any_call() {
2254 let (mut sandbox, recorder, _) = planted();
2255 answer_launch(&recorder);
2256 let mut request = RunRequest::new().with_image("arn:image").with_egress();
2257 request.deny_egress = true;
2258
2259 let error = sandbox
2260 .run(request)
2261 .await
2262 .expect_err("opposite intents are not resolved by a precedence rule");
2263 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2264 assert!(error.to_string().contains("opposite things"), "{error}");
2265 assert_eq!(
2266 recorder.calls().len(),
2267 0,
2268 "a locally-refused launch costs nothing"
2269 );
2270 }
2271
2272 /// **A pinned `imageVersion` reaches the wire through the sandbox**, and an unpinned
2273 /// launch omits the member entirely.
2274 ///
2275 /// This is the layer a caller actually uses, so the forwarding is what needs asserting:
2276 /// [`RunRequest::image_version`] is a field on a struct and proves nothing about the
2277 /// emitted body until it is read off the wire member.
2278 ///
2279 /// **Falsification** — run 2026-08-16. Drop `wire.image_version = ...` from `run` and the
2280 /// pinned assertion goes red with the member absent, while every other launch test still
2281 /// passes — which is why this one is here.
2282 #[tokio::test]
2283 async fn a_pinned_image_version_is_forwarded_to_the_launch_request() {
2284 let (mut sandbox, recorder, _) = planted();
2285 answer_launch(&recorder);
2286 sandbox
2287 .run(
2288 RunRequest::new()
2289 .with_image("arn:image")
2290 .with_image_version("2.0"),
2291 )
2292 .await
2293 .expect("launches");
2294 assert_eq!(
2295 recorder.first_body("RunMicrovm")["imageVersion"],
2296 "2.0",
2297 "a canary has to launch against the version it means to test"
2298 );
2299
2300 let (mut sandbox, recorder, _) = planted();
2301 answer_launch(&recorder);
2302 sandbox
2303 .run(RunRequest::new().with_image("arn:image"))
2304 .await
2305 .expect("launches");
2306 assert!(
2307 recorder
2308 .first_body("RunMicrovm")
2309 .get("imageVersion")
2310 .is_none(),
2311 "an unpinned launch sends what this client always sent"
2312 );
2313 }
2314
2315 /// **The local refusal, with zero control-plane calls.**
2316 ///
2317 /// The observable difference between this client and one that lets AWS answer, and the
2318 /// same shape as STATE-5's suspend refusal: an over-budget launch env fails before the
2319 /// launch rather than as a `ValidationException` on a member the caller did not know
2320 /// they were filling. botocore does not enforce the ceiling client-side, so without
2321 /// this there is no local signal at all.
2322 ///
2323 /// **Falsification** — build the payload after `run_microvm` instead of before, and the
2324 /// call count assertion goes red.
2325 #[tokio::test]
2326 async fn an_over_budget_launch_env_is_refused_before_any_control_plane_call() {
2327 let (mut sandbox, recorder, _) = planted();
2328 answer_launch(&recorder);
2329
2330 let error = sandbox
2331 .run(
2332 RunRequest::new()
2333 .with_image("arn:image")
2334 .with_launch_env("AWS_SESSION_TOKEN", "t".repeat(4096)),
2335 )
2336 .await
2337 .expect_err("a credential-scale launch env does not fit the payload");
2338
2339 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2340 let message = error.to_string();
2341 assert!(message.contains("ceiling of 4096"), "{message}");
2342 assert!(message.contains("launch env contributed"), "{message}");
2343 assert_eq!(
2344 recorder.calls().len(),
2345 0,
2346 "the refusal has to be local: a launch that reached AWS costs a VM and a bill"
2347 );
2348 // And the sandbox is left usable rather than half-launched, so a caller can trim
2349 // the env and retry through the same handle.
2350 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
2351 assert_eq!(sandbox.bootstrap_count(), 0);
2352 assert!(sandbox.microvm().is_none());
2353 }
2354
2355 /// **Issue #24's guards reach through `Sandbox`, which is the layer both bindings use.**
2356 ///
2357 /// `microvms-py`'s `run` and `microvms-js`'s `run` both build a [`RunRequest`] and hand it to
2358 /// [`Sandbox::run`], which fills a `RunMicrovmRequest` and calls
2359 /// [`crate::control::ControlPlane::run_microvm`]. So the guards there cover the bindings — but
2360 /// "cover" is a claim about a call chain, and a chain is what a refactor breaks. This asserts
2361 /// it at the layer the bindings actually enter, so a `Sandbox::run` that stopped delegating
2362 /// (or a binding-side default that bypassed the request type) fails a test rather than shipping
2363 /// an unguarded surface to Python and Node.
2364 ///
2365 /// `max_idle_sec` is the case that matters most here, because it is a plain `u32` keyword
2366 /// argument on both bindings with no type narrowing it: `sandbox.run(max_idle_sec=59)` is what
2367 /// a Python caller writes, and before this it reached the wire.
2368 ///
2369 /// **Guard proof.** Delete `require_idle_duration` from `run_microvm` and the idle row goes red
2370 /// on the call count; delete `require_valid_role_arn` and the role row does. Neither guard is in
2371 /// this file, which is the point — the test is about the chain.
2372 #[tokio::test]
2373 async fn the_bindings_layer_refuses_an_illegal_launch_with_zero_calls() {
2374 let cases: [(&str, RunRequest, &str); 3] = [
2375 (
2376 "max_idle_sec=59, which is what a Python keyword argument passes",
2377 RunRequest {
2378 max_idle_sec: 59,
2379 ..RunRequest::new().with_image("arn:image")
2380 },
2381 "maxIdleDurationSeconds",
2382 ),
2383 (
2384 "an execution role that is a bare name",
2385 RunRequest {
2386 execution_role_arn: Some("execution-role".to_string()),
2387 ..RunRequest::new().with_image("arn:image")
2388 },
2389 "role *name*",
2390 ),
2391 (
2392 "a pinned version with a trailing newline",
2393 RunRequest::new()
2394 .with_image("arn:image")
2395 .with_image_version("2.0\n"),
2396 "contains whitespace",
2397 ),
2398 ];
2399
2400 for (label, request, expected) in cases {
2401 let (mut sandbox, recorder, _) = planted();
2402 // The launch answered, so a missing guard is a *successful* launch — a real VM and a
2403 // real bill — rather than a different failure.
2404 answer_launch(&recorder);
2405
2406 // `expect_err` needs the Ok type to be `Debug`, and `run` answers `&mut Session`
2407 // (which is not) — so the discriminant is matched instead. That is also the more
2408 // honest assertion: it says the launch produced no session at all.
2409 let Err(error) = sandbox.run(request).await else {
2410 panic!("{label} must be refused");
2411 };
2412 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
2413 assert!(
2414 error.to_string().contains(expected),
2415 "{label}: wanted {expected:?}, got {error}"
2416 );
2417 assert_eq!(
2418 recorder.calls().len(),
2419 0,
2420 "{label}: the refusal has to be local, because a launch that reached AWS costs a \
2421 VM and a bill"
2422 );
2423 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
2424 assert!(sandbox.microvm().is_none());
2425 }
2426 }
2427
2428 /// A launch whose VM dies during startup leaves the token **not** installed, because
2429 /// the run hook is what delivers it and a dead VM ran no hook.
2430 ///
2431 /// The distinction this pins is the one the test above cannot: both a correct client and
2432 /// one that installs the token at the launch call reach RUNNING with the token set, and
2433 /// only this case separates them.
2434 #[tokio::test]
2435 async fn a_launch_that_dies_during_startup_installs_no_token() {
2436 let (mut sandbox, recorder, _) = planted();
2437 recorder
2438 .answer(
2439 "RunMicrovm",
2440 Answer::ok(fake::microvm_response("PENDING", None)),
2441 )
2442 .answer(
2443 "GetMicrovm",
2444 Answer::ok(fake::microvm_response(
2445 "TERMINATED",
2446 Some("run hook returned 500"),
2447 )),
2448 );
2449
2450 let error = sandbox
2451 .run(RunRequest::new().with_image("arn:image"))
2452 .await
2453 .expect_err("a VM that died during startup is not a launch");
2454 assert_eq!(error.kind(), ErrorKind::LaunchDied);
2455 assert!(
2456 error.to_string().contains("run hook returned 500"),
2457 "{error}"
2458 );
2459
2460 assert!(
2461 !sandbox.token_installed(),
2462 "the run hook never answered, so nothing was installed"
2463 );
2464 assert_eq!(
2465 sandbox.bootstrap_count(),
2466 0,
2467 "a bootstrap counted here would make STATE-3's ceiling unreachable for a retry"
2468 );
2469 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
2470 assert!(
2471 sandbox.image_exists(),
2472 "STATE-1 still holds: the launch was accepted against a real image"
2473 );
2474 }
2475
2476 /// **STATE-3, the guard proof.** A second `run` on one sandbox is refused, and it is
2477 /// refused with **zero** further control-plane calls.
2478 ///
2479 /// The call count is the assertion that matters. A client that let the second launch
2480 /// through would create a second VM and silently address two guests through one handle,
2481 /// and it would also deliver a second run-hook payload to a daemon whose one-shot
2482 /// bootstrap answers 409 — so "an error came back" is not enough to tell the two apart.
2483 ///
2484 /// **Falsification** — delete the `bootstrap_count > 0` branch from `run` and this test
2485 /// is red on both the count and the call total. Verified; see the packet's guard proofs.
2486 #[tokio::test]
2487 async fn a_second_run_on_one_sandbox_is_refused_before_any_call() {
2488 let (mut sandbox, recorder, _) = launched().await;
2489 assert_eq!(sandbox.bootstrap_count(), 1);
2490 let before = recorder.calls().len();
2491
2492 let error = sandbox
2493 .run(RunRequest::new().with_image("arn:image"))
2494 .await
2495 .expect_err("the token is installed at most once per VM lifetime");
2496 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2497 assert!(error.to_string().contains("STATE-3"), "{error}");
2498 assert!(
2499 error.to_string().contains("needs a second Sandbox"),
2500 "the remedy has to be nameable: {error}"
2501 );
2502
2503 assert_eq!(
2504 sandbox.bootstrap_count(),
2505 1,
2506 "bootstrap_count <= 1 is the Z3-proved invariant"
2507 );
2508 assert_eq!(
2509 recorder.calls().len(),
2510 before,
2511 "the refusal must cost no control-plane call"
2512 );
2513 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2514 }
2515
2516 /// **STATE-4 and STATE-6.** A suspend from RUNNING moves to SUSPENDING before the wait,
2517 /// and the platform reporting SUSPENDED is what moves it to SUSPENDED.
2518 #[tokio::test]
2519 async fn a_suspend_from_running_passes_through_suspending_to_suspended() {
2520 let (sandbox, recorder, _) = suspended().await;
2521 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
2522 assert!(
2523 sandbox.token_installed(),
2524 "a freeze keeps the guest's memory, so the token survives"
2525 );
2526 assert_eq!(sandbox.bootstrap_count(), 1, "no re-bootstrap on a freeze");
2527
2528 let calls = recorder.calls();
2529 let suspend = calls
2530 .iter()
2531 .find(|call| call.operation == "SuspendMicrovm")
2532 .expect("the suspend went out");
2533 assert_eq!(suspend.method, Method::Post);
2534 assert_eq!(suspend.path, "/2025-09-09/microvms/mvm-abc123/suspend");
2535 }
2536
2537 /// A suspend whose VM dies while suspending reports TERMINATED rather than raising, and
2538 /// records it — which is what stops a later resume.
2539 ///
2540 /// TERMINATED is *wanted* by the suspend wait for a reason: a VM that dies mid-suspend
2541 /// is a state to report, not an exception out of the middle of a teardown.
2542 #[tokio::test]
2543 async fn a_vm_that_dies_while_suspending_is_recorded_rather_than_raised() {
2544 let (mut sandbox, recorder, _) = launched().await;
2545 recorder
2546 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
2547 .answer(
2548 "GetMicrovm",
2549 Answer::ok(fake::microvm_response("TERMINATED", Some("idle policy"))),
2550 );
2551
2552 sandbox
2553 .suspend()
2554 .await
2555 .expect("a death mid-suspend is a state, not an error");
2556 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
2557 assert!(
2558 sandbox.was_terminated(),
2559 "STATE-11's precondition is recorded"
2560 );
2561 }
2562
2563 /// A suspend whose wire call fails leaves the sandbox in RUNNING, and a retry works.
2564 ///
2565 /// STATE-4's "accepted" is the wire call succeeding. A client that moved to SUSPENDING
2566 /// before the call would strand a failed call there — a state neither suspend nor
2567 /// resume accepts, so one throttled or dropped request would brick the handle.
2568 /// **Falsification** — move `self.lifecycle = Lifecycle::Suspending` back above the
2569 /// `control.suspend` call and the first assertion here reads SUSPENDING, and the retry
2570 /// is refused by the STATE-5 guard.
2571 #[tokio::test]
2572 async fn a_suspend_whose_call_fails_stays_running_and_can_be_retried() {
2573 let (mut sandbox, recorder, _) = planted();
2574 recorder
2575 .answer(
2576 "RunMicrovm",
2577 Answer::ok(fake::microvm_response("PENDING", None)),
2578 )
2579 .answer(
2580 "GetMicrovm",
2581 Answer::ok(fake::microvm_response("RUNNING", None)),
2582 )
2583 .answer(
2584 "GetMicrovm",
2585 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2586 )
2587 .answer(
2588 "CreateMicrovmAuthToken",
2589 Answer::ok(fake::auth_token_response("proxy-token")),
2590 )
2591 // A 409 rather than a 429 or a transport cut, because those two are retried
2592 // inside `send_with_retry` and this test wants the call to fail once, fast.
2593 .answer("SuspendMicrovm", Answer::failure(409, "ConflictException"))
2594 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2595
2596 sandbox
2597 .run(RunRequest::new().with_image("arn:image"))
2598 .await
2599 .expect("launches");
2600
2601 sandbox
2602 .suspend()
2603 .await
2604 .expect_err("the control plane refused the suspend");
2605 assert_eq!(
2606 sandbox.lifecycle(),
2607 Lifecycle::Running,
2608 "a refused suspend must not move the lifecycle: SUSPENDING accepts neither a \
2609 suspend nor a resume, so recording it here would brick the handle"
2610 );
2611
2612 sandbox.suspend().await.expect("the retry suspends");
2613 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
2614 assert_eq!(recorder.call_count("SuspendMicrovm"), 2);
2615 }
2616
2617 /// **STATE-5, the guard proof.** A suspend from anything but RUNNING is refused with
2618 /// **zero** control-plane calls.
2619 ///
2620 /// Every non-RUNNING state, so a state added to `Lifecycle` later is not silently
2621 /// exempt. The count is the load-bearing assertion: letting the call go and reading the
2622 /// service's answer also produces an error, and that error is about a non-running id
2623 /// rather than about which of two things the caller got wrong.
2624 ///
2625 /// **Falsification** — delete the `lifecycle != Running` branch from `suspend` and the
2626 /// SUSPENDED case emits a `SuspendMicrovm` call, turning the count assertion red.
2627 /// Verified; see the packet's guard proofs.
2628 #[tokio::test]
2629 async fn a_suspend_from_a_non_running_state_reaches_no_control_plane_call() {
2630 // From SUSPENDED — the caller who believes they resumed.
2631 let (mut sandbox, recorder, _) = suspended().await;
2632 let before = recorder.call_count("SuspendMicrovm");
2633
2634 let error = sandbox
2635 .suspend()
2636 .await
2637 .expect_err("a suspend is only issued from RUNNING");
2638 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2639 assert!(error.to_string().contains("STATE-5"), "{error}");
2640 assert!(error.to_string().contains("SUSPENDED"), "{error}");
2641 assert_eq!(
2642 recorder.call_count("SuspendMicrovm"),
2643 before,
2644 "the refusal must be local: no second suspend went to the wire"
2645 );
2646
2647 // And from PENDING, before anything launched at all.
2648 let (mut fresh, fresh_recorder, _) = planted();
2649 let error = fresh.suspend().await.expect_err("nothing to suspend");
2650 assert_eq!(error.kind(), ErrorKind::Precondition);
2651 assert_eq!(fresh_recorder.calls().len(), 0);
2652 }
2653
2654 /// **STATE-7 and STATE-8.** A resume from SUSPENDED returns to RUNNING, re-delivers
2655 /// **nothing**, and drops the cached proxy token.
2656 ///
2657 /// Three assertions, and each one is a different failure mode. The bootstrap count says
2658 /// no run-hook payload was re-delivered — which a daemon's one-shot bootstrap would
2659 /// answer 409 to, reading like a broken VM. The `RunMicrovm` count says no second launch
2660 /// happened. And `is_cached` says the token was invalidated, which is the only
2661 /// observable difference on a path where the endpoint URL does not change: a client that
2662 /// kept the token would produce identical requests until the pre-suspend token stopped
2663 /// validating, and that rejection reads exactly like a dead daemon.
2664 #[tokio::test]
2665 async fn a_resume_reuses_the_token_redelivers_nothing_and_drops_the_proxy_token() {
2666 let (mut sandbox, recorder, _) = suspended().await;
2667
2668 // The session mints once before the suspend, so there is a cached token to drop.
2669 let auth = Arc::clone(
2670 sandbox
2671 .session()
2672 .expect("launched")
2673 .proxy_auth()
2674 .expect("the launch wired a minter"),
2675 );
2676 auth.headers().await.expect("mints");
2677 assert!(
2678 auth.is_cached(),
2679 "there must be a token for the drop to matter"
2680 );
2681 assert_eq!(auth.mint_count(), 1);
2682
2683 recorder
2684 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
2685 .answer(
2686 "GetMicrovm",
2687 Answer::ok(fake::microvm_response("RUNNING", None)),
2688 );
2689 let session = sandbox.resume().await.expect("resumes");
2690 assert_eq!(
2691 session.endpoint(),
2692 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
2693 );
2694
2695 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2696 assert!(
2697 sandbox.token_installed(),
2698 "STATE-7: the installed token is reused"
2699 );
2700 assert_eq!(
2701 sandbox.bootstrap_count(),
2702 1,
2703 "STATE-7: a resume re-delivers no run-hook payload"
2704 );
2705 assert_eq!(
2706 recorder.call_count("RunMicrovm"),
2707 1,
2708 "a resume is not a second launch"
2709 );
2710 assert!(
2711 !auth.is_cached(),
2712 "STATE-8: the cached proxy token survived the resume"
2713 );
2714 }
2715
2716 /// **STATE-8's guard proof, on the mint count.** The request after a resume mints a
2717 /// fresh token rather than reusing the pre-suspend one.
2718 ///
2719 /// Separate from the test above because `is_cached` is a fact about the cache and this is
2720 /// a fact about the wire: a client that dropped the cache but kept emitting the old
2721 /// header would pass the first assertion and fail every request.
2722 ///
2723 /// **Falsification** — delete the `proxy.invalidate()` line from `Session::rebind` and
2724 /// the mint count stays at 1 here. Verified; see the packet's guard proofs.
2725 #[tokio::test]
2726 async fn the_request_after_a_resume_mints_a_fresh_proxy_token() {
2727 let (mut sandbox, recorder, _) = suspended().await;
2728 let auth = Arc::clone(
2729 sandbox
2730 .session()
2731 .expect("launched")
2732 .proxy_auth()
2733 .expect("wired"),
2734 );
2735 auth.headers().await.expect("mints");
2736 assert_eq!(auth.mint_count(), 1);
2737
2738 recorder
2739 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
2740 .answer(
2741 "GetMicrovm",
2742 Answer::ok(fake::microvm_response("RUNNING", None)),
2743 );
2744 sandbox.resume().await.expect("resumes");
2745
2746 // No clock movement at all, so a re-mint here is caused by the invalidation rather
2747 // than by the refresh window rolling over.
2748 auth.headers().await.expect("re-mints");
2749 assert_eq!(
2750 auth.mint_count(),
2751 2,
2752 "STATE-8: the resume did not force a fresh mint"
2753 );
2754 }
2755
2756 /// **STATE-12, the guard proof.** A resume past the launch-time suspended window is
2757 /// refused locally, with **zero** resume calls, and the message names the window.
2758 ///
2759 /// Four assertions, and the packet asks for all four because each rules out a different
2760 /// wrong implementation. `ErrorKind::WindowClosed` rather than `Timeout` separates this
2761 /// from the client that waits. Zero `ResumeMicrovm` calls is what "before any wire call"
2762 /// means. The elapsed reading rules out an implementation that polled to the deadline
2763 /// first — the fake's clock advances on every `sleep`, so a poll loop here would show up
2764 /// as time passed. And the message naming both numbers plus `idlePolicy` is what sends a
2765 /// reader to the finding rather than looking for the flag that reopens the window.
2766 ///
2767 /// **Falsification** — delete the `require_open_suspended_window` call from `resume` and
2768 /// the fake answers TERMINATED, so this test fails with an `ErrorKind::LaunchDied` after
2769 /// a `ResumeMicrovm` call, naming neither the window nor the seconds elapsed. Verified;
2770 /// see the packet's guard proofs.
2771 /// A keepalive reads this receiver instead of the lock, so every transition must reach
2772 /// it: a suspend it missed would be undone by the keepalive's next poll auto-resuming
2773 /// the VM.
2774 #[tokio::test]
2775 async fn the_lifecycle_watch_follows_every_transition() {
2776 let (mut sandbox, recorder, _clock) = planted();
2777 let watch = sandbox.watch_lifecycle();
2778 assert_eq!(*watch.borrow(), Lifecycle::Pending);
2779 recorder
2780 .answer(
2781 "RunMicrovm",
2782 Answer::ok(fake::microvm_response("PENDING", None)),
2783 )
2784 .answer(
2785 "GetMicrovm",
2786 Answer::ok(fake::microvm_response("RUNNING", None)),
2787 )
2788 .answer(
2789 "GetMicrovm",
2790 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2791 )
2792 .answer(
2793 "CreateMicrovmAuthToken",
2794 Answer::ok(fake::auth_token_response("proxy-token")),
2795 )
2796 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2797 sandbox
2798 .run(RunRequest::new().with_image("arn:image"))
2799 .await
2800 .expect("launches");
2801 assert_eq!(*watch.borrow(), Lifecycle::Running);
2802 sandbox.suspend().await.expect("suspends");
2803 assert_eq!(*watch.borrow(), Lifecycle::Suspended);
2804 assert_eq!(*watch.borrow(), sandbox.lifecycle());
2805 }
2806
2807 #[tokio::test]
2808 async fn a_resume_past_the_suspended_window_is_refused_before_the_wire() {
2809 // A short window, so the arithmetic is legible: sixty seconds asked for at launch.
2810 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
2811 assert_eq!(sandbox.suspended_window(), Some(Duration::from_secs(60)));
2812 assert_eq!(
2813 sandbox.idle_window(),
2814 Some(Duration::from_secs(600)),
2815 "the launch's own maxIdleDurationSeconds, which a keepalive checks its cadence against"
2816 );
2817
2818 // The window closes. This is the whole of what the clock injection buys.
2819 clock.advance(Duration::from_secs(61));
2820
2821 // What a service says about a VM the idlePolicy already terminated — which is what a
2822 // client without the local check would spend its poll timeout discovering.
2823 recorder
2824 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
2825 .answer(
2826 "GetMicrovm",
2827 Answer::ok(fake::microvm_response(
2828 "TERMINATED",
2829 Some("suspended window elapsed"),
2830 )),
2831 );
2832 let elapsed_before = clock.now();
2833 let resumes_before = recorder.call_count("ResumeMicrovm");
2834
2835 let error = sandbox
2836 .resume()
2837 .await
2838 .expect_err("the window the launch set has closed");
2839
2840 assert_eq!(error.kind(), ErrorKind::WindowClosed);
2841 assert_eq!(error.code(), "ERR_WINDOW_CLOSED");
2842 let message = error.to_string();
2843 assert!(message.contains("61s"), "the elapsed time: {message}");
2844 assert!(
2845 message.contains("60s suspendedDurationSeconds"),
2846 "the window has to be named: {message}"
2847 );
2848 assert!(message.contains("idlePolicy"), "the finding: {message}");
2849 assert!(
2850 message.contains("Refused before ResumeMicrovm"),
2851 "why the refusal is local: {message}"
2852 );
2853 assert_eq!(
2854 recorder.call_count("ResumeMicrovm"),
2855 resumes_before,
2856 "the refusal must come before ResumeMicrovm"
2857 );
2858 assert_eq!(
2859 clock.now(),
2860 elapsed_before,
2861 "the refusal must be immediate rather than polled to the deadline"
2862 );
2863 }
2864
2865 /// **STATE-12's fallback.** With no window from its own request, the sandbox uses the
2866 /// `suspendedDurationSeconds` that `GetMicrovm` reported (600 in the fake).
2867 #[tokio::test]
2868 async fn the_window_falls_back_to_the_idle_policy_the_service_reported() {
2869 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
2870 sandbox.suspended_window = None;
2871 recorder.answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
2872 clock.advance(Duration::from_secs(601));
2873 let error = sandbox
2874 .resume()
2875 .await
2876 .expect_err("past the reported window");
2877 assert_eq!(error.kind(), ErrorKind::WindowClosed);
2878 assert!(error.to_string().contains("600s"), "{error}");
2879 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
2880 }
2881
2882 /// **#195 at the sandbox.** A client-token launch that adopts an idle-suspended VM is
2883 /// resumed and reaches RUNNING with the token counted once. The same poll on a launch
2884 /// without a caller token is still a startup death (TRAP-8).
2885 #[tokio::test]
2886 async fn a_client_token_launch_resumes_the_vm_it_adopted() {
2887 let (mut sandbox, recorder, _) = planted();
2888 recorder
2889 .answer(
2890 "RunMicrovm",
2891 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2892 )
2893 .answer(
2894 "GetMicrovm",
2895 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2896 )
2897 .answer(
2898 "GetMicrovm",
2899 Answer::ok(fake::microvm_response("RUNNING", None)),
2900 )
2901 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
2902 .answer(
2903 "CreateMicrovmAuthToken",
2904 Answer::ok(fake::auth_token_response("proxy-token")),
2905 );
2906 let mut request = RunRequest::new().with_image("arn:image");
2907 request.client_token = Some("job-195".into());
2908 request.agent_token = Some("agent-token".into());
2909 sandbox
2910 .run(request)
2911 .await
2912 .expect("the adopted VM is resumed");
2913 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2914 assert_eq!(sandbox.bootstrap_count(), 1);
2915 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
2916
2917 let (mut fresh, recorder, _) = planted();
2918 recorder
2919 .answer(
2920 "RunMicrovm",
2921 Answer::ok(fake::microvm_response("PENDING", None)),
2922 )
2923 .answer(
2924 "GetMicrovm",
2925 Answer::ok(fake::microvm_response("SUSPENDED", Some("hook failed"))),
2926 );
2927 let error = fresh
2928 .run(RunRequest::new().with_image("arn:image"))
2929 .await
2930 .expect_err("a fresh launch that reads SUSPENDED died during startup");
2931 assert_eq!(error.kind(), ErrorKind::LaunchDied);
2932 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
2933 }
2934
2935 /// **`wait: false`.** The launch returns once accepted, with an addressable session and
2936 /// the lifecycle still PENDING; `wait_until_running` finishes it, exactly once.
2937 #[tokio::test]
2938 async fn a_launch_that_does_not_wait_is_finished_by_wait_until_running() {
2939 let (mut sandbox, recorder, _) = planted();
2940 answer_launch(&recorder);
2941 let mut request = RunRequest::new().with_image("arn:image");
2942 request.wait = false;
2943 let endpoint = sandbox
2944 .run(request)
2945 .await
2946 .expect("accepted")
2947 .endpoint()
2948 .to_string();
2949 assert!(endpoint.starts_with("https://"), "{endpoint}");
2950 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
2951 assert_eq!(sandbox.bootstrap_count(), 0);
2952 assert_eq!(recorder.call_count("GetMicrovm"), 0, "nothing polled yet");
2953
2954 sandbox
2955 .wait_until_running(DEFAULT_READY_TIMEOUT)
2956 .await
2957 .expect("reaches RUNNING");
2958 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2959 assert_eq!(sandbox.bootstrap_count(), 1);
2960 let error = sandbox
2961 .wait_until_running(DEFAULT_READY_TIMEOUT)
2962 .await
2963 .expect_err("a running launch has nothing to wait for");
2964 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2965 assert_eq!(sandbox.bootstrap_count(), 1, "counted once (STATE-3)");
2966 }
2967
2968 /// A resume **inside** the window goes through, so the guard is a comparison rather than
2969 /// a blanket refusal of every resume.
2970 ///
2971 /// The boundary is inclusive: elapsed exactly equal to the window is still open, matching
2972 /// the Python's `elapsed <= window`.
2973 #[tokio::test]
2974 async fn a_resume_inside_the_window_is_allowed_and_the_boundary_is_inclusive() {
2975 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
2976
2977 // Exactly at the window.
2978 clock.advance(Duration::from_secs(60));
2979 recorder
2980 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
2981 .answer(
2982 "GetMicrovm",
2983 Answer::ok(fake::microvm_response("RUNNING", None)),
2984 );
2985 sandbox
2986 .resume()
2987 .await
2988 .expect("60s == the 60s window is open");
2989 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2990 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
2991 }
2992
2993 /// A successful resume clears the stamp, so the next cycle's window is measured from the
2994 /// next suspend rather than from the first one.
2995 ///
2996 /// The failure this rules out is subtle and would only show on a second cycle:
2997 /// accumulating every suspension's elapsed time into one total refuses a resume whose own
2998 /// window is wide open. The clock is advanced past the window *between* the two cycles,
2999 /// which is what makes the accumulating implementation fail here.
3000 #[tokio::test]
3001 async fn a_successful_resume_clears_the_stamp_so_a_second_cycle_measures_its_own_window() {
3002 let (mut sandbox, recorder, clock) = planted();
3003 // Every `GetMicrovm` answer for both cycles, in the order the waits consume them:
3004 // the launch's RUNNING, then SUSPENDED / RUNNING twice. Queued up front for the
3005 // reason `suspended_with_window` documents — a stale answer left at the head makes a
3006 // wait poll twice, and the fake's `sleep` moves the very clock this test is measuring.
3007 recorder
3008 .answer(
3009 "RunMicrovm",
3010 Answer::ok(fake::microvm_response("PENDING", None)),
3011 )
3012 .answer(
3013 "CreateMicrovmAuthToken",
3014 Answer::ok(fake::auth_token_response("proxy-token")),
3015 )
3016 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3017 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3018 .answer(
3019 "GetMicrovm",
3020 Answer::ok(fake::microvm_response("RUNNING", None)),
3021 )
3022 .answer(
3023 "GetMicrovm",
3024 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3025 )
3026 .answer(
3027 "GetMicrovm",
3028 Answer::ok(fake::microvm_response("RUNNING", None)),
3029 )
3030 .answer(
3031 "GetMicrovm",
3032 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3033 )
3034 .answer(
3035 "GetMicrovm",
3036 Answer::ok(fake::microvm_response("RUNNING", None)),
3037 );
3038 sandbox
3039 .run(
3040 RunRequest::new()
3041 .with_image("arn:image")
3042 .with_suspended_sec(60),
3043 )
3044 .await
3045 .expect("launches");
3046
3047 for cycle in 0..2 {
3048 sandbox
3049 .suspend()
3050 .await
3051 .unwrap_or_else(|error| panic!("cycle {cycle} suspend: {error}"));
3052 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended, "cycle {cycle}");
3053
3054 // Forty-five seconds each time — inside the sixty-second window on its own, and
3055 // ninety in total. An implementation that never cleared the stamp measures the
3056 // total and refuses the second resume, which is the whole point of the loop.
3057 clock.advance(Duration::from_secs(45));
3058
3059 sandbox.resume().await.unwrap_or_else(|error| {
3060 panic!("cycle {cycle} resume, whose own window is wide open: {error}")
3061 });
3062 assert_eq!(sandbox.lifecycle(), Lifecycle::Running, "cycle {cycle}");
3063 assert_eq!(
3064 sandbox.bootstrap_count(),
3065 1,
3066 "cycle {cycle}: no re-bootstrap"
3067 );
3068 }
3069
3070 assert_eq!(
3071 clock.now(),
3072 Duration::from_secs(90),
3073 "ninety seconds of accumulated suspension against a sixty-second window: this is \
3074 the total an implementation that never cleared the stamp would have compared"
3075 );
3076 assert_eq!(
3077 recorder.call_count("ResumeMicrovm"),
3078 2,
3079 "both cycles resumed"
3080 );
3081 }
3082
3083 /// **STATE-11, the resume-after-terminate case the packet names.** A terminated VM never
3084 /// returns to RUNNING, and the recorder sees **zero** resume wire calls.
3085 ///
3086 /// The zero count is the assertion the packet asks for, and it is the right one: a client
3087 /// that called and read the answer would also fail, with whatever the service says about
3088 /// a terminated id — which is a different statement from "this VM is gone and even a
3089 /// successful call would hand you a different machine".
3090 #[tokio::test]
3091 async fn a_resume_after_terminate_records_zero_resume_calls() {
3092 let (mut sandbox, recorder, _) = launched().await;
3093 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
3094 let report = sandbox.terminate(TeardownOpts::default()).await;
3095 assert!(report.terminate_accepted);
3096 assert!(sandbox.was_terminated());
3097
3098 let error = sandbox
3099 .resume()
3100 .await
3101 .expect_err("a terminated VM never returns to RUNNING");
3102 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3103 assert!(error.to_string().contains("STATE-11"), "{error}");
3104 assert_eq!(
3105 recorder.call_count("ResumeMicrovm"),
3106 0,
3107 "no resume may reach the wire once the VM was terminated"
3108 );
3109 assert_ne!(
3110 sandbox.lifecycle(),
3111 Lifecycle::Running,
3112 "the lifecycle must not return to RUNNING"
3113 );
3114 }
3115
3116 /// **STATE-9 and STATE-10.** A terminate accepted is TERMINATING with the VM recorded
3117 /// terminated; the platform reporting termination complete is what marks TERMINATED.
3118 ///
3119 /// The two are separate states rather than one because the default teardown does not
3120 /// wait — so TERMINATING is a state a report really ends in, and calling it TERMINATED
3121 /// would claim an observation nobody made.
3122 #[tokio::test]
3123 async fn a_terminate_records_terminating_and_the_completion_report_marks_terminated() {
3124 let (mut sandbox, recorder, _) = launched().await;
3125 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
3126
3127 let report = sandbox.terminate(TeardownOpts::default()).await;
3128 assert!(report.terminate_accepted, "STATE-9: the call was accepted");
3129 assert!(sandbox.was_terminated(), "STATE-9: recorded terminated");
3130 assert_eq!(
3131 report.lifecycle,
3132 Some(Lifecycle::Terminating),
3133 "the default teardown does not wait, so TERMINATED is not claimed"
3134 );
3135 assert!(!report.leaked(), "{report:?}");
3136
3137 // And with the wait asked for, STATE-10 is observed.
3138 let (mut sandbox, recorder, _) = launched().await;
3139 recorder
3140 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
3141 .answer(
3142 "GetMicrovm",
3143 Answer::ok(fake::microvm_response("TERMINATED", None)),
3144 );
3145 let report = sandbox
3146 .terminate(TeardownOpts::default().waiting_for_terminated())
3147 .await;
3148 assert_eq!(report.lifecycle, Some(Lifecycle::Terminated));
3149 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3150 }
3151
3152 /// The teardown order is VM, then image, then the log group **last** — asserted on the
3153 /// recorder's ledger rather than on the report, because the order is a property of what
3154 /// was emitted.
3155 ///
3156 /// The log group is last because the service can recreate a group deleted before its
3157 /// image. This crate cannot delete it at all, so the assertion is that it is *named
3158 /// after* the image work rather than that a delete call went out — see
3159 /// `TeardownReport::undeleted`.
3160 #[tokio::test]
3161 async fn teardown_deletes_the_vm_then_the_image_and_names_the_log_group_last() {
3162 let (mut sandbox, recorder, _) = planted();
3163 recorder
3164 .answer(
3165 "CreateMicrovmImage",
3166 Answer::created(fake::create_image_response("agentd-conformance")),
3167 )
3168 .answer(
3169 "GetMicrovmImage",
3170 Answer::ok(fake::get_image_response("agentd-conformance", "CREATED")),
3171 );
3172 sandbox
3173 .build_image(CreateImageRequest::new(
3174 "agentd-conformance",
3175 b"binary".to_vec(),
3176 "s3://bucket/img.zip",
3177 "arn:aws:iam::123456789012:role/build",
3178 ))
3179 .await
3180 .expect("builds");
3181 answer_launch(&recorder);
3182 sandbox
3183 .run(RunRequest::new())
3184 .await
3185 .expect("launches from the built image");
3186
3187 recorder
3188 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
3189 .answer(
3190 "ListMicrovmImageVersions",
3191 Answer::ok(fake::list_versions_response("1")),
3192 )
3193 .answer(
3194 "DeleteMicrovmImage",
3195 Answer::ok(fake::delete_image_response()),
3196 );
3197
3198 let report = sandbox
3199 .terminate(
3200 TeardownOpts::default()
3201 .deleting_image()
3202 .deleting_log_group(),
3203 )
3204 .await;
3205 assert!(report.terminate_accepted);
3206 assert_eq!(report.image_deleted, Some(true));
3207
3208 // The ledger, filtered to the three destructive operations.
3209 let order: Vec<&str> = recorder
3210 .operations()
3211 .into_iter()
3212 .filter(|operation| matches!(*operation, "TerminateMicrovm" | "DeleteMicrovmImage"))
3213 .collect();
3214 assert_eq!(
3215 order,
3216 ["TerminateMicrovm", "DeleteMicrovmImage"],
3217 "the VM goes before the image: a terminating VM holds a reference to it"
3218 );
3219
3220 // The log group is named, and named last.
3221 assert_eq!(
3222 report.undeleted,
3223 ["/aws/lambda-microvms/agentd-conformance"],
3224 "{report:?}"
3225 );
3226 assert!(report.leaked());
3227 assert!(
3228 report
3229 .failures
3230 .last()
3231 .expect("a failure")
3232 .contains("log group"),
3233 "the log group's failure has to be the last one recorded: {report:?}"
3234 );
3235 assert!(
3236 report
3237 .failures
3238 .last()
3239 .expect("a failure")
3240 .contains("terraform destroy"),
3241 "the reason it leaks has to be named: {report:?}"
3242 );
3243 }
3244
3245 /// An image delete that fails every attempt lands its identifier in the report rather
3246 /// than raising, so the CLI can emit what was left behind (CLI-6).
3247 ///
3248 /// The identifier rather than a flag is the whole point: a leak nobody can name is a leak
3249 /// nobody can clean up.
3250 #[tokio::test]
3251 async fn a_failing_image_delete_names_what_it_left_behind() {
3252 let (mut sandbox, recorder, _) = planted();
3253 recorder
3254 .answer(
3255 "CreateMicrovmImage",
3256 Answer::created(fake::create_image_response("img")),
3257 )
3258 .answer(
3259 "GetMicrovmImage",
3260 Answer::ok(fake::get_image_response("img", "CREATED")),
3261 );
3262 sandbox
3263 .build_image(CreateImageRequest::new(
3264 "img",
3265 b"binary".to_vec(),
3266 "s3://bucket/img.zip",
3267 "arn:aws:iam::123456789012:role/build",
3268 ))
3269 .await
3270 .expect("builds");
3271 answer_launch(&recorder);
3272 sandbox.run(RunRequest::new()).await.expect("launches");
3273
3274 recorder
3275 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
3276 .answer(
3277 "ListMicrovmImageVersions",
3278 Answer::ok(fake::list_versions_response("1")),
3279 )
3280 .answer(
3281 "DeleteMicrovmImage",
3282 Answer::failure(409, "image is in CREATING"),
3283 );
3284
3285 let mut opts = TeardownOpts::default().deleting_image();
3286 opts.delete_attempts = 3;
3287 opts.delete_backoff = Duration::from_secs(1);
3288 let report = sandbox.terminate(opts).await;
3289
3290 assert_eq!(report.image_deleted, Some(false));
3291 assert_eq!(
3292 report.undeleted,
3293 ["arn:aws:lambda:us-east-1:123456789012:microvm-image:img"],
3294 "{report:?}"
3295 );
3296 assert!(
3297 sandbox.image_exists(),
3298 "the image really is still there, so the flag must say so"
3299 );
3300 assert_eq!(
3301 recorder.call_count("DeleteMicrovmImage"),
3302 3,
3303 "every attempt ran"
3304 );
3305 }
3306
3307 /// A terminate whose own call fails still records the VM as one this client asked to
3308 /// destroy, and names it as undeleted.
3309 ///
3310 /// Recording it is what stops a later resume from being offered against a VM the caller
3311 /// already tried to kill (STATE-11) — the alternative leaves the sandbox looking
3312 /// resumable, which is the worse of the two wrong answers.
3313 #[tokio::test]
3314 async fn a_terminate_whose_call_fails_still_records_the_intent_and_the_leak() {
3315 let (mut sandbox, recorder, _) = launched().await;
3316 recorder.answer(
3317 "TerminateMicrovm",
3318 Answer::failure(500, "InternalServerException"),
3319 );
3320
3321 let report = sandbox.terminate(TeardownOpts::default()).await;
3322 assert!(!report.terminate_accepted);
3323 assert_eq!(report.undeleted, ["mvm-abc123"], "{report:?}");
3324 assert!(sandbox.was_terminated());
3325 assert_eq!(report.lifecycle, Some(Lifecycle::Terminating));
3326 assert!(
3327 !report.failures.is_empty(),
3328 "the swallowed failure is recorded"
3329 );
3330 }
3331
3332 /// A teardown on a sandbox that never launched is a no-op report rather than a panic.
3333 ///
3334 /// The path a caller reaches by tearing down in a `finally` after a launch that failed
3335 /// before any VM existed, which is exactly when a teardown that assumed a VM would turn
3336 /// the real failure into a panic.
3337 #[tokio::test]
3338 async fn a_teardown_before_any_launch_is_an_empty_report() {
3339 let (mut sandbox, recorder, _) = planted();
3340 let report = sandbox.terminate(TeardownOpts::default()).await;
3341 assert!(!report.terminate_accepted);
3342 assert!(!report.leaked());
3343 assert_eq!(report.lifecycle, Some(Lifecycle::Pending));
3344 assert_eq!(recorder.calls().len(), 0);
3345 }
3346
3347 /// A run with no image at all is a precondition error before any call, rather than a
3348 /// launch of an empty identifier.
3349 #[tokio::test]
3350 async fn a_run_with_no_image_is_refused_before_any_call() {
3351 let (mut sandbox, recorder, _) = planted();
3352 let error = sandbox
3353 .run(RunRequest::new())
3354 .await
3355 .expect_err("there is nothing to launch");
3356 assert_eq!(error.kind(), ErrorKind::Precondition);
3357 assert!(error.to_string().contains("build_image first"), "{error}");
3358 assert_eq!(recorder.calls().len(), 0);
3359 }
3360
3361 /// The lifecycle's six states are the symspec's `vm_state` domain, spelled as the service
3362 /// spells them.
3363 ///
3364 /// Asserted rather than assumed because the spelling reaches an error message a reader
3365 /// compares against a `GetMicrovm` response: a lifecycle rendering `Suspended` beside a
3366 /// service saying `SUSPENDED` reads like two different facts.
3367 #[test]
3368 fn the_lifecycle_states_are_the_state_models_domain_in_the_services_spelling() {
3369 let all = [
3370 Lifecycle::Pending,
3371 Lifecycle::Running,
3372 Lifecycle::Suspending,
3373 Lifecycle::Suspended,
3374 Lifecycle::Terminating,
3375 Lifecycle::Terminated,
3376 ];
3377 assert_eq!(
3378 all.map(Lifecycle::as_str),
3379 [
3380 "PENDING",
3381 "RUNNING",
3382 "SUSPENDING",
3383 "SUSPENDED",
3384 "TERMINATING",
3385 "TERMINATED"
3386 ]
3387 );
3388 for state in all {
3389 assert_eq!(state.to_string(), state.as_str());
3390 }
3391 // The four live states are the ones a `Drop` warns about, and TERMINATING is not one
3392 // of them: the platform accepted the terminate, so nobody needs telling.
3393 assert_eq!(
3394 all.iter().filter(|state| state.is_live()).count(),
3395 4,
3396 "a state added to the enum defaults to not-live, which would silence the warning"
3397 );
3398 assert!(!Lifecycle::Terminating.is_live());
3399 assert!(!Lifecycle::Terminated.is_live());
3400 }
3401
3402 /// An agent token is 64 hex characters and never repeats.
3403 ///
3404 /// The distinctness matters because the token is the only thing separating one VM's
3405 /// control API from another's: two VMs sharing a token means either one's caller can
3406 /// drive the other's guest.
3407 #[test]
3408 fn a_minted_agent_token_is_sixty_four_hex_characters_and_never_repeats() {
3409 let minted: std::collections::HashSet<String> =
3410 (0..200).map(|_| mint_agent_token()).collect();
3411 assert_eq!(minted.len(), 200, "a repeated token is a shared guest");
3412 for token in &minted {
3413 assert_eq!(token.len(), 64, "{token}");
3414 assert!(token.chars().all(|c| c.is_ascii_hexdigit()), "{token}");
3415 assert_ne!(
3416 token,
3417 &"0".repeat(64),
3418 "an all-zero draw is a read that did nothing"
3419 );
3420 }
3421 }
3422
3423 /// A sandbox's `Debug` does not print the agent token.
3424 #[tokio::test]
3425 async fn a_sandbox_debug_does_not_print_the_agent_token() {
3426 let (mut sandbox, recorder, _) = planted();
3427 answer_launch(&recorder);
3428 sandbox
3429 .run(
3430 RunRequest::new()
3431 .with_image("arn:image")
3432 .with_agent_token("super-secret-agent-token"),
3433 )
3434 .await
3435 .expect("launches");
3436
3437 let rendered = format!("{sandbox:?}");
3438 assert!(rendered.contains("mvm-abc123"), "{rendered}");
3439 assert!(rendered.contains("Running"), "{rendered}");
3440 assert!(
3441 !rendered.contains("super-secret"),
3442 "the agent token reached a Debug string: {rendered}"
3443 );
3444 }
3445
3446 /// A torn-down sandbox does not warn on drop, and an abandoned one is the case the
3447 /// warning exists for.
3448 ///
3449 /// Asserted on the flag rather than on stderr — capturing another thread's stderr from a
3450 /// test is not something this crate can do without a dependency — so what is pinned is
3451 /// the condition the warning branches on. Both halves, so the branch cannot be
3452 /// vacuously true.
3453 #[tokio::test]
3454 async fn a_torn_down_sandbox_is_distinguishable_from_an_abandoned_one() {
3455 let (mut sandbox, recorder, _) = launched().await;
3456 assert!(
3457 sandbox.lifecycle().is_live() && !sandbox.torn_down,
3458 "an abandoned live VM is what the drop warning is for"
3459 );
3460
3461 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
3462 sandbox.terminate(TeardownOpts::default()).await;
3463 assert!(
3464 sandbox.torn_down,
3465 "terminate must mark the sandbox torn down"
3466 );
3467 assert!(
3468 !sandbox.lifecycle().is_live(),
3469 "and the lifecycle must no longer read as live"
3470 );
3471 }
3472
3473 #[tokio::test]
3474 async fn stable_launch_key_and_payload_survive_a_fresh_supervisor() {
3475 let mut bodies = Vec::new();
3476 for _ in 0..2 {
3477 let (mut sandbox, recorder, _) = planted();
3478 answer_launch(&recorder);
3479 let mut request = RunRequest::new().with_image("arn:image");
3480 request.client_token = Some("one-unique-job-launch".into());
3481 request.agent_token = Some("persisted-private-secret".into());
3482 sandbox.run(request).await.expect("accepted");
3483 bodies.push(recorder.first_body("RunMicrovm"));
3484 }
3485 assert_eq!(bodies[0]["clientToken"], "one-unique-job-launch");
3486 assert_eq!(bodies[0], bodies[1]);
3487 }
3488
3489 #[tokio::test]
3490 async fn unsafe_or_malformed_stable_launch_is_refused_without_calls() {
3491 for (token, secret, identity) in [
3492 ("key".to_string(), None, false),
3493 ("key".to_string(), Some("secret".into()), true),
3494 ("".to_string(), Some("secret".into()), false),
3495 ("x".repeat(129), Some("secret".into()), false),
3496 ("line\nbreak".to_string(), Some("secret".into()), false),
3497 ] {
3498 let (mut sandbox, recorder, _) = planted();
3499 let mut request = RunRequest::new().with_image("arn:image");
3500 request.client_token = Some(token);
3501 request.agent_token = secret;
3502 request.identity = identity;
3503 let error = sandbox.run(request).await.expect_err("invalid");
3504 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3505 assert!(recorder.calls().is_empty());
3506 }
3507 }
3508}