microvms_app/sandbox.rs
1// SPDX-License-Identifier: Apache-2.0
2//! The `Sandbox` lifecycle: one VM's whole life, with the state machine the symspec
3//! model describes made true of the code.
4//!
5//! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the
6//! bindings use: build, run, suspend, resume, terminate, the launch-time suspended
7//! window (STATE-12), and teardown in the order that does not leak.
8//!
9//! # The lifecycle is a field, and every transition is guarded
10//!
11//! [`Lifecycle`] is the symspec's `vm_state` verbatim, and [`Sandbox`] carries the other
12//! four variables beside it — `token_installed`, `image_exists`, `was_terminated`,
13//! `bootstrap_count`. The Z3 proofs over that model (bootstrap at most once, suspend from
14//! non-RUNNING unreachable, TERMINATED never returns to RUNNING) are proofs about *this*
15//! struct's reachable states, which is only worth something if the transitions here are
16//! the only way to move it. Those fields are private and transitions are guarded here.
17//! Adoption initializes an existing VM from service-observed state and is covered by
18//! the adoption tests; it does not replay the launch bootstrap.
19//!
20//! # Runtime-checked rather than typestate, deliberately
21//!
22//! The packet offered `Sandbox<Running>` returning a `Suspended` handle, which would make
23//! STATE-5's wrong call a compile error rather than a local refusal — strictly stronger on
24//! the ladder in [`crate`]. It is not what landed, for the reason the packet names as
25//! acceptable: T-W3-8 wraps **one** object for PyO3 and napi-rs, and a type whose Rust
26//! identity changes on every transition cannot be one `#[pyclass]`. A typestate sandbox
27//! would be re-erased into a runtime-checked enum at the binding boundary, so the check
28//! would exist twice with the binding's copy being the one most callers actually hit.
29//!
30//! What is kept from the typestate idea is the part that costs nothing: the check happens
31//! **before** the wire call, so a suspend from SUSPENDED is refused with zero
32//! control-plane calls rather than answered by AWS. The test asserts the call count, which
33//! is the observable that distinguishes the two.
34//!
35//! # The suspended window is checked before the call (STATE-12)
36//!
37//! The launch-time `idlePolicy` *terminates* a suspended VM once `suspendedDurationSeconds`
38//! passes, which means "resume later" silently stops working. A resume past the window is
39//! refused locally, before `ResumeMicrovm`, because the alternative is calling and reading
40//! the failure: the service answers about a terminated id, which is not the same statement
41//! as "the window you set at launch closed", and getting there costs the full poll timeout
42//! first. The window comes from this sandbox's own `RunMicrovm` request, falling back to the
43//! `idlePolicy` that `GetMicrovm` reports (measured 2026-08-15, `docs/PLATFORM.md`). Only
44//! the suspend's start time is the client's alone.
45//!
46//! # Teardown never raises, and the log group is last
47//!
48//! [`Sandbox::terminate`] returns a [`TeardownReport`] rather than a `Result`. It runs
49//! where a caller's `finally` would, and an error raised there replaces the real failure
50//! with a teardown failure — the real one being the one worth reading.
51//!
52//! The order is VM, then image (retrying), then the log group **last**, because the
53//! service can recreate a group deleted before its image. See
54//! [`TeardownReport::undeleted`] for what this crate can and cannot delete.
55//!
56//! # There is no `Drop` that tears down
57//!
58//! Rust has no context manager and `Drop` cannot await. A `Drop` that blocked on a runtime
59//! would deadlock inside one; a `Drop` that spawned would race the process exit. So
60//! [`Sandbox`]'s `Drop` only **warns** about a live VM, naming the id, and the rule is
61//! that a caller calls [`Sandbox::terminate`] explicitly.
62
63use std::sync::Arc;
64use std::time::Duration;
65
66use crate::control::{
67 ControlPlane, CreateImageRequest, Image, Microvm, RunHookPayload, RunMicrovmRequest, WaitOpts,
68};
69use crate::error::{Error, ErrorKind};
70use crate::region::Region;
71use crate::session::{Session, TokenMinter};
72
73/// The default wait for a launch to reach RUNNING: five minutes, matching the Python client's
74/// `ready_timeout_sec`. The daemon wait after it is
75/// [`crate::session::DEFAULT_BOOTSTRAP_TIMEOUT`].
76pub const DEFAULT_RUNNING_TIMEOUT: Duration = Duration::from_secs(300);
77
78/// The old name of [`DEFAULT_RUNNING_TIMEOUT`], kept for one release.
79#[deprecated(
80 note = "renamed DEFAULT_RUNNING_TIMEOUT: `session::DEFAULT_READY_TIMEOUT` shared this name \
81 for a different wait, the daemon's"
82)]
83pub const DEFAULT_READY_TIMEOUT: Duration = DEFAULT_RUNNING_TIMEOUT;
84
85/// The default lifecycle wait for suspend, resume, and terminate, and for a state wait the
86/// caller gives no deadline (the bindings' `wait_for_state`, the CLI's `terminate --wait`).
87pub const DEFAULT_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(300);
88
89/// How many times a teardown retries the image delete.
90///
91/// Twenty, from the Python client: an image in `CREATING` refuses deletion and a VM still
92/// terminating holds a reference, so this is the difference between a clean account and a
93/// billed leak rather than politeness.
94pub const DEFAULT_DELETE_ATTEMPTS: u32 = 20;
95
96/// The gap between image-delete attempts.
97pub const DEFAULT_DELETE_BACKOFF: Duration = Duration::from_secs(15);
98
99/// Where the advisory deny points a well-behaved client: a loopback port nothing serves.
100///
101/// Loopback rather than an unroutable public address, because a refused connection is
102/// immediate and a black-holed one costs the workload a connect timeout per request. Port 1
103/// is privileged, so a demoted workload cannot answer it by accident.
104pub const DENY_EGRESS_PROXY_URL: &str = "http://127.0.0.1:1";
105
106/// The environment variables the advisory deny sets, in both spellings clients read.
107///
108/// Both cases on purpose: `curl` and most Unix clients read the lowercase names, Python's
109/// `requests`/`urllib3`, npm and the AWS SDKs read the uppercase ones, and a client that
110/// reads only the spelling this list omits is a hole in a mechanism that is advisory to
111/// begin with. `no_proxy` is deliberately absent: the launch environment starts empty
112/// (`agentd` clears it and applies only what the hook delivered), so there is no inherited
113/// exemption to overwrite, and each key spends payload budget.
114pub const DENY_EGRESS_ENV_KEYS: [&str; 6] = [
115 "http_proxy",
116 "https_proxy",
117 "all_proxy",
118 "HTTP_PROXY",
119 "HTTPS_PROXY",
120 "ALL_PROXY",
121];
122
123/// How often a lifecycle wait polls. Public so an adapter that builds its own [`WaitOpts`]
124/// polls at the same rate rather than retyping it.
125pub const LIFECYCLE_POLL_INTERVAL: Duration = Duration::from_secs(5);
126
127/// The symspec's `vm_state`, verbatim.
128///
129/// Six states and no others, which is the S1 half of this module: a lifecycle held as a
130/// `String` would let `"RUNNING "` and `"Running"` both exist, and every guard below would
131/// have to decide which it meant.
132#[derive(Clone, Copy, Debug, Eq, PartialEq)]
133pub enum Lifecycle {
134 /// The initial state, and the state a launch is accepted into (STATE-1).
135 Pending,
136 /// The service reports RUNNING; launch has completed bootstrap, or adoption supplied
137 /// the existing token (STATE-2).
138 Running,
139 /// A suspend was accepted and the platform has not yet reported it complete (STATE-4).
140 Suspending,
141 /// The platform reported suspension complete (STATE-6).
142 Suspended,
143 /// A terminate was accepted (STATE-9).
144 Terminating,
145 /// The platform reported termination complete (STATE-10).
146 Terminated,
147}
148
149impl Lifecycle {
150 /// The name the service uses for this state, which is also what an error message says.
151 pub fn as_str(self) -> &'static str {
152 match self {
153 Lifecycle::Pending => "PENDING",
154 Lifecycle::Running => "RUNNING",
155 Lifecycle::Suspending => "SUSPENDING",
156 Lifecycle::Suspended => "SUSPENDED",
157 Lifecycle::Terminating => "TERMINATING",
158 Lifecycle::Terminated => "TERMINATED",
159 }
160 }
161
162 /// The lifecycle the service's `state` names, or `None` for a spelling this client does
163 /// not know. `constants::MICROVM_STATES` is the closed set, and a test ties the two.
164 pub fn from_service(state: &str) -> Option<Self> {
165 Some(match state {
166 "PENDING" => Lifecycle::Pending,
167 "RUNNING" => Lifecycle::Running,
168 "SUSPENDING" => Lifecycle::Suspending,
169 "SUSPENDED" => Lifecycle::Suspended,
170 "TERMINATING" => Lifecycle::Terminating,
171 "TERMINATED" => Lifecycle::Terminated,
172 _ => return None,
173 })
174 }
175
176 /// Whether a VM in this state is still billing, which is what a `Drop` warning is for.
177 pub fn is_live(self) -> bool {
178 matches!(
179 self,
180 Lifecycle::Pending | Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended
181 )
182 }
183}
184
185impl std::fmt::Display for Lifecycle {
186 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
187 f.write_str(self.as_str())
188 }
189}
190
191/// Everything a launch needs, with the defaults the Python client measured.
192///
193/// `agent_token` is optional because the common case is a per-VM secret nobody needs to
194/// see; a caller who has one already — a harness minting its own, or a retry that must
195/// reuse the first attempt's — passes it.
196#[derive(Clone, Debug)]
197pub struct RunRequest {
198 /// The image to launch, or `None` for the one [`Sandbox::build_image`] built.
199 ///
200 /// An image ARN, or a bare image name, which [`Sandbox::run`] resolves to its ARN with one
201 /// `ListMicrovmImages` read. A name no image carries is refused with `ERR_PRECONDITION`
202 /// before anything launches.
203 pub image_identifier: Option<String>,
204 /// `imageVersion`, or `None` for the image's own latest active version.
205 ///
206 /// Pinning is what makes a canary a canary: without it a launch takes whatever version is
207 /// latest at the moment the call lands, which is not necessarily the one the caller just
208 /// built. Paired with [`crate::control::ControlPlane::set_image_version_status`] it is also
209 /// the rollback — set the bad version INACTIVE, re-pin here to the good one — and a version
210 /// set INACTIVE refuses to launch when named here.
211 ///
212 /// See [`crate::control::RunMicrovmRequest::image_version`], which this forwards to.
213 pub image_version: Option<String>,
214 /// The execution role. Optional in the model; every real launch needs one.
215 pub execution_role_arn: Option<String>,
216 /// The bearer token the daemon will accept, or `None` to mint one.
217 pub agent_token: Option<String>,
218 /// Persist this unique launch key before sending a request; reuse only for that same
219 /// launch and identical parameters. None mints a fresh key. A stable key requires
220 /// an explicit agent_token and identity=false so retries carry identical payloads.
221 pub client_token: Option<String>,
222 /// Base environment for every exec in the launched VM, delivered in the same
223 /// `runHookPayload` as the token.
224 ///
225 /// The daemon applies this *under* each request's own `env`, so a per-exec value
226 /// wins on a key both set. Empty by default, and an empty map produces byte-for-byte
227 /// the payload this client always sent — a caller who never touches this field
228 /// cannot be affected by the field existing.
229 ///
230 /// It shares the token's 4096-byte payload budget, and the check is local:
231 /// [`crate::control::RunHookPayload::for_launch`] refuses an over-ceiling payload
232 /// before any call, naming the byte count and how much of it the env is. That
233 /// matters here more than for the token, because one bearer token has always fit
234 /// with room to spare and a map of credentials does not.
235 pub launch_env: std::collections::HashMap<String, String>,
236 /// Whether to generate and deliver a tunnel identity (#70 layer 3).
237 ///
238 /// On, the launch generates two fresh x25519 seeds, delivers the VM's seed and the
239 /// host's public key in the same payload as the token, and [`RunOutcome`]-equivalent
240 /// state on the session keeps the [`crate::identity::TunnelIdentity`] — the host secret
241 /// and the VM's public pin — for `tunnel --verify-identity` to prove the far end.
242 /// Off (the default) sends byte-for-byte the payload this client always sent.
243 ///
244 /// A flag rather than always-on, deliberately: the material spends 137 bytes of the
245 /// payload's measured 4096-byte budget, and a caller near the ceiling with a launch env
246 /// deserves to choose which feature gets the room.
247 pub identity: bool,
248 /// Whether to request the egress connector. Off omits it from the request; measured
249 /// 2026-09-12 the platform gave a connector-less VM outbound network anyway
250 /// (`docs/PLATFORM.md`).
251 pub egress: bool,
252 /// Customer-managed Lambda VPC egress connector ARNs. See
253 /// [`RunMicrovmRequest::egress_network_connectors`].
254 pub egress_network_connectors: Vec<String>,
255 /// Sets advisory proxy-deny environment variables for clients that honor them.
256 /// Workloads can bypass these variables. Internet isolation requires a VPC egress
257 /// connector using a VPC without an internet gateway or NAT gateway.
258 ///
259 /// Refused with [`RunRequest::egress`]. Existing caller-supplied proxy variables win;
260 /// the added variables share the 4096-byte run-hook payload budget.
261 pub deny_egress: bool,
262 /// Whether to launch shell-capable: the ingress set becomes the measured pair
263 /// `[HTTP_INGRESS, SHELL_INGRESS]` instead of `ALL_INGRESS`, which is what
264 /// `CreateMicrovmShellAuthToken` requires and what `microvm shell` attaches to.
265 /// Off (the default) launches byte-for-byte what this client always sent.
266 pub shell: bool,
267 /// `idlePolicy.maxIdleDurationSeconds`.
268 pub max_idle_sec: u32,
269 /// `idlePolicy.suspendedDurationSeconds` — the window STATE-12 refuses past.
270 pub suspended_sec: u32,
271 /// `idlePolicy.autoResumeEnabled`.
272 pub auto_resume: bool,
273 /// `maximumDurationInSeconds`, checked against 1..=28800 before the call.
274 pub max_duration_sec: u32,
275 /// How long to wait for RUNNING. The wait for the daemon to answer, after it, is
276 /// [`crate::session::DEFAULT_BOOTSTRAP_TIMEOUT`].
277 pub ready_timeout: Duration,
278 /// A label for the run token (TRAP-1). Never the token.
279 pub token_scope: Option<String>,
280 /// Per-VM `logging` (#201). See [`RunMicrovmRequest::logging`].
281 pub logging: Option<crate::control::ops::Logging>,
282 /// Whether [`Sandbox::run`] waits for RUNNING and for the daemon to answer before
283 /// returning (the default).
284 ///
285 /// Off returns as soon as `RunMicrovm` is accepted, with the lifecycle still PENDING:
286 /// the session is addressable, and [`Sandbox::wait_until_running`] finishes both waits
287 /// later, from this process or after a durable workflow's next step.
288 pub wait: bool,
289}
290
291impl Default for RunRequest {
292 fn default() -> Self {
293 Self {
294 image_identifier: None,
295 image_version: None,
296 execution_role_arn: None,
297 agent_token: None,
298 client_token: None,
299 launch_env: std::collections::HashMap::new(),
300 identity: false,
301 egress: false,
302 egress_network_connectors: Vec::new(),
303 deny_egress: false,
304 shell: false,
305 max_idle_sec: 600,
306 suspended_sec: 600,
307 auto_resume: false,
308 max_duration_sec: 3_600,
309 ready_timeout: DEFAULT_RUNNING_TIMEOUT,
310 token_scope: None,
311 logging: None,
312 wait: true,
313 }
314 }
315}
316
317impl RunRequest {
318 /// A launch with the measured defaults: ingress only, ten-minute idle and suspended
319 /// windows, a one-hour maximum duration, no auto-resume.
320 pub fn new() -> Self {
321 Self::default()
322 }
323
324 /// Launches `identifier`, an image ARN or a bare image name, rather than the built image.
325 #[must_use]
326 pub fn with_image(mut self, identifier: impl Into<String>) -> Self {
327 self.image_identifier = Some(identifier.into());
328 self
329 }
330
331 /// Requests the managed internet egress connector. Omission does not block internet access.
332 #[must_use]
333 pub fn with_egress(mut self) -> Self {
334 self.egress = true;
335 self
336 }
337
338 /// Routes egress through a customer-managed Lambda network connector.
339 #[must_use]
340 pub fn with_egress_network_connector(mut self, arn: impl Into<String>) -> Self {
341 self.egress_network_connectors.push(arn.into());
342 self
343 }
344
345 /// Replaces managed internet egress with customer-managed VPC connectors, when any are
346 /// given; an empty list leaves the request unchanged.
347 ///
348 /// The two cannot be combined, so a launch surface that requests managed egress by
349 /// default (an agent VM) uses this to switch rather than failing the launch.
350 #[must_use]
351 pub fn with_vpc_egress(mut self, arns: Vec<String>) -> Self {
352 if !arns.is_empty() {
353 self.egress = false;
354 self.egress_network_connectors = arns;
355 }
356 self
357 }
358
359 /// Applies the advisory in-guest deny. See [`RunRequest::deny_egress`] for what it is
360 /// and, more importantly, for what it is not.
361 #[must_use]
362 pub fn with_deny_egress(mut self) -> Self {
363 self.deny_egress = true;
364 self
365 }
366
367 /// What this launch's outbound network **is**, as opposed to what it asked for.
368 ///
369 /// The one derivation, so the CLI's envelope, the CLI's human line and any embedding
370 /// consumer cannot disagree about the same launch.
371 pub fn egress_posture(&self) -> crate::control::EgressPosture {
372 crate::control::EgressPosture::for_launch(self.egress, self.deny_egress)
373 }
374
375 /// The launch environment as the guest will see it: the caller's, plus the advisory
376 /// proxy deny when [`RunRequest::deny_egress`] is on.
377 ///
378 /// The caller's own value wins on a shared key. A launch that already routes through a
379 /// proxy of its own has said where its traffic goes, and overwriting that with a black
380 /// hole would break a working configuration in the name of a posture this client
381 /// already declines to call a seal.
382 pub fn effective_launch_env(&self) -> std::collections::HashMap<String, String> {
383 let mut env = self.launch_env.clone();
384 if self.deny_egress {
385 for key in DENY_EGRESS_ENV_KEYS {
386 env.entry(key.to_string())
387 .or_insert_with(|| DENY_EGRESS_PROXY_URL.to_string());
388 }
389 }
390 env
391 }
392
393 /// Requests a shell-capable launch. See [`RunRequest::shell`].
394 #[must_use]
395 pub fn with_shell(mut self) -> Self {
396 self.shell = true;
397 self
398 }
399
400 /// Pins the launch to one `imageVersion`. See the field for why.
401 #[must_use]
402 pub fn with_image_version(mut self, version: impl Into<String>) -> Self {
403 self.image_version = Some(version.into());
404 self
405 }
406
407 /// Sets the suspended window this sandbox will refuse a resume past (STATE-12).
408 #[must_use]
409 pub fn with_suspended_sec(mut self, seconds: u32) -> Self {
410 self.suspended_sec = seconds;
411 self
412 }
413
414 /// Reuses a caller-supplied agent token rather than minting one.
415 #[must_use]
416 pub fn with_agent_token(mut self, token: impl Into<String>) -> Self {
417 self.agent_token = Some(token.into());
418 self
419 }
420
421 /// Generates and delivers a tunnel identity with the launch (#70 layer 3).
422 ///
423 /// After a `run` with this set, [`Sandbox::tunnel_identity`] holds what
424 /// `tunnel --verify-identity` needs: the host's secret and the VM's public pin.
425 #[must_use]
426 pub fn with_identity(mut self) -> Self {
427 self.identity = true;
428 self
429 }
430
431 /// Adds one launch-environment variable, which every exec in the VM starts with.
432 ///
433 /// One pair per call rather than a whole map, because that is how a caller builds
434 /// one — from flags, from a config file, one credential at a time — and a
435 /// map-taking setter makes the second call silently discard the first. The field is
436 /// public for a caller who really does hold a map.
437 #[must_use]
438 pub fn with_launch_env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
439 self.launch_env.insert(key.into(), value.into());
440 self
441 }
442}
443
444/// What a teardown should delete beyond the VM itself.
445///
446/// Both deletions are opt-in, because both destroy something a caller may still want: the
447/// image is reusable across runs, and the log group is where a failed build's only evidence
448/// lives.
449#[derive(Clone, Copy, Debug)]
450pub struct TeardownOpts {
451 /// Whether to delete the image the sandbox built.
452 pub delete_image: bool,
453 /// Whether the build log group should be deleted.
454 ///
455 /// This crate **cannot** delete it — CloudWatch is not in the dependency set — so
456 /// asking names the group in [`TeardownReport::undeleted`] rather than removing it.
457 /// See that field for why naming is the honest answer rather than a silent success.
458 pub delete_log_group: bool,
459 /// How many times the image delete is retried.
460 pub delete_attempts: u32,
461 /// The gap between image-delete attempts.
462 pub delete_backoff: Duration,
463 /// How long to wait for TERMINATED, or `None` to return as soon as the terminate call
464 /// is accepted.
465 ///
466 /// `None` by default, matching the Python client: the caller is on the way out, and a
467 /// teardown that blocked five minutes on a state nobody reads is five minutes of a CI
468 /// job. A caller that needs STATE-10 *observed* passes
469 /// [`TeardownOpts::waiting_for_terminated`].
470 pub wait_for_terminated: Option<Duration>,
471}
472
473impl Default for TeardownOpts {
474 fn default() -> Self {
475 Self {
476 delete_image: false,
477 delete_log_group: false,
478 delete_attempts: DEFAULT_DELETE_ATTEMPTS,
479 delete_backoff: DEFAULT_DELETE_BACKOFF,
480 wait_for_terminated: None,
481 }
482 }
483}
484
485impl TeardownOpts {
486 /// Deletes the image as well as the VM.
487 #[must_use]
488 pub fn deleting_image(mut self) -> Self {
489 self.delete_image = true;
490 self
491 }
492
493 /// Asks for the log group too, which names it rather than deleting it.
494 #[must_use]
495 pub fn deleting_log_group(mut self) -> Self {
496 self.delete_log_group = true;
497 self
498 }
499
500 /// Waits for TERMINATED before returning (STATE-10).
501 #[must_use]
502 pub fn waiting_for_terminated(mut self) -> Self {
503 self.wait_for_terminated = Some(DEFAULT_LIFECYCLE_TIMEOUT);
504 self
505 }
506}
507
508/// What a teardown did, and what it left behind.
509///
510/// Returned rather than raised. See the module docs: this runs where a `finally` would.
511#[derive(Clone, Debug, Default, Eq, PartialEq)]
512pub struct TeardownReport {
513 /// Identifiers of everything a caller asked to have deleted that still exists.
514 ///
515 /// The CLI emits these (CLI-6), which is the whole reason they are identifiers rather
516 /// than a boolean: a leak nobody can name is a leak nobody can clean up.
517 ///
518 /// Two things land here. A delete that was attempted and failed — an image whose twenty
519 /// attempts all hit a conflict. And the build **log group**, which this crate cannot
520 /// delete at all: no CloudWatch client exists anywhere in this workspace, and a logs
521 /// API for one delete call is a bigger surface than the leak it would hide (an
522 /// `aws-sdk-cloudwatchlogs` edge is a reasonable future add if teardown grows more
523 /// log-group work). Naming it is strictly better than the
524 /// alternatives: deleting it is unavailable, and silently succeeding would report a
525 /// clean teardown over six accumulated log groups — which is how the leak was found in
526 /// the first place.
527 pub undeleted: Vec<String>,
528 /// Whether the terminate call was accepted.
529 pub terminate_accepted: bool,
530 /// Whether the image was deleted, or `None` when deletion was not asked for.
531 pub image_deleted: Option<bool>,
532 /// The lifecycle state the sandbox ended in.
533 pub lifecycle: Option<Lifecycle>,
534 /// Every failure the teardown swallowed, in the order it hit them.
535 ///
536 /// Kept because a teardown that never raises is a teardown whose failures are invisible
537 /// otherwise, and the first one is usually the cause of the rest.
538 pub failures: Vec<String>,
539}
540
541impl TeardownReport {
542 /// Whether anything a caller asked for was left behind.
543 pub fn leaked(&self) -> bool {
544 !self.undeleted.is_empty()
545 }
546}
547
548/// Mints proxy tokens for one MicroVM through the control plane.
549///
550/// The bridge between the two lanes: `ControlPlane::mint_auth_token` answers a
551/// `control::ProxyToken`, [`TokenMinter`] wants a `session::ProxyToken`, and the `From`
552/// impl at `session/proxy.rs` is the conversion — one `.into()` at the boundary, which is
553/// why that impl is `From` rather than a named function.
554///
555/// The plane sits behind an `Arc` rather than a reference because the minter must outlive
556/// the call that built the session: minting happens inside the request path on every later
557/// request, which is what makes it happen at all (TRAP-9).
558///
559/// Public so a driving adapter that builds its own session mints through this rather than a
560/// copy of it.
561pub struct ControlPlaneMinter {
562 control: Arc<ControlPlane>,
563 microvm_id: String,
564}
565
566impl ControlPlaneMinter {
567 /// A minter for `microvm_id`'s tokens, through `control`.
568 pub fn new(control: Arc<ControlPlane>, microvm_id: impl Into<String>) -> Self {
569 Self {
570 control,
571 microvm_id: microvm_id.into(),
572 }
573 }
574}
575
576impl std::fmt::Debug for ControlPlaneMinter {
577 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
578 f.debug_struct("ControlPlaneMinter")
579 .field("microvm_id", &self.microvm_id)
580 .finish_non_exhaustive()
581 }
582}
583
584impl TokenMinter for ControlPlaneMinter {
585 fn mint(
586 &self,
587 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
588 Box::pin(async move {
589 let minted = self.control.mint_auth_token(&self.microvm_id).await?;
590 Ok(minted.into())
591 })
592 }
593
594 /// Overrides the default, which ignores the ports and delegates.
595 ///
596 /// This is the minter that has a control plane behind it, so it is the one that can
597 /// actually widen a token's scope — and without this override
598 /// `Session::connect_headers(8080)` would keep answering a header pair behind a token
599 /// scoped to 9000 only, which the proxy refuses with 403 `Access to port denied`. See
600 /// [`TokenMinter::mint_for_ports`] for the measurement.
601 fn mint_for_ports(
602 &self,
603 ports: &[u16],
604 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
605 let specs: Vec<crate::control::ops::PortSpecification> = ports
606 .iter()
607 .map(|port| crate::control::ops::PortSpecification::port(*port))
608 .collect();
609 Box::pin(async move {
610 let minted = self
611 .control
612 .mint_auth_token_for(&self.microvm_id, &specs)
613 .await?;
614 Ok(minted.into())
615 })
616 }
617}
618
619/// One MicroVM's whole life.
620///
621/// See the module docs for the state machine, the window, and why teardown is explicit.
622pub struct Sandbox {
623 control: Arc<ControlPlane>,
624 image: Option<Image>,
625 /// The image ARN [`Sandbox::run`] resolved and sent in `RunMicrovm`, set before the call.
626 launch_image_arn: Option<String>,
627 microvm: Option<Microvm>,
628 session: Option<Session>,
629 /// A backend for the session [`Sandbox::run`] builds, or `None` for the real HTTP one.
630 ///
631 /// The daemon-side sibling of `with_control_plane`'s transport seam: a test that
632 /// scripts the control plane can launch, but the session `run` then builds would dial
633 /// a real endpoint. This lets the same test script the daemon too. Never set in
634 /// production — `Session::builder` picks reqwest when no backend is given.
635 session_backend: Option<crate::session::SharedBackend>,
636
637 // ── the symspec's five variables ─────────────────────────────────────────
638 lifecycle: Lifecycle,
639 /// Every transition of `lifecycle`, for a reader that must not take the caller's lock
640 /// (a keepalive task running while the same sandbox is busy inside a long exec).
641 lifecycle_watch: tokio::sync::watch::Sender<Lifecycle>,
642 token_installed: bool,
643 image_exists: bool,
644 was_terminated: bool,
645 bootstrap_count: u32,
646
647 /// The window from *our own* `RunMicrovm` request. `GetMicrovm` also reports it (in
648 /// `idlePolicy`), which is the fallback when this sandbox has no request of its own.
649 suspended_window: Option<Duration>,
650 /// Whether the accepted launch carried a caller client token, and so may have adopted
651 /// a VM an earlier attempt launched (#195).
652 launch_adoptable: bool,
653 /// The egress posture of this sandbox's own accepted launch, or `None` for a sandbox
654 /// that launched nothing (an adopted one never saw the launch options).
655 launch_posture: Option<crate::control::EgressPosture>,
656 /// The launch's agent token, held until the session that carries it is built. Kept out
657 /// of `Debug` like every other credential here.
658 pending_agent_token: Option<String>,
659 /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence
660 /// is checked against it.
661 idle_window: Option<Duration>,
662 /// The clock reading when the suspend call was accepted, or `None` when not suspended.
663 suspended_at: Option<Duration>,
664 /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down
665 /// one.
666 torn_down: bool,
667 /// Built by [`Sandbox::adopt`] around a VM another process launched. That process owns
668 /// the VM's teardown, so dropping this handle is not a leak and `Drop` stays quiet.
669 adopted: bool,
670 /// Set by [`Sandbox::detach`]: the VM was handed to another process, which now owns its
671 /// lifecycle and teardown. Every transition is refused and `Drop` stays quiet.
672 detached: bool,
673 /// The launch's tunnel identity, when [`RunRequest::identity`] asked for one.
674 ///
675 /// Holds the host secret and the VM's *public* pin — the VM's own secret was dropped
676 /// the moment the payload was built ([`crate::identity::LaunchIdentity::keep`]), so
677 /// nothing on the host side can impersonate the VM it launched.
678 tunnel_identity: Option<crate::identity::TunnelIdentity>,
679 /// The STS and S3 calls [`Sandbox::ensure_image`] makes, or `None` until the first
680 /// call builds the real ones. The seam a test replaces them at.
681 build_services: Option<Arc<dyn crate::control::BuildServices>>,
682 /// The caller's account, resolved once per sandbox for image ARNs (IMAGE-8).
683 account: Option<String>,
684}
685
686impl std::fmt::Debug for Sandbox {
687 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688 // No agent token: it is a credential, and a `Debug` that printed it would put it in
689 // every log line that formats a sandbox.
690 f.debug_struct("Sandbox")
691 .field("lifecycle", &self.lifecycle)
692 .field("microvm", &self.microvm.as_ref().map(|vm| &vm.id))
693 .field("image", &self.image.as_ref().map(|image| &image.identifier))
694 .field("token_installed", &self.token_installed)
695 .field("bootstrap_count", &self.bootstrap_count)
696 .field("was_terminated", &self.was_terminated)
697 .field("suspended_window", &self.suspended_window)
698 .field("adopted", &self.adopted)
699 .field("detached", &self.detached)
700 .finish_non_exhaustive()
701 }
702}
703
704/// What another process needs to adopt a VM this sandbox handed off with
705/// [`Sandbox::detach`]: pass the fields to [`Sandbox::adopt`] (or the prelude's `Sandbox::adopt_in`).
706///
707/// The agent token is a credential: readable through [`Detached::agent_token`] so it can be
708/// persisted, and absent from `Debug`, so a log line that formats the record cannot leak it.
709/// Store it the way [`crate::names::NameRecord`] asks: privately, encrypted.
710#[derive(Clone)]
711pub struct Detached {
712 /// The VM's identifier.
713 pub microvm_id: String,
714 /// The HTTPS endpoint its daemon answers on.
715 pub endpoint: String,
716 /// The region the VM runs in.
717 pub region: Region,
718 /// The daemon port the endpoint's proxy tokens are minted for.
719 pub port: u16,
720 agent_token: String,
721}
722
723impl Detached {
724 /// The bearer the VM's daemon accepts; required by [`Sandbox::adopt`].
725 pub fn agent_token(&self) -> &str {
726 &self.agent_token
727 }
728}
729
730impl std::fmt::Debug for Detached {
731 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
732 f.debug_struct("Detached")
733 .field("microvm_id", &self.microvm_id)
734 .field("endpoint", &self.endpoint)
735 .field("region", &self.region)
736 .field("port", &self.port)
737 .field("agent_token", &"<redacted>")
738 .finish()
739 }
740}
741
742impl Sandbox {
743 /// A sandbox over an already-built [`ControlPlane`].
744 ///
745 /// The seam every test uses, and public for the same reason `ControlPlane::from_ports`
746 /// is: a caller who wants a non-default port, or a fake transport, has already built the
747 /// plane, and a sandbox constructible only from a [`Region`] would be a sandbox no test
748 /// can drive without AWS. `Sandbox::new(region)`, `Sandbox::adopt_in` and
749 /// `Sandbox::from_name`, which resolve credentials for a region, are in
750 /// `microvms_core::prelude`.
751 ///
752 /// The plane's own clock is what times the suspended window, rather than a second one
753 /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil
754 /// tests record: a window measured on one clock while the poll loop sleeps on another
755 /// calls a closed window open.
756 pub fn with_control_plane(control: ControlPlane) -> Self {
757 Self {
758 control: Arc::new(control),
759 image: None,
760 launch_image_arn: None,
761 microvm: None,
762 session: None,
763 session_backend: None,
764 lifecycle: Lifecycle::Pending,
765 lifecycle_watch: tokio::sync::watch::Sender::new(Lifecycle::Pending),
766 token_installed: false,
767 image_exists: false,
768 was_terminated: false,
769 bootstrap_count: 0,
770 suspended_window: None,
771 launch_adoptable: false,
772 launch_posture: None,
773 pending_agent_token: None,
774 idle_window: None,
775 suspended_at: None,
776 torn_down: false,
777 adopted: false,
778 detached: false,
779 tunnel_identity: None,
780 build_services: None,
781 account: None,
782 }
783 }
784
785 /// Routes [`Sandbox::ensure_image`]'s STS and S3 calls through `services`.
786 ///
787 /// The seam beside `with_control_plane`'s transport: a test that scripts the control
788 /// plane replaces the two calls outside it here, and production builds
789 /// [`crate::control::SignedBuildServices`] on the first `ensure_image`.
790 pub fn with_build_services(mut self, services: Arc<dyn crate::control::BuildServices>) -> Self {
791 self.build_services = Some(services);
792 self
793 }
794
795 /// Routes the session [`Sandbox::run`] builds through `backend` instead of real HTTP.
796 ///
797 /// The daemon-side half of the test seam `with_control_plane` opens: scripting the
798 /// control plane gets a test through the launch, and this gets it through everything
799 /// the launched session then does — uploads, execs, downloads — without a daemon.
800 pub fn with_session_backend(mut self, backend: crate::session::SharedBackend) -> Self {
801 self.session_backend = Some(backend);
802 self
803 }
804
805 // ── the symspec's five variables, readable ───────────────────────────────
806
807 /// The lifecycle state, which is the symspec's `vm_state`.
808 pub fn lifecycle(&self) -> Lifecycle {
809 self.lifecycle
810 }
811
812 /// Whether the agent token has been installed (STATE-2).
813 pub fn token_installed(&self) -> bool {
814 self.token_installed
815 }
816
817 /// Whether an image is recorded as existing (STATE-1).
818 pub fn image_exists(&self) -> bool {
819 self.image_exists
820 }
821
822 /// Whether this VM was ever terminated (STATE-11).
823 pub fn was_terminated(&self) -> bool {
824 self.was_terminated
825 }
826
827 /// How many times the token has been installed. Never above one (STATE-3).
828 pub fn bootstrap_count(&self) -> u32 {
829 self.bootstrap_count
830 }
831
832 /// The image, once built.
833 pub fn image(&self) -> Option<&Image> {
834 self.image.as_ref()
835 }
836
837 /// The image ARN [`Sandbox::run`] sent in `RunMicrovm`: the caller's identifier with a
838 /// bare name resolved, or the built image's.
839 ///
840 /// Recorded before the call rather than read back from [`Sandbox::microvm`], which holds
841 /// only what an accepted launch answered: a launch the service refused, or one a caller
842 /// interrupted, still names the image it asked for here. `None` until `run` has resolved
843 /// an image, and on an adopted sandbox. Not the image [`Sandbox::terminate`] deletes, which
844 /// is only ever one this sandbox built ([`Sandbox::image`]).
845 pub fn launch_image_arn(&self) -> Option<&str> {
846 self.launch_image_arn.as_deref()
847 }
848
849 /// The VM as the service last described it.
850 pub fn microvm(&self) -> Option<&Microvm> {
851 self.microvm.as_ref()
852 }
853
854 /// The session, once launched.
855 pub fn session(&self) -> Option<&Session> {
856 self.session.as_ref()
857 }
858
859 /// The suspended window this sandbox asked for at launch, once it has launched.
860 pub fn suspended_window(&self) -> Option<Duration> {
861 self.suspended_window
862 }
863
864 /// A receiver that sees every lifecycle transition without taking this sandbox's lock.
865 ///
866 /// A binding's keepalive reads it between polls: a sandbox busy inside a long exec
867 /// holds the lock the whole time, which is exactly when the keepalive must keep polling,
868 /// and a suspend or terminate must end the keepalive before its next poll auto-resumes
869 /// the VM.
870 pub fn watch_lifecycle(&self) -> tokio::sync::watch::Receiver<Lifecycle> {
871 self.lifecycle_watch.subscribe()
872 }
873
874 fn set_lifecycle(&mut self, lifecycle: Lifecycle) {
875 self.lifecycle = lifecycle;
876 self.lifecycle_watch.send_replace(lifecycle);
877 }
878
879 /// The idle window this sandbox asked for at launch, once it has launched.
880 pub fn idle_window(&self) -> Option<Duration> {
881 self.idle_window
882 }
883
884 /// The tunnel identity, when the launch asked for one ([`RunRequest::identity`]).
885 ///
886 /// What `tunnel --verify-identity` verifies with: the host's secret and the VM's public
887 /// pin. `None` on a launch that did not ask, and the tunnel route then refuses an
888 /// identity request with close code 4401 rather than downgrading.
889 pub fn tunnel_identity(&self) -> Option<&crate::identity::TunnelIdentity> {
890 self.tunnel_identity.as_ref()
891 }
892
893 /// The egress posture of this sandbox's launch (BIND-12), which its session carries.
894 ///
895 /// The launch's own [`RunRequest::egress_posture`] once [`Sandbox::run`] was accepted.
896 /// Otherwise [`EgressPosture::default`], `unsealed`: an adopted sandbox never saw the
897 /// launch options, and the weakest true claim is the only one it can make.
898 ///
899 /// [`EgressPosture::default`]: crate::control::EgressPosture::default
900 pub fn egress_posture(&self) -> crate::control::EgressPosture {
901 self.launch_posture.unwrap_or_default()
902 }
903
904 /// The agent port the control plane was built with: the hooks port on every image
905 /// this sandbox builds, and the port the daemon listens on in every VM it launches.
906 /// Read-only; `agents::AgentVm` needs it to derive a Dockerfile whose `AGENTD_PORT`
907 /// agrees with the launch.
908 pub fn port(&self) -> u16 {
909 self.control.port()
910 }
911
912 // ── build ────────────────────────────────────────────────────────────────
913
914 /// Builds an image and waits for it to become usable.
915 ///
916 /// Every local guard runs inside [`ControlPlane::create_image`], before the call —
917 /// which matters because the create happens *after* the caller's artifact upload, so a
918 /// rejection AWS raises costs the upload first.
919 pub async fn build_image(&mut self, request: CreateImageRequest) -> Result<&Image, Error> {
920 let size = request.size;
921 let created = self.control.create_image(request).await?;
922 let mut built = self
923 .control
924 .wait_for_image(&created.identifier, size, WaitOpts::default())
925 .await?;
926 // The logging config survives the wait from the create's own record, because the
927 // readback cannot carry it: `GetMicrovmImage` reports no logging member, and the
928 // resolved log stream's discriminator was minted inside `create_image` — this is
929 // the only copy, and dropping it here would leave the caller unable to name the
930 // stream their build wrote to.
931 built.log_group = created.log_group;
932 built.log_stream = created.log_stream;
933 // Recorded before any launch, because a built image exists whether or not anything
934 // is ever run from it — and the teardown has to be able to name it either way.
935 self.image_exists = true;
936 self.image = Some(built);
937 Ok(self.image.as_ref().expect("just assigned"))
938 }
939
940 /// Builds or reuses the content-addressed image for `request` (#221): the name, the
941 /// decision after each describe, the upload and the create race are
942 /// [`crate::control::ensure`]'s.
943 pub async fn ensure_image(
944 &mut self,
945 request: crate::control::EnsureImageRequest,
946 ) -> Result<crate::control::EnsuredImage, Error> {
947 // Everything local first: a request this client refuses costs no call at all, the
948 // caller-identity lookup included.
949 //
950 // On the blocking pool, not on this task: `prepare` hashes and zips the daemon and
951 // the build context, about a second of CPU for the 2 MB daemon in a debug build.
952 // Inline, it stalls every task on this worker, and a concurrent ensure's credential
953 // fetch parked behind it past the instance-metadata provider's one-second timeout
954 // fails as a missing identity (#309).
955 let control = Arc::clone(&self.control);
956 let prepared =
957 tokio::task::spawn_blocking(move || crate::control::ensure::prepare(&control, request))
958 .await
959 .map_err(|joined| {
960 Error::new(
961 ErrorKind::Unexpected,
962 format!("preparing the image's artifact did not complete: {joined}"),
963 )
964 })??;
965 let services = self.build_services().await?;
966 let account = match &self.account {
967 Some(account) => account.clone(),
968 None => {
969 let account = services.caller_account().await?;
970 self.account = Some(account.clone());
971 account
972 }
973 };
974 let ensured =
975 crate::control::ensure::ensure(&self.control, services.as_ref(), &account, prepared)
976 .await?;
977 self.image_exists = true;
978 self.image = Some(ensured.image.clone());
979 Ok(ensured)
980 }
981
982 /// The STS and S3 client `ensure_image` and `upload_artifact_for` reach AWS through: the one
983 /// `with_build_services` set, or the plane's adapters' for its region, built once.
984 async fn build_services(&mut self) -> Result<Arc<dyn crate::control::BuildServices>, Error> {
985 if let Some(services) = &self.build_services {
986 return Ok(Arc::clone(services));
987 }
988 let built = self
989 .control
990 .adapters()
991 .build_services(self.control.region().clone())
992 .await?;
993 self.build_services = Some(Arc::clone(&built));
994 Ok(built)
995 }
996
997 /// The artifact bytes to upload to the request's `code_artifact_uri`.
998 ///
999 /// The upload is the caller's on this path; [`Sandbox::ensure_image`] uploads for
1000 /// itself.
1001 pub fn build_artifact_for(&self, request: &CreateImageRequest) -> Result<Vec<u8>, Error> {
1002 self.control.build_artifact_for(request)
1003 }
1004
1005 /// Builds the artifact for `request` and puts it at its `code_artifact_uri`, through the
1006 /// build services `ensure_image` uploads with (#258): for a build under a name the caller
1007 /// chose, where no ensure derives the key. The URI must be `s3://<bucket>/<key>`.
1008 ///
1009 /// Run [`Self::preflight`] first: a request this client refuses then costs no upload.
1010 pub async fn upload_artifact_for(&mut self, request: &CreateImageRequest) -> Result<(), Error> {
1011 let (bucket, key) = crate::control::ensure::s3_location(&request.code_artifact_uri)?;
1012 let bytes = self.control.build_artifact_for(request)?;
1013 let services = self.build_services().await?;
1014 services.put_object(bucket, key, bytes).await
1015 }
1016
1017 /// Every local guard [`Self::build_image`] runs, callable **before** the artifact upload.
1018 ///
1019 /// A delegation to [`ControlPlane::preflight`], on this type so the CLI's build path
1020 /// reaches it through the one sandbox it already holds. Local; zero calls. The point is
1021 /// ordering: `build_image`'s guards run after the caller's upload, so a request this
1022 /// client itself refuses would still cost the S3 PUT unless the caller checks first.
1023 pub fn preflight(&self, request: &CreateImageRequest) -> Result<(), Error> {
1024 self.control.preflight(request)
1025 }
1026
1027 /// The ARN for `identifier`: an ARN passes through with zero calls, a bare name is
1028 /// resolved through the image listing by exact match.
1029 ///
1030 /// A read-only delegation to [`ControlPlane::resolve_image_arn`], the resolution
1031 /// [`Sandbox::run`] makes itself, for a caller that wants the ARN without launching. It
1032 /// touches none of the five state-machine variables: resolution is a question about the
1033 /// account, not about this VM's lifecycle.
1034 pub async fn resolve_image_arn(&self, identifier: &str) -> Result<String, Error> {
1035 self.control.resolve_image_arn(identifier).await
1036 }
1037
1038 /// The image with exactly `name`, or `None`. Read-only; see
1039 /// [`ControlPlane::find_image_by_name`] for the pagination and exact-match rules.
1040 pub async fn find_image_by_name(
1041 &self,
1042 name: &str,
1043 ) -> Result<Option<crate::control::ops::MicrovmImageSummaryWire>, Error> {
1044 self.control.find_image_by_name(name).await
1045 }
1046
1047 /// The content hash `build --reuse` keys an image name to. Local; zero calls.
1048 pub fn artifact_content_hash_for(&self, request: &CreateImageRequest) -> String {
1049 self.control.artifact_content_hash_for(request)
1050 }
1051
1052 /// The Dockerfile `request` builds with; see [`ControlPlane::dockerfile_for`]. Local;
1053 /// zero calls.
1054 pub fn dockerfile_for(&self, request: &CreateImageRequest) -> String {
1055 self.control.dockerfile_for(request)
1056 }
1057
1058 /// The versions of a managed base image, for pinning
1059 /// [`CreateImageRequest::base_image_version`].
1060 ///
1061 /// A read-only delegation to [`crate::control::ControlPlane::managed_base_versions`], on
1062 /// this type for the reason [`Self::resolve_image_arn`] gives: it touches none of the five
1063 /// state-machine variables, because a question about AWS's published bases is not a
1064 /// question about this VM's lifecycle.
1065 pub async fn managed_base_versions(
1066 &self,
1067 base_image_arn: &str,
1068 ) -> Result<Vec<crate::control::ops::ManagedMicrovmImageVersionWire>, Error> {
1069 self.control.managed_base_versions(base_image_arn).await
1070 }
1071
1072 /// One version's configuration, state, and availability status. Read-only.
1073 pub async fn image_version(
1074 &self,
1075 identifier: &str,
1076 version: &str,
1077 ) -> Result<crate::control::ops::MicrovmImageVersionSummaryWire, Error> {
1078 self.control.get_image_version(identifier, version).await
1079 }
1080
1081 // ── run (STATE-1, STATE-2, STATE-3) ──────────────────────────────────────
1082
1083 /// Launches a MicroVM and returns its session. By default it waits for RUNNING
1084 /// and for the daemon to answer; setting [`RunRequest::wait`] to false skips
1085 /// those readiness waits.
1086 ///
1087 /// # The three state requirements this is
1088 ///
1089 /// STATE-1: the accepted launch moves the lifecycle to PENDING and records the image as
1090 /// existing. STATE-2: the platform reporting RUNNING is what marks the token installed
1091 /// — not the launch call, because the run hook is what delivers it and a launch that
1092 /// dies during startup delivered nothing. STATE-3: `bootstrap_count` is incremented
1093 /// exactly here, and a second `run` on the same sandbox is refused, which is what makes
1094 /// "at most once per VM lifetime" a property of the type rather than of a caller's
1095 /// discipline.
1096 ///
1097 /// The agent token rides in `runHookPayload`, which is what keeps it out of the shared
1098 /// image snapshot. That is safe because the platform forwards no external traffic until
1099 /// the run hook returns 200, so a per-VM secret delivered at launch wins the
1100 /// first-writer race through the endpoint.
1101 ///
1102 /// # A bare image name
1103 ///
1104 /// [`RunRequest::image_identifier`] may be a name: after the local refusals it is resolved
1105 /// through [`ControlPlane::resolve_image_arn`], and [`Sandbox::launch_image_arn`] records
1106 /// the ARN sent. A name that resolves to nothing fails with `ERR_PRECONDITION` and leaves
1107 /// the sandbox as it was, so a later `run` on it can still launch.
1108 pub async fn run(&mut self, request: RunRequest) -> Result<&mut Session, Error> {
1109 self.refuse_detached("run")?;
1110 // STATE-3's local half. A sandbox that has already bootstrapped cannot bootstrap
1111 // again, and the refusal is here rather than in a comment because `run` twice is
1112 // the plausible mistake — a retry loop around a launch that timed out.
1113 if self.bootstrap_count > 0 || self.microvm.is_some() {
1114 return Err(Error::invalid_arg(format!(
1115 "this sandbox has already launched a VM ({} bootstrap(s), lifecycle {}), and the \
1116 agent token is installed at most once per VM lifetime (STATE-3). A second VM \
1117 needs a second Sandbox — reusing this one would either re-deliver a run-hook \
1118 payload to a daemon whose one-shot bootstrap refuses it, or silently address \
1119 two guests through one handle.",
1120 self.bootstrap_count, self.lifecycle,
1121 )));
1122 }
1123
1124 // The network options, refused and classified by the one function a harness asks
1125 // before a launch (BIND-13), so its answer and this launch cannot disagree. Opposite
1126 // intents are refused rather than resolved by a precedence rule nobody would find:
1127 // `--egress` asks the platform for outbound network and `--deny-egress` asks the
1128 // guest's own clients to refuse it, so a launch carrying both would report `open`
1129 // while its workload's tools failed closed.
1130 let posture = crate::control::egress_posture_for(
1131 request.egress,
1132 &request.egress_network_connectors,
1133 request.deny_egress,
1134 Some(self.control.region()),
1135 )?;
1136
1137 let Some(identifier) = request
1138 .image_identifier
1139 .clone()
1140 .or_else(|| self.image.as_ref().map(|image| image.identifier.clone()))
1141 else {
1142 return Err(Error::new(
1143 ErrorKind::Precondition,
1144 "no image to launch: pass RunRequest::with_image or call build_image first."
1145 .to_string(),
1146 ));
1147 };
1148
1149 if request.client_token.is_some() && (request.agent_token.is_none() || request.identity) {
1150 return Err(Error::invalid_arg(
1151 "a stable client_token requires an explicit agent_token and identity=false; persist identical launch parameters before retrying",
1152 ));
1153 }
1154 let agent_token = match request.agent_token.clone() {
1155 Some(token) => token,
1156 None => mint_agent_token(self.control.entropy())?,
1157 };
1158 // Generated before the payload because the payload carries its two public-facing
1159 // fields. The VM's secret half lives exactly as long as this binding: `keep()` below
1160 // drops it before the launch call returns.
1161 let launch_identity = if request.identity {
1162 Some(crate::entropy::launch_identity(self.control.entropy())?)
1163 } else {
1164 None
1165 };
1166 // Checked even though this builds the JSON itself, because neither half is ours:
1167 // the token may be caller-supplied, so someone passing a signed blob rather than a
1168 // bearer token is exactly who this catches (TRAP-5), and the launch env is entirely
1169 // the caller's. This is the pre-flight refusal — over-ceiling fails here with the
1170 // byte count, before the launch, rather than as a `ValidationException` on a member
1171 // the caller did not know they were filling.
1172 let payload = RunHookPayload::for_launch_with_identity(
1173 &agent_token,
1174 &request.effective_launch_env(),
1175 launch_identity.as_ref(),
1176 )?;
1177
1178 let mut wire = RunMicrovmRequest::new(identifier, payload);
1179 wire.image_version = request.image_version.clone();
1180 wire.execution_role_arn = request.execution_role_arn.clone();
1181 wire.egress_network_connectors = request.egress_network_connectors.clone();
1182 wire.max_idle_sec = request.max_idle_sec;
1183 wire.suspended_sec = request.suspended_sec;
1184 wire.auto_resume = request.auto_resume;
1185 wire.max_duration_sec = request.max_duration_sec;
1186 wire.token_scope = request.token_scope.clone();
1187 wire.client_token = request.client_token.clone();
1188 wire.logging = request.logging.clone();
1189 if request.egress {
1190 wire = wire.with_egress();
1191 }
1192 if request.shell {
1193 wire = wire.with_shell();
1194 }
1195
1196 // `imageIdentifier` takes an ARN, and the service answers a bare name with HTTP 400
1197 // "Malformed ARN", so a name is resolved here, where every surface's launch passes
1198 // (the CLI's `run --image`, both bindings' `Sandbox.run` and `AgentVm.launch`). The
1199 // launch's local refusals go first: the listing a name costs is a call, and a launch
1200 // `run_microvm` would refuse anyway costs none. An ARN passes through with no call.
1201 self.control.launch_connectors(&wire)?;
1202 wire.image_identifier = self
1203 .control
1204 .resolve_image_arn(&wire.image_identifier)
1205 .await?;
1206 self.launch_image_arn = Some(wire.image_identifier.clone());
1207
1208 let launched = self.control.run_microvm(wire).await?;
1209
1210 // The durable half of the identity, kept the moment the launch is accepted: the host
1211 // secret and the VM's public pin. The VM's own secret ends here — `keep` drops it —
1212 // so from this line on the host can verify its VM and impersonate it never.
1213 self.tunnel_identity = launch_identity.map(crate::identity::LaunchIdentity::keep);
1214
1215 // STATE-1: the launch was accepted.
1216 self.set_lifecycle(Lifecycle::Pending);
1217 self.image_exists = true;
1218 // Recorded here rather than after the wait, because the window the idlePolicy
1219 // enforces was set by *this* request and a launch that then fails still leaves a VM
1220 // the caller may have to reason about.
1221 self.suspended_window = Some(Duration::from_secs(u64::from(request.suspended_sec)));
1222 self.idle_window = Some(Duration::from_secs(u64::from(request.max_idle_sec)));
1223 self.microvm = Some(launched);
1224 // A caller-supplied client token can adopt the VM an earlier attempt launched, which
1225 // may have idle-suspended since (#195); a minted token always launches afresh.
1226 self.launch_adoptable = request.client_token.is_some();
1227 self.launch_posture = Some(posture);
1228 self.pending_agent_token = Some(agent_token);
1229
1230 if request.wait {
1231 return self.wait_until_running(request.ready_timeout).await;
1232 }
1233 // Not waiting: the endpoint is in the launch reply, so the session is addressable
1234 // now, and a daemon request before RUNNING fails like any other unready request.
1235 self.build_session()?;
1236 Ok(self.session.as_mut().expect("just assigned"))
1237 }
1238
1239 /// The session for the launched VM, from its endpoint and the launch's agent token.
1240 fn build_session(&mut self) -> Result<(), Error> {
1241 let (Some(vm), Some(agent_token)) = (&self.microvm, &self.pending_agent_token) else {
1242 return Err(Error::new(
1243 ErrorKind::Precondition,
1244 "no launch to build a session for",
1245 ));
1246 };
1247 let minter = Arc::new(ControlPlaneMinter::new(
1248 Arc::clone(&self.control),
1249 vm.id.clone(),
1250 ));
1251 // The plane's clock, so the session's exec ids and token refreshes read the same time
1252 // source as the lifecycle that launched it.
1253 let mut builder = Session::builder(vm.endpoint.clone(), agent_token.clone())
1254 .with_minter(minter)
1255 .with_port(self.control.port())
1256 .with_egress_posture(self.egress_posture())
1257 .with_clock(self.control.shared_clock());
1258 if let Some(backend) = &self.session_backend {
1259 builder = builder.with_backend(Arc::clone(backend));
1260 }
1261 self.session = Some(builder.build_with(self.control.adapters())?);
1262 Ok(())
1263 }
1264
1265 /// Waits for a launch accepted by [`Sandbox::run`] to reach RUNNING, up to `timeout`, and
1266 /// then for its daemon to answer, up to [`crate::session::DEFAULT_BOOTSTRAP_TIMEOUT`].
1267 ///
1268 /// `run` calls this itself unless [`RunRequest::wait`] was off. A launch that adopted an
1269 /// existing VM through its client token resumes it if it idle-suspended (#195); a fresh
1270 /// launch that reaches any terminal state first fails fast with `stateReason` (TRAP-8).
1271 ///
1272 /// # Why RUNNING isn't enough
1273 ///
1274 /// RUNNING says the run hook answered, so the daemon is up and the token installed
1275 /// (STATE-2), but the endpoint's proxy path commonly refuses a connection or two just
1276 /// after. The session is handed back once one health poll through it has answered, so a
1277 /// first exec after a launch doesn't meet that window, on any surface (#254). A daemon that
1278 /// never answers fails this with `ERR_TIMEOUT`, and the lifecycle stays RUNNING: the VM is
1279 /// there and billing, and [`Sandbox::session`] can still wait for it.
1280 pub async fn wait_until_running(&mut self, timeout: Duration) -> Result<&mut Session, Error> {
1281 self.refuse_detached("wait until running")?;
1282 let id = self.require_microvm("wait_until_running")?;
1283 if self.lifecycle != Lifecycle::Pending {
1284 return Err(Error::invalid_arg(format!(
1285 "microvm {id} is {}; wait_until_running finishes a launch that is still PENDING",
1286 self.lifecycle,
1287 )));
1288 }
1289 let opts = WaitOpts::for_lifecycle(timeout);
1290 let running = if self.launch_adoptable {
1291 self.control.wait_for_launch(&id, opts).await?
1292 } else {
1293 self.control.wait_for_running(&id, opts).await?
1294 };
1295
1296 // STATE-2. The platform reported the run hook succeeded, so the token is in the
1297 // guest's memory — and this is the one place that counts it (STATE-3).
1298 self.set_lifecycle(Lifecycle::Running);
1299 self.token_installed = true;
1300 self.bootstrap_count += 1;
1301 self.microvm = Some(running);
1302 if self.session.is_none() {
1303 self.build_session()?;
1304 }
1305 let session = self.session.as_mut().expect("just assigned");
1306 session
1307 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1308 .await?;
1309 Ok(session)
1310 }
1311
1312 /// Waits until the VM this sandbox addresses answers, whatever a launch still has to wait
1313 /// for: from PENDING, [`Sandbox::wait_until_running`] with `running_timeout` (RUNNING, then
1314 /// the daemon); from RUNNING, the daemon alone, up to
1315 /// [`crate::session::DEFAULT_BOOTSTRAP_TIMEOUT`].
1316 ///
1317 /// The shape for a process that adopted a VM another one launched without waiting: it can't
1318 /// know which of the two waits is left, and the lifecycle [`Sandbox::adopt`] read from the
1319 /// service says. Any other lifecycle is refused, because a suspended or terminated VM isn't
1320 /// on its way to answering.
1321 pub async fn wait_until_ready(
1322 &mut self,
1323 running_timeout: Duration,
1324 ) -> Result<&mut Session, Error> {
1325 if self.lifecycle == Lifecycle::Running {
1326 self.refuse_detached("wait until ready")?;
1327 let session = self.session.as_mut().ok_or_else(|| {
1328 Error::new(ErrorKind::Precondition, "a RUNNING sandbox with no session")
1329 })?;
1330 session
1331 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1332 .await?;
1333 return Ok(session);
1334 }
1335 self.wait_until_running(running_timeout).await
1336 }
1337
1338 // ── adopt (STATE-3, with the lifecycle read from the service) ────────────
1339
1340 /// A sandbox for a VM another process launched, rebuilt from its private record.
1341 ///
1342 /// The durable-workflow shape: every step runs in a fresh process, and each one needs
1343 /// suspend, resume, and terminate, not just the exec and file surface `Session::attach`
1344 /// (`microvms_core::prelude::SessionExt::attach`) gives. The lifecycle is read from `GetMicrovm`
1345 /// rather than assumed, so every guard below starts from what the service reports.
1346 ///
1347 /// # Bootstrap is counted, never repeated (STATE-3)
1348 ///
1349 /// The VM was bootstrapped by the launch that created it, so an adopted sandbox refuses
1350 /// [`Sandbox::run`] and never sends a run-hook payload. A VM adopted while still PENDING
1351 /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once
1352 /// when the service reports RUNNING, the same as a launch this sandbox made itself.
1353 ///
1354 /// # The suspended window (STATE-12)
1355 ///
1356 /// The window comes from the `idlePolicy` `GetMicrovm` reports. A VM adopted while
1357 /// already SUSPENDED has no suspend time this client observed, so the local check has
1358 /// nothing to measure and the service answers the resume instead.
1359 ///
1360 /// `endpoint` must agree with the service's when the service reports one: a mismatch
1361 /// means the identifiers came from two different records. `agent_token` is the bearer
1362 /// credential the VM was launched with; it never appears in `Debug` or an error.
1363 pub async fn adopt(
1364 control: ControlPlane,
1365 microvm_id: impl Into<String>,
1366 endpoint: impl Into<String>,
1367 agent_token: impl Into<String>,
1368 ) -> Result<Self, Error> {
1369 let microvm_id = microvm_id.into();
1370 let endpoint = endpoint.into();
1371 let agent_token = agent_token.into();
1372 if agent_token.is_empty() {
1373 return Err(Error::invalid_arg(
1374 "adopt needs the agent token the VM was launched with: its daemon accepts only \
1375 that bearer, and adopting without it would build a handle every exec refuses.",
1376 ));
1377 }
1378 let mut vm = control.get_microvm(µvm_id).await?;
1379 if !endpoint.is_empty() && !vm.endpoint.is_empty() && vm.endpoint != endpoint {
1380 return Err(Error::invalid_arg(format!(
1381 "microvm {microvm_id} reports endpoint {}, not {endpoint}: the id and endpoint \
1382 came from different records, and a session built from them would address one \
1383 VM while its lifecycle calls reached another.",
1384 vm.endpoint,
1385 )));
1386 }
1387 if vm.endpoint.is_empty() {
1388 vm.endpoint = endpoint;
1389 }
1390 let Some(lifecycle) = Lifecycle::from_service(&vm.state) else {
1391 return Err(Error::new(
1392 ErrorKind::Platform,
1393 format!(
1394 "microvm {microvm_id} reports state {}, which this client does not know; \
1395 refusing to guess which lifecycle guards apply.",
1396 vm.state,
1397 ),
1398 ));
1399 };
1400
1401 let mut sandbox = Self::with_control_plane(control);
1402 sandbox.adopted = true;
1403 sandbox.set_lifecycle(lifecycle);
1404 // The window the VM was launched with, so a keepalive on the adopted session paces
1405 // itself against the real idle policy rather than the platform minimum.
1406 sandbox.idle_window = vm
1407 .idle_policy
1408 .as_ref()
1409 .map(|policy| Duration::from_secs(u64::from(policy.max_idle_duration_seconds)));
1410 // A VM exists, so the image it was launched from did (STATE-1).
1411 sandbox.image_exists = true;
1412 // PENDING may still idle-suspend before this handle sees RUNNING (#195).
1413 sandbox.launch_adoptable = true;
1414 sandbox.pending_agent_token = Some(agent_token);
1415 match lifecycle {
1416 Lifecycle::Pending => {}
1417 Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended => {
1418 sandbox.token_installed = true;
1419 sandbox.bootstrap_count = 1;
1420 }
1421 Lifecycle::Terminating | Lifecycle::Terminated => {
1422 sandbox.bootstrap_count = 1;
1423 sandbox.was_terminated = true;
1424 }
1425 }
1426 sandbox.microvm = Some(vm);
1427 if lifecycle.is_live() {
1428 sandbox.build_session()?;
1429 }
1430 Ok(sandbox)
1431 }
1432
1433 /// A [`crate::names::NameRecord`] for this VM, to register under `name`.
1434 ///
1435 /// The egress posture is left unknown: the sandbox does not keep its launch request, and
1436 /// a record must not claim a network it cannot vouch for. The tunnel identity is kept when
1437 /// the sandbox holds one, launched with it or adopted from a record that carried it: a
1438 /// record without it can't verify the VM later, which is what the pair is for (#263).
1439 pub fn name_record(&self, name: &str) -> Result<crate::names::NameRecord, Error> {
1440 let Some(vm) = self.microvm() else {
1441 return Err(Error::new(
1442 ErrorKind::Precondition,
1443 "there is no VM to name yet: launch or adopt one first",
1444 ));
1445 };
1446 let Some(session) = self.session() else {
1447 return Err(Error::new(
1448 ErrorKind::Precondition,
1449 format!(
1450 "microvm {} has no session to take its agent token from",
1451 vm.id
1452 ),
1453 ));
1454 };
1455 // Stamped on the plane's clock, the one every other time this sandbox reads comes from.
1456 let mut record = crate::names::NameRecord::new_at(
1457 name,
1458 vm.id.as_str(),
1459 vm.endpoint.as_str(),
1460 session.agent_token(),
1461 self.control.region().as_str(),
1462 self.control.clock().unix_now().as_secs(),
1463 )?;
1464 record.set_tunnel_identity(self.tunnel_identity.as_ref());
1465 Ok(record)
1466 }
1467
1468 /// [`Sandbox::adopt`] from a [`crate::names::NameRecord`] kept in any store.
1469 ///
1470 /// Keeps the record's tunnel identity, so [`Sandbox::tunnel_identity`] and a later
1471 /// [`Sandbox::name_record`] carry it. A record holding one half of the pair, or a pair that
1472 /// doesn't decode, is refused before any AWS call rather than adopted without it: the
1473 /// record claims a verifiable VM, and dropping the claim would hide that it's broken.
1474 pub async fn adopt_record(
1475 control: ControlPlane,
1476 record: crate::names::NameRecord,
1477 ) -> Result<Self, Error> {
1478 let identity = record.tunnel_identity()?;
1479 let mut sandbox = Self::adopt(
1480 control,
1481 record.microvm_id,
1482 record.endpoint,
1483 record.agent_token,
1484 )
1485 .await?;
1486 sandbox.tunnel_identity = identity;
1487 Ok(sandbox)
1488 }
1489
1490 /// Hands the VM off to another process and returns what that process needs to adopt it.
1491 ///
1492 /// For a workflow whose steps run in different processes (a durable function's launch
1493 /// step, then later steps that [`Sandbox::adopt`]): the launching process calls this
1494 /// instead of dropping the sandbox, which would warn that a live VM was abandoned. The
1495 /// VM keeps running and nothing is sent to AWS. From here on this sandbox is inert — its
1496 /// session is dropped and `run`, `wait_until_running`, `suspend`, `resume`, and
1497 /// `terminate` are refused — because the adopter now owns the lifecycle, and a second
1498 /// handle driving it would bypass the adopter's guards.
1499 ///
1500 /// Refused (`Precondition`) when there is no live VM to hand off: nothing launched yet,
1501 /// already torn down or terminated, or already detached.
1502 pub fn detach(&mut self) -> Result<Detached, Error> {
1503 self.refuse_detached("detach")?;
1504 let Some(vm) = self.microvm.as_ref() else {
1505 return Err(Error::new(
1506 ErrorKind::Precondition,
1507 "nothing to detach: this sandbox has not launched a VM.",
1508 ));
1509 };
1510 if self.torn_down || !self.lifecycle.is_live() {
1511 return Err(Error::new(
1512 ErrorKind::Precondition,
1513 format!(
1514 "microvm {} is {} and there is no live VM to hand off; only a PENDING, \
1515 RUNNING, or SUSPENDED VM can be adopted.",
1516 vm.id, self.lifecycle,
1517 ),
1518 ));
1519 }
1520 let agent_token = self
1521 .session
1522 .as_ref()
1523 .map(|session| session.agent_token().to_string())
1524 .or_else(|| self.pending_agent_token.clone())
1525 .ok_or_else(|| {
1526 Error::new(
1527 ErrorKind::Precondition,
1528 format!(
1529 "microvm {} has no agent token to hand off; an adopter could not \
1530 reach its daemon.",
1531 vm.id
1532 ),
1533 )
1534 })?;
1535 let detached = Detached {
1536 microvm_id: vm.id.clone(),
1537 endpoint: vm.endpoint.clone(),
1538 region: self.control.region().clone(),
1539 port: self.control.port(),
1540 agent_token,
1541 };
1542 self.detached = true;
1543 self.session = None;
1544 self.pending_agent_token = None;
1545 Ok(detached)
1546 }
1547
1548 /// Whether [`Sandbox::detach`] handed this sandbox's VM to another process.
1549 pub fn detached(&self) -> bool {
1550 self.detached
1551 }
1552
1553 /// Refuses `what` on a sandbox whose VM was handed off.
1554 fn refuse_detached(&self, what: &str) -> Result<(), Error> {
1555 if !self.detached {
1556 return Ok(());
1557 }
1558 let id = self.microvm.as_ref().map_or("?", |vm| vm.id.as_str());
1559 Err(Error::new(
1560 ErrorKind::Precondition,
1561 format!(
1562 "cannot {what}: microvm {id} was detached and belongs to whoever adopts it. \
1563 Adopt it with `Sandbox::adopt` and {what} through that handle."
1564 ),
1565 ))
1566 }
1567
1568 /// Whether this sandbox was built by [`Sandbox::adopt`] rather than by its own launch.
1569 pub fn adopted(&self) -> bool {
1570 self.adopted
1571 }
1572
1573 // ── suspend (STATE-4, STATE-5, STATE-6) ──────────────────────────────────
1574
1575 /// Freezes the VM and waits for the platform to report it.
1576 ///
1577 /// A freeze and restore rather than a stop and start: the guest keeps its memory, so the
1578 /// token, the filesystem, and every exec record survive. The one thing that does not is
1579 /// the guest's view of time — it observes the whole suspension as a single jump, so any
1580 /// timeout, lease, or TLS session a running command holds expires at once on resume.
1581 ///
1582 /// # STATE-5 is checked before the wire, not after
1583 ///
1584 /// A suspend from anything but RUNNING is refused here with **zero** control-plane
1585 /// calls. That is the observable difference between this and a client that lets AWS
1586 /// answer, and it is what the test asserts on.
1587 pub async fn suspend(&mut self) -> Result<(), Error> {
1588 self.refuse_detached("suspend")?;
1589 let id = self.require_microvm("suspend")?;
1590
1591 // STATE-5.
1592 if self.lifecycle != Lifecycle::Running {
1593 return Err(Error::invalid_arg(format!(
1594 "microvm {id} is {} and a suspend is only issued from RUNNING (STATE-5). Refused \
1595 here rather than by the service, because the service's answer about a \
1596 non-running id does not say which of the two things went wrong — and a suspend \
1597 issued from SUSPENDED is a caller who believes they resumed.",
1598 self.lifecycle,
1599 )));
1600 }
1601
1602 // STATE-4: accepted, so the lifecycle moves before the wait. Acceptance is the
1603 // wire call succeeding, so the assignment comes after it: moving to SUSPENDING
1604 // first would leave a failed call (a throttle, a dead transport) stuck in a state
1605 // neither suspend nor resume accepts, bricking the handle over one bad request.
1606 self.control.suspend(&id).await?;
1607 self.set_lifecycle(Lifecycle::Suspending);
1608 // Stamped after the call and before the wait, not after the wait: the idlePolicy's
1609 // window starts when the platform begins suspending, so timing it from SUSPENDED
1610 // would under-count the transition and call a closed window open.
1611 self.suspended_at = Some(self.control.clock().elapsed());
1612
1613 let settled = self
1614 .control
1615 .wait_for_state(
1616 &id,
1617 &crate::control::microvm::SUSPEND_WANTED,
1618 &[],
1619 self.lifecycle_wait(),
1620 )
1621 .await?;
1622
1623 // STATE-6, and the TERMINATED case beside it. A VM that dies while suspending is a
1624 // state to report rather than an exception out of the middle of a teardown, so the
1625 // wait *wants* TERMINATED — and recording it here is what stops a resume from being
1626 // offered afterwards (STATE-11).
1627 let settled_lifecycle = match settled.state.as_str() {
1628 "SUSPENDED" => Lifecycle::Suspended,
1629 "TERMINATED" => {
1630 self.was_terminated = true;
1631 Lifecycle::Terminated
1632 }
1633 other => {
1634 return Err(Error::new(
1635 ErrorKind::Platform,
1636 format!(
1637 "the suspend wait returned {other}, which is neither SUSPENDED nor \
1638 TERMINATED — and those two are what this client asked for."
1639 ),
1640 ));
1641 }
1642 };
1643 self.set_lifecycle(settled_lifecycle);
1644 self.microvm = Some(settled);
1645 Ok(())
1646 }
1647
1648 // ── resume (STATE-7, STATE-8, STATE-12) ──────────────────────────────────
1649
1650 /// Thaws the VM and returns a usable session.
1651 ///
1652 /// # What is deliberately not re-delivered (STATE-7)
1653 ///
1654 /// Nothing. No run-hook payload, no token, no bootstrap. The in-memory token survived
1655 /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be
1656 /// refused — a 409 that reads like a broken VM.
1657 ///
1658 /// # The window is checked first (STATE-12)
1659 ///
1660 /// Before any wire call, because the answer is already known and calling costs the poll
1661 /// timeout to learn something worse. The falsification is a service reporting
1662 /// TERMINATED: without this check the resume burns the full deadline and then reports a
1663 /// state the client could have named at once.
1664 ///
1665 /// # The proxy token is dropped (STATE-8)
1666 ///
1667 /// [`Session::rebind`] invalidates it. The endpoint URL does not change across
1668 /// suspend/resume, so the rebind is usually a no-op on the URL — but a token minted
1669 /// against the pre-suspend instance may no longer validate, and that rejection reads
1670 /// exactly like a dead daemon.
1671 pub async fn resume(&mut self) -> Result<&mut Session, Error> {
1672 self.refuse_detached("resume")?;
1673 let id = self.require_microvm("resume")?;
1674
1675 // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal
1676 // comes before the window check and before any call.
1677 if self.was_terminated || self.lifecycle == Lifecycle::Terminated {
1678 return Err(Error::invalid_arg(format!(
1679 "microvm {id} was terminated, and a terminated VM never returns to RUNNING \
1680 (STATE-11). There is nothing to resume: the guest's memory is gone, so even a \
1681 call the service accepted would hand back a different machine."
1682 )));
1683 }
1684 if self.lifecycle != Lifecycle::Suspended {
1685 return Err(Error::invalid_arg(format!(
1686 "microvm {id} is {} and a resume is only issued from SUSPENDED (STATE-7).",
1687 self.lifecycle,
1688 )));
1689 }
1690
1691 // STATE-12, first and locally.
1692 self.require_open_suspended_window(&id)?;
1693
1694 self.control.resume(&id).await?;
1695 // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the
1696 // state this call was made from, so failing on it would fail every resume. A VM the
1697 // idlePolicy terminated during suspension never reaches RUNNING, and waiting only
1698 // for RUNNING there burns the full timeout and then reports a timeout message
1699 // hiding a cause the service had already stated in `stateReason`.
1700 let running = self
1701 .control
1702 .wait_for_state(
1703 &id,
1704 &["RUNNING"],
1705 &crate::constants::DEAD_STATES,
1706 self.lifecycle_wait(),
1707 )
1708 .await?;
1709
1710 self.set_lifecycle(Lifecycle::Running);
1711 // STATE-8, through the endpoint the service just reported rather than the one held:
1712 // the URL is measured not to change, and reading it from the response is what makes
1713 // that a fact this code depends on rather than an assumption it encodes.
1714 let endpoint = running.endpoint.clone();
1715 self.microvm = Some(running);
1716 if let Some(session) = self.session.as_mut() {
1717 session.rebind(endpoint);
1718 }
1719 // Cleared on success so the next cycle's window is measured from the next suspend.
1720 // Leaving it set would accumulate every suspension's elapsed time into one total and
1721 // reject a resume whose own window is wide open.
1722 self.suspended_at = None;
1723
1724 self.session.as_mut().ok_or_else(|| {
1725 Error::new(
1726 ErrorKind::Unexpected,
1727 "the VM resumed but this sandbox holds no session, which run() always builds"
1728 .to_string(),
1729 )
1730 })
1731 }
1732
1733 /// Rejects a resume the launch-time `idlePolicy` has already made impossible (STATE-12).
1734 ///
1735 /// The message names the elapsed time, the window, and the `idlePolicy` finding, because
1736 /// "cannot resume" alone sends a reader looking for the flag that reopens it.
1737 fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> {
1738 let reported = self
1739 .microvm
1740 .as_ref()
1741 .and_then(|vm| vm.idle_policy.as_ref())
1742 .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds)));
1743 let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at)
1744 else {
1745 // No window or no suspend time means this sandbox cannot know how long the VM
1746 // has been suspended, and guessing would refuse a resume the service would honour.
1747 return Ok(());
1748 };
1749 let elapsed = self.control.clock().elapsed().saturating_sub(since);
1750 if elapsed <= window {
1751 return Ok(());
1752 }
1753 Err(Error::new(
1754 ErrorKind::WindowClosed,
1755 format!(
1756 "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \
1757 window set at launch — the idlePolicy terminates a suspended VM once that window \
1758 passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \
1759 Refused before ResumeMicrovm, because calling would cost the full poll timeout \
1760 to learn the same thing less clearly. A longer window has to be set at launch \
1761 on the next VM; there is no call that extends this one.",
1762 elapsed.as_secs(),
1763 window.as_secs(),
1764 ),
1765 ))
1766 }
1767
1768 // ── terminate (STATE-9, STATE-10) ────────────────────────────────────────
1769
1770 /// Tears down, best-effort, never erroring.
1771 ///
1772 /// Order: VM, then image, then the log group **last**. The log group is last because the
1773 /// service can recreate a group deleted before its image — and see
1774 /// [`TeardownReport::undeleted`] for why this crate names it rather than deleting it.
1775 pub async fn terminate(&mut self, opts: TeardownOpts) -> TeardownReport {
1776 let mut report = TeardownReport::default();
1777 if let Err(error) = self.refuse_detached("terminate") {
1778 // Never erroring is `terminate`'s contract, so the refusal is reported rather
1779 // than raised; the VM is untouched and still belongs to its adopter.
1780 report.failures.push(error.to_string());
1781 report.lifecycle = Some(self.lifecycle);
1782 return report;
1783 }
1784 self.torn_down = true;
1785
1786 // The session first: it holds a cached proxy token whose only remaining use would be
1787 // a request against a VM that is going away.
1788 self.session = None;
1789
1790 // 1. The VM.
1791 if let Some(id) = self.microvm.as_ref().map(|vm| vm.id.clone()) {
1792 // STATE-9. Recorded before the call, so a terminate whose call fails still marks
1793 // the VM as one this client asked to destroy — which is what stops a later
1794 // resume (STATE-11) rather than leaving the sandbox looking resumable.
1795 self.set_lifecycle(Lifecycle::Terminating);
1796 self.was_terminated = true;
1797
1798 match self.control.terminate(&id).await {
1799 Ok(()) => report.terminate_accepted = true,
1800 Err(error) => {
1801 report.failures.push(format!("terminate {id}: {error}"));
1802 report.undeleted.push(id.clone());
1803 }
1804 }
1805
1806 if report.terminate_accepted
1807 && let Some(timeout) = opts.wait_for_terminated
1808 {
1809 let wait = WaitOpts::for_lifecycle(timeout);
1810 match self
1811 .control
1812 .wait_for_state(&id, &["TERMINATED"], &[], wait)
1813 .await
1814 {
1815 // STATE-10.
1816 Ok(settled) => {
1817 self.set_lifecycle(Lifecycle::Terminated);
1818 self.microvm = Some(settled);
1819 }
1820 // Not a leak: the platform accepted the terminate, so the VM is on its
1821 // way out and the lifecycle stays TERMINATING honestly.
1822 Err(error) => report
1823 .failures
1824 .push(format!("waiting for {id} to reach TERMINATED: {error}")),
1825 }
1826 }
1827 }
1828
1829 // 2. The image, retrying — an image in CREATING refuses deletion and a VM still
1830 // terminating holds a reference.
1831 if opts.delete_image {
1832 match self.image.as_ref().map(|image| image.identifier.clone()) {
1833 Some(identifier) => {
1834 let deleted = self
1835 .control
1836 .delete_image(&identifier, opts.delete_attempts, opts.delete_backoff)
1837 .await;
1838 report.image_deleted = Some(deleted);
1839 if deleted {
1840 self.image_exists = false;
1841 } else {
1842 report.failures.push(format!(
1843 "the image {identifier} survived {} delete attempts",
1844 opts.delete_attempts.max(1)
1845 ));
1846 report.undeleted.push(identifier);
1847 }
1848 }
1849 None => report.image_deleted = Some(false),
1850 }
1851 }
1852
1853 // 3. The log group, LAST. Named rather than deleted; see TeardownReport::undeleted.
1854 if opts.delete_log_group
1855 && let Some(group) = self.image.as_ref().map(Image::build_log_group)
1856 {
1857 report.failures.push(format!(
1858 "the build log group {group} was not deleted: CloudWatch Logs is not in this \
1859 crate's dependency set, so it is reported rather than removed. It is \
1860 service-created, which means no Terraform stack owns it and `terraform destroy` \
1861 leaves it behind — six accumulated before anyone noticed."
1862 ));
1863 report.undeleted.push(group);
1864 }
1865
1866 report.lifecycle = Some(self.lifecycle);
1867 report
1868 }
1869
1870 // ── internals ────────────────────────────────────────────────────────────
1871
1872 /// The VM id, or a precondition error naming what was attempted.
1873 fn require_microvm(&self, what: &str) -> Result<String, Error> {
1874 self.microvm
1875 .as_ref()
1876 .map(|vm| vm.id.clone())
1877 .ok_or_else(|| {
1878 Error::new(
1879 ErrorKind::Precondition,
1880 format!("nothing to {what}: this sandbox has not launched a VM."),
1881 )
1882 })
1883 }
1884
1885 /// Five minutes at five-second polls, for suspend, resume, and terminate.
1886 fn lifecycle_wait(&self) -> WaitOpts {
1887 WaitOpts::for_lifecycle(DEFAULT_LIFECYCLE_TIMEOUT)
1888 }
1889}
1890
1891/// Warns about a live VM rather than tearing it down.
1892///
1893/// See the module docs: `Drop` cannot await, so a teardown here would either deadlock
1894/// inside a runtime or race the process exit. The warning names the id, because the only
1895/// useful thing a drop can do is tell whoever reads stderr what to go delete.
1896///
1897/// The warning goes through the plane's [`Adapters::warn`](crate::adapters::Adapters::warn),
1898/// which in production writes it to stderr.
1899impl Sandbox {
1900 /// The warning `Drop` writes for a live VM nobody took responsibility for, or `None`
1901 /// when dropping is not a leak: torn down, adopted (its launcher owns it), or detached
1902 /// (its adopter owns it).
1903 fn drop_warning(&self) -> Option<String> {
1904 if self.torn_down || self.adopted || self.detached {
1905 return None;
1906 }
1907 let vm = self.microvm.as_ref()?;
1908 self.lifecycle.is_live().then(|| {
1909 format!(
1910 "warning: the Sandbox for microvm {} was dropped in {} without terminate() or \
1911 detach(). Nothing was torn down — Drop cannot await, so a teardown here would \
1912 deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \
1913 ceiling: terminate it with `microvm terminate {}`.",
1914 vm.id, self.lifecycle, vm.id,
1915 )
1916 })
1917 }
1918}
1919
1920impl Drop for Sandbox {
1921 fn drop(&mut self) {
1922 if let Some(warning) = self.drop_warning() {
1923 self.control.adapters().warn(&warning);
1924 }
1925 }
1926}
1927
1928/// A fresh per-VM bearer token: 32 bytes from `entropy` as 64 hex characters.
1929///
1930/// The hand-rolled `/dev/urandom` read this once replaced carried a clock-mixing fallback
1931/// for a failed read. An unavailable source refuses the launch instead: minting a bearer
1932/// token from a clock would be worse than not launching.
1933fn mint_agent_token(entropy: &dyn crate::entropy::Entropy) -> Result<String, Error> {
1934 let mut bytes = [0u8; 32];
1935 entropy.fill(&mut bytes)?;
1936 Ok(const_hex::encode(bytes))
1937}
1938
1939#[cfg(test)]
1940mod tests {
1941 use super::*;
1942 use crate::control::fake::{self as fake, Answer, FakeControlPlane, TestClock};
1943 use crate::control::transport::Method;
1944
1945 /// A sandbox over the contract recorder, plus the two handles a test asserts through.
1946 ///
1947 /// The recorder is T-W2-4's, deliberately: its answers are literal JSON in the service
1948 /// model's spelling rather than values serialized from this crate's types, so a member
1949 /// this lane misreads cannot be misread identically by the fake.
1950 fn planted() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
1951 let recorder = Arc::new(FakeControlPlane::new());
1952 let clock = Arc::new(TestClock::new());
1953 let plane = crate::testing::control_plane(
1954 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
1955 Region::UsEast1,
1956 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
1957 );
1958 // A daemon that answers health, because a launch waits for one (#254).
1959 let sandbox = Sandbox::with_control_plane(plane)
1960 .with_session_backend(crate::testing::HealthyDaemon::new());
1961 (sandbox, recorder, clock)
1962 }
1963
1964 /// Suspend, resume and terminate wait five minutes at five-second polls, with no stall
1965 /// probe: core's lifecycle wait, not the image build's 45-minute default (#266).
1966 #[test]
1967 fn the_suspend_resume_and_terminate_waits_are_cores_lifecycle_wait() {
1968 let (sandbox, _, _) = planted();
1969 let wait = sandbox.lifecycle_wait();
1970 assert_eq!(
1971 (wait.timeout, wait.poll_interval, wait.stall_grace),
1972 (
1973 DEFAULT_LIFECYCLE_TIMEOUT,
1974 LIFECYCLE_POLL_INTERVAL,
1975 Duration::MAX
1976 )
1977 );
1978 }
1979
1980 /// Queues everything a launch to RUNNING needs.
1981 fn answer_launch(recorder: &FakeControlPlane) {
1982 recorder
1983 .answer(
1984 "RunMicrovm",
1985 Answer::ok(fake::microvm_response("PENDING", None)),
1986 )
1987 .answer(
1988 "GetMicrovm",
1989 Answer::ok(fake::microvm_response("RUNNING", None)),
1990 )
1991 .answer(
1992 "CreateMicrovmAuthToken",
1993 Answer::ok(fake::auth_token_response("proxy-token")),
1994 );
1995 }
1996
1997 #[tokio::test]
1998 async fn vpc_egress_configuration_survives_the_sandbox_launch() {
1999 let (mut sandbox, recorder, _) = planted();
2000 answer_launch(&recorder);
2001 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2002 let request = RunRequest::new()
2003 .with_image("arn:image")
2004 .with_egress_network_connector(arn);
2005 assert_eq!(
2006 request.egress_posture(),
2007 crate::control::EgressPosture::Unsealed
2008 );
2009 sandbox.run(request).await.expect("launch");
2010 let body = recorder.first_body("RunMicrovm");
2011 assert_eq!(body["egressNetworkConnectors"], serde_json::json!([arn]));
2012 }
2013
2014 /// **A sandbox's session reads the plane's clock.** Its exec ids carry the plane's wall
2015 /// reading, so the lifecycle and the session it hands out share one time source.
2016 ///
2017 /// **Falsification**: drop `with_clock(self.control.shared_clock())` from `build_session`
2018 /// and the session takes the adapters' clock, whose wall reading is another instant.
2019 #[tokio::test]
2020 async fn a_sandbox_session_reads_the_planes_clock() {
2021 let (mut sandbox, recorder, clock) = planted();
2022 answer_launch(&recorder);
2023 let session = sandbox
2024 .run(RunRequest::new().with_image("arn:image"))
2025 .await
2026 .expect("the launch reaches RUNNING");
2027 let pinned = Duration::from_secs(1_767_225_600) + Duration::from_nanos(987_654_321);
2028 clock.set_unix_now(pinned);
2029 let id = session.mint_exec_id();
2030 let value =
2031 u64::from_str_radix(id.strip_prefix("x-").expect("the x- shape"), 16).expect("hex");
2032 let low_bits = (1_u64 << 40) - 1;
2033 assert_eq!(
2034 value & low_bits,
2035 (pinned.as_nanos() as u64) & low_bits,
2036 "{id} wasn't minted from the plane's clock"
2037 );
2038 }
2039
2040 /// **`ensure_image` builds its STS and S3 client through the plane's adapters**, so a
2041 /// plane over test adapters never reaches a real credential chain.
2042 ///
2043 /// **Falsification**: build `SignedBuildServices` inline again, and the refusal (or, with
2044 /// credentials in the environment, the call) is the real chain's rather than the
2045 /// adapters'.
2046 #[tokio::test]
2047 async fn ensure_image_builds_its_services_through_the_planes_adapters() {
2048 let (mut sandbox, recorder, _) = planted();
2049 let dockerfile = crate::control::artifact::wrap_dockerfile(
2050 "FROM python:3.12-slim\nWORKDIR /app\n",
2051 &crate::control::artifact::WrapOptions::default(),
2052 )
2053 .expect("a wrappable Dockerfile");
2054 let request = crate::control::EnsureImageRequest::new(
2055 "task",
2056 crate::testing::aarch64_daemon(b"daemon"),
2057 dockerfile,
2058 "artifact-bucket",
2059 "arn:aws:iam::123456789012:role/build",
2060 );
2061 let error = sandbox
2062 .ensure_image(request)
2063 .await
2064 .expect_err("the test adapters build no services");
2065 assert_eq!(error.kind(), ErrorKind::Credentials, "{error}");
2066 assert!(
2067 error
2068 .to_string()
2069 .contains("the test adapters build no STS or S3 client for us-east-1"),
2070 "{error}"
2071 );
2072 assert!(
2073 recorder.calls().is_empty(),
2074 "no control-plane call came first"
2075 );
2076 }
2077
2078 /// **A launch draws its agent token, identity seeds and client token from the plane's
2079 /// entropy**, in that order, so the one source a caller or a test swaps in decides all
2080 /// three.
2081 ///
2082 /// **Falsification**: mint any of the three from `OsEntropy` instead of
2083 /// `self.control.entropy()` and its assertion below reads a value the scripted source
2084 /// never produced.
2085 #[tokio::test]
2086 async fn a_launch_draws_every_random_value_from_the_planes_entropy() {
2087 use crate::entropy::testing::SequenceEntropy;
2088 use base64::Engine as _;
2089
2090 let recorder = Arc::new(FakeControlPlane::new());
2091 let entropy = Arc::new(SequenceEntropy::new());
2092 let plane = crate::testing::control_plane(
2093 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2094 Region::UsEast1,
2095 Arc::new(TestClock::new()),
2096 )
2097 .with_entropy(Arc::clone(&entropy) as Arc<dyn crate::entropy::Entropy>);
2098 let mut sandbox = Sandbox::with_control_plane(plane)
2099 .with_session_backend(crate::testing::HealthyDaemon::new());
2100 answer_launch(&recorder);
2101 let session = sandbox
2102 .run(RunRequest::new().with_image("arn:image").with_identity())
2103 .await
2104 .expect("the launch reaches RUNNING");
2105
2106 assert_eq!(
2107 session.agent_token(),
2108 const_hex::encode(SequenceEntropy::draw(1, 32)),
2109 "the agent token"
2110 );
2111 let body = recorder.first_body("RunMicrovm");
2112 let payload: serde_json::Value =
2113 serde_json::from_str(body["runHookPayload"].as_str().expect("a string payload"))
2114 .expect("the payload is JSON");
2115 assert_eq!(
2116 payload[protocol::identity::SEED_KEY],
2117 base64::engine::general_purpose::STANDARD
2118 .encode(SequenceEntropy::draw(2, protocol::identity::SEED_BYTES)),
2119 "the VM seed"
2120 );
2121 let client_token = body["clientToken"].as_str().expect("a client token");
2122 assert!(
2123 client_token.ends_with(&const_hex::encode(SequenceEntropy::draw(4, 8))),
2124 "the client token's nonce: {client_token}"
2125 );
2126 assert_eq!(
2127 entropy.calls(),
2128 4,
2129 "one draw per value, the host seed included"
2130 );
2131 sandbox.detach().expect("hand the scripted VM off quietly");
2132 }
2133
2134 /// An unavailable entropy source refuses the launch before anything reaches the plane.
2135 #[tokio::test]
2136 async fn an_unavailable_entropy_source_refuses_the_launch() {
2137 use crate::entropy::testing::SequenceEntropy;
2138
2139 let recorder = Arc::new(FakeControlPlane::new());
2140 let plane = crate::testing::control_plane(
2141 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2142 Region::UsEast1,
2143 Arc::new(TestClock::new()),
2144 )
2145 .with_entropy(Arc::new(SequenceEntropy::unavailable()));
2146 let mut sandbox = Sandbox::with_control_plane(plane)
2147 .with_session_backend(crate::testing::HealthyDaemon::new());
2148 answer_launch(&recorder);
2149 let err = sandbox
2150 .run(RunRequest::new().with_image("arn:image"))
2151 .await
2152 .expect_err("no pool, no bearer token");
2153 assert_eq!(err.kind(), ErrorKind::Unexpected, "{err}");
2154 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2155 }
2156
2157 // ── a launch waits for the daemon (#254) ──────────────────────────────────
2158
2159 /// A sandbox whose daemon answers health from `replies`, over a scripted plane that
2160 /// answers a launch to RUNNING.
2161 fn launching_to(
2162 replies: Vec<crate::testing::Reply>,
2163 ) -> (
2164 Sandbox,
2165 Arc<FakeControlPlane>,
2166 Arc<crate::testing::Recorder>,
2167 ) {
2168 let recorder = Arc::new(FakeControlPlane::new());
2169 let plane = crate::testing::control_plane(
2170 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2171 Region::UsEast1,
2172 Arc::new(TestClock::new()),
2173 );
2174 let daemon = crate::testing::Recorder::with(replies);
2175 let sandbox = Sandbox::with_control_plane(plane).with_session_backend(daemon.clone());
2176 answer_launch(&recorder);
2177 (sandbox, recorder, daemon)
2178 }
2179
2180 /// Not yet bootstrapped, then bootstrapped: the two health answers a launch waits through.
2181 fn not_ready_then_ready() -> Vec<crate::testing::Reply> {
2182 vec![
2183 crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()),
2184 crate::testing::Reply::ok(crate::testing::health_body(true)),
2185 ]
2186 }
2187
2188 /// The paths a scripted daemon was asked for, in order.
2189 fn daemon_paths(daemon: &crate::testing::Recorder) -> Vec<String> {
2190 daemon
2191 .requests()
2192 .iter()
2193 .map(|request| format!("{} {}", request.method, request.path))
2194 .collect()
2195 }
2196
2197 /// **`run` returns once the daemon answers, not at RUNNING (#254).** The plane reports
2198 /// RUNNING and the daemon answers 503 and then 200 bootstrapped: `run` hands back the
2199 /// session after the second poll, as the CLI's `run` and `AgentVm.launch` did on their own.
2200 ///
2201 /// **Falsification**: drop the `wait_until_ready` call from `Sandbox::wait_until_running`
2202 /// and `run` returns at RUNNING having asked the daemon nothing.
2203 #[tokio::test(start_paused = true)]
2204 async fn a_launch_returns_once_its_daemon_answers() {
2205 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2206 sandbox
2207 .run(RunRequest::new().with_image("arn:image"))
2208 .await
2209 .expect("the launch reaches RUNNING and its daemon answers");
2210 assert_eq!(
2211 daemon_paths(&daemon),
2212 ["GET /v1/health", "GET /v1/health"],
2213 "one poll refused, one bootstrapped, and nothing after"
2214 );
2215 sandbox.detach().expect("hand the scripted VM off quietly");
2216 }
2217
2218 /// **A launch that didn't wait is finished the same way (#254):** `wait_until_running`
2219 /// after `run(wait = false)` returns once the daemon answers too, and the accepted launch
2220 /// itself asked the daemon nothing.
2221 #[tokio::test(start_paused = true)]
2222 async fn wait_until_running_returns_once_the_daemon_answers() {
2223 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2224 let mut request = RunRequest::new().with_image("arn:image");
2225 request.wait = false;
2226 sandbox.run(request).await.expect("the launch is accepted");
2227 assert!(
2228 daemon_paths(&daemon).is_empty(),
2229 "an accepted launch doesn't wait"
2230 );
2231 sandbox
2232 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
2233 .await
2234 .expect("RUNNING, then the daemon answers");
2235 assert_eq!(daemon_paths(&daemon), ["GET /v1/health", "GET /v1/health"]);
2236 sandbox.detach().expect("hand the scripted VM off quietly");
2237 }
2238
2239 /// **A daemon that never answers fails the launch with `ERR_TIMEOUT` (#254),** after the
2240 /// bootstrap wait, rather than the first exec failing later. The VM did reach RUNNING, so
2241 /// the lifecycle says so and the token counts as installed (STATE-2).
2242 #[tokio::test(start_paused = true)]
2243 async fn a_launch_whose_daemon_never_answers_times_out_running() {
2244 let never = (0..128)
2245 .map(|_| crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()))
2246 .collect();
2247 let (mut sandbox, _recorder, daemon) = launching_to(never);
2248 let error = sandbox
2249 .run(RunRequest::new().with_image("arn:image"))
2250 .await
2251 .expect_err("the daemon never answered");
2252 assert_eq!(error.kind(), ErrorKind::Timeout, "{error}");
2253 assert!(daemon.requests().len() > 1, "it kept asking");
2254 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2255 assert!(sandbox.token_installed());
2256 sandbox.detach().expect("hand the scripted VM off quietly");
2257 }
2258
2259 // ── a launch by bare image name (#253) ───────────────────────────────────
2260
2261 /// The ARN the fake's listing gives `wanted-image`.
2262 const WANTED_ARN: &str = "arn:aws:lambda:us-east-1:123456789012:microvm-image:wanted-image";
2263
2264 /// A listing that holds `wanted-image` beside a name it's a prefix of.
2265 fn answer_listing(recorder: &FakeControlPlane) {
2266 recorder.answer(
2267 "ListMicrovmImages",
2268 Answer::ok(fake::list_images_response(
2269 &["wanted-image-old", "wanted-image"],
2270 None,
2271 )),
2272 );
2273 }
2274
2275 /// **A launch by bare image name sends the listed ARN**, resolved before `RunMicrovm`, as
2276 /// `run --image NAME` did. Every surface's launch is this `run`, and the SDKs used to send
2277 /// the name verbatim, which the service answers with HTTP 400 "Malformed ARN".
2278 ///
2279 /// **Falsification**: drop the `resolve_image_arn` call from `Sandbox::run` and the calls
2280 /// start at `RunMicrovm`, whose body carries `wanted-image`.
2281 #[tokio::test]
2282 async fn a_launch_by_bare_image_name_sends_the_listed_arn() {
2283 let (mut sandbox, recorder, _) = planted();
2284 answer_listing(&recorder);
2285 answer_launch(&recorder);
2286 sandbox
2287 .run(RunRequest::new().with_image("wanted-image"))
2288 .await
2289 .expect("the launch reaches RUNNING");
2290
2291 let operations = recorder.operations();
2292 assert_eq!(
2293 operations.get(..2),
2294 Some(&["ListMicrovmImages", "RunMicrovm"][..]),
2295 "the name is resolved, once, before the launch: {operations:?}"
2296 );
2297 assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2298 assert_eq!(
2299 recorder.first_body("RunMicrovm")["imageIdentifier"],
2300 WANTED_ARN,
2301 "the launch carries the exact match's ARN, never the name"
2302 );
2303 sandbox.detach().expect("hand the scripted VM off quietly");
2304 }
2305
2306 /// **An image ARN launches with no listing call**, and is the ARN the sandbox records.
2307 #[tokio::test]
2308 async fn a_launch_by_image_arn_makes_no_listing_call() {
2309 let (mut sandbox, recorder, _) = planted();
2310 answer_launch(&recorder);
2311 sandbox
2312 .run(RunRequest::new().with_image(WANTED_ARN))
2313 .await
2314 .expect("the launch reaches RUNNING");
2315
2316 assert_eq!(recorder.call_count("ListMicrovmImages"), 0);
2317 assert_eq!(recorder.operations().first(), Some(&"RunMicrovm"));
2318 assert_eq!(
2319 recorder.first_body("RunMicrovm")["imageIdentifier"],
2320 WANTED_ARN
2321 );
2322 assert_eq!(sandbox.launch_image_arn(), Some(WANTED_ARN));
2323 sandbox.detach().expect("hand the scripted VM off quietly");
2324 }
2325
2326 /// **The resolved ARN is recorded before `RunMicrovm`, so a launch the service refuses
2327 /// still names the image it asked for.** The CLI's failure envelope reads it from here:
2328 /// `microvm()` holds only what an accepted launch answered.
2329 ///
2330 /// **Falsification**: drop the `launch_image_arn` assignment from `Sandbox::run` and the
2331 /// accessor reads `None` after the refusal.
2332 #[tokio::test]
2333 async fn a_refused_launch_by_name_still_records_the_resolved_arn() {
2334 let (mut sandbox, recorder, _) = planted();
2335 answer_listing(&recorder);
2336 recorder.answer("RunMicrovm", Answer::failure(400, "scripted refusal"));
2337 sandbox
2338 .run(RunRequest::new().with_image("wanted-image"))
2339 .await
2340 .expect_err("the service refused the launch");
2341
2342 assert_eq!(
2343 sandbox.launch_image_arn(),
2344 Some(WANTED_ARN),
2345 "the ARN the refused launch asked for"
2346 );
2347 assert!(sandbox.microvm().is_none(), "no launch was accepted");
2348 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2349 }
2350
2351 /// **A name no image carries is `ERR_PRECONDITION` before any launch, and the sandbox is
2352 /// left as it was**: nothing recorded (STATE-1 counts only an accepted launch), so the
2353 /// same sandbox launches once the caller passes an image that exists.
2354 #[tokio::test]
2355 async fn an_unknown_image_name_is_refused_before_the_launch_and_the_sandbox_still_runs() {
2356 let (mut sandbox, recorder, _) = planted();
2357 recorder.answer(
2358 "ListMicrovmImages",
2359 Answer::ok(fake::list_images_response(&[], None)),
2360 );
2361 let error = sandbox
2362 .run(RunRequest::new().with_image("no-such-image"))
2363 .await
2364 .expect_err("nothing to launch from");
2365 assert_eq!(error.code(), "ERR_PRECONDITION");
2366 assert!(error.to_string().contains("no-such-image"), "{error}");
2367 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2368 assert_eq!(sandbox.launch_image_arn(), None);
2369 assert!(!sandbox.image_exists(), "STATE-1: no launch was accepted");
2370 assert!(sandbox.microvm().is_none());
2371 assert_eq!(sandbox.bootstrap_count(), 0);
2372
2373 answer_launch(&recorder);
2374 sandbox
2375 .run(RunRequest::new().with_image(WANTED_ARN))
2376 .await
2377 .expect("a failed resolution doesn't use up the sandbox's one launch");
2378 assert_eq!(sandbox.bootstrap_count(), 1);
2379 sandbox.detach().expect("hand the scripted VM off quietly");
2380 }
2381
2382 /// **A launch this client refuses costs no call, a bare name included**: the local
2383 /// refusals come before the listing a name costs.
2384 ///
2385 /// **Falsification**: drop the `launch_connectors` call from `Sandbox::run` and the
2386 /// out-of-range duration is refused by `run_microvm` only after the listing went out.
2387 #[tokio::test]
2388 async fn a_launch_refused_locally_makes_no_listing_call_for_a_name() {
2389 let (mut sandbox, recorder, _) = planted();
2390 answer_listing(&recorder);
2391 let mut request = RunRequest::new().with_image("wanted-image");
2392 request.max_duration_sec = 0;
2393 let error = sandbox
2394 .run(request)
2395 .await
2396 .expect_err("a zero maximum duration is refused locally");
2397 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
2398 assert!(
2399 recorder.calls().is_empty(),
2400 "the refusal came after a call: {:?}",
2401 recorder.operations()
2402 );
2403 assert_eq!(sandbox.launch_image_arn(), None);
2404 }
2405
2406 /// **BIND-12: the launched session carries its request's posture**, the value
2407 /// `egress_posture_for` answers for the same options and the CLI envelope reports
2408 /// (the CLI's `guards/run.rs` holds that half). Each of the four launchable shapes.
2409 ///
2410 /// **Falsification** — 2026-09-24. Build the session without `with_egress_posture` and
2411 /// the `open` and `best-effort` rows read `unsealed`; restored.
2412 #[tokio::test]
2413 async fn a_launched_session_reports_its_requests_posture() {
2414 use crate::control::EgressPosture;
2415 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2416 let rows = [
2417 (RunRequest::new(), EgressPosture::Unsealed),
2418 (RunRequest::new().with_egress(), EgressPosture::Open),
2419 (
2420 RunRequest::new().with_deny_egress(),
2421 EgressPosture::BestEffort,
2422 ),
2423 (
2424 RunRequest::new().with_egress_network_connector(arn),
2425 EgressPosture::Unsealed,
2426 ),
2427 ];
2428 for (request, expected) in rows {
2429 let answered = crate::control::egress_posture_for(
2430 request.egress,
2431 &request.egress_network_connectors,
2432 request.deny_egress,
2433 Some(&Region::UsEast1),
2434 )
2435 .expect("a launchable request");
2436 assert_eq!(answered, expected);
2437 let (mut sandbox, recorder, _) = planted();
2438 answer_launch(&recorder);
2439 let session = sandbox
2440 .run(request.with_image("arn:image"))
2441 .await
2442 .expect("the launch reaches RUNNING");
2443 assert_eq!(session.egress_posture(), expected, "BIND-12");
2444 assert_eq!(sandbox.egress_posture(), expected);
2445 sandbox.detach().expect("hand the scripted VM off quietly");
2446 }
2447 }
2448
2449 /// **BIND-12: a session that does not hold its launch options reports `unsealed`**, the
2450 /// weakest true claim, whatever the VM was launched with: an adopted VM and a session
2451 /// attached directly both lack the request.
2452 #[tokio::test]
2453 async fn a_session_without_its_launch_options_reports_unsealed() {
2454 use crate::control::EgressPosture;
2455 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2456 assert_eq!(
2457 sandbox.session().expect("RUNNING").egress_posture(),
2458 EgressPosture::Unsealed
2459 );
2460 assert_eq!(sandbox.egress_posture(), EgressPosture::Unsealed);
2461 let direct = Session::builder(ADOPT_ENDPOINT, ADOPT_TOKEN)
2462 .build_with(&crate::testing::TestAdapters::new())
2463 .expect("a direct session");
2464 assert_eq!(direct.egress_posture(), EgressPosture::Unsealed);
2465 }
2466
2467 /// A launched sandbox in RUNNING, which is where four of the twelve keys start.
2468 async fn launched() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2469 let (mut sandbox, recorder, clock) = planted();
2470 answer_launch(&recorder);
2471 sandbox
2472 .run(RunRequest::new().with_image("arn:image"))
2473 .await
2474 .expect("the launch reaches RUNNING");
2475 (sandbox, recorder, clock)
2476 }
2477
2478 /// **A dropped live sandbox warns through the plane's adapters**, not a stream of its
2479 /// own, so the driving adapter decides where the warning goes (ARCH-7).
2480 ///
2481 /// **Falsification**: write the warning straight to stderr in `Drop` and the recorded
2482 /// list comes back empty (and the crate's `clippy.toml` refuses `std::io::stderr`).
2483 #[tokio::test]
2484 async fn a_dropped_live_sandbox_warns_through_the_planes_adapters() {
2485 let recorder = Arc::new(FakeControlPlane::new());
2486 let clock = Arc::new(TestClock::new());
2487 let adapters = Arc::new(crate::testing::TestAdapters::new());
2488 let plane = crate::control::ControlPlane::from_ports(
2489 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2490 Region::UsEast1,
2491 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2492 Arc::new(crate::testing::SequenceEntropy::new()),
2493 Arc::clone(&adapters) as Arc<dyn crate::adapters::Adapters>,
2494 );
2495 let mut sandbox = Sandbox::with_control_plane(plane)
2496 .with_session_backend(crate::testing::HealthyDaemon::new());
2497 answer_launch(&recorder);
2498 sandbox
2499 .run(RunRequest::new().with_image("arn:image"))
2500 .await
2501 .expect("the launch reaches RUNNING");
2502 let id = sandbox.microvm().expect("launched").id.clone();
2503 drop(sandbox);
2504 let warnings = adapters.warnings();
2505 assert_eq!(warnings.len(), 1, "{warnings:?}");
2506 assert!(warnings[0].contains(&id), "{warnings:?}");
2507 }
2508
2509 /// **Detach hands off, and nothing warns or moves afterwards.** A launched sandbox
2510 /// warns on drop; once detached it returns what an adopter needs (with the token kept
2511 /// out of `Debug`), stops warning, makes no further control-plane call, and refuses every
2512 /// transition — including `terminate`, which reports the refusal instead of raising.
2513 ///
2514 /// **Falsification** — 2026-09-24. Leave `detached` out of `drop_warning`'s early
2515 /// return and the "no warning after detach" assertion goes red; drop the
2516 /// `refuse_detached` guard from `suspend` and the zero-calls assertion goes red (the
2517 /// suspend reaches `SuspendMicrovm`). Both restored.
2518 #[tokio::test]
2519 async fn a_detached_sandbox_hands_off_its_vm_and_stays_quiet() {
2520 let (mut sandbox, recorder, _clock) = launched().await;
2521 assert!(
2522 sandbox.drop_warning().is_some(),
2523 "a launched, live VM warns on drop"
2524 );
2525 let token = sandbox
2526 .session()
2527 .expect("launched")
2528 .agent_token()
2529 .to_string();
2530 let vm = sandbox.microvm().expect("launched").clone();
2531
2532 let detached = sandbox.detach().expect("a RUNNING VM detaches");
2533 assert_eq!(detached.microvm_id, vm.id);
2534 assert_eq!(detached.endpoint, vm.endpoint);
2535 assert_eq!(detached.agent_token(), token);
2536 assert_eq!(detached.region, Region::UsEast1);
2537 let printed = format!("{detached:?} {sandbox:?}");
2538 assert!(!printed.contains(&token), "{printed}");
2539 assert!(printed.contains("detached: true"), "{printed}");
2540
2541 assert!(sandbox.detached());
2542 assert!(
2543 sandbox.drop_warning().is_none(),
2544 "detached is not abandoned"
2545 );
2546 assert!(
2547 sandbox.session().is_none(),
2548 "the session went to the adopter"
2549 );
2550
2551 let before = recorder.calls().len();
2552 for error in [
2553 sandbox.suspend().await.expect_err("detached"),
2554 sandbox.resume().await.map(|_| ()).expect_err("detached"),
2555 sandbox
2556 .run(RunRequest::new().with_image("arn:image"))
2557 .await
2558 .map(|_| ())
2559 .expect_err("detached"),
2560 sandbox
2561 .wait_until_running(Duration::from_secs(1))
2562 .await
2563 .map(|_| ())
2564 .expect_err("detached"),
2565 sandbox.detach().map(|_| ()).expect_err("already detached"),
2566 ] {
2567 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2568 assert!(error.to_string().contains("detached"), "{error}");
2569 assert!(!error.to_string().contains(&token), "{error}");
2570 }
2571 let report = sandbox.terminate(TeardownOpts::default()).await;
2572 assert!(!report.terminate_accepted);
2573 assert!(report.failures[0].contains("detached"), "{report:?}");
2574 assert_eq!(recorder.calls().len(), before, "detached makes no AWS call");
2575 assert!(
2576 sandbox.drop_warning().is_none(),
2577 "a refused terminate is still quiet"
2578 );
2579 }
2580
2581 /// Detach needs a live VM: nothing launched, or already torn down, is refused.
2582 #[tokio::test]
2583 async fn detach_refuses_without_a_live_vm() {
2584 let (mut unlaunched, _recorder, _clock) = planted();
2585 let error = unlaunched.detach().expect_err("nothing launched");
2586 assert_eq!(error.kind(), ErrorKind::Precondition);
2587 assert!(!unlaunched.detached());
2588
2589 let (mut sandbox, recorder, _clock) = launched().await;
2590 recorder
2591 .answer("TerminateMicrovm", Answer::ok("{}"))
2592 .answer(
2593 "GetMicrovm",
2594 Answer::ok(fake::microvm_response("TERMINATED", None)),
2595 );
2596 sandbox.terminate(TeardownOpts::default()).await;
2597 let error = sandbox.detach().expect_err("torn down");
2598 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2599 assert!(!sandbox.detached());
2600 }
2601
2602 /// Drives a launched sandbox to SUSPENDED, which is where resume starts.
2603 ///
2604 /// Both `GetMicrovm` answers are queued **before** the launch rather than one before
2605 /// each wait, and that is not a style choice. The recorder repeats its last queued
2606 /// answer, so a queue left at `[RUNNING]` after the launch and then appended to becomes
2607 /// `[RUNNING, SUSPENDED]` — the suspend wait pops the stale RUNNING, sleeps a poll
2608 /// interval, and the fake's `sleep` **advances the clock**. That five seconds is
2609 /// invisible until it lands in the middle of the STATE-12 window arithmetic, which is
2610 /// exactly where it first showed up. Queueing both up front means every wait matches on
2611 /// its first poll and no test clock moves except when a test moves it.
2612 async fn suspended_with_window(
2613 suspended_sec: u32,
2614 ) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2615 let (mut sandbox, recorder, clock) = planted();
2616 recorder
2617 .answer(
2618 "RunMicrovm",
2619 Answer::ok(fake::microvm_response("PENDING", None)),
2620 )
2621 .answer(
2622 "GetMicrovm",
2623 Answer::ok(fake::microvm_response("RUNNING", None)),
2624 )
2625 .answer(
2626 "GetMicrovm",
2627 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2628 )
2629 .answer(
2630 "CreateMicrovmAuthToken",
2631 Answer::ok(fake::auth_token_response("proxy-token")),
2632 )
2633 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2634
2635 sandbox
2636 .run(
2637 RunRequest::new()
2638 .with_image("arn:image")
2639 .with_suspended_sec(suspended_sec),
2640 )
2641 .await
2642 .expect("launches");
2643 sandbox.suspend().await.expect("suspends");
2644 assert_eq!(
2645 clock.now(),
2646 Duration::ZERO,
2647 "no wait may have slept, or the window arithmetic below is measuring the fake's \
2648 poll interval as well as the elapsed suspension"
2649 );
2650 (sandbox, recorder, clock)
2651 }
2652
2653 /// A suspended sandbox at the default ten-minute window.
2654 async fn suspended() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2655 suspended_with_window(600).await
2656 }
2657
2658 // ── adopt (#196) ─────────────────────────────────────────────────────────
2659
2660 /// A canary agent token, so a test can assert it appears in no `Debug` or error.
2661 const ADOPT_TOKEN: &str = "adopt-canary-token-9f1c";
2662 const ADOPT_ENDPOINT: &str = "https://mvm-abc123.microvm.us-east-1.amazonaws.com";
2663
2664 /// A plane over the recorder, whose `GetMicrovm` answers the caller queues.
2665 fn adopt_plane() -> (ControlPlane, Arc<FakeControlPlane>, Arc<TestClock>) {
2666 let recorder = Arc::new(FakeControlPlane::new());
2667 let clock = Arc::new(TestClock::new());
2668 let plane = crate::testing::control_plane(
2669 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2670 Region::UsEast1,
2671 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2672 );
2673 (plane, recorder, clock)
2674 }
2675
2676 async fn adopted_in(state: &str) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2677 let (plane, recorder, clock) = adopt_plane();
2678 recorder.answer(
2679 "GetMicrovm",
2680 Answer::ok(fake::microvm_response(state, None)),
2681 );
2682 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2683 .await
2684 .expect("adopts");
2685 (sandbox, recorder, clock)
2686 }
2687
2688 /// **An adopted sandbox feeds its keepalive.** The idle window is the one `GetMicrovm`
2689 /// reports, and the lifecycle watch starts at the adopted state, so a keepalive on an
2690 /// adopted session paces against the real policy and ends when that sandbox suspends.
2691 ///
2692 /// **Falsification** — 2026-09-24. Assign `sandbox.lifecycle` directly instead of
2693 /// through `set_lifecycle`, or drop the `idle_window` line, and this goes red; restored.
2694 #[tokio::test]
2695 async fn an_adopted_sandbox_reports_the_services_idle_window_and_watches_its_lifecycle() {
2696 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2697 assert_eq!(sandbox.idle_window(), Some(Duration::from_secs(1800)));
2698 assert_eq!(*sandbox.watch_lifecycle().borrow(), Lifecycle::Running);
2699 }
2700
2701 /// Every service state has a lifecycle, and nothing else parses.
2702 #[test]
2703 fn every_service_state_maps_onto_one_lifecycle() {
2704 for state in crate::constants::MICROVM_STATES {
2705 let lifecycle = Lifecycle::from_service(state).expect("a known state");
2706 assert_eq!(lifecycle.as_str(), state);
2707 }
2708 assert_eq!(Lifecycle::from_service("Running"), None);
2709 }
2710
2711 /// **STATE-3 for an adopted VM.** The lifecycle is the service's, the bootstrap is
2712 /// counted once, and `run` is refused before any wire call, so no run-hook payload is
2713 /// ever sent.
2714 ///
2715 /// **Falsification** — 2026-09-24. Leave the adopted lifecycle at PENDING instead of
2716 /// reading it from `GetMicrovm` and the lifecycle and `token_installed` assertions here
2717 /// go red, as does the suspend in the next test; restored after.
2718 #[tokio::test]
2719 async fn an_adopted_vm_takes_its_lifecycle_from_the_service_and_refuses_run() {
2720 let (mut sandbox, recorder, _) = adopted_in("RUNNING").await;
2721 assert!(sandbox.adopted());
2722 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2723 assert!(sandbox.token_installed());
2724 assert_eq!(sandbox.bootstrap_count(), 1);
2725 assert!(sandbox.image_exists());
2726 let session = sandbox.session().expect("a live VM gets a session");
2727 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
2728 assert_eq!(session.agent_token(), ADOPT_TOKEN);
2729
2730 let error = sandbox
2731 .run(RunRequest::new().with_image("arn:image"))
2732 .await
2733 .expect_err("an adopted VM was bootstrapped by its own launch");
2734 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2735 assert!(error.to_string().contains("STATE-3"), "{error}");
2736 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2737 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
2738 }
2739
2740 /// A name resolves to the record's VM and adopts it; a missing name or a record from
2741 /// another region is refused before any AWS call.
2742 ///
2743 /// `Sandbox::from_name` itself builds a production plane, so its refusals are pinned in
2744 /// the prelude's tests. What's here is the resolve it runs first, and the adopt after.
2745 #[tokio::test]
2746 async fn a_name_adopts_its_records_vm_and_foreign_names_are_refused_locally() {
2747 struct One(crate::names::NameRecord);
2748 impl crate::names::NameStore for One {
2749 fn get(&self, name: &str) -> Result<Option<crate::names::NameRecord>, Error> {
2750 Ok((name == self.0.name).then(|| self.0.clone()))
2751 }
2752 fn put(&self, _: &crate::names::NameRecord) -> Result<(), Error> {
2753 unreachable!()
2754 }
2755 fn delete(&self, _: &str) -> Result<bool, Error> {
2756 unreachable!()
2757 }
2758 fn list(&self) -> Result<Vec<crate::names::NameRecord>, Error> {
2759 Ok(vec![self.0.clone()])
2760 }
2761 }
2762 let record = crate::names::NameRecord::new_at(
2763 "ci",
2764 "mvm-abc123",
2765 ADOPT_ENDPOINT,
2766 ADOPT_TOKEN,
2767 "us-west-2",
2768 1_767_225_600,
2769 )
2770 .expect("record");
2771 let store = One(record.clone());
2772 let foreign = crate::names::resolve(&store, "ci", Some(&Region::UsEast1))
2773 .expect_err("registered in another region");
2774 assert_eq!(foreign.kind(), ErrorKind::InvalidArg);
2775 let missing = crate::names::resolve(&store, "nope", None).expect_err("no such name");
2776 assert_eq!(missing.kind(), ErrorKind::Precondition);
2777 assert!(!format!("{foreign} {missing}").contains(ADOPT_TOKEN));
2778
2779 let (plane, recorder, _) = adopt_plane();
2780 recorder.answer(
2781 "GetMicrovm",
2782 Answer::ok(fake::microvm_response("RUNNING", None)),
2783 );
2784 let resolved = crate::names::resolve(&store, "ci", Some(&Region::UsWest2)).expect("ok");
2785 let sandbox = Sandbox::adopt_record(plane, resolved)
2786 .await
2787 .expect("adopts");
2788 assert!(sandbox.adopted());
2789 assert_eq!(
2790 sandbox.microvm().map(|vm| vm.id.as_str()),
2791 Some("mvm-abc123")
2792 );
2793 assert_eq!(
2794 sandbox.session().expect("session").agent_token(),
2795 ADOPT_TOKEN
2796 );
2797 let named = sandbox
2798 .name_record("again")
2799 .expect("an adopted VM can be named");
2800 assert_eq!(
2801 (
2802 named.microvm_id.as_str(),
2803 named.agent_token.as_str(),
2804 named.region.as_str()
2805 ),
2806 ("mvm-abc123", ADOPT_TOKEN, "us-east-1")
2807 );
2808 let unlaunched = Sandbox::with_control_plane(adopt_plane().0);
2809 assert_eq!(
2810 unlaunched
2811 .name_record("x")
2812 .expect_err("nothing to name")
2813 .kind(),
2814 ErrorKind::Precondition
2815 );
2816 }
2817
2818 /// **A record's tunnel identity survives adoption and naming** (#263).
2819 ///
2820 /// `adopt_record` used to drop the pair and `name_record` never wrote it, so a VM named
2821 /// through the SDKs lost what `tunnel --verify-identity` needs to check it. A record with
2822 /// half the pair is refused before any AWS call.
2823 #[tokio::test]
2824 async fn a_records_tunnel_identity_survives_adoption_and_naming() {
2825 let identity = crate::identity::LaunchIdentity::from_seeds([7; 32], [9; 32])
2826 .expect("valid seeds")
2827 .keep();
2828 let mut record = crate::names::NameRecord::new_at(
2829 "ci",
2830 "mvm-abc123",
2831 ADOPT_ENDPOINT,
2832 ADOPT_TOKEN,
2833 "us-east-1",
2834 1_767_225_600,
2835 )
2836 .expect("record");
2837 record.identity_host_seed = Some(identity.host_seed_base64());
2838 record.identity_vm_public_key = Some(identity.vm_public_key_base64());
2839
2840 let (plane, recorder, _) = adopt_plane();
2841 recorder.answer(
2842 "GetMicrovm",
2843 Answer::ok(fake::microvm_response("RUNNING", None)),
2844 );
2845 let sandbox = Sandbox::adopt_record(plane, record.clone())
2846 .await
2847 .expect("adopts");
2848 let kept = sandbox
2849 .tunnel_identity()
2850 .expect("the record's pair is kept");
2851 assert_eq!(kept.host_seed_base64(), identity.host_seed_base64());
2852 assert_eq!(kept.vm_public_key_base64(), identity.vm_public_key_base64());
2853 let named = sandbox.name_record("again").expect("names");
2854 assert_eq!(named.identity_host_seed, record.identity_host_seed);
2855 assert_eq!(named.identity_vm_public_key, record.identity_vm_public_key);
2856
2857 let (plane, recorder, _) = adopt_plane();
2858 let mut torn = record;
2859 torn.identity_vm_public_key = None;
2860 let refused = Sandbox::adopt_record(plane, torn)
2861 .await
2862 .expect_err("half a pair is refused");
2863 assert_eq!(refused.kind(), ErrorKind::InvalidArg);
2864 assert!(refused.to_string().contains("one half"), "{refused}");
2865 assert!(recorder.calls().is_empty(), "refused before any AWS call");
2866 }
2867
2868 /// Suspend, resume, and terminate all work through an adopted handle with the usual
2869 /// guards, and nothing about the launch is re-delivered (STATE-7).
2870 #[tokio::test]
2871 async fn an_adopted_vm_suspends_resumes_and_terminates_through_its_handle() {
2872 let (plane, recorder, _) = adopt_plane();
2873 recorder
2874 .answer(
2875 "GetMicrovm",
2876 Answer::ok(fake::microvm_response("RUNNING", None)),
2877 )
2878 .answer(
2879 "GetMicrovm",
2880 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2881 )
2882 .answer(
2883 "GetMicrovm",
2884 Answer::ok(fake::microvm_response("RUNNING", None)),
2885 )
2886 .answer(
2887 "GetMicrovm",
2888 Answer::ok(fake::microvm_response("TERMINATED", None)),
2889 )
2890 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
2891 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
2892 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
2893 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2894 .await
2895 .expect("adopts");
2896
2897 sandbox.suspend().await.expect("suspends from RUNNING");
2898 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
2899 sandbox.resume().await.expect("resumes from SUSPENDED");
2900 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2901 let report = sandbox
2902 .terminate(TeardownOpts::default().waiting_for_terminated())
2903 .await;
2904 assert!(report.terminate_accepted, "{report:?}");
2905 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
2906 assert!(sandbox.was_terminated());
2907
2908 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2909 assert!(
2910 recorder
2911 .bodies_as_text()
2912 .iter()
2913 .all(|body| !body.contains("runHookPayload") && !body.contains(ADOPT_TOKEN)),
2914 "no launch payload or agent token may reach the control plane"
2915 );
2916 assert_eq!(sandbox.bootstrap_count(), 1, "STATE-3: never above one");
2917 }
2918
2919 /// **STATE-12 for an adopted VM.** With no window from a request of its own, the
2920 /// sandbox refuses a late resume using the `idlePolicy` window `GetMicrovm` reported.
2921 ///
2922 /// **Falsification** — 2026-09-24. Drop the reported-window fallback in
2923 /// `require_open_suspended_window` and this resume reaches `ResumeMicrovm`; restored
2924 /// after.
2925 #[tokio::test]
2926 async fn an_adopted_vm_refuses_a_resume_past_the_reported_window() {
2927 let (plane, recorder, clock) = adopt_plane();
2928 recorder
2929 .answer(
2930 "GetMicrovm",
2931 Answer::ok(fake::microvm_response("RUNNING", None)),
2932 )
2933 .answer(
2934 "GetMicrovm",
2935 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2936 )
2937 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2938 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2939 .await
2940 .expect("adopts");
2941 assert_eq!(sandbox.suspended_window(), None, "no request of its own");
2942 sandbox.suspend().await.expect("suspends");
2943
2944 clock.advance(Duration::from_secs(601));
2945 let error = sandbox
2946 .resume()
2947 .await
2948 .expect_err("past the 600 s window the service reported");
2949 assert_eq!(error.kind(), ErrorKind::WindowClosed);
2950 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
2951 }
2952
2953 /// A VM adopted while already SUSPENDED has no locally observed suspend time, so the
2954 /// resume goes to the service rather than being refused on a guess.
2955 #[tokio::test]
2956 async fn an_adopted_suspended_vm_resumes_through_the_service() {
2957 let (plane, recorder, _) = adopt_plane();
2958 recorder
2959 .answer(
2960 "GetMicrovm",
2961 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2962 )
2963 .answer(
2964 "GetMicrovm",
2965 Answer::ok(fake::microvm_response("RUNNING", None)),
2966 )
2967 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
2968 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2969 .await
2970 .expect("adopts");
2971 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
2972 let session = sandbox
2973 .resume()
2974 .await
2975 .expect("the service answers the resume");
2976 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
2977 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
2978 }
2979
2980 /// **STATE-11 for an adopted VM.** A terminated VM is offered neither a resume nor a
2981 /// suspend, and gets no session.
2982 #[tokio::test]
2983 async fn an_adopted_terminated_vm_refuses_every_transition_locally() {
2984 let (mut sandbox, recorder, _) = adopted_in("TERMINATED").await;
2985 assert!(sandbox.was_terminated());
2986 assert!(sandbox.session().is_none());
2987 let resume = sandbox.resume().await.expect_err("STATE-11");
2988 assert!(resume.to_string().contains("STATE-11"), "{resume}");
2989 let suspend = sandbox.suspend().await.expect_err("STATE-5");
2990 assert!(suspend.to_string().contains("STATE-5"), "{suspend}");
2991 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
2992 }
2993
2994 /// A sandbox adopted in `state` whose plane and session are scripted: `GetMicrovm` answers
2995 /// `state` and then RUNNING, and the daemon answers every health poll.
2996 async fn adopted_with_a_daemon(
2997 state: &str,
2998 ) -> (
2999 Sandbox,
3000 Arc<FakeControlPlane>,
3001 Arc<crate::testing::HealthyDaemon>,
3002 ) {
3003 let recorder = Arc::new(FakeControlPlane::new());
3004 let daemon = crate::testing::HealthyDaemon::new();
3005 let plane = crate::control::ControlPlane::from_ports(
3006 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3007 Region::UsEast1,
3008 Arc::new(TestClock::new()),
3009 Arc::new(crate::testing::SequenceEntropy::new()),
3010 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3011 );
3012 recorder
3013 .answer(
3014 "GetMicrovm",
3015 Answer::ok(fake::microvm_response(state, None)),
3016 )
3017 .answer(
3018 "GetMicrovm",
3019 Answer::ok(fake::microvm_response("RUNNING", None)),
3020 )
3021 .answer(
3022 "CreateMicrovmAuthToken",
3023 Answer::ok(fake::auth_token_response("proxy-token")),
3024 );
3025 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3026 .await
3027 .expect("adopts");
3028 (sandbox, recorder, daemon)
3029 }
3030
3031 /// **`wait_until_ready` finishes whichever wait is left (#269):** from an adopted PENDING VM,
3032 /// RUNNING and then the daemon; from an adopted RUNNING one, the daemon alone, with no
3033 /// further control-plane read.
3034 ///
3035 /// **Falsification**: drop the RUNNING branch from `Sandbox::wait_until_ready`, and the
3036 /// RUNNING VM is refused as not PENDING.
3037 #[tokio::test]
3038 async fn wait_until_ready_finishes_whichever_wait_is_left() {
3039 let (mut pending, recorder, daemon) = adopted_with_a_daemon("PENDING").await;
3040 pending
3041 .wait_until_ready(Duration::from_secs(60))
3042 .await
3043 .expect("RUNNING, then the daemon");
3044 assert_eq!(pending.lifecycle(), Lifecycle::Running);
3045 assert_eq!(pending.bootstrap_count(), 1);
3046 assert_eq!(daemon.polls(), 1);
3047 assert_eq!(
3048 recorder.call_count("GetMicrovm"),
3049 2,
3050 "the adopt's read and the wait's"
3051 );
3052
3053 let (mut running, recorder, daemon) = adopted_with_a_daemon("RUNNING").await;
3054 running
3055 .wait_until_ready(Duration::from_secs(60))
3056 .await
3057 .expect("the daemon answers");
3058 assert_eq!(daemon.polls(), 1);
3059 assert_eq!(
3060 recorder.call_count("GetMicrovm"),
3061 1,
3062 "RUNNING needs no second read"
3063 );
3064 }
3065
3066 /// A VM that isn't on its way to answering is refused, with no daemon poll.
3067 #[tokio::test]
3068 async fn wait_until_ready_refuses_a_suspended_vm() {
3069 let (mut suspended, _, daemon) = adopted_with_a_daemon("SUSPENDED").await;
3070 let error = suspended
3071 .wait_until_ready(Duration::from_secs(60))
3072 .await
3073 .expect_err("a suspended VM isn't starting");
3074 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
3075 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3076 assert_eq!(daemon.polls(), 0);
3077 }
3078
3079 /// A VM adopted while PENDING finishes through `wait_until_running`, which counts the
3080 /// bootstrap exactly once and, like a launch, waits for the daemon to answer.
3081 #[tokio::test]
3082 async fn an_adopted_pending_vm_counts_its_bootstrap_once_on_running() {
3083 let recorder = Arc::new(FakeControlPlane::new());
3084 let daemon = crate::testing::HealthyDaemon::new();
3085 let plane = crate::control::ControlPlane::from_ports(
3086 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3087 Region::UsEast1,
3088 Arc::new(TestClock::new()),
3089 Arc::new(crate::testing::SequenceEntropy::new()),
3090 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3091 );
3092 recorder
3093 .answer(
3094 "GetMicrovm",
3095 Answer::ok(fake::microvm_response("PENDING", None)),
3096 )
3097 .answer(
3098 "GetMicrovm",
3099 Answer::ok(fake::microvm_response("RUNNING", None)),
3100 )
3101 .answer(
3102 "CreateMicrovmAuthToken",
3103 Answer::ok(fake::auth_token_response("proxy-token")),
3104 );
3105 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3106 .await
3107 .expect("adopts");
3108 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3109 assert_eq!(sandbox.bootstrap_count(), 0);
3110 sandbox
3111 .wait_until_running(Duration::from_secs(60))
3112 .await
3113 .expect("reaches RUNNING");
3114 assert_eq!(sandbox.bootstrap_count(), 1);
3115 assert!(sandbox.token_installed());
3116 assert_eq!(
3117 daemon.polls(),
3118 1,
3119 "the daemon answered before the session came back"
3120 );
3121 assert!(
3122 sandbox
3123 .run(RunRequest::new().with_image("arn:image"))
3124 .await
3125 .is_err(),
3126 "still one bootstrap per VM"
3127 );
3128 assert_eq!(recorder.call_count("RunMicrovm"), 0);
3129 }
3130
3131 /// The refusals: an empty token before any call, a mismatched endpoint, and a state
3132 /// this client does not know. None of them, and no `Debug`, prints the token.
3133 ///
3134 /// **Falsification** — 2026-09-24. Remove the endpoint comparison and the mismatch case
3135 /// adopts; remove the empty-token check and that case reaches `GetMicrovm`; print the
3136 /// held token in `Debug` and the canary assertion fails. Each restored after.
3137 #[tokio::test]
3138 async fn adopt_refuses_bad_records_and_never_prints_the_token() {
3139 let (plane, recorder, _) = adopt_plane();
3140 let empty = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, "")
3141 .await
3142 .expect_err("no token");
3143 assert_eq!(empty.kind(), ErrorKind::InvalidArg);
3144 assert_eq!(recorder.calls().len(), 0, "refused before any call");
3145
3146 let (plane, recorder, _) = adopt_plane();
3147 recorder.answer(
3148 "GetMicrovm",
3149 Answer::ok(fake::microvm_response("RUNNING", None)),
3150 );
3151 let mismatch = Sandbox::adopt(plane, "mvm-abc123", "https://other.example", ADOPT_TOKEN)
3152 .await
3153 .expect_err("the endpoint belongs to another record");
3154 assert_eq!(mismatch.kind(), ErrorKind::InvalidArg);
3155 assert!(!mismatch.to_string().contains(ADOPT_TOKEN));
3156
3157 let (plane, recorder, _) = adopt_plane();
3158 recorder.answer(
3159 "GetMicrovm",
3160 Answer::ok(fake::microvm_response("REBOOTING", None)),
3161 );
3162 let unknown = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3163 .await
3164 .expect_err("an unknown state");
3165 assert_eq!(unknown.kind(), ErrorKind::Platform);
3166 assert!(!unknown.to_string().contains(ADOPT_TOKEN));
3167
3168 let (sandbox, _, _) = adopted_in("RUNNING").await;
3169 let printed = format!("{sandbox:?}");
3170 assert!(!printed.contains(ADOPT_TOKEN), "{printed}");
3171 assert!(printed.contains("adopted: true"), "{printed}");
3172 }
3173
3174 /// **STATE-1 and STATE-2.** A launch accepted is PENDING with the image recorded as
3175 /// existing; the platform reporting the run hook succeeded is what marks the token
3176 /// installed.
3177 ///
3178 /// The two halves are asserted with the *same* fake because they are the same call, and
3179 /// what distinguishes them is which fact is recorded when: `image_exists` is true from
3180 /// the accepted launch, `token_installed` only from the RUNNING report. A client that
3181 /// set both at the launch call would pass an end-state assertion and be wrong about a
3182 /// VM that died during startup — which is the next test.
3183 #[tokio::test]
3184 async fn a_launch_records_the_image_and_the_running_report_installs_the_token() {
3185 let (mut sandbox, recorder, _) = planted();
3186 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3187 assert!(
3188 !sandbox.image_exists(),
3189 "nothing is recorded before a launch"
3190 );
3191 assert!(!sandbox.token_installed());
3192
3193 answer_launch(&recorder);
3194 let session = sandbox
3195 .run(RunRequest::new().with_image("arn:image"))
3196 .await
3197 .expect("launches");
3198 assert_eq!(
3199 session.endpoint(),
3200 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
3201 );
3202
3203 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3204 assert!(sandbox.image_exists(), "STATE-1: the image is recorded");
3205 assert!(sandbox.token_installed(), "STATE-2: the token is installed");
3206 assert_eq!(sandbox.bootstrap_count(), 1);
3207 assert_eq!(sandbox.microvm().expect("launched").id, "mvm-abc123");
3208
3209 // The payload really carried the token, read off the wire member rather than from
3210 // the request type — a field on a struct proves nothing about what was emitted.
3211 let body = recorder.first_body("RunMicrovm");
3212 let payload = body["runHookPayload"].as_str().expect("a string");
3213 assert!(payload.starts_with(r#"{"agent_token":"#), "{payload}");
3214 assert_eq!(body["idlePolicy"]["suspendedDurationSeconds"], 600);
3215 }
3216
3217 /// A launch env reaches the wire member the daemon parses, and it is the only thing
3218 /// that changes about the request.
3219 ///
3220 /// Read off `runHookPayload` rather than off the request struct: a field on a struct
3221 /// proves nothing about what was emitted, which is the same argument the test above
3222 /// makes about the token.
3223 #[tokio::test]
3224 async fn a_launch_env_reaches_the_run_hook_payload_on_the_wire() {
3225 let (mut sandbox, recorder, _) = planted();
3226 answer_launch(&recorder);
3227 sandbox
3228 .run(
3229 RunRequest::new()
3230 .with_image("arn:image")
3231 .with_launch_env("ANTHROPIC_BASE_URL", "https://gateway.example")
3232 .with_launch_env("PATH", "/usr/local/bin:/usr/bin:/bin"),
3233 )
3234 .await
3235 .expect("launches");
3236
3237 let body = recorder.first_body("RunMicrovm");
3238 let payload = body["runHookPayload"].as_str().expect("a string");
3239 // One parse deeper, because that is where the daemon reads it from too.
3240 let inner: serde_json::Value =
3241 serde_json::from_str(payload).expect("the payload is itself JSON");
3242 assert!(
3243 inner["agent_token"].as_str().is_some_and(|t| !t.is_empty()),
3244 "the token still rides alongside: {payload}"
3245 );
3246 assert_eq!(
3247 inner["env"]["ANTHROPIC_BASE_URL"],
3248 "https://gateway.example"
3249 );
3250 assert_eq!(inner["env"]["PATH"], "/usr/local/bin:/usr/bin:/bin");
3251 }
3252
3253 /// **The advisory deny reaches the guest's environment, and it changes nothing else
3254 /// about the request.**
3255 ///
3256 /// Read off `runHookPayload` for the reason the launch-env test gives, and the connector
3257 /// member is asserted absent in the same body: `deny_egress` must not quietly ask the
3258 /// platform for anything, because the whole claim is that the platform has nothing to
3259 /// ask for.
3260 ///
3261 /// **Falsification** — 2026-09-13. Make `run` build the payload from
3262 /// `request.launch_env` instead of `request.effective_launch_env()` and the two
3263 /// `https_proxy` assertions go red while every other launch test stays green; restored
3264 /// after.
3265 #[tokio::test]
3266 async fn the_advisory_deny_reaches_the_launch_environment_in_both_spellings() {
3267 let (mut sandbox, recorder, _) = planted();
3268 answer_launch(&recorder);
3269 let request = RunRequest::new().with_image("arn:image").with_deny_egress();
3270 assert_eq!(
3271 request.egress_posture(),
3272 crate::control::EgressPosture::BestEffort
3273 );
3274 sandbox.run(request).await.expect("launches");
3275
3276 let body = recorder.first_body("RunMicrovm");
3277 let inner: serde_json::Value =
3278 serde_json::from_str(body["runHookPayload"].as_str().expect("a string"))
3279 .expect("the payload is itself JSON");
3280 for key in crate::sandbox::DENY_EGRESS_ENV_KEYS {
3281 assert_eq!(
3282 inner["env"][key], DENY_EGRESS_PROXY_URL,
3283 "{key} must point at the black hole: {inner}"
3284 );
3285 }
3286 assert!(
3287 body.get("egressNetworkConnectors").is_none(),
3288 "the advisory deny asks the platform for nothing: {body}"
3289 );
3290 }
3291
3292 /// A caller's own proxy value survives the deny, and a launch that never asked for the
3293 /// deny carries none of the keys.
3294 #[test]
3295 fn the_deny_never_overwrites_a_callers_own_proxy_and_is_absent_by_default() {
3296 let kept = RunRequest::new()
3297 .with_deny_egress()
3298 .with_launch_env("https_proxy", "http://proxy.internal:3128")
3299 .effective_launch_env();
3300 assert_eq!(kept["https_proxy"], "http://proxy.internal:3128");
3301 assert_eq!(kept["http_proxy"], DENY_EGRESS_PROXY_URL);
3302
3303 let plain = RunRequest::new().effective_launch_env();
3304 assert!(
3305 plain.is_empty(),
3306 "a launch that did not ask for the deny sends byte-for-byte what it always sent: \
3307 {plain:?}"
3308 );
3309 assert_eq!(
3310 RunRequest::new().egress_posture(),
3311 crate::control::EgressPosture::Unsealed,
3312 "and its posture is the measured one, not a seal"
3313 );
3314 }
3315
3316 /// **`egress` and `deny_egress` together are refused locally, with zero calls.**
3317 ///
3318 /// The same shape as the over-budget refusal below: a launch carrying both would report
3319 /// `open` while the workload's own tools failed closed, and AWS has no opinion to
3320 /// return about it.
3321 ///
3322 /// **Falsification** — 2026-09-13. Delete the refusal in `run` and this test fails on
3323 /// `expect_err`; the launch then succeeds and reports posture `open`. Restored after.
3324 #[tokio::test]
3325 async fn asking_for_egress_and_the_deny_together_is_refused_before_any_call() {
3326 let (mut sandbox, recorder, _) = planted();
3327 answer_launch(&recorder);
3328 let mut request = RunRequest::new().with_image("arn:image").with_egress();
3329 request.deny_egress = true;
3330
3331 let error = sandbox
3332 .run(request)
3333 .await
3334 .expect_err("opposite intents are not resolved by a precedence rule");
3335 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3336 assert!(error.to_string().contains("opposite things"), "{error}");
3337 assert_eq!(
3338 recorder.calls().len(),
3339 0,
3340 "a locally-refused launch costs nothing"
3341 );
3342 }
3343
3344 /// **A pinned `imageVersion` reaches the wire through the sandbox**, and an unpinned
3345 /// launch omits the member entirely.
3346 ///
3347 /// This is the layer a caller actually uses, so the forwarding is what needs asserting:
3348 /// [`RunRequest::image_version`] is a field on a struct and proves nothing about the
3349 /// emitted body until it is read off the wire member.
3350 ///
3351 /// **Falsification** — run 2026-08-16. Drop `wire.image_version = ...` from `run` and the
3352 /// pinned assertion goes red with the member absent, while every other launch test still
3353 /// passes — which is why this one is here.
3354 #[tokio::test]
3355 async fn a_pinned_image_version_is_forwarded_to_the_launch_request() {
3356 let (mut sandbox, recorder, _) = planted();
3357 answer_launch(&recorder);
3358 sandbox
3359 .run(
3360 RunRequest::new()
3361 .with_image("arn:image")
3362 .with_image_version("2.0"),
3363 )
3364 .await
3365 .expect("launches");
3366 assert_eq!(
3367 recorder.first_body("RunMicrovm")["imageVersion"],
3368 "2.0",
3369 "a canary has to launch against the version it means to test"
3370 );
3371
3372 let (mut sandbox, recorder, _) = planted();
3373 answer_launch(&recorder);
3374 sandbox
3375 .run(RunRequest::new().with_image("arn:image"))
3376 .await
3377 .expect("launches");
3378 assert!(
3379 recorder
3380 .first_body("RunMicrovm")
3381 .get("imageVersion")
3382 .is_none(),
3383 "an unpinned launch sends what this client always sent"
3384 );
3385 }
3386
3387 /// **The local refusal, with zero control-plane calls.**
3388 ///
3389 /// The observable difference between this client and one that lets AWS answer, and the
3390 /// same shape as STATE-5's suspend refusal: an over-budget launch env fails before the
3391 /// launch rather than as a `ValidationException` on a member the caller did not know
3392 /// they were filling. botocore does not enforce the ceiling client-side, so without
3393 /// this there is no local signal at all.
3394 ///
3395 /// **Falsification** — build the payload after `run_microvm` instead of before, and the
3396 /// call count assertion goes red.
3397 #[tokio::test]
3398 async fn an_over_budget_launch_env_is_refused_before_any_control_plane_call() {
3399 let (mut sandbox, recorder, _) = planted();
3400 answer_launch(&recorder);
3401
3402 let error = sandbox
3403 .run(
3404 RunRequest::new()
3405 .with_image("arn:image")
3406 .with_launch_env("AWS_SESSION_TOKEN", "t".repeat(4096)),
3407 )
3408 .await
3409 .expect_err("a credential-scale launch env does not fit the payload");
3410
3411 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3412 let message = error.to_string();
3413 assert!(message.contains("ceiling of 4096"), "{message}");
3414 assert!(message.contains("launch env contributed"), "{message}");
3415 assert_eq!(
3416 recorder.calls().len(),
3417 0,
3418 "the refusal has to be local: a launch that reached AWS costs a VM and a bill"
3419 );
3420 // And the sandbox is left usable rather than half-launched, so a caller can trim
3421 // the env and retry through the same handle.
3422 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3423 assert_eq!(sandbox.bootstrap_count(), 0);
3424 assert!(sandbox.microvm().is_none());
3425 }
3426
3427 /// **Issue #24's guards reach through `Sandbox`, which is the layer both bindings use.**
3428 ///
3429 /// `microvms-py`'s `run` and `microvms-js`'s `run` both build a [`RunRequest`] and hand it to
3430 /// [`Sandbox::run`], which fills a `RunMicrovmRequest` and calls
3431 /// [`crate::control::ControlPlane::run_microvm`]. So the guards there cover the bindings — but
3432 /// "cover" is a claim about a call chain, and a chain is what a refactor breaks. This asserts
3433 /// it at the layer the bindings actually enter, so a `Sandbox::run` that stopped delegating
3434 /// (or a binding-side default that bypassed the request type) fails a test rather than shipping
3435 /// an unguarded surface to Python and Node.
3436 ///
3437 /// `max_idle_sec` is the case that matters most here, because it is a plain `u32` keyword
3438 /// argument on both bindings with no type narrowing it: `sandbox.run(max_idle_sec=59)` is what
3439 /// a Python caller writes, and before this it reached the wire.
3440 ///
3441 /// **Guard proof.** Delete `require_idle_duration` from `run_microvm` and the idle row goes red
3442 /// on the call count; delete `require_valid_role_arn` and the role row does. Neither guard is in
3443 /// this file, which is the point — the test is about the chain.
3444 #[tokio::test]
3445 async fn the_bindings_layer_refuses_an_illegal_launch_with_zero_calls() {
3446 let cases: [(&str, RunRequest, &str); 3] = [
3447 (
3448 "max_idle_sec=59, which is what a Python keyword argument passes",
3449 RunRequest {
3450 max_idle_sec: 59,
3451 ..RunRequest::new().with_image("arn:image")
3452 },
3453 "maxIdleDurationSeconds",
3454 ),
3455 (
3456 "an execution role that is a bare name",
3457 RunRequest {
3458 execution_role_arn: Some("execution-role".to_string()),
3459 ..RunRequest::new().with_image("arn:image")
3460 },
3461 "role *name*",
3462 ),
3463 (
3464 "a pinned version with a trailing newline",
3465 RunRequest::new()
3466 .with_image("arn:image")
3467 .with_image_version("2.0\n"),
3468 "contains whitespace",
3469 ),
3470 ];
3471
3472 for (label, request, expected) in cases {
3473 let (mut sandbox, recorder, _) = planted();
3474 // The launch answered, so a missing guard is a *successful* launch — a real VM and a
3475 // real bill — rather than a different failure.
3476 answer_launch(&recorder);
3477
3478 // `expect_err` needs the Ok type to be `Debug`, and `run` answers `&mut Session`
3479 // (which is not) — so the discriminant is matched instead. That is also the more
3480 // honest assertion: it says the launch produced no session at all.
3481 let Err(error) = sandbox.run(request).await else {
3482 panic!("{label} must be refused");
3483 };
3484 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
3485 assert!(
3486 error.to_string().contains(expected),
3487 "{label}: wanted {expected:?}, got {error}"
3488 );
3489 assert_eq!(
3490 recorder.calls().len(),
3491 0,
3492 "{label}: the refusal has to be local, because a launch that reached AWS costs a \
3493 VM and a bill"
3494 );
3495 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3496 assert!(sandbox.microvm().is_none());
3497 }
3498 }
3499
3500 /// A launch whose VM dies during startup leaves the token **not** installed, because
3501 /// the run hook is what delivers it and a dead VM ran no hook.
3502 ///
3503 /// The distinction this pins is the one the test above cannot: both a correct client and
3504 /// one that installs the token at the launch call reach RUNNING with the token set, and
3505 /// only this case separates them.
3506 #[tokio::test]
3507 async fn a_launch_that_dies_during_startup_installs_no_token() {
3508 let (mut sandbox, recorder, _) = planted();
3509 recorder
3510 .answer(
3511 "RunMicrovm",
3512 Answer::ok(fake::microvm_response("PENDING", None)),
3513 )
3514 .answer(
3515 "GetMicrovm",
3516 Answer::ok(fake::microvm_response(
3517 "TERMINATED",
3518 Some("run hook returned 500"),
3519 )),
3520 );
3521
3522 let error = sandbox
3523 .run(RunRequest::new().with_image("arn:image"))
3524 .await
3525 .expect_err("a VM that died during startup is not a launch");
3526 assert_eq!(error.kind(), ErrorKind::LaunchDied);
3527 assert!(
3528 error.to_string().contains("run hook returned 500"),
3529 "{error}"
3530 );
3531
3532 assert!(
3533 !sandbox.token_installed(),
3534 "the run hook never answered, so nothing was installed"
3535 );
3536 assert_eq!(
3537 sandbox.bootstrap_count(),
3538 0,
3539 "a bootstrap counted here would make STATE-3's ceiling unreachable for a retry"
3540 );
3541 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3542 assert!(
3543 sandbox.image_exists(),
3544 "STATE-1 still holds: the launch was accepted against a real image"
3545 );
3546 }
3547
3548 /// **STATE-3, the guard proof.** A second `run` on one sandbox is refused, and it is
3549 /// refused with **zero** further control-plane calls.
3550 ///
3551 /// The call count is the assertion that matters. A client that let the second launch
3552 /// through would create a second VM and silently address two guests through one handle,
3553 /// and it would also deliver a second run-hook payload to a daemon whose one-shot
3554 /// bootstrap answers 409 — so "an error came back" is not enough to tell the two apart.
3555 ///
3556 /// **Falsification** — delete the `bootstrap_count > 0` branch from `run` and this test
3557 /// is red on both the count and the call total. Verified; see the packet's guard proofs.
3558 #[tokio::test]
3559 async fn a_second_run_on_one_sandbox_is_refused_before_any_call() {
3560 let (mut sandbox, recorder, _) = launched().await;
3561 assert_eq!(sandbox.bootstrap_count(), 1);
3562 let before = recorder.calls().len();
3563
3564 let error = sandbox
3565 .run(RunRequest::new().with_image("arn:image"))
3566 .await
3567 .expect_err("the token is installed at most once per VM lifetime");
3568 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3569 assert!(error.to_string().contains("STATE-3"), "{error}");
3570 assert!(
3571 error.to_string().contains("needs a second Sandbox"),
3572 "the remedy has to be nameable: {error}"
3573 );
3574
3575 assert_eq!(
3576 sandbox.bootstrap_count(),
3577 1,
3578 "bootstrap_count <= 1 is the Z3-proved invariant"
3579 );
3580 assert_eq!(
3581 recorder.calls().len(),
3582 before,
3583 "the refusal must cost no control-plane call"
3584 );
3585 assert_eq!(recorder.call_count("RunMicrovm"), 1);
3586 }
3587
3588 /// **STATE-4 and STATE-6.** A suspend from RUNNING moves to SUSPENDING before the wait,
3589 /// and the platform reporting SUSPENDED is what moves it to SUSPENDED.
3590 #[tokio::test]
3591 async fn a_suspend_from_running_passes_through_suspending_to_suspended() {
3592 let (sandbox, recorder, _) = suspended().await;
3593 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3594 assert!(
3595 sandbox.token_installed(),
3596 "a freeze keeps the guest's memory, so the token survives"
3597 );
3598 assert_eq!(sandbox.bootstrap_count(), 1, "no re-bootstrap on a freeze");
3599
3600 let calls = recorder.calls();
3601 let suspend = calls
3602 .iter()
3603 .find(|call| call.operation == "SuspendMicrovm")
3604 .expect("the suspend went out");
3605 assert_eq!(suspend.method, Method::Post);
3606 assert_eq!(suspend.path, "/2025-09-09/microvms/mvm-abc123/suspend");
3607 }
3608
3609 /// A suspend whose VM dies while suspending reports TERMINATED rather than raising, and
3610 /// records it — which is what stops a later resume.
3611 ///
3612 /// TERMINATED is *wanted* by the suspend wait for a reason: a VM that dies mid-suspend
3613 /// is a state to report, not an exception out of the middle of a teardown.
3614 #[tokio::test]
3615 async fn a_vm_that_dies_while_suspending_is_recorded_rather_than_raised() {
3616 let (mut sandbox, recorder, _) = launched().await;
3617 recorder
3618 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3619 .answer(
3620 "GetMicrovm",
3621 Answer::ok(fake::microvm_response("TERMINATED", Some("idle policy"))),
3622 );
3623
3624 sandbox
3625 .suspend()
3626 .await
3627 .expect("a death mid-suspend is a state, not an error");
3628 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3629 assert!(
3630 sandbox.was_terminated(),
3631 "STATE-11's precondition is recorded"
3632 );
3633 }
3634
3635 /// A suspend whose wire call fails leaves the sandbox in RUNNING, and a retry works.
3636 ///
3637 /// STATE-4's "accepted" is the wire call succeeding. A client that moved to SUSPENDING
3638 /// before the call would strand a failed call there — a state neither suspend nor
3639 /// resume accepts, so one throttled or dropped request would brick the handle.
3640 /// **Falsification** — move `self.lifecycle = Lifecycle::Suspending` back above the
3641 /// `control.suspend` call and the first assertion here reads SUSPENDING, and the retry
3642 /// is refused by the STATE-5 guard.
3643 #[tokio::test]
3644 async fn a_suspend_whose_call_fails_stays_running_and_can_be_retried() {
3645 let (mut sandbox, recorder, _) = planted();
3646 recorder
3647 .answer(
3648 "RunMicrovm",
3649 Answer::ok(fake::microvm_response("PENDING", None)),
3650 )
3651 .answer(
3652 "GetMicrovm",
3653 Answer::ok(fake::microvm_response("RUNNING", None)),
3654 )
3655 .answer(
3656 "GetMicrovm",
3657 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3658 )
3659 .answer(
3660 "CreateMicrovmAuthToken",
3661 Answer::ok(fake::auth_token_response("proxy-token")),
3662 )
3663 // A 409 rather than a 429 or a transport cut, because those two are retried
3664 // inside `send_with_retry` and this test wants the call to fail once, fast.
3665 .answer("SuspendMicrovm", Answer::failure(409, "ConflictException"))
3666 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
3667
3668 sandbox
3669 .run(RunRequest::new().with_image("arn:image"))
3670 .await
3671 .expect("launches");
3672
3673 sandbox
3674 .suspend()
3675 .await
3676 .expect_err("the control plane refused the suspend");
3677 assert_eq!(
3678 sandbox.lifecycle(),
3679 Lifecycle::Running,
3680 "a refused suspend must not move the lifecycle: SUSPENDING accepts neither a \
3681 suspend nor a resume, so recording it here would brick the handle"
3682 );
3683
3684 sandbox.suspend().await.expect("the retry suspends");
3685 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3686 assert_eq!(recorder.call_count("SuspendMicrovm"), 2);
3687 }
3688
3689 /// **STATE-5, the guard proof.** A suspend from anything but RUNNING is refused with
3690 /// **zero** control-plane calls.
3691 ///
3692 /// Every non-RUNNING state, so a state added to `Lifecycle` later is not silently
3693 /// exempt. The count is the load-bearing assertion: letting the call go and reading the
3694 /// service's answer also produces an error, and that error is about a non-running id
3695 /// rather than about which of two things the caller got wrong.
3696 ///
3697 /// **Falsification** — delete the `lifecycle != Running` branch from `suspend` and the
3698 /// SUSPENDED case emits a `SuspendMicrovm` call, turning the count assertion red.
3699 /// Verified; see the packet's guard proofs.
3700 #[tokio::test]
3701 async fn a_suspend_from_a_non_running_state_reaches_no_control_plane_call() {
3702 // From SUSPENDED — the caller who believes they resumed.
3703 let (mut sandbox, recorder, _) = suspended().await;
3704 let before = recorder.call_count("SuspendMicrovm");
3705
3706 let error = sandbox
3707 .suspend()
3708 .await
3709 .expect_err("a suspend is only issued from RUNNING");
3710 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3711 assert!(error.to_string().contains("STATE-5"), "{error}");
3712 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3713 assert_eq!(
3714 recorder.call_count("SuspendMicrovm"),
3715 before,
3716 "the refusal must be local: no second suspend went to the wire"
3717 );
3718
3719 // And from PENDING, before anything launched at all.
3720 let (mut fresh, fresh_recorder, _) = planted();
3721 let error = fresh.suspend().await.expect_err("nothing to suspend");
3722 assert_eq!(error.kind(), ErrorKind::Precondition);
3723 assert_eq!(fresh_recorder.calls().len(), 0);
3724 }
3725
3726 /// **STATE-7 and STATE-8.** A resume from SUSPENDED returns to RUNNING, re-delivers
3727 /// **nothing**, and drops the cached proxy token.
3728 ///
3729 /// Three assertions, and each one is a different failure mode. The bootstrap count says
3730 /// no run-hook payload was re-delivered — which a daemon's one-shot bootstrap would
3731 /// answer 409 to, reading like a broken VM. The `RunMicrovm` count says no second launch
3732 /// happened. And `is_cached` says the token was invalidated, which is the only
3733 /// observable difference on a path where the endpoint URL does not change: a client that
3734 /// kept the token would produce identical requests until the pre-suspend token stopped
3735 /// validating, and that rejection reads exactly like a dead daemon.
3736 #[tokio::test]
3737 async fn a_resume_reuses_the_token_redelivers_nothing_and_drops_the_proxy_token() {
3738 let (mut sandbox, recorder, _) = suspended().await;
3739
3740 // The session mints once before the suspend, so there is a cached token to drop.
3741 let auth = Arc::clone(
3742 sandbox
3743 .session()
3744 .expect("launched")
3745 .proxy_auth()
3746 .expect("the launch wired a minter"),
3747 );
3748 auth.headers().await.expect("mints");
3749 assert!(
3750 auth.is_cached(),
3751 "there must be a token for the drop to matter"
3752 );
3753 assert_eq!(auth.mint_count(), 1);
3754
3755 recorder
3756 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3757 .answer(
3758 "GetMicrovm",
3759 Answer::ok(fake::microvm_response("RUNNING", None)),
3760 );
3761 let session = sandbox.resume().await.expect("resumes");
3762 assert_eq!(
3763 session.endpoint(),
3764 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
3765 );
3766
3767 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3768 assert!(
3769 sandbox.token_installed(),
3770 "STATE-7: the installed token is reused"
3771 );
3772 assert_eq!(
3773 sandbox.bootstrap_count(),
3774 1,
3775 "STATE-7: a resume re-delivers no run-hook payload"
3776 );
3777 assert_eq!(
3778 recorder.call_count("RunMicrovm"),
3779 1,
3780 "a resume is not a second launch"
3781 );
3782 assert!(
3783 !auth.is_cached(),
3784 "STATE-8: the cached proxy token survived the resume"
3785 );
3786 }
3787
3788 /// **STATE-8's guard proof, on the mint count.** The request after a resume mints a
3789 /// fresh token rather than reusing the pre-suspend one.
3790 ///
3791 /// Separate from the test above because `is_cached` is a fact about the cache and this is
3792 /// a fact about the wire: a client that dropped the cache but kept emitting the old
3793 /// header would pass the first assertion and fail every request.
3794 ///
3795 /// **Falsification** — delete the `proxy.invalidate()` line from `Session::rebind` and
3796 /// the mint count stays at 1 here. Verified; see the packet's guard proofs.
3797 #[tokio::test]
3798 async fn the_request_after_a_resume_mints_a_fresh_proxy_token() {
3799 let (mut sandbox, recorder, _) = suspended().await;
3800 let auth = Arc::clone(
3801 sandbox
3802 .session()
3803 .expect("launched")
3804 .proxy_auth()
3805 .expect("wired"),
3806 );
3807 auth.headers().await.expect("mints");
3808 assert_eq!(auth.mint_count(), 1);
3809
3810 recorder
3811 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3812 .answer(
3813 "GetMicrovm",
3814 Answer::ok(fake::microvm_response("RUNNING", None)),
3815 );
3816 sandbox.resume().await.expect("resumes");
3817
3818 // No clock movement at all, so a re-mint here is caused by the invalidation rather
3819 // than by the refresh window rolling over.
3820 auth.headers().await.expect("re-mints");
3821 assert_eq!(
3822 auth.mint_count(),
3823 2,
3824 "STATE-8: the resume did not force a fresh mint"
3825 );
3826 }
3827
3828 /// **STATE-12, the guard proof.** A resume past the launch-time suspended window is
3829 /// refused locally, with **zero** resume calls, and the message names the window.
3830 ///
3831 /// Four assertions, and the packet asks for all four because each rules out a different
3832 /// wrong implementation. `ErrorKind::WindowClosed` rather than `Timeout` separates this
3833 /// from the client that waits. Zero `ResumeMicrovm` calls is what "before any wire call"
3834 /// means. The elapsed reading rules out an implementation that polled to the deadline
3835 /// first — the fake's clock advances on every `sleep`, so a poll loop here would show up
3836 /// as time passed. And the message naming both numbers plus `idlePolicy` is what sends a
3837 /// reader to the finding rather than looking for the flag that reopens the window.
3838 ///
3839 /// **Falsification** — delete the `require_open_suspended_window` call from `resume` and
3840 /// the fake answers TERMINATED, so this test fails with an `ErrorKind::LaunchDied` after
3841 /// a `ResumeMicrovm` call, naming neither the window nor the seconds elapsed. Verified;
3842 /// see the packet's guard proofs.
3843 /// A keepalive reads this receiver instead of the lock, so every transition must reach
3844 /// it: a suspend it missed would be undone by the keepalive's next poll auto-resuming
3845 /// the VM.
3846 #[tokio::test]
3847 async fn the_lifecycle_watch_follows_every_transition() {
3848 let (mut sandbox, recorder, _clock) = planted();
3849 let watch = sandbox.watch_lifecycle();
3850 assert_eq!(*watch.borrow(), Lifecycle::Pending);
3851 recorder
3852 .answer(
3853 "RunMicrovm",
3854 Answer::ok(fake::microvm_response("PENDING", None)),
3855 )
3856 .answer(
3857 "GetMicrovm",
3858 Answer::ok(fake::microvm_response("RUNNING", None)),
3859 )
3860 .answer(
3861 "GetMicrovm",
3862 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3863 )
3864 .answer(
3865 "CreateMicrovmAuthToken",
3866 Answer::ok(fake::auth_token_response("proxy-token")),
3867 )
3868 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
3869 sandbox
3870 .run(RunRequest::new().with_image("arn:image"))
3871 .await
3872 .expect("launches");
3873 assert_eq!(*watch.borrow(), Lifecycle::Running);
3874 sandbox.suspend().await.expect("suspends");
3875 assert_eq!(*watch.borrow(), Lifecycle::Suspended);
3876 assert_eq!(*watch.borrow(), sandbox.lifecycle());
3877 }
3878
3879 #[tokio::test]
3880 async fn a_resume_past_the_suspended_window_is_refused_before_the_wire() {
3881 // A short window, so the arithmetic is legible: sixty seconds asked for at launch.
3882 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
3883 assert_eq!(sandbox.suspended_window(), Some(Duration::from_secs(60)));
3884 assert_eq!(
3885 sandbox.idle_window(),
3886 Some(Duration::from_secs(600)),
3887 "the launch's own maxIdleDurationSeconds, which a keepalive checks its cadence against"
3888 );
3889
3890 // The window closes. This is the whole of what the clock injection buys.
3891 clock.advance(Duration::from_secs(61));
3892
3893 // What a service says about a VM the idlePolicy already terminated — which is what a
3894 // client without the local check would spend its poll timeout discovering.
3895 recorder
3896 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3897 .answer(
3898 "GetMicrovm",
3899 Answer::ok(fake::microvm_response(
3900 "TERMINATED",
3901 Some("suspended window elapsed"),
3902 )),
3903 );
3904 let elapsed_before = clock.now();
3905 let resumes_before = recorder.call_count("ResumeMicrovm");
3906
3907 let error = sandbox
3908 .resume()
3909 .await
3910 .expect_err("the window the launch set has closed");
3911
3912 assert_eq!(error.kind(), ErrorKind::WindowClosed);
3913 assert_eq!(error.code(), "ERR_WINDOW_CLOSED");
3914 let message = error.to_string();
3915 assert!(message.contains("61s"), "the elapsed time: {message}");
3916 assert!(
3917 message.contains("60s suspendedDurationSeconds"),
3918 "the window has to be named: {message}"
3919 );
3920 assert!(message.contains("idlePolicy"), "the finding: {message}");
3921 assert!(
3922 message.contains("Refused before ResumeMicrovm"),
3923 "why the refusal is local: {message}"
3924 );
3925 assert_eq!(
3926 recorder.call_count("ResumeMicrovm"),
3927 resumes_before,
3928 "the refusal must come before ResumeMicrovm"
3929 );
3930 assert_eq!(
3931 clock.now(),
3932 elapsed_before,
3933 "the refusal must be immediate rather than polled to the deadline"
3934 );
3935 }
3936
3937 /// **STATE-12's fallback.** With no window from its own request, the sandbox uses the
3938 /// `suspendedDurationSeconds` that `GetMicrovm` reported (600 in the fake).
3939 #[tokio::test]
3940 async fn the_window_falls_back_to_the_idle_policy_the_service_reported() {
3941 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
3942 sandbox.suspended_window = None;
3943 recorder.answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3944 clock.advance(Duration::from_secs(601));
3945 let error = sandbox
3946 .resume()
3947 .await
3948 .expect_err("past the reported window");
3949 assert_eq!(error.kind(), ErrorKind::WindowClosed);
3950 assert!(error.to_string().contains("600s"), "{error}");
3951 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
3952 }
3953
3954 /// **#195 at the sandbox.** A client-token launch that adopts an idle-suspended VM is
3955 /// resumed and reaches RUNNING with the token counted once. The same poll on a launch
3956 /// without a caller token is still a startup death (TRAP-8).
3957 #[tokio::test]
3958 async fn a_client_token_launch_resumes_the_vm_it_adopted() {
3959 let (mut sandbox, recorder, _) = planted();
3960 recorder
3961 .answer(
3962 "RunMicrovm",
3963 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3964 )
3965 .answer(
3966 "GetMicrovm",
3967 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3968 )
3969 .answer(
3970 "GetMicrovm",
3971 Answer::ok(fake::microvm_response("RUNNING", None)),
3972 )
3973 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3974 .answer(
3975 "CreateMicrovmAuthToken",
3976 Answer::ok(fake::auth_token_response("proxy-token")),
3977 );
3978 let mut request = RunRequest::new().with_image("arn:image");
3979 request.client_token = Some("job-195".into());
3980 request.agent_token = Some("agent-token".into());
3981 sandbox
3982 .run(request)
3983 .await
3984 .expect("the adopted VM is resumed");
3985 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3986 assert_eq!(sandbox.bootstrap_count(), 1);
3987 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3988
3989 let (mut fresh, recorder, _) = planted();
3990 recorder
3991 .answer(
3992 "RunMicrovm",
3993 Answer::ok(fake::microvm_response("PENDING", None)),
3994 )
3995 .answer(
3996 "GetMicrovm",
3997 Answer::ok(fake::microvm_response("SUSPENDED", Some("hook failed"))),
3998 );
3999 let error = fresh
4000 .run(RunRequest::new().with_image("arn:image"))
4001 .await
4002 .expect_err("a fresh launch that reads SUSPENDED died during startup");
4003 assert_eq!(error.kind(), ErrorKind::LaunchDied);
4004 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4005 }
4006
4007 /// **`wait: false`.** The launch returns once accepted, with an addressable session and
4008 /// the lifecycle still PENDING; `wait_until_running` finishes it, exactly once.
4009 #[tokio::test]
4010 async fn a_launch_that_does_not_wait_is_finished_by_wait_until_running() {
4011 let (mut sandbox, recorder, _) = planted();
4012 answer_launch(&recorder);
4013 let mut request = RunRequest::new().with_image("arn:image");
4014 request.wait = false;
4015 let endpoint = sandbox
4016 .run(request)
4017 .await
4018 .expect("accepted")
4019 .endpoint()
4020 .to_string();
4021 assert!(endpoint.starts_with("https://"), "{endpoint}");
4022 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
4023 assert_eq!(sandbox.bootstrap_count(), 0);
4024 assert_eq!(recorder.call_count("GetMicrovm"), 0, "nothing polled yet");
4025
4026 sandbox
4027 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4028 .await
4029 .expect("reaches RUNNING");
4030 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4031 assert_eq!(sandbox.bootstrap_count(), 1);
4032 let error = sandbox
4033 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4034 .await
4035 .expect_err("a running launch has nothing to wait for");
4036 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4037 assert_eq!(sandbox.bootstrap_count(), 1, "counted once (STATE-3)");
4038 }
4039
4040 /// A resume **inside** the window goes through, so the guard is a comparison rather than
4041 /// a blanket refusal of every resume.
4042 ///
4043 /// The boundary is inclusive: elapsed exactly equal to the window is still open, matching
4044 /// the Python's `elapsed <= window`.
4045 #[tokio::test]
4046 async fn a_resume_inside_the_window_is_allowed_and_the_boundary_is_inclusive() {
4047 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
4048
4049 // Exactly at the window.
4050 clock.advance(Duration::from_secs(60));
4051 recorder
4052 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4053 .answer(
4054 "GetMicrovm",
4055 Answer::ok(fake::microvm_response("RUNNING", None)),
4056 );
4057 sandbox
4058 .resume()
4059 .await
4060 .expect("60s == the 60s window is open");
4061 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4062 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4063 }
4064
4065 /// A successful resume clears the stamp, so the next cycle's window is measured from the
4066 /// next suspend rather than from the first one.
4067 ///
4068 /// The failure this rules out is subtle and would only show on a second cycle:
4069 /// accumulating every suspension's elapsed time into one total refuses a resume whose own
4070 /// window is wide open. The clock is advanced past the window *between* the two cycles,
4071 /// which is what makes the accumulating implementation fail here.
4072 #[tokio::test]
4073 async fn a_successful_resume_clears_the_stamp_so_a_second_cycle_measures_its_own_window() {
4074 let (mut sandbox, recorder, clock) = planted();
4075 // Every `GetMicrovm` answer for both cycles, in the order the waits consume them:
4076 // the launch's RUNNING, then SUSPENDED / RUNNING twice. Queued up front for the
4077 // reason `suspended_with_window` documents — a stale answer left at the head makes a
4078 // wait poll twice, and the fake's `sleep` moves the very clock this test is measuring.
4079 recorder
4080 .answer(
4081 "RunMicrovm",
4082 Answer::ok(fake::microvm_response("PENDING", None)),
4083 )
4084 .answer(
4085 "CreateMicrovmAuthToken",
4086 Answer::ok(fake::auth_token_response("proxy-token")),
4087 )
4088 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
4089 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4090 .answer(
4091 "GetMicrovm",
4092 Answer::ok(fake::microvm_response("RUNNING", None)),
4093 )
4094 .answer(
4095 "GetMicrovm",
4096 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4097 )
4098 .answer(
4099 "GetMicrovm",
4100 Answer::ok(fake::microvm_response("RUNNING", None)),
4101 )
4102 .answer(
4103 "GetMicrovm",
4104 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4105 )
4106 .answer(
4107 "GetMicrovm",
4108 Answer::ok(fake::microvm_response("RUNNING", None)),
4109 );
4110 sandbox
4111 .run(
4112 RunRequest::new()
4113 .with_image("arn:image")
4114 .with_suspended_sec(60),
4115 )
4116 .await
4117 .expect("launches");
4118
4119 for cycle in 0..2 {
4120 sandbox
4121 .suspend()
4122 .await
4123 .unwrap_or_else(|error| panic!("cycle {cycle} suspend: {error}"));
4124 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended, "cycle {cycle}");
4125
4126 // Forty-five seconds each time — inside the sixty-second window on its own, and
4127 // ninety in total. An implementation that never cleared the stamp measures the
4128 // total and refuses the second resume, which is the whole point of the loop.
4129 clock.advance(Duration::from_secs(45));
4130
4131 sandbox.resume().await.unwrap_or_else(|error| {
4132 panic!("cycle {cycle} resume, whose own window is wide open: {error}")
4133 });
4134 assert_eq!(sandbox.lifecycle(), Lifecycle::Running, "cycle {cycle}");
4135 assert_eq!(
4136 sandbox.bootstrap_count(),
4137 1,
4138 "cycle {cycle}: no re-bootstrap"
4139 );
4140 }
4141
4142 assert_eq!(
4143 clock.now(),
4144 Duration::from_secs(90),
4145 "ninety seconds of accumulated suspension against a sixty-second window: this is \
4146 the total an implementation that never cleared the stamp would have compared"
4147 );
4148 assert_eq!(
4149 recorder.call_count("ResumeMicrovm"),
4150 2,
4151 "both cycles resumed"
4152 );
4153 }
4154
4155 /// **STATE-11, the resume-after-terminate case the packet names.** A terminated VM never
4156 /// returns to RUNNING, and the recorder sees **zero** resume wire calls.
4157 ///
4158 /// The zero count is the assertion the packet asks for, and it is the right one: a client
4159 /// that called and read the answer would also fail, with whatever the service says about
4160 /// a terminated id — which is a different statement from "this VM is gone and even a
4161 /// successful call would hand you a different machine".
4162 #[tokio::test]
4163 async fn a_resume_after_terminate_records_zero_resume_calls() {
4164 let (mut sandbox, recorder, _) = launched().await;
4165 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4166 let report = sandbox.terminate(TeardownOpts::default()).await;
4167 assert!(report.terminate_accepted);
4168 assert!(sandbox.was_terminated());
4169
4170 let error = sandbox
4171 .resume()
4172 .await
4173 .expect_err("a terminated VM never returns to RUNNING");
4174 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4175 assert!(error.to_string().contains("STATE-11"), "{error}");
4176 assert_eq!(
4177 recorder.call_count("ResumeMicrovm"),
4178 0,
4179 "no resume may reach the wire once the VM was terminated"
4180 );
4181 assert_ne!(
4182 sandbox.lifecycle(),
4183 Lifecycle::Running,
4184 "the lifecycle must not return to RUNNING"
4185 );
4186 }
4187
4188 /// **STATE-9 and STATE-10.** A terminate accepted is TERMINATING with the VM recorded
4189 /// terminated; the platform reporting termination complete is what marks TERMINATED.
4190 ///
4191 /// The two are separate states rather than one because the default teardown does not
4192 /// wait — so TERMINATING is a state a report really ends in, and calling it TERMINATED
4193 /// would claim an observation nobody made.
4194 #[tokio::test]
4195 async fn a_terminate_records_terminating_and_the_completion_report_marks_terminated() {
4196 let (mut sandbox, recorder, _) = launched().await;
4197 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4198
4199 let report = sandbox.terminate(TeardownOpts::default()).await;
4200 assert!(report.terminate_accepted, "STATE-9: the call was accepted");
4201 assert!(sandbox.was_terminated(), "STATE-9: recorded terminated");
4202 assert_eq!(
4203 report.lifecycle,
4204 Some(Lifecycle::Terminating),
4205 "the default teardown does not wait, so TERMINATED is not claimed"
4206 );
4207 assert!(!report.leaked(), "{report:?}");
4208
4209 // And with the wait asked for, STATE-10 is observed.
4210 let (mut sandbox, recorder, _) = launched().await;
4211 recorder
4212 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4213 .answer(
4214 "GetMicrovm",
4215 Answer::ok(fake::microvm_response("TERMINATED", None)),
4216 );
4217 let report = sandbox
4218 .terminate(TeardownOpts::default().waiting_for_terminated())
4219 .await;
4220 assert_eq!(report.lifecycle, Some(Lifecycle::Terminated));
4221 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4222 }
4223
4224 /// The teardown order is VM, then image, then the log group **last** — asserted on the
4225 /// recorder's ledger rather than on the report, because the order is a property of what
4226 /// was emitted.
4227 ///
4228 /// The log group is last because the service can recreate a group deleted before its
4229 /// image. This crate cannot delete it at all, so the assertion is that it is *named
4230 /// after* the image work rather than that a delete call went out — see
4231 /// `TeardownReport::undeleted`.
4232 #[tokio::test]
4233 async fn teardown_deletes_the_vm_then_the_image_and_names_the_log_group_last() {
4234 let (mut sandbox, recorder, _) = planted();
4235 recorder
4236 .answer(
4237 "CreateMicrovmImage",
4238 Answer::created(fake::create_image_response("agentd-conformance")),
4239 )
4240 .answer(
4241 "GetMicrovmImage",
4242 Answer::ok(fake::get_image_response("agentd-conformance", "CREATED")),
4243 );
4244 sandbox
4245 .build_image(CreateImageRequest::new(
4246 "agentd-conformance",
4247 crate::testing::aarch64_daemon(b"binary"),
4248 "s3://bucket/img.zip",
4249 "arn:aws:iam::123456789012:role/build",
4250 ))
4251 .await
4252 .expect("builds");
4253 answer_launch(&recorder);
4254 sandbox
4255 .run(RunRequest::new())
4256 .await
4257 .expect("launches from the built image");
4258
4259 recorder
4260 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4261 .answer(
4262 "ListMicrovmImageVersions",
4263 Answer::ok(fake::list_versions_response("1")),
4264 )
4265 .answer(
4266 "DeleteMicrovmImage",
4267 Answer::ok(fake::delete_image_response()),
4268 );
4269
4270 let report = sandbox
4271 .terminate(
4272 TeardownOpts::default()
4273 .deleting_image()
4274 .deleting_log_group(),
4275 )
4276 .await;
4277 assert!(report.terminate_accepted);
4278 assert_eq!(report.image_deleted, Some(true));
4279
4280 // The ledger, filtered to the three destructive operations.
4281 let order: Vec<&str> = recorder
4282 .operations()
4283 .into_iter()
4284 .filter(|operation| matches!(*operation, "TerminateMicrovm" | "DeleteMicrovmImage"))
4285 .collect();
4286 assert_eq!(
4287 order,
4288 ["TerminateMicrovm", "DeleteMicrovmImage"],
4289 "the VM goes before the image: a terminating VM holds a reference to it"
4290 );
4291
4292 // The log group is named, and named last.
4293 assert_eq!(
4294 report.undeleted,
4295 ["/aws/lambda-microvms/agentd-conformance"],
4296 "{report:?}"
4297 );
4298 assert!(report.leaked());
4299 assert!(
4300 report
4301 .failures
4302 .last()
4303 .expect("a failure")
4304 .contains("log group"),
4305 "the log group's failure has to be the last one recorded: {report:?}"
4306 );
4307 assert!(
4308 report
4309 .failures
4310 .last()
4311 .expect("a failure")
4312 .contains("terraform destroy"),
4313 "the reason it leaks has to be named: {report:?}"
4314 );
4315 }
4316
4317 /// An image delete that fails every attempt lands its identifier in the report rather
4318 /// than raising, so the CLI can emit what was left behind (CLI-6).
4319 ///
4320 /// The identifier rather than a flag is the whole point: a leak nobody can name is a leak
4321 /// nobody can clean up.
4322 #[tokio::test]
4323 async fn a_failing_image_delete_names_what_it_left_behind() {
4324 let (mut sandbox, recorder, _) = planted();
4325 recorder
4326 .answer(
4327 "CreateMicrovmImage",
4328 Answer::created(fake::create_image_response("img")),
4329 )
4330 .answer(
4331 "GetMicrovmImage",
4332 Answer::ok(fake::get_image_response("img", "CREATED")),
4333 );
4334 sandbox
4335 .build_image(CreateImageRequest::new(
4336 "img",
4337 crate::testing::aarch64_daemon(b"binary"),
4338 "s3://bucket/img.zip",
4339 "arn:aws:iam::123456789012:role/build",
4340 ))
4341 .await
4342 .expect("builds");
4343 answer_launch(&recorder);
4344 sandbox.run(RunRequest::new()).await.expect("launches");
4345
4346 recorder
4347 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4348 .answer(
4349 "ListMicrovmImageVersions",
4350 Answer::ok(fake::list_versions_response("1")),
4351 )
4352 .answer(
4353 "DeleteMicrovmImage",
4354 Answer::failure(409, "image is in CREATING"),
4355 );
4356
4357 let mut opts = TeardownOpts::default().deleting_image();
4358 opts.delete_attempts = 3;
4359 opts.delete_backoff = Duration::from_secs(1);
4360 let report = sandbox.terminate(opts).await;
4361
4362 assert_eq!(report.image_deleted, Some(false));
4363 assert_eq!(
4364 report.undeleted,
4365 ["arn:aws:lambda:us-east-1:123456789012:microvm-image:img"],
4366 "{report:?}"
4367 );
4368 assert!(
4369 sandbox.image_exists(),
4370 "the image really is still there, so the flag must say so"
4371 );
4372 assert_eq!(
4373 recorder.call_count("DeleteMicrovmImage"),
4374 3,
4375 "every attempt ran"
4376 );
4377 }
4378
4379 /// A terminate whose own call fails still records the VM as one this client asked to
4380 /// destroy, and names it as undeleted.
4381 ///
4382 /// Recording it is what stops a later resume from being offered against a VM the caller
4383 /// already tried to kill (STATE-11) — the alternative leaves the sandbox looking
4384 /// resumable, which is the worse of the two wrong answers.
4385 #[tokio::test]
4386 async fn a_terminate_whose_call_fails_still_records_the_intent_and_the_leak() {
4387 let (mut sandbox, recorder, _) = launched().await;
4388 recorder.answer(
4389 "TerminateMicrovm",
4390 Answer::failure(500, "InternalServerException"),
4391 );
4392
4393 let report = sandbox.terminate(TeardownOpts::default()).await;
4394 assert!(!report.terminate_accepted);
4395 assert_eq!(report.undeleted, ["mvm-abc123"], "{report:?}");
4396 assert!(sandbox.was_terminated());
4397 assert_eq!(report.lifecycle, Some(Lifecycle::Terminating));
4398 assert!(
4399 !report.failures.is_empty(),
4400 "the swallowed failure is recorded"
4401 );
4402 }
4403
4404 /// A teardown on a sandbox that never launched is a no-op report rather than a panic.
4405 ///
4406 /// The path a caller reaches by tearing down in a `finally` after a launch that failed
4407 /// before any VM existed, which is exactly when a teardown that assumed a VM would turn
4408 /// the real failure into a panic.
4409 #[tokio::test]
4410 async fn a_teardown_before_any_launch_is_an_empty_report() {
4411 let (mut sandbox, recorder, _) = planted();
4412 let report = sandbox.terminate(TeardownOpts::default()).await;
4413 assert!(!report.terminate_accepted);
4414 assert!(!report.leaked());
4415 assert_eq!(report.lifecycle, Some(Lifecycle::Pending));
4416 assert_eq!(recorder.calls().len(), 0);
4417 }
4418
4419 /// A run with no image at all is a precondition error before any call, rather than a
4420 /// launch of an empty identifier.
4421 #[tokio::test]
4422 async fn a_run_with_no_image_is_refused_before_any_call() {
4423 let (mut sandbox, recorder, _) = planted();
4424 let error = sandbox
4425 .run(RunRequest::new())
4426 .await
4427 .expect_err("there is nothing to launch");
4428 assert_eq!(error.kind(), ErrorKind::Precondition);
4429 assert!(error.to_string().contains("build_image first"), "{error}");
4430 assert_eq!(recorder.calls().len(), 0);
4431 }
4432
4433 /// The lifecycle's six states are the symspec's `vm_state` domain, spelled as the service
4434 /// spells them.
4435 ///
4436 /// Asserted rather than assumed because the spelling reaches an error message a reader
4437 /// compares against a `GetMicrovm` response: a lifecycle rendering `Suspended` beside a
4438 /// service saying `SUSPENDED` reads like two different facts.
4439 #[test]
4440 fn the_lifecycle_states_are_the_state_models_domain_in_the_services_spelling() {
4441 let all = [
4442 Lifecycle::Pending,
4443 Lifecycle::Running,
4444 Lifecycle::Suspending,
4445 Lifecycle::Suspended,
4446 Lifecycle::Terminating,
4447 Lifecycle::Terminated,
4448 ];
4449 assert_eq!(
4450 all.map(Lifecycle::as_str),
4451 [
4452 "PENDING",
4453 "RUNNING",
4454 "SUSPENDING",
4455 "SUSPENDED",
4456 "TERMINATING",
4457 "TERMINATED"
4458 ]
4459 );
4460 for state in all {
4461 assert_eq!(state.to_string(), state.as_str());
4462 }
4463 // The four live states are the ones a `Drop` warns about, and TERMINATING is not one
4464 // of them: the platform accepted the terminate, so nobody needs telling.
4465 assert_eq!(
4466 all.iter().filter(|state| state.is_live()).count(),
4467 4,
4468 "a state added to the enum defaults to not-live, which would silence the warning"
4469 );
4470 assert!(!Lifecycle::Terminating.is_live());
4471 assert!(!Lifecycle::Terminated.is_live());
4472 }
4473
4474 /// An agent token is 64 hex characters and never repeats across draws.
4475 ///
4476 /// The distinctness matters because the token is the only thing separating one VM's
4477 /// control API from another's: two VMs sharing a token means either one's caller can
4478 /// drive the other's guest. The token is the source's draw, so a fresh source per launch
4479 /// is fresh tokens; the OS pool's own distinctness is `microvms-edges`'s
4480 /// `the_os_source_draws_distinct_bytes`.
4481 #[test]
4482 fn a_minted_agent_token_is_sixty_four_hex_characters_and_never_repeats() {
4483 let entropy = crate::entropy::testing::SequenceEntropy::new();
4484 let minted: std::collections::HashSet<String> = (0..200)
4485 .map(|_| mint_agent_token(&entropy).expect("scripted"))
4486 .collect();
4487 assert_eq!(minted.len(), 200, "a repeated token is a shared guest");
4488 for token in &minted {
4489 assert_eq!(token.len(), 64, "{token}");
4490 assert!(token.chars().all(|c| c.is_ascii_hexdigit()), "{token}");
4491 assert_ne!(
4492 token,
4493 &"0".repeat(64),
4494 "an all-zero draw is a read that did nothing"
4495 );
4496 }
4497 }
4498
4499 /// A sandbox's `Debug` does not print the agent token.
4500 #[tokio::test]
4501 async fn a_sandbox_debug_does_not_print_the_agent_token() {
4502 let (mut sandbox, recorder, _) = planted();
4503 answer_launch(&recorder);
4504 sandbox
4505 .run(
4506 RunRequest::new()
4507 .with_image("arn:image")
4508 .with_agent_token("super-secret-agent-token"),
4509 )
4510 .await
4511 .expect("launches");
4512
4513 let rendered = format!("{sandbox:?}");
4514 assert!(rendered.contains("mvm-abc123"), "{rendered}");
4515 assert!(rendered.contains("Running"), "{rendered}");
4516 assert!(
4517 !rendered.contains("super-secret"),
4518 "the agent token reached a Debug string: {rendered}"
4519 );
4520 }
4521
4522 /// A torn-down sandbox does not warn on drop, and an abandoned one is the case the
4523 /// warning exists for.
4524 ///
4525 /// Asserted on the flag rather than on stderr — capturing another thread's stderr from a
4526 /// test is not something this crate can do without a dependency — so what is pinned is
4527 /// the condition the warning branches on. Both halves, so the branch cannot be
4528 /// vacuously true.
4529 #[tokio::test]
4530 async fn a_torn_down_sandbox_is_distinguishable_from_an_abandoned_one() {
4531 let (mut sandbox, recorder, _) = launched().await;
4532 assert!(
4533 sandbox.lifecycle().is_live() && !sandbox.torn_down,
4534 "an abandoned live VM is what the drop warning is for"
4535 );
4536
4537 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4538 sandbox.terminate(TeardownOpts::default()).await;
4539 assert!(
4540 sandbox.torn_down,
4541 "terminate must mark the sandbox torn down"
4542 );
4543 assert!(
4544 !sandbox.lifecycle().is_live(),
4545 "and the lifecycle must no longer read as live"
4546 );
4547 }
4548
4549 #[tokio::test]
4550 async fn stable_launch_key_and_payload_survive_a_fresh_supervisor() {
4551 let mut bodies = Vec::new();
4552 for _ in 0..2 {
4553 let (mut sandbox, recorder, _) = planted();
4554 answer_launch(&recorder);
4555 let mut request = RunRequest::new().with_image("arn:image");
4556 request.client_token = Some("one-unique-job-launch".into());
4557 request.agent_token = Some("persisted-private-secret".into());
4558 sandbox.run(request).await.expect("accepted");
4559 bodies.push(recorder.first_body("RunMicrovm"));
4560 }
4561 assert_eq!(bodies[0]["clientToken"], "one-unique-job-launch");
4562 assert_eq!(bodies[0], bodies[1]);
4563 }
4564
4565 #[tokio::test]
4566 async fn unsafe_or_malformed_stable_launch_is_refused_without_calls() {
4567 for (token, secret, identity) in [
4568 ("key".to_string(), None, false),
4569 ("key".to_string(), Some("secret".into()), true),
4570 ("".to_string(), Some("secret".into()), false),
4571 ("x".repeat(129), Some("secret".into()), false),
4572 ("line\nbreak".to_string(), Some("secret".into()), false),
4573 ] {
4574 let (mut sandbox, recorder, _) = planted();
4575 let mut request = RunRequest::new().with_image("arn:image");
4576 request.client_token = Some(token);
4577 request.agent_token = secret;
4578 request.identity = identity;
4579 let error = sandbox.run(request).await.expect_err("invalid");
4580 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4581 assert!(recorder.calls().is_empty());
4582 }
4583 }
4584}