Skip to main content

microvms_app/
sandbox.rs

1// SPDX-License-Identifier: Apache-2.0
2//! The `Sandbox` lifecycle: one VM's whole life, with the state machine the symspec
3//! model describes made true of the code.
4//!
5//! Composes [`crate::control`] and [`crate::session`] into the surface the CLI and the
6//! bindings use: build, run, suspend, resume, terminate, a resume of a VM that ended
7//! before it (STATE-12), and teardown in the order that does not leak.
8//!
9//! # The lifecycle is a field, and every transition is guarded
10//!
11//! [`Lifecycle`] is the symspec's `vm_state` verbatim, and [`Sandbox`] carries the other
12//! four variables beside it — `token_installed`, `image_exists`, `was_terminated`,
13//! `bootstrap_count`. The Z3 proofs over that model (bootstrap at most once, suspend from
14//! non-RUNNING unreachable, TERMINATED never returns to RUNNING) are proofs about *this*
15//! struct's reachable states, which is only worth something if the transitions here are
16//! the only way to move it. Those fields are private and transitions are guarded here.
17//! Adoption initializes an existing VM from service-observed state and is covered by
18//! the adoption tests; it does not replay the launch bootstrap.
19//!
20//! # Runtime-checked rather than typestate, deliberately
21//!
22//! The packet offered `Sandbox<Running>` returning a `Suspended` handle, which would make
23//! STATE-5's wrong call a compile error rather than a local refusal — strictly stronger on
24//! the ladder in [`crate`]. It is not what landed, for the reason the packet names as
25//! acceptable: T-W3-8 wraps **one** object for PyO3 and napi-rs, and a type whose Rust
26//! identity changes on every transition cannot be one `#[pyclass]`. A typestate sandbox
27//! would be re-erased into a runtime-checked enum at the binding boundary, so the check
28//! would exist twice with the binding's copy being the one most callers actually hit.
29//!
30//! What is kept from the typestate idea is the part that costs nothing: the check happens
31//! **before** the wire call, so a suspend from SUSPENDED is refused with zero
32//! control-plane calls rather than answered by AWS. The test asserts the call count, which
33//! is the observable that distinguishes the two.
34//!
35//! # The service decides whether a suspended VM resumes (STATE-12)
36//!
37//! The launch-time `idlePolicy` *terminates* a VM the idle policy suspended once
38//! `suspendedDurationSeconds` passes, and the service doesn't hold an explicit
39//! `SuspendMicrovm` to that window: measured 2026-10-12 in us-east-1, a VM this client
40//! suspended resumed 1283 s into a 60 s window (#597). So a resume is never refused here for
41//! the time it has spent suspended. It goes to the service, and the service's refusal for a
42//! VM that ended, whatever ended it, comes back from [`ControlPlane::resume`] as
43//! [`ErrorKind::WindowClosed`], which this sandbox records as a termination (STATE-11).
44//!
45//! A sandbox held as RUNNING reads `GetMicrovm` before deciding whether it can resume.
46//! This observes a suspend or termination made by the platform idle policy. A service
47//! state of SUSPENDED follows the normal resume path; an ended VM is `WindowClosed`.
48//!
49//! # Teardown never raises, and the log group is last
50//!
51//! [`Sandbox::terminate`] returns a [`TeardownReport`] rather than a `Result`. It runs
52//! where a caller's `finally` would, and an error raised there replaces the real failure
53//! with a teardown failure — the real one being the one worth reading.
54//!
55//! The order is VM, then image (retrying), then the log group **last**, because the
56//! service can recreate a group deleted before its image. See
57//! [`TeardownReport::undeleted`] for what this crate can and cannot delete.
58//!
59//! # There is no `Drop` that tears down
60//!
61//! Rust has no context manager and `Drop` cannot await. A `Drop` that blocked on a runtime
62//! would deadlock inside one; a `Drop` that spawned would race the process exit. So
63//! [`Sandbox`]'s `Drop` only **warns** about a live VM, naming the id, and the rule is
64//! that a caller calls [`Sandbox::terminate`] explicitly.
65
66use std::sync::Arc;
67use std::time::Duration;
68
69use crate::control::{
70    ControlPlane, CreateImageRequest, Image, Microvm, RunHookPayload, RunMicrovmRequest, WaitOpts,
71};
72use crate::error::{Error, ErrorKind};
73use crate::region::Region;
74use crate::session::{Session, TokenMinter};
75
76/// The default wait for a launch to reach RUNNING: five minutes, matching the Python client's
77/// `ready_timeout_sec`. The daemon wait after it is
78/// [`crate::session::DEFAULT_BOOTSTRAP_TIMEOUT`].
79pub const DEFAULT_RUNNING_TIMEOUT: Duration = Duration::from_secs(300);
80
81/// The old name of [`DEFAULT_RUNNING_TIMEOUT`], kept for one release.
82#[deprecated(
83    note = "renamed DEFAULT_RUNNING_TIMEOUT: `session::DEFAULT_READY_TIMEOUT` shared this name \
84            for a different wait, the daemon's"
85)]
86pub const DEFAULT_READY_TIMEOUT: Duration = DEFAULT_RUNNING_TIMEOUT;
87
88/// The default lifecycle wait for suspend, resume, and terminate, and for a state wait the
89/// caller gives no deadline (the bindings' `wait_for_state`, the CLI's `terminate --wait`).
90pub const DEFAULT_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(300);
91
92/// How many times a teardown retries the image delete.
93///
94/// Twenty, from the Python client: an image in `CREATING` refuses deletion and a VM still
95/// terminating holds a reference, so this is the difference between a clean account and a
96/// billed leak rather than politeness.
97pub const DEFAULT_DELETE_ATTEMPTS: u32 = 20;
98
99/// The gap between image-delete attempts.
100pub const DEFAULT_DELETE_BACKOFF: Duration = Duration::from_secs(15);
101
102/// Where the advisory deny points a well-behaved client: a loopback port nothing serves.
103///
104/// Loopback rather than an unroutable public address, because a refused connection is
105/// immediate and a black-holed one costs the workload a connect timeout per request. Port 1
106/// is privileged, so a demoted workload cannot answer it by accident.
107pub const DENY_EGRESS_PROXY_URL: &str = "http://127.0.0.1:1";
108
109/// The environment variables the advisory deny sets, in both spellings clients read.
110///
111/// Both cases on purpose: `curl` and most Unix clients read the lowercase names, Python's
112/// `requests`/`urllib3`, npm and the AWS SDKs read the uppercase ones, and a client that
113/// reads only the spelling this list omits is a hole in a mechanism that is advisory to
114/// begin with. `no_proxy` is deliberately absent: the launch environment starts empty
115/// (`agentd` clears it and applies only what the hook delivered), so there is no inherited
116/// exemption to overwrite, and each key spends payload budget.
117pub const DENY_EGRESS_ENV_KEYS: [&str; 6] = [
118    "http_proxy",
119    "https_proxy",
120    "all_proxy",
121    "HTTP_PROXY",
122    "HTTPS_PROXY",
123    "ALL_PROXY",
124];
125
126/// Default lifecycle polling interval. Public so an adapter that builds its own [`WaitOpts`]
127/// reuses the interval. Sandbox launches poll faster during their initial retries.
128pub const LIFECYCLE_POLL_INTERVAL: Duration = Duration::from_secs(5);
129
130/// The symspec's `vm_state`, verbatim.
131///
132/// Six states and no others, which is the S1 half of this module: a lifecycle held as a
133/// `String` would let `"RUNNING "` and `"Running"` both exist, and every guard below would
134/// have to decide which it meant.
135#[derive(Clone, Copy, Debug, Eq, PartialEq)]
136pub enum Lifecycle {
137    /// The initial state, and the state a launch is accepted into (STATE-1).
138    Pending,
139    /// The service reports RUNNING; launch has completed bootstrap, or adoption supplied
140    /// the existing token (STATE-2).
141    Running,
142    /// A suspend was accepted and the platform has not yet reported it complete (STATE-4).
143    Suspending,
144    /// The platform reported suspension complete (STATE-6).
145    Suspended,
146    /// A terminate was accepted (STATE-9).
147    Terminating,
148    /// The platform reported termination complete (STATE-10).
149    Terminated,
150}
151
152impl Lifecycle {
153    /// The name the service uses for this state, which is also what an error message says.
154    pub fn as_str(self) -> &'static str {
155        match self {
156            Lifecycle::Pending => "PENDING",
157            Lifecycle::Running => "RUNNING",
158            Lifecycle::Suspending => "SUSPENDING",
159            Lifecycle::Suspended => "SUSPENDED",
160            Lifecycle::Terminating => "TERMINATING",
161            Lifecycle::Terminated => "TERMINATED",
162        }
163    }
164
165    /// The lifecycle the service's `state` names, or `None` for a spelling this client does
166    /// not know. `constants::MICROVM_STATES` is the closed set, and a test ties the two.
167    pub fn from_service(state: &str) -> Option<Self> {
168        Some(match state {
169            "PENDING" => Lifecycle::Pending,
170            "RUNNING" => Lifecycle::Running,
171            "SUSPENDING" => Lifecycle::Suspending,
172            "SUSPENDED" => Lifecycle::Suspended,
173            "TERMINATING" => Lifecycle::Terminating,
174            "TERMINATED" => Lifecycle::Terminated,
175            _ => return None,
176        })
177    }
178
179    /// Whether a VM in this state is still billing, which is what a `Drop` warning is for.
180    pub fn is_live(self) -> bool {
181        matches!(
182            self,
183            Lifecycle::Pending | Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended
184        )
185    }
186}
187
188impl std::fmt::Display for Lifecycle {
189    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
190        f.write_str(self.as_str())
191    }
192}
193
194/// Everything a launch needs, with the defaults the Python client measured.
195///
196/// `agent_token` is optional because the common case is a per-VM secret nobody needs to
197/// see; a caller who has one already — a harness minting its own, or a retry that must
198/// reuse the first attempt's — passes it.
199#[derive(Clone, Debug)]
200pub struct RunRequest {
201    /// The image to launch, or `None` for the one [`Sandbox::build_image`] built.
202    ///
203    /// An image ARN, or a bare image name, which [`Sandbox::run`] resolves to its ARN with one
204    /// `ListMicrovmImages` read. A name no image carries is refused with `ERR_PRECONDITION`
205    /// before anything launches.
206    pub image_identifier: Option<String>,
207    /// `imageVersion`, or `None` for the image's own latest active version.
208    ///
209    /// Pinning is what makes a canary a canary: without it a launch takes whatever version is
210    /// latest at the moment the call lands, which is not necessarily the one the caller just
211    /// built. Paired with [`crate::control::ControlPlane::set_image_version_status`] it is also
212    /// the rollback — set the bad version INACTIVE, re-pin here to the good one — and a version
213    /// set INACTIVE refuses to launch when named here.
214    ///
215    /// See [`crate::control::RunMicrovmRequest::image_version`], which this forwards to.
216    pub image_version: Option<String>,
217    /// The execution role. Optional in the model; every real launch needs one.
218    pub execution_role_arn: Option<String>,
219    /// The bearer token the daemon will accept, or `None` to mint one.
220    pub agent_token: Option<String>,
221    /// Persist this unique launch key before sending a request; reuse only for that same
222    /// launch and identical parameters. None mints a fresh key. A stable key requires
223    /// an explicit agent_token and identity=false so retries carry identical payloads.
224    pub client_token: Option<String>,
225    /// Base environment for every exec in the launched VM, delivered in the same
226    /// `runHookPayload` as the token.
227    ///
228    /// The daemon applies this *under* each request's own `env`, so a per-exec value
229    /// wins on a key both set. Empty by default, and an empty map produces byte-for-byte
230    /// the payload this client always sent — a caller who never touches this field
231    /// cannot be affected by the field existing.
232    ///
233    /// It shares the token's 4096-byte payload budget, and the check is local:
234    /// [`crate::control::RunHookPayload::for_launch`] refuses an over-ceiling payload
235    /// before any call, naming the byte count and how much of it the env is. That
236    /// matters here more than for the token, because one bearer token has always fit
237    /// with room to spare and a map of credentials does not.
238    pub launch_env: std::collections::HashMap<String, String>,
239    /// Whether to generate and deliver a tunnel identity (#70 layer 3).
240    ///
241    /// On, the launch generates two fresh x25519 seeds, delivers the VM's seed and the
242    /// host's public key in the same payload as the token, and [`RunOutcome`]-equivalent
243    /// state on the session keeps the [`crate::identity::TunnelIdentity`] — the host secret
244    /// and the VM's public pin — for `tunnel --verify-identity` to prove the far end.
245    /// Off (the default) sends byte-for-byte the payload this client always sent.
246    ///
247    /// A flag rather than always-on, deliberately: the material spends 137 bytes of the
248    /// payload's measured 4096-byte budget, and a caller near the ceiling with a launch env
249    /// deserves to choose which feature gets the room.
250    pub identity: bool,
251    /// Whether to request the egress connector. Off omits it from the request; measured
252    /// 2026-09-12 the platform gave a connector-less VM outbound network anyway
253    /// (`docs/PLATFORM.md`).
254    pub egress: bool,
255    /// Customer-managed Lambda VPC egress connector ARNs. See
256    /// [`RunMicrovmRequest::egress_network_connectors`].
257    pub egress_network_connectors: Vec<String>,
258    /// Sets advisory proxy-deny environment variables for clients that honor them.
259    /// Workloads can bypass these variables. Internet isolation requires a VPC egress
260    /// connector using a VPC without an internet gateway or NAT gateway.
261    ///
262    /// Refused with [`RunRequest::egress`]. Existing caller-supplied proxy variables win;
263    /// the added variables share the 4096-byte run-hook payload budget.
264    pub deny_egress: bool,
265    /// Whether to launch shell-capable: the ingress set becomes the measured pair
266    /// `[HTTP_INGRESS, SHELL_INGRESS]` instead of `ALL_INGRESS`, which is what
267    /// `CreateMicrovmShellAuthToken` requires and what `microvm shell` attaches to.
268    /// Off (the default) launches byte-for-byte what this client always sent.
269    pub shell: bool,
270    /// `idlePolicy.maxIdleDurationSeconds`.
271    pub max_idle_sec: u32,
272    /// `idlePolicy.suspendedDurationSeconds`: how long a VM the idle policy suspended lasts
273    /// before the service terminates it. An explicit suspend isn't held to it (STATE-12).
274    pub suspended_sec: u32,
275    /// `idlePolicy.autoResumeEnabled`.
276    pub auto_resume: bool,
277    /// `maximumDurationInSeconds`, checked against 1..=28800 before the call.
278    pub max_duration_sec: u32,
279    /// How long to wait for RUNNING. The wait for the daemon to answer, after it, is
280    /// [`crate::session::DEFAULT_BOOTSTRAP_TIMEOUT`].
281    pub ready_timeout: Duration,
282    /// A label for the run token (TRAP-1). Never the token.
283    pub token_scope: Option<String>,
284    /// Per-VM `logging` (#201). See [`RunMicrovmRequest::logging`].
285    pub logging: Option<crate::control::ops::Logging>,
286    /// Whether [`Sandbox::run`] waits for RUNNING and for the daemon to answer before
287    /// returning (the default).
288    ///
289    /// Off returns as soon as `RunMicrovm` is accepted, with the lifecycle still PENDING:
290    /// the session is addressable, and [`Sandbox::wait_until_running`] finishes both waits
291    /// later, from this process or after a durable workflow's next step.
292    pub wait: bool,
293}
294
295impl Default for RunRequest {
296    fn default() -> Self {
297        Self {
298            image_identifier: None,
299            image_version: None,
300            execution_role_arn: None,
301            agent_token: None,
302            client_token: None,
303            launch_env: std::collections::HashMap::new(),
304            identity: false,
305            egress: false,
306            egress_network_connectors: Vec::new(),
307            deny_egress: false,
308            shell: false,
309            max_idle_sec: 600,
310            suspended_sec: 600,
311            auto_resume: false,
312            max_duration_sec: 3_600,
313            ready_timeout: DEFAULT_RUNNING_TIMEOUT,
314            token_scope: None,
315            logging: None,
316            wait: true,
317        }
318    }
319}
320
321impl RunRequest {
322    /// A launch with the measured defaults: ingress only, ten-minute idle and suspended
323    /// windows, a one-hour maximum duration, no auto-resume.
324    pub fn new() -> Self {
325        Self::default()
326    }
327
328    /// Launches `identifier`, an image ARN or a bare image name, rather than the built image.
329    #[must_use]
330    pub fn with_image(mut self, identifier: impl Into<String>) -> Self {
331        self.image_identifier = Some(identifier.into());
332        self
333    }
334
335    /// Requests the managed internet egress connector. Omission does not block internet access.
336    #[must_use]
337    pub fn with_egress(mut self) -> Self {
338        self.egress = true;
339        self
340    }
341
342    /// Routes egress through a customer-managed Lambda network connector.
343    #[must_use]
344    pub fn with_egress_network_connector(mut self, arn: impl Into<String>) -> Self {
345        self.egress_network_connectors.push(arn.into());
346        self
347    }
348
349    /// Replaces managed internet egress with customer-managed VPC connectors, when any are
350    /// given; an empty list leaves the request unchanged.
351    ///
352    /// The two cannot be combined, so a launch surface that requests managed egress by
353    /// default (an agent VM) uses this to switch rather than failing the launch.
354    #[must_use]
355    pub fn with_vpc_egress(mut self, arns: Vec<String>) -> Self {
356        if !arns.is_empty() {
357            self.egress = false;
358            self.egress_network_connectors = arns;
359        }
360        self
361    }
362
363    /// Applies the advisory in-guest deny. See [`RunRequest::deny_egress`] for what it is
364    /// and, more importantly, for what it is not.
365    #[must_use]
366    pub fn with_deny_egress(mut self) -> Self {
367        self.deny_egress = true;
368        self
369    }
370
371    /// What this launch's outbound network **is**, as opposed to what it asked for.
372    ///
373    /// The one derivation, so the CLI's envelope, the CLI's human line and any embedding
374    /// consumer cannot disagree about the same launch.
375    pub fn egress_posture(&self) -> crate::control::EgressPosture {
376        crate::control::EgressPosture::for_launch(self.egress, self.deny_egress)
377    }
378
379    /// The launch environment as the guest will see it: the caller's, plus the advisory
380    /// proxy deny when [`RunRequest::deny_egress`] is on.
381    ///
382    /// The caller's own value wins on a shared key. A launch that already routes through a
383    /// proxy of its own has said where its traffic goes, and overwriting that with a black
384    /// hole would break a working configuration in the name of a posture this client
385    /// already declines to call a seal.
386    pub fn effective_launch_env(&self) -> std::collections::HashMap<String, String> {
387        let mut env = self.launch_env.clone();
388        if self.deny_egress {
389            for key in DENY_EGRESS_ENV_KEYS {
390                env.entry(key.to_string())
391                    .or_insert_with(|| DENY_EGRESS_PROXY_URL.to_string());
392            }
393        }
394        env
395    }
396
397    /// Requests a shell-capable launch. See [`RunRequest::shell`].
398    #[must_use]
399    pub fn with_shell(mut self) -> Self {
400        self.shell = true;
401        self
402    }
403
404    /// Pins the launch to one `imageVersion`. See the field for why.
405    #[must_use]
406    pub fn with_image_version(mut self, version: impl Into<String>) -> Self {
407        self.image_version = Some(version.into());
408        self
409    }
410
411    /// Sets `idlePolicy.suspendedDurationSeconds`, how long a VM the idle policy suspended
412    /// lasts before the service terminates it. See [`RunRequest::suspended_sec`].
413    #[must_use]
414    pub fn with_suspended_sec(mut self, seconds: u32) -> Self {
415        self.suspended_sec = seconds;
416        self
417    }
418
419    /// Reuses a caller-supplied agent token rather than minting one.
420    #[must_use]
421    pub fn with_agent_token(mut self, token: impl Into<String>) -> Self {
422        self.agent_token = Some(token.into());
423        self
424    }
425
426    /// Generates and delivers a tunnel identity with the launch (#70 layer 3).
427    ///
428    /// After a `run` with this set, [`Sandbox::tunnel_identity`] holds what
429    /// `tunnel --verify-identity` needs: the host's secret and the VM's public pin.
430    #[must_use]
431    pub fn with_identity(mut self) -> Self {
432        self.identity = true;
433        self
434    }
435
436    /// Adds one launch-environment variable, which every exec in the VM starts with.
437    ///
438    /// One pair per call rather than a whole map, because that is how a caller builds
439    /// one — from flags, from a config file, one credential at a time — and a
440    /// map-taking setter makes the second call silently discard the first. The field is
441    /// public for a caller who really does hold a map.
442    #[must_use]
443    pub fn with_launch_env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
444        self.launch_env.insert(key.into(), value.into());
445        self
446    }
447}
448
449/// What a teardown should delete beyond the VM itself.
450///
451/// Both deletions are opt-in, because both destroy something a caller may still want: the
452/// image is reusable across runs, and the log group is where a failed build's only evidence
453/// lives.
454#[derive(Clone, Copy, Debug)]
455pub struct TeardownOpts {
456    /// Whether to delete the image the sandbox built.
457    pub delete_image: bool,
458    /// Whether the build log group should be deleted.
459    ///
460    /// This crate **cannot** delete it — CloudWatch is not in the dependency set — so
461    /// asking names the group in [`TeardownReport::undeleted`] rather than removing it.
462    /// See that field for why naming is the honest answer rather than a silent success.
463    pub delete_log_group: bool,
464    /// How many times the image delete is retried.
465    pub delete_attempts: u32,
466    /// The gap between image-delete attempts.
467    pub delete_backoff: Duration,
468    /// How long to wait for TERMINATED, or `None` to return as soon as the terminate call
469    /// is accepted.
470    ///
471    /// `None` by default, matching the Python client: the caller is on the way out, and a
472    /// teardown that blocked five minutes on a state nobody reads is five minutes of a CI
473    /// job. A caller that needs STATE-10 *observed* passes
474    /// [`TeardownOpts::waiting_for_terminated`].
475    pub wait_for_terminated: Option<Duration>,
476}
477
478impl Default for TeardownOpts {
479    fn default() -> Self {
480        Self {
481            delete_image: false,
482            delete_log_group: false,
483            delete_attempts: DEFAULT_DELETE_ATTEMPTS,
484            delete_backoff: DEFAULT_DELETE_BACKOFF,
485            wait_for_terminated: None,
486        }
487    }
488}
489
490impl TeardownOpts {
491    /// Deletes the image as well as the VM.
492    #[must_use]
493    pub fn deleting_image(mut self) -> Self {
494        self.delete_image = true;
495        self
496    }
497
498    /// Asks for the log group too, which names it rather than deleting it.
499    #[must_use]
500    pub fn deleting_log_group(mut self) -> Self {
501        self.delete_log_group = true;
502        self
503    }
504
505    /// Waits for TERMINATED before returning (STATE-10).
506    #[must_use]
507    pub fn waiting_for_terminated(mut self) -> Self {
508        self.wait_for_terminated = Some(DEFAULT_LIFECYCLE_TIMEOUT);
509        self
510    }
511}
512
513/// What a teardown did, and what it left behind.
514///
515/// Returned rather than raised. See the module docs: this runs where a `finally` would.
516#[derive(Clone, Debug, Default, Eq, PartialEq)]
517pub struct TeardownReport {
518    /// Identifiers of everything a caller asked to have deleted that still exists.
519    ///
520    /// The CLI emits these (CLI-6), which is the whole reason they are identifiers rather
521    /// than a boolean: a leak nobody can name is a leak nobody can clean up.
522    ///
523    /// Two things land here. A delete that was attempted and failed — an image whose twenty
524    /// attempts all hit a conflict. And the build **log group**, which this crate cannot
525    /// delete at all: no CloudWatch client exists anywhere in this workspace, and a logs
526    /// API for one delete call is a bigger surface than the leak it would hide (an
527    /// `aws-sdk-cloudwatchlogs` edge is a reasonable future add if teardown grows more
528    /// log-group work). Naming it is strictly better than the
529    /// alternatives: deleting it is unavailable, and silently succeeding would report a
530    /// clean teardown over six accumulated log groups — which is how the leak was found in
531    /// the first place.
532    pub undeleted: Vec<String>,
533    /// Whether the terminate call was accepted.
534    pub terminate_accepted: bool,
535    /// Whether the image was deleted, or `None` when deletion was not asked for.
536    pub image_deleted: Option<bool>,
537    /// The lifecycle state the sandbox ended in.
538    pub lifecycle: Option<Lifecycle>,
539    /// Every failure the teardown swallowed, in the order it hit them.
540    ///
541    /// Kept because a teardown that never raises is a teardown whose failures are invisible
542    /// otherwise, and the first one is usually the cause of the rest.
543    pub failures: Vec<String>,
544}
545
546impl TeardownReport {
547    /// Whether anything a caller asked for was left behind.
548    pub fn leaked(&self) -> bool {
549        !self.undeleted.is_empty()
550    }
551}
552
553/// Mints proxy tokens for one MicroVM through the control plane.
554///
555/// The bridge between the two lanes: `ControlPlane::mint_auth_token` answers a
556/// `control::ProxyToken`, [`TokenMinter`] wants a `session::ProxyToken`, and the `From`
557/// impl at `session/proxy.rs` is the conversion — one `.into()` at the boundary, which is
558/// why that impl is `From` rather than a named function.
559///
560/// The plane sits behind an `Arc` rather than a reference because the minter must outlive
561/// the call that built the session: minting happens inside the request path on every later
562/// request, which is what makes it happen at all (TRAP-9).
563///
564/// Public so a driving adapter that builds its own session mints through this rather than a
565/// copy of it.
566pub struct ControlPlaneMinter {
567    control: Arc<ControlPlane>,
568    microvm_id: String,
569}
570
571impl ControlPlaneMinter {
572    /// A minter for `microvm_id`'s tokens, through `control`.
573    pub fn new(control: Arc<ControlPlane>, microvm_id: impl Into<String>) -> Self {
574        Self {
575            control,
576            microvm_id: microvm_id.into(),
577        }
578    }
579}
580
581impl std::fmt::Debug for ControlPlaneMinter {
582    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
583        f.debug_struct("ControlPlaneMinter")
584            .field("microvm_id", &self.microvm_id)
585            .finish_non_exhaustive()
586    }
587}
588
589impl TokenMinter for ControlPlaneMinter {
590    fn mint(
591        &self,
592    ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
593        Box::pin(async move {
594            let minted = self.control.mint_auth_token(&self.microvm_id).await?;
595            Ok(minted.into())
596        })
597    }
598
599    /// Overrides the default, which ignores the ports and delegates.
600    ///
601    /// This is the minter that has a control plane behind it, so it is the one that can
602    /// actually widen a token's scope — and without this override
603    /// `Session::connect_headers(8080)` would keep answering a header pair behind a token
604    /// scoped to 9000 only, which the proxy refuses with 403 `Access to port denied`. See
605    /// [`TokenMinter::mint_for_ports`] for the measurement.
606    fn mint_for_ports(
607        &self,
608        ports: &[u16],
609    ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
610        let specs: Vec<crate::control::ops::PortSpecification> = ports
611            .iter()
612            .map(|port| crate::control::ops::PortSpecification::port(*port))
613            .collect();
614        Box::pin(async move {
615            let minted = self
616                .control
617                .mint_auth_token_for(&self.microvm_id, &specs)
618                .await?;
619            Ok(minted.into())
620        })
621    }
622}
623
624/// One MicroVM's whole life.
625///
626/// See the module docs for the state machine, the resume of an ended VM, and why teardown is
627/// explicit.
628pub struct Sandbox {
629    control: Arc<ControlPlane>,
630    image: Option<Image>,
631    /// The image ARN [`Sandbox::run`] resolved and sent in `RunMicrovm`, set before the call.
632    launch_image_arn: Option<String>,
633    launch_metrics: Option<Arc<std::sync::Mutex<crate::session::LaunchMetrics>>>,
634    microvm: Option<Microvm>,
635    session: Option<Session>,
636    /// A backend for the session [`Sandbox::run`] builds, or `None` for the real HTTP one.
637    ///
638    /// The daemon-side sibling of `with_control_plane`'s transport seam: a test that
639    /// scripts the control plane can launch, but the session `run` then builds would dial
640    /// a real endpoint. This lets the same test script the daemon too. Never set in
641    /// production — `Session::builder` picks reqwest when no backend is given.
642    session_backend: Option<crate::session::SharedBackend>,
643
644    // ── the symspec's five variables ─────────────────────────────────────────
645    lifecycle: Lifecycle,
646    /// Every transition of `lifecycle`, for a reader that must not take the caller's lock
647    /// (a keepalive task running while the same sandbox is busy inside a long exec).
648    lifecycle_watch: tokio::sync::watch::Sender<Lifecycle>,
649    token_installed: bool,
650    image_exists: bool,
651    was_terminated: bool,
652    bootstrap_count: u32,
653
654    /// The suspended window from *our own* `RunMicrovm` request, or `None` for a sandbox that
655    /// launched nothing. `GetMicrovm` reports the service's in `idlePolicy`.
656    suspended_window: Option<Duration>,
657    /// Whether the accepted launch carried a caller client token, and so may have adopted
658    /// a VM an earlier attempt launched (#195).
659    launch_adoptable: bool,
660    /// The egress posture of this sandbox's own accepted launch, or `None` for a sandbox
661    /// that launched nothing (an adopted one never saw the launch options).
662    launch_posture: Option<crate::control::EgressPosture>,
663    /// The launch's agent token, held until the session that carries it is built. Kept out
664    /// of `Debug` like every other credential here.
665    pending_agent_token: Option<String>,
666    /// `maxIdleDurationSeconds` from our own `RunMicrovm` request: a keepalive's cadence
667    /// is checked against it.
668    idle_window: Option<Duration>,
669    /// Set by [`Sandbox::terminate`], so `Drop` can tell an abandoned VM from a torn-down
670    /// one.
671    torn_down: bool,
672    /// Built by [`Sandbox::adopt`] around a VM another process launched. That process owns
673    /// the VM's teardown, so dropping this handle is not a leak and `Drop` stays quiet.
674    adopted: bool,
675    /// Set by [`Sandbox::detach`]: the VM was handed to another process, which now owns its
676    /// lifecycle and teardown. Every transition is refused and `Drop` stays quiet.
677    detached: bool,
678    /// The launch's tunnel identity, when [`RunRequest::identity`] asked for one.
679    ///
680    /// Holds the host secret and the VM's *public* pin — the VM's own secret was dropped
681    /// the moment the payload was built ([`crate::identity::LaunchIdentity::keep`]), so
682    /// nothing on the host side can impersonate the VM it launched.
683    tunnel_identity: Option<crate::identity::TunnelIdentity>,
684    /// The STS and S3 calls [`Sandbox::ensure_image`] makes, or `None` until the first
685    /// call builds the real ones. The seam a test replaces them at.
686    build_services: Option<Arc<dyn crate::control::BuildServices>>,
687    /// The caller's account, resolved once per sandbox for image ARNs (IMAGE-8).
688    account: Option<String>,
689}
690
691impl std::fmt::Debug for Sandbox {
692    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
693        // No agent token: it is a credential, and a `Debug` that printed it would put it in
694        // every log line that formats a sandbox.
695        f.debug_struct("Sandbox")
696            .field("lifecycle", &self.lifecycle)
697            .field("microvm", &self.microvm.as_ref().map(|vm| &vm.id))
698            .field("image", &self.image.as_ref().map(|image| &image.identifier))
699            .field("token_installed", &self.token_installed)
700            .field("bootstrap_count", &self.bootstrap_count)
701            .field("was_terminated", &self.was_terminated)
702            .field("suspended_window", &self.suspended_window)
703            .field("adopted", &self.adopted)
704            .field("detached", &self.detached)
705            .finish_non_exhaustive()
706    }
707}
708
709/// What another process needs to adopt a VM this sandbox handed off with
710/// [`Sandbox::detach`]: pass the fields to [`Sandbox::adopt`] (or the prelude's `Sandbox::adopt_in`).
711///
712/// The agent token is a credential: readable through [`Detached::agent_token`] so it can be
713/// persisted, and absent from `Debug`, so a log line that formats the record cannot leak it.
714/// Store it the way [`crate::names::NameRecord`] asks: privately, encrypted.
715#[derive(Clone)]
716pub struct Detached {
717    /// The VM's identifier.
718    pub microvm_id: String,
719    /// The HTTPS endpoint its daemon answers on.
720    pub endpoint: String,
721    /// The region the VM runs in.
722    pub region: Region,
723    /// The daemon port the endpoint's proxy tokens are minted for.
724    pub port: u16,
725    agent_token: String,
726}
727
728impl Detached {
729    /// The bearer the VM's daemon accepts; required by [`Sandbox::adopt`].
730    pub fn agent_token(&self) -> &str {
731        &self.agent_token
732    }
733}
734
735impl std::fmt::Debug for Detached {
736    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
737        f.debug_struct("Detached")
738            .field("microvm_id", &self.microvm_id)
739            .field("endpoint", &self.endpoint)
740            .field("region", &self.region)
741            .field("port", &self.port)
742            .field("agent_token", &"<redacted>")
743            .finish()
744    }
745}
746
747impl Sandbox {
748    /// A sandbox over an already-built [`ControlPlane`].
749    ///
750    /// The seam every test uses, and public for the same reason `ControlPlane::from_ports`
751    /// is: a caller who wants a non-default port, or a fake transport, has already built the
752    /// plane, and a sandbox constructible only from a [`Region`] would be a sandbox no test
753    /// can drive without AWS. `Sandbox::new(region)`, `Sandbox::adopt_in` and
754    /// `Sandbox::from_name`, which resolve credentials for a region, are in
755    /// `microvms_core::prelude`.
756    ///
757    /// The plane's own clock is what every wait here sleeps on, rather than a second one
758    /// this type would own. Two clocks in one lifecycle is the trap the daemon's turmoil
759    /// tests record: a deadline measured on one clock while the poll loop sleeps on another
760    /// reads the wrong elapsed time.
761    pub fn with_control_plane(control: ControlPlane) -> Self {
762        Self {
763            control: Arc::new(control),
764            image: None,
765            launch_image_arn: None,
766            launch_metrics: None,
767            microvm: None,
768            session: None,
769            session_backend: None,
770            lifecycle: Lifecycle::Pending,
771            lifecycle_watch: tokio::sync::watch::Sender::new(Lifecycle::Pending),
772            token_installed: false,
773            image_exists: false,
774            was_terminated: false,
775            bootstrap_count: 0,
776            suspended_window: None,
777            launch_adoptable: false,
778            launch_posture: None,
779            pending_agent_token: None,
780            idle_window: None,
781            torn_down: false,
782            adopted: false,
783            detached: false,
784            tunnel_identity: None,
785            build_services: None,
786            account: None,
787        }
788    }
789
790    /// Routes [`Sandbox::ensure_image`]'s STS and S3 calls through `services`.
791    ///
792    /// The seam beside `with_control_plane`'s transport: a test that scripts the control
793    /// plane replaces the two calls outside it here, and production builds
794    /// [`crate::control::SignedBuildServices`] on the first `ensure_image`.
795    pub fn with_build_services(mut self, services: Arc<dyn crate::control::BuildServices>) -> Self {
796        self.build_services = Some(services);
797        self
798    }
799
800    /// Routes the session [`Sandbox::run`] builds through `backend` instead of real HTTP.
801    ///
802    /// The daemon-side half of the test seam `with_control_plane` opens: scripting the
803    /// control plane gets a test through the launch, and this gets it through everything
804    /// the launched session then does — uploads, execs, downloads — without a daemon.
805    pub fn with_session_backend(mut self, backend: crate::session::SharedBackend) -> Self {
806        self.session_backend = Some(backend);
807        self
808    }
809
810    // ── the symspec's five variables, readable ───────────────────────────────
811
812    /// The lifecycle state, which is the symspec's `vm_state`.
813    pub fn lifecycle(&self) -> Lifecycle {
814        self.lifecycle
815    }
816
817    /// Whether the agent token has been installed (STATE-2).
818    pub fn token_installed(&self) -> bool {
819        self.token_installed
820    }
821
822    /// Whether an image is recorded as existing (STATE-1).
823    pub fn image_exists(&self) -> bool {
824        self.image_exists
825    }
826
827    /// Whether this VM was ever terminated (STATE-11).
828    pub fn was_terminated(&self) -> bool {
829        self.was_terminated
830    }
831
832    /// How many times the token has been installed. Never above one (STATE-3).
833    pub fn bootstrap_count(&self) -> u32 {
834        self.bootstrap_count
835    }
836
837    /// The image, once built.
838    pub fn image(&self) -> Option<&Image> {
839        self.image.as_ref()
840    }
841
842    /// The image ARN [`Sandbox::run`] sent in `RunMicrovm`: the caller's identifier with a
843    /// bare name resolved, or the built image's.
844    ///
845    /// Recorded before the call rather than read back from [`Sandbox::microvm`], which holds
846    /// only what an accepted launch answered: a launch the service refused, or one a caller
847    /// interrupted, still names the image it asked for here. `None` until `run` has resolved
848    /// an image, and on an adopted sandbox. Not the image [`Sandbox::terminate`] deletes, which
849    /// is only ever one this sandbox built ([`Sandbox::image`]).
850    pub fn launch_image_arn(&self) -> Option<&str> {
851        self.launch_image_arn.as_deref()
852    }
853
854    /// The VM as the service last described it.
855    pub fn microvm(&self) -> Option<&Microvm> {
856        self.microvm.as_ref()
857    }
858
859    /// The session, once launched.
860    pub fn session(&self) -> Option<&Session> {
861        self.session.as_ref()
862    }
863
864    /// The suspended window this sandbox asked for at launch, once it has launched.
865    pub fn suspended_window(&self) -> Option<Duration> {
866        self.suspended_window
867    }
868
869    /// A receiver that sees every lifecycle transition without taking this sandbox's lock.
870    ///
871    /// A binding's keepalive reads it between polls: a sandbox busy inside a long exec
872    /// holds the lock the whole time, which is exactly when the keepalive must keep polling,
873    /// and a suspend or terminate must end the keepalive before its next poll auto-resumes
874    /// the VM.
875    pub fn watch_lifecycle(&self) -> tokio::sync::watch::Receiver<Lifecycle> {
876        self.lifecycle_watch.subscribe()
877    }
878
879    fn set_lifecycle(&mut self, lifecycle: Lifecycle) {
880        self.lifecycle = lifecycle;
881        self.lifecycle_watch.send_replace(lifecycle);
882    }
883
884    /// The idle window this sandbox asked for at launch, once it has launched.
885    pub fn idle_window(&self) -> Option<Duration> {
886        self.idle_window
887    }
888
889    /// The tunnel identity, when the launch asked for one ([`RunRequest::identity`]).
890    ///
891    /// What `tunnel --verify-identity` verifies with: the host's secret and the VM's public
892    /// pin. `None` on a launch that did not ask, and the tunnel route then refuses an
893    /// identity request with close code 4401 rather than downgrading.
894    pub fn tunnel_identity(&self) -> Option<&crate::identity::TunnelIdentity> {
895        self.tunnel_identity.as_ref()
896    }
897
898    /// The egress posture of this sandbox's launch (BIND-12), which its session carries.
899    ///
900    /// The launch's own [`RunRequest::egress_posture`] once [`Sandbox::run`] was accepted.
901    /// Otherwise [`EgressPosture::default`], `unsealed`: an adopted sandbox never saw the
902    /// launch options, and the weakest true claim is the only one it can make.
903    ///
904    /// [`EgressPosture::default`]: crate::control::EgressPosture::default
905    pub fn egress_posture(&self) -> crate::control::EgressPosture {
906        self.launch_posture.unwrap_or_default()
907    }
908
909    /// The agent port the control plane was built with: the hooks port on every image
910    /// this sandbox builds, and the port the daemon listens on in every VM it launches.
911    /// Read-only; `agents::AgentVm` needs it to derive a Dockerfile whose `AGENTD_PORT`
912    /// agrees with the launch.
913    pub fn port(&self) -> u16 {
914        self.control.port()
915    }
916
917    // ── build ────────────────────────────────────────────────────────────────
918
919    /// Builds an image and waits for it to become usable.
920    ///
921    /// Every local guard runs inside [`ControlPlane::create_image`], before the call —
922    /// which matters because the create happens *after* the caller's artifact upload, so a
923    /// rejection AWS raises costs the upload first.
924    pub async fn build_image(&mut self, request: CreateImageRequest) -> Result<&Image, Error> {
925        let size = request.size;
926        let created = self.control.create_image(request).await?;
927        let mut built = self
928            .control
929            .wait_for_image(&created.identifier, size, WaitOpts::default())
930            .await?;
931        // The logging config survives the wait from the create's own record, because the
932        // readback cannot carry it: `GetMicrovmImage` reports no logging member, and the
933        // resolved log stream's discriminator was minted inside `create_image` — this is
934        // the only copy, and dropping it here would leave the caller unable to name the
935        // stream their build wrote to.
936        built.log_group = created.log_group;
937        built.log_stream = created.log_stream;
938        // Recorded before any launch, because a built image exists whether or not anything
939        // is ever run from it — and the teardown has to be able to name it either way.
940        self.image_exists = true;
941        self.image = Some(built);
942        Ok(self.image.as_ref().expect("just assigned"))
943    }
944
945    /// Builds or reuses the content-addressed image for `request` (#221): the name, the
946    /// decision after each describe, the upload and the create race are
947    /// [`crate::control::ensure`]'s.
948    pub async fn ensure_image(
949        &mut self,
950        request: crate::control::EnsureImageRequest,
951    ) -> Result<crate::control::EnsuredImage, Error> {
952        // Everything local first: a request this client refuses costs no call at all, the
953        // caller-identity lookup included.
954        //
955        // On the blocking pool, not on this task: `prepare` hashes and zips the daemon and
956        // the build context, about a second of CPU for the 2 MB daemon in a debug build.
957        // Inline, it stalls every task on this worker, and a concurrent ensure's credential
958        // fetch parked behind it past the instance-metadata provider's one-second timeout
959        // fails as a missing identity (#309).
960        let control = Arc::clone(&self.control);
961        let prepared =
962            tokio::task::spawn_blocking(move || crate::control::ensure::prepare(&control, request))
963                .await
964                .map_err(|joined| {
965                    Error::new(
966                        ErrorKind::Unexpected,
967                        format!("preparing the image's artifact did not complete: {joined}"),
968                    )
969                })??;
970        let services = self.build_services().await?;
971        let account = match &self.account {
972            Some(account) => account.clone(),
973            None => {
974                let account = services.caller_account().await?;
975                self.account = Some(account.clone());
976                account
977            }
978        };
979        let ensured =
980            crate::control::ensure::ensure(&self.control, services.as_ref(), &account, prepared)
981                .await?;
982        self.image_exists = true;
983        self.image = Some(ensured.image.clone());
984        Ok(ensured)
985    }
986
987    /// The STS and S3 client `ensure_image` and `upload_artifact_for` reach AWS through: the one
988    /// `with_build_services` set, or the plane's adapters' for its region, built once.
989    async fn build_services(&mut self) -> Result<Arc<dyn crate::control::BuildServices>, Error> {
990        if let Some(services) = &self.build_services {
991            return Ok(Arc::clone(services));
992        }
993        let built = self
994            .control
995            .adapters()
996            .build_services(self.control.region().clone())
997            .await?;
998        self.build_services = Some(Arc::clone(&built));
999        Ok(built)
1000    }
1001
1002    /// The artifact bytes to upload to the request's `code_artifact_uri`.
1003    ///
1004    /// The upload is the caller's on this path; [`Sandbox::ensure_image`] uploads for
1005    /// itself.
1006    pub fn build_artifact_for(&self, request: &CreateImageRequest) -> Result<Vec<u8>, Error> {
1007        self.control.build_artifact_for(request)
1008    }
1009
1010    /// Builds the artifact for `request` and puts it at its `code_artifact_uri`, through the
1011    /// build services `ensure_image` uploads with (#258): for a build under a name the caller
1012    /// chose, where no ensure derives the key. The URI must be `s3://<bucket>/<key>`.
1013    ///
1014    /// Run [`Self::preflight`] first: a request this client refuses then costs no upload.
1015    pub async fn upload_artifact_for(&mut self, request: &CreateImageRequest) -> Result<(), Error> {
1016        let (bucket, key) = crate::control::ensure::s3_location(&request.code_artifact_uri)?;
1017        let bytes = self.control.build_artifact_for(request)?;
1018        let services = self.build_services().await?;
1019        services.put_object(bucket, key, bytes).await
1020    }
1021
1022    /// Every local guard [`Self::build_image`] runs, callable **before** the artifact upload.
1023    ///
1024    /// A delegation to [`ControlPlane::preflight`], on this type so the CLI's build path
1025    /// reaches it through the one sandbox it already holds. Local; zero calls. The point is
1026    /// ordering: `build_image`'s guards run after the caller's upload, so a request this
1027    /// client itself refuses would still cost the S3 PUT unless the caller checks first.
1028    pub fn preflight(&self, request: &CreateImageRequest) -> Result<(), Error> {
1029        self.control.preflight(request)
1030    }
1031
1032    /// The ARN for `identifier`: an ARN passes through with zero calls, a bare name is
1033    /// resolved through the image listing by exact match.
1034    ///
1035    /// A read-only delegation to [`ControlPlane::resolve_image_arn`], the resolution
1036    /// [`Sandbox::run`] makes itself, for a caller that wants the ARN without launching. It
1037    /// touches none of the five state-machine variables: resolution is a question about the
1038    /// account, not about this VM's lifecycle.
1039    pub async fn resolve_image_arn(&self, identifier: &str) -> Result<String, Error> {
1040        self.control.resolve_image_arn(identifier).await
1041    }
1042
1043    /// The image with exactly `name`, or `None`. Read-only; see
1044    /// [`ControlPlane::find_image_by_name`] for the pagination and exact-match rules.
1045    pub async fn find_image_by_name(
1046        &self,
1047        name: &str,
1048    ) -> Result<Option<crate::control::ops::MicrovmImageSummaryWire>, Error> {
1049        self.control.find_image_by_name(name).await
1050    }
1051
1052    /// The content hash `build --reuse` keys an image name to. Local; zero calls.
1053    pub fn artifact_content_hash_for(&self, request: &CreateImageRequest) -> String {
1054        self.control.artifact_content_hash_for(request)
1055    }
1056
1057    /// The Dockerfile `request` builds with; see [`ControlPlane::dockerfile_for`]. Local;
1058    /// zero calls.
1059    pub fn dockerfile_for(&self, request: &CreateImageRequest) -> String {
1060        self.control.dockerfile_for(request)
1061    }
1062
1063    /// The versions of a managed base image, for pinning
1064    /// [`CreateImageRequest::base_image_version`].
1065    ///
1066    /// A read-only delegation to [`crate::control::ControlPlane::managed_base_versions`], on
1067    /// this type for the reason [`Self::resolve_image_arn`] gives: it touches none of the five
1068    /// state-machine variables, because a question about AWS's published bases is not a
1069    /// question about this VM's lifecycle.
1070    pub async fn managed_base_versions(
1071        &self,
1072        base_image_arn: &str,
1073    ) -> Result<Vec<crate::control::ops::ManagedMicrovmImageVersionWire>, Error> {
1074        self.control.managed_base_versions(base_image_arn).await
1075    }
1076
1077    /// One version's configuration, state, and availability status. Read-only.
1078    pub async fn image_version(
1079        &self,
1080        identifier: &str,
1081        version: &str,
1082    ) -> Result<crate::control::ops::MicrovmImageVersionSummaryWire, Error> {
1083        self.control.get_image_version(identifier, version).await
1084    }
1085
1086    // ── run (STATE-1, STATE-2, STATE-3) ──────────────────────────────────────
1087
1088    /// Launches a MicroVM and returns its session. By default it waits for RUNNING
1089    /// and for the daemon to answer; setting [`RunRequest::wait`] to false skips
1090    /// those readiness waits.
1091    ///
1092    /// # The three state requirements this is
1093    ///
1094    /// STATE-1: the accepted launch moves the lifecycle to PENDING and records the image as
1095    /// existing. STATE-2: the platform reporting RUNNING is what marks the token installed
1096    /// — not the launch call, because the run hook is what delivers it and a launch that
1097    /// dies during startup delivered nothing. STATE-3: `bootstrap_count` is incremented
1098    /// exactly here, and a second `run` on the same sandbox is refused, which is what makes
1099    /// "at most once per VM lifetime" a property of the type rather than of a caller's
1100    /// discipline.
1101    ///
1102    /// The agent token rides in `runHookPayload`, which is what keeps it out of the shared
1103    /// image snapshot. That is safe because the platform forwards no external traffic until
1104    /// the run hook returns 200, so a per-VM secret delivered at launch wins the
1105    /// first-writer race through the endpoint.
1106    ///
1107    /// # A bare image name
1108    ///
1109    /// [`RunRequest::image_identifier`] may be a name: after the local refusals it is resolved
1110    /// through [`ControlPlane::resolve_image_arn`], and [`Sandbox::launch_image_arn`] records
1111    /// the ARN sent. A name that resolves to nothing fails with `ERR_PRECONDITION` and leaves
1112    /// the sandbox as it was, so a later `run` on it can still launch.
1113    pub async fn run(&mut self, request: RunRequest) -> Result<&mut Session, Error> {
1114        self.run_inner(request, None).await
1115    }
1116
1117    /// Like `run`, but stops retries when signaled. An in-flight request is allowed to
1118    /// finish so an accepted VM is recorded; cancellation never discards its identity.
1119    pub async fn run_cancellable(
1120        &mut self,
1121        request: RunRequest,
1122        cancellation: &crate::control::transport::RetryCancellation,
1123    ) -> Result<&mut Session, Error> {
1124        self.run_inner(request, Some(cancellation)).await
1125    }
1126
1127    async fn run_inner(
1128        &mut self,
1129        request: RunRequest,
1130        cancellation: Option<&crate::control::transport::RetryCancellation>,
1131    ) -> Result<&mut Session, Error> {
1132        self.refuse_detached("run")?;
1133        // STATE-3's local half. A sandbox that has already bootstrapped cannot bootstrap
1134        // again, and the refusal is here rather than in a comment because `run` twice is
1135        // the plausible mistake — a retry loop around a launch that timed out.
1136        if self.bootstrap_count > 0 || self.microvm.is_some() {
1137            return Err(Error::invalid_arg(format!(
1138                "this sandbox has already launched a VM ({} bootstrap(s), lifecycle {}), and the \
1139                 agent token is installed at most once per VM lifetime (STATE-3). A second VM \
1140                 needs a second Sandbox — reusing this one would either re-deliver a run-hook \
1141                 payload to a daemon whose one-shot bootstrap refuses it, or silently address \
1142                 two guests through one handle.",
1143                self.bootstrap_count, self.lifecycle,
1144            )));
1145        }
1146
1147        // The network options, refused and classified by the one function a harness asks
1148        // before a launch (BIND-13), so its answer and this launch cannot disagree. Opposite
1149        // intents are refused rather than resolved by a precedence rule nobody would find:
1150        // `--egress` asks the platform for outbound network and `--deny-egress` asks the
1151        // guest's own clients to refuse it, so a launch carrying both would report `open`
1152        // while its workload's tools failed closed.
1153        let posture = crate::control::egress_posture_for(
1154            request.egress,
1155            &request.egress_network_connectors,
1156            request.deny_egress,
1157            Some(self.control.region()),
1158        )?;
1159
1160        let Some(identifier) = request
1161            .image_identifier
1162            .clone()
1163            .or_else(|| self.image.as_ref().map(|image| image.identifier.clone()))
1164        else {
1165            return Err(Error::new(
1166                ErrorKind::Precondition,
1167                "no image to launch: pass RunRequest::with_image or call build_image first."
1168                    .to_string(),
1169            ));
1170        };
1171
1172        if request.client_token.is_some() && (request.agent_token.is_none() || request.identity) {
1173            return Err(Error::invalid_arg(
1174                "a stable client_token requires an explicit agent_token and identity=false; persist identical launch parameters before retrying",
1175            ));
1176        }
1177        let agent_token = match request.agent_token.clone() {
1178            Some(token) => token,
1179            None => mint_agent_token(self.control.entropy())?,
1180        };
1181        // Generated before the payload because the payload carries its two public-facing
1182        // fields. The VM's secret half lives exactly as long as this binding: `keep()` below
1183        // drops it before the launch call returns.
1184        let launch_identity = if request.identity {
1185            Some(crate::entropy::launch_identity(self.control.entropy())?)
1186        } else {
1187            None
1188        };
1189        // Checked even though this builds the JSON itself, because neither half is ours:
1190        // the token may be caller-supplied, so someone passing a signed blob rather than a
1191        // bearer token is exactly who this catches (TRAP-5), and the launch env is entirely
1192        // the caller's. This is the pre-flight refusal — over-ceiling fails here with the
1193        // byte count, before the launch, rather than as a `ValidationException` on a member
1194        // the caller did not know they were filling.
1195        let payload = RunHookPayload::for_launch_with_identity(
1196            &agent_token,
1197            &request.effective_launch_env(),
1198            launch_identity.as_ref(),
1199        )?;
1200
1201        let mut wire = RunMicrovmRequest::new(identifier, payload);
1202        wire.image_version = request.image_version.clone();
1203        wire.execution_role_arn = request.execution_role_arn.clone();
1204        wire.egress_network_connectors = request.egress_network_connectors.clone();
1205        wire.max_idle_sec = request.max_idle_sec;
1206        wire.suspended_sec = request.suspended_sec;
1207        wire.auto_resume = request.auto_resume;
1208        wire.max_duration_sec = request.max_duration_sec;
1209        wire.token_scope = request.token_scope.clone();
1210        wire.client_token = request.client_token.clone();
1211        wire.logging = request.logging.clone();
1212        if request.egress {
1213            wire = wire.with_egress();
1214        }
1215        if request.shell {
1216            wire = wire.with_shell();
1217        }
1218
1219        // `imageIdentifier` takes an ARN, and the service answers a bare name with HTTP 400
1220        // "Malformed ARN", so a name is resolved here, where every surface's launch passes
1221        // (the CLI's `run --image`, both bindings' `Sandbox.run` and `AgentVm.launch`). The
1222        // launch's local refusals go first: the listing a name costs is a call, and a launch
1223        // `run_microvm` would refuse anyway costs none. An ARN passes through with no call.
1224        self.control.launch_connectors(&wire)?;
1225        let resolution = self.control.resolve_image_arn(&wire.image_identifier);
1226        wire.image_identifier = if let Some(cancellation) = cancellation {
1227            // This lookup only reads images, so cancelling it cannot abandon a VM.
1228            // The launch below remains shielded while its request is in flight.
1229            tokio::select! {
1230                biased;
1231                _ = cancellation.cancelled() => {
1232                    return Err(Error::new(
1233                        ErrorKind::Interrupted,
1234                        "launch cancelled before a VM was assigned",
1235                    ));
1236                }
1237                resolved = resolution => resolved?,
1238            }
1239        } else {
1240            resolution.await?
1241        };
1242        self.launch_image_arn = Some(wire.image_identifier.clone());
1243
1244        let (launched, details) = self
1245            .control
1246            .run_microvm_reported(wire, cancellation)
1247            .await?;
1248        self.launch_metrics = Some(Arc::new(std::sync::Mutex::new(
1249            crate::session::LaunchMetrics {
1250                attempt_count: details.attempt_count,
1251                backoff_seconds: details.backoff.as_secs_f64(),
1252                ..Default::default()
1253            },
1254        )));
1255
1256        // The durable half of the identity, kept the moment the launch is accepted: the host
1257        // secret and the VM's public pin. The VM's own secret ends here — `keep` drops it —
1258        // so from this line on the host can verify its VM and impersonate it never.
1259        self.tunnel_identity = launch_identity.map(crate::identity::LaunchIdentity::keep);
1260
1261        // STATE-1: the launch was accepted.
1262        self.set_lifecycle(Lifecycle::Pending);
1263        self.image_exists = true;
1264        // Recorded here rather than after the wait, because the window the idlePolicy
1265        // enforces was set by *this* request and a launch that then fails still leaves a VM
1266        // the caller may have to reason about.
1267        self.suspended_window = Some(Duration::from_secs(u64::from(request.suspended_sec)));
1268        self.idle_window = Some(Duration::from_secs(u64::from(request.max_idle_sec)));
1269        self.microvm = Some(launched);
1270        // A caller-supplied client token can adopt the VM an earlier attempt launched, which
1271        // may have idle-suspended since (#195); a minted token always launches afresh.
1272        self.launch_adoptable = request.client_token.is_some();
1273        self.launch_posture = Some(posture);
1274        self.pending_agent_token = Some(agent_token);
1275
1276        if request.wait {
1277            return self.wait_until_running(request.ready_timeout).await;
1278        }
1279        // Not waiting: the endpoint is in the launch reply, so the session is addressable
1280        // now, and a daemon request before RUNNING fails like any other unready request.
1281        self.build_session()?;
1282        Ok(self.session.as_mut().expect("just assigned"))
1283    }
1284
1285    /// The session for the launched VM, from its endpoint and the launch's agent token.
1286    fn build_session(&mut self) -> Result<(), Error> {
1287        let (Some(vm), Some(agent_token)) = (&self.microvm, &self.pending_agent_token) else {
1288            return Err(Error::new(
1289                ErrorKind::Precondition,
1290                "no launch to build a session for",
1291            ));
1292        };
1293        let minter = Arc::new(ControlPlaneMinter::new(
1294            Arc::clone(&self.control),
1295            vm.id.clone(),
1296        ));
1297        // The plane's clock, so the session's exec ids and token refreshes read the same time
1298        // source as the lifecycle that launched it.
1299        let mut builder = Session::builder(vm.endpoint.clone(), agent_token.clone())
1300            .with_minter(minter)
1301            .with_port(self.control.port())
1302            .with_egress_posture(self.egress_posture())
1303            .with_clock(self.control.shared_clock());
1304        if let Some(backend) = &self.session_backend {
1305            builder = builder.with_backend(Arc::clone(backend));
1306        }
1307        let mut session = builder.build_with(self.control.adapters())?;
1308        session.launch_metrics = self.launch_metrics.clone();
1309        self.session = Some(session);
1310        Ok(())
1311    }
1312
1313    /// Waits for a launch accepted by [`Sandbox::run`] to reach RUNNING, up to `timeout`, and
1314    /// then for its daemon to answer, up to [`crate::session::DEFAULT_BOOTSTRAP_TIMEOUT`].
1315    ///
1316    /// `run` calls this itself unless [`RunRequest::wait`] was off. A launch that adopted an
1317    /// existing VM through its client token resumes it if it idle-suspended (#195); a fresh
1318    /// launch that reaches any terminal state first fails fast with `stateReason` (TRAP-8).
1319    ///
1320    /// # Why RUNNING isn't enough
1321    ///
1322    /// RUNNING says the run hook answered, so the daemon is up and the token installed
1323    /// (STATE-2), but the endpoint's proxy path commonly refuses a connection or two just
1324    /// after. The session is handed back once one health poll through it has answered, so a
1325    /// first exec after a launch doesn't meet that window, on any surface (#254). A daemon that
1326    /// never answers fails this with `ERR_TIMEOUT`, and the lifecycle stays RUNNING: the VM is
1327    /// there and billing, and [`Sandbox::session`] can still wait for it.
1328    pub async fn wait_until_running(&mut self, timeout: Duration) -> Result<&mut Session, Error> {
1329        self.refuse_detached("wait until running")?;
1330        let id = self.require_microvm("wait_until_running")?;
1331        if self.lifecycle != Lifecycle::Pending {
1332            return Err(Error::invalid_arg(format!(
1333                "microvm {id} is {}; wait_until_running finishes a launch that is still PENDING",
1334                self.lifecycle,
1335            )));
1336        }
1337        let opts = WaitOpts::for_lifecycle(timeout);
1338        let running = self
1339            .control
1340            .wait_for_sandbox_launch(&id, opts, self.launch_adoptable)
1341            .await?;
1342
1343        // STATE-2. The platform reported the run hook succeeded, so the token is in the
1344        // guest's memory — and this is the one place that counts it (STATE-3).
1345        self.set_lifecycle(Lifecycle::Running);
1346        if let Some(metrics) = &self.launch_metrics {
1347            metrics
1348                .lock()
1349                .unwrap_or_else(std::sync::PoisonError::into_inner)
1350                .running_at
1351                .get_or_insert_with(|| self.control.clock().unix_now().as_secs_f64());
1352        }
1353        self.token_installed = true;
1354        self.bootstrap_count += 1;
1355        self.microvm = Some(running);
1356        if self.session.is_none() {
1357            self.build_session()?;
1358        }
1359        let session = self.session.as_mut().expect("just assigned");
1360        session
1361            .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1362            .await?;
1363        Ok(session)
1364    }
1365
1366    /// Waits until the VM this sandbox addresses answers, whatever a launch still has to wait
1367    /// for: from PENDING, [`Sandbox::wait_until_running`] with `running_timeout` (RUNNING, then
1368    /// the daemon); from RUNNING, the daemon alone, up to
1369    /// [`crate::session::DEFAULT_BOOTSTRAP_TIMEOUT`].
1370    ///
1371    /// The shape for a process that adopted a VM another one launched without waiting: it can't
1372    /// know which of the two waits is left, and the lifecycle [`Sandbox::adopt`] read from the
1373    /// service says. Any other lifecycle is refused, because a suspended or terminated VM isn't
1374    /// on its way to answering.
1375    pub async fn wait_until_ready(
1376        &mut self,
1377        running_timeout: Duration,
1378    ) -> Result<&mut Session, Error> {
1379        if self.lifecycle == Lifecycle::Running {
1380            self.refuse_detached("wait until ready")?;
1381            let session = self.session.as_mut().ok_or_else(|| {
1382                Error::new(ErrorKind::Precondition, "a RUNNING sandbox with no session")
1383            })?;
1384            session
1385                .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1386                .await?;
1387            return Ok(session);
1388        }
1389        self.wait_until_running(running_timeout).await
1390    }
1391
1392    // ── adopt (STATE-3, with the lifecycle read from the service) ────────────
1393
1394    /// A sandbox for a VM another process launched, rebuilt from its private record.
1395    ///
1396    /// The durable-workflow shape: every step runs in a fresh process, and each one needs
1397    /// suspend, resume, and terminate, not just the exec and file surface `Session::attach`
1398    /// (`microvms_core::prelude::SessionExt::attach`) gives. The lifecycle is read from `GetMicrovm`
1399    /// rather than assumed, so every guard below starts from what the service reports.
1400    ///
1401    /// # Bootstrap is counted, never repeated (STATE-3)
1402    ///
1403    /// The VM was bootstrapped by the launch that created it, so an adopted sandbox refuses
1404    /// [`Sandbox::run`] and never sends a run-hook payload. A VM adopted while still PENDING
1405    /// finishes through [`Sandbox::wait_until_running`], which counts the bootstrap once
1406    /// when the service reports RUNNING, the same as a launch this sandbox made itself.
1407    ///
1408    /// `endpoint` must agree with the service's when the service reports one: a mismatch
1409    /// means the identifiers came from two different records. `agent_token` is the bearer
1410    /// credential the VM was launched with; it never appears in `Debug` or an error.
1411    pub async fn adopt(
1412        control: ControlPlane,
1413        microvm_id: impl Into<String>,
1414        endpoint: impl Into<String>,
1415        agent_token: impl Into<String>,
1416    ) -> Result<Self, Error> {
1417        let microvm_id = microvm_id.into();
1418        let endpoint = endpoint.into();
1419        let agent_token = agent_token.into();
1420        if agent_token.is_empty() {
1421            return Err(Error::invalid_arg(
1422                "adopt needs the agent token the VM was launched with: its daemon accepts only \
1423                 that bearer, and adopting without it would build a handle every exec refuses.",
1424            ));
1425        }
1426        let mut vm = control.get_microvm(&microvm_id).await?;
1427        if !endpoint.is_empty() && !vm.endpoint.is_empty() && vm.endpoint != endpoint {
1428            return Err(Error::invalid_arg(format!(
1429                "microvm {microvm_id} reports endpoint {}, not {endpoint}: the id and endpoint \
1430                 came from different records, and a session built from them would address one \
1431                 VM while its lifecycle calls reached another.",
1432                vm.endpoint,
1433            )));
1434        }
1435        if vm.endpoint.is_empty() {
1436            vm.endpoint = endpoint;
1437        }
1438        let Some(lifecycle) = Lifecycle::from_service(&vm.state) else {
1439            return Err(Error::new(
1440                ErrorKind::Platform,
1441                format!(
1442                    "microvm {microvm_id} reports state {}, which this client does not know; \
1443                     refusing to guess which lifecycle guards apply.",
1444                    vm.state,
1445                ),
1446            ));
1447        };
1448
1449        let mut sandbox = Self::with_control_plane(control);
1450        sandbox.adopted = true;
1451        sandbox.set_lifecycle(lifecycle);
1452        // The window the VM was launched with, so a keepalive on the adopted session paces
1453        // itself against the real idle policy rather than the platform minimum.
1454        sandbox.idle_window = vm
1455            .idle_policy
1456            .as_ref()
1457            .map(|policy| Duration::from_secs(u64::from(policy.max_idle_duration_seconds)));
1458        // A VM exists, so the image it was launched from did (STATE-1).
1459        sandbox.image_exists = true;
1460        // PENDING may still idle-suspend before this handle sees RUNNING (#195).
1461        sandbox.launch_adoptable = true;
1462        sandbox.pending_agent_token = Some(agent_token);
1463        match lifecycle {
1464            Lifecycle::Pending => {}
1465            Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended => {
1466                sandbox.token_installed = true;
1467                sandbox.bootstrap_count = 1;
1468            }
1469            Lifecycle::Terminating | Lifecycle::Terminated => {
1470                sandbox.bootstrap_count = 1;
1471                sandbox.was_terminated = true;
1472            }
1473        }
1474        sandbox.microvm = Some(vm);
1475        if lifecycle.is_live() {
1476            sandbox.build_session()?;
1477        }
1478        Ok(sandbox)
1479    }
1480
1481    /// A [`crate::names::NameRecord`] for this VM, to register under `name`.
1482    ///
1483    /// The egress posture is left unknown: the sandbox does not keep its launch request, and
1484    /// a record must not claim a network it cannot vouch for. The tunnel identity is kept when
1485    /// the sandbox holds one, launched with it or adopted from a record that carried it: a
1486    /// record without it can't verify the VM later, which is what the pair is for (#263).
1487    pub fn name_record(&self, name: &str) -> Result<crate::names::NameRecord, Error> {
1488        let Some(vm) = self.microvm() else {
1489            return Err(Error::new(
1490                ErrorKind::Precondition,
1491                "there is no VM to name yet: launch or adopt one first",
1492            ));
1493        };
1494        let Some(session) = self.session() else {
1495            return Err(Error::new(
1496                ErrorKind::Precondition,
1497                format!(
1498                    "microvm {} has no session to take its agent token from",
1499                    vm.id
1500                ),
1501            ));
1502        };
1503        // Stamped on the plane's clock, the one every other time this sandbox reads comes from.
1504        let mut record = crate::names::NameRecord::new_at(
1505            name,
1506            vm.id.as_str(),
1507            vm.endpoint.as_str(),
1508            session.agent_token(),
1509            self.control.region().as_str(),
1510            self.control.clock().unix_now().as_secs(),
1511        )?;
1512        record.set_tunnel_identity(self.tunnel_identity.as_ref());
1513        Ok(record)
1514    }
1515
1516    /// [`Sandbox::adopt`] from a [`crate::names::NameRecord`] kept in any store.
1517    ///
1518    /// Keeps the record's tunnel identity, so [`Sandbox::tunnel_identity`] and a later
1519    /// [`Sandbox::name_record`] carry it. A record holding one half of the pair, or a pair that
1520    /// doesn't decode, is refused before any AWS call rather than adopted without it: the
1521    /// record claims a verifiable VM, and dropping the claim would hide that it's broken.
1522    pub async fn adopt_record(
1523        control: ControlPlane,
1524        record: crate::names::NameRecord,
1525    ) -> Result<Self, Error> {
1526        let identity = record.tunnel_identity()?;
1527        let mut sandbox = Self::adopt(
1528            control,
1529            record.microvm_id,
1530            record.endpoint,
1531            record.agent_token,
1532        )
1533        .await?;
1534        sandbox.tunnel_identity = identity;
1535        Ok(sandbox)
1536    }
1537
1538    /// Hands the VM off to another process and returns what that process needs to adopt it.
1539    ///
1540    /// For a workflow whose steps run in different processes (a durable function's launch
1541    /// step, then later steps that [`Sandbox::adopt`]): the launching process calls this
1542    /// instead of dropping the sandbox, which would warn that a live VM was abandoned. The
1543    /// VM keeps running and nothing is sent to AWS. From here on this sandbox is inert — its
1544    /// session is dropped and `run`, `wait_until_running`, `suspend`, `resume`, and
1545    /// `terminate` are refused — because the adopter now owns the lifecycle, and a second
1546    /// handle driving it would bypass the adopter's guards.
1547    ///
1548    /// Refused (`Precondition`) when there is no live VM to hand off: nothing launched yet,
1549    /// already torn down or terminated, or already detached.
1550    pub fn detach(&mut self) -> Result<Detached, Error> {
1551        self.refuse_detached("detach")?;
1552        let Some(vm) = self.microvm.as_ref() else {
1553            return Err(Error::new(
1554                ErrorKind::Precondition,
1555                "nothing to detach: this sandbox has not launched a VM.",
1556            ));
1557        };
1558        if self.torn_down || !self.lifecycle.is_live() {
1559            return Err(Error::new(
1560                ErrorKind::Precondition,
1561                format!(
1562                    "microvm {} is {} and there is no live VM to hand off; only a PENDING, \
1563                     RUNNING, or SUSPENDED VM can be adopted.",
1564                    vm.id, self.lifecycle,
1565                ),
1566            ));
1567        }
1568        let agent_token = self
1569            .session
1570            .as_ref()
1571            .map(|session| session.agent_token().to_string())
1572            .or_else(|| self.pending_agent_token.clone())
1573            .ok_or_else(|| {
1574                Error::new(
1575                    ErrorKind::Precondition,
1576                    format!(
1577                        "microvm {} has no agent token to hand off; an adopter could not \
1578                         reach its daemon.",
1579                        vm.id
1580                    ),
1581                )
1582            })?;
1583        let detached = Detached {
1584            microvm_id: vm.id.clone(),
1585            endpoint: vm.endpoint.clone(),
1586            region: self.control.region().clone(),
1587            port: self.control.port(),
1588            agent_token,
1589        };
1590        self.detached = true;
1591        self.session = None;
1592        self.pending_agent_token = None;
1593        Ok(detached)
1594    }
1595
1596    /// Whether [`Sandbox::detach`] handed this sandbox's VM to another process.
1597    pub fn detached(&self) -> bool {
1598        self.detached
1599    }
1600
1601    /// Refuses `what` on a sandbox whose VM was handed off.
1602    fn refuse_detached(&self, what: &str) -> Result<(), Error> {
1603        if !self.detached {
1604            return Ok(());
1605        }
1606        let id = self.microvm.as_ref().map_or("?", |vm| vm.id.as_str());
1607        Err(Error::new(
1608            ErrorKind::Precondition,
1609            format!(
1610                "cannot {what}: microvm {id} was detached and belongs to whoever adopts it. \
1611                 Adopt it with `Sandbox::adopt` and {what} through that handle."
1612            ),
1613        ))
1614    }
1615
1616    /// Whether this sandbox was built by [`Sandbox::adopt`] rather than by its own launch.
1617    pub fn adopted(&self) -> bool {
1618        self.adopted
1619    }
1620
1621    // ── suspend (STATE-4, STATE-5, STATE-6) ──────────────────────────────────
1622
1623    /// Freezes the VM and waits for the platform to report it.
1624    ///
1625    /// A freeze and restore rather than a stop and start: the guest keeps its memory, so the
1626    /// token, the filesystem, and every exec record survive. The one thing that does not is
1627    /// the guest's view of time — it observes the whole suspension as a single jump, so any
1628    /// timeout, lease, or TLS session a running command holds expires at once on resume.
1629    ///
1630    /// # STATE-5 is checked before the wire, not after
1631    ///
1632    /// A suspend from anything but RUNNING is refused here with **zero** control-plane
1633    /// calls. That is the observable difference between this and a client that lets AWS
1634    /// answer, and it is what the test asserts on.
1635    pub async fn suspend(&mut self) -> Result<(), Error> {
1636        self.refuse_detached("suspend")?;
1637        let id = self.require_microvm("suspend")?;
1638
1639        // STATE-5.
1640        if self.lifecycle != Lifecycle::Running {
1641            return Err(Error::invalid_arg(format!(
1642                "microvm {id} is {} and a suspend is only issued from RUNNING (STATE-5). Refused \
1643                 here rather than by the service, because the service's answer about a \
1644                 non-running id does not say which of the two things went wrong — and a suspend \
1645                 issued from SUSPENDED is a caller who believes they resumed.",
1646                self.lifecycle,
1647            )));
1648        }
1649
1650        // STATE-4: accepted, so the lifecycle moves before the wait. Acceptance is the
1651        // wire call succeeding, so the assignment comes after it: moving to SUSPENDING
1652        // first would leave a failed call (a throttle, a dead transport) stuck in a state
1653        // neither suspend nor resume accepts, bricking the handle over one bad request.
1654        self.control.suspend(&id).await?;
1655        self.set_lifecycle(Lifecycle::Suspending);
1656
1657        let settled = self
1658            .control
1659            .wait_for_state(
1660                &id,
1661                &crate::control::microvm::SUSPEND_WANTED,
1662                &[],
1663                self.lifecycle_wait(),
1664            )
1665            .await?;
1666
1667        // STATE-6, and the TERMINATED case beside it. A VM that dies while suspending is a
1668        // state to report rather than an exception out of the middle of a teardown, so the
1669        // wait *wants* TERMINATED — and recording it here is what stops a resume from being
1670        // offered afterwards (STATE-11).
1671        let settled_lifecycle = match settled.state.as_str() {
1672            "SUSPENDED" => Lifecycle::Suspended,
1673            "TERMINATED" => {
1674                self.was_terminated = true;
1675                Lifecycle::Terminated
1676            }
1677            other => {
1678                return Err(Error::new(
1679                    ErrorKind::Platform,
1680                    format!(
1681                        "the suspend wait returned {other}, which is neither SUSPENDED nor \
1682                         TERMINATED — and those two are what this client asked for."
1683                    ),
1684                ));
1685            }
1686        };
1687        self.set_lifecycle(settled_lifecycle);
1688        self.microvm = Some(settled);
1689        Ok(())
1690    }
1691
1692    // ── resume (STATE-7, STATE-8, STATE-12) ──────────────────────────────────
1693
1694    /// Thaws the VM and returns a usable session.
1695    ///
1696    /// # What is deliberately not re-delivered (STATE-7)
1697    ///
1698    /// Nothing. No run-hook payload, no token, no bootstrap. The in-memory token survived
1699    /// the freeze, and re-delivering it would hit the daemon's one-shot bootstrap and be
1700    /// refused — a 409 that reads like a broken VM.
1701    ///
1702    /// # The service decides, and a VM it ended is `WindowClosed` (STATE-12)
1703    ///
1704    /// No resume is refused here for the time the VM has spent suspended. The service ends a
1705    /// VM the idle policy suspended once `suspendedDurationSeconds` passes, and it kept a VM
1706    /// this client suspended resumable 1283 s into a 60 s window (measured 2026-10-12,
1707    /// us-east-1, #597). A resume the service refuses because the VM ended, whatever ended
1708    /// it, is [`ErrorKind::WindowClosed`] ([`ControlPlane::resume`]), and this records the VM
1709    /// as terminated, so a second resume is refused here with no call (STATE-11). A VM the
1710    /// idle policy suspended may still be RUNNING here, so resume first reads its service
1711    /// state. It resumes from SUSPENDED and reports `WindowClosed` for an ended VM.
1712    ///
1713    /// # The proxy token is dropped (STATE-8)
1714    ///
1715    /// [`Session::rebind`] invalidates it. The endpoint URL does not change across
1716    /// suspend/resume, so the rebind is usually a no-op on the URL — but a token minted
1717    /// against the pre-suspend instance may no longer validate, and that rejection reads
1718    /// exactly like a dead daemon.
1719    pub async fn resume(&mut self) -> Result<&mut Session, Error> {
1720        self.refuse_detached("resume")?;
1721        let id = self.require_microvm("resume")?;
1722
1723        // STATE-11's local half: a terminated VM never returns to RUNNING, so the refusal
1724        // comes before any call.
1725        if self.was_terminated || self.lifecycle == Lifecycle::Terminated {
1726            return Err(Error::invalid_arg(format!(
1727                "microvm {id} was terminated, and a terminated VM never returns to RUNNING \
1728                 (STATE-11). There is nothing to resume: the guest's memory is gone, so even a \
1729                 call the service accepted would hand back a different machine."
1730            )));
1731        }
1732        let mut resume_state = self.lifecycle;
1733        // Idle suspension happens without a client call. Refresh a RUNNING handle before
1734        // applying STATE-7, but keep STATE-11 above entirely local once termination is known.
1735        if self.lifecycle == Lifecycle::Running {
1736            let observed = self.control.get_microvm(&id).await?;
1737            resume_state = Lifecycle::from_service(&observed.state).ok_or_else(|| {
1738                Error::new(
1739                    ErrorKind::Platform,
1740                    format!("microvm {id} reports unknown state {}", observed.state),
1741                )
1742            })?;
1743            let ended = crate::constants::DEAD_STATES.contains(&observed.state.as_str());
1744            let error = ended.then(|| {
1745                Error::new(
1746                    ErrorKind::WindowClosed,
1747                    format!(
1748                        "microvm {id} can't be resumed: GetMicrovm reports the VM {} ({}). \
1749                         A VM that has ended doesn't come back; launch a new one.",
1750                        observed.state,
1751                        observed.reason(),
1752                    ),
1753                )
1754            });
1755            self.microvm = Some(observed);
1756            // A transient state must not strand the handle: only RUNNING handles refresh
1757            // here, so leave it eligible to observe the service again on the next call.
1758            if resume_state == Lifecycle::Suspended {
1759                self.set_lifecycle(resume_state);
1760            }
1761            if let Some(error) = error {
1762                self.was_terminated = true;
1763                self.set_lifecycle(Lifecycle::Terminated);
1764                return Err(error);
1765            }
1766        }
1767        if resume_state != Lifecycle::Suspended {
1768            return Err(Error::invalid_arg(format!(
1769                "microvm {id} is {} and a resume is only issued from SUSPENDED (STATE-7).",
1770                resume_state,
1771            )));
1772        }
1773
1774        // STATE-12: a `WindowClosed` refusal is a VM that ended before this resume. It's
1775        // recorded as TERMINATED whichever of TERMINATING or TERMINATED `GetMicrovm` read,
1776        // because the service has said the VM's state can't change, and as a termination, so
1777        // the next resume is refused before any call (STATE-11). Any other refusal leaves the
1778        // sandbox SUSPENDED, and a retry goes to the service again.
1779        if let Err(refused) = self.control.resume(&id).await {
1780            if refused.kind() == ErrorKind::WindowClosed {
1781                self.was_terminated = true;
1782                self.set_lifecycle(Lifecycle::Terminated);
1783            }
1784            return Err(refused);
1785        }
1786        // `fail_on` is the *dead* states rather than the terminal ones: SUSPENDED is the
1787        // state this call was made from, so failing on it would fail every resume. A VM that
1788        // terminates once the resume is accepted never reaches RUNNING, and waiting only for
1789        // RUNNING there burns the full timeout and then reports a timeout message hiding a
1790        // cause the service had already stated in `stateReason`.
1791        let running = self
1792            .control
1793            .wait_for_state(
1794                &id,
1795                &["RUNNING"],
1796                &crate::constants::DEAD_STATES,
1797                self.lifecycle_wait(),
1798            )
1799            .await?;
1800
1801        self.set_lifecycle(Lifecycle::Running);
1802        // STATE-8, through the endpoint the service just reported rather than the one held:
1803        // the URL is measured not to change, and reading it from the response is what makes
1804        // that a fact this code depends on rather than an assumption it encodes.
1805        let endpoint = running.endpoint.clone();
1806        self.microvm = Some(running);
1807        if let Some(session) = self.session.as_mut() {
1808            session.rebind(endpoint);
1809        }
1810
1811        self.session.as_mut().ok_or_else(|| {
1812            Error::new(
1813                ErrorKind::Unexpected,
1814                "the VM resumed but this sandbox holds no session, which run() always builds"
1815                    .to_string(),
1816            )
1817        })
1818    }
1819
1820    // ── terminate (STATE-9, STATE-10) ────────────────────────────────────────
1821
1822    /// Tears down, best-effort, never erroring.
1823    ///
1824    /// Order: VM, then image, then the log group **last**. The log group is last because the
1825    /// service can recreate a group deleted before its image — and see
1826    /// [`TeardownReport::undeleted`] for why this crate names it rather than deleting it.
1827    pub async fn terminate(&mut self, opts: TeardownOpts) -> TeardownReport {
1828        let mut report = TeardownReport::default();
1829        if let Err(error) = self.refuse_detached("terminate") {
1830            // Never erroring is `terminate`'s contract, so the refusal is reported rather
1831            // than raised; the VM is untouched and still belongs to its adopter.
1832            report.failures.push(error.to_string());
1833            report.lifecycle = Some(self.lifecycle);
1834            return report;
1835        }
1836        self.torn_down = true;
1837
1838        // The session first: it holds a cached proxy token whose only remaining use would be
1839        // a request against a VM that is going away.
1840        self.session = None;
1841
1842        // 1. The VM.
1843        if let Some(id) = self.microvm.as_ref().map(|vm| vm.id.clone()) {
1844            // STATE-9. Recorded before the call, so a terminate whose call fails still marks
1845            // the VM as one this client asked to destroy — which is what stops a later
1846            // resume (STATE-11) rather than leaving the sandbox looking resumable.
1847            self.set_lifecycle(Lifecycle::Terminating);
1848            self.was_terminated = true;
1849
1850            match self.control.terminate(&id).await {
1851                Ok(()) => report.terminate_accepted = true,
1852                Err(error) => {
1853                    report.failures.push(format!("terminate {id}: {error}"));
1854                    report.undeleted.push(id.clone());
1855                }
1856            }
1857
1858            if report.terminate_accepted
1859                && let Some(timeout) = opts.wait_for_terminated
1860            {
1861                let wait = WaitOpts::for_lifecycle(timeout);
1862                match self
1863                    .control
1864                    .wait_for_state(&id, &["TERMINATED"], &[], wait)
1865                    .await
1866                {
1867                    // STATE-10.
1868                    Ok(settled) => {
1869                        self.set_lifecycle(Lifecycle::Terminated);
1870                        self.microvm = Some(settled);
1871                    }
1872                    // Not a leak: the platform accepted the terminate, so the VM is on its
1873                    // way out and the lifecycle stays TERMINATING honestly.
1874                    Err(error) => report
1875                        .failures
1876                        .push(format!("waiting for {id} to reach TERMINATED: {error}")),
1877                }
1878            }
1879        }
1880
1881        // 2. The image, retrying — an image in CREATING refuses deletion and a VM still
1882        //    terminating holds a reference.
1883        if opts.delete_image {
1884            match self.image.as_ref().map(|image| image.identifier.clone()) {
1885                Some(identifier) => {
1886                    let deleted = self
1887                        .control
1888                        .delete_image(&identifier, opts.delete_attempts, opts.delete_backoff)
1889                        .await;
1890                    report.image_deleted = Some(deleted);
1891                    if deleted {
1892                        self.image_exists = false;
1893                    } else {
1894                        report.failures.push(format!(
1895                            "the image {identifier} survived {} delete attempts",
1896                            opts.delete_attempts.max(1)
1897                        ));
1898                        report.undeleted.push(identifier);
1899                    }
1900                }
1901                None => report.image_deleted = Some(false),
1902            }
1903        }
1904
1905        // 3. The log group, LAST. Named rather than deleted; see TeardownReport::undeleted.
1906        if opts.delete_log_group
1907            && let Some(group) = self.image.as_ref().map(Image::build_log_group)
1908        {
1909            report.failures.push(format!(
1910                "the build log group {group} was not deleted: CloudWatch Logs is not in this \
1911                 crate's dependency set, so it is reported rather than removed. It is \
1912                 service-created, which means no Terraform stack owns it and `terraform destroy` \
1913                 leaves it behind — six accumulated before anyone noticed."
1914            ));
1915            report.undeleted.push(group);
1916        }
1917
1918        report.lifecycle = Some(self.lifecycle);
1919        report
1920    }
1921
1922    // ── internals ────────────────────────────────────────────────────────────
1923
1924    /// The VM id, or a precondition error naming what was attempted.
1925    fn require_microvm(&self, what: &str) -> Result<String, Error> {
1926        self.microvm
1927            .as_ref()
1928            .map(|vm| vm.id.clone())
1929            .ok_or_else(|| {
1930                Error::new(
1931                    ErrorKind::Precondition,
1932                    format!("nothing to {what}: this sandbox has not launched a VM."),
1933                )
1934            })
1935    }
1936
1937    /// Five minutes at five-second polls, for suspend, resume, and terminate.
1938    fn lifecycle_wait(&self) -> WaitOpts {
1939        WaitOpts::for_lifecycle(DEFAULT_LIFECYCLE_TIMEOUT)
1940    }
1941}
1942
1943/// Warns about a live VM rather than tearing it down.
1944///
1945/// See the module docs: `Drop` cannot await, so a teardown here would either deadlock
1946/// inside a runtime or race the process exit. The warning names the id, because the only
1947/// useful thing a drop can do is tell whoever reads stderr what to go delete.
1948///
1949/// The warning goes through the plane's [`Adapters::warn`](crate::adapters::Adapters::warn),
1950/// which in production writes it to stderr.
1951impl Sandbox {
1952    /// The warning `Drop` writes for a live VM nobody took responsibility for, or `None`
1953    /// when dropping is not a leak: torn down, adopted (its launcher owns it), or detached
1954    /// (its adopter owns it).
1955    fn drop_warning(&self) -> Option<String> {
1956        if self.torn_down || self.adopted || self.detached {
1957            return None;
1958        }
1959        let vm = self.microvm.as_ref()?;
1960        self.lifecycle.is_live().then(|| {
1961            format!(
1962                "warning: the Sandbox for microvm {id} was dropped in {state} without \
1963                 terminate() or detach(). Nothing was torn down — Drop cannot await, so a \
1964                 teardown here would deadlock inside a runtime. {state} is the state this \
1965                 sandbox last saw, and the platform may have suspended or ended the VM since. A \
1966                 VM that's still live bills until it ends, at its maximumDurationInSeconds \
1967                 ceiling at the latest: terminate it with `microvm terminate {id}`.",
1968                id = vm.id,
1969                state = self.lifecycle,
1970            )
1971        })
1972    }
1973}
1974
1975impl Drop for Sandbox {
1976    fn drop(&mut self) {
1977        if let Some(warning) = self.drop_warning() {
1978            self.control.adapters().warn(&warning);
1979        }
1980    }
1981}
1982
1983/// A fresh per-VM bearer token: 32 bytes from `entropy` as 64 hex characters.
1984///
1985/// The hand-rolled `/dev/urandom` read this once replaced carried a clock-mixing fallback
1986/// for a failed read. An unavailable source refuses the launch instead: minting a bearer
1987/// token from a clock would be worse than not launching.
1988fn mint_agent_token(entropy: &dyn crate::entropy::Entropy) -> Result<String, Error> {
1989    let mut bytes = [0u8; 32];
1990    entropy.fill(&mut bytes)?;
1991    Ok(const_hex::encode(bytes))
1992}
1993
1994#[cfg(test)]
1995mod tests {
1996    use super::*;
1997    use crate::control::fake::{self as fake, Answer, FakeControlPlane, TestClock};
1998    use crate::control::transport::Method;
1999
2000    /// A sandbox over the contract recorder, plus the two handles a test asserts through.
2001    ///
2002    /// Tests that reach daemon readiness pause Tokio time so a broken readiness condition
2003    /// reaches its deadline without spending the bootstrap budget on wall-clock sleeps.
2004    /// The separate `TestClock` continues to measure the scripted control-plane waits.
2005    ///
2006    /// The recorder is T-W2-4's, deliberately: its answers are literal JSON in the service
2007    /// model's spelling rather than values serialized from this crate's types, so a member
2008    /// this lane misreads cannot be misread identically by the fake.
2009    fn planted() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2010        let recorder = Arc::new(FakeControlPlane::new());
2011        let clock = Arc::new(TestClock::new());
2012        let plane = crate::testing::control_plane(
2013            Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2014            Region::US_EAST_1,
2015            Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2016        );
2017        // A daemon that answers health, because a launch waits for one (#254).
2018        let sandbox = Sandbox::with_control_plane(plane)
2019            .with_session_backend(crate::testing::HealthyDaemon::new());
2020        (sandbox, recorder, clock)
2021    }
2022
2023    /// Suspend, resume and terminate wait five minutes at five-second polls, with no stall
2024    /// probe: core's lifecycle wait, not the image build's 45-minute default (#266).
2025    #[test]
2026    fn the_suspend_resume_and_terminate_waits_are_cores_lifecycle_wait() {
2027        let (sandbox, _, _) = planted();
2028        let wait = sandbox.lifecycle_wait();
2029        assert_eq!(
2030            (wait.timeout, wait.poll_interval, wait.stall_grace),
2031            (
2032                DEFAULT_LIFECYCLE_TIMEOUT,
2033                LIFECYCLE_POLL_INTERVAL,
2034                Duration::MAX
2035            )
2036        );
2037    }
2038
2039    /// Queues everything a launch to RUNNING needs.
2040    fn answer_launch(recorder: &FakeControlPlane) {
2041        recorder
2042            .answer(
2043                "RunMicrovm",
2044                Answer::ok(fake::microvm_response("PENDING", None)),
2045            )
2046            .answer(
2047                "GetMicrovm",
2048                Answer::ok(fake::microvm_response("RUNNING", None)),
2049            )
2050            .answer(
2051                "CreateMicrovmAuthToken",
2052                Answer::ok(fake::auth_token_response("proxy-token")),
2053            );
2054    }
2055
2056    #[tokio::test(start_paused = true)]
2057    async fn sandbox_launch_polls_at_one_second_then_backs_off() {
2058        for pending_polls in 0..=6 {
2059            let (mut sandbox, recorder, clock) = planted();
2060            recorder.answer(
2061                "RunMicrovm",
2062                Answer::ok(fake::microvm_response("PENDING", None)),
2063            );
2064            for _ in 0..pending_polls {
2065                recorder.answer(
2066                    "GetMicrovm",
2067                    Answer::ok(fake::microvm_response("PENDING", None)),
2068                );
2069            }
2070            recorder
2071                .answer(
2072                    "GetMicrovm",
2073                    Answer::ok(fake::microvm_response("RUNNING", None)),
2074                )
2075                .answer(
2076                    "CreateMicrovmAuthToken",
2077                    Answer::ok(fake::auth_token_response("proxy-token")),
2078                );
2079            sandbox
2080                .run(RunRequest::new().with_image("arn:image"))
2081                .await
2082                .expect("ready");
2083            let expected_seconds = [0, 1, 2, 3, 8, 13, 18][pending_polls];
2084            assert_eq!(clock.now(), Duration::from_secs(expected_seconds));
2085            assert_eq!(recorder.call_count("GetMicrovm"), pending_polls + 1);
2086            assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2087            sandbox.detach().expect("hand the scripted VM off quietly");
2088        }
2089
2090        // The public control-plane wait still honors a fixed interval.
2091        let (sandbox, recorder, clock) = planted();
2092        recorder
2093            .answer(
2094                "GetMicrovm",
2095                Answer::ok(fake::microvm_response("PENDING", None)),
2096            )
2097            .answer(
2098                "GetMicrovm",
2099                Answer::ok(fake::microvm_response("RUNNING", None)),
2100            );
2101        sandbox
2102            .control
2103            .wait_for_launch("mvm-abc123", WaitOpts::for_launch())
2104            .await
2105            .expect("ready");
2106        assert_eq!(clock.now(), Duration::from_secs(5));
2107
2108        // Only client-token adoption can resume a suspended launch.
2109        let (mut sandbox, recorder, _) = planted();
2110        recorder
2111            .answer(
2112                "RunMicrovm",
2113                Answer::ok(fake::microvm_response("PENDING", None)),
2114            )
2115            .answer(
2116                "GetMicrovm",
2117                Answer::ok(fake::microvm_response("SUSPENDED", None)),
2118            );
2119        let error = sandbox
2120            .run(RunRequest::new().with_image("arn:image"))
2121            .await
2122            .expect_err("a fresh launch fails fast");
2123        assert_eq!(error.kind(), ErrorKind::LaunchDied);
2124        assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
2125        sandbox.detach().expect("hand the scripted VM off quietly");
2126    }
2127
2128    #[tokio::test(start_paused = true)]
2129    async fn vpc_egress_configuration_survives_the_sandbox_launch() {
2130        let (mut sandbox, recorder, _) = planted();
2131        answer_launch(&recorder);
2132        let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2133        let request = RunRequest::new()
2134            .with_image("arn:image")
2135            .with_egress_network_connector(arn);
2136        assert_eq!(
2137            request.egress_posture(),
2138            crate::control::EgressPosture::Unsealed
2139        );
2140        sandbox.run(request).await.expect("launch");
2141        let body = recorder.first_body("RunMicrovm");
2142        assert_eq!(body["egressNetworkConnectors"], serde_json::json!([arn]));
2143    }
2144
2145    /// **A sandbox's session reads the plane's clock.** Its exec ids carry the plane's wall
2146    /// reading, so the lifecycle and the session it hands out share one time source.
2147    ///
2148    /// **Falsification**: drop `with_clock(self.control.shared_clock())` from `build_session`
2149    /// and the session takes the adapters' clock, whose wall reading is another instant.
2150    #[tokio::test(start_paused = true)]
2151    async fn a_sandbox_session_reads_the_planes_clock() {
2152        let (mut sandbox, recorder, clock) = planted();
2153        answer_launch(&recorder);
2154        let session = sandbox
2155            .run(RunRequest::new().with_image("arn:image"))
2156            .await
2157            .expect("the launch reaches RUNNING");
2158        let pinned = Duration::from_secs(1_767_225_600) + Duration::from_nanos(987_654_321);
2159        clock.set_unix_now(pinned);
2160        let id = session.mint_exec_id();
2161        let value =
2162            u64::from_str_radix(id.strip_prefix("x-").expect("the x- shape"), 16).expect("hex");
2163        let low_bits = (1_u64 << 40) - 1;
2164        assert_eq!(
2165            value & low_bits,
2166            (pinned.as_nanos() as u64) & low_bits,
2167            "{id} wasn't minted from the plane's clock"
2168        );
2169    }
2170
2171    /// **`ensure_image` builds its STS and S3 client through the plane's adapters**, so a
2172    /// plane over test adapters never reaches a real credential chain.
2173    ///
2174    /// **Falsification**: build `SignedBuildServices` inline again, and the refusal (or, with
2175    /// credentials in the environment, the call) is the real chain's rather than the
2176    /// adapters'.
2177    #[tokio::test]
2178    async fn ensure_image_builds_its_services_through_the_planes_adapters() {
2179        let (mut sandbox, recorder, _) = planted();
2180        let dockerfile = crate::control::artifact::wrap_dockerfile(
2181            "FROM python:3.12-slim\nWORKDIR /app\n",
2182            &crate::control::artifact::WrapOptions::default(),
2183        )
2184        .expect("a wrappable Dockerfile");
2185        let request = crate::control::EnsureImageRequest::new(
2186            "task",
2187            crate::testing::aarch64_daemon(b"daemon"),
2188            dockerfile,
2189            "artifact-bucket",
2190            "arn:aws:iam::123456789012:role/build",
2191        );
2192        let error = sandbox
2193            .ensure_image(request)
2194            .await
2195            .expect_err("the test adapters build no services");
2196        assert_eq!(error.kind(), ErrorKind::Credentials, "{error}");
2197        assert!(
2198            error
2199                .to_string()
2200                .contains("the test adapters build no STS or S3 client for us-east-1"),
2201            "{error}"
2202        );
2203        assert!(
2204            recorder.calls().is_empty(),
2205            "no control-plane call came first"
2206        );
2207    }
2208
2209    /// **A launch draws its agent token, identity seeds and client token from the plane's
2210    /// entropy**, in that order, so the one source a caller or a test swaps in decides all
2211    /// three.
2212    ///
2213    /// **Falsification**: mint any of the three from `OsEntropy` instead of
2214    /// `self.control.entropy()` and its assertion below reads a value the scripted source
2215    /// never produced.
2216    #[tokio::test(start_paused = true)]
2217    async fn a_launch_draws_every_random_value_from_the_planes_entropy() {
2218        use crate::entropy::testing::SequenceEntropy;
2219        use base64::Engine as _;
2220
2221        let recorder = Arc::new(FakeControlPlane::new());
2222        let entropy = Arc::new(SequenceEntropy::new());
2223        let plane = crate::testing::control_plane(
2224            Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2225            Region::US_EAST_1,
2226            Arc::new(TestClock::new()),
2227        )
2228        .with_entropy(Arc::clone(&entropy) as Arc<dyn crate::entropy::Entropy>);
2229        let mut sandbox = Sandbox::with_control_plane(plane)
2230            .with_session_backend(crate::testing::HealthyDaemon::new());
2231        answer_launch(&recorder);
2232        let session = sandbox
2233            .run(RunRequest::new().with_image("arn:image").with_identity())
2234            .await
2235            .expect("the launch reaches RUNNING");
2236
2237        assert_eq!(
2238            session.agent_token(),
2239            const_hex::encode(SequenceEntropy::draw(1, 32)),
2240            "the agent token"
2241        );
2242        let body = recorder.first_body("RunMicrovm");
2243        let payload: serde_json::Value =
2244            serde_json::from_str(body["runHookPayload"].as_str().expect("a string payload"))
2245                .expect("the payload is JSON");
2246        assert_eq!(
2247            payload[protocol::identity::SEED_KEY],
2248            base64::engine::general_purpose::STANDARD
2249                .encode(SequenceEntropy::draw(2, protocol::identity::SEED_BYTES)),
2250            "the VM seed"
2251        );
2252        let client_token = body["clientToken"].as_str().expect("a client token");
2253        assert!(
2254            client_token.ends_with(&const_hex::encode(SequenceEntropy::draw(4, 8))),
2255            "the client token's nonce: {client_token}"
2256        );
2257        assert_eq!(
2258            entropy.calls(),
2259            4,
2260            "one draw per value, the host seed included"
2261        );
2262        sandbox.detach().expect("hand the scripted VM off quietly");
2263    }
2264
2265    /// An unavailable entropy source refuses the launch before anything reaches the plane.
2266    #[tokio::test]
2267    async fn an_unavailable_entropy_source_refuses_the_launch() {
2268        use crate::entropy::testing::SequenceEntropy;
2269
2270        let recorder = Arc::new(FakeControlPlane::new());
2271        let plane = crate::testing::control_plane(
2272            Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2273            Region::US_EAST_1,
2274            Arc::new(TestClock::new()),
2275        )
2276        .with_entropy(Arc::new(SequenceEntropy::unavailable()));
2277        let mut sandbox = Sandbox::with_control_plane(plane)
2278            .with_session_backend(crate::testing::HealthyDaemon::new());
2279        answer_launch(&recorder);
2280        let err = sandbox
2281            .run(RunRequest::new().with_image("arn:image"))
2282            .await
2283            .expect_err("no pool, no bearer token");
2284        assert_eq!(err.kind(), ErrorKind::Unexpected, "{err}");
2285        assert_eq!(recorder.call_count("RunMicrovm"), 0);
2286    }
2287
2288    // ── a launch waits for the daemon (#254) ──────────────────────────────────
2289
2290    /// A sandbox whose daemon answers health from `replies`, over a scripted plane that
2291    /// answers a launch to RUNNING.
2292    fn launching_to(
2293        replies: Vec<crate::testing::Reply>,
2294    ) -> (
2295        Sandbox,
2296        Arc<FakeControlPlane>,
2297        Arc<crate::testing::Recorder>,
2298    ) {
2299        let recorder = Arc::new(FakeControlPlane::new());
2300        let plane = crate::testing::control_plane(
2301            Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2302            Region::US_EAST_1,
2303            Arc::new(TestClock::new()),
2304        );
2305        let daemon = crate::testing::Recorder::with(replies);
2306        let sandbox = Sandbox::with_control_plane(plane).with_session_backend(daemon.clone());
2307        answer_launch(&recorder);
2308        (sandbox, recorder, daemon)
2309    }
2310
2311    #[tokio::test(start_paused = true)]
2312    async fn launch_metrics_track_retries_running_and_ready_without_changing_on_clones() {
2313        use crate::clock::Clock;
2314        let recorder = Arc::new(FakeControlPlane::new());
2315        recorder.answer("RunMicrovm", Answer::throttled_as_400());
2316        answer_launch(&recorder);
2317        let clock = Arc::new(TestClock::new());
2318        let plane =
2319            crate::testing::control_plane(recorder.clone(), Region::US_EAST_1, clock.clone());
2320        let mut sandbox = Sandbox::with_control_plane(plane)
2321            .with_session_backend(crate::testing::HealthyDaemon::new());
2322        let mut request = RunRequest::new().with_image("arn:image");
2323        request.wait = false;
2324        let session = sandbox.run(request).await.unwrap().clone();
2325        let accepted = session.launch_metrics().unwrap();
2326        assert_eq!(accepted.attempt_count, 2);
2327        assert!(accepted.backoff_seconds > 0.0);
2328        assert_eq!(accepted.running_at, None);
2329        assert_eq!(accepted.ready_at, None);
2330        clock.advance(Duration::from_secs(7));
2331        sandbox
2332            .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
2333            .await
2334            .unwrap();
2335        let ready = session.launch_metrics().unwrap();
2336        assert_eq!(ready.attempt_count, accepted.attempt_count);
2337        assert_eq!(ready.backoff_seconds, accepted.backoff_seconds);
2338        assert_eq!(ready.running_at, Some(clock.unix_now().as_secs_f64()));
2339        assert_eq!(ready.ready_at, ready.running_at);
2340        clock.advance(Duration::from_secs(3));
2341        session
2342            .wait_until_ready(Duration::from_secs(1))
2343            .await
2344            .unwrap();
2345        assert_eq!(session.launch_metrics(), Some(ready));
2346        sandbox.detach().unwrap();
2347    }
2348
2349    #[tokio::test]
2350    async fn cancellation_preserves_an_inflight_accepted_launch() {
2351        use crate::control::transport::RetryCancellation;
2352        let (mut sandbox, recorder, _) = planted();
2353        answer_launch(&recorder);
2354        let gate = recorder.hold_responses();
2355        let cancellation = RetryCancellation::default();
2356        let mut request = RunRequest::new().with_image("arn:image");
2357        request.wait = false;
2358        {
2359            let pending = sandbox.run_cancellable(request, &cancellation);
2360            tokio::pin!(pending);
2361            tokio::select! {
2362                biased;
2363                _ = &mut pending => panic!("response must wait for the gate"),
2364                _ = std::future::ready(()) => {}
2365            }
2366            assert_eq!(recorder.call_count("RunMicrovm"), 1);
2367            cancellation.cancel();
2368            gate.notify_one();
2369            assert_eq!(
2370                pending
2371                    .await
2372                    .unwrap()
2373                    .launch_metrics()
2374                    .unwrap()
2375                    .attempt_count,
2376                1
2377            );
2378        }
2379        assert!(sandbox.microvm().is_some());
2380        assert_eq!(recorder.call_count("RunMicrovm"), 1);
2381        sandbox.detach().unwrap();
2382    }
2383
2384    #[tokio::test]
2385    async fn cancelled_retry_does_not_launch_before_any_request() {
2386        use crate::control::transport::RetryCancellation;
2387        let (mut sandbox, recorder, _) = planted();
2388        recorder.answer(
2389            "RunMicrovm",
2390            Answer::failure(403, "pre-cancelled request reached the transport"),
2391        );
2392        let cancellation = RetryCancellation::default();
2393        cancellation.cancel();
2394        let error = sandbox
2395            .run_cancellable(RunRequest::new().with_image("arn:image"), &cancellation)
2396            .await
2397            .unwrap_err();
2398        assert_eq!(recorder.call_count("RunMicrovm"), 0);
2399        assert_eq!(error.kind(), ErrorKind::Interrupted);
2400        assert!(sandbox.microvm().is_none());
2401    }
2402
2403    #[tokio::test]
2404    async fn cancellation_interrupts_image_resolution_and_releases_the_sandbox() {
2405        use crate::control::transport::{RetryCancellation, RetryPolicy};
2406        let recorder = Arc::new(FakeControlPlane::new());
2407        recorder.answer("ListMicrovmImages", Answer::throttled_as_400());
2408        let plane = crate::testing::control_plane(
2409            recorder.clone(),
2410            Region::US_EAST_1,
2411            Arc::new(crate::testing::YieldingClock::new()),
2412        )
2413        .with_retry(RetryPolicy::new(100, Duration::from_secs(3600)).unwrap());
2414        let sandbox = tokio::sync::Mutex::new(Sandbox::with_control_plane(plane));
2415        let cancellation = RetryCancellation::default();
2416        let pending = async {
2417            sandbox
2418                .lock()
2419                .await
2420                .run_cancellable(RunRequest::new().with_image("named-image"), &cancellation)
2421                .await
2422                .map(|_| ())
2423        };
2424        tokio::pin!(pending);
2425        tokio::select! {
2426            biased;
2427            _ = &mut pending => panic!("image resolution must be retrying"),
2428            _ = std::future::ready(()) => {}
2429        }
2430        assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2431        assert!(sandbox.try_lock().is_err());
2432        cancellation.cancel();
2433        let error = tokio::select! {
2434            biased;
2435            result = &mut pending => result.unwrap_err(),
2436            _ = std::future::ready(()) => panic!("cancellation must finish without another sleep"),
2437        };
2438        assert_eq!(error.kind(), ErrorKind::Interrupted);
2439        assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2440        assert_eq!(recorder.call_count("RunMicrovm"), 0);
2441        let sandbox = sandbox.try_lock().expect("cancellation releases the lock");
2442        assert!(sandbox.microvm().is_none());
2443    }
2444
2445    /// Not yet bootstrapped, then bootstrapped: the two health answers a launch waits through.
2446    fn not_ready_then_ready() -> Vec<crate::testing::Reply> {
2447        vec![
2448            crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()),
2449            crate::testing::Reply::ok(crate::testing::health_body(true)),
2450        ]
2451    }
2452
2453    /// The paths a scripted daemon was asked for, in order.
2454    fn daemon_paths(daemon: &crate::testing::Recorder) -> Vec<String> {
2455        daemon
2456            .requests()
2457            .iter()
2458            .map(|request| format!("{} {}", request.method, request.path))
2459            .collect()
2460    }
2461
2462    /// **`run` returns once the daemon answers, not at RUNNING (#254).** The plane reports
2463    /// RUNNING and the daemon answers 503 and then 200 bootstrapped: `run` hands back the
2464    /// session after the second poll, as the CLI's `run` and `AgentVm.launch` did on their own.
2465    ///
2466    /// **Falsification**: drop the `wait_until_ready` call from `Sandbox::wait_until_running`
2467    /// and `run` returns at RUNNING having asked the daemon nothing.
2468    #[tokio::test(start_paused = true)]
2469    async fn a_launch_returns_once_its_daemon_answers() {
2470        let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2471        sandbox
2472            .run(RunRequest::new().with_image("arn:image"))
2473            .await
2474            .expect("the launch reaches RUNNING and its daemon answers");
2475        assert_eq!(
2476            daemon_paths(&daemon),
2477            ["GET /v1/health", "GET /v1/health"],
2478            "one poll refused, one bootstrapped, and nothing after"
2479        );
2480        sandbox.detach().expect("hand the scripted VM off quietly");
2481    }
2482
2483    /// **A launch that didn't wait is finished the same way (#254):** `wait_until_running`
2484    /// after `run(wait = false)` returns once the daemon answers too, and the accepted launch
2485    /// itself asked the daemon nothing.
2486    #[tokio::test(start_paused = true)]
2487    async fn wait_until_running_returns_once_the_daemon_answers() {
2488        let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2489        let mut request = RunRequest::new().with_image("arn:image");
2490        request.wait = false;
2491        sandbox.run(request).await.expect("the launch is accepted");
2492        assert!(
2493            daemon_paths(&daemon).is_empty(),
2494            "an accepted launch doesn't wait"
2495        );
2496        sandbox
2497            .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
2498            .await
2499            .expect("RUNNING, then the daemon answers");
2500        assert_eq!(daemon_paths(&daemon), ["GET /v1/health", "GET /v1/health"]);
2501        sandbox.detach().expect("hand the scripted VM off quietly");
2502    }
2503
2504    /// **A daemon that never answers fails the launch with `ERR_TIMEOUT` (#254),** after the
2505    /// bootstrap wait, rather than the first exec failing later. The VM did reach RUNNING, so
2506    /// the lifecycle says so and the token counts as installed (STATE-2).
2507    #[tokio::test(start_paused = true)]
2508    async fn a_launch_whose_daemon_never_answers_times_out_running() {
2509        let never = (0..128)
2510            .map(|_| crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()))
2511            .collect();
2512        let (mut sandbox, _recorder, daemon) = launching_to(never);
2513        let error = sandbox
2514            .run(RunRequest::new().with_image("arn:image"))
2515            .await
2516            .expect_err("the daemon never answered");
2517        assert_eq!(error.kind(), ErrorKind::Timeout, "{error}");
2518        assert!(daemon.requests().len() > 1, "it kept asking");
2519        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2520        assert!(sandbox.token_installed());
2521        sandbox.detach().expect("hand the scripted VM off quietly");
2522    }
2523
2524    // ── a launch by bare image name (#253) ───────────────────────────────────
2525
2526    /// The ARN the fake's listing gives `wanted-image`.
2527    const WANTED_ARN: &str = "arn:aws:lambda:us-east-1:123456789012:microvm-image:wanted-image";
2528
2529    /// A listing that holds `wanted-image` beside a name it's a prefix of.
2530    fn answer_listing(recorder: &FakeControlPlane) {
2531        recorder.answer(
2532            "ListMicrovmImages",
2533            Answer::ok(fake::list_images_response(
2534                &["wanted-image-old", "wanted-image"],
2535                None,
2536            )),
2537        );
2538    }
2539
2540    /// **A launch by bare image name sends the listed ARN**, resolved before `RunMicrovm`, as
2541    /// `run --image NAME` did. Every surface's launch is this `run`, and the SDKs used to send
2542    /// the name verbatim, which the service answers with HTTP 400 "Malformed ARN".
2543    ///
2544    /// **Falsification**: drop the `resolve_image_arn` call from `Sandbox::run` and the calls
2545    /// start at `RunMicrovm`, whose body carries `wanted-image`.
2546    #[tokio::test(start_paused = true)]
2547    async fn a_launch_by_bare_image_name_sends_the_listed_arn() {
2548        let (mut sandbox, recorder, _) = planted();
2549        answer_listing(&recorder);
2550        answer_launch(&recorder);
2551        sandbox
2552            .run(RunRequest::new().with_image("wanted-image"))
2553            .await
2554            .expect("the launch reaches RUNNING");
2555
2556        let operations = recorder.operations();
2557        assert_eq!(
2558            operations.get(..2),
2559            Some(&["ListMicrovmImages", "RunMicrovm"][..]),
2560            "the name is resolved, once, before the launch: {operations:?}"
2561        );
2562        assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2563        assert_eq!(
2564            recorder.first_body("RunMicrovm")["imageIdentifier"],
2565            WANTED_ARN,
2566            "the launch carries the exact match's ARN, never the name"
2567        );
2568        sandbox.detach().expect("hand the scripted VM off quietly");
2569    }
2570
2571    /// **An image ARN launches with no listing call**, and is the ARN the sandbox records.
2572    #[tokio::test(start_paused = true)]
2573    async fn a_launch_by_image_arn_makes_no_listing_call() {
2574        let (mut sandbox, recorder, _) = planted();
2575        answer_launch(&recorder);
2576        sandbox
2577            .run(RunRequest::new().with_image(WANTED_ARN))
2578            .await
2579            .expect("the launch reaches RUNNING");
2580
2581        assert_eq!(recorder.call_count("ListMicrovmImages"), 0);
2582        assert_eq!(recorder.operations().first(), Some(&"RunMicrovm"));
2583        assert_eq!(
2584            recorder.first_body("RunMicrovm")["imageIdentifier"],
2585            WANTED_ARN
2586        );
2587        assert_eq!(sandbox.launch_image_arn(), Some(WANTED_ARN));
2588        sandbox.detach().expect("hand the scripted VM off quietly");
2589    }
2590
2591    /// **The resolved ARN is recorded before `RunMicrovm`, so a launch the service refuses
2592    /// still names the image it asked for.** The CLI's failure envelope reads it from here:
2593    /// `microvm()` holds only what an accepted launch answered.
2594    ///
2595    /// **Falsification**: drop the `launch_image_arn` assignment from `Sandbox::run` and the
2596    /// accessor reads `None` after the refusal.
2597    #[tokio::test]
2598    async fn a_refused_launch_by_name_still_records_the_resolved_arn() {
2599        let (mut sandbox, recorder, _) = planted();
2600        answer_listing(&recorder);
2601        recorder.answer("RunMicrovm", Answer::failure(400, "scripted refusal"));
2602        sandbox
2603            .run(RunRequest::new().with_image("wanted-image"))
2604            .await
2605            .expect_err("the service refused the launch");
2606
2607        assert_eq!(
2608            sandbox.launch_image_arn(),
2609            Some(WANTED_ARN),
2610            "the ARN the refused launch asked for"
2611        );
2612        assert!(sandbox.microvm().is_none(), "no launch was accepted");
2613        assert_eq!(recorder.call_count("RunMicrovm"), 1);
2614    }
2615
2616    /// **A name no image carries is `ERR_PRECONDITION` before any launch, and the sandbox is
2617    /// left as it was**: nothing recorded (STATE-1 counts only an accepted launch), so the
2618    /// same sandbox launches once the caller passes an image that exists.
2619    #[tokio::test(start_paused = true)]
2620    async fn an_unknown_image_name_is_refused_before_the_launch_and_the_sandbox_still_runs() {
2621        let (mut sandbox, recorder, _) = planted();
2622        recorder.answer(
2623            "ListMicrovmImages",
2624            Answer::ok(fake::list_images_response(&[], None)),
2625        );
2626        let error = sandbox
2627            .run(RunRequest::new().with_image("no-such-image"))
2628            .await
2629            .expect_err("nothing to launch from");
2630        assert_eq!(error.code(), "ERR_PRECONDITION");
2631        assert!(error.to_string().contains("no-such-image"), "{error}");
2632        assert_eq!(recorder.call_count("RunMicrovm"), 0);
2633        assert_eq!(sandbox.launch_image_arn(), None);
2634        assert!(!sandbox.image_exists(), "STATE-1: no launch was accepted");
2635        assert!(sandbox.microvm().is_none());
2636        assert_eq!(sandbox.bootstrap_count(), 0);
2637
2638        answer_launch(&recorder);
2639        sandbox
2640            .run(RunRequest::new().with_image(WANTED_ARN))
2641            .await
2642            .expect("a failed resolution doesn't use up the sandbox's one launch");
2643        assert_eq!(sandbox.bootstrap_count(), 1);
2644        sandbox.detach().expect("hand the scripted VM off quietly");
2645    }
2646
2647    /// **A launch this client refuses costs no call, a bare name included**: the local
2648    /// refusals come before the listing a name costs.
2649    ///
2650    /// **Falsification**: drop the `launch_connectors` call from `Sandbox::run` and the
2651    /// out-of-range duration is refused by `run_microvm` only after the listing went out.
2652    #[tokio::test]
2653    async fn a_launch_refused_locally_makes_no_listing_call_for_a_name() {
2654        let (mut sandbox, recorder, _) = planted();
2655        answer_listing(&recorder);
2656        let mut request = RunRequest::new().with_image("wanted-image");
2657        request.max_duration_sec = 0;
2658        let error = sandbox
2659            .run(request)
2660            .await
2661            .expect_err("a zero maximum duration is refused locally");
2662        assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
2663        assert!(
2664            recorder.calls().is_empty(),
2665            "the refusal came after a call: {:?}",
2666            recorder.operations()
2667        );
2668        assert_eq!(sandbox.launch_image_arn(), None);
2669    }
2670
2671    /// **BIND-12: the launched session carries its request's posture**, the value
2672    /// `egress_posture_for` answers for the same options and the CLI envelope reports
2673    /// (the CLI's `guards/run.rs` holds that half). Each of the four launchable shapes.
2674    ///
2675    /// **Falsification** — 2026-09-24. Build the session without `with_egress_posture` and
2676    /// the `open` and `best-effort` rows read `unsealed`; restored.
2677    #[tokio::test(start_paused = true)]
2678    async fn a_launched_session_reports_its_requests_posture() {
2679        use crate::control::EgressPosture;
2680        let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2681        let rows = [
2682            (RunRequest::new(), EgressPosture::Unsealed),
2683            (RunRequest::new().with_egress(), EgressPosture::Open),
2684            (
2685                RunRequest::new().with_deny_egress(),
2686                EgressPosture::BestEffort,
2687            ),
2688            (
2689                RunRequest::new().with_egress_network_connector(arn),
2690                EgressPosture::Unsealed,
2691            ),
2692        ];
2693        for (request, expected) in rows {
2694            let answered = crate::control::egress_posture_for(
2695                request.egress,
2696                &request.egress_network_connectors,
2697                request.deny_egress,
2698                Some(&Region::US_EAST_1),
2699            )
2700            .expect("a launchable request");
2701            assert_eq!(answered, expected);
2702            let (mut sandbox, recorder, _) = planted();
2703            answer_launch(&recorder);
2704            let session = sandbox
2705                .run(request.with_image("arn:image"))
2706                .await
2707                .expect("the launch reaches RUNNING");
2708            assert_eq!(session.egress_posture(), expected, "BIND-12");
2709            assert_eq!(sandbox.egress_posture(), expected);
2710            sandbox.detach().expect("hand the scripted VM off quietly");
2711        }
2712    }
2713
2714    /// **BIND-12: a session that does not hold its launch options reports `unsealed`**, the
2715    /// weakest true claim, whatever the VM was launched with: an adopted VM and a session
2716    /// attached directly both lack the request.
2717    #[tokio::test]
2718    async fn a_session_without_its_launch_options_reports_unsealed() {
2719        use crate::control::EgressPosture;
2720        let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2721        assert_eq!(
2722            sandbox.session().expect("RUNNING").egress_posture(),
2723            EgressPosture::Unsealed
2724        );
2725        assert_eq!(sandbox.egress_posture(), EgressPosture::Unsealed);
2726        let direct = Session::builder(ADOPT_ENDPOINT, ADOPT_TOKEN)
2727            .build_with(&crate::testing::TestAdapters::new())
2728            .expect("a direct session");
2729        assert_eq!(direct.egress_posture(), EgressPosture::Unsealed);
2730    }
2731
2732    /// A launched sandbox in RUNNING, which is where four of the twelve keys start.
2733    async fn launched() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2734        let (mut sandbox, recorder, clock) = planted();
2735        answer_launch(&recorder);
2736        sandbox
2737            .run(RunRequest::new().with_image("arn:image"))
2738            .await
2739            .expect("the launch reaches RUNNING");
2740        (sandbox, recorder, clock)
2741    }
2742
2743    /// **A dropped live sandbox warns through the plane's adapters**, not a stream of its
2744    /// own, so the driving adapter decides where the warning goes (ARCH-7). The warning names
2745    /// the state as the one the sandbox last saw, since the platform can suspend or end the VM
2746    /// without telling it, and says only that a VM still live bills.
2747    ///
2748    /// **Falsification**: write the warning straight to stderr in `Drop` and the recorded
2749    /// list comes back empty (and the crate's `clippy.toml` refuses `std::io::stderr`).
2750    #[tokio::test(start_paused = true)]
2751    async fn a_dropped_live_sandbox_warns_through_the_planes_adapters() {
2752        let recorder = Arc::new(FakeControlPlane::new());
2753        let clock = Arc::new(TestClock::new());
2754        let adapters = Arc::new(crate::testing::TestAdapters::new());
2755        let plane = crate::control::ControlPlane::from_ports(
2756            Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2757            Region::US_EAST_1,
2758            Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2759            Arc::new(crate::testing::SequenceEntropy::new()),
2760            Arc::clone(&adapters) as Arc<dyn crate::adapters::Adapters>,
2761        );
2762        let mut sandbox = Sandbox::with_control_plane(plane)
2763            .with_session_backend(crate::testing::HealthyDaemon::new());
2764        answer_launch(&recorder);
2765        sandbox
2766            .run(RunRequest::new().with_image("arn:image"))
2767            .await
2768            .expect("the launch reaches RUNNING");
2769        let id = sandbox.microvm().expect("launched").id.clone();
2770        drop(sandbox);
2771        let warnings = adapters.warnings();
2772        assert_eq!(warnings.len(), 1, "{warnings:?}");
2773        assert!(warnings[0].contains(&id), "{warnings:?}");
2774        assert!(
2775            warnings[0].contains("RUNNING is the state this sandbox last saw")
2776                && warnings[0].contains("A VM that's still live bills"),
2777            "{warnings:?}"
2778        );
2779    }
2780
2781    /// **Detach hands off, and nothing warns or moves afterwards.** A launched sandbox
2782    /// warns on drop; once detached it returns what an adopter needs (with the token kept
2783    /// out of `Debug`), stops warning, makes no further control-plane call, and refuses every
2784    /// transition — including `terminate`, which reports the refusal instead of raising.
2785    ///
2786    /// **Falsification** — 2026-09-24. Leave `detached` out of `drop_warning`'s early
2787    /// return and the "no warning after detach" assertion goes red; drop the
2788    /// `refuse_detached` guard from `suspend` and the zero-calls assertion goes red (the
2789    /// suspend reaches `SuspendMicrovm`). Both restored.
2790    #[tokio::test(start_paused = true)]
2791    async fn a_detached_sandbox_hands_off_its_vm_and_stays_quiet() {
2792        let (mut sandbox, recorder, _clock) = launched().await;
2793        assert!(
2794            sandbox.drop_warning().is_some(),
2795            "a launched, live VM warns on drop"
2796        );
2797        let token = sandbox
2798            .session()
2799            .expect("launched")
2800            .agent_token()
2801            .to_string();
2802        let vm = sandbox.microvm().expect("launched").clone();
2803
2804        let detached = sandbox.detach().expect("a RUNNING VM detaches");
2805        assert_eq!(detached.microvm_id, vm.id);
2806        assert_eq!(detached.endpoint, vm.endpoint);
2807        assert_eq!(detached.agent_token(), token);
2808        assert_eq!(detached.region, Region::US_EAST_1);
2809        let printed = format!("{detached:?} {sandbox:?}");
2810        assert!(!printed.contains(&token), "{printed}");
2811        assert!(printed.contains("detached: true"), "{printed}");
2812
2813        assert!(sandbox.detached());
2814        assert!(
2815            sandbox.drop_warning().is_none(),
2816            "detached is not abandoned"
2817        );
2818        assert!(
2819            sandbox.session().is_none(),
2820            "the session went to the adopter"
2821        );
2822
2823        let before = recorder.calls().len();
2824        for error in [
2825            sandbox.suspend().await.expect_err("detached"),
2826            sandbox.resume().await.map(|_| ()).expect_err("detached"),
2827            sandbox
2828                .run(RunRequest::new().with_image("arn:image"))
2829                .await
2830                .map(|_| ())
2831                .expect_err("detached"),
2832            sandbox
2833                .wait_until_running(Duration::from_secs(1))
2834                .await
2835                .map(|_| ())
2836                .expect_err("detached"),
2837            sandbox.detach().map(|_| ()).expect_err("already detached"),
2838        ] {
2839            assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2840            assert!(error.to_string().contains("detached"), "{error}");
2841            assert!(!error.to_string().contains(&token), "{error}");
2842        }
2843        let report = sandbox.terminate(TeardownOpts::default()).await;
2844        assert!(!report.terminate_accepted);
2845        assert!(report.failures[0].contains("detached"), "{report:?}");
2846        assert_eq!(recorder.calls().len(), before, "detached makes no AWS call");
2847        assert!(
2848            sandbox.drop_warning().is_none(),
2849            "a refused terminate is still quiet"
2850        );
2851    }
2852
2853    /// Detach needs a live VM: nothing launched, or already torn down, is refused.
2854    #[tokio::test(start_paused = true)]
2855    async fn detach_refuses_without_a_live_vm() {
2856        let (mut unlaunched, _recorder, _clock) = planted();
2857        let error = unlaunched.detach().expect_err("nothing launched");
2858        assert_eq!(error.kind(), ErrorKind::Precondition);
2859        assert!(!unlaunched.detached());
2860
2861        let (mut sandbox, recorder, _clock) = launched().await;
2862        recorder
2863            .answer("TerminateMicrovm", Answer::ok("{}"))
2864            .answer(
2865                "GetMicrovm",
2866                Answer::ok(fake::microvm_response("TERMINATED", None)),
2867            );
2868        sandbox.terminate(TeardownOpts::default()).await;
2869        let error = sandbox.detach().expect_err("torn down");
2870        assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2871        assert!(!sandbox.detached());
2872    }
2873
2874    /// Drives a launched sandbox to SUSPENDED, which is where resume starts.
2875    ///
2876    /// Both `GetMicrovm` answers are queued **before** the launch rather than one before
2877    /// each wait, and that is not a style choice. The recorder repeats its last queued
2878    /// answer, so a queue left at `[RUNNING]` after the launch and then appended to becomes
2879    /// `[RUNNING, SUSPENDED]` — the suspend wait pops the stale RUNNING, sleeps a poll
2880    /// interval, and the fake's `sleep` **advances the clock**. That five seconds is
2881    /// invisible until a test reads the clock, which then counts the fake's poll interval as
2882    /// time the VM spent suspended. Queueing both up front means every wait matches on its
2883    /// first poll and no test clock moves except when a test moves it.
2884    async fn suspended_with_window(
2885        suspended_sec: u32,
2886    ) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2887        let (mut sandbox, recorder, clock) = planted();
2888        recorder
2889            .answer(
2890                "RunMicrovm",
2891                Answer::ok(fake::microvm_response("PENDING", None)),
2892            )
2893            .answer(
2894                "GetMicrovm",
2895                Answer::ok(fake::microvm_response("RUNNING", None)),
2896            )
2897            .answer(
2898                "GetMicrovm",
2899                Answer::ok(fake::microvm_response("SUSPENDED", None)),
2900            )
2901            .answer(
2902                "CreateMicrovmAuthToken",
2903                Answer::ok(fake::auth_token_response("proxy-token")),
2904            )
2905            .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2906
2907        sandbox
2908            .run(
2909                RunRequest::new()
2910                    .with_image("arn:image")
2911                    .with_suspended_sec(suspended_sec),
2912            )
2913            .await
2914            .expect("launches");
2915        sandbox.suspend().await.expect("suspends");
2916        assert_eq!(
2917            clock.now(),
2918            Duration::ZERO,
2919            "no wait may have slept, or a test that reads the clock is measuring the fake's \
2920             poll interval as well as the elapsed suspension"
2921        );
2922        (sandbox, recorder, clock)
2923    }
2924
2925    /// A suspended sandbox at the default ten-minute window.
2926    async fn suspended() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2927        suspended_with_window(600).await
2928    }
2929
2930    // ── adopt (#196) ─────────────────────────────────────────────────────────
2931
2932    /// A canary agent token, so a test can assert it appears in no `Debug` or error.
2933    const ADOPT_TOKEN: &str = "adopt-canary-token-9f1c";
2934    const ADOPT_ENDPOINT: &str = "https://mvm-abc123.microvm.us-east-1.amazonaws.com";
2935
2936    /// A plane over the recorder, whose `GetMicrovm` answers the caller queues.
2937    fn adopt_plane() -> (ControlPlane, Arc<FakeControlPlane>, Arc<TestClock>) {
2938        let recorder = Arc::new(FakeControlPlane::new());
2939        let clock = Arc::new(TestClock::new());
2940        let plane = crate::testing::control_plane(
2941            Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2942            Region::US_EAST_1,
2943            Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2944        );
2945        (plane, recorder, clock)
2946    }
2947
2948    async fn adopted_in(state: &str) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2949        let (plane, recorder, clock) = adopt_plane();
2950        recorder.answer(
2951            "GetMicrovm",
2952            Answer::ok(fake::microvm_response(state, None)),
2953        );
2954        let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2955            .await
2956            .expect("adopts");
2957        (sandbox, recorder, clock)
2958    }
2959
2960    /// **An adopted sandbox feeds its keepalive.** The idle window is the one `GetMicrovm`
2961    /// reports, and the lifecycle watch starts at the adopted state, so a keepalive on an
2962    /// adopted session paces against the real policy and ends when that sandbox suspends.
2963    ///
2964    /// **Falsification** — 2026-09-24. Assign `sandbox.lifecycle` directly instead of
2965    /// through `set_lifecycle`, or drop the `idle_window` line, and this goes red; restored.
2966    #[tokio::test]
2967    async fn an_adopted_sandbox_reports_the_services_idle_window_and_watches_its_lifecycle() {
2968        let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2969        assert_eq!(sandbox.idle_window(), Some(Duration::from_secs(1800)));
2970        assert_eq!(*sandbox.watch_lifecycle().borrow(), Lifecycle::Running);
2971    }
2972
2973    /// Every service state has a lifecycle, and nothing else parses.
2974    #[test]
2975    fn every_service_state_maps_onto_one_lifecycle() {
2976        for state in crate::constants::MICROVM_STATES {
2977            let lifecycle = Lifecycle::from_service(state).expect("a known state");
2978            assert_eq!(lifecycle.as_str(), state);
2979        }
2980        assert_eq!(Lifecycle::from_service("Running"), None);
2981    }
2982
2983    /// **STATE-3 for an adopted VM.** The lifecycle is the service's, the bootstrap is
2984    /// counted once, and `run` is refused before any wire call, so no run-hook payload is
2985    /// ever sent.
2986    ///
2987    /// **Falsification** — 2026-09-24. Leave the adopted lifecycle at PENDING instead of
2988    /// reading it from `GetMicrovm` and the lifecycle and `token_installed` assertions here
2989    /// go red, as does the suspend in the next test; restored after.
2990    #[tokio::test]
2991    async fn an_adopted_vm_takes_its_lifecycle_from_the_service_and_refuses_run() {
2992        let (mut sandbox, recorder, _) = adopted_in("RUNNING").await;
2993        assert!(sandbox.adopted());
2994        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2995        assert!(sandbox.token_installed());
2996        assert_eq!(sandbox.bootstrap_count(), 1);
2997        assert!(sandbox.image_exists());
2998        let session = sandbox.session().expect("a live VM gets a session");
2999        assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
3000        assert_eq!(session.agent_token(), ADOPT_TOKEN);
3001
3002        let error = sandbox
3003            .run(RunRequest::new().with_image("arn:image"))
3004            .await
3005            .expect_err("an adopted VM was bootstrapped by its own launch");
3006        assert_eq!(error.kind(), ErrorKind::InvalidArg);
3007        assert!(error.to_string().contains("STATE-3"), "{error}");
3008        assert_eq!(recorder.call_count("RunMicrovm"), 0);
3009        assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
3010    }
3011
3012    /// A name resolves to the record's VM and adopts it; a missing name or a record from
3013    /// another region is refused before any AWS call.
3014    ///
3015    /// `Sandbox::from_name` itself builds a production plane, so its refusals are pinned in
3016    /// the prelude's tests. What's here is the resolve it runs first, and the adopt after.
3017    #[tokio::test]
3018    async fn a_name_adopts_its_records_vm_and_foreign_names_are_refused_locally() {
3019        struct One(crate::names::NameRecord);
3020        impl crate::names::NameStore for One {
3021            fn get(&self, name: &str) -> Result<Option<crate::names::NameRecord>, Error> {
3022                Ok((name == self.0.name).then(|| self.0.clone()))
3023            }
3024            fn put(&self, _: &crate::names::NameRecord) -> Result<(), Error> {
3025                unreachable!()
3026            }
3027            fn delete(&self, _: &str) -> Result<bool, Error> {
3028                unreachable!()
3029            }
3030            fn list(&self) -> Result<Vec<crate::names::NameRecord>, Error> {
3031                Ok(vec![self.0.clone()])
3032            }
3033        }
3034        let record = crate::names::NameRecord::new_at(
3035            "ci",
3036            "mvm-abc123",
3037            ADOPT_ENDPOINT,
3038            ADOPT_TOKEN,
3039            "us-west-2",
3040            1_767_225_600,
3041        )
3042        .expect("record");
3043        let store = One(record.clone());
3044        let foreign = crate::names::resolve(&store, "ci", Some(&Region::US_EAST_1))
3045            .expect_err("registered in another region");
3046        assert_eq!(foreign.kind(), ErrorKind::InvalidArg);
3047        let missing = crate::names::resolve(&store, "nope", None).expect_err("no such name");
3048        assert_eq!(missing.kind(), ErrorKind::Precondition);
3049        assert!(!format!("{foreign} {missing}").contains(ADOPT_TOKEN));
3050
3051        let (plane, recorder, _) = adopt_plane();
3052        recorder.answer(
3053            "GetMicrovm",
3054            Answer::ok(fake::microvm_response("RUNNING", None)),
3055        );
3056        let resolved = crate::names::resolve(&store, "ci", Some(&Region::US_WEST_2)).expect("ok");
3057        let sandbox = Sandbox::adopt_record(plane, resolved)
3058            .await
3059            .expect("adopts");
3060        assert!(sandbox.adopted());
3061        assert_eq!(
3062            sandbox.microvm().map(|vm| vm.id.as_str()),
3063            Some("mvm-abc123")
3064        );
3065        assert_eq!(
3066            sandbox.session().expect("session").agent_token(),
3067            ADOPT_TOKEN
3068        );
3069        let named = sandbox
3070            .name_record("again")
3071            .expect("an adopted VM can be named");
3072        assert_eq!(
3073            (
3074                named.microvm_id.as_str(),
3075                named.agent_token.as_str(),
3076                named.region.as_str()
3077            ),
3078            ("mvm-abc123", ADOPT_TOKEN, "us-east-1")
3079        );
3080        let unlaunched = Sandbox::with_control_plane(adopt_plane().0);
3081        assert_eq!(
3082            unlaunched
3083                .name_record("x")
3084                .expect_err("nothing to name")
3085                .kind(),
3086            ErrorKind::Precondition
3087        );
3088    }
3089
3090    /// **A record's tunnel identity survives adoption and naming** (#263).
3091    ///
3092    /// `adopt_record` used to drop the pair and `name_record` never wrote it, so a VM named
3093    /// through the SDKs lost what `tunnel --verify-identity` needs to check it. A record with
3094    /// half the pair is refused before any AWS call.
3095    #[tokio::test]
3096    async fn a_records_tunnel_identity_survives_adoption_and_naming() {
3097        let identity = crate::identity::LaunchIdentity::from_seeds([7; 32], [9; 32])
3098            .expect("valid seeds")
3099            .keep();
3100        let mut record = crate::names::NameRecord::new_at(
3101            "ci",
3102            "mvm-abc123",
3103            ADOPT_ENDPOINT,
3104            ADOPT_TOKEN,
3105            "us-east-1",
3106            1_767_225_600,
3107        )
3108        .expect("record");
3109        record.identity_host_seed = Some(identity.host_seed_base64());
3110        record.identity_vm_public_key = Some(identity.vm_public_key_base64());
3111
3112        let (plane, recorder, _) = adopt_plane();
3113        recorder.answer(
3114            "GetMicrovm",
3115            Answer::ok(fake::microvm_response("RUNNING", None)),
3116        );
3117        let sandbox = Sandbox::adopt_record(plane, record.clone())
3118            .await
3119            .expect("adopts");
3120        let kept = sandbox
3121            .tunnel_identity()
3122            .expect("the record's pair is kept");
3123        assert_eq!(kept.host_seed_base64(), identity.host_seed_base64());
3124        assert_eq!(kept.vm_public_key_base64(), identity.vm_public_key_base64());
3125        let named = sandbox.name_record("again").expect("names");
3126        assert_eq!(named.identity_host_seed, record.identity_host_seed);
3127        assert_eq!(named.identity_vm_public_key, record.identity_vm_public_key);
3128
3129        let (plane, recorder, _) = adopt_plane();
3130        let mut torn = record;
3131        torn.identity_vm_public_key = None;
3132        let refused = Sandbox::adopt_record(plane, torn)
3133            .await
3134            .expect_err("half a pair is refused");
3135        assert_eq!(refused.kind(), ErrorKind::InvalidArg);
3136        assert!(refused.to_string().contains("one half"), "{refused}");
3137        assert!(recorder.calls().is_empty(), "refused before any AWS call");
3138    }
3139
3140    /// Suspend, resume, and terminate all work through an adopted handle with the usual
3141    /// guards, and nothing about the launch is re-delivered (STATE-7).
3142    #[tokio::test]
3143    async fn an_adopted_vm_suspends_resumes_and_terminates_through_its_handle() {
3144        let (plane, recorder, _) = adopt_plane();
3145        recorder
3146            .answer(
3147                "GetMicrovm",
3148                Answer::ok(fake::microvm_response("RUNNING", None)),
3149            )
3150            .answer(
3151                "GetMicrovm",
3152                Answer::ok(fake::microvm_response("SUSPENDED", None)),
3153            )
3154            .answer(
3155                "GetMicrovm",
3156                Answer::ok(fake::microvm_response("RUNNING", None)),
3157            )
3158            .answer(
3159                "GetMicrovm",
3160                Answer::ok(fake::microvm_response("TERMINATED", None)),
3161            )
3162            .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3163            .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3164            .answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
3165        let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3166            .await
3167            .expect("adopts");
3168
3169        sandbox.suspend().await.expect("suspends from RUNNING");
3170        assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3171        sandbox.resume().await.expect("resumes from SUSPENDED");
3172        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3173        let report = sandbox
3174            .terminate(TeardownOpts::default().waiting_for_terminated())
3175            .await;
3176        assert!(report.terminate_accepted, "{report:?}");
3177        assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3178        assert!(sandbox.was_terminated());
3179
3180        assert_eq!(recorder.call_count("RunMicrovm"), 0);
3181        assert!(
3182            recorder
3183                .bodies_as_text()
3184                .iter()
3185                .all(|body| !body.contains("runHookPayload") && !body.contains(ADOPT_TOKEN)),
3186            "no launch payload or agent token may reach the control plane"
3187        );
3188        assert_eq!(sandbox.bootstrap_count(), 1, "STATE-3: never above one");
3189    }
3190
3191    /// **STATE-12 for an adopted VM.** An adopted VM this sandbox suspended resumes past the
3192    /// 600 s `idlePolicy` window `GetMicrovm` reported: the resume goes to the service, which
3193    /// doesn't hold an explicit suspend to that window (#597).
3194    #[tokio::test]
3195    async fn an_adopted_vm_resumes_past_the_reported_window_through_the_service() {
3196        let (plane, recorder, clock) = adopt_plane();
3197        recorder
3198            .answer(
3199                "GetMicrovm",
3200                Answer::ok(fake::microvm_response("RUNNING", None)),
3201            )
3202            .answer(
3203                "GetMicrovm",
3204                Answer::ok(fake::microvm_response("SUSPENDED", None)),
3205            )
3206            .answer(
3207                "GetMicrovm",
3208                Answer::ok(fake::microvm_response("RUNNING", None)),
3209            )
3210            .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3211            .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3212        let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3213            .await
3214            .expect("adopts");
3215        assert_eq!(sandbox.suspended_window(), None, "no request of its own");
3216        sandbox.suspend().await.expect("suspends");
3217
3218        clock.advance(Duration::from_secs(601));
3219        sandbox
3220            .resume()
3221            .await
3222            .expect("past the 600 s window the service reported, the service resumes it");
3223        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3224        assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3225    }
3226
3227    /// A VM adopted while already SUSPENDED has no locally observed suspend time, so the
3228    /// resume goes to the service rather than being refused on a guess.
3229    #[tokio::test]
3230    async fn an_adopted_suspended_vm_resumes_through_the_service() {
3231        let (plane, recorder, _) = adopt_plane();
3232        recorder
3233            .answer(
3234                "GetMicrovm",
3235                Answer::ok(fake::microvm_response("SUSPENDED", None)),
3236            )
3237            .answer(
3238                "GetMicrovm",
3239                Answer::ok(fake::microvm_response("RUNNING", None)),
3240            )
3241            .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3242        let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3243            .await
3244            .expect("adopts");
3245        assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3246        let session = sandbox
3247            .resume()
3248            .await
3249            .expect("the service answers the resume");
3250        assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
3251        assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3252    }
3253
3254    /// **STATE-11 for an adopted VM.** A terminated VM is offered neither a resume nor a
3255    /// suspend, and gets no session.
3256    #[tokio::test]
3257    async fn an_adopted_terminated_vm_refuses_every_transition_locally() {
3258        let (mut sandbox, recorder, _) = adopted_in("TERMINATED").await;
3259        assert!(sandbox.was_terminated());
3260        assert!(sandbox.session().is_none());
3261        let resume = sandbox.resume().await.expect_err("STATE-11");
3262        assert!(resume.to_string().contains("STATE-11"), "{resume}");
3263        let suspend = sandbox.suspend().await.expect_err("STATE-5");
3264        assert!(suspend.to_string().contains("STATE-5"), "{suspend}");
3265        assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
3266    }
3267
3268    /// A sandbox adopted in `state` whose plane and session are scripted: `GetMicrovm` answers
3269    /// `state` and then RUNNING, and the daemon answers every health poll.
3270    async fn adopted_with_a_daemon(
3271        state: &str,
3272    ) -> (
3273        Sandbox,
3274        Arc<FakeControlPlane>,
3275        Arc<crate::testing::HealthyDaemon>,
3276    ) {
3277        let recorder = Arc::new(FakeControlPlane::new());
3278        let daemon = crate::testing::HealthyDaemon::new();
3279        let plane = crate::control::ControlPlane::from_ports(
3280            Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3281            Region::US_EAST_1,
3282            Arc::new(TestClock::new()),
3283            Arc::new(crate::testing::SequenceEntropy::new()),
3284            Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3285        );
3286        recorder
3287            .answer(
3288                "GetMicrovm",
3289                Answer::ok(fake::microvm_response(state, None)),
3290            )
3291            .answer(
3292                "GetMicrovm",
3293                Answer::ok(fake::microvm_response("RUNNING", None)),
3294            )
3295            .answer(
3296                "CreateMicrovmAuthToken",
3297                Answer::ok(fake::auth_token_response("proxy-token")),
3298            );
3299        let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3300            .await
3301            .expect("adopts");
3302        (sandbox, recorder, daemon)
3303    }
3304
3305    /// **`wait_until_ready` finishes whichever wait is left (#269):** from an adopted PENDING VM,
3306    /// RUNNING and then the daemon; from an adopted RUNNING one, the daemon alone, with no
3307    /// further control-plane read.
3308    ///
3309    /// **Falsification**: drop the RUNNING branch from `Sandbox::wait_until_ready`, and the
3310    /// RUNNING VM is refused as not PENDING.
3311    #[tokio::test(start_paused = true)]
3312    async fn wait_until_ready_finishes_whichever_wait_is_left() {
3313        let (mut pending, recorder, daemon) = adopted_with_a_daemon("PENDING").await;
3314        pending
3315            .wait_until_ready(Duration::from_secs(60))
3316            .await
3317            .expect("RUNNING, then the daemon");
3318        assert_eq!(pending.lifecycle(), Lifecycle::Running);
3319        assert_eq!(pending.bootstrap_count(), 1);
3320        assert_eq!(daemon.polls(), 1);
3321        assert_eq!(
3322            recorder.call_count("GetMicrovm"),
3323            2,
3324            "the adopt's read and the wait's"
3325        );
3326
3327        let (mut running, recorder, daemon) = adopted_with_a_daemon("RUNNING").await;
3328        running
3329            .wait_until_ready(Duration::from_secs(60))
3330            .await
3331            .expect("the daemon answers");
3332        assert_eq!(daemon.polls(), 1);
3333        assert_eq!(
3334            recorder.call_count("GetMicrovm"),
3335            1,
3336            "RUNNING needs no second read"
3337        );
3338    }
3339
3340    /// A VM that isn't on its way to answering is refused, with no daemon poll.
3341    #[tokio::test]
3342    async fn wait_until_ready_refuses_a_suspended_vm() {
3343        let (mut suspended, _, daemon) = adopted_with_a_daemon("SUSPENDED").await;
3344        let error = suspended
3345            .wait_until_ready(Duration::from_secs(60))
3346            .await
3347            .expect_err("a suspended VM isn't starting");
3348        assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
3349        assert!(error.to_string().contains("SUSPENDED"), "{error}");
3350        assert_eq!(daemon.polls(), 0);
3351    }
3352
3353    /// A VM adopted while PENDING finishes through `wait_until_running`, which counts the
3354    /// bootstrap exactly once and, like a launch, waits for the daemon to answer.
3355    #[tokio::test(start_paused = true)]
3356    async fn an_adopted_pending_vm_counts_its_bootstrap_once_on_running() {
3357        let recorder = Arc::new(FakeControlPlane::new());
3358        let daemon = crate::testing::HealthyDaemon::new();
3359        let plane = crate::control::ControlPlane::from_ports(
3360            Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3361            Region::US_EAST_1,
3362            Arc::new(TestClock::new()),
3363            Arc::new(crate::testing::SequenceEntropy::new()),
3364            Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3365        );
3366        recorder
3367            .answer(
3368                "GetMicrovm",
3369                Answer::ok(fake::microvm_response("PENDING", None)),
3370            )
3371            .answer(
3372                "GetMicrovm",
3373                Answer::ok(fake::microvm_response("RUNNING", None)),
3374            )
3375            .answer(
3376                "CreateMicrovmAuthToken",
3377                Answer::ok(fake::auth_token_response("proxy-token")),
3378            );
3379        let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3380            .await
3381            .expect("adopts");
3382        assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3383        assert_eq!(sandbox.bootstrap_count(), 0);
3384        sandbox
3385            .wait_until_running(Duration::from_secs(60))
3386            .await
3387            .expect("reaches RUNNING");
3388        assert_eq!(sandbox.bootstrap_count(), 1);
3389        assert!(sandbox.token_installed());
3390        assert_eq!(
3391            daemon.polls(),
3392            1,
3393            "the daemon answered before the session came back"
3394        );
3395        assert!(
3396            sandbox
3397                .run(RunRequest::new().with_image("arn:image"))
3398                .await
3399                .is_err(),
3400            "still one bootstrap per VM"
3401        );
3402        assert_eq!(recorder.call_count("RunMicrovm"), 0);
3403    }
3404
3405    /// The refusals: an empty token before any call, a mismatched endpoint, and a state
3406    /// this client does not know. None of them, and no `Debug`, prints the token.
3407    ///
3408    /// **Falsification** — 2026-09-24. Remove the endpoint comparison and the mismatch case
3409    /// adopts; remove the empty-token check and that case reaches `GetMicrovm`; print the
3410    /// held token in `Debug` and the canary assertion fails. Each restored after.
3411    #[tokio::test]
3412    async fn adopt_refuses_bad_records_and_never_prints_the_token() {
3413        let (plane, recorder, _) = adopt_plane();
3414        let empty = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, "")
3415            .await
3416            .expect_err("no token");
3417        assert_eq!(empty.kind(), ErrorKind::InvalidArg);
3418        assert_eq!(recorder.calls().len(), 0, "refused before any call");
3419
3420        let (plane, recorder, _) = adopt_plane();
3421        recorder.answer(
3422            "GetMicrovm",
3423            Answer::ok(fake::microvm_response("RUNNING", None)),
3424        );
3425        let mismatch = Sandbox::adopt(plane, "mvm-abc123", "https://other.example", ADOPT_TOKEN)
3426            .await
3427            .expect_err("the endpoint belongs to another record");
3428        assert_eq!(mismatch.kind(), ErrorKind::InvalidArg);
3429        assert!(!mismatch.to_string().contains(ADOPT_TOKEN));
3430
3431        let (plane, recorder, _) = adopt_plane();
3432        recorder.answer(
3433            "GetMicrovm",
3434            Answer::ok(fake::microvm_response("REBOOTING", None)),
3435        );
3436        let unknown = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3437            .await
3438            .expect_err("an unknown state");
3439        assert_eq!(unknown.kind(), ErrorKind::Platform);
3440        assert!(!unknown.to_string().contains(ADOPT_TOKEN));
3441
3442        let (sandbox, _, _) = adopted_in("RUNNING").await;
3443        let printed = format!("{sandbox:?}");
3444        assert!(!printed.contains(ADOPT_TOKEN), "{printed}");
3445        assert!(printed.contains("adopted: true"), "{printed}");
3446    }
3447
3448    /// **STATE-1 and STATE-2.** A launch accepted is PENDING with the image recorded as
3449    /// existing; the platform reporting the run hook succeeded is what marks the token
3450    /// installed.
3451    ///
3452    /// The two halves are asserted with the *same* fake because they are the same call, and
3453    /// what distinguishes them is which fact is recorded when: `image_exists` is true from
3454    /// the accepted launch, `token_installed` only from the RUNNING report. A client that
3455    /// set both at the launch call would pass an end-state assertion and be wrong about a
3456    /// VM that died during startup — which is the next test.
3457    #[tokio::test(start_paused = true)]
3458    async fn a_launch_records_the_image_and_the_running_report_installs_the_token() {
3459        let (mut sandbox, recorder, _) = planted();
3460        assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3461        assert!(
3462            !sandbox.image_exists(),
3463            "nothing is recorded before a launch"
3464        );
3465        assert!(!sandbox.token_installed());
3466
3467        answer_launch(&recorder);
3468        let session = sandbox
3469            .run(RunRequest::new().with_image("arn:image"))
3470            .await
3471            .expect("launches");
3472        assert_eq!(
3473            session.endpoint(),
3474            "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
3475        );
3476
3477        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3478        assert!(sandbox.image_exists(), "STATE-1: the image is recorded");
3479        assert!(sandbox.token_installed(), "STATE-2: the token is installed");
3480        assert_eq!(sandbox.bootstrap_count(), 1);
3481        assert_eq!(sandbox.microvm().expect("launched").id, "mvm-abc123");
3482
3483        // The payload really carried the token, read off the wire member rather than from
3484        // the request type — a field on a struct proves nothing about what was emitted.
3485        let body = recorder.first_body("RunMicrovm");
3486        let payload = body["runHookPayload"].as_str().expect("a string");
3487        assert!(payload.starts_with(r#"{"agent_token":"#), "{payload}");
3488        assert_eq!(body["idlePolicy"]["suspendedDurationSeconds"], 600);
3489    }
3490
3491    /// A launch env reaches the wire member the daemon parses, and it is the only thing
3492    /// that changes about the request.
3493    ///
3494    /// Read off `runHookPayload` rather than off the request struct: a field on a struct
3495    /// proves nothing about what was emitted, which is the same argument the test above
3496    /// makes about the token.
3497    #[tokio::test(start_paused = true)]
3498    async fn a_launch_env_reaches_the_run_hook_payload_on_the_wire() {
3499        let (mut sandbox, recorder, _) = planted();
3500        answer_launch(&recorder);
3501        sandbox
3502            .run(
3503                RunRequest::new()
3504                    .with_image("arn:image")
3505                    .with_launch_env("ANTHROPIC_BASE_URL", "https://gateway.example")
3506                    .with_launch_env("PATH", "/usr/local/bin:/usr/bin:/bin"),
3507            )
3508            .await
3509            .expect("launches");
3510
3511        let body = recorder.first_body("RunMicrovm");
3512        let payload = body["runHookPayload"].as_str().expect("a string");
3513        // One parse deeper, because that is where the daemon reads it from too.
3514        let inner: serde_json::Value =
3515            serde_json::from_str(payload).expect("the payload is itself JSON");
3516        assert!(
3517            inner["agent_token"].as_str().is_some_and(|t| !t.is_empty()),
3518            "the token still rides alongside: {payload}"
3519        );
3520        assert_eq!(
3521            inner["env"]["ANTHROPIC_BASE_URL"],
3522            "https://gateway.example"
3523        );
3524        assert_eq!(inner["env"]["PATH"], "/usr/local/bin:/usr/bin:/bin");
3525    }
3526
3527    /// **The advisory deny reaches the guest's environment, and it changes nothing else
3528    /// about the request.**
3529    ///
3530    /// Read off `runHookPayload` for the reason the launch-env test gives, and the connector
3531    /// member is asserted absent in the same body: `deny_egress` must not quietly ask the
3532    /// platform for anything, because the whole claim is that the platform has nothing to
3533    /// ask for.
3534    ///
3535    /// **Falsification** — 2026-09-13. Make `run` build the payload from
3536    /// `request.launch_env` instead of `request.effective_launch_env()` and the two
3537    /// `https_proxy` assertions go red while every other launch test stays green; restored
3538    /// after.
3539    #[tokio::test(start_paused = true)]
3540    async fn the_advisory_deny_reaches_the_launch_environment_in_both_spellings() {
3541        let (mut sandbox, recorder, _) = planted();
3542        answer_launch(&recorder);
3543        let request = RunRequest::new().with_image("arn:image").with_deny_egress();
3544        assert_eq!(
3545            request.egress_posture(),
3546            crate::control::EgressPosture::BestEffort
3547        );
3548        sandbox.run(request).await.expect("launches");
3549
3550        let body = recorder.first_body("RunMicrovm");
3551        let inner: serde_json::Value =
3552            serde_json::from_str(body["runHookPayload"].as_str().expect("a string"))
3553                .expect("the payload is itself JSON");
3554        for key in crate::sandbox::DENY_EGRESS_ENV_KEYS {
3555            assert_eq!(
3556                inner["env"][key], DENY_EGRESS_PROXY_URL,
3557                "{key} must point at the black hole: {inner}"
3558            );
3559        }
3560        assert!(
3561            body.get("egressNetworkConnectors").is_none(),
3562            "the advisory deny asks the platform for nothing: {body}"
3563        );
3564    }
3565
3566    /// A caller's own proxy value survives the deny, and a launch that never asked for the
3567    /// deny carries none of the keys.
3568    #[test]
3569    fn the_deny_never_overwrites_a_callers_own_proxy_and_is_absent_by_default() {
3570        let kept = RunRequest::new()
3571            .with_deny_egress()
3572            .with_launch_env("https_proxy", "http://proxy.internal:3128")
3573            .effective_launch_env();
3574        assert_eq!(kept["https_proxy"], "http://proxy.internal:3128");
3575        assert_eq!(kept["http_proxy"], DENY_EGRESS_PROXY_URL);
3576
3577        let plain = RunRequest::new().effective_launch_env();
3578        assert!(
3579            plain.is_empty(),
3580            "a launch that did not ask for the deny sends byte-for-byte what it always sent: \
3581             {plain:?}"
3582        );
3583        assert_eq!(
3584            RunRequest::new().egress_posture(),
3585            crate::control::EgressPosture::Unsealed,
3586            "and its posture is the measured one, not a seal"
3587        );
3588    }
3589
3590    /// **`egress` and `deny_egress` together are refused locally, with zero calls.**
3591    ///
3592    /// The same shape as the over-budget refusal below: a launch carrying both would report
3593    /// `open` while the workload's own tools failed closed, and AWS has no opinion to
3594    /// return about it.
3595    ///
3596    /// **Falsification** — 2026-09-13. Delete the refusal in `run` and this test fails on
3597    /// `expect_err`; the launch then succeeds and reports posture `open`. Restored after.
3598    #[tokio::test]
3599    async fn asking_for_egress_and_the_deny_together_is_refused_before_any_call() {
3600        let (mut sandbox, recorder, _) = planted();
3601        answer_launch(&recorder);
3602        let mut request = RunRequest::new().with_image("arn:image").with_egress();
3603        request.deny_egress = true;
3604
3605        let error = sandbox
3606            .run(request)
3607            .await
3608            .expect_err("opposite intents are not resolved by a precedence rule");
3609        assert_eq!(error.kind(), ErrorKind::InvalidArg);
3610        assert!(error.to_string().contains("opposite things"), "{error}");
3611        assert_eq!(
3612            recorder.calls().len(),
3613            0,
3614            "a locally-refused launch costs nothing"
3615        );
3616    }
3617
3618    /// **A pinned `imageVersion` reaches the wire through the sandbox**, and an unpinned
3619    /// launch omits the member entirely.
3620    ///
3621    /// This is the layer a caller actually uses, so the forwarding is what needs asserting:
3622    /// [`RunRequest::image_version`] is a field on a struct and proves nothing about the
3623    /// emitted body until it is read off the wire member.
3624    ///
3625    /// **Falsification** — run 2026-08-16. Drop `wire.image_version = ...` from `run` and the
3626    /// pinned assertion goes red with the member absent, while every other launch test still
3627    /// passes — which is why this one is here.
3628    #[tokio::test(start_paused = true)]
3629    async fn a_pinned_image_version_is_forwarded_to_the_launch_request() {
3630        let (mut sandbox, recorder, _) = planted();
3631        answer_launch(&recorder);
3632        sandbox
3633            .run(
3634                RunRequest::new()
3635                    .with_image("arn:image")
3636                    .with_image_version("2.0"),
3637            )
3638            .await
3639            .expect("launches");
3640        assert_eq!(
3641            recorder.first_body("RunMicrovm")["imageVersion"],
3642            "2.0",
3643            "a canary has to launch against the version it means to test"
3644        );
3645
3646        let (mut sandbox, recorder, _) = planted();
3647        answer_launch(&recorder);
3648        sandbox
3649            .run(RunRequest::new().with_image("arn:image"))
3650            .await
3651            .expect("launches");
3652        assert!(
3653            recorder
3654                .first_body("RunMicrovm")
3655                .get("imageVersion")
3656                .is_none(),
3657            "an unpinned launch sends what this client always sent"
3658        );
3659    }
3660
3661    /// **The local refusal, with zero control-plane calls.**
3662    ///
3663    /// The observable difference between this client and one that lets AWS answer, and the
3664    /// same shape as STATE-5's suspend refusal: an over-budget launch env fails before the
3665    /// launch rather than as a `ValidationException` on a member the caller did not know
3666    /// they were filling. botocore does not enforce the ceiling client-side, so without
3667    /// this there is no local signal at all.
3668    ///
3669    /// **Falsification** — build the payload after `run_microvm` instead of before, and the
3670    /// call count assertion goes red.
3671    #[tokio::test]
3672    async fn an_over_budget_launch_env_is_refused_before_any_control_plane_call() {
3673        let (mut sandbox, recorder, _) = planted();
3674        answer_launch(&recorder);
3675
3676        let error = sandbox
3677            .run(
3678                RunRequest::new()
3679                    .with_image("arn:image")
3680                    .with_launch_env("AWS_SESSION_TOKEN", "t".repeat(4096)),
3681            )
3682            .await
3683            .expect_err("a credential-scale launch env does not fit the payload");
3684
3685        assert_eq!(error.kind(), ErrorKind::InvalidArg);
3686        let message = error.to_string();
3687        assert!(message.contains("ceiling of 4096"), "{message}");
3688        assert!(message.contains("launch env contributed"), "{message}");
3689        assert_eq!(
3690            recorder.calls().len(),
3691            0,
3692            "the refusal has to be local: a launch that reached AWS costs a VM and a bill"
3693        );
3694        // And the sandbox is left usable rather than half-launched, so a caller can trim
3695        // the env and retry through the same handle.
3696        assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3697        assert_eq!(sandbox.bootstrap_count(), 0);
3698        assert!(sandbox.microvm().is_none());
3699    }
3700
3701    /// **Issue #24's guards reach through `Sandbox`, which is the layer both bindings use.**
3702    ///
3703    /// `microvms-py`'s `run` and `microvms-js`'s `run` both build a [`RunRequest`] and hand it to
3704    /// [`Sandbox::run`], which fills a `RunMicrovmRequest` and calls
3705    /// [`crate::control::ControlPlane::run_microvm`]. So the guards there cover the bindings — but
3706    /// "cover" is a claim about a call chain, and a chain is what a refactor breaks. This asserts
3707    /// it at the layer the bindings actually enter, so a `Sandbox::run` that stopped delegating
3708    /// (or a binding-side default that bypassed the request type) fails a test rather than shipping
3709    /// an unguarded surface to Python and Node.
3710    ///
3711    /// `max_idle_sec` is the case that matters most here, because it is a plain `u32` keyword
3712    /// argument on both bindings with no type narrowing it: `sandbox.run(max_idle_sec=59)` is what
3713    /// a Python caller writes, and before this it reached the wire.
3714    ///
3715    /// **Guard proof.** Delete `require_idle_duration` from `run_microvm` and the idle row goes red
3716    /// on the call count; delete `require_valid_role_arn` and the role row does. Neither guard is in
3717    /// this file, which is the point — the test is about the chain.
3718    #[tokio::test]
3719    async fn the_bindings_layer_refuses_an_illegal_launch_with_zero_calls() {
3720        let cases: [(&str, RunRequest, &str); 3] = [
3721            (
3722                "max_idle_sec=59, which is what a Python keyword argument passes",
3723                RunRequest {
3724                    max_idle_sec: 59,
3725                    ..RunRequest::new().with_image("arn:image")
3726                },
3727                "maxIdleDurationSeconds",
3728            ),
3729            (
3730                "an execution role that is a bare name",
3731                RunRequest {
3732                    execution_role_arn: Some("execution-role".to_string()),
3733                    ..RunRequest::new().with_image("arn:image")
3734                },
3735                "role *name*",
3736            ),
3737            (
3738                "a pinned version with a trailing newline",
3739                RunRequest::new()
3740                    .with_image("arn:image")
3741                    .with_image_version("2.0\n"),
3742                "contains whitespace",
3743            ),
3744        ];
3745
3746        for (label, request, expected) in cases {
3747            let (mut sandbox, recorder, _) = planted();
3748            // The launch answered, so a missing guard is a *successful* launch — a real VM and a
3749            // real bill — rather than a different failure.
3750            answer_launch(&recorder);
3751
3752            // `expect_err` needs the Ok type to be `Debug`, and `run` answers `&mut Session`
3753            // (which is not) — so the discriminant is matched instead. That is also the more
3754            // honest assertion: it says the launch produced no session at all.
3755            let Err(error) = sandbox.run(request).await else {
3756                panic!("{label} must be refused");
3757            };
3758            assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
3759            assert!(
3760                error.to_string().contains(expected),
3761                "{label}: wanted {expected:?}, got {error}"
3762            );
3763            assert_eq!(
3764                recorder.calls().len(),
3765                0,
3766                "{label}: the refusal has to be local, because a launch that reached AWS costs a \
3767                 VM and a bill"
3768            );
3769            assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3770            assert!(sandbox.microvm().is_none());
3771        }
3772    }
3773
3774    /// A launch whose VM dies during startup leaves the token **not** installed, because
3775    /// the run hook is what delivers it and a dead VM ran no hook.
3776    ///
3777    /// The distinction this pins is the one the test above cannot: both a correct client and
3778    /// one that installs the token at the launch call reach RUNNING with the token set, and
3779    /// only this case separates them.
3780    #[tokio::test]
3781    async fn a_launch_that_dies_during_startup_installs_no_token() {
3782        let (mut sandbox, recorder, _) = planted();
3783        recorder
3784            .answer(
3785                "RunMicrovm",
3786                Answer::ok(fake::microvm_response("PENDING", None)),
3787            )
3788            .answer(
3789                "GetMicrovm",
3790                Answer::ok(fake::microvm_response(
3791                    "TERMINATED",
3792                    Some("run hook returned 500"),
3793                )),
3794            );
3795
3796        let error = sandbox
3797            .run(RunRequest::new().with_image("arn:image"))
3798            .await
3799            .expect_err("a VM that died during startup is not a launch");
3800        assert_eq!(error.kind(), ErrorKind::LaunchDied);
3801        assert!(
3802            error.to_string().contains("run hook returned 500"),
3803            "{error}"
3804        );
3805
3806        assert!(
3807            !sandbox.token_installed(),
3808            "the run hook never answered, so nothing was installed"
3809        );
3810        assert_eq!(
3811            sandbox.bootstrap_count(),
3812            0,
3813            "a bootstrap counted here would make STATE-3's ceiling unreachable for a retry"
3814        );
3815        assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3816        assert!(
3817            sandbox.image_exists(),
3818            "STATE-1 still holds: the launch was accepted against a real image"
3819        );
3820    }
3821
3822    /// **STATE-3, the guard proof.** A second `run` on one sandbox is refused, and it is
3823    /// refused with **zero** further control-plane calls.
3824    ///
3825    /// The call count is the assertion that matters. A client that let the second launch
3826    /// through would create a second VM and silently address two guests through one handle,
3827    /// and it would also deliver a second run-hook payload to a daemon whose one-shot
3828    /// bootstrap answers 409 — so "an error came back" is not enough to tell the two apart.
3829    ///
3830    /// **Falsification** — delete the `bootstrap_count > 0` branch from `run` and this test
3831    /// is red on both the count and the call total. Verified; see the packet's guard proofs.
3832    #[tokio::test(start_paused = true)]
3833    async fn a_second_run_on_one_sandbox_is_refused_before_any_call() {
3834        let (mut sandbox, recorder, _) = launched().await;
3835        assert_eq!(sandbox.bootstrap_count(), 1);
3836        let before = recorder.calls().len();
3837
3838        let error = sandbox
3839            .run(RunRequest::new().with_image("arn:image"))
3840            .await
3841            .expect_err("the token is installed at most once per VM lifetime");
3842        assert_eq!(error.kind(), ErrorKind::InvalidArg);
3843        assert!(error.to_string().contains("STATE-3"), "{error}");
3844        assert!(
3845            error.to_string().contains("needs a second Sandbox"),
3846            "the remedy has to be nameable: {error}"
3847        );
3848
3849        assert_eq!(
3850            sandbox.bootstrap_count(),
3851            1,
3852            "bootstrap_count <= 1 is the Z3-proved invariant"
3853        );
3854        assert_eq!(
3855            recorder.calls().len(),
3856            before,
3857            "the refusal must cost no control-plane call"
3858        );
3859        assert_eq!(recorder.call_count("RunMicrovm"), 1);
3860    }
3861
3862    /// **STATE-4 and STATE-6.** A suspend from RUNNING moves to SUSPENDING before the wait,
3863    /// and the platform reporting SUSPENDED is what moves it to SUSPENDED.
3864    #[tokio::test(start_paused = true)]
3865    async fn a_suspend_from_running_passes_through_suspending_to_suspended() {
3866        let (sandbox, recorder, _) = suspended().await;
3867        assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3868        assert!(
3869            sandbox.token_installed(),
3870            "a freeze keeps the guest's memory, so the token survives"
3871        );
3872        assert_eq!(sandbox.bootstrap_count(), 1, "no re-bootstrap on a freeze");
3873
3874        let calls = recorder.calls();
3875        let suspend = calls
3876            .iter()
3877            .find(|call| call.operation == "SuspendMicrovm")
3878            .expect("the suspend went out");
3879        assert_eq!(suspend.method, Method::Post);
3880        assert_eq!(suspend.path, "/2025-09-09/microvms/mvm-abc123/suspend");
3881    }
3882
3883    /// A suspend whose VM dies while suspending reports TERMINATED rather than raising, and
3884    /// records it — which is what stops a later resume.
3885    ///
3886    /// TERMINATED is *wanted* by the suspend wait for a reason: a VM that dies mid-suspend
3887    /// is a state to report, not an exception out of the middle of a teardown.
3888    #[tokio::test(start_paused = true)]
3889    async fn a_vm_that_dies_while_suspending_is_recorded_rather_than_raised() {
3890        let (mut sandbox, recorder, _) = launched().await;
3891        recorder
3892            .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3893            .answer(
3894                "GetMicrovm",
3895                Answer::ok(fake::microvm_response("TERMINATED", Some("idle policy"))),
3896            );
3897
3898        sandbox
3899            .suspend()
3900            .await
3901            .expect("a death mid-suspend is a state, not an error");
3902        assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3903        assert!(
3904            sandbox.was_terminated(),
3905            "STATE-11's precondition is recorded"
3906        );
3907    }
3908
3909    /// A suspend whose wire call fails leaves the sandbox in RUNNING, and a retry works.
3910    ///
3911    /// STATE-4's "accepted" is the wire call succeeding. A client that moved to SUSPENDING
3912    /// before the call would strand a failed call there — a state neither suspend nor
3913    /// resume accepts, so one throttled or dropped request would brick the handle.
3914    /// **Falsification** — move `self.lifecycle = Lifecycle::Suspending` back above the
3915    /// `control.suspend` call and the first assertion here reads SUSPENDING, and the retry
3916    /// is refused by the STATE-5 guard.
3917    #[tokio::test(start_paused = true)]
3918    async fn a_suspend_whose_call_fails_stays_running_and_can_be_retried() {
3919        let (mut sandbox, recorder, _) = planted();
3920        recorder
3921            .answer(
3922                "RunMicrovm",
3923                Answer::ok(fake::microvm_response("PENDING", None)),
3924            )
3925            .answer(
3926                "GetMicrovm",
3927                Answer::ok(fake::microvm_response("RUNNING", None)),
3928            )
3929            .answer(
3930                "GetMicrovm",
3931                Answer::ok(fake::microvm_response("SUSPENDED", None)),
3932            )
3933            .answer(
3934                "CreateMicrovmAuthToken",
3935                Answer::ok(fake::auth_token_response("proxy-token")),
3936            )
3937            // A 409 rather than a 429 or a transport cut, because those two are retried
3938            // inside `send_with_retry` and this test wants the call to fail once, fast.
3939            .answer("SuspendMicrovm", Answer::failure(409, "ConflictException"))
3940            .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
3941
3942        sandbox
3943            .run(RunRequest::new().with_image("arn:image"))
3944            .await
3945            .expect("launches");
3946
3947        sandbox
3948            .suspend()
3949            .await
3950            .expect_err("the control plane refused the suspend");
3951        assert_eq!(
3952            sandbox.lifecycle(),
3953            Lifecycle::Running,
3954            "a refused suspend must not move the lifecycle: SUSPENDING accepts neither a \
3955             suspend nor a resume, so recording it here would brick the handle"
3956        );
3957
3958        sandbox.suspend().await.expect("the retry suspends");
3959        assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3960        assert_eq!(recorder.call_count("SuspendMicrovm"), 2);
3961    }
3962
3963    /// **STATE-5, the guard proof.** A suspend from anything but RUNNING is refused with
3964    /// **zero** control-plane calls.
3965    ///
3966    /// Every non-RUNNING state, so a state added to `Lifecycle` later is not silently
3967    /// exempt. The count is the load-bearing assertion: letting the call go and reading the
3968    /// service's answer also produces an error, and that error is about a non-running id
3969    /// rather than about which of two things the caller got wrong.
3970    ///
3971    /// **Falsification** — delete the `lifecycle != Running` branch from `suspend` and the
3972    /// SUSPENDED case emits a `SuspendMicrovm` call, turning the count assertion red.
3973    /// Verified; see the packet's guard proofs.
3974    #[tokio::test(start_paused = true)]
3975    async fn a_suspend_from_a_non_running_state_reaches_no_control_plane_call() {
3976        // From SUSPENDED — the caller who believes they resumed.
3977        let (mut sandbox, recorder, _) = suspended().await;
3978        let before = recorder.call_count("SuspendMicrovm");
3979
3980        let error = sandbox
3981            .suspend()
3982            .await
3983            .expect_err("a suspend is only issued from RUNNING");
3984        assert_eq!(error.kind(), ErrorKind::InvalidArg);
3985        assert!(error.to_string().contains("STATE-5"), "{error}");
3986        assert!(error.to_string().contains("SUSPENDED"), "{error}");
3987        assert_eq!(
3988            recorder.call_count("SuspendMicrovm"),
3989            before,
3990            "the refusal must be local: no second suspend went to the wire"
3991        );
3992
3993        // And from PENDING, before anything launched at all.
3994        let (mut fresh, fresh_recorder, _) = planted();
3995        let error = fresh.suspend().await.expect_err("nothing to suspend");
3996        assert_eq!(error.kind(), ErrorKind::Precondition);
3997        assert_eq!(fresh_recorder.calls().len(), 0);
3998    }
3999
4000    /// **STATE-7 and STATE-8.** A resume from SUSPENDED returns to RUNNING, re-delivers
4001    /// **nothing**, and drops the cached proxy token.
4002    ///
4003    /// Three assertions, and each one is a different failure mode. The bootstrap count says
4004    /// no run-hook payload was re-delivered — which a daemon's one-shot bootstrap would
4005    /// answer 409 to, reading like a broken VM. The `RunMicrovm` count says no second launch
4006    /// happened. And `is_cached` says the token was invalidated, which is the only
4007    /// observable difference on a path where the endpoint URL does not change: a client that
4008    /// kept the token would produce identical requests until the pre-suspend token stopped
4009    /// validating, and that rejection reads exactly like a dead daemon.
4010    #[tokio::test(start_paused = true)]
4011    async fn a_resume_reuses_the_token_redelivers_nothing_and_drops_the_proxy_token() {
4012        let (mut sandbox, recorder, _) = suspended().await;
4013
4014        // The session mints once before the suspend, so there is a cached token to drop.
4015        let auth = Arc::clone(
4016            sandbox
4017                .session()
4018                .expect("launched")
4019                .proxy_auth()
4020                .expect("the launch wired a minter"),
4021        );
4022        auth.headers().await.expect("mints");
4023        assert!(
4024            auth.is_cached(),
4025            "there must be a token for the drop to matter"
4026        );
4027        assert_eq!(auth.mint_count(), 1);
4028
4029        recorder
4030            .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4031            .answer(
4032                "GetMicrovm",
4033                Answer::ok(fake::microvm_response("RUNNING", None)),
4034            );
4035        let session = sandbox.resume().await.expect("resumes");
4036        assert_eq!(
4037            session.endpoint(),
4038            "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
4039        );
4040
4041        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4042        assert!(
4043            sandbox.token_installed(),
4044            "STATE-7: the installed token is reused"
4045        );
4046        assert_eq!(
4047            sandbox.bootstrap_count(),
4048            1,
4049            "STATE-7: a resume re-delivers no run-hook payload"
4050        );
4051        assert_eq!(
4052            recorder.call_count("RunMicrovm"),
4053            1,
4054            "a resume is not a second launch"
4055        );
4056        assert!(
4057            !auth.is_cached(),
4058            "STATE-8: the cached proxy token survived the resume"
4059        );
4060    }
4061
4062    /// **STATE-8's guard proof, on the mint count.** The request after a resume mints a
4063    /// fresh token rather than reusing the pre-suspend one.
4064    ///
4065    /// Separate from the test above because `is_cached` is a fact about the cache and this is
4066    /// a fact about the wire: a client that dropped the cache but kept emitting the old
4067    /// header would pass the first assertion and fail every request.
4068    ///
4069    /// **Falsification** — delete the `proxy.invalidate()` line from `Session::rebind` and
4070    /// the mint count stays at 1 here. Verified; see the packet's guard proofs.
4071    #[tokio::test(start_paused = true)]
4072    async fn the_request_after_a_resume_mints_a_fresh_proxy_token() {
4073        let (mut sandbox, recorder, _) = suspended().await;
4074        let auth = Arc::clone(
4075            sandbox
4076                .session()
4077                .expect("launched")
4078                .proxy_auth()
4079                .expect("wired"),
4080        );
4081        auth.headers().await.expect("mints");
4082        assert_eq!(auth.mint_count(), 1);
4083
4084        recorder
4085            .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4086            .answer(
4087                "GetMicrovm",
4088                Answer::ok(fake::microvm_response("RUNNING", None)),
4089            );
4090        sandbox.resume().await.expect("resumes");
4091
4092        // No clock movement at all, so a re-mint here is caused by the invalidation rather
4093        // than by the refresh window rolling over.
4094        auth.headers().await.expect("re-mints");
4095        assert_eq!(
4096            auth.mint_count(),
4097            2,
4098            "STATE-8: the resume did not force a fresh mint"
4099        );
4100    }
4101
4102    /// A keepalive reads this receiver instead of the lock, so every transition must reach
4103    /// it: a suspend it missed would be undone by the keepalive's next poll auto-resuming
4104    /// the VM.
4105    #[tokio::test(start_paused = true)]
4106    async fn the_lifecycle_watch_follows_every_transition() {
4107        let (mut sandbox, recorder, _clock) = planted();
4108        let watch = sandbox.watch_lifecycle();
4109        assert_eq!(*watch.borrow(), Lifecycle::Pending);
4110        recorder
4111            .answer(
4112                "RunMicrovm",
4113                Answer::ok(fake::microvm_response("PENDING", None)),
4114            )
4115            .answer(
4116                "GetMicrovm",
4117                Answer::ok(fake::microvm_response("RUNNING", None)),
4118            )
4119            .answer(
4120                "GetMicrovm",
4121                Answer::ok(fake::microvm_response("SUSPENDED", None)),
4122            )
4123            .answer(
4124                "CreateMicrovmAuthToken",
4125                Answer::ok(fake::auth_token_response("proxy-token")),
4126            )
4127            .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
4128        sandbox
4129            .run(RunRequest::new().with_image("arn:image"))
4130            .await
4131            .expect("launches");
4132        assert_eq!(*watch.borrow(), Lifecycle::Running);
4133        sandbox.suspend().await.expect("suspends");
4134        assert_eq!(*watch.borrow(), Lifecycle::Suspended);
4135        assert_eq!(*watch.borrow(), sandbox.lifecycle());
4136    }
4137
4138    /// `ResumeMicrovm`'s refusal of a VM the service terminated, as it answered on 2026-10-12
4139    /// in us-east-1 (#597): a 400 whose error type is `ValidationException`.
4140    fn terminated_refusal() -> Answer {
4141        Answer {
4142            status: 400,
4143            body: r#"{"message": "The MicroVM mvm-abc123 has been terminated and its state cannot be changed."}"#
4144                .to_string(),
4145            error_type: Some("ValidationException".to_string()),
4146        }
4147    }
4148
4149    /// STATE-7 and STATE-8: idle suspension is learned from the service, then resumed
4150    /// without a second bootstrap. The cached proxy token is dropped as for an explicit
4151    /// suspend, and both the held VM and the lifecycle watch reflect the new state.
4152    #[tokio::test(start_paused = true)]
4153    async fn resume_observes_platform_idle_suspension_and_resumes() {
4154        let (mut sandbox, recorder, _) = planted();
4155        recorder
4156            .answer(
4157                "RunMicrovm",
4158                Answer::ok(fake::microvm_response("PENDING", None)),
4159            )
4160            .answer(
4161                "GetMicrovm",
4162                Answer::ok(fake::microvm_response("RUNNING", None)),
4163            )
4164            .answer(
4165                "GetMicrovm",
4166                Answer::ok(fake::microvm_response("SUSPENDED", None)),
4167            )
4168            .answer(
4169                "GetMicrovm",
4170                Answer::ok(fake::microvm_response("RUNNING", None)),
4171            )
4172            .answer(
4173                "CreateMicrovmAuthToken",
4174                Answer::ok(fake::auth_token_response("proxy-token")),
4175            )
4176            .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4177        sandbox
4178            .run(RunRequest::new().with_image("arn:image"))
4179            .await
4180            .expect("launch");
4181        let token = sandbox
4182            .session()
4183            .expect("session")
4184            .agent_token()
4185            .to_string();
4186        let watch = sandbox.watch_lifecycle();
4187        let calls = recorder.calls().len();
4188        sandbox.resume().await.expect("idle-suspended VM resumes");
4189        assert_eq!(
4190            &recorder.operations()[calls..],
4191            &["GetMicrovm", "ResumeMicrovm", "GetMicrovm"]
4192        );
4193        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4194        assert_eq!(*watch.borrow(), Lifecycle::Running);
4195        assert_eq!(sandbox.microvm().expect("VM").state, "RUNNING");
4196        assert_eq!(sandbox.bootstrap_count(), 1);
4197        assert_eq!(recorder.call_count("RunMicrovm"), 1);
4198        assert_eq!(recorder.call_count("SuspendMicrovm"), 0);
4199        assert_eq!(sandbox.session().expect("session").agent_token(), token);
4200        assert!(
4201            !sandbox
4202                .session()
4203                .expect("session")
4204                .proxy_auth()
4205                .expect("auth")
4206                .is_cached()
4207        );
4208    }
4209
4210    /// STATE-12 and STATE-11: an idle-suspended VM may already have ended before resume.
4211    /// The observation names its state and reason, records termination, and avoids a
4212    /// pointless resume call or a drop warning about a live VM.
4213    #[tokio::test(start_paused = true)]
4214    async fn resume_observes_platform_termination_and_records_window_closed() {
4215        for state in ["TERMINATING", "TERMINATED"] {
4216            let (mut sandbox, recorder, _) = planted();
4217            recorder
4218                .answer(
4219                    "RunMicrovm",
4220                    Answer::ok(fake::microvm_response("PENDING", None)),
4221                )
4222                .answer(
4223                    "GetMicrovm",
4224                    Answer::ok(fake::microvm_response("RUNNING", None)),
4225                )
4226                .answer(
4227                    "GetMicrovm",
4228                    Answer::ok(fake::microvm_response(state, Some("idle window elapsed"))),
4229                )
4230                .answer(
4231                    "CreateMicrovmAuthToken",
4232                    Answer::ok(fake::auth_token_response("proxy-token")),
4233                );
4234            sandbox
4235                .run(RunRequest::new().with_image("arn:image"))
4236                .await
4237                .expect("launch");
4238            let watch = sandbox.watch_lifecycle();
4239            let error = sandbox.resume().await.expect_err("the VM ended");
4240            assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}");
4241            assert!(error.to_string().contains(state), "{error}");
4242            assert!(error.to_string().contains("idle window elapsed"), "{error}");
4243            assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4244            assert_eq!(*watch.borrow(), Lifecycle::Terminated);
4245            assert_eq!(sandbox.microvm().expect("VM").state, state);
4246            assert!(sandbox.was_terminated());
4247            assert_eq!(sandbox.drop_warning(), None);
4248            let calls = recorder.calls().len();
4249            assert_eq!(
4250                sandbox.resume().await.expect_err("STATE-11").kind(),
4251                ErrorKind::InvalidArg
4252            );
4253            assert_eq!(
4254                recorder.calls().len(),
4255                calls,
4256                "known termination stays local"
4257            );
4258            assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4259        }
4260    }
4261
4262    /// STATE-7: a service that still reports RUNNING must not receive ResumeMicrovm.
4263    /// A failed observation also leaves the handle available for a later retry.
4264    #[tokio::test(start_paused = true)]
4265    async fn resume_observes_running_and_propagates_observation_failures() {
4266        for (response, expected) in [
4267            (
4268                Answer::ok(fake::microvm_response("RUNNING", None)),
4269                ErrorKind::InvalidArg,
4270            ),
4271            (
4272                Answer::ok(fake::microvm_response("UNKNOWN", None)),
4273                ErrorKind::Platform,
4274            ),
4275            (
4276                Answer::failure(403, "access denied"),
4277                ErrorKind::Credentials,
4278            ),
4279        ] {
4280            let (mut sandbox, recorder, _) = planted();
4281            recorder
4282                .answer(
4283                    "RunMicrovm",
4284                    Answer::ok(fake::microvm_response("PENDING", None)),
4285                )
4286                .answer(
4287                    "GetMicrovm",
4288                    Answer::ok(fake::microvm_response("RUNNING", None)),
4289                )
4290                .answer("GetMicrovm", response)
4291                .answer(
4292                    "CreateMicrovmAuthToken",
4293                    Answer::ok(fake::auth_token_response("proxy-token")),
4294                );
4295            sandbox
4296                .run(RunRequest::new().with_image("arn:image"))
4297                .await
4298                .expect("launch");
4299            let calls = recorder.calls().len();
4300            let error = sandbox.resume().await.expect_err("cannot resume");
4301            assert_eq!(error.kind(), expected, "{error}");
4302            assert_eq!(&recorder.operations()[calls..], &["GetMicrovm"]);
4303            assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4304            assert!(!sandbox.was_terminated());
4305        }
4306    }
4307
4308    /// A transient service state must not strand a locally RUNNING handle: the next
4309    /// resume still observes the service and can resume once suspension completes.
4310    #[tokio::test(start_paused = true)]
4311    async fn resume_retries_observation_after_transient_service_states() {
4312        for state in ["SUSPENDING", "PENDING", "RESUMING"] {
4313            let (mut sandbox, recorder, _) = planted();
4314            recorder
4315                .answer(
4316                    "RunMicrovm",
4317                    Answer::ok(fake::microvm_response("PENDING", None)),
4318                )
4319                .answer(
4320                    "GetMicrovm",
4321                    Answer::ok(fake::microvm_response("RUNNING", None)),
4322                )
4323                .answer(
4324                    "GetMicrovm",
4325                    Answer::ok(fake::microvm_response(state, None)),
4326                )
4327                .answer(
4328                    "GetMicrovm",
4329                    Answer::ok(fake::microvm_response("SUSPENDED", None)),
4330                )
4331                .answer(
4332                    "GetMicrovm",
4333                    Answer::ok(fake::microvm_response("RUNNING", None)),
4334                )
4335                .answer(
4336                    "CreateMicrovmAuthToken",
4337                    Answer::ok(fake::auth_token_response("proxy-token")),
4338                )
4339                .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4340            sandbox
4341                .run(RunRequest::new().with_image("arn:image"))
4342                .await
4343                .expect("launch");
4344            let error = sandbox.resume().await.expect_err("still changing state");
4345            assert!(error.to_string().contains(state), "{error}");
4346            assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4347            assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4348            sandbox
4349                .resume()
4350                .await
4351                .expect("the next observation can resume");
4352            assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4353            assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4354        }
4355    }
4356
4357    /// **STATE-12, the regression (#597).** A resume 1283 s into a 60 s window, on a sandbox
4358    /// that suspended the VM itself, reaches `ResumeMicrovm` and comes back RUNNING. Those are
4359    /// the numbers measured on 2026-10-12 in us-east-1: the service doesn't hold an explicit
4360    /// suspend to `suspendedDurationSeconds`, so a client that refused here refused a resume
4361    /// the service honors.
4362    #[tokio::test(start_paused = true)]
4363    async fn a_resume_past_the_window_of_an_explicit_suspend_reaches_the_service_and_resumes() {
4364        let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
4365        assert_eq!(sandbox.suspended_window(), Some(Duration::from_secs(60)));
4366        assert_eq!(
4367            sandbox.idle_window(),
4368            Some(Duration::from_secs(600)),
4369            "the launch's own maxIdleDurationSeconds, which a keepalive checks its cadence against"
4370        );
4371
4372        clock.advance(Duration::from_secs(1283));
4373        recorder
4374            .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4375            .answer(
4376                "GetMicrovm",
4377                Answer::ok(fake::microvm_response("RUNNING", None)),
4378            );
4379        sandbox
4380            .resume()
4381            .await
4382            .expect("the service resumes an explicit suspend past its window");
4383
4384        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4385        assert!(!sandbox.was_terminated());
4386        assert_eq!(
4387            recorder.call_count("ResumeMicrovm"),
4388            1,
4389            "the resume has to reach the service"
4390        );
4391    }
4392
4393    /// **STATE-12, a VM the service ended.** The service refuses the resume with the
4394    /// `ValidationException` it gave on 2026-10-12 for a VM the idle policy terminated, and
4395    /// `GetMicrovm` reports TERMINATED. That's `ERR_WINDOW_CLOSED` naming the state and the
4396    /// `stateReason`, and the sandbox records the termination: a second resume is refused
4397    /// with no call (STATE-11), and dropping it warns about no live VM.
4398    #[tokio::test(start_paused = true)]
4399    async fn a_resume_the_service_refuses_for_an_ended_vm_is_window_closed_and_recorded() {
4400        let (mut sandbox, recorder, _clock) = planted();
4401        // Every `GetMicrovm` answer queued up front, in the order the launch wait, the suspend
4402        // wait and the refused resume's read take them; `suspended_with_window` says why.
4403        recorder
4404            .answer(
4405                "RunMicrovm",
4406                Answer::ok(fake::microvm_response("PENDING", None)),
4407            )
4408            .answer(
4409                "GetMicrovm",
4410                Answer::ok(fake::microvm_response("RUNNING", None)),
4411            )
4412            .answer(
4413                "GetMicrovm",
4414                Answer::ok(fake::microvm_response("SUSPENDED", None)),
4415            )
4416            .answer(
4417                "GetMicrovm",
4418                Answer::ok(fake::microvm_response(
4419                    "TERMINATED",
4420                    Some("MicroVM exceeded maximum lifetime."),
4421                )),
4422            )
4423            .answer(
4424                "CreateMicrovmAuthToken",
4425                Answer::ok(fake::auth_token_response("proxy-token")),
4426            )
4427            .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
4428            .answer("ResumeMicrovm", terminated_refusal());
4429        sandbox
4430            .run(
4431                RunRequest::new()
4432                    .with_image("arn:image")
4433                    .with_suspended_sec(60),
4434            )
4435            .await
4436            .expect("launches");
4437        sandbox.suspend().await.expect("suspends");
4438
4439        let error = sandbox
4440            .resume()
4441            .await
4442            .expect_err("the service ended the VM");
4443        assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}");
4444        assert_eq!(error.code(), "ERR_WINDOW_CLOSED");
4445        let message = error.to_string();
4446        assert!(message.contains("TERMINATED"), "the state: {message}");
4447        assert!(
4448            message.contains("MicroVM exceeded maximum lifetime."),
4449            "the stateReason: {message}"
4450        );
4451        assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4452        assert!(sandbox.was_terminated());
4453        assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4454
4455        let again = sandbox.resume().await.expect_err("STATE-11");
4456        assert!(again.to_string().contains("STATE-11"), "{again}");
4457        assert_eq!(
4458            recorder.call_count("ResumeMicrovm"),
4459            1,
4460            "the second resume makes no call"
4461        );
4462        assert_eq!(
4463            sandbox.drop_warning(),
4464            None,
4465            "a VM the service ended bills nothing, so dropping its sandbox isn't a leak"
4466        );
4467    }
4468
4469    /// A resume the service refuses while `GetMicrovm` still reports SUSPENDED is the
4470    /// service's answer, and the sandbox stays SUSPENDED: only a VM the service reports dead
4471    /// is recorded as terminated (STATE-12), so a retry reaches the service and resumes.
4472    #[tokio::test(start_paused = true)]
4473    async fn a_resume_refused_for_a_live_vm_leaves_the_sandbox_suspended_and_resumable() {
4474        let (mut sandbox, recorder, _clock) = suspended().await;
4475        recorder
4476            .answer(
4477                "ResumeMicrovm",
4478                Answer::failure(409, "The MicroVM is changing state."),
4479            )
4480            .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4481
4482        let error = sandbox.resume().await.expect_err("the service refused");
4483        assert_eq!(error.kind(), ErrorKind::Platform, "{error}");
4484        assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
4485        assert!(!sandbox.was_terminated());
4486
4487        recorder.answer(
4488            "GetMicrovm",
4489            Answer::ok(fake::microvm_response("RUNNING", None)),
4490        );
4491        sandbox.resume().await.expect("the retry resumes");
4492        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4493        assert_eq!(recorder.call_count("ResumeMicrovm"), 2);
4494    }
4495
4496    /// **#195 at the sandbox.** A client-token launch that adopts an idle-suspended VM is
4497    /// resumed and reaches RUNNING with the token counted once. The same poll on a launch
4498    /// without a caller token is still a startup death (TRAP-8).
4499    #[tokio::test(start_paused = true)]
4500    async fn a_client_token_launch_resumes_the_vm_it_adopted() {
4501        let (mut sandbox, recorder, _) = planted();
4502        recorder
4503            .answer(
4504                "RunMicrovm",
4505                Answer::ok(fake::microvm_response("SUSPENDED", None)),
4506            )
4507            .answer(
4508                "GetMicrovm",
4509                Answer::ok(fake::microvm_response("SUSPENDED", None)),
4510            )
4511            .answer(
4512                "GetMicrovm",
4513                Answer::ok(fake::microvm_response("RUNNING", None)),
4514            )
4515            .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4516            .answer(
4517                "CreateMicrovmAuthToken",
4518                Answer::ok(fake::auth_token_response("proxy-token")),
4519            );
4520        let mut request = RunRequest::new().with_image("arn:image");
4521        request.client_token = Some("job-195".into());
4522        request.agent_token = Some("agent-token".into());
4523        sandbox
4524            .run(request)
4525            .await
4526            .expect("the adopted VM is resumed");
4527        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4528        assert_eq!(sandbox.bootstrap_count(), 1);
4529        assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4530
4531        let (mut fresh, recorder, _) = planted();
4532        recorder
4533            .answer(
4534                "RunMicrovm",
4535                Answer::ok(fake::microvm_response("PENDING", None)),
4536            )
4537            .answer(
4538                "GetMicrovm",
4539                Answer::ok(fake::microvm_response("SUSPENDED", Some("hook failed"))),
4540            );
4541        let error = fresh
4542            .run(RunRequest::new().with_image("arn:image"))
4543            .await
4544            .expect_err("a fresh launch that reads SUSPENDED died during startup");
4545        assert_eq!(error.kind(), ErrorKind::LaunchDied);
4546        assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4547    }
4548
4549    /// **`wait: false`.** The launch returns once accepted, with an addressable session and
4550    /// the lifecycle still PENDING; `wait_until_running` finishes it, exactly once.
4551    #[tokio::test(start_paused = true)]
4552    async fn a_launch_that_does_not_wait_is_finished_by_wait_until_running() {
4553        let (mut sandbox, recorder, _) = planted();
4554        answer_launch(&recorder);
4555        let mut request = RunRequest::new().with_image("arn:image");
4556        request.wait = false;
4557        let endpoint = sandbox
4558            .run(request)
4559            .await
4560            .expect("accepted")
4561            .endpoint()
4562            .to_string();
4563        assert!(endpoint.starts_with("https://"), "{endpoint}");
4564        assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
4565        assert_eq!(sandbox.bootstrap_count(), 0);
4566        assert_eq!(recorder.call_count("GetMicrovm"), 0, "nothing polled yet");
4567
4568        sandbox
4569            .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4570            .await
4571            .expect("reaches RUNNING");
4572        assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4573        assert_eq!(sandbox.bootstrap_count(), 1);
4574        let error = sandbox
4575            .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4576            .await
4577            .expect_err("a running launch has nothing to wait for");
4578        assert_eq!(error.kind(), ErrorKind::InvalidArg);
4579        assert_eq!(sandbox.bootstrap_count(), 1, "counted once (STATE-3)");
4580    }
4581
4582    /// **STATE-11, the resume-after-terminate case the packet names.** A terminated VM never
4583    /// returns to RUNNING, and the recorder sees **zero** resume wire calls.
4584    ///
4585    /// The zero count is the assertion the packet asks for, and it is the right one: a client
4586    /// that called and read the answer would also fail, with whatever the service says about
4587    /// a terminated id — which is a different statement from "this VM is gone and even a
4588    /// successful call would hand you a different machine".
4589    #[tokio::test(start_paused = true)]
4590    async fn a_resume_after_terminate_records_zero_resume_calls() {
4591        let (mut sandbox, recorder, _) = launched().await;
4592        recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4593        let report = sandbox.terminate(TeardownOpts::default()).await;
4594        assert!(report.terminate_accepted);
4595        assert!(sandbox.was_terminated());
4596
4597        let error = sandbox
4598            .resume()
4599            .await
4600            .expect_err("a terminated VM never returns to RUNNING");
4601        assert_eq!(error.kind(), ErrorKind::InvalidArg);
4602        assert!(error.to_string().contains("STATE-11"), "{error}");
4603        assert_eq!(
4604            recorder.call_count("ResumeMicrovm"),
4605            0,
4606            "no resume may reach the wire once the VM was terminated"
4607        );
4608        assert_ne!(
4609            sandbox.lifecycle(),
4610            Lifecycle::Running,
4611            "the lifecycle must not return to RUNNING"
4612        );
4613    }
4614
4615    /// **STATE-9 and STATE-10.** A terminate accepted is TERMINATING with the VM recorded
4616    /// terminated; the platform reporting termination complete is what marks TERMINATED.
4617    ///
4618    /// The two are separate states rather than one because the default teardown does not
4619    /// wait — so TERMINATING is a state a report really ends in, and calling it TERMINATED
4620    /// would claim an observation nobody made.
4621    #[tokio::test(start_paused = true)]
4622    async fn a_terminate_records_terminating_and_the_completion_report_marks_terminated() {
4623        let (mut sandbox, recorder, _) = launched().await;
4624        recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4625
4626        let report = sandbox.terminate(TeardownOpts::default()).await;
4627        assert!(report.terminate_accepted, "STATE-9: the call was accepted");
4628        assert!(sandbox.was_terminated(), "STATE-9: recorded terminated");
4629        assert_eq!(
4630            report.lifecycle,
4631            Some(Lifecycle::Terminating),
4632            "the default teardown does not wait, so TERMINATED is not claimed"
4633        );
4634        assert!(!report.leaked(), "{report:?}");
4635
4636        // And with the wait asked for, STATE-10 is observed.
4637        let (mut sandbox, recorder, _) = launched().await;
4638        recorder
4639            .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4640            .answer(
4641                "GetMicrovm",
4642                Answer::ok(fake::microvm_response("TERMINATED", None)),
4643            );
4644        let report = sandbox
4645            .terminate(TeardownOpts::default().waiting_for_terminated())
4646            .await;
4647        assert_eq!(report.lifecycle, Some(Lifecycle::Terminated));
4648        assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4649    }
4650
4651    /// The teardown order is VM, then image, then the log group **last** — asserted on the
4652    /// recorder's ledger rather than on the report, because the order is a property of what
4653    /// was emitted.
4654    ///
4655    /// The log group is last because the service can recreate a group deleted before its
4656    /// image. This crate cannot delete it at all, so the assertion is that it is *named
4657    /// after* the image work rather than that a delete call went out — see
4658    /// `TeardownReport::undeleted`.
4659    #[tokio::test(start_paused = true)]
4660    async fn teardown_deletes_the_vm_then_the_image_and_names_the_log_group_last() {
4661        let (mut sandbox, recorder, _) = planted();
4662        recorder
4663            .answer(
4664                "CreateMicrovmImage",
4665                Answer::created(fake::create_image_response("agentd-conformance")),
4666            )
4667            .answer(
4668                "GetMicrovmImage",
4669                Answer::ok(fake::get_image_response("agentd-conformance", "CREATED")),
4670            );
4671        sandbox
4672            .build_image(CreateImageRequest::new(
4673                "agentd-conformance",
4674                crate::testing::aarch64_daemon(b"binary"),
4675                "s3://bucket/img.zip",
4676                "arn:aws:iam::123456789012:role/build",
4677            ))
4678            .await
4679            .expect("builds");
4680        answer_launch(&recorder);
4681        sandbox
4682            .run(RunRequest::new())
4683            .await
4684            .expect("launches from the built image");
4685
4686        recorder
4687            .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4688            .answer(
4689                "ListMicrovmImageVersions",
4690                Answer::ok(fake::list_versions_response("1")),
4691            )
4692            .answer(
4693                "DeleteMicrovmImage",
4694                Answer::ok(fake::delete_image_response()),
4695            );
4696
4697        let report = sandbox
4698            .terminate(
4699                TeardownOpts::default()
4700                    .deleting_image()
4701                    .deleting_log_group(),
4702            )
4703            .await;
4704        assert!(report.terminate_accepted);
4705        assert_eq!(report.image_deleted, Some(true));
4706
4707        // The ledger, filtered to the three destructive operations.
4708        let order: Vec<&str> = recorder
4709            .operations()
4710            .into_iter()
4711            .filter(|operation| matches!(*operation, "TerminateMicrovm" | "DeleteMicrovmImage"))
4712            .collect();
4713        assert_eq!(
4714            order,
4715            ["TerminateMicrovm", "DeleteMicrovmImage"],
4716            "the VM goes before the image: a terminating VM holds a reference to it"
4717        );
4718
4719        // The log group is named, and named last.
4720        assert_eq!(
4721            report.undeleted,
4722            ["/aws/lambda-microvms/agentd-conformance"],
4723            "{report:?}"
4724        );
4725        assert!(report.leaked());
4726        assert!(
4727            report
4728                .failures
4729                .last()
4730                .expect("a failure")
4731                .contains("log group"),
4732            "the log group's failure has to be the last one recorded: {report:?}"
4733        );
4734        assert!(
4735            report
4736                .failures
4737                .last()
4738                .expect("a failure")
4739                .contains("terraform destroy"),
4740            "the reason it leaks has to be named: {report:?}"
4741        );
4742    }
4743
4744    /// An image delete that fails every attempt lands its identifier in the report rather
4745    /// than raising, so the CLI can emit what was left behind (CLI-6).
4746    ///
4747    /// The identifier rather than a flag is the whole point: a leak nobody can name is a leak
4748    /// nobody can clean up.
4749    #[tokio::test(start_paused = true)]
4750    async fn a_failing_image_delete_names_what_it_left_behind() {
4751        let (mut sandbox, recorder, _) = planted();
4752        recorder
4753            .answer(
4754                "CreateMicrovmImage",
4755                Answer::created(fake::create_image_response("img")),
4756            )
4757            .answer(
4758                "GetMicrovmImage",
4759                Answer::ok(fake::get_image_response("img", "CREATED")),
4760            );
4761        sandbox
4762            .build_image(CreateImageRequest::new(
4763                "img",
4764                crate::testing::aarch64_daemon(b"binary"),
4765                "s3://bucket/img.zip",
4766                "arn:aws:iam::123456789012:role/build",
4767            ))
4768            .await
4769            .expect("builds");
4770        answer_launch(&recorder);
4771        sandbox.run(RunRequest::new()).await.expect("launches");
4772
4773        recorder
4774            .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4775            .answer(
4776                "ListMicrovmImageVersions",
4777                Answer::ok(fake::list_versions_response("1")),
4778            )
4779            .answer(
4780                "DeleteMicrovmImage",
4781                Answer::failure(409, "image is in CREATING"),
4782            );
4783
4784        let mut opts = TeardownOpts::default().deleting_image();
4785        opts.delete_attempts = 3;
4786        opts.delete_backoff = Duration::from_secs(1);
4787        let report = sandbox.terminate(opts).await;
4788
4789        assert_eq!(report.image_deleted, Some(false));
4790        assert_eq!(
4791            report.undeleted,
4792            ["arn:aws:lambda:us-east-1:123456789012:microvm-image:img"],
4793            "{report:?}"
4794        );
4795        assert!(
4796            sandbox.image_exists(),
4797            "the image really is still there, so the flag must say so"
4798        );
4799        assert_eq!(
4800            recorder.call_count("DeleteMicrovmImage"),
4801            3,
4802            "every attempt ran"
4803        );
4804    }
4805
4806    /// A terminate whose own call fails still records the VM as one this client asked to
4807    /// destroy, and names it as undeleted.
4808    ///
4809    /// Recording it is what stops a later resume from being offered against a VM the caller
4810    /// already tried to kill (STATE-11) — the alternative leaves the sandbox looking
4811    /// resumable, which is the worse of the two wrong answers.
4812    #[tokio::test(start_paused = true)]
4813    async fn a_terminate_whose_call_fails_still_records_the_intent_and_the_leak() {
4814        let (mut sandbox, recorder, _) = launched().await;
4815        for _ in 0..6 {
4816            recorder.answer(
4817                "TerminateMicrovm",
4818                Answer::failure(500, "InternalServerException"),
4819            );
4820        }
4821        // The failure remains terminal even if the retry budget regresses.
4822        recorder.answer("TerminateMicrovm", Answer::failure(403, "denied"));
4823
4824        let report = sandbox.terminate(TeardownOpts::default()).await;
4825        assert_eq!(recorder.call_count("TerminateMicrovm"), 6);
4826        assert!(!report.terminate_accepted);
4827        assert_eq!(report.undeleted, ["mvm-abc123"], "{report:?}");
4828        assert!(sandbox.was_terminated());
4829        assert_eq!(report.lifecycle, Some(Lifecycle::Terminating));
4830        assert!(
4831            !report.failures.is_empty(),
4832            "the swallowed failure is recorded"
4833        );
4834    }
4835
4836    /// A teardown on a sandbox that never launched is a no-op report rather than a panic.
4837    ///
4838    /// The path a caller reaches by tearing down in a `finally` after a launch that failed
4839    /// before any VM existed, which is exactly when a teardown that assumed a VM would turn
4840    /// the real failure into a panic.
4841    #[tokio::test]
4842    async fn a_teardown_before_any_launch_is_an_empty_report() {
4843        let (mut sandbox, recorder, _) = planted();
4844        let report = sandbox.terminate(TeardownOpts::default()).await;
4845        assert!(!report.terminate_accepted);
4846        assert!(!report.leaked());
4847        assert_eq!(report.lifecycle, Some(Lifecycle::Pending));
4848        assert_eq!(recorder.calls().len(), 0);
4849    }
4850
4851    /// A run with no image at all is a precondition error before any call, rather than a
4852    /// launch of an empty identifier.
4853    #[tokio::test]
4854    async fn a_run_with_no_image_is_refused_before_any_call() {
4855        let (mut sandbox, recorder, _) = planted();
4856        let error = sandbox
4857            .run(RunRequest::new())
4858            .await
4859            .expect_err("there is nothing to launch");
4860        assert_eq!(error.kind(), ErrorKind::Precondition);
4861        assert!(error.to_string().contains("build_image first"), "{error}");
4862        assert_eq!(recorder.calls().len(), 0);
4863    }
4864
4865    /// The lifecycle's six states are the symspec's `vm_state` domain, spelled as the service
4866    /// spells them.
4867    ///
4868    /// Asserted rather than assumed because the spelling reaches an error message a reader
4869    /// compares against a `GetMicrovm` response: a lifecycle rendering `Suspended` beside a
4870    /// service saying `SUSPENDED` reads like two different facts.
4871    #[test]
4872    fn the_lifecycle_states_are_the_state_models_domain_in_the_services_spelling() {
4873        let all = [
4874            Lifecycle::Pending,
4875            Lifecycle::Running,
4876            Lifecycle::Suspending,
4877            Lifecycle::Suspended,
4878            Lifecycle::Terminating,
4879            Lifecycle::Terminated,
4880        ];
4881        assert_eq!(
4882            all.map(Lifecycle::as_str),
4883            [
4884                "PENDING",
4885                "RUNNING",
4886                "SUSPENDING",
4887                "SUSPENDED",
4888                "TERMINATING",
4889                "TERMINATED"
4890            ]
4891        );
4892        for state in all {
4893            assert_eq!(state.to_string(), state.as_str());
4894        }
4895        // The four live states are the ones a `Drop` warns about, and TERMINATING is not one
4896        // of them: the platform accepted the terminate, so nobody needs telling.
4897        assert_eq!(
4898            all.iter().filter(|state| state.is_live()).count(),
4899            4,
4900            "a state added to the enum defaults to not-live, which would silence the warning"
4901        );
4902        assert!(!Lifecycle::Terminating.is_live());
4903        assert!(!Lifecycle::Terminated.is_live());
4904    }
4905
4906    /// An agent token is 64 hex characters and never repeats across draws.
4907    ///
4908    /// The distinctness matters because the token is the only thing separating one VM's
4909    /// control API from another's: two VMs sharing a token means either one's caller can
4910    /// drive the other's guest. The token is the source's draw, so a fresh source per launch
4911    /// is fresh tokens; the OS pool's own distinctness is `microvms-edges`'s
4912    /// `the_os_source_draws_distinct_bytes`.
4913    #[test]
4914    fn a_minted_agent_token_is_sixty_four_hex_characters_and_never_repeats() {
4915        let entropy = crate::entropy::testing::SequenceEntropy::new();
4916        let minted: std::collections::HashSet<String> = (0..200)
4917            .map(|_| mint_agent_token(&entropy).expect("scripted"))
4918            .collect();
4919        assert_eq!(minted.len(), 200, "a repeated token is a shared guest");
4920        for token in &minted {
4921            assert_eq!(token.len(), 64, "{token}");
4922            assert!(token.chars().all(|c| c.is_ascii_hexdigit()), "{token}");
4923            assert_ne!(
4924                token,
4925                &"0".repeat(64),
4926                "an all-zero draw is a read that did nothing"
4927            );
4928        }
4929    }
4930
4931    /// A sandbox's `Debug` does not print the agent token.
4932    #[tokio::test(start_paused = true)]
4933    async fn a_sandbox_debug_does_not_print_the_agent_token() {
4934        let (mut sandbox, recorder, _) = planted();
4935        answer_launch(&recorder);
4936        sandbox
4937            .run(
4938                RunRequest::new()
4939                    .with_image("arn:image")
4940                    .with_agent_token("super-secret-agent-token"),
4941            )
4942            .await
4943            .expect("launches");
4944
4945        let rendered = format!("{sandbox:?}");
4946        assert!(rendered.contains("mvm-abc123"), "{rendered}");
4947        assert!(rendered.contains("Running"), "{rendered}");
4948        assert!(
4949            !rendered.contains("super-secret"),
4950            "the agent token reached a Debug string: {rendered}"
4951        );
4952    }
4953
4954    /// A torn-down sandbox does not warn on drop, and an abandoned one is the case the
4955    /// warning exists for.
4956    ///
4957    /// Asserted on the flag rather than on stderr — capturing another thread's stderr from a
4958    /// test is not something this crate can do without a dependency — so what is pinned is
4959    /// the condition the warning branches on. Both halves, so the branch cannot be
4960    /// vacuously true.
4961    #[tokio::test(start_paused = true)]
4962    async fn a_torn_down_sandbox_is_distinguishable_from_an_abandoned_one() {
4963        let (mut sandbox, recorder, _) = launched().await;
4964        assert!(
4965            sandbox.lifecycle().is_live() && !sandbox.torn_down,
4966            "an abandoned live VM is what the drop warning is for"
4967        );
4968
4969        recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4970        sandbox.terminate(TeardownOpts::default()).await;
4971        assert!(
4972            sandbox.torn_down,
4973            "terminate must mark the sandbox torn down"
4974        );
4975        assert!(
4976            !sandbox.lifecycle().is_live(),
4977            "and the lifecycle must no longer read as live"
4978        );
4979    }
4980
4981    #[tokio::test(start_paused = true)]
4982    async fn stable_launch_key_and_payload_survive_a_fresh_supervisor() {
4983        let mut bodies = Vec::new();
4984        for _ in 0..2 {
4985            let (mut sandbox, recorder, _) = planted();
4986            answer_launch(&recorder);
4987            let mut request = RunRequest::new().with_image("arn:image");
4988            request.client_token = Some("one-unique-job-launch".into());
4989            request.agent_token = Some("persisted-private-secret".into());
4990            sandbox.run(request).await.expect("accepted");
4991            bodies.push(recorder.first_body("RunMicrovm"));
4992        }
4993        assert_eq!(bodies[0]["clientToken"], "one-unique-job-launch");
4994        assert_eq!(bodies[0], bodies[1]);
4995    }
4996
4997    #[tokio::test]
4998    async fn unsafe_or_malformed_stable_launch_is_refused_without_calls() {
4999        for (token, secret, identity) in [
5000            ("key".to_string(), None, false),
5001            ("key".to_string(), Some("secret".into()), true),
5002            ("".to_string(), Some("secret".into()), false),
5003            ("x".repeat(129), Some("secret".into()), false),
5004            ("line\nbreak".to_string(), Some("secret".into()), false),
5005        ] {
5006            let (mut sandbox, recorder, _) = planted();
5007            let mut request = RunRequest::new().with_image("arn:image");
5008            request.client_token = Some(token);
5009            request.agent_token = secret;
5010            request.identity = identity;
5011            let error = sandbox.run(request).await.expect_err("invalid");
5012            assert_eq!(error.kind(), ErrorKind::InvalidArg);
5013            assert!(recorder.calls().is_empty());
5014        }
5015    }
5016}