1use std::sync::Arc;
67use std::time::Duration;
68
69use crate::control::{
70 ControlPlane, CreateImageRequest, Image, Microvm, RunHookPayload, RunMicrovmRequest, WaitOpts,
71};
72use crate::error::{Error, ErrorKind};
73use crate::region::Region;
74use crate::session::{Session, TokenMinter};
75
76pub const DEFAULT_RUNNING_TIMEOUT: Duration = Duration::from_secs(300);
80
81#[deprecated(
83 note = "renamed DEFAULT_RUNNING_TIMEOUT: `session::DEFAULT_READY_TIMEOUT` shared this name \
84 for a different wait, the daemon's"
85)]
86pub const DEFAULT_READY_TIMEOUT: Duration = DEFAULT_RUNNING_TIMEOUT;
87
88pub const DEFAULT_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(300);
91
92pub const DEFAULT_DELETE_ATTEMPTS: u32 = 20;
98
99pub const DEFAULT_DELETE_BACKOFF: Duration = Duration::from_secs(15);
101
102pub const DENY_EGRESS_PROXY_URL: &str = "http://127.0.0.1:1";
108
109pub const DENY_EGRESS_ENV_KEYS: [&str; 6] = [
118 "http_proxy",
119 "https_proxy",
120 "all_proxy",
121 "HTTP_PROXY",
122 "HTTPS_PROXY",
123 "ALL_PROXY",
124];
125
126pub const LIFECYCLE_POLL_INTERVAL: Duration = Duration::from_secs(5);
129
130#[derive(Clone, Copy, Debug, Eq, PartialEq)]
136pub enum Lifecycle {
137 Pending,
139 Running,
142 Suspending,
144 Suspended,
146 Terminating,
148 Terminated,
150}
151
152impl Lifecycle {
153 pub fn as_str(self) -> &'static str {
155 match self {
156 Lifecycle::Pending => "PENDING",
157 Lifecycle::Running => "RUNNING",
158 Lifecycle::Suspending => "SUSPENDING",
159 Lifecycle::Suspended => "SUSPENDED",
160 Lifecycle::Terminating => "TERMINATING",
161 Lifecycle::Terminated => "TERMINATED",
162 }
163 }
164
165 pub fn from_service(state: &str) -> Option<Self> {
168 Some(match state {
169 "PENDING" => Lifecycle::Pending,
170 "RUNNING" => Lifecycle::Running,
171 "SUSPENDING" => Lifecycle::Suspending,
172 "SUSPENDED" => Lifecycle::Suspended,
173 "TERMINATING" => Lifecycle::Terminating,
174 "TERMINATED" => Lifecycle::Terminated,
175 _ => return None,
176 })
177 }
178
179 pub fn is_live(self) -> bool {
181 matches!(
182 self,
183 Lifecycle::Pending | Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended
184 )
185 }
186}
187
188impl std::fmt::Display for Lifecycle {
189 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
190 f.write_str(self.as_str())
191 }
192}
193
194#[derive(Clone, Debug)]
200pub struct RunRequest {
201 pub image_identifier: Option<String>,
207 pub image_version: Option<String>,
217 pub execution_role_arn: Option<String>,
219 pub agent_token: Option<String>,
221 pub client_token: Option<String>,
225 pub launch_env: std::collections::HashMap<String, String>,
239 pub identity: bool,
251 pub egress: bool,
255 pub egress_network_connectors: Vec<String>,
258 pub deny_egress: bool,
265 pub shell: bool,
270 pub max_idle_sec: u32,
272 pub suspended_sec: u32,
275 pub auto_resume: bool,
277 pub max_duration_sec: u32,
279 pub ready_timeout: Duration,
282 pub token_scope: Option<String>,
284 pub logging: Option<crate::control::ops::Logging>,
286 pub wait: bool,
293}
294
295impl Default for RunRequest {
296 fn default() -> Self {
297 Self {
298 image_identifier: None,
299 image_version: None,
300 execution_role_arn: None,
301 agent_token: None,
302 client_token: None,
303 launch_env: std::collections::HashMap::new(),
304 identity: false,
305 egress: false,
306 egress_network_connectors: Vec::new(),
307 deny_egress: false,
308 shell: false,
309 max_idle_sec: 600,
310 suspended_sec: 600,
311 auto_resume: false,
312 max_duration_sec: 3_600,
313 ready_timeout: DEFAULT_RUNNING_TIMEOUT,
314 token_scope: None,
315 logging: None,
316 wait: true,
317 }
318 }
319}
320
321impl RunRequest {
322 pub fn new() -> Self {
325 Self::default()
326 }
327
328 #[must_use]
330 pub fn with_image(mut self, identifier: impl Into<String>) -> Self {
331 self.image_identifier = Some(identifier.into());
332 self
333 }
334
335 #[must_use]
337 pub fn with_egress(mut self) -> Self {
338 self.egress = true;
339 self
340 }
341
342 #[must_use]
344 pub fn with_egress_network_connector(mut self, arn: impl Into<String>) -> Self {
345 self.egress_network_connectors.push(arn.into());
346 self
347 }
348
349 #[must_use]
355 pub fn with_vpc_egress(mut self, arns: Vec<String>) -> Self {
356 if !arns.is_empty() {
357 self.egress = false;
358 self.egress_network_connectors = arns;
359 }
360 self
361 }
362
363 #[must_use]
366 pub fn with_deny_egress(mut self) -> Self {
367 self.deny_egress = true;
368 self
369 }
370
371 pub fn egress_posture(&self) -> crate::control::EgressPosture {
376 crate::control::EgressPosture::for_launch(self.egress, self.deny_egress)
377 }
378
379 pub fn effective_launch_env(&self) -> std::collections::HashMap<String, String> {
387 let mut env = self.launch_env.clone();
388 if self.deny_egress {
389 for key in DENY_EGRESS_ENV_KEYS {
390 env.entry(key.to_string())
391 .or_insert_with(|| DENY_EGRESS_PROXY_URL.to_string());
392 }
393 }
394 env
395 }
396
397 #[must_use]
399 pub fn with_shell(mut self) -> Self {
400 self.shell = true;
401 self
402 }
403
404 #[must_use]
406 pub fn with_image_version(mut self, version: impl Into<String>) -> Self {
407 self.image_version = Some(version.into());
408 self
409 }
410
411 #[must_use]
414 pub fn with_suspended_sec(mut self, seconds: u32) -> Self {
415 self.suspended_sec = seconds;
416 self
417 }
418
419 #[must_use]
421 pub fn with_agent_token(mut self, token: impl Into<String>) -> Self {
422 self.agent_token = Some(token.into());
423 self
424 }
425
426 #[must_use]
431 pub fn with_identity(mut self) -> Self {
432 self.identity = true;
433 self
434 }
435
436 #[must_use]
443 pub fn with_launch_env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
444 self.launch_env.insert(key.into(), value.into());
445 self
446 }
447}
448
449#[derive(Clone, Copy, Debug)]
455pub struct TeardownOpts {
456 pub delete_image: bool,
458 pub delete_log_group: bool,
464 pub delete_attempts: u32,
466 pub delete_backoff: Duration,
468 pub wait_for_terminated: Option<Duration>,
476}
477
478impl Default for TeardownOpts {
479 fn default() -> Self {
480 Self {
481 delete_image: false,
482 delete_log_group: false,
483 delete_attempts: DEFAULT_DELETE_ATTEMPTS,
484 delete_backoff: DEFAULT_DELETE_BACKOFF,
485 wait_for_terminated: None,
486 }
487 }
488}
489
490impl TeardownOpts {
491 #[must_use]
493 pub fn deleting_image(mut self) -> Self {
494 self.delete_image = true;
495 self
496 }
497
498 #[must_use]
500 pub fn deleting_log_group(mut self) -> Self {
501 self.delete_log_group = true;
502 self
503 }
504
505 #[must_use]
507 pub fn waiting_for_terminated(mut self) -> Self {
508 self.wait_for_terminated = Some(DEFAULT_LIFECYCLE_TIMEOUT);
509 self
510 }
511}
512
513#[derive(Clone, Debug, Default, Eq, PartialEq)]
517pub struct TeardownReport {
518 pub undeleted: Vec<String>,
533 pub terminate_accepted: bool,
535 pub image_deleted: Option<bool>,
537 pub lifecycle: Option<Lifecycle>,
539 pub failures: Vec<String>,
544}
545
546impl TeardownReport {
547 pub fn leaked(&self) -> bool {
549 !self.undeleted.is_empty()
550 }
551}
552
553pub struct ControlPlaneMinter {
567 control: Arc<ControlPlane>,
568 microvm_id: String,
569}
570
571impl ControlPlaneMinter {
572 pub fn new(control: Arc<ControlPlane>, microvm_id: impl Into<String>) -> Self {
574 Self {
575 control,
576 microvm_id: microvm_id.into(),
577 }
578 }
579}
580
581impl std::fmt::Debug for ControlPlaneMinter {
582 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
583 f.debug_struct("ControlPlaneMinter")
584 .field("microvm_id", &self.microvm_id)
585 .finish_non_exhaustive()
586 }
587}
588
589impl TokenMinter for ControlPlaneMinter {
590 fn mint(
591 &self,
592 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
593 Box::pin(async move {
594 let minted = self.control.mint_auth_token(&self.microvm_id).await?;
595 Ok(minted.into())
596 })
597 }
598
599 fn mint_for_ports(
607 &self,
608 ports: &[u16],
609 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
610 let specs: Vec<crate::control::ops::PortSpecification> = ports
611 .iter()
612 .map(|port| crate::control::ops::PortSpecification::port(*port))
613 .collect();
614 Box::pin(async move {
615 let minted = self
616 .control
617 .mint_auth_token_for(&self.microvm_id, &specs)
618 .await?;
619 Ok(minted.into())
620 })
621 }
622}
623
624pub struct Sandbox {
629 control: Arc<ControlPlane>,
630 image: Option<Image>,
631 launch_image_arn: Option<String>,
633 launch_metrics: Option<Arc<std::sync::Mutex<crate::session::LaunchMetrics>>>,
634 microvm: Option<Microvm>,
635 session: Option<Session>,
636 session_backend: Option<crate::session::SharedBackend>,
643
644 lifecycle: Lifecycle,
646 lifecycle_watch: tokio::sync::watch::Sender<Lifecycle>,
649 token_installed: bool,
650 image_exists: bool,
651 was_terminated: bool,
652 bootstrap_count: u32,
653
654 suspended_window: Option<Duration>,
657 launch_adoptable: bool,
660 launch_posture: Option<crate::control::EgressPosture>,
663 pending_agent_token: Option<String>,
666 idle_window: Option<Duration>,
669 torn_down: bool,
672 adopted: bool,
675 detached: bool,
678 tunnel_identity: Option<crate::identity::TunnelIdentity>,
684 build_services: Option<Arc<dyn crate::control::BuildServices>>,
687 account: Option<String>,
689}
690
691impl std::fmt::Debug for Sandbox {
692 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
693 f.debug_struct("Sandbox")
696 .field("lifecycle", &self.lifecycle)
697 .field("microvm", &self.microvm.as_ref().map(|vm| &vm.id))
698 .field("image", &self.image.as_ref().map(|image| &image.identifier))
699 .field("token_installed", &self.token_installed)
700 .field("bootstrap_count", &self.bootstrap_count)
701 .field("was_terminated", &self.was_terminated)
702 .field("suspended_window", &self.suspended_window)
703 .field("adopted", &self.adopted)
704 .field("detached", &self.detached)
705 .finish_non_exhaustive()
706 }
707}
708
709#[derive(Clone)]
716pub struct Detached {
717 pub microvm_id: String,
719 pub endpoint: String,
721 pub region: Region,
723 pub port: u16,
725 agent_token: String,
726}
727
728impl Detached {
729 pub fn agent_token(&self) -> &str {
731 &self.agent_token
732 }
733}
734
735impl std::fmt::Debug for Detached {
736 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
737 f.debug_struct("Detached")
738 .field("microvm_id", &self.microvm_id)
739 .field("endpoint", &self.endpoint)
740 .field("region", &self.region)
741 .field("port", &self.port)
742 .field("agent_token", &"<redacted>")
743 .finish()
744 }
745}
746
747impl Sandbox {
748 pub fn with_control_plane(control: ControlPlane) -> Self {
762 Self {
763 control: Arc::new(control),
764 image: None,
765 launch_image_arn: None,
766 launch_metrics: None,
767 microvm: None,
768 session: None,
769 session_backend: None,
770 lifecycle: Lifecycle::Pending,
771 lifecycle_watch: tokio::sync::watch::Sender::new(Lifecycle::Pending),
772 token_installed: false,
773 image_exists: false,
774 was_terminated: false,
775 bootstrap_count: 0,
776 suspended_window: None,
777 launch_adoptable: false,
778 launch_posture: None,
779 pending_agent_token: None,
780 idle_window: None,
781 torn_down: false,
782 adopted: false,
783 detached: false,
784 tunnel_identity: None,
785 build_services: None,
786 account: None,
787 }
788 }
789
790 pub fn with_build_services(mut self, services: Arc<dyn crate::control::BuildServices>) -> Self {
796 self.build_services = Some(services);
797 self
798 }
799
800 pub fn with_session_backend(mut self, backend: crate::session::SharedBackend) -> Self {
806 self.session_backend = Some(backend);
807 self
808 }
809
810 pub fn lifecycle(&self) -> Lifecycle {
814 self.lifecycle
815 }
816
817 pub fn token_installed(&self) -> bool {
819 self.token_installed
820 }
821
822 pub fn image_exists(&self) -> bool {
824 self.image_exists
825 }
826
827 pub fn was_terminated(&self) -> bool {
829 self.was_terminated
830 }
831
832 pub fn bootstrap_count(&self) -> u32 {
834 self.bootstrap_count
835 }
836
837 pub fn image(&self) -> Option<&Image> {
839 self.image.as_ref()
840 }
841
842 pub fn launch_image_arn(&self) -> Option<&str> {
851 self.launch_image_arn.as_deref()
852 }
853
854 pub fn microvm(&self) -> Option<&Microvm> {
856 self.microvm.as_ref()
857 }
858
859 pub fn session(&self) -> Option<&Session> {
861 self.session.as_ref()
862 }
863
864 pub fn suspended_window(&self) -> Option<Duration> {
866 self.suspended_window
867 }
868
869 pub fn watch_lifecycle(&self) -> tokio::sync::watch::Receiver<Lifecycle> {
876 self.lifecycle_watch.subscribe()
877 }
878
879 fn set_lifecycle(&mut self, lifecycle: Lifecycle) {
880 self.lifecycle = lifecycle;
881 self.lifecycle_watch.send_replace(lifecycle);
882 }
883
884 pub fn idle_window(&self) -> Option<Duration> {
886 self.idle_window
887 }
888
889 pub fn tunnel_identity(&self) -> Option<&crate::identity::TunnelIdentity> {
895 self.tunnel_identity.as_ref()
896 }
897
898 pub fn egress_posture(&self) -> crate::control::EgressPosture {
906 self.launch_posture.unwrap_or_default()
907 }
908
909 pub fn port(&self) -> u16 {
914 self.control.port()
915 }
916
917 pub async fn build_image(&mut self, request: CreateImageRequest) -> Result<&Image, Error> {
925 let size = request.size;
926 let created = self.control.create_image(request).await?;
927 let mut built = self
928 .control
929 .wait_for_image(&created.identifier, size, WaitOpts::default())
930 .await?;
931 built.log_group = created.log_group;
937 built.log_stream = created.log_stream;
938 self.image_exists = true;
941 self.image = Some(built);
942 Ok(self.image.as_ref().expect("just assigned"))
943 }
944
945 pub async fn ensure_image(
949 &mut self,
950 request: crate::control::EnsureImageRequest,
951 ) -> Result<crate::control::EnsuredImage, Error> {
952 let control = Arc::clone(&self.control);
961 let prepared =
962 tokio::task::spawn_blocking(move || crate::control::ensure::prepare(&control, request))
963 .await
964 .map_err(|joined| {
965 Error::new(
966 ErrorKind::Unexpected,
967 format!("preparing the image's artifact did not complete: {joined}"),
968 )
969 })??;
970 let services = self.build_services().await?;
971 let account = match &self.account {
972 Some(account) => account.clone(),
973 None => {
974 let account = services.caller_account().await?;
975 self.account = Some(account.clone());
976 account
977 }
978 };
979 let ensured =
980 crate::control::ensure::ensure(&self.control, services.as_ref(), &account, prepared)
981 .await?;
982 self.image_exists = true;
983 self.image = Some(ensured.image.clone());
984 Ok(ensured)
985 }
986
987 async fn build_services(&mut self) -> Result<Arc<dyn crate::control::BuildServices>, Error> {
990 if let Some(services) = &self.build_services {
991 return Ok(Arc::clone(services));
992 }
993 let built = self
994 .control
995 .adapters()
996 .build_services(self.control.region().clone())
997 .await?;
998 self.build_services = Some(Arc::clone(&built));
999 Ok(built)
1000 }
1001
1002 pub fn build_artifact_for(&self, request: &CreateImageRequest) -> Result<Vec<u8>, Error> {
1007 self.control.build_artifact_for(request)
1008 }
1009
1010 pub async fn upload_artifact_for(&mut self, request: &CreateImageRequest) -> Result<(), Error> {
1016 let (bucket, key) = crate::control::ensure::s3_location(&request.code_artifact_uri)?;
1017 let bytes = self.control.build_artifact_for(request)?;
1018 let services = self.build_services().await?;
1019 services.put_object(bucket, key, bytes).await
1020 }
1021
1022 pub fn preflight(&self, request: &CreateImageRequest) -> Result<(), Error> {
1029 self.control.preflight(request)
1030 }
1031
1032 pub async fn resolve_image_arn(&self, identifier: &str) -> Result<String, Error> {
1040 self.control.resolve_image_arn(identifier).await
1041 }
1042
1043 pub async fn find_image_by_name(
1046 &self,
1047 name: &str,
1048 ) -> Result<Option<crate::control::ops::MicrovmImageSummaryWire>, Error> {
1049 self.control.find_image_by_name(name).await
1050 }
1051
1052 pub fn artifact_content_hash_for(&self, request: &CreateImageRequest) -> String {
1054 self.control.artifact_content_hash_for(request)
1055 }
1056
1057 pub fn dockerfile_for(&self, request: &CreateImageRequest) -> String {
1060 self.control.dockerfile_for(request)
1061 }
1062
1063 pub async fn managed_base_versions(
1071 &self,
1072 base_image_arn: &str,
1073 ) -> Result<Vec<crate::control::ops::ManagedMicrovmImageVersionWire>, Error> {
1074 self.control.managed_base_versions(base_image_arn).await
1075 }
1076
1077 pub async fn image_version(
1079 &self,
1080 identifier: &str,
1081 version: &str,
1082 ) -> Result<crate::control::ops::MicrovmImageVersionSummaryWire, Error> {
1083 self.control.get_image_version(identifier, version).await
1084 }
1085
1086 pub async fn run(&mut self, request: RunRequest) -> Result<&mut Session, Error> {
1114 self.run_inner(request, None).await
1115 }
1116
1117 pub async fn run_cancellable(
1120 &mut self,
1121 request: RunRequest,
1122 cancellation: &crate::control::transport::RetryCancellation,
1123 ) -> Result<&mut Session, Error> {
1124 self.run_inner(request, Some(cancellation)).await
1125 }
1126
1127 async fn run_inner(
1128 &mut self,
1129 request: RunRequest,
1130 cancellation: Option<&crate::control::transport::RetryCancellation>,
1131 ) -> Result<&mut Session, Error> {
1132 self.refuse_detached("run")?;
1133 if self.bootstrap_count > 0 || self.microvm.is_some() {
1137 return Err(Error::invalid_arg(format!(
1138 "this sandbox has already launched a VM ({} bootstrap(s), lifecycle {}), and the \
1139 agent token is installed at most once per VM lifetime (STATE-3). A second VM \
1140 needs a second Sandbox — reusing this one would either re-deliver a run-hook \
1141 payload to a daemon whose one-shot bootstrap refuses it, or silently address \
1142 two guests through one handle.",
1143 self.bootstrap_count, self.lifecycle,
1144 )));
1145 }
1146
1147 let posture = crate::control::egress_posture_for(
1154 request.egress,
1155 &request.egress_network_connectors,
1156 request.deny_egress,
1157 Some(self.control.region()),
1158 )?;
1159
1160 let Some(identifier) = request
1161 .image_identifier
1162 .clone()
1163 .or_else(|| self.image.as_ref().map(|image| image.identifier.clone()))
1164 else {
1165 return Err(Error::new(
1166 ErrorKind::Precondition,
1167 "no image to launch: pass RunRequest::with_image or call build_image first."
1168 .to_string(),
1169 ));
1170 };
1171
1172 if request.client_token.is_some() && (request.agent_token.is_none() || request.identity) {
1173 return Err(Error::invalid_arg(
1174 "a stable client_token requires an explicit agent_token and identity=false; persist identical launch parameters before retrying",
1175 ));
1176 }
1177 let agent_token = match request.agent_token.clone() {
1178 Some(token) => token,
1179 None => mint_agent_token(self.control.entropy())?,
1180 };
1181 let launch_identity = if request.identity {
1185 Some(crate::entropy::launch_identity(self.control.entropy())?)
1186 } else {
1187 None
1188 };
1189 let payload = RunHookPayload::for_launch_with_identity(
1196 &agent_token,
1197 &request.effective_launch_env(),
1198 launch_identity.as_ref(),
1199 )?;
1200
1201 let mut wire = RunMicrovmRequest::new(identifier, payload);
1202 wire.image_version = request.image_version.clone();
1203 wire.execution_role_arn = request.execution_role_arn.clone();
1204 wire.egress_network_connectors = request.egress_network_connectors.clone();
1205 wire.max_idle_sec = request.max_idle_sec;
1206 wire.suspended_sec = request.suspended_sec;
1207 wire.auto_resume = request.auto_resume;
1208 wire.max_duration_sec = request.max_duration_sec;
1209 wire.token_scope = request.token_scope.clone();
1210 wire.client_token = request.client_token.clone();
1211 wire.logging = request.logging.clone();
1212 if request.egress {
1213 wire = wire.with_egress();
1214 }
1215 if request.shell {
1216 wire = wire.with_shell();
1217 }
1218
1219 self.control.launch_connectors(&wire)?;
1225 let resolution = self.control.resolve_image_arn(&wire.image_identifier);
1226 wire.image_identifier = if let Some(cancellation) = cancellation {
1227 tokio::select! {
1230 biased;
1231 _ = cancellation.cancelled() => {
1232 return Err(Error::new(
1233 ErrorKind::Interrupted,
1234 "launch cancelled before a VM was assigned",
1235 ));
1236 }
1237 resolved = resolution => resolved?,
1238 }
1239 } else {
1240 resolution.await?
1241 };
1242 self.launch_image_arn = Some(wire.image_identifier.clone());
1243
1244 let (launched, details) = self
1245 .control
1246 .run_microvm_reported(wire, cancellation)
1247 .await?;
1248 self.launch_metrics = Some(Arc::new(std::sync::Mutex::new(
1249 crate::session::LaunchMetrics {
1250 attempt_count: details.attempt_count,
1251 backoff_seconds: details.backoff.as_secs_f64(),
1252 ..Default::default()
1253 },
1254 )));
1255
1256 self.tunnel_identity = launch_identity.map(crate::identity::LaunchIdentity::keep);
1260
1261 self.set_lifecycle(Lifecycle::Pending);
1263 self.image_exists = true;
1264 self.suspended_window = Some(Duration::from_secs(u64::from(request.suspended_sec)));
1268 self.idle_window = Some(Duration::from_secs(u64::from(request.max_idle_sec)));
1269 self.microvm = Some(launched);
1270 self.launch_adoptable = request.client_token.is_some();
1273 self.launch_posture = Some(posture);
1274 self.pending_agent_token = Some(agent_token);
1275
1276 if request.wait {
1277 return self.wait_until_running(request.ready_timeout).await;
1278 }
1279 self.build_session()?;
1282 Ok(self.session.as_mut().expect("just assigned"))
1283 }
1284
1285 fn build_session(&mut self) -> Result<(), Error> {
1287 let (Some(vm), Some(agent_token)) = (&self.microvm, &self.pending_agent_token) else {
1288 return Err(Error::new(
1289 ErrorKind::Precondition,
1290 "no launch to build a session for",
1291 ));
1292 };
1293 let minter = Arc::new(ControlPlaneMinter::new(
1294 Arc::clone(&self.control),
1295 vm.id.clone(),
1296 ));
1297 let mut builder = Session::builder(vm.endpoint.clone(), agent_token.clone())
1300 .with_minter(minter)
1301 .with_port(self.control.port())
1302 .with_egress_posture(self.egress_posture())
1303 .with_clock(self.control.shared_clock());
1304 if let Some(backend) = &self.session_backend {
1305 builder = builder.with_backend(Arc::clone(backend));
1306 }
1307 let mut session = builder.build_with(self.control.adapters())?;
1308 session.launch_metrics = self.launch_metrics.clone();
1309 self.session = Some(session);
1310 Ok(())
1311 }
1312
1313 pub async fn wait_until_running(&mut self, timeout: Duration) -> Result<&mut Session, Error> {
1329 self.refuse_detached("wait until running")?;
1330 let id = self.require_microvm("wait_until_running")?;
1331 if self.lifecycle != Lifecycle::Pending {
1332 return Err(Error::invalid_arg(format!(
1333 "microvm {id} is {}; wait_until_running finishes a launch that is still PENDING",
1334 self.lifecycle,
1335 )));
1336 }
1337 let opts = WaitOpts::for_lifecycle(timeout);
1338 let running = self
1339 .control
1340 .wait_for_sandbox_launch(&id, opts, self.launch_adoptable)
1341 .await?;
1342
1343 self.set_lifecycle(Lifecycle::Running);
1346 if let Some(metrics) = &self.launch_metrics {
1347 metrics
1348 .lock()
1349 .unwrap_or_else(std::sync::PoisonError::into_inner)
1350 .running_at
1351 .get_or_insert_with(|| self.control.clock().unix_now().as_secs_f64());
1352 }
1353 self.token_installed = true;
1354 self.bootstrap_count += 1;
1355 self.microvm = Some(running);
1356 if self.session.is_none() {
1357 self.build_session()?;
1358 }
1359 let session = self.session.as_mut().expect("just assigned");
1360 session
1361 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1362 .await?;
1363 Ok(session)
1364 }
1365
1366 pub async fn wait_until_ready(
1376 &mut self,
1377 running_timeout: Duration,
1378 ) -> Result<&mut Session, Error> {
1379 if self.lifecycle == Lifecycle::Running {
1380 self.refuse_detached("wait until ready")?;
1381 let session = self.session.as_mut().ok_or_else(|| {
1382 Error::new(ErrorKind::Precondition, "a RUNNING sandbox with no session")
1383 })?;
1384 session
1385 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1386 .await?;
1387 return Ok(session);
1388 }
1389 self.wait_until_running(running_timeout).await
1390 }
1391
1392 pub async fn adopt(
1412 control: ControlPlane,
1413 microvm_id: impl Into<String>,
1414 endpoint: impl Into<String>,
1415 agent_token: impl Into<String>,
1416 ) -> Result<Self, Error> {
1417 let microvm_id = microvm_id.into();
1418 let endpoint = endpoint.into();
1419 let agent_token = agent_token.into();
1420 if agent_token.is_empty() {
1421 return Err(Error::invalid_arg(
1422 "adopt needs the agent token the VM was launched with: its daemon accepts only \
1423 that bearer, and adopting without it would build a handle every exec refuses.",
1424 ));
1425 }
1426 let mut vm = control.get_microvm(µvm_id).await?;
1427 if !endpoint.is_empty() && !vm.endpoint.is_empty() && vm.endpoint != endpoint {
1428 return Err(Error::invalid_arg(format!(
1429 "microvm {microvm_id} reports endpoint {}, not {endpoint}: the id and endpoint \
1430 came from different records, and a session built from them would address one \
1431 VM while its lifecycle calls reached another.",
1432 vm.endpoint,
1433 )));
1434 }
1435 if vm.endpoint.is_empty() {
1436 vm.endpoint = endpoint;
1437 }
1438 let Some(lifecycle) = Lifecycle::from_service(&vm.state) else {
1439 return Err(Error::new(
1440 ErrorKind::Platform,
1441 format!(
1442 "microvm {microvm_id} reports state {}, which this client does not know; \
1443 refusing to guess which lifecycle guards apply.",
1444 vm.state,
1445 ),
1446 ));
1447 };
1448
1449 let mut sandbox = Self::with_control_plane(control);
1450 sandbox.adopted = true;
1451 sandbox.set_lifecycle(lifecycle);
1452 sandbox.idle_window = vm
1455 .idle_policy
1456 .as_ref()
1457 .map(|policy| Duration::from_secs(u64::from(policy.max_idle_duration_seconds)));
1458 sandbox.image_exists = true;
1460 sandbox.launch_adoptable = true;
1462 sandbox.pending_agent_token = Some(agent_token);
1463 match lifecycle {
1464 Lifecycle::Pending => {}
1465 Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended => {
1466 sandbox.token_installed = true;
1467 sandbox.bootstrap_count = 1;
1468 }
1469 Lifecycle::Terminating | Lifecycle::Terminated => {
1470 sandbox.bootstrap_count = 1;
1471 sandbox.was_terminated = true;
1472 }
1473 }
1474 sandbox.microvm = Some(vm);
1475 if lifecycle.is_live() {
1476 sandbox.build_session()?;
1477 }
1478 Ok(sandbox)
1479 }
1480
1481 pub fn name_record(&self, name: &str) -> Result<crate::names::NameRecord, Error> {
1488 let Some(vm) = self.microvm() else {
1489 return Err(Error::new(
1490 ErrorKind::Precondition,
1491 "there is no VM to name yet: launch or adopt one first",
1492 ));
1493 };
1494 let Some(session) = self.session() else {
1495 return Err(Error::new(
1496 ErrorKind::Precondition,
1497 format!(
1498 "microvm {} has no session to take its agent token from",
1499 vm.id
1500 ),
1501 ));
1502 };
1503 let mut record = crate::names::NameRecord::new_at(
1505 name,
1506 vm.id.as_str(),
1507 vm.endpoint.as_str(),
1508 session.agent_token(),
1509 self.control.region().as_str(),
1510 self.control.clock().unix_now().as_secs(),
1511 )?;
1512 record.set_tunnel_identity(self.tunnel_identity.as_ref());
1513 Ok(record)
1514 }
1515
1516 pub async fn adopt_record(
1523 control: ControlPlane,
1524 record: crate::names::NameRecord,
1525 ) -> Result<Self, Error> {
1526 let identity = record.tunnel_identity()?;
1527 let mut sandbox = Self::adopt(
1528 control,
1529 record.microvm_id,
1530 record.endpoint,
1531 record.agent_token,
1532 )
1533 .await?;
1534 sandbox.tunnel_identity = identity;
1535 Ok(sandbox)
1536 }
1537
1538 pub fn detach(&mut self) -> Result<Detached, Error> {
1551 self.refuse_detached("detach")?;
1552 let Some(vm) = self.microvm.as_ref() else {
1553 return Err(Error::new(
1554 ErrorKind::Precondition,
1555 "nothing to detach: this sandbox has not launched a VM.",
1556 ));
1557 };
1558 if self.torn_down || !self.lifecycle.is_live() {
1559 return Err(Error::new(
1560 ErrorKind::Precondition,
1561 format!(
1562 "microvm {} is {} and there is no live VM to hand off; only a PENDING, \
1563 RUNNING, or SUSPENDED VM can be adopted.",
1564 vm.id, self.lifecycle,
1565 ),
1566 ));
1567 }
1568 let agent_token = self
1569 .session
1570 .as_ref()
1571 .map(|session| session.agent_token().to_string())
1572 .or_else(|| self.pending_agent_token.clone())
1573 .ok_or_else(|| {
1574 Error::new(
1575 ErrorKind::Precondition,
1576 format!(
1577 "microvm {} has no agent token to hand off; an adopter could not \
1578 reach its daemon.",
1579 vm.id
1580 ),
1581 )
1582 })?;
1583 let detached = Detached {
1584 microvm_id: vm.id.clone(),
1585 endpoint: vm.endpoint.clone(),
1586 region: self.control.region().clone(),
1587 port: self.control.port(),
1588 agent_token,
1589 };
1590 self.detached = true;
1591 self.session = None;
1592 self.pending_agent_token = None;
1593 Ok(detached)
1594 }
1595
1596 pub fn detached(&self) -> bool {
1598 self.detached
1599 }
1600
1601 fn refuse_detached(&self, what: &str) -> Result<(), Error> {
1603 if !self.detached {
1604 return Ok(());
1605 }
1606 let id = self.microvm.as_ref().map_or("?", |vm| vm.id.as_str());
1607 Err(Error::new(
1608 ErrorKind::Precondition,
1609 format!(
1610 "cannot {what}: microvm {id} was detached and belongs to whoever adopts it. \
1611 Adopt it with `Sandbox::adopt` and {what} through that handle."
1612 ),
1613 ))
1614 }
1615
1616 pub fn adopted(&self) -> bool {
1618 self.adopted
1619 }
1620
1621 pub async fn suspend(&mut self) -> Result<(), Error> {
1636 self.refuse_detached("suspend")?;
1637 let id = self.require_microvm("suspend")?;
1638
1639 if self.lifecycle != Lifecycle::Running {
1641 return Err(Error::invalid_arg(format!(
1642 "microvm {id} is {} and a suspend is only issued from RUNNING (STATE-5). Refused \
1643 here rather than by the service, because the service's answer about a \
1644 non-running id does not say which of the two things went wrong — and a suspend \
1645 issued from SUSPENDED is a caller who believes they resumed.",
1646 self.lifecycle,
1647 )));
1648 }
1649
1650 self.control.suspend(&id).await?;
1655 self.set_lifecycle(Lifecycle::Suspending);
1656
1657 let settled = self
1658 .control
1659 .wait_for_state(
1660 &id,
1661 &crate::control::microvm::SUSPEND_WANTED,
1662 &[],
1663 self.lifecycle_wait(),
1664 )
1665 .await?;
1666
1667 let settled_lifecycle = match settled.state.as_str() {
1672 "SUSPENDED" => Lifecycle::Suspended,
1673 "TERMINATED" => {
1674 self.was_terminated = true;
1675 Lifecycle::Terminated
1676 }
1677 other => {
1678 return Err(Error::new(
1679 ErrorKind::Platform,
1680 format!(
1681 "the suspend wait returned {other}, which is neither SUSPENDED nor \
1682 TERMINATED — and those two are what this client asked for."
1683 ),
1684 ));
1685 }
1686 };
1687 self.set_lifecycle(settled_lifecycle);
1688 self.microvm = Some(settled);
1689 Ok(())
1690 }
1691
1692 pub async fn resume(&mut self) -> Result<&mut Session, Error> {
1720 self.refuse_detached("resume")?;
1721 let id = self.require_microvm("resume")?;
1722
1723 if self.was_terminated || self.lifecycle == Lifecycle::Terminated {
1726 return Err(Error::invalid_arg(format!(
1727 "microvm {id} was terminated, and a terminated VM never returns to RUNNING \
1728 (STATE-11). There is nothing to resume: the guest's memory is gone, so even a \
1729 call the service accepted would hand back a different machine."
1730 )));
1731 }
1732 let mut resume_state = self.lifecycle;
1733 if self.lifecycle == Lifecycle::Running {
1736 let observed = self.control.get_microvm(&id).await?;
1737 resume_state = Lifecycle::from_service(&observed.state).ok_or_else(|| {
1738 Error::new(
1739 ErrorKind::Platform,
1740 format!("microvm {id} reports unknown state {}", observed.state),
1741 )
1742 })?;
1743 let ended = crate::constants::DEAD_STATES.contains(&observed.state.as_str());
1744 let error = ended.then(|| {
1745 Error::new(
1746 ErrorKind::WindowClosed,
1747 format!(
1748 "microvm {id} can't be resumed: GetMicrovm reports the VM {} ({}). \
1749 A VM that has ended doesn't come back; launch a new one.",
1750 observed.state,
1751 observed.reason(),
1752 ),
1753 )
1754 });
1755 self.microvm = Some(observed);
1756 if resume_state == Lifecycle::Suspended {
1759 self.set_lifecycle(resume_state);
1760 }
1761 if let Some(error) = error {
1762 self.was_terminated = true;
1763 self.set_lifecycle(Lifecycle::Terminated);
1764 return Err(error);
1765 }
1766 }
1767 if resume_state != Lifecycle::Suspended {
1768 return Err(Error::invalid_arg(format!(
1769 "microvm {id} is {} and a resume is only issued from SUSPENDED (STATE-7).",
1770 resume_state,
1771 )));
1772 }
1773
1774 if let Err(refused) = self.control.resume(&id).await {
1780 if refused.kind() == ErrorKind::WindowClosed {
1781 self.was_terminated = true;
1782 self.set_lifecycle(Lifecycle::Terminated);
1783 }
1784 return Err(refused);
1785 }
1786 let running = self
1792 .control
1793 .wait_for_state(
1794 &id,
1795 &["RUNNING"],
1796 &crate::constants::DEAD_STATES,
1797 self.lifecycle_wait(),
1798 )
1799 .await?;
1800
1801 self.set_lifecycle(Lifecycle::Running);
1802 let endpoint = running.endpoint.clone();
1806 self.microvm = Some(running);
1807 if let Some(session) = self.session.as_mut() {
1808 session.rebind(endpoint);
1809 }
1810
1811 self.session.as_mut().ok_or_else(|| {
1812 Error::new(
1813 ErrorKind::Unexpected,
1814 "the VM resumed but this sandbox holds no session, which run() always builds"
1815 .to_string(),
1816 )
1817 })
1818 }
1819
1820 pub async fn terminate(&mut self, opts: TeardownOpts) -> TeardownReport {
1828 let mut report = TeardownReport::default();
1829 if let Err(error) = self.refuse_detached("terminate") {
1830 report.failures.push(error.to_string());
1833 report.lifecycle = Some(self.lifecycle);
1834 return report;
1835 }
1836 self.torn_down = true;
1837
1838 self.session = None;
1841
1842 if let Some(id) = self.microvm.as_ref().map(|vm| vm.id.clone()) {
1844 self.set_lifecycle(Lifecycle::Terminating);
1848 self.was_terminated = true;
1849
1850 match self.control.terminate(&id).await {
1851 Ok(()) => report.terminate_accepted = true,
1852 Err(error) => {
1853 report.failures.push(format!("terminate {id}: {error}"));
1854 report.undeleted.push(id.clone());
1855 }
1856 }
1857
1858 if report.terminate_accepted
1859 && let Some(timeout) = opts.wait_for_terminated
1860 {
1861 let wait = WaitOpts::for_lifecycle(timeout);
1862 match self
1863 .control
1864 .wait_for_state(&id, &["TERMINATED"], &[], wait)
1865 .await
1866 {
1867 Ok(settled) => {
1869 self.set_lifecycle(Lifecycle::Terminated);
1870 self.microvm = Some(settled);
1871 }
1872 Err(error) => report
1875 .failures
1876 .push(format!("waiting for {id} to reach TERMINATED: {error}")),
1877 }
1878 }
1879 }
1880
1881 if opts.delete_image {
1884 match self.image.as_ref().map(|image| image.identifier.clone()) {
1885 Some(identifier) => {
1886 let deleted = self
1887 .control
1888 .delete_image(&identifier, opts.delete_attempts, opts.delete_backoff)
1889 .await;
1890 report.image_deleted = Some(deleted);
1891 if deleted {
1892 self.image_exists = false;
1893 } else {
1894 report.failures.push(format!(
1895 "the image {identifier} survived {} delete attempts",
1896 opts.delete_attempts.max(1)
1897 ));
1898 report.undeleted.push(identifier);
1899 }
1900 }
1901 None => report.image_deleted = Some(false),
1902 }
1903 }
1904
1905 if opts.delete_log_group
1907 && let Some(group) = self.image.as_ref().map(Image::build_log_group)
1908 {
1909 report.failures.push(format!(
1910 "the build log group {group} was not deleted: CloudWatch Logs is not in this \
1911 crate's dependency set, so it is reported rather than removed. It is \
1912 service-created, which means no Terraform stack owns it and `terraform destroy` \
1913 leaves it behind — six accumulated before anyone noticed."
1914 ));
1915 report.undeleted.push(group);
1916 }
1917
1918 report.lifecycle = Some(self.lifecycle);
1919 report
1920 }
1921
1922 fn require_microvm(&self, what: &str) -> Result<String, Error> {
1926 self.microvm
1927 .as_ref()
1928 .map(|vm| vm.id.clone())
1929 .ok_or_else(|| {
1930 Error::new(
1931 ErrorKind::Precondition,
1932 format!("nothing to {what}: this sandbox has not launched a VM."),
1933 )
1934 })
1935 }
1936
1937 fn lifecycle_wait(&self) -> WaitOpts {
1939 WaitOpts::for_lifecycle(DEFAULT_LIFECYCLE_TIMEOUT)
1940 }
1941}
1942
1943impl Sandbox {
1952 fn drop_warning(&self) -> Option<String> {
1956 if self.torn_down || self.adopted || self.detached {
1957 return None;
1958 }
1959 let vm = self.microvm.as_ref()?;
1960 self.lifecycle.is_live().then(|| {
1961 format!(
1962 "warning: the Sandbox for microvm {id} was dropped in {state} without \
1963 terminate() or detach(). Nothing was torn down — Drop cannot await, so a \
1964 teardown here would deadlock inside a runtime. {state} is the state this \
1965 sandbox last saw, and the platform may have suspended or ended the VM since. A \
1966 VM that's still live bills until it ends, at its maximumDurationInSeconds \
1967 ceiling at the latest: terminate it with `microvm terminate {id}`.",
1968 id = vm.id,
1969 state = self.lifecycle,
1970 )
1971 })
1972 }
1973}
1974
1975impl Drop for Sandbox {
1976 fn drop(&mut self) {
1977 if let Some(warning) = self.drop_warning() {
1978 self.control.adapters().warn(&warning);
1979 }
1980 }
1981}
1982
1983fn mint_agent_token(entropy: &dyn crate::entropy::Entropy) -> Result<String, Error> {
1989 let mut bytes = [0u8; 32];
1990 entropy.fill(&mut bytes)?;
1991 Ok(const_hex::encode(bytes))
1992}
1993
1994#[cfg(test)]
1995mod tests {
1996 use super::*;
1997 use crate::control::fake::{self as fake, Answer, FakeControlPlane, TestClock};
1998 use crate::control::transport::Method;
1999
2000 fn planted() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2010 let recorder = Arc::new(FakeControlPlane::new());
2011 let clock = Arc::new(TestClock::new());
2012 let plane = crate::testing::control_plane(
2013 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2014 Region::UsEast1,
2015 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2016 );
2017 let sandbox = Sandbox::with_control_plane(plane)
2019 .with_session_backend(crate::testing::HealthyDaemon::new());
2020 (sandbox, recorder, clock)
2021 }
2022
2023 #[test]
2026 fn the_suspend_resume_and_terminate_waits_are_cores_lifecycle_wait() {
2027 let (sandbox, _, _) = planted();
2028 let wait = sandbox.lifecycle_wait();
2029 assert_eq!(
2030 (wait.timeout, wait.poll_interval, wait.stall_grace),
2031 (
2032 DEFAULT_LIFECYCLE_TIMEOUT,
2033 LIFECYCLE_POLL_INTERVAL,
2034 Duration::MAX
2035 )
2036 );
2037 }
2038
2039 fn answer_launch(recorder: &FakeControlPlane) {
2041 recorder
2042 .answer(
2043 "RunMicrovm",
2044 Answer::ok(fake::microvm_response("PENDING", None)),
2045 )
2046 .answer(
2047 "GetMicrovm",
2048 Answer::ok(fake::microvm_response("RUNNING", None)),
2049 )
2050 .answer(
2051 "CreateMicrovmAuthToken",
2052 Answer::ok(fake::auth_token_response("proxy-token")),
2053 );
2054 }
2055
2056 #[tokio::test(start_paused = true)]
2057 async fn sandbox_launch_polls_at_one_second_then_backs_off() {
2058 for pending_polls in 0..=6 {
2059 let (mut sandbox, recorder, clock) = planted();
2060 recorder.answer(
2061 "RunMicrovm",
2062 Answer::ok(fake::microvm_response("PENDING", None)),
2063 );
2064 for _ in 0..pending_polls {
2065 recorder.answer(
2066 "GetMicrovm",
2067 Answer::ok(fake::microvm_response("PENDING", None)),
2068 );
2069 }
2070 recorder
2071 .answer(
2072 "GetMicrovm",
2073 Answer::ok(fake::microvm_response("RUNNING", None)),
2074 )
2075 .answer(
2076 "CreateMicrovmAuthToken",
2077 Answer::ok(fake::auth_token_response("proxy-token")),
2078 );
2079 sandbox
2080 .run(RunRequest::new().with_image("arn:image"))
2081 .await
2082 .expect("ready");
2083 let expected_seconds = [0, 1, 2, 3, 8, 13, 18][pending_polls];
2084 assert_eq!(clock.now(), Duration::from_secs(expected_seconds));
2085 assert_eq!(recorder.call_count("GetMicrovm"), pending_polls + 1);
2086 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2087 sandbox.detach().expect("hand the scripted VM off quietly");
2088 }
2089
2090 let (sandbox, recorder, clock) = planted();
2092 recorder
2093 .answer(
2094 "GetMicrovm",
2095 Answer::ok(fake::microvm_response("PENDING", None)),
2096 )
2097 .answer(
2098 "GetMicrovm",
2099 Answer::ok(fake::microvm_response("RUNNING", None)),
2100 );
2101 sandbox
2102 .control
2103 .wait_for_launch("mvm-abc123", WaitOpts::for_launch())
2104 .await
2105 .expect("ready");
2106 assert_eq!(clock.now(), Duration::from_secs(5));
2107
2108 let (mut sandbox, recorder, _) = planted();
2110 recorder
2111 .answer(
2112 "RunMicrovm",
2113 Answer::ok(fake::microvm_response("PENDING", None)),
2114 )
2115 .answer(
2116 "GetMicrovm",
2117 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2118 );
2119 let error = sandbox
2120 .run(RunRequest::new().with_image("arn:image"))
2121 .await
2122 .expect_err("a fresh launch fails fast");
2123 assert_eq!(error.kind(), ErrorKind::LaunchDied);
2124 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
2125 sandbox.detach().expect("hand the scripted VM off quietly");
2126 }
2127
2128 #[tokio::test(start_paused = true)]
2129 async fn vpc_egress_configuration_survives_the_sandbox_launch() {
2130 let (mut sandbox, recorder, _) = planted();
2131 answer_launch(&recorder);
2132 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2133 let request = RunRequest::new()
2134 .with_image("arn:image")
2135 .with_egress_network_connector(arn);
2136 assert_eq!(
2137 request.egress_posture(),
2138 crate::control::EgressPosture::Unsealed
2139 );
2140 sandbox.run(request).await.expect("launch");
2141 let body = recorder.first_body("RunMicrovm");
2142 assert_eq!(body["egressNetworkConnectors"], serde_json::json!([arn]));
2143 }
2144
2145 #[tokio::test(start_paused = true)]
2151 async fn a_sandbox_session_reads_the_planes_clock() {
2152 let (mut sandbox, recorder, clock) = planted();
2153 answer_launch(&recorder);
2154 let session = sandbox
2155 .run(RunRequest::new().with_image("arn:image"))
2156 .await
2157 .expect("the launch reaches RUNNING");
2158 let pinned = Duration::from_secs(1_767_225_600) + Duration::from_nanos(987_654_321);
2159 clock.set_unix_now(pinned);
2160 let id = session.mint_exec_id();
2161 let value =
2162 u64::from_str_radix(id.strip_prefix("x-").expect("the x- shape"), 16).expect("hex");
2163 let low_bits = (1_u64 << 40) - 1;
2164 assert_eq!(
2165 value & low_bits,
2166 (pinned.as_nanos() as u64) & low_bits,
2167 "{id} wasn't minted from the plane's clock"
2168 );
2169 }
2170
2171 #[tokio::test]
2178 async fn ensure_image_builds_its_services_through_the_planes_adapters() {
2179 let (mut sandbox, recorder, _) = planted();
2180 let dockerfile = crate::control::artifact::wrap_dockerfile(
2181 "FROM python:3.12-slim\nWORKDIR /app\n",
2182 &crate::control::artifact::WrapOptions::default(),
2183 )
2184 .expect("a wrappable Dockerfile");
2185 let request = crate::control::EnsureImageRequest::new(
2186 "task",
2187 crate::testing::aarch64_daemon(b"daemon"),
2188 dockerfile,
2189 "artifact-bucket",
2190 "arn:aws:iam::123456789012:role/build",
2191 );
2192 let error = sandbox
2193 .ensure_image(request)
2194 .await
2195 .expect_err("the test adapters build no services");
2196 assert_eq!(error.kind(), ErrorKind::Credentials, "{error}");
2197 assert!(
2198 error
2199 .to_string()
2200 .contains("the test adapters build no STS or S3 client for us-east-1"),
2201 "{error}"
2202 );
2203 assert!(
2204 recorder.calls().is_empty(),
2205 "no control-plane call came first"
2206 );
2207 }
2208
2209 #[tokio::test(start_paused = true)]
2217 async fn a_launch_draws_every_random_value_from_the_planes_entropy() {
2218 use crate::entropy::testing::SequenceEntropy;
2219 use base64::Engine as _;
2220
2221 let recorder = Arc::new(FakeControlPlane::new());
2222 let entropy = Arc::new(SequenceEntropy::new());
2223 let plane = crate::testing::control_plane(
2224 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2225 Region::UsEast1,
2226 Arc::new(TestClock::new()),
2227 )
2228 .with_entropy(Arc::clone(&entropy) as Arc<dyn crate::entropy::Entropy>);
2229 let mut sandbox = Sandbox::with_control_plane(plane)
2230 .with_session_backend(crate::testing::HealthyDaemon::new());
2231 answer_launch(&recorder);
2232 let session = sandbox
2233 .run(RunRequest::new().with_image("arn:image").with_identity())
2234 .await
2235 .expect("the launch reaches RUNNING");
2236
2237 assert_eq!(
2238 session.agent_token(),
2239 const_hex::encode(SequenceEntropy::draw(1, 32)),
2240 "the agent token"
2241 );
2242 let body = recorder.first_body("RunMicrovm");
2243 let payload: serde_json::Value =
2244 serde_json::from_str(body["runHookPayload"].as_str().expect("a string payload"))
2245 .expect("the payload is JSON");
2246 assert_eq!(
2247 payload[protocol::identity::SEED_KEY],
2248 base64::engine::general_purpose::STANDARD
2249 .encode(SequenceEntropy::draw(2, protocol::identity::SEED_BYTES)),
2250 "the VM seed"
2251 );
2252 let client_token = body["clientToken"].as_str().expect("a client token");
2253 assert!(
2254 client_token.ends_with(&const_hex::encode(SequenceEntropy::draw(4, 8))),
2255 "the client token's nonce: {client_token}"
2256 );
2257 assert_eq!(
2258 entropy.calls(),
2259 4,
2260 "one draw per value, the host seed included"
2261 );
2262 sandbox.detach().expect("hand the scripted VM off quietly");
2263 }
2264
2265 #[tokio::test]
2267 async fn an_unavailable_entropy_source_refuses_the_launch() {
2268 use crate::entropy::testing::SequenceEntropy;
2269
2270 let recorder = Arc::new(FakeControlPlane::new());
2271 let plane = crate::testing::control_plane(
2272 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2273 Region::UsEast1,
2274 Arc::new(TestClock::new()),
2275 )
2276 .with_entropy(Arc::new(SequenceEntropy::unavailable()));
2277 let mut sandbox = Sandbox::with_control_plane(plane)
2278 .with_session_backend(crate::testing::HealthyDaemon::new());
2279 answer_launch(&recorder);
2280 let err = sandbox
2281 .run(RunRequest::new().with_image("arn:image"))
2282 .await
2283 .expect_err("no pool, no bearer token");
2284 assert_eq!(err.kind(), ErrorKind::Unexpected, "{err}");
2285 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2286 }
2287
2288 fn launching_to(
2293 replies: Vec<crate::testing::Reply>,
2294 ) -> (
2295 Sandbox,
2296 Arc<FakeControlPlane>,
2297 Arc<crate::testing::Recorder>,
2298 ) {
2299 let recorder = Arc::new(FakeControlPlane::new());
2300 let plane = crate::testing::control_plane(
2301 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2302 Region::UsEast1,
2303 Arc::new(TestClock::new()),
2304 );
2305 let daemon = crate::testing::Recorder::with(replies);
2306 let sandbox = Sandbox::with_control_plane(plane).with_session_backend(daemon.clone());
2307 answer_launch(&recorder);
2308 (sandbox, recorder, daemon)
2309 }
2310
2311 #[tokio::test(start_paused = true)]
2312 async fn launch_metrics_track_retries_running_and_ready_without_changing_on_clones() {
2313 use crate::clock::Clock;
2314 let recorder = Arc::new(FakeControlPlane::new());
2315 recorder.answer("RunMicrovm", Answer::throttled_as_400());
2316 answer_launch(&recorder);
2317 let clock = Arc::new(TestClock::new());
2318 let plane = crate::testing::control_plane(recorder.clone(), Region::UsEast1, clock.clone());
2319 let mut sandbox = Sandbox::with_control_plane(plane)
2320 .with_session_backend(crate::testing::HealthyDaemon::new());
2321 let mut request = RunRequest::new().with_image("arn:image");
2322 request.wait = false;
2323 let session = sandbox.run(request).await.unwrap().clone();
2324 let accepted = session.launch_metrics().unwrap();
2325 assert_eq!(accepted.attempt_count, 2);
2326 assert!(accepted.backoff_seconds > 0.0);
2327 assert_eq!(accepted.running_at, None);
2328 assert_eq!(accepted.ready_at, None);
2329 clock.advance(Duration::from_secs(7));
2330 sandbox
2331 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
2332 .await
2333 .unwrap();
2334 let ready = session.launch_metrics().unwrap();
2335 assert_eq!(ready.attempt_count, accepted.attempt_count);
2336 assert_eq!(ready.backoff_seconds, accepted.backoff_seconds);
2337 assert_eq!(ready.running_at, Some(clock.unix_now().as_secs_f64()));
2338 assert_eq!(ready.ready_at, ready.running_at);
2339 clock.advance(Duration::from_secs(3));
2340 session
2341 .wait_until_ready(Duration::from_secs(1))
2342 .await
2343 .unwrap();
2344 assert_eq!(session.launch_metrics(), Some(ready));
2345 sandbox.detach().unwrap();
2346 }
2347
2348 #[tokio::test]
2349 async fn cancellation_preserves_an_inflight_accepted_launch() {
2350 use crate::control::transport::RetryCancellation;
2351 let (mut sandbox, recorder, _) = planted();
2352 answer_launch(&recorder);
2353 let gate = recorder.hold_responses();
2354 let cancellation = RetryCancellation::default();
2355 let mut request = RunRequest::new().with_image("arn:image");
2356 request.wait = false;
2357 {
2358 let pending = sandbox.run_cancellable(request, &cancellation);
2359 tokio::pin!(pending);
2360 tokio::select! {
2361 biased;
2362 _ = &mut pending => panic!("response must wait for the gate"),
2363 _ = std::future::ready(()) => {}
2364 }
2365 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2366 cancellation.cancel();
2367 gate.notify_one();
2368 assert_eq!(
2369 pending
2370 .await
2371 .unwrap()
2372 .launch_metrics()
2373 .unwrap()
2374 .attempt_count,
2375 1
2376 );
2377 }
2378 assert!(sandbox.microvm().is_some());
2379 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2380 sandbox.detach().unwrap();
2381 }
2382
2383 #[tokio::test]
2384 async fn cancelled_retry_does_not_launch_before_any_request() {
2385 use crate::control::transport::RetryCancellation;
2386 let (mut sandbox, recorder, _) = planted();
2387 recorder.answer(
2388 "RunMicrovm",
2389 Answer::failure(403, "pre-cancelled request reached the transport"),
2390 );
2391 let cancellation = RetryCancellation::default();
2392 cancellation.cancel();
2393 let error = sandbox
2394 .run_cancellable(RunRequest::new().with_image("arn:image"), &cancellation)
2395 .await
2396 .unwrap_err();
2397 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2398 assert_eq!(error.kind(), ErrorKind::Interrupted);
2399 assert!(sandbox.microvm().is_none());
2400 }
2401
2402 #[tokio::test]
2403 async fn cancellation_interrupts_image_resolution_and_releases_the_sandbox() {
2404 use crate::control::transport::{RetryCancellation, RetryPolicy};
2405 let recorder = Arc::new(FakeControlPlane::new());
2406 recorder.answer("ListMicrovmImages", Answer::throttled_as_400());
2407 let plane = crate::testing::control_plane(
2408 recorder.clone(),
2409 Region::UsEast1,
2410 Arc::new(crate::testing::YieldingClock::new()),
2411 )
2412 .with_retry(RetryPolicy::new(100, Duration::from_secs(3600)).unwrap());
2413 let sandbox = tokio::sync::Mutex::new(Sandbox::with_control_plane(plane));
2414 let cancellation = RetryCancellation::default();
2415 let pending = async {
2416 sandbox
2417 .lock()
2418 .await
2419 .run_cancellable(RunRequest::new().with_image("named-image"), &cancellation)
2420 .await
2421 .map(|_| ())
2422 };
2423 tokio::pin!(pending);
2424 tokio::select! {
2425 biased;
2426 _ = &mut pending => panic!("image resolution must be retrying"),
2427 _ = std::future::ready(()) => {}
2428 }
2429 assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2430 assert!(sandbox.try_lock().is_err());
2431 cancellation.cancel();
2432 let error = tokio::select! {
2433 biased;
2434 result = &mut pending => result.unwrap_err(),
2435 _ = std::future::ready(()) => panic!("cancellation must finish without another sleep"),
2436 };
2437 assert_eq!(error.kind(), ErrorKind::Interrupted);
2438 assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2439 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2440 let sandbox = sandbox.try_lock().expect("cancellation releases the lock");
2441 assert!(sandbox.microvm().is_none());
2442 }
2443
2444 fn not_ready_then_ready() -> Vec<crate::testing::Reply> {
2446 vec![
2447 crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()),
2448 crate::testing::Reply::ok(crate::testing::health_body(true)),
2449 ]
2450 }
2451
2452 fn daemon_paths(daemon: &crate::testing::Recorder) -> Vec<String> {
2454 daemon
2455 .requests()
2456 .iter()
2457 .map(|request| format!("{} {}", request.method, request.path))
2458 .collect()
2459 }
2460
2461 #[tokio::test(start_paused = true)]
2468 async fn a_launch_returns_once_its_daemon_answers() {
2469 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2470 sandbox
2471 .run(RunRequest::new().with_image("arn:image"))
2472 .await
2473 .expect("the launch reaches RUNNING and its daemon answers");
2474 assert_eq!(
2475 daemon_paths(&daemon),
2476 ["GET /v1/health", "GET /v1/health"],
2477 "one poll refused, one bootstrapped, and nothing after"
2478 );
2479 sandbox.detach().expect("hand the scripted VM off quietly");
2480 }
2481
2482 #[tokio::test(start_paused = true)]
2486 async fn wait_until_running_returns_once_the_daemon_answers() {
2487 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2488 let mut request = RunRequest::new().with_image("arn:image");
2489 request.wait = false;
2490 sandbox.run(request).await.expect("the launch is accepted");
2491 assert!(
2492 daemon_paths(&daemon).is_empty(),
2493 "an accepted launch doesn't wait"
2494 );
2495 sandbox
2496 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
2497 .await
2498 .expect("RUNNING, then the daemon answers");
2499 assert_eq!(daemon_paths(&daemon), ["GET /v1/health", "GET /v1/health"]);
2500 sandbox.detach().expect("hand the scripted VM off quietly");
2501 }
2502
2503 #[tokio::test(start_paused = true)]
2507 async fn a_launch_whose_daemon_never_answers_times_out_running() {
2508 let never = (0..128)
2509 .map(|_| crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()))
2510 .collect();
2511 let (mut sandbox, _recorder, daemon) = launching_to(never);
2512 let error = sandbox
2513 .run(RunRequest::new().with_image("arn:image"))
2514 .await
2515 .expect_err("the daemon never answered");
2516 assert_eq!(error.kind(), ErrorKind::Timeout, "{error}");
2517 assert!(daemon.requests().len() > 1, "it kept asking");
2518 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2519 assert!(sandbox.token_installed());
2520 sandbox.detach().expect("hand the scripted VM off quietly");
2521 }
2522
2523 const WANTED_ARN: &str = "arn:aws:lambda:us-east-1:123456789012:microvm-image:wanted-image";
2527
2528 fn answer_listing(recorder: &FakeControlPlane) {
2530 recorder.answer(
2531 "ListMicrovmImages",
2532 Answer::ok(fake::list_images_response(
2533 &["wanted-image-old", "wanted-image"],
2534 None,
2535 )),
2536 );
2537 }
2538
2539 #[tokio::test(start_paused = true)]
2546 async fn a_launch_by_bare_image_name_sends_the_listed_arn() {
2547 let (mut sandbox, recorder, _) = planted();
2548 answer_listing(&recorder);
2549 answer_launch(&recorder);
2550 sandbox
2551 .run(RunRequest::new().with_image("wanted-image"))
2552 .await
2553 .expect("the launch reaches RUNNING");
2554
2555 let operations = recorder.operations();
2556 assert_eq!(
2557 operations.get(..2),
2558 Some(&["ListMicrovmImages", "RunMicrovm"][..]),
2559 "the name is resolved, once, before the launch: {operations:?}"
2560 );
2561 assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2562 assert_eq!(
2563 recorder.first_body("RunMicrovm")["imageIdentifier"],
2564 WANTED_ARN,
2565 "the launch carries the exact match's ARN, never the name"
2566 );
2567 sandbox.detach().expect("hand the scripted VM off quietly");
2568 }
2569
2570 #[tokio::test(start_paused = true)]
2572 async fn a_launch_by_image_arn_makes_no_listing_call() {
2573 let (mut sandbox, recorder, _) = planted();
2574 answer_launch(&recorder);
2575 sandbox
2576 .run(RunRequest::new().with_image(WANTED_ARN))
2577 .await
2578 .expect("the launch reaches RUNNING");
2579
2580 assert_eq!(recorder.call_count("ListMicrovmImages"), 0);
2581 assert_eq!(recorder.operations().first(), Some(&"RunMicrovm"));
2582 assert_eq!(
2583 recorder.first_body("RunMicrovm")["imageIdentifier"],
2584 WANTED_ARN
2585 );
2586 assert_eq!(sandbox.launch_image_arn(), Some(WANTED_ARN));
2587 sandbox.detach().expect("hand the scripted VM off quietly");
2588 }
2589
2590 #[tokio::test]
2597 async fn a_refused_launch_by_name_still_records_the_resolved_arn() {
2598 let (mut sandbox, recorder, _) = planted();
2599 answer_listing(&recorder);
2600 recorder.answer("RunMicrovm", Answer::failure(400, "scripted refusal"));
2601 sandbox
2602 .run(RunRequest::new().with_image("wanted-image"))
2603 .await
2604 .expect_err("the service refused the launch");
2605
2606 assert_eq!(
2607 sandbox.launch_image_arn(),
2608 Some(WANTED_ARN),
2609 "the ARN the refused launch asked for"
2610 );
2611 assert!(sandbox.microvm().is_none(), "no launch was accepted");
2612 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2613 }
2614
2615 #[tokio::test(start_paused = true)]
2619 async fn an_unknown_image_name_is_refused_before_the_launch_and_the_sandbox_still_runs() {
2620 let (mut sandbox, recorder, _) = planted();
2621 recorder.answer(
2622 "ListMicrovmImages",
2623 Answer::ok(fake::list_images_response(&[], None)),
2624 );
2625 let error = sandbox
2626 .run(RunRequest::new().with_image("no-such-image"))
2627 .await
2628 .expect_err("nothing to launch from");
2629 assert_eq!(error.code(), "ERR_PRECONDITION");
2630 assert!(error.to_string().contains("no-such-image"), "{error}");
2631 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2632 assert_eq!(sandbox.launch_image_arn(), None);
2633 assert!(!sandbox.image_exists(), "STATE-1: no launch was accepted");
2634 assert!(sandbox.microvm().is_none());
2635 assert_eq!(sandbox.bootstrap_count(), 0);
2636
2637 answer_launch(&recorder);
2638 sandbox
2639 .run(RunRequest::new().with_image(WANTED_ARN))
2640 .await
2641 .expect("a failed resolution doesn't use up the sandbox's one launch");
2642 assert_eq!(sandbox.bootstrap_count(), 1);
2643 sandbox.detach().expect("hand the scripted VM off quietly");
2644 }
2645
2646 #[tokio::test]
2652 async fn a_launch_refused_locally_makes_no_listing_call_for_a_name() {
2653 let (mut sandbox, recorder, _) = planted();
2654 answer_listing(&recorder);
2655 let mut request = RunRequest::new().with_image("wanted-image");
2656 request.max_duration_sec = 0;
2657 let error = sandbox
2658 .run(request)
2659 .await
2660 .expect_err("a zero maximum duration is refused locally");
2661 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
2662 assert!(
2663 recorder.calls().is_empty(),
2664 "the refusal came after a call: {:?}",
2665 recorder.operations()
2666 );
2667 assert_eq!(sandbox.launch_image_arn(), None);
2668 }
2669
2670 #[tokio::test(start_paused = true)]
2677 async fn a_launched_session_reports_its_requests_posture() {
2678 use crate::control::EgressPosture;
2679 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2680 let rows = [
2681 (RunRequest::new(), EgressPosture::Unsealed),
2682 (RunRequest::new().with_egress(), EgressPosture::Open),
2683 (
2684 RunRequest::new().with_deny_egress(),
2685 EgressPosture::BestEffort,
2686 ),
2687 (
2688 RunRequest::new().with_egress_network_connector(arn),
2689 EgressPosture::Unsealed,
2690 ),
2691 ];
2692 for (request, expected) in rows {
2693 let answered = crate::control::egress_posture_for(
2694 request.egress,
2695 &request.egress_network_connectors,
2696 request.deny_egress,
2697 Some(&Region::UsEast1),
2698 )
2699 .expect("a launchable request");
2700 assert_eq!(answered, expected);
2701 let (mut sandbox, recorder, _) = planted();
2702 answer_launch(&recorder);
2703 let session = sandbox
2704 .run(request.with_image("arn:image"))
2705 .await
2706 .expect("the launch reaches RUNNING");
2707 assert_eq!(session.egress_posture(), expected, "BIND-12");
2708 assert_eq!(sandbox.egress_posture(), expected);
2709 sandbox.detach().expect("hand the scripted VM off quietly");
2710 }
2711 }
2712
2713 #[tokio::test]
2717 async fn a_session_without_its_launch_options_reports_unsealed() {
2718 use crate::control::EgressPosture;
2719 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2720 assert_eq!(
2721 sandbox.session().expect("RUNNING").egress_posture(),
2722 EgressPosture::Unsealed
2723 );
2724 assert_eq!(sandbox.egress_posture(), EgressPosture::Unsealed);
2725 let direct = Session::builder(ADOPT_ENDPOINT, ADOPT_TOKEN)
2726 .build_with(&crate::testing::TestAdapters::new())
2727 .expect("a direct session");
2728 assert_eq!(direct.egress_posture(), EgressPosture::Unsealed);
2729 }
2730
2731 async fn launched() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2733 let (mut sandbox, recorder, clock) = planted();
2734 answer_launch(&recorder);
2735 sandbox
2736 .run(RunRequest::new().with_image("arn:image"))
2737 .await
2738 .expect("the launch reaches RUNNING");
2739 (sandbox, recorder, clock)
2740 }
2741
2742 #[tokio::test(start_paused = true)]
2750 async fn a_dropped_live_sandbox_warns_through_the_planes_adapters() {
2751 let recorder = Arc::new(FakeControlPlane::new());
2752 let clock = Arc::new(TestClock::new());
2753 let adapters = Arc::new(crate::testing::TestAdapters::new());
2754 let plane = crate::control::ControlPlane::from_ports(
2755 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2756 Region::UsEast1,
2757 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2758 Arc::new(crate::testing::SequenceEntropy::new()),
2759 Arc::clone(&adapters) as Arc<dyn crate::adapters::Adapters>,
2760 );
2761 let mut sandbox = Sandbox::with_control_plane(plane)
2762 .with_session_backend(crate::testing::HealthyDaemon::new());
2763 answer_launch(&recorder);
2764 sandbox
2765 .run(RunRequest::new().with_image("arn:image"))
2766 .await
2767 .expect("the launch reaches RUNNING");
2768 let id = sandbox.microvm().expect("launched").id.clone();
2769 drop(sandbox);
2770 let warnings = adapters.warnings();
2771 assert_eq!(warnings.len(), 1, "{warnings:?}");
2772 assert!(warnings[0].contains(&id), "{warnings:?}");
2773 assert!(
2774 warnings[0].contains("RUNNING is the state this sandbox last saw")
2775 && warnings[0].contains("A VM that's still live bills"),
2776 "{warnings:?}"
2777 );
2778 }
2779
2780 #[tokio::test(start_paused = true)]
2790 async fn a_detached_sandbox_hands_off_its_vm_and_stays_quiet() {
2791 let (mut sandbox, recorder, _clock) = launched().await;
2792 assert!(
2793 sandbox.drop_warning().is_some(),
2794 "a launched, live VM warns on drop"
2795 );
2796 let token = sandbox
2797 .session()
2798 .expect("launched")
2799 .agent_token()
2800 .to_string();
2801 let vm = sandbox.microvm().expect("launched").clone();
2802
2803 let detached = sandbox.detach().expect("a RUNNING VM detaches");
2804 assert_eq!(detached.microvm_id, vm.id);
2805 assert_eq!(detached.endpoint, vm.endpoint);
2806 assert_eq!(detached.agent_token(), token);
2807 assert_eq!(detached.region, Region::UsEast1);
2808 let printed = format!("{detached:?} {sandbox:?}");
2809 assert!(!printed.contains(&token), "{printed}");
2810 assert!(printed.contains("detached: true"), "{printed}");
2811
2812 assert!(sandbox.detached());
2813 assert!(
2814 sandbox.drop_warning().is_none(),
2815 "detached is not abandoned"
2816 );
2817 assert!(
2818 sandbox.session().is_none(),
2819 "the session went to the adopter"
2820 );
2821
2822 let before = recorder.calls().len();
2823 for error in [
2824 sandbox.suspend().await.expect_err("detached"),
2825 sandbox.resume().await.map(|_| ()).expect_err("detached"),
2826 sandbox
2827 .run(RunRequest::new().with_image("arn:image"))
2828 .await
2829 .map(|_| ())
2830 .expect_err("detached"),
2831 sandbox
2832 .wait_until_running(Duration::from_secs(1))
2833 .await
2834 .map(|_| ())
2835 .expect_err("detached"),
2836 sandbox.detach().map(|_| ()).expect_err("already detached"),
2837 ] {
2838 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2839 assert!(error.to_string().contains("detached"), "{error}");
2840 assert!(!error.to_string().contains(&token), "{error}");
2841 }
2842 let report = sandbox.terminate(TeardownOpts::default()).await;
2843 assert!(!report.terminate_accepted);
2844 assert!(report.failures[0].contains("detached"), "{report:?}");
2845 assert_eq!(recorder.calls().len(), before, "detached makes no AWS call");
2846 assert!(
2847 sandbox.drop_warning().is_none(),
2848 "a refused terminate is still quiet"
2849 );
2850 }
2851
2852 #[tokio::test(start_paused = true)]
2854 async fn detach_refuses_without_a_live_vm() {
2855 let (mut unlaunched, _recorder, _clock) = planted();
2856 let error = unlaunched.detach().expect_err("nothing launched");
2857 assert_eq!(error.kind(), ErrorKind::Precondition);
2858 assert!(!unlaunched.detached());
2859
2860 let (mut sandbox, recorder, _clock) = launched().await;
2861 recorder
2862 .answer("TerminateMicrovm", Answer::ok("{}"))
2863 .answer(
2864 "GetMicrovm",
2865 Answer::ok(fake::microvm_response("TERMINATED", None)),
2866 );
2867 sandbox.terminate(TeardownOpts::default()).await;
2868 let error = sandbox.detach().expect_err("torn down");
2869 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2870 assert!(!sandbox.detached());
2871 }
2872
2873 async fn suspended_with_window(
2884 suspended_sec: u32,
2885 ) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2886 let (mut sandbox, recorder, clock) = planted();
2887 recorder
2888 .answer(
2889 "RunMicrovm",
2890 Answer::ok(fake::microvm_response("PENDING", None)),
2891 )
2892 .answer(
2893 "GetMicrovm",
2894 Answer::ok(fake::microvm_response("RUNNING", None)),
2895 )
2896 .answer(
2897 "GetMicrovm",
2898 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2899 )
2900 .answer(
2901 "CreateMicrovmAuthToken",
2902 Answer::ok(fake::auth_token_response("proxy-token")),
2903 )
2904 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2905
2906 sandbox
2907 .run(
2908 RunRequest::new()
2909 .with_image("arn:image")
2910 .with_suspended_sec(suspended_sec),
2911 )
2912 .await
2913 .expect("launches");
2914 sandbox.suspend().await.expect("suspends");
2915 assert_eq!(
2916 clock.now(),
2917 Duration::ZERO,
2918 "no wait may have slept, or a test that reads the clock is measuring the fake's \
2919 poll interval as well as the elapsed suspension"
2920 );
2921 (sandbox, recorder, clock)
2922 }
2923
2924 async fn suspended() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2926 suspended_with_window(600).await
2927 }
2928
2929 const ADOPT_TOKEN: &str = "adopt-canary-token-9f1c";
2933 const ADOPT_ENDPOINT: &str = "https://mvm-abc123.microvm.us-east-1.amazonaws.com";
2934
2935 fn adopt_plane() -> (ControlPlane, Arc<FakeControlPlane>, Arc<TestClock>) {
2937 let recorder = Arc::new(FakeControlPlane::new());
2938 let clock = Arc::new(TestClock::new());
2939 let plane = crate::testing::control_plane(
2940 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2941 Region::UsEast1,
2942 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2943 );
2944 (plane, recorder, clock)
2945 }
2946
2947 async fn adopted_in(state: &str) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2948 let (plane, recorder, clock) = adopt_plane();
2949 recorder.answer(
2950 "GetMicrovm",
2951 Answer::ok(fake::microvm_response(state, None)),
2952 );
2953 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2954 .await
2955 .expect("adopts");
2956 (sandbox, recorder, clock)
2957 }
2958
2959 #[tokio::test]
2966 async fn an_adopted_sandbox_reports_the_services_idle_window_and_watches_its_lifecycle() {
2967 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2968 assert_eq!(sandbox.idle_window(), Some(Duration::from_secs(1800)));
2969 assert_eq!(*sandbox.watch_lifecycle().borrow(), Lifecycle::Running);
2970 }
2971
2972 #[test]
2974 fn every_service_state_maps_onto_one_lifecycle() {
2975 for state in crate::constants::MICROVM_STATES {
2976 let lifecycle = Lifecycle::from_service(state).expect("a known state");
2977 assert_eq!(lifecycle.as_str(), state);
2978 }
2979 assert_eq!(Lifecycle::from_service("Running"), None);
2980 }
2981
2982 #[tokio::test]
2990 async fn an_adopted_vm_takes_its_lifecycle_from_the_service_and_refuses_run() {
2991 let (mut sandbox, recorder, _) = adopted_in("RUNNING").await;
2992 assert!(sandbox.adopted());
2993 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2994 assert!(sandbox.token_installed());
2995 assert_eq!(sandbox.bootstrap_count(), 1);
2996 assert!(sandbox.image_exists());
2997 let session = sandbox.session().expect("a live VM gets a session");
2998 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
2999 assert_eq!(session.agent_token(), ADOPT_TOKEN);
3000
3001 let error = sandbox
3002 .run(RunRequest::new().with_image("arn:image"))
3003 .await
3004 .expect_err("an adopted VM was bootstrapped by its own launch");
3005 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3006 assert!(error.to_string().contains("STATE-3"), "{error}");
3007 assert_eq!(recorder.call_count("RunMicrovm"), 0);
3008 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
3009 }
3010
3011 #[tokio::test]
3017 async fn a_name_adopts_its_records_vm_and_foreign_names_are_refused_locally() {
3018 struct One(crate::names::NameRecord);
3019 impl crate::names::NameStore for One {
3020 fn get(&self, name: &str) -> Result<Option<crate::names::NameRecord>, Error> {
3021 Ok((name == self.0.name).then(|| self.0.clone()))
3022 }
3023 fn put(&self, _: &crate::names::NameRecord) -> Result<(), Error> {
3024 unreachable!()
3025 }
3026 fn delete(&self, _: &str) -> Result<bool, Error> {
3027 unreachable!()
3028 }
3029 fn list(&self) -> Result<Vec<crate::names::NameRecord>, Error> {
3030 Ok(vec![self.0.clone()])
3031 }
3032 }
3033 let record = crate::names::NameRecord::new_at(
3034 "ci",
3035 "mvm-abc123",
3036 ADOPT_ENDPOINT,
3037 ADOPT_TOKEN,
3038 "us-west-2",
3039 1_767_225_600,
3040 )
3041 .expect("record");
3042 let store = One(record.clone());
3043 let foreign = crate::names::resolve(&store, "ci", Some(&Region::UsEast1))
3044 .expect_err("registered in another region");
3045 assert_eq!(foreign.kind(), ErrorKind::InvalidArg);
3046 let missing = crate::names::resolve(&store, "nope", None).expect_err("no such name");
3047 assert_eq!(missing.kind(), ErrorKind::Precondition);
3048 assert!(!format!("{foreign} {missing}").contains(ADOPT_TOKEN));
3049
3050 let (plane, recorder, _) = adopt_plane();
3051 recorder.answer(
3052 "GetMicrovm",
3053 Answer::ok(fake::microvm_response("RUNNING", None)),
3054 );
3055 let resolved = crate::names::resolve(&store, "ci", Some(&Region::UsWest2)).expect("ok");
3056 let sandbox = Sandbox::adopt_record(plane, resolved)
3057 .await
3058 .expect("adopts");
3059 assert!(sandbox.adopted());
3060 assert_eq!(
3061 sandbox.microvm().map(|vm| vm.id.as_str()),
3062 Some("mvm-abc123")
3063 );
3064 assert_eq!(
3065 sandbox.session().expect("session").agent_token(),
3066 ADOPT_TOKEN
3067 );
3068 let named = sandbox
3069 .name_record("again")
3070 .expect("an adopted VM can be named");
3071 assert_eq!(
3072 (
3073 named.microvm_id.as_str(),
3074 named.agent_token.as_str(),
3075 named.region.as_str()
3076 ),
3077 ("mvm-abc123", ADOPT_TOKEN, "us-east-1")
3078 );
3079 let unlaunched = Sandbox::with_control_plane(adopt_plane().0);
3080 assert_eq!(
3081 unlaunched
3082 .name_record("x")
3083 .expect_err("nothing to name")
3084 .kind(),
3085 ErrorKind::Precondition
3086 );
3087 }
3088
3089 #[tokio::test]
3095 async fn a_records_tunnel_identity_survives_adoption_and_naming() {
3096 let identity = crate::identity::LaunchIdentity::from_seeds([7; 32], [9; 32])
3097 .expect("valid seeds")
3098 .keep();
3099 let mut record = crate::names::NameRecord::new_at(
3100 "ci",
3101 "mvm-abc123",
3102 ADOPT_ENDPOINT,
3103 ADOPT_TOKEN,
3104 "us-east-1",
3105 1_767_225_600,
3106 )
3107 .expect("record");
3108 record.identity_host_seed = Some(identity.host_seed_base64());
3109 record.identity_vm_public_key = Some(identity.vm_public_key_base64());
3110
3111 let (plane, recorder, _) = adopt_plane();
3112 recorder.answer(
3113 "GetMicrovm",
3114 Answer::ok(fake::microvm_response("RUNNING", None)),
3115 );
3116 let sandbox = Sandbox::adopt_record(plane, record.clone())
3117 .await
3118 .expect("adopts");
3119 let kept = sandbox
3120 .tunnel_identity()
3121 .expect("the record's pair is kept");
3122 assert_eq!(kept.host_seed_base64(), identity.host_seed_base64());
3123 assert_eq!(kept.vm_public_key_base64(), identity.vm_public_key_base64());
3124 let named = sandbox.name_record("again").expect("names");
3125 assert_eq!(named.identity_host_seed, record.identity_host_seed);
3126 assert_eq!(named.identity_vm_public_key, record.identity_vm_public_key);
3127
3128 let (plane, recorder, _) = adopt_plane();
3129 let mut torn = record;
3130 torn.identity_vm_public_key = None;
3131 let refused = Sandbox::adopt_record(plane, torn)
3132 .await
3133 .expect_err("half a pair is refused");
3134 assert_eq!(refused.kind(), ErrorKind::InvalidArg);
3135 assert!(refused.to_string().contains("one half"), "{refused}");
3136 assert!(recorder.calls().is_empty(), "refused before any AWS call");
3137 }
3138
3139 #[tokio::test]
3142 async fn an_adopted_vm_suspends_resumes_and_terminates_through_its_handle() {
3143 let (plane, recorder, _) = adopt_plane();
3144 recorder
3145 .answer(
3146 "GetMicrovm",
3147 Answer::ok(fake::microvm_response("RUNNING", None)),
3148 )
3149 .answer(
3150 "GetMicrovm",
3151 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3152 )
3153 .answer(
3154 "GetMicrovm",
3155 Answer::ok(fake::microvm_response("RUNNING", None)),
3156 )
3157 .answer(
3158 "GetMicrovm",
3159 Answer::ok(fake::microvm_response("TERMINATED", None)),
3160 )
3161 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3162 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3163 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
3164 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3165 .await
3166 .expect("adopts");
3167
3168 sandbox.suspend().await.expect("suspends from RUNNING");
3169 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3170 sandbox.resume().await.expect("resumes from SUSPENDED");
3171 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3172 let report = sandbox
3173 .terminate(TeardownOpts::default().waiting_for_terminated())
3174 .await;
3175 assert!(report.terminate_accepted, "{report:?}");
3176 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3177 assert!(sandbox.was_terminated());
3178
3179 assert_eq!(recorder.call_count("RunMicrovm"), 0);
3180 assert!(
3181 recorder
3182 .bodies_as_text()
3183 .iter()
3184 .all(|body| !body.contains("runHookPayload") && !body.contains(ADOPT_TOKEN)),
3185 "no launch payload or agent token may reach the control plane"
3186 );
3187 assert_eq!(sandbox.bootstrap_count(), 1, "STATE-3: never above one");
3188 }
3189
3190 #[tokio::test]
3194 async fn an_adopted_vm_resumes_past_the_reported_window_through_the_service() {
3195 let (plane, recorder, clock) = adopt_plane();
3196 recorder
3197 .answer(
3198 "GetMicrovm",
3199 Answer::ok(fake::microvm_response("RUNNING", None)),
3200 )
3201 .answer(
3202 "GetMicrovm",
3203 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3204 )
3205 .answer(
3206 "GetMicrovm",
3207 Answer::ok(fake::microvm_response("RUNNING", None)),
3208 )
3209 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3210 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3211 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3212 .await
3213 .expect("adopts");
3214 assert_eq!(sandbox.suspended_window(), None, "no request of its own");
3215 sandbox.suspend().await.expect("suspends");
3216
3217 clock.advance(Duration::from_secs(601));
3218 sandbox
3219 .resume()
3220 .await
3221 .expect("past the 600 s window the service reported, the service resumes it");
3222 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3223 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3224 }
3225
3226 #[tokio::test]
3229 async fn an_adopted_suspended_vm_resumes_through_the_service() {
3230 let (plane, recorder, _) = adopt_plane();
3231 recorder
3232 .answer(
3233 "GetMicrovm",
3234 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3235 )
3236 .answer(
3237 "GetMicrovm",
3238 Answer::ok(fake::microvm_response("RUNNING", None)),
3239 )
3240 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3241 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3242 .await
3243 .expect("adopts");
3244 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3245 let session = sandbox
3246 .resume()
3247 .await
3248 .expect("the service answers the resume");
3249 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
3250 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3251 }
3252
3253 #[tokio::test]
3256 async fn an_adopted_terminated_vm_refuses_every_transition_locally() {
3257 let (mut sandbox, recorder, _) = adopted_in("TERMINATED").await;
3258 assert!(sandbox.was_terminated());
3259 assert!(sandbox.session().is_none());
3260 let resume = sandbox.resume().await.expect_err("STATE-11");
3261 assert!(resume.to_string().contains("STATE-11"), "{resume}");
3262 let suspend = sandbox.suspend().await.expect_err("STATE-5");
3263 assert!(suspend.to_string().contains("STATE-5"), "{suspend}");
3264 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
3265 }
3266
3267 async fn adopted_with_a_daemon(
3270 state: &str,
3271 ) -> (
3272 Sandbox,
3273 Arc<FakeControlPlane>,
3274 Arc<crate::testing::HealthyDaemon>,
3275 ) {
3276 let recorder = Arc::new(FakeControlPlane::new());
3277 let daemon = crate::testing::HealthyDaemon::new();
3278 let plane = crate::control::ControlPlane::from_ports(
3279 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3280 Region::UsEast1,
3281 Arc::new(TestClock::new()),
3282 Arc::new(crate::testing::SequenceEntropy::new()),
3283 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3284 );
3285 recorder
3286 .answer(
3287 "GetMicrovm",
3288 Answer::ok(fake::microvm_response(state, None)),
3289 )
3290 .answer(
3291 "GetMicrovm",
3292 Answer::ok(fake::microvm_response("RUNNING", None)),
3293 )
3294 .answer(
3295 "CreateMicrovmAuthToken",
3296 Answer::ok(fake::auth_token_response("proxy-token")),
3297 );
3298 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3299 .await
3300 .expect("adopts");
3301 (sandbox, recorder, daemon)
3302 }
3303
3304 #[tokio::test(start_paused = true)]
3311 async fn wait_until_ready_finishes_whichever_wait_is_left() {
3312 let (mut pending, recorder, daemon) = adopted_with_a_daemon("PENDING").await;
3313 pending
3314 .wait_until_ready(Duration::from_secs(60))
3315 .await
3316 .expect("RUNNING, then the daemon");
3317 assert_eq!(pending.lifecycle(), Lifecycle::Running);
3318 assert_eq!(pending.bootstrap_count(), 1);
3319 assert_eq!(daemon.polls(), 1);
3320 assert_eq!(
3321 recorder.call_count("GetMicrovm"),
3322 2,
3323 "the adopt's read and the wait's"
3324 );
3325
3326 let (mut running, recorder, daemon) = adopted_with_a_daemon("RUNNING").await;
3327 running
3328 .wait_until_ready(Duration::from_secs(60))
3329 .await
3330 .expect("the daemon answers");
3331 assert_eq!(daemon.polls(), 1);
3332 assert_eq!(
3333 recorder.call_count("GetMicrovm"),
3334 1,
3335 "RUNNING needs no second read"
3336 );
3337 }
3338
3339 #[tokio::test]
3341 async fn wait_until_ready_refuses_a_suspended_vm() {
3342 let (mut suspended, _, daemon) = adopted_with_a_daemon("SUSPENDED").await;
3343 let error = suspended
3344 .wait_until_ready(Duration::from_secs(60))
3345 .await
3346 .expect_err("a suspended VM isn't starting");
3347 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
3348 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3349 assert_eq!(daemon.polls(), 0);
3350 }
3351
3352 #[tokio::test(start_paused = true)]
3355 async fn an_adopted_pending_vm_counts_its_bootstrap_once_on_running() {
3356 let recorder = Arc::new(FakeControlPlane::new());
3357 let daemon = crate::testing::HealthyDaemon::new();
3358 let plane = crate::control::ControlPlane::from_ports(
3359 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3360 Region::UsEast1,
3361 Arc::new(TestClock::new()),
3362 Arc::new(crate::testing::SequenceEntropy::new()),
3363 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3364 );
3365 recorder
3366 .answer(
3367 "GetMicrovm",
3368 Answer::ok(fake::microvm_response("PENDING", None)),
3369 )
3370 .answer(
3371 "GetMicrovm",
3372 Answer::ok(fake::microvm_response("RUNNING", None)),
3373 )
3374 .answer(
3375 "CreateMicrovmAuthToken",
3376 Answer::ok(fake::auth_token_response("proxy-token")),
3377 );
3378 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3379 .await
3380 .expect("adopts");
3381 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3382 assert_eq!(sandbox.bootstrap_count(), 0);
3383 sandbox
3384 .wait_until_running(Duration::from_secs(60))
3385 .await
3386 .expect("reaches RUNNING");
3387 assert_eq!(sandbox.bootstrap_count(), 1);
3388 assert!(sandbox.token_installed());
3389 assert_eq!(
3390 daemon.polls(),
3391 1,
3392 "the daemon answered before the session came back"
3393 );
3394 assert!(
3395 sandbox
3396 .run(RunRequest::new().with_image("arn:image"))
3397 .await
3398 .is_err(),
3399 "still one bootstrap per VM"
3400 );
3401 assert_eq!(recorder.call_count("RunMicrovm"), 0);
3402 }
3403
3404 #[tokio::test]
3411 async fn adopt_refuses_bad_records_and_never_prints_the_token() {
3412 let (plane, recorder, _) = adopt_plane();
3413 let empty = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, "")
3414 .await
3415 .expect_err("no token");
3416 assert_eq!(empty.kind(), ErrorKind::InvalidArg);
3417 assert_eq!(recorder.calls().len(), 0, "refused before any call");
3418
3419 let (plane, recorder, _) = adopt_plane();
3420 recorder.answer(
3421 "GetMicrovm",
3422 Answer::ok(fake::microvm_response("RUNNING", None)),
3423 );
3424 let mismatch = Sandbox::adopt(plane, "mvm-abc123", "https://other.example", ADOPT_TOKEN)
3425 .await
3426 .expect_err("the endpoint belongs to another record");
3427 assert_eq!(mismatch.kind(), ErrorKind::InvalidArg);
3428 assert!(!mismatch.to_string().contains(ADOPT_TOKEN));
3429
3430 let (plane, recorder, _) = adopt_plane();
3431 recorder.answer(
3432 "GetMicrovm",
3433 Answer::ok(fake::microvm_response("REBOOTING", None)),
3434 );
3435 let unknown = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3436 .await
3437 .expect_err("an unknown state");
3438 assert_eq!(unknown.kind(), ErrorKind::Platform);
3439 assert!(!unknown.to_string().contains(ADOPT_TOKEN));
3440
3441 let (sandbox, _, _) = adopted_in("RUNNING").await;
3442 let printed = format!("{sandbox:?}");
3443 assert!(!printed.contains(ADOPT_TOKEN), "{printed}");
3444 assert!(printed.contains("adopted: true"), "{printed}");
3445 }
3446
3447 #[tokio::test(start_paused = true)]
3457 async fn a_launch_records_the_image_and_the_running_report_installs_the_token() {
3458 let (mut sandbox, recorder, _) = planted();
3459 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3460 assert!(
3461 !sandbox.image_exists(),
3462 "nothing is recorded before a launch"
3463 );
3464 assert!(!sandbox.token_installed());
3465
3466 answer_launch(&recorder);
3467 let session = sandbox
3468 .run(RunRequest::new().with_image("arn:image"))
3469 .await
3470 .expect("launches");
3471 assert_eq!(
3472 session.endpoint(),
3473 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
3474 );
3475
3476 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3477 assert!(sandbox.image_exists(), "STATE-1: the image is recorded");
3478 assert!(sandbox.token_installed(), "STATE-2: the token is installed");
3479 assert_eq!(sandbox.bootstrap_count(), 1);
3480 assert_eq!(sandbox.microvm().expect("launched").id, "mvm-abc123");
3481
3482 let body = recorder.first_body("RunMicrovm");
3485 let payload = body["runHookPayload"].as_str().expect("a string");
3486 assert!(payload.starts_with(r#"{"agent_token":"#), "{payload}");
3487 assert_eq!(body["idlePolicy"]["suspendedDurationSeconds"], 600);
3488 }
3489
3490 #[tokio::test(start_paused = true)]
3497 async fn a_launch_env_reaches_the_run_hook_payload_on_the_wire() {
3498 let (mut sandbox, recorder, _) = planted();
3499 answer_launch(&recorder);
3500 sandbox
3501 .run(
3502 RunRequest::new()
3503 .with_image("arn:image")
3504 .with_launch_env("ANTHROPIC_BASE_URL", "https://gateway.example")
3505 .with_launch_env("PATH", "/usr/local/bin:/usr/bin:/bin"),
3506 )
3507 .await
3508 .expect("launches");
3509
3510 let body = recorder.first_body("RunMicrovm");
3511 let payload = body["runHookPayload"].as_str().expect("a string");
3512 let inner: serde_json::Value =
3514 serde_json::from_str(payload).expect("the payload is itself JSON");
3515 assert!(
3516 inner["agent_token"].as_str().is_some_and(|t| !t.is_empty()),
3517 "the token still rides alongside: {payload}"
3518 );
3519 assert_eq!(
3520 inner["env"]["ANTHROPIC_BASE_URL"],
3521 "https://gateway.example"
3522 );
3523 assert_eq!(inner["env"]["PATH"], "/usr/local/bin:/usr/bin:/bin");
3524 }
3525
3526 #[tokio::test(start_paused = true)]
3539 async fn the_advisory_deny_reaches_the_launch_environment_in_both_spellings() {
3540 let (mut sandbox, recorder, _) = planted();
3541 answer_launch(&recorder);
3542 let request = RunRequest::new().with_image("arn:image").with_deny_egress();
3543 assert_eq!(
3544 request.egress_posture(),
3545 crate::control::EgressPosture::BestEffort
3546 );
3547 sandbox.run(request).await.expect("launches");
3548
3549 let body = recorder.first_body("RunMicrovm");
3550 let inner: serde_json::Value =
3551 serde_json::from_str(body["runHookPayload"].as_str().expect("a string"))
3552 .expect("the payload is itself JSON");
3553 for key in crate::sandbox::DENY_EGRESS_ENV_KEYS {
3554 assert_eq!(
3555 inner["env"][key], DENY_EGRESS_PROXY_URL,
3556 "{key} must point at the black hole: {inner}"
3557 );
3558 }
3559 assert!(
3560 body.get("egressNetworkConnectors").is_none(),
3561 "the advisory deny asks the platform for nothing: {body}"
3562 );
3563 }
3564
3565 #[test]
3568 fn the_deny_never_overwrites_a_callers_own_proxy_and_is_absent_by_default() {
3569 let kept = RunRequest::new()
3570 .with_deny_egress()
3571 .with_launch_env("https_proxy", "http://proxy.internal:3128")
3572 .effective_launch_env();
3573 assert_eq!(kept["https_proxy"], "http://proxy.internal:3128");
3574 assert_eq!(kept["http_proxy"], DENY_EGRESS_PROXY_URL);
3575
3576 let plain = RunRequest::new().effective_launch_env();
3577 assert!(
3578 plain.is_empty(),
3579 "a launch that did not ask for the deny sends byte-for-byte what it always sent: \
3580 {plain:?}"
3581 );
3582 assert_eq!(
3583 RunRequest::new().egress_posture(),
3584 crate::control::EgressPosture::Unsealed,
3585 "and its posture is the measured one, not a seal"
3586 );
3587 }
3588
3589 #[tokio::test]
3598 async fn asking_for_egress_and_the_deny_together_is_refused_before_any_call() {
3599 let (mut sandbox, recorder, _) = planted();
3600 answer_launch(&recorder);
3601 let mut request = RunRequest::new().with_image("arn:image").with_egress();
3602 request.deny_egress = true;
3603
3604 let error = sandbox
3605 .run(request)
3606 .await
3607 .expect_err("opposite intents are not resolved by a precedence rule");
3608 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3609 assert!(error.to_string().contains("opposite things"), "{error}");
3610 assert_eq!(
3611 recorder.calls().len(),
3612 0,
3613 "a locally-refused launch costs nothing"
3614 );
3615 }
3616
3617 #[tokio::test(start_paused = true)]
3628 async fn a_pinned_image_version_is_forwarded_to_the_launch_request() {
3629 let (mut sandbox, recorder, _) = planted();
3630 answer_launch(&recorder);
3631 sandbox
3632 .run(
3633 RunRequest::new()
3634 .with_image("arn:image")
3635 .with_image_version("2.0"),
3636 )
3637 .await
3638 .expect("launches");
3639 assert_eq!(
3640 recorder.first_body("RunMicrovm")["imageVersion"],
3641 "2.0",
3642 "a canary has to launch against the version it means to test"
3643 );
3644
3645 let (mut sandbox, recorder, _) = planted();
3646 answer_launch(&recorder);
3647 sandbox
3648 .run(RunRequest::new().with_image("arn:image"))
3649 .await
3650 .expect("launches");
3651 assert!(
3652 recorder
3653 .first_body("RunMicrovm")
3654 .get("imageVersion")
3655 .is_none(),
3656 "an unpinned launch sends what this client always sent"
3657 );
3658 }
3659
3660 #[tokio::test]
3671 async fn an_over_budget_launch_env_is_refused_before_any_control_plane_call() {
3672 let (mut sandbox, recorder, _) = planted();
3673 answer_launch(&recorder);
3674
3675 let error = sandbox
3676 .run(
3677 RunRequest::new()
3678 .with_image("arn:image")
3679 .with_launch_env("AWS_SESSION_TOKEN", "t".repeat(4096)),
3680 )
3681 .await
3682 .expect_err("a credential-scale launch env does not fit the payload");
3683
3684 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3685 let message = error.to_string();
3686 assert!(message.contains("ceiling of 4096"), "{message}");
3687 assert!(message.contains("launch env contributed"), "{message}");
3688 assert_eq!(
3689 recorder.calls().len(),
3690 0,
3691 "the refusal has to be local: a launch that reached AWS costs a VM and a bill"
3692 );
3693 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3696 assert_eq!(sandbox.bootstrap_count(), 0);
3697 assert!(sandbox.microvm().is_none());
3698 }
3699
3700 #[tokio::test]
3718 async fn the_bindings_layer_refuses_an_illegal_launch_with_zero_calls() {
3719 let cases: [(&str, RunRequest, &str); 3] = [
3720 (
3721 "max_idle_sec=59, which is what a Python keyword argument passes",
3722 RunRequest {
3723 max_idle_sec: 59,
3724 ..RunRequest::new().with_image("arn:image")
3725 },
3726 "maxIdleDurationSeconds",
3727 ),
3728 (
3729 "an execution role that is a bare name",
3730 RunRequest {
3731 execution_role_arn: Some("execution-role".to_string()),
3732 ..RunRequest::new().with_image("arn:image")
3733 },
3734 "role *name*",
3735 ),
3736 (
3737 "a pinned version with a trailing newline",
3738 RunRequest::new()
3739 .with_image("arn:image")
3740 .with_image_version("2.0\n"),
3741 "contains whitespace",
3742 ),
3743 ];
3744
3745 for (label, request, expected) in cases {
3746 let (mut sandbox, recorder, _) = planted();
3747 answer_launch(&recorder);
3750
3751 let Err(error) = sandbox.run(request).await else {
3755 panic!("{label} must be refused");
3756 };
3757 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
3758 assert!(
3759 error.to_string().contains(expected),
3760 "{label}: wanted {expected:?}, got {error}"
3761 );
3762 assert_eq!(
3763 recorder.calls().len(),
3764 0,
3765 "{label}: the refusal has to be local, because a launch that reached AWS costs a \
3766 VM and a bill"
3767 );
3768 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3769 assert!(sandbox.microvm().is_none());
3770 }
3771 }
3772
3773 #[tokio::test]
3780 async fn a_launch_that_dies_during_startup_installs_no_token() {
3781 let (mut sandbox, recorder, _) = planted();
3782 recorder
3783 .answer(
3784 "RunMicrovm",
3785 Answer::ok(fake::microvm_response("PENDING", None)),
3786 )
3787 .answer(
3788 "GetMicrovm",
3789 Answer::ok(fake::microvm_response(
3790 "TERMINATED",
3791 Some("run hook returned 500"),
3792 )),
3793 );
3794
3795 let error = sandbox
3796 .run(RunRequest::new().with_image("arn:image"))
3797 .await
3798 .expect_err("a VM that died during startup is not a launch");
3799 assert_eq!(error.kind(), ErrorKind::LaunchDied);
3800 assert!(
3801 error.to_string().contains("run hook returned 500"),
3802 "{error}"
3803 );
3804
3805 assert!(
3806 !sandbox.token_installed(),
3807 "the run hook never answered, so nothing was installed"
3808 );
3809 assert_eq!(
3810 sandbox.bootstrap_count(),
3811 0,
3812 "a bootstrap counted here would make STATE-3's ceiling unreachable for a retry"
3813 );
3814 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3815 assert!(
3816 sandbox.image_exists(),
3817 "STATE-1 still holds: the launch was accepted against a real image"
3818 );
3819 }
3820
3821 #[tokio::test(start_paused = true)]
3832 async fn a_second_run_on_one_sandbox_is_refused_before_any_call() {
3833 let (mut sandbox, recorder, _) = launched().await;
3834 assert_eq!(sandbox.bootstrap_count(), 1);
3835 let before = recorder.calls().len();
3836
3837 let error = sandbox
3838 .run(RunRequest::new().with_image("arn:image"))
3839 .await
3840 .expect_err("the token is installed at most once per VM lifetime");
3841 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3842 assert!(error.to_string().contains("STATE-3"), "{error}");
3843 assert!(
3844 error.to_string().contains("needs a second Sandbox"),
3845 "the remedy has to be nameable: {error}"
3846 );
3847
3848 assert_eq!(
3849 sandbox.bootstrap_count(),
3850 1,
3851 "bootstrap_count <= 1 is the Z3-proved invariant"
3852 );
3853 assert_eq!(
3854 recorder.calls().len(),
3855 before,
3856 "the refusal must cost no control-plane call"
3857 );
3858 assert_eq!(recorder.call_count("RunMicrovm"), 1);
3859 }
3860
3861 #[tokio::test(start_paused = true)]
3864 async fn a_suspend_from_running_passes_through_suspending_to_suspended() {
3865 let (sandbox, recorder, _) = suspended().await;
3866 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3867 assert!(
3868 sandbox.token_installed(),
3869 "a freeze keeps the guest's memory, so the token survives"
3870 );
3871 assert_eq!(sandbox.bootstrap_count(), 1, "no re-bootstrap on a freeze");
3872
3873 let calls = recorder.calls();
3874 let suspend = calls
3875 .iter()
3876 .find(|call| call.operation == "SuspendMicrovm")
3877 .expect("the suspend went out");
3878 assert_eq!(suspend.method, Method::Post);
3879 assert_eq!(suspend.path, "/2025-09-09/microvms/mvm-abc123/suspend");
3880 }
3881
3882 #[tokio::test(start_paused = true)]
3888 async fn a_vm_that_dies_while_suspending_is_recorded_rather_than_raised() {
3889 let (mut sandbox, recorder, _) = launched().await;
3890 recorder
3891 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3892 .answer(
3893 "GetMicrovm",
3894 Answer::ok(fake::microvm_response("TERMINATED", Some("idle policy"))),
3895 );
3896
3897 sandbox
3898 .suspend()
3899 .await
3900 .expect("a death mid-suspend is a state, not an error");
3901 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3902 assert!(
3903 sandbox.was_terminated(),
3904 "STATE-11's precondition is recorded"
3905 );
3906 }
3907
3908 #[tokio::test(start_paused = true)]
3917 async fn a_suspend_whose_call_fails_stays_running_and_can_be_retried() {
3918 let (mut sandbox, recorder, _) = planted();
3919 recorder
3920 .answer(
3921 "RunMicrovm",
3922 Answer::ok(fake::microvm_response("PENDING", None)),
3923 )
3924 .answer(
3925 "GetMicrovm",
3926 Answer::ok(fake::microvm_response("RUNNING", None)),
3927 )
3928 .answer(
3929 "GetMicrovm",
3930 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3931 )
3932 .answer(
3933 "CreateMicrovmAuthToken",
3934 Answer::ok(fake::auth_token_response("proxy-token")),
3935 )
3936 .answer("SuspendMicrovm", Answer::failure(409, "ConflictException"))
3939 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
3940
3941 sandbox
3942 .run(RunRequest::new().with_image("arn:image"))
3943 .await
3944 .expect("launches");
3945
3946 sandbox
3947 .suspend()
3948 .await
3949 .expect_err("the control plane refused the suspend");
3950 assert_eq!(
3951 sandbox.lifecycle(),
3952 Lifecycle::Running,
3953 "a refused suspend must not move the lifecycle: SUSPENDING accepts neither a \
3954 suspend nor a resume, so recording it here would brick the handle"
3955 );
3956
3957 sandbox.suspend().await.expect("the retry suspends");
3958 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3959 assert_eq!(recorder.call_count("SuspendMicrovm"), 2);
3960 }
3961
3962 #[tokio::test(start_paused = true)]
3974 async fn a_suspend_from_a_non_running_state_reaches_no_control_plane_call() {
3975 let (mut sandbox, recorder, _) = suspended().await;
3977 let before = recorder.call_count("SuspendMicrovm");
3978
3979 let error = sandbox
3980 .suspend()
3981 .await
3982 .expect_err("a suspend is only issued from RUNNING");
3983 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3984 assert!(error.to_string().contains("STATE-5"), "{error}");
3985 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3986 assert_eq!(
3987 recorder.call_count("SuspendMicrovm"),
3988 before,
3989 "the refusal must be local: no second suspend went to the wire"
3990 );
3991
3992 let (mut fresh, fresh_recorder, _) = planted();
3994 let error = fresh.suspend().await.expect_err("nothing to suspend");
3995 assert_eq!(error.kind(), ErrorKind::Precondition);
3996 assert_eq!(fresh_recorder.calls().len(), 0);
3997 }
3998
3999 #[tokio::test(start_paused = true)]
4010 async fn a_resume_reuses_the_token_redelivers_nothing_and_drops_the_proxy_token() {
4011 let (mut sandbox, recorder, _) = suspended().await;
4012
4013 let auth = Arc::clone(
4015 sandbox
4016 .session()
4017 .expect("launched")
4018 .proxy_auth()
4019 .expect("the launch wired a minter"),
4020 );
4021 auth.headers().await.expect("mints");
4022 assert!(
4023 auth.is_cached(),
4024 "there must be a token for the drop to matter"
4025 );
4026 assert_eq!(auth.mint_count(), 1);
4027
4028 recorder
4029 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4030 .answer(
4031 "GetMicrovm",
4032 Answer::ok(fake::microvm_response("RUNNING", None)),
4033 );
4034 let session = sandbox.resume().await.expect("resumes");
4035 assert_eq!(
4036 session.endpoint(),
4037 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
4038 );
4039
4040 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4041 assert!(
4042 sandbox.token_installed(),
4043 "STATE-7: the installed token is reused"
4044 );
4045 assert_eq!(
4046 sandbox.bootstrap_count(),
4047 1,
4048 "STATE-7: a resume re-delivers no run-hook payload"
4049 );
4050 assert_eq!(
4051 recorder.call_count("RunMicrovm"),
4052 1,
4053 "a resume is not a second launch"
4054 );
4055 assert!(
4056 !auth.is_cached(),
4057 "STATE-8: the cached proxy token survived the resume"
4058 );
4059 }
4060
4061 #[tokio::test(start_paused = true)]
4071 async fn the_request_after_a_resume_mints_a_fresh_proxy_token() {
4072 let (mut sandbox, recorder, _) = suspended().await;
4073 let auth = Arc::clone(
4074 sandbox
4075 .session()
4076 .expect("launched")
4077 .proxy_auth()
4078 .expect("wired"),
4079 );
4080 auth.headers().await.expect("mints");
4081 assert_eq!(auth.mint_count(), 1);
4082
4083 recorder
4084 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4085 .answer(
4086 "GetMicrovm",
4087 Answer::ok(fake::microvm_response("RUNNING", None)),
4088 );
4089 sandbox.resume().await.expect("resumes");
4090
4091 auth.headers().await.expect("re-mints");
4094 assert_eq!(
4095 auth.mint_count(),
4096 2,
4097 "STATE-8: the resume did not force a fresh mint"
4098 );
4099 }
4100
4101 #[tokio::test(start_paused = true)]
4105 async fn the_lifecycle_watch_follows_every_transition() {
4106 let (mut sandbox, recorder, _clock) = planted();
4107 let watch = sandbox.watch_lifecycle();
4108 assert_eq!(*watch.borrow(), Lifecycle::Pending);
4109 recorder
4110 .answer(
4111 "RunMicrovm",
4112 Answer::ok(fake::microvm_response("PENDING", None)),
4113 )
4114 .answer(
4115 "GetMicrovm",
4116 Answer::ok(fake::microvm_response("RUNNING", None)),
4117 )
4118 .answer(
4119 "GetMicrovm",
4120 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4121 )
4122 .answer(
4123 "CreateMicrovmAuthToken",
4124 Answer::ok(fake::auth_token_response("proxy-token")),
4125 )
4126 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
4127 sandbox
4128 .run(RunRequest::new().with_image("arn:image"))
4129 .await
4130 .expect("launches");
4131 assert_eq!(*watch.borrow(), Lifecycle::Running);
4132 sandbox.suspend().await.expect("suspends");
4133 assert_eq!(*watch.borrow(), Lifecycle::Suspended);
4134 assert_eq!(*watch.borrow(), sandbox.lifecycle());
4135 }
4136
4137 fn terminated_refusal() -> Answer {
4140 Answer {
4141 status: 400,
4142 body: r#"{"message": "The MicroVM mvm-abc123 has been terminated and its state cannot be changed."}"#
4143 .to_string(),
4144 error_type: Some("ValidationException".to_string()),
4145 }
4146 }
4147
4148 #[tokio::test(start_paused = true)]
4152 async fn resume_observes_platform_idle_suspension_and_resumes() {
4153 let (mut sandbox, recorder, _) = planted();
4154 recorder
4155 .answer(
4156 "RunMicrovm",
4157 Answer::ok(fake::microvm_response("PENDING", None)),
4158 )
4159 .answer(
4160 "GetMicrovm",
4161 Answer::ok(fake::microvm_response("RUNNING", None)),
4162 )
4163 .answer(
4164 "GetMicrovm",
4165 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4166 )
4167 .answer(
4168 "GetMicrovm",
4169 Answer::ok(fake::microvm_response("RUNNING", None)),
4170 )
4171 .answer(
4172 "CreateMicrovmAuthToken",
4173 Answer::ok(fake::auth_token_response("proxy-token")),
4174 )
4175 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4176 sandbox
4177 .run(RunRequest::new().with_image("arn:image"))
4178 .await
4179 .expect("launch");
4180 let token = sandbox
4181 .session()
4182 .expect("session")
4183 .agent_token()
4184 .to_string();
4185 let watch = sandbox.watch_lifecycle();
4186 let calls = recorder.calls().len();
4187 sandbox.resume().await.expect("idle-suspended VM resumes");
4188 assert_eq!(
4189 &recorder.operations()[calls..],
4190 &["GetMicrovm", "ResumeMicrovm", "GetMicrovm"]
4191 );
4192 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4193 assert_eq!(*watch.borrow(), Lifecycle::Running);
4194 assert_eq!(sandbox.microvm().expect("VM").state, "RUNNING");
4195 assert_eq!(sandbox.bootstrap_count(), 1);
4196 assert_eq!(recorder.call_count("RunMicrovm"), 1);
4197 assert_eq!(recorder.call_count("SuspendMicrovm"), 0);
4198 assert_eq!(sandbox.session().expect("session").agent_token(), token);
4199 assert!(
4200 !sandbox
4201 .session()
4202 .expect("session")
4203 .proxy_auth()
4204 .expect("auth")
4205 .is_cached()
4206 );
4207 }
4208
4209 #[tokio::test(start_paused = true)]
4213 async fn resume_observes_platform_termination_and_records_window_closed() {
4214 for state in ["TERMINATING", "TERMINATED"] {
4215 let (mut sandbox, recorder, _) = planted();
4216 recorder
4217 .answer(
4218 "RunMicrovm",
4219 Answer::ok(fake::microvm_response("PENDING", None)),
4220 )
4221 .answer(
4222 "GetMicrovm",
4223 Answer::ok(fake::microvm_response("RUNNING", None)),
4224 )
4225 .answer(
4226 "GetMicrovm",
4227 Answer::ok(fake::microvm_response(state, Some("idle window elapsed"))),
4228 )
4229 .answer(
4230 "CreateMicrovmAuthToken",
4231 Answer::ok(fake::auth_token_response("proxy-token")),
4232 );
4233 sandbox
4234 .run(RunRequest::new().with_image("arn:image"))
4235 .await
4236 .expect("launch");
4237 let watch = sandbox.watch_lifecycle();
4238 let error = sandbox.resume().await.expect_err("the VM ended");
4239 assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}");
4240 assert!(error.to_string().contains(state), "{error}");
4241 assert!(error.to_string().contains("idle window elapsed"), "{error}");
4242 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4243 assert_eq!(*watch.borrow(), Lifecycle::Terminated);
4244 assert_eq!(sandbox.microvm().expect("VM").state, state);
4245 assert!(sandbox.was_terminated());
4246 assert_eq!(sandbox.drop_warning(), None);
4247 let calls = recorder.calls().len();
4248 assert_eq!(
4249 sandbox.resume().await.expect_err("STATE-11").kind(),
4250 ErrorKind::InvalidArg
4251 );
4252 assert_eq!(
4253 recorder.calls().len(),
4254 calls,
4255 "known termination stays local"
4256 );
4257 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4258 }
4259 }
4260
4261 #[tokio::test(start_paused = true)]
4264 async fn resume_observes_running_and_propagates_observation_failures() {
4265 for (response, expected) in [
4266 (
4267 Answer::ok(fake::microvm_response("RUNNING", None)),
4268 ErrorKind::InvalidArg,
4269 ),
4270 (
4271 Answer::ok(fake::microvm_response("UNKNOWN", None)),
4272 ErrorKind::Platform,
4273 ),
4274 (
4275 Answer::failure(403, "access denied"),
4276 ErrorKind::Credentials,
4277 ),
4278 ] {
4279 let (mut sandbox, recorder, _) = planted();
4280 recorder
4281 .answer(
4282 "RunMicrovm",
4283 Answer::ok(fake::microvm_response("PENDING", None)),
4284 )
4285 .answer(
4286 "GetMicrovm",
4287 Answer::ok(fake::microvm_response("RUNNING", None)),
4288 )
4289 .answer("GetMicrovm", response)
4290 .answer(
4291 "CreateMicrovmAuthToken",
4292 Answer::ok(fake::auth_token_response("proxy-token")),
4293 );
4294 sandbox
4295 .run(RunRequest::new().with_image("arn:image"))
4296 .await
4297 .expect("launch");
4298 let calls = recorder.calls().len();
4299 let error = sandbox.resume().await.expect_err("cannot resume");
4300 assert_eq!(error.kind(), expected, "{error}");
4301 assert_eq!(&recorder.operations()[calls..], &["GetMicrovm"]);
4302 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4303 assert!(!sandbox.was_terminated());
4304 }
4305 }
4306
4307 #[tokio::test(start_paused = true)]
4310 async fn resume_retries_observation_after_transient_service_states() {
4311 for state in ["SUSPENDING", "PENDING", "RESUMING"] {
4312 let (mut sandbox, recorder, _) = planted();
4313 recorder
4314 .answer(
4315 "RunMicrovm",
4316 Answer::ok(fake::microvm_response("PENDING", None)),
4317 )
4318 .answer(
4319 "GetMicrovm",
4320 Answer::ok(fake::microvm_response("RUNNING", None)),
4321 )
4322 .answer(
4323 "GetMicrovm",
4324 Answer::ok(fake::microvm_response(state, None)),
4325 )
4326 .answer(
4327 "GetMicrovm",
4328 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4329 )
4330 .answer(
4331 "GetMicrovm",
4332 Answer::ok(fake::microvm_response("RUNNING", None)),
4333 )
4334 .answer(
4335 "CreateMicrovmAuthToken",
4336 Answer::ok(fake::auth_token_response("proxy-token")),
4337 )
4338 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4339 sandbox
4340 .run(RunRequest::new().with_image("arn:image"))
4341 .await
4342 .expect("launch");
4343 let error = sandbox.resume().await.expect_err("still changing state");
4344 assert!(error.to_string().contains(state), "{error}");
4345 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4346 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4347 sandbox
4348 .resume()
4349 .await
4350 .expect("the next observation can resume");
4351 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4352 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4353 }
4354 }
4355
4356 #[tokio::test(start_paused = true)]
4362 async fn a_resume_past_the_window_of_an_explicit_suspend_reaches_the_service_and_resumes() {
4363 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
4364 assert_eq!(sandbox.suspended_window(), Some(Duration::from_secs(60)));
4365 assert_eq!(
4366 sandbox.idle_window(),
4367 Some(Duration::from_secs(600)),
4368 "the launch's own maxIdleDurationSeconds, which a keepalive checks its cadence against"
4369 );
4370
4371 clock.advance(Duration::from_secs(1283));
4372 recorder
4373 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4374 .answer(
4375 "GetMicrovm",
4376 Answer::ok(fake::microvm_response("RUNNING", None)),
4377 );
4378 sandbox
4379 .resume()
4380 .await
4381 .expect("the service resumes an explicit suspend past its window");
4382
4383 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4384 assert!(!sandbox.was_terminated());
4385 assert_eq!(
4386 recorder.call_count("ResumeMicrovm"),
4387 1,
4388 "the resume has to reach the service"
4389 );
4390 }
4391
4392 #[tokio::test(start_paused = true)]
4398 async fn a_resume_the_service_refuses_for_an_ended_vm_is_window_closed_and_recorded() {
4399 let (mut sandbox, recorder, _clock) = planted();
4400 recorder
4403 .answer(
4404 "RunMicrovm",
4405 Answer::ok(fake::microvm_response("PENDING", None)),
4406 )
4407 .answer(
4408 "GetMicrovm",
4409 Answer::ok(fake::microvm_response("RUNNING", None)),
4410 )
4411 .answer(
4412 "GetMicrovm",
4413 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4414 )
4415 .answer(
4416 "GetMicrovm",
4417 Answer::ok(fake::microvm_response(
4418 "TERMINATED",
4419 Some("MicroVM exceeded maximum lifetime."),
4420 )),
4421 )
4422 .answer(
4423 "CreateMicrovmAuthToken",
4424 Answer::ok(fake::auth_token_response("proxy-token")),
4425 )
4426 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
4427 .answer("ResumeMicrovm", terminated_refusal());
4428 sandbox
4429 .run(
4430 RunRequest::new()
4431 .with_image("arn:image")
4432 .with_suspended_sec(60),
4433 )
4434 .await
4435 .expect("launches");
4436 sandbox.suspend().await.expect("suspends");
4437
4438 let error = sandbox
4439 .resume()
4440 .await
4441 .expect_err("the service ended the VM");
4442 assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}");
4443 assert_eq!(error.code(), "ERR_WINDOW_CLOSED");
4444 let message = error.to_string();
4445 assert!(message.contains("TERMINATED"), "the state: {message}");
4446 assert!(
4447 message.contains("MicroVM exceeded maximum lifetime."),
4448 "the stateReason: {message}"
4449 );
4450 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4451 assert!(sandbox.was_terminated());
4452 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4453
4454 let again = sandbox.resume().await.expect_err("STATE-11");
4455 assert!(again.to_string().contains("STATE-11"), "{again}");
4456 assert_eq!(
4457 recorder.call_count("ResumeMicrovm"),
4458 1,
4459 "the second resume makes no call"
4460 );
4461 assert_eq!(
4462 sandbox.drop_warning(),
4463 None,
4464 "a VM the service ended bills nothing, so dropping its sandbox isn't a leak"
4465 );
4466 }
4467
4468 #[tokio::test(start_paused = true)]
4472 async fn a_resume_refused_for_a_live_vm_leaves_the_sandbox_suspended_and_resumable() {
4473 let (mut sandbox, recorder, _clock) = suspended().await;
4474 recorder
4475 .answer(
4476 "ResumeMicrovm",
4477 Answer::failure(409, "The MicroVM is changing state."),
4478 )
4479 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4480
4481 let error = sandbox.resume().await.expect_err("the service refused");
4482 assert_eq!(error.kind(), ErrorKind::Platform, "{error}");
4483 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
4484 assert!(!sandbox.was_terminated());
4485
4486 recorder.answer(
4487 "GetMicrovm",
4488 Answer::ok(fake::microvm_response("RUNNING", None)),
4489 );
4490 sandbox.resume().await.expect("the retry resumes");
4491 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4492 assert_eq!(recorder.call_count("ResumeMicrovm"), 2);
4493 }
4494
4495 #[tokio::test(start_paused = true)]
4499 async fn a_client_token_launch_resumes_the_vm_it_adopted() {
4500 let (mut sandbox, recorder, _) = planted();
4501 recorder
4502 .answer(
4503 "RunMicrovm",
4504 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4505 )
4506 .answer(
4507 "GetMicrovm",
4508 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4509 )
4510 .answer(
4511 "GetMicrovm",
4512 Answer::ok(fake::microvm_response("RUNNING", None)),
4513 )
4514 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4515 .answer(
4516 "CreateMicrovmAuthToken",
4517 Answer::ok(fake::auth_token_response("proxy-token")),
4518 );
4519 let mut request = RunRequest::new().with_image("arn:image");
4520 request.client_token = Some("job-195".into());
4521 request.agent_token = Some("agent-token".into());
4522 sandbox
4523 .run(request)
4524 .await
4525 .expect("the adopted VM is resumed");
4526 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4527 assert_eq!(sandbox.bootstrap_count(), 1);
4528 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4529
4530 let (mut fresh, recorder, _) = planted();
4531 recorder
4532 .answer(
4533 "RunMicrovm",
4534 Answer::ok(fake::microvm_response("PENDING", None)),
4535 )
4536 .answer(
4537 "GetMicrovm",
4538 Answer::ok(fake::microvm_response("SUSPENDED", Some("hook failed"))),
4539 );
4540 let error = fresh
4541 .run(RunRequest::new().with_image("arn:image"))
4542 .await
4543 .expect_err("a fresh launch that reads SUSPENDED died during startup");
4544 assert_eq!(error.kind(), ErrorKind::LaunchDied);
4545 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4546 }
4547
4548 #[tokio::test(start_paused = true)]
4551 async fn a_launch_that_does_not_wait_is_finished_by_wait_until_running() {
4552 let (mut sandbox, recorder, _) = planted();
4553 answer_launch(&recorder);
4554 let mut request = RunRequest::new().with_image("arn:image");
4555 request.wait = false;
4556 let endpoint = sandbox
4557 .run(request)
4558 .await
4559 .expect("accepted")
4560 .endpoint()
4561 .to_string();
4562 assert!(endpoint.starts_with("https://"), "{endpoint}");
4563 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
4564 assert_eq!(sandbox.bootstrap_count(), 0);
4565 assert_eq!(recorder.call_count("GetMicrovm"), 0, "nothing polled yet");
4566
4567 sandbox
4568 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4569 .await
4570 .expect("reaches RUNNING");
4571 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4572 assert_eq!(sandbox.bootstrap_count(), 1);
4573 let error = sandbox
4574 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4575 .await
4576 .expect_err("a running launch has nothing to wait for");
4577 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4578 assert_eq!(sandbox.bootstrap_count(), 1, "counted once (STATE-3)");
4579 }
4580
4581 #[tokio::test(start_paused = true)]
4589 async fn a_resume_after_terminate_records_zero_resume_calls() {
4590 let (mut sandbox, recorder, _) = launched().await;
4591 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4592 let report = sandbox.terminate(TeardownOpts::default()).await;
4593 assert!(report.terminate_accepted);
4594 assert!(sandbox.was_terminated());
4595
4596 let error = sandbox
4597 .resume()
4598 .await
4599 .expect_err("a terminated VM never returns to RUNNING");
4600 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4601 assert!(error.to_string().contains("STATE-11"), "{error}");
4602 assert_eq!(
4603 recorder.call_count("ResumeMicrovm"),
4604 0,
4605 "no resume may reach the wire once the VM was terminated"
4606 );
4607 assert_ne!(
4608 sandbox.lifecycle(),
4609 Lifecycle::Running,
4610 "the lifecycle must not return to RUNNING"
4611 );
4612 }
4613
4614 #[tokio::test(start_paused = true)]
4621 async fn a_terminate_records_terminating_and_the_completion_report_marks_terminated() {
4622 let (mut sandbox, recorder, _) = launched().await;
4623 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4624
4625 let report = sandbox.terminate(TeardownOpts::default()).await;
4626 assert!(report.terminate_accepted, "STATE-9: the call was accepted");
4627 assert!(sandbox.was_terminated(), "STATE-9: recorded terminated");
4628 assert_eq!(
4629 report.lifecycle,
4630 Some(Lifecycle::Terminating),
4631 "the default teardown does not wait, so TERMINATED is not claimed"
4632 );
4633 assert!(!report.leaked(), "{report:?}");
4634
4635 let (mut sandbox, recorder, _) = launched().await;
4637 recorder
4638 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4639 .answer(
4640 "GetMicrovm",
4641 Answer::ok(fake::microvm_response("TERMINATED", None)),
4642 );
4643 let report = sandbox
4644 .terminate(TeardownOpts::default().waiting_for_terminated())
4645 .await;
4646 assert_eq!(report.lifecycle, Some(Lifecycle::Terminated));
4647 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4648 }
4649
4650 #[tokio::test(start_paused = true)]
4659 async fn teardown_deletes_the_vm_then_the_image_and_names_the_log_group_last() {
4660 let (mut sandbox, recorder, _) = planted();
4661 recorder
4662 .answer(
4663 "CreateMicrovmImage",
4664 Answer::created(fake::create_image_response("agentd-conformance")),
4665 )
4666 .answer(
4667 "GetMicrovmImage",
4668 Answer::ok(fake::get_image_response("agentd-conformance", "CREATED")),
4669 );
4670 sandbox
4671 .build_image(CreateImageRequest::new(
4672 "agentd-conformance",
4673 crate::testing::aarch64_daemon(b"binary"),
4674 "s3://bucket/img.zip",
4675 "arn:aws:iam::123456789012:role/build",
4676 ))
4677 .await
4678 .expect("builds");
4679 answer_launch(&recorder);
4680 sandbox
4681 .run(RunRequest::new())
4682 .await
4683 .expect("launches from the built image");
4684
4685 recorder
4686 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4687 .answer(
4688 "ListMicrovmImageVersions",
4689 Answer::ok(fake::list_versions_response("1")),
4690 )
4691 .answer(
4692 "DeleteMicrovmImage",
4693 Answer::ok(fake::delete_image_response()),
4694 );
4695
4696 let report = sandbox
4697 .terminate(
4698 TeardownOpts::default()
4699 .deleting_image()
4700 .deleting_log_group(),
4701 )
4702 .await;
4703 assert!(report.terminate_accepted);
4704 assert_eq!(report.image_deleted, Some(true));
4705
4706 let order: Vec<&str> = recorder
4708 .operations()
4709 .into_iter()
4710 .filter(|operation| matches!(*operation, "TerminateMicrovm" | "DeleteMicrovmImage"))
4711 .collect();
4712 assert_eq!(
4713 order,
4714 ["TerminateMicrovm", "DeleteMicrovmImage"],
4715 "the VM goes before the image: a terminating VM holds a reference to it"
4716 );
4717
4718 assert_eq!(
4720 report.undeleted,
4721 ["/aws/lambda-microvms/agentd-conformance"],
4722 "{report:?}"
4723 );
4724 assert!(report.leaked());
4725 assert!(
4726 report
4727 .failures
4728 .last()
4729 .expect("a failure")
4730 .contains("log group"),
4731 "the log group's failure has to be the last one recorded: {report:?}"
4732 );
4733 assert!(
4734 report
4735 .failures
4736 .last()
4737 .expect("a failure")
4738 .contains("terraform destroy"),
4739 "the reason it leaks has to be named: {report:?}"
4740 );
4741 }
4742
4743 #[tokio::test(start_paused = true)]
4749 async fn a_failing_image_delete_names_what_it_left_behind() {
4750 let (mut sandbox, recorder, _) = planted();
4751 recorder
4752 .answer(
4753 "CreateMicrovmImage",
4754 Answer::created(fake::create_image_response("img")),
4755 )
4756 .answer(
4757 "GetMicrovmImage",
4758 Answer::ok(fake::get_image_response("img", "CREATED")),
4759 );
4760 sandbox
4761 .build_image(CreateImageRequest::new(
4762 "img",
4763 crate::testing::aarch64_daemon(b"binary"),
4764 "s3://bucket/img.zip",
4765 "arn:aws:iam::123456789012:role/build",
4766 ))
4767 .await
4768 .expect("builds");
4769 answer_launch(&recorder);
4770 sandbox.run(RunRequest::new()).await.expect("launches");
4771
4772 recorder
4773 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4774 .answer(
4775 "ListMicrovmImageVersions",
4776 Answer::ok(fake::list_versions_response("1")),
4777 )
4778 .answer(
4779 "DeleteMicrovmImage",
4780 Answer::failure(409, "image is in CREATING"),
4781 );
4782
4783 let mut opts = TeardownOpts::default().deleting_image();
4784 opts.delete_attempts = 3;
4785 opts.delete_backoff = Duration::from_secs(1);
4786 let report = sandbox.terminate(opts).await;
4787
4788 assert_eq!(report.image_deleted, Some(false));
4789 assert_eq!(
4790 report.undeleted,
4791 ["arn:aws:lambda:us-east-1:123456789012:microvm-image:img"],
4792 "{report:?}"
4793 );
4794 assert!(
4795 sandbox.image_exists(),
4796 "the image really is still there, so the flag must say so"
4797 );
4798 assert_eq!(
4799 recorder.call_count("DeleteMicrovmImage"),
4800 3,
4801 "every attempt ran"
4802 );
4803 }
4804
4805 #[tokio::test(start_paused = true)]
4812 async fn a_terminate_whose_call_fails_still_records_the_intent_and_the_leak() {
4813 let (mut sandbox, recorder, _) = launched().await;
4814 for _ in 0..6 {
4815 recorder.answer(
4816 "TerminateMicrovm",
4817 Answer::failure(500, "InternalServerException"),
4818 );
4819 }
4820 recorder.answer("TerminateMicrovm", Answer::failure(403, "denied"));
4822
4823 let report = sandbox.terminate(TeardownOpts::default()).await;
4824 assert_eq!(recorder.call_count("TerminateMicrovm"), 6);
4825 assert!(!report.terminate_accepted);
4826 assert_eq!(report.undeleted, ["mvm-abc123"], "{report:?}");
4827 assert!(sandbox.was_terminated());
4828 assert_eq!(report.lifecycle, Some(Lifecycle::Terminating));
4829 assert!(
4830 !report.failures.is_empty(),
4831 "the swallowed failure is recorded"
4832 );
4833 }
4834
4835 #[tokio::test]
4841 async fn a_teardown_before_any_launch_is_an_empty_report() {
4842 let (mut sandbox, recorder, _) = planted();
4843 let report = sandbox.terminate(TeardownOpts::default()).await;
4844 assert!(!report.terminate_accepted);
4845 assert!(!report.leaked());
4846 assert_eq!(report.lifecycle, Some(Lifecycle::Pending));
4847 assert_eq!(recorder.calls().len(), 0);
4848 }
4849
4850 #[tokio::test]
4853 async fn a_run_with_no_image_is_refused_before_any_call() {
4854 let (mut sandbox, recorder, _) = planted();
4855 let error = sandbox
4856 .run(RunRequest::new())
4857 .await
4858 .expect_err("there is nothing to launch");
4859 assert_eq!(error.kind(), ErrorKind::Precondition);
4860 assert!(error.to_string().contains("build_image first"), "{error}");
4861 assert_eq!(recorder.calls().len(), 0);
4862 }
4863
4864 #[test]
4871 fn the_lifecycle_states_are_the_state_models_domain_in_the_services_spelling() {
4872 let all = [
4873 Lifecycle::Pending,
4874 Lifecycle::Running,
4875 Lifecycle::Suspending,
4876 Lifecycle::Suspended,
4877 Lifecycle::Terminating,
4878 Lifecycle::Terminated,
4879 ];
4880 assert_eq!(
4881 all.map(Lifecycle::as_str),
4882 [
4883 "PENDING",
4884 "RUNNING",
4885 "SUSPENDING",
4886 "SUSPENDED",
4887 "TERMINATING",
4888 "TERMINATED"
4889 ]
4890 );
4891 for state in all {
4892 assert_eq!(state.to_string(), state.as_str());
4893 }
4894 assert_eq!(
4897 all.iter().filter(|state| state.is_live()).count(),
4898 4,
4899 "a state added to the enum defaults to not-live, which would silence the warning"
4900 );
4901 assert!(!Lifecycle::Terminating.is_live());
4902 assert!(!Lifecycle::Terminated.is_live());
4903 }
4904
4905 #[test]
4913 fn a_minted_agent_token_is_sixty_four_hex_characters_and_never_repeats() {
4914 let entropy = crate::entropy::testing::SequenceEntropy::new();
4915 let minted: std::collections::HashSet<String> = (0..200)
4916 .map(|_| mint_agent_token(&entropy).expect("scripted"))
4917 .collect();
4918 assert_eq!(minted.len(), 200, "a repeated token is a shared guest");
4919 for token in &minted {
4920 assert_eq!(token.len(), 64, "{token}");
4921 assert!(token.chars().all(|c| c.is_ascii_hexdigit()), "{token}");
4922 assert_ne!(
4923 token,
4924 &"0".repeat(64),
4925 "an all-zero draw is a read that did nothing"
4926 );
4927 }
4928 }
4929
4930 #[tokio::test(start_paused = true)]
4932 async fn a_sandbox_debug_does_not_print_the_agent_token() {
4933 let (mut sandbox, recorder, _) = planted();
4934 answer_launch(&recorder);
4935 sandbox
4936 .run(
4937 RunRequest::new()
4938 .with_image("arn:image")
4939 .with_agent_token("super-secret-agent-token"),
4940 )
4941 .await
4942 .expect("launches");
4943
4944 let rendered = format!("{sandbox:?}");
4945 assert!(rendered.contains("mvm-abc123"), "{rendered}");
4946 assert!(rendered.contains("Running"), "{rendered}");
4947 assert!(
4948 !rendered.contains("super-secret"),
4949 "the agent token reached a Debug string: {rendered}"
4950 );
4951 }
4952
4953 #[tokio::test(start_paused = true)]
4961 async fn a_torn_down_sandbox_is_distinguishable_from_an_abandoned_one() {
4962 let (mut sandbox, recorder, _) = launched().await;
4963 assert!(
4964 sandbox.lifecycle().is_live() && !sandbox.torn_down,
4965 "an abandoned live VM is what the drop warning is for"
4966 );
4967
4968 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4969 sandbox.terminate(TeardownOpts::default()).await;
4970 assert!(
4971 sandbox.torn_down,
4972 "terminate must mark the sandbox torn down"
4973 );
4974 assert!(
4975 !sandbox.lifecycle().is_live(),
4976 "and the lifecycle must no longer read as live"
4977 );
4978 }
4979
4980 #[tokio::test(start_paused = true)]
4981 async fn stable_launch_key_and_payload_survive_a_fresh_supervisor() {
4982 let mut bodies = Vec::new();
4983 for _ in 0..2 {
4984 let (mut sandbox, recorder, _) = planted();
4985 answer_launch(&recorder);
4986 let mut request = RunRequest::new().with_image("arn:image");
4987 request.client_token = Some("one-unique-job-launch".into());
4988 request.agent_token = Some("persisted-private-secret".into());
4989 sandbox.run(request).await.expect("accepted");
4990 bodies.push(recorder.first_body("RunMicrovm"));
4991 }
4992 assert_eq!(bodies[0]["clientToken"], "one-unique-job-launch");
4993 assert_eq!(bodies[0], bodies[1]);
4994 }
4995
4996 #[tokio::test]
4997 async fn unsafe_or_malformed_stable_launch_is_refused_without_calls() {
4998 for (token, secret, identity) in [
4999 ("key".to_string(), None, false),
5000 ("key".to_string(), Some("secret".into()), true),
5001 ("".to_string(), Some("secret".into()), false),
5002 ("x".repeat(129), Some("secret".into()), false),
5003 ("line\nbreak".to_string(), Some("secret".into()), false),
5004 ] {
5005 let (mut sandbox, recorder, _) = planted();
5006 let mut request = RunRequest::new().with_image("arn:image");
5007 request.client_token = Some(token);
5008 request.agent_token = secret;
5009 request.identity = identity;
5010 let error = sandbox.run(request).await.expect_err("invalid");
5011 assert_eq!(error.kind(), ErrorKind::InvalidArg);
5012 assert!(recorder.calls().is_empty());
5013 }
5014 }
5015}