1use std::sync::Arc;
67use std::time::Duration;
68
69use crate::control::{
70 ControlPlane, CreateImageRequest, Image, Microvm, RunHookPayload, RunMicrovmRequest, WaitOpts,
71};
72use crate::error::{Error, ErrorKind};
73use crate::region::Region;
74use crate::session::{Session, TokenMinter};
75
76pub const DEFAULT_RUNNING_TIMEOUT: Duration = Duration::from_secs(300);
80
81#[deprecated(
83 note = "renamed DEFAULT_RUNNING_TIMEOUT: `session::DEFAULT_READY_TIMEOUT` shared this name \
84 for a different wait, the daemon's"
85)]
86pub const DEFAULT_READY_TIMEOUT: Duration = DEFAULT_RUNNING_TIMEOUT;
87
88pub const DEFAULT_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(300);
91
92pub const DEFAULT_DELETE_ATTEMPTS: u32 = 20;
98
99pub const DEFAULT_DELETE_BACKOFF: Duration = Duration::from_secs(15);
101
102pub const DENY_EGRESS_PROXY_URL: &str = "http://127.0.0.1:1";
108
109pub const DENY_EGRESS_ENV_KEYS: [&str; 6] = [
118 "http_proxy",
119 "https_proxy",
120 "all_proxy",
121 "HTTP_PROXY",
122 "HTTPS_PROXY",
123 "ALL_PROXY",
124];
125
126pub const LIFECYCLE_POLL_INTERVAL: Duration = Duration::from_secs(5);
129
130#[derive(Clone, Copy, Debug, Eq, PartialEq)]
136pub enum Lifecycle {
137 Pending,
139 Running,
142 Suspending,
144 Suspended,
146 Terminating,
148 Terminated,
150}
151
152impl Lifecycle {
153 pub fn as_str(self) -> &'static str {
155 match self {
156 Lifecycle::Pending => "PENDING",
157 Lifecycle::Running => "RUNNING",
158 Lifecycle::Suspending => "SUSPENDING",
159 Lifecycle::Suspended => "SUSPENDED",
160 Lifecycle::Terminating => "TERMINATING",
161 Lifecycle::Terminated => "TERMINATED",
162 }
163 }
164
165 pub fn from_service(state: &str) -> Option<Self> {
168 Some(match state {
169 "PENDING" => Lifecycle::Pending,
170 "RUNNING" => Lifecycle::Running,
171 "SUSPENDING" => Lifecycle::Suspending,
172 "SUSPENDED" => Lifecycle::Suspended,
173 "TERMINATING" => Lifecycle::Terminating,
174 "TERMINATED" => Lifecycle::Terminated,
175 _ => return None,
176 })
177 }
178
179 pub fn is_live(self) -> bool {
181 matches!(
182 self,
183 Lifecycle::Pending | Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended
184 )
185 }
186}
187
188impl std::fmt::Display for Lifecycle {
189 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
190 f.write_str(self.as_str())
191 }
192}
193
194#[derive(Clone, Debug)]
200pub struct RunRequest {
201 pub image_identifier: Option<String>,
207 pub image_version: Option<String>,
217 pub execution_role_arn: Option<String>,
219 pub agent_token: Option<String>,
221 pub client_token: Option<String>,
225 pub launch_env: std::collections::HashMap<String, String>,
239 pub identity: bool,
251 pub egress: bool,
255 pub egress_network_connectors: Vec<String>,
258 pub deny_egress: bool,
265 pub shell: bool,
270 pub max_idle_sec: u32,
272 pub suspended_sec: u32,
275 pub auto_resume: bool,
277 pub max_duration_sec: u32,
279 pub ready_timeout: Duration,
282 pub token_scope: Option<String>,
284 pub logging: Option<crate::control::ops::Logging>,
286 pub wait: bool,
293}
294
295impl Default for RunRequest {
296 fn default() -> Self {
297 Self {
298 image_identifier: None,
299 image_version: None,
300 execution_role_arn: None,
301 agent_token: None,
302 client_token: None,
303 launch_env: std::collections::HashMap::new(),
304 identity: false,
305 egress: false,
306 egress_network_connectors: Vec::new(),
307 deny_egress: false,
308 shell: false,
309 max_idle_sec: 600,
310 suspended_sec: 600,
311 auto_resume: false,
312 max_duration_sec: 3_600,
313 ready_timeout: DEFAULT_RUNNING_TIMEOUT,
314 token_scope: None,
315 logging: None,
316 wait: true,
317 }
318 }
319}
320
321impl RunRequest {
322 pub fn new() -> Self {
325 Self::default()
326 }
327
328 #[must_use]
330 pub fn with_image(mut self, identifier: impl Into<String>) -> Self {
331 self.image_identifier = Some(identifier.into());
332 self
333 }
334
335 #[must_use]
337 pub fn with_egress(mut self) -> Self {
338 self.egress = true;
339 self
340 }
341
342 #[must_use]
344 pub fn with_egress_network_connector(mut self, arn: impl Into<String>) -> Self {
345 self.egress_network_connectors.push(arn.into());
346 self
347 }
348
349 #[must_use]
355 pub fn with_vpc_egress(mut self, arns: Vec<String>) -> Self {
356 if !arns.is_empty() {
357 self.egress = false;
358 self.egress_network_connectors = arns;
359 }
360 self
361 }
362
363 #[must_use]
366 pub fn with_deny_egress(mut self) -> Self {
367 self.deny_egress = true;
368 self
369 }
370
371 pub fn egress_posture(&self) -> crate::control::EgressPosture {
376 crate::control::EgressPosture::for_launch(self.egress, self.deny_egress)
377 }
378
379 pub fn effective_launch_env(&self) -> std::collections::HashMap<String, String> {
387 let mut env = self.launch_env.clone();
388 if self.deny_egress {
389 for key in DENY_EGRESS_ENV_KEYS {
390 env.entry(key.to_string())
391 .or_insert_with(|| DENY_EGRESS_PROXY_URL.to_string());
392 }
393 }
394 env
395 }
396
397 #[must_use]
399 pub fn with_shell(mut self) -> Self {
400 self.shell = true;
401 self
402 }
403
404 #[must_use]
406 pub fn with_image_version(mut self, version: impl Into<String>) -> Self {
407 self.image_version = Some(version.into());
408 self
409 }
410
411 #[must_use]
414 pub fn with_suspended_sec(mut self, seconds: u32) -> Self {
415 self.suspended_sec = seconds;
416 self
417 }
418
419 #[must_use]
421 pub fn with_agent_token(mut self, token: impl Into<String>) -> Self {
422 self.agent_token = Some(token.into());
423 self
424 }
425
426 #[must_use]
431 pub fn with_identity(mut self) -> Self {
432 self.identity = true;
433 self
434 }
435
436 #[must_use]
443 pub fn with_launch_env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
444 self.launch_env.insert(key.into(), value.into());
445 self
446 }
447}
448
449#[derive(Clone, Copy, Debug)]
455pub struct TeardownOpts {
456 pub delete_image: bool,
458 pub delete_log_group: bool,
464 pub delete_attempts: u32,
466 pub delete_backoff: Duration,
468 pub wait_for_terminated: Option<Duration>,
476}
477
478impl Default for TeardownOpts {
479 fn default() -> Self {
480 Self {
481 delete_image: false,
482 delete_log_group: false,
483 delete_attempts: DEFAULT_DELETE_ATTEMPTS,
484 delete_backoff: DEFAULT_DELETE_BACKOFF,
485 wait_for_terminated: None,
486 }
487 }
488}
489
490impl TeardownOpts {
491 #[must_use]
493 pub fn deleting_image(mut self) -> Self {
494 self.delete_image = true;
495 self
496 }
497
498 #[must_use]
500 pub fn deleting_log_group(mut self) -> Self {
501 self.delete_log_group = true;
502 self
503 }
504
505 #[must_use]
507 pub fn waiting_for_terminated(mut self) -> Self {
508 self.wait_for_terminated = Some(DEFAULT_LIFECYCLE_TIMEOUT);
509 self
510 }
511}
512
513#[derive(Clone, Debug, Default, Eq, PartialEq)]
517pub struct TeardownReport {
518 pub undeleted: Vec<String>,
533 pub terminate_accepted: bool,
535 pub image_deleted: Option<bool>,
537 pub lifecycle: Option<Lifecycle>,
539 pub failures: Vec<String>,
544}
545
546impl TeardownReport {
547 pub fn leaked(&self) -> bool {
549 !self.undeleted.is_empty()
550 }
551}
552
553pub struct ControlPlaneMinter {
567 control: Arc<ControlPlane>,
568 microvm_id: String,
569}
570
571impl ControlPlaneMinter {
572 pub fn new(control: Arc<ControlPlane>, microvm_id: impl Into<String>) -> Self {
574 Self {
575 control,
576 microvm_id: microvm_id.into(),
577 }
578 }
579}
580
581impl std::fmt::Debug for ControlPlaneMinter {
582 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
583 f.debug_struct("ControlPlaneMinter")
584 .field("microvm_id", &self.microvm_id)
585 .finish_non_exhaustive()
586 }
587}
588
589impl TokenMinter for ControlPlaneMinter {
590 fn mint(
591 &self,
592 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
593 Box::pin(async move {
594 let minted = self.control.mint_auth_token(&self.microvm_id).await?;
595 Ok(minted.into())
596 })
597 }
598
599 fn mint_for_ports(
607 &self,
608 ports: &[u16],
609 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
610 let specs: Vec<crate::control::ops::PortSpecification> = ports
611 .iter()
612 .map(|port| crate::control::ops::PortSpecification::port(*port))
613 .collect();
614 Box::pin(async move {
615 let minted = self
616 .control
617 .mint_auth_token_for(&self.microvm_id, &specs)
618 .await?;
619 Ok(minted.into())
620 })
621 }
622}
623
624pub struct Sandbox {
629 control: Arc<ControlPlane>,
630 image: Option<Image>,
631 launch_image_arn: Option<String>,
633 launch_metrics: Option<Arc<std::sync::Mutex<crate::session::LaunchMetrics>>>,
634 microvm: Option<Microvm>,
635 session: Option<Session>,
636 session_backend: Option<crate::session::SharedBackend>,
643
644 lifecycle: Lifecycle,
646 lifecycle_watch: tokio::sync::watch::Sender<Lifecycle>,
649 token_installed: bool,
650 image_exists: bool,
651 was_terminated: bool,
652 bootstrap_count: u32,
653
654 suspended_window: Option<Duration>,
657 launch_adoptable: bool,
660 launch_posture: Option<crate::control::EgressPosture>,
663 pending_agent_token: Option<String>,
666 idle_window: Option<Duration>,
669 torn_down: bool,
672 adopted: bool,
675 detached: bool,
678 tunnel_identity: Option<crate::identity::TunnelIdentity>,
684 build_services: Option<Arc<dyn crate::control::BuildServices>>,
687 account: Option<String>,
689}
690
691impl std::fmt::Debug for Sandbox {
692 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
693 f.debug_struct("Sandbox")
696 .field("lifecycle", &self.lifecycle)
697 .field("microvm", &self.microvm.as_ref().map(|vm| &vm.id))
698 .field("image", &self.image.as_ref().map(|image| &image.identifier))
699 .field("token_installed", &self.token_installed)
700 .field("bootstrap_count", &self.bootstrap_count)
701 .field("was_terminated", &self.was_terminated)
702 .field("suspended_window", &self.suspended_window)
703 .field("adopted", &self.adopted)
704 .field("detached", &self.detached)
705 .finish_non_exhaustive()
706 }
707}
708
709#[derive(Clone)]
716pub struct Detached {
717 pub microvm_id: String,
719 pub endpoint: String,
721 pub region: Region,
723 pub port: u16,
725 agent_token: String,
726}
727
728impl Detached {
729 pub fn agent_token(&self) -> &str {
731 &self.agent_token
732 }
733}
734
735impl std::fmt::Debug for Detached {
736 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
737 f.debug_struct("Detached")
738 .field("microvm_id", &self.microvm_id)
739 .field("endpoint", &self.endpoint)
740 .field("region", &self.region)
741 .field("port", &self.port)
742 .field("agent_token", &"<redacted>")
743 .finish()
744 }
745}
746
747impl Sandbox {
748 pub fn with_control_plane(control: ControlPlane) -> Self {
762 Self {
763 control: Arc::new(control),
764 image: None,
765 launch_image_arn: None,
766 launch_metrics: None,
767 microvm: None,
768 session: None,
769 session_backend: None,
770 lifecycle: Lifecycle::Pending,
771 lifecycle_watch: tokio::sync::watch::Sender::new(Lifecycle::Pending),
772 token_installed: false,
773 image_exists: false,
774 was_terminated: false,
775 bootstrap_count: 0,
776 suspended_window: None,
777 launch_adoptable: false,
778 launch_posture: None,
779 pending_agent_token: None,
780 idle_window: None,
781 torn_down: false,
782 adopted: false,
783 detached: false,
784 tunnel_identity: None,
785 build_services: None,
786 account: None,
787 }
788 }
789
790 pub fn with_build_services(mut self, services: Arc<dyn crate::control::BuildServices>) -> Self {
796 self.build_services = Some(services);
797 self
798 }
799
800 pub fn with_session_backend(mut self, backend: crate::session::SharedBackend) -> Self {
806 self.session_backend = Some(backend);
807 self
808 }
809
810 pub fn lifecycle(&self) -> Lifecycle {
814 self.lifecycle
815 }
816
817 pub fn token_installed(&self) -> bool {
819 self.token_installed
820 }
821
822 pub fn image_exists(&self) -> bool {
824 self.image_exists
825 }
826
827 pub fn was_terminated(&self) -> bool {
829 self.was_terminated
830 }
831
832 pub fn bootstrap_count(&self) -> u32 {
834 self.bootstrap_count
835 }
836
837 pub fn image(&self) -> Option<&Image> {
839 self.image.as_ref()
840 }
841
842 pub fn launch_image_arn(&self) -> Option<&str> {
851 self.launch_image_arn.as_deref()
852 }
853
854 pub fn microvm(&self) -> Option<&Microvm> {
856 self.microvm.as_ref()
857 }
858
859 pub fn session(&self) -> Option<&Session> {
861 self.session.as_ref()
862 }
863
864 pub fn suspended_window(&self) -> Option<Duration> {
866 self.suspended_window
867 }
868
869 pub fn watch_lifecycle(&self) -> tokio::sync::watch::Receiver<Lifecycle> {
876 self.lifecycle_watch.subscribe()
877 }
878
879 fn set_lifecycle(&mut self, lifecycle: Lifecycle) {
880 self.lifecycle = lifecycle;
881 self.lifecycle_watch.send_replace(lifecycle);
882 }
883
884 pub fn idle_window(&self) -> Option<Duration> {
886 self.idle_window
887 }
888
889 pub fn tunnel_identity(&self) -> Option<&crate::identity::TunnelIdentity> {
895 self.tunnel_identity.as_ref()
896 }
897
898 pub fn egress_posture(&self) -> crate::control::EgressPosture {
906 self.launch_posture.unwrap_or_default()
907 }
908
909 pub fn port(&self) -> u16 {
914 self.control.port()
915 }
916
917 pub async fn build_image(&mut self, request: CreateImageRequest) -> Result<&Image, Error> {
925 let size = request.size;
926 let created = self.control.create_image(request).await?;
927 let mut built = self
928 .control
929 .wait_for_image(&created.identifier, size, WaitOpts::default())
930 .await?;
931 built.log_group = created.log_group;
937 built.log_stream = created.log_stream;
938 self.image_exists = true;
941 self.image = Some(built);
942 Ok(self.image.as_ref().expect("just assigned"))
943 }
944
945 pub async fn ensure_image(
949 &mut self,
950 request: crate::control::EnsureImageRequest,
951 ) -> Result<crate::control::EnsuredImage, Error> {
952 let control = Arc::clone(&self.control);
961 let prepared =
962 tokio::task::spawn_blocking(move || crate::control::ensure::prepare(&control, request))
963 .await
964 .map_err(|joined| {
965 Error::new(
966 ErrorKind::Unexpected,
967 format!("preparing the image's artifact did not complete: {joined}"),
968 )
969 })??;
970 let services = self.build_services().await?;
971 let account = match &self.account {
972 Some(account) => account.clone(),
973 None => {
974 let account = services.caller_account().await?;
975 self.account = Some(account.clone());
976 account
977 }
978 };
979 let ensured =
980 crate::control::ensure::ensure(&self.control, services.as_ref(), &account, prepared)
981 .await?;
982 self.image_exists = true;
983 self.image = Some(ensured.image.clone());
984 Ok(ensured)
985 }
986
987 async fn build_services(&mut self) -> Result<Arc<dyn crate::control::BuildServices>, Error> {
990 if let Some(services) = &self.build_services {
991 return Ok(Arc::clone(services));
992 }
993 let built = self
994 .control
995 .adapters()
996 .build_services(self.control.region().clone())
997 .await?;
998 self.build_services = Some(Arc::clone(&built));
999 Ok(built)
1000 }
1001
1002 pub fn build_artifact_for(&self, request: &CreateImageRequest) -> Result<Vec<u8>, Error> {
1007 self.control.build_artifact_for(request)
1008 }
1009
1010 pub async fn upload_artifact_for(&mut self, request: &CreateImageRequest) -> Result<(), Error> {
1016 let (bucket, key) = crate::control::ensure::s3_location(&request.code_artifact_uri)?;
1017 let bytes = self.control.build_artifact_for(request)?;
1018 let services = self.build_services().await?;
1019 services.put_object(bucket, key, bytes).await
1020 }
1021
1022 pub fn preflight(&self, request: &CreateImageRequest) -> Result<(), Error> {
1029 self.control.preflight(request)
1030 }
1031
1032 pub async fn resolve_image_arn(&self, identifier: &str) -> Result<String, Error> {
1040 self.control.resolve_image_arn(identifier).await
1041 }
1042
1043 pub async fn find_image_by_name(
1046 &self,
1047 name: &str,
1048 ) -> Result<Option<crate::control::ops::MicrovmImageSummaryWire>, Error> {
1049 self.control.find_image_by_name(name).await
1050 }
1051
1052 pub fn artifact_content_hash_for(&self, request: &CreateImageRequest) -> String {
1054 self.control.artifact_content_hash_for(request)
1055 }
1056
1057 pub fn dockerfile_for(&self, request: &CreateImageRequest) -> String {
1060 self.control.dockerfile_for(request)
1061 }
1062
1063 pub async fn managed_base_versions(
1071 &self,
1072 base_image_arn: &str,
1073 ) -> Result<Vec<crate::control::ops::ManagedMicrovmImageVersionWire>, Error> {
1074 self.control.managed_base_versions(base_image_arn).await
1075 }
1076
1077 pub async fn image_version(
1079 &self,
1080 identifier: &str,
1081 version: &str,
1082 ) -> Result<crate::control::ops::MicrovmImageVersionSummaryWire, Error> {
1083 self.control.get_image_version(identifier, version).await
1084 }
1085
1086 pub async fn run(&mut self, request: RunRequest) -> Result<&mut Session, Error> {
1114 self.run_inner(request, None).await
1115 }
1116
1117 pub async fn run_cancellable(
1120 &mut self,
1121 request: RunRequest,
1122 cancellation: &crate::control::transport::RetryCancellation,
1123 ) -> Result<&mut Session, Error> {
1124 self.run_inner(request, Some(cancellation)).await
1125 }
1126
1127 async fn run_inner(
1128 &mut self,
1129 request: RunRequest,
1130 cancellation: Option<&crate::control::transport::RetryCancellation>,
1131 ) -> Result<&mut Session, Error> {
1132 self.refuse_detached("run")?;
1133 if self.bootstrap_count > 0 || self.microvm.is_some() {
1137 return Err(Error::invalid_arg(format!(
1138 "this sandbox has already launched a VM ({} bootstrap(s), lifecycle {}), and the \
1139 agent token is installed at most once per VM lifetime (STATE-3). A second VM \
1140 needs a second Sandbox — reusing this one would either re-deliver a run-hook \
1141 payload to a daemon whose one-shot bootstrap refuses it, or silently address \
1142 two guests through one handle.",
1143 self.bootstrap_count, self.lifecycle,
1144 )));
1145 }
1146
1147 let posture = crate::control::egress_posture_for(
1154 request.egress,
1155 &request.egress_network_connectors,
1156 request.deny_egress,
1157 Some(self.control.region()),
1158 )?;
1159
1160 let Some(identifier) = request
1161 .image_identifier
1162 .clone()
1163 .or_else(|| self.image.as_ref().map(|image| image.identifier.clone()))
1164 else {
1165 return Err(Error::new(
1166 ErrorKind::Precondition,
1167 "no image to launch: pass RunRequest::with_image or call build_image first."
1168 .to_string(),
1169 ));
1170 };
1171
1172 if request.client_token.is_some() && (request.agent_token.is_none() || request.identity) {
1173 return Err(Error::invalid_arg(
1174 "a stable client_token requires an explicit agent_token and identity=false; persist identical launch parameters before retrying",
1175 ));
1176 }
1177 let agent_token = match request.agent_token.clone() {
1178 Some(token) => token,
1179 None => mint_agent_token(self.control.entropy())?,
1180 };
1181 let launch_identity = if request.identity {
1185 Some(crate::entropy::launch_identity(self.control.entropy())?)
1186 } else {
1187 None
1188 };
1189 let payload = RunHookPayload::for_launch_with_identity(
1196 &agent_token,
1197 &request.effective_launch_env(),
1198 launch_identity.as_ref(),
1199 )?;
1200
1201 let mut wire = RunMicrovmRequest::new(identifier, payload);
1202 wire.image_version = request.image_version.clone();
1203 wire.execution_role_arn = request.execution_role_arn.clone();
1204 wire.egress_network_connectors = request.egress_network_connectors.clone();
1205 wire.max_idle_sec = request.max_idle_sec;
1206 wire.suspended_sec = request.suspended_sec;
1207 wire.auto_resume = request.auto_resume;
1208 wire.max_duration_sec = request.max_duration_sec;
1209 wire.token_scope = request.token_scope.clone();
1210 wire.client_token = request.client_token.clone();
1211 wire.logging = request.logging.clone();
1212 if request.egress {
1213 wire = wire.with_egress();
1214 }
1215 if request.shell {
1216 wire = wire.with_shell();
1217 }
1218
1219 self.control.launch_connectors(&wire)?;
1225 let resolution = self.control.resolve_image_arn(&wire.image_identifier);
1226 wire.image_identifier = if let Some(cancellation) = cancellation {
1227 tokio::select! {
1230 biased;
1231 _ = cancellation.cancelled() => {
1232 return Err(Error::new(
1233 ErrorKind::Interrupted,
1234 "launch cancelled before a VM was assigned",
1235 ));
1236 }
1237 resolved = resolution => resolved?,
1238 }
1239 } else {
1240 resolution.await?
1241 };
1242 self.launch_image_arn = Some(wire.image_identifier.clone());
1243
1244 let (launched, details) = self
1245 .control
1246 .run_microvm_reported(wire, cancellation)
1247 .await?;
1248 self.launch_metrics = Some(Arc::new(std::sync::Mutex::new(
1249 crate::session::LaunchMetrics {
1250 attempt_count: details.attempt_count,
1251 backoff_seconds: details.backoff.as_secs_f64(),
1252 ..Default::default()
1253 },
1254 )));
1255
1256 self.tunnel_identity = launch_identity.map(crate::identity::LaunchIdentity::keep);
1260
1261 self.set_lifecycle(Lifecycle::Pending);
1263 self.image_exists = true;
1264 self.suspended_window = Some(Duration::from_secs(u64::from(request.suspended_sec)));
1268 self.idle_window = Some(Duration::from_secs(u64::from(request.max_idle_sec)));
1269 self.microvm = Some(launched);
1270 self.launch_adoptable = request.client_token.is_some();
1273 self.launch_posture = Some(posture);
1274 self.pending_agent_token = Some(agent_token);
1275
1276 if request.wait {
1277 return self.wait_until_running(request.ready_timeout).await;
1278 }
1279 self.build_session()?;
1282 Ok(self.session.as_mut().expect("just assigned"))
1283 }
1284
1285 fn build_session(&mut self) -> Result<(), Error> {
1287 let (Some(vm), Some(agent_token)) = (&self.microvm, &self.pending_agent_token) else {
1288 return Err(Error::new(
1289 ErrorKind::Precondition,
1290 "no launch to build a session for",
1291 ));
1292 };
1293 let minter = Arc::new(ControlPlaneMinter::new(
1294 Arc::clone(&self.control),
1295 vm.id.clone(),
1296 ));
1297 let mut builder = Session::builder(vm.endpoint.clone(), agent_token.clone())
1300 .with_minter(minter)
1301 .with_port(self.control.port())
1302 .with_egress_posture(self.egress_posture())
1303 .with_clock(self.control.shared_clock());
1304 if let Some(backend) = &self.session_backend {
1305 builder = builder.with_backend(Arc::clone(backend));
1306 }
1307 let mut session = builder.build_with(self.control.adapters())?;
1308 session.launch_metrics = self.launch_metrics.clone();
1309 self.session = Some(session);
1310 Ok(())
1311 }
1312
1313 pub async fn wait_until_running(&mut self, timeout: Duration) -> Result<&mut Session, Error> {
1329 self.refuse_detached("wait until running")?;
1330 let id = self.require_microvm("wait_until_running")?;
1331 if self.lifecycle != Lifecycle::Pending {
1332 return Err(Error::invalid_arg(format!(
1333 "microvm {id} is {}; wait_until_running finishes a launch that is still PENDING",
1334 self.lifecycle,
1335 )));
1336 }
1337 let opts = WaitOpts::for_lifecycle(timeout);
1338 let running = self
1339 .control
1340 .wait_for_sandbox_launch(&id, opts, self.launch_adoptable)
1341 .await?;
1342
1343 self.set_lifecycle(Lifecycle::Running);
1346 if let Some(metrics) = &self.launch_metrics {
1347 metrics
1348 .lock()
1349 .unwrap_or_else(std::sync::PoisonError::into_inner)
1350 .running_at
1351 .get_or_insert_with(|| self.control.clock().unix_now().as_secs_f64());
1352 }
1353 self.token_installed = true;
1354 self.bootstrap_count += 1;
1355 self.microvm = Some(running);
1356 if self.session.is_none() {
1357 self.build_session()?;
1358 }
1359 let session = self.session.as_mut().expect("just assigned");
1360 session
1361 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1362 .await?;
1363 Ok(session)
1364 }
1365
1366 pub async fn wait_until_ready(
1376 &mut self,
1377 running_timeout: Duration,
1378 ) -> Result<&mut Session, Error> {
1379 if self.lifecycle == Lifecycle::Running {
1380 self.refuse_detached("wait until ready")?;
1381 let session = self.session.as_mut().ok_or_else(|| {
1382 Error::new(ErrorKind::Precondition, "a RUNNING sandbox with no session")
1383 })?;
1384 session
1385 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1386 .await?;
1387 return Ok(session);
1388 }
1389 self.wait_until_running(running_timeout).await
1390 }
1391
1392 pub async fn adopt(
1412 control: ControlPlane,
1413 microvm_id: impl Into<String>,
1414 endpoint: impl Into<String>,
1415 agent_token: impl Into<String>,
1416 ) -> Result<Self, Error> {
1417 let microvm_id = microvm_id.into();
1418 let endpoint = endpoint.into();
1419 let agent_token = agent_token.into();
1420 if agent_token.is_empty() {
1421 return Err(Error::invalid_arg(
1422 "adopt needs the agent token the VM was launched with: its daemon accepts only \
1423 that bearer, and adopting without it would build a handle every exec refuses.",
1424 ));
1425 }
1426 let mut vm = control.get_microvm(µvm_id).await?;
1427 if !endpoint.is_empty() && !vm.endpoint.is_empty() && vm.endpoint != endpoint {
1428 return Err(Error::invalid_arg(format!(
1429 "microvm {microvm_id} reports endpoint {}, not {endpoint}: the id and endpoint \
1430 came from different records, and a session built from them would address one \
1431 VM while its lifecycle calls reached another.",
1432 vm.endpoint,
1433 )));
1434 }
1435 if vm.endpoint.is_empty() {
1436 vm.endpoint = endpoint;
1437 }
1438 let Some(lifecycle) = Lifecycle::from_service(&vm.state) else {
1439 return Err(Error::new(
1440 ErrorKind::Platform,
1441 format!(
1442 "microvm {microvm_id} reports state {}, which this client does not know; \
1443 refusing to guess which lifecycle guards apply.",
1444 vm.state,
1445 ),
1446 ));
1447 };
1448
1449 let mut sandbox = Self::with_control_plane(control);
1450 sandbox.adopted = true;
1451 sandbox.set_lifecycle(lifecycle);
1452 sandbox.idle_window = vm
1455 .idle_policy
1456 .as_ref()
1457 .map(|policy| Duration::from_secs(u64::from(policy.max_idle_duration_seconds)));
1458 sandbox.image_exists = true;
1460 sandbox.launch_adoptable = true;
1462 sandbox.pending_agent_token = Some(agent_token);
1463 match lifecycle {
1464 Lifecycle::Pending => {}
1465 Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended => {
1466 sandbox.token_installed = true;
1467 sandbox.bootstrap_count = 1;
1468 }
1469 Lifecycle::Terminating | Lifecycle::Terminated => {
1470 sandbox.bootstrap_count = 1;
1471 sandbox.was_terminated = true;
1472 }
1473 }
1474 sandbox.microvm = Some(vm);
1475 if lifecycle.is_live() {
1476 sandbox.build_session()?;
1477 }
1478 Ok(sandbox)
1479 }
1480
1481 pub fn name_record(&self, name: &str) -> Result<crate::names::NameRecord, Error> {
1488 let Some(vm) = self.microvm() else {
1489 return Err(Error::new(
1490 ErrorKind::Precondition,
1491 "there is no VM to name yet: launch or adopt one first",
1492 ));
1493 };
1494 let Some(session) = self.session() else {
1495 return Err(Error::new(
1496 ErrorKind::Precondition,
1497 format!(
1498 "microvm {} has no session to take its agent token from",
1499 vm.id
1500 ),
1501 ));
1502 };
1503 let mut record = crate::names::NameRecord::new_at(
1505 name,
1506 vm.id.as_str(),
1507 vm.endpoint.as_str(),
1508 session.agent_token(),
1509 self.control.region().as_str(),
1510 self.control.clock().unix_now().as_secs(),
1511 )?;
1512 record.set_tunnel_identity(self.tunnel_identity.as_ref());
1513 Ok(record)
1514 }
1515
1516 pub async fn adopt_record(
1523 control: ControlPlane,
1524 record: crate::names::NameRecord,
1525 ) -> Result<Self, Error> {
1526 let identity = record.tunnel_identity()?;
1527 let mut sandbox = Self::adopt(
1528 control,
1529 record.microvm_id,
1530 record.endpoint,
1531 record.agent_token,
1532 )
1533 .await?;
1534 sandbox.tunnel_identity = identity;
1535 Ok(sandbox)
1536 }
1537
1538 pub fn detach(&mut self) -> Result<Detached, Error> {
1551 self.refuse_detached("detach")?;
1552 let Some(vm) = self.microvm.as_ref() else {
1553 return Err(Error::new(
1554 ErrorKind::Precondition,
1555 "nothing to detach: this sandbox has not launched a VM.",
1556 ));
1557 };
1558 if self.torn_down || !self.lifecycle.is_live() {
1559 return Err(Error::new(
1560 ErrorKind::Precondition,
1561 format!(
1562 "microvm {} is {} and there is no live VM to hand off; only a PENDING, \
1563 RUNNING, or SUSPENDED VM can be adopted.",
1564 vm.id, self.lifecycle,
1565 ),
1566 ));
1567 }
1568 let agent_token = self
1569 .session
1570 .as_ref()
1571 .map(|session| session.agent_token().to_string())
1572 .or_else(|| self.pending_agent_token.clone())
1573 .ok_or_else(|| {
1574 Error::new(
1575 ErrorKind::Precondition,
1576 format!(
1577 "microvm {} has no agent token to hand off; an adopter could not \
1578 reach its daemon.",
1579 vm.id
1580 ),
1581 )
1582 })?;
1583 let detached = Detached {
1584 microvm_id: vm.id.clone(),
1585 endpoint: vm.endpoint.clone(),
1586 region: self.control.region().clone(),
1587 port: self.control.port(),
1588 agent_token,
1589 };
1590 self.detached = true;
1591 self.session = None;
1592 self.pending_agent_token = None;
1593 Ok(detached)
1594 }
1595
1596 pub fn detached(&self) -> bool {
1598 self.detached
1599 }
1600
1601 fn refuse_detached(&self, what: &str) -> Result<(), Error> {
1603 if !self.detached {
1604 return Ok(());
1605 }
1606 let id = self.microvm.as_ref().map_or("?", |vm| vm.id.as_str());
1607 Err(Error::new(
1608 ErrorKind::Precondition,
1609 format!(
1610 "cannot {what}: microvm {id} was detached and belongs to whoever adopts it. \
1611 Adopt it with `Sandbox::adopt` and {what} through that handle."
1612 ),
1613 ))
1614 }
1615
1616 pub fn adopted(&self) -> bool {
1618 self.adopted
1619 }
1620
1621 pub async fn suspend(&mut self) -> Result<(), Error> {
1636 self.refuse_detached("suspend")?;
1637 let id = self.require_microvm("suspend")?;
1638
1639 if self.lifecycle != Lifecycle::Running {
1641 return Err(Error::invalid_arg(format!(
1642 "microvm {id} is {} and a suspend is only issued from RUNNING (STATE-5). Refused \
1643 here rather than by the service, because the service's answer about a \
1644 non-running id does not say which of the two things went wrong — and a suspend \
1645 issued from SUSPENDED is a caller who believes they resumed.",
1646 self.lifecycle,
1647 )));
1648 }
1649
1650 self.control.suspend(&id).await?;
1655 self.set_lifecycle(Lifecycle::Suspending);
1656
1657 let settled = self
1658 .control
1659 .wait_for_state(
1660 &id,
1661 &crate::control::microvm::SUSPEND_WANTED,
1662 &[],
1663 self.lifecycle_wait(),
1664 )
1665 .await?;
1666
1667 let settled_lifecycle = match settled.state.as_str() {
1672 "SUSPENDED" => Lifecycle::Suspended,
1673 "TERMINATED" => {
1674 self.was_terminated = true;
1675 Lifecycle::Terminated
1676 }
1677 other => {
1678 return Err(Error::new(
1679 ErrorKind::Platform,
1680 format!(
1681 "the suspend wait returned {other}, which is neither SUSPENDED nor \
1682 TERMINATED — and those two are what this client asked for."
1683 ),
1684 ));
1685 }
1686 };
1687 self.set_lifecycle(settled_lifecycle);
1688 self.microvm = Some(settled);
1689 Ok(())
1690 }
1691
1692 pub async fn resume(&mut self) -> Result<&mut Session, Error> {
1720 self.refuse_detached("resume")?;
1721 let id = self.require_microvm("resume")?;
1722
1723 if self.was_terminated || self.lifecycle == Lifecycle::Terminated {
1726 return Err(Error::invalid_arg(format!(
1727 "microvm {id} was terminated, and a terminated VM never returns to RUNNING \
1728 (STATE-11). There is nothing to resume: the guest's memory is gone, so even a \
1729 call the service accepted would hand back a different machine."
1730 )));
1731 }
1732 let mut resume_state = self.lifecycle;
1733 if self.lifecycle == Lifecycle::Running {
1736 let observed = self.control.get_microvm(&id).await?;
1737 resume_state = Lifecycle::from_service(&observed.state).ok_or_else(|| {
1738 Error::new(
1739 ErrorKind::Platform,
1740 format!("microvm {id} reports unknown state {}", observed.state),
1741 )
1742 })?;
1743 let ended = crate::constants::DEAD_STATES.contains(&observed.state.as_str());
1744 let error = ended.then(|| {
1745 Error::new(
1746 ErrorKind::WindowClosed,
1747 format!(
1748 "microvm {id} can't be resumed: GetMicrovm reports the VM {} ({}). \
1749 A VM that has ended doesn't come back; launch a new one.",
1750 observed.state,
1751 observed.reason(),
1752 ),
1753 )
1754 });
1755 self.microvm = Some(observed);
1756 if resume_state == Lifecycle::Suspended {
1759 self.set_lifecycle(resume_state);
1760 }
1761 if let Some(error) = error {
1762 self.was_terminated = true;
1763 self.set_lifecycle(Lifecycle::Terminated);
1764 return Err(error);
1765 }
1766 }
1767 if resume_state != Lifecycle::Suspended {
1768 return Err(Error::invalid_arg(format!(
1769 "microvm {id} is {} and a resume is only issued from SUSPENDED (STATE-7).",
1770 resume_state,
1771 )));
1772 }
1773
1774 if let Err(refused) = self.control.resume(&id).await {
1780 if refused.kind() == ErrorKind::WindowClosed {
1781 self.was_terminated = true;
1782 self.set_lifecycle(Lifecycle::Terminated);
1783 }
1784 return Err(refused);
1785 }
1786 let running = self
1792 .control
1793 .wait_for_state(
1794 &id,
1795 &["RUNNING"],
1796 &crate::constants::DEAD_STATES,
1797 self.lifecycle_wait(),
1798 )
1799 .await?;
1800
1801 self.set_lifecycle(Lifecycle::Running);
1802 let endpoint = running.endpoint.clone();
1806 self.microvm = Some(running);
1807 if let Some(session) = self.session.as_mut() {
1808 session.rebind(endpoint);
1809 }
1810
1811 self.session.as_mut().ok_or_else(|| {
1812 Error::new(
1813 ErrorKind::Unexpected,
1814 "the VM resumed but this sandbox holds no session, which run() always builds"
1815 .to_string(),
1816 )
1817 })
1818 }
1819
1820 pub async fn terminate(&mut self, opts: TeardownOpts) -> TeardownReport {
1828 let mut report = TeardownReport::default();
1829 if let Err(error) = self.refuse_detached("terminate") {
1830 report.failures.push(error.to_string());
1833 report.lifecycle = Some(self.lifecycle);
1834 return report;
1835 }
1836 self.torn_down = true;
1837
1838 self.session = None;
1841
1842 if let Some(id) = self.microvm.as_ref().map(|vm| vm.id.clone()) {
1844 self.set_lifecycle(Lifecycle::Terminating);
1848 self.was_terminated = true;
1849
1850 match self.control.terminate(&id).await {
1851 Ok(()) => report.terminate_accepted = true,
1852 Err(error) => {
1853 report.failures.push(format!("terminate {id}: {error}"));
1854 report.undeleted.push(id.clone());
1855 }
1856 }
1857
1858 if report.terminate_accepted
1859 && let Some(timeout) = opts.wait_for_terminated
1860 {
1861 let wait = WaitOpts::for_lifecycle(timeout);
1862 match self
1863 .control
1864 .wait_for_state(&id, &["TERMINATED"], &[], wait)
1865 .await
1866 {
1867 Ok(settled) => {
1869 self.set_lifecycle(Lifecycle::Terminated);
1870 self.microvm = Some(settled);
1871 }
1872 Err(error) => report
1875 .failures
1876 .push(format!("waiting for {id} to reach TERMINATED: {error}")),
1877 }
1878 }
1879 }
1880
1881 if opts.delete_image {
1884 match self.image.as_ref().map(|image| image.identifier.clone()) {
1885 Some(identifier) => {
1886 let deleted = self
1887 .control
1888 .delete_image(&identifier, opts.delete_attempts, opts.delete_backoff)
1889 .await;
1890 report.image_deleted = Some(deleted);
1891 if deleted {
1892 self.image_exists = false;
1893 } else {
1894 report.failures.push(format!(
1895 "the image {identifier} survived {} delete attempts",
1896 opts.delete_attempts.max(1)
1897 ));
1898 report.undeleted.push(identifier);
1899 }
1900 }
1901 None => report.image_deleted = Some(false),
1902 }
1903 }
1904
1905 if opts.delete_log_group
1907 && let Some(group) = self.image.as_ref().map(Image::build_log_group)
1908 {
1909 report.failures.push(format!(
1910 "the build log group {group} was not deleted: CloudWatch Logs is not in this \
1911 crate's dependency set, so it is reported rather than removed. It is \
1912 service-created, which means no Terraform stack owns it and `terraform destroy` \
1913 leaves it behind — six accumulated before anyone noticed."
1914 ));
1915 report.undeleted.push(group);
1916 }
1917
1918 report.lifecycle = Some(self.lifecycle);
1919 report
1920 }
1921
1922 fn require_microvm(&self, what: &str) -> Result<String, Error> {
1926 self.microvm
1927 .as_ref()
1928 .map(|vm| vm.id.clone())
1929 .ok_or_else(|| {
1930 Error::new(
1931 ErrorKind::Precondition,
1932 format!("nothing to {what}: this sandbox has not launched a VM."),
1933 )
1934 })
1935 }
1936
1937 fn lifecycle_wait(&self) -> WaitOpts {
1939 WaitOpts::for_lifecycle(DEFAULT_LIFECYCLE_TIMEOUT)
1940 }
1941}
1942
1943impl Sandbox {
1952 fn drop_warning(&self) -> Option<String> {
1956 if self.torn_down || self.adopted || self.detached {
1957 return None;
1958 }
1959 let vm = self.microvm.as_ref()?;
1960 self.lifecycle.is_live().then(|| {
1961 format!(
1962 "warning: the Sandbox for microvm {id} was dropped in {state} without \
1963 terminate() or detach(). Nothing was torn down — Drop cannot await, so a \
1964 teardown here would deadlock inside a runtime. {state} is the state this \
1965 sandbox last saw, and the platform may have suspended or ended the VM since. A \
1966 VM that's still live bills until it ends, at its maximumDurationInSeconds \
1967 ceiling at the latest: terminate it with `microvm terminate {id}`.",
1968 id = vm.id,
1969 state = self.lifecycle,
1970 )
1971 })
1972 }
1973}
1974
1975impl Drop for Sandbox {
1976 fn drop(&mut self) {
1977 if let Some(warning) = self.drop_warning() {
1978 self.control.adapters().warn(&warning);
1979 }
1980 }
1981}
1982
1983fn mint_agent_token(entropy: &dyn crate::entropy::Entropy) -> Result<String, Error> {
1989 let mut bytes = [0u8; 32];
1990 entropy.fill(&mut bytes)?;
1991 Ok(const_hex::encode(bytes))
1992}
1993
1994#[cfg(test)]
1995mod tests {
1996 use super::*;
1997 use crate::control::fake::{self as fake, Answer, FakeControlPlane, TestClock};
1998 use crate::control::transport::Method;
1999
2000 fn planted() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2010 let recorder = Arc::new(FakeControlPlane::new());
2011 let clock = Arc::new(TestClock::new());
2012 let plane = crate::testing::control_plane(
2013 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2014 Region::US_EAST_1,
2015 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2016 );
2017 let sandbox = Sandbox::with_control_plane(plane)
2019 .with_session_backend(crate::testing::HealthyDaemon::new());
2020 (sandbox, recorder, clock)
2021 }
2022
2023 #[test]
2026 fn the_suspend_resume_and_terminate_waits_are_cores_lifecycle_wait() {
2027 let (sandbox, _, _) = planted();
2028 let wait = sandbox.lifecycle_wait();
2029 assert_eq!(
2030 (wait.timeout, wait.poll_interval, wait.stall_grace),
2031 (
2032 DEFAULT_LIFECYCLE_TIMEOUT,
2033 LIFECYCLE_POLL_INTERVAL,
2034 Duration::MAX
2035 )
2036 );
2037 }
2038
2039 fn answer_launch(recorder: &FakeControlPlane) {
2041 recorder
2042 .answer(
2043 "RunMicrovm",
2044 Answer::ok(fake::microvm_response("PENDING", None)),
2045 )
2046 .answer(
2047 "GetMicrovm",
2048 Answer::ok(fake::microvm_response("RUNNING", None)),
2049 )
2050 .answer(
2051 "CreateMicrovmAuthToken",
2052 Answer::ok(fake::auth_token_response("proxy-token")),
2053 );
2054 }
2055
2056 #[tokio::test(start_paused = true)]
2057 async fn sandbox_launch_polls_at_one_second_then_backs_off() {
2058 for pending_polls in 0..=6 {
2059 let (mut sandbox, recorder, clock) = planted();
2060 recorder.answer(
2061 "RunMicrovm",
2062 Answer::ok(fake::microvm_response("PENDING", None)),
2063 );
2064 for _ in 0..pending_polls {
2065 recorder.answer(
2066 "GetMicrovm",
2067 Answer::ok(fake::microvm_response("PENDING", None)),
2068 );
2069 }
2070 recorder
2071 .answer(
2072 "GetMicrovm",
2073 Answer::ok(fake::microvm_response("RUNNING", None)),
2074 )
2075 .answer(
2076 "CreateMicrovmAuthToken",
2077 Answer::ok(fake::auth_token_response("proxy-token")),
2078 );
2079 sandbox
2080 .run(RunRequest::new().with_image("arn:image"))
2081 .await
2082 .expect("ready");
2083 let expected_seconds = [0, 1, 2, 3, 8, 13, 18][pending_polls];
2084 assert_eq!(clock.now(), Duration::from_secs(expected_seconds));
2085 assert_eq!(recorder.call_count("GetMicrovm"), pending_polls + 1);
2086 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2087 sandbox.detach().expect("hand the scripted VM off quietly");
2088 }
2089
2090 let (sandbox, recorder, clock) = planted();
2092 recorder
2093 .answer(
2094 "GetMicrovm",
2095 Answer::ok(fake::microvm_response("PENDING", None)),
2096 )
2097 .answer(
2098 "GetMicrovm",
2099 Answer::ok(fake::microvm_response("RUNNING", None)),
2100 );
2101 sandbox
2102 .control
2103 .wait_for_launch("mvm-abc123", WaitOpts::for_launch())
2104 .await
2105 .expect("ready");
2106 assert_eq!(clock.now(), Duration::from_secs(5));
2107
2108 let (mut sandbox, recorder, _) = planted();
2110 recorder
2111 .answer(
2112 "RunMicrovm",
2113 Answer::ok(fake::microvm_response("PENDING", None)),
2114 )
2115 .answer(
2116 "GetMicrovm",
2117 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2118 );
2119 let error = sandbox
2120 .run(RunRequest::new().with_image("arn:image"))
2121 .await
2122 .expect_err("a fresh launch fails fast");
2123 assert_eq!(error.kind(), ErrorKind::LaunchDied);
2124 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
2125 sandbox.detach().expect("hand the scripted VM off quietly");
2126 }
2127
2128 #[tokio::test(start_paused = true)]
2129 async fn vpc_egress_configuration_survives_the_sandbox_launch() {
2130 let (mut sandbox, recorder, _) = planted();
2131 answer_launch(&recorder);
2132 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2133 let request = RunRequest::new()
2134 .with_image("arn:image")
2135 .with_egress_network_connector(arn);
2136 assert_eq!(
2137 request.egress_posture(),
2138 crate::control::EgressPosture::Unsealed
2139 );
2140 sandbox.run(request).await.expect("launch");
2141 let body = recorder.first_body("RunMicrovm");
2142 assert_eq!(body["egressNetworkConnectors"], serde_json::json!([arn]));
2143 }
2144
2145 #[tokio::test(start_paused = true)]
2151 async fn a_sandbox_session_reads_the_planes_clock() {
2152 let (mut sandbox, recorder, clock) = planted();
2153 answer_launch(&recorder);
2154 let session = sandbox
2155 .run(RunRequest::new().with_image("arn:image"))
2156 .await
2157 .expect("the launch reaches RUNNING");
2158 let pinned = Duration::from_secs(1_767_225_600) + Duration::from_nanos(987_654_321);
2159 clock.set_unix_now(pinned);
2160 let id = session.mint_exec_id();
2161 let value =
2162 u64::from_str_radix(id.strip_prefix("x-").expect("the x- shape"), 16).expect("hex");
2163 let low_bits = (1_u64 << 40) - 1;
2164 assert_eq!(
2165 value & low_bits,
2166 (pinned.as_nanos() as u64) & low_bits,
2167 "{id} wasn't minted from the plane's clock"
2168 );
2169 }
2170
2171 #[tokio::test]
2178 async fn ensure_image_builds_its_services_through_the_planes_adapters() {
2179 let (mut sandbox, recorder, _) = planted();
2180 let dockerfile = crate::control::artifact::wrap_dockerfile(
2181 "FROM python:3.12-slim\nWORKDIR /app\n",
2182 &crate::control::artifact::WrapOptions::default(),
2183 )
2184 .expect("a wrappable Dockerfile");
2185 let request = crate::control::EnsureImageRequest::new(
2186 "task",
2187 crate::testing::aarch64_daemon(b"daemon"),
2188 dockerfile,
2189 "artifact-bucket",
2190 "arn:aws:iam::123456789012:role/build",
2191 );
2192 let error = sandbox
2193 .ensure_image(request)
2194 .await
2195 .expect_err("the test adapters build no services");
2196 assert_eq!(error.kind(), ErrorKind::Credentials, "{error}");
2197 assert!(
2198 error
2199 .to_string()
2200 .contains("the test adapters build no STS or S3 client for us-east-1"),
2201 "{error}"
2202 );
2203 assert!(
2204 recorder.calls().is_empty(),
2205 "no control-plane call came first"
2206 );
2207 }
2208
2209 #[tokio::test(start_paused = true)]
2217 async fn a_launch_draws_every_random_value_from_the_planes_entropy() {
2218 use crate::entropy::testing::SequenceEntropy;
2219 use base64::Engine as _;
2220
2221 let recorder = Arc::new(FakeControlPlane::new());
2222 let entropy = Arc::new(SequenceEntropy::new());
2223 let plane = crate::testing::control_plane(
2224 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2225 Region::US_EAST_1,
2226 Arc::new(TestClock::new()),
2227 )
2228 .with_entropy(Arc::clone(&entropy) as Arc<dyn crate::entropy::Entropy>);
2229 let mut sandbox = Sandbox::with_control_plane(plane)
2230 .with_session_backend(crate::testing::HealthyDaemon::new());
2231 answer_launch(&recorder);
2232 let session = sandbox
2233 .run(RunRequest::new().with_image("arn:image").with_identity())
2234 .await
2235 .expect("the launch reaches RUNNING");
2236
2237 assert_eq!(
2238 session.agent_token(),
2239 const_hex::encode(SequenceEntropy::draw(1, 32)),
2240 "the agent token"
2241 );
2242 let body = recorder.first_body("RunMicrovm");
2243 let payload: serde_json::Value =
2244 serde_json::from_str(body["runHookPayload"].as_str().expect("a string payload"))
2245 .expect("the payload is JSON");
2246 assert_eq!(
2247 payload[protocol::identity::SEED_KEY],
2248 base64::engine::general_purpose::STANDARD
2249 .encode(SequenceEntropy::draw(2, protocol::identity::SEED_BYTES)),
2250 "the VM seed"
2251 );
2252 let client_token = body["clientToken"].as_str().expect("a client token");
2253 assert!(
2254 client_token.ends_with(&const_hex::encode(SequenceEntropy::draw(4, 8))),
2255 "the client token's nonce: {client_token}"
2256 );
2257 assert_eq!(
2258 entropy.calls(),
2259 4,
2260 "one draw per value, the host seed included"
2261 );
2262 sandbox.detach().expect("hand the scripted VM off quietly");
2263 }
2264
2265 #[tokio::test]
2267 async fn an_unavailable_entropy_source_refuses_the_launch() {
2268 use crate::entropy::testing::SequenceEntropy;
2269
2270 let recorder = Arc::new(FakeControlPlane::new());
2271 let plane = crate::testing::control_plane(
2272 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2273 Region::US_EAST_1,
2274 Arc::new(TestClock::new()),
2275 )
2276 .with_entropy(Arc::new(SequenceEntropy::unavailable()));
2277 let mut sandbox = Sandbox::with_control_plane(plane)
2278 .with_session_backend(crate::testing::HealthyDaemon::new());
2279 answer_launch(&recorder);
2280 let err = sandbox
2281 .run(RunRequest::new().with_image("arn:image"))
2282 .await
2283 .expect_err("no pool, no bearer token");
2284 assert_eq!(err.kind(), ErrorKind::Unexpected, "{err}");
2285 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2286 }
2287
2288 fn launching_to(
2293 replies: Vec<crate::testing::Reply>,
2294 ) -> (
2295 Sandbox,
2296 Arc<FakeControlPlane>,
2297 Arc<crate::testing::Recorder>,
2298 ) {
2299 let recorder = Arc::new(FakeControlPlane::new());
2300 let plane = crate::testing::control_plane(
2301 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2302 Region::US_EAST_1,
2303 Arc::new(TestClock::new()),
2304 );
2305 let daemon = crate::testing::Recorder::with(replies);
2306 let sandbox = Sandbox::with_control_plane(plane).with_session_backend(daemon.clone());
2307 answer_launch(&recorder);
2308 (sandbox, recorder, daemon)
2309 }
2310
2311 #[tokio::test(start_paused = true)]
2312 async fn launch_metrics_track_retries_running_and_ready_without_changing_on_clones() {
2313 use crate::clock::Clock;
2314 let recorder = Arc::new(FakeControlPlane::new());
2315 recorder.answer("RunMicrovm", Answer::throttled_as_400());
2316 answer_launch(&recorder);
2317 let clock = Arc::new(TestClock::new());
2318 let plane =
2319 crate::testing::control_plane(recorder.clone(), Region::US_EAST_1, clock.clone());
2320 let mut sandbox = Sandbox::with_control_plane(plane)
2321 .with_session_backend(crate::testing::HealthyDaemon::new());
2322 let mut request = RunRequest::new().with_image("arn:image");
2323 request.wait = false;
2324 let session = sandbox.run(request).await.unwrap().clone();
2325 let accepted = session.launch_metrics().unwrap();
2326 assert_eq!(accepted.attempt_count, 2);
2327 assert!(accepted.backoff_seconds > 0.0);
2328 assert_eq!(accepted.running_at, None);
2329 assert_eq!(accepted.ready_at, None);
2330 clock.advance(Duration::from_secs(7));
2331 sandbox
2332 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
2333 .await
2334 .unwrap();
2335 let ready = session.launch_metrics().unwrap();
2336 assert_eq!(ready.attempt_count, accepted.attempt_count);
2337 assert_eq!(ready.backoff_seconds, accepted.backoff_seconds);
2338 assert_eq!(ready.running_at, Some(clock.unix_now().as_secs_f64()));
2339 assert_eq!(ready.ready_at, ready.running_at);
2340 clock.advance(Duration::from_secs(3));
2341 session
2342 .wait_until_ready(Duration::from_secs(1))
2343 .await
2344 .unwrap();
2345 assert_eq!(session.launch_metrics(), Some(ready));
2346 sandbox.detach().unwrap();
2347 }
2348
2349 #[tokio::test]
2350 async fn cancellation_preserves_an_inflight_accepted_launch() {
2351 use crate::control::transport::RetryCancellation;
2352 let (mut sandbox, recorder, _) = planted();
2353 answer_launch(&recorder);
2354 let gate = recorder.hold_responses();
2355 let cancellation = RetryCancellation::default();
2356 let mut request = RunRequest::new().with_image("arn:image");
2357 request.wait = false;
2358 {
2359 let pending = sandbox.run_cancellable(request, &cancellation);
2360 tokio::pin!(pending);
2361 tokio::select! {
2362 biased;
2363 _ = &mut pending => panic!("response must wait for the gate"),
2364 _ = std::future::ready(()) => {}
2365 }
2366 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2367 cancellation.cancel();
2368 gate.notify_one();
2369 assert_eq!(
2370 pending
2371 .await
2372 .unwrap()
2373 .launch_metrics()
2374 .unwrap()
2375 .attempt_count,
2376 1
2377 );
2378 }
2379 assert!(sandbox.microvm().is_some());
2380 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2381 sandbox.detach().unwrap();
2382 }
2383
2384 #[tokio::test]
2385 async fn cancelled_retry_does_not_launch_before_any_request() {
2386 use crate::control::transport::RetryCancellation;
2387 let (mut sandbox, recorder, _) = planted();
2388 recorder.answer(
2389 "RunMicrovm",
2390 Answer::failure(403, "pre-cancelled request reached the transport"),
2391 );
2392 let cancellation = RetryCancellation::default();
2393 cancellation.cancel();
2394 let error = sandbox
2395 .run_cancellable(RunRequest::new().with_image("arn:image"), &cancellation)
2396 .await
2397 .unwrap_err();
2398 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2399 assert_eq!(error.kind(), ErrorKind::Interrupted);
2400 assert!(sandbox.microvm().is_none());
2401 }
2402
2403 #[tokio::test]
2404 async fn cancellation_interrupts_image_resolution_and_releases_the_sandbox() {
2405 use crate::control::transport::{RetryCancellation, RetryPolicy};
2406 let recorder = Arc::new(FakeControlPlane::new());
2407 recorder.answer("ListMicrovmImages", Answer::throttled_as_400());
2408 let plane = crate::testing::control_plane(
2409 recorder.clone(),
2410 Region::US_EAST_1,
2411 Arc::new(crate::testing::YieldingClock::new()),
2412 )
2413 .with_retry(RetryPolicy::new(100, Duration::from_secs(3600)).unwrap());
2414 let sandbox = tokio::sync::Mutex::new(Sandbox::with_control_plane(plane));
2415 let cancellation = RetryCancellation::default();
2416 let pending = async {
2417 sandbox
2418 .lock()
2419 .await
2420 .run_cancellable(RunRequest::new().with_image("named-image"), &cancellation)
2421 .await
2422 .map(|_| ())
2423 };
2424 tokio::pin!(pending);
2425 tokio::select! {
2426 biased;
2427 _ = &mut pending => panic!("image resolution must be retrying"),
2428 _ = std::future::ready(()) => {}
2429 }
2430 assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2431 assert!(sandbox.try_lock().is_err());
2432 cancellation.cancel();
2433 let error = tokio::select! {
2434 biased;
2435 result = &mut pending => result.unwrap_err(),
2436 _ = std::future::ready(()) => panic!("cancellation must finish without another sleep"),
2437 };
2438 assert_eq!(error.kind(), ErrorKind::Interrupted);
2439 assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2440 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2441 let sandbox = sandbox.try_lock().expect("cancellation releases the lock");
2442 assert!(sandbox.microvm().is_none());
2443 }
2444
2445 fn not_ready_then_ready() -> Vec<crate::testing::Reply> {
2447 vec![
2448 crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()),
2449 crate::testing::Reply::ok(crate::testing::health_body(true)),
2450 ]
2451 }
2452
2453 fn daemon_paths(daemon: &crate::testing::Recorder) -> Vec<String> {
2455 daemon
2456 .requests()
2457 .iter()
2458 .map(|request| format!("{} {}", request.method, request.path))
2459 .collect()
2460 }
2461
2462 #[tokio::test(start_paused = true)]
2469 async fn a_launch_returns_once_its_daemon_answers() {
2470 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2471 sandbox
2472 .run(RunRequest::new().with_image("arn:image"))
2473 .await
2474 .expect("the launch reaches RUNNING and its daemon answers");
2475 assert_eq!(
2476 daemon_paths(&daemon),
2477 ["GET /v1/health", "GET /v1/health"],
2478 "one poll refused, one bootstrapped, and nothing after"
2479 );
2480 sandbox.detach().expect("hand the scripted VM off quietly");
2481 }
2482
2483 #[tokio::test(start_paused = true)]
2487 async fn wait_until_running_returns_once_the_daemon_answers() {
2488 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2489 let mut request = RunRequest::new().with_image("arn:image");
2490 request.wait = false;
2491 sandbox.run(request).await.expect("the launch is accepted");
2492 assert!(
2493 daemon_paths(&daemon).is_empty(),
2494 "an accepted launch doesn't wait"
2495 );
2496 sandbox
2497 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
2498 .await
2499 .expect("RUNNING, then the daemon answers");
2500 assert_eq!(daemon_paths(&daemon), ["GET /v1/health", "GET /v1/health"]);
2501 sandbox.detach().expect("hand the scripted VM off quietly");
2502 }
2503
2504 #[tokio::test(start_paused = true)]
2508 async fn a_launch_whose_daemon_never_answers_times_out_running() {
2509 let never = (0..128)
2510 .map(|_| crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()))
2511 .collect();
2512 let (mut sandbox, _recorder, daemon) = launching_to(never);
2513 let error = sandbox
2514 .run(RunRequest::new().with_image("arn:image"))
2515 .await
2516 .expect_err("the daemon never answered");
2517 assert_eq!(error.kind(), ErrorKind::Timeout, "{error}");
2518 assert!(daemon.requests().len() > 1, "it kept asking");
2519 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2520 assert!(sandbox.token_installed());
2521 sandbox.detach().expect("hand the scripted VM off quietly");
2522 }
2523
2524 const WANTED_ARN: &str = "arn:aws:lambda:us-east-1:123456789012:microvm-image:wanted-image";
2528
2529 fn answer_listing(recorder: &FakeControlPlane) {
2531 recorder.answer(
2532 "ListMicrovmImages",
2533 Answer::ok(fake::list_images_response(
2534 &["wanted-image-old", "wanted-image"],
2535 None,
2536 )),
2537 );
2538 }
2539
2540 #[tokio::test(start_paused = true)]
2547 async fn a_launch_by_bare_image_name_sends_the_listed_arn() {
2548 let (mut sandbox, recorder, _) = planted();
2549 answer_listing(&recorder);
2550 answer_launch(&recorder);
2551 sandbox
2552 .run(RunRequest::new().with_image("wanted-image"))
2553 .await
2554 .expect("the launch reaches RUNNING");
2555
2556 let operations = recorder.operations();
2557 assert_eq!(
2558 operations.get(..2),
2559 Some(&["ListMicrovmImages", "RunMicrovm"][..]),
2560 "the name is resolved, once, before the launch: {operations:?}"
2561 );
2562 assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2563 assert_eq!(
2564 recorder.first_body("RunMicrovm")["imageIdentifier"],
2565 WANTED_ARN,
2566 "the launch carries the exact match's ARN, never the name"
2567 );
2568 sandbox.detach().expect("hand the scripted VM off quietly");
2569 }
2570
2571 #[tokio::test(start_paused = true)]
2573 async fn a_launch_by_image_arn_makes_no_listing_call() {
2574 let (mut sandbox, recorder, _) = planted();
2575 answer_launch(&recorder);
2576 sandbox
2577 .run(RunRequest::new().with_image(WANTED_ARN))
2578 .await
2579 .expect("the launch reaches RUNNING");
2580
2581 assert_eq!(recorder.call_count("ListMicrovmImages"), 0);
2582 assert_eq!(recorder.operations().first(), Some(&"RunMicrovm"));
2583 assert_eq!(
2584 recorder.first_body("RunMicrovm")["imageIdentifier"],
2585 WANTED_ARN
2586 );
2587 assert_eq!(sandbox.launch_image_arn(), Some(WANTED_ARN));
2588 sandbox.detach().expect("hand the scripted VM off quietly");
2589 }
2590
2591 #[tokio::test]
2598 async fn a_refused_launch_by_name_still_records_the_resolved_arn() {
2599 let (mut sandbox, recorder, _) = planted();
2600 answer_listing(&recorder);
2601 recorder.answer("RunMicrovm", Answer::failure(400, "scripted refusal"));
2602 sandbox
2603 .run(RunRequest::new().with_image("wanted-image"))
2604 .await
2605 .expect_err("the service refused the launch");
2606
2607 assert_eq!(
2608 sandbox.launch_image_arn(),
2609 Some(WANTED_ARN),
2610 "the ARN the refused launch asked for"
2611 );
2612 assert!(sandbox.microvm().is_none(), "no launch was accepted");
2613 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2614 }
2615
2616 #[tokio::test(start_paused = true)]
2620 async fn an_unknown_image_name_is_refused_before_the_launch_and_the_sandbox_still_runs() {
2621 let (mut sandbox, recorder, _) = planted();
2622 recorder.answer(
2623 "ListMicrovmImages",
2624 Answer::ok(fake::list_images_response(&[], None)),
2625 );
2626 let error = sandbox
2627 .run(RunRequest::new().with_image("no-such-image"))
2628 .await
2629 .expect_err("nothing to launch from");
2630 assert_eq!(error.code(), "ERR_PRECONDITION");
2631 assert!(error.to_string().contains("no-such-image"), "{error}");
2632 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2633 assert_eq!(sandbox.launch_image_arn(), None);
2634 assert!(!sandbox.image_exists(), "STATE-1: no launch was accepted");
2635 assert!(sandbox.microvm().is_none());
2636 assert_eq!(sandbox.bootstrap_count(), 0);
2637
2638 answer_launch(&recorder);
2639 sandbox
2640 .run(RunRequest::new().with_image(WANTED_ARN))
2641 .await
2642 .expect("a failed resolution doesn't use up the sandbox's one launch");
2643 assert_eq!(sandbox.bootstrap_count(), 1);
2644 sandbox.detach().expect("hand the scripted VM off quietly");
2645 }
2646
2647 #[tokio::test]
2653 async fn a_launch_refused_locally_makes_no_listing_call_for_a_name() {
2654 let (mut sandbox, recorder, _) = planted();
2655 answer_listing(&recorder);
2656 let mut request = RunRequest::new().with_image("wanted-image");
2657 request.max_duration_sec = 0;
2658 let error = sandbox
2659 .run(request)
2660 .await
2661 .expect_err("a zero maximum duration is refused locally");
2662 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
2663 assert!(
2664 recorder.calls().is_empty(),
2665 "the refusal came after a call: {:?}",
2666 recorder.operations()
2667 );
2668 assert_eq!(sandbox.launch_image_arn(), None);
2669 }
2670
2671 #[tokio::test(start_paused = true)]
2678 async fn a_launched_session_reports_its_requests_posture() {
2679 use crate::control::EgressPosture;
2680 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2681 let rows = [
2682 (RunRequest::new(), EgressPosture::Unsealed),
2683 (RunRequest::new().with_egress(), EgressPosture::Open),
2684 (
2685 RunRequest::new().with_deny_egress(),
2686 EgressPosture::BestEffort,
2687 ),
2688 (
2689 RunRequest::new().with_egress_network_connector(arn),
2690 EgressPosture::Unsealed,
2691 ),
2692 ];
2693 for (request, expected) in rows {
2694 let answered = crate::control::egress_posture_for(
2695 request.egress,
2696 &request.egress_network_connectors,
2697 request.deny_egress,
2698 Some(&Region::US_EAST_1),
2699 )
2700 .expect("a launchable request");
2701 assert_eq!(answered, expected);
2702 let (mut sandbox, recorder, _) = planted();
2703 answer_launch(&recorder);
2704 let session = sandbox
2705 .run(request.with_image("arn:image"))
2706 .await
2707 .expect("the launch reaches RUNNING");
2708 assert_eq!(session.egress_posture(), expected, "BIND-12");
2709 assert_eq!(sandbox.egress_posture(), expected);
2710 sandbox.detach().expect("hand the scripted VM off quietly");
2711 }
2712 }
2713
2714 #[tokio::test]
2718 async fn a_session_without_its_launch_options_reports_unsealed() {
2719 use crate::control::EgressPosture;
2720 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2721 assert_eq!(
2722 sandbox.session().expect("RUNNING").egress_posture(),
2723 EgressPosture::Unsealed
2724 );
2725 assert_eq!(sandbox.egress_posture(), EgressPosture::Unsealed);
2726 let direct = Session::builder(ADOPT_ENDPOINT, ADOPT_TOKEN)
2727 .build_with(&crate::testing::TestAdapters::new())
2728 .expect("a direct session");
2729 assert_eq!(direct.egress_posture(), EgressPosture::Unsealed);
2730 }
2731
2732 async fn launched() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2734 let (mut sandbox, recorder, clock) = planted();
2735 answer_launch(&recorder);
2736 sandbox
2737 .run(RunRequest::new().with_image("arn:image"))
2738 .await
2739 .expect("the launch reaches RUNNING");
2740 (sandbox, recorder, clock)
2741 }
2742
2743 #[tokio::test(start_paused = true)]
2751 async fn a_dropped_live_sandbox_warns_through_the_planes_adapters() {
2752 let recorder = Arc::new(FakeControlPlane::new());
2753 let clock = Arc::new(TestClock::new());
2754 let adapters = Arc::new(crate::testing::TestAdapters::new());
2755 let plane = crate::control::ControlPlane::from_ports(
2756 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2757 Region::US_EAST_1,
2758 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2759 Arc::new(crate::testing::SequenceEntropy::new()),
2760 Arc::clone(&adapters) as Arc<dyn crate::adapters::Adapters>,
2761 );
2762 let mut sandbox = Sandbox::with_control_plane(plane)
2763 .with_session_backend(crate::testing::HealthyDaemon::new());
2764 answer_launch(&recorder);
2765 sandbox
2766 .run(RunRequest::new().with_image("arn:image"))
2767 .await
2768 .expect("the launch reaches RUNNING");
2769 let id = sandbox.microvm().expect("launched").id.clone();
2770 drop(sandbox);
2771 let warnings = adapters.warnings();
2772 assert_eq!(warnings.len(), 1, "{warnings:?}");
2773 assert!(warnings[0].contains(&id), "{warnings:?}");
2774 assert!(
2775 warnings[0].contains("RUNNING is the state this sandbox last saw")
2776 && warnings[0].contains("A VM that's still live bills"),
2777 "{warnings:?}"
2778 );
2779 }
2780
2781 #[tokio::test(start_paused = true)]
2791 async fn a_detached_sandbox_hands_off_its_vm_and_stays_quiet() {
2792 let (mut sandbox, recorder, _clock) = launched().await;
2793 assert!(
2794 sandbox.drop_warning().is_some(),
2795 "a launched, live VM warns on drop"
2796 );
2797 let token = sandbox
2798 .session()
2799 .expect("launched")
2800 .agent_token()
2801 .to_string();
2802 let vm = sandbox.microvm().expect("launched").clone();
2803
2804 let detached = sandbox.detach().expect("a RUNNING VM detaches");
2805 assert_eq!(detached.microvm_id, vm.id);
2806 assert_eq!(detached.endpoint, vm.endpoint);
2807 assert_eq!(detached.agent_token(), token);
2808 assert_eq!(detached.region, Region::US_EAST_1);
2809 let printed = format!("{detached:?} {sandbox:?}");
2810 assert!(!printed.contains(&token), "{printed}");
2811 assert!(printed.contains("detached: true"), "{printed}");
2812
2813 assert!(sandbox.detached());
2814 assert!(
2815 sandbox.drop_warning().is_none(),
2816 "detached is not abandoned"
2817 );
2818 assert!(
2819 sandbox.session().is_none(),
2820 "the session went to the adopter"
2821 );
2822
2823 let before = recorder.calls().len();
2824 for error in [
2825 sandbox.suspend().await.expect_err("detached"),
2826 sandbox.resume().await.map(|_| ()).expect_err("detached"),
2827 sandbox
2828 .run(RunRequest::new().with_image("arn:image"))
2829 .await
2830 .map(|_| ())
2831 .expect_err("detached"),
2832 sandbox
2833 .wait_until_running(Duration::from_secs(1))
2834 .await
2835 .map(|_| ())
2836 .expect_err("detached"),
2837 sandbox.detach().map(|_| ()).expect_err("already detached"),
2838 ] {
2839 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2840 assert!(error.to_string().contains("detached"), "{error}");
2841 assert!(!error.to_string().contains(&token), "{error}");
2842 }
2843 let report = sandbox.terminate(TeardownOpts::default()).await;
2844 assert!(!report.terminate_accepted);
2845 assert!(report.failures[0].contains("detached"), "{report:?}");
2846 assert_eq!(recorder.calls().len(), before, "detached makes no AWS call");
2847 assert!(
2848 sandbox.drop_warning().is_none(),
2849 "a refused terminate is still quiet"
2850 );
2851 }
2852
2853 #[tokio::test(start_paused = true)]
2855 async fn detach_refuses_without_a_live_vm() {
2856 let (mut unlaunched, _recorder, _clock) = planted();
2857 let error = unlaunched.detach().expect_err("nothing launched");
2858 assert_eq!(error.kind(), ErrorKind::Precondition);
2859 assert!(!unlaunched.detached());
2860
2861 let (mut sandbox, recorder, _clock) = launched().await;
2862 recorder
2863 .answer("TerminateMicrovm", Answer::ok("{}"))
2864 .answer(
2865 "GetMicrovm",
2866 Answer::ok(fake::microvm_response("TERMINATED", None)),
2867 );
2868 sandbox.terminate(TeardownOpts::default()).await;
2869 let error = sandbox.detach().expect_err("torn down");
2870 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2871 assert!(!sandbox.detached());
2872 }
2873
2874 async fn suspended_with_window(
2885 suspended_sec: u32,
2886 ) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2887 let (mut sandbox, recorder, clock) = planted();
2888 recorder
2889 .answer(
2890 "RunMicrovm",
2891 Answer::ok(fake::microvm_response("PENDING", None)),
2892 )
2893 .answer(
2894 "GetMicrovm",
2895 Answer::ok(fake::microvm_response("RUNNING", None)),
2896 )
2897 .answer(
2898 "GetMicrovm",
2899 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2900 )
2901 .answer(
2902 "CreateMicrovmAuthToken",
2903 Answer::ok(fake::auth_token_response("proxy-token")),
2904 )
2905 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2906
2907 sandbox
2908 .run(
2909 RunRequest::new()
2910 .with_image("arn:image")
2911 .with_suspended_sec(suspended_sec),
2912 )
2913 .await
2914 .expect("launches");
2915 sandbox.suspend().await.expect("suspends");
2916 assert_eq!(
2917 clock.now(),
2918 Duration::ZERO,
2919 "no wait may have slept, or a test that reads the clock is measuring the fake's \
2920 poll interval as well as the elapsed suspension"
2921 );
2922 (sandbox, recorder, clock)
2923 }
2924
2925 async fn suspended() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2927 suspended_with_window(600).await
2928 }
2929
2930 const ADOPT_TOKEN: &str = "adopt-canary-token-9f1c";
2934 const ADOPT_ENDPOINT: &str = "https://mvm-abc123.microvm.us-east-1.amazonaws.com";
2935
2936 fn adopt_plane() -> (ControlPlane, Arc<FakeControlPlane>, Arc<TestClock>) {
2938 let recorder = Arc::new(FakeControlPlane::new());
2939 let clock = Arc::new(TestClock::new());
2940 let plane = crate::testing::control_plane(
2941 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2942 Region::US_EAST_1,
2943 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2944 );
2945 (plane, recorder, clock)
2946 }
2947
2948 async fn adopted_in(state: &str) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2949 let (plane, recorder, clock) = adopt_plane();
2950 recorder.answer(
2951 "GetMicrovm",
2952 Answer::ok(fake::microvm_response(state, None)),
2953 );
2954 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2955 .await
2956 .expect("adopts");
2957 (sandbox, recorder, clock)
2958 }
2959
2960 #[tokio::test]
2967 async fn an_adopted_sandbox_reports_the_services_idle_window_and_watches_its_lifecycle() {
2968 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2969 assert_eq!(sandbox.idle_window(), Some(Duration::from_secs(1800)));
2970 assert_eq!(*sandbox.watch_lifecycle().borrow(), Lifecycle::Running);
2971 }
2972
2973 #[test]
2975 fn every_service_state_maps_onto_one_lifecycle() {
2976 for state in crate::constants::MICROVM_STATES {
2977 let lifecycle = Lifecycle::from_service(state).expect("a known state");
2978 assert_eq!(lifecycle.as_str(), state);
2979 }
2980 assert_eq!(Lifecycle::from_service("Running"), None);
2981 }
2982
2983 #[tokio::test]
2991 async fn an_adopted_vm_takes_its_lifecycle_from_the_service_and_refuses_run() {
2992 let (mut sandbox, recorder, _) = adopted_in("RUNNING").await;
2993 assert!(sandbox.adopted());
2994 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2995 assert!(sandbox.token_installed());
2996 assert_eq!(sandbox.bootstrap_count(), 1);
2997 assert!(sandbox.image_exists());
2998 let session = sandbox.session().expect("a live VM gets a session");
2999 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
3000 assert_eq!(session.agent_token(), ADOPT_TOKEN);
3001
3002 let error = sandbox
3003 .run(RunRequest::new().with_image("arn:image"))
3004 .await
3005 .expect_err("an adopted VM was bootstrapped by its own launch");
3006 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3007 assert!(error.to_string().contains("STATE-3"), "{error}");
3008 assert_eq!(recorder.call_count("RunMicrovm"), 0);
3009 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
3010 }
3011
3012 #[tokio::test]
3018 async fn a_name_adopts_its_records_vm_and_foreign_names_are_refused_locally() {
3019 struct One(crate::names::NameRecord);
3020 impl crate::names::NameStore for One {
3021 fn get(&self, name: &str) -> Result<Option<crate::names::NameRecord>, Error> {
3022 Ok((name == self.0.name).then(|| self.0.clone()))
3023 }
3024 fn put(&self, _: &crate::names::NameRecord) -> Result<(), Error> {
3025 unreachable!()
3026 }
3027 fn delete(&self, _: &str) -> Result<bool, Error> {
3028 unreachable!()
3029 }
3030 fn list(&self) -> Result<Vec<crate::names::NameRecord>, Error> {
3031 Ok(vec![self.0.clone()])
3032 }
3033 }
3034 let record = crate::names::NameRecord::new_at(
3035 "ci",
3036 "mvm-abc123",
3037 ADOPT_ENDPOINT,
3038 ADOPT_TOKEN,
3039 "us-west-2",
3040 1_767_225_600,
3041 )
3042 .expect("record");
3043 let store = One(record.clone());
3044 let foreign = crate::names::resolve(&store, "ci", Some(&Region::US_EAST_1))
3045 .expect_err("registered in another region");
3046 assert_eq!(foreign.kind(), ErrorKind::InvalidArg);
3047 let missing = crate::names::resolve(&store, "nope", None).expect_err("no such name");
3048 assert_eq!(missing.kind(), ErrorKind::Precondition);
3049 assert!(!format!("{foreign} {missing}").contains(ADOPT_TOKEN));
3050
3051 let (plane, recorder, _) = adopt_plane();
3052 recorder.answer(
3053 "GetMicrovm",
3054 Answer::ok(fake::microvm_response("RUNNING", None)),
3055 );
3056 let resolved = crate::names::resolve(&store, "ci", Some(&Region::US_WEST_2)).expect("ok");
3057 let sandbox = Sandbox::adopt_record(plane, resolved)
3058 .await
3059 .expect("adopts");
3060 assert!(sandbox.adopted());
3061 assert_eq!(
3062 sandbox.microvm().map(|vm| vm.id.as_str()),
3063 Some("mvm-abc123")
3064 );
3065 assert_eq!(
3066 sandbox.session().expect("session").agent_token(),
3067 ADOPT_TOKEN
3068 );
3069 let named = sandbox
3070 .name_record("again")
3071 .expect("an adopted VM can be named");
3072 assert_eq!(
3073 (
3074 named.microvm_id.as_str(),
3075 named.agent_token.as_str(),
3076 named.region.as_str()
3077 ),
3078 ("mvm-abc123", ADOPT_TOKEN, "us-east-1")
3079 );
3080 let unlaunched = Sandbox::with_control_plane(adopt_plane().0);
3081 assert_eq!(
3082 unlaunched
3083 .name_record("x")
3084 .expect_err("nothing to name")
3085 .kind(),
3086 ErrorKind::Precondition
3087 );
3088 }
3089
3090 #[tokio::test]
3096 async fn a_records_tunnel_identity_survives_adoption_and_naming() {
3097 let identity = crate::identity::LaunchIdentity::from_seeds([7; 32], [9; 32])
3098 .expect("valid seeds")
3099 .keep();
3100 let mut record = crate::names::NameRecord::new_at(
3101 "ci",
3102 "mvm-abc123",
3103 ADOPT_ENDPOINT,
3104 ADOPT_TOKEN,
3105 "us-east-1",
3106 1_767_225_600,
3107 )
3108 .expect("record");
3109 record.identity_host_seed = Some(identity.host_seed_base64());
3110 record.identity_vm_public_key = Some(identity.vm_public_key_base64());
3111
3112 let (plane, recorder, _) = adopt_plane();
3113 recorder.answer(
3114 "GetMicrovm",
3115 Answer::ok(fake::microvm_response("RUNNING", None)),
3116 );
3117 let sandbox = Sandbox::adopt_record(plane, record.clone())
3118 .await
3119 .expect("adopts");
3120 let kept = sandbox
3121 .tunnel_identity()
3122 .expect("the record's pair is kept");
3123 assert_eq!(kept.host_seed_base64(), identity.host_seed_base64());
3124 assert_eq!(kept.vm_public_key_base64(), identity.vm_public_key_base64());
3125 let named = sandbox.name_record("again").expect("names");
3126 assert_eq!(named.identity_host_seed, record.identity_host_seed);
3127 assert_eq!(named.identity_vm_public_key, record.identity_vm_public_key);
3128
3129 let (plane, recorder, _) = adopt_plane();
3130 let mut torn = record;
3131 torn.identity_vm_public_key = None;
3132 let refused = Sandbox::adopt_record(plane, torn)
3133 .await
3134 .expect_err("half a pair is refused");
3135 assert_eq!(refused.kind(), ErrorKind::InvalidArg);
3136 assert!(refused.to_string().contains("one half"), "{refused}");
3137 assert!(recorder.calls().is_empty(), "refused before any AWS call");
3138 }
3139
3140 #[tokio::test]
3143 async fn an_adopted_vm_suspends_resumes_and_terminates_through_its_handle() {
3144 let (plane, recorder, _) = adopt_plane();
3145 recorder
3146 .answer(
3147 "GetMicrovm",
3148 Answer::ok(fake::microvm_response("RUNNING", None)),
3149 )
3150 .answer(
3151 "GetMicrovm",
3152 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3153 )
3154 .answer(
3155 "GetMicrovm",
3156 Answer::ok(fake::microvm_response("RUNNING", None)),
3157 )
3158 .answer(
3159 "GetMicrovm",
3160 Answer::ok(fake::microvm_response("TERMINATED", None)),
3161 )
3162 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3163 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3164 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
3165 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3166 .await
3167 .expect("adopts");
3168
3169 sandbox.suspend().await.expect("suspends from RUNNING");
3170 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3171 sandbox.resume().await.expect("resumes from SUSPENDED");
3172 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3173 let report = sandbox
3174 .terminate(TeardownOpts::default().waiting_for_terminated())
3175 .await;
3176 assert!(report.terminate_accepted, "{report:?}");
3177 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3178 assert!(sandbox.was_terminated());
3179
3180 assert_eq!(recorder.call_count("RunMicrovm"), 0);
3181 assert!(
3182 recorder
3183 .bodies_as_text()
3184 .iter()
3185 .all(|body| !body.contains("runHookPayload") && !body.contains(ADOPT_TOKEN)),
3186 "no launch payload or agent token may reach the control plane"
3187 );
3188 assert_eq!(sandbox.bootstrap_count(), 1, "STATE-3: never above one");
3189 }
3190
3191 #[tokio::test]
3195 async fn an_adopted_vm_resumes_past_the_reported_window_through_the_service() {
3196 let (plane, recorder, clock) = adopt_plane();
3197 recorder
3198 .answer(
3199 "GetMicrovm",
3200 Answer::ok(fake::microvm_response("RUNNING", None)),
3201 )
3202 .answer(
3203 "GetMicrovm",
3204 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3205 )
3206 .answer(
3207 "GetMicrovm",
3208 Answer::ok(fake::microvm_response("RUNNING", None)),
3209 )
3210 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3211 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3212 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3213 .await
3214 .expect("adopts");
3215 assert_eq!(sandbox.suspended_window(), None, "no request of its own");
3216 sandbox.suspend().await.expect("suspends");
3217
3218 clock.advance(Duration::from_secs(601));
3219 sandbox
3220 .resume()
3221 .await
3222 .expect("past the 600 s window the service reported, the service resumes it");
3223 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3224 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3225 }
3226
3227 #[tokio::test]
3230 async fn an_adopted_suspended_vm_resumes_through_the_service() {
3231 let (plane, recorder, _) = adopt_plane();
3232 recorder
3233 .answer(
3234 "GetMicrovm",
3235 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3236 )
3237 .answer(
3238 "GetMicrovm",
3239 Answer::ok(fake::microvm_response("RUNNING", None)),
3240 )
3241 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3242 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3243 .await
3244 .expect("adopts");
3245 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3246 let session = sandbox
3247 .resume()
3248 .await
3249 .expect("the service answers the resume");
3250 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
3251 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3252 }
3253
3254 #[tokio::test]
3257 async fn an_adopted_terminated_vm_refuses_every_transition_locally() {
3258 let (mut sandbox, recorder, _) = adopted_in("TERMINATED").await;
3259 assert!(sandbox.was_terminated());
3260 assert!(sandbox.session().is_none());
3261 let resume = sandbox.resume().await.expect_err("STATE-11");
3262 assert!(resume.to_string().contains("STATE-11"), "{resume}");
3263 let suspend = sandbox.suspend().await.expect_err("STATE-5");
3264 assert!(suspend.to_string().contains("STATE-5"), "{suspend}");
3265 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
3266 }
3267
3268 async fn adopted_with_a_daemon(
3271 state: &str,
3272 ) -> (
3273 Sandbox,
3274 Arc<FakeControlPlane>,
3275 Arc<crate::testing::HealthyDaemon>,
3276 ) {
3277 let recorder = Arc::new(FakeControlPlane::new());
3278 let daemon = crate::testing::HealthyDaemon::new();
3279 let plane = crate::control::ControlPlane::from_ports(
3280 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3281 Region::US_EAST_1,
3282 Arc::new(TestClock::new()),
3283 Arc::new(crate::testing::SequenceEntropy::new()),
3284 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3285 );
3286 recorder
3287 .answer(
3288 "GetMicrovm",
3289 Answer::ok(fake::microvm_response(state, None)),
3290 )
3291 .answer(
3292 "GetMicrovm",
3293 Answer::ok(fake::microvm_response("RUNNING", None)),
3294 )
3295 .answer(
3296 "CreateMicrovmAuthToken",
3297 Answer::ok(fake::auth_token_response("proxy-token")),
3298 );
3299 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3300 .await
3301 .expect("adopts");
3302 (sandbox, recorder, daemon)
3303 }
3304
3305 #[tokio::test(start_paused = true)]
3312 async fn wait_until_ready_finishes_whichever_wait_is_left() {
3313 let (mut pending, recorder, daemon) = adopted_with_a_daemon("PENDING").await;
3314 pending
3315 .wait_until_ready(Duration::from_secs(60))
3316 .await
3317 .expect("RUNNING, then the daemon");
3318 assert_eq!(pending.lifecycle(), Lifecycle::Running);
3319 assert_eq!(pending.bootstrap_count(), 1);
3320 assert_eq!(daemon.polls(), 1);
3321 assert_eq!(
3322 recorder.call_count("GetMicrovm"),
3323 2,
3324 "the adopt's read and the wait's"
3325 );
3326
3327 let (mut running, recorder, daemon) = adopted_with_a_daemon("RUNNING").await;
3328 running
3329 .wait_until_ready(Duration::from_secs(60))
3330 .await
3331 .expect("the daemon answers");
3332 assert_eq!(daemon.polls(), 1);
3333 assert_eq!(
3334 recorder.call_count("GetMicrovm"),
3335 1,
3336 "RUNNING needs no second read"
3337 );
3338 }
3339
3340 #[tokio::test]
3342 async fn wait_until_ready_refuses_a_suspended_vm() {
3343 let (mut suspended, _, daemon) = adopted_with_a_daemon("SUSPENDED").await;
3344 let error = suspended
3345 .wait_until_ready(Duration::from_secs(60))
3346 .await
3347 .expect_err("a suspended VM isn't starting");
3348 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
3349 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3350 assert_eq!(daemon.polls(), 0);
3351 }
3352
3353 #[tokio::test(start_paused = true)]
3356 async fn an_adopted_pending_vm_counts_its_bootstrap_once_on_running() {
3357 let recorder = Arc::new(FakeControlPlane::new());
3358 let daemon = crate::testing::HealthyDaemon::new();
3359 let plane = crate::control::ControlPlane::from_ports(
3360 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3361 Region::US_EAST_1,
3362 Arc::new(TestClock::new()),
3363 Arc::new(crate::testing::SequenceEntropy::new()),
3364 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3365 );
3366 recorder
3367 .answer(
3368 "GetMicrovm",
3369 Answer::ok(fake::microvm_response("PENDING", None)),
3370 )
3371 .answer(
3372 "GetMicrovm",
3373 Answer::ok(fake::microvm_response("RUNNING", None)),
3374 )
3375 .answer(
3376 "CreateMicrovmAuthToken",
3377 Answer::ok(fake::auth_token_response("proxy-token")),
3378 );
3379 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3380 .await
3381 .expect("adopts");
3382 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3383 assert_eq!(sandbox.bootstrap_count(), 0);
3384 sandbox
3385 .wait_until_running(Duration::from_secs(60))
3386 .await
3387 .expect("reaches RUNNING");
3388 assert_eq!(sandbox.bootstrap_count(), 1);
3389 assert!(sandbox.token_installed());
3390 assert_eq!(
3391 daemon.polls(),
3392 1,
3393 "the daemon answered before the session came back"
3394 );
3395 assert!(
3396 sandbox
3397 .run(RunRequest::new().with_image("arn:image"))
3398 .await
3399 .is_err(),
3400 "still one bootstrap per VM"
3401 );
3402 assert_eq!(recorder.call_count("RunMicrovm"), 0);
3403 }
3404
3405 #[tokio::test]
3412 async fn adopt_refuses_bad_records_and_never_prints_the_token() {
3413 let (plane, recorder, _) = adopt_plane();
3414 let empty = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, "")
3415 .await
3416 .expect_err("no token");
3417 assert_eq!(empty.kind(), ErrorKind::InvalidArg);
3418 assert_eq!(recorder.calls().len(), 0, "refused before any call");
3419
3420 let (plane, recorder, _) = adopt_plane();
3421 recorder.answer(
3422 "GetMicrovm",
3423 Answer::ok(fake::microvm_response("RUNNING", None)),
3424 );
3425 let mismatch = Sandbox::adopt(plane, "mvm-abc123", "https://other.example", ADOPT_TOKEN)
3426 .await
3427 .expect_err("the endpoint belongs to another record");
3428 assert_eq!(mismatch.kind(), ErrorKind::InvalidArg);
3429 assert!(!mismatch.to_string().contains(ADOPT_TOKEN));
3430
3431 let (plane, recorder, _) = adopt_plane();
3432 recorder.answer(
3433 "GetMicrovm",
3434 Answer::ok(fake::microvm_response("REBOOTING", None)),
3435 );
3436 let unknown = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3437 .await
3438 .expect_err("an unknown state");
3439 assert_eq!(unknown.kind(), ErrorKind::Platform);
3440 assert!(!unknown.to_string().contains(ADOPT_TOKEN));
3441
3442 let (sandbox, _, _) = adopted_in("RUNNING").await;
3443 let printed = format!("{sandbox:?}");
3444 assert!(!printed.contains(ADOPT_TOKEN), "{printed}");
3445 assert!(printed.contains("adopted: true"), "{printed}");
3446 }
3447
3448 #[tokio::test(start_paused = true)]
3458 async fn a_launch_records_the_image_and_the_running_report_installs_the_token() {
3459 let (mut sandbox, recorder, _) = planted();
3460 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3461 assert!(
3462 !sandbox.image_exists(),
3463 "nothing is recorded before a launch"
3464 );
3465 assert!(!sandbox.token_installed());
3466
3467 answer_launch(&recorder);
3468 let session = sandbox
3469 .run(RunRequest::new().with_image("arn:image"))
3470 .await
3471 .expect("launches");
3472 assert_eq!(
3473 session.endpoint(),
3474 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
3475 );
3476
3477 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3478 assert!(sandbox.image_exists(), "STATE-1: the image is recorded");
3479 assert!(sandbox.token_installed(), "STATE-2: the token is installed");
3480 assert_eq!(sandbox.bootstrap_count(), 1);
3481 assert_eq!(sandbox.microvm().expect("launched").id, "mvm-abc123");
3482
3483 let body = recorder.first_body("RunMicrovm");
3486 let payload = body["runHookPayload"].as_str().expect("a string");
3487 assert!(payload.starts_with(r#"{"agent_token":"#), "{payload}");
3488 assert_eq!(body["idlePolicy"]["suspendedDurationSeconds"], 600);
3489 }
3490
3491 #[tokio::test(start_paused = true)]
3498 async fn a_launch_env_reaches_the_run_hook_payload_on_the_wire() {
3499 let (mut sandbox, recorder, _) = planted();
3500 answer_launch(&recorder);
3501 sandbox
3502 .run(
3503 RunRequest::new()
3504 .with_image("arn:image")
3505 .with_launch_env("ANTHROPIC_BASE_URL", "https://gateway.example")
3506 .with_launch_env("PATH", "/usr/local/bin:/usr/bin:/bin"),
3507 )
3508 .await
3509 .expect("launches");
3510
3511 let body = recorder.first_body("RunMicrovm");
3512 let payload = body["runHookPayload"].as_str().expect("a string");
3513 let inner: serde_json::Value =
3515 serde_json::from_str(payload).expect("the payload is itself JSON");
3516 assert!(
3517 inner["agent_token"].as_str().is_some_and(|t| !t.is_empty()),
3518 "the token still rides alongside: {payload}"
3519 );
3520 assert_eq!(
3521 inner["env"]["ANTHROPIC_BASE_URL"],
3522 "https://gateway.example"
3523 );
3524 assert_eq!(inner["env"]["PATH"], "/usr/local/bin:/usr/bin:/bin");
3525 }
3526
3527 #[tokio::test(start_paused = true)]
3540 async fn the_advisory_deny_reaches_the_launch_environment_in_both_spellings() {
3541 let (mut sandbox, recorder, _) = planted();
3542 answer_launch(&recorder);
3543 let request = RunRequest::new().with_image("arn:image").with_deny_egress();
3544 assert_eq!(
3545 request.egress_posture(),
3546 crate::control::EgressPosture::BestEffort
3547 );
3548 sandbox.run(request).await.expect("launches");
3549
3550 let body = recorder.first_body("RunMicrovm");
3551 let inner: serde_json::Value =
3552 serde_json::from_str(body["runHookPayload"].as_str().expect("a string"))
3553 .expect("the payload is itself JSON");
3554 for key in crate::sandbox::DENY_EGRESS_ENV_KEYS {
3555 assert_eq!(
3556 inner["env"][key], DENY_EGRESS_PROXY_URL,
3557 "{key} must point at the black hole: {inner}"
3558 );
3559 }
3560 assert!(
3561 body.get("egressNetworkConnectors").is_none(),
3562 "the advisory deny asks the platform for nothing: {body}"
3563 );
3564 }
3565
3566 #[test]
3569 fn the_deny_never_overwrites_a_callers_own_proxy_and_is_absent_by_default() {
3570 let kept = RunRequest::new()
3571 .with_deny_egress()
3572 .with_launch_env("https_proxy", "http://proxy.internal:3128")
3573 .effective_launch_env();
3574 assert_eq!(kept["https_proxy"], "http://proxy.internal:3128");
3575 assert_eq!(kept["http_proxy"], DENY_EGRESS_PROXY_URL);
3576
3577 let plain = RunRequest::new().effective_launch_env();
3578 assert!(
3579 plain.is_empty(),
3580 "a launch that did not ask for the deny sends byte-for-byte what it always sent: \
3581 {plain:?}"
3582 );
3583 assert_eq!(
3584 RunRequest::new().egress_posture(),
3585 crate::control::EgressPosture::Unsealed,
3586 "and its posture is the measured one, not a seal"
3587 );
3588 }
3589
3590 #[tokio::test]
3599 async fn asking_for_egress_and_the_deny_together_is_refused_before_any_call() {
3600 let (mut sandbox, recorder, _) = planted();
3601 answer_launch(&recorder);
3602 let mut request = RunRequest::new().with_image("arn:image").with_egress();
3603 request.deny_egress = true;
3604
3605 let error = sandbox
3606 .run(request)
3607 .await
3608 .expect_err("opposite intents are not resolved by a precedence rule");
3609 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3610 assert!(error.to_string().contains("opposite things"), "{error}");
3611 assert_eq!(
3612 recorder.calls().len(),
3613 0,
3614 "a locally-refused launch costs nothing"
3615 );
3616 }
3617
3618 #[tokio::test(start_paused = true)]
3629 async fn a_pinned_image_version_is_forwarded_to_the_launch_request() {
3630 let (mut sandbox, recorder, _) = planted();
3631 answer_launch(&recorder);
3632 sandbox
3633 .run(
3634 RunRequest::new()
3635 .with_image("arn:image")
3636 .with_image_version("2.0"),
3637 )
3638 .await
3639 .expect("launches");
3640 assert_eq!(
3641 recorder.first_body("RunMicrovm")["imageVersion"],
3642 "2.0",
3643 "a canary has to launch against the version it means to test"
3644 );
3645
3646 let (mut sandbox, recorder, _) = planted();
3647 answer_launch(&recorder);
3648 sandbox
3649 .run(RunRequest::new().with_image("arn:image"))
3650 .await
3651 .expect("launches");
3652 assert!(
3653 recorder
3654 .first_body("RunMicrovm")
3655 .get("imageVersion")
3656 .is_none(),
3657 "an unpinned launch sends what this client always sent"
3658 );
3659 }
3660
3661 #[tokio::test]
3672 async fn an_over_budget_launch_env_is_refused_before_any_control_plane_call() {
3673 let (mut sandbox, recorder, _) = planted();
3674 answer_launch(&recorder);
3675
3676 let error = sandbox
3677 .run(
3678 RunRequest::new()
3679 .with_image("arn:image")
3680 .with_launch_env("AWS_SESSION_TOKEN", "t".repeat(4096)),
3681 )
3682 .await
3683 .expect_err("a credential-scale launch env does not fit the payload");
3684
3685 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3686 let message = error.to_string();
3687 assert!(message.contains("ceiling of 4096"), "{message}");
3688 assert!(message.contains("launch env contributed"), "{message}");
3689 assert_eq!(
3690 recorder.calls().len(),
3691 0,
3692 "the refusal has to be local: a launch that reached AWS costs a VM and a bill"
3693 );
3694 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3697 assert_eq!(sandbox.bootstrap_count(), 0);
3698 assert!(sandbox.microvm().is_none());
3699 }
3700
3701 #[tokio::test]
3719 async fn the_bindings_layer_refuses_an_illegal_launch_with_zero_calls() {
3720 let cases: [(&str, RunRequest, &str); 3] = [
3721 (
3722 "max_idle_sec=59, which is what a Python keyword argument passes",
3723 RunRequest {
3724 max_idle_sec: 59,
3725 ..RunRequest::new().with_image("arn:image")
3726 },
3727 "maxIdleDurationSeconds",
3728 ),
3729 (
3730 "an execution role that is a bare name",
3731 RunRequest {
3732 execution_role_arn: Some("execution-role".to_string()),
3733 ..RunRequest::new().with_image("arn:image")
3734 },
3735 "role *name*",
3736 ),
3737 (
3738 "a pinned version with a trailing newline",
3739 RunRequest::new()
3740 .with_image("arn:image")
3741 .with_image_version("2.0\n"),
3742 "contains whitespace",
3743 ),
3744 ];
3745
3746 for (label, request, expected) in cases {
3747 let (mut sandbox, recorder, _) = planted();
3748 answer_launch(&recorder);
3751
3752 let Err(error) = sandbox.run(request).await else {
3756 panic!("{label} must be refused");
3757 };
3758 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
3759 assert!(
3760 error.to_string().contains(expected),
3761 "{label}: wanted {expected:?}, got {error}"
3762 );
3763 assert_eq!(
3764 recorder.calls().len(),
3765 0,
3766 "{label}: the refusal has to be local, because a launch that reached AWS costs a \
3767 VM and a bill"
3768 );
3769 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3770 assert!(sandbox.microvm().is_none());
3771 }
3772 }
3773
3774 #[tokio::test]
3781 async fn a_launch_that_dies_during_startup_installs_no_token() {
3782 let (mut sandbox, recorder, _) = planted();
3783 recorder
3784 .answer(
3785 "RunMicrovm",
3786 Answer::ok(fake::microvm_response("PENDING", None)),
3787 )
3788 .answer(
3789 "GetMicrovm",
3790 Answer::ok(fake::microvm_response(
3791 "TERMINATED",
3792 Some("run hook returned 500"),
3793 )),
3794 );
3795
3796 let error = sandbox
3797 .run(RunRequest::new().with_image("arn:image"))
3798 .await
3799 .expect_err("a VM that died during startup is not a launch");
3800 assert_eq!(error.kind(), ErrorKind::LaunchDied);
3801 assert!(
3802 error.to_string().contains("run hook returned 500"),
3803 "{error}"
3804 );
3805
3806 assert!(
3807 !sandbox.token_installed(),
3808 "the run hook never answered, so nothing was installed"
3809 );
3810 assert_eq!(
3811 sandbox.bootstrap_count(),
3812 0,
3813 "a bootstrap counted here would make STATE-3's ceiling unreachable for a retry"
3814 );
3815 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3816 assert!(
3817 sandbox.image_exists(),
3818 "STATE-1 still holds: the launch was accepted against a real image"
3819 );
3820 }
3821
3822 #[tokio::test(start_paused = true)]
3833 async fn a_second_run_on_one_sandbox_is_refused_before_any_call() {
3834 let (mut sandbox, recorder, _) = launched().await;
3835 assert_eq!(sandbox.bootstrap_count(), 1);
3836 let before = recorder.calls().len();
3837
3838 let error = sandbox
3839 .run(RunRequest::new().with_image("arn:image"))
3840 .await
3841 .expect_err("the token is installed at most once per VM lifetime");
3842 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3843 assert!(error.to_string().contains("STATE-3"), "{error}");
3844 assert!(
3845 error.to_string().contains("needs a second Sandbox"),
3846 "the remedy has to be nameable: {error}"
3847 );
3848
3849 assert_eq!(
3850 sandbox.bootstrap_count(),
3851 1,
3852 "bootstrap_count <= 1 is the Z3-proved invariant"
3853 );
3854 assert_eq!(
3855 recorder.calls().len(),
3856 before,
3857 "the refusal must cost no control-plane call"
3858 );
3859 assert_eq!(recorder.call_count("RunMicrovm"), 1);
3860 }
3861
3862 #[tokio::test(start_paused = true)]
3865 async fn a_suspend_from_running_passes_through_suspending_to_suspended() {
3866 let (sandbox, recorder, _) = suspended().await;
3867 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3868 assert!(
3869 sandbox.token_installed(),
3870 "a freeze keeps the guest's memory, so the token survives"
3871 );
3872 assert_eq!(sandbox.bootstrap_count(), 1, "no re-bootstrap on a freeze");
3873
3874 let calls = recorder.calls();
3875 let suspend = calls
3876 .iter()
3877 .find(|call| call.operation == "SuspendMicrovm")
3878 .expect("the suspend went out");
3879 assert_eq!(suspend.method, Method::Post);
3880 assert_eq!(suspend.path, "/2025-09-09/microvms/mvm-abc123/suspend");
3881 }
3882
3883 #[tokio::test(start_paused = true)]
3889 async fn a_vm_that_dies_while_suspending_is_recorded_rather_than_raised() {
3890 let (mut sandbox, recorder, _) = launched().await;
3891 recorder
3892 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3893 .answer(
3894 "GetMicrovm",
3895 Answer::ok(fake::microvm_response("TERMINATED", Some("idle policy"))),
3896 );
3897
3898 sandbox
3899 .suspend()
3900 .await
3901 .expect("a death mid-suspend is a state, not an error");
3902 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3903 assert!(
3904 sandbox.was_terminated(),
3905 "STATE-11's precondition is recorded"
3906 );
3907 }
3908
3909 #[tokio::test(start_paused = true)]
3918 async fn a_suspend_whose_call_fails_stays_running_and_can_be_retried() {
3919 let (mut sandbox, recorder, _) = planted();
3920 recorder
3921 .answer(
3922 "RunMicrovm",
3923 Answer::ok(fake::microvm_response("PENDING", None)),
3924 )
3925 .answer(
3926 "GetMicrovm",
3927 Answer::ok(fake::microvm_response("RUNNING", None)),
3928 )
3929 .answer(
3930 "GetMicrovm",
3931 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3932 )
3933 .answer(
3934 "CreateMicrovmAuthToken",
3935 Answer::ok(fake::auth_token_response("proxy-token")),
3936 )
3937 .answer("SuspendMicrovm", Answer::failure(409, "ConflictException"))
3940 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
3941
3942 sandbox
3943 .run(RunRequest::new().with_image("arn:image"))
3944 .await
3945 .expect("launches");
3946
3947 sandbox
3948 .suspend()
3949 .await
3950 .expect_err("the control plane refused the suspend");
3951 assert_eq!(
3952 sandbox.lifecycle(),
3953 Lifecycle::Running,
3954 "a refused suspend must not move the lifecycle: SUSPENDING accepts neither a \
3955 suspend nor a resume, so recording it here would brick the handle"
3956 );
3957
3958 sandbox.suspend().await.expect("the retry suspends");
3959 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3960 assert_eq!(recorder.call_count("SuspendMicrovm"), 2);
3961 }
3962
3963 #[tokio::test(start_paused = true)]
3975 async fn a_suspend_from_a_non_running_state_reaches_no_control_plane_call() {
3976 let (mut sandbox, recorder, _) = suspended().await;
3978 let before = recorder.call_count("SuspendMicrovm");
3979
3980 let error = sandbox
3981 .suspend()
3982 .await
3983 .expect_err("a suspend is only issued from RUNNING");
3984 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3985 assert!(error.to_string().contains("STATE-5"), "{error}");
3986 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3987 assert_eq!(
3988 recorder.call_count("SuspendMicrovm"),
3989 before,
3990 "the refusal must be local: no second suspend went to the wire"
3991 );
3992
3993 let (mut fresh, fresh_recorder, _) = planted();
3995 let error = fresh.suspend().await.expect_err("nothing to suspend");
3996 assert_eq!(error.kind(), ErrorKind::Precondition);
3997 assert_eq!(fresh_recorder.calls().len(), 0);
3998 }
3999
4000 #[tokio::test(start_paused = true)]
4011 async fn a_resume_reuses_the_token_redelivers_nothing_and_drops_the_proxy_token() {
4012 let (mut sandbox, recorder, _) = suspended().await;
4013
4014 let auth = Arc::clone(
4016 sandbox
4017 .session()
4018 .expect("launched")
4019 .proxy_auth()
4020 .expect("the launch wired a minter"),
4021 );
4022 auth.headers().await.expect("mints");
4023 assert!(
4024 auth.is_cached(),
4025 "there must be a token for the drop to matter"
4026 );
4027 assert_eq!(auth.mint_count(), 1);
4028
4029 recorder
4030 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4031 .answer(
4032 "GetMicrovm",
4033 Answer::ok(fake::microvm_response("RUNNING", None)),
4034 );
4035 let session = sandbox.resume().await.expect("resumes");
4036 assert_eq!(
4037 session.endpoint(),
4038 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
4039 );
4040
4041 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4042 assert!(
4043 sandbox.token_installed(),
4044 "STATE-7: the installed token is reused"
4045 );
4046 assert_eq!(
4047 sandbox.bootstrap_count(),
4048 1,
4049 "STATE-7: a resume re-delivers no run-hook payload"
4050 );
4051 assert_eq!(
4052 recorder.call_count("RunMicrovm"),
4053 1,
4054 "a resume is not a second launch"
4055 );
4056 assert!(
4057 !auth.is_cached(),
4058 "STATE-8: the cached proxy token survived the resume"
4059 );
4060 }
4061
4062 #[tokio::test(start_paused = true)]
4072 async fn the_request_after_a_resume_mints_a_fresh_proxy_token() {
4073 let (mut sandbox, recorder, _) = suspended().await;
4074 let auth = Arc::clone(
4075 sandbox
4076 .session()
4077 .expect("launched")
4078 .proxy_auth()
4079 .expect("wired"),
4080 );
4081 auth.headers().await.expect("mints");
4082 assert_eq!(auth.mint_count(), 1);
4083
4084 recorder
4085 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4086 .answer(
4087 "GetMicrovm",
4088 Answer::ok(fake::microvm_response("RUNNING", None)),
4089 );
4090 sandbox.resume().await.expect("resumes");
4091
4092 auth.headers().await.expect("re-mints");
4095 assert_eq!(
4096 auth.mint_count(),
4097 2,
4098 "STATE-8: the resume did not force a fresh mint"
4099 );
4100 }
4101
4102 #[tokio::test(start_paused = true)]
4106 async fn the_lifecycle_watch_follows_every_transition() {
4107 let (mut sandbox, recorder, _clock) = planted();
4108 let watch = sandbox.watch_lifecycle();
4109 assert_eq!(*watch.borrow(), Lifecycle::Pending);
4110 recorder
4111 .answer(
4112 "RunMicrovm",
4113 Answer::ok(fake::microvm_response("PENDING", None)),
4114 )
4115 .answer(
4116 "GetMicrovm",
4117 Answer::ok(fake::microvm_response("RUNNING", None)),
4118 )
4119 .answer(
4120 "GetMicrovm",
4121 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4122 )
4123 .answer(
4124 "CreateMicrovmAuthToken",
4125 Answer::ok(fake::auth_token_response("proxy-token")),
4126 )
4127 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
4128 sandbox
4129 .run(RunRequest::new().with_image("arn:image"))
4130 .await
4131 .expect("launches");
4132 assert_eq!(*watch.borrow(), Lifecycle::Running);
4133 sandbox.suspend().await.expect("suspends");
4134 assert_eq!(*watch.borrow(), Lifecycle::Suspended);
4135 assert_eq!(*watch.borrow(), sandbox.lifecycle());
4136 }
4137
4138 fn terminated_refusal() -> Answer {
4141 Answer {
4142 status: 400,
4143 body: r#"{"message": "The MicroVM mvm-abc123 has been terminated and its state cannot be changed."}"#
4144 .to_string(),
4145 error_type: Some("ValidationException".to_string()),
4146 }
4147 }
4148
4149 #[tokio::test(start_paused = true)]
4153 async fn resume_observes_platform_idle_suspension_and_resumes() {
4154 let (mut sandbox, recorder, _) = planted();
4155 recorder
4156 .answer(
4157 "RunMicrovm",
4158 Answer::ok(fake::microvm_response("PENDING", None)),
4159 )
4160 .answer(
4161 "GetMicrovm",
4162 Answer::ok(fake::microvm_response("RUNNING", None)),
4163 )
4164 .answer(
4165 "GetMicrovm",
4166 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4167 )
4168 .answer(
4169 "GetMicrovm",
4170 Answer::ok(fake::microvm_response("RUNNING", None)),
4171 )
4172 .answer(
4173 "CreateMicrovmAuthToken",
4174 Answer::ok(fake::auth_token_response("proxy-token")),
4175 )
4176 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4177 sandbox
4178 .run(RunRequest::new().with_image("arn:image"))
4179 .await
4180 .expect("launch");
4181 let token = sandbox
4182 .session()
4183 .expect("session")
4184 .agent_token()
4185 .to_string();
4186 let watch = sandbox.watch_lifecycle();
4187 let calls = recorder.calls().len();
4188 sandbox.resume().await.expect("idle-suspended VM resumes");
4189 assert_eq!(
4190 &recorder.operations()[calls..],
4191 &["GetMicrovm", "ResumeMicrovm", "GetMicrovm"]
4192 );
4193 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4194 assert_eq!(*watch.borrow(), Lifecycle::Running);
4195 assert_eq!(sandbox.microvm().expect("VM").state, "RUNNING");
4196 assert_eq!(sandbox.bootstrap_count(), 1);
4197 assert_eq!(recorder.call_count("RunMicrovm"), 1);
4198 assert_eq!(recorder.call_count("SuspendMicrovm"), 0);
4199 assert_eq!(sandbox.session().expect("session").agent_token(), token);
4200 assert!(
4201 !sandbox
4202 .session()
4203 .expect("session")
4204 .proxy_auth()
4205 .expect("auth")
4206 .is_cached()
4207 );
4208 }
4209
4210 #[tokio::test(start_paused = true)]
4214 async fn resume_observes_platform_termination_and_records_window_closed() {
4215 for state in ["TERMINATING", "TERMINATED"] {
4216 let (mut sandbox, recorder, _) = planted();
4217 recorder
4218 .answer(
4219 "RunMicrovm",
4220 Answer::ok(fake::microvm_response("PENDING", None)),
4221 )
4222 .answer(
4223 "GetMicrovm",
4224 Answer::ok(fake::microvm_response("RUNNING", None)),
4225 )
4226 .answer(
4227 "GetMicrovm",
4228 Answer::ok(fake::microvm_response(state, Some("idle window elapsed"))),
4229 )
4230 .answer(
4231 "CreateMicrovmAuthToken",
4232 Answer::ok(fake::auth_token_response("proxy-token")),
4233 );
4234 sandbox
4235 .run(RunRequest::new().with_image("arn:image"))
4236 .await
4237 .expect("launch");
4238 let watch = sandbox.watch_lifecycle();
4239 let error = sandbox.resume().await.expect_err("the VM ended");
4240 assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}");
4241 assert!(error.to_string().contains(state), "{error}");
4242 assert!(error.to_string().contains("idle window elapsed"), "{error}");
4243 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4244 assert_eq!(*watch.borrow(), Lifecycle::Terminated);
4245 assert_eq!(sandbox.microvm().expect("VM").state, state);
4246 assert!(sandbox.was_terminated());
4247 assert_eq!(sandbox.drop_warning(), None);
4248 let calls = recorder.calls().len();
4249 assert_eq!(
4250 sandbox.resume().await.expect_err("STATE-11").kind(),
4251 ErrorKind::InvalidArg
4252 );
4253 assert_eq!(
4254 recorder.calls().len(),
4255 calls,
4256 "known termination stays local"
4257 );
4258 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4259 }
4260 }
4261
4262 #[tokio::test(start_paused = true)]
4265 async fn resume_observes_running_and_propagates_observation_failures() {
4266 for (response, expected) in [
4267 (
4268 Answer::ok(fake::microvm_response("RUNNING", None)),
4269 ErrorKind::InvalidArg,
4270 ),
4271 (
4272 Answer::ok(fake::microvm_response("UNKNOWN", None)),
4273 ErrorKind::Platform,
4274 ),
4275 (
4276 Answer::failure(403, "access denied"),
4277 ErrorKind::Credentials,
4278 ),
4279 ] {
4280 let (mut sandbox, recorder, _) = planted();
4281 recorder
4282 .answer(
4283 "RunMicrovm",
4284 Answer::ok(fake::microvm_response("PENDING", None)),
4285 )
4286 .answer(
4287 "GetMicrovm",
4288 Answer::ok(fake::microvm_response("RUNNING", None)),
4289 )
4290 .answer("GetMicrovm", response)
4291 .answer(
4292 "CreateMicrovmAuthToken",
4293 Answer::ok(fake::auth_token_response("proxy-token")),
4294 );
4295 sandbox
4296 .run(RunRequest::new().with_image("arn:image"))
4297 .await
4298 .expect("launch");
4299 let calls = recorder.calls().len();
4300 let error = sandbox.resume().await.expect_err("cannot resume");
4301 assert_eq!(error.kind(), expected, "{error}");
4302 assert_eq!(&recorder.operations()[calls..], &["GetMicrovm"]);
4303 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4304 assert!(!sandbox.was_terminated());
4305 }
4306 }
4307
4308 #[tokio::test(start_paused = true)]
4311 async fn resume_retries_observation_after_transient_service_states() {
4312 for state in ["SUSPENDING", "PENDING", "RESUMING"] {
4313 let (mut sandbox, recorder, _) = planted();
4314 recorder
4315 .answer(
4316 "RunMicrovm",
4317 Answer::ok(fake::microvm_response("PENDING", None)),
4318 )
4319 .answer(
4320 "GetMicrovm",
4321 Answer::ok(fake::microvm_response("RUNNING", None)),
4322 )
4323 .answer(
4324 "GetMicrovm",
4325 Answer::ok(fake::microvm_response(state, None)),
4326 )
4327 .answer(
4328 "GetMicrovm",
4329 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4330 )
4331 .answer(
4332 "GetMicrovm",
4333 Answer::ok(fake::microvm_response("RUNNING", None)),
4334 )
4335 .answer(
4336 "CreateMicrovmAuthToken",
4337 Answer::ok(fake::auth_token_response("proxy-token")),
4338 )
4339 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4340 sandbox
4341 .run(RunRequest::new().with_image("arn:image"))
4342 .await
4343 .expect("launch");
4344 let error = sandbox.resume().await.expect_err("still changing state");
4345 assert!(error.to_string().contains(state), "{error}");
4346 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4347 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4348 sandbox
4349 .resume()
4350 .await
4351 .expect("the next observation can resume");
4352 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4353 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4354 }
4355 }
4356
4357 #[tokio::test(start_paused = true)]
4363 async fn a_resume_past_the_window_of_an_explicit_suspend_reaches_the_service_and_resumes() {
4364 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
4365 assert_eq!(sandbox.suspended_window(), Some(Duration::from_secs(60)));
4366 assert_eq!(
4367 sandbox.idle_window(),
4368 Some(Duration::from_secs(600)),
4369 "the launch's own maxIdleDurationSeconds, which a keepalive checks its cadence against"
4370 );
4371
4372 clock.advance(Duration::from_secs(1283));
4373 recorder
4374 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4375 .answer(
4376 "GetMicrovm",
4377 Answer::ok(fake::microvm_response("RUNNING", None)),
4378 );
4379 sandbox
4380 .resume()
4381 .await
4382 .expect("the service resumes an explicit suspend past its window");
4383
4384 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4385 assert!(!sandbox.was_terminated());
4386 assert_eq!(
4387 recorder.call_count("ResumeMicrovm"),
4388 1,
4389 "the resume has to reach the service"
4390 );
4391 }
4392
4393 #[tokio::test(start_paused = true)]
4399 async fn a_resume_the_service_refuses_for_an_ended_vm_is_window_closed_and_recorded() {
4400 let (mut sandbox, recorder, _clock) = planted();
4401 recorder
4404 .answer(
4405 "RunMicrovm",
4406 Answer::ok(fake::microvm_response("PENDING", None)),
4407 )
4408 .answer(
4409 "GetMicrovm",
4410 Answer::ok(fake::microvm_response("RUNNING", None)),
4411 )
4412 .answer(
4413 "GetMicrovm",
4414 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4415 )
4416 .answer(
4417 "GetMicrovm",
4418 Answer::ok(fake::microvm_response(
4419 "TERMINATED",
4420 Some("MicroVM exceeded maximum lifetime."),
4421 )),
4422 )
4423 .answer(
4424 "CreateMicrovmAuthToken",
4425 Answer::ok(fake::auth_token_response("proxy-token")),
4426 )
4427 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
4428 .answer("ResumeMicrovm", terminated_refusal());
4429 sandbox
4430 .run(
4431 RunRequest::new()
4432 .with_image("arn:image")
4433 .with_suspended_sec(60),
4434 )
4435 .await
4436 .expect("launches");
4437 sandbox.suspend().await.expect("suspends");
4438
4439 let error = sandbox
4440 .resume()
4441 .await
4442 .expect_err("the service ended the VM");
4443 assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}");
4444 assert_eq!(error.code(), "ERR_WINDOW_CLOSED");
4445 let message = error.to_string();
4446 assert!(message.contains("TERMINATED"), "the state: {message}");
4447 assert!(
4448 message.contains("MicroVM exceeded maximum lifetime."),
4449 "the stateReason: {message}"
4450 );
4451 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4452 assert!(sandbox.was_terminated());
4453 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4454
4455 let again = sandbox.resume().await.expect_err("STATE-11");
4456 assert!(again.to_string().contains("STATE-11"), "{again}");
4457 assert_eq!(
4458 recorder.call_count("ResumeMicrovm"),
4459 1,
4460 "the second resume makes no call"
4461 );
4462 assert_eq!(
4463 sandbox.drop_warning(),
4464 None,
4465 "a VM the service ended bills nothing, so dropping its sandbox isn't a leak"
4466 );
4467 }
4468
4469 #[tokio::test(start_paused = true)]
4473 async fn a_resume_refused_for_a_live_vm_leaves_the_sandbox_suspended_and_resumable() {
4474 let (mut sandbox, recorder, _clock) = suspended().await;
4475 recorder
4476 .answer(
4477 "ResumeMicrovm",
4478 Answer::failure(409, "The MicroVM is changing state."),
4479 )
4480 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4481
4482 let error = sandbox.resume().await.expect_err("the service refused");
4483 assert_eq!(error.kind(), ErrorKind::Platform, "{error}");
4484 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
4485 assert!(!sandbox.was_terminated());
4486
4487 recorder.answer(
4488 "GetMicrovm",
4489 Answer::ok(fake::microvm_response("RUNNING", None)),
4490 );
4491 sandbox.resume().await.expect("the retry resumes");
4492 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4493 assert_eq!(recorder.call_count("ResumeMicrovm"), 2);
4494 }
4495
4496 #[tokio::test(start_paused = true)]
4500 async fn a_client_token_launch_resumes_the_vm_it_adopted() {
4501 let (mut sandbox, recorder, _) = planted();
4502 recorder
4503 .answer(
4504 "RunMicrovm",
4505 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4506 )
4507 .answer(
4508 "GetMicrovm",
4509 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4510 )
4511 .answer(
4512 "GetMicrovm",
4513 Answer::ok(fake::microvm_response("RUNNING", None)),
4514 )
4515 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4516 .answer(
4517 "CreateMicrovmAuthToken",
4518 Answer::ok(fake::auth_token_response("proxy-token")),
4519 );
4520 let mut request = RunRequest::new().with_image("arn:image");
4521 request.client_token = Some("job-195".into());
4522 request.agent_token = Some("agent-token".into());
4523 sandbox
4524 .run(request)
4525 .await
4526 .expect("the adopted VM is resumed");
4527 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4528 assert_eq!(sandbox.bootstrap_count(), 1);
4529 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4530
4531 let (mut fresh, recorder, _) = planted();
4532 recorder
4533 .answer(
4534 "RunMicrovm",
4535 Answer::ok(fake::microvm_response("PENDING", None)),
4536 )
4537 .answer(
4538 "GetMicrovm",
4539 Answer::ok(fake::microvm_response("SUSPENDED", Some("hook failed"))),
4540 );
4541 let error = fresh
4542 .run(RunRequest::new().with_image("arn:image"))
4543 .await
4544 .expect_err("a fresh launch that reads SUSPENDED died during startup");
4545 assert_eq!(error.kind(), ErrorKind::LaunchDied);
4546 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4547 }
4548
4549 #[tokio::test(start_paused = true)]
4552 async fn a_launch_that_does_not_wait_is_finished_by_wait_until_running() {
4553 let (mut sandbox, recorder, _) = planted();
4554 answer_launch(&recorder);
4555 let mut request = RunRequest::new().with_image("arn:image");
4556 request.wait = false;
4557 let endpoint = sandbox
4558 .run(request)
4559 .await
4560 .expect("accepted")
4561 .endpoint()
4562 .to_string();
4563 assert!(endpoint.starts_with("https://"), "{endpoint}");
4564 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
4565 assert_eq!(sandbox.bootstrap_count(), 0);
4566 assert_eq!(recorder.call_count("GetMicrovm"), 0, "nothing polled yet");
4567
4568 sandbox
4569 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4570 .await
4571 .expect("reaches RUNNING");
4572 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4573 assert_eq!(sandbox.bootstrap_count(), 1);
4574 let error = sandbox
4575 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4576 .await
4577 .expect_err("a running launch has nothing to wait for");
4578 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4579 assert_eq!(sandbox.bootstrap_count(), 1, "counted once (STATE-3)");
4580 }
4581
4582 #[tokio::test(start_paused = true)]
4590 async fn a_resume_after_terminate_records_zero_resume_calls() {
4591 let (mut sandbox, recorder, _) = launched().await;
4592 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4593 let report = sandbox.terminate(TeardownOpts::default()).await;
4594 assert!(report.terminate_accepted);
4595 assert!(sandbox.was_terminated());
4596
4597 let error = sandbox
4598 .resume()
4599 .await
4600 .expect_err("a terminated VM never returns to RUNNING");
4601 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4602 assert!(error.to_string().contains("STATE-11"), "{error}");
4603 assert_eq!(
4604 recorder.call_count("ResumeMicrovm"),
4605 0,
4606 "no resume may reach the wire once the VM was terminated"
4607 );
4608 assert_ne!(
4609 sandbox.lifecycle(),
4610 Lifecycle::Running,
4611 "the lifecycle must not return to RUNNING"
4612 );
4613 }
4614
4615 #[tokio::test(start_paused = true)]
4622 async fn a_terminate_records_terminating_and_the_completion_report_marks_terminated() {
4623 let (mut sandbox, recorder, _) = launched().await;
4624 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4625
4626 let report = sandbox.terminate(TeardownOpts::default()).await;
4627 assert!(report.terminate_accepted, "STATE-9: the call was accepted");
4628 assert!(sandbox.was_terminated(), "STATE-9: recorded terminated");
4629 assert_eq!(
4630 report.lifecycle,
4631 Some(Lifecycle::Terminating),
4632 "the default teardown does not wait, so TERMINATED is not claimed"
4633 );
4634 assert!(!report.leaked(), "{report:?}");
4635
4636 let (mut sandbox, recorder, _) = launched().await;
4638 recorder
4639 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4640 .answer(
4641 "GetMicrovm",
4642 Answer::ok(fake::microvm_response("TERMINATED", None)),
4643 );
4644 let report = sandbox
4645 .terminate(TeardownOpts::default().waiting_for_terminated())
4646 .await;
4647 assert_eq!(report.lifecycle, Some(Lifecycle::Terminated));
4648 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4649 }
4650
4651 #[tokio::test(start_paused = true)]
4660 async fn teardown_deletes_the_vm_then_the_image_and_names_the_log_group_last() {
4661 let (mut sandbox, recorder, _) = planted();
4662 recorder
4663 .answer(
4664 "CreateMicrovmImage",
4665 Answer::created(fake::create_image_response("agentd-conformance")),
4666 )
4667 .answer(
4668 "GetMicrovmImage",
4669 Answer::ok(fake::get_image_response("agentd-conformance", "CREATED")),
4670 );
4671 sandbox
4672 .build_image(CreateImageRequest::new(
4673 "agentd-conformance",
4674 crate::testing::aarch64_daemon(b"binary"),
4675 "s3://bucket/img.zip",
4676 "arn:aws:iam::123456789012:role/build",
4677 ))
4678 .await
4679 .expect("builds");
4680 answer_launch(&recorder);
4681 sandbox
4682 .run(RunRequest::new())
4683 .await
4684 .expect("launches from the built image");
4685
4686 recorder
4687 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4688 .answer(
4689 "ListMicrovmImageVersions",
4690 Answer::ok(fake::list_versions_response("1")),
4691 )
4692 .answer(
4693 "DeleteMicrovmImage",
4694 Answer::ok(fake::delete_image_response()),
4695 );
4696
4697 let report = sandbox
4698 .terminate(
4699 TeardownOpts::default()
4700 .deleting_image()
4701 .deleting_log_group(),
4702 )
4703 .await;
4704 assert!(report.terminate_accepted);
4705 assert_eq!(report.image_deleted, Some(true));
4706
4707 let order: Vec<&str> = recorder
4709 .operations()
4710 .into_iter()
4711 .filter(|operation| matches!(*operation, "TerminateMicrovm" | "DeleteMicrovmImage"))
4712 .collect();
4713 assert_eq!(
4714 order,
4715 ["TerminateMicrovm", "DeleteMicrovmImage"],
4716 "the VM goes before the image: a terminating VM holds a reference to it"
4717 );
4718
4719 assert_eq!(
4721 report.undeleted,
4722 ["/aws/lambda-microvms/agentd-conformance"],
4723 "{report:?}"
4724 );
4725 assert!(report.leaked());
4726 assert!(
4727 report
4728 .failures
4729 .last()
4730 .expect("a failure")
4731 .contains("log group"),
4732 "the log group's failure has to be the last one recorded: {report:?}"
4733 );
4734 assert!(
4735 report
4736 .failures
4737 .last()
4738 .expect("a failure")
4739 .contains("terraform destroy"),
4740 "the reason it leaks has to be named: {report:?}"
4741 );
4742 }
4743
4744 #[tokio::test(start_paused = true)]
4750 async fn a_failing_image_delete_names_what_it_left_behind() {
4751 let (mut sandbox, recorder, _) = planted();
4752 recorder
4753 .answer(
4754 "CreateMicrovmImage",
4755 Answer::created(fake::create_image_response("img")),
4756 )
4757 .answer(
4758 "GetMicrovmImage",
4759 Answer::ok(fake::get_image_response("img", "CREATED")),
4760 );
4761 sandbox
4762 .build_image(CreateImageRequest::new(
4763 "img",
4764 crate::testing::aarch64_daemon(b"binary"),
4765 "s3://bucket/img.zip",
4766 "arn:aws:iam::123456789012:role/build",
4767 ))
4768 .await
4769 .expect("builds");
4770 answer_launch(&recorder);
4771 sandbox.run(RunRequest::new()).await.expect("launches");
4772
4773 recorder
4774 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4775 .answer(
4776 "ListMicrovmImageVersions",
4777 Answer::ok(fake::list_versions_response("1")),
4778 )
4779 .answer(
4780 "DeleteMicrovmImage",
4781 Answer::failure(409, "image is in CREATING"),
4782 );
4783
4784 let mut opts = TeardownOpts::default().deleting_image();
4785 opts.delete_attempts = 3;
4786 opts.delete_backoff = Duration::from_secs(1);
4787 let report = sandbox.terminate(opts).await;
4788
4789 assert_eq!(report.image_deleted, Some(false));
4790 assert_eq!(
4791 report.undeleted,
4792 ["arn:aws:lambda:us-east-1:123456789012:microvm-image:img"],
4793 "{report:?}"
4794 );
4795 assert!(
4796 sandbox.image_exists(),
4797 "the image really is still there, so the flag must say so"
4798 );
4799 assert_eq!(
4800 recorder.call_count("DeleteMicrovmImage"),
4801 3,
4802 "every attempt ran"
4803 );
4804 }
4805
4806 #[tokio::test(start_paused = true)]
4813 async fn a_terminate_whose_call_fails_still_records_the_intent_and_the_leak() {
4814 let (mut sandbox, recorder, _) = launched().await;
4815 for _ in 0..6 {
4816 recorder.answer(
4817 "TerminateMicrovm",
4818 Answer::failure(500, "InternalServerException"),
4819 );
4820 }
4821 recorder.answer("TerminateMicrovm", Answer::failure(403, "denied"));
4823
4824 let report = sandbox.terminate(TeardownOpts::default()).await;
4825 assert_eq!(recorder.call_count("TerminateMicrovm"), 6);
4826 assert!(!report.terminate_accepted);
4827 assert_eq!(report.undeleted, ["mvm-abc123"], "{report:?}");
4828 assert!(sandbox.was_terminated());
4829 assert_eq!(report.lifecycle, Some(Lifecycle::Terminating));
4830 assert!(
4831 !report.failures.is_empty(),
4832 "the swallowed failure is recorded"
4833 );
4834 }
4835
4836 #[tokio::test]
4842 async fn a_teardown_before_any_launch_is_an_empty_report() {
4843 let (mut sandbox, recorder, _) = planted();
4844 let report = sandbox.terminate(TeardownOpts::default()).await;
4845 assert!(!report.terminate_accepted);
4846 assert!(!report.leaked());
4847 assert_eq!(report.lifecycle, Some(Lifecycle::Pending));
4848 assert_eq!(recorder.calls().len(), 0);
4849 }
4850
4851 #[tokio::test]
4854 async fn a_run_with_no_image_is_refused_before_any_call() {
4855 let (mut sandbox, recorder, _) = planted();
4856 let error = sandbox
4857 .run(RunRequest::new())
4858 .await
4859 .expect_err("there is nothing to launch");
4860 assert_eq!(error.kind(), ErrorKind::Precondition);
4861 assert!(error.to_string().contains("build_image first"), "{error}");
4862 assert_eq!(recorder.calls().len(), 0);
4863 }
4864
4865 #[test]
4872 fn the_lifecycle_states_are_the_state_models_domain_in_the_services_spelling() {
4873 let all = [
4874 Lifecycle::Pending,
4875 Lifecycle::Running,
4876 Lifecycle::Suspending,
4877 Lifecycle::Suspended,
4878 Lifecycle::Terminating,
4879 Lifecycle::Terminated,
4880 ];
4881 assert_eq!(
4882 all.map(Lifecycle::as_str),
4883 [
4884 "PENDING",
4885 "RUNNING",
4886 "SUSPENDING",
4887 "SUSPENDED",
4888 "TERMINATING",
4889 "TERMINATED"
4890 ]
4891 );
4892 for state in all {
4893 assert_eq!(state.to_string(), state.as_str());
4894 }
4895 assert_eq!(
4898 all.iter().filter(|state| state.is_live()).count(),
4899 4,
4900 "a state added to the enum defaults to not-live, which would silence the warning"
4901 );
4902 assert!(!Lifecycle::Terminating.is_live());
4903 assert!(!Lifecycle::Terminated.is_live());
4904 }
4905
4906 #[test]
4914 fn a_minted_agent_token_is_sixty_four_hex_characters_and_never_repeats() {
4915 let entropy = crate::entropy::testing::SequenceEntropy::new();
4916 let minted: std::collections::HashSet<String> = (0..200)
4917 .map(|_| mint_agent_token(&entropy).expect("scripted"))
4918 .collect();
4919 assert_eq!(minted.len(), 200, "a repeated token is a shared guest");
4920 for token in &minted {
4921 assert_eq!(token.len(), 64, "{token}");
4922 assert!(token.chars().all(|c| c.is_ascii_hexdigit()), "{token}");
4923 assert_ne!(
4924 token,
4925 &"0".repeat(64),
4926 "an all-zero draw is a read that did nothing"
4927 );
4928 }
4929 }
4930
4931 #[tokio::test(start_paused = true)]
4933 async fn a_sandbox_debug_does_not_print_the_agent_token() {
4934 let (mut sandbox, recorder, _) = planted();
4935 answer_launch(&recorder);
4936 sandbox
4937 .run(
4938 RunRequest::new()
4939 .with_image("arn:image")
4940 .with_agent_token("super-secret-agent-token"),
4941 )
4942 .await
4943 .expect("launches");
4944
4945 let rendered = format!("{sandbox:?}");
4946 assert!(rendered.contains("mvm-abc123"), "{rendered}");
4947 assert!(rendered.contains("Running"), "{rendered}");
4948 assert!(
4949 !rendered.contains("super-secret"),
4950 "the agent token reached a Debug string: {rendered}"
4951 );
4952 }
4953
4954 #[tokio::test(start_paused = true)]
4962 async fn a_torn_down_sandbox_is_distinguishable_from_an_abandoned_one() {
4963 let (mut sandbox, recorder, _) = launched().await;
4964 assert!(
4965 sandbox.lifecycle().is_live() && !sandbox.torn_down,
4966 "an abandoned live VM is what the drop warning is for"
4967 );
4968
4969 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4970 sandbox.terminate(TeardownOpts::default()).await;
4971 assert!(
4972 sandbox.torn_down,
4973 "terminate must mark the sandbox torn down"
4974 );
4975 assert!(
4976 !sandbox.lifecycle().is_live(),
4977 "and the lifecycle must no longer read as live"
4978 );
4979 }
4980
4981 #[tokio::test(start_paused = true)]
4982 async fn stable_launch_key_and_payload_survive_a_fresh_supervisor() {
4983 let mut bodies = Vec::new();
4984 for _ in 0..2 {
4985 let (mut sandbox, recorder, _) = planted();
4986 answer_launch(&recorder);
4987 let mut request = RunRequest::new().with_image("arn:image");
4988 request.client_token = Some("one-unique-job-launch".into());
4989 request.agent_token = Some("persisted-private-secret".into());
4990 sandbox.run(request).await.expect("accepted");
4991 bodies.push(recorder.first_body("RunMicrovm"));
4992 }
4993 assert_eq!(bodies[0]["clientToken"], "one-unique-job-launch");
4994 assert_eq!(bodies[0], bodies[1]);
4995 }
4996
4997 #[tokio::test]
4998 async fn unsafe_or_malformed_stable_launch_is_refused_without_calls() {
4999 for (token, secret, identity) in [
5000 ("key".to_string(), None, false),
5001 ("key".to_string(), Some("secret".into()), true),
5002 ("".to_string(), Some("secret".into()), false),
5003 ("x".repeat(129), Some("secret".into()), false),
5004 ("line\nbreak".to_string(), Some("secret".into()), false),
5005 ] {
5006 let (mut sandbox, recorder, _) = planted();
5007 let mut request = RunRequest::new().with_image("arn:image");
5008 request.client_token = Some(token);
5009 request.agent_token = secret;
5010 request.identity = identity;
5011 let error = sandbox.run(request).await.expect_err("invalid");
5012 assert_eq!(error.kind(), ErrorKind::InvalidArg);
5013 assert!(recorder.calls().is_empty());
5014 }
5015 }
5016}