1use std::sync::Arc;
67use std::time::Duration;
68
69use crate::control::{
70 ControlPlane, CreateImageRequest, Image, Microvm, RunHookPayload, RunMicrovmRequest, WaitOpts,
71};
72use crate::error::{Error, ErrorKind};
73use crate::region::Region;
74use crate::session::{Session, TokenMinter};
75
76pub const DEFAULT_RUNNING_TIMEOUT: Duration = Duration::from_secs(300);
80
81#[deprecated(
83 note = "renamed DEFAULT_RUNNING_TIMEOUT: `session::DEFAULT_READY_TIMEOUT` shared this name \
84 for a different wait, the daemon's"
85)]
86pub const DEFAULT_READY_TIMEOUT: Duration = DEFAULT_RUNNING_TIMEOUT;
87
88pub const DEFAULT_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(300);
91
92pub const DEFAULT_DELETE_ATTEMPTS: u32 = 20;
98
99pub const DEFAULT_DELETE_BACKOFF: Duration = Duration::from_secs(15);
101
102pub const DENY_EGRESS_PROXY_URL: &str = "http://127.0.0.1:1";
108
109pub const DENY_EGRESS_ENV_KEYS: [&str; 6] = [
118 "http_proxy",
119 "https_proxy",
120 "all_proxy",
121 "HTTP_PROXY",
122 "HTTPS_PROXY",
123 "ALL_PROXY",
124];
125
126pub const LIFECYCLE_POLL_INTERVAL: Duration = Duration::from_secs(5);
129
130#[derive(Clone, Copy, Debug, Eq, PartialEq)]
136pub enum Lifecycle {
137 Pending,
139 Running,
142 Suspending,
144 Suspended,
146 Terminating,
148 Terminated,
150}
151
152impl Lifecycle {
153 pub fn as_str(self) -> &'static str {
155 match self {
156 Lifecycle::Pending => "PENDING",
157 Lifecycle::Running => "RUNNING",
158 Lifecycle::Suspending => "SUSPENDING",
159 Lifecycle::Suspended => "SUSPENDED",
160 Lifecycle::Terminating => "TERMINATING",
161 Lifecycle::Terminated => "TERMINATED",
162 }
163 }
164
165 pub fn from_service(state: &str) -> Option<Self> {
168 Some(match state {
169 "PENDING" => Lifecycle::Pending,
170 "RUNNING" => Lifecycle::Running,
171 "SUSPENDING" => Lifecycle::Suspending,
172 "SUSPENDED" => Lifecycle::Suspended,
173 "TERMINATING" => Lifecycle::Terminating,
174 "TERMINATED" => Lifecycle::Terminated,
175 _ => return None,
176 })
177 }
178
179 pub fn is_live(self) -> bool {
181 matches!(
182 self,
183 Lifecycle::Pending | Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended
184 )
185 }
186}
187
188impl std::fmt::Display for Lifecycle {
189 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
190 f.write_str(self.as_str())
191 }
192}
193
194#[derive(Clone, Debug)]
200pub struct RunRequest {
201 pub image_identifier: Option<String>,
207 pub image_version: Option<String>,
217 pub execution_role_arn: Option<String>,
219 pub agent_token: Option<String>,
221 pub client_token: Option<String>,
225 pub launch_env: std::collections::HashMap<String, String>,
239 pub identity: bool,
251 pub egress: bool,
255 pub egress_network_connectors: Vec<String>,
258 pub deny_egress: bool,
265 pub shell: bool,
270 pub max_idle_sec: u32,
272 pub suspended_sec: u32,
275 pub auto_resume: bool,
277 pub max_duration_sec: u32,
279 pub ready_timeout: Duration,
282 pub token_scope: Option<String>,
284 pub logging: Option<crate::control::ops::Logging>,
286 pub wait: bool,
293}
294
295impl Default for RunRequest {
296 fn default() -> Self {
297 Self {
298 image_identifier: None,
299 image_version: None,
300 execution_role_arn: None,
301 agent_token: None,
302 client_token: None,
303 launch_env: std::collections::HashMap::new(),
304 identity: false,
305 egress: false,
306 egress_network_connectors: Vec::new(),
307 deny_egress: false,
308 shell: false,
309 max_idle_sec: 600,
310 suspended_sec: 600,
311 auto_resume: false,
312 max_duration_sec: 3_600,
313 ready_timeout: DEFAULT_RUNNING_TIMEOUT,
314 token_scope: None,
315 logging: None,
316 wait: true,
317 }
318 }
319}
320
321impl RunRequest {
322 pub fn new() -> Self {
325 Self::default()
326 }
327
328 #[must_use]
330 pub fn with_image(mut self, identifier: impl Into<String>) -> Self {
331 self.image_identifier = Some(identifier.into());
332 self
333 }
334
335 #[must_use]
337 pub fn with_egress(mut self) -> Self {
338 self.egress = true;
339 self
340 }
341
342 #[must_use]
344 pub fn with_egress_network_connector(mut self, arn: impl Into<String>) -> Self {
345 self.egress_network_connectors.push(arn.into());
346 self
347 }
348
349 #[must_use]
355 pub fn with_vpc_egress(mut self, arns: Vec<String>) -> Self {
356 if !arns.is_empty() {
357 self.egress = false;
358 self.egress_network_connectors = arns;
359 }
360 self
361 }
362
363 #[must_use]
366 pub fn with_deny_egress(mut self) -> Self {
367 self.deny_egress = true;
368 self
369 }
370
371 pub fn egress_posture(&self) -> crate::control::EgressPosture {
376 crate::control::EgressPosture::for_launch(self.egress, self.deny_egress)
377 }
378
379 pub fn effective_launch_env(&self) -> std::collections::HashMap<String, String> {
387 let mut env = self.launch_env.clone();
388 if self.deny_egress {
389 for key in DENY_EGRESS_ENV_KEYS {
390 env.entry(key.to_string())
391 .or_insert_with(|| DENY_EGRESS_PROXY_URL.to_string());
392 }
393 }
394 env
395 }
396
397 #[must_use]
399 pub fn with_shell(mut self) -> Self {
400 self.shell = true;
401 self
402 }
403
404 #[must_use]
406 pub fn with_image_version(mut self, version: impl Into<String>) -> Self {
407 self.image_version = Some(version.into());
408 self
409 }
410
411 #[must_use]
414 pub fn with_suspended_sec(mut self, seconds: u32) -> Self {
415 self.suspended_sec = seconds;
416 self
417 }
418
419 #[must_use]
421 pub fn with_agent_token(mut self, token: impl Into<String>) -> Self {
422 self.agent_token = Some(token.into());
423 self
424 }
425
426 #[must_use]
431 pub fn with_identity(mut self) -> Self {
432 self.identity = true;
433 self
434 }
435
436 #[must_use]
443 pub fn with_launch_env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
444 self.launch_env.insert(key.into(), value.into());
445 self
446 }
447}
448
449#[derive(Clone, Copy, Debug)]
455pub struct TeardownOpts {
456 pub delete_image: bool,
458 pub delete_log_group: bool,
464 pub delete_attempts: u32,
466 pub delete_backoff: Duration,
468 pub wait_for_terminated: Option<Duration>,
476}
477
478impl Default for TeardownOpts {
479 fn default() -> Self {
480 Self {
481 delete_image: false,
482 delete_log_group: false,
483 delete_attempts: DEFAULT_DELETE_ATTEMPTS,
484 delete_backoff: DEFAULT_DELETE_BACKOFF,
485 wait_for_terminated: None,
486 }
487 }
488}
489
490impl TeardownOpts {
491 #[must_use]
493 pub fn deleting_image(mut self) -> Self {
494 self.delete_image = true;
495 self
496 }
497
498 #[must_use]
500 pub fn deleting_log_group(mut self) -> Self {
501 self.delete_log_group = true;
502 self
503 }
504
505 #[must_use]
507 pub fn waiting_for_terminated(mut self) -> Self {
508 self.wait_for_terminated = Some(DEFAULT_LIFECYCLE_TIMEOUT);
509 self
510 }
511}
512
513#[derive(Clone, Debug, Default, Eq, PartialEq)]
517pub struct TeardownReport {
518 pub undeleted: Vec<String>,
533 pub terminate_accepted: bool,
535 pub image_deleted: Option<bool>,
537 pub lifecycle: Option<Lifecycle>,
539 pub failures: Vec<String>,
544}
545
546impl TeardownReport {
547 pub fn leaked(&self) -> bool {
549 !self.undeleted.is_empty()
550 }
551}
552
553pub struct ControlPlaneMinter {
567 control: Arc<ControlPlane>,
568 microvm_id: String,
569}
570
571impl ControlPlaneMinter {
572 pub fn new(control: Arc<ControlPlane>, microvm_id: impl Into<String>) -> Self {
574 Self {
575 control,
576 microvm_id: microvm_id.into(),
577 }
578 }
579}
580
581impl std::fmt::Debug for ControlPlaneMinter {
582 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
583 f.debug_struct("ControlPlaneMinter")
584 .field("microvm_id", &self.microvm_id)
585 .finish_non_exhaustive()
586 }
587}
588
589impl TokenMinter for ControlPlaneMinter {
590 fn mint(
591 &self,
592 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
593 Box::pin(async move {
594 let minted = self.control.mint_auth_token(&self.microvm_id).await?;
595 Ok(minted.into())
596 })
597 }
598
599 fn mint_for_ports(
607 &self,
608 ports: &[u16],
609 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
610 let specs: Vec<crate::control::ops::PortSpecification> = ports
611 .iter()
612 .map(|port| crate::control::ops::PortSpecification::port(*port))
613 .collect();
614 Box::pin(async move {
615 let minted = self
616 .control
617 .mint_auth_token_for(&self.microvm_id, &specs)
618 .await?;
619 Ok(minted.into())
620 })
621 }
622}
623
624pub struct Sandbox {
629 control: Arc<ControlPlane>,
630 image: Option<Image>,
631 launch_image_arn: Option<String>,
633 microvm: Option<Microvm>,
634 session: Option<Session>,
635 session_backend: Option<crate::session::SharedBackend>,
642
643 lifecycle: Lifecycle,
645 lifecycle_watch: tokio::sync::watch::Sender<Lifecycle>,
648 token_installed: bool,
649 image_exists: bool,
650 was_terminated: bool,
651 bootstrap_count: u32,
652
653 suspended_window: Option<Duration>,
656 launch_adoptable: bool,
659 launch_posture: Option<crate::control::EgressPosture>,
662 pending_agent_token: Option<String>,
665 idle_window: Option<Duration>,
668 torn_down: bool,
671 adopted: bool,
674 detached: bool,
677 tunnel_identity: Option<crate::identity::TunnelIdentity>,
683 build_services: Option<Arc<dyn crate::control::BuildServices>>,
686 account: Option<String>,
688}
689
690impl std::fmt::Debug for Sandbox {
691 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
692 f.debug_struct("Sandbox")
695 .field("lifecycle", &self.lifecycle)
696 .field("microvm", &self.microvm.as_ref().map(|vm| &vm.id))
697 .field("image", &self.image.as_ref().map(|image| &image.identifier))
698 .field("token_installed", &self.token_installed)
699 .field("bootstrap_count", &self.bootstrap_count)
700 .field("was_terminated", &self.was_terminated)
701 .field("suspended_window", &self.suspended_window)
702 .field("adopted", &self.adopted)
703 .field("detached", &self.detached)
704 .finish_non_exhaustive()
705 }
706}
707
708#[derive(Clone)]
715pub struct Detached {
716 pub microvm_id: String,
718 pub endpoint: String,
720 pub region: Region,
722 pub port: u16,
724 agent_token: String,
725}
726
727impl Detached {
728 pub fn agent_token(&self) -> &str {
730 &self.agent_token
731 }
732}
733
734impl std::fmt::Debug for Detached {
735 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
736 f.debug_struct("Detached")
737 .field("microvm_id", &self.microvm_id)
738 .field("endpoint", &self.endpoint)
739 .field("region", &self.region)
740 .field("port", &self.port)
741 .field("agent_token", &"<redacted>")
742 .finish()
743 }
744}
745
746impl Sandbox {
747 pub fn with_control_plane(control: ControlPlane) -> Self {
761 Self {
762 control: Arc::new(control),
763 image: None,
764 launch_image_arn: None,
765 microvm: None,
766 session: None,
767 session_backend: None,
768 lifecycle: Lifecycle::Pending,
769 lifecycle_watch: tokio::sync::watch::Sender::new(Lifecycle::Pending),
770 token_installed: false,
771 image_exists: false,
772 was_terminated: false,
773 bootstrap_count: 0,
774 suspended_window: None,
775 launch_adoptable: false,
776 launch_posture: None,
777 pending_agent_token: None,
778 idle_window: None,
779 torn_down: false,
780 adopted: false,
781 detached: false,
782 tunnel_identity: None,
783 build_services: None,
784 account: None,
785 }
786 }
787
788 pub fn with_build_services(mut self, services: Arc<dyn crate::control::BuildServices>) -> Self {
794 self.build_services = Some(services);
795 self
796 }
797
798 pub fn with_session_backend(mut self, backend: crate::session::SharedBackend) -> Self {
804 self.session_backend = Some(backend);
805 self
806 }
807
808 pub fn lifecycle(&self) -> Lifecycle {
812 self.lifecycle
813 }
814
815 pub fn token_installed(&self) -> bool {
817 self.token_installed
818 }
819
820 pub fn image_exists(&self) -> bool {
822 self.image_exists
823 }
824
825 pub fn was_terminated(&self) -> bool {
827 self.was_terminated
828 }
829
830 pub fn bootstrap_count(&self) -> u32 {
832 self.bootstrap_count
833 }
834
835 pub fn image(&self) -> Option<&Image> {
837 self.image.as_ref()
838 }
839
840 pub fn launch_image_arn(&self) -> Option<&str> {
849 self.launch_image_arn.as_deref()
850 }
851
852 pub fn microvm(&self) -> Option<&Microvm> {
854 self.microvm.as_ref()
855 }
856
857 pub fn session(&self) -> Option<&Session> {
859 self.session.as_ref()
860 }
861
862 pub fn suspended_window(&self) -> Option<Duration> {
864 self.suspended_window
865 }
866
867 pub fn watch_lifecycle(&self) -> tokio::sync::watch::Receiver<Lifecycle> {
874 self.lifecycle_watch.subscribe()
875 }
876
877 fn set_lifecycle(&mut self, lifecycle: Lifecycle) {
878 self.lifecycle = lifecycle;
879 self.lifecycle_watch.send_replace(lifecycle);
880 }
881
882 pub fn idle_window(&self) -> Option<Duration> {
884 self.idle_window
885 }
886
887 pub fn tunnel_identity(&self) -> Option<&crate::identity::TunnelIdentity> {
893 self.tunnel_identity.as_ref()
894 }
895
896 pub fn egress_posture(&self) -> crate::control::EgressPosture {
904 self.launch_posture.unwrap_or_default()
905 }
906
907 pub fn port(&self) -> u16 {
912 self.control.port()
913 }
914
915 pub async fn build_image(&mut self, request: CreateImageRequest) -> Result<&Image, Error> {
923 let size = request.size;
924 let created = self.control.create_image(request).await?;
925 let mut built = self
926 .control
927 .wait_for_image(&created.identifier, size, WaitOpts::default())
928 .await?;
929 built.log_group = created.log_group;
935 built.log_stream = created.log_stream;
936 self.image_exists = true;
939 self.image = Some(built);
940 Ok(self.image.as_ref().expect("just assigned"))
941 }
942
943 pub async fn ensure_image(
947 &mut self,
948 request: crate::control::EnsureImageRequest,
949 ) -> Result<crate::control::EnsuredImage, Error> {
950 let control = Arc::clone(&self.control);
959 let prepared =
960 tokio::task::spawn_blocking(move || crate::control::ensure::prepare(&control, request))
961 .await
962 .map_err(|joined| {
963 Error::new(
964 ErrorKind::Unexpected,
965 format!("preparing the image's artifact did not complete: {joined}"),
966 )
967 })??;
968 let services = self.build_services().await?;
969 let account = match &self.account {
970 Some(account) => account.clone(),
971 None => {
972 let account = services.caller_account().await?;
973 self.account = Some(account.clone());
974 account
975 }
976 };
977 let ensured =
978 crate::control::ensure::ensure(&self.control, services.as_ref(), &account, prepared)
979 .await?;
980 self.image_exists = true;
981 self.image = Some(ensured.image.clone());
982 Ok(ensured)
983 }
984
985 async fn build_services(&mut self) -> Result<Arc<dyn crate::control::BuildServices>, Error> {
988 if let Some(services) = &self.build_services {
989 return Ok(Arc::clone(services));
990 }
991 let built = self
992 .control
993 .adapters()
994 .build_services(self.control.region().clone())
995 .await?;
996 self.build_services = Some(Arc::clone(&built));
997 Ok(built)
998 }
999
1000 pub fn build_artifact_for(&self, request: &CreateImageRequest) -> Result<Vec<u8>, Error> {
1005 self.control.build_artifact_for(request)
1006 }
1007
1008 pub async fn upload_artifact_for(&mut self, request: &CreateImageRequest) -> Result<(), Error> {
1014 let (bucket, key) = crate::control::ensure::s3_location(&request.code_artifact_uri)?;
1015 let bytes = self.control.build_artifact_for(request)?;
1016 let services = self.build_services().await?;
1017 services.put_object(bucket, key, bytes).await
1018 }
1019
1020 pub fn preflight(&self, request: &CreateImageRequest) -> Result<(), Error> {
1027 self.control.preflight(request)
1028 }
1029
1030 pub async fn resolve_image_arn(&self, identifier: &str) -> Result<String, Error> {
1038 self.control.resolve_image_arn(identifier).await
1039 }
1040
1041 pub async fn find_image_by_name(
1044 &self,
1045 name: &str,
1046 ) -> Result<Option<crate::control::ops::MicrovmImageSummaryWire>, Error> {
1047 self.control.find_image_by_name(name).await
1048 }
1049
1050 pub fn artifact_content_hash_for(&self, request: &CreateImageRequest) -> String {
1052 self.control.artifact_content_hash_for(request)
1053 }
1054
1055 pub fn dockerfile_for(&self, request: &CreateImageRequest) -> String {
1058 self.control.dockerfile_for(request)
1059 }
1060
1061 pub async fn managed_base_versions(
1069 &self,
1070 base_image_arn: &str,
1071 ) -> Result<Vec<crate::control::ops::ManagedMicrovmImageVersionWire>, Error> {
1072 self.control.managed_base_versions(base_image_arn).await
1073 }
1074
1075 pub async fn image_version(
1077 &self,
1078 identifier: &str,
1079 version: &str,
1080 ) -> Result<crate::control::ops::MicrovmImageVersionSummaryWire, Error> {
1081 self.control.get_image_version(identifier, version).await
1082 }
1083
1084 pub async fn run(&mut self, request: RunRequest) -> Result<&mut Session, Error> {
1112 self.refuse_detached("run")?;
1113 if self.bootstrap_count > 0 || self.microvm.is_some() {
1117 return Err(Error::invalid_arg(format!(
1118 "this sandbox has already launched a VM ({} bootstrap(s), lifecycle {}), and the \
1119 agent token is installed at most once per VM lifetime (STATE-3). A second VM \
1120 needs a second Sandbox — reusing this one would either re-deliver a run-hook \
1121 payload to a daemon whose one-shot bootstrap refuses it, or silently address \
1122 two guests through one handle.",
1123 self.bootstrap_count, self.lifecycle,
1124 )));
1125 }
1126
1127 let posture = crate::control::egress_posture_for(
1134 request.egress,
1135 &request.egress_network_connectors,
1136 request.deny_egress,
1137 Some(self.control.region()),
1138 )?;
1139
1140 let Some(identifier) = request
1141 .image_identifier
1142 .clone()
1143 .or_else(|| self.image.as_ref().map(|image| image.identifier.clone()))
1144 else {
1145 return Err(Error::new(
1146 ErrorKind::Precondition,
1147 "no image to launch: pass RunRequest::with_image or call build_image first."
1148 .to_string(),
1149 ));
1150 };
1151
1152 if request.client_token.is_some() && (request.agent_token.is_none() || request.identity) {
1153 return Err(Error::invalid_arg(
1154 "a stable client_token requires an explicit agent_token and identity=false; persist identical launch parameters before retrying",
1155 ));
1156 }
1157 let agent_token = match request.agent_token.clone() {
1158 Some(token) => token,
1159 None => mint_agent_token(self.control.entropy())?,
1160 };
1161 let launch_identity = if request.identity {
1165 Some(crate::entropy::launch_identity(self.control.entropy())?)
1166 } else {
1167 None
1168 };
1169 let payload = RunHookPayload::for_launch_with_identity(
1176 &agent_token,
1177 &request.effective_launch_env(),
1178 launch_identity.as_ref(),
1179 )?;
1180
1181 let mut wire = RunMicrovmRequest::new(identifier, payload);
1182 wire.image_version = request.image_version.clone();
1183 wire.execution_role_arn = request.execution_role_arn.clone();
1184 wire.egress_network_connectors = request.egress_network_connectors.clone();
1185 wire.max_idle_sec = request.max_idle_sec;
1186 wire.suspended_sec = request.suspended_sec;
1187 wire.auto_resume = request.auto_resume;
1188 wire.max_duration_sec = request.max_duration_sec;
1189 wire.token_scope = request.token_scope.clone();
1190 wire.client_token = request.client_token.clone();
1191 wire.logging = request.logging.clone();
1192 if request.egress {
1193 wire = wire.with_egress();
1194 }
1195 if request.shell {
1196 wire = wire.with_shell();
1197 }
1198
1199 self.control.launch_connectors(&wire)?;
1205 wire.image_identifier = self
1206 .control
1207 .resolve_image_arn(&wire.image_identifier)
1208 .await?;
1209 self.launch_image_arn = Some(wire.image_identifier.clone());
1210
1211 let launched = self.control.run_microvm(wire).await?;
1212
1213 self.tunnel_identity = launch_identity.map(crate::identity::LaunchIdentity::keep);
1217
1218 self.set_lifecycle(Lifecycle::Pending);
1220 self.image_exists = true;
1221 self.suspended_window = Some(Duration::from_secs(u64::from(request.suspended_sec)));
1225 self.idle_window = Some(Duration::from_secs(u64::from(request.max_idle_sec)));
1226 self.microvm = Some(launched);
1227 self.launch_adoptable = request.client_token.is_some();
1230 self.launch_posture = Some(posture);
1231 self.pending_agent_token = Some(agent_token);
1232
1233 if request.wait {
1234 return self.wait_until_running(request.ready_timeout).await;
1235 }
1236 self.build_session()?;
1239 Ok(self.session.as_mut().expect("just assigned"))
1240 }
1241
1242 fn build_session(&mut self) -> Result<(), Error> {
1244 let (Some(vm), Some(agent_token)) = (&self.microvm, &self.pending_agent_token) else {
1245 return Err(Error::new(
1246 ErrorKind::Precondition,
1247 "no launch to build a session for",
1248 ));
1249 };
1250 let minter = Arc::new(ControlPlaneMinter::new(
1251 Arc::clone(&self.control),
1252 vm.id.clone(),
1253 ));
1254 let mut builder = Session::builder(vm.endpoint.clone(), agent_token.clone())
1257 .with_minter(minter)
1258 .with_port(self.control.port())
1259 .with_egress_posture(self.egress_posture())
1260 .with_clock(self.control.shared_clock());
1261 if let Some(backend) = &self.session_backend {
1262 builder = builder.with_backend(Arc::clone(backend));
1263 }
1264 self.session = Some(builder.build_with(self.control.adapters())?);
1265 Ok(())
1266 }
1267
1268 pub async fn wait_until_running(&mut self, timeout: Duration) -> Result<&mut Session, Error> {
1284 self.refuse_detached("wait until running")?;
1285 let id = self.require_microvm("wait_until_running")?;
1286 if self.lifecycle != Lifecycle::Pending {
1287 return Err(Error::invalid_arg(format!(
1288 "microvm {id} is {}; wait_until_running finishes a launch that is still PENDING",
1289 self.lifecycle,
1290 )));
1291 }
1292 let opts = WaitOpts::for_lifecycle(timeout);
1293 let running = self
1294 .control
1295 .wait_for_sandbox_launch(&id, opts, self.launch_adoptable)
1296 .await?;
1297
1298 self.set_lifecycle(Lifecycle::Running);
1301 self.token_installed = true;
1302 self.bootstrap_count += 1;
1303 self.microvm = Some(running);
1304 if self.session.is_none() {
1305 self.build_session()?;
1306 }
1307 let session = self.session.as_mut().expect("just assigned");
1308 session
1309 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1310 .await?;
1311 Ok(session)
1312 }
1313
1314 pub async fn wait_until_ready(
1324 &mut self,
1325 running_timeout: Duration,
1326 ) -> Result<&mut Session, Error> {
1327 if self.lifecycle == Lifecycle::Running {
1328 self.refuse_detached("wait until ready")?;
1329 let session = self.session.as_mut().ok_or_else(|| {
1330 Error::new(ErrorKind::Precondition, "a RUNNING sandbox with no session")
1331 })?;
1332 session
1333 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1334 .await?;
1335 return Ok(session);
1336 }
1337 self.wait_until_running(running_timeout).await
1338 }
1339
1340 pub async fn adopt(
1360 control: ControlPlane,
1361 microvm_id: impl Into<String>,
1362 endpoint: impl Into<String>,
1363 agent_token: impl Into<String>,
1364 ) -> Result<Self, Error> {
1365 let microvm_id = microvm_id.into();
1366 let endpoint = endpoint.into();
1367 let agent_token = agent_token.into();
1368 if agent_token.is_empty() {
1369 return Err(Error::invalid_arg(
1370 "adopt needs the agent token the VM was launched with: its daemon accepts only \
1371 that bearer, and adopting without it would build a handle every exec refuses.",
1372 ));
1373 }
1374 let mut vm = control.get_microvm(µvm_id).await?;
1375 if !endpoint.is_empty() && !vm.endpoint.is_empty() && vm.endpoint != endpoint {
1376 return Err(Error::invalid_arg(format!(
1377 "microvm {microvm_id} reports endpoint {}, not {endpoint}: the id and endpoint \
1378 came from different records, and a session built from them would address one \
1379 VM while its lifecycle calls reached another.",
1380 vm.endpoint,
1381 )));
1382 }
1383 if vm.endpoint.is_empty() {
1384 vm.endpoint = endpoint;
1385 }
1386 let Some(lifecycle) = Lifecycle::from_service(&vm.state) else {
1387 return Err(Error::new(
1388 ErrorKind::Platform,
1389 format!(
1390 "microvm {microvm_id} reports state {}, which this client does not know; \
1391 refusing to guess which lifecycle guards apply.",
1392 vm.state,
1393 ),
1394 ));
1395 };
1396
1397 let mut sandbox = Self::with_control_plane(control);
1398 sandbox.adopted = true;
1399 sandbox.set_lifecycle(lifecycle);
1400 sandbox.idle_window = vm
1403 .idle_policy
1404 .as_ref()
1405 .map(|policy| Duration::from_secs(u64::from(policy.max_idle_duration_seconds)));
1406 sandbox.image_exists = true;
1408 sandbox.launch_adoptable = true;
1410 sandbox.pending_agent_token = Some(agent_token);
1411 match lifecycle {
1412 Lifecycle::Pending => {}
1413 Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended => {
1414 sandbox.token_installed = true;
1415 sandbox.bootstrap_count = 1;
1416 }
1417 Lifecycle::Terminating | Lifecycle::Terminated => {
1418 sandbox.bootstrap_count = 1;
1419 sandbox.was_terminated = true;
1420 }
1421 }
1422 sandbox.microvm = Some(vm);
1423 if lifecycle.is_live() {
1424 sandbox.build_session()?;
1425 }
1426 Ok(sandbox)
1427 }
1428
1429 pub fn name_record(&self, name: &str) -> Result<crate::names::NameRecord, Error> {
1436 let Some(vm) = self.microvm() else {
1437 return Err(Error::new(
1438 ErrorKind::Precondition,
1439 "there is no VM to name yet: launch or adopt one first",
1440 ));
1441 };
1442 let Some(session) = self.session() else {
1443 return Err(Error::new(
1444 ErrorKind::Precondition,
1445 format!(
1446 "microvm {} has no session to take its agent token from",
1447 vm.id
1448 ),
1449 ));
1450 };
1451 let mut record = crate::names::NameRecord::new_at(
1453 name,
1454 vm.id.as_str(),
1455 vm.endpoint.as_str(),
1456 session.agent_token(),
1457 self.control.region().as_str(),
1458 self.control.clock().unix_now().as_secs(),
1459 )?;
1460 record.set_tunnel_identity(self.tunnel_identity.as_ref());
1461 Ok(record)
1462 }
1463
1464 pub async fn adopt_record(
1471 control: ControlPlane,
1472 record: crate::names::NameRecord,
1473 ) -> Result<Self, Error> {
1474 let identity = record.tunnel_identity()?;
1475 let mut sandbox = Self::adopt(
1476 control,
1477 record.microvm_id,
1478 record.endpoint,
1479 record.agent_token,
1480 )
1481 .await?;
1482 sandbox.tunnel_identity = identity;
1483 Ok(sandbox)
1484 }
1485
1486 pub fn detach(&mut self) -> Result<Detached, Error> {
1499 self.refuse_detached("detach")?;
1500 let Some(vm) = self.microvm.as_ref() else {
1501 return Err(Error::new(
1502 ErrorKind::Precondition,
1503 "nothing to detach: this sandbox has not launched a VM.",
1504 ));
1505 };
1506 if self.torn_down || !self.lifecycle.is_live() {
1507 return Err(Error::new(
1508 ErrorKind::Precondition,
1509 format!(
1510 "microvm {} is {} and there is no live VM to hand off; only a PENDING, \
1511 RUNNING, or SUSPENDED VM can be adopted.",
1512 vm.id, self.lifecycle,
1513 ),
1514 ));
1515 }
1516 let agent_token = self
1517 .session
1518 .as_ref()
1519 .map(|session| session.agent_token().to_string())
1520 .or_else(|| self.pending_agent_token.clone())
1521 .ok_or_else(|| {
1522 Error::new(
1523 ErrorKind::Precondition,
1524 format!(
1525 "microvm {} has no agent token to hand off; an adopter could not \
1526 reach its daemon.",
1527 vm.id
1528 ),
1529 )
1530 })?;
1531 let detached = Detached {
1532 microvm_id: vm.id.clone(),
1533 endpoint: vm.endpoint.clone(),
1534 region: self.control.region().clone(),
1535 port: self.control.port(),
1536 agent_token,
1537 };
1538 self.detached = true;
1539 self.session = None;
1540 self.pending_agent_token = None;
1541 Ok(detached)
1542 }
1543
1544 pub fn detached(&self) -> bool {
1546 self.detached
1547 }
1548
1549 fn refuse_detached(&self, what: &str) -> Result<(), Error> {
1551 if !self.detached {
1552 return Ok(());
1553 }
1554 let id = self.microvm.as_ref().map_or("?", |vm| vm.id.as_str());
1555 Err(Error::new(
1556 ErrorKind::Precondition,
1557 format!(
1558 "cannot {what}: microvm {id} was detached and belongs to whoever adopts it. \
1559 Adopt it with `Sandbox::adopt` and {what} through that handle."
1560 ),
1561 ))
1562 }
1563
1564 pub fn adopted(&self) -> bool {
1566 self.adopted
1567 }
1568
1569 pub async fn suspend(&mut self) -> Result<(), Error> {
1584 self.refuse_detached("suspend")?;
1585 let id = self.require_microvm("suspend")?;
1586
1587 if self.lifecycle != Lifecycle::Running {
1589 return Err(Error::invalid_arg(format!(
1590 "microvm {id} is {} and a suspend is only issued from RUNNING (STATE-5). Refused \
1591 here rather than by the service, because the service's answer about a \
1592 non-running id does not say which of the two things went wrong — and a suspend \
1593 issued from SUSPENDED is a caller who believes they resumed.",
1594 self.lifecycle,
1595 )));
1596 }
1597
1598 self.control.suspend(&id).await?;
1603 self.set_lifecycle(Lifecycle::Suspending);
1604
1605 let settled = self
1606 .control
1607 .wait_for_state(
1608 &id,
1609 &crate::control::microvm::SUSPEND_WANTED,
1610 &[],
1611 self.lifecycle_wait(),
1612 )
1613 .await?;
1614
1615 let settled_lifecycle = match settled.state.as_str() {
1620 "SUSPENDED" => Lifecycle::Suspended,
1621 "TERMINATED" => {
1622 self.was_terminated = true;
1623 Lifecycle::Terminated
1624 }
1625 other => {
1626 return Err(Error::new(
1627 ErrorKind::Platform,
1628 format!(
1629 "the suspend wait returned {other}, which is neither SUSPENDED nor \
1630 TERMINATED — and those two are what this client asked for."
1631 ),
1632 ));
1633 }
1634 };
1635 self.set_lifecycle(settled_lifecycle);
1636 self.microvm = Some(settled);
1637 Ok(())
1638 }
1639
1640 pub async fn resume(&mut self) -> Result<&mut Session, Error> {
1668 self.refuse_detached("resume")?;
1669 let id = self.require_microvm("resume")?;
1670
1671 if self.was_terminated || self.lifecycle == Lifecycle::Terminated {
1674 return Err(Error::invalid_arg(format!(
1675 "microvm {id} was terminated, and a terminated VM never returns to RUNNING \
1676 (STATE-11). There is nothing to resume: the guest's memory is gone, so even a \
1677 call the service accepted would hand back a different machine."
1678 )));
1679 }
1680 let mut resume_state = self.lifecycle;
1681 if self.lifecycle == Lifecycle::Running {
1684 let observed = self.control.get_microvm(&id).await?;
1685 resume_state = Lifecycle::from_service(&observed.state).ok_or_else(|| {
1686 Error::new(
1687 ErrorKind::Platform,
1688 format!("microvm {id} reports unknown state {}", observed.state),
1689 )
1690 })?;
1691 let ended = crate::constants::DEAD_STATES.contains(&observed.state.as_str());
1692 let error = ended.then(|| {
1693 Error::new(
1694 ErrorKind::WindowClosed,
1695 format!(
1696 "microvm {id} can't be resumed: GetMicrovm reports the VM {} ({}). \
1697 A VM that has ended doesn't come back; launch a new one.",
1698 observed.state,
1699 observed.reason(),
1700 ),
1701 )
1702 });
1703 self.microvm = Some(observed);
1704 if resume_state == Lifecycle::Suspended {
1707 self.set_lifecycle(resume_state);
1708 }
1709 if let Some(error) = error {
1710 self.was_terminated = true;
1711 self.set_lifecycle(Lifecycle::Terminated);
1712 return Err(error);
1713 }
1714 }
1715 if resume_state != Lifecycle::Suspended {
1716 return Err(Error::invalid_arg(format!(
1717 "microvm {id} is {} and a resume is only issued from SUSPENDED (STATE-7).",
1718 resume_state,
1719 )));
1720 }
1721
1722 if let Err(refused) = self.control.resume(&id).await {
1728 if refused.kind() == ErrorKind::WindowClosed {
1729 self.was_terminated = true;
1730 self.set_lifecycle(Lifecycle::Terminated);
1731 }
1732 return Err(refused);
1733 }
1734 let running = self
1740 .control
1741 .wait_for_state(
1742 &id,
1743 &["RUNNING"],
1744 &crate::constants::DEAD_STATES,
1745 self.lifecycle_wait(),
1746 )
1747 .await?;
1748
1749 self.set_lifecycle(Lifecycle::Running);
1750 let endpoint = running.endpoint.clone();
1754 self.microvm = Some(running);
1755 if let Some(session) = self.session.as_mut() {
1756 session.rebind(endpoint);
1757 }
1758
1759 self.session.as_mut().ok_or_else(|| {
1760 Error::new(
1761 ErrorKind::Unexpected,
1762 "the VM resumed but this sandbox holds no session, which run() always builds"
1763 .to_string(),
1764 )
1765 })
1766 }
1767
1768 pub async fn terminate(&mut self, opts: TeardownOpts) -> TeardownReport {
1776 let mut report = TeardownReport::default();
1777 if let Err(error) = self.refuse_detached("terminate") {
1778 report.failures.push(error.to_string());
1781 report.lifecycle = Some(self.lifecycle);
1782 return report;
1783 }
1784 self.torn_down = true;
1785
1786 self.session = None;
1789
1790 if let Some(id) = self.microvm.as_ref().map(|vm| vm.id.clone()) {
1792 self.set_lifecycle(Lifecycle::Terminating);
1796 self.was_terminated = true;
1797
1798 match self.control.terminate(&id).await {
1799 Ok(()) => report.terminate_accepted = true,
1800 Err(error) => {
1801 report.failures.push(format!("terminate {id}: {error}"));
1802 report.undeleted.push(id.clone());
1803 }
1804 }
1805
1806 if report.terminate_accepted
1807 && let Some(timeout) = opts.wait_for_terminated
1808 {
1809 let wait = WaitOpts::for_lifecycle(timeout);
1810 match self
1811 .control
1812 .wait_for_state(&id, &["TERMINATED"], &[], wait)
1813 .await
1814 {
1815 Ok(settled) => {
1817 self.set_lifecycle(Lifecycle::Terminated);
1818 self.microvm = Some(settled);
1819 }
1820 Err(error) => report
1823 .failures
1824 .push(format!("waiting for {id} to reach TERMINATED: {error}")),
1825 }
1826 }
1827 }
1828
1829 if opts.delete_image {
1832 match self.image.as_ref().map(|image| image.identifier.clone()) {
1833 Some(identifier) => {
1834 let deleted = self
1835 .control
1836 .delete_image(&identifier, opts.delete_attempts, opts.delete_backoff)
1837 .await;
1838 report.image_deleted = Some(deleted);
1839 if deleted {
1840 self.image_exists = false;
1841 } else {
1842 report.failures.push(format!(
1843 "the image {identifier} survived {} delete attempts",
1844 opts.delete_attempts.max(1)
1845 ));
1846 report.undeleted.push(identifier);
1847 }
1848 }
1849 None => report.image_deleted = Some(false),
1850 }
1851 }
1852
1853 if opts.delete_log_group
1855 && let Some(group) = self.image.as_ref().map(Image::build_log_group)
1856 {
1857 report.failures.push(format!(
1858 "the build log group {group} was not deleted: CloudWatch Logs is not in this \
1859 crate's dependency set, so it is reported rather than removed. It is \
1860 service-created, which means no Terraform stack owns it and `terraform destroy` \
1861 leaves it behind — six accumulated before anyone noticed."
1862 ));
1863 report.undeleted.push(group);
1864 }
1865
1866 report.lifecycle = Some(self.lifecycle);
1867 report
1868 }
1869
1870 fn require_microvm(&self, what: &str) -> Result<String, Error> {
1874 self.microvm
1875 .as_ref()
1876 .map(|vm| vm.id.clone())
1877 .ok_or_else(|| {
1878 Error::new(
1879 ErrorKind::Precondition,
1880 format!("nothing to {what}: this sandbox has not launched a VM."),
1881 )
1882 })
1883 }
1884
1885 fn lifecycle_wait(&self) -> WaitOpts {
1887 WaitOpts::for_lifecycle(DEFAULT_LIFECYCLE_TIMEOUT)
1888 }
1889}
1890
1891impl Sandbox {
1900 fn drop_warning(&self) -> Option<String> {
1904 if self.torn_down || self.adopted || self.detached {
1905 return None;
1906 }
1907 let vm = self.microvm.as_ref()?;
1908 self.lifecycle.is_live().then(|| {
1909 format!(
1910 "warning: the Sandbox for microvm {id} was dropped in {state} without \
1911 terminate() or detach(). Nothing was torn down — Drop cannot await, so a \
1912 teardown here would deadlock inside a runtime. {state} is the state this \
1913 sandbox last saw, and the platform may have suspended or ended the VM since. A \
1914 VM that's still live bills until it ends, at its maximumDurationInSeconds \
1915 ceiling at the latest: terminate it with `microvm terminate {id}`.",
1916 id = vm.id,
1917 state = self.lifecycle,
1918 )
1919 })
1920 }
1921}
1922
1923impl Drop for Sandbox {
1924 fn drop(&mut self) {
1925 if let Some(warning) = self.drop_warning() {
1926 self.control.adapters().warn(&warning);
1927 }
1928 }
1929}
1930
1931fn mint_agent_token(entropy: &dyn crate::entropy::Entropy) -> Result<String, Error> {
1937 let mut bytes = [0u8; 32];
1938 entropy.fill(&mut bytes)?;
1939 Ok(const_hex::encode(bytes))
1940}
1941
1942#[cfg(test)]
1943mod tests {
1944 use super::*;
1945 use crate::control::fake::{self as fake, Answer, FakeControlPlane, TestClock};
1946 use crate::control::transport::Method;
1947
1948 fn planted() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
1954 let recorder = Arc::new(FakeControlPlane::new());
1955 let clock = Arc::new(TestClock::new());
1956 let plane = crate::testing::control_plane(
1957 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
1958 Region::UsEast1,
1959 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
1960 );
1961 let sandbox = Sandbox::with_control_plane(plane)
1963 .with_session_backend(crate::testing::HealthyDaemon::new());
1964 (sandbox, recorder, clock)
1965 }
1966
1967 #[test]
1970 fn the_suspend_resume_and_terminate_waits_are_cores_lifecycle_wait() {
1971 let (sandbox, _, _) = planted();
1972 let wait = sandbox.lifecycle_wait();
1973 assert_eq!(
1974 (wait.timeout, wait.poll_interval, wait.stall_grace),
1975 (
1976 DEFAULT_LIFECYCLE_TIMEOUT,
1977 LIFECYCLE_POLL_INTERVAL,
1978 Duration::MAX
1979 )
1980 );
1981 }
1982
1983 fn answer_launch(recorder: &FakeControlPlane) {
1985 recorder
1986 .answer(
1987 "RunMicrovm",
1988 Answer::ok(fake::microvm_response("PENDING", None)),
1989 )
1990 .answer(
1991 "GetMicrovm",
1992 Answer::ok(fake::microvm_response("RUNNING", None)),
1993 )
1994 .answer(
1995 "CreateMicrovmAuthToken",
1996 Answer::ok(fake::auth_token_response("proxy-token")),
1997 );
1998 }
1999
2000 #[tokio::test]
2001 async fn sandbox_launch_polls_at_one_second_then_backs_off() {
2002 for pending_polls in 0..=6 {
2003 let (mut sandbox, recorder, clock) = planted();
2004 recorder.answer(
2005 "RunMicrovm",
2006 Answer::ok(fake::microvm_response("PENDING", None)),
2007 );
2008 for _ in 0..pending_polls {
2009 recorder.answer(
2010 "GetMicrovm",
2011 Answer::ok(fake::microvm_response("PENDING", None)),
2012 );
2013 }
2014 recorder
2015 .answer(
2016 "GetMicrovm",
2017 Answer::ok(fake::microvm_response("RUNNING", None)),
2018 )
2019 .answer(
2020 "CreateMicrovmAuthToken",
2021 Answer::ok(fake::auth_token_response("proxy-token")),
2022 );
2023 sandbox
2024 .run(RunRequest::new().with_image("arn:image"))
2025 .await
2026 .expect("ready");
2027 let expected_seconds = [0, 1, 2, 3, 8, 13, 18][pending_polls];
2028 assert_eq!(clock.now(), Duration::from_secs(expected_seconds));
2029 assert_eq!(recorder.call_count("GetMicrovm"), pending_polls + 1);
2030 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2031 sandbox.detach().expect("hand the scripted VM off quietly");
2032 }
2033
2034 let (sandbox, recorder, clock) = planted();
2036 recorder
2037 .answer(
2038 "GetMicrovm",
2039 Answer::ok(fake::microvm_response("PENDING", None)),
2040 )
2041 .answer(
2042 "GetMicrovm",
2043 Answer::ok(fake::microvm_response("RUNNING", None)),
2044 );
2045 sandbox
2046 .control
2047 .wait_for_launch("mvm-abc123", WaitOpts::for_launch())
2048 .await
2049 .expect("ready");
2050 assert_eq!(clock.now(), Duration::from_secs(5));
2051
2052 let (mut sandbox, recorder, _) = planted();
2054 recorder
2055 .answer(
2056 "RunMicrovm",
2057 Answer::ok(fake::microvm_response("PENDING", None)),
2058 )
2059 .answer(
2060 "GetMicrovm",
2061 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2062 );
2063 let error = sandbox
2064 .run(RunRequest::new().with_image("arn:image"))
2065 .await
2066 .expect_err("a fresh launch fails fast");
2067 assert_eq!(error.kind(), ErrorKind::LaunchDied);
2068 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
2069 sandbox.detach().expect("hand the scripted VM off quietly");
2070 }
2071
2072 #[tokio::test]
2073 async fn vpc_egress_configuration_survives_the_sandbox_launch() {
2074 let (mut sandbox, recorder, _) = planted();
2075 answer_launch(&recorder);
2076 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2077 let request = RunRequest::new()
2078 .with_image("arn:image")
2079 .with_egress_network_connector(arn);
2080 assert_eq!(
2081 request.egress_posture(),
2082 crate::control::EgressPosture::Unsealed
2083 );
2084 sandbox.run(request).await.expect("launch");
2085 let body = recorder.first_body("RunMicrovm");
2086 assert_eq!(body["egressNetworkConnectors"], serde_json::json!([arn]));
2087 }
2088
2089 #[tokio::test]
2095 async fn a_sandbox_session_reads_the_planes_clock() {
2096 let (mut sandbox, recorder, clock) = planted();
2097 answer_launch(&recorder);
2098 let session = sandbox
2099 .run(RunRequest::new().with_image("arn:image"))
2100 .await
2101 .expect("the launch reaches RUNNING");
2102 let pinned = Duration::from_secs(1_767_225_600) + Duration::from_nanos(987_654_321);
2103 clock.set_unix_now(pinned);
2104 let id = session.mint_exec_id();
2105 let value =
2106 u64::from_str_radix(id.strip_prefix("x-").expect("the x- shape"), 16).expect("hex");
2107 let low_bits = (1_u64 << 40) - 1;
2108 assert_eq!(
2109 value & low_bits,
2110 (pinned.as_nanos() as u64) & low_bits,
2111 "{id} wasn't minted from the plane's clock"
2112 );
2113 }
2114
2115 #[tokio::test]
2122 async fn ensure_image_builds_its_services_through_the_planes_adapters() {
2123 let (mut sandbox, recorder, _) = planted();
2124 let dockerfile = crate::control::artifact::wrap_dockerfile(
2125 "FROM python:3.12-slim\nWORKDIR /app\n",
2126 &crate::control::artifact::WrapOptions::default(),
2127 )
2128 .expect("a wrappable Dockerfile");
2129 let request = crate::control::EnsureImageRequest::new(
2130 "task",
2131 crate::testing::aarch64_daemon(b"daemon"),
2132 dockerfile,
2133 "artifact-bucket",
2134 "arn:aws:iam::123456789012:role/build",
2135 );
2136 let error = sandbox
2137 .ensure_image(request)
2138 .await
2139 .expect_err("the test adapters build no services");
2140 assert_eq!(error.kind(), ErrorKind::Credentials, "{error}");
2141 assert!(
2142 error
2143 .to_string()
2144 .contains("the test adapters build no STS or S3 client for us-east-1"),
2145 "{error}"
2146 );
2147 assert!(
2148 recorder.calls().is_empty(),
2149 "no control-plane call came first"
2150 );
2151 }
2152
2153 #[tokio::test]
2161 async fn a_launch_draws_every_random_value_from_the_planes_entropy() {
2162 use crate::entropy::testing::SequenceEntropy;
2163 use base64::Engine as _;
2164
2165 let recorder = Arc::new(FakeControlPlane::new());
2166 let entropy = Arc::new(SequenceEntropy::new());
2167 let plane = crate::testing::control_plane(
2168 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2169 Region::UsEast1,
2170 Arc::new(TestClock::new()),
2171 )
2172 .with_entropy(Arc::clone(&entropy) as Arc<dyn crate::entropy::Entropy>);
2173 let mut sandbox = Sandbox::with_control_plane(plane)
2174 .with_session_backend(crate::testing::HealthyDaemon::new());
2175 answer_launch(&recorder);
2176 let session = sandbox
2177 .run(RunRequest::new().with_image("arn:image").with_identity())
2178 .await
2179 .expect("the launch reaches RUNNING");
2180
2181 assert_eq!(
2182 session.agent_token(),
2183 const_hex::encode(SequenceEntropy::draw(1, 32)),
2184 "the agent token"
2185 );
2186 let body = recorder.first_body("RunMicrovm");
2187 let payload: serde_json::Value =
2188 serde_json::from_str(body["runHookPayload"].as_str().expect("a string payload"))
2189 .expect("the payload is JSON");
2190 assert_eq!(
2191 payload[protocol::identity::SEED_KEY],
2192 base64::engine::general_purpose::STANDARD
2193 .encode(SequenceEntropy::draw(2, protocol::identity::SEED_BYTES)),
2194 "the VM seed"
2195 );
2196 let client_token = body["clientToken"].as_str().expect("a client token");
2197 assert!(
2198 client_token.ends_with(&const_hex::encode(SequenceEntropy::draw(4, 8))),
2199 "the client token's nonce: {client_token}"
2200 );
2201 assert_eq!(
2202 entropy.calls(),
2203 4,
2204 "one draw per value, the host seed included"
2205 );
2206 sandbox.detach().expect("hand the scripted VM off quietly");
2207 }
2208
2209 #[tokio::test]
2211 async fn an_unavailable_entropy_source_refuses_the_launch() {
2212 use crate::entropy::testing::SequenceEntropy;
2213
2214 let recorder = Arc::new(FakeControlPlane::new());
2215 let plane = crate::testing::control_plane(
2216 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2217 Region::UsEast1,
2218 Arc::new(TestClock::new()),
2219 )
2220 .with_entropy(Arc::new(SequenceEntropy::unavailable()));
2221 let mut sandbox = Sandbox::with_control_plane(plane)
2222 .with_session_backend(crate::testing::HealthyDaemon::new());
2223 answer_launch(&recorder);
2224 let err = sandbox
2225 .run(RunRequest::new().with_image("arn:image"))
2226 .await
2227 .expect_err("no pool, no bearer token");
2228 assert_eq!(err.kind(), ErrorKind::Unexpected, "{err}");
2229 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2230 }
2231
2232 fn launching_to(
2237 replies: Vec<crate::testing::Reply>,
2238 ) -> (
2239 Sandbox,
2240 Arc<FakeControlPlane>,
2241 Arc<crate::testing::Recorder>,
2242 ) {
2243 let recorder = Arc::new(FakeControlPlane::new());
2244 let plane = crate::testing::control_plane(
2245 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2246 Region::UsEast1,
2247 Arc::new(TestClock::new()),
2248 );
2249 let daemon = crate::testing::Recorder::with(replies);
2250 let sandbox = Sandbox::with_control_plane(plane).with_session_backend(daemon.clone());
2251 answer_launch(&recorder);
2252 (sandbox, recorder, daemon)
2253 }
2254
2255 fn not_ready_then_ready() -> Vec<crate::testing::Reply> {
2257 vec![
2258 crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()),
2259 crate::testing::Reply::ok(crate::testing::health_body(true)),
2260 ]
2261 }
2262
2263 fn daemon_paths(daemon: &crate::testing::Recorder) -> Vec<String> {
2265 daemon
2266 .requests()
2267 .iter()
2268 .map(|request| format!("{} {}", request.method, request.path))
2269 .collect()
2270 }
2271
2272 #[tokio::test(start_paused = true)]
2279 async fn a_launch_returns_once_its_daemon_answers() {
2280 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2281 sandbox
2282 .run(RunRequest::new().with_image("arn:image"))
2283 .await
2284 .expect("the launch reaches RUNNING and its daemon answers");
2285 assert_eq!(
2286 daemon_paths(&daemon),
2287 ["GET /v1/health", "GET /v1/health"],
2288 "one poll refused, one bootstrapped, and nothing after"
2289 );
2290 sandbox.detach().expect("hand the scripted VM off quietly");
2291 }
2292
2293 #[tokio::test(start_paused = true)]
2297 async fn wait_until_running_returns_once_the_daemon_answers() {
2298 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2299 let mut request = RunRequest::new().with_image("arn:image");
2300 request.wait = false;
2301 sandbox.run(request).await.expect("the launch is accepted");
2302 assert!(
2303 daemon_paths(&daemon).is_empty(),
2304 "an accepted launch doesn't wait"
2305 );
2306 sandbox
2307 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
2308 .await
2309 .expect("RUNNING, then the daemon answers");
2310 assert_eq!(daemon_paths(&daemon), ["GET /v1/health", "GET /v1/health"]);
2311 sandbox.detach().expect("hand the scripted VM off quietly");
2312 }
2313
2314 #[tokio::test(start_paused = true)]
2318 async fn a_launch_whose_daemon_never_answers_times_out_running() {
2319 let never = (0..128)
2320 .map(|_| crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()))
2321 .collect();
2322 let (mut sandbox, _recorder, daemon) = launching_to(never);
2323 let error = sandbox
2324 .run(RunRequest::new().with_image("arn:image"))
2325 .await
2326 .expect_err("the daemon never answered");
2327 assert_eq!(error.kind(), ErrorKind::Timeout, "{error}");
2328 assert!(daemon.requests().len() > 1, "it kept asking");
2329 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2330 assert!(sandbox.token_installed());
2331 sandbox.detach().expect("hand the scripted VM off quietly");
2332 }
2333
2334 const WANTED_ARN: &str = "arn:aws:lambda:us-east-1:123456789012:microvm-image:wanted-image";
2338
2339 fn answer_listing(recorder: &FakeControlPlane) {
2341 recorder.answer(
2342 "ListMicrovmImages",
2343 Answer::ok(fake::list_images_response(
2344 &["wanted-image-old", "wanted-image"],
2345 None,
2346 )),
2347 );
2348 }
2349
2350 #[tokio::test]
2357 async fn a_launch_by_bare_image_name_sends_the_listed_arn() {
2358 let (mut sandbox, recorder, _) = planted();
2359 answer_listing(&recorder);
2360 answer_launch(&recorder);
2361 sandbox
2362 .run(RunRequest::new().with_image("wanted-image"))
2363 .await
2364 .expect("the launch reaches RUNNING");
2365
2366 let operations = recorder.operations();
2367 assert_eq!(
2368 operations.get(..2),
2369 Some(&["ListMicrovmImages", "RunMicrovm"][..]),
2370 "the name is resolved, once, before the launch: {operations:?}"
2371 );
2372 assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2373 assert_eq!(
2374 recorder.first_body("RunMicrovm")["imageIdentifier"],
2375 WANTED_ARN,
2376 "the launch carries the exact match's ARN, never the name"
2377 );
2378 sandbox.detach().expect("hand the scripted VM off quietly");
2379 }
2380
2381 #[tokio::test]
2383 async fn a_launch_by_image_arn_makes_no_listing_call() {
2384 let (mut sandbox, recorder, _) = planted();
2385 answer_launch(&recorder);
2386 sandbox
2387 .run(RunRequest::new().with_image(WANTED_ARN))
2388 .await
2389 .expect("the launch reaches RUNNING");
2390
2391 assert_eq!(recorder.call_count("ListMicrovmImages"), 0);
2392 assert_eq!(recorder.operations().first(), Some(&"RunMicrovm"));
2393 assert_eq!(
2394 recorder.first_body("RunMicrovm")["imageIdentifier"],
2395 WANTED_ARN
2396 );
2397 assert_eq!(sandbox.launch_image_arn(), Some(WANTED_ARN));
2398 sandbox.detach().expect("hand the scripted VM off quietly");
2399 }
2400
2401 #[tokio::test]
2408 async fn a_refused_launch_by_name_still_records_the_resolved_arn() {
2409 let (mut sandbox, recorder, _) = planted();
2410 answer_listing(&recorder);
2411 recorder.answer("RunMicrovm", Answer::failure(400, "scripted refusal"));
2412 sandbox
2413 .run(RunRequest::new().with_image("wanted-image"))
2414 .await
2415 .expect_err("the service refused the launch");
2416
2417 assert_eq!(
2418 sandbox.launch_image_arn(),
2419 Some(WANTED_ARN),
2420 "the ARN the refused launch asked for"
2421 );
2422 assert!(sandbox.microvm().is_none(), "no launch was accepted");
2423 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2424 }
2425
2426 #[tokio::test]
2430 async fn an_unknown_image_name_is_refused_before_the_launch_and_the_sandbox_still_runs() {
2431 let (mut sandbox, recorder, _) = planted();
2432 recorder.answer(
2433 "ListMicrovmImages",
2434 Answer::ok(fake::list_images_response(&[], None)),
2435 );
2436 let error = sandbox
2437 .run(RunRequest::new().with_image("no-such-image"))
2438 .await
2439 .expect_err("nothing to launch from");
2440 assert_eq!(error.code(), "ERR_PRECONDITION");
2441 assert!(error.to_string().contains("no-such-image"), "{error}");
2442 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2443 assert_eq!(sandbox.launch_image_arn(), None);
2444 assert!(!sandbox.image_exists(), "STATE-1: no launch was accepted");
2445 assert!(sandbox.microvm().is_none());
2446 assert_eq!(sandbox.bootstrap_count(), 0);
2447
2448 answer_launch(&recorder);
2449 sandbox
2450 .run(RunRequest::new().with_image(WANTED_ARN))
2451 .await
2452 .expect("a failed resolution doesn't use up the sandbox's one launch");
2453 assert_eq!(sandbox.bootstrap_count(), 1);
2454 sandbox.detach().expect("hand the scripted VM off quietly");
2455 }
2456
2457 #[tokio::test]
2463 async fn a_launch_refused_locally_makes_no_listing_call_for_a_name() {
2464 let (mut sandbox, recorder, _) = planted();
2465 answer_listing(&recorder);
2466 let mut request = RunRequest::new().with_image("wanted-image");
2467 request.max_duration_sec = 0;
2468 let error = sandbox
2469 .run(request)
2470 .await
2471 .expect_err("a zero maximum duration is refused locally");
2472 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
2473 assert!(
2474 recorder.calls().is_empty(),
2475 "the refusal came after a call: {:?}",
2476 recorder.operations()
2477 );
2478 assert_eq!(sandbox.launch_image_arn(), None);
2479 }
2480
2481 #[tokio::test]
2488 async fn a_launched_session_reports_its_requests_posture() {
2489 use crate::control::EgressPosture;
2490 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2491 let rows = [
2492 (RunRequest::new(), EgressPosture::Unsealed),
2493 (RunRequest::new().with_egress(), EgressPosture::Open),
2494 (
2495 RunRequest::new().with_deny_egress(),
2496 EgressPosture::BestEffort,
2497 ),
2498 (
2499 RunRequest::new().with_egress_network_connector(arn),
2500 EgressPosture::Unsealed,
2501 ),
2502 ];
2503 for (request, expected) in rows {
2504 let answered = crate::control::egress_posture_for(
2505 request.egress,
2506 &request.egress_network_connectors,
2507 request.deny_egress,
2508 Some(&Region::UsEast1),
2509 )
2510 .expect("a launchable request");
2511 assert_eq!(answered, expected);
2512 let (mut sandbox, recorder, _) = planted();
2513 answer_launch(&recorder);
2514 let session = sandbox
2515 .run(request.with_image("arn:image"))
2516 .await
2517 .expect("the launch reaches RUNNING");
2518 assert_eq!(session.egress_posture(), expected, "BIND-12");
2519 assert_eq!(sandbox.egress_posture(), expected);
2520 sandbox.detach().expect("hand the scripted VM off quietly");
2521 }
2522 }
2523
2524 #[tokio::test]
2528 async fn a_session_without_its_launch_options_reports_unsealed() {
2529 use crate::control::EgressPosture;
2530 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2531 assert_eq!(
2532 sandbox.session().expect("RUNNING").egress_posture(),
2533 EgressPosture::Unsealed
2534 );
2535 assert_eq!(sandbox.egress_posture(), EgressPosture::Unsealed);
2536 let direct = Session::builder(ADOPT_ENDPOINT, ADOPT_TOKEN)
2537 .build_with(&crate::testing::TestAdapters::new())
2538 .expect("a direct session");
2539 assert_eq!(direct.egress_posture(), EgressPosture::Unsealed);
2540 }
2541
2542 async fn launched() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2544 let (mut sandbox, recorder, clock) = planted();
2545 answer_launch(&recorder);
2546 sandbox
2547 .run(RunRequest::new().with_image("arn:image"))
2548 .await
2549 .expect("the launch reaches RUNNING");
2550 (sandbox, recorder, clock)
2551 }
2552
2553 #[tokio::test]
2561 async fn a_dropped_live_sandbox_warns_through_the_planes_adapters() {
2562 let recorder = Arc::new(FakeControlPlane::new());
2563 let clock = Arc::new(TestClock::new());
2564 let adapters = Arc::new(crate::testing::TestAdapters::new());
2565 let plane = crate::control::ControlPlane::from_ports(
2566 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2567 Region::UsEast1,
2568 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2569 Arc::new(crate::testing::SequenceEntropy::new()),
2570 Arc::clone(&adapters) as Arc<dyn crate::adapters::Adapters>,
2571 );
2572 let mut sandbox = Sandbox::with_control_plane(plane)
2573 .with_session_backend(crate::testing::HealthyDaemon::new());
2574 answer_launch(&recorder);
2575 sandbox
2576 .run(RunRequest::new().with_image("arn:image"))
2577 .await
2578 .expect("the launch reaches RUNNING");
2579 let id = sandbox.microvm().expect("launched").id.clone();
2580 drop(sandbox);
2581 let warnings = adapters.warnings();
2582 assert_eq!(warnings.len(), 1, "{warnings:?}");
2583 assert!(warnings[0].contains(&id), "{warnings:?}");
2584 assert!(
2585 warnings[0].contains("RUNNING is the state this sandbox last saw")
2586 && warnings[0].contains("A VM that's still live bills"),
2587 "{warnings:?}"
2588 );
2589 }
2590
2591 #[tokio::test]
2601 async fn a_detached_sandbox_hands_off_its_vm_and_stays_quiet() {
2602 let (mut sandbox, recorder, _clock) = launched().await;
2603 assert!(
2604 sandbox.drop_warning().is_some(),
2605 "a launched, live VM warns on drop"
2606 );
2607 let token = sandbox
2608 .session()
2609 .expect("launched")
2610 .agent_token()
2611 .to_string();
2612 let vm = sandbox.microvm().expect("launched").clone();
2613
2614 let detached = sandbox.detach().expect("a RUNNING VM detaches");
2615 assert_eq!(detached.microvm_id, vm.id);
2616 assert_eq!(detached.endpoint, vm.endpoint);
2617 assert_eq!(detached.agent_token(), token);
2618 assert_eq!(detached.region, Region::UsEast1);
2619 let printed = format!("{detached:?} {sandbox:?}");
2620 assert!(!printed.contains(&token), "{printed}");
2621 assert!(printed.contains("detached: true"), "{printed}");
2622
2623 assert!(sandbox.detached());
2624 assert!(
2625 sandbox.drop_warning().is_none(),
2626 "detached is not abandoned"
2627 );
2628 assert!(
2629 sandbox.session().is_none(),
2630 "the session went to the adopter"
2631 );
2632
2633 let before = recorder.calls().len();
2634 for error in [
2635 sandbox.suspend().await.expect_err("detached"),
2636 sandbox.resume().await.map(|_| ()).expect_err("detached"),
2637 sandbox
2638 .run(RunRequest::new().with_image("arn:image"))
2639 .await
2640 .map(|_| ())
2641 .expect_err("detached"),
2642 sandbox
2643 .wait_until_running(Duration::from_secs(1))
2644 .await
2645 .map(|_| ())
2646 .expect_err("detached"),
2647 sandbox.detach().map(|_| ()).expect_err("already detached"),
2648 ] {
2649 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2650 assert!(error.to_string().contains("detached"), "{error}");
2651 assert!(!error.to_string().contains(&token), "{error}");
2652 }
2653 let report = sandbox.terminate(TeardownOpts::default()).await;
2654 assert!(!report.terminate_accepted);
2655 assert!(report.failures[0].contains("detached"), "{report:?}");
2656 assert_eq!(recorder.calls().len(), before, "detached makes no AWS call");
2657 assert!(
2658 sandbox.drop_warning().is_none(),
2659 "a refused terminate is still quiet"
2660 );
2661 }
2662
2663 #[tokio::test]
2665 async fn detach_refuses_without_a_live_vm() {
2666 let (mut unlaunched, _recorder, _clock) = planted();
2667 let error = unlaunched.detach().expect_err("nothing launched");
2668 assert_eq!(error.kind(), ErrorKind::Precondition);
2669 assert!(!unlaunched.detached());
2670
2671 let (mut sandbox, recorder, _clock) = launched().await;
2672 recorder
2673 .answer("TerminateMicrovm", Answer::ok("{}"))
2674 .answer(
2675 "GetMicrovm",
2676 Answer::ok(fake::microvm_response("TERMINATED", None)),
2677 );
2678 sandbox.terminate(TeardownOpts::default()).await;
2679 let error = sandbox.detach().expect_err("torn down");
2680 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2681 assert!(!sandbox.detached());
2682 }
2683
2684 async fn suspended_with_window(
2695 suspended_sec: u32,
2696 ) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2697 let (mut sandbox, recorder, clock) = planted();
2698 recorder
2699 .answer(
2700 "RunMicrovm",
2701 Answer::ok(fake::microvm_response("PENDING", None)),
2702 )
2703 .answer(
2704 "GetMicrovm",
2705 Answer::ok(fake::microvm_response("RUNNING", None)),
2706 )
2707 .answer(
2708 "GetMicrovm",
2709 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2710 )
2711 .answer(
2712 "CreateMicrovmAuthToken",
2713 Answer::ok(fake::auth_token_response("proxy-token")),
2714 )
2715 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2716
2717 sandbox
2718 .run(
2719 RunRequest::new()
2720 .with_image("arn:image")
2721 .with_suspended_sec(suspended_sec),
2722 )
2723 .await
2724 .expect("launches");
2725 sandbox.suspend().await.expect("suspends");
2726 assert_eq!(
2727 clock.now(),
2728 Duration::ZERO,
2729 "no wait may have slept, or a test that reads the clock is measuring the fake's \
2730 poll interval as well as the elapsed suspension"
2731 );
2732 (sandbox, recorder, clock)
2733 }
2734
2735 async fn suspended() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2737 suspended_with_window(600).await
2738 }
2739
2740 const ADOPT_TOKEN: &str = "adopt-canary-token-9f1c";
2744 const ADOPT_ENDPOINT: &str = "https://mvm-abc123.microvm.us-east-1.amazonaws.com";
2745
2746 fn adopt_plane() -> (ControlPlane, Arc<FakeControlPlane>, Arc<TestClock>) {
2748 let recorder = Arc::new(FakeControlPlane::new());
2749 let clock = Arc::new(TestClock::new());
2750 let plane = crate::testing::control_plane(
2751 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2752 Region::UsEast1,
2753 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2754 );
2755 (plane, recorder, clock)
2756 }
2757
2758 async fn adopted_in(state: &str) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2759 let (plane, recorder, clock) = adopt_plane();
2760 recorder.answer(
2761 "GetMicrovm",
2762 Answer::ok(fake::microvm_response(state, None)),
2763 );
2764 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2765 .await
2766 .expect("adopts");
2767 (sandbox, recorder, clock)
2768 }
2769
2770 #[tokio::test]
2777 async fn an_adopted_sandbox_reports_the_services_idle_window_and_watches_its_lifecycle() {
2778 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2779 assert_eq!(sandbox.idle_window(), Some(Duration::from_secs(1800)));
2780 assert_eq!(*sandbox.watch_lifecycle().borrow(), Lifecycle::Running);
2781 }
2782
2783 #[test]
2785 fn every_service_state_maps_onto_one_lifecycle() {
2786 for state in crate::constants::MICROVM_STATES {
2787 let lifecycle = Lifecycle::from_service(state).expect("a known state");
2788 assert_eq!(lifecycle.as_str(), state);
2789 }
2790 assert_eq!(Lifecycle::from_service("Running"), None);
2791 }
2792
2793 #[tokio::test]
2801 async fn an_adopted_vm_takes_its_lifecycle_from_the_service_and_refuses_run() {
2802 let (mut sandbox, recorder, _) = adopted_in("RUNNING").await;
2803 assert!(sandbox.adopted());
2804 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2805 assert!(sandbox.token_installed());
2806 assert_eq!(sandbox.bootstrap_count(), 1);
2807 assert!(sandbox.image_exists());
2808 let session = sandbox.session().expect("a live VM gets a session");
2809 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
2810 assert_eq!(session.agent_token(), ADOPT_TOKEN);
2811
2812 let error = sandbox
2813 .run(RunRequest::new().with_image("arn:image"))
2814 .await
2815 .expect_err("an adopted VM was bootstrapped by its own launch");
2816 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2817 assert!(error.to_string().contains("STATE-3"), "{error}");
2818 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2819 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
2820 }
2821
2822 #[tokio::test]
2828 async fn a_name_adopts_its_records_vm_and_foreign_names_are_refused_locally() {
2829 struct One(crate::names::NameRecord);
2830 impl crate::names::NameStore for One {
2831 fn get(&self, name: &str) -> Result<Option<crate::names::NameRecord>, Error> {
2832 Ok((name == self.0.name).then(|| self.0.clone()))
2833 }
2834 fn put(&self, _: &crate::names::NameRecord) -> Result<(), Error> {
2835 unreachable!()
2836 }
2837 fn delete(&self, _: &str) -> Result<bool, Error> {
2838 unreachable!()
2839 }
2840 fn list(&self) -> Result<Vec<crate::names::NameRecord>, Error> {
2841 Ok(vec![self.0.clone()])
2842 }
2843 }
2844 let record = crate::names::NameRecord::new_at(
2845 "ci",
2846 "mvm-abc123",
2847 ADOPT_ENDPOINT,
2848 ADOPT_TOKEN,
2849 "us-west-2",
2850 1_767_225_600,
2851 )
2852 .expect("record");
2853 let store = One(record.clone());
2854 let foreign = crate::names::resolve(&store, "ci", Some(&Region::UsEast1))
2855 .expect_err("registered in another region");
2856 assert_eq!(foreign.kind(), ErrorKind::InvalidArg);
2857 let missing = crate::names::resolve(&store, "nope", None).expect_err("no such name");
2858 assert_eq!(missing.kind(), ErrorKind::Precondition);
2859 assert!(!format!("{foreign} {missing}").contains(ADOPT_TOKEN));
2860
2861 let (plane, recorder, _) = adopt_plane();
2862 recorder.answer(
2863 "GetMicrovm",
2864 Answer::ok(fake::microvm_response("RUNNING", None)),
2865 );
2866 let resolved = crate::names::resolve(&store, "ci", Some(&Region::UsWest2)).expect("ok");
2867 let sandbox = Sandbox::adopt_record(plane, resolved)
2868 .await
2869 .expect("adopts");
2870 assert!(sandbox.adopted());
2871 assert_eq!(
2872 sandbox.microvm().map(|vm| vm.id.as_str()),
2873 Some("mvm-abc123")
2874 );
2875 assert_eq!(
2876 sandbox.session().expect("session").agent_token(),
2877 ADOPT_TOKEN
2878 );
2879 let named = sandbox
2880 .name_record("again")
2881 .expect("an adopted VM can be named");
2882 assert_eq!(
2883 (
2884 named.microvm_id.as_str(),
2885 named.agent_token.as_str(),
2886 named.region.as_str()
2887 ),
2888 ("mvm-abc123", ADOPT_TOKEN, "us-east-1")
2889 );
2890 let unlaunched = Sandbox::with_control_plane(adopt_plane().0);
2891 assert_eq!(
2892 unlaunched
2893 .name_record("x")
2894 .expect_err("nothing to name")
2895 .kind(),
2896 ErrorKind::Precondition
2897 );
2898 }
2899
2900 #[tokio::test]
2906 async fn a_records_tunnel_identity_survives_adoption_and_naming() {
2907 let identity = crate::identity::LaunchIdentity::from_seeds([7; 32], [9; 32])
2908 .expect("valid seeds")
2909 .keep();
2910 let mut record = crate::names::NameRecord::new_at(
2911 "ci",
2912 "mvm-abc123",
2913 ADOPT_ENDPOINT,
2914 ADOPT_TOKEN,
2915 "us-east-1",
2916 1_767_225_600,
2917 )
2918 .expect("record");
2919 record.identity_host_seed = Some(identity.host_seed_base64());
2920 record.identity_vm_public_key = Some(identity.vm_public_key_base64());
2921
2922 let (plane, recorder, _) = adopt_plane();
2923 recorder.answer(
2924 "GetMicrovm",
2925 Answer::ok(fake::microvm_response("RUNNING", None)),
2926 );
2927 let sandbox = Sandbox::adopt_record(plane, record.clone())
2928 .await
2929 .expect("adopts");
2930 let kept = sandbox
2931 .tunnel_identity()
2932 .expect("the record's pair is kept");
2933 assert_eq!(kept.host_seed_base64(), identity.host_seed_base64());
2934 assert_eq!(kept.vm_public_key_base64(), identity.vm_public_key_base64());
2935 let named = sandbox.name_record("again").expect("names");
2936 assert_eq!(named.identity_host_seed, record.identity_host_seed);
2937 assert_eq!(named.identity_vm_public_key, record.identity_vm_public_key);
2938
2939 let (plane, recorder, _) = adopt_plane();
2940 let mut torn = record;
2941 torn.identity_vm_public_key = None;
2942 let refused = Sandbox::adopt_record(plane, torn)
2943 .await
2944 .expect_err("half a pair is refused");
2945 assert_eq!(refused.kind(), ErrorKind::InvalidArg);
2946 assert!(refused.to_string().contains("one half"), "{refused}");
2947 assert!(recorder.calls().is_empty(), "refused before any AWS call");
2948 }
2949
2950 #[tokio::test]
2953 async fn an_adopted_vm_suspends_resumes_and_terminates_through_its_handle() {
2954 let (plane, recorder, _) = adopt_plane();
2955 recorder
2956 .answer(
2957 "GetMicrovm",
2958 Answer::ok(fake::microvm_response("RUNNING", None)),
2959 )
2960 .answer(
2961 "GetMicrovm",
2962 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2963 )
2964 .answer(
2965 "GetMicrovm",
2966 Answer::ok(fake::microvm_response("RUNNING", None)),
2967 )
2968 .answer(
2969 "GetMicrovm",
2970 Answer::ok(fake::microvm_response("TERMINATED", None)),
2971 )
2972 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
2973 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
2974 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
2975 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2976 .await
2977 .expect("adopts");
2978
2979 sandbox.suspend().await.expect("suspends from RUNNING");
2980 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
2981 sandbox.resume().await.expect("resumes from SUSPENDED");
2982 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2983 let report = sandbox
2984 .terminate(TeardownOpts::default().waiting_for_terminated())
2985 .await;
2986 assert!(report.terminate_accepted, "{report:?}");
2987 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
2988 assert!(sandbox.was_terminated());
2989
2990 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2991 assert!(
2992 recorder
2993 .bodies_as_text()
2994 .iter()
2995 .all(|body| !body.contains("runHookPayload") && !body.contains(ADOPT_TOKEN)),
2996 "no launch payload or agent token may reach the control plane"
2997 );
2998 assert_eq!(sandbox.bootstrap_count(), 1, "STATE-3: never above one");
2999 }
3000
3001 #[tokio::test]
3005 async fn an_adopted_vm_resumes_past_the_reported_window_through_the_service() {
3006 let (plane, recorder, clock) = adopt_plane();
3007 recorder
3008 .answer(
3009 "GetMicrovm",
3010 Answer::ok(fake::microvm_response("RUNNING", None)),
3011 )
3012 .answer(
3013 "GetMicrovm",
3014 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3015 )
3016 .answer(
3017 "GetMicrovm",
3018 Answer::ok(fake::microvm_response("RUNNING", None)),
3019 )
3020 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3021 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3022 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3023 .await
3024 .expect("adopts");
3025 assert_eq!(sandbox.suspended_window(), None, "no request of its own");
3026 sandbox.suspend().await.expect("suspends");
3027
3028 clock.advance(Duration::from_secs(601));
3029 sandbox
3030 .resume()
3031 .await
3032 .expect("past the 600 s window the service reported, the service resumes it");
3033 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3034 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3035 }
3036
3037 #[tokio::test]
3040 async fn an_adopted_suspended_vm_resumes_through_the_service() {
3041 let (plane, recorder, _) = adopt_plane();
3042 recorder
3043 .answer(
3044 "GetMicrovm",
3045 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3046 )
3047 .answer(
3048 "GetMicrovm",
3049 Answer::ok(fake::microvm_response("RUNNING", None)),
3050 )
3051 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3052 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3053 .await
3054 .expect("adopts");
3055 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3056 let session = sandbox
3057 .resume()
3058 .await
3059 .expect("the service answers the resume");
3060 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
3061 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3062 }
3063
3064 #[tokio::test]
3067 async fn an_adopted_terminated_vm_refuses_every_transition_locally() {
3068 let (mut sandbox, recorder, _) = adopted_in("TERMINATED").await;
3069 assert!(sandbox.was_terminated());
3070 assert!(sandbox.session().is_none());
3071 let resume = sandbox.resume().await.expect_err("STATE-11");
3072 assert!(resume.to_string().contains("STATE-11"), "{resume}");
3073 let suspend = sandbox.suspend().await.expect_err("STATE-5");
3074 assert!(suspend.to_string().contains("STATE-5"), "{suspend}");
3075 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
3076 }
3077
3078 async fn adopted_with_a_daemon(
3081 state: &str,
3082 ) -> (
3083 Sandbox,
3084 Arc<FakeControlPlane>,
3085 Arc<crate::testing::HealthyDaemon>,
3086 ) {
3087 let recorder = Arc::new(FakeControlPlane::new());
3088 let daemon = crate::testing::HealthyDaemon::new();
3089 let plane = crate::control::ControlPlane::from_ports(
3090 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3091 Region::UsEast1,
3092 Arc::new(TestClock::new()),
3093 Arc::new(crate::testing::SequenceEntropy::new()),
3094 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3095 );
3096 recorder
3097 .answer(
3098 "GetMicrovm",
3099 Answer::ok(fake::microvm_response(state, None)),
3100 )
3101 .answer(
3102 "GetMicrovm",
3103 Answer::ok(fake::microvm_response("RUNNING", None)),
3104 )
3105 .answer(
3106 "CreateMicrovmAuthToken",
3107 Answer::ok(fake::auth_token_response("proxy-token")),
3108 );
3109 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3110 .await
3111 .expect("adopts");
3112 (sandbox, recorder, daemon)
3113 }
3114
3115 #[tokio::test]
3122 async fn wait_until_ready_finishes_whichever_wait_is_left() {
3123 let (mut pending, recorder, daemon) = adopted_with_a_daemon("PENDING").await;
3124 pending
3125 .wait_until_ready(Duration::from_secs(60))
3126 .await
3127 .expect("RUNNING, then the daemon");
3128 assert_eq!(pending.lifecycle(), Lifecycle::Running);
3129 assert_eq!(pending.bootstrap_count(), 1);
3130 assert_eq!(daemon.polls(), 1);
3131 assert_eq!(
3132 recorder.call_count("GetMicrovm"),
3133 2,
3134 "the adopt's read and the wait's"
3135 );
3136
3137 let (mut running, recorder, daemon) = adopted_with_a_daemon("RUNNING").await;
3138 running
3139 .wait_until_ready(Duration::from_secs(60))
3140 .await
3141 .expect("the daemon answers");
3142 assert_eq!(daemon.polls(), 1);
3143 assert_eq!(
3144 recorder.call_count("GetMicrovm"),
3145 1,
3146 "RUNNING needs no second read"
3147 );
3148 }
3149
3150 #[tokio::test]
3152 async fn wait_until_ready_refuses_a_suspended_vm() {
3153 let (mut suspended, _, daemon) = adopted_with_a_daemon("SUSPENDED").await;
3154 let error = suspended
3155 .wait_until_ready(Duration::from_secs(60))
3156 .await
3157 .expect_err("a suspended VM isn't starting");
3158 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
3159 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3160 assert_eq!(daemon.polls(), 0);
3161 }
3162
3163 #[tokio::test]
3166 async fn an_adopted_pending_vm_counts_its_bootstrap_once_on_running() {
3167 let recorder = Arc::new(FakeControlPlane::new());
3168 let daemon = crate::testing::HealthyDaemon::new();
3169 let plane = crate::control::ControlPlane::from_ports(
3170 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3171 Region::UsEast1,
3172 Arc::new(TestClock::new()),
3173 Arc::new(crate::testing::SequenceEntropy::new()),
3174 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3175 );
3176 recorder
3177 .answer(
3178 "GetMicrovm",
3179 Answer::ok(fake::microvm_response("PENDING", None)),
3180 )
3181 .answer(
3182 "GetMicrovm",
3183 Answer::ok(fake::microvm_response("RUNNING", None)),
3184 )
3185 .answer(
3186 "CreateMicrovmAuthToken",
3187 Answer::ok(fake::auth_token_response("proxy-token")),
3188 );
3189 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3190 .await
3191 .expect("adopts");
3192 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3193 assert_eq!(sandbox.bootstrap_count(), 0);
3194 sandbox
3195 .wait_until_running(Duration::from_secs(60))
3196 .await
3197 .expect("reaches RUNNING");
3198 assert_eq!(sandbox.bootstrap_count(), 1);
3199 assert!(sandbox.token_installed());
3200 assert_eq!(
3201 daemon.polls(),
3202 1,
3203 "the daemon answered before the session came back"
3204 );
3205 assert!(
3206 sandbox
3207 .run(RunRequest::new().with_image("arn:image"))
3208 .await
3209 .is_err(),
3210 "still one bootstrap per VM"
3211 );
3212 assert_eq!(recorder.call_count("RunMicrovm"), 0);
3213 }
3214
3215 #[tokio::test]
3222 async fn adopt_refuses_bad_records_and_never_prints_the_token() {
3223 let (plane, recorder, _) = adopt_plane();
3224 let empty = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, "")
3225 .await
3226 .expect_err("no token");
3227 assert_eq!(empty.kind(), ErrorKind::InvalidArg);
3228 assert_eq!(recorder.calls().len(), 0, "refused before any call");
3229
3230 let (plane, recorder, _) = adopt_plane();
3231 recorder.answer(
3232 "GetMicrovm",
3233 Answer::ok(fake::microvm_response("RUNNING", None)),
3234 );
3235 let mismatch = Sandbox::adopt(plane, "mvm-abc123", "https://other.example", ADOPT_TOKEN)
3236 .await
3237 .expect_err("the endpoint belongs to another record");
3238 assert_eq!(mismatch.kind(), ErrorKind::InvalidArg);
3239 assert!(!mismatch.to_string().contains(ADOPT_TOKEN));
3240
3241 let (plane, recorder, _) = adopt_plane();
3242 recorder.answer(
3243 "GetMicrovm",
3244 Answer::ok(fake::microvm_response("REBOOTING", None)),
3245 );
3246 let unknown = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3247 .await
3248 .expect_err("an unknown state");
3249 assert_eq!(unknown.kind(), ErrorKind::Platform);
3250 assert!(!unknown.to_string().contains(ADOPT_TOKEN));
3251
3252 let (sandbox, _, _) = adopted_in("RUNNING").await;
3253 let printed = format!("{sandbox:?}");
3254 assert!(!printed.contains(ADOPT_TOKEN), "{printed}");
3255 assert!(printed.contains("adopted: true"), "{printed}");
3256 }
3257
3258 #[tokio::test]
3268 async fn a_launch_records_the_image_and_the_running_report_installs_the_token() {
3269 let (mut sandbox, recorder, _) = planted();
3270 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3271 assert!(
3272 !sandbox.image_exists(),
3273 "nothing is recorded before a launch"
3274 );
3275 assert!(!sandbox.token_installed());
3276
3277 answer_launch(&recorder);
3278 let session = sandbox
3279 .run(RunRequest::new().with_image("arn:image"))
3280 .await
3281 .expect("launches");
3282 assert_eq!(
3283 session.endpoint(),
3284 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
3285 );
3286
3287 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3288 assert!(sandbox.image_exists(), "STATE-1: the image is recorded");
3289 assert!(sandbox.token_installed(), "STATE-2: the token is installed");
3290 assert_eq!(sandbox.bootstrap_count(), 1);
3291 assert_eq!(sandbox.microvm().expect("launched").id, "mvm-abc123");
3292
3293 let body = recorder.first_body("RunMicrovm");
3296 let payload = body["runHookPayload"].as_str().expect("a string");
3297 assert!(payload.starts_with(r#"{"agent_token":"#), "{payload}");
3298 assert_eq!(body["idlePolicy"]["suspendedDurationSeconds"], 600);
3299 }
3300
3301 #[tokio::test]
3308 async fn a_launch_env_reaches_the_run_hook_payload_on_the_wire() {
3309 let (mut sandbox, recorder, _) = planted();
3310 answer_launch(&recorder);
3311 sandbox
3312 .run(
3313 RunRequest::new()
3314 .with_image("arn:image")
3315 .with_launch_env("ANTHROPIC_BASE_URL", "https://gateway.example")
3316 .with_launch_env("PATH", "/usr/local/bin:/usr/bin:/bin"),
3317 )
3318 .await
3319 .expect("launches");
3320
3321 let body = recorder.first_body("RunMicrovm");
3322 let payload = body["runHookPayload"].as_str().expect("a string");
3323 let inner: serde_json::Value =
3325 serde_json::from_str(payload).expect("the payload is itself JSON");
3326 assert!(
3327 inner["agent_token"].as_str().is_some_and(|t| !t.is_empty()),
3328 "the token still rides alongside: {payload}"
3329 );
3330 assert_eq!(
3331 inner["env"]["ANTHROPIC_BASE_URL"],
3332 "https://gateway.example"
3333 );
3334 assert_eq!(inner["env"]["PATH"], "/usr/local/bin:/usr/bin:/bin");
3335 }
3336
3337 #[tokio::test]
3350 async fn the_advisory_deny_reaches_the_launch_environment_in_both_spellings() {
3351 let (mut sandbox, recorder, _) = planted();
3352 answer_launch(&recorder);
3353 let request = RunRequest::new().with_image("arn:image").with_deny_egress();
3354 assert_eq!(
3355 request.egress_posture(),
3356 crate::control::EgressPosture::BestEffort
3357 );
3358 sandbox.run(request).await.expect("launches");
3359
3360 let body = recorder.first_body("RunMicrovm");
3361 let inner: serde_json::Value =
3362 serde_json::from_str(body["runHookPayload"].as_str().expect("a string"))
3363 .expect("the payload is itself JSON");
3364 for key in crate::sandbox::DENY_EGRESS_ENV_KEYS {
3365 assert_eq!(
3366 inner["env"][key], DENY_EGRESS_PROXY_URL,
3367 "{key} must point at the black hole: {inner}"
3368 );
3369 }
3370 assert!(
3371 body.get("egressNetworkConnectors").is_none(),
3372 "the advisory deny asks the platform for nothing: {body}"
3373 );
3374 }
3375
3376 #[test]
3379 fn the_deny_never_overwrites_a_callers_own_proxy_and_is_absent_by_default() {
3380 let kept = RunRequest::new()
3381 .with_deny_egress()
3382 .with_launch_env("https_proxy", "http://proxy.internal:3128")
3383 .effective_launch_env();
3384 assert_eq!(kept["https_proxy"], "http://proxy.internal:3128");
3385 assert_eq!(kept["http_proxy"], DENY_EGRESS_PROXY_URL);
3386
3387 let plain = RunRequest::new().effective_launch_env();
3388 assert!(
3389 plain.is_empty(),
3390 "a launch that did not ask for the deny sends byte-for-byte what it always sent: \
3391 {plain:?}"
3392 );
3393 assert_eq!(
3394 RunRequest::new().egress_posture(),
3395 crate::control::EgressPosture::Unsealed,
3396 "and its posture is the measured one, not a seal"
3397 );
3398 }
3399
3400 #[tokio::test]
3409 async fn asking_for_egress_and_the_deny_together_is_refused_before_any_call() {
3410 let (mut sandbox, recorder, _) = planted();
3411 answer_launch(&recorder);
3412 let mut request = RunRequest::new().with_image("arn:image").with_egress();
3413 request.deny_egress = true;
3414
3415 let error = sandbox
3416 .run(request)
3417 .await
3418 .expect_err("opposite intents are not resolved by a precedence rule");
3419 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3420 assert!(error.to_string().contains("opposite things"), "{error}");
3421 assert_eq!(
3422 recorder.calls().len(),
3423 0,
3424 "a locally-refused launch costs nothing"
3425 );
3426 }
3427
3428 #[tokio::test]
3439 async fn a_pinned_image_version_is_forwarded_to_the_launch_request() {
3440 let (mut sandbox, recorder, _) = planted();
3441 answer_launch(&recorder);
3442 sandbox
3443 .run(
3444 RunRequest::new()
3445 .with_image("arn:image")
3446 .with_image_version("2.0"),
3447 )
3448 .await
3449 .expect("launches");
3450 assert_eq!(
3451 recorder.first_body("RunMicrovm")["imageVersion"],
3452 "2.0",
3453 "a canary has to launch against the version it means to test"
3454 );
3455
3456 let (mut sandbox, recorder, _) = planted();
3457 answer_launch(&recorder);
3458 sandbox
3459 .run(RunRequest::new().with_image("arn:image"))
3460 .await
3461 .expect("launches");
3462 assert!(
3463 recorder
3464 .first_body("RunMicrovm")
3465 .get("imageVersion")
3466 .is_none(),
3467 "an unpinned launch sends what this client always sent"
3468 );
3469 }
3470
3471 #[tokio::test]
3482 async fn an_over_budget_launch_env_is_refused_before_any_control_plane_call() {
3483 let (mut sandbox, recorder, _) = planted();
3484 answer_launch(&recorder);
3485
3486 let error = sandbox
3487 .run(
3488 RunRequest::new()
3489 .with_image("arn:image")
3490 .with_launch_env("AWS_SESSION_TOKEN", "t".repeat(4096)),
3491 )
3492 .await
3493 .expect_err("a credential-scale launch env does not fit the payload");
3494
3495 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3496 let message = error.to_string();
3497 assert!(message.contains("ceiling of 4096"), "{message}");
3498 assert!(message.contains("launch env contributed"), "{message}");
3499 assert_eq!(
3500 recorder.calls().len(),
3501 0,
3502 "the refusal has to be local: a launch that reached AWS costs a VM and a bill"
3503 );
3504 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3507 assert_eq!(sandbox.bootstrap_count(), 0);
3508 assert!(sandbox.microvm().is_none());
3509 }
3510
3511 #[tokio::test]
3529 async fn the_bindings_layer_refuses_an_illegal_launch_with_zero_calls() {
3530 let cases: [(&str, RunRequest, &str); 3] = [
3531 (
3532 "max_idle_sec=59, which is what a Python keyword argument passes",
3533 RunRequest {
3534 max_idle_sec: 59,
3535 ..RunRequest::new().with_image("arn:image")
3536 },
3537 "maxIdleDurationSeconds",
3538 ),
3539 (
3540 "an execution role that is a bare name",
3541 RunRequest {
3542 execution_role_arn: Some("execution-role".to_string()),
3543 ..RunRequest::new().with_image("arn:image")
3544 },
3545 "role *name*",
3546 ),
3547 (
3548 "a pinned version with a trailing newline",
3549 RunRequest::new()
3550 .with_image("arn:image")
3551 .with_image_version("2.0\n"),
3552 "contains whitespace",
3553 ),
3554 ];
3555
3556 for (label, request, expected) in cases {
3557 let (mut sandbox, recorder, _) = planted();
3558 answer_launch(&recorder);
3561
3562 let Err(error) = sandbox.run(request).await else {
3566 panic!("{label} must be refused");
3567 };
3568 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
3569 assert!(
3570 error.to_string().contains(expected),
3571 "{label}: wanted {expected:?}, got {error}"
3572 );
3573 assert_eq!(
3574 recorder.calls().len(),
3575 0,
3576 "{label}: the refusal has to be local, because a launch that reached AWS costs a \
3577 VM and a bill"
3578 );
3579 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3580 assert!(sandbox.microvm().is_none());
3581 }
3582 }
3583
3584 #[tokio::test]
3591 async fn a_launch_that_dies_during_startup_installs_no_token() {
3592 let (mut sandbox, recorder, _) = planted();
3593 recorder
3594 .answer(
3595 "RunMicrovm",
3596 Answer::ok(fake::microvm_response("PENDING", None)),
3597 )
3598 .answer(
3599 "GetMicrovm",
3600 Answer::ok(fake::microvm_response(
3601 "TERMINATED",
3602 Some("run hook returned 500"),
3603 )),
3604 );
3605
3606 let error = sandbox
3607 .run(RunRequest::new().with_image("arn:image"))
3608 .await
3609 .expect_err("a VM that died during startup is not a launch");
3610 assert_eq!(error.kind(), ErrorKind::LaunchDied);
3611 assert!(
3612 error.to_string().contains("run hook returned 500"),
3613 "{error}"
3614 );
3615
3616 assert!(
3617 !sandbox.token_installed(),
3618 "the run hook never answered, so nothing was installed"
3619 );
3620 assert_eq!(
3621 sandbox.bootstrap_count(),
3622 0,
3623 "a bootstrap counted here would make STATE-3's ceiling unreachable for a retry"
3624 );
3625 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3626 assert!(
3627 sandbox.image_exists(),
3628 "STATE-1 still holds: the launch was accepted against a real image"
3629 );
3630 }
3631
3632 #[tokio::test]
3643 async fn a_second_run_on_one_sandbox_is_refused_before_any_call() {
3644 let (mut sandbox, recorder, _) = launched().await;
3645 assert_eq!(sandbox.bootstrap_count(), 1);
3646 let before = recorder.calls().len();
3647
3648 let error = sandbox
3649 .run(RunRequest::new().with_image("arn:image"))
3650 .await
3651 .expect_err("the token is installed at most once per VM lifetime");
3652 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3653 assert!(error.to_string().contains("STATE-3"), "{error}");
3654 assert!(
3655 error.to_string().contains("needs a second Sandbox"),
3656 "the remedy has to be nameable: {error}"
3657 );
3658
3659 assert_eq!(
3660 sandbox.bootstrap_count(),
3661 1,
3662 "bootstrap_count <= 1 is the Z3-proved invariant"
3663 );
3664 assert_eq!(
3665 recorder.calls().len(),
3666 before,
3667 "the refusal must cost no control-plane call"
3668 );
3669 assert_eq!(recorder.call_count("RunMicrovm"), 1);
3670 }
3671
3672 #[tokio::test]
3675 async fn a_suspend_from_running_passes_through_suspending_to_suspended() {
3676 let (sandbox, recorder, _) = suspended().await;
3677 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3678 assert!(
3679 sandbox.token_installed(),
3680 "a freeze keeps the guest's memory, so the token survives"
3681 );
3682 assert_eq!(sandbox.bootstrap_count(), 1, "no re-bootstrap on a freeze");
3683
3684 let calls = recorder.calls();
3685 let suspend = calls
3686 .iter()
3687 .find(|call| call.operation == "SuspendMicrovm")
3688 .expect("the suspend went out");
3689 assert_eq!(suspend.method, Method::Post);
3690 assert_eq!(suspend.path, "/2025-09-09/microvms/mvm-abc123/suspend");
3691 }
3692
3693 #[tokio::test]
3699 async fn a_vm_that_dies_while_suspending_is_recorded_rather_than_raised() {
3700 let (mut sandbox, recorder, _) = launched().await;
3701 recorder
3702 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3703 .answer(
3704 "GetMicrovm",
3705 Answer::ok(fake::microvm_response("TERMINATED", Some("idle policy"))),
3706 );
3707
3708 sandbox
3709 .suspend()
3710 .await
3711 .expect("a death mid-suspend is a state, not an error");
3712 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3713 assert!(
3714 sandbox.was_terminated(),
3715 "STATE-11's precondition is recorded"
3716 );
3717 }
3718
3719 #[tokio::test]
3728 async fn a_suspend_whose_call_fails_stays_running_and_can_be_retried() {
3729 let (mut sandbox, recorder, _) = planted();
3730 recorder
3731 .answer(
3732 "RunMicrovm",
3733 Answer::ok(fake::microvm_response("PENDING", None)),
3734 )
3735 .answer(
3736 "GetMicrovm",
3737 Answer::ok(fake::microvm_response("RUNNING", None)),
3738 )
3739 .answer(
3740 "GetMicrovm",
3741 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3742 )
3743 .answer(
3744 "CreateMicrovmAuthToken",
3745 Answer::ok(fake::auth_token_response("proxy-token")),
3746 )
3747 .answer("SuspendMicrovm", Answer::failure(409, "ConflictException"))
3750 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
3751
3752 sandbox
3753 .run(RunRequest::new().with_image("arn:image"))
3754 .await
3755 .expect("launches");
3756
3757 sandbox
3758 .suspend()
3759 .await
3760 .expect_err("the control plane refused the suspend");
3761 assert_eq!(
3762 sandbox.lifecycle(),
3763 Lifecycle::Running,
3764 "a refused suspend must not move the lifecycle: SUSPENDING accepts neither a \
3765 suspend nor a resume, so recording it here would brick the handle"
3766 );
3767
3768 sandbox.suspend().await.expect("the retry suspends");
3769 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3770 assert_eq!(recorder.call_count("SuspendMicrovm"), 2);
3771 }
3772
3773 #[tokio::test]
3785 async fn a_suspend_from_a_non_running_state_reaches_no_control_plane_call() {
3786 let (mut sandbox, recorder, _) = suspended().await;
3788 let before = recorder.call_count("SuspendMicrovm");
3789
3790 let error = sandbox
3791 .suspend()
3792 .await
3793 .expect_err("a suspend is only issued from RUNNING");
3794 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3795 assert!(error.to_string().contains("STATE-5"), "{error}");
3796 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3797 assert_eq!(
3798 recorder.call_count("SuspendMicrovm"),
3799 before,
3800 "the refusal must be local: no second suspend went to the wire"
3801 );
3802
3803 let (mut fresh, fresh_recorder, _) = planted();
3805 let error = fresh.suspend().await.expect_err("nothing to suspend");
3806 assert_eq!(error.kind(), ErrorKind::Precondition);
3807 assert_eq!(fresh_recorder.calls().len(), 0);
3808 }
3809
3810 #[tokio::test]
3821 async fn a_resume_reuses_the_token_redelivers_nothing_and_drops_the_proxy_token() {
3822 let (mut sandbox, recorder, _) = suspended().await;
3823
3824 let auth = Arc::clone(
3826 sandbox
3827 .session()
3828 .expect("launched")
3829 .proxy_auth()
3830 .expect("the launch wired a minter"),
3831 );
3832 auth.headers().await.expect("mints");
3833 assert!(
3834 auth.is_cached(),
3835 "there must be a token for the drop to matter"
3836 );
3837 assert_eq!(auth.mint_count(), 1);
3838
3839 recorder
3840 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3841 .answer(
3842 "GetMicrovm",
3843 Answer::ok(fake::microvm_response("RUNNING", None)),
3844 );
3845 let session = sandbox.resume().await.expect("resumes");
3846 assert_eq!(
3847 session.endpoint(),
3848 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
3849 );
3850
3851 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3852 assert!(
3853 sandbox.token_installed(),
3854 "STATE-7: the installed token is reused"
3855 );
3856 assert_eq!(
3857 sandbox.bootstrap_count(),
3858 1,
3859 "STATE-7: a resume re-delivers no run-hook payload"
3860 );
3861 assert_eq!(
3862 recorder.call_count("RunMicrovm"),
3863 1,
3864 "a resume is not a second launch"
3865 );
3866 assert!(
3867 !auth.is_cached(),
3868 "STATE-8: the cached proxy token survived the resume"
3869 );
3870 }
3871
3872 #[tokio::test]
3882 async fn the_request_after_a_resume_mints_a_fresh_proxy_token() {
3883 let (mut sandbox, recorder, _) = suspended().await;
3884 let auth = Arc::clone(
3885 sandbox
3886 .session()
3887 .expect("launched")
3888 .proxy_auth()
3889 .expect("wired"),
3890 );
3891 auth.headers().await.expect("mints");
3892 assert_eq!(auth.mint_count(), 1);
3893
3894 recorder
3895 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3896 .answer(
3897 "GetMicrovm",
3898 Answer::ok(fake::microvm_response("RUNNING", None)),
3899 );
3900 sandbox.resume().await.expect("resumes");
3901
3902 auth.headers().await.expect("re-mints");
3905 assert_eq!(
3906 auth.mint_count(),
3907 2,
3908 "STATE-8: the resume did not force a fresh mint"
3909 );
3910 }
3911
3912 #[tokio::test]
3916 async fn the_lifecycle_watch_follows_every_transition() {
3917 let (mut sandbox, recorder, _clock) = planted();
3918 let watch = sandbox.watch_lifecycle();
3919 assert_eq!(*watch.borrow(), Lifecycle::Pending);
3920 recorder
3921 .answer(
3922 "RunMicrovm",
3923 Answer::ok(fake::microvm_response("PENDING", None)),
3924 )
3925 .answer(
3926 "GetMicrovm",
3927 Answer::ok(fake::microvm_response("RUNNING", None)),
3928 )
3929 .answer(
3930 "GetMicrovm",
3931 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3932 )
3933 .answer(
3934 "CreateMicrovmAuthToken",
3935 Answer::ok(fake::auth_token_response("proxy-token")),
3936 )
3937 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
3938 sandbox
3939 .run(RunRequest::new().with_image("arn:image"))
3940 .await
3941 .expect("launches");
3942 assert_eq!(*watch.borrow(), Lifecycle::Running);
3943 sandbox.suspend().await.expect("suspends");
3944 assert_eq!(*watch.borrow(), Lifecycle::Suspended);
3945 assert_eq!(*watch.borrow(), sandbox.lifecycle());
3946 }
3947
3948 fn terminated_refusal() -> Answer {
3951 Answer {
3952 status: 400,
3953 body: r#"{"message": "The MicroVM mvm-abc123 has been terminated and its state cannot be changed."}"#
3954 .to_string(),
3955 error_type: Some("ValidationException".to_string()),
3956 }
3957 }
3958
3959 #[tokio::test]
3963 async fn resume_observes_platform_idle_suspension_and_resumes() {
3964 let (mut sandbox, recorder, _) = planted();
3965 recorder
3966 .answer(
3967 "RunMicrovm",
3968 Answer::ok(fake::microvm_response("PENDING", None)),
3969 )
3970 .answer(
3971 "GetMicrovm",
3972 Answer::ok(fake::microvm_response("RUNNING", None)),
3973 )
3974 .answer(
3975 "GetMicrovm",
3976 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3977 )
3978 .answer(
3979 "GetMicrovm",
3980 Answer::ok(fake::microvm_response("RUNNING", None)),
3981 )
3982 .answer(
3983 "CreateMicrovmAuthToken",
3984 Answer::ok(fake::auth_token_response("proxy-token")),
3985 )
3986 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3987 sandbox
3988 .run(RunRequest::new().with_image("arn:image"))
3989 .await
3990 .expect("launch");
3991 let token = sandbox
3992 .session()
3993 .expect("session")
3994 .agent_token()
3995 .to_string();
3996 let watch = sandbox.watch_lifecycle();
3997 let calls = recorder.calls().len();
3998 sandbox.resume().await.expect("idle-suspended VM resumes");
3999 assert_eq!(
4000 &recorder.operations()[calls..],
4001 &["GetMicrovm", "ResumeMicrovm", "GetMicrovm"]
4002 );
4003 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4004 assert_eq!(*watch.borrow(), Lifecycle::Running);
4005 assert_eq!(sandbox.microvm().expect("VM").state, "RUNNING");
4006 assert_eq!(sandbox.bootstrap_count(), 1);
4007 assert_eq!(recorder.call_count("RunMicrovm"), 1);
4008 assert_eq!(recorder.call_count("SuspendMicrovm"), 0);
4009 assert_eq!(sandbox.session().expect("session").agent_token(), token);
4010 assert!(
4011 !sandbox
4012 .session()
4013 .expect("session")
4014 .proxy_auth()
4015 .expect("auth")
4016 .is_cached()
4017 );
4018 }
4019
4020 #[tokio::test]
4024 async fn resume_observes_platform_termination_and_records_window_closed() {
4025 for state in ["TERMINATING", "TERMINATED"] {
4026 let (mut sandbox, recorder, _) = planted();
4027 recorder
4028 .answer(
4029 "RunMicrovm",
4030 Answer::ok(fake::microvm_response("PENDING", None)),
4031 )
4032 .answer(
4033 "GetMicrovm",
4034 Answer::ok(fake::microvm_response("RUNNING", None)),
4035 )
4036 .answer(
4037 "GetMicrovm",
4038 Answer::ok(fake::microvm_response(state, Some("idle window elapsed"))),
4039 )
4040 .answer(
4041 "CreateMicrovmAuthToken",
4042 Answer::ok(fake::auth_token_response("proxy-token")),
4043 );
4044 sandbox
4045 .run(RunRequest::new().with_image("arn:image"))
4046 .await
4047 .expect("launch");
4048 let watch = sandbox.watch_lifecycle();
4049 let error = sandbox.resume().await.expect_err("the VM ended");
4050 assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}");
4051 assert!(error.to_string().contains(state), "{error}");
4052 assert!(error.to_string().contains("idle window elapsed"), "{error}");
4053 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4054 assert_eq!(*watch.borrow(), Lifecycle::Terminated);
4055 assert_eq!(sandbox.microvm().expect("VM").state, state);
4056 assert!(sandbox.was_terminated());
4057 assert_eq!(sandbox.drop_warning(), None);
4058 let calls = recorder.calls().len();
4059 assert_eq!(
4060 sandbox.resume().await.expect_err("STATE-11").kind(),
4061 ErrorKind::InvalidArg
4062 );
4063 assert_eq!(
4064 recorder.calls().len(),
4065 calls,
4066 "known termination stays local"
4067 );
4068 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4069 }
4070 }
4071
4072 #[tokio::test]
4075 async fn resume_observes_running_and_propagates_observation_failures() {
4076 for (response, expected) in [
4077 (
4078 Answer::ok(fake::microvm_response("RUNNING", None)),
4079 ErrorKind::InvalidArg,
4080 ),
4081 (
4082 Answer::ok(fake::microvm_response("UNKNOWN", None)),
4083 ErrorKind::Platform,
4084 ),
4085 (
4086 Answer::failure(403, "access denied"),
4087 ErrorKind::Credentials,
4088 ),
4089 ] {
4090 let (mut sandbox, recorder, _) = planted();
4091 recorder
4092 .answer(
4093 "RunMicrovm",
4094 Answer::ok(fake::microvm_response("PENDING", None)),
4095 )
4096 .answer(
4097 "GetMicrovm",
4098 Answer::ok(fake::microvm_response("RUNNING", None)),
4099 )
4100 .answer("GetMicrovm", response)
4101 .answer(
4102 "CreateMicrovmAuthToken",
4103 Answer::ok(fake::auth_token_response("proxy-token")),
4104 );
4105 sandbox
4106 .run(RunRequest::new().with_image("arn:image"))
4107 .await
4108 .expect("launch");
4109 let calls = recorder.calls().len();
4110 let error = sandbox.resume().await.expect_err("cannot resume");
4111 assert_eq!(error.kind(), expected, "{error}");
4112 assert_eq!(&recorder.operations()[calls..], &["GetMicrovm"]);
4113 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4114 assert!(!sandbox.was_terminated());
4115 }
4116 }
4117
4118 #[tokio::test]
4121 async fn resume_retries_observation_after_transient_service_states() {
4122 for state in ["SUSPENDING", "PENDING", "RESUMING"] {
4123 let (mut sandbox, recorder, _) = planted();
4124 recorder
4125 .answer(
4126 "RunMicrovm",
4127 Answer::ok(fake::microvm_response("PENDING", None)),
4128 )
4129 .answer(
4130 "GetMicrovm",
4131 Answer::ok(fake::microvm_response("RUNNING", None)),
4132 )
4133 .answer(
4134 "GetMicrovm",
4135 Answer::ok(fake::microvm_response(state, None)),
4136 )
4137 .answer(
4138 "GetMicrovm",
4139 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4140 )
4141 .answer(
4142 "GetMicrovm",
4143 Answer::ok(fake::microvm_response("RUNNING", None)),
4144 )
4145 .answer(
4146 "CreateMicrovmAuthToken",
4147 Answer::ok(fake::auth_token_response("proxy-token")),
4148 )
4149 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4150 sandbox
4151 .run(RunRequest::new().with_image("arn:image"))
4152 .await
4153 .expect("launch");
4154 let error = sandbox.resume().await.expect_err("still changing state");
4155 assert!(error.to_string().contains(state), "{error}");
4156 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4157 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4158 sandbox
4159 .resume()
4160 .await
4161 .expect("the next observation can resume");
4162 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4163 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4164 }
4165 }
4166
4167 #[tokio::test]
4173 async fn a_resume_past_the_window_of_an_explicit_suspend_reaches_the_service_and_resumes() {
4174 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
4175 assert_eq!(sandbox.suspended_window(), Some(Duration::from_secs(60)));
4176 assert_eq!(
4177 sandbox.idle_window(),
4178 Some(Duration::from_secs(600)),
4179 "the launch's own maxIdleDurationSeconds, which a keepalive checks its cadence against"
4180 );
4181
4182 clock.advance(Duration::from_secs(1283));
4183 recorder
4184 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4185 .answer(
4186 "GetMicrovm",
4187 Answer::ok(fake::microvm_response("RUNNING", None)),
4188 );
4189 sandbox
4190 .resume()
4191 .await
4192 .expect("the service resumes an explicit suspend past its window");
4193
4194 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4195 assert!(!sandbox.was_terminated());
4196 assert_eq!(
4197 recorder.call_count("ResumeMicrovm"),
4198 1,
4199 "the resume has to reach the service"
4200 );
4201 }
4202
4203 #[tokio::test]
4209 async fn a_resume_the_service_refuses_for_an_ended_vm_is_window_closed_and_recorded() {
4210 let (mut sandbox, recorder, _clock) = planted();
4211 recorder
4214 .answer(
4215 "RunMicrovm",
4216 Answer::ok(fake::microvm_response("PENDING", None)),
4217 )
4218 .answer(
4219 "GetMicrovm",
4220 Answer::ok(fake::microvm_response("RUNNING", None)),
4221 )
4222 .answer(
4223 "GetMicrovm",
4224 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4225 )
4226 .answer(
4227 "GetMicrovm",
4228 Answer::ok(fake::microvm_response(
4229 "TERMINATED",
4230 Some("MicroVM exceeded maximum lifetime."),
4231 )),
4232 )
4233 .answer(
4234 "CreateMicrovmAuthToken",
4235 Answer::ok(fake::auth_token_response("proxy-token")),
4236 )
4237 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
4238 .answer("ResumeMicrovm", terminated_refusal());
4239 sandbox
4240 .run(
4241 RunRequest::new()
4242 .with_image("arn:image")
4243 .with_suspended_sec(60),
4244 )
4245 .await
4246 .expect("launches");
4247 sandbox.suspend().await.expect("suspends");
4248
4249 let error = sandbox
4250 .resume()
4251 .await
4252 .expect_err("the service ended the VM");
4253 assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}");
4254 assert_eq!(error.code(), "ERR_WINDOW_CLOSED");
4255 let message = error.to_string();
4256 assert!(message.contains("TERMINATED"), "the state: {message}");
4257 assert!(
4258 message.contains("MicroVM exceeded maximum lifetime."),
4259 "the stateReason: {message}"
4260 );
4261 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4262 assert!(sandbox.was_terminated());
4263 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4264
4265 let again = sandbox.resume().await.expect_err("STATE-11");
4266 assert!(again.to_string().contains("STATE-11"), "{again}");
4267 assert_eq!(
4268 recorder.call_count("ResumeMicrovm"),
4269 1,
4270 "the second resume makes no call"
4271 );
4272 assert_eq!(
4273 sandbox.drop_warning(),
4274 None,
4275 "a VM the service ended bills nothing, so dropping its sandbox isn't a leak"
4276 );
4277 }
4278
4279 #[tokio::test]
4283 async fn a_resume_refused_for_a_live_vm_leaves_the_sandbox_suspended_and_resumable() {
4284 let (mut sandbox, recorder, _clock) = suspended().await;
4285 recorder
4286 .answer(
4287 "ResumeMicrovm",
4288 Answer::failure(409, "The MicroVM is changing state."),
4289 )
4290 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
4291
4292 let error = sandbox.resume().await.expect_err("the service refused");
4293 assert_eq!(error.kind(), ErrorKind::Platform, "{error}");
4294 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
4295 assert!(!sandbox.was_terminated());
4296
4297 recorder.answer(
4298 "GetMicrovm",
4299 Answer::ok(fake::microvm_response("RUNNING", None)),
4300 );
4301 sandbox.resume().await.expect("the retry resumes");
4302 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4303 assert_eq!(recorder.call_count("ResumeMicrovm"), 2);
4304 }
4305
4306 #[tokio::test]
4310 async fn a_client_token_launch_resumes_the_vm_it_adopted() {
4311 let (mut sandbox, recorder, _) = planted();
4312 recorder
4313 .answer(
4314 "RunMicrovm",
4315 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4316 )
4317 .answer(
4318 "GetMicrovm",
4319 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4320 )
4321 .answer(
4322 "GetMicrovm",
4323 Answer::ok(fake::microvm_response("RUNNING", None)),
4324 )
4325 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4326 .answer(
4327 "CreateMicrovmAuthToken",
4328 Answer::ok(fake::auth_token_response("proxy-token")),
4329 );
4330 let mut request = RunRequest::new().with_image("arn:image");
4331 request.client_token = Some("job-195".into());
4332 request.agent_token = Some("agent-token".into());
4333 sandbox
4334 .run(request)
4335 .await
4336 .expect("the adopted VM is resumed");
4337 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4338 assert_eq!(sandbox.bootstrap_count(), 1);
4339 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4340
4341 let (mut fresh, recorder, _) = planted();
4342 recorder
4343 .answer(
4344 "RunMicrovm",
4345 Answer::ok(fake::microvm_response("PENDING", None)),
4346 )
4347 .answer(
4348 "GetMicrovm",
4349 Answer::ok(fake::microvm_response("SUSPENDED", Some("hook failed"))),
4350 );
4351 let error = fresh
4352 .run(RunRequest::new().with_image("arn:image"))
4353 .await
4354 .expect_err("a fresh launch that reads SUSPENDED died during startup");
4355 assert_eq!(error.kind(), ErrorKind::LaunchDied);
4356 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4357 }
4358
4359 #[tokio::test]
4362 async fn a_launch_that_does_not_wait_is_finished_by_wait_until_running() {
4363 let (mut sandbox, recorder, _) = planted();
4364 answer_launch(&recorder);
4365 let mut request = RunRequest::new().with_image("arn:image");
4366 request.wait = false;
4367 let endpoint = sandbox
4368 .run(request)
4369 .await
4370 .expect("accepted")
4371 .endpoint()
4372 .to_string();
4373 assert!(endpoint.starts_with("https://"), "{endpoint}");
4374 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
4375 assert_eq!(sandbox.bootstrap_count(), 0);
4376 assert_eq!(recorder.call_count("GetMicrovm"), 0, "nothing polled yet");
4377
4378 sandbox
4379 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4380 .await
4381 .expect("reaches RUNNING");
4382 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4383 assert_eq!(sandbox.bootstrap_count(), 1);
4384 let error = sandbox
4385 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4386 .await
4387 .expect_err("a running launch has nothing to wait for");
4388 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4389 assert_eq!(sandbox.bootstrap_count(), 1, "counted once (STATE-3)");
4390 }
4391
4392 #[tokio::test]
4400 async fn a_resume_after_terminate_records_zero_resume_calls() {
4401 let (mut sandbox, recorder, _) = launched().await;
4402 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4403 let report = sandbox.terminate(TeardownOpts::default()).await;
4404 assert!(report.terminate_accepted);
4405 assert!(sandbox.was_terminated());
4406
4407 let error = sandbox
4408 .resume()
4409 .await
4410 .expect_err("a terminated VM never returns to RUNNING");
4411 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4412 assert!(error.to_string().contains("STATE-11"), "{error}");
4413 assert_eq!(
4414 recorder.call_count("ResumeMicrovm"),
4415 0,
4416 "no resume may reach the wire once the VM was terminated"
4417 );
4418 assert_ne!(
4419 sandbox.lifecycle(),
4420 Lifecycle::Running,
4421 "the lifecycle must not return to RUNNING"
4422 );
4423 }
4424
4425 #[tokio::test]
4432 async fn a_terminate_records_terminating_and_the_completion_report_marks_terminated() {
4433 let (mut sandbox, recorder, _) = launched().await;
4434 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4435
4436 let report = sandbox.terminate(TeardownOpts::default()).await;
4437 assert!(report.terminate_accepted, "STATE-9: the call was accepted");
4438 assert!(sandbox.was_terminated(), "STATE-9: recorded terminated");
4439 assert_eq!(
4440 report.lifecycle,
4441 Some(Lifecycle::Terminating),
4442 "the default teardown does not wait, so TERMINATED is not claimed"
4443 );
4444 assert!(!report.leaked(), "{report:?}");
4445
4446 let (mut sandbox, recorder, _) = launched().await;
4448 recorder
4449 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4450 .answer(
4451 "GetMicrovm",
4452 Answer::ok(fake::microvm_response("TERMINATED", None)),
4453 );
4454 let report = sandbox
4455 .terminate(TeardownOpts::default().waiting_for_terminated())
4456 .await;
4457 assert_eq!(report.lifecycle, Some(Lifecycle::Terminated));
4458 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4459 }
4460
4461 #[tokio::test]
4470 async fn teardown_deletes_the_vm_then_the_image_and_names_the_log_group_last() {
4471 let (mut sandbox, recorder, _) = planted();
4472 recorder
4473 .answer(
4474 "CreateMicrovmImage",
4475 Answer::created(fake::create_image_response("agentd-conformance")),
4476 )
4477 .answer(
4478 "GetMicrovmImage",
4479 Answer::ok(fake::get_image_response("agentd-conformance", "CREATED")),
4480 );
4481 sandbox
4482 .build_image(CreateImageRequest::new(
4483 "agentd-conformance",
4484 crate::testing::aarch64_daemon(b"binary"),
4485 "s3://bucket/img.zip",
4486 "arn:aws:iam::123456789012:role/build",
4487 ))
4488 .await
4489 .expect("builds");
4490 answer_launch(&recorder);
4491 sandbox
4492 .run(RunRequest::new())
4493 .await
4494 .expect("launches from the built image");
4495
4496 recorder
4497 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4498 .answer(
4499 "ListMicrovmImageVersions",
4500 Answer::ok(fake::list_versions_response("1")),
4501 )
4502 .answer(
4503 "DeleteMicrovmImage",
4504 Answer::ok(fake::delete_image_response()),
4505 );
4506
4507 let report = sandbox
4508 .terminate(
4509 TeardownOpts::default()
4510 .deleting_image()
4511 .deleting_log_group(),
4512 )
4513 .await;
4514 assert!(report.terminate_accepted);
4515 assert_eq!(report.image_deleted, Some(true));
4516
4517 let order: Vec<&str> = recorder
4519 .operations()
4520 .into_iter()
4521 .filter(|operation| matches!(*operation, "TerminateMicrovm" | "DeleteMicrovmImage"))
4522 .collect();
4523 assert_eq!(
4524 order,
4525 ["TerminateMicrovm", "DeleteMicrovmImage"],
4526 "the VM goes before the image: a terminating VM holds a reference to it"
4527 );
4528
4529 assert_eq!(
4531 report.undeleted,
4532 ["/aws/lambda-microvms/agentd-conformance"],
4533 "{report:?}"
4534 );
4535 assert!(report.leaked());
4536 assert!(
4537 report
4538 .failures
4539 .last()
4540 .expect("a failure")
4541 .contains("log group"),
4542 "the log group's failure has to be the last one recorded: {report:?}"
4543 );
4544 assert!(
4545 report
4546 .failures
4547 .last()
4548 .expect("a failure")
4549 .contains("terraform destroy"),
4550 "the reason it leaks has to be named: {report:?}"
4551 );
4552 }
4553
4554 #[tokio::test]
4560 async fn a_failing_image_delete_names_what_it_left_behind() {
4561 let (mut sandbox, recorder, _) = planted();
4562 recorder
4563 .answer(
4564 "CreateMicrovmImage",
4565 Answer::created(fake::create_image_response("img")),
4566 )
4567 .answer(
4568 "GetMicrovmImage",
4569 Answer::ok(fake::get_image_response("img", "CREATED")),
4570 );
4571 sandbox
4572 .build_image(CreateImageRequest::new(
4573 "img",
4574 crate::testing::aarch64_daemon(b"binary"),
4575 "s3://bucket/img.zip",
4576 "arn:aws:iam::123456789012:role/build",
4577 ))
4578 .await
4579 .expect("builds");
4580 answer_launch(&recorder);
4581 sandbox.run(RunRequest::new()).await.expect("launches");
4582
4583 recorder
4584 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4585 .answer(
4586 "ListMicrovmImageVersions",
4587 Answer::ok(fake::list_versions_response("1")),
4588 )
4589 .answer(
4590 "DeleteMicrovmImage",
4591 Answer::failure(409, "image is in CREATING"),
4592 );
4593
4594 let mut opts = TeardownOpts::default().deleting_image();
4595 opts.delete_attempts = 3;
4596 opts.delete_backoff = Duration::from_secs(1);
4597 let report = sandbox.terminate(opts).await;
4598
4599 assert_eq!(report.image_deleted, Some(false));
4600 assert_eq!(
4601 report.undeleted,
4602 ["arn:aws:lambda:us-east-1:123456789012:microvm-image:img"],
4603 "{report:?}"
4604 );
4605 assert!(
4606 sandbox.image_exists(),
4607 "the image really is still there, so the flag must say so"
4608 );
4609 assert_eq!(
4610 recorder.call_count("DeleteMicrovmImage"),
4611 3,
4612 "every attempt ran"
4613 );
4614 }
4615
4616 #[tokio::test]
4623 async fn a_terminate_whose_call_fails_still_records_the_intent_and_the_leak() {
4624 let (mut sandbox, recorder, _) = launched().await;
4625 recorder.answer(
4626 "TerminateMicrovm",
4627 Answer::failure(500, "InternalServerException"),
4628 );
4629
4630 let report = sandbox.terminate(TeardownOpts::default()).await;
4631 assert!(!report.terminate_accepted);
4632 assert_eq!(report.undeleted, ["mvm-abc123"], "{report:?}");
4633 assert!(sandbox.was_terminated());
4634 assert_eq!(report.lifecycle, Some(Lifecycle::Terminating));
4635 assert!(
4636 !report.failures.is_empty(),
4637 "the swallowed failure is recorded"
4638 );
4639 }
4640
4641 #[tokio::test]
4647 async fn a_teardown_before_any_launch_is_an_empty_report() {
4648 let (mut sandbox, recorder, _) = planted();
4649 let report = sandbox.terminate(TeardownOpts::default()).await;
4650 assert!(!report.terminate_accepted);
4651 assert!(!report.leaked());
4652 assert_eq!(report.lifecycle, Some(Lifecycle::Pending));
4653 assert_eq!(recorder.calls().len(), 0);
4654 }
4655
4656 #[tokio::test]
4659 async fn a_run_with_no_image_is_refused_before_any_call() {
4660 let (mut sandbox, recorder, _) = planted();
4661 let error = sandbox
4662 .run(RunRequest::new())
4663 .await
4664 .expect_err("there is nothing to launch");
4665 assert_eq!(error.kind(), ErrorKind::Precondition);
4666 assert!(error.to_string().contains("build_image first"), "{error}");
4667 assert_eq!(recorder.calls().len(), 0);
4668 }
4669
4670 #[test]
4677 fn the_lifecycle_states_are_the_state_models_domain_in_the_services_spelling() {
4678 let all = [
4679 Lifecycle::Pending,
4680 Lifecycle::Running,
4681 Lifecycle::Suspending,
4682 Lifecycle::Suspended,
4683 Lifecycle::Terminating,
4684 Lifecycle::Terminated,
4685 ];
4686 assert_eq!(
4687 all.map(Lifecycle::as_str),
4688 [
4689 "PENDING",
4690 "RUNNING",
4691 "SUSPENDING",
4692 "SUSPENDED",
4693 "TERMINATING",
4694 "TERMINATED"
4695 ]
4696 );
4697 for state in all {
4698 assert_eq!(state.to_string(), state.as_str());
4699 }
4700 assert_eq!(
4703 all.iter().filter(|state| state.is_live()).count(),
4704 4,
4705 "a state added to the enum defaults to not-live, which would silence the warning"
4706 );
4707 assert!(!Lifecycle::Terminating.is_live());
4708 assert!(!Lifecycle::Terminated.is_live());
4709 }
4710
4711 #[test]
4719 fn a_minted_agent_token_is_sixty_four_hex_characters_and_never_repeats() {
4720 let entropy = crate::entropy::testing::SequenceEntropy::new();
4721 let minted: std::collections::HashSet<String> = (0..200)
4722 .map(|_| mint_agent_token(&entropy).expect("scripted"))
4723 .collect();
4724 assert_eq!(minted.len(), 200, "a repeated token is a shared guest");
4725 for token in &minted {
4726 assert_eq!(token.len(), 64, "{token}");
4727 assert!(token.chars().all(|c| c.is_ascii_hexdigit()), "{token}");
4728 assert_ne!(
4729 token,
4730 &"0".repeat(64),
4731 "an all-zero draw is a read that did nothing"
4732 );
4733 }
4734 }
4735
4736 #[tokio::test]
4738 async fn a_sandbox_debug_does_not_print_the_agent_token() {
4739 let (mut sandbox, recorder, _) = planted();
4740 answer_launch(&recorder);
4741 sandbox
4742 .run(
4743 RunRequest::new()
4744 .with_image("arn:image")
4745 .with_agent_token("super-secret-agent-token"),
4746 )
4747 .await
4748 .expect("launches");
4749
4750 let rendered = format!("{sandbox:?}");
4751 assert!(rendered.contains("mvm-abc123"), "{rendered}");
4752 assert!(rendered.contains("Running"), "{rendered}");
4753 assert!(
4754 !rendered.contains("super-secret"),
4755 "the agent token reached a Debug string: {rendered}"
4756 );
4757 }
4758
4759 #[tokio::test]
4767 async fn a_torn_down_sandbox_is_distinguishable_from_an_abandoned_one() {
4768 let (mut sandbox, recorder, _) = launched().await;
4769 assert!(
4770 sandbox.lifecycle().is_live() && !sandbox.torn_down,
4771 "an abandoned live VM is what the drop warning is for"
4772 );
4773
4774 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4775 sandbox.terminate(TeardownOpts::default()).await;
4776 assert!(
4777 sandbox.torn_down,
4778 "terminate must mark the sandbox torn down"
4779 );
4780 assert!(
4781 !sandbox.lifecycle().is_live(),
4782 "and the lifecycle must no longer read as live"
4783 );
4784 }
4785
4786 #[tokio::test]
4787 async fn stable_launch_key_and_payload_survive_a_fresh_supervisor() {
4788 let mut bodies = Vec::new();
4789 for _ in 0..2 {
4790 let (mut sandbox, recorder, _) = planted();
4791 answer_launch(&recorder);
4792 let mut request = RunRequest::new().with_image("arn:image");
4793 request.client_token = Some("one-unique-job-launch".into());
4794 request.agent_token = Some("persisted-private-secret".into());
4795 sandbox.run(request).await.expect("accepted");
4796 bodies.push(recorder.first_body("RunMicrovm"));
4797 }
4798 assert_eq!(bodies[0]["clientToken"], "one-unique-job-launch");
4799 assert_eq!(bodies[0], bodies[1]);
4800 }
4801
4802 #[tokio::test]
4803 async fn unsafe_or_malformed_stable_launch_is_refused_without_calls() {
4804 for (token, secret, identity) in [
4805 ("key".to_string(), None, false),
4806 ("key".to_string(), Some("secret".into()), true),
4807 ("".to_string(), Some("secret".into()), false),
4808 ("x".repeat(129), Some("secret".into()), false),
4809 ("line\nbreak".to_string(), Some("secret".into()), false),
4810 ] {
4811 let (mut sandbox, recorder, _) = planted();
4812 let mut request = RunRequest::new().with_image("arn:image");
4813 request.client_token = Some(token);
4814 request.agent_token = secret;
4815 request.identity = identity;
4816 let error = sandbox.run(request).await.expect_err("invalid");
4817 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4818 assert!(recorder.calls().is_empty());
4819 }
4820 }
4821}